chore(scanner): merge latest origin/main

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

.config/e2e-distributed-selection.txt

@@ -1,2 +1,2 @@
-sha256-linux=74a1d6975e2d6dfe01363585cb9bfa3232d79556a154d4f599653fe2ed1ce696
-sha256-darwin=74a1d6975e2d6dfe01363585cb9bfa3232d79556a154d4f599653fe2ed1ce696
+sha256-linux=e31182a075f38b8a6680f4563ed674f0cb92d7ad9e764d3da667c99510656325
+sha256-darwin=e31182a075f38b8a6680f4563ed674f0cb92d7ad9e764d3da667c99510656325

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=b53fe6d46cdffedfad6a83da0dfffa9aec10cd7c8ace53469f129587bc9f4d50
-sha256-linux=6be90f5c3886e9852ac27d51f2d47e6d2fa8bb424e0d03ee931268dcd0020db8
+sha256-darwin=359b16f0bba8baa6e995fd3701062a61088fd4c0196ade3bac9e8fda7a050833
+sha256-linux=137098f8bc31a5ca38cac35a5b628346c1082b0277ec3982e2e0ffb69d0ea2e4

.config/e2e-smoke-selection.txt

@@ -1 +1 @@
-sha256=002ec5d973f82288e50a096a2c8b3b38749f62a7a3e169775566874d8ee98b4a
+sha256=eaa140bb7c7fff74c92a0cd191805f7fdee3174eaa6a649a1b807e1e6b497a31

.github/workflows/e2e-upgrade.yml

@@ -25,7 +25,6 @@ on:
       - "crates/e2e_test/src/upgrade_compatibility_test.rs"
       - "crates/ecstore/**"
       - "crates/filemeta/**"
-      - "crates/heal/**"
       - "crates/kms/**"
       - "crates/storage-api/**"
       - "rustfs/**"
@@ -161,7 +160,6 @@ jobs:
           )
           if [[ "$UPGRADE_SOURCE_VERSION" == "1.0.0-rc.5" ]]; then
             cases+=(odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration)
-            cases+=(checkpoint-upgrade:rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects)
           else
             cases+=(legacy-delete-reservation:stable_upgrade_deletes_foreign_reservations_without_losing_live_objects)
           fi

.github/workflows/windows-filesystem.yml

@@ -25,8 +25,6 @@ on:
       - "crates/ecstore/src/object_api/write_commit_context.rs"
       - "crates/ecstore/src/bucket/utils.rs"
       - "crates/ecstore/Cargo.toml"
-      - "crates/storage-api/src/bucket.rs"
-      - "crates/storage-api/Cargo.toml"
       - "crates/utils/src/dunce.rs"
       - "crates/utils/src/os/**"
       - "crates/utils/Cargo.toml"
@@ -50,8 +48,6 @@ on:
       - "crates/ecstore/src/object_api/write_commit_context.rs"
       - "crates/ecstore/src/bucket/utils.rs"
       - "crates/ecstore/Cargo.toml"
-      - "crates/storage-api/src/bucket.rs"
-      - "crates/storage-api/Cargo.toml"
       - "crates/utils/src/dunce.rs"
       - "crates/utils/src/os/**"
       - "crates/utils/Cargo.toml"
@@ -144,17 +140,6 @@ jobs:
         shell: pwsh
         run: "cargo test -p rustfs-ecstore --lib bucket::utils:: -- --nocapture"
 
-      - name: Test artifact prefix Windows file-name rules
-        shell: pwsh
-        run: |
-          $testName = 'bucket::tests::artifact_prefix_preserves_host_file_name_rules'
-          cargo test --locked -p rustfs-storage-api --lib $testName -- --exact --nocapture --color never 2>&1 |
-            Tee-Object -Variable testOutput
-          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
-          if (($testOutput -join "`n") -notmatch 'test result: ok\. 1 passed; 0 failed; 0 ignored;') {
-            throw 'Expected exactly one passing artifact prefix test on Windows'
-          }
-
       - name: Test Windows file-name rules
         shell: pwsh
         run: "cargo test -p rustfs-utils --no-default-features --features path --lib dunce:: -- --nocapture"

Cargo.toml

@@ -252,9 +252,9 @@ atoi = "3.1.1"
 atomic_enum = "0.3.0"
 aws-config = { version = "1.13.0" }
 aws-credential-types = { version = "1.3.1" }
-aws-sdk-kms = { default-features = false, version = "1.126.0" }
-aws-sdk-s3 = { default-features = false, version = "1.155.0" }
-aws-sdk-sts = { default-features = false, version = "1.122.0" }
+aws-sdk-kms = { default-features = false, version = "1.125.0" }
+aws-sdk-s3 = { default-features = false, version = "1.154.0" }
+aws-sdk-sts = { default-features = false, version = "1.121.0" }
 aws-smithy-async = { version = "1.3.1" }
 aws-smithy-http-client = { default-features = false, version = "1.5.1" }
 aws-smithy-runtime-api = { version = "1.19.1" }
@@ -272,7 +272,7 @@ crossbeam-utils = "0.8.23"
 datafusion = { default-features = false, version = "55.2.0" }
 derive_builder = "0.20.2"
 ed25519-dalek = "3.0.0"
-enumset = "1.1.15"
+enumset = "1.1.14"
 faster-hex = "1.0.0"
 flate2 = "1.1.10"
 glob = "0.3.4"
@@ -317,7 +317,7 @@ rustfs-erasure-codec = { version = "9.0.0" }
 reed-solomon-simd = "3.1.0"
 regex = { version = "1.13.1" }
 rumqttc = { package = "rumqttc-next", version = "0.34.0" }
-redis = { version = "1.8.0" }
+redis = { version = "1.7.1" }
 rustify = { version = "0.7", default-features = false }
 rustix = { version = "1.1.5" }
 rust-embed = { version = "8.13.0" }
@@ -327,7 +327,7 @@ s3s-sigv4 = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fc
 serial_test = "4.0.1"
 shadow-rs = { default-features = false, version = "2.0.0" }
 siphasher = "1.0.4"
-smallvec = { version = "1.16.4" }
+smallvec = { version = "1.16.3" }
 compact_str = "0.10.0"
 snap = "1.1.2"
 starshard = { version = "2.4.0" }
@@ -346,7 +346,7 @@ tracing-subscriber = { version = "0.3.23" }
 transform-stream = "0.3.1"
 url = "2.5.8"
 urlencoding = "2.1.3"
-uuid = { version = "1.29.0" }
+uuid = { version = "1.28.0" }
 vaultrs = { version = "0.8.0" }
 tar = "0.4.46"
 walkdir = "2.5.0"
@@ -354,7 +354,7 @@ winapi-util = "0.1.11"
 windows = { version = "0.62.2" }
 windows-sys = "0.61.2"
 xxhash-rust = { version = "0.8.19" }
-zip = "9.1.0"
+zip = "9.0.5"
 zstd = "0.14.3"
 
 # Observability and Metrics
@@ -368,7 +368,7 @@ opentelemetry-proto = { version = "0.33.1", default-features = false, features =
 opentelemetry_sdk = { version = "0.33.1" }
 opentelemetry-semantic-conventions = { version = "0.33.1" }
 opentelemetry-stdout = { version = "0.33.1" }
-pyroscope = { version = "2.1.5" }
+pyroscope = { version = "2.1.4" }
 
 # FTP and SFTP
 libunftp = { version = "0.23.1" }
@@ -382,9 +382,9 @@ russh-sftp = "3.0.1"
 dav-server = "0.11.0"
 
 # Performance Analysis and Memory Profiling
-rustfs-mimalloc = { version = "0.7.2" }
+rustfs-mimalloc = { version = "0.7.1" }
 # Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
-hotpath = { version = "0.29.1", default-features = false }
+hotpath = { version = "0.28.7", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.49" }
 
@@ -421,19 +421,3 @@ inherits = "release"
 inherits = "release"
 debug = true
 strip = "none"
-
-# Pin upstream fixes until compatible crates.io releases are available.
-[patch.crates-io]
-# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
-# Upstream: openidconnect-rs#244 and oauth2-rs#352.
-openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
-oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" }
-aws-sigv4 = { git = "https://github.com/smithy-lang/smithy-rs", rev = "5c1c4bca2ea7af97496e077c9f1a718eaac64bf8" }
-
-# Keep the signer's path dependencies on the SDK's existing registry types.
-[patch."https://github.com/smithy-lang/smithy-rs"]
-aws-credential-types = { version = "=1.3.1" }
-aws-smithy-eventstream = { version = "=0.61.6" }
-aws-smithy-http = { version = "=0.64.2" }
-aws-smithy-runtime-api = { version = "=1.19.1" }
-aws-smithy-types = { version = "=1.9.0" }

crates/concurrency/src/adaptive.rs

@@ -1,957 +0,0 @@
-// Copyright 2026 RustFS Team
-// Licensed under the Apache License, Version 2.0.
-
-//! Mutable admission capacity without replacing or revoking held permits.
-
-use std::num::NonZeroU16;
-use std::sync::Arc;
-use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
-use std::time::Duration;
-use tokio::sync::{Notify, Semaphore};
-use tokio::time::Instant;
-
-/// A resource owner publishes a capacity; running units retain their permits
-/// after a reduction. Lease slots are reserved once, with no per-unit Arc
-/// allocation. Tokio wait queues may lazily initialize platform mutex storage
-/// on first contention. It does not discover devices, start
-/// workers, or decide resource policy. Cancellation drops the acquire future.
-///
-/// Uncontended admission is atomic. Parked work takes a FIFO admission turn,
-/// so repeated fast acquisitions cannot starve an older waiter. Source/age
-/// priority still belongs to the caller's scheduler, before joining this queue.
-pub struct AdaptiveAdmission {
-    // One CAS observes both capacity and active count, so a concurrent shrink
-    // cannot admit a new unit using an already superseded capacity.
-    state: AtomicU64,
-    origin: Instant,
-    valid_until_millis: AtomicU64,
-    // Only the FIFO turn holder subscribes. A stored notification is harmless:
-    // capacity and freshness are always checked again after waking.
-    changed: Notify,
-    queued: AtomicUsize,
-    // FIFO admission turn, not a mutable-state lock. No storage lock or
-    // admitted permit may be held while joining. This private semaphore is
-    // never closed; release and resize do not acquire it.
-    turn: Semaphore,
-    maximum: u32,
-    slots: Box<[PermitSlots]>,
-    next_chunk: AtomicUsize,
-}
-
-struct PermitSlots {
-    free: AtomicU64,
-    references: [AtomicUsize; 64],
-}
-
-struct QueuedAdmission<'a>(&'a AdaptiveAdmission);
-
-impl Drop for QueuedAdmission<'_> {
-    fn drop(&mut self) {
-        self.0.queued.fetch_sub(1, Ordering::AcqRel);
-    }
-}
-
-/// One active unit. Clones share that unit without allocating; the final drop
-/// returns it to the same owner, including after resize or waiter cancellation.
-#[must_use = "hold the permit for the admitted unit's complete lifetime"]
-pub struct AdaptivePermit {
-    owner: Arc<AdaptiveAdmission>,
-    slot: usize,
-}
-
-impl Clone for AdaptivePermit {
-    fn clone(&self) -> Self {
-        // Every slot reference owns a strong owner reference. Clone the Arc
-        // first, so its overflow protection also bounds the slot refcount.
-        let owner = Arc::clone(&self.owner);
-        owner.slots[self.slot / 64].references[self.slot % 64].fetch_add(1, Ordering::Relaxed);
-        Self { owner, slot: self.slot }
-    }
-}
-
-impl std::fmt::Debug for AdaptivePermit {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("AdaptivePermit")
-            .field("slot", &self.slot)
-            .field("owner", &self.owner.snapshot())
-            .finish()
-    }
-}
-
-/// One coherent view of configured capacity and actual active units.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub struct AdaptiveAdmissionSnapshot {
-    /// Current cap on new admissions. Zero pauses all new units.
-    pub limit: u32,
-    /// Existing units, which may temporarily exceed a reduced limit.
-    pub active: u32,
-}
-
-/// Node scheduling capacity, separate from physical I/O and buffer ownership.
-/// Its non-cloneable units end with the async operation; a native tail must
-/// retain its own physical-resource unit, never this scheduling unit.
-pub struct NodeAdmission {
-    capacity: Arc<AdaptiveAdmission>,
-    completed: AtomicU64,
-}
-
-/// One async operation's scheduling unit. Dropping a cancelled or failed
-/// operation releases capacity without claiming completed work.
-#[must_use = "retain the unit until the async operation ends"]
-pub struct NodePermit {
-    _capacity: AdaptivePermit,
-    owner: Arc<NodeAdmission>,
-}
-
-impl NodeAdmission {
-    /// Construct one node owner with no capacity until a resource sample arrives.
-    pub fn new(maximum: NonZeroU16) -> Arc<Self> {
-        Arc::new(Self {
-            capacity: AdaptiveAdmission::new(maximum, 0),
-            completed: AtomicU64::new(0),
-        })
-    }
-
-    /// Publish feedback with this owner's actual completed-work counter.
-    pub fn update(&self, controller: &mut FeedbackController, observation: Option<FeedbackObservation>) -> FeedbackDecision {
-        let observation = observation.map(|mut observation| {
-            observation.completed_units = self.completed();
-            observation
-        });
-        controller.update(&self.capacity, observation)
-    }
-
-    /// Current leased scheduling cap and held async units.
-    pub fn snapshot(&self) -> AdaptiveAdmissionSnapshot {
-        self.capacity.snapshot()
-    }
-
-    /// Completed operations, or unknown after the counter exhausts its range.
-    pub fn completed(&self) -> Option<u64> {
-        let value = self.completed.load(Ordering::Acquire);
-        (value != u64::MAX).then_some(value)
-    }
-
-    /// Try without waiting, including when the caller already owns a lock.
-    pub fn try_acquire(self: &Arc<Self>) -> Option<NodePermit> {
-        self.capacity.try_acquire().map(|capacity| NodePermit {
-            _capacity: capacity,
-            owner: self.clone(),
-        })
-    }
-
-    /// Wait in FIFO order. Callers must provide a deadline/cancellation and
-    /// acquire before storage locks, not while retaining another node unit.
-    pub async fn acquire(self: &Arc<Self>) -> NodePermit {
-        NodePermit {
-            _capacity: self.capacity.acquire().await,
-            owner: self.clone(),
-        }
-    }
-}
-
-impl NodePermit {
-    /// Count a completed operation exactly once, then release its node unit.
-    /// This counter is scheduling progress, not an authoritative heal receipt.
-    pub fn complete(self) {
-        let _ = self
-            .owner
-            .completed
-            .try_update(Ordering::AcqRel, Ordering::Acquire, |value| value.checked_add(1));
-    }
-}
-
-/// Confidence-qualified pressure for one complete sampling window.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum ResourcePressure {
-    /// Missing, stale, reset, or insufficient observations; never evidence of idle.
-    Unknown,
-    /// A resource or foreground-protection threshold was reached.
-    Busy,
-    /// All required observations were fresh and below their low watermarks.
-    Stable,
-}
-
-/// Resource-owner input to the controller, after applying operator and kernel
-/// caps. This describes one owner; callers must not duplicate a device's budget
-/// for aliases, pools, or remote requesters.
-#[derive(Debug, Clone, Copy)]
-pub struct FeedbackObservation {
-    /// Identity of the resource/telemetry generation. Replacement resets probes.
-    pub generation: u64,
-    /// Upper bound imposed by resources and explicit configuration; zero blocks.
-    pub max_concurrency: u32,
-    /// Complete-window pressure, including the confidence of its sources.
-    pub pressure: ResourcePressure,
-    /// Cumulative completed work in this generation, not just scheduled work.
-    pub completed_units: Option<u64>,
-}
-
-/// Why the latest concurrency limit was selected.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum FeedbackReason {
-    /// The resource cap is unavailable; no new unit can be bounded safely.
-    Unavailable,
-    /// A changed owner or reset work counter requires a new baseline.
-    NewBaseline,
-    /// The resource or operator cap reduced the permitted concurrency.
-    CapReduced,
-    /// Pressure caused an immediate multiplicative reduction.
-    Pressure,
-    /// Incomplete telemetry retains at most one maintenance unit.
-    LowConfidence,
-    /// Waiting for consecutive stable windows, work progress and cooldown.
-    Recovering,
-    /// One additional unit is being probed after a stable recovery period.
-    Probe,
-}
-
-/// One controller decision, suitable for metrics without dynamic labels.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub struct FeedbackDecision {
-    /// New admissions use this cap; existing permits remain valid.
-    pub limit: u32,
-    /// Decision cause, distinct from a proof of work completion or storage health.
-    pub reason: FeedbackReason,
-}
-
-/// Single-owner additive probe/multiplicative decrease controller. It adjusts
-/// only concurrency. Every source must be fresh before declaring `Stable`.
-///
-/// The sampler runs every second. A gap over two seconds resets recovery; eight
-/// consecutive stable seconds and observed completed work allow one extra unit.
-/// Pressure or lower caps apply immediately. This policy is not a performance
-/// claim; its parameters require the same workload/foreground-SLO acceptance.
-pub struct FeedbackController {
-    previous: Option<(Instant, FeedbackObservation)>,
-    stable_since: Option<Instant>,
-    last_change: Instant,
-    limit: u32,
-}
-
-impl Default for FeedbackController {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl FeedbackController {
-    /// Start without a resource observation or an admission grant.
-    pub fn new() -> Self {
-        Self {
-            previous: None,
-            stable_since: None,
-            last_change: Instant::now(),
-            limit: 0,
-        }
-    }
-
-    /// Consume one sample and publish a three-second lease to the same gate.
-    /// If this function stops being called, admission expires independently.
-    pub fn update(&mut self, gate: &AdaptiveAdmission, observation: Option<FeedbackObservation>) -> FeedbackDecision {
-        let observation = observation.map(|mut observation| {
-            observation.max_concurrency = observation.max_concurrency.min(gate.maximum);
-            observation
-        });
-        let decision = self.decide(Instant::now(), observation);
-        gate.set_sampled_limit(decision.limit, Duration::from_secs(3));
-        decision
-    }
-
-    fn decide(&mut self, now: Instant, observation: Option<FeedbackObservation>) -> FeedbackDecision {
-        let Some(observation) = observation else {
-            self.previous = None;
-            self.stable_since = None;
-            self.last_change = now;
-            self.limit = 0;
-            return FeedbackDecision {
-                limit: 0,
-                reason: FeedbackReason::Unavailable,
-            };
-        };
-        let previous = self.previous.replace((now, observation));
-        let contiguous = previous.is_some_and(|(at, old)| {
-            now.saturating_duration_since(at) <= Duration::from_secs(2)
-                && old.generation == observation.generation
-                && matches!((old.completed_units, observation.completed_units), (Some(a), Some(b)) if b >= a)
-        });
-        let advanced = previous
-            .is_some_and(|(_, old)| matches!((old.completed_units, observation.completed_units), (Some(a), Some(b)) if b > a));
-        let old_limit = self.limit;
-        let reason = if observation.max_concurrency == 0 {
-            self.limit = 0;
-            self.stable_since = None;
-            FeedbackReason::CapReduced
-        } else if observation.pressure == ResourcePressure::Busy {
-            self.limit = (self.limit / 2).max(1).min(observation.max_concurrency);
-            self.stable_since = None;
-            // Repeated busy windows restart cooldown, even at the minimum cap.
-            self.last_change = now;
-            FeedbackReason::Pressure
-        } else if observation.max_concurrency < self.limit {
-            self.limit = observation.max_concurrency;
-            self.stable_since = None;
-            FeedbackReason::CapReduced
-        } else if observation.pressure == ResourcePressure::Unknown || observation.completed_units.is_none() {
-            self.limit = 1.min(observation.max_concurrency);
-            self.stable_since = None;
-            FeedbackReason::LowConfidence
-        } else if !contiguous || self.limit == 0 {
-            self.limit = 1.min(observation.max_concurrency);
-            self.stable_since = None;
-            self.last_change = now;
-            FeedbackReason::NewBaseline
-        } else {
-            let since = *self.stable_since.get_or_insert(now);
-            if advanced
-                && now.saturating_duration_since(since) >= Duration::from_secs(8)
-                && now.saturating_duration_since(self.last_change) >= Duration::from_secs(8)
-                && self.limit < observation.max_concurrency
-            {
-                self.limit += 1;
-                FeedbackReason::Probe
-            } else {
-                FeedbackReason::Recovering
-            }
-        };
-        if old_limit != self.limit {
-            self.last_change = now;
-        }
-        FeedbackDecision {
-            limit: self.limit,
-            reason,
-        }
-    }
-}
-
-impl AdaptiveAdmission {
-    /// Construct a bounded owner, with zero initial capacity allowed for unknown
-    /// input. The immutable maximum reserves at most 65,535 lease slots once;
-    /// acquiring or cloning a permit does not allocate a per-unit Arc.
-    pub fn new(maximum: NonZeroU16, limit: u32) -> Arc<Self> {
-        let maximum_u32 = u32::from(maximum.get());
-        let maximum = usize::from(maximum.get());
-        let slots = (0..maximum.div_ceil(64))
-            .map(|chunk| {
-                let count = (maximum - chunk * 64).min(64);
-                PermitSlots {
-                    free: AtomicU64::new(if count == 64 { u64::MAX } else { (1_u64 << count) - 1 }),
-                    references: std::array::from_fn(|_| AtomicUsize::new(0)),
-                }
-            })
-            .collect::<Vec<_>>()
-            .into_boxed_slice();
-        Arc::new(Self {
-            state: AtomicU64::new(u64::from(limit.min(maximum_u32)) << 32),
-            origin: Instant::now(),
-            valid_until_millis: AtomicU64::new(u64::MAX),
-            changed: Notify::new(),
-            queued: AtomicUsize::new(0),
-            turn: Semaphore::new(1),
-            maximum: maximum_u32,
-            slots,
-            next_chunk: AtomicUsize::new(0),
-        })
-    }
-
-    /// Publish a cap bounded by the immutable maximum, without losing held work
-    /// or changing owner identity.
-    pub fn set_limit(&self, limit: u32) {
-        let limit = limit.min(self.maximum);
-        let _ = self.state.try_update(Ordering::AcqRel, Ordering::Acquire, |state| {
-            Some((u64::from(limit) << 32) | (state & u64::from(u32::MAX)))
-        });
-        self.changed.notify_one();
-    }
-
-    /// Publish a cap with a finite observation lifetime. Expired observations
-    /// block new admissions even if the sampling worker has stopped. Existing
-    /// permits are retained. A single resource owner must serialize publishers.
-    ///
-    /// Capacity is published before freshness, so a concurrent reader can
-    /// temporarily reject a fresh update but cannot use an increased lifetime
-    /// with the previous, potentially larger capacity.
-    pub fn set_sampled_limit(&self, limit: u32, valid_for: Duration) {
-        self.set_limit(limit);
-        let deadline = self
-            .elapsed_millis()
-            .saturating_add(u64::try_from(valid_for.as_millis()).unwrap_or(u64::MAX));
-        self.valid_until_millis.store(deadline, Ordering::Release);
-        self.changed.notify_one();
-    }
-
-    fn elapsed_millis(&self) -> u64 {
-        u64::try_from(self.origin.elapsed().as_millis()).unwrap_or(u64::MAX)
-    }
-
-    fn fresh(&self) -> bool {
-        self.elapsed_millis() < self.valid_until_millis.load(Ordering::Acquire)
-    }
-
-    /// Read the cap and active count from the same atomic observation.
-    pub fn snapshot(&self) -> AdaptiveAdmissionSnapshot {
-        // Acquire freshness first: a newly published lease must not be paired
-        // with capacity loaded before its preceding reduction.
-        let fresh = self.fresh();
-        let state = self.state.load(Ordering::Acquire);
-        AdaptiveAdmissionSnapshot {
-            limit: if fresh {
-                u32::try_from(state >> 32).unwrap_or(u32::MAX)
-            } else {
-                0
-            },
-            active: u32::try_from(state & u64::from(u32::MAX)).unwrap_or(u32::MAX),
-        }
-    }
-
-    /// Admit immediately if capacity remains, without a per-unit allocation.
-    /// The uncontended path uses atomics; refunding a racing acquisition may
-    /// wake the FIFO head through Tokio's notification queue.
-    pub fn try_acquire(self: &Arc<Self>) -> Option<AdaptivePermit> {
-        if self.queued.load(Ordering::Acquire) != 0 {
-            return None;
-        }
-        let permit = self.acquire_capacity()?;
-        // A waiter can register between the first queue check and the capacity
-        // CAS. Refund before exposing the permit rather than stealing its turn.
-        (self.queued.load(Ordering::Acquire) == 0).then_some(permit)
-    }
-
-    fn acquire_capacity(self: &Arc<Self>) -> Option<AdaptivePermit> {
-        if !self.fresh() {
-            return None;
-        }
-        let state = self.state.load(Ordering::Acquire);
-        if state & u64::from(u32::MAX) >= state >> 32 {
-            return None;
-        }
-        let slot = self.reserve_slot()?;
-        if self
-            .state
-            .try_update(Ordering::AcqRel, Ordering::Acquire, |state| {
-                let active = state & u64::from(u32::MAX);
-                (active < state >> 32).then(|| state + 1)
-            })
-            .is_err()
-        {
-            self.release_slot(slot);
-            self.changed.notify_one();
-            return None;
-        }
-        self.slots[slot / 64].references[slot % 64].store(1, Ordering::Release);
-        let permit = AdaptivePermit {
-            owner: self.clone(),
-            slot,
-        };
-        // The capacity CAS can retry across an observation expiry. Recheck
-        // before giving ownership to the caller; dropping refunds this unit.
-        self.fresh().then_some(permit)
-    }
-
-    fn reserve_slot(&self) -> Option<usize> {
-        let first = self.next_chunk.fetch_add(1, Ordering::Relaxed) % self.slots.len();
-        for offset in 0..self.slots.len() {
-            let chunk_index = (first + offset) % self.slots.len();
-            let free = &self.slots[chunk_index].free;
-            if let Ok(bits) = free.try_update(Ordering::AcqRel, Ordering::Acquire, |bits| (bits != 0).then(|| bits & (bits - 1)))
-            {
-                // trailing_zeros is in 0..64 because the zero word is rejected.
-                return Some(chunk_index * 64 + bits.trailing_zeros() as usize);
-            }
-        }
-        None
-    }
-
-    fn release_slot(&self, slot: usize) {
-        self.slots[slot / 64].free.fetch_or(1_u64 << (slot % 64), Ordering::Release);
-    }
-
-    /// Wait for capacity. The future owns no permit until it returns; dropping
-    /// a pending future leaves capacity untouched. Wrap in the caller's existing
-    /// cancellation/deadline select before acquiring any storage commit locks.
-    pub async fn acquire(self: &Arc<Self>) -> AdaptivePermit {
-        if let Some(permit) = self.try_acquire() {
-            return permit;
-        }
-        self.queued.fetch_add(1, Ordering::AcqRel);
-        let _queued = QueuedAdmission(self);
-        // Retain the successful permit inside its Result. No code can close
-        // this private semaphore, so acquisition cannot return an error.
-        let _turn = self.turn.acquire().await;
-        loop {
-            let notified = self.changed.notified();
-            tokio::pin!(notified);
-            // Register before checking capacity to avoid a lost release/resize.
-            notified.as_mut().enable();
-            if let Some(permit) = self.acquire_capacity() {
-                return permit;
-            }
-            notified.await;
-        }
-    }
-}
-
-impl Drop for AdaptivePermit {
-    fn drop(&mut self) {
-        if self.owner.slots[self.slot / 64].references[self.slot % 64].fetch_sub(1, Ordering::AcqRel) != 1 {
-            return;
-        }
-        // Publish the free slot before exposing another admission capacity.
-        self.owner.release_slot(self.slot);
-        self.owner.state.fetch_sub(1, Ordering::AcqRel);
-        self.owner.changed.notify_one();
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[tokio::test]
-    async fn node_progress_requires_explicit_completion_and_never_wraps() {
-        let node = NodeAdmission::new(NonZeroU16::new(2).expect("two slots"));
-        node.capacity.set_limit(2);
-        drop(node.acquire().await);
-        assert_eq!(node.completed(), Some(0), "cancellation is not progress");
-        node.acquire().await.complete();
-        assert_eq!(node.completed(), Some(1));
-        node.completed.store(u64::MAX - 1, Ordering::Release);
-        node.acquire().await.complete();
-        node.acquire().await.complete();
-        assert_eq!(node.completed(), None, "exhausted progress stays unknown");
-        assert_eq!(node.snapshot().active, 0);
-    }
-
-    #[tokio::test]
-    async fn cancelled_node_work_does_not_reserve_capacity_for_unrelated_native_tail() {
-        let node = NodeAdmission::new(NonZeroU16::new(1).expect("one slot"));
-        node.capacity.set_limit(1);
-        let physical = AdaptiveAdmission::new(NonZeroU16::new(1).expect("one disk unit"), 1);
-        let scheduling = node.acquire().await;
-        let native_tail = physical.acquire().await;
-        drop(scheduling);
-        assert_eq!(node.completed(), Some(0));
-        assert!(node.try_acquire().is_some(), "healthy resources can still be scheduled");
-        assert!(physical.try_acquire().is_none(), "the stalled disk remains charged");
-        drop(native_tail);
-        assert!(physical.try_acquire().is_some());
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn node_feedback_expires_without_revoking_held_work_or_fabricating_progress() {
-        let node = NodeAdmission::new(NonZeroU16::new(2).expect("two slots"));
-        let mut controller = FeedbackController::new();
-        let input = FeedbackObservation {
-            generation: 1,
-            max_concurrency: 2,
-            pressure: ResourcePressure::Stable,
-            completed_units: Some(99),
-        };
-        node.update(&mut controller, Some(input));
-        let held = node.acquire().await;
-        assert_eq!(node.completed(), Some(0));
-        tokio::time::advance(Duration::from_secs(4)).await;
-        assert_eq!(node.snapshot(), AdaptiveAdmissionSnapshot { limit: 0, active: 1 });
-        assert!(node.try_acquire().is_none());
-        held.complete();
-        assert_eq!(node.completed(), Some(1));
-        assert_eq!(node.snapshot().active, 0);
-    }
-
-    #[tokio::test]
-    async fn shared_unit_stays_admitted_until_native_worker_releases_its_last_clone() {
-        let gate = AdaptiveAdmission::new(NonZeroU16::new(1).expect("one unit"), 1);
-        let outer = gate.acquire().await;
-        let native = outer.clone();
-        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
-        let (release_tx, release_rx) = tokio::sync::oneshot::channel();
-        let worker = tokio::task::spawn_blocking(move || {
-            started_tx.send(()).expect("worker started");
-            release_rx.blocking_recv().expect("release native worker");
-            drop(native);
-        });
-        started_rx.await.expect("native worker is live");
-        drop(outer);
-        assert_eq!(gate.snapshot().active, 1);
-        assert!(gate.try_acquire().is_none());
-        gate.set_limit(0);
-        assert_eq!(gate.snapshot().active, 1);
-        release_tx.send(()).expect("release tail");
-        worker.await.expect("native tail completed");
-        assert_eq!(gate.snapshot().active, 0);
-        gate.set_limit(1);
-        assert!(gate.try_acquire().is_some());
-    }
-
-    #[tokio::test]
-    async fn bounded_slots_cross_word_boundaries_and_recycle_only_after_final_drop() {
-        let maximum = NonZeroU16::new(65).expect("partial second chunk");
-        let gate = AdaptiveAdmission::new(maximum, u32::MAX);
-        assert_eq!(gate.snapshot().limit, 65);
-        for _ in 0..4 {
-            let units: Vec<_> = (0..65).map(|_| gate.try_acquire().expect("free slot")).collect();
-            let tails = units.clone();
-            assert_eq!(gate.snapshot().active, 65);
-            assert!(gate.try_acquire().is_none());
-            drop(units);
-            assert_eq!(gate.snapshot().active, 65);
-            gate.set_limit(0);
-            for (remaining, tail) in (0..65).rev().zip(tails) {
-                drop(tail);
-                assert_eq!(gate.snapshot().active, remaining);
-            }
-            gate.set_limit(u32::MAX);
-            assert_eq!(gate.snapshot().limit, 65);
-        }
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-    async fn shared_units_survive_concurrent_clone_drop_reuse_and_resize() {
-        let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded owner"), 8);
-        let mut workers = Vec::new();
-        for _ in 0..16 {
-            let gate = gate.clone();
-            workers.push(tokio::spawn(async move {
-                for _ in 0..64 {
-                    let root = gate.acquire().await;
-                    let one = root.clone();
-                    let two = one.clone();
-                    drop(root);
-                    tokio::task::yield_now().await;
-                    drop(two);
-                    tokio::task::yield_now().await;
-                    drop(one);
-                }
-            }));
-        }
-        for limit in [0, 1, 8, 2, 0, 8] {
-            gate.set_limit(limit);
-            tokio::task::yield_now().await;
-        }
-        tokio::time::timeout(Duration::from_secs(5), async {
-            for worker in workers {
-                worker.await.expect("lease worker");
-            }
-        })
-        .await
-        .expect("all lease workers complete");
-        assert_eq!(gate.snapshot(), AdaptiveAdmissionSnapshot { limit: 8, active: 0 });
-        let full: Vec<_> = (0..8).map(|_| gate.try_acquire().expect("all slots returned")).collect();
-        assert!(gate.try_acquire().is_none());
-        drop(full);
-    }
-
-    #[tokio::test]
-    async fn queued_work_cannot_be_starved_by_new_fast_acquisitions() {
-        use std::task::{Context, Poll, Waker};
-        let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 1);
-        let held = gate.acquire().await;
-        let mut first = Box::pin(gate.acquire());
-        let mut second = Box::pin(gate.acquire());
-        let mut context = Context::from_waker(Waker::noop());
-        assert!(first.as_mut().poll(&mut context).is_pending());
-        assert!(second.as_mut().poll(&mut context).is_pending());
-        drop(held);
-        assert!(gate.try_acquire().is_none(), "new work must not steal a parked worker's turn");
-        assert!(second.as_mut().poll(&mut context).is_pending(), "later work cannot pass the queue head");
-        let Poll::Ready(first_permit) = first.as_mut().poll(&mut context) else {
-            panic!("queue head must run")
-        };
-        drop(first_permit);
-        let Poll::Ready(second_permit) = second.as_mut().poll(&mut context) else {
-            panic!("second waiter must run")
-        };
-        drop(second_permit);
-        assert!(gate.try_acquire().is_some(), "drained queue restores the atomic fast path");
-    }
-
-    #[tokio::test]
-    async fn cancelling_queued_work_preserves_fifo_capacity_and_fast_path() {
-        use std::task::{Context, Poll, Waker};
-        let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 0);
-        let mut head = Box::pin(gate.acquire());
-        let mut middle = Box::pin(gate.acquire());
-        let mut tail = Box::pin(gate.acquire());
-        let mut context = Context::from_waker(Waker::noop());
-        assert!(head.as_mut().poll(&mut context).is_pending());
-        assert!(middle.as_mut().poll(&mut context).is_pending());
-        assert!(tail.as_mut().poll(&mut context).is_pending());
-        drop(middle);
-        drop(head);
-        gate.set_limit(1);
-        assert!(
-            gate.try_acquire().is_none(),
-            "cancelled head must not expose the surviving waiter's capacity"
-        );
-        let Poll::Ready(permit) = tail.as_mut().poll(&mut context) else {
-            panic!("surviving waiter must run")
-        };
-        drop(permit);
-        assert!(gate.try_acquire().is_some());
-
-        gate.set_limit(0);
-        let mut cancelled = Box::pin(gate.acquire());
-        assert!(cancelled.as_mut().poll(&mut context).is_pending());
-        drop(cancelled);
-        gate.set_limit(1);
-        assert!(
-            gate.try_acquire().is_some(),
-            "cancelling the final waiter must not strand an admission turn"
-        );
-    }
-
-    #[tokio::test]
-    async fn cancelling_a_notified_head_hands_capacity_to_the_next_waiter() {
-        use std::task::{Context, Poll, Waker};
-        let gate = AdaptiveAdmission::new(NonZeroU16::new(2).expect("two units"), 0);
-        let mut head = Box::pin(gate.acquire());
-        let mut next = Box::pin(gate.acquire());
-        let mut context = Context::from_waker(Waker::noop());
-        assert!(head.as_mut().poll(&mut context).is_pending());
-        assert!(next.as_mut().poll(&mut context).is_pending());
-        gate.set_limit(1);
-        // Cancel after notification, before the head consumes its capacity.
-        drop(head);
-        let Poll::Ready(permit) = next.as_mut().poll(&mut context) else {
-            panic!("successor must observe capacity without another notification")
-        };
-        assert_eq!(gate.snapshot(), AdaptiveAdmissionSnapshot { limit: 1, active: 1 });
-        drop(permit);
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn one_resize_serves_multiple_fifo_turns_but_stored_wakes_cannot_bypass_expiry() {
-        use std::task::{Context, Poll, Waker};
-        let gate = AdaptiveAdmission::new(NonZeroU16::new(2).expect("two units"), 0);
-        let mut first = Box::pin(gate.acquire());
-        let mut second = Box::pin(gate.acquire());
-        let mut context = Context::from_waker(Waker::noop());
-        assert!(first.as_mut().poll(&mut context).is_pending());
-        assert!(second.as_mut().poll(&mut context).is_pending());
-        gate.set_sampled_limit(2, Duration::from_secs(1));
-        let Poll::Ready(one) = first.as_mut().poll(&mut context) else {
-            panic!("first turn must observe resize")
-        };
-        let Poll::Ready(two) = second.as_mut().poll(&mut context) else {
-            panic!("second turn must observe resize without releasing the first unit")
-        };
-        tokio::time::advance(Duration::from_secs(1)).await;
-        drop(one);
-        drop(two);
-        let mut expired = Box::pin(gate.acquire());
-        assert!(expired.as_mut().poll(&mut context).is_pending(), "stored wake is not a fresh lease");
-        gate.set_sampled_limit(1, Duration::from_secs(1));
-        let Poll::Ready(renewed) = expired.as_mut().poll(&mut context) else {
-            panic!("fresh observation must wake expired admission")
-        };
-        drop(renewed);
-        assert_eq!(gate.snapshot().active, 0);
-    }
-    use std::future::Future;
-    use std::task::Poll;
-
-    #[tokio::test]
-    async fn shrink_preserves_all_held_units_and_blocks_until_below_new_limit() {
-        let owner = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 3);
-        let first = owner.acquire().await;
-        let second = owner.acquire().await;
-        let third = owner.acquire().await;
-        owner.set_limit(1);
-        assert_eq!(owner.snapshot(), AdaptiveAdmissionSnapshot { limit: 1, active: 3 });
-        drop(first);
-        assert!(owner.try_acquire().is_none());
-        drop(second);
-        assert!(owner.try_acquire().is_none());
-        drop(third);
-        let fourth = owner.try_acquire().expect("last old permit returned to original owner");
-        assert_eq!(owner.snapshot(), AdaptiveAdmissionSnapshot { limit: 1, active: 1 });
-        drop(fourth);
-        assert_eq!(owner.snapshot().active, 0);
-    }
-
-    #[tokio::test]
-    async fn zero_capacity_and_cancelled_waiter_do_not_leak_or_bypass() {
-        let owner = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 0);
-        let mut waiting = Box::pin(owner.acquire());
-        std::future::poll_fn(|cx| {
-            assert!(waiting.as_mut().poll(cx).is_pending());
-            Poll::Ready(())
-        })
-        .await;
-        drop(waiting);
-        assert_eq!(owner.snapshot(), AdaptiveAdmissionSnapshot { limit: 0, active: 0 });
-        assert!(owner.try_acquire().is_none());
-        owner.set_limit(1);
-        let permit = owner.acquire().await;
-        assert!(owner.try_acquire().is_none());
-        drop(permit);
-        assert_eq!(owner.snapshot().active, 0);
-    }
-
-    #[tokio::test]
-    async fn resize_wakes_a_registered_waiter_without_an_unrelated_release() {
-        let owner = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 0);
-        let mut waiting = Box::pin(owner.acquire());
-        std::future::poll_fn(|cx| {
-            assert!(waiting.as_mut().poll(cx).is_pending());
-            Poll::Ready(())
-        })
-        .await;
-        owner.set_limit(1);
-        let permit = tokio::time::timeout(std::time::Duration::from_secs(1), waiting)
-            .await
-            .expect("resize notification");
-        assert_eq!(owner.snapshot().active, 1);
-        drop(permit);
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-    async fn concurrent_resize_release_and_acquire_keep_the_owner_count_exact() {
-        let owner = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 8);
-        let mut tasks = Vec::new();
-        for _ in 0..16 {
-            let owner = owner.clone();
-            tasks.push(tokio::spawn(async move {
-                for _ in 0..100 {
-                    let permit = owner.acquire().await;
-                    tokio::task::yield_now().await;
-                    drop(permit);
-                }
-            }));
-        }
-        for limit in [1, 0, 4, 2, 8, 0, u32::MAX, 1] {
-            owner.set_limit(limit);
-            tokio::task::yield_now().await;
-        }
-        owner.set_limit(8);
-        tokio::time::timeout(std::time::Duration::from_secs(5), async {
-            for task in tasks {
-                task.await.expect("admission task");
-            }
-        })
-        .await
-        .expect("all waiters make progress");
-        assert_eq!(owner.snapshot(), AdaptiveAdmissionSnapshot { limit: 8, active: 0 });
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn expired_observation_blocks_new_work_without_revoking_held_permits() {
-        let owner = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 0);
-        owner.set_sampled_limit(2, Duration::from_secs(2));
-        let held = owner.acquire().await;
-        tokio::time::advance(Duration::from_secs(2)).await;
-        assert!(owner.try_acquire().is_none());
-        assert_eq!(owner.snapshot(), AdaptiveAdmissionSnapshot { limit: 0, active: 1 });
-        drop(held);
-        assert_eq!(owner.snapshot().active, 0);
-        owner.set_sampled_limit(1, Duration::from_secs(2));
-        let fresh = owner.try_acquire().expect("fresh observation reopens same owner");
-        owner.set_sampled_limit(8, Duration::ZERO);
-        assert!(owner.try_acquire().is_none());
-        drop(fresh);
-        assert_eq!(owner.snapshot().active, 0);
-    }
-
-    fn healthy(completed: u64) -> FeedbackObservation {
-        FeedbackObservation {
-            generation: 1,
-            max_concurrency: 8,
-            pressure: ResourcePressure::Stable,
-            completed_units: Some(completed),
-        }
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn controller_grows_one_unit_per_cooldown_and_cuts_pressure_immediately() {
-        let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 0);
-        let mut controller = FeedbackController::new();
-        assert_eq!(controller.update(&gate, Some(healthy(0))).limit, 1);
-        for completed in 1..=25 {
-            tokio::time::advance(Duration::from_secs(1)).await;
-            let limit = controller.update(&gate, Some(healthy(completed))).limit;
-            assert_eq!(
-                limit,
-                if completed < 9 {
-                    1
-                } else if completed < 17 {
-                    2
-                } else if completed < 25 {
-                    3
-                } else {
-                    4
-                }
-            );
-        }
-        let held = gate.try_acquire().expect("existing work");
-        let mut pressure = healthy(26);
-        pressure.pressure = ResourcePressure::Busy;
-        assert_eq!(
-            controller.update(&gate, Some(pressure)),
-            FeedbackDecision {
-                limit: 2,
-                reason: FeedbackReason::Pressure
-            }
-        );
-        pressure.max_concurrency = 0;
-        assert_eq!(controller.update(&gate, Some(pressure)).limit, 0);
-        assert_eq!(gate.snapshot().active, 1);
-        assert!(gate.try_acquire().is_none());
-        drop(held);
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn controller_missing_low_sample_stalled_or_changed_sources_never_expand() {
-        for kind in 0..5 {
-            let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 0);
-            let mut controller = FeedbackController::new();
-            for tick in 0..32 {
-                let mut observation = healthy(tick);
-                match kind {
-                    0 => observation.pressure = ResourcePressure::Unknown,
-                    1 => observation.completed_units = None,
-                    2 => observation.completed_units = Some(0),
-                    3 => observation.generation = tick,
-                    _ => {}
-                }
-                tokio::time::advance(Duration::from_secs(if kind == 4 { 3 } else { 1 })).await;
-                assert!(controller.update(&gate, Some(observation)).limit <= 1, "kind {kind}");
-            }
-            assert_eq!(
-                controller.update(&gate, None),
-                FeedbackDecision {
-                    limit: 0,
-                    reason: FeedbackReason::Unavailable
-                }
-            );
-            assert!(gate.try_acquire().is_none());
-        }
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn controller_resumes_after_zero_cap_without_requiring_impossible_progress() {
-        let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 0);
-        let mut controller = FeedbackController::new();
-        let mut sample = healthy(0);
-        sample.max_concurrency = 0;
-        assert_eq!(controller.update(&gate, Some(sample)).limit, 0);
-        tokio::time::advance(Duration::from_secs(1)).await;
-        sample.max_concurrency = 8;
-        assert_eq!(controller.update(&gate, Some(sample)).limit, 1);
-        assert!(gate.try_acquire().is_some(), "minimum progress restarts inside the restored cap");
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn controller_applies_pressure_reduction_and_lower_cap_in_the_same_window() {
-        let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 0);
-        let mut controller = FeedbackController::new();
-        for completed in 0..=25 {
-            controller.update(&gate, Some(healthy(completed)));
-            tokio::time::advance(Duration::from_secs(1)).await;
-        }
-        assert_eq!(gate.snapshot().limit, 4);
-        let mut sample = healthy(26);
-        sample.max_concurrency = 3;
-        sample.pressure = ResourcePressure::Busy;
-        assert_eq!(controller.update(&gate, Some(sample)).limit, 2);
-    }
-}

crates/concurrency/src/foreground.rs

@@ -1,568 +0,0 @@
-// Copyright 2026 RustFS Team
-// Licensed under the Apache License, Version 2.0.
-
-//! Bounded foreground completion windows for background resource feedback.
-//! Recording uses fixed atomic storage, without an allocation, mutex or task
-//! per request. A single reader rotates banks and waits for the retired bank's
-//! writers before reading it. Delayed writers delay publication, not the
-//! recorded window boundary. Percentiles are intervals, never exact values.
-
-use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
-use std::time::{Duration, Instant};
-
-const SUB_BUCKETS: usize = 64;
-const FINITE_BUCKETS: usize = 27 * SUB_BUCKETS;
-const BUCKETS: usize = FINITE_BUCKETS + 1;
-const MAX_MICROS: u64 = u32::MAX as u64;
-
-/// Bounded foreground categories; no bucket, object or request identifiers.
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub enum ForegroundClass {
-    /// Object body reads.
-    Read,
-    /// Object writes, including multipart and copy operations.
-    Write,
-    /// Metadata and listing operations.
-    Metadata,
-    /// Other operations, including rejection before typed dispatch.
-    Other,
-}
-
-impl ForegroundClass {
-    /// Stable order shared by window summaries.
-    pub const ALL: [Self; 4] = [Self::Read, Self::Write, Self::Metadata, Self::Other];
-
-    /// Fixed metric label.
-    pub const fn as_str(self) -> &'static str {
-        match self {
-            Self::Read => "read",
-            Self::Write => "write",
-            Self::Metadata => "metadata",
-            Self::Other => "other",
-        }
-    }
-
-    const fn index(self) -> usize {
-        match self {
-            Self::Read => 0,
-            Self::Write => 1,
-            Self::Metadata => 2,
-            Self::Other => 3,
-        }
-    }
-}
-
-/// Outcome at the observed response-body boundary, not a client ACK.
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub enum CompletionOutcome {
-    /// Successful response body completed.
-    Success,
-    /// Error response, service error, or body-stream failure.
-    Failed,
-    /// The request/body was dropped before completion.
-    Cancelled,
-}
-
-/// Inclusive bounds containing the observed percentile.
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub struct LatencyBounds {
-    /// Inclusive lower bound in nanoseconds.
-    pub lower_nanos: u64,
-    /// Inclusive upper bound; absent for the overflow bucket.
-    pub upper_nanos: Option<u64>,
-}
-
-/// One class in a complete observation window.
-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
-pub struct ClassCompletions {
-    /// All finishes, including errors and cancellation.
-    pub finished: u64,
-    /// Error responses, service errors and body-stream failures.
-    pub failed: u64,
-    /// Censored observations; these must not be treated as successful latency.
-    pub cancelled: u64,
-    /// P99 of non-cancelled finishes, when at least 100 samples exist.
-    /// This count permits computing the rank; it is not statistical confidence.
-    pub p99: Option<LatencyBounds>,
-}
-
-/// Clock readings bracketing an atomic event-window cut. A scheduler pause
-/// between these readings is visible, rather than disguised as precise timing.
-#[derive(Clone, Copy, Debug)]
-pub struct WindowBoundary {
-    /// Read immediately before the event cut.
-    pub earliest: Instant,
-    /// Read immediately after the event cut.
-    pub latest: Instant,
-}
-
-/// Complete windows retain their actual cut brackets, even if publication waits.
-#[derive(Clone, Copy, Debug)]
-pub struct ForegroundWindow {
-    /// Start of this event window.
-    pub start: WindowBoundary,
-    /// Cut of the retired bank, possibly older than publication time.
-    pub end: WindowBoundary,
-    /// Request starts in this window, including pre-dispatch rejection.
-    pub started: Option<u64>,
-    /// Requests still alive at the cut; absent after accounting overflow.
-    pub in_flight: Option<u64>,
-    /// Per-class completions in ForegroundClass::ALL order.
-    pub classes: [Option<ClassCompletions>; 4],
-    /// False after a counter overflow or an inconsistent event count.
-    pub valid: bool,
-}
-
-struct ClassBank {
-    latencies: [AtomicU64; BUCKETS],
-    failed: AtomicU64,
-    cancelled: AtomicU64,
-}
-
-impl ClassBank {
-    fn new() -> Self {
-        Self {
-            latencies: std::array::from_fn(|_| AtomicU64::new(0)),
-            failed: AtomicU64::new(0),
-            cancelled: AtomicU64::new(0),
-        }
-    }
-}
-
-struct Bank {
-    writers: AtomicUsize,
-    started: AtomicU64,
-    overflowed: AtomicBool,
-    classes: [ClassBank; 4],
-}
-
-impl Bank {
-    fn new() -> Self {
-        Self {
-            writers: AtomicUsize::new(0),
-            started: AtomicU64::new(0),
-            overflowed: AtomicBool::new(false),
-            classes: std::array::from_fn(|_| ClassBank::new()),
-        }
-    }
-
-    fn increment(&self, counter: &AtomicU64) {
-        if counter.fetch_add(1, Ordering::Relaxed) == u64::MAX {
-            self.overflowed.store(true, Ordering::Relaxed);
-        }
-    }
-}
-
-struct Writer<'a>(&'a Bank);
-
-impl Drop for Writer<'_> {
-    fn drop(&mut self) {
-        self.0.writers.fetch_sub(1, Ordering::SeqCst);
-    }
-}
-
-/// One process-wide recorder. Its fixed storage is allocated with the owner.
-/// A request holds only a reference, timestamps and bounded enum state.
-pub struct ForegroundRecorder {
-    created_at: Instant,
-    epoch: AtomicU64,
-    reader_taken: AtomicBool,
-    banks: [Bank; 2],
-}
-
-impl Default for ForegroundRecorder {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl ForegroundRecorder {
-    /// Construct empty accounting for one lifetime of the owning process.
-    pub fn new() -> Self {
-        Self {
-            created_at: Instant::now(),
-            epoch: AtomicU64::new(0),
-            reader_taken: AtomicBool::new(false),
-            banks: std::array::from_fn(|_| Bank::new()),
-        }
-    }
-
-    fn writer(&self) -> Writer<'_> {
-        loop {
-            let epoch = self.epoch.load(Ordering::SeqCst);
-            let bank = &self.banks[usize::from(epoch & 1 != 0)];
-            bank.writers.fetch_add(1, Ordering::SeqCst);
-            let writer = Writer(bank);
-            // Sequential ordering joins registration, the reader's cut, and
-            // its zero-writer check. A late old-epoch registrant cannot write
-            // into a bank that the reader has begun draining or reusing.
-            if self.epoch.load(Ordering::SeqCst) == epoch {
-                return writer;
-            }
-        }
-    }
-
-    /// Begin at the external service-call boundary. Drop records cancellation
-    /// unless finish has already recorded a response-body outcome.
-    pub fn start(&self) -> ForegroundObservation<'_> {
-        let started_at = Instant::now();
-        let writer = self.writer();
-        writer.0.increment(&writer.0.started);
-        ForegroundObservation {
-            recorder: self,
-            started_at: Some(started_at),
-            class: ForegroundClass::Other,
-        }
-    }
-
-    fn finish(&self, class: ForegroundClass, elapsed: Duration, outcome: CompletionOutcome) {
-        let writer = self.writer();
-        let bank = writer.0;
-        let class = &bank.classes[class.index()];
-        if outcome == CompletionOutcome::Cancelled {
-            bank.increment(&class.cancelled);
-            return;
-        }
-        let bucket = latency_bucket(elapsed);
-        bank.increment(&class.latencies[bucket]);
-        if outcome == CompletionOutcome::Failed {
-            bank.increment(&class.failed);
-        }
-    }
-
-    /// Claim the sole reader. It cannot be replaced after being dropped:
-    /// resetting its carried in-flight count would falsify later windows.
-    pub fn take_reader(&self) -> Option<ForegroundReader<'_>> {
-        self.reader_taken
-            .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
-            .ok()?;
-        Some(ForegroundReader {
-            recorder: self,
-            start: WindowBoundary {
-                earliest: self.created_at,
-                latest: self.created_at,
-            },
-            retired: None,
-            in_flight: Some(0),
-        })
-    }
-}
-
-/// Non-cloneable request ownership, moved from the service future to its body.
-pub struct ForegroundObservation<'a> {
-    recorder: &'a ForegroundRecorder,
-    started_at: Option<Instant>,
-    class: ForegroundClass,
-}
-
-impl ForegroundObservation<'_> {
-    /// Set the first dispatched operation's bounded category.
-    pub fn set_class(&mut self, class: ForegroundClass) {
-        self.class = class;
-    }
-
-    /// Finish once. Later calls or Drop do not produce another observation.
-    pub fn finish(&mut self, outcome: CompletionOutcome) {
-        if let Some(started_at) = self.started_at.take() {
-            self.recorder.finish(self.class, started_at.elapsed(), outcome);
-        }
-    }
-}
-
-impl Drop for ForegroundObservation<'_> {
-    fn drop(&mut self) {
-        self.finish(CompletionOutcome::Cancelled);
-    }
-}
-
-/// Single-reader state; never acquired by a request path.
-pub struct ForegroundReader<'a> {
-    recorder: &'a ForegroundRecorder,
-    start: WindowBoundary,
-    retired: Option<(usize, WindowBoundary)>,
-    in_flight: Option<u64>,
-}
-
-impl ForegroundReader<'_> {
-    /// Cut a new window, or finish publishing an earlier cut. A live retired
-    /// writer returns None without blocking, discarding data or moving its cut.
-    pub fn sample(&mut self) -> Option<ForegroundWindow> {
-        if self.retired.is_none() {
-            let epoch = self.recorder.epoch.load(Ordering::SeqCst);
-            let next = epoch.checked_add(1)?;
-            let earliest = Instant::now();
-            self.recorder.epoch.store(next, Ordering::SeqCst);
-            let cut = WindowBoundary {
-                earliest,
-                latest: Instant::now(),
-            };
-            self.retired = Some((usize::from(epoch & 1 != 0), cut));
-        }
-        let (index, end) = self.retired?;
-        let bank = &self.recorder.banks[index];
-        if bank.writers.load(Ordering::SeqCst) != 0 {
-            return None;
-        }
-
-        let started = bank.started.swap(0, Ordering::Relaxed);
-        let overflowed = bank.overflowed.swap(false, Ordering::Relaxed);
-        let classes = std::array::from_fn(|index| {
-            let class = &bank.classes[index];
-            let failed = class.failed.swap(0, Ordering::Relaxed);
-            let cancelled = class.cancelled.swap(0, Ordering::Relaxed);
-            let buckets = std::array::from_fn::<_, BUCKETS, _>(|i| class.latencies[i].swap(0, Ordering::Relaxed));
-            let non_cancelled = buckets.iter().try_fold(0_u64, |sum, count| sum.checked_add(*count));
-            let finished = non_cancelled.and_then(|total| total.checked_add(cancelled));
-            let finished = finished.filter(|_| !overflowed && non_cancelled.is_some_and(|total| failed <= total))?;
-            Some(ClassCompletions {
-                finished,
-                failed,
-                cancelled,
-                p99: non_cancelled
-                    .filter(|total| *total >= 100)
-                    .and_then(|total| percentile_99(&buckets, total)),
-            })
-        });
-        let started = (!overflowed).then_some(started);
-        let finished = classes
-            .iter()
-            .try_fold(0_u64, |sum, class| sum.checked_add(class.as_ref()?.finished));
-        self.in_flight = self
-            .in_flight
-            .zip(finished)
-            .and_then(|(active, finished)| active.checked_add(started?)?.checked_sub(finished));
-        let window = ForegroundWindow {
-            start: self.start,
-            end,
-            started,
-            in_flight: self.in_flight,
-            classes,
-            valid: self.in_flight.is_some(),
-        };
-        self.start = end;
-        self.retired = None;
-        Some(window)
-    }
-}
-
-fn latency_bucket(elapsed: Duration) -> usize {
-    let micros = elapsed.as_nanos().div_ceil(1_000);
-    let Ok(micros) = u64::try_from(micros) else {
-        return FINITE_BUCKETS;
-    };
-    if micros > MAX_MICROS {
-        return FINITE_BUCKETS;
-    }
-    if micros < SUB_BUCKETS as u64 {
-        return usize::try_from(micros).unwrap_or(FINITE_BUCKETS);
-    }
-    let shift = 63 - micros.leading_zeros() - 6;
-    usize::try_from((u64::from(shift) + 1) * SUB_BUCKETS as u64 + (micros >> shift) - SUB_BUCKETS as u64)
-        .unwrap_or(FINITE_BUCKETS)
-}
-
-fn bucket_bounds(index: usize) -> LatencyBounds {
-    if index == FINITE_BUCKETS {
-        return LatencyBounds {
-            lower_nanos: MAX_MICROS * 1_000 + 1,
-            upper_nanos: None,
-        };
-    }
-    let (lower, upper) = if index < SUB_BUCKETS {
-        (index as u64, index as u64)
-    } else {
-        let shift = index / SUB_BUCKETS - 1;
-        let lower = (SUB_BUCKETS + index % SUB_BUCKETS) as u64 * (1_u64 << shift);
-        (lower, lower + (1_u64 << shift) - 1)
-    };
-    LatencyBounds {
-        lower_nanos: lower.saturating_sub(1) * 1_000 + u64::from(lower != 0),
-        upper_nanos: Some(upper * 1_000),
-    }
-}
-
-fn percentile_99(buckets: &[u64; BUCKETS], total: u64) -> Option<LatencyBounds> {
-    let rank = (u128::from(total) * 99).div_ceil(100);
-    let mut cumulative = 0_u128;
-    for (index, count) in buckets.iter().enumerate() {
-        cumulative += u128::from(*count);
-        if cumulative >= rank {
-            return Some(bucket_bounds(index));
-        }
-    }
-    None
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::sync::Arc;
-
-    #[test]
-    fn percentile_intervals_contain_each_bucket_edge_and_neighbour() {
-        for index in 0..BUCKETS {
-            let bounds = bucket_bounds(index);
-            assert_eq!(latency_bucket(Duration::from_nanos(bounds.lower_nanos)), index);
-            if let Some(upper) = bounds.upper_nanos {
-                assert!(upper >= bounds.lower_nanos);
-                assert_eq!(latency_bucket(Duration::from_nanos(upper)), index);
-                assert_eq!(latency_bucket(Duration::from_nanos(upper + 1)), index + 1);
-            }
-        }
-        assert_eq!(latency_bucket(Duration::MAX), FINITE_BUCKETS);
-    }
-
-    #[test]
-    fn body_ownership_carries_in_flight_counts_across_windows() {
-        let recorder = ForegroundRecorder::new();
-        let mut reader = recorder.take_reader().expect("first reader");
-        let mut request = recorder.start();
-        request.set_class(ForegroundClass::Read);
-        let first = reader.sample().expect("cut with a live request");
-        assert!(first.valid);
-        assert_eq!(first.started, Some(1));
-        assert_eq!(first.in_flight, Some(1));
-        assert_eq!(first.classes[0].expect("valid read count").finished, 0);
-
-        request.finish(CompletionOutcome::Success);
-        request.finish(CompletionOutcome::Failed);
-        drop(request);
-        let second = reader.sample().expect("body finished");
-        assert!(second.valid);
-        assert_eq!(second.started, Some(0));
-        assert_eq!(second.in_flight, Some(0));
-        let read = second.classes[0].expect("valid read summary");
-        assert_eq!((read.finished, read.failed, read.cancelled, read.p99), (1, 0, 0, None));
-        assert_eq!(first.end.earliest, second.start.earliest);
-        assert_eq!(first.end.latest, second.start.latest);
-    }
-
-    #[test]
-    fn cancellations_are_counted_without_fabricating_success_latencies() {
-        let recorder = ForegroundRecorder::new();
-        let mut reader = recorder.take_reader().expect("first reader");
-        for _ in 0..150 {
-            let mut request = recorder.start();
-            request.set_class(ForegroundClass::Write);
-        }
-        let window = reader.sample().expect("cancelled requests");
-        let write = window.classes[1].expect("known write count");
-        assert!(window.valid);
-        assert_eq!(window.in_flight, Some(0));
-        assert_eq!((write.finished, write.cancelled, write.failed, write.p99), (150, 150, 0, None));
-    }
-
-    #[test]
-    fn delayed_retired_writer_preserves_the_original_cut_and_next_bank() {
-        let recorder = ForegroundRecorder::new();
-        let mut reader = recorder.take_reader().expect("first reader");
-        let writer = recorder.writer();
-        assert!(reader.sample().is_none(), "never read a live writer's bank");
-        let original_cut = reader.retired.expect("pending cut").1;
-        let request = recorder.start(); // a start after the cut belongs to the new bank
-        writer.0.increment(&writer.0.started);
-        drop(writer);
-        let previous = reader.sample().expect("retired writer drained");
-        assert_eq!(previous.end.earliest, original_cut.earliest);
-        assert_eq!(previous.end.latest, original_cut.latest);
-        assert_eq!(previous.started, Some(1));
-        assert_eq!(previous.in_flight, Some(1));
-        drop(request);
-        let current = reader.sample().expect("next bank");
-        assert_eq!(current.started, Some(1));
-        assert_eq!(current.in_flight, Some(1));
-        assert_eq!(current.classes[3].expect("unclassified cancellation").cancelled, 1);
-    }
-
-    #[test]
-    fn completed_percentile_uses_rank_and_never_turns_maximum_into_p99() {
-        let recorder = ForegroundRecorder::new();
-        let mut reader = recorder.take_reader().expect("first reader");
-        for i in 0..100 {
-            let writer = recorder.writer();
-            writer.0.increment(&writer.0.started);
-            drop(writer);
-            recorder.finish(
-                ForegroundClass::Read,
-                if i == 99 {
-                    Duration::from_secs(60)
-                } else {
-                    Duration::from_millis(2)
-                },
-                if i == 99 {
-                    CompletionOutcome::Failed
-                } else {
-                    CompletionOutcome::Success
-                },
-            );
-        }
-        let window = reader.sample().expect("complete window");
-        let read = window.classes[0].expect("read summary");
-        assert_eq!((read.finished, read.failed, read.cancelled), (100, 1, 0));
-        let p99 = read.p99.expect("100 non-cancelled samples");
-        assert!(p99.lower_nanos <= 2_000_000);
-        assert!(p99.upper_nanos.is_some_and(|upper| (2_000_000..3_000_000).contains(&upper)));
-    }
-
-    #[test]
-    fn overflow_stays_unknown_and_reader_cannot_reset_event_accounting() {
-        let recorder = ForegroundRecorder::new();
-        {
-            let mut reader = recorder.take_reader().expect("first reader");
-            recorder.banks[0].started.store(u64::MAX, Ordering::Relaxed);
-            drop(recorder.start());
-            let invalid = reader.sample().expect("overflow window");
-            assert!(!invalid.valid);
-            assert_eq!(invalid.started, None);
-            assert_eq!(invalid.in_flight, None);
-            assert_eq!(invalid.classes, [None; 4]);
-            let later = reader.sample().expect("later empty window");
-            assert!(!later.valid, "a lost carried count cannot recover by inventing zero active work");
-        }
-        assert!(recorder.take_reader().is_none(), "reader replacement would lose live request ownership");
-    }
-
-    #[test]
-    fn concurrent_rotation_preserves_every_start_finish_and_cancellation() {
-        let recorder = Arc::new(ForegroundRecorder::new());
-        let mut reader = recorder.take_reader().expect("first reader");
-        let workers: Vec<_> = (0..4)
-            .map(|_| {
-                let recorder = Arc::clone(&recorder);
-                std::thread::spawn(move || {
-                    for index in 0..10_000 {
-                        let mut request = recorder.start();
-                        request.set_class(ForegroundClass::Read);
-                        if index % 10 != 0 {
-                            request.finish(CompletionOutcome::Success);
-                        }
-                    }
-                })
-            })
-            .collect();
-        let (mut started, mut finished, mut cancelled) = (0, 0, 0);
-        let mut add_window = |window: ForegroundWindow| {
-            assert!(window.valid, "concurrent event cuts must stay balanced");
-            started += window.started.expect("valid start count");
-            let reads = window.classes[0].expect("valid read count");
-            finished += reads.finished;
-            cancelled += reads.cancelled;
-        };
-        while workers.iter().any(|worker| !worker.is_finished()) {
-            if let Some(window) = reader.sample() {
-                add_window(window);
-            }
-            std::thread::yield_now();
-        }
-        for worker in workers {
-            worker.join().expect("writer must finish");
-        }
-        add_window(reader.sample().expect("drain after writers join"));
-        // A previously retired window may have been published by the last
-        // sample; cut once more to include the current, still open bank.
-        let final_window = reader.sample().expect("final open bank");
-        assert_eq!(final_window.in_flight, Some(0));
-        add_window(final_window);
-        assert_eq!((started, finished, cancelled), (40_000, 40_000, 4_000));
-    }
-}

crates/concurrency/src/lib.rs

@@ -36,10 +36,8 @@
 // Re-exported io-core type used by downstream timeout implementations.
 pub use rustfs_io_core::OperationProgress;
 
-pub mod adaptive;
 mod backpressure;
 mod deadlock;
-pub mod foreground;
 mod queue;
 pub mod workers;
 pub mod workload;

crates/concurrency/src/workers.rs

@@ -49,28 +49,12 @@ impl Workers {
 
     /// Acquire a worker slot, waiting until one becomes available.
     pub async fn take(&self) {
-        struct AcquireCancelGuard<'a> {
-            workers: &'a Workers,
-            completed: bool,
-        }
-        impl Drop for AcquireCancelGuard<'_> {
-            fn drop(&mut self) {
-                if !self.completed {
-                    self.workers.in_use.send_modify(|_| {});
-                }
-            }
-        }
-        let mut guard = AcquireCancelGuard {
-            workers: self,
-            completed: false,
-        };
         let permit = match self.semaphore.acquire().await {
             Ok(permit) => permit,
             // The semaphore is owned by `self` and never closed.
             Err(_) => unreachable!("worker semaphore is never closed"),
         };
         permit.forget(); // returned manually via give()
-        guard.completed = true;
         self.in_use.send_modify(|held| *held += 1);
         trace!(
             event = "worker_slot.acquire",
@@ -98,7 +82,6 @@ impl Workers {
         });
         if released {
             self.semaphore.add_permits(1);
-            self.in_use.send_modify(|_| {});
         }
         trace!(
             event = "worker_slot.release",
@@ -116,9 +99,7 @@ impl Workers {
     pub async fn wait(&self) {
         let mut rx = self.in_use.subscribe();
         // The sender is owned by `self`, so it outlives this borrow.
-        let _ = rx
-            .wait_for(|&held| held == 0 && self.semaphore.available_permits() == self.limit)
-            .await;
+        let _ = rx.wait_for(|&held| held == 0).await;
         debug!(
             event = "worker_slot.wait",
             component = "concurrency",
@@ -133,9 +114,7 @@ impl Workers {
 
     /// Return the current number of available worker slots.
     pub async fn available(&self) -> usize {
-        self.limit
-            .saturating_sub(*self.in_use.borrow())
-            .min(self.semaphore.available_permits())
+        self.limit.saturating_sub(*self.in_use.borrow())
     }
 }
 
@@ -238,36 +217,4 @@ mod tests {
         }
         assert_eq!(workers.available().await, 0);
     }
-
-    #[tokio::test]
-    async fn test_wait_does_not_complete_while_permit_is_in_transit_to_queued_taker() {
-        use std::future::Future;
-        use std::pin::pin;
-        use std::task::{Context, Poll, Waker};
-
-        let workers = Workers::new(1).unwrap();
-        workers.take().await;
-
-        let mut taker = pin!(workers.take());
-        let mut cx = Context::from_waker(Waker::noop());
-        assert!(matches!(taker.as_mut().poll(&mut cx), Poll::Pending));
-
-        let mut waiter = pin!(workers.wait());
-        assert!(matches!(waiter.as_mut().poll(&mut cx), Poll::Pending));
-
-        // Hand off the slot to the queued taker before `taker` is polled again.
-        workers.give().await;
-        assert_eq!(workers.available().await, 0);
-        assert!(
-            matches!(waiter.as_mut().poll(&mut cx), Poll::Pending),
-            "wait() must not complete while a permit is in transit to a woken taker"
-        );
-
-        assert!(matches!(taker.as_mut().poll(&mut cx), Poll::Ready(())));
-        assert!(matches!(waiter.as_mut().poll(&mut cx), Poll::Pending));
-
-        workers.give().await;
-        assert!(matches!(waiter.as_mut().poll(&mut cx), Poll::Ready(())));
-        assert_eq!(workers.available().await, 1);
-    }
 }

crates/config/README.md

@@ -116,7 +116,6 @@ when restoring `/rustfs/console` for a custom OEM build.
 - `RUSTFS_DATA_SCANNER_START_DELAY_SECS` (deprecated alias for compatibility)
 - `RUSTFS_SCANNER_IDLE_MODE` (canonical)
 - `RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS` (canonical)
-- `RUSTFS_SCANNER_RECOVERY_BACKOFF_CAP_SECS` (canonical)
 - `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` (canonical)
 - `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` (canonical)
 - `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` (canonical)
@@ -128,12 +127,6 @@ Scanner cycle budget controls:
 
 - When `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` is unset, the finite default is 1800 seconds (30 minutes), matching the scanner benchmark guidance.
 - An explicit `0` preserves the compatibility behavior of an unbounded runtime budget. Object and directory budgets likewise remain unbounded when explicitly set to `0`.
-
-Scanner recovery retry controls:
-
-- `RUSTFS_SCANNER_RECOVERY_BACKOFF_CAP_SECS` caps the wait between automatic startup recovery retries. The persistent admin key is `scanner.recovery_backoff_cap`; the environment takes precedence over the persistent value, and the unset default remains 600 seconds (10 minutes). Values must be positive integer seconds within the platform timer range; `0`, malformed values, and overflow are rejected. A valid environment override does not hide an invalid persistent value from `admin config set` validation.
-- The cap bounds the existing exponential retry schedule. Updating the persistent value applies to the next computed retry; it does not reschedule a wait already in progress. Retryable observations continue retrying after the short retry budget. Admin scanner status reports the effective value and source at `runtime_config.recovery_backoff_cap_seconds`.
-
 Scanner walker controls:
 
 - `RUSTFS_SCANNER_WALKERS_PER_DISK` sets how many bucket walkers each local disk of an erasure set runs; the default `1` walks one bucket per disk at a time, which is the right value for rotational media. On flash, `2` to `4` lets one disk walk several buckets concurrently, which matters most for single-disk deployments with many buckets, where the walk is otherwise fully serial. Values outside `1` through `16` are rejected by `admin config set`; an out-of-range or non-numeric environment value is logged as a scanner runtime config error (`state = "startup_apply_failed"`) and the scanner keeps the default. The walkers of one disk share that disk's scanner admission, so peers still see one scan session per physical disk, and `RUSTFS_SCANNER_MAX_CONCURRENT_DISK_SCANS` still caps the total.

crates/config/src/constants/internode.rs

@@ -276,9 +276,6 @@ pub const DEFAULT_INTERNODE_BULK_CHANNELS: usize = 2;
 pub const ENV_INTERNODE_HTTP_TUNING_PROFILE: &str = "RUSTFS_INTERNODE_HTTP_TUNING_PROFILE";
 pub const DEFAULT_INTERNODE_HTTP_TUNING_PROFILE: &str = "legacy";
 
-/// Internode HTTP data-plane protocol: `http1` (default) or `auto` (TLS ALPN).
-pub const ENV_INTERNODE_HTTP_PROTOCOL: &str = "RUSTFS_INTERNODE_HTTP_PROTOCOL";
-
 /// Internode HTTP connection pool maximum idle connections per host.
 pub const ENV_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST: &str = "RUSTFS_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST";
 
@@ -290,9 +287,8 @@ pub const ENV_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_HT
 /// NOTE (backlog#805-C3): all `RUSTFS_INTERNODE_HTTP2_*` tuning (window sizes,
 /// adaptive window, keepalive timeout) only takes effect when the internode
 /// connection actually negotiates HTTP/2, which happens via TLS-ALPN. Over
-/// plaintext internode transport or with the default `http1` protocol policy,
-/// these settings are inert. Select `RUSTFS_INTERNODE_HTTP_PROTOCOL=auto` and
-/// enable internode TLS to use them.
+/// plaintext internode transport the connection is HTTP/1.1 and these settings
+/// are silently inert — enable internode TLS to use them.
 pub const ENV_INTERNODE_HTTP2_INITIAL_STREAM_WINDOW_SIZE: &str = "RUSTFS_INTERNODE_HTTP2_INITIAL_STREAM_WINDOW_SIZE";
 
 /// Internode HTTP/2 initial connection window size in bytes.

crates/config/src/constants/scanner.rs

@@ -53,9 +53,6 @@ pub const SCANNER_IDLE_MODE: &str = "idle_mode";
 /// Scanner config key controlling scanner cache save timeout in seconds.
 pub const SCANNER_CACHE_SAVE_TIMEOUT: &str = "cache_save_timeout";
 
-/// Scanner config key capping automatic recovery retry waits in seconds.
-pub const SCANNER_RECOVERY_BACKOFF_CAP: &str = "recovery_backoff_cap";
-
 /// Scanner config key capping concurrent scanner set tasks.
 pub const SCANNER_MAX_CONCURRENT_SET_SCANS: &str = "max_concurrent_set_scans";
 
@@ -96,7 +93,6 @@ pub const SCANNER_KEYS: &[&str] = &[
     SCANNER_BITROT_CYCLE,
     SCANNER_IDLE_MODE,
     SCANNER_CACHE_SAVE_TIMEOUT,
-    SCANNER_RECOVERY_BACKOFF_CAP,
     SCANNER_MAX_CONCURRENT_SET_SCANS,
     SCANNER_MAX_CONCURRENT_DISK_SCANS,
     SCANNER_WALKERS_PER_DISK,
@@ -220,14 +216,6 @@ pub const ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS: &str = "RUSTFS_SCANNER_CACHE_SAVE
 /// Default scanner cache save timeout in seconds.
 pub const DEFAULT_SCANNER_CACHE_SAVE_TIMEOUT_SECS: u64 = 14;
 
-/// Environment variable that caps automatic scanner recovery retry waits.
-/// The value must be a positive integer number of seconds within the timer range.
-/// - Example: `export RUSTFS_SCANNER_RECOVERY_BACKOFF_CAP_SECS=600`
-pub const ENV_SCANNER_RECOVERY_BACKOFF_CAP_SECS: &str = "RUSTFS_SCANNER_RECOVERY_BACKOFF_CAP_SECS";
-
-/// Default maximum wait between automatic scanner recovery retries.
-pub const DEFAULT_SCANNER_RECOVERY_BACKOFF_CAP_SECS: u64 = 10 * 60;
-
 /// Environment variable that caps concurrent scanner set tasks.
 /// A value of `0` keeps the existing topology-based concurrency.
 /// - Example: `export RUSTFS_SCANNER_MAX_CONCURRENT_SET_SCANS=2`

crates/e2e_test/Cargo.toml

@@ -107,7 +107,7 @@ flate2.workspace = true
 http.workspace = true
 http-body-util.workspace = true
 hyper = { workspace = true, features = ["http2", "http1", "server"] }
-hyper-util = { workspace = true, features = ["tokio", "server-auto", "server-graceful", "client-legacy", "tracing"] }
+hyper-util = { workspace = true, features = ["tokio", "server-auto", "server-graceful", "tracing"] }
 reqwest = { workspace = true, features = ["json", "multipart", "stream"] }
 rustfs-signer.workspace = true
 
@@ -147,6 +147,5 @@ clap = { workspace = true, features = ["derive", "env"] }
 
 [dev-dependencies]
 aws-smithy-http-client = { workspace = true, default-features = false, features = ["test-util"] }
-rustfs-heal.workspace = true
 tempfile.workspace = true
 tokio = { workspace = true, features = ["test-util"] }

crates/e2e_test/README.md

@@ -200,8 +200,8 @@ python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run
 python3 scripts/e2e_binary.py run --binary "$RUSTFS_E2E_STARTUP_CAS_BINARY" --features e2e-test-hooks -- cargo nextest run --profile e2e-full -p e2e_test
 python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-nightly -p e2e_test
 
-# Distributed 4-node 4-disk lane includes physical-worker fault injection and
-# scanner_heal_topology_ cases; both require the fault-test hooks server.
+# Distributed 4-node 4-disk lane uses the fault-test hooks server; the
+# scanner_heal_topology_ fault cases fail closed without it.
 # Upgrade cases require RUSTFS_UPGRADE_SOURCE_BINARY and fail closed without it.
 python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
 python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-distributed -p e2e_test

crates/e2e_test/src/common.rs

@@ -34,7 +34,7 @@ use rustfs_signer::sign_v4;
 use s3s::Body;
 use serde_json;
 use std::fs as stdfs;
-use std::io::{self, ErrorKind};
+use std::io::ErrorKind;
 use std::net::{Ipv6Addr, SocketAddr};
 use std::path::{Path, PathBuf};
 use std::process::{Child, Command, Stdio};
@@ -1272,7 +1272,6 @@ pub struct RustFSTestClusterEnvironment {
     /// Optional socket proxies used for the corresponding node's volume
     /// endpoints. Proxies must be installed before [`Self::start`].
     volume_proxy_addresses: Vec<Option<SocketAddr>>,
-    tls_ca_pem: Option<Vec<u8>>,
 }
 
 impl RustFSTestClusterEnvironment {
@@ -1313,10 +1312,7 @@ impl RustFSTestClusterEnvironment {
     pub async fn with_topology(topology: ClusterTopology) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
         topology.validate()?;
 
-        let temp_dir = std::env::temp_dir()
-            .join(format!("rustfs_cluster_test_{}", Uuid::new_v4()))
-            .to_string_lossy()
-            .into_owned();
+        let temp_dir = format!("/tmp/rustfs_cluster_test_{}", Uuid::new_v4());
         fs::create_dir_all(&temp_dir).await?;
 
         // Map every node to its owning pool index.
@@ -1378,35 +1374,9 @@ impl RustFSTestClusterEnvironment {
             node_capture_log_paths: vec![None; topology.node_count],
             topology,
             volume_proxy_addresses: vec![None; node_count],
-            tls_ca_pem: None,
         })
     }
 
-    /// Configure verified HTTPS for both volume endpoints and S3 clients.
-    /// The bundle must be generated before any node starts.
-    pub async fn enable_tls(&mut self, directory: &Path) -> std::io::Result<()> {
-        if self.nodes.iter().any(|node| node.process.is_some()) {
-            return Err(std::io::Error::new(
-                ErrorKind::InvalidInput,
-                "configure cluster TLS before starting nodes",
-            ));
-        }
-        let directory = fs::canonicalize(directory).await?;
-        let ca_pem = fs::read(directory.join("ca.crt")).await?;
-        reqwest::Certificate::from_pem(&ca_pem).map_err(|error| std::io::Error::new(ErrorKind::InvalidData, error))?;
-        self.set_env(
-            "RUSTFS_TLS_PATH",
-            directory
-                .to_str()
-                .ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, "TLS directory is not UTF-8"))?,
-        );
-        for node in &mut self.nodes {
-            node.url = format!("https://{}", node.address);
-        }
-        self.tls_ca_pem = Some(ca_pem);
-        Ok(())
-    }
-
     /// Add an extra environment variable applied to every cluster node process.
     pub fn set_env<K, V>(&mut self, key: K, value: V)
     where
@@ -1497,29 +1467,6 @@ impl RustFSTestClusterEnvironment {
         Ok(proxy)
     }
 
-    /// Route internode traffic through a method-aware HTTP proxy.
-    pub(crate) async fn start_rpc_proxy_for_node(
-        &mut self,
-        node_idx: usize,
-    ) -> io::Result<crate::rpc_fault_proxy::RpcFaultProxy> {
-        self.ensure_node_index(node_idx)
-            .map_err(|error| io::Error::new(ErrorKind::InvalidInput, error))?;
-        if self.volume_proxy_addresses[node_idx].is_some() {
-            return Err(io::Error::new(
-                ErrorKind::AlreadyExists,
-                format!("a volume proxy is already configured for node {node_idx}"),
-            ));
-        }
-        let target = self.nodes[node_idx]
-            .address
-            .parse::<SocketAddr>()
-            .map_err(|error| io::Error::new(ErrorKind::InvalidInput, error))?;
-        let listen = SocketAddr::from((Ipv6Addr::LOCALHOST, target.port()));
-        let proxy = crate::rpc_fault_proxy::RpcFaultProxy::start_on(listen, target).await?;
-        self.volume_proxy_addresses[node_idx] = Some(proxy.local_addr());
-        Ok(proxy)
-    }
-
     fn volume_address(&self, node_idx: usize) -> String {
         self.volume_proxy_addresses[node_idx]
             .map(|address| address.to_string())
@@ -1528,7 +1475,6 @@ impl RustFSTestClusterEnvironment {
 
     fn build_volumes_arg(&self) -> String {
         let pools = self.topology.normalized_pools();
-        let scheme = if self.tls_ca_pem.is_some() { "https" } else { "http" };
 
         if pools.len() <= 1 {
             // Single pool: explicit enumeration of every drive on every node.
@@ -1538,7 +1484,7 @@ impl RustFSTestClusterEnvironment {
                 .enumerate()
                 .flat_map(|(node_idx, n)| {
                     let address = self.volume_address(node_idx);
-                    n.data_dirs.iter().map(move |dir| format!("{scheme}://{address}{dir}"))
+                    n.data_dirs.iter().map(move |dir| format!("http://{}{}", address, dir))
                 })
                 .collect::<Vec<_>>()
                 .join(" ");
@@ -1558,7 +1504,7 @@ impl RustFSTestClusterEnvironment {
                     .and_then(|d| d.rsplit_once('/').map(|(parent, _)| parent))
                     .unwrap_or(&node.data_dir);
                 format!(
-                    "{scheme}://{}{}/drive{{0...{}}}",
+                    "http://{}{}/drive{{0...{}}}",
                     self.volume_address(node_idx),
                     base,
                     self.topology.drives_per_node - 1
@@ -1747,20 +1693,13 @@ impl RustFSTestClusterEnvironment {
         if node_idx >= self.nodes.len() {
             return Err("node_idx is invalid".into());
         }
-        let mut config =
-            build_test_s3_config(&self.nodes[node_idx].url, &self.access_key, &self.secret_key, None, "cluster-test");
-        if let Some(ca_pem) = &self.tls_ca_pem {
-            use aws_smithy_http_client::tls::{Provider, TlsContext, TrustStore, rustls_provider::CryptoMode};
-            let tls = TlsContext::builder()
-                .with_trust_store(TrustStore::empty().with_pem_certificate(ca_pem.clone()))
-                .build()?;
-            let client = SmithyHttpClientBuilder::new()
-                .tls_provider(Provider::Rustls(CryptoMode::AwsLc))
-                .tls_context(tls)
-                .build_https();
-            config = config.to_builder().http_client(client).build();
-        }
-        Ok(Client::from_conf(config))
+        Ok(Client::from_conf(build_test_s3_config(
+            &self.nodes[node_idx].url,
+            &self.access_key,
+            &self.secret_key,
+            None,
+            "cluster-test",
+        )))
     }
 
     /// Create S3 clients for all nodes in the RustFS cluster and collect them into a vector.
@@ -1885,7 +1824,7 @@ impl RustFSTestClusterEnvironment {
         }
 
         self.nodes.push(ClusterNode {
-            url: format!("{}://{address}", if self.tls_ca_pem.is_some() { "https" } else { "http" }),
+            url: format!("http://{address}"),
             address,
             data_dir: data_dirs[0].clone(),
             data_dirs,
@@ -2499,7 +2438,6 @@ mod tests {
             node_capture_log_paths: vec![None; topology.node_count],
             topology,
             volume_proxy_addresses: vec![None; node_count],
-            tls_ca_pem: None,
         }
     }
 
@@ -2517,8 +2455,8 @@ mod tests {
         );
     }
 
-    #[tokio::test]
-    async fn volumes_single_pool_multidrive_enumerates_every_drive() {
+    #[test]
+    fn volumes_single_pool_multidrive_enumerates_every_drive() {
         // 4 nodes x 2 drives -> 8 explicit endpoints, one legacy pool. No
         // ellipses, so the server parser keeps this as a single DistErasure pool.
         let env = fake_cluster(ClusterTopology::single_pool_multidrive(4, 2));
@@ -2530,25 +2468,6 @@ mod tests {
             .join(" ");
         assert_eq!(env.build_volumes_arg(), expected);
         assert_eq!(env.build_volumes_arg().split(' ').count(), 8);
-
-        let directory = tempfile::tempdir().expect("TLS fixture directory");
-        crate::tls_gen::run(crate::tls_gen::Args {
-            out_dir: directory.path().to_owned(),
-            days: 1,
-            force: false,
-        })
-        .expect("generate TLS bundle");
-        for topology in [
-            ClusterTopology::single_pool_multidrive(4, 4),
-            ClusterTopology::per_node_pools(4, vec![vec![0], vec![1]]),
-        ] {
-            let mut cluster = fake_cluster(topology);
-            let plaintext_volumes = cluster.build_volumes_arg();
-            cluster.enable_tls(directory.path()).await.expect("configure TLS");
-            assert_eq!(cluster.build_volumes_arg(), plaintext_volumes.replace("http://", "https://"));
-            assert!(cluster.nodes.iter().all(|node| node.url.starts_with("https://")));
-            assert!(cluster.create_s3_client(0).is_ok(), "trusted SDK client should build");
-        }
     }
 
     #[test]
@@ -2636,22 +2555,6 @@ mod tests {
         proxy.shutdown().await;
     }
 
-    #[tokio::test]
-    async fn cluster_data_dirs_use_os_temporary_directory() {
-        let cluster = RustFSTestClusterEnvironment::new(1).await.expect("allocate isolated cluster");
-        let root = PathBuf::from(&cluster.temp_dir);
-        assert!(
-            root.starts_with(std::env::temp_dir()),
-            "cluster must honor the selected temporary filesystem"
-        );
-        for node in &cluster.nodes {
-            assert!(Path::new(&node.data_dir).starts_with(&root));
-            assert!(node.data_dirs.iter().all(|drive| Path::new(drive).starts_with(&root)));
-        }
-        drop(cluster);
-        assert!(!root.exists(), "dropping the fixture must remove its owned directories");
-    }
-
     #[test]
     fn cluster_node_env_supports_per_node_overrides() {
         let mut env = fake_cluster(ClusterTopology::single_pool(4));

crates/e2e_test/src/data_usage_test.rs

@@ -14,7 +14,6 @@
 
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
-use futures::{StreamExt, TryStreamExt, stream};
 use rustfs_data_usage::DataUsageInfo;
 use tokio::time::{Duration, sleep};
 
@@ -75,23 +74,16 @@ async fn data_usage_reports_all_objects() -> Result<(), Box<dyn std::error::Erro
     // Create bucket and upload objects
     client.create_bucket().bucket(TEST_BUCKET).send().await?;
 
-    // Keep the full fixture and await every durable PUT before checking publication.
-    stream::iter(0..1000)
-        .map(Ok)
-        .try_for_each_concurrent(16, |i| {
-            let client = &client;
-            async move {
-                client
-                    .put_object()
-                    .bucket(TEST_BUCKET)
-                    .key(format!("obj-{i:04}"))
-                    .body(ByteStream::from_static(b"hello-world"))
-                    .send()
-                    .await
-                    .map(|_| ())
-            }
-        })
-        .await?;
+    for i in 0..1000 {
+        let key = format!("obj-{i:04}");
+        client
+            .put_object()
+            .bucket(TEST_BUCKET)
+            .key(key)
+            .body(ByteStream::from_static(b"hello-world"))
+            .send()
+            .await?;
+    }
 
     let usage = wait_for_bucket_usage(&env, TEST_BUCKET, |usage| {
         usage

crates/e2e_test/src/delete_objects_versioning_test.rs

@@ -24,163 +24,6 @@
 //! `xl.meta` immediately visible, and the old file-cache shortcut has been
 //! removed from the read path.
 
-use aws_sdk_s3::Client;
-use aws_sdk_s3::config::retry::RetryConfig;
-use aws_sdk_s3::types::{BucketVersioningStatus, Delete, ObjectIdentifier, VersioningConfiguration};
-use std::collections::BTreeMap;
-
-async fn delete_exact_marker_versions(
-    client: &Client,
-    bucket: &str,
-    key: &str,
-    versions: &[(String, bool)],
-) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let objects = versions
-        .iter()
-        .map(|(version, _)| ObjectIdentifier::builder().key(key).version_id(version).build())
-        .collect::<Result<Vec<_>, _>>()?;
-    let response = client
-        .delete_objects()
-        .bucket(bucket)
-        .delete(Delete::builder().set_objects(Some(objects)).build()?)
-        .send()
-        .await?;
-    assert!(response.errors().is_empty(), "explicit batch returned errors: {response:?}");
-    let mut actual = BTreeMap::new();
-    for deleted in response.deleted() {
-        assert_eq!(deleted.key(), Some(key));
-        *actual
-            .entry((
-                deleted.version_id().ok_or("Deleted entry omitted version ID")?.to_string(),
-                deleted.delete_marker().unwrap_or(false),
-                deleted.delete_marker_version_id().map(str::to_string),
-            ))
-            .or_insert(0_usize) += 1;
-    }
-    let mut expected = BTreeMap::new();
-    for (version, marker) in versions {
-        *expected
-            .entry((version.clone(), *marker, marker.then(|| version.clone())))
-            .or_insert(0_usize) += 1;
-    }
-    assert_eq!(actual, expected, "every requested occurrence must report its own version identity");
-    Ok(())
-}
-
-pub(crate) async fn assert_explicit_marker_batch_cleanup(
-    clients: &[Client],
-    bucket: &str,
-) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    assert!(!clients.is_empty());
-    let clients: Vec<_> = clients
-        .iter()
-        .map(|client| {
-            Client::from_conf(
-                client
-                    .config()
-                    .to_builder()
-                    .retry_config(RetryConfig::standard().with_max_attempts(1))
-                    .build(),
-            )
-        })
-        .collect();
-    let writer = &clients[0];
-    let marker_writer = &clients[1 % clients.len()];
-    let deleter = &clients[2 % clients.len()];
-    let key = "same-key-history";
-    writer.create_bucket().bucket(bucket).send().await?;
-    writer
-        .put_bucket_versioning()
-        .bucket(bucket)
-        .versioning_configuration(
-            VersioningConfiguration::builder()
-                .status(BucketVersioningStatus::Enabled)
-                .build(),
-        )
-        .send()
-        .await?;
-    let mut data = Vec::new();
-    for byte in [41_u8, 73_u8] {
-        let body = vec![byte; 1024 * 1024];
-        let uploaded = writer
-            .put_object()
-            .bucket(bucket)
-            .key(key)
-            .body(body.clone().into())
-            .send()
-            .await?;
-        data.push((uploaded.version_id().ok_or("versioned PUT omitted ID")?.to_string(), body));
-    }
-    let mut markers = Vec::with_capacity(16);
-    for _ in 0..16 {
-        let deleted = marker_writer.delete_object().bucket(bucket).key(key).send().await?;
-        assert_eq!(deleted.delete_marker(), Some(true));
-        markers.push(deleted.version_id().ok_or("marker creation omitted ID")?.to_string());
-    }
-    let mut requested: Vec<_> = markers.iter().map(|id| (id.clone(), true)).collect();
-    requested.push((markers[0].clone(), true));
-    requested.push((markers[15].clone(), true));
-    delete_exact_marker_versions(deleter, bucket, key, &requested).await?;
-    for client in &clients {
-        let versions = client.list_object_versions().bucket(bucket).send().await?;
-        assert!(versions.delete_markers().is_empty(), "marker survived successful batch: {versions:?}");
-        let mut actual: Vec<_> = versions
-            .versions()
-            .iter()
-            .filter_map(|version| version.version_id())
-            .collect();
-        actual.sort_unstable();
-        let mut expected: Vec<_> = data.iter().map(|(id, _)| id.as_str()).collect();
-        expected.sort_unstable();
-        assert_eq!(actual, expected);
-        let current = client.get_object().bucket(bucket).key(key).send().await?;
-        assert_eq!(current.body.collect().await?.into_bytes().as_ref(), data[1].1.as_slice());
-        let historical = client
-            .get_object()
-            .bucket(bucket)
-            .key(key)
-            .version_id(&data[0].0)
-            .send()
-            .await?;
-        assert_eq!(historical.body.collect().await?.into_bytes().as_ref(), data[0].1.as_slice());
-    }
-    let uuid_marker = marker_writer.delete_object().bucket(bucket).key(key).send().await?;
-    let uuid_marker = uuid_marker.version_id().ok_or("marker creation omitted UUID")?.to_string();
-    writer
-        .put_bucket_versioning()
-        .bucket(bucket)
-        .versioning_configuration(
-            VersioningConfiguration::builder()
-                .status(BucketVersioningStatus::Suspended)
-                .build(),
-        )
-        .send()
-        .await?;
-    let null_marker = marker_writer.delete_object().bucket(bucket).key(key).send().await?;
-    assert_eq!(null_marker.delete_marker(), Some(true));
-    assert_eq!(null_marker.version_id(), Some("null"));
-    let requested = [
-        (uuid_marker.clone(), true),
-        (data[1].0.clone(), false),
-        ("null".to_string(), true),
-        (data[0].0.clone(), false),
-        (uuid_marker, true),
-    ];
-    delete_exact_marker_versions(deleter, bucket, key, &requested).await?;
-    for client in &clients {
-        let versions = client.list_object_versions().bucket(bucket).send().await?;
-        assert!(versions.versions().is_empty() && versions.delete_markers().is_empty(), "{versions:?}");
-    }
-    clients
-        .last()
-        .expect("nonempty clients")
-        .delete_bucket()
-        .bucket(bucket)
-        .send()
-        .await?;
-    Ok(())
-}
-
 #[cfg(test)]
 mod tests {
     use crate::common::{RustFSTestEnvironment, init_logging};
@@ -192,16 +35,6 @@ mod tests {
         env.create_s3_client()
     }
 
-    #[tokio::test]
-    async fn test_explicit_delete_marker_batch_removes_versions_without_client_retries()
-    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        init_logging();
-        let mut env = RustFSTestEnvironment::new().await?;
-        env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
-            .await?;
-        super::assert_explicit_marker_batch_cleanup(&[env.create_s3_client()], "explicit-marker-batch").await
-    }
-
     /// Regression test for Issue #1878.
     ///
     /// Verifies that after calling `delete_objects` (the batch plural API) on

crates/e2e_test/src/distributed/artifact_delete_test.rs

@@ -1,378 +0,0 @@
-// Copyright 2026 RustFS Team
-// SPDX-License-Identifier: Apache-2.0
-
-#![cfg(unix)]
-
-use super::artifact_seal_test::{assert_exact_version, confirm_seal, create_artifact_attempt};
-use super::harness::{
-    DistCluster, DistLayout, TestResult, enable_versioning, payload_for, s3_probe_client, unique_bucket, wait_for_ready,
-};
-use crate::chaos::{VersionShardCensus, census_object_version_on_disk};
-use crate::common::init_logging;
-use aws_sdk_s3::error::ProvideErrorMetadata;
-use aws_sdk_s3::types::{Delete, ObjectIdentifier};
-use serde_json::json;
-use std::collections::{BTreeMap, BTreeSet};
-use std::os::unix::fs::MetadataExt;
-use std::path::{Path, PathBuf};
-use std::time::Duration;
-use tokio::time::timeout;
-
-const PUBLIC_INTERNAL_ERROR: &str = "We encountered an internal error, please try again.";
-
-#[derive(Debug, PartialEq, Eq)]
-pub(super) struct PhysicalEntry {
-    identity: (u64, u64),
-    changed: (i64, i64),
-    modified: (i64, i64),
-    bytes: Option<Vec<u8>>,
-}
-
-// Include directory identity/ctime as well as file bytes. Creating and removing
-// a temporary marker/rollback file cannot hide behind a restored final xl.meta.
-// Reads change atime, which is deliberately excluded.
-pub(super) fn physical_snapshot(dist: &DistCluster, bucket: &str) -> TestResult<BTreeMap<(usize, PathBuf), PhysicalEntry>> {
-    let mut snapshot = BTreeMap::new();
-    for (node, instance) in dist.cluster.nodes.iter().enumerate() {
-        let root = Path::new(&instance.data_dirs[0]).join(bucket);
-        for entry in walkdir::WalkDir::new(&root) {
-            let entry = entry?;
-            let metadata = std::fs::symlink_metadata(entry.path())?;
-            assert!(!metadata.file_type().is_symlink(), "unexpected fixture symlink");
-            snapshot.insert(
-                (node, entry.path().strip_prefix(&root)?.to_path_buf()),
-                PhysicalEntry {
-                    identity: (metadata.dev(), metadata.ino()),
-                    changed: (metadata.ctime(), metadata.ctime_nsec()),
-                    modified: (metadata.mtime(), metadata.mtime_nsec()),
-                    bytes: if metadata.is_dir() {
-                        None
-                    } else {
-                        Some(std::fs::read(entry.path())?)
-                    },
-                },
-            );
-        }
-    }
-    Ok(snapshot)
-}
-
-struct Version {
-    key: String,
-    id: String,
-    body: Vec<u8>,
-    manifests: Vec<VersionShardCensus>,
-}
-
-impl Version {
-    async fn put(dist: &DistCluster, bucket: &str, key: &str, seed: &str) -> TestResult<Self> {
-        let body = payload_for(seed, 2 * 1024 * 1024);
-        let output = timeout(
-            Duration::from_secs(30),
-            dist.client(1)?
-                .put_object()
-                .bucket(bucket)
-                .key(key)
-                .body(body.clone().into())
-                .send(),
-        )
-        .await??;
-        let id = output.version_id().ok_or("fixture PUT omitted version ID")?.to_owned();
-        let manifests = dist
-            .cluster
-            .nodes
-            .iter()
-            .map(|node| census_object_version_on_disk(Path::new(&node.data_dirs[0]), bucket, key, Some(&id)))
-            .collect::<TestResult<Vec<_>>>()?;
-        assert!(
-            manifests
-                .iter()
-                .all(|manifest| manifest.is_complete() && !manifest.expected_part_numbers.is_empty())
-        );
-        Ok(Self {
-            key: key.to_owned(),
-            id,
-            body,
-            manifests,
-        })
-    }
-}
-
-fn target(key: &str, version: Option<&str>) -> TestResult<ObjectIdentifier> {
-    Ok(ObjectIdentifier::builder()
-        .key(key)
-        .set_version_id(version.map(str::to_owned))
-        .build()?)
-}
-
-async fn assert_fixture(
-    dist: &DistCluster,
-    bucket: &str,
-    prefix: &str,
-    versions: &[Version],
-    marker: (&str, &str),
-) -> TestResult {
-    let expected: BTreeSet<_> = versions
-        .iter()
-        .map(|version| (version.key.clone(), version.id.clone()))
-        .collect();
-    for node in 0..dist.cluster.nodes.len() {
-        let client = dist.client(node)?;
-        let listed = timeout(
-            Duration::from_secs(30),
-            client.list_object_versions().bucket(bucket).prefix(prefix).send(),
-        )
-        .await??;
-        let actual: BTreeSet<_> = listed
-            .versions()
-            .iter()
-            .map(|version| {
-                (
-                    version.key().unwrap_or_default().to_owned(),
-                    version.version_id().unwrap_or_default().to_owned(),
-                )
-            })
-            .collect();
-        assert_eq!(listed.versions().len(), versions.len());
-        assert_eq!(actual, expected, "delete changed exact data versions on node {node}");
-        assert_eq!(listed.delete_markers().len(), 1);
-        assert_eq!(listed.delete_markers()[0].key(), Some(marker.0));
-        assert_eq!(listed.delete_markers()[0].version_id(), Some(marker.1));
-        assert_eq!(listed.delete_markers()[0].is_latest(), Some(true));
-        for version in versions {
-            assert_exact_version(&client, bucket, &version.key, &version.id, &version.body).await?;
-            let disk = Path::new(&dist.cluster.nodes[node].data_dirs[0]);
-            assert!(
-                census_object_version_on_disk(disk, bucket, &version.key, Some(&version.id))?
-                    .matches_manifest(&version.manifests[node])
-            );
-        }
-    }
-    Ok(())
-}
-
-async fn assert_single_denied(dist: &DistCluster, node: usize, bucket: &str, key: &str, version: Option<&str>) -> TestResult {
-    let before = physical_snapshot(dist, bucket)?;
-    // One attempt: a later retry must not conceal a first local mutation.
-    let client = s3_probe_client(&dist.client(node)?);
-    let error = timeout(
-        Duration::from_secs(30),
-        client
-            .delete_object()
-            .bucket(bucket)
-            .key(key)
-            .set_version_id(version.map(str::to_owned))
-            .send(),
-    )
-    .await?
-    .expect_err("ordinary deletion succeeded in a managed bucket");
-    let service = error
-        .as_service_error()
-        .ok_or("delete failed without an S3 service rejection")?;
-    assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(500));
-    assert_eq!(service.code(), Some("InternalError"));
-    assert_eq!(service.message(), Some(PUBLIC_INTERNAL_ERROR));
-    assert!(
-        physical_snapshot(dist, bucket)? == before,
-        "first delete through node {node} wrote or rolled back managed data"
-    );
-    Ok(())
-}
-
-async fn assert_batch_denied(dist: &DistCluster, node: usize, bucket: &str, objects: &[ObjectIdentifier]) -> TestResult {
-    let before = physical_snapshot(dist, bucket)?;
-    let client = s3_probe_client(&dist.client(node)?);
-    let response = timeout(
-        Duration::from_secs(30),
-        client
-            .delete_objects()
-            .bucket(bucket)
-            .delete(Delete::builder().set_objects(Some(objects.to_vec())).quiet(false).build()?)
-            .send(),
-    )
-    .await??;
-    assert!(response.deleted().is_empty(), "managed batch reported a successful delete");
-    assert_eq!(response.errors().len(), objects.len(), "batch must reject every requested occurrence");
-    for (error, requested) in response.errors().iter().zip(objects) {
-        assert_eq!(error.key(), Some(requested.key()));
-        assert_eq!(error.version_id(), requested.version_id());
-        assert_eq!(error.code(), Some("InternalError"));
-        assert_eq!(error.message(), Some(PUBLIC_INTERNAL_ERROR));
-    }
-    assert!(
-        physical_snapshot(dist, bucket)? == before,
-        "first batch through node {node} wrote or rolled back managed data"
-    );
-    Ok(())
-}
-
-fn copy_existing_object(source: &Path, destination: &Path) -> TestResult {
-    assert!(!destination.exists());
-    for entry in walkdir::WalkDir::new(source) {
-        let entry = entry?;
-        let target = destination.join(entry.path().strip_prefix(source)?);
-        if entry.file_type().is_dir() {
-            std::fs::create_dir_all(&target)?;
-        } else {
-            assert!(entry.file_type().is_file());
-            std::fs::copy(entry.path(), target)?;
-        }
-    }
-    Ok(())
-}
-
-async fn ordinary_delete_control(dist: &DistCluster, bucket: &str) -> TestResult {
-    let client = s3_probe_client(&dist.client(0)?);
-    let key = "ordinary-observer-control.bin";
-    let version = Version::put(dist, bucket, key, key).await?;
-    let before = physical_snapshot(dist, bucket)?;
-    let deleted = client.delete_object().bucket(bucket).key(key).send().await?;
-    assert_eq!(deleted.delete_marker(), Some(true));
-    let marker = deleted.version_id().ok_or("ordinary delete omitted marker ID")?;
-    assert!(
-        physical_snapshot(dist, bucket)? != before,
-        "physical observer missed real marker creation"
-    );
-    client
-        .delete_object()
-        .bucket(bucket)
-        .key(key)
-        .version_id(marker)
-        .send()
-        .await?;
-    // The logical version inventory is restored, but ctime/identity still
-    // expose the two writes. This checks the observer used by managed negatives.
-    let listed = client.list_object_versions().bucket(bucket).prefix(key).send().await?;
-    assert_eq!(listed.versions().len(), 1);
-    assert_eq!(listed.versions()[0].version_id(), Some(version.id.as_str()));
-    assert!(listed.delete_markers().is_empty());
-    assert!(
-        physical_snapshot(dist, bucket)? != before,
-        "marker round-trip hid transient physical writes"
-    );
-    assert_exact_version(&client, bucket, key, &version.id, &version.body).await?;
-    client
-        .delete_object()
-        .bucket(bucket)
-        .key(key)
-        .version_id(&version.id)
-        .send()
-        .await?;
-    assert!(
-        client
-            .list_object_versions()
-            .bucket(bucket)
-            .prefix(key)
-            .send()
-            .await?
-            .versions()
-            .is_empty()
-    );
-
-    let current = Version::put(dist, bucket, "ordinary-batch-current.bin", "current").await?;
-    let exact = Version::put(dist, bucket, "ordinary-batch-exact.bin", "exact").await?;
-    let response = client
-        .delete_objects()
-        .bucket(bucket)
-        .delete(
-            Delete::builder()
-                .objects(target(&current.key, None)?)
-                .objects(target(&exact.key, Some(&exact.id))?)
-                .objects(target(&exact.key, Some(&exact.id))?)
-                .build()?,
-        )
-        .send()
-        .await?;
-    assert!(response.errors().is_empty());
-    assert_eq!(response.deleted().len(), 3);
-    let listed = client
-        .list_object_versions()
-        .bucket(bucket)
-        .prefix(&current.key)
-        .send()
-        .await?;
-    assert_eq!(listed.versions().len(), 1);
-    assert_eq!(listed.delete_markers().len(), 1);
-    assert!(
-        client
-            .list_object_versions()
-            .bucket(bucket)
-            .prefix(&exact.key)
-            .send()
-            .await?
-            .versions()
-            .is_empty()
-    );
-    Ok(())
-}
-
-#[tokio::test]
-async fn four_node_artifact_delete_entrypoints_preserve_versions_and_physical_state() -> TestResult {
-    init_logging();
-    let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
-    let (identity, admin) = create_artifact_attempt(&dist).await?;
-    let bucket = identity["bucket"]
-        .as_str()
-        .ok_or("managed receipt omitted bucket")?
-        .to_owned();
-    let prefix = identity["attempt_prefix"]
-        .as_str()
-        .ok_or("managed receipt omitted prefix")?
-        .to_owned();
-    let key = format!("{prefix}native.bin");
-    let mut versions = vec![
-        Version::put(&dist, &bucket, &key, "native-old").await?,
-        Version::put(&dist, &bucket, &key, "native-new").await?,
-    ];
-    let ordinary = unique_bucket("delete-control");
-    dist.create_bucket(&ordinary).await?;
-    enable_versioning(&dist.client(0)?, &ordinary).await?;
-
-    // A new managed bucket cannot legitimately create a marker. Model existing
-    // persisted state from before adoption/upgrade using a real S3-generated
-    // ordinary object+marker, copied disk-for-disk in the same erasure set.
-    // Only this object's files move; bucket authority/configuration is untouched.
-    let marker_key = format!("{prefix}existing-marker.bin");
-    versions.push(Version::put(&dist, &ordinary, &marker_key, "existing-data").await?);
-    let output = dist
-        .client(2)?
-        .delete_object()
-        .bucket(&ordinary)
-        .key(&marker_key)
-        .send()
-        .await?;
-    assert_eq!(output.delete_marker(), Some(true));
-    let marker = output.version_id().ok_or("ordinary fixture omitted marker ID")?.to_owned();
-    dist.cluster.stop();
-    for node in &dist.cluster.nodes {
-        let disk = Path::new(&node.data_dirs[0]);
-        copy_existing_object(&disk.join(&ordinary).join(&marker_key), &disk.join(&bucket).join(&marker_key))?;
-    }
-    dist.cluster.start().await?;
-    wait_for_ready(&dist.cluster).await?;
-    assert_fixture(&dist, &bucket, &prefix, &versions, (&marker_key, &marker)).await?;
-    ordinary_delete_control(&dist, &ordinary).await?;
-    let batch = vec![
-        target(&key, None)?,
-        target(&key, Some(&versions[0].id))?,
-        target(&key, Some(&versions[1].id))?,
-        target(&marker_key, Some(&marker))?,
-        target(&key, Some(&versions[0].id))?,
-        target(&marker_key, Some(&versions[2].id))?,
-    ];
-    for phase in ["open", "sealed", "restarted"] {
-        if phase == "sealed" {
-            confirm_seal(&dist, &admin, &json!({"version": 1, "target": identity}).to_string(), &identity).await?;
-        } else if phase == "restarted" {
-            dist.restart_with_current_binary().await?;
-        }
-        for node in 0..dist.cluster.nodes.len() {
-            assert_single_denied(&dist, node, &bucket, &key, None).await?;
-            assert_single_denied(&dist, node, &bucket, &key, Some(&versions[0].id)).await?;
-            assert_single_denied(&dist, node, &bucket, &marker_key, Some(&marker)).await?;
-            assert_batch_denied(&dist, node, &bucket, &batch).await?;
-        }
-        assert_fixture(&dist, &bucket, &prefix, &versions, (&marker_key, &marker)).await?;
-    }
-    Ok(())
-}

crates/e2e_test/src/distributed/artifact_inventory_test.rs

@@ -1,434 +0,0 @@
-// Copyright 2026 RustFS Team
-// SPDX-License-Identifier: Apache-2.0
-
-#![cfg(unix)]
-
-use super::artifact_delete_test::physical_snapshot;
-use super::artifact_seal_test::{ArtifactAdmin, create_artifact_attempt};
-use super::harness::{DistCluster, DistLayout, TestResult, wait_for_ready};
-use crate::common::{
-    AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
-    signed_s3_request,
-};
-use futures::{StreamExt as _, TryStreamExt as _};
-use http::{Method, StatusCode};
-use serde::Deserialize;
-use serde_json::{Value, json};
-use std::{collections::BTreeSet, time::Duration};
-use tokio::time::timeout;
-use uuid::Uuid;
-
-const ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/inventory";
-const BUDGET: Duration = Duration::from_secs(45);
-
-struct Reader {
-    access: String,
-    secret: String,
-}
-
-impl Reader {
-    async fn create(dist: &DistCluster, target: &Value, action: &str) -> TestResult<Self> {
-        let reader = Self {
-            access: format!("inventory{}", &Uuid::new_v4().simple().to_string()[..12]),
-            secret: Uuid::new_v4().simple().to_string(),
-        };
-        let policy_name = format!("inventory{}", &Uuid::new_v4().simple().to_string()[..12]);
-        let bucket = target["bucket"].as_str().ok_or("target bucket")?;
-        let prefix = target["attempt_prefix"]
-            .as_str()
-            .ok_or("target prefix")?
-            .trim_end_matches('/');
-        let resource = if action == "s3:ListBucket" {
-            format!("arn:aws:s3:::{bucket}")
-        } else {
-            format!("arn:aws:s3:::{bucket}/{prefix}")
-        };
-        let policy =
-            json!({"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":action,"Resource":resource}]}).to_string();
-        let url = &dist.cluster.nodes[0].url;
-        let access = &dist.cluster.access_key;
-        let secret = &dist.cluster.secret_key;
-        admin_create_user_via(AdminTransport::Signed, url, access, secret, &reader.access, &reader.secret).await?;
-        admin_add_canned_policy_via(AdminTransport::Signed, url, access, secret, &policy_name, &policy).await?;
-        admin_attach_user_policy_via(AdminTransport::Signed, url, access, secret, &policy_name, &reader.access).await?;
-        Ok(reader)
-    }
-
-    async fn request(&self, dist: &DistCluster, target: &Value, invocation: Uuid) -> TestResult<reqwest::Response> {
-        timeout(
-            BUDGET,
-            signed_s3_request(
-                Method::POST,
-                &format!("{}{ROUTE}", dist.cluster.nodes[0].url),
-                Some(json!({"version":1,"invocation_id":invocation,"target":target}).to_string()),
-                Some("application/json"),
-                &self.access,
-                &self.secret,
-            ),
-        )
-        .await?
-    }
-}
-
-#[derive(Debug, Deserialize)]
-#[serde(deny_unknown_fields)]
-struct Frame {
-    version: u32,
-    invocation_id: Uuid,
-    target: Value,
-    sequence: u64,
-    payload: Payload,
-}
-
-#[derive(Debug, Deserialize)]
-#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
-enum Payload {
-    Observation {
-        disk_id: Uuid,
-        key: String,
-        version_id: Uuid,
-        delete_marker: bool,
-    },
-    Complete {
-        members: usize,
-        observations: u64,
-    },
-}
-
-async fn read_frames(mut response: reqwest::Response) -> TestResult<(Vec<Frame>, bool)> {
-    let mut pending = Vec::new();
-    let mut frames = Vec::new();
-    let mut clean = true;
-    loop {
-        match timeout(BUDGET, response.chunk()).await? {
-            Ok(Some(chunk)) => {
-                pending.extend_from_slice(&chunk);
-                while let Some(end) = pending.iter().position(|byte| *byte == b'\n') {
-                    assert!(end < 1024 * 1024, "HTTP inventory line exceeds encoded bound");
-                    let line: Vec<_> = pending.drain(..=end).collect();
-                    frames.push(serde_json::from_slice(&line[..end])?);
-                }
-                assert!(pending.len() < 1024 * 1024, "unbounded incomplete HTTP line");
-            }
-            Ok(None) => break,
-            Err(_) => {
-                clean = false;
-                break;
-            }
-        }
-    }
-    Ok((frames, clean && pending.is_empty()))
-}
-
-async fn seal(dist: &DistCluster, user: &ArtifactAdmin, target: &Value) -> TestResult {
-    let (status, body) = user
-        .request(
-            dist,
-            0,
-            "/rustfs/admin/v3/plugin-artifacts/seal-writes",
-            json!({"version":1,"target":target}).to_string(),
-        )
-        .await?;
-    assert_eq!(status, StatusCode::OK, "actual all-member closure: {body}");
-    assert_eq!(serde_json::from_str::<Value>(&body)?["all_writes_sealed"], true);
-    Ok(())
-}
-
-fn exact_inventory(frames: &[Frame], target: &Value, invocation: Uuid, versions: &BTreeSet<(String, Uuid, bool)>) {
-    let mut observed = BTreeSet::new();
-    let mut disks = BTreeSet::new();
-    let mut count = 0u64;
-    for (sequence, frame) in frames.iter().enumerate() {
-        assert_eq!(frame.version, 1);
-        assert_eq!(frame.invocation_id, invocation);
-        assert_eq!(&frame.target, target);
-        assert_eq!(frame.sequence, u64::try_from(sequence).expect("bounded fixture sequence"));
-        match &frame.payload {
-            Payload::Observation {
-                disk_id,
-                key,
-                version_id,
-                delete_marker,
-            } => {
-                assert!(!disk_id.is_nil() && !version_id.is_nil());
-                assert!(
-                    observed.insert((*disk_id, key.clone(), *version_id, *delete_marker)),
-                    "duplicate physical observation"
-                );
-                disks.insert(*disk_id);
-                count += 1;
-            }
-            Payload::Complete { members, observations } => {
-                assert_eq!(sequence + 1, frames.len(), "Complete must be the last HTTP frame");
-                assert_eq!(*members, 4);
-                assert_eq!(*observations, count);
-            }
-        }
-    }
-    assert!(
-        matches!(frames.last().map(|frame| &frame.payload), Some(Payload::Complete { .. })),
-        "A03I3A_HTTP_COMPLETE"
-    );
-    if !versions.is_empty() {
-        assert_eq!(disks.len(), 4);
-    }
-    for disk in disks {
-        let actual: BTreeSet<_> = observed
-            .iter()
-            .filter(|(id, ..)| *id == disk)
-            .map(|(_, key, version, marker)| (key.clone(), *version, *marker))
-            .collect();
-        assert_eq!(&actual, versions, "A03I3A_EXACT_VERSIONS");
-    }
-    assert_eq!(observed.len(), versions.len() * 4);
-}
-
-#[tokio::test]
-async fn artifact_inventory_http_exact_versions() -> TestResult {
-    crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks"));
-    init_logging();
-    let dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
-    let (target, admin) = create_artifact_attempt(&dist).await?;
-    let bucket = target["bucket"].as_str().ok_or("bucket")?;
-    let prefix = target["attempt_prefix"].as_str().ok_or("prefix")?;
-    let mut versions = BTreeSet::new();
-    for (name, body) in [
-        ("same", b"first".to_vec()),
-        ("same", b"second".to_vec()),
-        ("zero", Vec::new()),
-    ] {
-        let key = format!("{prefix}{name}");
-        let output = timeout(BUDGET, dist.client(1)?.put_object().bucket(bucket).key(&key).body(body.into()).send()).await??;
-        let version = Uuid::parse_str(output.version_id().ok_or("actual PUT version")?)?;
-        versions.insert((key, version, false));
-    }
-    seal(&dist, &admin, &target).await?;
-    // A wildcard action is still restricted to this exact attempt resource.
-    // The separate authority case exercises the dedicated action alone.
-    let reader = Reader::create(&dist, &target, "admin:*").await?;
-    let before = physical_snapshot(&dist, bucket)?;
-    let invocation = Uuid::new_v4();
-    let response = reader.request(&dist, &target, invocation).await?;
-    assert_eq!(response.status(), StatusCode::OK, "A03I3A_HTTP_ROUTE");
-    assert_eq!(
-        response.headers().get("content-type").and_then(|value| value.to_str().ok()),
-        Some("application/x-ndjson")
-    );
-    let (frames, clean) = read_frames(response).await?;
-    assert!(clean, "Complete requires clean HTTP EOF");
-    assert!(frames.len() > 2, "real multi-frame HTTP body");
-    exact_inventory(&frames, &target, invocation, &versions);
-    assert!(
-        before == physical_snapshot(&dist, bucket)?,
-        "inventory mutated physical artifact namespace"
-    );
-    let (empty, empty_admin) = create_artifact_attempt(&dist).await?;
-    seal(&dist, &empty_admin, &empty).await?;
-    let empty_reader = Reader::create(&dist, &empty, "admin:*").await?;
-    let invocation = Uuid::new_v4();
-    let response = empty_reader.request(&dist, &empty, invocation).await?;
-    assert_eq!(response.status(), StatusCode::OK);
-    let (frames, clean) = read_frames(response).await?;
-    assert!(clean);
-    exact_inventory(&frames, &empty, invocation, &BTreeSet::new());
-    Ok(())
-}
-
-#[tokio::test]
-async fn artifact_inventory_http_scoped_authority() -> TestResult {
-    crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks"));
-    init_logging();
-    let dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
-    let (target, admin) = create_artifact_attempt(&dist).await?;
-    seal(&dist, &admin, &target).await?;
-    let reader = Reader::create(&dist, &target, "admin:InventoryPluginArtifactAttempt").await?;
-    let invocation = Uuid::new_v4();
-    let response = reader.request(&dist, &target, invocation).await?;
-    assert_eq!(response.status(), StatusCode::OK, "dedicated inventory authority");
-    let (frames, clean) = read_frames(response).await?;
-    assert!(clean);
-    exact_inventory(&frames, &target, invocation, &BTreeSet::new());
-    let mut other_prefix = target.clone();
-    other_prefix["attempt_prefix"] = json!(format!("outputs/6169/{}/{}/", Uuid::new_v4(), Uuid::new_v4()));
-    let mut other_bucket = target.clone();
-    other_bucket["bucket"] = json!("another-artifact-bucket");
-    for denied in [&other_prefix, &other_bucket] {
-        assert_eq!(
-            reader.request(&dist, denied, Uuid::new_v4()).await?.status(),
-            StatusCode::FORBIDDEN,
-            "A03I3A_SCOPED_AUTHORITY"
-        );
-    }
-    let (status, _) = admin
-        .request(
-            &dist,
-            0,
-            ROUTE,
-            json!({"version":1,"invocation_id":Uuid::new_v4(),"target":target}).to_string(),
-        )
-        .await?;
-    assert_eq!(status, StatusCode::FORBIDDEN, "A03I3A_SCOPED_AUTHORITY seal-only user");
-    let listed = Reader::create(&dist, &target, "s3:ListBucket").await?;
-    dist.client_with_credentials(0, &listed.access, &listed.secret)?
-        .list_objects_v2()
-        .bucket(target["bucket"].as_str().ok_or("bucket")?)
-        .send()
-        .await?;
-    assert_eq!(
-        listed.request(&dist, &target, Uuid::new_v4()).await?.status(),
-        StatusCode::FORBIDDEN,
-        "A03I3A_SCOPED_AUTHORITY S3 list-only user"
-    );
-    let root_response = signed_s3_request(
-        Method::POST,
-        &format!("{}{ROUTE}", dist.cluster.nodes[0].url),
-        Some(json!({"version":1,"invocation_id":Uuid::new_v4(),"target":target}).to_string()),
-        Some("application/json"),
-        &dist.cluster.access_key,
-        &dist.cluster.secret_key,
-    )
-    .await?;
-    assert_eq!(root_response.status(), StatusCode::FORBIDDEN, "original scoped user required");
-    for body in [
-        json!({"version":1,"invocation_id":Uuid::nil(),"target":target}),
-        json!({"version":1,"invocation_id":Uuid::new_v4(),"target":target,"extra":true}),
-    ] {
-        let response = signed_s3_request(
-            Method::POST,
-            &format!("{}{ROUTE}", dist.cluster.nodes[0].url),
-            Some(body.to_string()),
-            Some("application/json"),
-            &reader.access,
-            &reader.secret,
-        )
-        .await?;
-        assert_eq!(response.status(), StatusCode::BAD_REQUEST, "strict inventory request");
-    }
-    Ok(())
-}
-
-async fn assert_incomplete(response: reqwest::Response) -> TestResult<Vec<Frame>> {
-    assert_eq!(
-        response.status(),
-        StatusCode::OK,
-        "valid request must exercise the streaming failure boundary"
-    );
-    let (frames, _) = read_frames(response).await?;
-    assert!(
-        !frames.iter().any(|frame| matches!(frame.payload, Payload::Complete { .. })),
-        "A03I3A_NO_FALSE_COMPLETE"
-    );
-    Ok(frames)
-}
-
-#[tokio::test]
-async fn artifact_inventory_http_incomplete_has_no_terminal() -> TestResult {
-    crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks"));
-    init_logging();
-    let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
-    let (target, admin) = create_artifact_attempt(&dist).await?;
-    let bucket = target["bucket"].as_str().ok_or("bucket")?;
-    let prefix = target["attempt_prefix"].as_str().ok_or("prefix")?;
-    // Real PUTs produce more data than a stalled HTTP/2 flow-control window.
-    futures::stream::iter(0..256)
-        .map(|index| {
-            let client = dist.client(1);
-            let key = format!("{prefix}slow/{index:04}/{}/{}", "a".repeat(190), "b".repeat(190));
-            async move {
-                let output = client?
-                    .put_object()
-                    .bucket(bucket)
-                    .key(key)
-                    .body(vec![1].into())
-                    .send()
-                    .await?;
-                assert!(!Uuid::parse_str(output.version_id().ok_or("actual PUT version")?)?.is_nil());
-                Ok::<_, Box<dyn std::error::Error + Send + Sync>>(())
-            }
-        })
-        .buffer_unordered(8)
-        .try_collect::<Vec<_>>()
-        .await?;
-    seal(&dist, &admin, &target).await?;
-    let reader = Reader::create(&dist, &target, "admin:InventoryPluginArtifactAttempt").await?;
-    dist.cluster.stop_node(3)?;
-    assert_incomplete(reader.request(&dist, &target, Uuid::new_v4()).await?).await?;
-    dist.cluster.start_node(3).await?;
-    wait_for_ready(&dist.cluster).await?;
-    let mut wrong = target.clone();
-    wrong["bucket_incarnation"] = json!(Uuid::new_v4());
-    assert_incomplete(reader.request(&dist, &wrong, Uuid::new_v4()).await?).await?;
-    // The configured endpoint map is ordered. Damage its final receiver, so
-    // earlier real observations are emitted before the physical scan fails.
-    let node = dist
-        .cluster
-        .nodes
-        .iter()
-        .max_by_key(|node| format!("{}{}", node.url, node.data_dirs[0]))
-        .ok_or("configured members")?;
-    let metadata = std::path::Path::new(&node.data_dirs[0])
-        .join(bucket)
-        .join(format!("{prefix}slow/0000/{}/{}", "a".repeat(190), "b".repeat(190)))
-        .join("xl.meta");
-    let original = std::fs::read(&metadata)?;
-    std::fs::write(&metadata, b"corrupt inventory metadata")?;
-    let before = physical_snapshot(&dist, bucket)?;
-    let outcome = assert_incomplete(reader.request(&dist, &target, Uuid::new_v4()).await?).await;
-    let after = physical_snapshot(&dist, bucket)?;
-    std::fs::write(&metadata, original)?;
-    let frames = outcome?;
-    assert!(before == after, "failed inventory repaired or mutated a physical object");
-    assert!(
-        frames
-            .iter()
-            .any(|frame| matches!(frame.payload, Payload::Observation { .. })),
-        "late physical failure must follow real HTTP observations"
-    );
-    let client = reqwest::Client::builder()
-        .no_proxy()
-        .http2_prior_knowledge()
-        .http2_initial_stream_window_size(1)
-        .http2_initial_connection_window_size(1)
-        .http2_adaptive_window(false)
-        .build()?;
-    let url = format!("{}{ROUTE}", dist.cluster.nodes[0].url);
-    let request = |invocation| {
-        json!({"version":1,"invocation_id":invocation,"target":target})
-            .to_string()
-            .into_bytes()
-    };
-    let response = crate::common::signed_request_with_client(
-        &client,
-        Method::POST,
-        &url,
-        &reader.access,
-        &reader.secret,
-        Some(request(Uuid::new_v4())),
-        Some("application/json"),
-    )
-    .await?;
-    assert_eq!(response.status(), StatusCode::OK);
-    // Do not poll the body: its one-byte HTTP/2 receive window stalls transport.
-    // This waits beyond the fixed30s service budget, not for readiness.
-    tokio::time::sleep(Duration::from_secs(35)).await;
-    assert_incomplete(response).await?;
-    let response = crate::common::signed_request_with_client(
-        &client,
-        Method::POST,
-        &url,
-        &reader.access,
-        &reader.secret,
-        Some(request(Uuid::new_v4())),
-        Some("application/json"),
-    )
-    .await?;
-    assert_eq!(response.status(), StatusCode::OK);
-    drop(response);
-    let response = timeout(Duration::from_secs(10), reader.request(&dist, &target, Uuid::new_v4())).await??;
-    let (frames, clean) = timeout(Duration::from_secs(10), read_frames(response)).await??;
-    assert!(
-        clean && matches!(frames.last().map(|frame| &frame.payload), Some(Payload::Complete { .. })),
-        "fresh scan must succeed promptly after an actual disconnected HTTP body"
-    );
-    Ok(())
-}

crates/e2e_test/src/distributed/artifact_put_tail_test.rs

@@ -1,269 +0,0 @@
-// Copyright 2026 RustFS Team
-// SPDX-License-Identifier: Apache-2.0
-
-use super::artifact_repair_tail_test::PhysicalBarrier;
-use super::artifact_seal_test::{assert_exact_version, confirm_seal, create_artifact_attempt};
-use super::harness::{DistCluster, DistLayout, TestResult, payload_for, wait_until};
-use crate::chaos::census_object_version_on_disk;
-use crate::common::init_logging;
-use crate::storage_api::artifact_rpc::{
-    DiskError, TONIC_RPC_PREFIX, gen_tonic_signature_headers, set_tonic_canonical_body_digest,
-};
-use http::StatusCode;
-use rustfs_filemeta::FileMeta;
-use rustfs_protos::proto_gen::node_service::{RenameDataRequest, node_service_client::NodeServiceClient};
-use serde_json::{Value, json};
-use std::collections::BTreeMap;
-use std::path::{Path, PathBuf};
-use std::time::Duration;
-use tokio::time::timeout;
-use tonic::Request;
-use uuid::Uuid;
-
-const PUT_SEAL_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/seal-attempt";
-
-fn signed_put_request(body: RenameDataRequest, audience: &str) -> TestResult<Request<RenameDataRequest>> {
-    let canonical = rustfs_protos::canonical_artifact_rename_data_request_body(&body)?;
-    let mut request = Request::new(body);
-    request.set_timeout(Duration::from_secs(15));
-    set_tonic_canonical_body_digest(&mut request, &canonical)?;
-    let digest = request
-        .metadata()
-        .get("x-rustfs-content-sha256")
-        .ok_or("missing canonical digest")?
-        .to_str()?
-        .to_owned();
-    request.metadata_mut().as_mut().extend(gen_tonic_signature_headers(
-        audience,
-        TONIC_RPC_PREFIX.trim_start_matches('/'),
-        "ArtifactRenameData",
-        Some(&digest),
-    )?);
-    Ok(request)
-}
-
-fn publication_snapshot(paths: &[&Path]) -> TestResult<BTreeMap<PathBuf, Option<Vec<u8>>>> {
-    let mut snapshot = BTreeMap::new();
-    for root in paths {
-        for entry in walkdir::WalkDir::new(root) {
-            let entry = entry?;
-            let contents = if entry.file_type().is_dir() {
-                None
-            } else {
-                Some(std::fs::read(entry.path())?)
-            };
-            snapshot.insert(entry.path().to_path_buf(), contents);
-        }
-    }
-    Ok(snapshot)
-}
-
-async fn assert_raw_put_closed(body: &RenameDataRequest, url: &str, audience: &str, paths: &[&Path]) -> TestResult {
-    let before = publication_snapshot(paths)?;
-    // Re-sign the exact original request; replay protection is not the rejection.
-    let request = signed_put_request(body.clone(), audience)?;
-    let response = timeout(Duration::from_secs(20), async {
-        let mut client = NodeServiceClient::connect(url.to_owned()).await?;
-        Ok::<_, Box<dyn std::error::Error + Send + Sync>>(client.artifact_rename_data(request).await?.into_inner())
-    })
-    .await??;
-    assert!(!response.success, "sealed attempt admitted another ordinary PUT");
-    let error = response.error.ok_or("rejected raw PUT omitted error")?;
-    assert_eq!(error.code, DiskError::Io(std::io::ErrorKind::PermissionDenied.into()).to_u32());
-    assert_eq!(error.error_info, "io error artifact attempt mutation domain is closed");
-    assert_eq!(publication_snapshot(paths)?, before, "rejected raw PUT mutated staged or published files");
-    Ok(())
-}
-
-#[tokio::test]
-async fn four_node_artifact_put_seal_waits_for_disconnected_physical_tail() -> TestResult {
-    crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks"));
-    init_logging();
-    let _ = rustfs_credentials::set_global_rpc_secret("rustfs-internode-signature-e2e-secret".to_owned());
-    let rpc_secret = rustfs_credentials::try_get_rpc_token()?;
-    let dist = DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &[("RUSTFS_RPC_SECRET", &rpc_secret)]).await?;
-    let (target, user) = create_artifact_attempt(&dist).await?;
-    let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
-    let prefix = target["attempt_prefix"].as_str().ok_or("target omitted prefix")?;
-    let key = format!("{prefix}detached-put.bin");
-    let payload = payload_for(&key, 2 * 1024 * 1024);
-    let output = timeout(
-        Duration::from_secs(30),
-        dist.client(1)?
-            .put_object()
-            .bucket(bucket)
-            .key(&key)
-            .body(payload.clone().into())
-            .send(),
-    )
-    .await??;
-    let version = output.version_id().ok_or("managed PUT omitted version ID")?;
-    let request_body = json!({"version": 1, "target": target}).to_string();
-    let baselines = dist
-        .cluster
-        .nodes
-        .iter()
-        .map(|node| census_object_version_on_disk(Path::new(&node.data_dirs[0]), bucket, &key, Some(version)))
-        .collect::<TestResult<Vec<_>>>()?;
-    assert!(
-        baselines
-            .iter()
-            .all(|baseline| baseline.is_complete() && !baseline.expected_part_numbers.is_empty())
-    );
-
-    let node = &dist.cluster.nodes[3];
-    let disk = PathBuf::from(&node.data_dirs[0]).canonicalize()?;
-    let object = disk.join(bucket).join(&key);
-    let metadata_bytes = std::fs::read(object.join("xl.meta"))?;
-    let fi = FileMeta::load(&metadata_bytes)?.into_fileinfo(bucket, &key, version, true, false, true)?;
-    let admitted_file_info = serde_json::to_value(&fi)?;
-    assert!(!fi.is_healing(), "ordinary publication must not carry the repair marker");
-    let data_dir = fi
-        .data_dir
-        .ok_or("physical-tail fixture requires external shards")?
-        .to_string();
-    let source = Uuid::new_v4().to_string();
-    let staging = disk.join(".rustfs.sys/tmp").join(&source);
-    std::fs::create_dir_all(&staging)?;
-    let destination = object.join(&data_dir);
-    // Replay the receiver publication of a real ordinary PUT while its attempt
-    // remains open. Preserve its referenced version metadata. The admitted
-    // owned operation may still finish metadata publication after RPC detach.
-    std::fs::rename(&destination, staging.join(&data_dir))?;
-    let barrier = PhysicalBarrier::arm(&disk, &destination)?;
-    let format: Value = serde_json::from_slice(&std::fs::read(disk.join(".rustfs.sys/format.json"))?)?;
-    let disk_id = Uuid::parse_str(format["xl"]["this"].as_str().ok_or("disk format omitted xl.this")?)?;
-    let incarnation = Uuid::parse_str(target["bucket_incarnation"].as_str().ok_or("target omitted incarnation")?)?;
-    let put = RenameDataRequest {
-        disk: disk_id.to_string(),
-        src_volume: ".rustfs.sys/tmp".to_owned(),
-        src_path: source,
-        dst_volume: bucket.to_owned(),
-        dst_path: key.clone(),
-        file_info: serde_json::to_string(&fi)?,
-        file_info_bin: Default::default(),
-        scanner_publication_lease_token: Default::default(),
-        bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
-    };
-    let request = signed_put_request(put.clone(), &node.address)?;
-    // Direct RPC takes no coordinator attempt lock and owns its own connection.
-    let url = node.url.clone();
-    let mut rpc = tokio::spawn(async move {
-        timeout(Duration::from_secs(30), async move {
-            let mut client = NodeServiceClient::connect(url).await?;
-            Ok::<_, Box<dyn std::error::Error + Send + Sync>>(client.artifact_rename_data(request).await?)
-        })
-        .await
-    });
-    tokio::select! {
-        result = &mut rpc => {
-            let response = result???.into_inner();
-            panic!(
-                "ordinary PUT receiver completed before the physical barrier: success={}, error={:?}",
-                response.success, response.error
-            );
-        }
-        arrived = barrier.observed("arrived") => arrived?,
-    }
-    assert!(!destination.exists(), "physical PUT escaped its barrier");
-    rpc.abort();
-    assert!(rpc.await.expect_err("disconnect the ordinary PUT waiter").is_cancelled());
-    // The ordinary PUT wrapper retains an owned task. Observe its RPC-facing
-    // waiter dropping, rather than cancellation of the continuing disk task.
-    barrier.observed("rpc-waiter-dropped").await?;
-
-    let seal = user.request(&dist, 0, PUT_SEAL_ROUTE, request_body.clone());
-    tokio::pin!(seal);
-    let closed = wait_until(
-        Duration::from_secs(15),
-        || async {
-            Ok(walkdir::WalkDir::new(disk.join(".rustfs-artifact-put-seals"))
-                .min_depth(2)
-                .max_depth(2)
-                .into_iter()
-                .filter_map(Result::ok)
-                .any(|entry| entry.file_name() == "closed"))
-        },
-        "PUT receiver persisted closing before physical drain",
-    );
-    tokio::pin!(closed);
-    tokio::select! {
-        result = &mut seal => {
-            let (status, response) = result?;
-            panic!("PUT seal returned before its blocked receiver closed admission: {status} {response}");
-        }
-        result = &mut closed => result?,
-    }
-    let (status, response) = timeout(Duration::from_secs(40), &mut seal).await??;
-    assert_eq!(
-        status,
-        StatusCode::SERVICE_UNAVAILABLE,
-        "PUT seal confirmed an unfinished physical tail: {response}"
-    );
-    assert!(response.contains("artifact operation unconfirmed"));
-    assert!(!destination.exists());
-    assert_raw_put_closed(&put, &node.url, &node.address, &[&staging, &object]).await?;
-    assert!(!destination.exists(), "a second admission published while the worker was paused");
-
-    barrier.release()?;
-    wait_until(
-        Duration::from_secs(15),
-        || async { Ok(destination.join("part.1").is_file()) },
-        "detached ordinary PUT worker finished",
-    )
-    .await?;
-    confirm_seal(&dist, &user, &request_body, &target).await?;
-    // Seal confirmation is the boundary: already admitted work may re-encode
-    // metadata before this point, but the complete FileInfo must be unchanged.
-    let published_objects: Vec<_> = dist
-        .cluster
-        .nodes
-        .iter()
-        .map(|node| PathBuf::from(&node.data_dirs[0]).join(bucket).join(&key))
-        .collect();
-    let published_paths: Vec<_> = published_objects.iter().map(PathBuf::as_path).collect();
-    let sealed_snapshot = publication_snapshot(&published_paths)?;
-    let sealed_file_info =
-        FileMeta::load(&std::fs::read(object.join("xl.meta"))?)?.into_fileinfo(bucket, &key, version, true, false, true)?;
-    assert_eq!(
-        serde_json::to_value(sealed_file_info)?,
-        admitted_file_info,
-        "admitted PUT changed FileInfo semantics"
-    );
-    for (index, baseline) in baselines.iter().enumerate() {
-        assert!(
-            census_object_version_on_disk(Path::new(&dist.cluster.nodes[index].data_dirs[0]), bucket, &key, Some(version))?
-                .matches_manifest(baseline)
-        );
-        assert_exact_version(&dist.client(index)?, bucket, &key, version, &payload).await?;
-    }
-
-    // Stage the exact request again. A source metadata sentinel makes a missing
-    // admission check observable even if the existing destination later causes
-    // ENOTEMPTY: ordinary rename rewrites source metadata before that syscall.
-    let staged_data = staging.join(&data_dir);
-    std::fs::create_dir_all(&staged_data)?;
-    for entry in std::fs::read_dir(&destination)? {
-        let entry = entry?;
-        assert!(entry.file_type()?.is_file(), "unexpected nested shard fixture");
-        std::fs::copy(entry.path(), staged_data.join(entry.file_name()))?;
-    }
-    std::fs::write(staging.join("xl.meta"), b"closed-admission-must-preserve-this-staged-metadata")?;
-    assert_raw_put_closed(&put, &node.url, &node.address, &[&staging, &object]).await?;
-    let listed = dist
-        .client(0)?
-        .list_object_versions()
-        .bucket(bucket)
-        .prefix(&key)
-        .send()
-        .await?;
-    assert_eq!(listed.versions().len(), 1, "raw PUT retries created another version");
-    assert_eq!(listed.versions()[0].version_id(), Some(version));
-    assert!(listed.delete_markers().is_empty());
-    confirm_seal(&dist, &user, &request_body, &target).await?;
-    assert!(
-        publication_snapshot(&published_paths)? == sealed_snapshot,
-        "published files changed after the first successful physical-drain confirmation"
-    );
-    Ok(())
-}

crates/e2e_test/src/distributed/artifact_repair_tail_test.rs

@@ -1,218 +0,0 @@
-// Copyright 2026 RustFS Team
-// SPDX-License-Identifier: Apache-2.0
-
-use super::artifact_seal_test::{assert_exact_version, confirm_seal, create_artifact_attempt};
-use super::harness::{DistCluster, DistLayout, TestResult, payload_for, wait_until};
-use crate::chaos::census_object_version_on_disk;
-use crate::common::init_logging;
-use crate::storage_api::artifact_rpc::{TONIC_RPC_PREFIX, gen_tonic_signature_headers, set_tonic_canonical_body_digest};
-use http::StatusCode;
-use rustfs_filemeta::FileMeta;
-use rustfs_protos::proto_gen::node_service::{
-    ArtifactRepairRenameDataRequest, RenameDataRequest, node_service_client::NodeServiceClient,
-};
-use serde_json::{Value, json};
-use std::path::{Path, PathBuf};
-use std::time::Duration;
-use tokio::time::timeout;
-use tonic::Request;
-use uuid::Uuid;
-
-const REPAIR_SEAL_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/seal-repairs";
-
-pub(super) struct PhysicalBarrier(PathBuf);
-
-impl PhysicalBarrier {
-    pub(super) fn arm(disk: &Path, destination: &Path) -> TestResult<Self> {
-        let path = disk.join(".rustfs.sys/e2e-physical-rename-barrier");
-        std::fs::write(&path, destination.to_str().ok_or("non-UTF8 fixture destination")?)?;
-        Ok(Self(path))
-    }
-
-    pub(super) async fn observed(&self, stage: &str) -> TestResult {
-        wait_until(Duration::from_secs(15), || async { Ok(self.0.with_extension(stage).is_file()) }, stage).await
-    }
-
-    pub(super) fn release(&self) -> TestResult {
-        std::fs::write(self.0.with_extension("release"), b"release physical worker")?;
-        Ok(())
-    }
-}
-
-impl Drop for PhysicalBarrier {
-    fn drop(&mut self) {
-        // Release even on assertion failure; the worker also has its own deadline.
-        let _ = self.release();
-        let _ = std::fs::remove_file(&self.0);
-    }
-}
-
-#[tokio::test]
-async fn four_node_artifact_repair_seal_waits_for_disconnected_physical_tail() -> TestResult {
-    crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks"));
-    init_logging();
-    // Match the other raw internode signature tests sharing the OnceLock.
-    let _ = rustfs_credentials::set_global_rpc_secret("rustfs-internode-signature-e2e-secret".to_owned());
-    let rpc_secret = rustfs_credentials::try_get_rpc_token()?;
-    let dist = DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &[("RUSTFS_RPC_SECRET", &rpc_secret)]).await?;
-    let (target, user) = create_artifact_attempt(&dist).await?;
-    let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
-    let prefix = target["attempt_prefix"].as_str().ok_or("target omitted prefix")?;
-    let key = format!("{prefix}detached-rename.bin");
-    let body = payload_for(&key, 2 * 1024 * 1024);
-    let output = timeout(
-        Duration::from_secs(30),
-        dist.client(1)?
-            .put_object()
-            .bucket(bucket)
-            .key(&key)
-            .body(body.clone().into())
-            .send(),
-    )
-    .await??;
-    let version = output.version_id().ok_or("managed PUT omitted version ID")?;
-    let request_body = json!({"version": 1, "target": target}).to_string();
-    confirm_seal(&dist, &user, &request_body, &target).await?;
-
-    let node = &dist.cluster.nodes[3];
-    let disk = PathBuf::from(&node.data_dirs[0]).canonicalize()?;
-    let object = disk.join(bucket).join(&key);
-    let metadata = FileMeta::load(&std::fs::read(object.join("xl.meta"))?)?;
-    let mut fi = metadata.into_fileinfo(bucket, &key, version, true, false, true)?;
-    fi.set_healing();
-    let data_dir = fi
-        .data_dir
-        .ok_or("physical-tail fixture requires external shard files")?
-        .to_string();
-    let baseline = census_object_version_on_disk(&disk, bucket, &key, Some(version))?;
-    assert!(baseline.is_complete() && !baseline.expected_part_numbers.is_empty());
-    let source = Uuid::new_v4().to_string();
-    let staging = disk.join(".rustfs.sys/tmp").join(&source);
-    std::fs::create_dir_all(&staging)?;
-    let destination = object.join(&data_dir);
-    std::fs::rename(&destination, staging.join(&data_dir))?;
-    let barrier = PhysicalBarrier::arm(&disk, &destination)?;
-    let format: Value = serde_json::from_slice(&std::fs::read(disk.join(".rustfs.sys/format.json"))?)?;
-    let disk_id = Uuid::parse_str(format["xl"]["this"].as_str().ok_or("disk format omitted xl.this")?)?;
-    let incarnation = Uuid::parse_str(target["bucket_incarnation"].as_str().ok_or("target omitted incarnation")?)?;
-    let repair = ArtifactRepairRenameDataRequest {
-        rename: Some(RenameDataRequest {
-            disk: disk_id.to_string(),
-            src_volume: ".rustfs.sys/tmp".to_owned(),
-            src_path: source,
-            dst_volume: bucket.to_owned(),
-            dst_path: key.clone(),
-            file_info: serde_json::to_string(&fi)?,
-            file_info_bin: Default::default(),
-            scanner_publication_lease_token: Default::default(),
-            bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
-        }),
-        expected_disk_id: disk_id.as_bytes().to_vec().into(),
-    };
-    let canonical = rustfs_protos::canonical_artifact_repair_rename_data_request_body(&repair)?;
-    let mut request = Request::new(repair);
-    request.set_timeout(Duration::from_secs(15));
-    set_tonic_canonical_body_digest(&mut request, &canonical)?;
-    let digest = request
-        .metadata()
-        .get("x-rustfs-content-sha256")
-        .ok_or("missing canonical digest")?
-        .to_str()?
-        .to_owned();
-    let headers = gen_tonic_signature_headers(
-        &node.address,
-        TONIC_RPC_PREFIX.trim_start_matches('/'),
-        "ArtifactRepairRenameData",
-        Some(&digest),
-    )?;
-    request.metadata_mut().as_mut().extend(headers);
-    // An independent channel and direct receiver RPC take no coordinator attempt
-    // lock. Dropping this task closes the client stream while rename is paused.
-    let url = node.url.clone();
-    let mut rpc = tokio::spawn(async move {
-        timeout(Duration::from_secs(30), async move {
-            let mut client = NodeServiceClient::connect(url).await?;
-            client
-                .artifact_repair_rename_data(request)
-                .await
-                .map_err(|error| -> Box<dyn std::error::Error + Send + Sync> { Box::new(error) })
-        })
-        .await
-    });
-    tokio::select! {
-        result = &mut rpc => {
-            let response = result???.into_inner();
-            panic!(
-                "repair receiver completed before the physical barrier: success={}, error={:?}",
-                response.success, response.error
-            );
-        }
-        arrived = barrier.observed("arrived") => arrived?,
-    }
-    assert!(!destination.exists(), "physical publication ran before its barrier");
-    rpc.abort();
-    assert!(rpc.await.expect_err("cancel the disconnected RPC waiter").is_cancelled());
-    // This acknowledgement is emitted by the receiver's dropped async waiter,
-    // not by the client and not by a delayed response.
-    barrier.observed("waiter-dropped").await?;
-
-    let seal = user.request(&dist, 0, REPAIR_SEAL_ROUTE, request_body.clone());
-    tokio::pin!(seal);
-    let closed = wait_until(
-        Duration::from_secs(15),
-        || async {
-            Ok(walkdir::WalkDir::new(disk.join(".rustfs-artifact-repair-seals"))
-                .min_depth(2)
-                .max_depth(2)
-                .into_iter()
-                .filter_map(Result::ok)
-                .any(|entry| entry.file_name() == "closed"))
-        },
-        "receiver persisted repair closure while its physical worker remains paused",
-    );
-    tokio::pin!(closed);
-    tokio::select! {
-        result = &mut seal => {
-            let (status, response) = result?;
-            panic!("repair seal returned before the blocked receiver closed admission: {status} {response}");
-        }
-        result = &mut closed => result?,
-    }
-    // Cover a complete receiver drain timeout, so early physical-owner release
-    // causes a successful response here and fails this runtime assertion.
-    let (status, response) = timeout(Duration::from_secs(40), &mut seal).await??;
-    assert_eq!(
-        status,
-        StatusCode::SERVICE_UNAVAILABLE,
-        "seal confirmed an unfinished physical tail: {response}"
-    );
-    assert!(response.contains("artifact operation unconfirmed"));
-    assert!(!destination.exists(), "worker escaped the physical barrier");
-    barrier.release()?;
-    wait_until(
-        Duration::from_secs(15),
-        || async { Ok(destination.join("part.1").is_file()) },
-        "detached physical rename completes after release",
-    )
-    .await?;
-    wait_until(
-        Duration::from_secs(45),
-        || async {
-            let (status, response) = user.request(&dist, 0, REPAIR_SEAL_ROUTE, request_body.clone()).await?;
-            if status == StatusCode::SERVICE_UNAVAILABLE && response.contains("artifact operation unconfirmed") {
-                return Ok(false);
-            }
-            assert_eq!(status, StatusCode::OK, "repair seal did not confirm after physical drain: {response}");
-            assert_eq!(
-                serde_json::from_str::<Value>(&response)?,
-                json!({"version": 1, "target": target, "attempt_repairs_sealed": true})
-            );
-            Ok(true)
-        },
-        "exact retry confirms all four receivers after physical drain",
-    )
-    .await?;
-    assert!(census_object_version_on_disk(&disk, bucket, &key, Some(version))?.matches_manifest(&baseline));
-    assert_exact_version(&dist.client(0)?, bucket, &key, version, &body).await?;
-    Ok(())
-}

crates/e2e_test/src/distributed/artifact_repair_test.rs

@@ -1,327 +0,0 @@
-// Copyright 2026 RustFS Team
-// SPDX-License-Identifier: Apache-2.0
-
-use super::artifact_seal_test::{ArtifactAdmin, assert_exact_version, confirm_seal, create_artifact_attempt};
-use super::harness::{
-    DistCluster, DistLayout, TestResult, enable_versioning, payload_for, unique_bucket, wait_for_ready, wait_until,
-};
-use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post, start_root_heal_when_control_ready};
-use crate::common::init_logging;
-use http::StatusCode;
-use serde_json::{Value, json};
-use std::path::{Path, PathBuf};
-use std::time::Duration;
-use tokio::time::timeout;
-
-const REPAIR_SEAL_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/seal-repairs";
-const HEAL_BODY: &str =
-    r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
-const HEAL_BUDGET: Duration = Duration::from_secs(120);
-
-pub(super) struct RepairObject {
-    pub(super) key: String,
-    pub(super) version: String,
-    pub(super) body: Vec<u8>,
-    manifests: Vec<VersionShardCensus>,
-}
-
-fn roots(dist: &DistCluster) -> Vec<PathBuf> {
-    dist.cluster
-        .nodes
-        .iter()
-        .map(|node| PathBuf::from(&node.data_dirs[0]))
-        .collect()
-}
-
-impl RepairObject {
-    pub(super) async fn put(dist: &DistCluster, bucket: &str, key: String) -> TestResult<Self> {
-        let body = payload_for(&key, 2 * 1024 * 1024);
-        let output = timeout(
-            Duration::from_secs(30),
-            dist.client(1)?
-                .put_object()
-                .bucket(bucket)
-                .key(&key)
-                .body(body.clone().into())
-                .send(),
-        )
-        .await??;
-        let version = output.version_id().ok_or("managed PUT omitted version ID")?.to_owned();
-        let manifests = roots(dist)
-            .iter()
-            .map(|disk| census_object_version_on_disk(disk, bucket, &key, Some(&version)))
-            .collect::<TestResult<Vec<_>>>()?;
-        for manifest in &manifests {
-            assert!(manifest.is_complete(), "repair needs a complete baseline: {manifest:?}");
-            assert!(!manifest.expected_part_numbers.is_empty(), "repair must use actual shard files");
-        }
-        Ok(Self {
-            key,
-            version,
-            body,
-            manifests,
-        })
-    }
-
-    pub(super) fn part(&self, disk: &Path, bucket: &str, node: usize) -> TestResult<PathBuf> {
-        let data_dir = self.manifests[node]
-            .data_dir
-            .as_ref()
-            .ok_or("external shard omitted data directory")?;
-        Ok(disk.join(bucket).join(&self.key).join(data_dir).join("part.1"))
-    }
-
-    pub(super) async fn assert_repaired(&self, dist: &DistCluster, bucket: &str) -> TestResult {
-        wait_until(
-            HEAL_BUDGET,
-            || async {
-                for (disk, baseline) in roots(dist).iter().zip(&self.manifests) {
-                    if !census_object_version_on_disk(disk, bucket, &self.key, Some(&self.version))?.matches_manifest(baseline) {
-                        return Ok(false);
-                    }
-                }
-                Ok(true)
-            },
-            "every configured receiver restored the exact version and original shard fingerprints",
-        )
-        .await?;
-        for node in 0..dist.cluster.nodes.len() {
-            assert_exact_version(&dist.client(node)?, bucket, &self.key, &self.version, &self.body).await?;
-        }
-        Ok(())
-    }
-}
-
-pub(super) async fn start_heal(dist: &DistCluster, bucket: &str, prefix: &str, remove: bool) -> TestResult<String> {
-    let url = format!("{}/rustfs/admin/v3/heal/{bucket}/{prefix}?forceStart=true", dist.cluster.nodes[0].url);
-    let body = if remove {
-        HEAL_BODY.replace("\"remove\":false", "\"remove\":true")
-    } else {
-        HEAL_BODY.to_owned()
-    };
-    let response = timeout(
-        HEAL_BUDGET,
-        start_root_heal_when_control_ready(&url, &body, &dist.cluster.access_key, &dist.cluster.secret_key),
-    )
-    .await??;
-    let response: Value = serde_json::from_str(&response)?;
-    Ok(response["clientToken"]
-        .as_str()
-        .ok_or("heal omitted its task token")?
-        .to_owned())
-}
-
-pub(super) async fn finish_heal(dist: &DistCluster, token: &str, rejected_key: Option<&str>) -> TestResult {
-    let url = format!("{}/rustfs/admin/v3/heal/?clientToken={token}", dist.cluster.nodes[0].url);
-    wait_until(
-        HEAL_BUDGET,
-        || async {
-            let response = signed_admin_post(&url, None, &dist.cluster.access_key, &dist.cluster.secret_key).await?;
-            let status: Value = serde_json::from_str(&response)?;
-            if status["summary"] == "running" {
-                return Ok(false);
-            }
-            if let Some(key) = rejected_key {
-                assert_eq!(status["summary"], "stopped", "closed repair must report failure: {status}");
-                assert!(
-                    status["outcome"]["counters"]["failed"]
-                        .as_u64()
-                        .is_some_and(|count| count > 0)
-                );
-                assert!(
-                    status["outcome"]["objects"].as_array().is_some_and(|objects| objects
-                        .iter()
-                        .any(|object| { object["identity"]["object"] == key && object["disposition"]["state"] == "failed" })),
-                    "heal must actually attempt the rejected object: {status}"
-                );
-            } else {
-                assert_eq!(status["summary"], "finished", "heal did not complete: {status}");
-            }
-            Ok(true)
-        },
-        "exact admin heal task reached its expected terminal outcome",
-    )
-    .await
-}
-
-async fn repair_seal(dist: &DistCluster, user: &ArtifactAdmin, target: &Value, confirmed: bool) -> TestResult {
-    wait_until(
-        Duration::from_secs(45),
-        || async {
-            let request = json!({"version": 1, "target": target}).to_string();
-            let (status, body) = user.request(dist, 0, REPAIR_SEAL_ROUTE, request).await?;
-            if !confirmed {
-                assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "offline receiver repair seal: {body}");
-                assert!(body.contains("<Code>ServiceUnavailable</Code>"));
-                assert!(body.contains("artifact operation unconfirmed"));
-                return Ok(true);
-            }
-            if status == StatusCode::SERVICE_UNAVAILABLE
-                && body.contains("<Code>ServiceUnavailable</Code>")
-                && body.contains("<Message>artifact operation unconfirmed; retry the same identity</Message>")
-            {
-                return Ok(false);
-            }
-            assert_eq!(status, StatusCode::OK, "repair seal failed: {body}");
-            assert_eq!(
-                serde_json::from_str::<Value>(&body)?,
-                json!({"version": 1, "target": target, "attempt_repairs_sealed": true})
-            );
-            Ok(true)
-        },
-        "repair seal confirms every configured receiver",
-    )
-    .await
-}
-
-#[tokio::test]
-async fn four_node_artifact_shard_and_minority_metadata_repair_after_put_seal() -> TestResult {
-    init_logging();
-    // Sub-quorum metadata recovery supports genuine legacy shards only.
-    let mut dist =
-        DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &[("RUSTFS_SHARD_INTEGRITY_WRITE", "false")]).await?;
-    let (target, user) = create_artifact_attempt(&dist).await?;
-    let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
-    let prefix = target["attempt_prefix"].as_str().ok_or("target omitted prefix")?;
-    let shard = RepairObject::put(&dist, bucket, format!("{prefix}shard.bin")).await?;
-    let metadata = RepairObject::put(&dist, bucket, format!("{prefix}metadata.bin")).await?;
-    let later = dist
-        .client(2)?
-        .put_object()
-        .bucket(bucket)
-        .key(&shard.key)
-        .body(b"newer-version".to_vec().into())
-        .send()
-        .await?;
-    let later_version = later.version_id().ok_or("second PUT omitted version ID")?.to_owned();
-    assert_ne!(shard.version, later_version);
-    confirm_seal(&dist, &user, &json!({"version": 1, "target": target}).to_string(), &target).await?;
-
-    dist.cluster.stop();
-    let disks = roots(&dist);
-    std::fs::remove_file(shard.part(&disks[3], bucket, 3)?)?;
-    for disk in &disks[1..] {
-        std::fs::remove_file(disk.join(bucket).join(&metadata.key).join("xl.meta"))?;
-    }
-    assert!(!census_object_version_on_disk(&disks[3], bucket, &shard.key, Some(&shard.version))?.is_complete());
-    assert!(disks[0].join(bucket).join(&metadata.key).join("xl.meta").is_file());
-    assert!(
-        disks[1..]
-            .iter()
-            .all(|disk| !disk.join(bucket).join(&metadata.key).join("xl.meta").exists())
-    );
-    dist.cluster.start().await?;
-    wait_for_ready(&dist.cluster).await?;
-    let token = start_heal(&dist, bucket, prefix, false).await?;
-    shard.assert_repaired(&dist, bucket).await?;
-    metadata.assert_repaired(&dist, bucket).await?;
-    finish_heal(&dist, &token, None).await?;
-    assert_exact_version(&dist.client(3)?, bucket, &shard.key, &later_version, b"newer-version").await?;
-    let listed = dist
-        .client(0)?
-        .list_object_versions()
-        .bucket(bucket)
-        .prefix(prefix)
-        .send()
-        .await?;
-    assert_eq!(listed.versions().len(), 3, "repair must not publish replacement versions");
-    assert!(listed.delete_markers().is_empty());
-    Ok(())
-}
-
-#[tokio::test]
-async fn four_node_artifact_repair_seal_denies_shard_and_metadata_repair_after_restart() -> TestResult {
-    init_logging();
-    let mut dist =
-        DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &[("RUSTFS_SHARD_INTEGRITY_WRITE", "false")]).await?;
-    let (target, user) = create_artifact_attempt(&dist).await?;
-    let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
-    let prefix = target["attempt_prefix"].as_str().ok_or("target omitted prefix")?;
-    let shard = RepairObject::put(&dist, bucket, format!("{prefix}shard.bin")).await?;
-    let metadata = RepairObject::put(&dist, bucket, format!("{prefix}metadata.bin")).await?;
-    confirm_seal(&dist, &user, &json!({"version": 1, "target": target}).to_string(), &target).await?;
-    repair_seal(&dist, &user, &target, true).await?;
-    let ordinary_bucket = unique_bucket("repair-control");
-    dist.create_bucket(&ordinary_bucket).await?;
-    enable_versioning(&dist.client(0)?, &ordinary_bucket).await?;
-    let ordinary = RepairObject::put(&dist, &ordinary_bucket, "ordinary.bin".to_owned()).await?;
-    let disks = roots(&dist);
-    let part = shard.part(&disks[3], bucket, 3)?;
-    std::fs::remove_file(&part)?;
-    for disk in &disks[1..] {
-        std::fs::remove_file(disk.join(bucket).join(&metadata.key).join("xl.meta"))?;
-    }
-    for restarted in [false, true] {
-        if restarted {
-            dist.restart_with_current_binary().await?;
-        }
-        // The same receivers must still execute an ordinary bucket repair.
-        std::fs::remove_file(ordinary.part(&disks[3], &ordinary_bucket, 3)?)?;
-        let control = start_heal(&dist, &ordinary_bucket, &ordinary.key, false).await?;
-        ordinary.assert_repaired(&dist, &ordinary_bucket).await?;
-        finish_heal(&dist, &control, None).await?;
-        for object in [&shard, &metadata] {
-            let token = start_heal(&dist, bucket, &object.key, false).await?;
-            finish_heal(&dist, &token, Some(&object.key)).await?;
-        }
-        assert!(!part.exists(), "sealed repair recreated the missing shard");
-        assert!(
-            disks[1..]
-                .iter()
-                .all(|disk| !disk.join(bucket).join(&metadata.key).join("xl.meta").exists())
-        );
-        assert_exact_version(&dist.client(2)?, bucket, &shard.key, &shard.version, &shard.body).await?;
-    }
-    Ok(())
-}
-
-#[tokio::test]
-async fn four_node_artifact_repair_seal_requires_every_receiver_on_exact_retry() -> TestResult {
-    init_logging();
-    let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
-    let (target, user) = create_artifact_attempt(&dist).await?;
-    let request = json!({"version": 1, "target": target}).to_string();
-    confirm_seal(&dist, &user, &request, &target).await?;
-    // Repeat after global confirmation: its persisted receipt cannot replace a
-    // fresh acknowledgement from an offline configured receiver.
-    for _ in 0..2 {
-        dist.cluster.stop_node(3)?;
-        repair_seal(&dist, &user, &target, false).await?;
-        dist.cluster.start_node(3).await?;
-        wait_for_ready(&dist.cluster).await?;
-        repair_seal(&dist, &user, &target, true).await?;
-    }
-    Ok(())
-}
-
-#[tokio::test]
-async fn four_node_artifact_dangling_repair_preserves_surviving_metadata() -> TestResult {
-    init_logging();
-    let mut dist =
-        DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &[("RUSTFS_SHARD_INTEGRITY_WRITE", "false")]).await?;
-    let (target, user) = create_artifact_attempt(&dist).await?;
-    let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
-    let prefix = target["attempt_prefix"].as_str().ok_or("target omitted prefix")?;
-    let object = RepairObject::put(&dist, bucket, format!("{prefix}dangling.bin")).await?;
-    confirm_seal(&dist, &user, &json!({"version": 1, "target": target}).to_string(), &target).await?;
-    dist.cluster.stop();
-    let disks = roots(&dist);
-    let survivor = disks[0].join(bucket).join(&object.key).join("xl.meta");
-    let before = std::fs::read(&survivor)?;
-    for (node, disk) in disks.iter().enumerate() {
-        std::fs::remove_file(object.part(disk, bucket, node)?)?;
-        if node != 0 {
-            std::fs::remove_file(disk.join(bucket).join(&object.key).join("xl.meta"))?;
-        }
-    }
-    dist.cluster.start().await?;
-    wait_for_ready(&dist.cluster).await?;
-    let token = start_heal(&dist, bucket, &object.key, true).await?;
-    finish_heal(&dist, &token, Some(&object.key)).await?;
-    assert_eq!(
-        std::fs::read(survivor)?,
-        before,
-        "distributed repair must not authorize managed dangling deletion"
-    );
-    Ok(())
-}

crates/e2e_test/src/distributed/artifact_seal_test.rs

@@ -1,347 +0,0 @@
-// Copyright 2026 RustFS Team
-// SPDX-License-Identifier: Apache-2.0
-
-use super::harness::{
-    DistCluster, DistLayout, TestResult, assert_object_bytes, put_object, s3_probe_client, unique_bucket, wait_for_ready,
-    wait_until,
-};
-use crate::common::{
-    AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, admin_request, init_logging,
-};
-use aws_sdk_s3::Client;
-use aws_sdk_s3::error::ProvideErrorMetadata;
-use http::{Method, StatusCode};
-use serde_json::{Value, json};
-use std::time::Duration;
-use tokio::time::timeout;
-use uuid::Uuid;
-
-const BUCKET_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/buckets";
-const INSPECT_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/buckets/inspect";
-const SEAL_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/seal-attempt";
-const ADMIN_BUDGET: Duration = Duration::from_secs(45);
-
-pub(super) struct ArtifactAdmin {
-    access_key: String,
-    secret_key: String,
-}
-
-impl ArtifactAdmin {
-    pub(super) async fn request(
-        &self,
-        dist: &DistCluster,
-        node: usize,
-        route: &str,
-        body: String,
-    ) -> TestResult<(StatusCode, String)> {
-        timeout(
-            ADMIN_BUDGET,
-            admin_request(
-                &dist.cluster.nodes[node].url,
-                Method::POST,
-                route,
-                Some(body),
-                &self.access_key,
-                &self.secret_key,
-            ),
-        )
-        .await?
-    }
-}
-
-async fn create_artifact_admin(dist: &DistCluster, bucket: &str, prefix: &str) -> TestResult<ArtifactAdmin> {
-    let user = ArtifactAdmin {
-        access_key: format!("artifact{}", &Uuid::new_v4().simple().to_string()[..12]),
-        secret_key: Uuid::new_v4().simple().to_string(),
-    };
-    let policy_name = format!("artifact{}", &Uuid::new_v4().simple().to_string()[..12]);
-    let policy = json!({
-        "Version": "2012-10-17",
-        "Statement": [
-            {
-                "Effect": "Allow",
-                "Action": ["admin:CreatePluginArtifactBucket", "admin:GetPluginArtifactBucket"],
-                "Resource": format!("arn:aws:s3:::{bucket}"),
-            },
-            {
-                "Effect": "Allow",
-                "Action": ["admin:SealPluginArtifactAttempt", "admin:SealPluginArtifactRepairs", "admin:SealPluginArtifactWrites"],
-                "Resource": format!("arn:aws:s3:::{bucket}/{}", prefix.trim_end_matches('/')),
-            },
-        ],
-    })
-    .to_string();
-    let url = &dist.cluster.nodes[0].url;
-    let access = &dist.cluster.access_key;
-    let secret = &dist.cluster.secret_key;
-    admin_create_user_via(AdminTransport::Signed, url, access, secret, &user.access_key, &user.secret_key).await?;
-    admin_add_canned_policy_via(AdminTransport::Signed, url, access, secret, &policy_name, &policy).await?;
-    admin_attach_user_policy_via(AdminTransport::Signed, url, access, secret, &policy_name, &user.access_key).await?;
-    Ok(user)
-}
-
-pub(super) async fn create_artifact_attempt(dist: &DistCluster) -> TestResult<(Value, ArtifactAdmin)> {
-    let bucket = unique_bucket("artifact");
-    let prefix = format!("outputs/6169/{}/{}/", Uuid::new_v4(), Uuid::new_v4());
-    let user = create_artifact_admin(dist, &bucket, &prefix).await?;
-    let body = json!({"version": 1, "bucket": bucket, "base_prefix": "outputs/"}).to_string();
-    let (status, response) = user.request(dist, 0, BUCKET_ROUTE, body).await?;
-    assert_eq!(status, StatusCode::OK, "artifact bucket creation failed: {response}");
-    let receipt: Value = serde_json::from_str(&response)?;
-    assert_eq!(receipt["version"], 1);
-    assert_eq!(receipt["bucket"], bucket);
-    assert_eq!(receipt["base_prefix"], "outputs/");
-    assert_eq!(receipt["versioning_enabled"], true);
-    let incarnation = receipt["bucket_incarnation"]
-        .as_str()
-        .ok_or("artifact receipt omitted incarnation")?;
-    assert!(!Uuid::parse_str(incarnation)?.is_nil());
-    let target = json!({
-        "bucket": bucket,
-        "bucket_incarnation": incarnation,
-        "attempt_prefix": prefix,
-    });
-    assert_bucket_identity(dist, &user, 1, &target).await?;
-    Ok((target, user))
-}
-
-async fn assert_bucket_identity(dist: &DistCluster, user: &ArtifactAdmin, node: usize, target: &Value) -> TestResult {
-    let body = json!({"version": 1, "bucket": target["bucket"], "base_prefix": "outputs/"}).to_string();
-    wait_until(
-        ADMIN_BUDGET,
-        || async {
-            let (status, response) = user.request(dist, node, INSPECT_ROUTE, body.clone()).await?;
-            // IAM peers may not have received the new user and policy yet.
-            // Storage failures must fail immediately, including the old gate's 503.
-            if matches!(status, StatusCode::FORBIDDEN | StatusCode::UNAUTHORIZED)
-                && (response.contains("<Code>AccessDenied</Code>") || response.contains("<Code>InvalidAccessKeyId</Code>"))
-            {
-                return Ok(false);
-            }
-            assert_eq!(status, StatusCode::OK, "cross-node artifact inspect failed: {response}");
-            assert_eq!(
-                serde_json::from_str::<Value>(&response)?,
-                json!({
-                    "version": 1,
-                    "bucket": target["bucket"],
-                    "base_prefix": "outputs/",
-                    "bucket_incarnation": target["bucket_incarnation"],
-                    "versioning_enabled": true,
-                }),
-                "inspect must return the created managed bucket identity"
-            );
-            Ok(true)
-        },
-        "artifact IAM user and policy propagation",
-    )
-    .await
-}
-
-pub(super) async fn confirm_seal(dist: &DistCluster, user: &ArtifactAdmin, request: &str, target: &Value) -> TestResult {
-    wait_until(
-        ADMIN_BUDGET,
-        || async {
-            let (status, response) = user.request(dist, 0, SEAL_ROUTE, request.to_owned()).await?;
-            // HTTP readiness does not confirm that remote disk handles have
-            // recovered. Retry only the API's explicit same-identity response.
-            if status == StatusCode::SERVICE_UNAVAILABLE
-                && response.contains("<Code>ServiceUnavailable</Code>")
-                && response.contains("<Message>artifact operation unconfirmed; retry the same identity</Message>")
-            {
-                return Ok(false);
-            }
-            assert_eq!(status, StatusCode::OK, "artifact seal failed: {response}");
-            let receipt: Value = serde_json::from_str(&response)?;
-            assert_eq!(receipt, json!({"version": 1, "target": target, "ordinary_put_sealed": true}));
-            Ok(true)
-        },
-        "exact artifact seal confirmation after receiver recovery",
-    )
-    .await
-}
-
-async fn assert_seal_unconfirmed(dist: &DistCluster, user: &ArtifactAdmin, request: &str) -> TestResult {
-    let (status, body) = user.request(dist, 0, SEAL_ROUTE, request.to_owned()).await?;
-    assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "offline receiver seal response: {body}");
-    assert!(
-        body.contains("<Code>ServiceUnavailable</Code>"),
-        "seal failure lost its service code: {body}"
-    );
-    assert!(
-        body.contains("artifact operation unconfirmed"),
-        "seal failure lost its retry contract: {body}"
-    );
-    Ok(())
-}
-
-pub(super) async fn assert_exact_version(client: &Client, bucket: &str, key: &str, version: &str, body: &[u8]) -> TestResult {
-    let output = timeout(
-        Duration::from_secs(30),
-        client.get_object().bucket(bucket).key(key).version_id(version).send(),
-    )
-    .await??;
-    assert_eq!(output.version_id(), Some(version));
-    let actual = timeout(Duration::from_secs(30), output.body.collect()).await??.into_bytes();
-    assert_eq!(actual.as_ref(), body);
-    Ok(())
-}
-
-async fn assert_put_denied(client: &Client, ordinary_bucket: &str, bucket: &str, key: &str) -> TestResult {
-    // The same receiver must still accept a normal write before its sealed probe.
-    put_object(client, ordinary_bucket, key, b"ordinary-control".to_vec()).await?;
-    assert_object_bytes(client, ordinary_bucket, key, b"ordinary-control").await?;
-    let client = s3_probe_client(client);
-    let result = timeout(
-        Duration::from_secs(30),
-        client
-            .put_object()
-            .bucket(bucket)
-            .key(key)
-            .body(b"must-not-publish".to_vec().into())
-            .send(),
-    )
-    .await?;
-    let error = result.expect_err("sealed artifact attempt accepted a new PUT");
-    let service = error
-        .as_service_error()
-        .ok_or("sealed PUT failed without a service rejection")?;
-    assert_eq!(
-        error.raw_response().map(|response| response.status().as_u16()),
-        Some(500),
-        "sealed PUT must return its current HTTP status"
-    );
-    assert_eq!(service.code(), Some("InternalError"), "sealed PUT must return its current service code");
-    // StorageError::Io is redacted at the API boundary; the sealed reason is
-    // not exposed. Success controls and exact inventory checks prove the effect.
-    assert_eq!(
-        service.message(),
-        Some("We encountered an internal error, please try again."),
-        "sealed PUT must retain the public redacted message"
-    );
-    Ok(())
-}
-
-#[tokio::test]
-async fn four_node_artifact_put_seal_survives_full_restart() -> TestResult {
-    init_logging();
-    let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
-    let (target, user) = create_artifact_attempt(&dist).await?;
-    let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
-    let prefix = target["attempt_prefix"].as_str().ok_or("target omitted attempt prefix")?;
-    let key = format!("{prefix}result.bin");
-    let mut versions = Vec::new();
-    for (node, body) in [(1, b"first-version".as_slice()), (2, b"second-version".as_slice())] {
-        let client = dist.client(node)?;
-        let output = timeout(
-            Duration::from_secs(30),
-            client.put_object().bucket(bucket).key(&key).body(body.to_vec().into()).send(),
-        )
-        .await??;
-        let version = output.version_id().ok_or("artifact PUT omitted version ID")?.to_owned();
-        assert!(!Uuid::parse_str(&version)?.is_nil());
-        assert_exact_version(&dist.client(3)?, bucket, &key, &version, body).await?;
-        versions.push((version, body.to_vec()));
-    }
-    assert_ne!(versions[0].0, versions[1].0);
-    let ordinary = unique_bucket("ordinary");
-    dist.create_bucket(&ordinary).await?;
-    put_object(&dist.client(2)?, &ordinary, "normal.bin", b"before-seal".to_vec()).await?;
-    assert_object_bytes(&dist.client(3)?, &ordinary, "normal.bin", b"before-seal").await?;
-
-    let request = json!({"version": 1, "target": target}).to_string();
-    confirm_seal(&dist, &user, &request, &target).await?;
-    let other_prefix = format!("outputs/6169/{}/{}/", Uuid::new_v4(), Uuid::new_v4());
-    for phase in ["sealed", "restarted"] {
-        if phase == "restarted" {
-            dist.cluster.stop();
-            dist.cluster.start().await?;
-            wait_for_ready(&dist.cluster).await?;
-        }
-        assert_bucket_identity(&dist, &user, 2, &target).await?;
-        for node in 0..dist.cluster.nodes.len() {
-            // A second attempt in the managed bucket must remain writable;
-            // a broken bucket-wide write path cannot stand in for sealing.
-            let client = dist.client(node)?;
-            let other_key = format!("{other_prefix}{phase}-{node}.bin");
-            let output = timeout(
-                Duration::from_secs(30),
-                client
-                    .put_object()
-                    .bucket(bucket)
-                    .key(&other_key)
-                    .body(b"other-attempt".to_vec().into())
-                    .send(),
-            )
-            .await??;
-            let version = output.version_id().ok_or("other attempt PUT omitted version ID")?;
-            assert!(!Uuid::parse_str(version)?.is_nil());
-            assert_exact_version(
-                &dist.client((node + 1) % dist.cluster.nodes.len())?,
-                bucket,
-                &other_key,
-                version,
-                b"other-attempt",
-            )
-            .await?;
-            assert_put_denied(&dist.client(node)?, &ordinary, bucket, &format!("{prefix}{phase}-{node}.bin")).await?;
-            for (version, body) in &versions {
-                assert_exact_version(&dist.client(node)?, bucket, &key, version, body).await?;
-            }
-        }
-        let listed = dist
-            .client(0)?
-            .list_object_versions()
-            .bucket(bucket)
-            .prefix(prefix)
-            .send()
-            .await?;
-        assert_eq!(listed.versions().len(), 2, "denied PUT left an artifact version: {listed:?}");
-        assert!(listed.delete_markers().is_empty(), "artifact seal created a delete marker");
-        put_object(&dist.client(1)?, &ordinary, "normal.bin", phase.as_bytes().to_vec()).await?;
-        assert_object_bytes(&dist.client(2)?, &ordinary, "normal.bin", phase.as_bytes()).await?;
-    }
-    Ok(())
-}
-
-#[tokio::test]
-async fn four_node_artifact_seal_requires_every_configured_receiver_on_exact_retry() -> TestResult {
-    init_logging();
-    let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
-    let (target, user) = create_artifact_attempt(&dist).await?;
-    let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
-    let prefix = target["attempt_prefix"].as_str().ok_or("target omitted attempt prefix")?;
-    let key = format!("{prefix}existing.bin");
-    let output = dist
-        .client(1)?
-        .put_object()
-        .bucket(bucket)
-        .key(&key)
-        .body(b"existing".to_vec().into())
-        .send()
-        .await?;
-    let version = output.version_id().ok_or("artifact PUT omitted version ID")?.to_owned();
-    assert_exact_version(&dist.client(3)?, bucket, &key, &version, b"existing").await?;
-    let ordinary = unique_bucket("ordinary");
-    dist.create_bucket(&ordinary).await?;
-    let request = json!({"version": 1, "target": target}).to_string();
-
-    // The second iteration starts from a fully acknowledged global seal. Its
-    // persisted record cannot replace a fresh acknowledgement from every disk.
-    for _ in 0..2 {
-        dist.cluster.stop_node(3)?;
-        assert_seal_unconfirmed(&dist, &user, &request).await?;
-        dist.cluster.start_node(3).await?;
-        wait_for_ready(&dist.cluster).await?;
-        confirm_seal(&dist, &user, &request, &target).await?;
-    }
-    assert_put_denied(&dist.client(3)?, &ordinary, bucket, &format!("{prefix}late.bin")).await?;
-    assert_exact_version(&dist.client(2)?, bucket, &key, &version, b"existing").await?;
-    let listed = timeout(
-        Duration::from_secs(30),
-        dist.client(0)?.list_object_versions().bucket(bucket).prefix(prefix).send(),
-    )
-    .await??;
-    assert_eq!(listed.versions().len(), 1, "rejected PUT added a version");
-    assert!(listed.delete_markers().is_empty(), "seal added a delete marker");
-    Ok(())
-}

crates/e2e_test/src/distributed/artifact_writes_test.rs

@@ -1,235 +0,0 @@
-// Copyright 2026 RustFS Team
-// SPDX-License-Identifier: Apache-2.0
-
-#![cfg(unix)]
-
-use super::artifact_delete_test::physical_snapshot;
-use super::artifact_repair_test::{RepairObject, finish_heal, start_heal};
-use super::artifact_seal_test::{ArtifactAdmin, assert_exact_version, confirm_seal, create_artifact_attempt};
-use super::harness::{
-    DistCluster, DistLayout, TestResult, enable_versioning, s3_probe_client, unique_bucket, wait_for_ready, wait_until,
-};
-use crate::common::init_logging;
-use aws_sdk_s3::error::ProvideErrorMetadata;
-use aws_sdk_s3::types::{
-    ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockRetention,
-    ObjectLockRetentionMode,
-};
-use http::StatusCode;
-use serde_json::{Value, json};
-use std::path::Path;
-use std::time::Duration;
-
-const WRITES_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/seal-writes";
-const INTERNAL_MESSAGE: &str = "We encountered an internal error, please try again.";
-
-async fn seal_writes(dist: &DistCluster, user: &ArtifactAdmin, target: &Value, confirmed: bool) -> TestResult {
-    wait_until(Duration::from_secs(45), || async {
-        let (status, response) = user.request(dist, 0, WRITES_ROUTE, json!({"version":1,"target":target}).to_string()).await?;
-        if status == StatusCode::SERVICE_UNAVAILABLE && response.contains("artifact operation unconfirmed; retry the same identity") {
-            return Ok(!confirmed);
-        }
-        assert!(confirmed, "offline configured receiver produced a complete receipt: {status} {response}");
-        assert_eq!(status, StatusCode::OK, "combined writes seal: {response}");
-        assert_eq!(serde_json::from_str::<Value>(&response)?, json!({"version":1,"target":target,"ordinary_put_sealed":true,"attempt_repairs_sealed":true,"all_writes_sealed":true}));
-        Ok(true)
-    }, "combined seal acknowledges both domains on every receiver").await
-}
-
-async fn assert_put_closed(dist: &DistCluster, bucket: &str, key: &str) -> TestResult {
-    let before = physical_snapshot(dist, bucket)?;
-    let client = s3_probe_client(&dist.client(2)?);
-    let error = client
-        .put_object()
-        .bucket(bucket)
-        .key(key)
-        .body(b"must-not-publish".to_vec().into())
-        .send()
-        .await
-        .expect_err("combined seal admitted a new ordinary PUT");
-    let service = error.as_service_error().ok_or("PUT was not a service rejection")?;
-    assert_eq!(service.code(), Some("InternalError"));
-    assert_eq!(service.message(), Some(INTERNAL_MESSAGE));
-    assert!(
-        physical_snapshot(dist, bucket)? == before,
-        "rejected PUT changed the managed physical namespace"
-    );
-    Ok(())
-}
-
-async fn lock_metadata_entrypoints(dist: &DistCluster, managed: &Value, object: &RepairObject, ordinary: &str) -> TestResult {
-    let bucket = managed["bucket"].as_str().ok_or("bucket identity")?;
-    let client = s3_probe_client(&dist.client(0)?);
-    for bucket in [bucket, ordinary] {
-        client
-            .put_object_lock_configuration()
-            .bucket(bucket)
-            .object_lock_configuration(
-                ObjectLockConfiguration::builder()
-                    .object_lock_enabled(ObjectLockEnabled::Enabled)
-                    .build(),
-            )
-            .send()
-            .await?;
-        wait_until(
-            Duration::from_secs(30),
-            || async {
-                for node in 0..4 {
-                    let output = match dist.client(node)?.get_object_lock_configuration().bucket(bucket).send().await {
-                        Ok(output) => output,
-                        Err(error)
-                            if error.as_service_error().and_then(ProvideErrorMetadata::code)
-                                == Some("ObjectLockConfigurationNotFoundError") =>
-                        {
-                            return Ok(false);
-                        }
-                        Err(error) => return Err(error.into()),
-                    };
-                    if output
-                        .object_lock_configuration()
-                        .and_then(|configuration| configuration.object_lock_enabled())
-                        != Some(&ObjectLockEnabled::Enabled)
-                    {
-                        return Ok(false);
-                    }
-                }
-                Ok(true)
-            },
-            "actual Object Lock configuration is enabled on all request nodes",
-        )
-        .await?;
-    }
-    let control = RepairObject::put(dist, ordinary, "lock-control.bin".to_owned()).await?;
-    let retention = ObjectLockRetention::builder()
-        .mode(ObjectLockRetentionMode::Governance)
-        .retain_until_date(aws_sdk_s3::primitives::DateTime::from_secs(chrono::Utc::now().timestamp() + 86400))
-        .build();
-    let hold = ObjectLockLegalHold::builder().status(ObjectLockLegalHoldStatus::On).build();
-    client
-        .put_object_retention()
-        .bucket(ordinary)
-        .key(&control.key)
-        .version_id(&control.version)
-        .retention(retention.clone())
-        .send()
-        .await?;
-    assert_eq!(
-        client
-            .get_object_retention()
-            .bucket(ordinary)
-            .key(&control.key)
-            .version_id(&control.version)
-            .send()
-            .await?
-            .retention()
-            .and_then(|value| value.mode()),
-        Some(&ObjectLockRetentionMode::Governance)
-    );
-    client
-        .put_object_legal_hold()
-        .bucket(ordinary)
-        .key(&control.key)
-        .version_id(&control.version)
-        .legal_hold(hold.clone())
-        .send()
-        .await?;
-    assert_eq!(
-        client
-            .get_object_legal_hold()
-            .bucket(ordinary)
-            .key(&control.key)
-            .version_id(&control.version)
-            .send()
-            .await?
-            .legal_hold()
-            .and_then(|value| value.status()),
-        Some(&ObjectLockLegalHoldStatus::On)
-    );
-    let before = physical_snapshot(dist, bucket)?;
-    let error = client
-        .put_object_retention()
-        .bucket(bucket)
-        .key(&object.key)
-        .version_id(&object.version)
-        .retention(retention)
-        .send()
-        .await
-        .expect_err("managed Retention must reach and fail its unsupported metadata write");
-    let service = error.as_service_error().ok_or("retention was not a service rejection")?;
-    assert_eq!(service.code(), Some("InternalError"));
-    assert_eq!(service.message(), Some(INTERNAL_MESSAGE));
-    assert!(physical_snapshot(dist, bucket)? == before, "retention changed managed metadata");
-    let error = client
-        .put_object_legal_hold()
-        .bucket(bucket)
-        .key(&object.key)
-        .version_id(&object.version)
-        .legal_hold(hold)
-        .send()
-        .await
-        .expect_err("managed LegalHold must reach and fail its unsupported metadata write");
-    let service = error.as_service_error().ok_or("legal hold was not a service rejection")?;
-    assert_eq!(service.code(), Some("InternalError"));
-    assert_eq!(service.message(), Some("Io error: artifact bucket does not support this write operation"));
-    assert!(physical_snapshot(dist, bucket)? == before, "legal hold changed managed metadata");
-    Ok(())
-}
-
-#[tokio::test]
-async fn four_node_artifact_writes_seal_first_partial_retry_offline_and_restart() -> TestResult {
-    init_logging();
-    let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
-    let ordinary = unique_bucket("writes-control");
-    dist.create_bucket(&ordinary).await?;
-    enable_versioning(&dist.client(0)?, &ordinary).await?;
-    let (fresh, fresh_user) = create_artifact_attempt(&dist).await?;
-    let fresh_bucket = fresh["bucket"].as_str().ok_or("fresh bucket")?;
-    let fresh_prefix = fresh["attempt_prefix"].as_str().ok_or("fresh prefix")?;
-    let first = RepairObject::put(&dist, fresh_bucket, format!("{fresh_prefix}existing.bin")).await?;
-    lock_metadata_entrypoints(&dist, &fresh, &first, &ordinary).await?;
-    seal_writes(&dist, &fresh_user, &fresh, true).await?;
-    seal_writes(&dist, &fresh_user, &fresh, true).await?;
-    assert_put_closed(&dist, fresh_bucket, &format!("{fresh_prefix}closed.bin")).await?;
-
-    let (partial, partial_user) = create_artifact_attempt(&dist).await?;
-    let bucket = partial["bucket"].as_str().ok_or("partial bucket")?;
-    let prefix = partial["attempt_prefix"].as_str().ok_or("partial prefix")?;
-    let repaired = RepairObject::put(&dist, bucket, format!("{prefix}repair.bin")).await?;
-    // Enter the combined operation with only its ordinary domain closed.
-    confirm_seal(&dist, &partial_user, &json!({"version":1,"target":partial}).to_string(), &partial).await?;
-    let part = repaired.part(Path::new(&dist.cluster.nodes[3].data_dirs[0]), bucket, 3)?;
-    std::fs::remove_file(&part)?;
-    let token = start_heal(&dist, bucket, &repaired.key, false).await?;
-    repaired.assert_repaired(&dist, bucket).await?;
-    finish_heal(&dist, &token, None).await?;
-    for _ in 0..2 {
-        dist.cluster.stop_node(3)?;
-        seal_writes(&dist, &partial_user, &partial, false).await?;
-        dist.cluster.start_node(3).await?;
-        wait_for_ready(&dist.cluster).await?;
-        seal_writes(&dist, &partial_user, &partial, true).await?;
-    }
-    let ordinary_object = RepairObject::put(&dist, &ordinary, "repair-control.bin".to_owned()).await?;
-    for restarted in [false, true] {
-        if restarted {
-            dist.restart_with_current_binary().await?;
-        }
-        seal_writes(&dist, &partial_user, &partial, true).await?;
-        assert_put_closed(&dist, bucket, &format!("{prefix}closed-{restarted}.bin")).await?;
-        if part.exists() {
-            std::fs::remove_file(&part)?;
-        }
-        let before = physical_snapshot(&dist, bucket)?;
-        let token = start_heal(&dist, bucket, &repaired.key, false).await?;
-        finish_heal(&dist, &token, Some(&repaired.key)).await?;
-        assert!(!part.exists(), "combined seal admitted a remote repair publication");
-        assert!(physical_snapshot(&dist, bucket)? == before, "rejected repair changed metadata or shards");
-        assert_exact_version(&dist.client(2)?, bucket, &repaired.key, &repaired.version, &repaired.body).await?;
-        let ordinary_part = ordinary_object.part(Path::new(&dist.cluster.nodes[3].data_dirs[0]), &ordinary, 3)?;
-        std::fs::remove_file(ordinary_part)?;
-        let token = start_heal(&dist, &ordinary, &ordinary_object.key, false).await?;
-        ordinary_object.assert_repaired(&dist, &ordinary).await?;
-        finish_heal(&dist, &token, None).await?;
-    }
-    Ok(())
-}

crates/e2e_test/src/distributed/harness.rs

@@ -34,7 +34,7 @@ use crate::common::{
 };
 use crate::replication_extension_test::LOOPBACK_REPLICATION_TARGET_ENV;
 use aws_sdk_s3::Client;
-use aws_sdk_s3::config::{Credentials, retry::RetryConfig};
+use aws_sdk_s3::config::retry::RetryConfig;
 use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
 use aws_sdk_s3::operation::{get_object::GetObjectError, put_object::PutObjectError};
 use aws_sdk_s3::primitives::ByteStream;
@@ -185,12 +185,13 @@ impl DistCluster {
         if node_idx >= self.cluster.nodes.len() {
             return Err("node_idx is invalid".into());
         }
-        let config = self.cluster.create_s3_client(node_idx)?.config().to_builder();
-        Ok(Client::from_conf(
-            config
-                .credentials_provider(Credentials::new(access_key, secret_key, None, None, "cluster-iam-test"))
-                .build(),
-        ))
+        Ok(Client::from_conf(build_test_s3_config(
+            &self.cluster.nodes[node_idx].url,
+            access_key,
+            secret_key,
+            None,
+            "cluster-iam-test",
+        )))
     }
 
     /// Stop the fleet, drop retired pools from the command line, and restart
@@ -346,7 +347,7 @@ fn relocate_pool_storage(cluster: &mut RustFSTestClusterEnvironment, node_idx: u
     Ok(pool_run_root)
 }
 
-pub(super) fn configure_node_logs(cluster: &mut RustFSTestClusterEnvironment, capture_dir: Option<&Path>) -> TestResult {
+fn configure_node_logs(cluster: &mut RustFSTestClusterEnvironment, capture_dir: Option<&Path>) -> TestResult {
     if let Some(directory) = capture_dir {
         std::fs::create_dir_all(directory)?;
         for node_idx in 0..cluster.nodes.len() {

crates/e2e_test/src/distributed/internode_tls_test.rs

@@ -1,286 +0,0 @@
-// Copyright 2026 RustFS Team
-// Licensed under the Apache License, Version 2.0.
-
-use super::harness::{DistCluster, DistLayout, TestResult, unique_bucket};
-use crate::tls_gen;
-use aws_sdk_s3::{Client, config::retry::RetryConfig, primitives::ByteStream};
-use rand::{Rng, SeedableRng, rngs::StdRng};
-use sha2::{Digest, Sha256};
-use std::io::{Read, Seek, SeekFrom, Write};
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
-use std::time::{Duration, Instant};
-use tokio::task::JoinSet;
-
-const MIB: u64 = 1024 * 1024;
-const READ_TIMEOUT: Duration = Duration::from_secs(180);
-
-struct ExpectedRead {
-    range: Option<(u64, u64)>,
-    length: u64,
-    sha256: String,
-}
-
-struct ObjectFixture {
-    key: String,
-    path: PathBuf,
-    length: u64,
-    reads: Vec<ExpectedRead>,
-}
-
-fn fixture(directory: &Path, length: u64) -> TestResult<ObjectFixture> {
-    let key = format!("object-{length}");
-    let path = directory.join(&key);
-    let mut file = std::fs::File::create(&path)?;
-    let mut rng = StdRng::seed_from_u64(length);
-    let mut block = vec![0; usize::try_from(MIB)?];
-    let mut remaining = length;
-    let mut digest = Sha256::new();
-    while remaining != 0 {
-        let count = usize::try_from(remaining.min(MIB))?;
-        rng.fill_bytes(&mut block[..count]);
-        file.write_all(&block[..count])?;
-        digest.update(&block[..count]);
-        remaining -= u64::try_from(count)?;
-    }
-    drop(file);
-    let mut reads = vec![ExpectedRead {
-        range: None,
-        length,
-        sha256: hex_simd::encode_to_string(digest.finalize(), hex_simd::AsciiCase::Lower),
-    }];
-    // Unaligned ranges cross TLS frames and EC blocks; the tail detects
-    // truncation and incorrect range offsets independently of a full GET.
-    for (start, end) in [(17, (MIB + 31).min(length - 1)), (length.saturating_sub(65537), length - 1)] {
-        let mut input = std::fs::File::open(&path)?;
-        input.seek(SeekFrom::Start(start))?;
-        let range_length = end - start + 1;
-        let mut hasher = Sha256::new();
-        let mut input = input.take(range_length);
-        loop {
-            let count = input.read(&mut block)?;
-            if count == 0 {
-                break;
-            }
-            hasher.update(&block[..count]);
-        }
-        reads.push(ExpectedRead {
-            range: Some((start, end)),
-            length: range_length,
-            sha256: hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower),
-        });
-    }
-    Ok(ObjectFixture {
-        key,
-        path,
-        length,
-        reads,
-    })
-}
-
-async fn verified_get(
-    client: &Client,
-    node: usize,
-    bucket: &str,
-    object: &ObjectFixture,
-    expected: &ExpectedRead,
-    concurrent: bool,
-) -> TestResult<serde_json::Value> {
-    let started = Instant::now();
-    let mut received = 0u64;
-    let mut stage = "response headers";
-    tokio::time::timeout(READ_TIMEOUT, async {
-        let mut request = client.get_object().bucket(bucket).key(&object.key);
-        if let Some((start, end)) = expected.range {
-            request = request.range(format!("bytes={start}-{end}"));
-        }
-        let mut output = request.send().await?;
-        assert_eq!(output.content_length(), Some(i64::try_from(expected.length)?));
-        if let Some((start, end)) = expected.range {
-            assert_eq!(output.content_range(), Some(format!("bytes {start}-{end}/{}", object.length).as_str()));
-        } else {
-            assert_eq!(output.content_range(), None);
-        }
-        stage = "response body";
-        let mut digest = Sha256::new();
-        while let Some(chunk) = output.body.next().await {
-            let chunk = chunk?;
-            received += u64::try_from(chunk.len())?;
-            assert!(received <= expected.length, "node {node}: GET exceeded the requested length");
-            digest.update(&chunk);
-        }
-        let actual_sha256 = hex_simd::encode_to_string(digest.finalize(), hex_simd::AsciiCase::Lower);
-        assert_eq!(received, expected.length, "node {node}: truncated GET {}", object.key);
-        assert_eq!(actual_sha256, expected.sha256, "node {node}: corrupt GET {}", object.key);
-        eprintln!(
-            "TLS acceptance: GET node={node}, key={}, range={:?}, concurrent={concurrent}, bytes={received}, elapsed={:?}",
-            object.key, expected.range, started.elapsed()
-        );
-        Ok(serde_json::json!({
-            "operation": "get", "node": node, "key": object.key,
-            "range": expected.range, "bytes": received, "sha256": actual_sha256,
-            "concurrent": concurrent, "elapsed_ms": started.elapsed().as_secs_f64() * 1000.0,
-        }))
-    })
-    .await
-    .map_err(|error| {
-        std::io::Error::new(
-            std::io::ErrorKind::TimedOut,
-            format!(
-                "TLS GET timed out: node={node}, key={}, range={:?}, concurrent={concurrent}, stage={stage}, received={received}/{}, elapsed={:?}: {error}",
-                object.key, expected.range, expected.length, started.elapsed()
-            ),
-        )
-    })?
-}
-
-/// This case always exercises the full acceptance sizes; do not substitute
-/// tiny objects and count that as a large distributed-read pass.
-#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-async fn four_node_tls_large_and_range_gets_preserve_bytes() -> TestResult {
-    let mode = std::env::var("RUSTFS_E2E_INTERNODE_HTTP_MODE").unwrap_or_else(|_| "http1".into());
-    let protocol = match mode.as_str() {
-        "http1" => "http1",
-        "auto" | "unpooled" => "auto",
-        _ => return Err("RUSTFS_E2E_INTERNODE_HTTP_MODE must be http1, auto or unpooled".into()),
-    };
-    let mut dist = DistCluster::new_stopped_with_env(
-        DistLayout::FourByFour,
-        &[
-            ("RUSTFS_INTERNODE_HTTP_PROTOCOL", protocol),
-            ("RUSTFS_SCANNER_ENABLED", "true"),
-            ("RUSTFS_SCANNER_START_DELAY_SECS", "0"),
-            ("RUSTFS_SCANNER_CYCLE", "1"),
-            ("RUSTFS_HEAL_ENABLED", "true"),
-            ("RUST_LOG", "warn"),
-        ],
-    )
-    .await?;
-    if mode == "unpooled" {
-        dist.cluster.set_env("RUSTFS_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST", "0");
-    }
-    let directory = PathBuf::from(&dist.cluster.temp_dir);
-    let tls_directory = directory.join("tls");
-    tls_gen::run(tls_gen::Args {
-        out_dir: tls_directory.clone(),
-        days: 1,
-        force: false,
-    })?;
-    dist.cluster.enable_tls(&tls_directory).await?;
-    let report_path = std::env::var_os("RUSTFS_INTERNODE_ACCEPTANCE_REPORT").map(PathBuf::from);
-    if let Some(path) = &report_path
-        && (path.exists() || path.with_extension("pids.json").exists())
-    {
-        return Err("acceptance report paths must be unused".into());
-    }
-    let log_directory = report_path.as_ref().map_or_else(
-        || {
-            std::env::var_os("RUSTFS_E2E_LOG_DIR")
-                .map_or_else(|| directory.join("logs"), |root| PathBuf::from(root).join(unique_bucket("internode-tls")))
-        },
-        |path| path.with_extension("logs"),
-    );
-    std::fs::create_dir(&log_directory)?;
-    dist.capture_logs_under(&log_directory)?;
-    dist.cluster.start().await?;
-    let clients = dist
-        .clients()?
-        .into_iter()
-        .map(|client| Client::from_conf(client.config().to_builder().retry_config(RetryConfig::disabled()).build()))
-        .collect::<Vec<_>>();
-    let bucket = unique_bucket("internode-tls");
-    clients[0].create_bucket().bucket(&bucket).send().await?;
-    let pids = dist
-        .cluster
-        .nodes
-        .iter()
-        .map(|node| node.process.as_ref().map(std::process::Child::id))
-        .collect::<Vec<_>>();
-    if let Some(path) = &report_path {
-        std::fs::write(path.with_extension("pids.json"), serde_json::to_vec(&pids)?)?;
-    }
-
-    let mut results = Vec::new();
-    for length in [32 * 1024, 100 * MIB, 1024 * MIB] {
-        let fixture_directory = directory.clone();
-        let object = Arc::new(tokio::task::spawn_blocking(move || fixture(&fixture_directory, length)).await??);
-        eprintln!("TLS acceptance: fixture key={}, bytes={length}", object.key);
-        let started = Instant::now();
-        tokio::time::timeout(
-            READ_TIMEOUT,
-            clients[0]
-                .put_object()
-                .bucket(&bucket)
-                .key(&object.key)
-                .body(ByteStream::from_path(&object.path).await?)
-                .send(),
-        )
-        .await
-        .map_err(|error| {
-            std::io::Error::new(
-                std::io::ErrorKind::TimedOut,
-                format!(
-                    "TLS PUT timed out: key={}, bytes={length}, elapsed={:?}: {error}",
-                    object.key,
-                    started.elapsed()
-                ),
-            )
-        })??;
-        eprintln!("TLS acceptance: PUT key={}, bytes={length}, elapsed={:?}", object.key, started.elapsed());
-        results.push(
-            serde_json::json!({"operation": "put", "bytes": length, "elapsed_ms": started.elapsed().as_secs_f64() * 1000.0}),
-        );
-        if length == 32 * 1024 {
-            let alternate = dist.client_with_credentials(0, &dist.cluster.access_key, &dist.cluster.secret_key)?;
-            let alternate = Client::from_conf(alternate.config().to_builder().retry_config(RetryConfig::disabled()).build());
-            results.push(verified_get(&alternate, 0, &bucket, &object, &object.reads[0], false).await?);
-            let invalid = dist.client_with_credentials(0, "invalid-tls-user", "invalid-tls-secret-password")?;
-            let invalid = Client::from_conf(invalid.config().to_builder().retry_config(RetryConfig::disabled()).build());
-            let denied = invalid
-                .list_buckets()
-                .send()
-                .await
-                .expect_err("alternate credentials must not inherit the root identity");
-            assert_eq!(denied.raw_response().map(|response| response.status().as_u16()), Some(403));
-        }
-        for expected in &object.reads {
-            for (node, client) in clients.iter().enumerate() {
-                results.push(verified_get(client, node, &bucket, &object, expected, false).await?);
-            }
-        }
-        // Each reader owns a Tokio task and its cooperative budget. Polling
-        // every body and synchronous digest in one parent task can let an
-        // always-ready reader starve its siblings and their deadlines.
-        let mut readers = JoinSet::new();
-        for (node, client) in clients.iter().enumerate() {
-            let client = client.clone();
-            let bucket = bucket.clone();
-            let object = Arc::clone(&object);
-            readers.spawn(async move { verified_get(&client, node, &bucket, &object, &object.reads[0], true).await });
-        }
-        while let Some(result) = readers.join_next().await {
-            results.push(result??);
-        }
-        tokio::fs::remove_file(&object.path).await?;
-    }
-    for node in 0..clients.len() {
-        let log = tokio::fs::read_to_string(log_directory.join(format!("node-{node}.log"))).await?;
-        assert!(
-            !log.contains("internode body stalled"),
-            "node {node}: internode body stall during verified reads"
-        );
-    }
-    let report = serde_json::json!({
-        "schema": 1, "mode": mode, "tls_verified": true, "nodes": 4, "drives_per_node": 4,
-        "source_revision": env!("RUSTFS_E2E_BUILD_COMMIT"),
-        "source_dirty": env!("RUSTFS_E2E_BUILD_DIRTY"),
-        "scanner_enabled": true, "heal_enabled": true,
-        "scanner_progress_verified": false, "heal_progress_verified": false,
-        "results": results,
-    });
-    if let Some(path) = report_path {
-        std::fs::write(path, serde_json::to_vec_pretty(&report)?)?;
-    }
-    eprintln!("internode_tls_acceptance={report}");
-    Ok(())
-}

crates/e2e_test/src/distributed/mod.rs

@@ -18,13 +18,6 @@
 //! `.github/workflows/e2e-distributed.yml`. Excluded from `e2e-full` because
 //! each case starts four real `rustfs` processes.
 
-mod artifact_delete_test;
-mod artifact_inventory_test;
-mod artifact_put_tail_test;
-mod artifact_repair_tail_test;
-mod artifact_repair_test;
-mod artifact_seal_test;
-mod artifact_writes_test;
 mod chaos_test;
 mod concurrency_stability_test;
 mod concurrent_data_movement_test;
@@ -33,8 +26,6 @@ mod expand_decommission_rebalance_test;
 mod extra_test;
 mod harness;
 mod heal_test;
-mod internode_tls_test;
-mod multipart_fence_acceptance_test;
 mod object_lock_test;
 mod observability_test;
 mod replication_delete_marker_test;

crates/e2e_test/src/distributed/multipart_fence_acceptance_test.rs

@@ -1,825 +0,0 @@
-// Copyright 2026 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use super::harness::{
-    TestResult, assert_object_bytes, cluster_admin_ok, configure_node_logs, payload_for, retrying_put, sha256_hex, unique_bucket,
-    wait_until,
-};
-use crate::common::{ClusterTopology, RustFSTestClusterEnvironment};
-use crate::rpc_fault_proxy::{RpcFaultProxy, RpcPause};
-use aws_sdk_s3::{
-    Client,
-    config::retry::RetryConfig,
-    error::{ProvideErrorMetadata, SdkError},
-    operation::create_multipart_upload::{CreateMultipartUploadError, CreateMultipartUploadOutput},
-    primitives::ByteStream,
-    types::{CompletedMultipartUpload, CompletedPart},
-};
-use futures::{StreamExt as _, TryStreamExt as _, stream};
-use std::time::{Duration, Instant};
-use tokio::task::JoinHandle;
-
-const DEADLINE: Duration = Duration::from_secs(45);
-
-async fn fixture(
-    drives_per_node: usize,
-    storage_class: &str,
-) -> TestResult<(RustFSTestClusterEnvironment, Vec<RpcFaultProxy>, Vec<Client>, String)> {
-    let mut cluster =
-        RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(4, drives_per_node)).await?;
-    for (name, value) in [
-        ("RUSTFS_STORAGE_CLASS_STANDARD", storage_class),
-        ("RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT", "5"),
-        ("RUSTFS_OBJECT_LOCK_RPC_TIMEOUT_MS", "3000"),
-        ("RUSTFS_DRIVE_METADATA_TIMEOUT_SECS", "2"),
-        ("RUSTFS_DRIVE_MAX_TIMEOUT_DURATION", "5"),
-        // Keep disk-health ejection out of a method-specific quorum test.
-        ("RUSTFS_DRIVE_SUSPECT_FAILURE_THRESHOLD", "1000"),
-        ("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false"),
-        ("NO_PROXY", "127.0.0.1,::1,localhost"),
-        ("HTTP_PROXY", ""),
-        ("HTTPS_PROXY", ""),
-        ("RUST_LOG", "warn"),
-    ] {
-        cluster.set_env(name, value);
-    }
-    let mut proxies = Vec::new();
-    for node in 0..4 {
-        proxies.push(cluster.start_rpc_proxy_for_node(node).await?);
-    }
-    configure_node_logs(&mut cluster, None)?;
-    cluster.start().await?;
-    let clients = cluster
-        .create_all_clients()?
-        .into_iter()
-        .map(|client| {
-            Client::from_conf(
-                client
-                    .config()
-                    .to_builder()
-                    .retry_config(RetryConfig::standard().with_max_attempts(1))
-                    .build(),
-            )
-        })
-        .collect();
-    let bucket = unique_bucket("mpu-fence");
-    cluster.create_test_bucket(&bucket).await?;
-    Ok((cluster, proxies, clients, bucket))
-}
-
-fn create(
-    client: &Client,
-    bucket: &str,
-    key: &str,
-) -> JoinHandle<Result<CreateMultipartUploadOutput, SdkError<CreateMultipartUploadError>>> {
-    let request = client.create_multipart_upload().bucket(bucket).key(key);
-    tokio::spawn(async move { request.send().await })
-}
-
-async fn wait_for_faults(pauses: &[RpcPause], methods: &[&str]) -> TestResult {
-    if methods.iter().any(|method| matches!(*method, "Lock" | "LockBatch")) {
-        // Adaptive lock fan-out may not contact an unnecessary voter. All
-        // selected peers remain gated; require an actual scoped request hit.
-        let (result, _, _) = futures::future::select_all(pauses.iter().map(|pause| Box::pin(pause.wait_for_hits(1)))).await;
-        result?;
-    } else {
-        // Storage mutations fan out eagerly. Every selected peer must really
-        // receive this upload's RPC, so an unrelated failure cannot fake quorum.
-        for pause in pauses {
-            pause.wait_for_hits(1).await?;
-        }
-    }
-    Ok(())
-}
-
-fn assert_unavailable<E: std::fmt::Debug + ProvideErrorMetadata>(error: &SdkError<E>, cause: &str, retry: Option<&str>) {
-    let response = error.raw_response().expect("an S3 service response");
-    assert_eq!(response.status().as_u16(), 503, "{error:?}");
-    assert_eq!(error.as_service_error().and_then(|e| e.meta().code()), Some("ServiceUnavailable"));
-    assert_eq!(response.headers().get("x-rustfs-error-cause"), Some(cause));
-    assert_eq!(response.headers().get("retry-after"), retry);
-    assert!(
-        response
-            .headers()
-            .get("content-type")
-            .is_some_and(|v| v.starts_with("application/xml"))
-    );
-    let request_id = response.headers().get("x-amz-request-id").expect("request correlation ID");
-    assert!(!request_id.is_empty());
-    eprintln!("MPU acceptance: cause={cause}, retry={retry:?}, request_id={request_id}");
-}
-
-async fn abort(client: &Client, bucket: &str, key: &str, upload: &str) -> TestResult {
-    client
-        .abort_multipart_upload()
-        .bucket(bucket)
-        .key(key)
-        .upload_id(upload)
-        .send()
-        .await?;
-    let missing = client
-        .list_parts()
-        .bucket(bucket)
-        .key(key)
-        .upload_id(upload)
-        .send()
-        .await
-        .expect_err("an aborted upload must no longer admit ListParts");
-    assert_eq!(missing.as_service_error().and_then(|e| e.meta().code()), Some("NoSuchUpload"));
-    Ok(())
-}
-
-// Fault removal does not synchronously clear existing server leases or drive
-// health state. Keep the injected operation single-attempt, then require
-// bounded recovery of the exact cleanup API instead of assuming instant heal.
-async fn recover_after_fault<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
-where
-    E: std::fmt::Debug + ProvideErrorMetadata,
-    F: FnMut() -> Fut,
-    Fut: std::future::Future<Output = Result<T, SdkError<E>>>,
-{
-    let started = Instant::now();
-    let mut attempts = 0;
-    let mut last_failure = String::new();
-    let outcome = tokio::time::timeout(DEADLINE, async {
-        loop {
-            attempts += 1;
-            let result = call().await;
-            let retry = result.as_ref().err().and_then(|error| {
-                let response = error.raw_response()?;
-                if response.status().as_u16() != 503
-                    || error.as_service_error()?.meta().code() != Some("ServiceUnavailable")
-                    || !matches!(
-                        response.headers().get("x-rustfs-error-cause"),
-                        Some(
-                            "namespace_lock_timeout"
-                                | "namespace_lock_unavailable"
-                                | "storage_write_quorum"
-                                | "storage_read_quorum"
-                        )
-                    )
-                {
-                    return None;
-                }
-                let delay = match response.headers().get("retry-after") {
-                    Some("1") => Duration::from_secs(1),
-                    None => Duration::from_millis(100),
-                    Some(_) => return None,
-                };
-                last_failure = format!(
-                    "cause={:?}, request_id={:?}",
-                    response.headers().get("x-rustfs-error-cause"),
-                    response.headers().get("x-amz-request-id")
-                );
-                Some(delay)
-            });
-            match retry {
-                Some(delay) => tokio::time::sleep(delay).await,
-                None => return result,
-            }
-        }
-    })
-    .await;
-    eprintln!(
-        "MPU recovery: operation={operation}, attempts={attempts}, elapsed={:?}, last={last_failure}",
-        started.elapsed()
-    );
-    outcome.map_err(|_| format!("{operation} did not recover within {DEADLINE:?}: {last_failure}").into())
-}
-
-async fn abort_after_fault(client: &Client, bucket: &str, key: &str, upload: &str, allow_missing: bool) -> TestResult {
-    match recover_after_fault(&format!("Abort {key}"), || {
-        client
-            .abort_multipart_upload()
-            .bucket(bucket)
-            .key(key)
-            .upload_id(upload)
-            .send()
-    })
-    .await?
-    {
-        Ok(_) => {}
-        Err(error) if allow_missing && error.as_service_error().and_then(|e| e.meta().code()) == Some("NoSuchUpload") => {}
-        Err(error) => return Err(error.into()),
-    }
-    let missing = recover_after_fault(&format!("ListParts {key}"), || {
-        client.list_parts().bucket(bucket).key(key).upload_id(upload).send()
-    })
-    .await?
-    .expect_err("a recovered Abort must remove the upload");
-    assert_eq!(missing.as_service_error().and_then(|e| e.meta().code()), Some("NoSuchUpload"));
-    Ok(())
-}
-
-async fn round_trip(clients: &[Client], bucket: &str, key: &str, size: usize) -> TestResult {
-    let upload = clients[0].create_multipart_upload().bucket(bucket).key(key).send().await?;
-    let upload_id = upload.upload_id().ok_or("missing upload ID")?;
-    let body = payload_for(key, size);
-    let mut parts: Vec<_> = stream::iter(body.chunks(8 * 1024 * 1024).enumerate())
-        .map(|(part, bytes)| async move {
-            let number = i32::try_from(part + 1)?;
-            let result = clients[part % clients.len()]
-                .upload_part()
-                .bucket(bucket)
-                .key(key)
-                .upload_id(upload_id)
-                .part_number(number)
-                .body(ByteStream::from(bytes.to_vec()))
-                .send()
-                .await?;
-            Ok::<_, Box<dyn std::error::Error + Send + Sync>>(
-                CompletedPart::builder()
-                    .part_number(number)
-                    .e_tag(result.e_tag().ok_or("missing part ETag")?)
-                    .build(),
-            )
-        })
-        .buffer_unordered(4)
-        .try_collect()
-        .await?;
-    parts.sort_by_key(|part| part.part_number());
-    clients[1]
-        .complete_multipart_upload()
-        .bucket(bucket)
-        .key(key)
-        .upload_id(upload_id)
-        .multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(parts)).build())
-        .send()
-        .await?;
-    for client in clients {
-        assert_object_bytes(client, bucket, key, &body).await?;
-    }
-    clients[0].delete_object().bucket(bucket).key(key).send().await?;
-    Ok(())
-}
-
-async fn assert_no_uploads(clients: &[Client], bucket: &str) -> TestResult {
-    for client in clients {
-        let uploads = client.list_multipart_uploads().bucket(bucket).send().await?;
-        assert!(uploads.uploads().is_empty(), "unexpected residual uploads: {:?}", uploads.uploads());
-    }
-    Ok(())
-}
-
-async fn assert_no_uploads_after_fault(clients: &[Client], bucket: &str) -> TestResult {
-    for client in clients {
-        let uploads = recover_after_fault("ListMultipartUploads after fault", || {
-            client.list_multipart_uploads().bucket(bucket).send()
-        })
-        .await??;
-        assert!(uploads.uploads().is_empty(), "unexpected residual uploads: {:?}", uploads.uploads());
-    }
-    Ok(())
-}
-
-async fn assert_object_bytes_after_fault(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
-    let output = recover_after_fault(&format!("GetObject {key} after fault"), || {
-        client.get_object().bucket(bucket).key(key).send()
-    })
-    .await??;
-    // A successful response must have a complete, correct body. Streaming
-    // failures are not retried or converted into a successful comparison.
-    let bytes = output.body.collect().await?.into_bytes();
-    if bytes.as_ref() != expected {
-        return Err(format!("object {key} bytes differ: expected {} bytes, got {}", expected.len(), bytes.len()).into());
-    }
-    Ok(())
-}
-
-// This fixture inspects RustFS RPCs, so use the server-generated upload's
-// canonical suffix and fail explicitly if its encoding changes.
-fn upload_scope(upload_id: &str) -> TestResult<String> {
-    let decoded = String::from_utf8(base64_simd::URL_SAFE_NO_PAD.decode_to_vec(upload_id.as_bytes())?)?;
-    let (_, suffix) = decoded.split_once('.').ok_or("server upload ID lacks deployment delimiter")?;
-    if suffix.is_empty() {
-        return Err("server upload ID has an empty suffix".into());
-    }
-    Ok(suffix.to_owned())
-}
-
-fn upload_path_scope(bucket: &str, key: &str, upload_id: &str) -> TestResult<String> {
-    Ok(format!(
-        "{}/{}",
-        sha256_hex(format!("{bucket}/{key}").as_bytes()),
-        upload_scope(upload_id)?
-    ))
-}
-
-async fn with_paused_peers<T: Send + 'static>(
-    proxies: &[RpcFaultProxy],
-    methods: &[&'static str],
-    limits: &[usize],
-    scope: &str,
-    request: impl std::future::Future<Output = T> + Send + 'static,
-) -> TestResult<T> {
-    let pauses: Vec<_> = proxies[1..=limits.len()]
-        .iter()
-        .zip(limits)
-        .map(|(proxy, limit)| proxy.pause(methods, *limit, Some(scope)))
-        .collect::<Result<_, _>>()?;
-    let task = tokio::spawn(request);
-    wait_for_faults(&pauses, methods).await?;
-    let result = tokio::time::timeout(DEADLINE, task).await??;
-    for pause in &pauses {
-        pause.discard_pending();
-    }
-    drop(pauses);
-    Ok(result)
-}
-
-fn quorum_outcome<T: std::fmt::Debug, E: std::fmt::Debug + ProvideErrorMetadata>(
-    result: Result<T, SdkError<E>>,
-    at_quorum: bool,
-    locks: bool,
-) -> TestResult<Option<T>> {
-    if at_quorum {
-        Ok(Some(result.map_err(|error| format!("exactly quorum must succeed: {error:?}"))?))
-    } else {
-        assert_unavailable(
-            &result.expect_err("quorum-minus-one must fail"),
-            if locks {
-                "namespace_lock_timeout"
-            } else {
-                "storage_write_quorum"
-            },
-            if locks { Some("1") } else { None },
-        );
-        Ok(None)
-    }
-}
-
-async fn wait_for_lock_peer_recovery(proxies: &[RpcFaultProxy], client: &Client, bucket: &str) -> TestResult {
-    let key = unique_bucket("peer-recovery");
-    let observers = proxies[1..]
-        .iter()
-        .map(|proxy| proxy.pause(&["Lock", "LockBatch"], usize::MAX, Some(&key)))
-        .collect::<Result<Vec<_>, _>>()?;
-    for observer in &observers {
-        observer.release();
-    }
-    wait_until(
-        DEADLINE,
-        || async {
-            retrying_put(client, bucket, &key, vec![0x42; 1024], DEADLINE).await?;
-            // Quorum success alone can leave one peer's request breaker open.
-            // Two replies require a half-open peer to finish its first probe
-            // before admitting another RPC; every peer must rejoin the matrix.
-            Ok(observers.iter().all(|observer| observer.forwarded() >= 2))
-        },
-        "all lock peers recover after fault retirement",
-    )
-    .await
-}
-
-async fn wait_for_storage_peer_recovery(
-    proxies: &[RpcFaultProxy],
-    client: &Client,
-    bucket: &str,
-    upload: &str,
-    drives_per_node: usize,
-) -> TestResult {
-    let scope = upload_scope(upload)?;
-    tokio::time::timeout(
-        DEADLINE,
-        wait_until(
-            DEADLINE,
-            || async {
-                let observers = proxies[1..]
-                    .iter()
-                    .map(|proxy| proxy.pause(&["ReadVersion"], usize::MAX, Some(&scope)))
-                    .collect::<Result<Vec<_>, _>>()?;
-                for observer in &observers {
-                    observer.release();
-                }
-                // A fresh absent key forces a full metadata read without mutating
-                // this upload. Count distinct configured disk routes, not retries
-                // from the same healthy disk or just lock-RPC reachability.
-                let key = format!("storage-ready-{}", uuid::Uuid::new_v4());
-                let _ = client.list_parts().bucket(bucket).key(key).upload_id(upload).send().await;
-                Ok(observers
-                    .iter()
-                    .all(|observer| observer.forwarded_routes() >= drives_per_node))
-            },
-            "all storage disk routes recover after fault retirement",
-        ),
-    )
-    .await
-    .map_err(|_| "storage route recovery deadline exceeded".to_string())?
-}
-
-async fn phase_quorum_matrix(
-    proxies: &[RpcFaultProxy],
-    clients: &[Client],
-    bucket: &str,
-    locks: bool,
-    drives_per_node: usize,
-) -> TestResult {
-    for phase in ["create", "part", "complete", "abort"] {
-        // Ordinary Create holds only the shared bucket lifecycle fence.
-        // Four-node read quorum is two; commit/write quorum is three.
-        let unavailable_at_quorum = if locks && phase == "create" { 2 } else { 1 };
-        for unavailable in [unavailable_at_quorum, unavailable_at_quorum + 1] {
-            let key = format!("{locks}-{phase}-{unavailable}");
-            let mut limits = vec![usize::MAX; unavailable];
-            // EC12+4: one peer contributes four shards; one packet on the
-            // second peer moves exactly twelve available shards to eleven.
-            if !locks && drives_per_node == 4 && unavailable > 1 {
-                limits[unavailable - 1] = 1;
-            }
-            let available_votes = if locks {
-                4 - unavailable
-            } else {
-                4 * drives_per_node - limits.iter().map(|limit| drives_per_node.min(*limit)).sum::<usize>()
-            };
-            let methods: &[&'static str] = if locks {
-                &["Lock", "LockBatch"]
-            } else {
-                match phase {
-                    "create" => &["WriteMetadata"],
-                    "part" => &["PreparePartTransaction"],
-                    "complete" => &["RenameData", "RenameDataAtIncarnation", "RenameDataWithQuotaFence"],
-                    "abort" => &["Delete"],
-                    _ => unreachable!(),
-                }
-            };
-            eprintln!("MPU quorum acceptance: phase={phase}, lock_fault={locks}, available_votes={available_votes}");
-            if phase == "create" {
-                let request = clients[0].create_multipart_upload().bucket(bucket).key(&key);
-                let result = with_paused_peers(proxies, methods, &limits, bucket, async move { request.send().await }).await?;
-                if let Some(upload) = quorum_outcome(result, unavailable == unavailable_at_quorum, locks)? {
-                    let upload_id = upload.upload_id().ok_or("missing upload ID")?;
-                    wait_for_storage_peer_recovery(proxies, &clients[0], bucket, upload_id, drives_per_node).await?;
-                    abort_after_fault(&clients[0], bucket, &key, upload_id, false).await?;
-                }
-            } else {
-                let upload = clients[0]
-                    .create_multipart_upload()
-                    .bucket(bucket)
-                    .key(&key)
-                    .send()
-                    .await
-                    .map_err(|error| format!("setup Create failed for {key}: {error:?}"))?;
-                let upload_id = upload.upload_id().ok_or("missing upload ID")?;
-                let scope = if !locks && phase == "abort" {
-                    upload_path_scope(bucket, &key, upload_id)?
-                } else {
-                    upload_scope(upload_id)?
-                };
-                let body = payload_for(&key, if drives_per_node == 4 { 9 * 1024 * 1024 } else { 1024 * 1024 });
-                let initial = if phase != "part" {
-                    Some(
-                        clients[0]
-                            .upload_part()
-                            .bucket(bucket)
-                            .key(&key)
-                            .upload_id(upload_id)
-                            .part_number(1)
-                            .body(ByteStream::from(body.clone()))
-                            .send()
-                            .await?,
-                    )
-                } else {
-                    None
-                };
-                match phase {
-                    "part" => {
-                        let request = clients[0]
-                            .upload_part()
-                            .bucket(bucket)
-                            .key(&key)
-                            .upload_id(upload_id)
-                            .part_number(1)
-                            .body(ByteStream::from(body.clone()));
-                        let result =
-                            with_paused_peers(proxies, methods, &limits, &scope, async move { request.send().await }).await?;
-                        if let Some(part) = quorum_outcome(result, unavailable == unavailable_at_quorum, locks)? {
-                            let completion = clients[0]
-                                .complete_multipart_upload()
-                                .bucket(bucket)
-                                .key(&key)
-                                .upload_id(upload_id)
-                                .multipart_upload(
-                                    CompletedMultipartUpload::builder()
-                                        .parts(
-                                            CompletedPart::builder()
-                                                .part_number(1)
-                                                .e_tag(part.e_tag().ok_or("missing ETag")?)
-                                                .build(),
-                                        )
-                                        .build(),
-                                );
-                            recover_after_fault(&format!("Complete {key} after part fault"), || completion.clone().send())
-                                .await??;
-                            for reader in clients {
-                                assert_object_bytes_after_fault(reader, bucket, &key, &body).await?;
-                            }
-                        }
-                    }
-                    "complete" => {
-                        let part = initial.as_ref().expect("complete fixture has one part");
-                        let request = clients[0]
-                            .complete_multipart_upload()
-                            .bucket(bucket)
-                            .key(&key)
-                            .upload_id(upload_id)
-                            .multipart_upload(
-                                CompletedMultipartUpload::builder()
-                                    .parts(
-                                        CompletedPart::builder()
-                                            .part_number(1)
-                                            .e_tag(part.e_tag().ok_or("missing ETag")?)
-                                            .build(),
-                                    )
-                                    .build(),
-                            );
-                        let result =
-                            with_paused_peers(proxies, methods, &limits, &scope, async move { request.send().await }).await?;
-                        if quorum_outcome(result, unavailable == unavailable_at_quorum, locks)?.is_some() {
-                            for reader in clients {
-                                assert_object_bytes_after_fault(reader, bucket, &key, &body).await?;
-                            }
-                        }
-                    }
-                    "abort" => {
-                        let request = clients[0]
-                            .abort_multipart_upload()
-                            .bucket(bucket)
-                            .key(&key)
-                            .upload_id(upload_id);
-                        let result =
-                            with_paused_peers(proxies, methods, &limits, &scope, async move { request.send().await }).await?;
-                        quorum_outcome(result, unavailable == unavailable_at_quorum, locks)?;
-                    }
-                    _ => unreachable!(),
-                }
-                wait_for_storage_peer_recovery(proxies, &clients[0], bucket, upload_id, drives_per_node).await?;
-                // A rejected Complete must retain authority for ordinary
-                // Abort. NoSuchUpload must not hide partial staging loss.
-                let allow_missing = phase != "complete" || unavailable == unavailable_at_quorum;
-                abort_after_fault(&clients[0], bucket, &key, upload_id, allow_missing).await?;
-                recover_after_fault(&format!("DeleteObject {key} after {phase}"), || {
-                    clients[0].delete_object().bucket(bucket).key(&key).send()
-                })
-                .await??;
-            }
-            wait_for_lock_peer_recovery(proxies, &clients[0], bucket).await?;
-            round_trip(clients, bucket, "recovered", 9 * 1024 * 1024).await?;
-            assert_no_uploads_after_fault(clients, bucket).await?;
-        }
-    }
-    Ok(())
-}
-
-#[tokio::test]
-async fn metadata_delay_preserves_write_quorum_and_rejects_quorum_minus_one() -> TestResult {
-    let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
-    phase_quorum_matrix(&proxies, &clients, &bucket, false, 1).await
-}
-
-#[tokio::test]
-async fn lock_delay_preserves_admission_quorum_and_reports_timeout() -> TestResult {
-    let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
-    phase_quorum_matrix(&proxies, &clients, &bucket, true, 1).await
-}
-
-#[tokio::test]
-async fn sixteen_shard_multipart_metadata_requires_twelve_votes() -> TestResult {
-    let (cluster, proxies, clients, bucket) = fixture(4, "EC:4").await?;
-    let storage = cluster_admin_ok(&cluster, http::Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
-    let storage: serde_json::Value = serde_json::from_str(&storage)?;
-    let storage: rustfs_madmin::StorageInfo = serde_json::from_value(storage["info"].clone())?;
-    assert_eq!(storage.disks.len(), 16);
-    assert_eq!(storage.backend.total_sets, vec![1]);
-    assert_eq!(storage.backend.drives_per_set, vec![16]);
-    assert_eq!(storage.backend.standard_sc_parity, Some(4));
-    phase_quorum_matrix(&proxies, &clients, &bucket, false, 4).await
-}
-
-#[tokio::test]
-async fn cancelled_metadata_wait_releases_fence_for_bucket_recreation() -> TestResult {
-    let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
-    let pauses: Vec<_> = proxies[1..]
-        .iter()
-        .map(|proxy| proxy.pause(&["WriteMetadata"], usize::MAX, Some(&bucket)))
-        .collect::<Result<_, _>>()?;
-    let task = create(&clients[0], &bucket, "cancelled");
-    wait_for_faults(&pauses, &["WriteMetadata"]).await?;
-    task.abort();
-    assert!(task.await.expect_err("cancelled S3 future").is_cancelled());
-    drop(pauses);
-    // Deletion must wait for admitted server work to finish, then erase its
-    // generation. Client cancellation alone does not prove server cancellation.
-    tokio::time::timeout(DEADLINE, clients[1].delete_bucket().bucket(&bucket).send()).await??;
-    clients[2].create_bucket().bucket(&bucket).send().await?;
-    assert_no_uploads(&clients, &bucket).await?;
-    round_trip(&clients, &bucket, "new-generation", 9 * 1024 * 1024).await?;
-    assert_no_uploads(&clients, &bucket).await
-}
-
-/// Same logical geometry as the incident: one pool, four 16-shard sets,
-/// EC12+4, 45 MiB files, 8 MiB parts, two files and four parts in flight.
-#[tokio::test]
-async fn eight_node_eight_drive_multipart_load_leaves_no_residual_uploads() -> TestResult {
-    let rounds = std::env::var("RUSTFS_E2E_MPU_LOAD_ROUNDS")
-        .unwrap_or_else(|_| "3".into())
-        .parse::<usize>()?;
-    if !(1..=256).contains(&rounds) {
-        return Err("RUSTFS_E2E_MPU_LOAD_ROUNDS must be in 1..=256".into());
-    }
-    let mut cluster = RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(8, 8)).await?;
-    for (name, value) in [
-        ("RUSTFS_STORAGE_CLASS_STANDARD", "EC:4"),
-        ("TOKIO_WORKER_THREADS", "2"),
-        ("RAYON_NUM_THREADS", "2"),
-        ("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false"),
-        ("NO_PROXY", "127.0.0.1,::1,localhost"),
-        ("HTTP_PROXY", ""),
-        ("HTTPS_PROXY", ""),
-        ("RUST_LOG", "warn"),
-    ] {
-        cluster.set_env(name, value);
-    }
-    configure_node_logs(&mut cluster, None)?;
-    cluster.start().await?;
-    let clients: Vec<_> = cluster
-        .create_all_clients()?
-        .into_iter()
-        .map(|client| {
-            Client::from_conf(
-                client
-                    .config()
-                    .to_builder()
-                    .retry_config(RetryConfig::standard().with_max_attempts(1))
-                    .build(),
-            )
-        })
-        .collect();
-    let storage = cluster_admin_ok(&cluster, http::Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
-    let storage: serde_json::Value = serde_json::from_str(&storage)?;
-    let storage: rustfs_madmin::StorageInfo = serde_json::from_value(storage["info"].clone())?;
-    assert_eq!(storage.disks.len(), 64);
-    assert_eq!(storage.backend.total_sets, vec![4]);
-    assert_eq!(storage.backend.drives_per_set, vec![16]);
-    assert_eq!(storage.backend.standard_sc_parity, Some(4));
-    let bucket = unique_bucket("mpu-8x8");
-    cluster.create_test_bucket(&bucket).await?;
-    let started = Instant::now();
-    for round in 0..rounds {
-        let first = format!("round-{round}-first");
-        let second = format!("round-{round}-second");
-        tokio::try_join!(
-            round_trip(&clients, &bucket, &first, 45 * 1024 * 1024),
-            round_trip(&clients, &bucket, &second, 45 * 1024 * 1024),
-        )?;
-        let aborted = clients[round % clients.len()]
-            .create_multipart_upload()
-            .bucket(&bucket)
-            .key("aborted")
-            .send()
-            .await?;
-        let upload_id = aborted.upload_id().ok_or("missing upload ID")?;
-        clients[(round + 1) % clients.len()]
-            .upload_part()
-            .bucket(&bucket)
-            .key("aborted")
-            .upload_id(upload_id)
-            .part_number(1)
-            .body(ByteStream::from(vec![0xa5; 8 * 1024 * 1024]))
-            .send()
-            .await?;
-        abort(&clients[(round + 2) % clients.len()], &bucket, "aborted", upload_id).await?;
-        assert_no_uploads(&clients, &bucket).await?;
-        eprintln!(
-            "MPU 8x8 acceptance: round={}/{rounds}, completed={}, aborted={}, elapsed={:?}",
-            round + 1,
-            2 * (round + 1),
-            round + 1,
-            started.elapsed()
-        );
-    }
-    Ok(())
-}
-
-async fn resume_at_rpc<T: Send + 'static>(
-    proxies: &[RpcFaultProxy],
-    methods: &[&'static str],
-    scope: &str,
-    request: impl std::future::Future<Output = TestResult<T>> + Send + 'static,
-) -> TestResult<T> {
-    let pauses: Vec<_> = proxies[1..]
-        .iter()
-        .map(|proxy| proxy.pause(methods, usize::MAX, Some(scope)))
-        .collect::<Result<_, _>>()?;
-    eprintln!("MPU delay acceptance: methods={methods:?}, scope={scope}");
-    let task = tokio::spawn(request);
-    wait_for_faults(&pauses, methods).await?;
-    assert!(
-        !task.is_finished(),
-        "the selected commit must wait while only one shard/voter is available"
-    );
-    drop(pauses);
-    tokio::time::timeout(DEADLINE, task).await??
-}
-
-#[tokio::test]
-async fn each_multipart_phase_resumes_after_metadata_and_lock_delays() -> TestResult {
-    let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
-    for locks in [false, true] {
-        let key = format!("phase-{locks}");
-        let client = clients[0].clone();
-        let create = client.create_multipart_upload().bucket(&bucket).key(&key);
-        let upload = resume_at_rpc(
-            &proxies,
-            if locks { &["Lock", "LockBatch"] } else { &["WriteMetadata"] },
-            bucket.as_str(),
-            async move { Ok(create.send().await?) },
-        )
-        .await?;
-        let upload_id = upload.upload_id().ok_or("missing upload ID")?;
-        let scope = upload_scope(upload_id)?;
-        let body = payload_for(&key, 1024 * 1024);
-        let part = client
-            .upload_part()
-            .bucket(&bucket)
-            .key(&key)
-            .upload_id(upload_id)
-            .part_number(1)
-            .body(ByteStream::from(body.clone()));
-        let part = resume_at_rpc(
-            &proxies,
-            if locks {
-                &["Lock", "LockBatch"]
-            } else {
-                &["PreparePartTransaction"]
-            },
-            &scope,
-            async move { Ok(part.send().await?) },
-        )
-        .await?;
-        let complete = client
-            .complete_multipart_upload()
-            .bucket(&bucket)
-            .key(&key)
-            .upload_id(upload_id)
-            .multipart_upload(
-                CompletedMultipartUpload::builder()
-                    .parts(
-                        CompletedPart::builder()
-                            .part_number(1)
-                            .e_tag(part.e_tag().ok_or("missing ETag")?)
-                            .build(),
-                    )
-                    .build(),
-            );
-        resume_at_rpc(
-            &proxies,
-            if locks {
-                &["Lock", "LockBatch"]
-            } else {
-                &["RenameData", "RenameDataAtIncarnation", "RenameDataWithQuotaFence"]
-            },
-            &scope,
-            async move { Ok(complete.send().await?) },
-        )
-        .await?;
-        for reader in &clients {
-            assert_object_bytes(reader, &bucket, &key, &body).await?;
-        }
-        client.delete_object().bucket(&bucket).key(&key).send().await?;
-        let upload = client.create_multipart_upload().bucket(&bucket).key(&key).send().await?;
-        let upload_id = upload.upload_id().ok_or("missing upload ID")?;
-        let scope = upload_scope(upload_id)?;
-        let scope = if locks {
-            scope
-        } else {
-            upload_path_scope(&bucket, &key, upload_id)?
-        };
-        let request = client.abort_multipart_upload().bucket(&bucket).key(&key).upload_id(upload_id);
-        resume_at_rpc(&proxies, if locks { &["Lock", "LockBatch"] } else { &["Delete"] }, &scope, async move {
-            Ok(request.send().await?)
-        })
-        .await?;
-        let missing = client
-            .list_parts()
-            .bucket(&bucket)
-            .key(&key)
-            .upload_id(upload_id)
-            .send()
-            .await
-            .expect_err("delayed Abort must remove the upload");
-        assert_eq!(missing.as_service_error().and_then(|e| e.meta().code()), Some("NoSuchUpload"));
-        assert_no_uploads(&clients, &bucket).await?;
-    }
-    Ok(())
-}

crates/e2e_test/src/distributed/scanner_heal_topology_test.rs

@@ -528,7 +528,7 @@ async fn scanner_heal_topology_decommission_kill_before_source_cleanup_resumes_w
     run.movement_barrier.disarm()?;
     let restarted_pid = run.restart_killed_node(SEED_POOL).await?;
     let label = "after the killed leader resumed decommission";
-    let outcome = settle_decommission(&run, &manifest, label, DECOMMISSION_RESUME_COMPLETE_TIMEOUT).await;
+    let outcome = settle_decommission(&run, &manifest, label, MOVEMENT_COMPLETE_TIMEOUT).await;
     let strict_nodes = assert_process_durability_strict(&run.node_log_paths());
     record(
         &run,

crates/e2e_test/src/distributed/versioning_test.rs

@@ -17,14 +17,6 @@ use crate::common::init_logging;
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
 
-#[tokio::test]
-async fn four_node_explicit_delete_marker_batch_is_immediately_visible() -> TestResult {
-    init_logging();
-    let dist = DistCluster::start(DistLayout::FourByFour).await?;
-    let bucket = unique_bucket("explicit-marker-batch");
-    crate::delete_objects_versioning_test::assert_explicit_marker_batch_cleanup(&dist.clients()?, &bucket).await
-}
-
 #[tokio::test]
 async fn four_node_four_drive_versioning_put_list_get_delete_marker() -> TestResult {
     init_logging();

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -1206,9 +1206,7 @@ mod tests {
         let interval = interval
             .filter(|interval| *interval != 0)
             .ok_or("scanner schedule has no positive interval")?;
-        // A new leader may already be sleeping before completing its first cycle.
-        let schedule_start = if cycle_end == 0 { now } else { cycle_end };
-        let next_cycle = schedule_start
+        let next_cycle = cycle_end
             .checked_add(interval)
             .ok_or("scanner schedule timestamp overflows")?;
         let budget = next_cycle
@@ -1225,11 +1223,6 @@ mod tests {
             Ok(Duration::from_secs(259)),
         );
         assert_eq!(scanner_cycle_completion_budget(100, Some(1), 110, false), Ok(Duration::from_secs(60)));
-        // A newly elected leader can advertise a sleep before its first completed cycle.
-        assert_eq!(
-            scanner_cycle_completion_budget(0, Some(289), 1_792_913_211, false),
-            Ok(Duration::from_secs(349)),
-        );
     }
 
     #[test]
@@ -1250,7 +1243,7 @@ mod tests {
         previous_cycle_end: u64,
     ) -> Result<u64, Box<dyn Error + Send + Sync>> {
         let mut deadline = Instant::now() + Duration::from_secs(60);
-        let mut accounted_schedules = HashSet::new();
+        let mut schedule_accounted = false;
         loop {
             let mut latest_cycle_end = 0;
             let mut versions_observed = false;
@@ -1297,27 +1290,15 @@ mod tests {
                     return Err("scanner cycle timestamp is in the future".into());
                 }
                 let schedule = &status["cycle_schedule"];
-                if schedule["execution_role"].as_str() == Some("leader") {
+                if !schedule_accounted && schedule["execution_role"].as_str() == Some("leader") {
                     let active = metrics["current_cycle_active"]
                         .as_bool()
                         .ok_or("scanner status is missing its active-cycle flag")?;
-                    if active || schedule["effective_interval_available"].as_bool() == Some(true) {
-                        let interval = if active {
-                            None
-                        } else {
-                            Some(
-                                schedule["effective_interval_seconds"]
-                                    .as_u64()
-                                    .ok_or("scanner schedule is missing its effective interval")?,
-                            )
-                        };
-                        // Reconsider a changed leader or retry plan, but never renew
-                        // the deadline on repeated observations of the same sleep.
-                        if accounted_schedules.insert((node_index, cycle_end, interval)) {
-                            let budget = scanner_cycle_completion_budget(cycle_end, interval, now, active)?;
-                            let scheduled_deadline = Instant::now().checked_add(budget).ok_or("scanner deadline overflows")?;
-                            deadline = deadline.max(scheduled_deadline);
-                        }
+                    if active || (cycle_end != 0 && schedule["effective_interval_available"].as_bool() == Some(true)) {
+                        let interval = schedule["effective_interval_seconds"].as_u64();
+                        let budget = scanner_cycle_completion_budget(cycle_end, interval, now, active)?;
+                        deadline = Instant::now().checked_add(budget).ok_or("scanner deadline overflows")?;
+                        schedule_accounted = true;
                     }
                 }
                 // Superseded walks complete all sets while withholding quota authority.

crates/e2e_test/src/lib.rs

@@ -45,9 +45,6 @@ pub mod on_demand_migration;
 #[cfg(test)]
 pub mod fault_proxy;
 
-#[cfg(test)]
-mod rpc_fault_proxy;
-
 // Reliability tests built on the fault-injection harness
 #[cfg(test)]
 mod reliability_disk_fault_test;

crates/e2e_test/src/negative_sigv4_test.rs

@@ -230,302 +230,6 @@ fn encode_unsigned_aws_chunked_with_sha256_trailer(decoded: &[u8]) -> Vec<u8> {
     encoded
 }
 
-/// Sign the data and terminating chunks independently of the server decoder.
-fn encode_signed_aws_chunked(signer: &SigV4, headers: &SignedHeaders, decoded: &[u8], trailer: bool) -> Vec<u8> {
-    let mut previous = headers
-        .authorization
-        .rsplit_once("Signature=")
-        .expect("seed signature")
-        .1
-        .to_string();
-    let mut encoded = Vec::new();
-    for chunk in [decoded, b"".as_slice()] {
-        let string_to_sign = format!(
-            "AWS4-HMAC-SHA256-PAYLOAD\n{}\n{}\n{previous}\n{}\n{}",
-            headers.amz_date,
-            get_scope(REGION, signer.time, "s3"),
-            sha256_hex(b""),
-            sha256_hex(chunk),
-        );
-        previous = get_signature(get_signing_key(&signer.secret_key, REGION, signer.time, "s3"), &string_to_sign);
-        encoded.extend_from_slice(format!("{:x};chunk-signature={previous}\r\n", chunk.len()).as_bytes());
-        encoded.extend_from_slice(chunk);
-        encoded.extend_from_slice(b"\r\n");
-    }
-    if trailer {
-        let checksum = sha256_base64(decoded);
-        let canonical = format!("x-amz-checksum-sha256:{checksum}\n");
-        let string_to_sign = format!(
-            "AWS4-HMAC-SHA256-TRAILER\n{}\n{}\n{previous}\n{}",
-            headers.amz_date,
-            get_scope(REGION, signer.time, "s3"),
-            sha256_hex(canonical.as_bytes()),
-        );
-        let signature = get_signature(get_signing_key(&signer.secret_key, REGION, signer.time, "s3"), &string_to_sign);
-        encoded.extend_from_slice(
-            format!("x-amz-checksum-sha256:{checksum}\r\nx-amz-trailer-signature:{signature}\r\n\r\n").as_bytes(),
-        );
-    }
-    encoded
-}
-
-/// Exercise decoder failures through both storage publication boundaries.
-#[tokio::test]
-async fn aws_chunked_named_errors_reject_put_and_upload_part() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let mut env = RustFSTestEnvironment::new().await?;
-    setup(&mut env).await?;
-    let client = env.create_s3_client();
-    let decoded = b"private-chunked-fixture-body";
-    let mut failures = Vec::new();
-    for multipart in [false, true] {
-        for (case, expected_status, expected_code) in [
-            ("signature", 403, "SignatureDoesNotMatch"),
-            ("final-signature", 403, "SignatureDoesNotMatch"),
-            ("format", 400, "IncompleteBody"),
-            ("metadata-at-limit", 400, "IncompleteBody"),
-            ("metadata-over-limit", 400, "EntityTooLarge"),
-            ("chunk-over-limit", 400, "EntityTooLarge"),
-            ("trailers-over-limit", 400, "IncompleteBody"),
-            ("too-many-trailers", 400, "IncompleteBody"),
-        ] {
-            let key = format!("chunked-{case}-{multipart}.bin");
-            let upload_id = if multipart {
-                Some(
-                    client
-                        .create_multipart_upload()
-                        .bucket(BUCKET)
-                        .key(&key)
-                        .send()
-                        .await?
-                        .upload_id()
-                        .ok_or("missing upload ID")?
-                        .to_owned(),
-                )
-            } else {
-                None
-            };
-            let query = upload_id
-                .as_ref()
-                .map(|id| format!("partNumber=1&uploadId={}", urlencoding::encode(id)))
-                .unwrap_or_default();
-            let path = format!("/{BUCKET}/{key}");
-            let target = if query.is_empty() {
-                path.clone()
-            } else {
-                format!("{path}?{query}")
-            };
-            let signer = SigV4::new(&env);
-            let unsigned = matches!(case, "trailers-over-limit" | "too-many-trailers");
-            let mode = if unsigned {
-                UNSIGNED_PAYLOAD_TRAILER
-            } else {
-                "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
-            };
-            let length = decoded.len().to_string();
-            let mut extra = vec![
-                ("content-encoding", "aws-chunked"),
-                ("x-amz-decoded-content-length", length.as_str()),
-            ];
-            if unsigned {
-                extra.push(("x-amz-trailer", "x-amz-checksum-sha256"));
-            }
-            let headers = signer.sign_with_extra_headers("PUT", &path, &query, mode, &extra);
-            let encoded = match case {
-                "signature" | "final-signature" => {
-                    let mut body = encode_signed_aws_chunked(&signer, &headers, decoded, false);
-                    let offset = if case == "signature" {
-                        body.windows(2).position(|w| w == b"\r\n").expect("chunk header") + 2
-                    } else {
-                        body.len() - 5
-                    };
-                    body[offset] = if body[offset] == b'a' { b'b' } else { b'a' };
-                    body
-                }
-                "format" => b"invalid-chunk-size\r\n".to_vec(),
-                "metadata-at-limit" | "metadata-over-limit" => {
-                    let size = if case == "metadata-at-limit" { 1024 } else { 1025 };
-                    let mut body = vec![b'z'; size - 2];
-                    body.extend_from_slice(b"\r\n");
-                    body
-                }
-                "chunk-over-limit" => format!("10000001;chunk-signature={}\r\n", "0".repeat(64)).into_bytes(),
-                "trailers-over-limit" | "too-many-trailers" => {
-                    let mut body = format!("{:x}\r\n", decoded.len()).into_bytes();
-                    body.extend_from_slice(decoded);
-                    body.extend_from_slice(b"\r\n0\r\n");
-                    if case == "trailers-over-limit" {
-                        body.extend_from_slice(format!("x-private:{}\r\n\r\n", "v".repeat(16385 - 14)).as_bytes());
-                    } else {
-                        body.extend_from_slice("x-private:v\r\n".repeat(101).as_bytes());
-                        body.extend_from_slice(b"\r\n");
-                    }
-                    body
-                }
-                _ => unreachable!("all fixtures are enumerated"),
-            };
-            let mut request = local_http_client()
-                .put(format!("{}{target}", env.url))
-                .header("authorization", &headers.authorization)
-                .header("x-amz-content-sha256", &headers.content_sha256)
-                .header("x-amz-date", &headers.amz_date);
-            for (name, value) in extra {
-                request = request.header(name, value);
-            }
-            let response = request
-                .body(encoded)
-                .timeout(std::time::Duration::from_secs(20))
-                .send()
-                .await?;
-            let status = response.status().as_u16();
-            let body = response.text().await?;
-            eprintln!("aws-chunked {case} multipart={multipart}: HTTP {status}, {body}");
-            if status != expected_status || !body.contains(&format!("<Code>{expected_code}</Code>")) {
-                failures.push(format!(
-                    "{case} multipart={multipart}: expected {expected_status} {expected_code}, got {status} {body}"
-                ));
-            }
-            for private in [
-                "private-chunked-fixture-body",
-                "chunk-signature=",
-                "AwsChunkedStreamError",
-                "x-private",
-            ] {
-                if body.contains(private) {
-                    failures.push(format!("{case}: response exposed {private}"));
-                }
-            }
-            if let Some(id) = upload_id {
-                let parts = client.list_parts().bucket(BUCKET).key(&key).upload_id(&id).send().await?;
-                assert!(parts.parts().is_empty(), "{case}: rejected UploadPart published a part");
-                client
-                    .abort_multipart_upload()
-                    .bucket(BUCKET)
-                    .key(&key)
-                    .upload_id(id)
-                    .send()
-                    .await?;
-            }
-            let absent = client
-                .get_object()
-                .bucket(BUCKET)
-                .key(&key)
-                .send()
-                .await
-                .expect_err("a rejected body must not publish an object");
-            assert_eq!(absent.as_service_error().and_then(ProvideErrorMetadata::code), Some("NoSuchKey"));
-        }
-    }
-    assert!(failures.is_empty(), "{}", failures.join("\n"));
-    Ok(())
-}
-
-#[tokio::test]
-async fn aws_chunked_valid_signed_and_trailer_boundaries_succeed() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let mut env = RustFSTestEnvironment::new().await?;
-    setup(&mut env).await?;
-    let client = env.create_s3_client();
-    let decoded = b"valid-chunked-control";
-    for (case, multipart) in ["signed", "signed-trailer", "trailer-size-limit", "trailer-count-limit"]
-        .into_iter()
-        .flat_map(|case| [false, true].map(|multipart| (case, multipart)))
-    {
-        let key = format!("{case}-{multipart}");
-        let upload_id = if multipart {
-            Some(
-                client
-                    .create_multipart_upload()
-                    .bucket(BUCKET)
-                    .key(&key)
-                    .send()
-                    .await?
-                    .upload_id()
-                    .ok_or("missing upload ID")?
-                    .to_owned(),
-            )
-        } else {
-            None
-        };
-        let query = upload_id
-            .as_ref()
-            .map(|id| format!("partNumber=1&uploadId={}", urlencoding::encode(id)))
-            .unwrap_or_default();
-        let signer = SigV4::new(&env);
-        let path = format!("/{BUCKET}/{key}");
-        let target = if query.is_empty() {
-            path.clone()
-        } else {
-            format!("{path}?{query}")
-        };
-        let length = decoded.len().to_string();
-        let mode = match case {
-            "signed" => "STREAMING-AWS4-HMAC-SHA256-PAYLOAD",
-            "signed-trailer" => "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER",
-            _ => UNSIGNED_PAYLOAD_TRAILER,
-        };
-        let mut extra = vec![
-            ("content-encoding", "aws-chunked"),
-            ("x-amz-decoded-content-length", length.as_str()),
-        ];
-        if case != "signed" {
-            extra.push(("x-amz-trailer", "x-amz-checksum-sha256"));
-        }
-        let headers = signer.sign_with_extra_headers("PUT", &path, &query, mode, &extra);
-        let mut encoded = if case.starts_with("signed") {
-            encode_signed_aws_chunked(&signer, &headers, decoded, case == "signed-trailer")
-        } else {
-            encode_unsigned_aws_chunked_with_sha256_trailer(decoded)
-        };
-        if !case.starts_with("signed") {
-            encoded.truncate(encoded.len() - 2);
-            if case == "trailer-size-limit" {
-                let checksum_line_size = format!("x-amz-checksum-sha256:{}\r\n", sha256_base64(decoded)).len();
-                encoded.extend_from_slice(format!("x-padding:{}\r\n", "v".repeat(16384 - checksum_line_size - 14)).as_bytes());
-            } else {
-                encoded.extend_from_slice("x-padding:v\r\n".repeat(99).as_bytes());
-            }
-            encoded.extend_from_slice(b"\r\n");
-        }
-        let mut request = local_http_client()
-            .put(format!("{}{target}", env.url))
-            .header("authorization", &headers.authorization)
-            .header("x-amz-content-sha256", &headers.content_sha256)
-            .header("x-amz-date", &headers.amz_date);
-        for (name, value) in extra {
-            request = request.header(name, value);
-        }
-        let response = request
-            .body(encoded)
-            .timeout(std::time::Duration::from_secs(20))
-            .send()
-            .await?;
-        let status = response.status();
-        let etag = response
-            .headers()
-            .get("etag")
-            .and_then(|value| value.to_str().ok())
-            .map(str::to_owned);
-        let body = response.text().await?;
-        assert_eq!(status.as_u16(), 200, "{case}: {body}");
-        if let Some(id) = upload_id {
-            let part = aws_sdk_s3::types::CompletedPart::builder()
-                .part_number(1)
-                .e_tag(etag.ok_or("successful UploadPart omitted ETag")?)
-                .build();
-            client
-                .complete_multipart_upload()
-                .bucket(BUCKET)
-                .key(&key)
-                .upload_id(id)
-                .multipart_upload(aws_sdk_s3::types::CompletedMultipartUpload::builder().parts(part).build())
-                .send()
-                .await?;
-        }
-        let object = client.get_object().bucket(BUCKET).key(&key).send().await?;
-        assert_eq!(object.body.collect().await?.into_bytes().as_ref(), decoded, "{case}");
-    }
-    Ok(())
-}
-
 /// Positive control: a correctly hand-signed request must succeed. Without
 /// this, every negative assertion below could pass for the wrong reason (a
 /// broken signer that never produces a valid signature).

crates/e2e_test/src/reliant/grpc_lock_server.rs

@@ -541,50 +541,6 @@ impl NodeService for MinimalLockNodeService {
         Err(Status::unimplemented("lock-only test server"))
     }
 
-    async fn artifact_rename_data(
-        &self,
-        _request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
-    ) -> Result<Response<rustfs_protos::proto_gen::node_service::RenameDataResponse>, Status> {
-        Err(Status::unimplemented("lock-only test server"))
-    }
-
-    async fn artifact_repair_rename_data(
-        &self,
-        _request: Request<rustfs_protos::proto_gen::node_service::ArtifactRepairRenameDataRequest>,
-    ) -> Result<Response<rustfs_protos::proto_gen::node_service::RenameDataResponse>, Status> {
-        Err(Status::unimplemented("lock-only test server"))
-    }
-
-    async fn artifact_repair_write_metadata(
-        &self,
-        _request: Request<rustfs_protos::proto_gen::node_service::ArtifactRepairWriteMetadataRequest>,
-    ) -> Result<Response<rustfs_protos::proto_gen::node_service::WriteMetadataResponse>, Status> {
-        Err(Status::unimplemented("lock-only test server"))
-    }
-
-    type InventoryArtifactAttemptStream =
-        Pin<Box<dyn Stream<Item = Result<rustfs_protos::proto_gen::node_service::ArtifactInventoryFrame, Status>> + Send>>;
-    async fn inventory_artifact_attempt(
-        &self,
-        _request: Request<rustfs_protos::proto_gen::node_service::ArtifactInventoryRequest>,
-    ) -> Result<Response<Self::InventoryArtifactAttemptStream>, Status> {
-        Err(Status::unimplemented("lock-only test server"))
-    }
-
-    async fn seal_artifact_attempt_repairs(
-        &self,
-        _request: Request<rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsRequest>,
-    ) -> Result<Response<rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsResponse>, Status> {
-        Err(Status::unimplemented("lock-only test server"))
-    }
-
-    async fn seal_artifact_attempt_puts(
-        &self,
-        _request: Request<rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsRequest>,
-    ) -> Result<Response<rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsResponse>, Status> {
-        Err(Status::unimplemented("lock-only test server"))
-    }
-
     async fn rename_data(
         &self,
         _request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,

crates/e2e_test/src/rpc_fault_proxy.rs

@@ -1,614 +0,0 @@
-// Copyright 2026 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Method-selective faults without changing the server or gRPC body/trailers.
-
-use bytes::Bytes;
-use http_body_util::{BodyExt as _, Full, Limited, combinators::BoxBody};
-use hyper::{Request, body::Incoming, service::service_fn};
-use hyper_util::{
-    client::legacy::{Client, connect::HttpConnector},
-    rt::{TokioExecutor, TokioIo},
-    server::conn::auto,
-};
-use prost::Message as _;
-use rustfs_protos::proto_gen::node_service::{DeleteRequest, ReadVersionRequest};
-use std::{
-    collections::HashSet,
-    io,
-    net::SocketAddr,
-    sync::{
-        Arc, Mutex,
-        atomic::{AtomicUsize, Ordering},
-    },
-    time::Duration,
-};
-use tokio::{
-    net::TcpListener,
-    sync::{Notify, watch},
-    task::{JoinHandle, JoinSet},
-};
-
-type ProxyBody = BoxBody<Bytes, io::Error>;
-type ProxyClient = Client<HttpConnector, ProxyBody>;
-
-#[derive(Clone, Copy, PartialEq, Eq)]
-enum Disposition {
-    Hold,
-    Forward,
-    Discard,
-}
-
-struct PauseState {
-    methods: Vec<&'static str>,
-    scope: Option<Vec<u8>>,
-    limit: usize,
-    matched: AtomicUsize,
-    blocked: AtomicUsize,
-    forwarded: AtomicUsize,
-    forwarded_routes: Mutex<HashSet<String>>,
-    reached: Notify,
-    release: watch::Sender<Disposition>,
-}
-
-type PauseSlot = Arc<Mutex<Option<Arc<PauseState>>>>;
-
-pub(crate) struct RpcPause {
-    state: Arc<PauseState>,
-    slot: PauseSlot,
-}
-
-impl RpcPause {
-    pub(crate) fn hits(&self) -> usize {
-        self.state.blocked.load(Ordering::SeqCst)
-    }
-
-    /// Selected RPCs whose backend returned a response, excluding discarded packets.
-    pub(crate) fn forwarded(&self) -> usize {
-        self.state.forwarded.load(Ordering::SeqCst)
-    }
-
-    pub(crate) fn forwarded_routes(&self) -> usize {
-        self.state.forwarded_routes.lock().expect("RPC route observations").len()
-    }
-
-    pub(crate) async fn wait_for_hits(&self, count: usize) -> io::Result<()> {
-        tokio::time::timeout(Duration::from_secs(20), async {
-            loop {
-                let notified = self.state.reached.notified();
-                if self.hits() >= count {
-                    return;
-                }
-                notified.await;
-            }
-        })
-        .await
-        .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "the selected RPC fault was not reached"))
-    }
-
-    pub(crate) fn release(&self) {
-        self.settle(Disposition::Forward);
-    }
-
-    /// Retire packets that outlived the caller's operation budget. A later
-    /// healthy request must not replay an expired lock grant or mutation.
-    pub(crate) fn discard_pending(&self) {
-        self.settle(Disposition::Discard);
-    }
-
-    fn settle(&self, disposition: Disposition) {
-        self.state.release.send_if_modified(|current| {
-            if *current == Disposition::Hold {
-                *current = disposition;
-                true
-            } else {
-                false
-            }
-        });
-    }
-}
-
-impl Drop for RpcPause {
-    fn drop(&mut self) {
-        self.release();
-        let mut slot = self.slot.lock().expect("RPC pause slot");
-        if slot.as_ref().is_some_and(|current| Arc::ptr_eq(current, &self.state)) {
-            *slot = None;
-        }
-    }
-}
-
-pub(crate) struct RpcFaultProxy {
-    address: SocketAddr,
-    slot: PauseSlot,
-    task: JoinHandle<()>,
-}
-
-impl RpcFaultProxy {
-    pub(crate) async fn start_on(address: SocketAddr, target: SocketAddr) -> io::Result<Self> {
-        let listener = TcpListener::bind(address).await?;
-        let address = listener.local_addr()?;
-        let slot: PauseSlot = Arc::new(Mutex::new(None));
-        let active = slot.clone();
-        let task = tokio::spawn(async move {
-            let h1: ProxyClient = Client::builder(TokioExecutor::new()).build_http();
-            let h2: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
-            let mut connections = JoinSet::new();
-            loop {
-                tokio::select! {
-                    accepted = listener.accept() => {
-                        let Ok((socket, _)) = accepted else { break };
-                        let h1 = h1.clone();
-                        let h2 = h2.clone();
-                        let active = active.clone();
-                        connections.spawn(async move {
-                            let service = service_fn(move |request: Request<Incoming>| {
-                                let client = if request.version() == http::Version::HTTP_2 { h2.clone() } else { h1.clone() };
-                                let pause = active.lock().expect("RPC pause slot").clone();
-                                forward_request(request, client, target, pause)
-
-                            });
-                            let _ = auto::Builder::new(TokioExecutor::new())
-                                .serve_connection(TokioIo::new(socket), service).await;
-                        });
-                    }
-                    _ = connections.join_next(), if !connections.is_empty() => {}
-                }
-            }
-        });
-        Ok(Self { address, slot, task })
-    }
-
-    pub(crate) fn local_addr(&self) -> SocketAddr {
-        self.address
-    }
-
-    /// Pause at most limit requests to the named unary methods; other streams
-    /// on the same HTTP/2 connection continue forwarding.
-    pub(crate) fn pause(&self, methods: &[&'static str], limit: usize, scope: Option<&str>) -> io::Result<RpcPause> {
-        let mut slot = self.slot.lock().expect("RPC pause slot");
-        if slot.is_some() || methods.is_empty() || limit == 0 || scope.is_some_and(str::is_empty) {
-            return Err(io::Error::new(
-                io::ErrorKind::InvalidInput,
-                "RPC pause needs an empty slot, methods and a positive limit",
-            ));
-        }
-        let (release, _) = watch::channel(Disposition::Hold);
-        let state = Arc::new(PauseState {
-            methods: methods.to_vec(),
-            scope: scope.map(|scope| scope.as_bytes().to_vec()),
-            limit,
-            matched: AtomicUsize::new(0),
-            blocked: AtomicUsize::new(0),
-            forwarded: AtomicUsize::new(0),
-            forwarded_routes: Mutex::new(HashSet::new()),
-            reached: Notify::new(),
-            release,
-        });
-        *slot = Some(state.clone());
-        Ok(RpcPause {
-            state,
-            slot: self.slot.clone(),
-        })
-    }
-}
-
-fn unary_payload(body: &[u8]) -> io::Result<&[u8]> {
-    if body.len() < 5 || body[0] != 0 {
-        return Err(io::Error::new(io::ErrorKind::InvalidData, "scoped RPC needs an uncompressed unary frame"));
-    }
-    let mut encoded_length = [0; 4];
-    encoded_length.copy_from_slice(&body[1..5]);
-    if u32::try_from(body.len() - 5).ok() != Some(u32::from_be_bytes(encoded_length)) {
-        return Err(io::Error::new(io::ErrorKind::InvalidData, "scoped RPC frame length mismatch"));
-    }
-    Ok(&body[5..])
-}
-
-fn scope_matches(method: &str, body: &[u8], scope: &[u8]) -> io::Result<bool> {
-    if method != "Delete" {
-        return Ok(body.windows(scope.len()).any(|window| window == scope));
-    }
-    let request = DeleteRequest::decode(unary_payload(body)?).map_err(io::Error::other)?;
-    let options: serde_json::Value = serde_json::from_str(&request.options).map_err(io::Error::other)?;
-    Ok(request.volume == ".rustfs.sys/multipart"
-        && request.path.trim_end_matches('/').as_bytes() == scope
-        && options.get("recursive").and_then(serde_json::Value::as_bool) == Some(true))
-}
-
-async fn forward_request(
-    request: Request<Incoming>,
-    client: ProxyClient,
-    target: SocketAddr,
-    pause: Option<Arc<PauseState>>,
-) -> io::Result<hyper::Response<Incoming>> {
-    let (mut parts, body) = request.into_parts();
-    let mut body = body.map_err(io::Error::other).boxed();
-    let mut observed = None;
-    if let Some(pause) = pause {
-        let method = parts.uri.path().strip_prefix("/node_service.NodeService/");
-        if let Some(method) = method.filter(|name| pause.methods.contains(name)) {
-            let mut route = None;
-            let matches = if let Some(scope) = pause.scope.as_deref() {
-                // Inspect bounded control routing while preserving the exact
-                // signed bytes and trailers forwarded to the backend.
-                let collected = Limited::new(body, 1024 * 1024).collect().await.map_err(io::Error::other)?;
-                let trailers = collected.trailers().cloned();
-                let bytes = collected.to_bytes();
-                let matches = scope_matches(method, &bytes, scope)?;
-                if matches && method == "ReadVersion" {
-                    let disk = ReadVersionRequest::decode(unary_payload(&bytes)?)
-                        .map_err(io::Error::other)?
-                        .disk;
-                    if !disk.is_empty() {
-                        route = Some(disk);
-                    }
-                }
-                body = Full::new(bytes)
-                    .map_err(|never| match never {})
-                    .with_trailers(async move { trailers.map(Ok::<_, io::Error>) })
-                    .boxed();
-                matches
-            } else {
-                true
-            };
-            if matches && pause.matched.fetch_add(1, Ordering::SeqCst) < pause.limit {
-                let mut released = pause.release.subscribe();
-                pause.blocked.fetch_add(1, Ordering::SeqCst);
-                pause.reached.notify_one();
-                let disposition = loop {
-                    let disposition = *released.borrow_and_update();
-                    if disposition != Disposition::Hold {
-                        break disposition;
-                    }
-                    if released.changed().await.is_err() {
-                        break Disposition::Discard;
-                    }
-                };
-                if disposition == Disposition::Discard {
-                    return Err(io::Error::new(io::ErrorKind::ConnectionAborted, "injected RPC packet loss"));
-                }
-                observed = Some((pause, route));
-            }
-        }
-    }
-    if !parts.headers.contains_key(http::header::HOST)
-        && let Some(authority) = parts.uri.authority()
-    {
-        parts.headers.insert(
-            http::header::HOST,
-            http::HeaderValue::from_str(authority.as_str()).map_err(io::Error::other)?,
-        );
-    }
-    parts.uri = http::Uri::builder()
-        .scheme("http")
-        .authority(target.to_string())
-        .path_and_query(parts.uri.path_and_query().map_or("/", |value| value.as_str()))
-        .build()
-        .map_err(io::Error::other)?;
-    let response = client
-        .request(Request::from_parts(parts, body))
-        .await
-        .map_err(io::Error::other)?;
-    if let Some((pause, route)) = observed
-        && response.status().is_success()
-        && response.headers().get("grpc-status").is_none_or(|status| status == "0")
-    {
-        pause.forwarded.fetch_add(1, Ordering::SeqCst);
-        if let Some(route) = route {
-            pause.forwarded_routes.lock().expect("RPC route observations").insert(route);
-        }
-    }
-    Ok(response)
-}
-
-impl Drop for RpcFaultProxy {
-    fn drop(&mut self) {
-        self.task.abort();
-    }
-}
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use http_body_util::{Full, StreamBody};
-    use hyper::{Response, body::Frame};
-    use std::convert::Infallible;
-
-    struct Backend {
-        received: Arc<AtomicUsize>,
-        address: SocketAddr,
-        task: JoinHandle<()>,
-    }
-
-    impl Drop for Backend {
-        fn drop(&mut self) {
-            self.task.abort();
-        }
-    }
-
-    async fn backend() -> io::Result<Backend> {
-        let listener = TcpListener::bind("127.0.0.1:0").await?;
-        let address = listener.local_addr()?;
-        let received = Arc::new(AtomicUsize::new(0));
-        let received_task = received.clone();
-        let task = tokio::spawn(async move {
-            let mut children = JoinSet::new();
-            loop {
-                tokio::select! {
-                    accepted = listener.accept() => {
-                        let Ok((socket, _)) = accepted else { break };
-                        let received = received_task.clone();
-                        children.spawn(async move {
-                            let service = service_fn(move |request: Request<Incoming>| {
-                                let received = received.clone();
-                                async move {
-                                received.fetch_add(1, Ordering::SeqCst);
-                                let authority = request.headers().get(http::header::HOST).cloned();
-                                let body = request.into_body().collect().await?.to_bytes();
-                                let mut trailers = http::HeaderMap::new();
-                                trailers.insert("grpc-status", http::HeaderValue::from_static("0"));
-                                trailers.insert("x-proof", http::HeaderValue::from_static("unchanged-proof"));
-                                let stream = futures::stream::iter([
-                                    Ok::<_, Infallible>(Frame::data(body)),
-                                    Ok(Frame::trailers(trailers)),
-                                ]);
-                                let mut response = Response::new(StreamBody::new(stream));
-                                response.headers_mut().insert(http::header::TRAILER, http::HeaderValue::from_static("grpc-status, x-proof"));
-                                if let Some(authority) = authority { response.headers_mut().insert("x-original-authority", authority); }
-                                Ok::<_, hyper::Error>(response)
-                            }});
-                            let _ = auto::Builder::new(TokioExecutor::new()).serve_connection(TokioIo::new(socket), service).await;
-                        });
-                    }
-                    _ = children.join_next(), if !children.is_empty() => {}
-                }
-            }
-        });
-        Ok(Backend { received, address, task })
-    }
-
-    async fn call(client: &ProxyClient, address: SocketAddr, method: &str) -> io::Result<hyper::Response<Incoming>> {
-        call_body(client, address, method, Bytes::from_static(b"opaque signed protobuf")).await
-    }
-
-    async fn call_body(
-        client: &ProxyClient,
-        address: SocketAddr,
-        method: &str,
-        bytes: Bytes,
-    ) -> io::Result<hyper::Response<Incoming>> {
-        let body: ProxyBody = Full::new(bytes).map_err(|never| match never {}).boxed();
-        let request = Request::builder()
-            .method("POST")
-            .uri(format!("http://{address}/node_service.NodeService/{method}"))
-            .header(http::header::TE, "trailers")
-            .body(body)
-            .map_err(io::Error::other)?;
-        client.request(request).await.map_err(io::Error::other)
-    }
-
-    #[tokio::test]
-    async fn preserves_protocol_authority_payload_and_grpc_trailers() {
-        tokio::time::timeout(Duration::from_secs(10), async {
-            let backend = backend().await.expect("backend listener");
-            let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback address"), backend.address)
-                .await
-                .expect("proxy");
-            for h2 in [false, true] {
-                let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(h2).build_http();
-                let response = call(&client, proxy.local_addr(), "ReadVersion")
-                    .await
-                    .expect("transparent request");
-                assert_eq!(response.version(), if h2 { http::Version::HTTP_2 } else { http::Version::HTTP_11 });
-                assert_eq!(response.headers()["x-original-authority"], proxy.local_addr().to_string());
-                let collected = response.into_body().collect().await.expect("complete body and trailers");
-                let trailers = collected.trailers().expect("gRPC trailers must survive both proxy hops");
-                assert_eq!(trailers["grpc-status"], "0");
-                assert_eq!(trailers["x-proof"], "unchanged-proof");
-                assert_eq!(collected.to_bytes(), Bytes::from_static(b"opaque signed protobuf"));
-            }
-        })
-        .await
-        .expect("transparent proxy deadline");
-    }
-
-    #[tokio::test]
-    async fn isolates_rpc_methods_limits_injection_and_releases_on_drop() {
-        tokio::time::timeout(Duration::from_secs(10), async {
-            let backend = backend().await.expect("backend listener");
-            let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback address"), backend.address)
-                .await
-                .expect("proxy");
-            let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
-            let pause = proxy.pause(&["WriteMetadata"], 1, None).expect("arm one metadata request");
-            let task_client = client.clone();
-            let address = proxy.local_addr();
-            let blocked = tokio::spawn(async move { call(&task_client, address, "WriteMetadata").await });
-            pause.wait_for_hits(1).await.expect("metadata barrier reached");
-            for method in ["Lock", "WriteMetadata"] {
-                let response = call(&client, address, method)
-                    .await
-                    .expect("unselected or over-limit request must progress");
-                assert_eq!(
-                    response.into_body().collect().await.expect("forwarded body").to_bytes(),
-                    Bytes::from_static(b"opaque signed protobuf")
-                );
-            }
-            assert_eq!(pause.hits(), 1);
-            assert!(!blocked.is_finished(), "the selected request must still be held");
-            drop(pause);
-            assert_eq!(
-                blocked
-                    .await
-                    .expect("blocked request task")
-                    .expect("released request")
-                    .into_body()
-                    .collect()
-                    .await
-                    .expect("released body")
-                    .to_bytes(),
-                Bytes::from_static(b"opaque signed protobuf")
-            );
-        })
-        .await
-        .expect("selective fault deadline");
-    }
-    #[tokio::test]
-    async fn released_observer_requires_a_selected_backend_response() {
-        tokio::time::timeout(Duration::from_secs(10), async {
-            let backend = backend().await.expect("backend listener");
-            let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
-                .await
-                .expect("proxy listener");
-            let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
-            let observer = proxy
-                .pause(&["Lock"], usize::MAX, Some("recovery-bucket"))
-                .expect("RPC observer");
-            let unrelated = call_body(&client, proxy.local_addr(), "Lock", Bytes::from_static(b"other-bucket"))
-                .await
-                .expect("unrelated request");
-            unrelated.into_body().collect().await.expect("unrelated body");
-            assert_eq!(observer.forwarded(), 0, "unrelated traffic cannot prove recovery");
-            let task_client = client.clone();
-            let address = proxy.local_addr();
-            let request =
-                tokio::spawn(
-                    async move { call_body(&task_client, address, "Lock", Bytes::from_static(b"recovery-bucket")).await },
-                );
-            observer.wait_for_hits(1).await.expect("selected request arrived");
-            assert_eq!(observer.forwarded(), 0, "an intercepted request has not reached its backend");
-            observer.release();
-            let body = request
-                .await
-                .expect("request task")
-                .expect("forwarded request")
-                .into_body()
-                .collect()
-                .await
-                .expect("forwarded body");
-            assert_eq!(body.trailers().expect("preserved trailers")["x-proof"], "unchanged-proof");
-            assert_eq!(body.to_bytes(), Bytes::from_static(b"recovery-bucket"));
-            assert_eq!(observer.forwarded(), 1);
-            let another = call_body(&client, address, "Lock", Bytes::from_static(b"recovery-bucket"))
-                .await
-                .expect("observer forwards without pausing");
-            another.into_body().collect().await.expect("second body");
-            assert_eq!(observer.forwarded(), 2);
-            drop(observer);
-            let routes = proxy
-                .pause(&["ReadVersion"], usize::MAX, Some("recovery-route"))
-                .expect("disk-route observer");
-            routes.release();
-            for (disk, expected) in [("drive-a", 1), ("drive-a", 1), ("drive-b", 2)] {
-                let request = ReadVersionRequest {
-                    disk: disk.to_owned(),
-                    path: "recovery-route".to_owned(),
-                    ..Default::default()
-                }
-                .encode_to_vec();
-                let mut frame = vec![0];
-                frame.extend_from_slice(&u32::try_from(request.len()).expect("small frame").to_be_bytes());
-                frame.extend_from_slice(&request);
-                call_body(&client, address, "ReadVersion", Bytes::from(frame))
-                    .await
-                    .expect("routed read")
-                    .into_body()
-                    .collect()
-                    .await
-                    .expect("routed response");
-                assert_eq!(routes.forwarded_routes(), expected, "repeated traffic from one disk is only one voter");
-            }
-        })
-        .await
-        .expect("passive observer deadline");
-    }
-
-    #[tokio::test]
-    async fn scoped_fault_discards_expired_packets_without_replaying_them() {
-        tokio::time::timeout(Duration::from_secs(10), async {
-            let backend = backend().await.expect("backend listener");
-            let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
-                .await
-                .expect("proxy listener");
-            let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
-            assert!(proxy.pause(&["Lock"], 1, Some("")).is_err(), "empty scope must not match every body");
-            let pause = proxy
-                .pause(&["Lock"], usize::MAX, Some("specific-bucket"))
-                .expect("bucket-scoped lock fault");
-            let unrelated = call_body(&client, proxy.local_addr(), "Lock", Bytes::from_static(b"other-bucket"))
-                .await
-                .expect("unrelated bucket must progress");
-            let unrelated = unrelated.into_body().collect().await.expect("unrelated response");
-            assert_eq!(unrelated.trailers().expect("unchanged trailers")["x-proof"], "unchanged-proof");
-            assert_eq!(unrelated.to_bytes(), Bytes::from_static(b"other-bucket"));
-            assert_eq!(pause.hits(), 0);
-            let task_client = client.clone();
-            let address = proxy.local_addr();
-            let blocked =
-                tokio::spawn(
-                    async move { call_body(&task_client, address, "Lock", Bytes::from_static(b"specific-bucket")).await },
-                );
-            pause.wait_for_hits(1).await.expect("selected bucket reached");
-            assert!(!blocked.is_finished());
-            pause.discard_pending();
-            drop(pause); // Drop must not replace Discard with Forward.
-            assert!(blocked.await.expect("request task").is_err());
-            assert_eq!(backend.received.load(Ordering::SeqCst), 1, "discard must never execute at the peer");
-            let recovered = call(&client, address, "Lock").await.expect("future requests recover");
-            recovered.into_body().collect().await.expect("recovered response");
-            assert_eq!(backend.received.load(Ordering::SeqCst), 2);
-        })
-        .await
-        .expect("scoped discard deadline");
-    }
-    #[test]
-    fn delete_scope_matches_only_the_upload_root() {
-        let frame = |volume: &str, path: &str| {
-            let message = DeleteRequest {
-                volume: volume.to_owned(),
-                path: path.to_owned(),
-                options: r#"{"recursive":true}"#.to_owned(),
-                ..Default::default()
-            };
-            let payload = message.encode_to_vec();
-            let length = u32::try_from(payload.len()).expect("small control packet");
-            let mut bytes = vec![0];
-            bytes.extend_from_slice(&length.to_be_bytes());
-            bytes.extend_from_slice(&payload);
-            bytes
-        };
-        let root = b"hash/upload";
-        assert!(scope_matches("Delete", &frame(".rustfs.sys/multipart", "hash/upload"), root).expect("root frame"));
-        assert!(scope_matches("Delete", &frame(".rustfs.sys/multipart", "hash/upload/"), root).expect("directory frame"));
-        for path in ["hash/upload/part.1", "hash/upload/.abort-authority", "hash/upload-other"] {
-            assert!(!scope_matches("Delete", &frame(".rustfs.sys/multipart", path), root).expect("other path"));
-        }
-        assert!(!scope_matches("Delete", &frame(".rustfs.sys/tmp", "hash/upload"), root).expect("other volume"));
-        let non_recursive = DeleteRequest {
-            volume: ".rustfs.sys/multipart".to_owned(),
-            path: "hash/upload".to_owned(),
-            options: r#"{"recursive":false}"#.to_owned(),
-            ..Default::default()
-        }
-        .encode_to_vec();
-        let mut non_recursive_frame = vec![0];
-        non_recursive_frame.extend_from_slice(&u32::try_from(non_recursive.len()).expect("small frame").to_be_bytes());
-        non_recursive_frame.extend_from_slice(&non_recursive);
-        assert!(!scope_matches("Delete", &non_recursive_frame, root).expect("non-recursive frame"));
-        assert!(
-            scope_matches("Delete", &[0, 0, 0, 0, 1], root).is_err(),
-            "truncation must not become a match"
-        );
-    }
-}

crates/e2e_test/src/storage_api.rs

@@ -15,14 +15,12 @@
 #[cfg(test)]
 pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys;
 #[cfg(test)]
-pub(crate) use rustfs_ecstore::api::disk::{
-    DiskError, DiskOption, Endpoint, RUSTFS_META_BUCKET, VolumeInfo, WalkDirOptions, new_disk,
-};
+pub(crate) use rustfs_ecstore::api::disk::{Endpoint, RUSTFS_META_BUCKET, VolumeInfo, WalkDirOptions};
 pub(crate) use rustfs_ecstore::api::rpc::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
 #[cfg(test)]
 pub(crate) use rustfs_ecstore::api::rpc::{
     TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
-    set_tonic_canonical_body_digest, verify_tonic_boot_epoch_response,
+    verify_tonic_boot_epoch_response,
 };
 #[cfg(test)]
 pub(crate) use rustfs_ecstore::api::rpc::{gen_tonic_signature_interceptor, node_service_time_out_client};
@@ -38,11 +36,6 @@ pub(crate) mod grpc_lock {
     pub(crate) use super::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
 }
 
-#[cfg(test)]
-pub(crate) mod artifact_rpc {
-    pub(crate) use super::{DiskError, TONIC_RPC_PREFIX, gen_tonic_signature_headers, set_tonic_canonical_body_digest};
-}
-
 /// Signing/transport surface used by the cross-process internode RPC signature
 /// acceptance tests (backlog#1327). The signing helpers are what let a test mint
 /// the one legitimately signed request an on-path attacker is assumed to have
@@ -59,8 +52,3 @@ pub(crate) mod internode_rpc_signature {
 pub(crate) mod replication_extension {
     pub(crate) use super::BucketTargetSys;
 }
-
-#[cfg(test)]
-pub(crate) mod upgrade_compatibility {
-    pub(crate) use super::{DiskOption, Endpoint, new_disk};
-}

crates/e2e_test/src/upgrade_compatibility_test.rs

@@ -14,7 +14,7 @@
 
 use crate::common::{
     RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging, replication_fast_env, rustfs_binary_path,
-    signal_process, signed_request,
+    signed_request,
 };
 use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, Operation as FakeTargetOperation};
 use crate::on_demand_migration::common::{ODM_SERVER_ENV, OdmTestEnv, SeedObject, fake_source_client};
@@ -1372,353 +1372,6 @@ async fn rc5_rollback_requires_restoring_odm_configuration() -> TestResult {
     Ok(())
 }
 
-struct Rc5CheckpointCapture {
-    disk: PathBuf,
-    path: PathBuf,
-    task_id: String,
-    bytes: Vec<u8>,
-}
-
-async fn capture_rc5_checkpoint(drives: &[PathBuf], writer_pid: u32) -> Result<Rc5CheckpointCapture, BoxError> {
-    let has_signed_progress = |value: &Value| {
-        value["schema_version"].as_u64() == Some(6)
-            && value["successful_objects"].as_u64().is_some_and(|count| count > 0)
-            && value["processed_objects"]
-                .as_array()
-                .is_some_and(|objects| !objects.is_empty())
-            && value["integrity_digest"].as_str().is_some_and(|digest| !digest.is_empty())
-    };
-    tokio::time::timeout(Duration::from_secs(60), async {
-        loop {
-            for drive in drives {
-                let mut entries = match tokio::fs::read_dir(drive.join(".rustfs.sys/buckets")).await {
-                    Ok(entries) => entries,
-                    Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
-                    Err(error) => return Err(error.into()),
-                };
-                while let Some(entry) = entries.next_entry().await? {
-                    if !entry.file_name().to_string_lossy().ends_with("_ahm_checkpoint.json") {
-                        continue;
-                    }
-                    let metadata = entry.metadata().await?;
-                    if !metadata.is_file() || metadata.len() > 1024 * 1024 {
-                        return Err("the old checkpoint is not a bounded regular file".into());
-                    }
-                    let bytes = match tokio::fs::read(entry.path()).await {
-                        Ok(bytes) => bytes,
-                        Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
-                        Err(error) => return Err(error.into()),
-                    };
-                    let Ok(value) = serde_json::from_slice::<Value>(&bytes) else {
-                        continue;
-                    };
-                    if !has_signed_progress(&value) {
-                        continue;
-                    }
-
-                    // Freeze only this fixture's writer before copying its actual
-                    // bytes. The environment's normal stop path kills a stopped child.
-                    signal_process(writer_pid, "STOP")?;
-                    let bytes = tokio::fs::read(entry.path()).await?;
-                    let value: Value = serde_json::from_slice(&bytes)?;
-                    if !has_signed_progress(&value) {
-                        return Err("the stopped rc.5 writer did not retain signed checkpoint progress".into());
-                    }
-                    let task_id = value["task_id"].as_str().ok_or("the old checkpoint has no task id")?;
-                    uuid::Uuid::parse_str(task_id)?;
-                    if entry.file_name() != std::ffi::OsStr::new(&format!("{task_id}_ahm_checkpoint.json")) {
-                        return Err("the old checkpoint task id does not match its filename".into());
-                    }
-                    return Ok(Rc5CheckpointCapture {
-                        disk: drive.clone(),
-                        path: entry.path(),
-                        task_id: task_id.to_owned(),
-                        bytes,
-                    });
-                }
-            }
-            sleep(Duration::from_millis(5)).await;
-        }
-    })
-    .await
-    .map_err(|_| "rc.5 did not produce a signed checkpoint with progress within 60 seconds")?
-}
-
-fn rc5_checkpoint_object(index: usize) -> (String, Vec<u8>) {
-    (
-        format!("object-{index:05}.bin"),
-        format!("rc.5 checkpoint upgrade object {index:05}\n")
-            .repeat(256)
-            .into_bytes(),
-    )
-}
-
-fn checkpoint_binary_sha256(path: &Path) -> Result<String, BoxError> {
-    use sha2::Digest;
-    use std::io::Read;
-
-    let mut input = std::fs::File::open(path)?;
-    let mut digest = sha2::Sha256::new();
-    let mut buffer = [0_u8; 64 * 1024];
-    loop {
-        let count = input.read(&mut buffer)?;
-        if count == 0 {
-            break;
-        }
-        digest.update(&buffer[..count]);
-    }
-    Ok(hex_simd::encode_to_string(digest.finalize(), hex_simd::AsciiCase::Lower))
-}
-
-#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-#[ignore = "requires the pinned 1.0.0-rc.5 release binary"]
-async fn rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects() -> TestResult {
-    use crate::storage_api::upgrade_compatibility::{DiskOption, Endpoint, new_disk};
-    use futures::{StreamExt, TryStreamExt, stream};
-    use rustfs_heal::heal::CheckpointManager;
-
-    const BUCKET: &str = "upgrade-rc5-checkpoint";
-    const OBJECTS: usize = 1600;
-
-    init_logging();
-    let previous_binary = source_binary()?;
-    assert_eq!(binary_source_revision(&previous_binary).await?, RC5_COMMIT);
-    let current_binary = rustfs_binary_path();
-    let evidence_dir = std::env::var_os(G09_EVIDENCE_DIR_ENV)
-        .map(|root| PathBuf::from(root).join(format!("rc5-checkpoint-upgrade-{}", uuid::Uuid::new_v4())));
-    if let Some(directory) = &evidence_dir {
-        std::fs::create_dir_all(directory)?;
-    }
-
-    // Declare the server after its extra drives so error unwinding stops it
-    // before the fixture directories are removed.
-    let fixture = tempfile::TempDir::new()?;
-    let extra_drives = (0..3)
-        .map(|index| {
-            let path = fixture.path().join(format!("drive-{index}"));
-            std::fs::create_dir(&path)?;
-            Ok(path.to_str().ok_or("fixture drive path is not UTF-8")?.to_owned())
-        })
-        .collect::<Result<Vec<_>, BoxError>>()?;
-    let mut env = RustFSTestEnvironment::new().await?;
-    let log_path = evidence_dir.as_deref().unwrap_or(fixture.path()).join("server.log");
-    env.capture_log_path = Some(log_path.to_str().ok_or("fixture log path is not UTF-8")?.to_owned());
-    let server_env = [
-        ("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", "true"),
-        ("RUSTFS_DURABILITY_MODE", "strict"),
-        ("RUSTFS_NEW_BUCKET_DURABILITY_MODE", "strict"),
-        ("RUST_LOG", "warn,rustfs::heal=debug"),
-    ];
-    env.start_rustfs_server_from_binary(&previous_binary, extra_drives.iter().map(String::as_str).collect(), &server_env)
-        .await?;
-    env.create_test_bucket(BUCKET).await?;
-    let (status, durability) = admin_request(
-        &env.url,
-        Method::GET,
-        &format!("/rustfs/admin/v3/bucket-durability/{BUCKET}"),
-        None,
-        &env.access_key,
-        &env.secret_key,
-    )
-    .await?;
-    assert_eq!(status, StatusCode::OK, "old bucket durability: {durability}");
-    assert_eq!(serde_json::from_str::<Value>(&durability)?["mode"], "strict");
-
-    let old_client = env.create_s3_client();
-    let manifest = stream::iter(0..OBJECTS)
-        .map(|index| {
-            let client = old_client.clone();
-            async move {
-                let (key, body) = rc5_checkpoint_object(index);
-                let entry = serde_json::json!({
-                    "bucket": BUCKET,
-                    "key": key,
-                    "length": body.len(),
-                    "sha256": rustfs_utils::hex_sha256(&body, str::to_owned),
-                });
-                client
-                    .put_object()
-                    .bucket(BUCKET)
-                    .key(&key)
-                    .body(ByteStream::from(body))
-                    .send()
-                    .await?;
-                Ok::<_, BoxError>(entry)
-            }
-        })
-        .buffered(4)
-        .try_collect::<Vec<_>>()
-        .await?;
-    let drives = extra_drives
-        .iter()
-        .map(PathBuf::from)
-        .chain(std::iter::once(PathBuf::from(&env.temp_dir)))
-        .collect::<Vec<_>>();
-    let writer_pid = env.process.as_ref().ok_or("rc.5 writer is not running")?.id();
-    let start_heal = async {
-        let (status, body) = tokio::time::timeout(
-            Duration::from_secs(15),
-            admin_request(
-                &env.url,
-                Method::POST,
-                "/rustfs/admin/v3/heal/",
-                Some(
-                    serde_json::json!({
-                        "recursive": true, "dryRun": false, "remove": false, "recreate": false,
-                        "scanMode": 2, "updateParity": false, "nolock": false, "pool": 0, "set": 0,
-                    })
-                    .to_string(),
-                ),
-                &env.access_key,
-                &env.secret_key,
-            ),
-        )
-        .await??;
-        if status != StatusCode::OK {
-            return Err(format!("rc.5 root heal was not accepted: {status} {body}").into());
-        }
-        Ok::<_, BoxError>(body)
-    };
-    let (capture, heal_start) = tokio::try_join!(capture_rc5_checkpoint(&drives, writer_pid), start_heal)?;
-    env.stop_server();
-    assert_eq!(
-        std::fs::read(&capture.path)?,
-        capture.bytes,
-        "stopping the old writer must preserve captured bytes"
-    );
-    let checkpoint_dir = capture.disk.join(".rustfs.sys/buckets");
-    let resume_path = checkpoint_dir.join(format!("{}_ahm_resume_state.json", capture.task_id));
-    let resume_bytes = std::fs::read(&resume_path)?;
-    let old_checkpoint: Value = serde_json::from_slice(&capture.bytes)?;
-    if let Some(directory) = &evidence_dir {
-        std::fs::write(directory.join("checkpoint.rc5.json"), &capture.bytes)?;
-        std::fs::write(directory.join("resume.rc5.json"), &resume_bytes)?;
-        std::fs::write(directory.join("manifest.json"), serde_json::to_vec_pretty(&manifest)?)?;
-        std::fs::write(
-            directory.join("source.json"),
-            serde_json::to_vec_pretty(&serde_json::json!({
-                "kind": "old-release-checkpoint-reader-upgrade",
-                "scope": "schema-6 reader authentication, safe rewind and object reads; not full G09 qualification",
-                "writer_revision": RC5_COMMIT,
-                "writer_binary_sha256": checkpoint_binary_sha256(&previous_binary)?,
-                "candidate_revision": current_source_revision()?,
-                "candidate_binary_sha256": checkpoint_binary_sha256(&current_binary)?,
-                "layout": "one process and four logical drives; not physical fault qualification",
-                "task_id": capture.task_id,
-                "checkpoint_sha256": rustfs_utils::hex_sha256(&capture.bytes, str::to_owned),
-                "resume_sha256": rustfs_utils::hex_sha256(&resume_bytes, str::to_owned),
-                "acknowledged_objects": manifest.len(),
-                "bucket_durability": serde_json::from_str::<Value>(&durability)?,
-                "heal_start": serde_json::from_str::<Value>(&heal_start)?,
-            }))?,
-        )?;
-    }
-
-    let blocked_path = checkpoint_dir.join(format!("{}_ahm_checkpoint.blocked", capture.task_id));
-    assert!(!blocked_path.try_exists()?, "the original checkpoint must not already be blocked");
-    let disk_options = DiskOption {
-        cleanup: false,
-        health_check: false,
-    };
-    // Reopen with the same member order passed to the old writer.
-    let disk_index = drives
-        .iter()
-        .position(|drive| drive == &capture.disk)
-        .ok_or("captured disk is not a fixture member")?;
-    let mut endpoint = Endpoint::try_from(capture.disk.to_str().ok_or("captured drive path is not UTF-8")?)?;
-    endpoint.set_pool_index(0);
-    endpoint.set_set_index(0);
-    endpoint.set_disk_index(disk_index);
-    let disk = new_disk(&endpoint, &disk_options).await?;
-    let manager = CheckpointManager::load_from_disk(disk.clone(), &capture.task_id).await?;
-    let rewound = manager.get_checkpoint().await;
-    assert_eq!(rewound.schema_version, 7);
-    assert_eq!(rewound.task_id, capture.task_id);
-    assert_eq!((rewound.current_bucket_index, rewound.current_object_index), (0, 0));
-    assert!(rewound.processed_objects.is_empty() && rewound.failed_objects.is_empty() && rewound.skipped_objects.is_empty());
-    assert_eq!(rewound.successful_objects, 0);
-    assert_eq!(rewound.failed_object_count, 0);
-    assert_eq!(rewound.skipped_object_count, 0);
-    assert_eq!(rewound.processed_bytes, 0);
-    assert_eq!((rewound.total_objects, rewound.total_bytes), (0, 0));
-    assert!(rewound.baseline_generation.is_none() && !rewound.baseline_known && !rewound.counter_unknown);
-    assert!(!blocked_path.try_exists()?, "an authentic old checkpoint must not be quarantined");
-    assert_eq!(
-        std::fs::read(&capture.path)?,
-        capture.bytes,
-        "the reader must not rewrite the old checkpoint"
-    );
-    assert_eq!(std::fs::read(&resume_path)?, resume_bytes, "the reader must not take over the old task");
-    drop(manager);
-    drop(disk);
-
-    // A separate copy with one digest byte changed proves that successful
-    // rewind above did not bypass authentication of the original wire bytes.
-    let corrupt_root = fixture.path().join("corrupt-reader-control");
-    let corrupt_metadata = corrupt_root.join(".rustfs.sys/buckets");
-    std::fs::create_dir_all(&corrupt_metadata)?;
-    let mut corrupted = capture.bytes.clone();
-    let digest = old_checkpoint["integrity_digest"]
-        .as_str()
-        .ok_or("old checkpoint digest is not a string")?;
-    let offset = corrupted
-        .windows(digest.len())
-        .position(|bytes| bytes == digest.as_bytes())
-        .ok_or("the digest is absent from the captured bytes")?;
-    corrupted[offset] = if corrupted[offset] == b'A' { b'B' } else { b'A' };
-    let corrupt_path = corrupt_metadata.join(format!("{}_ahm_checkpoint.json", capture.task_id));
-    std::fs::write(&corrupt_path, &corrupted)?;
-    let endpoint = Endpoint::try_from(corrupt_root.to_str().ok_or("control disk path is not UTF-8")?)?;
-    let corrupt_disk = new_disk(&endpoint, &disk_options).await?;
-    let Err(error) = CheckpointManager::load_from_disk(corrupt_disk.clone(), &capture.task_id).await else {
-        return Err("the current checkpoint reader accepted a corrupted signature".into());
-    };
-    assert!(matches!(error, rustfs_heal::Error::InvalidCheckpoint(ref reason) if reason.contains("digest")));
-    let corrupt_blocked = corrupt_metadata.join(format!("{}_ahm_checkpoint.blocked", capture.task_id));
-    assert!(corrupt_blocked.try_exists()?, "the corrupted copy must retain a blocked marker");
-    assert_eq!(std::fs::read(&corrupt_path)?, corrupted, "quarantine must preserve the invalid bytes");
-    if let Some(directory) = &evidence_dir {
-        std::fs::write(directory.join("checkpoint.corrupted.json"), &corrupted)?;
-        std::fs::copy(&corrupt_blocked, directory.join("checkpoint.corrupted.blocked"))?;
-    }
-    drop(corrupt_disk);
-
-    env.restart_server_preserving_data(extra_drives.iter().map(String::as_str).collect(), &server_env)
-        .await?;
-    let current_client = env.create_s3_client();
-    stream::iter(0..OBJECTS)
-        .map(|index| {
-            let client = current_client.clone();
-            async move {
-                let (key, expected) = rc5_checkpoint_object(index);
-                let (_, actual) = read_object(&client, BUCKET, &key, None).await?;
-                assert_eq!(actual, expected, "old object {key} changed during the upgrade");
-                Ok::<_, BoxError>(())
-            }
-        })
-        .buffer_unordered(4)
-        .try_collect::<Vec<_>>()
-        .await?;
-    env.stop_server();
-    if let Some(directory) = &evidence_dir {
-        std::fs::write(
-            directory.join("result.json"),
-            serde_json::to_vec_pretty(&serde_json::json!({
-                "status": "passed",
-                "scope": "old-release checkpoint reader and object-data upgrade only",
-                "authenticated_original_bytes": true,
-                "original_checkpoint_unchanged_after_read": true,
-                "old_successful_objects": old_checkpoint["successful_objects"],
-                "rewound_schema": rewound.schema_version,
-                "rewound_successful_objects": rewound.successful_objects,
-                "corrupt_digest_rejected_and_retained": true,
-                "verified_objects": OBJECTS,
-            }))?,
-        )?;
-    }
-    Ok(())
-}
-
 #[tokio::test]
 #[ignore = "requires a pinned previous RustFS release binary"]
 async fn direct_upgrade_from_previous_release_preserves_object_contracts() -> TestResult {

crates/ecstore/src/api/mod.rs

@@ -382,43 +382,6 @@ pub mod config {
             with_config_object_read_lock, with_config_object_write_lock, with_server_config_read_lock,
             with_server_config_write_lock,
         };
-
-        /// Observe persisted config bytes and their reader-associated metadata within a byte limit.
-        pub async fn read_config_observation(
-            api: std::sync::Arc<crate::store::ECStore>,
-            file: &str,
-            max_bytes: usize,
-        ) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
-            crate::config::com::read_config_observation(api, file, max_bytes).await
-        }
-
-        /// Observe bounded config bytes while the caller retains the object's write owner.
-        pub async fn read_config_observation_no_lock(
-            api: std::sync::Arc<crate::store::ECStore>,
-            file: &str,
-            max_bytes: usize,
-            guard: &crate::object_api::WriteCommitGuard,
-        ) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
-            crate::config::com::read_config_observation_no_lock(api, file, max_bytes, guard).await
-        }
-
-        /// Read config bytes and metadata without treating an empty payload as absent.
-        pub async fn read_config_preserve_empty_with_metadata_opts(
-            api: std::sync::Arc<crate::store::ECStore>,
-            file: &str,
-            opts: &crate::object_api::ObjectOptions,
-        ) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
-            crate::config::com::read_config_preserve_empty_with_metadata_opts(api, file, opts).await
-        }
-
-        pub async fn read_config_limited_preserve_empty_with_metadata_opts(
-            api: std::sync::Arc<crate::store::ECStore>,
-            file: &str,
-            opts: &crate::object_api::ObjectOptions,
-            max_bytes: usize,
-        ) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
-            crate::config::com::read_config_limited_preserve_empty_with_metadata_opts(api, file, opts, max_bytes).await
-        }
     }
 
     pub mod storageclass {
@@ -468,19 +431,16 @@ pub mod disk {
     pub use crate::disk::local::{ReplacementExecutionLease, ScanGuard};
     #[cfg(all(feature = "test-util", not(windows)))]
     pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
-    pub use crate::disk::{ArtifactInventoryFrame, ArtifactInventoryVersion};
     pub use crate::disk::{
         BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
         CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration, CurrentDeleteTransition,
         DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
         DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DeleteRecoveryState,
         Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, FileReader, FileWriter,
-        GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEALING_MARKER_PATH, LegacyFileStream, LegacyFileWrite,
-        LegacyMetadataWrite, LegacyPartMutation, LegacyPartOperation, LegacyPathDelete, LegacyPathDeleteOperation,
-        LegacyRenameData, LegacyRenameFile, NsScannerOpenRequest, OldCurrentSize, PartTransactionAction,
-        QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
-        STORAGE_FORMAT_FILE, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path,
-        new_disk, validate_batch_read_version_item_count,
+        GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
+        PartTransactionAction, QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp,
+        ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
+        is_quota_mutation_fence_path, new_disk, validate_batch_read_version_item_count,
     };
     pub use bytes::Bytes;
     pub use endpoint::Endpoint;
@@ -531,8 +491,6 @@ pub mod global {
 }
 
 pub mod runtime {
-    #[cfg(any(test, feature = "test-util"))]
-    pub use crate::runtime::global::set_object_store_resolver;
     pub use crate::runtime::instance::{InstanceContext, bootstrap_ctx};
     pub use crate::runtime::sources::{
         boot_time, bucket_monitor, deployment_id, endpoint_pools, expiry_state_handle, first_cluster_node_is_local,
@@ -629,14 +587,12 @@ pub mod rpc {
         decode_heal_bucket_rpc_options, encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers,
         gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
         node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
-        sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
-        sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
-        tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
-        verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
-        verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer, verify_put_file_capability,
-        verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
-        verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
-        verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
+        sign_ns_scanner_capability, sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability,
+        sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
+        tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation,
+        verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
+        verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned,
+        verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
     };
 }
 
@@ -675,7 +631,6 @@ pub mod store_list {
 }
 
 pub mod storage {
-    pub use crate::bucket::artifact::inventory::ArtifactAttemptInventoryFrame;
     pub use crate::core::pools::{HealLifecycleExpiryContext, POOL_META_NAME};
     pub use crate::store::HealWalkVersion;
     pub use crate::store::{

crates/ecstore/src/bucket/artifact/backup_retention.rs

@@ -1,120 +0,0 @@
-// Copyright 2026 RustFS Team
-// SPDX-License-Identifier: Apache-2.0
-
-use super::*;
-use crate::bucket::metadata::BUCKET_INCARNATION_FILE;
-use crate::config::com::read_config_observation;
-use crate::object_api::WriteCompletion;
-
-impl ECStore {
-    /// Permanently retain this managed bucket's versions for historical restores.
-    /// Lifecycle WRITE drains earlier exact deletes. The detached owner keeps
-    /// lifecycle -> metadata transaction locks until every metadata write drains.
-    pub async fn retain_artifact_backups(self: Arc<Self>, bucket: &str, incarnation: Uuid, base_prefix: &str) -> Result<()> {
-        let bucket = bucket.to_owned();
-        let base_prefix = base_prefix.to_owned();
-        let ctx = Arc::clone(&self.ctx);
-        ctx.run_detached_mutation(async move {
-            require_put_seal_topology(&self).await?;
-            if self.ctx.lock_manager().is_disabled() {
-                return Err(Error::other("artifact backup retention requires namespace locking"));
-            }
-            let lifecycle = self.acquire_bucket_lifecycle_write_lock(&bucket).await?;
-            let transaction = metadata_sys::acquire_bucket_metadata_transaction_lock_in(&self.ctx, &bucket).await?;
-            let mut metadata = self.backup_retention_metadata(&bucket, incarnation, &base_prefix).await?;
-            let check = || {
-                if lifecycle.is_lock_lost() || transaction.is_lock_lost() {
-                    Err(Error::other("artifact backup retention fence was lost"))
-                } else {
-                    Ok(())
-                }
-            };
-            check()?;
-            // Even an idempotent retry rewrites with TailDrained: an earlier
-            // interrupted call may have reached only some metadata replicas.
-            metadata.artifact_backup_retained = true;
-            let mut opts = ObjectOptions {
-                max_parity: true,
-                write_completion: WriteCompletion::TailDrained,
-                ..Default::default()
-            };
-            opts.add_bucket_lifecycle_lock_guard(&lifecycle);
-            opts.add_namespace_lock_guard(&transaction);
-            metadata.save_with_store_options(self.clone(), &opts).await?;
-            check()?;
-            let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
-            let sys = sys.read().await.clone();
-            sys.set_validated(metadata).await?;
-            check()
-        })
-        .await
-        .map_err(|_| Error::other("artifact backup retention owner failed"))?
-    }
-
-    /// Observe the immutable bucket identity and retention in one locked read.
-    pub async fn inspect_artifact_backup_retention(&self, bucket: &str, incarnation: Uuid, base_prefix: &str) -> Result<bool> {
-        require_put_seal_topology(self).await?;
-        if self.ctx.lock_manager().is_disabled() {
-            return Err(Error::other("artifact backup retention requires namespace locking"));
-        }
-        let lifecycle = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
-        let metadata = self.backup_retention_metadata(bucket, incarnation, base_prefix).await?;
-        if lifecycle.is_lock_lost() {
-            return Err(Error::other("artifact backup retention observation fence was lost"));
-        }
-        Ok(metadata.artifact_backup_retained)
-    }
-
-    async fn backup_retention_metadata(&self, bucket: &str, incarnation: Uuid, base_prefix: &str) -> Result<BucketMetadata> {
-        if incarnation.is_nil() || !crate::storage_api_contracts::bucket::valid_artifact_base_prefix(base_prefix) {
-            return Err(Error::other("artifact backup retention identity is invalid"));
-        }
-        let mut metadata = self.read_backup_retention_metadata(bucket).await?;
-        let store = metadata_sys::object_store_in(&self.ctx).await?;
-        let (sidecar, _) =
-            read_config_observation(store, &format!("{BUCKET_META_PREFIX}/{bucket}/{BUCKET_INCARNATION_FILE}"), 16).await?;
-        metadata.parse_all_configs()?;
-        if metadata.bucket_incarnation_id != incarnation
-            || Uuid::from_slice(&sidecar).ok() != Some(incarnation)
-            || metadata.artifact_base_prefix.as_deref() != Some(base_prefix)
-            || !metadata.bucket_creation_committed
-            || !metadata.versioning_config.as_ref().is_some_and(|versioning| {
-                versioning.enabled()
-                    && !versioning.exclude_folders.unwrap_or(false)
-                    && versioning.excluded_prefixes.as_ref().is_none_or(Vec::is_empty)
-            })
-        {
-            return Err(Error::other("artifact backup retention identity mismatch"));
-        }
-        metadata.bucket_incarnation_sidecar = true;
-        Ok(metadata)
-    }
-
-    /// Caller must hold the bucket lifecycle fence. An unreadable or missing
-    /// record cannot prove that historical backup protection was never enabled.
-    pub(crate) async fn require_no_artifact_backup_retention(&self, bucket: &str) -> Result<()> {
-        if self.read_backup_retention_metadata(bucket).await?.artifact_backup_retained {
-            return Err(Error::other("artifact versions are retained for backups"));
-        }
-        Ok(())
-    }
-
-    async fn read_backup_retention_metadata(&self, bucket: &str) -> Result<BucketMetadata> {
-        if bucket.is_empty()
-            || super::super::utils::has_bad_path_component(bucket)
-            || super::super::utils::is_meta_bucketname(bucket)
-        {
-            return Err(Error::other("artifact backup retention bucket is invalid"));
-        }
-        let store = metadata_sys::object_store_in(&self.ctx).await?;
-        if store.id != self.id {
-            return Err(Error::other("artifact backup retention belongs to another store"));
-        }
-        let (bytes, _) = read_config_observation(store, &BucketMetadata::new(bucket).save_file_path(), 64 * 1024 * 1024).await?;
-        let metadata = BucketMetadata::decode_file(&bytes)?;
-        if metadata.name != bucket {
-            return Err(Error::other("artifact backup retention bucket mismatch"));
-        }
-        Ok(metadata)
-    }
-}

crates/ecstore/src/bucket/artifact/delete.rs

@@ -1,154 +0,0 @@
-// Copyright 2026 RustFS Team
-// SPDX-License-Identifier: Apache-2.0
-
-use super::*;
-use crate::storage_api_contracts::object::ObjectOperations as _;
-
-/// In-process authority for exactly one drained managed version. No caller can
-/// construct this capability or serialize it into a disk/RPC request.
-#[doc(hidden)]
-pub struct ArtifactDeleteTarget {
-    store: Uuid,
-    record: ArtifactVersionRetirementRecord,
-    lifecycle: Arc<rustfs_lock::NamespaceLockGuard>,
-    attempt: WriteCommitGuard,
-    repair: WriteCommitGuard,
-}
-
-impl ArtifactDeleteTarget {
-    fn check_live(&self) -> Result<()> {
-        if self.lifecycle.is_lock_lost() || self.attempt.is_lock_lost() || self.repair.is_lock_lost() {
-            return Err(Error::other("artifact exact delete owned fence was lost"));
-        }
-        Ok(())
-    }
-
-    pub(crate) fn lifecycle(&self) -> Arc<rustfs_lock::NamespaceLockGuard> {
-        Arc::clone(&self.lifecycle)
-    }
-
-    pub(crate) fn validate(&self, store: &ECStore, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
-        if self.store != store.id
-            || bucket != self.record.bucket
-            || rustfs_utils::path::encode_dir_object(object) != self.record.object
-            || opts.expected_bucket_incarnation_id != Some(self.record.incarnation_id)
-            || opts.version_id.as_deref() != Some(self.record.version_id.to_string().as_str())
-            || !opts.versioned
-            || opts.version_suspended
-            || opts.delete_prefix
-            || opts.delete_marker
-            || opts.data_movement
-            || opts.lifecycle_delete_all.is_some()
-            || self.lifecycle.is_lock_lost()
-            || self.attempt.is_lock_lost()
-            || self.repair.is_lock_lost()
-        {
-            return Err(Error::other("artifact exact delete target or owned fence mismatch"));
-        }
-        Ok(())
-    }
-}
-
-impl ECStore {
-    /// Read-only capability observation; each mutation revalidates the topology.
-    pub async fn supports_artifact_exact_delete(&self) -> bool {
-        require_local_put_seal_topology(self).await.is_ok() && !self.ctx.lock_manager().is_disabled()
-    }
-
-    /// Delete an exact version in a fully sealed attempt. This authenticates the
-    /// storage identity and drains writers; the caller still owns GC eligibility
-    /// and cleanup IAM. Distributed receiver deletion is not yet supported.
-    pub async fn delete_artifact_object_version(
-        self: Arc<Self>,
-        bucket: &str,
-        expected_incarnation: Uuid,
-        attempt_prefix: &str,
-        object: &str,
-        version: Uuid,
-    ) -> Result<()> {
-        require_local_put_seal_topology(&self).await?;
-        if self.ctx.lock_manager().is_disabled() {
-            return Err(Error::other("artifact exact deletion requires namespace locking"));
-        }
-        let lifecycle = Arc::new(self.acquire_bucket_lifecycle_read_lock(bucket).await?);
-        self.require_no_artifact_backup_retention(bucket).await?;
-        let put = self
-            .inventory_artifact_record(bucket, expected_incarnation, attempt_prefix)
-            .await?;
-        let repair = ArtifactRepairSealRecord::new(&put);
-        let record = ArtifactVersionRetirementRecord::new(&put, object, version)?;
-        let object = object.to_owned();
-        let store = Arc::clone(&self);
-        // Lifecycle -> attempt -> repair seal -> metadata transaction -> object.
-        // The spawned owner, and the disk deletion tail's typed capability, keep
-        // these locks after cancellation of the HTTP caller.
-        with_config_object_write_lock(self, put.path(), move |attempt| async move {
-            if !put.exists(Arc::clone(&store), &attempt).await? {
-                return Err(Error::other("artifact exact deletion requires sealed PUTs"));
-            }
-            let lock = store.new_ns_lock(RUSTFS_META_BUCKET, &repair.path()).await?;
-            let repair_owner = WriteCommitGuard::acquire(&lock, get_lock_acquire_timeout()).await?;
-            if !repair.exists(Arc::clone(&store), &repair_owner).await? {
-                return Err(Error::other("artifact exact deletion requires sealed repairs"));
-            }
-            let receivers = store.capture_artifact_receivers().await?;
-            store
-                .close_artifact_receiver_snapshot(&put, crate::disk::local::ArtifactLeaseDomain::Put, &receivers)
-                .await?;
-            store
-                .close_artifact_receiver_snapshot(&put, crate::disk::local::ArtifactLeaseDomain::Repair, &receivers)
-                .await?;
-            store.verify_artifact_receiver_snapshot(&put, &receivers).await?;
-            let target = Arc::new(ArtifactDeleteTarget {
-                store: store.id,
-                record,
-                lifecycle,
-                attempt,
-                repair: repair_owner,
-            });
-            let mut opts = ObjectOptions {
-                versioned: true,
-                version_id: Some(version.to_string()),
-                expected_bucket_incarnation_id: Some(expected_incarnation),
-                artifact_delete_target: Some(Arc::clone(&target)),
-                delete_mutation_owner: Some(target.clone()),
-                ..Default::default()
-            };
-            opts.add_write_commit_guard(&target.attempt);
-            opts.add_write_commit_guard(&target.repair);
-            // These owners fence seal records, not the object. Ordinary DELETE
-            // must still acquire and retain its exact object WRITE owner.
-            opts.no_lock = false;
-            target.check_live()?;
-            // Preserve the normal exact DELETE's Object Lock, quorum, journal,
-            // cache and capacity behavior. Its success is logical deletion, not
-            // a claim that every offline shard has been physically erased.
-            match store.delete_object(&put.bucket, &object, opts).await {
-                Ok(info) => {
-                    // Match the S3 request layer's committed-generation quota
-                    // delta. A strict absence retry must not subtract twice.
-                    if !info.name.is_empty()
-                        && !info.delete_marker
-                        && let Ok(size) = crate::data_usage::quota_object_size(&info)
-                    {
-                        crate::data_usage::record_bucket_object_delete_memory(&put.bucket, size, false).await;
-                    }
-                }
-                Err(Error::VersionNotFound(bucket, object, version))
-                    if bucket == target.record.bucket
-                        && rustfs_utils::path::encode_dir_object(&object) == target.record.object
-                        && version == target.record.version_id.to_string() => {}
-                Err(Error::ObjectNotFound(bucket, object))
-                    if bucket == target.record.bucket
-                        && rustfs_utils::path::encode_dir_object(&object) == target.record.object => {}
-                Err(error) => return Err(error),
-            }
-            store.verify_artifact_receiver_snapshot(&put, &receivers).await?;
-            store
-                .inventory_artifact_record(&put.bucket, expected_incarnation, &put.attempt_prefix)
-                .await?;
-            target.check_live()
-        })
-        .await?
-    }
-}

crates/ecstore/src/bucket/artifact/inventory.rs

@@ -1,207 +0,0 @@
-// Copyright 2026 RustFS Team
-// SPDX-License-Identifier: Apache-2.0
-
-use super::*;
-use crate::disk::{ArtifactInventoryFrame, ArtifactInventoryVersion, Disk};
-use rustfs_protos::proto_gen::node_service::ArtifactInventoryPhase;
-use tokio::{sync::mpsc, time::Instant};
-
-/// Physical observations retain their member. Replicas are not extra business
-/// versions and this stream makes no quorum-readability or GC claim.
-#[derive(Debug)]
-pub enum ArtifactAttemptInventoryFrame {
-    Observations {
-        endpoint: String,
-        disk_id: Uuid,
-        versions: Vec<ArtifactInventoryVersion>,
-    },
-    Complete {
-        members: usize,
-        observations: u64,
-    },
-}
-
-impl ECStore {
-    /// Inventory one fully sealed attempt across every configured receiver.
-    /// Earlier output is staging only; only the final frame plus Ok is complete.
-    pub async fn inventory_artifact_attempt(
-        self: Arc<Self>,
-        bucket: &str,
-        incarnation: Uuid,
-        prefix: &str,
-        deadline: Instant,
-        output: mpsc::Sender<ArtifactAttemptInventoryFrame>,
-    ) -> Result<()> {
-        tokio::time::timeout_at(deadline, Box::pin(async move {
-            require_put_seal_topology(&self).await?;
-            if self.ctx.lock_manager().is_disabled() { return Err(Error::other("artifact inventory requires namespace locking")); }
-            let lifecycle = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
-            let put = self.inventory_artifact_record(bucket, incarnation, prefix).await?;
-            let repair = ArtifactRepairSealRecord::new(&put);
-            // Lifecycle -> attempt -> repair record. Receivers do not reenter
-            // distributed locks; actual two-domain disk owners guard their reads.
-            let store = self.clone();
-            let bucket = bucket.to_owned();
-            let prefix = prefix.to_owned();
-            with_config_object_write_lock(self, put.path(), move |attempt| async move {
-                tokio::time::timeout_at(deadline, async {
-                    if !put.exists(store.clone(), &attempt).await? { return Err(Error::other("artifact inventory requires PUT seal record")); }
-                    let lock = store.new_ns_lock(RUSTFS_META_BUCKET, &repair.path()).await?;
-                    let repair_owner = WriteCommitGuard::acquire(&lock, get_lock_acquire_timeout()).await?;
-                    if !repair.exists(store.clone(), &repair_owner).await? { return Err(Error::other("artifact inventory requires repair seal record")); }
-                    let receivers = store.capture_artifact_receivers().await?;
-                    let mut observations = 0u64;
-                    for (endpoint, (disk, expected)) in &receivers {
-                        if lifecycle.is_lock_lost() || attempt.is_lock_lost() || repair_owner.is_lock_lost() {
-                            return Err(Error::other("artifact inventory fence was lost"));
-                        }
-                        let (sender, mut frames) = mpsc::channel(2);
-                        let scan = async {
-                            match disk.as_ref() {
-                                Disk::Local(local) => local.get_disk().inventory_artifact_attempt(*expected, &bucket, &prefix, put.identity(), deadline, sender).await,
-                                Disk::Remote(remote) => remote.inventory_artifact_attempt(*expected, (&bucket, incarnation), &prefix, ArtifactInventoryPhase::ArtifactInventoryScan, deadline, sender).await,
-                            }
-                        };
-                        tokio::pin!(scan);
-                        let mut sequence = 0;
-                        let mut count = 0u64;
-                        let mut terminal = false;
-                        let mut ended = false;
-                        let mut done = false;
-                        while !ended || !done {
-                            tokio::select! {
-                                result = &mut scan, if !done => { result.map_err(Error::other)?; done = true; },
-                                frame = frames.recv(), if !ended => match frame {
-                                    Some(ArtifactInventoryFrame::Entries { sequence: actual, versions }) if !terminal && actual == sequence => {
-                                        count = count.checked_add(u64::try_from(versions.len()).map_err(Error::other)?).ok_or_else(|| Error::other("artifact inventory count overflow"))?;
-                                        output.send(ArtifactAttemptInventoryFrame::Observations { endpoint: endpoint.clone(), disk_id: *expected, versions }).await
-                                            .map_err(|_| Error::other("artifact inventory consumer closed"))?;
-                                        sequence += 1;
-                                    }
-                                    Some(ArtifactInventoryFrame::Complete { disk_id, incarnation: actual, attempt_prefix, entries })
-                                        if !terminal && disk_id == *expected && actual == incarnation && attempt_prefix == prefix && entries == count => terminal = true,
-                                    Some(_) => return Err(Error::other("artifact inventory member stream is invalid")),
-                                    None => ended = true,
-                                },
-                            }
-                        }
-                        if !terminal { return Err(Error::other("artifact inventory member is incomplete")); }
-                        observations = observations.checked_add(count).ok_or_else(|| Error::other("artifact inventory count overflow"))?;
-                    }
-                    store.verify_inventory_members(&put, &receivers, deadline).await?;
-                    let current = store.artifact_inventory_receiver_identity(&bucket, incarnation, &prefix).await?;
-                    if current.receiver_key != put.identity().receiver_key || lifecycle.is_lock_lost() || attempt.is_lock_lost() || repair_owner.is_lock_lost() {
-                        return Err(Error::other("artifact inventory authority changed"));
-                    }
-                    output.send(ArtifactAttemptInventoryFrame::Complete { members: receivers.len(), observations }).await
-                        .map_err(|_| Error::other("artifact inventory consumer closed"))?;
-                    Ok(())
-                }).await.map_err(|_| Error::other("artifact inventory deadline exceeded"))?
-            }).await?
-        })).await.map_err(|_| Error::other("artifact inventory deadline exceeded"))?
-    }
-
-    pub(super) async fn inventory_artifact_record(&self, bucket: &str, expected: Uuid, prefix: &str) -> Result<PutSealRecord> {
-        use crate::bucket::metadata::BUCKET_INCARNATION_FILE;
-        use crate::config::com::read_config_observation;
-        if expected.is_nil()
-            || crate::bucket::utils::has_bad_path_component(bucket)
-            || bucket.is_empty()
-            || crate::bucket::utils::is_meta_bucketname(bucket)
-        {
-            return Err(Error::other("artifact inventory authority is invalid"));
-        }
-        let store = metadata_sys::object_store_in(&self.ctx).await?;
-        if store.id != self.id {
-            return Err(Error::other("artifact inventory authority belongs to another store"));
-        }
-        // Do not enter metadata_sys's lazy bucket READ/migration path: the
-        // coordinator already owns lifecycle READ and a writer may be queued.
-        // Observation reads bypass cache and suppress physical read repair.
-        let path = BucketMetadata::new(bucket).save_file_path();
-        let (bytes, _) = read_config_observation(store.clone(), &path, 64 * 1024 * 1024).await?;
-        let mut metadata = BucketMetadata::decode_file(&bytes)?;
-        let (sidecar, _) =
-            read_config_observation(store, &format!("{BUCKET_META_PREFIX}/{bucket}/{BUCKET_INCARNATION_FILE}"), 16).await?;
-        if metadata.name != bucket
-            || !metadata.bucket_creation_committed
-            || metadata.bucket_incarnation_id != expected
-            || Uuid::from_slice(&sidecar).ok() != Some(expected)
-        {
-            return Err(Error::other("artifact inventory incarnation mismatch"));
-        }
-        metadata.parse_all_configs()?;
-        if !metadata.versioning_config.as_ref().is_some_and(|config| {
-            config.enabled()
-                && !config.exclude_folders.unwrap_or(false)
-                && config.excluded_prefixes.as_ref().is_none_or(Vec::is_empty)
-        }) {
-            return Err(Error::other("artifact inventory requires managed versioning"));
-        }
-        PutSealRecord::new(&metadata, prefix)
-    }
-
-    pub(crate) async fn artifact_inventory_receiver_identity(
-        &self,
-        bucket: &str,
-        expected: Uuid,
-        prefix: &str,
-    ) -> Result<ArtifactPutIdentity> {
-        Ok(self.inventory_artifact_record(bucket, expected, prefix).await?.identity())
-    }
-
-    async fn verify_inventory_members(
-        &self,
-        record: &PutSealRecord,
-        captured: &ArtifactReceiverSnapshot,
-        deadline: Instant,
-    ) -> Result<()> {
-        for final_pass in [false, true] {
-            let current = self.artifact_receiver_inventory().await?;
-            if current.len() != captured.len()
-                || current
-                    .iter()
-                    .any(|(endpoint, disk)| !captured.get(endpoint).is_some_and(|(old, _)| Arc::ptr_eq(old, disk)))
-            {
-                return Err(Error::other("artifact inventory membership changed"));
-            }
-            if final_pass {
-                for (disk, expected) in captured.values() {
-                    if disk.cached_disk_id().await != Some(*expected) {
-                        return Err(Error::other("artifact inventory expected identity changed"));
-                    }
-                }
-                break;
-            }
-            for (disk, expected) in captured.values() {
-                if disk.cached_disk_id().await != Some(*expected) {
-                    return Err(Error::other("artifact inventory expected identity changed"));
-                }
-                match disk.as_ref() {
-                    Disk::Local(local) => {
-                        let _verification = local
-                            .get_disk()
-                            .verify_artifact_inventory_closed(*expected, record.identity(), deadline)
-                            .await
-                            .map_err(Error::other)?;
-                    }
-                    Disk::Remote(remote) => {
-                        let (output, _unused) = mpsc::channel(1);
-                        remote
-                            .inventory_artifact_attempt(
-                                *expected,
-                                (&record.bucket, record.incarnation_id),
-                                &record.attempt_prefix,
-                                ArtifactInventoryPhase::ArtifactInventoryVerify,
-                                deadline,
-                                output,
-                            )
-                            .await
-                            .map_err(Error::other)?;
-                    }
-                }
-            }
-        }
-        Ok(())
-    }
-}

crates/ecstore/src/bucket/artifact/ordinary_classification.rs

@@ -1,287 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::HashMap;
-use std::future::Future;
-use std::sync::atomic::{AtomicU64, Ordering};
-use std::sync::{Arc, Mutex, Weak};
-
-/// Shares only reads started after a caller arrives; retains no idle results.
-#[derive(Default)]
-pub(crate) struct OrdinaryClassificationRegistry {
-    gates: Mutex<HashMap<String, Weak<Gate>>>,
-    #[cfg(test)]
-    after_unregister: Option<std::sync::Barrier>,
-}
-
-#[derive(Default)]
-struct Gate {
-    started_generation: AtomicU64,
-    completed: tokio::sync::Mutex<Option<CompletedRead>>,
-}
-
-struct CompletedRead {
-    generation: u64,
-    allowed: bool,
-}
-
-struct Registration<'a> {
-    registry: &'a OrdinaryClassificationRegistry,
-    bucket: &'a str,
-    gate: Option<Arc<Gate>>,
-    observed_generation: u64,
-}
-
-impl OrdinaryClassificationRegistry {
-    pub(crate) async fn classify(&self, bucket: &str, read: impl Future<Output = bool>) -> bool {
-        self.register(bucket).classify(read).await
-    }
-
-    fn register<'a>(&'a self, bucket: &'a str) -> Registration<'a> {
-        // Callers requiring physical admission must register after acquiring it.
-        // Release the registry lock before awaiting the gate, then acquire the
-        // existing metadata locks.
-        let mut gates = self.gates.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
-        let gate = gates.get(bucket).and_then(Weak::upgrade).unwrap_or_else(|| {
-            let gate = Arc::new(Gate::default());
-            gates.insert(bucket.to_owned(), Arc::downgrade(&gate));
-            gate
-        });
-        let observed_generation = gate.started_generation.load(Ordering::Acquire);
-        Registration {
-            registry: self,
-            bucket,
-            gate: Some(gate),
-            observed_generation,
-        }
-    }
-}
-
-impl Registration<'_> {
-    async fn classify(self, read: impl Future<Output = bool>) -> bool {
-        let Some(gate) = self.gate.as_ref() else {
-            return false;
-        };
-        let mut completed = gate.completed.lock().await;
-        // Only a read started after this caller registered can authorize it.
-        // Arrivals during a read therefore wait for the next fresh read.
-        if let Some(result) = completed
-            .as_ref()
-            .filter(|result| result.generation > self.observed_generation)
-        {
-            return result.allowed;
-        }
-        let Ok(previous) = gate
-            .started_generation
-            .try_update(Ordering::AcqRel, Ordering::Acquire, |generation| generation.checked_add(1))
-        else {
-            return false;
-        };
-        // Keep the gate through the original bounded metadata read. Cancellation
-        // releases it without publishing a result; another caller can read again.
-        let allowed = read.await;
-        *completed = Some(CompletedRead {
-            generation: previous + 1,
-            allowed,
-        });
-        allowed
-    }
-}
-
-impl Drop for Registration<'_> {
-    fn drop(&mut self) {
-        let Some(gate) = self.gate.take() else {
-            return;
-        };
-        let mut gates = self.registry.gates.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
-        // Strong references are acquired and released under this registry lock,
-        // so concurrent final callers cannot both miss the last reference.
-        if Arc::strong_count(&gate) == 1
-            && gates
-                .get(self.bucket)
-                .is_some_and(|registered| registered.ptr_eq(&Arc::downgrade(&gate)))
-        {
-            gates.remove(self.bucket);
-        }
-        drop(gate);
-        drop(gates);
-        #[cfg(test)]
-        if let Some(barrier) = &self.registry.after_unregister {
-            barrier.wait();
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use futures::poll;
-    use std::task::Poll;
-    use tokio::sync::oneshot;
-
-    fn assert_empty(registry: &OrdinaryClassificationRegistry) {
-        assert!(registry.gates.lock().expect("classification registry").is_empty());
-    }
-
-    #[tokio::test]
-    async fn late_arrivals_share_only_the_next_fail_closed_read() {
-        let registry = OrdinaryClassificationRegistry::default();
-        let reads = AtomicU64::new(0);
-        let (release, waiting) = oneshot::channel();
-        let mut first = Box::pin(registry.classify("bucket", async {
-            reads.fetch_add(1, Ordering::Relaxed);
-            waiting.await.expect("release first read");
-            true
-        }));
-        assert!(poll!(first.as_mut()).is_pending());
-        let mut followers = (0..16)
-            .map(|_| {
-                Box::pin(registry.classify("bucket", async {
-                    reads.fetch_add(1, Ordering::Relaxed);
-                    false
-                }))
-            })
-            .collect::<Vec<_>>();
-        for follower in &mut followers {
-            assert!(poll!(follower.as_mut()).is_pending());
-        }
-        release.send(()).expect("first reader still alive");
-        assert_eq!(poll!(first.as_mut()), Poll::Ready(true));
-        for follower in &mut followers {
-            assert_eq!(poll!(follower.as_mut()), Poll::Ready(false));
-        }
-        assert_eq!(reads.load(Ordering::Relaxed), 2);
-        assert_empty(&registry);
-    }
-
-    #[tokio::test]
-    async fn successive_waves_always_read_again() {
-        let registry = OrdinaryClassificationRegistry::default();
-        let reads = AtomicU64::new(0);
-        for expected in [true, false, true] {
-            assert_eq!(
-                registry
-                    .classify("bucket", async {
-                        reads.fetch_add(1, Ordering::Relaxed);
-                        expected
-                    })
-                    .await,
-                expected
-            );
-            assert_empty(&registry);
-        }
-        assert_eq!(reads.load(Ordering::Relaxed), 3);
-    }
-
-    #[tokio::test]
-    async fn canceled_leader_does_not_publish_or_strand_waiters() {
-        let registry = OrdinaryClassificationRegistry::default();
-        let reads = AtomicU64::new(0);
-        let old_registration = registry.register("bucket");
-        assert!(
-            registry
-                .classify("bucket", async {
-                    reads.fetch_add(1, Ordering::Relaxed);
-                    true
-                })
-                .await
-        );
-        let mut leader = Box::pin(registry.classify("bucket", async {
-            reads.fetch_add(1, Ordering::Relaxed);
-            std::future::pending::<bool>().await
-        }));
-        assert!(poll!(leader.as_mut()).is_pending());
-        let mut old_waiter =
-            Box::pin(old_registration.classify(async { panic!("old waiter must reuse the completed post-registration read") }));
-        assert!(poll!(old_waiter.as_mut()).is_pending());
-        let mut late_waiter = Box::pin(registry.classify("bucket", async {
-            reads.fetch_add(1, Ordering::Relaxed);
-            false
-        }));
-        assert!(poll!(late_waiter.as_mut()).is_pending());
-        drop(leader);
-        assert_eq!(poll!(old_waiter.as_mut()), Poll::Ready(true));
-        assert_eq!(poll!(late_waiter.as_mut()), Poll::Ready(false));
-        assert_eq!(reads.load(Ordering::Relaxed), 3);
-        assert_empty(&registry);
-    }
-
-    #[tokio::test]
-    async fn canceled_follower_does_not_retain_the_gate() {
-        let registry = OrdinaryClassificationRegistry::default();
-        let (release, waiting) = oneshot::channel();
-        let mut leader = Box::pin(registry.classify("bucket", async {
-            waiting.await.expect("release leader");
-            true
-        }));
-        assert!(poll!(leader.as_mut()).is_pending());
-        let mut follower = Box::pin(registry.classify("bucket", async { panic!("canceled follower must not read") }));
-        assert!(poll!(follower.as_mut()).is_pending());
-        drop(follower);
-        assert_eq!(registry.gates.lock().expect("classification registry").len(), 1);
-        release.send(()).expect("leader still alive");
-        assert_eq!(poll!(leader.as_mut()), Poll::Ready(true));
-        assert_empty(&registry);
-    }
-
-    #[test]
-    fn concurrent_final_registrations_remove_the_gate() {
-        let registry = OrdinaryClassificationRegistry {
-            after_unregister: Some(std::sync::Barrier::new(2)),
-            ..Default::default()
-        };
-        let first = registry.register("bucket");
-        let second = registry.register("bucket");
-        // Both destructors reach the unlocked barrier before either returns.
-        // Moving the Arc release after this barrier leaves a stale registry key.
-        std::thread::scope(|scope| {
-            scope.spawn(move || drop(first));
-            scope.spawn(move || drop(second));
-        });
-        assert_empty(&registry);
-    }
-
-    #[tokio::test]
-    async fn buckets_and_stores_do_not_share_gates() {
-        let first_store = OrdinaryClassificationRegistry::default();
-        let second_store = OrdinaryClassificationRegistry::default();
-        let mut blocked = Box::pin(first_store.classify("first", std::future::pending()));
-        assert!(poll!(blocked.as_mut()).is_pending());
-        assert_eq!(poll!(Box::pin(first_store.classify("second", async { true }))), Poll::Ready(true));
-        assert_eq!(poll!(Box::pin(second_store.classify("first", async { false }))), Poll::Ready(false));
-        assert_empty(&second_store);
-        drop(blocked);
-        assert_empty(&first_store);
-    }
-
-    #[tokio::test]
-    async fn generation_exhaustion_fails_closed_without_reading() {
-        let registry = OrdinaryClassificationRegistry::default();
-        let registration = registry.register("bucket");
-        let gate = registration.gate.as_ref().expect("registered gate");
-        gate.started_generation.store(u64::MAX, Ordering::Release);
-        *gate.completed.lock().await = Some(CompletedRead {
-            generation: u64::MAX,
-            allowed: true,
-        });
-        assert!(
-            !registry
-                .classify("bucket", async { panic!("exhausted generation must deny without reading") })
-                .await
-        );
-        assert_eq!(gate.started_generation.load(Ordering::Acquire), u64::MAX);
-        drop(registration);
-        assert_empty(&registry);
-    }
-}

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -890,7 +890,6 @@ pub(crate) async fn cleanup_free_version_exact(
     let bucket_guard = api
         .acquire_bucket_lifecycle_read_lock(&oi.bucket)
         .await
-        .map(Arc::new)
         .map_err(std::io::Error::other)?;
     let (lease, delete_plan) = acquire_free_version_tier_lease(oi, &api.tier_config_mgr()).await?;
     let local_object = encode_dir_object(&oi.name);
@@ -959,7 +958,7 @@ pub(crate) async fn cleanup_free_version_exact(
         };
         delete_request.set_tier_free_version();
         if let Err(err) = set
-            .delete_object_version_with_lifecycle_owner(&oi.bucket, &local_object, &delete_request, Arc::clone(&bucket_guard))
+            .delete_object_version(&oi.bucket, &local_object, &delete_request, false)
             .await
             && first_error.is_none()
         {
@@ -8497,104 +8496,6 @@ mod tests {
         }
     }
 
-    #[cfg(feature = "test-util")]
-    #[tokio::test]
-    #[serial]
-    async fn artifact_delete_entrypoints_free_version_cleaner_completes_with_queued_lifecycle_writer() {
-        let (disk_paths, ecstore) = setup_test_env().await;
-        let bucket = format!("free-version-queued-writer-{}", Uuid::new_v4());
-        let object = "free-version";
-        create_test_bucket(&ecstore, &bucket).await;
-        let (backend, identity) = register_recovery_mock_tier(&ecstore).await;
-        seed_recoverable_free_version(&disk_paths, &bucket, object, None, Some(identity)).await;
-        let oi = list_tier_free_versions(Arc::clone(&ecstore), 100, None, None, CancellationToken::new())
-            .await
-            .expect("list real persisted free version")
-            .items
-            .into_iter()
-            .find(|oi| oi.bucket == bucket)
-            .expect("free-version task must address this fixture");
-        assert!(oi.transitioned_object.free_version);
-        backend
-            .set_put_remote_version(Some(oi.transitioned_object.version_id.clone()))
-            .await;
-        let remote = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
-            .await
-            .expect("remote fixture lease");
-        remote
-            .put(&oi.transitioned_object.name, ReaderImpl::Body(Bytes::from_static(b"body")), 4)
-            .await
-            .expect("seed exact remote object");
-        drop(remote);
-
-        let barrier = Arc::new(super::FreeVersionPostRemoteDeleteTestBarrier::default());
-        let cleanup_barrier = Arc::clone(&barrier);
-        let cleanup_store = Arc::clone(&ecstore);
-        let cleanup_oi = oi.clone();
-        let mut cleanup = tokio::spawn(async move {
-            super::FREE_VERSION_POST_REMOTE_DELETE_TEST_BARRIER
-                .scope(cleanup_barrier, async move {
-                    super::cleanup_free_version_exact(cleanup_store, &cleanup_oi, &CancellationToken::new()).await
-                })
-                .await
-        });
-        tokio::time::timeout(StdDuration::from_secs(10), barrier.arrived.notified())
-            .await
-            .expect("cleaner owns lifecycle READ and has completed remote DELETE");
-        assert!(!backend.contains(&oi.transitioned_object.name).await);
-        for disk in &disk_paths {
-            assert!(
-                disk.join(&bucket).join(object).join(STORAGE_FORMAT_FILE).is_file(),
-                "local exact cleanup must still be pending at the boundary"
-            );
-        }
-
-        // Resolve first so the pending poll is the actual local lock wait.
-        // A later READ would normally be admitted alongside the cleaner's READ;
-        // its pending state independently proves this WRITE is now queued.
-        let lifecycle = ecstore
-            .new_ns_lock(&bucket, crate::storage_api_contracts::bucket::BUCKET_LIFECYCLE_LOCK_OBJECT)
-            .await
-            .expect("lifecycle namespace");
-        let mut writer = Box::pin(lifecycle.get_write_lock(StdDuration::from_secs(30)));
-        let writer_queued = futures::poll!(&mut writer).is_pending();
-        let mut later_reader = Box::pin(lifecycle.get_read_lock(StdDuration::from_secs(30)));
-        let later_reader_blocked = futures::poll!(&mut later_reader).is_pending();
-        drop(later_reader);
-        barrier.release.notify_one();
-        assert!(writer_queued, "cleaner must retain the real lifecycle READ owner");
-        assert!(later_reader_blocked, "queued lifecycle WRITE must block a later READ");
-
-        // Keep the same WRITE future queued, without cancelling/retrying it.
-        // Re-entering READ here would deadlock behind this waiting writer.
-        let finished = tokio::time::timeout(StdDuration::from_secs(10), &mut cleanup).await;
-        if finished.is_err() {
-            cleanup.abort();
-            drop(writer);
-            let _ = tokio::time::timeout(StdDuration::from_secs(5), &mut cleanup).await;
-            panic!("exact local free-version cleanup re-entered lifecycle READ behind the queued writer");
-        }
-        assert!(
-            finished
-                .expect("bounded cleaner completion")
-                .expect("cleaner task")
-                .expect("exact local cleanup")
-        );
-        let writer = tokio::time::timeout(StdDuration::from_secs(10), writer)
-            .await
-            .expect("queued lifecycle WRITE proceeds after real cleanup drains")
-            .expect("lifecycle WRITE acquired");
-        for disk in &disk_paths {
-            assert!(!disk.join(&bucket).join(object).exists(), "free version remains on a real local disk");
-        }
-        assert_eq!(backend.remove_count().await, 1, "cleaner must delete the exact remote tuple once");
-        drop(writer);
-        ecstore
-            .delete_bucket(&bucket, &DeleteBucketOptions::default())
-            .await
-            .expect("remove empty fixture bucket");
-    }
-
     #[cfg(feature = "test-util")]
     #[tokio::test]
     #[serial]
@@ -14375,14 +14276,9 @@ mod tests {
 
         crate::store::init_local_disks(endpoint_pools.clone()).await.unwrap();
 
-        // Tests start the recovery loop they observe. An automatic startup loop
-        // would race that loop for the same unconsumed worker queue.
-        let ecstore = temp_env::async_with_vars(
-            [(super::ENV_TIER_FREE_VERSION_RECOVERY_ENABLED, Some("false"))],
-            ECStore::new("127.0.0.1:0".parse().unwrap(), endpoint_pools, CancellationToken::new()),
-        )
-        .await
-        .unwrap();
+        let ecstore = ECStore::new("127.0.0.1:0".parse().unwrap(), endpoint_pools, CancellationToken::new())
+            .await
+            .unwrap();
 
         let buckets = ecstore
             .list_bucket(&BucketOptions {
@@ -14954,13 +14850,7 @@ mod tests {
     #[tokio::test]
     #[serial]
     async fn tier_free_version_recovery_loop_observes_tier_configuration_changes() {
-        let automatic_recovery_started = super::TIER_FREE_VERSION_RECOVERY_STARTED.get().copied();
         let (disk_paths, ecstore) = setup_test_env().await;
-        assert_eq!(
-            super::TIER_FREE_VERSION_RECOVERY_STARTED.get().copied(),
-            automatic_recovery_started,
-            "fixture must not start an unowned automatic recovery loop"
-        );
         let manager = ecstore.tier_config_mgr();
         let saved_tiers = std::mem::take(&mut manager.write().await.tiers);
         let runtime_state = install_unconsumed_runtime_expiry_worker(&ecstore, 1).await;
@@ -14970,8 +14860,6 @@ mod tests {
         create_test_bucket(&ecstore, &bucket).await;
         seed_recoverable_free_version(&disk_paths, &bucket, object, None, None).await;
 
-        let page_permits = Arc::new(tokio::sync::Semaphore::new(1));
-        let worker_permits = Arc::clone(&page_permits);
         let (page_tx, mut page_rx) = tokio::sync::mpsc::unbounded_channel();
         let cancel = CancellationToken::new();
         let loop_cancel = cancel.clone();
@@ -14984,16 +14872,7 @@ mod tests {
                 move |bucket, object, cancel| {
                     let api = Arc::clone(&recovery_store);
                     let page_tx = page_tx.clone();
-                    let page_permits = Arc::clone(&worker_permits);
                     async move {
-                        // Advance the scheduler clock only while real disk I/O is
-                        // parked. Otherwise its timeout can win while the blocking
-                        // filesystem operation is still running on wall-clock time.
-                        let permit = tokio::select! {
-                            _ = cancel.cancelled() => return Err(crate::error::Error::other("test recovery canceled")),
-                            permit = page_permits.acquire() => permit.expect("test controls page admission"),
-                        };
-                        permit.forget();
                         let stats =
                             super::recover_configured_tier_free_versions(api, bucket.clone(), object.clone(), cancel).await?;
                         page_tx
@@ -15021,14 +14900,13 @@ mod tests {
         tokio::task::yield_now().await;
         tokio::time::advance(TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL).await;
         tokio::time::resume();
-        page_permits.add_permits(1);
         let (bucket_marker, object_marker, active) = tokio::time::timeout(StdDuration::from_secs(30), page_rx.recv())
             .await
             .expect("new tier should enable the next recovery page")
             .expect("worker should remain running");
         assert!(bucket_marker.is_none());
         assert!(object_marker.is_none());
-        assert_eq!(active.enqueued, 1, "recovery page: {active:?}");
+        assert_eq!(active.enqueued, 1);
         let task = recovery_rx
             .lock()
             .await
@@ -15049,7 +14927,6 @@ mod tests {
         tokio::task::yield_now().await;
         tokio::time::advance(TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL).await;
         tokio::time::resume();
-        page_permits.add_permits(1);
         let (_, _, idle_again) = tokio::time::timeout(StdDuration::from_secs(30), page_rx.recv())
             .await
             .expect("empty configuration should still produce an idle check")

crates/ecstore/src/bucket/lifecycle/metadata_boundary.rs

@@ -63,14 +63,7 @@ pub(crate) async fn get_expiry_configs_for_options(
     bucket: &str,
     opts: &crate::object_api::ObjectOptions,
 ) -> Result<LifecycleExpiryConfigs> {
-    let bucket_incarnation_id = match opts
-        .object_lock_config_snapshot
-        .as_ref()
-        .and_then(|snapshot| snapshot.validated_bucket_incarnation_for(api.id, bucket, opts.expected_bucket_incarnation_id))
-    {
-        Some(incarnation_id) => incarnation_id,
-        None => metadata_sys::get_bucket_incarnation_id_for_options_in(&api.ctx, bucket, opts).await?,
-    };
+    let bucket_incarnation_id = metadata_sys::get_bucket_incarnation_id_for_options_in(&api.ctx, bucket, opts).await?;
     let metadata = get_authoritative_metadata(api, bucket, bucket_incarnation_id).await?;
     let table_bucket_enabled = metadata.table_bucket_enabled();
 
@@ -181,36 +174,5 @@ mod tests {
                 .unwrap()
         );
         assert!(get_expiry_configs(&store_b, bucket).await.unwrap().lifecycle.is_none());
-
-        let snapshot = store_a.object_lock_config_snapshot(bucket).await.unwrap();
-        assert_eq!(
-            snapshot.validated_bucket_incarnation_for(store_a.id, bucket, Some(configs.bucket_incarnation_id)),
-            Some(configs.bucket_incarnation_id)
-        );
-        assert_eq!(
-            snapshot.validated_bucket_incarnation_for(store_a.id, bucket, Some(uuid::Uuid::new_v4())),
-            None
-        );
-        assert_eq!(snapshot.validated_bucket_incarnation_for(store_b.id, bucket, None), None);
-        let opts = crate::object_api::ObjectOptions {
-            object_lock_config_snapshot: Some(snapshot.clone()),
-            expected_bucket_incarnation_id: Some(configs.bucket_incarnation_id),
-            ..Default::default()
-        };
-        assert_eq!(
-            get_expiry_configs_for_options(&store_a, bucket, &opts)
-                .await
-                .unwrap()
-                .bucket_incarnation_id,
-            configs.bucket_incarnation_id
-        );
-        let unguarded = crate::object_api::ObjectLockConfigSnapshot::for_store_bucket(
-            store_a.id,
-            bucket,
-            configs.bucket_incarnation_id,
-            time::OffsetDateTime::UNIX_EPOCH,
-            metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
-        );
-        assert_eq!(unguarded.validated_bucket_incarnation_for(store_a.id, bucket, None), None);
     }
 }

crates/ecstore/src/bucket/metadata.rs

@@ -437,8 +437,6 @@ pub struct BucketMetadata {
     pub created: OffsetDateTime,
     pub lock_enabled: bool, // While marked as unused, it may need to be retained
     pub bucket_incarnation_id: Uuid,
-    pub artifact_base_prefix: Option<String>,
-    pub artifact_backup_retained: bool,
     pub(crate) bucket_incarnation_sidecar: bool,
     pub(crate) bucket_creation_committed: bool,
     pub policy_config_json: Vec<u8>,
@@ -514,8 +512,6 @@ impl Default for BucketMetadata {
             created: OffsetDateTime::UNIX_EPOCH,
             lock_enabled: Default::default(),
             bucket_incarnation_id: Uuid::nil(),
-            artifact_base_prefix: None,
-            artifact_backup_retained: false,
             bucket_incarnation_sidecar: false,
             bucket_creation_committed: false,
             policy_config_json: Default::default(),
@@ -611,7 +607,7 @@ impl BucketMetadata {
     /// legacy migration path, which mints the sidecar first; it must not be
     /// rewritten by this migration.
     pub(crate) fn needs_bucket_creation_commit(&self) -> bool {
-        (self.lock_enabled || self.artifact_base_prefix.is_some())
+        self.lock_enabled
             && !self.bucket_creation_committed
             && self.bucket_incarnation_sidecar
             && !self.bucket_incarnation_id.is_nil()
@@ -708,22 +704,6 @@ impl BucketMetadata {
                     self.bucket_incarnation_id =
                         Uuid::from_slice(&bytes).map_err(|err| Error::other(format!("invalid BucketIncarnationID: {err}")))?;
                 }
-                "ArtifactBackupRetained" => {
-                    if self.artifact_backup_retained || !rmp::decode::read_bool(rd)? {
-                        return Err(Error::other("invalid artifact backup retention marker"));
-                    }
-                    self.artifact_backup_retained = true;
-                }
-                "ArtifactBasePrefix" => {
-                    if self.artifact_base_prefix.is_some() {
-                        return Err(Error::other("duplicate artifact namespace"));
-                    }
-                    let prefix = read_msgp_str(rd)?;
-                    if !crate::storage_api_contracts::bucket::valid_artifact_base_prefix(&prefix) {
-                        return Err(Error::other("invalid artifact namespace"));
-                    }
-                    self.artifact_base_prefix = Some(prefix);
-                }
                 "PolicyConfigJSON" | "PolicyConfigJson" => self.policy_config_json = read_msgp_bin(rd)?,
                 "NotificationConfigXML" | "NotificationConfigXml" => self.notification_config_xml = read_msgp_bin(rd)?,
                 "LifecycleConfigXML" | "LifecycleConfigXml" => self.lifecycle_config_xml = read_msgp_bin(rd)?,
@@ -786,7 +766,7 @@ impl BucketMetadata {
     /// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
     pub fn encode_to<W: Write>(&self, wr: &mut W) -> Result<()> {
         // Map size: MinIO fields (25) + RustFS extensions (22)
-        let map_len: u32 = 47 + u32::from(self.artifact_base_prefix.is_some()) + u32::from(self.artifact_backup_retained);
+        let map_len: u32 = 47;
         rmp::encode::write_map_len(wr, map_len)?;
 
         // MinIO field order (same as Go struct)
@@ -800,15 +780,6 @@ impl BucketMetadata {
         rmp::encode::write_bool(wr, self.lock_enabled)?;
 
         write_bin_field(wr, "BucketIncarnationID", self.bucket_incarnation_id.as_bytes())?;
-        if let Some(prefix) = &self.artifact_base_prefix {
-            rmp::encode::write_str(wr, "ArtifactBasePrefix")?;
-            rmp::encode::write_str(wr, prefix)?;
-        }
-
-        if self.artifact_backup_retained {
-            rmp::encode::write_str(wr, "ArtifactBackupRetained")?;
-            rmp::encode::write_bool(wr, true)?;
-        }
 
         write_bin_field(wr, "PolicyConfigJSON", &self.policy_config_json)?;
         write_bin_field(wr, "NotificationConfigXML", &self.notification_config_xml)?;
@@ -909,28 +880,13 @@ impl BucketMetadata {
         }
 
         match version {
-            BUCKET_METADATA_VERSION | 2 => {}
+            BUCKET_METADATA_VERSION => {}
             _ => return Err(Error::other("read_bucket_metadata: version invalid")),
         }
 
         Ok(())
     }
 
-    /// Version two is reserved for permanent artifact backup retention. Older
-    /// readers reject its header instead of skipping the protection field.
-    pub(crate) fn decode_file(bytes: &[u8]) -> Result<Self> {
-        Self::check_header(bytes)?;
-        let metadata = Self::unmarshal(&bytes[4..])?;
-        let retained_header = LittleEndian::read_u16(&bytes[2..4]) == 2;
-        if retained_header != metadata.artifact_backup_retained {
-            return Err(Error::other("artifact backup retention header mismatch"));
-        }
-        if metadata.artifact_backup_retained {
-            metadata.validate_artifact_configuration()?;
-        }
-        Ok(metadata)
-    }
-
     pub(crate) fn default_timestamps(&mut self) {
         if self.policy_config_updated_at == OffsetDateTime::UNIX_EPOCH {
             self.policy_config_updated_at = self.created
@@ -1069,9 +1025,6 @@ impl BucketMetadata {
                     }
                     Some(config)
                 };
-                if self.artifact_base_prefix.is_some() && !config.as_ref().is_some_and(artifact_versioning_enabled) {
-                    return Err(Error::other("artifact bucket versioning must remain enabled"));
-                }
                 self.versioning_config_xml = data;
                 self.versioning_config = config;
                 self.versioning_config_updated_at = updated;
@@ -1174,34 +1127,13 @@ impl BucketMetadata {
         &mut self,
         store: std::sync::Arc<crate::store::ECStore>,
         write_completion: WriteCompletion,
-    ) -> Result<()> {
-        let opts = crate::object_api::ObjectOptions {
-            max_parity: true,
-            write_completion,
-            ..Default::default()
-        };
-        self.save_with_store_options(store, &opts).await
-    }
-
-    pub(crate) async fn save_with_store_options(
-        &mut self,
-        store: Arc<ECStore>,
-        opts: &crate::object_api::ObjectOptions,
     ) -> Result<()> {
         self.parse_all_configs()?;
-        self.validate_artifact_update(store.clone()).await?;
         let mut buf: Vec<u8> = vec![0; 4];
 
         LittleEndian::write_u16(&mut buf[0..2], BUCKET_METADATA_FORMAT);
 
-        LittleEndian::write_u16(
-            &mut buf[2..4],
-            if self.artifact_backup_retained {
-                2
-            } else {
-                BUCKET_METADATA_VERSION
-            },
-        );
+        LittleEndian::write_u16(&mut buf[2..4], BUCKET_METADATA_VERSION);
 
         let data = self
             .marshal_msg()
@@ -1209,52 +1141,21 @@ impl BucketMetadata {
 
         buf.extend_from_slice(&data);
 
-        save_config_with_opts(store, self.save_file_path().as_str(), buf, opts).await?;
+        save_config_with_opts(
+            store,
+            self.save_file_path().as_str(),
+            buf,
+            &crate::object_api::ObjectOptions {
+                max_parity: true,
+                write_completion,
+                ..Default::default()
+            },
+        )
+        .await?;
 
         Ok(())
     }
 
-    fn validate_artifact_configuration(&self) -> Result<()> {
-        if self.artifact_backup_retained && (self.artifact_base_prefix.is_none() || !self.bucket_creation_committed) {
-            return Err(Error::other("artifact backup retention requires a committed managed bucket"));
-        }
-        if let Some(prefix) = &self.artifact_base_prefix {
-            if !crate::storage_api_contracts::bucket::valid_artifact_base_prefix(prefix) || self.bucket_incarnation_id.is_nil() {
-                return Err(Error::other("invalid artifact bucket identity"));
-            }
-            let versioning = deserialize::<VersioningConfiguration>(&self.versioning_config_xml)?;
-            if !artifact_versioning_enabled(&versioning) {
-                return Err(Error::other("artifact bucket versioning must remain enabled"));
-            }
-        }
-        Ok(())
-    }
-
-    /// Full metadata replacement must not change a generation's creation identity.
-    pub(crate) async fn validate_artifact_update(&self, store: Arc<ECStore>) -> Result<()> {
-        self.validate_artifact_configuration()?;
-        // An existing empty/corrupt metadata object is not permission to create an identity.
-        match read_config_preserve_empty(store, &self.save_file_path()).await {
-            Ok(data) => {
-                let previous = Self::decode_file(&data)?;
-                previous.validate_artifact_configuration()?;
-                if previous.artifact_backup_retained && !self.artifact_backup_retained {
-                    return Err(Error::other("artifact backup retention cannot be removed"));
-                }
-                // Deletion removes this record before recreation. A caller-supplied
-                // incarnation cannot turn an existing bucket into a new generation.
-                if previous.artifact_base_prefix != self.artifact_base_prefix
-                    || (previous.artifact_base_prefix.is_some() && previous.bucket_incarnation_id != self.bucket_incarnation_id)
-                {
-                    return Err(Error::other("artifact bucket identity is immutable"));
-                }
-            }
-            Err(Error::ConfigNotFound) => {}
-            Err(error) => return Err(error),
-        }
-        Ok(())
-    }
-
     fn parse_policy_config(&mut self) -> Result<()> {
         if !self.policy_config_json.is_empty() {
             self.policy_config = Some(serde_json::from_slice(&self.policy_config_json)?);
@@ -1298,7 +1199,6 @@ impl BucketMetadata {
     /// `rustfs_bucket_metadata_parse_failed_total`, reflected in
     /// `rustfs_bucket_metadata_unparsable_current`, and logged at error level.
     pub(super) fn parse_all_configs(&mut self) -> Result<()> {
-        self.validate_artifact_configuration()?;
         if let Err(e) = self.parse_policy_config() {
             tracing::warn!(
                 event = "bucket_metadata_parse_failed",
@@ -1564,10 +1464,6 @@ impl BucketMetadata {
     }
 }
 
-fn artifact_versioning_enabled(config: &VersioningConfiguration) -> bool {
-    config.enabled() && !config.exclude_folders.unwrap_or(false) && config.excluded_prefixes.as_ref().is_none_or(Vec::is_empty)
-}
-
 pub(crate) async fn load_bucket_incarnation(api: Arc<ECStore>, bucket: &str) -> Result<Option<Uuid>> {
     let path = format!("{BUCKET_META_PREFIX}/{bucket}/{BUCKET_INCARNATION_FILE}");
     let data = match read_config_preserve_empty(api, &path).await {
@@ -1664,7 +1560,9 @@ async fn read_bucket_metadata(api: Arc<ECStore>, bucket: &str) -> Result<BucketM
 
     let data = read_config(api, &file_path).await?;
 
-    let bm = BucketMetadata::decode_file(&data)?;
+    BucketMetadata::check_header(&data)?;
+
+    let bm = BucketMetadata::unmarshal(&data[4..])?;
 
     Ok(bm)
 }
@@ -1688,98 +1586,6 @@ where
 mod test {
     use super::*;
 
-    #[test]
-    fn artifact_metadata_roundtrip_and_corrupt_prefix_rejection() {
-        let mut metadata = BucketMetadata::new("artifacts");
-        metadata.artifact_base_prefix = Some("images/".to_owned());
-        let encoded = metadata.marshal_msg().expect("encode artifact metadata");
-        let decoded = BucketMetadata::unmarshal(&encoded).expect("decode artifact metadata");
-        assert_eq!(decoded.artifact_base_prefix, metadata.artifact_base_prefix, "ARTIFACT_METADATA_ROUNDTRIP");
-        assert_eq!(decoded.bucket_incarnation_id, metadata.bucket_incarnation_id);
-        let legacy = BucketMetadata::new("ordinary")
-            .marshal_msg()
-            .expect("encode ordinary metadata");
-        assert!(
-            BucketMetadata::unmarshal(&legacy)
-                .expect("decode ordinary metadata")
-                .artifact_base_prefix
-                .is_none()
-        );
-        for prefix in ["", "../", "outputs", "a b/"] {
-            let mut invalid = Vec::new();
-            rmp::encode::write_map_len(&mut invalid, 1).expect("encode map");
-            rmp::encode::write_str(&mut invalid, "ArtifactBasePrefix").expect("encode field");
-            rmp::encode::write_str(&mut invalid, prefix).expect("encode invalid prefix");
-            assert!(BucketMetadata::unmarshal(&invalid).is_err(), "ARTIFACT_CORRUPT_PREFIX");
-        }
-    }
-
-    #[test]
-    fn artifact_backup_retention_marker_rejects_false_and_duplicates() {
-        for count in [1, 2] {
-            let mut bytes = Vec::new();
-            rmp::encode::write_map_len(&mut bytes, count).unwrap();
-            for _ in 0..count {
-                rmp::encode::write_str(&mut bytes, "ArtifactBackupRetained").unwrap();
-                rmp::encode::write_bool(&mut bytes, count == 2).unwrap();
-            }
-            assert!(BucketMetadata::unmarshal(&bytes).is_err(), "O02P3_INVALID_RETENTION_MARKER");
-        }
-    }
-
-    #[test]
-    fn artifact_backup_retention_requires_matching_header_and_identity() {
-        let mut metadata = BucketMetadata::new("retained");
-        metadata.bucket_incarnation_id = Uuid::new_v4();
-        metadata.bucket_creation_committed = true;
-        metadata.artifact_base_prefix = Some("outputs/".to_owned());
-        metadata.versioning_config_xml = b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec();
-        let file = |metadata: &BucketMetadata, version: u16| {
-            let mut bytes = Vec::new();
-            bytes.extend_from_slice(&BUCKET_METADATA_FORMAT.to_le_bytes());
-            bytes.extend_from_slice(&version.to_le_bytes());
-            bytes.extend_from_slice(&metadata.marshal_msg().unwrap());
-            bytes
-        };
-        assert!(BucketMetadata::decode_file(&file(&metadata, 1)).is_ok(), "O02P3_ORDINARY_V1_COMPATIBLE");
-        assert!(BucketMetadata::decode_file(&file(&metadata, 2)).is_err(), "O02P3_MARKER_REQUIRED");
-        metadata.artifact_backup_retained = true;
-        assert!(
-            BucketMetadata::decode_file(&file(&metadata, 1)).is_err(),
-            "O02P3_PROTECTION_CANNOT_DOWNGRADE_HEADER"
-        );
-        assert!(
-            BucketMetadata::decode_file(&file(&metadata, 2))
-                .unwrap()
-                .artifact_backup_retained,
-            "O02P3_PROTECTED_V2_ROUNDTRIP"
-        );
-        metadata.artifact_base_prefix = None;
-        assert!(
-            BucketMetadata::decode_file(&file(&metadata, 2)).is_err(),
-            "O02P3_PROTECTION_REQUIRES_MANAGED_BUCKET"
-        );
-    }
-
-    #[test]
-    fn artifact_versioning_cannot_be_disabled_or_excluded() {
-        let mut metadata = BucketMetadata::new("artifacts");
-        metadata.artifact_base_prefix = Some("images/".to_owned());
-        for xml in [
-            "",
-            "<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>",
-            "<VersioningConfiguration><Status>Enabled</Status><ExcludeFolders>true</ExcludeFolders></VersioningConfiguration>",
-            "<VersioningConfiguration><Status>Enabled</Status><ExcludedPrefixes><Prefix>images/</Prefix></ExcludedPrefixes></VersioningConfiguration>",
-        ] {
-            assert!(
-                metadata
-                    .update_config(BUCKET_VERSIONING_CONFIG, xml.as_bytes().to_vec())
-                    .is_err(),
-                "ARTIFACT_VERSIONING_REQUIRED"
-            );
-        }
-    }
-
     /// rustfs/backlog#1734: `StorageError::clone` rebuilds I/O errors from
     /// their text. The unreadable-config refusal must stay typed across that
     /// clone, or a cloned error stops mapping to its retryable S3 response.

crates/ecstore/src/bucket/metadata_sys.rs

@@ -293,10 +293,8 @@ pub(super) fn get_bucket_metadata_sys() -> Result<Arc<RwLock<BucketMetadataSys>>
 pub async fn set_bucket_metadata(bucket: String, bm: BucketMetadata) -> Result<()> {
     let sys = get_bucket_metadata_sys()?;
     let lock = sys.write().await;
-    if bucket != bm.name {
-        return Err(Error::other("bucket metadata identity mismatch"));
-    }
-    lock.set_validated(bm).await
+    lock.set(bucket, Arc::new(bm)).await;
+    Ok(())
 }
 
 pub async fn reload_bucket_metadata(api: Arc<ECStore>, bucket: &str) -> Result<()> {
@@ -587,15 +585,6 @@ pub(crate) async fn get_cached_bucket_incarnation_id_in(
     sys.get_bucket_incarnation_id(bucket).await
 }
 
-pub(crate) async fn get_bucket_incarnation_metadata_in(
-    ctx: &crate::runtime::instance::InstanceContext,
-    bucket: &str,
-) -> Result<Arc<BucketMetadata>> {
-    let sys = bucket_metadata_sys_of(ctx)?;
-    let sys = sys.read().await.clone();
-    sys.get_bucket_incarnation_metadata(bucket).await
-}
-
 pub(crate) async fn set_bucket_metadata_in(ctx: &crate::runtime::instance::InstanceContext, bm: BucketMetadata) -> Result<()> {
     let sys = bucket_metadata_sys_of(ctx)?;
     let lock = sys.read().await;
@@ -2143,16 +2132,7 @@ impl BucketMetadataSys {
         }
     }
 
-    pub(crate) async fn set_validated(&self, mut bm: BucketMetadata) -> Result<()> {
-        if bm.artifact_base_prefix.is_some() {
-            bm.parse_all_configs()?;
-        }
-        bm.validate_artifact_update(self.object_store()).await?;
-        self.set(bm.name.clone(), Arc::new(bm)).await;
-        Ok(())
-    }
-
-    async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
+    pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
         if !is_meta_bucketname(&bucket) {
             let publish_lock = self.metadata_publish_lock(&bucket);
             let _publish_guard = publish_lock.lock().await;
@@ -2167,11 +2147,6 @@ impl BucketMetadataSys {
         }
     }
 
-    #[cfg(any(test, feature = "test-util"))]
-    pub async fn set_for_test(&self, bucket: String, bm: Arc<BucketMetadata>) {
-        self.set(bucket, bm).await;
-    }
-
     /// Remove a bucket's cached metadata from the in-memory map.
     ///
     /// Returns `true` if an entry was present. Reserved meta buckets are ignored.
@@ -2676,16 +2651,12 @@ impl BucketMetadataSys {
     }
 
     async fn get_bucket_incarnation_id(&self, bucket: &str) -> Result<Uuid> {
-        Ok(self.get_bucket_incarnation_metadata(bucket).await?.bucket_incarnation_id)
-    }
-
-    async fn get_bucket_incarnation_metadata(&self, bucket: &str) -> Result<Arc<BucketMetadata>> {
         let authority = self.get_metadata_authority(bucket).await?;
         if let BucketMetadataAuthority::Authoritative(metadata) = &authority
             && metadata.bucket_incarnation_sidecar
             && !metadata.bucket_incarnation_id.is_nil()
         {
-            return Ok(Arc::clone(metadata));
+            return Ok(metadata.bucket_incarnation_id);
         }
 
         match authority {
@@ -2694,7 +2665,7 @@ impl BucketMetadataSys {
                     BucketMetadataAuthority::Authoritative(metadata)
                         if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
                     {
-                        Ok(metadata)
+                        Ok(metadata.bucket_incarnation_id)
                     }
                     BucketMetadataAuthority::Authoritative(_) | BucketMetadataAuthority::Fabricated => {
                         Err(Error::other(format!("bucket incarnation metadata is not authoritative: {bucket}")))
@@ -3022,8 +2993,7 @@ impl BucketMetadataSys {
             load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
         )
         .await?;
-        if persisted && (metadata.lock_enabled || metadata.artifact_base_prefix.is_some()) && !metadata.bucket_creation_committed
-        {
+        if persisted && metadata.lock_enabled && !metadata.bucket_creation_committed {
             return Err(Error::ErasureWriteQuorum);
         }
         if persisted {

crates/ecstore/src/bucket/mod.rs

@@ -14,7 +14,6 @@
 
 // #730: bucket subsystems still contain staged ECStore migration code.
 
-pub mod artifact;
 pub mod bandwidth;
 pub mod bucket_target_sys;
 pub mod config_parse_mode;

crates/ecstore/src/bucket/quota/checker.rs

@@ -235,8 +235,6 @@ impl QuotaChecker {
         Ok(updated_at)
     }
 
-    /// Return quota configuration with optional scanner usage, independently
-    /// of the usage requirements enforced by write admission.
     pub async fn get_quota_stats(&self, bucket: &str) -> Result<(BucketQuota, Option<u64>), QuotaError> {
         // If bucket doesn't exist, return ConfigNotFound error
         if !self.bucket_exists(bucket).await {
@@ -246,28 +244,37 @@ impl QuotaChecker {
         }
 
         let quota = self.get_quota_config(bucket).await?;
-        let current_usage = self.report_only_usage(bucket).await;
+        let current_usage = if quota.uses_durable_reservations() {
+            self.report_only_usage(bucket).await
+        } else {
+            Some(self.get_real_time_usage(bucket).await?)
+        };
 
         Ok((quota, current_usage))
     }
 
-    /// Best-effort usage for statistics and durable admission reports.
-    /// Missing scanner coverage is unknown, not an error or a zero-sized bucket.
+    /// Scanner usage for a bucket the reservation ledger admits is reporting
+    /// input only, so a failed lookup degrades to "unknown" and is counted
+    /// instead of becoming an error the write path would fail on.
     async fn report_only_usage(&self, bucket: &str) -> Option<u64> {
         // The plain lookup: a report must not wait for an early snapshot reload.
         let usage = get_bucket_usage_memory(bucket).await;
-        if usage.is_none() {
-            metrics::counter!(METRIC_QUOTA_USAGE_REPORT_UNAVAILABLE_TOTAL).increment(1);
-            debug!(
-                event = EVENT_QUOTA_USAGE_REPORT,
-                component = LOG_COMPONENT_ECSTORE,
-                subsystem = LOG_SUBSYSTEM_QUOTA,
-                state = "usage_unknown",
-                bucket = %bucket,
-                "quota usage report degraded"
-            );
+        match self.usage_or_degraded_baseline(bucket, usage).await {
+            Ok(usage) => Some(usage),
+            Err(err) => {
+                metrics::counter!(METRIC_QUOTA_USAGE_REPORT_UNAVAILABLE_TOTAL).increment(1);
+                debug!(
+                    event = EVENT_QUOTA_USAGE_REPORT,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_QUOTA,
+                    state = "usage_unknown",
+                    bucket = %bucket,
+                    error = %err,
+                    "durable quota usage report degraded"
+                );
+                None
+            }
         }
-        usage
     }
 
     /// Re-save `bucket`'s quota under the sharded durable reservation
@@ -537,8 +544,6 @@ mod tests {
     use serial_test::serial;
     use uuid::Uuid;
 
-    mod reporting_retirement;
-
     /// Regression (PR #5307 / s3-tests `test_100_continue_error_retry`): a
     /// bucket with no persisted metadata has no quota, so the admission check
     /// must pass and let the request reach its NoSuchBucket answer. With the
@@ -739,12 +744,8 @@ mod tests {
             1,
             "fail-closed admission takes the early-reload lookup"
         );
-        let (_, usage) = checker
-            .get_quota_stats(&legacy)
-            .await
-            .expect("legacy quota statistics are reporting only");
-        assert_eq!(usage, None);
-        assert_eq!(crate::data_usage::reload_unknown_bucket_usage_calls_for_test(&legacy), 1);
+        assert!(matches!(checker.get_quota_stats(&legacy).await, Err(QuotaError::UsageUnavailable { .. })));
+        assert_eq!(crate::data_usage::reload_unknown_bucket_usage_calls_for_test(&legacy), 2);
         assert_eq!(
             crate::data_usage::reload_unknown_bucket_usage_calls_for_test(&bucket),
             0,
@@ -895,7 +896,7 @@ mod tests {
             .update_bucket_metadata_config(&bucket, QUOTA_CONFIG_FILE, legacy_quota_json(8192))
             .await
             .expect("newer quota save");
-        sys.read().await.set_for_test(bucket.clone(), stale).await;
+        sys.read().await.set(bucket.clone(), stale).await;
         assert_eq!(
             checker
                 .migrate_reservation_protocol(&bucket)

crates/ecstore/src/bucket/quota/checker/tests/reporting_retirement.rs

@@ -1,330 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use super::*;
-use crate::disk::DiskAPI as _;
-use crate::object_api::{ObjectOptions, PutObjReader};
-use crate::storage_api_contracts::object::ObjectIO as _;
-use crate::test_metrics::CapturingRecorder;
-use rustfs_data_usage::{BucketUsageInfo, DataUsageInfo};
-use std::time::SystemTime;
-
-async fn stop_reporting_fixture(store: &crate::store::ECStore) {
-    if let Some(cancel) = store.ctx.background_cancel_token() {
-        cancel.cancel();
-    }
-    for pool in &store.pools {
-        for set in &pool.disk_set {
-            let disks = set.disks.read().await.clone();
-            for disk in disks.iter().flatten() {
-                disk.close().await.expect("stop fixture disk monitoring");
-            }
-        }
-    }
-}
-
-async fn install_reporting_snapshot(
-    store: Arc<crate::store::ECStore>,
-    bucket: &str,
-    source: &str,
-    size: u64,
-) -> Option<(String, Vec<u8>)> {
-    if source == "missing" {
-        return None;
-    }
-    let mut info = DataUsageInfo {
-        last_update: Some(SystemTime::now()),
-        scanner_epoch: Some(1),
-        scanner_cycle: Some(1),
-        buckets_count: 1,
-        ..Default::default()
-    };
-    info.buckets_usage.insert(
-        bucket.to_owned(),
-        BucketUsageInfo {
-            size,
-            objects_count: 1,
-            versions_count: 1,
-            ..Default::default()
-        },
-    );
-    info.bucket_sizes.insert(bucket.to_owned(), size);
-    info.calculate_totals();
-    let object = match source {
-        "legacy" => rustfs_data_usage::LEGACY_DATA_USAGE_OBJECT_NAME,
-        "incomplete" => rustfs_data_usage::DATA_USAGE_OBJECT_NAME,
-        "complete" => {
-            info.usage_snapshot_complete = true;
-            rustfs_data_usage::DATA_USAGE_OBJECT_NAME
-        }
-        "observed" => {
-            info.usage_snapshot_complete = true;
-            info.usage_snapshot_converged = Some(false);
-            info.usage_snapshot_authoritative_baseline = Some(DataUsageInfo::default().snapshot_identity());
-            rustfs_data_usage::DATA_USAGE_OBSERVED_OBJECT_NAME
-        }
-        _ => panic!("unknown scanner fixture source"),
-    };
-    let path = format!("{}/{object}", crate::disk::BUCKET_META_PREFIX);
-    let bytes = serde_json::to_vec(&info).expect("scanner fixture encoding");
-    crate::config::com::save_config(store, &path, bytes.clone())
-        .await
-        .expect("persist scanner fixture");
-    if source == "complete" {
-        crate::data_usage::replace_bucket_usage_memory_from_info(&info).await;
-    }
-    Some((path, bytes))
-}
-
-#[tokio::test(flavor = "current_thread")]
-#[serial]
-async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger_admission() {
-    let recorder = CapturingRecorder::default();
-    let _recorder = metrics::set_default_local_recorder(&recorder);
-    for protocol in [1, 2] {
-        for source in ["missing", "legacy", "incomplete", "observed", "complete"] {
-            let (_dirs, store, sys) = isolated_durable_quota_store().await;
-            let checker = QuotaChecker::new(sys);
-            let bucket = format!("quota-report-{protocol}-{}", Uuid::new_v4().simple());
-            let quota = if protocol == 1 {
-                BucketQuota::new(Some(4096))
-            } else {
-                BucketQuota::new_sharded(4096)
-            };
-            create_bucket_with_quota_json(&store, &bucket, serde_json::to_vec(&quota).expect("quota fixture encoding")).await;
-            let snapshot = install_reporting_snapshot(store.clone(), &bucket, source, 1_000_000).await;
-            crate::data_usage::invalidate_data_usage_snapshot_cache().await;
-            let unavailable_before = recorder.counter_value(METRIC_QUOTA_USAGE_REPORT_UNAVAILABLE_TOTAL, &[]);
-            let check = checker
-                .check_quota(&bucket, QuotaOperation::PutObject, 512)
-                .await
-                .expect("reporting cannot fail ledger admission");
-            let (_, usage) = checker.get_quota_stats(&bucket).await.expect("best-effort quota statistics");
-            let preview = checker
-                .check_quota_with_usage_reporting(&bucket, QuotaOperation::PutObject, 512, true)
-                .await
-                .expect("admin preview uses the same optional report");
-
-            // Exercise the real storage entry point before testing the new
-            // reporting semantics: the same ledger must admit and refuse
-            // these writes even when a scanner floor claims a million bytes.
-            let mut too_large = PutObjReader::from_vec(vec![0x5a; 8192]);
-            assert!(
-                matches!(
-                    store
-                        .put_object(&bucket, "too-large", &mut too_large, &ObjectOptions::default())
-                        .await,
-                    Err(StorageError::QuotaExceeded { .. })
-                ),
-                "protocol={protocol}, source={source}"
-            );
-            let mut fits = PutObjReader::from_vec(vec![0x5a; 512]);
-            store
-                .put_object(&bucket, "fits", &mut fits, &ObjectOptions::default())
-                .await
-                .expect("ledger admits 512 bytes");
-            let mut fills = PutObjReader::from_vec(vec![0x5a; 3584]);
-            store
-                .put_object(&bucket, "fills", &mut fills, &ObjectOptions::default())
-                .await
-                .expect("ledger admits exactly the remaining quota");
-            let mut exceeds = PutObjReader::from_vec(vec![0x5a; 1]);
-            assert!(
-                matches!(
-                    store
-                        .put_object(&bucket, "exceeds", &mut exceeds, &ObjectOptions::default())
-                        .await,
-                    Err(StorageError::QuotaExceeded { .. })
-                ),
-                "ledger cannot undercount protocol={protocol}, source={source}"
-            );
-
-            assert!(check.allowed && check.uses_durable_reservations);
-            let expected_usage = (source == "complete").then_some(1_000_000);
-            assert_eq!(
-                check.current_usage, expected_usage,
-                "protocol={protocol}, source={source}: a floor is not a current report"
-            );
-            assert_eq!(check.remaining, expected_usage.map(|_| 0));
-            assert_eq!(usage, expected_usage, "statistics use the same best-effort reporting contract");
-            assert_eq!(preview.current_usage, expected_usage);
-            assert_eq!(preview.remaining, expected_usage.map(|_| 0));
-            assert_eq!(preview.allowed, source != "complete", "unknown preview is advisory, not a reservation");
-            assert_eq!(
-                recorder.counter_value(METRIC_QUOTA_USAGE_REPORT_UNAVAILABLE_TOTAL, &[]) - unavailable_before,
-                if source == "complete" { 0 } else { 3 },
-                "each unknown check/statistics/preview report is counted"
-            );
-            assert_eq!(crate::data_usage::reload_unknown_bucket_usage_calls_for_test(&bucket), 0);
-            assert_eq!(persisted_quota(&store, &bucket).await.reservation_protocol, Some(protocol));
-            if let Some((path, original)) = snapshot {
-                assert_eq!(
-                    crate::config::com::read_config(store.clone(), &path)
-                        .await
-                        .expect("retained scanner evidence"),
-                    original
-                );
-            }
-            crate::data_usage::prepare_bucket_usage_for_namespace_change(&bucket, None)
-                .await
-                .expect("fixture cache cleanup");
-            crate::data_usage::invalidate_data_usage_snapshot_cache().await;
-            stop_reporting_fixture(&store).await;
-        }
-    }
-}
-
-#[tokio::test]
-#[serial]
-async fn snapshot_quota_admission_retains_degraded_floors_until_migration() {
-    for source in ["missing", "legacy", "incomplete", "observed"] {
-        let (_dirs, store, _) = isolated_durable_quota_store().await;
-        let bucket = format!("quota-compat-{}", Uuid::new_v4().simple());
-        create_bucket_with_quota_json(&store, &bucket, legacy_quota_json(4096)).await;
-        // An unregistered checker models a snapshot-protocol reader without
-        // starting the optional background migration during this assertion.
-        let checker = QuotaChecker::new(Arc::new(RwLock::new(BucketMetadataSys::new(store.clone()))));
-        install_reporting_snapshot(store.clone(), &bucket, source, 1000).await;
-        crate::data_usage::invalidate_data_usage_snapshot_cache().await;
-        let check = checker.check_quota(&bucket, QuotaOperation::PutObject, 512).await;
-        if source == "missing" {
-            assert!(matches!(check, Err(QuotaError::UsageUnavailable { .. })));
-        } else {
-            let check = check.expect("legacy admission retains its grounded compatibility fallback");
-            assert_eq!(check.current_usage, Some(1000));
-            assert!(check.allowed && !check.uses_durable_reservations);
-            assert!(
-                !checker
-                    .check_quota(&bucket, QuotaOperation::PutObject, 4096)
-                    .await
-                    .expect("legacy quota decision")
-                    .allowed
-            );
-        }
-        assert_eq!(persisted_quota(&store, &bucket).await.reservation_protocol, None);
-        crate::data_usage::prepare_bucket_usage_for_namespace_change(&bucket, None)
-            .await
-            .expect("fixture cache cleanup");
-        crate::data_usage::invalidate_data_usage_snapshot_cache().await;
-        stop_reporting_fixture(&store).await;
-    }
-}
-
-#[tokio::test(flavor = "current_thread")]
-#[serial]
-async fn quota_without_limit_keeps_the_zero_usage_io_fast_path() {
-    let (_dirs, store, sys) = isolated_durable_quota_store().await;
-    let bucket = format!("quota-unlimited-{}", Uuid::new_v4().simple());
-    create_bucket_with_quota_json(
-        &store,
-        &bucket,
-        serde_json::to_vec(&BucketQuota::new(None)).expect("unlimited quota encoding"),
-    )
-    .await;
-    let recorder = CapturingRecorder::default();
-    let _recorder = metrics::set_default_local_recorder(&recorder);
-    let check = QuotaChecker::new(sys)
-        .check_quota(&bucket, QuotaOperation::PutObject, u64::MAX)
-        .await
-        .expect("unlimited quota fast path");
-    assert!(check.allowed);
-    assert_eq!(check.current_usage, None);
-    assert_eq!(check.quota_limit, None);
-    assert_eq!(recorder.counter_value(METRIC_QUOTA_USAGE_REPORT_UNAVAILABLE_TOTAL, &[]), 0);
-    assert_eq!(crate::data_usage::reload_unknown_bucket_usage_calls_for_test(&bucket), 0);
-    stop_reporting_fixture(&store).await;
-}
-
-#[tokio::test]
-#[serial]
-async fn quota_statistics_remain_available_without_a_complete_scanner_snapshot() {
-    use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
-
-    for configuration in ["unconfigured", "unlimited", "snapshot"] {
-        for source in ["missing", "legacy", "incomplete", "observed", "complete", "empty"] {
-            let (_dirs, store, _) = isolated_durable_quota_store().await;
-            let bucket = format!("quota-statistics-{}", Uuid::new_v4().simple());
-            match configuration {
-                "unconfigured" => store
-                    .make_bucket(&bucket, &MakeBucketOptions::default())
-                    .await
-                    .expect("create bucket without a quota configuration"),
-                "unlimited" => {
-                    create_bucket_with_quota_json(
-                        &store,
-                        &bucket,
-                        serde_json::to_vec(&BucketQuota::new(None)).expect("unlimited quota encoding"),
-                    )
-                    .await;
-                }
-                "snapshot" => create_bucket_with_quota_json(&store, &bucket, legacy_quota_json(4096)).await,
-                _ => unreachable!(),
-            }
-            // An unregistered checker keeps legacy migration out of the
-            // assertion: reporting must work before a protocol upgrade.
-            let metadata_sys = BucketMetadataSys::new(store.clone());
-            let (metadata, _) = metadata_sys.get_config(&bucket).await.expect("load existing bucket metadata");
-            metadata_sys.set_for_test(bucket.clone(), metadata).await;
-            let checker = QuotaChecker::new(Arc::new(RwLock::new(metadata_sys)));
-            let (snapshot_source, size) = if source == "empty" { ("complete", 0) } else { (source, 1000) };
-            install_reporting_snapshot(store.clone(), &bucket, snapshot_source, size).await;
-            crate::data_usage::invalidate_data_usage_snapshot_cache().await;
-
-            let statistics = checker.get_quota_stats(&bucket).await;
-            let early_reloads = crate::data_usage::reload_unknown_bucket_usage_calls_for_test(&bucket);
-            let missing_bucket = format!("missing-{}", Uuid::new_v4().simple());
-            let missing_statistics = checker.get_quota_stats(&missing_bucket).await;
-            crate::data_usage::prepare_bucket_usage_for_namespace_change(&bucket, None)
-                .await
-                .expect("fixture cache cleanup");
-            crate::data_usage::invalidate_data_usage_snapshot_cache().await;
-            stop_reporting_fixture(&store).await;
-
-            let (quota, usage) = statistics.unwrap_or_else(|error| {
-                panic!("configuration={configuration}, source={source}: quota statistics must remain available: {error}")
-            });
-            assert_eq!(quota.quota, (configuration == "snapshot").then_some(4096));
-            assert_eq!(
-                usage,
-                (snapshot_source == "complete").then_some(size),
-                "configuration={configuration}, source={source}: only known usage is numeric"
-            );
-            assert_eq!(early_reloads, 0, "read-only statistics must not force an early snapshot reload");
-            assert!(
-                matches!(missing_statistics, Err(QuotaError::ConfigNotFound { bucket }) if bucket == missing_bucket),
-                "reporting must still reject a missing bucket"
-            );
-        }
-    }
-}
-
-#[tokio::test]
-async fn quota_statistics_preserve_configuration_errors() {
-    let (_dirs, store, _) = isolated_durable_quota_store().await;
-    let bucket = format!("quota-invalid-{}", Uuid::new_v4().simple());
-    create_bucket_with_quota_json(&store, &bucket, legacy_quota_json(4096)).await;
-    let metadata_sys = BucketMetadataSys::new(store.clone());
-    let (mut metadata, _) = metadata_sys.get_config(&bucket).await.expect("load existing bucket metadata");
-    Arc::make_mut(&mut metadata).quota_config_json = b"{not-json".to_vec();
-    metadata_sys.set_for_test(bucket.clone(), metadata).await;
-    let checker = QuotaChecker::new(Arc::new(RwLock::new(metadata_sys)));
-
-    let statistics = checker.get_quota_stats(&bucket).await;
-    stop_reporting_fixture(&store).await;
-
-    assert!(
-        matches!(statistics, Err(QuotaError::InvalidConfig { .. })),
-        "unavailable reporting must not hide malformed quota configuration"
-    );
-}

crates/ecstore/src/bucket/quota/reservation.rs

@@ -595,7 +595,7 @@ async fn reserve_sharded(context: QuotaContext, old_size: u64, new_size: u64) ->
         let allow_create = shard_may_be_created(&allocator, data.shard_index, bootstrap);
         let shard_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &data.shard_object).await?;
         let shard_guard = Arc::new(shard_lock.get_write_lock(get_lock_acquire_timeout()).await?);
-        let (mut shard, shard_created) = load_current_shard_locked(
+        let mut shard = load_current_shard_locked(
             Arc::clone(&store),
             &data.shard_object,
             bucket_incarnation,
@@ -640,7 +640,7 @@ async fn reserve_sharded(context: QuotaContext, old_size: u64, new_size: u64) ->
             });
         }
         let shard_was_empty = shard.grants.is_empty() && shard.reservations.is_empty() && shard.accounted_usage == 0;
-        if shard_created || shard_was_empty {
+        if shard_was_empty {
             fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &data.shard_object, None).await?;
             save_shard_locked(Arc::clone(&store), &data.shard_object, &shard, &shard_guard).await?;
         }
@@ -1510,18 +1510,18 @@ async fn load_current_shard_locked(
     shard_index: u16,
     allow_create: bool,
     allocator: &QuotaAllocatorLedger,
-) -> Result<(QuotaShardLedger, bool)> {
-    let (mut shard, created) = match load_shard_locked(Arc::clone(&store), object).await {
-        Ok(shard) if shard.matches(bucket_incarnation, quota_revision, shard_index) => (shard, false),
+) -> Result<QuotaShardLedger> {
+    let mut shard = match load_shard_locked(Arc::clone(&store), object).await {
+        Ok(shard) if shard.matches(bucket_incarnation, quota_revision, shard_index) => shard,
         Ok(_) => return Err(StorageError::PartMissingOrCorrupt),
         Err(StorageError::ConfigNotFound) if allow_create => {
-            (QuotaShardLedger::new(bucket_incarnation, quota_revision, shard_index), true)
+            QuotaShardLedger::new(bucket_incarnation, quota_revision, shard_index)
         }
         Err(StorageError::ConfigNotFound) => return Err(StorageError::PartMissingOrCorrupt),
         Err(err) => return Err(err),
     };
     shard.adopt_grants(allocator)?;
-    Ok((shard, created))
+    Ok(shard)
 }
 
 async fn save_allocator_locked(
@@ -1646,7 +1646,7 @@ async fn reap_sharded_reservations(
     .await?;
     let shard_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &data.shard_object).await?;
     let shard_guard = shard_lock.get_write_lock(get_lock_acquire_timeout()).await?;
-    let (shard, _) = load_current_shard_locked(
+    let shard = load_current_shard_locked(
         Arc::clone(&store),
         &data.shard_object,
         bucket_incarnation,
@@ -1709,7 +1709,7 @@ async fn reap_sharded_reservations(
     .await?;
     let shard_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &data.shard_object).await?;
     let shard_guard = Arc::new(shard_lock.get_write_lock(get_lock_acquire_timeout()).await?);
-    let (mut shard, _) = load_current_shard_locked(
+    let mut shard = load_current_shard_locked(
         Arc::clone(&store),
         &data.shard_object,
         bucket_incarnation,
@@ -1774,7 +1774,7 @@ async fn mark_commit_started_sharded(data: &ShardedReservationData) -> Result<()
         let allocator = load_allocator_locked(Arc::clone(&store), &data.allocator_object).await?;
         let shard_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &data.shard_object).await?;
         let shard_guard = Arc::new(shard_lock.get_write_lock(get_lock_acquire_timeout()).await?);
-        let (mut shard, _) = load_current_shard_locked(
+        let mut shard = load_current_shard_locked(
             Arc::clone(&store),
             &data.shard_object,
             data.bucket_incarnation,
@@ -1819,7 +1819,7 @@ async fn settle_sharded(data: &ShardedReservationData, committed: bool) -> Resul
         let allocator = load_allocator_locked(Arc::clone(&store), &data.allocator_object).await?;
         let shard_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &data.shard_object).await?;
         let shard_guard = Arc::new(shard_lock.get_write_lock(get_lock_acquire_timeout()).await?);
-        let (mut shard, _) = load_current_shard_locked(
+        let mut shard = load_current_shard_locked(
             Arc::clone(&store),
             &data.shard_object,
             data.bucket_incarnation,
@@ -2535,70 +2535,4 @@ mod tests {
             1
         );
     }
-
-    #[tokio::test]
-    #[serial]
-    async fn sharded_reservation_persists_a_populated_lazy_shard_before_refill() {
-        use crate::object_api::PutObjReader;
-        use crate::storage_api_contracts::object::ObjectIO;
-
-        let (_directories, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await;
-        metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
-        crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
-        let bucket = format!("quota-populated-bootstrap-{}", Uuid::new_v4().simple());
-        store
-            .make_bucket(&bucket, &MakeBucketOptions::default())
-            .await
-            .expect("create quota bucket");
-
-        let mut seed_shard = None;
-        let mut seed_name = None;
-        let mut candidate_name = None;
-        let mut bootstrap_name = None;
-        for index in 0..10_000 {
-            let object = format!("quota-populated-object-{index}");
-            let object_shard = shard_index(&object);
-            if seed_shard.is_none() {
-                seed_shard = Some(object_shard);
-                seed_name = Some(object);
-            } else if Some(object_shard) == seed_shard && candidate_name.is_none() {
-                candidate_name = Some(object);
-            } else if Some(object_shard) != seed_shard && bootstrap_name.is_none() {
-                bootstrap_name = Some(object);
-            }
-            if candidate_name.is_some() && bootstrap_name.is_some() {
-                break;
-            }
-        }
-        let seed_name = seed_name.expect("find a seeded shard");
-        let candidate_name = candidate_name.expect("find a second object in the seeded shard");
-        let bootstrap_name = bootstrap_name.expect("find an empty bootstrap shard");
-
-        let mut seed = PutObjReader::from_vec(vec![1]);
-        store
-            .put_object(&bucket, &seed_name, &mut seed, &ObjectOptions::default())
-            .await
-            .expect("seed existing usage before enabling v2");
-        store
-            .update_bucket_metadata_config(
-                &bucket,
-                "quota.json",
-                serde_json::to_vec(&super::super::BucketQuota::new_sharded(100_000)).expect("serialize v2 quota"),
-            )
-            .await
-            .expect("enable v2 quota");
-
-        let mut bootstrap = PutObjReader::from_vec(vec![2]);
-        store
-            .put_object(&bucket, &bootstrap_name, &mut bootstrap, &ObjectOptions::default())
-            .await
-            .expect("bootstrap an empty shard");
-
-        let mut candidate = PutObjReader::from_vec(vec![3]);
-        let result = store
-            .put_object(&bucket, &candidate_name, &mut candidate, &ObjectOptions::default())
-            .await
-            .expect("a populated shard must persist before refill and accept its first growth");
-        assert_eq!(result.size, 1);
-    }
 }

crates/ecstore/src/cache_value/metacache_set.rs

@@ -25,7 +25,7 @@ use std::{
     future::Future,
     io::ErrorKind,
     pin::Pin,
-    sync::{Arc, OnceLock},
+    sync::{Arc, OnceLock, atomic::AtomicBool},
     task::{Context, Poll},
     time::Duration,
 };
@@ -40,7 +40,7 @@ const LOG_COMPONENT_ECSTORE: &str = "ecstore";
 const LOG_SUBSYSTEM_METACACHE: &str = "metacache";
 const EVENT_METACACHE_LISTING: &str = "metacache_listing";
 
-pub type AgreedFn = Box<dyn Fn(MetaCacheEntry) -> Pin<Box<dyn Future<Output = Result<(), DiskError>> + Send>> + Send + 'static>;
+pub type AgreedFn = Box<dyn Fn(MetaCacheEntry) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
 pub type PartialFn =
     Box<dyn Fn(MetaCacheEntries, &[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
 pub(crate) type PartialResultFn = Box<
@@ -198,8 +198,6 @@ pub(crate) enum TestReaderBehavior {
         stopped: Arc<tokio::sync::Notify>,
     },
     IgnoreCancel,
-    /// Entries followed by a terminator trailer saying the producer stopped at its limit.
-    LimitReached(Vec<MetaCacheEntry>),
     ProducerError(DiskError),
     PrimaryErrorThenFallback(DiskError),
     PartialThenTimeout(Vec<MetaCacheEntry>),
@@ -238,6 +236,8 @@ pub struct ListPathRawOptions {
     pub skip_walkdir_total_timeout: bool,
     pub walkdir_timeout: Option<Duration>,
     pub walkdir_stall_timeout: Option<Duration>,
+    /// Shared terminal state for bounded producer walks.
+    pub producer_limit_reached: Option<Arc<AtomicBool>>,
     pub agreed: Option<AgreedFn>,
     pub partial: Option<PartialFn>,
     pub finished: Option<FinishedFn>,
@@ -248,7 +248,7 @@ pub struct ListPathRawOptions {
     #[cfg(test)]
     pub(crate) peek_timeout: Option<Duration>,
     #[cfg(test)]
-    pub(crate) test_disk_io_context_observed: Option<Arc<std::sync::atomic::AtomicBool>>,
+    pub(crate) test_disk_io_context_observed: Option<Arc<AtomicBool>>,
     // pub agreed: Option<Arc<dyn Fn(MetaCacheEntry) + Send + Sync>>,
     // pub partial: Option<Arc<dyn Fn(MetaCacheEntries, &[Option<Error>]) + Send + Sync>>,
     // pub finished: Option<Arc<dyn Fn(&[Option<Error>]) + Send + Sync>>,
@@ -273,6 +273,7 @@ impl Clone for ListPathRawOptions {
             skip_walkdir_total_timeout: self.skip_walkdir_total_timeout,
             walkdir_timeout: self.walkdir_timeout,
             walkdir_stall_timeout: self.walkdir_stall_timeout,
+            producer_limit_reached: self.producer_limit_reached.clone(),
             #[cfg(test)]
             test_reader_behaviors: self.test_reader_behaviors.clone(),
             #[cfg(test)]
@@ -300,31 +301,22 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
         skip_total_timeout: opts.skip_walkdir_total_timeout,
         timeout_ms: opts.walkdir_timeout.map(duration_millis),
         stall_timeout_ms: opts.walkdir_stall_timeout.map(duration_millis),
+        producer_limit_reached: opts.producer_limit_reached.clone(),
         ..Default::default()
     }
 }
 
-/// How a merged raw listing stream ended.
-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
-pub struct ListPathRawOutcome {
-    /// At least one producer stopped at its per-disk limit instead of at the
-    /// end of its namespace, so the closed stream is not proof of exhaustion.
-    /// Carried in-band by the metacache stream trailer, so remote producers
-    /// report it as well.
-    pub limit_reached: bool,
-}
-
 pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
     let rx = rx.child_token();
     let _cancel_guard = rx.clone().drop_guard();
-    list_path_raw_inner(rx, opts, None, None).await.map(|_| ())
+    list_path_raw_inner(rx, opts, None, None).await
 }
 
 pub(crate) async fn list_path_raw_with_claim_tracker(
     rx: CancellationToken,
     opts: ListPathRawOptions,
     claim_tracker: FallbackClaimTracker,
-) -> disk::error::Result<ListPathRawOutcome> {
+) -> disk::error::Result<()> {
     let rx = rx.child_token();
     let _cancel_guard = rx.clone().drop_guard();
     list_path_raw_inner(rx, opts, Some(claim_tracker), None).await
@@ -335,7 +327,7 @@ pub(crate) async fn list_path_raw_with_partial_result(
     opts: ListPathRawOptions,
     claim_tracker: FallbackClaimTracker,
     partial_result: PartialResultFn,
-) -> disk::error::Result<ListPathRawOutcome> {
+) -> disk::error::Result<()> {
     let rx = rx.child_token();
     let _cancel_guard = rx.clone().drop_guard();
     list_path_raw_inner(rx, opts, Some(claim_tracker), Some(partial_result)).await
@@ -346,7 +338,7 @@ async fn list_path_raw_inner(
     opts: ListPathRawOptions,
     fallback_claim_tracker: Option<FallbackClaimTracker>,
     partial_result: Option<PartialResultFn>,
-) -> disk::error::Result<ListPathRawOutcome> {
+) -> disk::error::Result<()> {
     if opts.disks.is_empty() {
         return Err(DiskError::ErasureReadQuorum);
     }
@@ -401,14 +393,6 @@ async fn list_path_raw_inner(
                         out.close().await.expect("test entries should close");
                         return Ok(());
                     }
-                    TestReaderBehavior::LimitReached(entries) => {
-                        let mut out = rustfs_filemeta::MetacacheWriter::new(&mut wr);
-                        out.write(&entries).await.expect("test entries should be written");
-                        out.close_with_limit_reached(true)
-                            .await
-                            .expect("test entries should close at the limit");
-                        return Ok(());
-                    }
                     TestReaderBehavior::Stall => {
                         let _held_writer = wr;
                         cancel_rx_clone.cancelled().await;
@@ -547,16 +531,6 @@ async fn list_path_raw_inner(
                             last_err = None;
                             continue;
                         }
-                        TestReaderBehavior::LimitReached(entries) => {
-                            let mut out = rustfs_filemeta::MetacacheWriter::new(&mut wr);
-                            out.write(&entries).await.expect("test fallback entries should be written");
-                            out.close_with_limit_reached(true)
-                                .await
-                                .expect("test fallback entries should close at the limit");
-                            need_fallback = false;
-                            last_err = None;
-                            continue;
-                        }
                         TestReaderBehavior::ProducerError(err) | TestReaderBehavior::PrimaryErrorThenFallback(err) => {
                             last_err = Some(err);
                             continue;
@@ -725,9 +699,6 @@ async fn list_path_raw_inner(
         }
         let mut pending_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
         let mut peek_outcomes: Vec<Option<PeekOutcome>> = std::iter::repeat_with(|| None).take(readers.len()).collect();
-        // Sticky across rounds: once any reader's trailer says its producer
-        // stopped at the per-disk limit, the merged stream may hide more keys.
-        let mut limit_reached = false;
 
         loop {
             let mut current = MetaCacheEntry::default();
@@ -739,7 +710,7 @@ async fn list_path_raw_inner(
             // );
 
             if revjob_rx.is_cancelled() {
-                return Ok(ListPathRawOutcome { limit_reached });
+                return Ok(());
             }
 
             let mut top_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
@@ -789,7 +760,6 @@ async fn list_path_raw_inner(
                                     continue;
                                 }
                                 // eof
-                                limit_reached |= r.limit_reached() == Some(true);
                                 at_eof += 1;
                                 // warn!("list_path_raw: peek eof, disk: {}", i);
                                 continue;
@@ -863,7 +833,7 @@ async fn list_path_raw_inner(
                             *r = MetacacheReader::new(detached_rd);
                             continue;
                         }
-                        PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached }),
+                        PeekOutcome::Cancelled => return Ok(()),
                     }
                 };
 
@@ -922,7 +892,7 @@ async fn list_path_raw_inner(
                 if let Some(finished_fn) = opts.finished.as_ref() {
                     tokio::select! {
                         biased;
-                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
+                        _ = revjob_rx.cancelled() => return Ok(()),
                         _ = finished_fn(&errs) => {}
                     }
                 }
@@ -979,7 +949,7 @@ async fn list_path_raw_inner(
                 {
                     tokio::select! {
                         biased;
-                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
+                        _ = revjob_rx.cancelled() => return Ok(()),
                         _ = finished_fn(&errs) => {}
                     }
                 }
@@ -999,8 +969,8 @@ async fn list_path_raw_inner(
                     // warn!("list_path_raw: agreed_fn start, current: {:?}", &current.name);
                     tokio::select! {
                         biased;
-                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
-                        result = agreed_fn(current) => result?
+                        _ = revjob_rx.cancelled() => return Ok(()),
+                        _ = agreed_fn(current) => {}
                     }
                     // warn!("list_path_raw: agreed_fn done");
                 }
@@ -1023,61 +993,58 @@ async fn list_path_raw_inner(
             if let Some(partial_fn) = partial_result.as_ref() {
                 tokio::select! {
                     biased;
-                    _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
+                    _ = revjob_rx.cancelled() => return Ok(()),
                     result = partial_fn(MetaCacheEntries(top_entries), &errs) => result?,
                 }
             } else if let Some(partial_fn) = opts.partial.as_ref() {
                 tokio::select! {
                     biased;
-                    _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
+                    _ = revjob_rx.cancelled() => return Ok(()),
                     _ = partial_fn(MetaCacheEntries(top_entries), &errs) => {}
                 }
             }
         }
-        Ok(ListPathRawOutcome { limit_reached })
+        Ok(())
     });
 
     let merge_started = std::time::Instant::now();
     let Some(revjob) = revjobs.join_next().await else {
         return Err(DiskError::Unexpected);
     };
-    let outcome = match revjob.map_err(std::io::Error::other)? {
-        Ok(outcome) => outcome,
-        Err(err) => {
-            rustfs_io_metrics::record_stage_duration("metacache_merge_failed", merge_started.elapsed().as_secs_f64() * 1000.0);
-            if is_missing_path_error(&err) {
-                debug!(
-                    event = EVENT_METACACHE_LISTING,
-                    component = LOG_COMPONENT_ECSTORE,
-                    subsystem = LOG_SUBSYSTEM_METACACHE,
-                    bucket = %log_bucket,
-                    path = %log_path,
-                    state = "merge_job_missing_path",
-                    error = ?err,
-                    "Metacache merge job missing path skipped"
-                );
-            } else {
-                error!(
-                    event = EVENT_METACACHE_LISTING,
-                    component = LOG_COMPONENT_ECSTORE,
-                    subsystem = LOG_SUBSYSTEM_METACACHE,
-                    bucket = %log_bucket,
-                    path = %log_path,
-                    state = "merge_job_failed",
-                    error = ?err,
-                    "Metacache merge job failed"
-                );
-            }
-            cancel_rx.cancel();
-            jobs.abort_all();
-
-            return Err(err);
+    if let Err(err) = revjob.map_err(std::io::Error::other)? {
+        rustfs_io_metrics::record_stage_duration("metacache_merge_failed", merge_started.elapsed().as_secs_f64() * 1000.0);
+        if is_missing_path_error(&err) {
+            debug!(
+                event = EVENT_METACACHE_LISTING,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_METACACHE,
+                bucket = %log_bucket,
+                path = %log_path,
+                state = "merge_job_missing_path",
+                error = ?err,
+                "Metacache merge job missing path skipped"
+            );
+        } else {
+            error!(
+                event = EVENT_METACACHE_LISTING,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_METACACHE,
+                bucket = %log_bucket,
+                path = %log_path,
+                state = "merge_job_failed",
+                error = ?err,
+                "Metacache merge job failed"
+            );
         }
-    };
+        cancel_rx.cancel();
+        jobs.abort_all();
+
+        return Err(err);
+    }
     if rx.is_cancelled() {
         cancel_rx.cancel();
         jobs.abort_all();
-        return Ok(outcome);
+        return Ok(());
     }
     rustfs_io_metrics::record_stage_duration("metacache_merge", merge_started.elapsed().as_secs_f64() * 1000.0);
 
@@ -1146,7 +1113,7 @@ async fn list_path_raw_inner(
     }
 
     // warn!("list_path_raw: done");
-    Ok(outcome)
+    Ok(())
 }
 
 #[inline]
@@ -1165,7 +1132,6 @@ mod tests {
     use rustfs_filemeta::MetacacheWriter;
     use rustfs_filemeta::{FileInfo, FileMeta, MetadataResolutionParams};
     use std::sync::Mutex;
-    use std::sync::atomic::AtomicBool;
     use time::OffsetDateTime;
     use uuid::Uuid;
 
@@ -1286,61 +1252,6 @@ mod tests {
         );
     }
 
-    #[tokio::test]
-    async fn list_path_raw_reports_limit_reached_from_any_reader_trailer() {
-        let first = fallback_test_entry();
-        let second = MetaCacheEntry {
-            name: "bucket/object-2".to_string(),
-            ..fallback_test_entry()
-        };
-        // One bounded producer, one that ends naturally, one legacy empty
-        // stream without any trailer.
-        let outcome = list_path_raw_inner(
-            CancellationToken::new(),
-            ListPathRawOptions {
-                disks: vec![None, None, None],
-                min_disks: 1,
-                test_reader_behaviors: vec![
-                    TestReaderBehavior::LimitReached(vec![first.clone()]),
-                    TestReaderBehavior::Entries(vec![first, second]),
-                    TestReaderBehavior::Eof,
-                ],
-                ..Default::default()
-            },
-            None,
-            None,
-        )
-        .await
-        .expect("listing should complete");
-
-        assert!(
-            outcome.limit_reached,
-            "a reader whose trailer reports a limit stop must mark the merged stream as possibly incomplete"
-        );
-    }
-
-    #[tokio::test]
-    async fn list_path_raw_reports_natural_end_when_no_reader_trailer_says_limit() {
-        let outcome = list_path_raw_inner(
-            CancellationToken::new(),
-            ListPathRawOptions {
-                disks: vec![None, None],
-                min_disks: 1,
-                test_reader_behaviors: vec![
-                    TestReaderBehavior::Entries(vec![fallback_test_entry()]),
-                    TestReaderBehavior::Eof,
-                ],
-                ..Default::default()
-            },
-            None,
-            None,
-        )
-        .await
-        .expect("listing should complete");
-
-        assert!(!outcome.limit_reached, "natural and legacy stream ends must not claim a limit stop");
-    }
-
     #[tokio::test]
     async fn list_path_raw_returns_typed_quorum_error_for_multiple_drive_failures() {
         let err = list_path_raw(
@@ -1799,34 +1710,6 @@ mod tests {
             .expect("aborting the listing should drop the blocked producer");
     }
 
-    #[tokio::test]
-    async fn list_path_raw_propagates_agreed_callback_failure() {
-        let entry = MetaCacheEntry {
-            name: "bucket/object".to_owned(),
-            metadata: vec![1],
-            ..Default::default()
-        };
-        let result = timeout(
-            Duration::from_secs(1),
-            list_path_raw(
-                CancellationToken::new(),
-                ListPathRawOptions {
-                    disks: vec![None],
-                    min_disks: 1,
-                    test_reader_behaviors: vec![TestReaderBehavior::Entries(vec![entry])],
-                    agreed: Some(Box::new(|_| Box::pin(async { Err(DiskError::ErasureReadQuorum) }))),
-                    ..Default::default()
-                },
-            ),
-        )
-        .await
-        .expect("callback failure must terminate the walk");
-        assert_eq!(
-            result.expect_err("agreed resolution must not silently omit the key"),
-            DiskError::ErasureReadQuorum
-        );
-    }
-
     #[tokio::test]
     async fn list_path_raw_propagates_partial_callback_failure() {
         let first = MetaCacheEntry {
@@ -1928,7 +1811,6 @@ mod tests {
                     Box::pin(async move {
                         callback_started.notify_one();
                         std::future::pending::<()>().await;
-                        Ok(())
                     })
                 })),
                 ..Default::default()
@@ -2080,7 +1962,6 @@ mod tests {
                     let seen = seen_clone.clone();
                     Box::pin(async move {
                         seen.lock().expect("seen mutex poisoned").push(entry.name);
-                        Ok(())
                     })
                 })),
                 ..Default::default()
@@ -2123,7 +2004,6 @@ mod tests {
                     let seen = seen_clone.clone();
                     Box::pin(async move {
                         seen.lock().expect("seen mutex poisoned").push(entry.name);
-                        Ok(())
                     })
                 })),
                 ..Default::default()
@@ -2196,7 +2076,6 @@ mod tests {
                     let seen = agreed_seen.clone();
                     Box::pin(async move {
                         seen.lock().expect("seen mutex poisoned").push(entry.name);
-                        Ok(())
                     })
                 })),
                 partial: Some(Box::new(move |entries, _| {
@@ -2362,7 +2241,6 @@ mod tests {
                     let agreed_events = agreed_events.clone();
                     Box::pin(async move {
                         agreed_events.lock().expect("events lock").push((entry.name, 4usize, true));
-                        Ok(())
                     })
                 })),
                 partial: Some(Box::new(move |entries, _| {
@@ -2519,7 +2397,6 @@ mod tests {
                         let name = entry.name;
                         Box::pin(async move {
                             agreed_obs.lock().expect("obs lock").push((name, vec![Some(desc); 4], true));
-                            Ok(())
                         })
                     })),
                     partial: Some(Box::new(move |entries, _| {

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -94,7 +94,6 @@ const REPLAY_CACHE_AUTO_MEMORY_PERCENT: u64 = 13;
 const REPLAY_CACHE_AUTO_RPC_RPS_PER_CPU: usize = 4096;
 const REPLAY_CACHE_AUTO_MAX_CAPACITY: usize = 33_554_432;
 const NS_SCANNER_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-capability-v3";
-const NS_SCANNER_CHECKPOINT_COVERAGE_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-checkpoint-coverage-v1";
 const NS_SCANNER_TIER_REGISTRY_GENERATION_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-tier-registry-generation-v1";
 pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
 static INTERNODE_RPC_SIGNATURE_STRICT: LazyLock<bool> = LazyLock::new(|| {
@@ -643,7 +642,6 @@ fn update_ns_scanner_capability_mac(
     challenge: Uuid,
     server_epoch: Uuid,
     supports_tier_registry_generation: bool,
-    supports_checkpoint_coverage: bool,
 ) {
     mac.update(NS_SCANNER_CAPABILITY_AUTH_DOMAIN);
     mac.update(&NS_SCANNER_PROTOCOL_VERSION.to_be_bytes());
@@ -655,9 +653,6 @@ fn update_ns_scanner_capability_mac(
         // that receives generation-fenced scanner work.
         mac.update(NS_SCANNER_TIER_REGISTRY_GENERATION_AUTH_DOMAIN);
     }
-    if supports_checkpoint_coverage {
-        mac.update(NS_SCANNER_CHECKPOINT_COVERAGE_AUTH_DOMAIN);
-    }
 }
 
 fn generate_ns_scanner_capability_proof(
@@ -665,20 +660,13 @@ fn generate_ns_scanner_capability_proof(
     challenge: Uuid,
     server_epoch: Uuid,
     supports_tier_registry_generation: bool,
-    supports_checkpoint_coverage: bool,
 ) -> std::io::Result<Vec<u8>> {
     if challenge.is_nil() || server_epoch.is_nil() {
         return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
     }
     let mut mac =
         <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
-    update_ns_scanner_capability_mac(
-        &mut mac,
-        challenge,
-        server_epoch,
-        supports_tier_registry_generation,
-        supports_checkpoint_coverage,
-    );
+    update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch, supports_tier_registry_generation);
     Ok(mac.finalize().into_bytes().to_vec())
 }
 
@@ -688,20 +676,13 @@ fn verify_ns_scanner_capability_proof(
     server_epoch: Uuid,
     proof: &[u8],
     supports_tier_registry_generation: bool,
-    supports_checkpoint_coverage: bool,
 ) -> std::io::Result<()> {
     if challenge.is_nil() || server_epoch.is_nil() {
         return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
     }
     let mut mac =
         <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
-    update_ns_scanner_capability_mac(
-        &mut mac,
-        challenge,
-        server_epoch,
-        supports_tier_registry_generation,
-        supports_checkpoint_coverage,
-    );
+    update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch, supports_tier_registry_generation);
     mac.verify_slice(proof)
         .map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid namespace scanner capability proof"))
 }
@@ -719,22 +700,7 @@ pub fn sign_ns_scanner_capability_with_tier_registry_generation(
     server_epoch: Uuid,
     supports_tier_registry_generation: bool,
 ) -> std::io::Result<Vec<u8>> {
-    sign_ns_scanner_capability_with_checkpoint_coverage(challenge, server_epoch, supports_tier_registry_generation, false)
-}
-
-pub fn sign_ns_scanner_capability_with_checkpoint_coverage(
-    challenge: Uuid,
-    server_epoch: Uuid,
-    supports_tier_registry_generation: bool,
-    supports_checkpoint_coverage: bool,
-) -> std::io::Result<Vec<u8>> {
-    generate_ns_scanner_capability_proof(
-        &get_shared_secret()?,
-        challenge,
-        server_epoch,
-        supports_tier_registry_generation,
-        supports_checkpoint_coverage,
-    )
+    generate_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, supports_tier_registry_generation)
 }
 
 pub fn verify_ns_scanner_capability_with_tier_registry_generation(
@@ -743,30 +709,7 @@ pub fn verify_ns_scanner_capability_with_tier_registry_generation(
     proof: &[u8],
     supports_tier_registry_generation: bool,
 ) -> std::io::Result<()> {
-    verify_ns_scanner_capability_with_checkpoint_coverage(
-        challenge,
-        server_epoch,
-        proof,
-        supports_tier_registry_generation,
-        false,
-    )
-}
-
-pub fn verify_ns_scanner_capability_with_checkpoint_coverage(
-    challenge: Uuid,
-    server_epoch: Uuid,
-    proof: &[u8],
-    supports_tier_registry_generation: bool,
-    supports_checkpoint_coverage: bool,
-) -> std::io::Result<()> {
-    verify_ns_scanner_capability_proof(
-        &get_shared_secret()?,
-        challenge,
-        server_epoch,
-        proof,
-        supports_tier_registry_generation,
-        supports_checkpoint_coverage,
-    )
+    verify_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, proof, supports_tier_registry_generation)
 }
 
 #[derive(Clone, Copy)]
@@ -1833,30 +1776,13 @@ mod tests {
         let secret = "test-scanner-capability-secret";
         let challenge = Uuid::new_v4();
         let server_epoch = Uuid::new_v4();
-        let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false, false)
+        let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false)
             .expect("capability proof should be generated");
 
-        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false, false).is_ok());
-        assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, false, false).is_err());
-        assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof, false, false).is_err());
-        assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof, false, false).is_err());
-    }
-
-    #[test]
-    fn namespace_scanner_capability_proof_binds_checkpoint_coverage_support() {
-        let secret = "checkpoint-capability-test-secret";
-        let challenge = Uuid::new_v4();
-        let server_epoch = Uuid::new_v4();
-        let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true, true)
-            .expect("checkpoint capability proof");
-        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true, true).is_ok());
-        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true, false).is_err());
-        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false, true).is_err());
-        let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true, false)
-            .expect("legacy generation capability proof");
-        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true, true).is_err());
-        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &[], true, true).is_err());
-        assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, true, true).is_err());
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false).is_ok());
+        assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, false).is_err());
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof, false).is_err());
+        assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof, false).is_err());
     }
 
     #[test]
@@ -1864,14 +1790,14 @@ mod tests {
         let secret = "test-scanner-capability-secret";
         let challenge = Uuid::new_v4();
         let server_epoch = Uuid::new_v4();
-        let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true, false)
+        let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true)
             .expect("generation capability proof should be generated");
 
-        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true, false).is_ok());
-        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false, false).is_err());
-        let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false, false)
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true).is_ok());
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false).is_err());
+        let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false)
             .expect("legacy capability proof should be generated");
-        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true, false).is_err());
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true).is_err());
     }
 
     /// Security regression for GHSA-r5qv-rc46-hv8q (internode RPC fail-closed,

crates/ecstore/src/cluster/rpc/internode_data_transport.rs

@@ -13,19 +13,18 @@
 // limitations under the License.
 
 use crate::cluster::rpc::{
-    build_auth_headers, build_put_file_auth_trailer, verify_ns_scanner_capability_with_checkpoint_coverage,
-    verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_capability,
+    build_auth_headers, build_put_file_auth_trailer, verify_ns_scanner_capability_with_tier_registry_generation,
+    verify_put_file_capability,
 };
 use crate::disk::error::{Error, Result};
 use crate::disk::{FileReader, FileWriter};
 use crate::storage_api_contracts::internode::{
-    NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CHECKPOINT_COVERAGE_QUERY,
-    NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY,
-    NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY,
-    NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY, NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_V1,
-    PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY,
-    PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY,
-    WALK_DIR_STREAM_COMPLETION_V1,
+    NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY,
+    NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY,
+    NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY,
+    NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY,
+    PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY,
+    PutFileCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
 };
 use async_trait::async_trait;
 use http::{HeaderMap, HeaderValue, Method, header::CONTENT_TYPE};
@@ -251,7 +250,6 @@ pub struct NsScannerStreamRequest {
 pub struct NsScannerCapabilityRequest {
     pub endpoint: String,
     pub supports_tier_registry_generation: bool,
-    pub supports_checkpoint_coverage: bool,
 }
 
 /// Data-plane stream opener used by `RemoteDisk`.
@@ -291,7 +289,6 @@ pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug {
             server_epoch,
             proof: Vec::new(),
             supports_tier_registry_generation: None,
-            supports_checkpoint_coverage: None,
         })
     }
     // Interface facet nobody calls yet: every transport implements both, but no
@@ -382,18 +379,6 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
     }
 
     async fn probe_ns_scanner_capability(&self, request: NsScannerCapabilityRequest) -> Result<NsScannerCapabilityResponse> {
-        let request = if request.supports_checkpoint_coverage {
-            match self.probe_ns_scanner_capability_once(&request).await {
-                Ok(response) => return Ok(response),
-                Err(error) if ns_scanner_capability_error_allows_legacy(&error) => NsScannerCapabilityRequest {
-                    supports_checkpoint_coverage: false,
-                    ..request
-                },
-                Err(error) => return Err(error),
-            }
-        } else {
-            request
-        };
         if request.supports_tier_registry_generation {
             return match self.probe_ns_scanner_capability_once(&request).await {
                 Ok(response) => Ok(response),
@@ -405,7 +390,6 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
                     let legacy_request = NsScannerCapabilityRequest {
                         endpoint: request.endpoint.clone(),
                         supports_tier_registry_generation: false,
-                        supports_checkpoint_coverage: false,
                     };
                     match self.probe_ns_scanner_capability_once(&legacy_request).await {
                         Ok(mut response) => {
@@ -463,32 +447,18 @@ impl TcpHttpInternodeDataTransport {
         if response.version != NS_SCANNER_PROTOCOL_VERSION || response.server_epoch.is_nil() {
             return Err(Error::other("incompatible remote namespace scanner capability response"));
         }
-        if let Err(err) = verify_ns_scanner_capability_with_checkpoint_coverage(
+        if let Err(err) = verify_ns_scanner_capability_with_tier_registry_generation(
             challenge,
             response.server_epoch,
             &response.proof,
             request.supports_tier_registry_generation,
-            request.supports_checkpoint_coverage,
         ) {
-            if request.supports_checkpoint_coverage
-                && verify_ns_scanner_capability_with_tier_registry_generation(
-                    challenge,
-                    response.server_epoch,
-                    &response.proof,
-                    request.supports_tier_registry_generation,
-                )
-                .is_ok()
-            {
-                normalize_ns_scanner_capability_response(&mut response, request.supports_tier_registry_generation, false);
-                return Ok(response);
-            }
             // A permissive older peer can ignore the additive marker and
             // return a valid legacy-scope proof with HTTP 200. Accept that
             // response only after independently authenticating the legacy
             // scope; all other verification failures remain fail-closed.
             if request.supports_tier_registry_generation && ns_scanner_capability_legacy_proof_is_valid(challenge, &response) {
                 response.supports_tier_registry_generation = None;
-                response.supports_checkpoint_coverage = None;
                 return Ok(response);
             }
             return Err(Error::other(format!("remote namespace scanner capability authentication failed: {err}")));
@@ -498,11 +468,7 @@ impl TcpHttpInternodeDataTransport {
         // verified scope so an intermediary cannot strip or rewrite the field
         // and force a silent downgrade after a successful generation-bound
         // handshake.
-        normalize_ns_scanner_capability_response(
-            &mut response,
-            request.supports_tier_registry_generation,
-            request.supports_checkpoint_coverage,
-        );
+        normalize_ns_scanner_capability_response(&mut response, request.supports_tier_registry_generation);
         Ok(response)
     }
 
@@ -832,13 +798,8 @@ fn build_walk_dir_url(request: &WalkDirStreamRequest) -> String {
     )
 }
 
-fn normalize_ns_scanner_capability_response(
-    response: &mut NsScannerCapabilityResponse,
-    requested_generation_support: bool,
-    requested_checkpoint_coverage: bool,
-) {
+fn normalize_ns_scanner_capability_response(response: &mut NsScannerCapabilityResponse, requested_generation_support: bool) {
     response.supports_tier_registry_generation = requested_generation_support.then_some(true);
-    response.supports_checkpoint_coverage = requested_checkpoint_coverage.then_some(true);
 }
 
 fn ns_scanner_capability_legacy_proof_is_valid(challenge: Uuid, response: &NsScannerCapabilityResponse) -> bool {
@@ -871,7 +832,7 @@ fn build_ns_scanner_url(request: &NsScannerStreamRequest) -> String {
 
 fn build_ns_scanner_capability_url(request: &NsScannerCapabilityRequest, challenge: Uuid) -> String {
     format!(
-        "{}{}?{}={}&{}={}{}{}",
+        "{}{}?{}={}&{}={}{}",
         request.endpoint,
         NS_SCANNER_PATH,
         NS_SCANNER_PROTOCOL_VERSION_QUERY,
@@ -882,11 +843,6 @@ fn build_ns_scanner_capability_url(request: &NsScannerCapabilityRequest, challen
             format!("&{}=true", NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY)
         } else {
             String::new()
-        },
-        if request.supports_checkpoint_coverage {
-            format!("&{}=true", NS_SCANNER_CHECKPOINT_COVERAGE_QUERY)
-        } else {
-            String::new()
         }
     )
 }
@@ -1045,7 +1001,6 @@ mod tests {
             .probe_ns_scanner(NsScannerCapabilityRequest {
                 endpoint: "http://node1:9000".to_string(),
                 supports_tier_registry_generation: false,
-                supports_checkpoint_coverage: false,
             })
             .await
             .expect_err("legacy transport should report namespace scanner as unsupported");
@@ -1072,97 +1027,6 @@ mod tests {
         assert!(matches!(open_err, Error::MethodNotAllowed));
     }
 
-    #[tokio::test]
-    async fn namespace_scanner_checkpoint_capability_authenticates_new_and_legacy_peers() {
-        crate::runtime::sources::ensure_test_rpc_secret();
-        for peer in ["current", "strict-old", "permissive-old", "invalid-proof", "missing-proof"] {
-            let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
-                .await
-                .expect("capability listener");
-            let address = listener.local_addr().expect("capability listener address");
-            let server_epoch = Uuid::new_v4();
-            let server = tokio::spawn(async move {
-                let requests = if peer == "strict-old" { 2 } else { 1 };
-                for index in 0..requests {
-                    let (mut socket, _) = listener.accept().await.expect("capability request");
-                    let mut bytes = Vec::new();
-                    while !bytes.windows(4).any(|window| window == b"\r\n\r\n") {
-                        let mut chunk = [0; 1024];
-                        let read = socket.read(&mut chunk).await.expect("capability headers");
-                        assert!(read > 0 && bytes.len() < 16 * 1024, "bounded complete capability headers");
-                        bytes.extend_from_slice(&chunk[..read]);
-                    }
-                    let request = std::str::from_utf8(&bytes).expect("HTTP headers");
-                    let target = request.split_whitespace().nth(1).expect("HTTP request target");
-                    let url = reqwest::Url::parse(&format!("http://{address}{target}")).expect("capability URL");
-                    let query: HashMap<_, _> = url.query_pairs().into_owned().collect();
-                    assert_eq!(query.get(NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY).map(String::as_str), Some("true"));
-                    assert_eq!(query.contains_key(NS_SCANNER_CHECKPOINT_COVERAGE_QUERY), index == 0);
-                    if peer == "strict-old" && index == 0 {
-                        socket
-                            .write_all(b"HTTP/1.1 400 Bad Request\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")
-                            .await
-                            .expect("legacy parser rejection");
-                        continue;
-                    }
-                    let challenge = query[NS_SCANNER_CAPABILITY_CHALLENGE_QUERY].parse().expect("challenge UUID");
-                    let proof = match peer {
-                        "invalid-proof" => vec![0; 32],
-                        "missing-proof" => Vec::new(),
-                        _ => crate::cluster::rpc::sign_ns_scanner_capability_with_checkpoint_coverage(
-                            challenge,
-                            server_epoch,
-                            true,
-                            peer == "current",
-                        )
-                        .expect("server capability proof"),
-                    };
-                    let body = rmp_serde::to_vec_named(&NsScannerCapabilityResponse {
-                        version: NS_SCANNER_PROTOCOL_VERSION,
-                        server_epoch,
-                        proof,
-                        supports_tier_registry_generation: Some(true),
-                        // Strip a supported bit, or forge it for an older peer.
-                        supports_checkpoint_coverage: (peer != "current").then_some(true),
-                    })
-                    .expect("capability response");
-                    socket
-                        .write_all(
-                            format!("HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n", body.len()).as_bytes(),
-                        )
-                        .await
-                        .expect("capability response headers");
-                    socket.write_all(&body).await.expect("capability response body");
-                }
-                requests
-            });
-            let response = tokio::time::timeout(
-                Duration::from_secs(10),
-                TcpHttpInternodeDataTransport.probe_ns_scanner_capability(NsScannerCapabilityRequest {
-                    endpoint: format!("http://{address}"),
-                    supports_tier_registry_generation: true,
-                    supports_checkpoint_coverage: true,
-                }),
-            )
-            .await
-            .expect("bounded capability probe");
-            if peer.ends_with("proof") {
-                assert!(
-                    response
-                        .expect_err("unverified capability must fail closed")
-                        .to_string()
-                        .contains("authentication failed")
-                );
-            } else {
-                let response = response.expect("authenticated capability");
-                assert_eq!(response.server_epoch, server_epoch);
-                assert_eq!(response.supports_tier_registry_generation, Some(true));
-                assert_eq!(response.supports_checkpoint_coverage, (peer == "current").then_some(true));
-            }
-            assert_eq!(server.await.expect("capability server"), if peer == "strict-old" { 2 } else { 1 });
-        }
-    }
-
     #[tokio::test]
     async fn walk_dir_restores_explicit_remote_missing_error() {
         let _ = rustfs_credentials::set_global_rpc_secret("walk-dir-error-test-secret".to_string());
@@ -1905,7 +1769,6 @@ mod tests {
             &NsScannerCapabilityRequest {
                 endpoint: "http://node1:9000".to_string(),
                 supports_tier_registry_generation: false,
-                supports_checkpoint_coverage: false,
             },
             challenge,
         );
@@ -1925,7 +1788,6 @@ mod tests {
             &NsScannerCapabilityRequest {
                 endpoint: "http://node1:9000".to_string(),
                 supports_tier_registry_generation: true,
-                supports_checkpoint_coverage: false,
             },
             challenge,
         );
@@ -1964,14 +1826,13 @@ mod tests {
             server_epoch: Uuid::new_v4(),
             proof: Vec::new(),
             supports_tier_registry_generation: None,
-            supports_checkpoint_coverage: None,
         };
 
-        normalize_ns_scanner_capability_response(&mut response, true, true);
+        normalize_ns_scanner_capability_response(&mut response, true);
         assert_eq!(response.supports_tier_registry_generation, Some(true));
 
         response.supports_tier_registry_generation = Some(false);
-        normalize_ns_scanner_capability_response(&mut response, false, false);
+        normalize_ns_scanner_capability_response(&mut response, false);
         assert_eq!(response.supports_tier_registry_generation, None);
     }
 
@@ -1985,7 +1846,6 @@ mod tests {
             proof: crate::cluster::rpc::sign_ns_scanner_capability(challenge, Uuid::new_v4())
                 .expect("placeholder proof should be generated"),
             supports_tier_registry_generation: None,
-            supports_checkpoint_coverage: None,
         };
         // A proof bound to a different challenge cannot authorize the legacy
         // fallback, even though the response has the expected shape.

crates/ecstore/src/cluster/rpc/mod.rs

@@ -36,13 +36,12 @@ pub use http_auth::{
     TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, gen_signature_headers,
     gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
     set_tonic_mutation_body_digest, set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest,
-    sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
-    sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
-    tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability,
-    verify_ns_scanner_capability_with_checkpoint_coverage, verify_ns_scanner_capability_with_tier_registry_generation,
-    verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
-    verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned,
-    verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
+    sign_ns_scanner_capability, sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability,
+    sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
+    verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
+    verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
+    verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
+    verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
 };
 #[cfg(test)]
 pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -631,17 +631,6 @@ fn decode_cross_pool_fence_capability(expected_member: &str, result: &[u8]) -> R
     Ok((version, epoch))
 }
 
-fn decode_mrf_lifecycle_reader_capability(expected_member: &str, result: &[u8]) -> Result<(u8, Uuid)> {
-    let (&maximum_format_version, identity) = result
-        .split_first()
-        .ok_or_else(|| Error::other("peer returned a truncated MRF lifecycle reader capability"))?;
-    if maximum_format_version == 0 || expected_member.trim().is_empty() {
-        return Err(Error::other("peer returned an invalid MRF lifecycle reader capability"));
-    }
-    let epoch = decode_remote_version_state_capability(expected_member, identity)?;
-    Ok((maximum_format_version, epoch))
-}
-
 #[derive(Clone, Debug)]
 pub struct PeerLiveEventsBatch {
     pub events: Vec<u8>,
@@ -1975,16 +1964,6 @@ impl PeerRestClient {
         Ok((self.topology_member.clone(), supported_version, epoch))
     }
 
-    /// Reports one peer's reader support without authorizing a lifecycle writer or rollback.
-    pub async fn probe_mrf_lifecycle_reader(&self, topology_fingerprint: String) -> Result<(String, u8, Uuid)> {
-        let probe = rustfs_protos::mrf_lifecycle_reader_capability_probe(Uuid::new_v4().as_bytes());
-        let result = self
-            .heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
-            .await?;
-        let (maximum_format_version, epoch) = decode_mrf_lifecycle_reader_capability(&self.topology_member, &result)?;
-        Ok((self.topology_member.clone(), maximum_format_version, epoch))
-    }
-
     pub async fn probe_ilm_recovery_export(&self, topology_fingerprint: String) -> Result<(String, Uuid)> {
         let probe = rustfs_protos::ilm_recovery_export_capability_probe(Uuid::new_v4().as_bytes());
         let result = self
@@ -4354,49 +4333,6 @@ mod tests {
         }
     }
 
-    #[test]
-    fn mrf_lifecycle_reader_capability_decoder_preserves_the_advertised_version() {
-        let epoch = Uuid::new_v4();
-        for version in [1, 2, u8::MAX] {
-            let result = rustfs_protos::encode_mrf_lifecycle_reader_capability(version, "node-a:9000", epoch.as_bytes())
-                .expect("capability frame");
-            assert_eq!(
-                decode_mrf_lifecycle_reader_capability("node-a:9000", &result).expect("reported capability"),
-                (version, epoch)
-            );
-        }
-    }
-
-    #[test]
-    fn mrf_lifecycle_reader_capability_decoder_rejects_unbound_or_malformed_identity() {
-        let epoch = Uuid::new_v4();
-        let valid =
-            rustfs_protos::encode_mrf_lifecycle_reader_capability(2, "node-a:9000", epoch.as_bytes()).expect("capability frame");
-        let mut invalid = vec![Vec::new(), vec![2], valid[..valid.len() - 1].to_vec()];
-        let mut zero = valid.clone();
-        zero[0] = 0;
-        invalid.push(zero);
-        let mut trailing = valid.clone();
-        trailing.push(0);
-        invalid.push(trailing);
-        let mut length_overflow = valid.clone();
-        length_overflow[1..9].copy_from_slice(&u64::MAX.to_be_bytes());
-        invalid.push(length_overflow);
-        let mut invalid_utf8 = valid.clone();
-        invalid_utf8[9] = 0xff;
-        invalid.push(invalid_utf8);
-        invalid.push(
-            rustfs_protos::encode_mrf_lifecycle_reader_capability(2, "node-a:9000", Uuid::nil().as_bytes())
-                .expect("nil epoch fixture"),
-        );
-        for result in invalid {
-            assert!(decode_mrf_lifecycle_reader_capability("node-a:9000", &result).is_err());
-        }
-        assert!(decode_mrf_lifecycle_reader_capability("node-b:9000", &valid).is_err());
-        let empty = rustfs_protos::encode_mrf_lifecycle_reader_capability(2, "", epoch.as_bytes()).expect("empty member fixture");
-        assert!(decode_mrf_lifecycle_reader_capability("", &empty).is_err());
-    }
-
     #[test]
     fn remote_version_state_capability_decoder_fails_closed() {
         let epoch = Uuid::new_v4();

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -1143,19 +1143,12 @@ impl RemoteDisk {
         let probe = self.data_transport.probe_ns_scanner_capability(NsScannerCapabilityRequest {
             endpoint: self.endpoint.grid_host(),
             supports_tier_registry_generation: true,
-            supports_checkpoint_coverage: true,
         });
         let result = timeout(NS_SCANNER_CAPABILITY_PROBE_TIMEOUT, probe)
             .await
             .map_err(|_| DiskError::other("remote namespace scanner capability probe timed out"))?;
         match result {
-            // RUSTFS_COMPAT_TODO(ns-scanner-checkpoint-coverage): old strict readers cannot accept the coverage field. Remove after every supported peer authenticates both scanner capabilities.
-            Ok(response)
-                if response.supports_tier_registry_generation == Some(true)
-                    && response.supports_checkpoint_coverage == Some(true) =>
-            {
-                Ok(Some(response.server_epoch))
-            }
+            Ok(response) if response.supports_tier_registry_generation == Some(true) => Ok(Some(response.server_epoch)),
             Ok(_) => Ok(None),
             // RUSTFS_COMPAT_TODO(ns-scanner-rpc-v3): old peers and legacy transports lack the authenticated startup-epoch handshake. Remove after every supported peer implements namespace scanner protocol v3.
             Err(DiskError::MethodNotAllowed) => Ok(None),
@@ -2375,300 +2368,7 @@ fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
     file_info.validate_for_metadata_read().map_err(Into::into)
 }
 
-fn validate_artifact_repair_seal_response(
-    request: &rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsRequest,
-    response: &rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsResponse,
-) -> Result<()> {
-    let canonical = rustfs_protos::canonical_seal_artifact_attempt_repairs_response_body(request, response)
-        .map_err(|_| Error::other("artifact repair seal response is too large"))?;
-    crate::cluster::rpc::verify_tonic_rpc_response_proof(&canonical, &response.response_proof)
-        .map_err(|_| Error::other("artifact repair seal response proof is invalid"))?;
-    if !response.success {
-        return Err(response.error.clone().unwrap_or_default().into());
-    }
-    let expected_disk = Uuid::from_slice(&request.expected_disk_id).ok().filter(|id| !id.is_nil());
-    if response.error.is_some() || expected_disk.is_none() || Uuid::from_slice(&response.disk_id).ok() != expected_disk {
-        return Err(Error::other("artifact repair seal response disk identity mismatch"));
-    }
-    Ok(())
-}
-
 impl RemoteDisk {
-    pub(crate) async fn rename_artifact_repair_data(
-        &self,
-        expected_disk: Uuid,
-        source: (&str, &str),
-        fi: &FileInfo,
-        destination: (&str, &str),
-        expected_incarnation: Uuid,
-    ) -> Result<RenameDataResp> {
-        use rustfs_protos::proto_gen::node_service::ArtifactRepairRenameDataRequest;
-        self.execute_with_timeout_for_op(
-            "artifact_repair_rename_data",
-            || async {
-                let body = ArtifactRepairRenameDataRequest {
-                    rename: Some(RenameDataRequest {
-                        disk: self.endpoint.to_string(),
-                        src_volume: source.0.to_owned(),
-                        src_path: source.1.to_owned(),
-                        dst_volume: destination.0.to_owned(),
-                        dst_path: destination.1.to_owned(),
-                        file_info: compat_json(fi)?,
-                        file_info_bin: encode_file_info_msgpack(fi)?.into(),
-                        scanner_publication_lease_token: Bytes::new(),
-                        bucket_incarnation_id: expected_incarnation.as_bytes().to_vec().into(),
-                    }),
-                    expected_disk_id: expected_disk.as_bytes().to_vec().into(),
-                };
-                let canonical = rustfs_protos::canonical_artifact_repair_rename_data_request_body(&body)
-                    .map_err(|_| Error::other("artifact repair rename request is too large"))?;
-                let mut request = Request::new(body);
-                crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
-                // Unimplemented is terminal: never downgrade repair publication to PUT or legacy RPCs.
-                let response = self
-                    .get_client()
-                    .await?
-                    .artifact_repair_rename_data(request)
-                    .await?
-                    .into_inner();
-                if !response.success {
-                    return Err(response.error.unwrap_or_default().into());
-                }
-                decode_msgpack_or_json::<RenameDataResp>(
-                    &response.rename_data_resp_bin,
-                    &response.rename_data_resp,
-                    "RenameDataResp",
-                )
-            },
-            get_max_timeout_duration(),
-        )
-        .await
-    }
-
-    pub(crate) async fn write_artifact_repair_metadata(
-        &self,
-        expected_disk: Uuid,
-        target: (&str, &str),
-        fi: &FileInfo,
-        expected_incarnation: Uuid,
-    ) -> Result<()> {
-        use rustfs_protos::proto_gen::node_service::ArtifactRepairWriteMetadataRequest;
-        self.execute_with_timeout_for_op(
-            "artifact_repair_write_metadata",
-            || async {
-                let body = ArtifactRepairWriteMetadataRequest {
-                    metadata: Some(WriteMetadataRequest {
-                        disk: self.endpoint.to_string(),
-                        volume: target.0.to_owned(),
-                        path: target.1.to_owned(),
-                        file_info: compat_json(fi)?,
-                        file_info_bin: encode_file_info_msgpack(fi)?.into(),
-                        bucket_incarnation_id: expected_incarnation.as_bytes().to_vec().into(),
-                    }),
-                    expected_disk_id: expected_disk.as_bytes().to_vec().into(),
-                };
-                let canonical = rustfs_protos::canonical_artifact_repair_write_metadata_request_body(&body)
-                    .map_err(|_| Error::other("artifact repair metadata request is too large"))?;
-                let mut request = Request::new(body);
-                crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
-                let response = self
-                    .get_client()
-                    .await?
-                    .artifact_repair_write_metadata(request)
-                    .await?
-                    .into_inner();
-                if !response.success {
-                    return Err(response.error.unwrap_or_default().into());
-                }
-                Ok(())
-            },
-            get_max_timeout_duration(),
-        )
-        .await
-    }
-
-    /// One authenticated scan (or explicitly distinct read-only identity probe).
-    /// Partial observations are never retried after output has begun.
-    pub async fn inventory_artifact_attempt(
-        &self,
-        expected_disk: Uuid,
-        target: (&str, Uuid),
-        prefix: &str,
-        phase: rustfs_protos::proto_gen::node_service::ArtifactInventoryPhase,
-        deadline: tokio::time::Instant,
-        output: tokio::sync::mpsc::Sender<crate::disk::ArtifactInventoryFrame>,
-    ) -> Result<()> {
-        use crate::disk::{ArtifactInventoryFrame, ArtifactInventoryVersion};
-        use rustfs_protos::{
-            artifact_inventory as protocol,
-            proto_gen::node_service::{ArtifactInventoryRequest, artifact_inventory_frame::Payload},
-        };
-        let (bucket, incarnation) = target;
-        let duration = deadline
-            .checked_duration_since(tokio::time::Instant::now())
-            .ok_or(Error::Timeout)?;
-        let body = ArtifactInventoryRequest {
-            protocol_version: protocol::VERSION,
-            invocation_id: Uuid::new_v4().as_bytes().to_vec().into(),
-            disk: self.endpoint.to_string(),
-            expected_disk_id: expected_disk.as_bytes().to_vec().into(),
-            bucket: bucket.to_owned(),
-            bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
-            attempt_prefix: prefix.to_owned(),
-            timeout_millis: u64::try_from(duration.as_millis().min(u128::from(protocol::MAX_TIMEOUT_MILLIS)))
-                .map_err(Error::other)?,
-            phase: phase.into(),
-        };
-        protocol::validate_request(&body).map_err(Error::other)?;
-        tokio::time::timeout_at(deadline, async {
-            let canonical = protocol::canonical_request(&body).map_err(Error::other)?;
-            let mut request = Request::new(body.clone());
-            request.set_timeout(duration);
-            crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
-            let mut stream = self
-                .get_client()
-                .await?
-                .max_decoding_message_size(protocol::MAX_FRAME_BYTES)
-                .inventory_artifact_attempt(request)
-                .await?
-                .into_inner();
-            let mut state = protocol::StreamState::new();
-            let mut terminal = None;
-            while let Some(frame) = stream.message().await? {
-                state
-                    .accept(&body, &frame, |canonical, proof| {
-                        crate::cluster::rpc::verify_tonic_rpc_response_proof(canonical, proof).is_ok()
-                    })
-                    .map_err(Error::other)?;
-                match frame.payload {
-                    Some(Payload::Entries(entries)) => {
-                        let versions = entries
-                            .versions
-                            .into_iter()
-                            .map(|entry| {
-                                Ok(ArtifactInventoryVersion {
-                                    key: entry.key,
-                                    version_id: Uuid::from_slice(&entry.version_id).map_err(Error::other)?,
-                                    delete_marker: entry.delete_marker,
-                                })
-                            })
-                            .collect::<Result<Vec<_>>>()?;
-                        output
-                            .send(ArtifactInventoryFrame::Entries {
-                                sequence: frame.sequence,
-                                versions,
-                            })
-                            .await
-                            .map_err(|_| Error::other("artifact inventory consumer closed"))?;
-                    }
-                    Some(Payload::Complete(complete)) => terminal = Some(complete.entries),
-                    Some(Payload::Verified(_)) => {}
-                    None => return Err(Error::other("artifact inventory payload is absent")),
-                }
-            }
-            // A valid Complete followed by a transport failure or extra frame is
-            // still incomplete. Publish the local terminal only after clean EOF.
-            state.finish().map_err(Error::other)?;
-            if let Some(entries) = terminal {
-                output
-                    .send(ArtifactInventoryFrame::Complete {
-                        disk_id: expected_disk,
-                        incarnation,
-                        attempt_prefix: prefix.to_owned(),
-                        entries,
-                    })
-                    .await
-                    .map_err(|_| Error::other("artifact inventory consumer closed"))?;
-            }
-            Ok(())
-        })
-        .await
-        .map_err(|_| Error::Timeout)?
-    }
-
-    pub(crate) async fn seal_artifact_puts(
-        &self,
-        expected_disk: Uuid,
-        bucket: &str,
-        incarnation: Uuid,
-        attempt_prefix: &str,
-    ) -> Result<()> {
-        use rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsRequest;
-        self.execute_with_timeout_for_op(
-            "seal_artifact_puts",
-            || async {
-                let body = SealArtifactAttemptPutsRequest {
-                    disk: self.endpoint.to_string(),
-                    bucket: bucket.to_owned(),
-                    bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
-                    attempt_prefix: attempt_prefix.to_owned(),
-                    expected_disk_id: expected_disk.as_bytes().to_vec().into(),
-                };
-                let canonical = rustfs_protos::canonical_seal_artifact_attempt_puts_request_body(&body)
-                    .map_err(|_| Error::other("artifact seal request is too large"))?;
-                let mut request = Request::new(body.clone());
-                crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
-                let response = self
-                    .get_client()
-                    .await?
-                    .seal_artifact_attempt_puts(request)
-                    .await?
-                    .into_inner();
-                let canonical = rustfs_protos::canonical_seal_artifact_attempt_puts_response_body(&body, &response)
-                    .map_err(|_| Error::other("artifact seal response is too large"))?;
-                crate::cluster::rpc::verify_tonic_rpc_response_proof(&canonical, &response.response_proof)
-                    .map_err(|_| Error::other("artifact seal response proof is invalid"))?;
-                if !response.success {
-                    return Err(response.error.unwrap_or_default().into());
-                }
-                if response.error.is_some() || Uuid::from_slice(&response.disk_id).ok() != Some(expected_disk) {
-                    return Err(Error::other("artifact seal response disk identity mismatch"));
-                }
-                Ok(())
-            },
-            get_max_timeout_duration(),
-        )
-        .await
-    }
-
-    pub(crate) async fn seal_artifact_repairs(
-        &self,
-        expected_disk: Uuid,
-        bucket: &str,
-        incarnation: Uuid,
-        attempt_prefix: &str,
-    ) -> Result<()> {
-        use rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsRequest;
-        if expected_disk.is_nil() || incarnation.is_nil() {
-            return Err(Error::other("artifact repair seal requires non-nil disk and incarnation identities"));
-        }
-        self.execute_with_timeout_for_op(
-            "seal_artifact_repairs",
-            || async {
-                let body = SealArtifactAttemptPutsRequest {
-                    disk: self.endpoint.to_string(),
-                    bucket: bucket.to_owned(),
-                    bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
-                    attempt_prefix: attempt_prefix.to_owned(),
-                    expected_disk_id: expected_disk.as_bytes().to_vec().into(),
-                };
-                let canonical = rustfs_protos::canonical_seal_artifact_attempt_repairs_request_body(&body)
-                    .map_err(|_| Error::other("artifact repair seal request is too large"))?;
-                let mut request = Request::new(body.clone());
-                crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
-                let response = self
-                    .get_client()
-                    .await?
-                    .seal_artifact_attempt_repairs(request)
-                    .await?
-                    .into_inner();
-                validate_artifact_repair_seal_response(&body, &response)
-            },
-            get_max_timeout_duration(),
-        )
-        .await
-    }
-
     #[tracing::instrument(level = "trace", skip_all)]
     pub(crate) async fn rename_data_borrowed(
         &self,
@@ -2678,39 +2378,20 @@ impl RemoteDisk {
         dst_volume: &str,
         dst_path: &str,
     ) -> Result<RenameDataResp> {
-        self.rename_data_borrowed_with_fence((src_volume, src_path), fi, (dst_volume, dst_path), None, None)
+        self.rename_data_borrowed_with_fence(src_volume, src_path, fi, dst_volume, dst_path, None)
             .await
     }
 
     #[tracing::instrument(level = "trace", skip_all)]
     pub(crate) async fn rename_data_borrowed_with_fence(
         &self,
-        source: (&str, &str),
+        src_volume: &str,
+        src_path: &str,
         fi: &FileInfo,
-        target: (&str, &str),
+        dst_volume: &str,
+        dst_path: &str,
         scanner_publication_lease_token: Option<Uuid>,
-        artifact_incarnation: Option<Uuid>,
     ) -> Result<RenameDataResp> {
-        let (src_volume, src_path) = source;
-        let (dst_volume, dst_path) = target;
-        if let Some(scope) = crate::bucket::artifact::artifact_repair_scope(dst_volume, dst_path)
-            && let Some(identity) = scope.receiver_identity()
-        {
-            scope.check().map_err(Error::other)?;
-            if artifact_incarnation.is_some() || scanner_publication_lease_token.is_some() {
-                return Err(Error::other("artifact repair cannot use a publication lease"));
-            }
-            let expected = self
-                .get_disk_id()
-                .await?
-                .filter(|id| !id.is_nil())
-                .ok_or_else(|| Error::other("artifact repair disk identity is unavailable"))?;
-            let result = self
-                .rename_artifact_repair_data(expected, (src_volume, src_path), fi, (dst_volume, dst_path), identity.incarnation)
-                .await;
-            scope.check().map_err(Error::other)?;
-            return result;
-        }
         trace!(
             event = EVENT_REMOTE_DISK_RPC,
             component = LOG_COMPONENT_ECSTORE,
@@ -2733,11 +2414,8 @@ impl RemoteDisk {
                 let file_info_bin = encode_file_info_msgpack(fi)?;
                 let mut client = self.get_client().await?;
                 let mut request = Request::new(RenameDataRequest {
-                    bucket_incarnation_id: artifact_incarnation
-                        .or_else(|| {
-                            crate::store::bucket_heal_scope_for_object(dst_volume, dst_path).map(|scope| scope.incarnation)
-                        })
-                        .map(|incarnation| incarnation.as_bytes().to_vec().into())
+                    bucket_incarnation_id: crate::store::bucket_heal_scope_for_object(dst_volume, dst_path)
+                        .map(|scope| scope.incarnation.as_bytes().to_vec().into())
                         .unwrap_or_default(),
                     disk: self.endpoint.to_string(),
                     src_volume: src_volume.to_string(),
@@ -2750,11 +2428,7 @@ impl RemoteDisk {
                         .map(|token| token.as_bytes().to_vec().into())
                         .unwrap_or_default(),
                 });
-                let canonical_body = if artifact_incarnation.is_some() {
-                    rustfs_protos::canonical_artifact_rename_data_request_body(request.get_ref())
-                } else {
-                    rustfs_protos::canonical_rename_data_request_body(request.get_ref())
-                };
+                let canonical_body = rustfs_protos::canonical_rename_data_request_body(request.get_ref());
                 let incarnation_bound = !request.get_ref().bucket_incarnation_id.is_empty();
                 let quota_fenced = rustfs_utils::http::metadata_compat::contains_key_str(
                     &fi.metadata,
@@ -2777,9 +2451,7 @@ impl RemoteDisk {
                 let rpc_started = rustfs_io_metrics::put_stage_timer();
                 let response = rustfs_io_metrics::observe_put_stage_future(
                     async {
-                        if artifact_incarnation.is_some() {
-                            client.artifact_rename_data(request).await
-                        } else if quota_fenced {
+                        if quota_fenced {
                             // The method itself proves quota-token enforcement; an old peer must fail closed.
                             client.rename_data_with_quota_fence(request).await
                         } else if incarnation_bound {
@@ -3525,21 +3197,6 @@ impl DiskAPI for RemoteDisk {
 
     #[tracing::instrument(level = "trace", skip_all)]
     async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
-        if let Some(scope) = crate::bucket::artifact::artifact_repair_scope(volume, path)
-            && let Some(identity) = scope.receiver_identity()
-        {
-            scope.check().map_err(Error::other)?;
-            let expected = self
-                .get_disk_id()
-                .await?
-                .filter(|id| !id.is_nil())
-                .ok_or_else(|| Error::other("artifact repair disk identity is unavailable"))?;
-            let result = self
-                .write_artifact_repair_metadata(expected, (volume, path), &fi, identity.incarnation)
-                .await;
-            scope.check().map_err(Error::other)?;
-            return result;
-        }
         trace!(
             event = EVENT_REMOTE_DISK_RPC,
             component = LOG_COMPONENT_ECSTORE,
@@ -4889,83 +4546,6 @@ mod tests {
 
     static INIT: Once = Once::new();
 
-    #[tokio::test]
-    async fn artifact_repair_old_peer_unimplemented_never_downgrades_mutations() {
-        runtime_sources::ensure_test_rpc_secret();
-        let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
-            .await
-            .expect("loopback test peer");
-        let endpoint = Endpoint {
-            url: url::Url::parse(&format!("{}/data/rustfs0", peer.addr)).expect("test disk endpoint"),
-            is_local: false,
-            pool_idx: 0,
-            set_idx: 0,
-            disk_idx: 0,
-        };
-        let disk = RemoteDisk::new(
-            &endpoint,
-            &DiskOption {
-                cleanup: false,
-                health_check: false,
-            },
-            Arc::new(TcpHttpInternodeDataTransport),
-        )
-        .await
-        .expect("remote test disk");
-        let expected_disk = Uuid::new_v4();
-        let incarnation = Uuid::new_v4();
-        let info = FileInfo::default();
-        let rename = disk
-            .rename_artifact_repair_data(
-                expected_disk,
-                ("tmp", "staged"),
-                &info,
-                ("artifacts", "outputs/tenant/run/attempt/image"),
-                incarnation,
-            )
-            .await
-            .expect_err("old peer cannot publish repair data");
-        let metadata = disk
-            .write_artifact_repair_metadata(expected_disk, ("artifacts", "outputs/tenant/run/attempt/image"), &info, incarnation)
-            .await
-            .expect_err("old peer cannot publish repair metadata");
-        let seal = disk
-            .seal_artifact_repairs(expected_disk, "artifacts", incarnation, "outputs/tenant/run/attempt")
-            .await
-            .expect_err("old peer cannot confirm repair closure");
-        for error in [rename, metadata, seal] {
-            let DiskError::Io(error) = error else {
-                panic!("expected gRPC status error");
-            };
-            let status = error
-                .get_ref()
-                .and_then(|error| error.downcast_ref::<crate::disk::error::RpcStatusError>())
-                .expect("preserved gRPC status");
-            assert_eq!(status.status().code(), Code::Unimplemented);
-        }
-        let paths = peer.peer.rpc_paths.lock().expect("RPC inventory").clone();
-        for method in [
-            "ArtifactRepairRenameData",
-            "ArtifactRepairWriteMetadata",
-            "SealArtifactAttemptRepairs",
-        ] {
-            assert_eq!(
-                paths.iter().filter(|path| path.ends_with(&format!("/{method}"))).count(),
-                1,
-                "each repair method is sent once"
-            );
-        }
-        assert!(
-            paths.iter().all(|path| matches!(
-                path.rsplit('/').next(),
-                Some("ArtifactRepairRenameData" | "ArtifactRepairWriteMetadata" | "SealArtifactAttemptRepairs" | "Ping")
-            )),
-            "repair cannot downgrade to a legacy mutation: {paths:?}"
-        );
-        disk.cancel_token.cancel();
-        peer.stop().await;
-    }
-
     #[tokio::test]
     async fn quota_fence_old_peer_rejects_without_legacy_fallback() {
         runtime_sources::ensure_test_rpc_secret();
@@ -5061,93 +4641,6 @@ mod tests {
         peer.stop().await;
     }
 
-    #[test]
-    fn artifact_repair_seal_response_requires_proof_full_request_and_physical_identity() {
-        use rustfs_protos::proto_gen::node_service::{
-            Error as WireError, SealArtifactAttemptPutsRequest, SealArtifactAttemptPutsResponse,
-        };
-        runtime_sources::ensure_test_rpc_secret();
-        let request = SealArtifactAttemptPutsRequest {
-            disk: "disk-a".into(),
-            bucket: "artifacts".into(),
-            bucket_incarnation_id: vec![1; 16].into(),
-            attempt_prefix: "outputs/tenant/run/attempt".into(),
-            expected_disk_id: vec![2; 16].into(),
-        };
-        let mut response = SealArtifactAttemptPutsResponse {
-            success: true,
-            disk_id: request.expected_disk_id.clone(),
-            ..Default::default()
-        };
-        let sign = |request: &SealArtifactAttemptPutsRequest, response: &mut SealArtifactAttemptPutsResponse| {
-            let canonical = rustfs_protos::canonical_seal_artifact_attempt_repairs_response_body(request, response)
-                .expect("repair response canonical");
-            response.response_proof = crate::cluster::rpc::http_auth::sign_tonic_rpc_response_proof(&canonical)
-                .expect("repair response proof")
-                .into();
-        };
-        assert!(validate_artifact_repair_seal_response(&request, &response).is_err(), "proof is mandatory");
-        sign(&request, &mut response);
-        validate_artifact_repair_seal_response(&request, &response).expect("signed exact physical receiver");
-        for mutate in [
-            |r: &mut SealArtifactAttemptPutsRequest| r.disk = "other-disk".into(),
-            |r: &mut SealArtifactAttemptPutsRequest| r.bucket = "other-bucket".into(),
-            |r: &mut SealArtifactAttemptPutsRequest| r.bucket_incarnation_id = vec![3; 16].into(),
-            |r: &mut SealArtifactAttemptPutsRequest| r.attempt_prefix = "other-attempt".into(),
-            |r: &mut SealArtifactAttemptPutsRequest| r.expected_disk_id = vec![4; 16].into(),
-        ] {
-            let mut changed = request.clone();
-            mutate(&mut changed);
-            assert!(
-                validate_artifact_repair_seal_response(&changed, &response).is_err(),
-                "another request cannot reuse the proof"
-            );
-        }
-        let legacy = rustfs_protos::canonical_seal_artifact_attempt_puts_response_body(&request, &response).expect("PUT domain");
-        let mut legacy_response = response.clone();
-        legacy_response.response_proof = crate::cluster::rpc::http_auth::sign_tonic_rpc_response_proof(&legacy)
-            .expect("PUT proof")
-            .into();
-        assert!(
-            validate_artifact_repair_seal_response(&request, &legacy_response).is_err(),
-            "PUT proof cannot acknowledge repair seal"
-        );
-        for disk in [vec![], vec![0; 16], vec![3; 16], vec![1; 15]] {
-            let mut changed = response.clone();
-            changed.disk_id = disk.into();
-            sign(&request, &mut changed);
-            assert!(
-                validate_artifact_repair_seal_response(&request, &changed).is_err(),
-                "signed wrong or absent physical identity is invalid"
-            );
-        }
-        for mutate in [
-            |r: &mut SealArtifactAttemptPutsResponse| r.success = false,
-            |r: &mut SealArtifactAttemptPutsResponse| r.error = Some(WireError::default()),
-        ] {
-            let mut changed = response.clone();
-            mutate(&mut changed);
-            assert!(
-                validate_artifact_repair_seal_response(&request, &changed).is_err(),
-                "response fields are bound"
-            );
-            sign(&request, &mut changed);
-            assert!(
-                validate_artifact_repair_seal_response(&request, &changed).is_err(),
-                "signed failure or inconsistent success is invalid"
-            );
-        }
-        let mut nil_request = request;
-        nil_request.expected_disk_id = vec![0; 16].into();
-        let mut nil_response = response;
-        nil_response.disk_id = nil_request.expected_disk_id.clone();
-        sign(&nil_request, &mut nil_response);
-        assert!(
-            validate_artifact_repair_seal_response(&nil_request, &nil_response).is_err(),
-            "nil physical identity is never a receipt"
-        );
-    }
-
     #[test]
     fn recoverable_batch_delete_rejects_invalid_error_codes_and_length() {
         let errors = decode_delete_recovery_batch_errors(vec![None, Some(DiskError::FileNotFound.to_u32())], 2)
@@ -5515,7 +5008,6 @@ mod tests {
         calls: Arc<StdMutex<Vec<RecordedTransportCall>>>,
         ns_scanner_probe_status: Arc<StdMutex<Option<u16>>>,
         ns_scanner_generation_support: Arc<StdMutex<Option<bool>>>,
-        ns_scanner_checkpoint_support: Arc<StdMutex<Option<bool>>>,
     }
 
     impl Default for RecordingInternodeDataTransport {
@@ -5524,7 +5016,6 @@ mod tests {
                 calls: Arc::default(),
                 ns_scanner_probe_status: Arc::default(),
                 ns_scanner_generation_support: Arc::new(StdMutex::new(Some(true))),
-                ns_scanner_checkpoint_support: Arc::new(StdMutex::new(Some(true))),
             }
         }
     }
@@ -5538,7 +5029,6 @@ mod tests {
         format_data: Bytes,
         read_all_data: Bytes,
         next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
-        rpc_paths: Arc<StdMutex<Vec<String>>>,
         unsupported_methods: Arc<StdMutex<Vec<String>>>,
     }
 
@@ -5558,7 +5048,6 @@ mod tests {
                 format_data,
                 read_all_data,
                 next_format_read_gate: Arc::default(),
-                rpc_paths: Arc::default(),
                 unsupported_methods: Arc::default(),
             }
         }
@@ -5614,11 +5103,6 @@ mod tests {
         }
 
         fn call(&mut self, request: http::Request<B>) -> Self::Future {
-            self.peer
-                .rpc_paths
-                .lock()
-                .expect("RPC path inventory")
-                .push(request.uri().path().to_owned());
             match request.uri().path() {
                 "/node_service.NodeService/DiskInfo" => {
                     #[derive(Clone)]
@@ -5782,7 +5266,6 @@ mod tests {
                 calls: Arc::default(),
                 ns_scanner_probe_status: Arc::new(StdMutex::new(Some(status))),
                 ns_scanner_generation_support: Arc::new(StdMutex::new(Some(true))),
-                ns_scanner_checkpoint_support: Arc::new(StdMutex::new(Some(true))),
             }
         }
 
@@ -5791,7 +5274,6 @@ mod tests {
                 calls: Arc::default(),
                 ns_scanner_probe_status: Arc::default(),
                 ns_scanner_generation_support: Arc::new(StdMutex::new(support)),
-                ns_scanner_checkpoint_support: Arc::new(StdMutex::new(Some(true))),
             }
         }
 
@@ -6492,7 +5974,6 @@ mod tests {
                 server_epoch,
                 proof: Vec::new(),
                 supports_tier_registry_generation,
-                supports_checkpoint_coverage: *self.ns_scanner_checkpoint_support.lock().expect("checkpoint capability lock"),
             })
         }
 
@@ -9588,28 +9069,6 @@ mod tests {
         }
     }
 
-    #[tokio::test]
-    async fn test_remote_disk_namespace_scanner_falls_back_until_checkpoint_support_is_authenticated() {
-        for support in [None, Some(false)] {
-            let transport = RecordingInternodeDataTransport::default();
-            *transport
-                .ns_scanner_checkpoint_support
-                .lock()
-                .expect("checkpoint capability lock") = support;
-            let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
-            assert_eq!(remote_disk.ns_scanner_server_epoch().await.expect("old peer fallback"), None);
-            assert!(matches!(transport.calls().as_slice(), [RecordedTransportCall::NsScannerProbe(_)]));
-            *transport
-                .ns_scanner_checkpoint_support
-                .lock()
-                .expect("checkpoint capability lock") = Some(true);
-            assert_eq!(
-                remote_disk.ns_scanner_server_epoch().await.expect("upgraded peer"),
-                Some(Uuid::from_u128(1))
-            );
-        }
-    }
-
     #[tokio::test]
     async fn test_remote_disk_namespace_scanner_capability_falls_back_without_generation_support() {
         for support in [None, Some(false)] {

crates/ecstore/src/config/com.rs

@@ -438,64 +438,6 @@ where
     Ok(data)
 }
 
-pub(crate) async fn read_config_observation<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<(Vec<u8>, ObjectInfo)>
-where
-    S: EcstoreObjectIO,
-{
-    let opts = ObjectOptions {
-        suppress_read_repair: true,
-        no_lock: false,
-        skip_verify_bitrot: false,
-        ..Default::default()
-    };
-    crate::object_api::without_get_object_body_cache_hook(read_config_limited_preserve_empty_with_metadata_opts(
-        api, file, &opts, max_bytes,
-    ))
-    .await
-}
-
-pub(crate) async fn read_config_observation_no_lock<S>(
-    api: Arc<S>,
-    file: &str,
-    max_bytes: usize,
-    guard: &crate::object_api::WriteCommitGuard,
-) -> Result<(Vec<u8>, ObjectInfo)>
-where
-    S: EcstoreObjectIO,
-{
-    if !guard.matches(RUSTFS_META_BUCKET, file) {
-        return Err(Error::InvalidArgument(
-            RUSTFS_META_BUCKET.to_owned(),
-            file.to_owned(),
-            "locked observation requires a matching write owner".to_owned(),
-        ));
-    }
-    let lock_lost_error = || Error::NamespaceLockQuorumUnavailable {
-        mode: "locked_config_observation",
-        bucket: RUSTFS_META_BUCKET.to_owned(),
-        object: file.to_owned(),
-        required: 1,
-        achieved: 0,
-    };
-    if guard.is_lock_lost() {
-        return Err(lock_lost_error());
-    }
-    let opts = ObjectOptions {
-        suppress_read_repair: true,
-        no_lock: true,
-        skip_verify_bitrot: false,
-        ..Default::default()
-    };
-    let result = crate::object_api::without_get_object_body_cache_hook(read_config_limited_preserve_empty_with_metadata_opts(
-        api, file, &opts, max_bytes,
-    ))
-    .await;
-    if guard.is_lock_lost() {
-        return Err(lock_lost_error());
-    }
-    result
-}
-
 pub(crate) async fn read_config_limited_preserve_empty<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
 where
     S: EcstoreObjectIO,
@@ -542,22 +484,10 @@ pub(crate) async fn read_config_preserve_empty<S>(api: Arc<S>, file: &str) -> Re
 where
     S: EcstoreObjectIO,
 {
-    let (data, _obj) = read_config_preserve_empty_with_metadata_opts(api, file, &ObjectOptions::default()).await?;
+    let (data, _obj) = read_config_with_metadata_inner(api, file, &ObjectOptions::default(), true, None).await?;
     Ok(data)
 }
 
-/// Preserve an empty config payload, its metadata, and caller-supplied read options.
-pub(crate) async fn read_config_preserve_empty_with_metadata_opts<S>(
-    api: Arc<S>,
-    file: &str,
-    opts: &ObjectOptions,
-) -> Result<(Vec<u8>, ObjectInfo)>
-where
-    S: EcstoreObjectIO,
-{
-    read_config_with_metadata_inner(api, file, opts, true, None).await
-}
-
 pub async fn read_config_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
 where
     S: ObjectIO<
@@ -586,13 +516,15 @@ pub(crate) async fn read_config_no_lock_preserve_empty_with_metadata<S>(api: Arc
 where
     S: EcstoreObjectIO,
 {
-    read_config_preserve_empty_with_metadata_opts(
+    read_config_with_metadata_inner(
         api,
         file,
         &ObjectOptions {
             no_lock: true,
             ..Default::default()
         },
+        true,
+        None,
     )
     .await
 }
@@ -2987,9 +2919,9 @@ mod tests {
         config_task_join_error, configs_semantically_equal, decode_server_config_blob, encode_server_config_blob,
         heal_config_descriptor, is_standard_object_server_config, lookup_configs, map_system_metadata_write_error,
         new_and_save_server_config, read_config, read_config_no_lock_preserve_empty_with_metadata, read_config_preserve_empty,
-        read_config_preserve_empty_with_metadata_opts, read_config_with_metadata, read_config_without_migrate,
-        read_server_config_snapshot, save_config_with_opts_inner, save_server_config, save_server_config_snapshot,
-        save_server_config_snapshot_with_generation, server_config_transaction_lock_path, storage_class_kvs_mut,
+        read_config_with_metadata, read_config_without_migrate, read_server_config_snapshot, save_config_with_opts_inner,
+        save_server_config, save_server_config_snapshot, save_server_config_snapshot_with_generation,
+        server_config_transaction_lock_path, storage_class_kvs_mut,
     };
     use crate::config::{audit, heal, notify, oidc, scanner};
     use crate::disk::{RUSTFS_META_BUCKET, endpoint::Endpoint};
@@ -5464,7 +5396,6 @@ mod tests {
         heal_replacement: Option<Vec<u8>>,
         heal_calls: AtomicUsize,
         write_calls: AtomicUsize,
-        last_get_no_lock: AtomicBool,
         last_put_no_lock: AtomicBool,
         last_put_preconditions: Mutex<Option<HTTPPreconditions>>,
         revision: AtomicUsize,
@@ -5480,7 +5411,6 @@ mod tests {
                 heal_replacement,
                 heal_calls: AtomicUsize::new(0),
                 write_calls: AtomicUsize::new(0),
-                last_get_no_lock: AtomicBool::new(false),
                 last_put_no_lock: AtomicBool::new(false),
                 last_put_preconditions: Mutex::new(None),
                 revision: AtomicUsize::new(1),
@@ -5536,9 +5466,8 @@ mod tests {
             _object: &str,
             _range: Option<HTTPRangeSpec>,
             _h: HeaderMap,
-            opts: &ObjectOptions,
+            _opts: &ObjectOptions,
         ) -> Result<GetObjectReader> {
-            self.last_get_no_lock.store(opts.no_lock, Ordering::SeqCst);
             let data = match &*self.state.lock().expect("state lock poisoned") {
                 RecoveryReadState::Missing => return Err(Error::ConfigNotFound),
                 RecoveryReadState::Blob(data) => data.clone(),
@@ -5850,325 +5779,6 @@ mod tests {
         assert_eq!(store.write_calls.load(Ordering::SeqCst), 0);
     }
 
-    type ObservationEofGate = (Option<tokio::sync::oneshot::Sender<()>>, tokio::sync::oneshot::Receiver<()>);
-
-    struct ObservationStream {
-        inner: Cursor<Vec<u8>>,
-        consumed: Arc<AtomicUsize>,
-        fail: bool,
-        eof_gate: Option<ObservationEofGate>,
-    }
-
-    impl AsyncRead for ObservationStream {
-        fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
-            if self.fail {
-                return Poll::Ready(Err(std::io::Error::other("observation stream failed")));
-            }
-            if self.inner.position() == u64::try_from(self.inner.get_ref().len()).expect("fixture length")
-                && let Some((ready, release)) = self.eof_gate.as_mut()
-            {
-                if let Some(ready) = ready.take() {
-                    ready.send(()).expect("EOF observer remains active");
-                }
-                match std::future::Future::poll(Pin::new(release), cx) {
-                    Poll::Pending => return Poll::Pending,
-                    Poll::Ready(result) => result.expect("EOF gate released"),
-                }
-                self.eof_gate = None;
-            }
-            let before = buf.filled().len();
-            let result = Pin::new(&mut self.inner).poll_read(cx, buf);
-            self.consumed.fetch_add(buf.filled().len() - before, Ordering::SeqCst);
-            result
-        }
-    }
-
-    struct ObservationStore {
-        body: Vec<u8>,
-        declared_size: i64,
-        error: Option<Error>,
-        stream_error: bool,
-        gets: AtomicUsize,
-        eof_gate: std::sync::Mutex<Option<ObservationEofGate>>,
-        consumed: Arc<AtomicUsize>,
-        options: std::sync::Mutex<Option<(bool, bool, bool)>>,
-        writes: AtomicUsize,
-    }
-
-    impl Debug for ObservationStore {
-        fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-            f.debug_struct("ObservationStore").finish()
-        }
-    }
-
-    impl ObservationStore {
-        fn new(body: Vec<u8>, declared_size: i64) -> Self {
-            Self {
-                body,
-                declared_size,
-                error: None,
-                stream_error: false,
-                gets: AtomicUsize::new(0),
-                eof_gate: std::sync::Mutex::new(None),
-                consumed: Arc::new(AtomicUsize::new(0)),
-                options: std::sync::Mutex::new(None),
-                writes: AtomicUsize::new(0),
-            }
-        }
-    }
-
-    #[async_trait::async_trait]
-    impl crate::storage_api_contracts::object::ObjectIO for ObservationStore {
-        type Error = Error;
-        type RangeSpec = HTTPRangeSpec;
-        type HeaderMap = HeaderMap;
-        type ObjectOptions = ObjectOptions;
-        type ObjectInfo = ObjectInfo;
-        type GetObjectReader = GetObjectReader;
-        type PutObjectReader = PutObjReader;
-
-        async fn get_object_reader(
-            &self,
-            bucket: &str,
-            object: &str,
-            range: Option<HTTPRangeSpec>,
-            _headers: HeaderMap,
-            opts: &ObjectOptions,
-        ) -> Result<GetObjectReader> {
-            assert_eq!(bucket, RUSTFS_META_BUCKET);
-            assert_eq!(object, "config/observation.bin");
-            self.gets.fetch_add(1, Ordering::SeqCst);
-            assert!(range.is_none());
-            *self.options.lock().expect("observation options") =
-                Some((opts.suppress_read_repair, opts.no_lock, opts.skip_verify_bitrot));
-            if let Some(error) = &self.error {
-                return Err(error.clone());
-            }
-            Ok(GetObjectReader {
-                stream: Box::new(ObservationStream {
-                    inner: Cursor::new(self.body.clone()),
-                    consumed: self.consumed.clone(),
-                    fail: self.stream_error,
-                    eof_gate: self.eof_gate.lock().expect("EOF gate").take(),
-                }),
-                object_info: ObjectInfo {
-                    size: self.declared_size,
-                    etag: Some("observation-etag".to_string()),
-                    ..Default::default()
-                },
-                buffered_body: None,
-                body_source: Default::default(),
-            })
-        }
-
-        async fn put_object(
-            &self,
-            _bucket: &str,
-            _object: &str,
-            _data: &mut PutObjReader,
-            _opts: &ObjectOptions,
-        ) -> Result<ObjectInfo> {
-            self.writes.fetch_add(1, Ordering::SeqCst);
-            Err(Error::other("observation must not write"))
-        }
-    }
-
-    #[tokio::test]
-    async fn locked_observation_preserves_options_and_typed_errors() {
-        let object = "config/observation.bin";
-        let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
-        let wrapper = rustfs_lock::NamespaceLockWrapper::new(
-            rustfs_lock::NamespaceLock::with_local_manager("locked-observation-options".to_owned(), manager.clone()),
-            rustfs_lock::ObjectKey::new(RUSTFS_META_BUCKET, object),
-            "locked-observation-options".to_owned(),
-        );
-        let guard = crate::object_api::WriteCommitGuard::acquire(&wrapper, std::time::Duration::from_secs(5))
-            .await
-            .expect("matching real owner");
-        let store = Arc::new(ObservationStore::new(b"raw".to_vec(), 3));
-        let result = super::read_config_observation_no_lock(store.clone(), object, 3, &guard).await;
-        assert!(result.is_ok(), "I06L_READ_FLAGS: {result:?}");
-        assert!(*store.options.lock().expect("options") == Some((true, true, false)), "I06L_READ_FLAGS");
-        assert_eq!(result.expect("raw observation").0, b"raw");
-        assert_eq!(store.writes.load(Ordering::SeqCst), 0);
-        for (bucket, key) in [(RUSTFS_META_BUCKET, "config/other.bin"), ("wrong-bucket", object)] {
-            let wrapper = rustfs_lock::NamespaceLockWrapper::new(
-                rustfs_lock::NamespaceLock::with_local_manager("locked-observation-target".to_owned(), manager.clone()),
-                rustfs_lock::ObjectKey::new(bucket, key),
-                "locked-observation-target".to_owned(),
-            );
-            let wrong = crate::object_api::WriteCommitGuard::acquire(&wrapper, std::time::Duration::from_secs(5))
-                .await
-                .expect("wrong-target real owner");
-            let store = Arc::new(ObservationStore::new(Vec::new(), 0));
-            let result = super::read_config_observation_no_lock(store.clone(), object, 8, &wrong).await;
-            assert!(matches!(result, Err(Error::InvalidArgument(..))), "I06L_TARGET_REJECTED");
-            assert!(store.gets.load(Ordering::SeqCst) == 0, "I06L_TARGET_REJECTED");
-        }
-        for error in [
-            Error::FileNotFound,
-            Error::ObjectNotFound(RUSTFS_META_BUCKET.to_owned(), object.to_owned()),
-            Error::ErasureReadQuorum,
-        ] {
-            let expected = if error == Error::FileNotFound || matches!(error, Error::ObjectNotFound(..)) {
-                Error::ConfigNotFound
-            } else {
-                error.clone()
-            };
-            let mut fixture = ObservationStore::new(Vec::new(), 0);
-            fixture.error = Some(error);
-            let store = Arc::new(fixture);
-            assert_eq!(
-                super::read_config_observation_no_lock(store.clone(), object, 8, &guard)
-                    .await
-                    .expect_err("typed error"),
-                expected
-            );
-            assert_eq!(store.writes.load(Ordering::SeqCst), 0);
-        }
-        let mut fixture = ObservationStore::new(Vec::new(), 0);
-        fixture.stream_error = true;
-        let store = Arc::new(fixture);
-        let err = super::read_config_observation_no_lock(store.clone(), object, 8, &guard)
-            .await
-            .expect_err("stream failure");
-        assert!(err.to_string().contains("observation stream failed"), "I06L_TYPED_STREAM_ERROR");
-        assert_eq!(store.writes.load(Ordering::SeqCst), 0);
-        let store = Arc::new(ObservationStore::new(vec![7; 64], 0));
-        let result = super::read_config_observation_no_lock(store.clone(), object, 8, &guard).await;
-        assert!(matches!(result, Err(Error::CorruptedFormat)), "I06L_STREAM_BOUND");
-        assert!(store.consumed.load(Ordering::SeqCst) == 9, "I06L_STREAM_BOUND");
-        assert_eq!(store.writes.load(Ordering::SeqCst), 0);
-    }
-
-    #[tokio::test]
-    async fn locked_observation_rejects_actual_owner_loss_before_and_after_read() {
-        let object = "config/observation.bin";
-        for after_read in [false, true] {
-            let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
-            let client = Arc::new(RefreshControlledClient::new(manager));
-            let wrapper = rustfs_lock::NamespaceLockWrapper::new(
-                rustfs_lock::NamespaceLock::new("locked-observation-loss".to_owned(), client.clone()),
-                rustfs_lock::ObjectKey::new(RUSTFS_META_BUCKET, object),
-                "locked-observation-loss".to_owned(),
-            );
-            let guard = crate::object_api::WriteCommitGuard::acquire(&wrapper, std::time::Duration::from_secs(5))
-                .await
-                .expect("real distributed owner");
-            let signal = guard.lock_lost_signal().expect("acquired distributed owner signal");
-            let store = Arc::new(ObservationStore::new(b"raw".to_vec(), 3));
-            if after_read {
-                let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
-                let (release_tx, release_rx) = tokio::sync::oneshot::channel();
-                *store.eof_gate.lock().expect("EOF gate") = Some((Some(ready_tx), release_rx));
-                let mut read = Box::pin(super::read_config_observation_no_lock(store.clone(), object, 3, &guard));
-                tokio::time::timeout(std::time::Duration::from_secs(60), async {
-                    tokio::select! {
-                        ready = ready_rx => ready.expect("read reached EOF"),
-                        result = &mut read => assert_eq!(false, true, "I06L_LOSS_AFTER_READ: returned before EOF gate: {result:?}"),
-                    }
-                }).await.expect("EOF watchdog");
-                client.reject_refreshes();
-                tokio::time::timeout(std::time::Duration::from_secs(60), signal.notified())
-                    .await
-                    .expect("actual owner loss watchdog");
-                assert!(guard.is_lock_lost(), "I06L_LOSS_AFTER_READ");
-                release_tx.send(()).expect("release EOF");
-                let result = tokio::time::timeout(std::time::Duration::from_secs(60), read)
-                    .await
-                    .expect("read watchdog");
-                assert!(
-                    matches!(
-                        result,
-                        Err(Error::NamespaceLockQuorumUnavailable {
-                            mode: "locked_config_observation",
-                            ..
-                        })
-                    ),
-                    "I06L_LOSS_AFTER_READ"
-                );
-                assert_eq!(store.gets.load(Ordering::SeqCst), 1);
-            } else {
-                client.reject_refreshes();
-                tokio::time::timeout(std::time::Duration::from_secs(60), signal.notified())
-                    .await
-                    .expect("actual owner loss watchdog");
-                assert!(guard.is_lock_lost(), "I06L_LOSS_BEFORE_GET");
-                let result = super::read_config_observation_no_lock(store.clone(), object, 3, &guard).await;
-                assert!(
-                    matches!(
-                        result,
-                        Err(Error::NamespaceLockQuorumUnavailable {
-                            mode: "locked_config_observation",
-                            ..
-                        })
-                    ),
-                    "I06L_LOSS_BEFORE_GET"
-                );
-                assert!(store.gets.load(Ordering::SeqCst) == 0, "I06L_LOSS_BEFORE_GET");
-            }
-            assert_eq!(store.writes.load(Ordering::SeqCst), 0);
-        }
-    }
-
-    #[tokio::test]
-    async fn observation_preserves_options_and_errors() {
-        for body in [Vec::new(), b"raw config".to_vec()] {
-            let store = Arc::new(ObservationStore::new(body.clone(), i64::try_from(body.len()).expect("fixture size")));
-            let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
-            assert!(result.is_ok(), "observation must preserve empty and nonempty objects: {result:?}");
-            let (observed, info) = result.expect("successful observation");
-            assert_eq!(observed, body);
-            assert_eq!(info.etag.as_deref(), Some("observation-etag"));
-            assert_eq!(
-                *store.options.lock().expect("options"),
-                Some((true, false, false)),
-                "I05_REPAIR_SUPPRESSED"
-            );
-            assert_eq!(store.writes.load(Ordering::SeqCst), 0);
-        }
-        for (source, expected) in [
-            (Error::FileNotFound, Error::ConfigNotFound),
-            (
-                Error::ObjectNotFound(RUSTFS_META_BUCKET.to_string(), "config/observation.bin".to_string()),
-                Error::ConfigNotFound,
-            ),
-            (Error::ErasureReadQuorum, Error::ErasureReadQuorum),
-        ] {
-            let mut fixture = ObservationStore::new(Vec::new(), 0);
-            fixture.error = Some(source);
-            let store = Arc::new(fixture);
-            let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
-            assert_eq!(result.expect_err("read error must propagate"), expected);
-            assert_eq!(store.writes.load(Ordering::SeqCst), 0);
-        }
-        let mut fixture = ObservationStore::new(Vec::new(), 0);
-        fixture.stream_error = true;
-        let store = Arc::new(fixture);
-        let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
-        let error = result.expect_err("stream error must propagate");
-        assert!(error.to_string().contains("observation stream failed"));
-        assert_eq!(store.writes.load(Ordering::SeqCst), 0);
-    }
-
-    #[tokio::test]
-    async fn observation_enforces_byte_limit() {
-        let store = Arc::new(ObservationStore::new(vec![7; 64], 0));
-        let result = super::read_config_observation(store.clone(), "config/observation.bin", 8).await;
-        assert!(result.is_err(), "I05_BOUND_ENFORCED");
-        assert_eq!(
-            result.expect_err("bounded observation error"),
-            Error::CorruptedFormat,
-            "I05_BOUND_ENFORCED"
-        );
-        assert_eq!(store.consumed.load(Ordering::SeqCst), 9, "I05_BOUND_ENFORCED: drain exactly limit + 1");
-        let store = Arc::new(ObservationStore::new(vec![7; 8], 8));
-        let result = super::read_config_observation(store.clone(), "config/observation.bin", 8).await;
-        assert!(result.is_ok(), "exact byte limit must succeed: {result:?}");
-        assert_eq!(result.expect("exact limit").0, vec![7; 8]);
-        assert_eq!(store.consumed.load(Ordering::SeqCst), 8);
-    }
-
     #[tokio::test]
     async fn read_config_preserve_empty_distinguishes_empty_object() {
         let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(Vec::new()), None));
@@ -6189,60 +5799,6 @@ mod tests {
         assert!(data.is_empty());
     }
 
-    #[tokio::test]
-    async fn read_config_preserve_empty_metadata_keeps_empty_and_options() {
-        let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(Vec::new()), None));
-        for no_lock in [false, true] {
-            let opts = ObjectOptions {
-                no_lock,
-                ..Default::default()
-            };
-            let (data, info) = read_config_preserve_empty_with_metadata_opts(store.clone(), "config/empty.json", &opts)
-                .await
-                .expect("an existing empty config must retain its payload and metadata");
-            assert!(data.is_empty());
-            assert_eq!(info.size, 0);
-            assert_eq!(info.actual_size, 0);
-            assert_eq!(info.etag.as_deref(), Some("config-1"));
-            assert_eq!(info.data_dir, Some(uuid::Uuid::from_u128(1)));
-            assert_eq!(store.last_get_no_lock.load(Ordering::SeqCst), no_lock);
-        }
-        let err = read_config(store, "config/empty.json")
-            .await
-            .expect_err("the existing reader must still treat empty configs as missing");
-        assert!(matches!(err, Error::ConfigNotFound));
-    }
-
-    #[tokio::test]
-    async fn read_config_preserve_empty_metadata_keeps_payload_and_metadata() {
-        let payload = b"original config bytes".to_vec();
-        let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(payload.clone()), None));
-        let (data, info) =
-            read_config_preserve_empty_with_metadata_opts(store, "config/nonempty.json", &ObjectOptions::default())
-                .await
-                .expect("a nonempty config must retain its payload and metadata");
-        assert_eq!(data, payload);
-        let size = i64::try_from(payload.len()).expect("test payload size fits i64");
-        assert_eq!(info.size, size);
-        assert_eq!(info.actual_size, size);
-        assert_eq!(info.etag.as_deref(), Some("config-1"));
-        assert_eq!(info.data_dir, Some(uuid::Uuid::from_u128(1)));
-    }
-
-    #[tokio::test]
-    async fn read_config_preserve_empty_metadata_preserves_read_errors() {
-        let missing = Arc::new(RecoveryMockStore::new(RecoveryReadState::Missing, None));
-        let err = read_config_preserve_empty_with_metadata_opts(missing, "config/missing.json", &ObjectOptions::default())
-            .await
-            .expect_err("physical absence must remain ConfigNotFound");
-        assert!(matches!(err, Error::ConfigNotFound));
-        let quorum = Arc::new(RecoveryMockStore::new(RecoveryReadState::QuorumError, None));
-        let err = read_config_preserve_empty_with_metadata_opts(quorum, "config/quorum.json", &ObjectOptions::default())
-            .await
-            .expect_err("a quorum failure must retain its typed read error");
-        assert!(matches!(err, Error::ErasureReadQuorum));
-    }
-
     #[async_trait::async_trait]
     impl StorageAdminApi for RecoveryMockStore {
         type BackendInfo = rustfs_madmin::BackendInfo;

crates/ecstore/src/config/scanner.rs

@@ -17,13 +17,12 @@ use rustfs_config::{
     DEFAULT_SCANNER_BITROT_CYCLE_SECS, DEFAULT_SCANNER_CACHE_SAVE_TIMEOUT_SECS, DEFAULT_SCANNER_CYCLE_MAX_DIRECTORIES,
     DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS, DEFAULT_SCANNER_CYCLE_MAX_OBJECTS, DEFAULT_SCANNER_DISK_IO_BUDGET,
     DEFAULT_SCANNER_IDLE_MODE, DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS, DEFAULT_SCANNER_MAX_CONCURRENT_SET_SCANS,
-    DEFAULT_SCANNER_RECOVERY_BACKOFF_CAP_SECS, DEFAULT_SCANNER_SPEED, DEFAULT_SCANNER_WALKERS_PER_DISK,
-    DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS, SCANNER_ALERT_EXCESS_FOLDERS, SCANNER_ALERT_EXCESS_VERSION_SIZE,
-    SCANNER_ALERT_EXCESS_VERSIONS, SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE,
-    SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY,
+    DEFAULT_SCANNER_SPEED, DEFAULT_SCANNER_WALKERS_PER_DISK, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS, SCANNER_ALERT_EXCESS_FOLDERS,
+    SCANNER_ALERT_EXCESS_VERSION_SIZE, SCANNER_ALERT_EXCESS_VERSIONS, SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT,
+    SCANNER_CYCLE, SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY,
     SCANNER_DISK_IO_BUDGET_BYTES_PER_SEC, SCANNER_DISK_IO_BUDGET_IOPS, SCANNER_IDLE_MODE, SCANNER_MAX_CONCURRENT_DISK_SCANS,
-    SCANNER_MAX_CONCURRENT_SET_SCANS, SCANNER_MAX_WAIT, SCANNER_RECOVERY_BACKOFF_CAP, SCANNER_SPEED, SCANNER_START_DELAY,
-    SCANNER_WALKERS_PER_DISK, SCANNER_YIELD_EVERY_N_OBJECTS,
+    SCANNER_MAX_CONCURRENT_SET_SCANS, SCANNER_MAX_WAIT, SCANNER_SPEED, SCANNER_START_DELAY, SCANNER_WALKERS_PER_DISK,
+    SCANNER_YIELD_EVERY_N_OBJECTS,
 };
 use std::sync::LazyLock;
 
@@ -84,11 +83,6 @@ pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
             value: DEFAULT_SCANNER_CACHE_SAVE_TIMEOUT_SECS.to_string(),
             hidden_if_empty: false,
         },
-        KV {
-            key: SCANNER_RECOVERY_BACKOFF_CAP.to_owned(),
-            value: DEFAULT_SCANNER_RECOVERY_BACKOFF_CAP_SECS.to_string(),
-            hidden_if_empty: false,
-        },
         KV {
             key: SCANNER_MAX_CONCURRENT_SET_SCANS.to_owned(),
             value: DEFAULT_SCANNER_MAX_CONCURRENT_SET_SCANS.to_string(),

crates/ecstore/src/core/pools.rs

@@ -196,56 +196,6 @@ const DECOMMISSION_RETRY_BACKOFF_CAP: std::time::Duration = std::time::Duration:
 const DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT: usize = 100;
 const DECOMMISSION_TERMINAL_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(1);
 const DECOMMISSION_CANCEL_TARGET_LOCK_MAX_ATTEMPTS: usize = 3;
-/// Relaunch delay after a durable pause. A first pause is relaunched by the
-/// next watchdog pass as before; each further pause on the same cause without
-/// progress holds the relaunch for this base, doubled per repeat.
-const DECOMMISSION_PAUSE_RELAUNCH_BASE: std::time::Duration = std::time::Duration::from_secs(30);
-const DECOMMISSION_PAUSE_RELAUNCH_MAX: std::time::Duration = std::time::Duration::from_secs(15 * 60);
-
-/// The pause record after `reason` paused a pool whose settled progress is
-/// `progress`. A pause keeps its exit: the pool is relaunched after the
-/// returned delay, never failed. Only a repeat of the same cause without
-/// progress in between grows the delay. Progress is what a resumed pass cannot
-/// redo: completed buckets and entries moved off the pool. Item counters are
-/// not, because a relisted retained entry is seen again on every pass.
-fn next_decommission_pause_record(
-    previous: Option<&crate::runtime::instance::DecommissionPauseRecord>,
-    reason: &str,
-    progress: u64,
-    now: tokio::time::Instant,
-) -> crate::runtime::instance::DecommissionPauseRecord {
-    let repeats = match previous {
-        Some(previous) if previous.reason == reason && previous.progress == progress => previous.repeats.saturating_add(1),
-        _ => 1,
-    };
-    let delay = if repeats < 2 {
-        std::time::Duration::ZERO
-    } else {
-        DECOMMISSION_PAUSE_RELAUNCH_BASE
-            .saturating_mul(1_u32 << (repeats - 2).min(5))
-            .min(DECOMMISSION_PAUSE_RELAUNCH_MAX)
-    };
-    crate::runtime::instance::DecommissionPauseRecord {
-        reason: reason.to_owned(),
-        progress,
-        repeats,
-        relaunch_after: now + delay,
-    }
-}
-
-/// Whether the watchdog must not relaunch this pool yet: it is still paused
-/// on the cause of its last recorded pause and that pause's delay has not
-/// run out.
-fn decommission_pause_relaunch_pending(
-    record: Option<&crate::runtime::instance::DecommissionPauseRecord>,
-    info: Option<&PoolDecommissionInfo>,
-    now: tokio::time::Instant,
-) -> bool {
-    record.is_some_and(|record| {
-        now < record.relaunch_after
-            && info.and_then(|info| info.capacity_blocked_reason.as_deref()) == Some(record.reason.as_str())
-    })
-}
 
 fn decommission_capacity_target_gate_retry_exhausted(attempt: usize) -> bool {
     attempt.saturating_add(1) >= DECOMMISSION_CAPACITY_TARGET_GATE_MAX_ATTEMPTS
@@ -769,25 +719,6 @@ pub(crate) fn local_decommission_queue_prefix(endpoints: &EndpointServerPools, i
     Ok(local)
 }
 
-/// Carry `current`'s progress counters for pool `idx` into `snapshot` when
-/// both describe the same decommission generation. Counters only grow within
-/// a generation, so the larger value of each is the newer one.
-fn carry_decommission_progress_counters(snapshot: &mut PoolMeta, current: &PoolMeta, idx: usize) {
-    let (Some(target), Some(source)) = (
-        snapshot.pools.get_mut(idx).and_then(|pool| pool.decommission.as_mut()),
-        current.pools.get(idx).and_then(|pool| pool.decommission.as_ref()),
-    ) else {
-        return;
-    };
-    if target.start_time.is_none() || target.start_time != source.start_time {
-        return;
-    }
-    target.items_decommissioned = target.items_decommissioned.max(source.items_decommissioned);
-    target.items_decommission_failed = target.items_decommission_failed.max(source.items_decommission_failed);
-    target.bytes_done = target.bytes_done.max(source.bytes_done);
-    target.bytes_failed = target.bytes_failed.max(source.bytes_failed);
-}
-
 fn resumable_decommission_queue_indices(meta: &PoolMeta) -> Vec<usize> {
     let mut indices = Vec::new();
     for (idx, pool) in meta.pools.iter().enumerate() {
@@ -5686,9 +5617,9 @@ enum DecommissionEntryAttemptOutcome {
 struct DeferredDecommissionEntry {
     name: String,
     attempts: usize,
-    /// Versions whose progress was already counted. Versions are counted
-    /// only when their entry settles, so a deferred entry has counted none;
-    /// this guards the replay against counting a settled version twice.
+    /// Versions whose progress was already counted. A deferral after the
+    /// copy (source cleanup) has counted every version; the replay must not
+    /// count them again.
     counted_versions: HashSet<(Option<uuid::Uuid>, bool)>,
 }
 
@@ -12431,22 +12362,6 @@ fn resolve_data_movement_lifecycle_expiry_result(action: IlmAction, apply_action
     )))
 }
 
-const DATA_MOVEMENT_DELETE_ALL_DEFERRED_REASON: &str = "delete_all_needs_bucket_write_lock";
-
-fn log_data_movement_lifecycle_expiry_deferred(bucket: &str, object: &str, action: IlmAction, reason: &str) {
-    warn!(
-        event = EVENT_DECOMMISSION_ENTRY,
-        component = LOG_COMPONENT_ECSTORE,
-        subsystem = LOG_SUBSYSTEM_POOLS,
-        state = "lifecycle_expiry_deferred",
-        bucket = %bucket,
-        object = %object,
-        action = ?action,
-        reason,
-        "Data movement deferred a version's lifecycle expiry; moving the version"
-    );
-}
-
 #[allow(clippy::too_many_arguments)]
 pub(crate) async fn should_skip_lifecycle_for_data_movement(
     store: Arc<ECStore>,
@@ -12506,21 +12421,6 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement(
             }
             let applied = !apply_actions
                 || match lock_lost_signal {
-                    // A delete-all is a prefix delete that takes the bucket
-                    // lifecycle write lock, which the lifecycle read locks held
-                    // by the inline apply and by the rebalance entry's bucket
-                    // incarnation fence would block until timeout. The version
-                    // is still live, so it moves like any other version and
-                    // lifecycle expires it on the target.
-                    Some(_) if action.delete_all() => {
-                        log_data_movement_lifecycle_expiry_deferred(
-                            bucket,
-                            &version.name,
-                            action,
-                            DATA_MOVEMENT_DELETE_ALL_DEFERRED_REASON,
-                        );
-                        return Ok(false);
-                    }
                     Some(signal) => {
                         apply_expiry_rule_for_data_movement(store, &event, event_source, &object_info, Some(signal)).await
                     }
@@ -12546,15 +12446,16 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement(
                             // The version is still live, so it moves like any
                             // other version and lifecycle expires it on the
                             // target.
-                            log_data_movement_lifecycle_expiry_deferred(
-                                bucket,
-                                &version.name,
-                                action,
-                                if delete_all {
-                                    DATA_MOVEMENT_DELETE_ALL_DEFERRED_REASON
-                                } else {
-                                    "not_applied"
-                                },
+                            warn!(
+                                event = EVENT_DECOMMISSION_ENTRY,
+                                component = LOG_COMPONENT_ECSTORE,
+                                subsystem = LOG_SUBSYSTEM_POOLS,
+                                state = "lifecycle_expiry_deferred",
+                                bucket = %bucket,
+                                object = %version.name,
+                                action = ?action,
+                                reason = if delete_all { "delete_all_needs_bucket_write_lock" } else { "not_applied" },
+                                "Data movement deferred a transitioned version's lifecycle expiry; moving the version"
                             );
                             return Ok(false);
                         }
@@ -15839,14 +15740,6 @@ impl ECStore {
         let (pool_meta_guard, mut snapshot) = self
             .acquire_pool_meta_write_guard(&mut save_guard, "decommission capacity pause failed")
             .await?;
-        // The pause is a checkpoint too. Without carrying the counters of this
-        // generation, everything counted since the last progress save would
-        // be dropped, and the relisted pass could not tell a version it
-        // already counted from one it never did.
-        {
-            let pool_meta = self.pool_meta.read().await;
-            carry_decommission_progress_counters(&mut snapshot, &pool_meta, idx);
-        }
         snapshot.mark_decommission_capacity_blocked(idx, decommission_pause_reason(err), OffsetDateTime::now_utc())?;
         let outcome = snapshot
             .save_no_lock_armed(
@@ -15857,40 +15750,12 @@ impl ECStore {
             )
             .await?;
         ensure_pool_meta_write_fence(&pool_meta_guard, "decommission capacity pause failed")?;
-        let (generation, completed_buckets) = {
+        {
             let mut pool_meta = self.pool_meta.write().await;
             publish_pool_meta_updates(&mut pool_meta, &outcome.committed, &[idx]);
-            pool_meta
-                .pools
-                .get(idx)
-                .and_then(|pool| pool.decommission.as_ref())
-                .map_or((None, 0), |info| (info.start_time, info.decommissioned_buckets.len()))
-        };
-        let cleaned_entries = generation.map_or(0, |generation| {
-            self.ctx
-                .with_decommission_count_ledger(idx, generation, |ledger| ledger.progress())
-        });
-        let progress = cleaned_entries.saturating_add(u64::try_from(completed_buckets).unwrap_or(u64::MAX));
+        }
         ensure_pool_meta_write_fence(&pool_meta_guard, "decommission capacity pause failed")?;
         outcome.disarm();
-        let reason = decommission_pause_reason(err);
-        let now = tokio::time::Instant::now();
-        let record = self
-            .ctx
-            .record_decommission_pause(idx, |previous| next_decommission_pause_record(previous, &reason, progress, now));
-        if record.repeats > 1 {
-            warn!(
-                event = EVENT_DECOMMISSION_STATE,
-                component = LOG_COMPONENT_ECSTORE,
-                subsystem = LOG_SUBSYSTEM_POOLS,
-                state = "pause_relaunch_backoff",
-                pool_index = idx,
-                repeats = record.repeats,
-                retry_delay_secs = record.relaunch_after.saturating_duration_since(now).as_secs(),
-                reason = %reason,
-                "Decommission paused again on the same cause without progress; backing off its relaunch"
-            );
-        }
         Ok(())
     }
 
@@ -15991,24 +15856,10 @@ impl ECStore {
         rx: &CancellationToken,
         endpoints: &EndpointServerPools,
     ) -> Result<Vec<(usize, DecommissionCancelerGuard)>> {
-        let (indices, relaunch_pending) = {
+        let indices = {
             let pool_meta = self.pool_meta.read().await;
-            let indices = resumable_decommission_queue_indices(&pool_meta);
-            let now = tokio::time::Instant::now();
-            let relaunch_pending = indices.iter().any(|idx| {
-                decommission_pause_relaunch_pending(
-                    self.ctx.decommission_pause(*idx).as_ref(),
-                    pool_meta.pools.get(*idx).and_then(|pool| pool.decommission.as_ref()),
-                    now,
-                )
-            });
-            (indices, relaunch_pending)
+            resumable_decommission_queue_indices(&pool_meta)
         };
-        // A paused pool resumes in queue order; while its backoff runs, no
-        // later pool may start ahead of it.
-        if relaunch_pending {
-            return Ok(Vec::new());
-        }
         let indices = local_decommission_queue_prefix(endpoints, &indices)?;
         self.reserve_decommission_routines(rx, indices.as_slice()).await
     }
@@ -17004,36 +16855,28 @@ impl ECStore {
                         DecommissionCapacityAdmission::ScannerBacklog,
                     )?;
                 }
-                let persist = |read: &data_movement::scanner_backlog::ScannerPauseBacklogRetirementRead| {
-                    let target = Arc::clone(&self.pools[read.replica.pool_index].disk_set[read.replica.set_index]);
-                    let record = record.clone();
-                    let preconditions = read.preconditions();
-                    let object_fence = &object_fence;
-                    let pool_meta_guard = &pool_meta_guard;
-                    async move {
-                        let mut opts = ObjectOptions {
-                            no_lock: self.pools[0].disk_set[0].shares_namespace_lock_domain(&target).await,
-                            ..Default::default()
-                        };
-                        object_fence.add_namespace_lock_fence(&mut opts);
-                        opts.add_namespace_lock_guard(pool_meta_guard);
-                        persist_native_scanner_pause_backlog_replica(target, record, preconditions, opts, phase).await
-                    }
-                };
-                let survivors = || {
+                let results = join_all(
                     replicas
                         .iter()
                         .filter(|read| read.replica.pool_index != idx && read.replica.writable)
-                };
-                // Create survivors that read as missing first, one at a time, and
-                // stop at the first failure. A failed create then leaves every
-                // stored survivor unchanged, so the retry re-plans from the same
-                // authority; a survivor that only read as missing rejects its
-                // create before any stored survivor is overwritten.
-                for read in survivors().filter(|read| read.replica.data.is_none()) {
-                    persist(read).await?;
-                }
-                let results = join_all(survivors().filter(|read| read.replica.data.is_some()).map(persist)).await;
+                        .map(|read| {
+                            let target = Arc::clone(&self.pools[read.replica.pool_index].disk_set[read.replica.set_index]);
+                            let record = record.clone();
+                            let object_fence = &object_fence;
+                            let pool_meta_guard = &pool_meta_guard;
+                            async move {
+                                let mut opts = ObjectOptions {
+                                    no_lock: self.pools[0].disk_set[0].shares_namespace_lock_domain(&target).await,
+                                    ..Default::default()
+                                };
+                                object_fence.add_namespace_lock_fence(&mut opts);
+                                opts.add_namespace_lock_guard(pool_meta_guard);
+                                persist_native_scanner_pause_backlog_replica(target, record, read.preconditions(), opts, phase)
+                                    .await
+                            }
+                        }),
+                )
+                .await;
                 // Every dispatched native CAS has finished before an error can
                 // release the owned task's object and durable membership fences.
                 for result in results {
@@ -17076,7 +16919,6 @@ impl ECStore {
             &[],
             data_movement::SourceCleanupBucketFence {
                 expected_incarnation_id: None,
-                lifecycle_owner: None,
                 lifecycle_guard: None,
                 namespace_lock_lost_signal: pool_meta_guard.lock_lost_signal(),
                 object_mutation_fence: Some(&object_fence),
@@ -17345,7 +17187,7 @@ impl ECStore {
         let operation_gate = self.ctx.data_movement_operation_gate();
 
         let bucket_incarnation_fence = match expected_bucket_incarnation_id {
-            Some(expected) => Some(Arc::new(self.acquire_bucket_incarnation_fence(&bucket, expected).await?)),
+            Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?),
             None => None,
         };
 
@@ -17440,13 +17282,10 @@ impl ECStore {
         let mut decommissioned: usize = 0;
         let mut expired: usize = 0;
         let mut free_version_disposition = DecommissionFreeVersionDisposition::default();
-        // Counted when the entry settles; see the flush before the progress save.
-        let mut pending_counts: Vec<((Option<uuid::Uuid>, bool), usize, bool)> = Vec::with_capacity(fivs.versions.len());
         let mut cleanup_preflight_allowed_missing = Vec::new();
         let mut cleanup_preflight_vanished = Vec::new();
         let mut cleanup_transferred_tier_versions = Vec::new();
         let mut entry_blocked = false;
-        let mut source_cleaned = false;
 
         for version in fivs.versions.iter() {
             if self.decommission_cancel_requested(idx, &rx).await {
@@ -17546,7 +17385,18 @@ impl ECStore {
                     ));
                 }
 
-                pending_counts.push(((version.version_id, version.deleted), 0, !migrated && !consumed));
+                if counted_versions.insert((version.version_id, version.deleted)) {
+                    let mut pool_meta = self.pool_meta.write().await;
+                    ensure_decommission_generation(&pool_meta, idx, generation)?;
+                    if let Err(err) = count_decommission_item(&mut pool_meta, idx, 0, !migrated && !consumed) {
+                        return Err(with_decommission_entry_context(
+                            "count_decommission_item",
+                            bucket.as_str(),
+                            entry.name.as_str(),
+                            err,
+                        ));
+                    }
+                }
 
                 if migrated || consumed {
                     decommissioned += 1;
@@ -17805,7 +17655,18 @@ impl ECStore {
                     continue;
                 }
 
-                pending_counts.push(((version.version_id, version.deleted), 0, failure));
+                if counted_versions.insert((version.version_id, version.deleted)) {
+                    let mut pool_meta = self.pool_meta.write().await;
+                    ensure_decommission_generation(&pool_meta, idx, generation)?;
+                    if let Err(err) = count_decommission_item(&mut pool_meta, idx, 0, failure) {
+                        return Err(with_decommission_entry_context(
+                            "count_decommission_item",
+                            bucket.as_str(),
+                            entry.name.as_str(),
+                            err,
+                        ));
+                    }
+                }
 
                 if !failure {
                     decommissioned += 1;
@@ -18198,7 +18059,18 @@ impl ECStore {
                 continue;
             }
 
-            pending_counts.push(((version.version_id, version.deleted), decommission_item_size(version.size), failure));
+            if counted_versions.insert((version.version_id, version.deleted)) {
+                let mut pool_meta = self.pool_meta.write().await;
+                ensure_decommission_generation(&pool_meta, idx, generation)?;
+                if let Err(err) = count_decommission_item(&mut pool_meta, idx, decommission_item_size(version.size), failure) {
+                    return Err(with_decommission_entry_context(
+                        "count_decommission_item",
+                        bucket.as_str(),
+                        entry.name.as_str(),
+                        err,
+                    ));
+                }
+            }
 
             if failure {
                 break;
@@ -18281,8 +18153,9 @@ impl ECStore {
                         &cleanup_transferred_tier_versions,
                         data_movement::SourceCleanupBucketFence {
                             expected_incarnation_id: expected_bucket_incarnation_id,
-                            lifecycle_owner: bucket_incarnation_fence.as_ref(),
-                            lifecycle_guard: None,
+                            lifecycle_guard: bucket_incarnation_fence
+                                .as_ref()
+                                .and_then(|guard| guard.namespace_lock_guard()),
                             namespace_lock_lost_signal: None,
                             object_mutation_fence: Some(&source_cleanup_mutation_fence),
                         },
@@ -18292,7 +18165,7 @@ impl ECStore {
                 })
                 .await;
             match cleanup_result {
-                Ok(_) => source_cleaned = true,
+                Ok(_) => {}
                 Err(data_movement::SourceCleanupError::Storage(err)) => {
                     resolve_decommission_entry_cleanup_delete_result(
                         Err::<(), Error>(err),
@@ -18372,48 +18245,6 @@ impl ECStore {
             let mut pool_meta = self.pool_meta.write().await;
             ensure_decommission_generation(&pool_meta, idx, generation)?;
 
-            // Versions are counted once their entry settles: copied and cleaned
-            // up, or retained. An attempt that is retried, deferred or resumed
-            // after a pause recounted nothing, because nothing of it was
-            // counted. A retained entry is relisted by every resumed pass; the
-            // ledger counts its versions once per process (rustfs/backlog#2873).
-            let had_failure = pending_counts.iter().any(|(_, _, failed)| *failed);
-            let admitted = self.ctx.with_decommission_count_ledger(idx, generation, |ledger| {
-                if source_cleaned {
-                    ledger.record_cleaned_entry();
-                }
-                pending_counts
-                    .into_iter()
-                    .filter(|(version, _, _)| ledger.admit(bucket.as_str(), entry.name.as_str(), *version, source_cleaned))
-                    .collect::<Vec<_>>()
-            });
-            for (version, size, failure) in admitted {
-                if counted_versions.insert(version)
-                    && let Err(err) = count_decommission_item(&mut pool_meta, idx, size, failure)
-                {
-                    return Err(with_decommission_entry_context(
-                        "count_decommission_item",
-                        bucket.as_str(),
-                        entry.name.as_str(),
-                        err,
-                    ));
-                }
-            }
-
-            // Failure presence also selects the terminal state. A full
-            // diagnostic ledger may undercount versions, but cannot erase the
-            // last evidence that this generation encountered a failed entry.
-            if had_failure
-                && pool_meta.pools[idx]
-                    .decommission
-                    .as_ref()
-                    .is_some_and(|info| info.items_decommission_failed == 0)
-            {
-                count_decommission_item(&mut pool_meta, idx, 0, true).map_err(|err| {
-                    with_decommission_entry_context("count_decommission_failure", bucket.as_str(), entry.name.as_str(), err)
-                })?;
-            }
-
             if let Err(err) = track_decommission_current_object(&mut pool_meta, idx, bucket.as_str(), entry.name.as_str()) {
                 return Err(with_decommission_entry_context(
                     "track_decommission_current_object",
@@ -27445,81 +27276,6 @@ mod tests {
         assert_eq!(named(&[]), None);
     }
 
-    /// A copy that lost drive quorum stays the named pause cause when a
-    /// transient contention settles after it (rustfs/backlog#2873, defect F).
-    #[test]
-    fn decommission_deferral_cause_keeps_a_copy_quorum_loss() {
-        let quorum = decommission_copy_quorum_deferral(
-            "delete_marker_copy",
-            "bucket",
-            "object",
-            Error::InsufficientWriteQuorum("bucket".to_owned(), "object".to_owned()),
-        );
-        let busy = decommission_target_gate_busy_error(2);
-        let named = |errors: &[&Error]| {
-            let mut recorded = None;
-            for err in errors {
-                record_decommission_deferral_cause(&mut recorded, err);
-            }
-            recorded.map(|(_, cause)| cause)
-        };
-        assert_eq!(named(&[&quorum, &busy]), Some(quorum.to_string()));
-        assert_eq!(named(&[&busy, &quorum]), Some(quorum.to_string()));
-        let reason = ensure_decommission_deferred_drained("bucket", 0, 1, named(&[&quorum, &busy]).as_deref())
-            .expect_err("a remaining deferral pauses the set")
-            .to_string();
-        assert!(reason.contains("lost drive quorum"), "{reason}");
-    }
-
-    /// A first pause relaunches on the next watchdog pass; only a repeat of
-    /// the same cause without a counted item backs off, and the delay is
-    /// bounded. Progress or a new cause starts over.
-    #[test]
-    fn decommission_pause_relaunch_backs_off_only_on_repeats_without_progress() {
-        let now = tokio::time::Instant::now();
-        let first = next_decommission_pause_record(None, "cause", 7, now);
-        assert_eq!((first.repeats, first.relaunch_after), (1, now));
-
-        let mut record = first;
-        let mut delays = Vec::new();
-        for _ in 0..8 {
-            record = next_decommission_pause_record(Some(&record), "cause", 7, now);
-            delays.push(record.relaunch_after - now);
-        }
-        assert_eq!(delays[0], DECOMMISSION_PAUSE_RELAUNCH_BASE);
-        assert_eq!(delays[1], 2 * DECOMMISSION_PAUSE_RELAUNCH_BASE);
-        assert!(delays.windows(2).all(|pair| pair[0] <= pair[1]), "{delays:?}");
-        assert_eq!(*delays.last().unwrap(), DECOMMISSION_PAUSE_RELAUNCH_MAX);
-
-        let progressed = next_decommission_pause_record(Some(&record), "cause", 8, now);
-        assert_eq!((progressed.repeats, progressed.relaunch_after), (1, now));
-        let other = next_decommission_pause_record(Some(&record), "other cause", 7, now);
-        assert_eq!((other.repeats, other.relaunch_after), (1, now));
-    }
-
-    /// The hold applies only while the pool is still paused on the recorded
-    /// cause and its delay has not run out.
-    #[test]
-    fn decommission_pause_relaunch_hold_requires_the_same_live_pause() {
-        let now = tokio::time::Instant::now();
-        let first = next_decommission_pause_record(None, "cause", 7, now);
-        let repeated = next_decommission_pause_record(Some(&first), "cause", 7, now);
-        let paused = PoolDecommissionInfo {
-            capacity_blocked_reason: Some("cause".to_owned()),
-            ..Default::default()
-        };
-        assert!(decommission_pause_relaunch_pending(Some(&repeated), Some(&paused), now));
-        assert!(!decommission_pause_relaunch_pending(
-            Some(&repeated),
-            Some(&paused),
-            repeated.relaunch_after
-        ));
-        assert!(!decommission_pause_relaunch_pending(Some(&first), Some(&paused), now));
-        let resumed = PoolDecommissionInfo::default();
-        assert!(!decommission_pause_relaunch_pending(Some(&repeated), Some(&resumed), now));
-        assert!(!decommission_pause_relaunch_pending(None, Some(&paused), now));
-    }
-
     /// A set that still holds a deferred entry must not report success.
     #[test]
     fn decommission_deferred_drain_invariant_rejects_unresolved_entries() {
@@ -28673,13 +28429,7 @@ impl SetDisks {
                 min_disks: listing_quorum,
                 skip_walkdir_total_timeout: true,
                 walkdir_stall_timeout: Some(DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT),
-                agreed: Some(Box::new(move |entry: MetaCacheEntry| {
-                    let future = cb1(entry);
-                    Box::pin(async move {
-                        future.await;
-                        Ok(())
-                    })
-                })),
+                agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry)))),
                 partial: Some(Box::new(move |entries: MetaCacheEntries, errs: &[Option<DiskError>]| {
                     let resolver = resolver.clone();
                     let cb_func = cb_func.clone();
@@ -29408,7 +29158,6 @@ mod pools_tests {
             pool_meta_save_gate: tokio::sync::Mutex::new(super::PoolMetaWriteState::for_test_bootstrap()),
             ctx,
             bucket_fence_registry: Arc::default(),
-            ordinary_classification: Default::default(),
         })
     }
 

crates/ecstore/src/core/pools_test.rs

@@ -499,7 +499,6 @@ mod decommission_lock_order_tests {
             ),
             ctx,
             bucket_fence_registry: Arc::default(),
-            ordinary_classification: Default::default(),
         });
         crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
         (store, refresh_calls)
@@ -2448,481 +2447,6 @@ mod decommission_lock_order_tests {
         });
     }
 
-    /// Seed `object` in pool 0 with three 2 KiB data versions and a newest
-    /// delete marker; returns the data versions oldest first.
-    async fn seed_versioned_entry_with_marker(
-        store: &Arc<crate::store::ECStore>,
-        bucket: &str,
-        object: &str,
-    ) -> Vec<crate::object_api::ObjectInfo> {
-        store
-            .make_bucket(
-                bucket,
-                &MakeBucketOptions {
-                    versioning_enabled: true,
-                    ..Default::default()
-                },
-            )
-            .await
-            .expect("create the versioned bucket");
-        let mut written = Vec::new();
-        for index in 0..3u8 {
-            written.push(
-                store.pools[0]
-                    .put_object(
-                        bucket,
-                        object,
-                        &mut PutObjReader::from_vec(vec![index; 2048]),
-                        &ObjectOptions {
-                            versioned: true,
-                            ..Default::default()
-                        },
-                    )
-                    .await
-                    .expect("write a source version"),
-            );
-        }
-        store.pools[0]
-            .delete_object(
-                bucket,
-                object,
-                ObjectOptions {
-                    versioned: true,
-                    ..Default::default()
-                },
-            )
-            .await
-            .expect("write the source delete marker");
-        written
-    }
-
-    /// Fail every target write of `object` after the first, before it
-    /// commits, while `armed` is set.
-    fn fail_target_writes_after_the_first(
-        bucket: &str,
-        object: &'static str,
-        armed: Arc<AtomicBool>,
-    ) -> DecommissionTestFaultGuard {
-        let writes = Arc::new(AtomicUsize::new(0));
-        let bucket = bucket.to_owned();
-        DecommissionTestFaultGuard::install_with_error(
-            Arc::new(move |stage, entry_bucket, entry_object, _, _| {
-                stage == "before_target_commit"
-                    && entry_bucket == bucket
-                    && entry_object == object
-                    && armed.load(Ordering::Acquire)
-                    && writes.fetch_add(1, Ordering::AcqRel) >= 1
-            }),
-            Some(Arc::new(|_, bucket, object| {
-                crate::error::Error::InsufficientWriteQuorum(bucket.to_owned(), object.to_owned())
-            })),
-        )
-    }
-
-    async fn assert_every_version_readable(
-        store: &Arc<crate::store::ECStore>,
-        bucket: &str,
-        object: &str,
-        written: &[crate::object_api::ObjectInfo],
-    ) {
-        for (index, version) in written.iter().enumerate() {
-            let opts = ObjectOptions {
-                versioned: true,
-                version_id: version.version_id.map(|id| id.to_string()),
-                ..Default::default()
-            };
-            let mut reader = store
-                .get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
-                .await
-                .unwrap_or_else(|err| panic!("version {:?} must stay readable: {err:?}", version.version_id));
-            let mut body = Vec::new();
-            reader.stream.read_to_end(&mut body).await.expect("read the version body");
-            assert_eq!(body, vec![index as u8; 2048], "version {:?}", version.version_id);
-        }
-    }
-
-    fn decommission_info(pool_meta: &crate::core::pools::PoolMeta) -> crate::core::pools::PoolDecommissionInfo {
-        pool_meta.pools[0]
-            .decommission
-            .clone()
-            .expect("the decommission state should remain present")
-    }
-
-    /// Replace the in-memory pool metadata with what the replicas persisted,
-    /// as a restarted leader would load it.
-    async fn reload_persisted_pool_meta(store: &Arc<crate::store::ECStore>) {
-        let mut persisted = crate::core::pools::PoolMeta::default();
-        persisted
-            .load_no_lock_from_replicas(store.pools.clone())
-            .await
-            .expect("reload the persisted pool metadata");
-        *store.pool_meta.write().await = persisted;
-    }
-
-    /// Wait until the relaunched routine recorded its `repeats`-th pause and
-    /// released its worker slot.
-    async fn wait_for_relaunched_pause(store: &Arc<crate::store::ECStore>, repeats: u32) {
-        tokio::time::timeout(Duration::from_secs(60), async {
-            loop {
-                let paused = store
-                    .ctx
-                    .decommission_pause(0)
-                    .is_some_and(|record| record.repeats >= repeats);
-                if paused && !store.has_active_local_decommission_worker().await {
-                    return;
-                }
-                tokio::time::sleep(Duration::from_millis(50)).await;
-            }
-        })
-        .await
-        .unwrap_or_else(|_| panic!("the relaunched routine should pause again: record={:?}", store.ctx.decommission_pause(0)));
-    }
-
-    /// A partially moved entry pauses the decommission, and the resumed
-    /// routine, started from the persisted metadata, finds its first versions
-    /// already on the target. Every version is counted exactly once
-    /// (rustfs/backlog#2873, defect F follow-up).
-    #[test]
-    #[serial_test::serial]
-    fn decommission_counts_each_version_once_across_a_pause_and_resume() {
-        run_large_stack_async_test("decommission-pause-resume-count", || async {
-            // Serialize this accounting fixture so the deferred entry precedes
-            // the later checkpoint, regardless of worker scheduling.
-            let env = [
-                ("RUSTFS_DECOMMISSION_BUCKET_CONCURRENCY", Some("1")),
-                ("RUSTFS_DECOMMISSION_ENTRY_CONCURRENCY", Some("1")),
-            ];
-            temp_env::async_with_vars(env, async {
-                let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
-                let bucket = test_bucket("pause-resume-count");
-                let object = "versioned/03.bin";
-                let written = seed_versioned_entry_with_marker(&store, &bucket, object).await;
-                // Listed after the partially moved entry: its completion saves
-                // a progress checkpoint while that entry's first versions are
-                // moved and its later ones are still deferred.
-                let source_set = store.pools[0].get_disks_by_key(object).set_index;
-                let later = (0..1024)
-                    .map(|index| format!("z/after-{index:04}.bin"))
-                    .find(|key| store.pools[0].get_disks_by_key(key).set_index == source_set)
-                    .expect("a later key in the same source set");
-                store.pools[0]
-                    .put_object(
-                        &bucket,
-                        &later,
-                        &mut PutObjReader::from_vec(vec![0x7a; 16]),
-                        &ObjectOptions {
-                            versioned: true,
-                            ..Default::default()
-                        },
-                    )
-                    .await
-                    .expect("write the later entry");
-
-                // Counters when the test bucket's first entry starts: only the
-                // system objects are in them.
-                let baseline = Arc::new(StdMutex::new(None::<(String, usize, usize)>));
-                let armed = Arc::new(AtomicBool::new(true));
-                let writes = Arc::new(AtomicUsize::new(0));
-                let _fault = DecommissionTestFaultGuard::install_with_error(
-                    Arc::new({
-                        let store = Arc::clone(&store);
-                        let bucket = bucket.clone();
-                        let baseline = Arc::clone(&baseline);
-                        let armed = Arc::clone(&armed);
-                        move |stage, entry_bucket, entry_object, _, _| {
-                            if entry_bucket != bucket {
-                                return false;
-                            }
-                            if stage == "exact_versions_read" {
-                                let mut baseline = baseline.lock().expect("baseline lock");
-                                if baseline.is_none() {
-                                    // This synchronous fault hook must not miss
-                                    // the baseline when metadata is briefly busy.
-                                    let pool_meta = tokio::task::block_in_place(|| store.pool_meta.blocking_read());
-                                    let info = pool_meta.pools[0].decommission.as_ref().expect("active decommission");
-                                    *baseline = Some((entry_object.to_owned(), info.items_decommissioned, info.bytes_done));
-                                }
-                                return false;
-                            }
-                            stage == "before_target_commit"
-                                && entry_object == object
-                                && armed.load(Ordering::Acquire)
-                                && writes.fetch_add(1, Ordering::AcqRel) >= 1
-                        }
-                    }),
-                    Some(Arc::new(|_, bucket, object| {
-                        crate::error::Error::InsufficientWriteQuorum(bucket.to_owned(), object.to_owned())
-                    })),
-                );
-                let _checkpoint = crate::core::pools::DecommissionCleanupMutationGuard::install(Arc::new({
-                    let store = Arc::clone(&store);
-                    let bucket = bucket.clone();
-                    move |hook_bucket, hook_object, _| {
-                        let store = Arc::clone(&store);
-                        let due = hook_bucket == bucket && hook_object == later;
-                        Box::pin(async move {
-                            if due {
-                                let stale = time::OffsetDateTime::now_utc() - time::Duration::hours(1);
-                                let mut pool_meta = store.pool_meta.write().await;
-                                pool_meta.pools[0].last_update = stale;
-                                if let Some(info) = pool_meta.pools[0].decommission.as_mut() {
-                                    info.progress_save_last_at = Some(stale);
-                                }
-                            }
-                        })
-                    }
-                }));
-
-                start_transitioned_entry_decommission(&store, &bucket).await;
-                spawn_decommission_routine(&store, 1)
-                    .await
-                    .expect("the decommission routine task should join")
-                    .expect("a paused decommission routine returns without an error");
-                let paused = decommission_info(&*store.pool_meta.read().await);
-                assert!(!paused.complete && !paused.failed, "{paused:?}");
-                assert!(paused.capacity_blocked_reason.is_some(), "{paused:?}");
-                assert_eq!(paused.items_decommission_failed, 0, "{paused:?}");
-                let (first_entry, base_items, base_bytes) = baseline
-                    .lock()
-                    .expect("baseline lock")
-                    .clone()
-                    .expect("the test bucket's first entry started");
-                assert_eq!(first_entry, object, "the baseline precedes every entry of the test bucket");
-
-                reload_persisted_pool_meta(&store).await;
-                armed.store(false, Ordering::Release);
-                spawn_decommission_routine(&store, 1)
-                    .await
-                    .expect("the resumed routine task should join")
-                    .expect("the resumed routine should return");
-                let done = decommission_info(&*store.pool_meta.read().await);
-                assert!(done.complete && !done.failed && done.capacity_blocked_reason.is_none(), "{done:?}");
-                assert_eq!(
-                    (done.items_decommissioned, done.bytes_done),
-                    (base_items + 5, base_bytes + 3 * 2048 + 16),
-                    "the entry's four versions and the later entry, each counted once: {done:?}"
-                );
-                assert_eq!(done.items_decommission_failed, 0, "{done:?}");
-                assert!(exact_versions(&store, 0, &bucket, object).await.is_empty(), "the source is cleaned up");
-                assert_every_version_readable(&store, &bucket, object, &written).await;
-            })
-            .await;
-        });
-    }
-
-    /// Leave `object`'s target metadata unusable on two of the target set's
-    /// four drives, below the read and write quorum of EC 3+1: a directory
-    /// where `xl.meta` belongs fails every read and write there.
-    async fn break_target_metadata_on_two_drives(
-        temp_dirs: &[tempfile::TempDir],
-        store: &Arc<crate::store::ECStore>,
-        target: usize,
-        bucket: &str,
-        object: &str,
-    ) {
-        let set_index = store.pools[target].get_disks_by_key(object).set_index;
-        for disk in 0..2 {
-            let meta = temp_dirs[target * 8 + set_index * 4 + disk]
-                .path()
-                .join(bucket)
-                .join(object)
-                .join("xl.meta");
-            tokio::fs::remove_file(&meta)
-                .await
-                .unwrap_or_else(|err| panic!("remove {}: {err}", meta.display()));
-            tokio::fs::create_dir(&meta)
-                .await
-                .unwrap_or_else(|err| panic!("block {}: {err}", meta.display()));
-        }
-    }
-
-    /// A target that holds the entry's metadata on fewer drives than its
-    /// quorum is unreadable. The intact source keeps serving every version,
-    /// and a pass that meets the sub-quorum target defers the entry: it pauses
-    /// without a failed item and counts nothing of the entry.
-    #[test]
-    #[serial_test::serial]
-    fn decommission_half_written_target_never_shadows_the_source() {
-        run_large_stack_async_test("decommission-half-written-target", || async {
-            let (temp_dirs, store, _other_store) =
-                crate::services::rebalance::test_three_pool_stores_with_four_disk_sets_with_isolated_node_contexts(None).await;
-            let bucket = test_bucket("half-written-target");
-            let object = "versioned/03.bin";
-            let written = seed_versioned_entry_with_marker(&store, &bucket, object).await;
-            let armed = Arc::new(AtomicBool::new(true));
-            let _fault = fail_target_writes_after_the_first(&bucket, object, Arc::clone(&armed));
-
-            start_transitioned_entry_decommission(&store, &bucket).await;
-            spawn_decommission_routine(&store, 1)
-                .await
-                .expect("the decommission routine task should join")
-                .expect("a paused decommission routine returns without an error");
-            let mut target = None;
-            for pool in [1, 2] {
-                if !exact_versions(&store, pool, &bucket, object).await.is_empty() {
-                    target = Some(pool);
-                }
-            }
-            let target = target.expect("the marker and the newest version reached a target");
-            break_target_metadata_on_two_drives(&temp_dirs, &store, target, &bucket, object).await;
-            let moved = store.pools[target]
-                .get_object_info(
-                    &bucket,
-                    object,
-                    &ObjectOptions {
-                        versioned: true,
-                        version_id: written[2].version_id.map(|id| id.to_string()),
-                        ..Default::default()
-                    },
-                )
-                .await;
-            assert!(moved.is_err(), "the target copy must be unreadable: {moved:?}");
-            assert_every_version_readable(&store, &bucket, object, &written).await;
-
-            // No injected fault any more: only the sub-quorum target remains.
-            armed.store(false, Ordering::Release);
-            let before = decommission_info(&*store.pool_meta.read().await);
-            spawn_decommission_routine(&store, 1)
-                .await
-                .expect("the resumed routine task should join")
-                .expect("a paused decommission routine returns without an error");
-            let info = decommission_info(&*store.pool_meta.read().await);
-            assert!(!info.complete && !info.failed, "the entry is deferred, not settled: {info:?}");
-            let reason = info.capacity_blocked_reason.as_deref().unwrap_or_default();
-            assert!(reason.contains("lost drive quorum"), "{reason}");
-            assert_eq!(info.items_decommission_failed, 0, "{info:?}");
-            assert_eq!(
-                (info.items_decommissioned, info.bytes_done),
-                (before.items_decommissioned, before.bytes_done),
-                "a deferred entry counts nothing: {info:?}"
-            );
-            assert_eq!(exact_versions(&store, 0, &bucket, object).await.len(), 4, "the source is retained");
-            assert_every_version_readable(&store, &bucket, object, &written).await;
-        });
-    }
-
-    /// A bucket holding a permanently retained entry and an entry that keeps
-    /// losing target quorum pauses on the same cause every pass. The retained
-    /// entry is not counted again, so the watchdog's relaunch backs off; a
-    /// first pause is relaunched by the watchdog path and a repeated one is
-    /// held (rustfs/backlog#2873, defect F follow-up).
-    #[test]
-    #[serial_test::serial]
-    fn decommission_relaunch_backs_off_while_a_retained_entry_is_relisted() {
-        assert_retained_entry_relaunch_backoff(false);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn decommission_full_count_ledger_preserves_failure_and_relaunch_backoff() {
-        assert_retained_entry_relaunch_backoff(true);
-    }
-
-    fn assert_retained_entry_relaunch_backoff(full_ledger: bool) {
-        run_large_stack_async_test("decommission-retained-backoff", move || async move {
-            let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
-            let bucket = test_bucket("retained-backoff");
-            let object = "versioned/03.bin";
-            let retained = "a/retained.bin";
-            seed_versioned_entry_with_marker(&store, &bucket, object).await;
-            store.pools[0]
-                .put_object(
-                    &bucket,
-                    retained,
-                    &mut PutObjReader::from_vec(vec![0x52; 64]),
-                    &ObjectOptions {
-                        versioned: true,
-                        ..Default::default()
-                    },
-                )
-                .await
-                .expect("write the retained entry");
-            let _fault = DecommissionTestFaultGuard::install_with_error(
-                Arc::new({
-                    let bucket = bucket.clone();
-                    move |stage, entry_bucket, entry_object, _, _| {
-                        entry_bucket == bucket
-                            && ((stage == "object_read" && entry_object == retained)
-                                || (stage == "before_target_commit" && entry_object == object))
-                    }
-                }),
-                Some(Arc::new(|stage, bucket, object| {
-                    if stage == "object_read" {
-                        crate::error::Error::other("injected permanent source read failure")
-                    } else {
-                        crate::error::Error::InsufficientWriteQuorum(bucket.to_owned(), object.to_owned())
-                    }
-                })),
-            );
-
-            start_transitioned_entry_decommission(&store, &bucket).await;
-            if full_ledger {
-                let generation = decommission_info(&*store.pool_meta.read().await)
-                    .start_time
-                    .expect("generation");
-                store.ctx.with_decommission_count_ledger(0, generation, |ledger| {
-                    for index in 0..2048 {
-                        ledger.admit("retained-ledger-fixture", &format!("key-{index}"), (None, false), false);
-                    }
-                    assert!(!ledger.admit("retained-ledger-fixture", "overflow", (None, false), false));
-                });
-            }
-            let pause_round = |round: u32| {
-                let store = Arc::clone(&store);
-                async move {
-                    let info = decommission_info(&*store.pool_meta.read().await);
-                    assert!(!info.complete && !info.failed, "round {round} pauses: {info:?}");
-                    assert!(info.capacity_blocked_reason.is_some(), "round {round}: {info:?}");
-                    assert_eq!(
-                        info.items_decommission_failed, 1,
-                        "the retained version is counted once (round {round}): {info:?}"
-                    );
-                    let record = store.ctx.decommission_pause(0).expect("the pause is recorded");
-                    assert_eq!(record.repeats, round, "round {round}: {record:?}");
-                    info
-                }
-            };
-
-            spawn_decommission_routine(&store, 1)
-                .await
-                .expect("the decommission routine task should join")
-                .expect("a paused decommission routine returns without an error");
-            let first = pause_round(1).await;
-            // The test helper keeps its worker slot; a finished routine frees it.
-            store.decommission_cancelers.write().await[0] = None;
-
-            // A first pause holds nothing: the watchdog path relaunches it.
-            store
-                .spawn_missing_local_decommission_routines()
-                .await
-                .expect("the watchdog relaunches a first pause");
-            assert!(
-                store.has_active_local_decommission_worker().await
-                    || store.ctx.decommission_pause(0).is_some_and(|record| record.repeats == 2),
-                "the first pause must be relaunched"
-            );
-            wait_for_relaunched_pause(&store, 2).await;
-            let second = pause_round(2).await;
-            assert_eq!(second.items_decommissioned, first.items_decommissioned, "{second:?}");
-
-            // A repeated pause is held: the watchdog path starts no worker.
-            store
-                .spawn_missing_local_decommission_routines()
-                .await
-                .expect("a held relaunch is not an error");
-            assert!(!store.has_active_local_decommission_worker().await, "a repeated pause must be held");
-            assert_eq!(store.ctx.decommission_pause(0).map(|record| record.repeats), Some(2));
-
-            spawn_decommission_routine(&store, 1)
-                .await
-                .expect("the third routine task should join")
-                .expect("a paused decommission routine returns without an error");
-            let third = pause_round(3).await;
-            assert_eq!(third.items_decommissioned, first.items_decommissioned, "{third:?}");
-            assert_eq!(exact_versions(&store, 0, &bucket, retained).await.len(), 1, "the retained source stays");
-        });
-    }
-
     /// Only a target write is deferred. A source version that cannot be read
     /// keeps its per-object failure, so an unreadable source ends the
     /// decommission as failed instead of pausing and resuming forever.

crates/ecstore/src/core/sets.rs

@@ -99,33 +99,6 @@ pub struct Sets {
     ctx: Arc<InstanceContext>,
 }
 
-#[cfg(test)]
-impl Sets {
-    pub(crate) fn from_test_set(
-        set: Arc<SetDisks>,
-        endpoints: PoolEndpoints,
-        format: FormatV3,
-        pool_idx: usize,
-        parity_count: usize,
-        ctx: Arc<InstanceContext>,
-    ) -> Arc<Self> {
-        Arc::new(Self {
-            id: format.id,
-            disk_set: vec![set],
-            pool_idx,
-            set_count: 1,
-            set_drive_count: endpoints.drives_per_set,
-            endpoints,
-            format,
-            parity_count,
-            default_parity_count: parity_count,
-            distribution_algo: DistributionAlgoVersion::V1,
-            exit_signal: None,
-            ctx,
-        })
-    }
-}
-
 impl Drop for Sets {
     fn drop(&mut self) {
         if let Some(exit_signal) = self.exit_signal.take() {
@@ -827,10 +800,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
             mod_time: dst_opts.mod_time,
             http_preconditions: dst_opts.http_preconditions.clone(),
             quota_admission: dst_opts.quota_admission,
-            expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id,
-            bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
-            bucket_incarnation_fence: dst_opts.bucket_incarnation_fence.clone(),
-            object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
             ..Default::default()
         };
 

crates/ecstore/src/data_movement/mod.rs

@@ -31,7 +31,7 @@ use crate::storage_api_contracts::{
     namespace::NamespaceLocking as _,
     object::{HTTPPreconditions, ObjectOperations as _},
 };
-use crate::store::{BucketIncarnationFenceGuard, DecommissionFixedReadAnchor, ECStore, SourceCleanupMutationFence};
+use crate::store::{DecommissionFixedReadAnchor, ECStore, SourceCleanupMutationFence};
 use bytes::Bytes;
 use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo};
 use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
@@ -1278,7 +1278,6 @@ pub(crate) enum SourceCleanupError {
 #[derive(Clone, Default)]
 pub(crate) struct SourceCleanupBucketFence<'a> {
     pub(crate) expected_incarnation_id: Option<uuid::Uuid>,
-    pub(crate) lifecycle_owner: Option<&'a Arc<BucketIncarnationFenceGuard>>,
     pub(crate) lifecycle_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
     pub(crate) namespace_lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
     pub(crate) object_mutation_fence: Option<&'a SourceCleanupMutationFence>,
@@ -1487,10 +1486,9 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
         )
     };
 
-    if bucket_fence.lifecycle_owner.is_some_and(|owner| owner.is_lock_lost())
-        || bucket_fence
-            .lifecycle_guard
-            .is_some_and(rustfs_lock::NamespaceLockGuard::is_lock_lost)
+    if bucket_fence
+        .lifecycle_guard
+        .is_some_and(rustfs_lock::NamespaceLockGuard::is_lock_lost)
     {
         return Err(SourceCleanupError::Storage(Error::other(format!(
             "{op_label}: bucket incarnation fence was lost before source cleanup"
@@ -1529,9 +1527,6 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
     if let Some(object_mutation_fence) = bucket_fence.object_mutation_fence {
         object_mutation_fence.add_namespace_lock_fence(&mut opts);
     }
-    if let Some(owner) = bucket_fence.lifecycle_owner {
-        owner.attach_owner(&mut opts);
-    }
     if let Some(bucket_lifecycle_guard) = bucket_fence.lifecycle_guard {
         opts.add_bucket_lifecycle_lock_guard(bucket_lifecycle_guard);
     }

crates/ecstore/src/data_usage/mod.rs

@@ -176,9 +176,6 @@ static USAGE_CACHE_UPDATING: OnceLock<CacheUpdating> = OnceLock::new();
 static LIVE_BUCKET_USAGE_CACHE: OnceLock<LiveBucketUsageCache> = OnceLock::new();
 static USAGE_MEMORY_GENERATION: AtomicU64 = AtomicU64::new(0);
 
-/// Compatibility input for snapshot-protocol quota admission and capacity
-/// reporting. Durable quota checks and statistics do not consult this floor.
-///
 /// Best-available persisted usage for `bucket` when no authoritative source
 /// exists yet (issue #5716): after an upgrade from a pre-v2 release the only
 /// persisted usage data is the legacy `.usage.json`, which is demoted to
@@ -216,7 +213,7 @@ pub async fn lookup_degraded_bucket_usage_baseline(store: Arc<ECStore>, bucket:
     // Stale or empty cache: refresh through the TTL-bounded loader. A failed
     // refresh carries the previous baseline forward, so quota admission keeps
     // its last grounded values through a backend read outage.
-    let _ = project_data_usage_from_backend_cached(store, |_| ()).await;
+    let _ = load_data_usage_from_backend_cached(store).await;
     let cache = data_usage_snapshot_cache().read().await;
     cache
         .as_ref()
@@ -231,40 +228,36 @@ struct CachedDataUsageSnapshot {
     loaded_at: tokio::time::Instant,
     /// Pre-discard per-bucket sizes from the same load, retained even when the
     /// snapshot is incomplete and its bucket data is discarded. Consumed only
-    /// by [`lookup_degraded_bucket_usage_baseline`] for snapshot-protocol
-    /// admission and capacity reporting until their compatibility windows end.
+    /// by [`lookup_degraded_bucket_usage_baseline`] for quota admission.
     degraded_baseline: HashMap<String, u64>,
 }
 
 impl CachedDataUsageSnapshot {
-    fn result<T>(&self, project: impl FnOnce(&DataUsageInfo) -> T) -> Result<T, Error> {
+    fn result(&self) -> Result<DataUsageInfo, Error> {
         self.info
-            .as_ref()
-            .map(project)
+            .clone()
             .ok_or_else(|| Error::other("data usage snapshot load recently failed"))
     }
 }
 
-fn fresh_cached_data_usage_snapshot<T>(
+fn fresh_cached_data_usage_snapshot(
     cache: &Option<CachedDataUsageSnapshot>,
     now: tokio::time::Instant,
     ttl: Duration,
-    project: impl FnOnce(&DataUsageInfo) -> T,
-) -> Option<Result<T, Error>> {
+) -> Option<Result<DataUsageInfo, Error>> {
     cache
         .as_ref()
         .filter(|cached| now.duration_since(cached.loaded_at) < ttl)
-        .map(|cached| cached.result(project))
+        .map(CachedDataUsageSnapshot::result)
 }
 
-fn cache_data_usage_snapshot_result<T>(
+fn cache_data_usage_snapshot_result(
     cache: &mut Option<CachedDataUsageSnapshot>,
     result: Result<LoadedUsageBaseline, Error>,
     loaded_at: tokio::time::Instant,
     refresh_generation: u64,
     current_generation: u64,
-    project: impl FnOnce(&DataUsageInfo) -> T,
-) -> Option<Result<T, Error>> {
+) -> Option<Result<DataUsageInfo, Error>> {
     if current_generation != refresh_generation {
         return None;
     }
@@ -283,13 +276,12 @@ fn cache_data_usage_snapshot_result<T>(
                     degraded_baseline.entry(bucket.clone()).or_insert(*size);
                 }
             }
-            let projected = project(&info);
             *cache = Some(CachedDataUsageSnapshot {
-                info: Some(info),
+                info: Some(info.clone()),
                 loaded_at,
                 degraded_baseline,
             });
-            Ok(projected)
+            Ok(info)
         }
         Err(e) => {
             // Keep the previous baseline through a failed refresh: quota
@@ -1490,21 +1482,12 @@ async fn normalize_loaded_data_usage(
 /// `save_data_usage_in_backend` invalidates the cache so a fresh scanner save
 /// is visible immediately in the saving process.
 pub async fn load_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
-    project_data_usage_from_backend_cached(store, DataUsageInfo::clone).await
-}
-
-// Project under the cache guard so scalar readers never copy the snapshot's
-// bucket maps. The owned API shares the same TTL, refresh and invalidation path.
-async fn project_data_usage_from_backend_cached<T>(
-    store: Arc<ECStore>,
-    project: impl Fn(&DataUsageInfo) -> T,
-) -> Result<T, Error> {
     let ttl = Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
 
     loop {
         {
             let cache = data_usage_snapshot_cache().read().await;
-            if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl, &project) {
+            if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
                 return result;
             }
         }
@@ -1516,7 +1499,7 @@ async fn project_data_usage_from_backend_cached<T>(
             .await;
         {
             let cache = data_usage_snapshot_cache().read().await;
-            if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl, &project) {
+            if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
                 return result;
             }
         }
@@ -1525,14 +1508,9 @@ async fn project_data_usage_from_backend_cached<T>(
         let result = load_data_usage_from_backend_with_baseline(store.clone()).await;
         let loaded_at = tokio::time::Instant::now();
         let mut cache = data_usage_snapshot_cache().write().await;
-        if let Some(result) = cache_data_usage_snapshot_result(
-            &mut cache,
-            result,
-            loaded_at,
-            refresh_generation,
-            data_usage_snapshot_generation(),
-            &project,
-        ) {
+        if let Some(result) =
+            cache_data_usage_snapshot_result(&mut cache, result, loaded_at, refresh_generation, data_usage_snapshot_generation())
+        {
             return result;
         }
         drop(cache);
@@ -1550,8 +1528,7 @@ pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> R
     loop {
         {
             let cache = admin_data_usage_snapshot_cache().read().await;
-            if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl, DataUsageInfo::clone)
-            {
+            if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
                 return result;
             }
         }
@@ -1562,8 +1539,7 @@ pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> R
             .await;
         {
             let cache = admin_data_usage_snapshot_cache().read().await;
-            if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl, DataUsageInfo::clone)
-            {
+            if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
                 return result;
             }
         }
@@ -1584,7 +1560,6 @@ pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> R
             loaded_at,
             refresh_generation,
             admin_data_usage_snapshot_generation(),
-            DataUsageInfo::clone,
         ) {
             return result;
         }
@@ -2298,10 +2273,8 @@ async fn get_persisted_bucket_usage(bucket: &str) -> Option<u64> {
 }
 
 async fn persisted_bucket_usage(store: Arc<ECStore>, bucket: &str) -> Option<u64> {
-    project_data_usage_from_backend_cached(store, |info| info.buckets_usage.get(bucket).map(|usage| usage.size))
-        .await
-        .ok()
-        .flatten()
+    let info = load_data_usage_from_backend_cached(store).await.ok()?;
+    info.buckets_usage.get(bucket).map(|usage| usage.size)
 }
 
 async fn reload_persisted_bucket_usage(store: Arc<ECStore>, bucket: &str) -> Option<u64> {
@@ -2398,7 +2371,6 @@ where
         tokio::time::Instant::now(),
         refresh_generation,
         data_usage_snapshot_generation(),
-        |_| (),
     )
     .is_some()
 }
@@ -2915,8 +2887,6 @@ pub async fn init_compression_total_memory_from_backend(store: Arc<ECStore>) {
 
 #[cfg(test)]
 mod tests {
-    mod snapshot_projection;
-
     use super::*;
     use crate::layout::endpoints::EndpointServerPools;
     use crate::runtime::instance::InstanceContext;
@@ -3360,7 +3330,6 @@ mod tests {
             pool_meta_save_gate: TokioMutex::default(),
             ctx,
             bucket_fence_registry: Arc::default(),
-            ordinary_classification: Default::default(),
         })
     }
 
@@ -4599,29 +4568,15 @@ mod tests {
             loaded_at,
             refresh_generation,
             data_usage_snapshot_generation(),
-            DataUsageInfo::clone,
         )
         .expect("an uninterrupted refresh should populate the cache");
         assert!(matches!(first, Err(Error::ErasureReadQuorum)));
 
-        let cached = fresh_cached_data_usage_snapshot(
-            &cache,
-            loaded_at + Duration::from_secs(1),
-            Duration::from_secs(30),
-            DataUsageInfo::clone,
-        )
-        .expect("the failed load must remain cached within the TTL");
+        let cached = fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(1), Duration::from_secs(30))
+            .expect("the failed load must remain cached within the TTL");
         assert!(cached.is_err());
 
-        assert!(
-            fresh_cached_data_usage_snapshot(
-                &cache,
-                loaded_at + Duration::from_secs(30),
-                Duration::from_secs(30),
-                DataUsageInfo::clone
-            )
-            .is_none()
-        );
+        assert!(fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(30), Duration::from_secs(30)).is_none());
     }
 
     #[test]
@@ -4642,20 +4597,14 @@ mod tests {
             loaded_at,
             refresh_generation,
             data_usage_snapshot_generation(),
-            DataUsageInfo::clone,
         )
         .expect("an uninterrupted refresh should populate the cache")
         .expect("successful load must be returned");
         assert_snapshot_bucket(&first, "bucket");
 
-        let cached = fresh_cached_data_usage_snapshot(
-            &cache,
-            loaded_at + Duration::from_secs(1),
-            Duration::from_secs(30),
-            DataUsageInfo::clone,
-        )
-        .expect("successful load must remain cached within the TTL")
-        .expect("cached success must remain successful");
+        let cached = fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(1), Duration::from_secs(30))
+            .expect("successful load must remain cached within the TTL")
+            .expect("cached success must remain successful");
         assert_snapshot_bucket(&cached, "bucket");
     }
 
@@ -4680,7 +4629,6 @@ mod tests {
             loaded_at,
             refresh_generation,
             data_usage_snapshot_generation(),
-            DataUsageInfo::clone,
         )
         .expect("an uninterrupted refresh should populate the cache")
         .expect("successful load must be returned");
@@ -4714,7 +4662,6 @@ mod tests {
             loaded_at,
             refresh_generation,
             data_usage_snapshot_generation(),
-            DataUsageInfo::clone,
         );
 
         assert!(stale_result.is_none());
@@ -6410,43 +6357,6 @@ mod tests {
         assert_eq!(get_bucket_usage_memory("bucket-a").await, None);
     }
 
-    #[tokio::test]
-    #[serial]
-    async fn complete_scanner_cycle_with_carried_bucket_preserves_legacy_quota_floor() {
-        clear_usage_memory_cache_for_test().await;
-        let bucket = "partial-carried-quota";
-        let mut baseline = data_usage_info_for_test(bucket, 2, 100, SystemTime::UNIX_EPOCH + Duration::from_secs(100));
-        baseline.scanner_epoch = Some(7);
-        baseline.scanner_cycle = Some(10);
-        baseline.usage_snapshot_converged = Some(true);
-        replace_bucket_usage_memory_from_info(&baseline).await;
-        record_bucket_object_write_memory(bucket, None, 25).await;
-        record_bucket_object_delete_memory(bucket, 20, true).await;
-        assert_eq!(get_bucket_usage_memory(bucket).await, Some(125));
-        let mutation_time = memory_cache().read().await[bucket].usage_updated_at;
-
-        let mut carried = baseline;
-        carried.last_update = Some(mutation_time + Duration::from_secs(1));
-        carried.scanner_cycle = Some(11);
-        carried.usage_snapshot_converged = Some(false);
-        carried.usage_snapshot_pending_buckets = vec![bucket.to_string()];
-        assert!(carried.usage_snapshot_complete);
-        replace_bucket_usage_memory_from_info(&carried).await;
-        assert_eq!(
-            get_bucket_usage_memory(bucket).await,
-            Some(125),
-            "Complete with a carried certified value must retain positive deltas and the delete hold"
-        );
-
-        let mut reconciled = data_usage_info_for_test(bucket, 2, 105, mutation_time + Duration::from_secs(2));
-        reconciled.scanner_epoch = Some(7);
-        reconciled.scanner_cycle = Some(12);
-        reconciled.usage_snapshot_converged = Some(true);
-        replace_bucket_usage_memory_from_info(&reconciled).await;
-        assert_eq!(get_bucket_usage_memory(bucket).await, Some(105));
-        clear_usage_memory_cache_for_test().await;
-    }
-
     #[tokio::test]
     #[serial]
     async fn stale_quota_uses_complete_baseline_plus_positive_deltas() {

crates/ecstore/src/data_usage/tests/snapshot_projection.rs

@@ -1,191 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use super::*;
-
-#[test]
-fn scalar_projection_borrows_snapshot_and_preserves_unknown_and_zero() {
-    let loaded_at = tokio::time::Instant::now();
-    let mut info = data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH);
-    info.buckets_usage.insert("empty".to_string(), BucketUsageInfo::default());
-    let cache = Some(CachedDataUsageSnapshot {
-        info: Some(info),
-        loaded_at,
-        degraded_baseline: HashMap::new(),
-    });
-    let stored = cache
-        .as_ref()
-        .expect("cached snapshot")
-        .info
-        .as_ref()
-        .expect("known snapshot");
-    let ttl = Duration::from_secs(30);
-
-    for (bucket, expected) in [("known", Some(42)), ("empty", Some(0)), ("missing", None)] {
-        let value = fresh_cached_data_usage_snapshot(&cache, loaded_at, ttl, |info| {
-            assert!(std::ptr::eq(info, stored), "projection must borrow the cached snapshot");
-            info.buckets_usage.get(bucket).map(|usage| usage.size)
-        })
-        .expect("fresh snapshot")
-        .expect("successful snapshot");
-        let owned = fresh_cached_data_usage_snapshot(&cache, loaded_at, ttl, DataUsageInfo::clone)
-            .expect("fresh snapshot")
-            .expect("successful snapshot");
-        assert_eq!(value, expected);
-        assert_eq!(value, owned.buckets_usage.get(bucket).map(|usage| usage.size));
-    }
-}
-
-#[test]
-fn projection_is_skipped_for_missing_failed_expired_or_invalidated_snapshots() {
-    let loaded_at = tokio::time::Instant::now();
-    let ttl = Duration::from_secs(30);
-    let never_project = |_: &DataUsageInfo| -> u64 { panic!("unusable snapshots must not be projected") };
-    assert!(fresh_cached_data_usage_snapshot(&None, loaded_at, ttl, never_project).is_none());
-
-    let mut cache = None;
-    let failure = cache_data_usage_snapshot_result(&mut cache, Err(Error::ErasureReadQuorum), loaded_at, 1, 1, never_project)
-        .expect("matching generation");
-    assert!(matches!(failure, Err(Error::ErasureReadQuorum)));
-    assert!(
-        fresh_cached_data_usage_snapshot(&cache, loaded_at, ttl, never_project)
-            .expect("negative cache remains fresh")
-            .is_err()
-    );
-
-    cache = Some(CachedDataUsageSnapshot {
-        info: Some(data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH)),
-        loaded_at,
-        degraded_baseline: HashMap::new(),
-    });
-    assert!(fresh_cached_data_usage_snapshot(&cache, loaded_at + ttl, ttl, never_project).is_none());
-    cache = None;
-    let rejected = cache_data_usage_snapshot_result(
-        &mut cache,
-        Ok(LoadedUsageBaseline {
-            info: data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH),
-            degraded_baseline: HashMap::new(),
-            observed_unavailable: false,
-        }),
-        loaded_at,
-        1,
-        2,
-        never_project,
-    );
-    assert!(rejected.is_none());
-    assert!(cache.is_none(), "rejected generation must not republish stale data");
-}
-
-#[test]
-fn projected_publication_moves_bucket_maps_into_cache() {
-    let info = data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH);
-    let original_key = info.buckets_usage.get_key_value("known").expect("source bucket").0.as_ptr();
-    let mut cache = None;
-    let usage = cache_data_usage_snapshot_result(
-        &mut cache,
-        Ok(LoadedUsageBaseline {
-            info,
-            degraded_baseline: HashMap::new(),
-            observed_unavailable: false,
-        }),
-        tokio::time::Instant::now(),
-        1,
-        1,
-        |info| {
-            assert_eq!(
-                info.buckets_usage
-                    .get_key_value("known")
-                    .expect("projected bucket")
-                    .0
-                    .as_ptr(),
-                original_key
-            );
-            info.buckets_usage.get("known").map(|usage| usage.size)
-        },
-    )
-    .expect("matching generation")
-    .expect("successful publication");
-    assert_eq!(usage, Some(42));
-    let cached = cache
-        .as_ref()
-        .expect("published cache")
-        .info
-        .as_ref()
-        .expect("published snapshot");
-    assert_eq!(
-        cached.buckets_usage.get_key_value("known").expect("cached bucket").0.as_ptr(),
-        original_key
-    );
-}
-
-#[tokio::test]
-#[serial]
-async fn persisted_scalar_projection_shares_cache_refresh_and_invalidation() {
-    use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
-    use crate::config::com::save_config;
-
-    let (_dirs, store) = isolated_store_over_temp_disks().await;
-    let mut info = data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH);
-    info.buckets_usage.insert("empty".to_string(), BucketUsageInfo::default());
-    info.bucket_sizes.insert("empty".to_string(), 0);
-    info.buckets_count = 2;
-    save_config(
-        store.clone(),
-        DATA_USAGE_OBJ_NAME_PATH.as_str(),
-        serde_json::to_vec(&info).expect("encode snapshot"),
-    )
-    .await
-    .expect("persist snapshot");
-    invalidate_data_usage_snapshot_cache().await;
-
-    for (bucket, expected) in [("known", Some(42)), ("empty", Some(0)), ("missing", None)] {
-        assert_eq!(persisted_bucket_usage(store.clone(), bucket).await, expected);
-        let owned = load_data_usage_from_backend_cached(store.clone())
-            .await
-            .expect("owned cached snapshot");
-        assert_eq!(expected, owned.buckets_usage.get(bucket).map(|usage| usage.size));
-    }
-    info.buckets_usage.get_mut("known").expect("known bucket").size = 84;
-    info.bucket_sizes.insert("known".to_string(), 84);
-    save_config(
-        store.clone(),
-        DATA_USAGE_OBJ_NAME_PATH.as_str(),
-        serde_json::to_vec(&info).expect("encode new snapshot"),
-    )
-    .await
-    .expect("persist new snapshot without invalidating cache");
-    assert_eq!(
-        persisted_bucket_usage(store.clone(), "known").await,
-        Some(42),
-        "warm reads reuse the TTL cache"
-    );
-    invalidate_data_usage_snapshot_cache().await;
-    assert_eq!(
-        persisted_bucket_usage(store.clone(), "known").await,
-        Some(84),
-        "invalidation exposes the new snapshot"
-    );
-    invalidate_data_usage_snapshot_cache().await;
-    if let Some(cancel) = store.ctx.background_cancel_token() {
-        cancel.cancel();
-    }
-    for pool in &store.pools {
-        for set in &pool.disk_set {
-            let disks = set.disks.read().await.clone();
-            for disk in disks.iter().flatten() {
-                disk.close().await.expect("stop fixture disk monitoring");
-            }
-        }
-    }
-}

crates/ecstore/src/diagnostics/admin_server_info.rs

@@ -393,10 +393,6 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage {
     }
 }
 
-fn is_online_drive_state(state: &str) -> bool {
-    state == DriveState::Ok.to_string() || state == DriveState::Unformatted.to_string()
-}
-
 /// Classify every drive into online / offline / unknown buckets.
 ///
 /// `unknown` holds drives synthesized for a member whose properties RPC could
@@ -416,7 +412,6 @@ fn get_online_offline_disks_stats(disks_info: &[Disk]) -> (BackendDisks, Backend
         unknown_disks.entry(ep.clone()).or_insert(0);
     }
 
-    let mut online_root_disk_count = 0;
     for disk in disks_info {
         let ep = &disk.endpoint;
         let state = &disk.state;
@@ -424,28 +419,30 @@ fn get_online_offline_disks_stats(disks_info: &[Disk]) -> (BackendDisks, Backend
             *unknown_disks.get_mut(ep).expect("endpoint should be in disk map") += 1;
             continue;
         }
-        if !is_online_drive_state(state) {
+        if *state != DriveState::Ok.to_string() && *state != DriveState::Unformatted.to_string() {
             *offline_disks.get_mut(ep).expect("endpoint should be in disk map") += 1;
             continue;
         }
         *online_disks.get_mut(ep).expect("endpoint should be in disk map") += 1;
-        if disk.root_disk {
-            online_root_disk_count += 1;
+    }
+
+    let mut root_disk_count = 0;
+    for di in disks_info {
+        if di.root_disk {
+            root_disk_count += 1;
         }
     }
 
     // When every non-offline, non-unknown drive is a root mount, leave the
     // online tally as-is instead of demoting all of them (matches the prior
     // behavior; the unknown bucket is simply carried through untouched).
-    if disks_info.len()
-        == (online_root_disk_count + offline_disks.values().sum::<usize>() + unknown_disks.values().sum::<usize>())
-    {
+    if disks_info.len() == (root_disk_count + offline_disks.values().sum::<usize>() + unknown_disks.values().sum::<usize>()) {
         return (BackendDisks(online_disks), BackendDisks(offline_disks), BackendDisks(unknown_disks));
     }
 
     for disk in disks_info {
         let ep = &disk.endpoint;
-        if disk.root_disk && is_online_drive_state(&disk.state) {
+        if disk.root_disk {
             *offline_disks.get_mut(ep).expect("endpoint should be in disk map") += 1;
             *online_disks.get_mut(ep).expect("endpoint should be in disk map") -= 1;
         }
@@ -838,35 +835,6 @@ mod tests {
         );
     }
 
-    #[test]
-    fn disk_stats_only_demote_online_root_disks() {
-        let mut offline_root = disk_with_state("http://n3:9000/data", ITEM_OFFLINE);
-        offline_root.root_disk = true;
-        let mut unknown_root = disk_with_state("http://n4:9000/data", ITEM_UNKNOWN);
-        unknown_root.root_disk = true;
-        let mut online_root = disk_with_state("http://n5:9000/data", "ok");
-        online_root.root_disk = true;
-
-        let disks = vec![
-            disk_with_state("http://n1:9000/data", "ok"),
-            disk_with_state("http://n2:9000/data", "ok"),
-            offline_root,
-            unknown_root,
-            online_root,
-        ];
-
-        let (online, offline, unknown) = get_online_offline_disks_stats(&disks);
-
-        assert_eq!(online.sum(), 2, "only the two non-root online drives remain online");
-        assert_eq!(offline.sum(), 2, "the offline root drive and demoted online root drive are offline");
-        assert_eq!(unknown.sum(), 1, "the unknown root drive stays in unknown without underflowing online");
-        assert_eq!(
-            online.sum() + offline.sum() + unknown.sum(),
-            disks.len(),
-            "online + offline + unknown must equal the total drive count"
-        );
-    }
-
     #[test]
     fn topology_reconcile_synthesizes_missing_member_as_unknown() {
         let endpoints = topology_with_hosts(&["rustfs-1", "rustfs-2", "rustfs-3", "rustfs-4"]);

crates/ecstore/src/disk/cleanup_runtime.rs

@@ -16,7 +16,6 @@
 //! PUT admission precedes staging; receiver and GC disk work share a separate
 //! budget so coordinators waiting for peers cannot exhaust receiver capacity.
 
-use crate::disk::io_admission::{current_resource_unit, with_optional_resource_unit};
 use crate::disk::{LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_DISK};
 use std::collections::BTreeSet;
 use std::future::Future;
@@ -423,7 +422,7 @@ impl CleanupRuntime {
         // the result receiver's lifetime to discard work cancelled during
         // runtime scheduling, before its first poll can start filesystem I/O.
         let (result_tx, result_rx) = tokio::sync::oneshot::channel();
-        let task = runtime.spawn(with_optional_resource_unit(current_resource_unit(), async move {
+        let task = runtime.spawn(async move {
             if result_tx.is_closed() {
                 return;
             }
@@ -431,7 +430,7 @@ impl CleanupRuntime {
             // through I/O even if the waiter cancels and closes result_rx.
             let result = execute_disk(pending, Some(active), execution, started, future).await;
             let _ = result_tx.send(result);
-        }));
+        });
         match result_rx.await {
             Ok(result) => result,
             Err(err) => {
@@ -470,11 +469,11 @@ impl CleanupRuntime {
             gates: Some((self.disk_admission.clone(), self.disk_execution.clone())),
         };
         handle
-            .spawn(with_optional_resource_unit(current_resource_unit(), async move {
+            .spawn(async move {
                 let active = execution.acquire_owned().await.map_err(super::error::DiskError::other)?;
                 let _active = Admission::new(active, "rustfs_cleanup_gc_active");
                 scan(budget).await
-            }))
+            })
             .await
             .map_err(super::error::DiskError::other)?
     }
@@ -583,7 +582,6 @@ where
     F::Output: Send + 'static,
 {
     let started = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
-    let future = with_optional_resource_unit(current_resource_unit(), future);
     match reservation {
         Some(reservation) => reservation.handle.clone().spawn(async move {
             // This semaphore is never closed. Both queued work and overflow
@@ -622,62 +620,6 @@ mod tests {
     use std::time::Duration;
     use tokio::sync::oneshot;
 
-    #[tokio::test]
-    async fn cleanup_dispatch_retains_resource_units_without_inheriting_disk_policy() {
-        use crate::disk::io_admission::{current_disk_io_context, with_background_disk_io, with_background_resource_unit};
-        use rustfs_concurrency::adaptive::AdaptiveAdmission;
-        use std::num::NonZeroU16;
-        use tokio_util::sync::CancellationToken;
-
-        for mode in 0..4 {
-            let runtime = Arc::new(CleanupRuntime::new(config(1, 1, 1)).expect("cleanup runtime"));
-            let worker_runtime = runtime.clone();
-            let gate = AdaptiveAdmission::new(NonZeroU16::new(1).expect("one unit"), 1);
-            let (started_tx, started_rx) = tokio::sync::oneshot::channel();
-            let (release_tx, release_rx) = tokio::sync::oneshot::channel();
-            let work = async move {
-                started_tx
-                    .send((current_resource_unit().is_some(), current_disk_io_context().is_none()))
-                    .expect("report worker context");
-                release_rx.await.expect("release owned worker");
-                Ok::<_, super::super::error::DiskError>(())
-            };
-            let task = tokio::spawn(with_background_disk_io(
-                CancellationToken::new(),
-                with_background_resource_unit(gate.acquire().await, async move {
-                    match mode {
-                        0 => worker_runtime
-                            .disk_operation(work, DiskAdmission::Request)
-                            .await
-                            .expect("disk operation"),
-                        1 => worker_runtime.gc_operation(move |_| work).await.expect("GC operation"),
-                        2 => {
-                            let reservation = worker_runtime.reserve().expect("cleanup reservation");
-                            run(Some(reservation), async { work.await.expect("cleanup operation") }).await;
-                        }
-                        _ => spawn(None, work)
-                            .await
-                            .expect("unisolated worker")
-                            .expect("unisolated operation"),
-                    }
-                }),
-            ));
-            let (inherited, original_policy) = deadline(started_rx).await.expect("owned work starts");
-            task.abort();
-            assert!(task.await.expect_err("waiter cancelled").is_cancelled());
-            let held = gate.snapshot().active;
-            release_tx.send(()).expect("owned work survives cancellation");
-            deadline(async {
-                while gate.snapshot().active != 0 {
-                    tokio::task::yield_now().await;
-                }
-            })
-            .await;
-            assert!(inherited, "mode {mode} lost resource ownership");
-            assert!(original_policy, "mode {mode} changed the detached worker's disk policy");
-            assert_eq!(held, 1, "mode {mode} released a live worker's unit");
-        }
-    }
     fn config(pending: usize, workers: usize, blocking_threads: usize) -> Config {
         Config {
             pending,

crates/ecstore/src/disk/fs.rs

@@ -159,6 +159,10 @@ pub fn lstat_std(path: impl AsRef<Path>) -> io::Result<Metadata> {
     std::fs::symlink_metadata(path)
 }
 
+pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
+    fs::create_dir_all(path.as_ref()).await
+}
+
 fn is_dir_error(e: &io::Error) -> bool {
     e.raw_os_error() == Some(libc::EISDIR)
         || e.kind() == io::ErrorKind::IsADirectory
@@ -206,6 +210,10 @@ pub fn remove_all_std(path: impl AsRef<Path>) -> io::Result<()> {
     }
 }
 
+pub async fn mkdir(path: impl AsRef<Path>) -> io::Result<()> {
+    fs::create_dir(path.as_ref()).await
+}
+
 pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
     fs::rename(from, to).await
 }
@@ -464,6 +472,17 @@ mod tests {
         assert!(metadata.file_type().is_symlink());
     }
 
+    #[tokio::test]
+    async fn test_make_dir_all() {
+        let temp_dir = TempDir::new().unwrap();
+        let nested_path = temp_dir.path().join("level1").join("level2").join("level3");
+
+        // Should create nested directories
+        assert!(make_dir_all(&nested_path).await.is_ok());
+        assert!(nested_path.exists());
+        assert!(nested_path.is_dir());
+    }
+
     #[tokio::test]
     async fn test_remove_file() {
         let temp_dir = TempDir::new().unwrap();
@@ -536,6 +555,17 @@ mod tests {
         assert!(!dir_path.exists());
     }
 
+    #[tokio::test]
+    async fn test_mkdir() {
+        let temp_dir = TempDir::new().unwrap();
+        let dir_path = temp_dir.path().join("test_mkdir");
+
+        // Create directory
+        assert!(mkdir(&dir_path).await.is_ok());
+        assert!(dir_path.exists());
+        assert!(dir_path.is_dir());
+    }
+
     #[tokio::test]
     async fn test_rename() {
         let temp_dir = TempDir::new().unwrap();

crates/ecstore/src/disk/io_admission.rs

@@ -24,7 +24,6 @@ use std::task::{Context, Poll};
 use std::time::Duration;
 
 use metrics::{counter, gauge, histogram};
-use rustfs_concurrency::adaptive::AdaptivePermit;
 use rustfs_rio::ChunkReader;
 use tokio::io::{AsyncRead, ReadBuf};
 use tokio::sync::Notify;
@@ -36,7 +35,6 @@ const BACKGROUND_MAX_WAIT: Duration = Duration::from_millis(100);
 
 tokio::task_local! {
     static DISK_IO_CONTEXT: DiskIoContext;
-    static RESOURCE_IO_UNIT: AdaptivePermit;
 }
 
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
@@ -51,7 +49,6 @@ pub(crate) struct DiskIoContext {
     pub(crate) cancellation: CancellationToken,
     pub(crate) admission: Option<Arc<DiskIoAdmission>>,
     pub(crate) permit: Option<DiskIoPermit>,
-    pub(crate) resource_unit: Option<AdaptivePermit>,
 }
 
 /// Mark disk I/O issued by `future` as background work for local-disk admission.
@@ -63,43 +60,12 @@ pub async fn with_background_disk_io<F: Future>(cancellation: CancellationToken,
             cancellation,
             admission: None,
             permit: None,
-            resource_unit: current_resource_unit(),
         },
         future,
     )
     .await
 }
 
-/// Retain an already admitted resource unit without changing disk classification,
-/// cancellation or admission. Acquire it before namespace, cache or commit locks.
-/// Existing background scopes and disk leases propagate it into native I/O.
-/// Use this for physical I/O ownership, not node-wide scheduling capacity that
-/// must return with a cancelled waiter so healthy disks can progress independently
-/// of a stalled native operation. Escaping buffers need their own memory leases.
-#[doc(hidden)]
-pub async fn with_background_resource_unit<F: Future>(unit: AdaptivePermit, future: F) -> F::Output {
-    RESOURCE_IO_UNIT.scope(unit, future).await
-}
-
-pub(crate) fn current_resource_unit() -> Option<AdaptivePermit> {
-    RESOURCE_IO_UNIT.try_with(Clone::clone).ok().or_else(|| {
-        DISK_IO_CONTEXT
-            .try_with(|context| context.resource_unit.clone())
-            .ok()
-            .flatten()
-    })
-}
-
-/// Transfer only resource ownership into an existing spawned task. Disk
-/// classification/permits are not inherited: changing those can introduce a
-/// wait behind a disk permit still held by the parent.
-pub(crate) async fn with_optional_resource_unit<F: Future>(unit: Option<AdaptivePermit>, future: F) -> F::Output {
-    match unit {
-        Some(unit) => with_background_resource_unit(unit, future).await,
-        None => future.await,
-    }
-}
-
 pub(crate) async fn with_disk_io_context<F: Future>(context: DiskIoContext, future: F) -> F::Output {
     DISK_IO_CONTEXT.scope(context, future).await
 }
@@ -113,11 +79,7 @@ pub(crate) async fn with_disk_io_permit<F: Future>(permit: Option<DiskIoPermit>,
         cancellation: CancellationToken::new(),
         admission: None,
         permit: None,
-        resource_unit: None,
     });
-    if context.resource_unit.is_none() {
-        context.resource_unit = permit.lease.resource_unit.clone();
-    }
     context.admission = Some(permit.admission());
     with_disk_io_context(
         DiskIoContext {
@@ -129,32 +91,15 @@ pub(crate) async fn with_disk_io_permit<F: Future>(permit: Option<DiskIoPermit>,
     .await
 }
 
-pub(crate) fn with_optional_disk_io_context<F: Future>(
-    context: Option<DiskIoContext>,
-    future: F,
-) -> impl Future<Output = F::Output> {
-    // Capture at future construction, before JoinSet/spawn moves polling to a
-    // different task. A resource-only scope has no DiskIoContext to carry it.
-    let unit = current_resource_unit();
-    async move {
-        let future = with_optional_resource_unit(unit, future);
-        match context {
-            Some(context) => with_disk_io_context(context, future).await,
-            None => future.await,
-        }
+pub(crate) async fn with_optional_disk_io_context<F: Future>(context: Option<DiskIoContext>, future: F) -> F::Output {
+    match context {
+        Some(context) => with_disk_io_context(context, future).await,
+        None => future.await,
     }
 }
 
 pub(crate) fn current_disk_io_context() -> Option<DiskIoContext> {
-    DISK_IO_CONTEXT
-        .try_with(|context| {
-            let mut context = context.clone();
-            if let Ok(unit) = RESOURCE_IO_UNIT.try_with(Clone::clone) {
-                context.resource_unit = Some(unit);
-            }
-            context
-        })
-        .ok()
+    DISK_IO_CONTEXT.try_with(Clone::clone).ok()
 }
 
 /// Acquire one low-level permit only when the current walker explicitly owns a gate.
@@ -230,10 +175,6 @@ impl DiskIoAdmission {
     }
 
     pub(crate) fn foreground(self: &Arc<Self>) -> DiskIoPermit {
-        self.foreground_with_resource(current_resource_unit())
-    }
-
-    fn foreground_with_resource(self: &Arc<Self>, resource_unit: Option<AdaptivePermit>) -> DiskIoPermit {
         self.foreground_active.fetch_add(1, Ordering::AcqRel);
         if self.background_waiters.load(Ordering::Acquire) > 0 {
             self.record_state();
@@ -242,20 +183,11 @@ impl DiskIoAdmission {
             lease: Arc::new(DiskIoPermitLease {
                 admission: self.clone(),
                 class: DiskIoClass::Foreground,
-                resource_unit,
             }),
         }
     }
 
     pub(crate) async fn background(self: &Arc<Self>, cancellation: &CancellationToken) -> std::io::Result<DiskIoPermit> {
-        self.background_with_resource(cancellation, current_resource_unit()).await
-    }
-
-    async fn background_with_resource(
-        self: &Arc<Self>,
-        cancellation: &CancellationToken,
-        resource_unit: Option<AdaptivePermit>,
-    ) -> std::io::Result<DiskIoPermit> {
         let started_at = Instant::now();
         let ticket = {
             let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
@@ -316,7 +248,6 @@ impl DiskIoAdmission {
                     lease: Arc::new(DiskIoPermitLease {
                         admission: self.clone(),
                         class: DiskIoClass::Background,
-                        resource_unit,
                     }),
                 });
             }
@@ -429,7 +360,6 @@ impl DiskIoPermit {
 struct DiskIoPermitLease {
     admission: Arc<DiskIoAdmission>,
     class: DiskIoClass,
-    resource_unit: Option<AdaptivePermit>,
 }
 
 pub(crate) fn spawn_blocking_with_disk_io_permit<T, F>(
@@ -440,12 +370,8 @@ where
     T: Send + 'static,
     F: FnOnce() -> T + Send + 'static,
 {
-    // A supplied disk permit can predate this resource scope or carry another
-    // owner's unit. Preserve the current unit independently until native I/O ends.
-    let resource_unit = current_resource_unit();
     tokio::task::spawn_blocking(move || {
         let _io_permit = io_permit;
-        let _resource_unit = resource_unit;
         operation()
     })
 }
@@ -510,7 +436,7 @@ impl<R> AdmissionReader<R> {
         }
 
         if self.context.class == DiskIoClass::Foreground {
-            self.permit = Some(self.admission.foreground_with_resource(self.context.resource_unit.clone()));
+            self.permit = Some(self.admission.foreground());
             return Poll::Ready(Ok(()));
         }
 
@@ -518,10 +444,7 @@ impl<R> AdmissionReader<R> {
         if wait.is_none() {
             let admission = self.admission.clone();
             let cancellation = self.context.cancellation.clone();
-            let resource_unit = self.context.resource_unit.clone();
-            *wait = Some(Box::pin(
-                async move { admission.background_with_resource(&cancellation, resource_unit).await },
-            ));
+            *wait = Some(Box::pin(async move { admission.background(&cancellation).await }));
         }
         let Some(wait_future) = wait.as_mut() else {
             return Poll::Ready(Err(io::Error::other("background disk I/O admission waiter is missing")));
@@ -591,166 +514,8 @@ impl ChunkReader for BoxedChunkReader {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use rustfs_concurrency::adaptive::AdaptiveAdmission;
-    use std::num::NonZeroU16;
     use tokio::io::{AsyncReadExt, AsyncWriteExt};
 
-    #[tokio::test]
-    async fn resource_unit_outlives_cancelled_scope_until_native_io_finishes() {
-        for permit_mode in 0..4 {
-            let gate = AdaptiveAdmission::new(NonZeroU16::new(1).expect("one unit"), 1);
-            let previous_gate = AdaptiveAdmission::new(NonZeroU16::new(1).expect("previous owner"), 1);
-            let unit = gate.acquire().await;
-            let disk = DiskIoAdmission::new("resource-unit-native");
-            let existing_permit = match permit_mode {
-                2 => Some(disk.foreground()),
-                3 => Some(with_background_resource_unit(previous_gate.acquire().await, async { disk.foreground() }).await),
-                _ => None,
-            };
-            let (started_tx, started_rx) = tokio::sync::oneshot::channel();
-            let (release_tx, release_rx) = tokio::sync::oneshot::channel();
-            let task = tokio::spawn(with_background_resource_unit(unit, async move {
-                with_background_disk_io(CancellationToken::new(), async {
-                    let permit = if existing_permit.is_some() {
-                        existing_permit
-                    } else if permit_mode == 1 {
-                        Some(disk.background(&CancellationToken::new()).await.expect("disk permit"))
-                    } else {
-                        None
-                    };
-                    spawn_blocking_with_disk_io_permit(permit, move || {
-                        started_tx.send(()).expect("native I/O started");
-                        release_rx.blocking_recv().expect("native I/O release");
-                    })
-                    .await
-                    .expect("native I/O completed");
-                })
-                .await;
-            }));
-            started_rx.await.expect("native worker owns unit");
-            task.abort();
-            assert!(task.await.expect_err("scope was cancelled").is_cancelled());
-            let held = gate.snapshot().active;
-            let previous_held = previous_gate.snapshot().active;
-            let rejected = gate.try_acquire().is_none();
-            gate.set_limit(0);
-            release_tx.send(()).expect("release native I/O");
-            tokio::time::timeout(Duration::from_secs(5), async {
-                while gate.snapshot().active != 0 || previous_gate.snapshot().active != 0 {
-                    tokio::task::yield_now().await;
-                }
-            })
-            .await
-            .expect("native completion releases the unit");
-            gate.set_limit(1);
-            assert!(gate.try_acquire().is_some());
-            assert_eq!(held, 1, "mode {permit_mode}: native I/O still owns the current unit");
-            assert!(rejected, "mode {permit_mode}: a live native worker must retain admission");
-            assert_eq!(previous_held, u32::from(permit_mode == 3), "preserve any older disk lease's unit");
-        }
-    }
-
-    #[tokio::test]
-    async fn escaped_disk_permit_restores_resource_context_without_a_new_unit() {
-        let gate = AdaptiveAdmission::new(NonZeroU16::new(1).expect("one unit"), 1);
-        let disk = DiskIoAdmission::new("resource-unit-escape");
-        let permit = with_background_resource_unit(gate.acquire().await, async {
-            disk.background(&CancellationToken::new()).await.expect("disk permit")
-        })
-        .await;
-        assert_eq!(gate.snapshot().active, 1);
-        with_background_disk_io(CancellationToken::new(), async {
-            with_disk_io_permit(Some(permit), async {
-                let unit = current_resource_unit().expect("escaped disk lease restores unit into an existing context");
-                assert_eq!(gate.snapshot().active, 1, "cloning retains one unit rather than charging twice");
-                drop(unit);
-            })
-            .await;
-        })
-        .await;
-        assert_eq!(gate.snapshot().active, 0);
-    }
-
-    #[tokio::test]
-    async fn escaped_stream_retains_resource_unit_outside_task_local_scope() {
-        let gate = AdaptiveAdmission::new(NonZeroU16::new(1).expect("one unit"), 1);
-        let disk = DiskIoAdmission::new("resource-unit-stream");
-        let (mut writer, inner) = tokio::io::duplex(8);
-        let mut reader = with_background_resource_unit(gate.acquire().await, async {
-            with_background_disk_io(CancellationToken::new(), async {
-                AdmissionReader::new(inner, disk, current_disk_io_context().expect("resource context"))
-            })
-            .await
-        })
-        .await;
-        assert_eq!(gate.snapshot().active, 1);
-        writer.write_all(b"rust").await.expect("write complete stream");
-        drop(writer);
-        let mut bytes = Vec::new();
-        reader.read_to_end(&mut bytes).await.expect("read outside original scope");
-        assert_eq!(bytes, b"rust");
-        assert_eq!(gate.snapshot().active, 1, "reader still owns its context");
-        drop(reader);
-        assert_eq!(gate.snapshot().active, 0);
-    }
-
-    #[tokio::test]
-    async fn resource_scope_preserves_existing_disk_policy_and_cancellation() {
-        let gate = AdaptiveAdmission::new(NonZeroU16::new(1).expect("one unit"), 1);
-        with_background_resource_unit(gate.acquire().await, async {
-            assert!(current_disk_io_context().is_none(), "resource ownership does not classify disk I/O");
-            assert!(current_resource_unit().is_some());
-        })
-        .await;
-        let cancellation = CancellationToken::new();
-        let existing = DiskIoContext {
-            class: DiskIoClass::Foreground,
-            cancellation: cancellation.clone(),
-            admission: None,
-            permit: None,
-            resource_unit: None,
-        };
-        with_disk_io_context(existing, async {
-            with_background_resource_unit(gate.acquire().await, async {
-                cancellation.cancel();
-                let context = current_disk_io_context().expect("existing disk context");
-                assert_eq!(context.class, DiskIoClass::Foreground);
-                assert!(
-                    context.resource_unit.is_some(),
-                    "captured disk context must retain the current resource unit"
-                );
-                assert!(context.cancellation.is_cancelled(), "retain the original cancellation token");
-                assert!(current_resource_unit().is_some());
-            })
-            .await;
-        })
-        .await;
-        assert_eq!(gate.snapshot().active, 0);
-    }
-
-    #[tokio::test]
-    async fn resource_only_scope_survives_existing_optional_context_spawn_boundary() {
-        let gate = AdaptiveAdmission::new(NonZeroU16::new(1).expect("one unit"), 1);
-        let (release_tx, release_rx) = tokio::sync::oneshot::channel();
-        let mut worker = None;
-        with_background_resource_unit(gate.acquire().await, async {
-            worker = Some(tokio::spawn(with_optional_disk_io_context(current_disk_io_context(), async {
-                release_rx.await.expect("release child after caller scope ends");
-                assert!(current_disk_io_context().is_none());
-                assert!(current_resource_unit().is_some());
-            })));
-        })
-        .await;
-        let held = gate.snapshot().active;
-        release_tx.send(()).expect("child remains alive");
-        worker
-            .expect("scoped child was spawned")
-            .await
-            .expect("child preserves resource-only scope");
-        assert_eq!(held, 1);
-        assert_eq!(gate.snapshot().active, 0);
-    }
-
     struct BlockingWorker(tokio::task::JoinHandle<()>);
 
     async fn wait_for_background_waiters(admission: &DiskIoAdmission, expected: usize) {
@@ -871,7 +636,6 @@ mod tests {
                 cancellation: CancellationToken::new(),
                 admission: None,
                 permit: None,
-                resource_unit: None,
             },
         );
         let read = tokio::spawn(async move {

crates/ecstore/src/disk/local/artifact_lease.rs

@@ -1,740 +0,0 @@
-// Copyright 2026 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use crate::disk::error::{DiskError, Result};
-use std::{fs::File, sync::Arc};
-
-/// A target disk's admitted artifact publication. Physical mutation workers
-/// retain this Arc until their final syscall and rollback have finished.
-pub struct ArtifactLease {
-    _active: File,
-    _key_directory: File,
-    _seal_directory: File,
-    _root: File,
-}
-
-pub type ArtifactPutLease = ArtifactLease;
-pub type ArtifactRepairLease = ArtifactLease;
-
-/// Read-only census owns both already-closed physical mutation domains.
-pub(super) struct ArtifactInventoryLease {
-    _put: Arc<ArtifactLease>,
-    _repair: Arc<ArtifactLease>,
-}
-
-pub(super) async fn drain_closed(root: File, key: [u8; 32], deadline: tokio::time::Instant) -> Result<ArtifactInventoryLease> {
-    #[cfg(unix)]
-    {
-        unix::drain_closed(root, key, deadline).await
-    }
-    #[cfg(not(unix))]
-    {
-        let _ = (root, key, deadline);
-        Err(DiskError::UnsupportedDisk)
-    }
-}
-
-#[derive(Clone, Copy)]
-pub(crate) enum Domain {
-    Put,
-    Repair,
-}
-
-impl Domain {
-    fn directory(self) -> &'static str {
-        match self {
-            Self::Put => ".rustfs-artifact-put-seals",
-            Self::Repair => ".rustfs-artifact-repair-seals",
-        }
-    }
-
-    fn closed(self) -> &'static [u8] {
-        match self {
-            Self::Put => b"rustfs-artifact-put-sealed-v1\n",
-            Self::Repair => b"rustfs-artifact-repair-sealed-v1\n",
-        }
-    }
-}
-
-pub(super) async fn pinned_root(disk: &super::LocalDisk) -> Result<File> {
-    #[cfg(target_os = "linux")]
-    let root = disk.mount_lease.clone();
-    #[cfg(all(unix, not(target_os = "linux")))]
-    let root = disk.artifact_identity_root.clone();
-    #[cfg(not(unix))]
-    let _ = disk;
-    tokio::task::spawn_blocking(move || {
-        #[cfg(unix)]
-        {
-            root.try_clone().map_err(DiskError::from)
-        }
-        #[cfg(not(unix))]
-        {
-            Err(DiskError::other("artifact disk leases require Unix"))
-        }
-    })
-    .await
-    .map_err(DiskError::from)?
-}
-
-pub(super) async fn acquire(root: File, key: [u8; 32]) -> Result<Arc<ArtifactPutLease>> {
-    acquire_in(root, key, Domain::Put).await
-}
-
-pub(super) async fn acquire_in(root: File, key: [u8; 32], domain: Domain) -> Result<Arc<ArtifactLease>> {
-    #[cfg(unix)]
-    {
-        unix::acquire(root, key, domain).await
-    }
-    #[cfg(not(unix))]
-    {
-        let _ = (root, key, domain);
-        Err(DiskError::other("artifact disk leases require Unix"))
-    }
-}
-
-pub(super) async fn seal(root: File, key: [u8; 32]) -> Result<()> {
-    seal_in(root, key, Domain::Put).await
-}
-
-pub(super) async fn seal_in(root: File, key: [u8; 32], domain: Domain) -> Result<()> {
-    #[cfg(unix)]
-    {
-        unix::seal(root, key, domain).await
-    }
-    #[cfg(not(unix))]
-    {
-        let _ = (root, key, domain);
-        Err(DiskError::other("artifact disk leases require Unix"))
-    }
-}
-
-#[cfg(unix)]
-mod unix {
-    use super::*;
-    use rustix::fs::{FlockOperation, Mode, OFlags, flock, mkdirat, openat};
-    use rustix::io::Errno;
-    use std::{io, io::Read, io::Write, os::unix::fs::MetadataExt, time::Duration};
-
-    const POLL_INTERVAL: Duration = Duration::from_millis(10);
-    const LOCK_TIMEOUT: Duration = Duration::from_secs(30);
-
-    struct Opened {
-        root: File,
-        seals: File,
-        key: File,
-        admission: File,
-        active: File,
-        domain: Domain,
-    }
-
-    fn invalid() -> io::Error {
-        io::Error::new(io::ErrorKind::InvalidData, "invalid artifact seal filesystem state")
-    }
-
-    fn directory(parent: &File, name: &str, device: u64) -> io::Result<File> {
-        match mkdirat(parent, name, Mode::RWXU) {
-            Ok(()) | Err(Errno::EXIST) => {}
-            Err(error) => return Err(error.into()),
-        }
-        let file = File::from(openat(
-            parent,
-            name,
-            OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
-            Mode::empty(),
-        )?);
-        let metadata = file.metadata()?;
-        if !metadata.is_dir() || metadata.dev() != device || metadata.mode() & 0o077 != 0 {
-            return Err(invalid());
-        }
-        Ok(file)
-    }
-
-    fn regular(file: &File, device: u64) -> io::Result<std::fs::Metadata> {
-        let metadata = file.metadata()?;
-        if !metadata.is_file() || metadata.dev() != device || metadata.nlink() != 1 || metadata.mode() & 0o077 != 0 {
-            return Err(invalid());
-        }
-        Ok(metadata)
-    }
-
-    fn lock_file(parent: &File, name: &str, device: u64) -> io::Result<File> {
-        let file = File::from(openat(
-            parent,
-            name,
-            OFlags::CREATE | OFlags::RDWR | OFlags::NOFOLLOW | OFlags::CLOEXEC | OFlags::NONBLOCK,
-            Mode::RUSR | Mode::WUSR,
-        )?);
-        if regular(&file, device)?.len() != 0 {
-            return Err(invalid());
-        }
-        Ok(file)
-    }
-
-    impl Opened {
-        fn open(root: File, key: [u8; 32], domain: Domain) -> io::Result<Self> {
-            let metadata = root.metadata()?;
-            if !metadata.is_dir() {
-                return Err(invalid());
-            }
-            let seals = directory(&root, domain.directory(), metadata.dev())?;
-            let name = hex_simd::encode_to_string(key, hex_simd::AsciiCase::Lower);
-            let key = directory(&seals, &name, metadata.dev())?;
-            // Neither lock inode is ever removed or replaced, including after
-            // closing. Independent processes must always lock the same inode.
-            let admission = lock_file(&key, "admission.lock", metadata.dev())?;
-            let active = lock_file(&key, "active.lock", metadata.dev())?;
-            Ok(Self {
-                root,
-                seals,
-                key,
-                admission,
-                active,
-                domain,
-            })
-        }
-
-        fn closed(&self) -> io::Result<Option<File>> {
-            self.closed_with_flags(OFlags::RDWR)
-        }
-
-        fn closed_with_flags(&self, access: OFlags) -> io::Result<Option<File>> {
-            let mut file = match openat(
-                &self.key,
-                "closed",
-                access | OFlags::NOFOLLOW | OFlags::CLOEXEC | OFlags::NONBLOCK,
-                Mode::empty(),
-            ) {
-                Ok(file) => File::from(file),
-                Err(Errno::NOENT) => return Ok(None),
-                Err(error) => return Err(error.into()),
-            };
-            let length = u64::try_from(self.domain.closed().len()).map_err(|_| invalid())?;
-            if regular(&file, self.root.metadata()?.dev())?.len() != length {
-                return Err(invalid());
-            }
-            let mut bytes = vec![0; self.domain.closed().len()];
-            file.read_exact(&mut bytes)?;
-            if bytes != self.domain.closed() {
-                return Err(invalid());
-            }
-            Ok(Some(file))
-        }
-
-        fn admit(self) -> io::Result<Arc<ArtifactLease>> {
-            if self.closed()?.is_some() {
-                return Err(io::Error::new(
-                    io::ErrorKind::PermissionDenied,
-                    "artifact attempt mutation domain is closed",
-                ));
-            }
-            // Admission serializes the closed check with taking this shared
-            // physical owner. A sealer cannot pass between those two steps.
-            flock(&self.active, FlockOperation::NonBlockingLockShared)?;
-            flock(&self.admission, FlockOperation::Unlock)?;
-            Ok(Arc::new(ArtifactLease {
-                _active: self.active,
-                _key_directory: self.key,
-                _seal_directory: self.seals,
-                _root: self.root,
-            }))
-        }
-
-        fn close(self) -> io::Result<Self> {
-            let marker = match self.closed()? {
-                Some(marker) => marker,
-                None => {
-                    let mut marker = File::from(openat(
-                        &self.key,
-                        "closed",
-                        OFlags::CREATE | OFlags::EXCL | OFlags::RDWR | OFlags::NOFOLLOW | OFlags::CLOEXEC,
-                        Mode::RUSR | Mode::WUSR,
-                    )?);
-                    regular(&marker, self.root.metadata()?.dev())?;
-                    marker.write_all(self.domain.closed())?;
-                    marker
-                }
-            };
-            // No admission is released until the permanent denial and every
-            // ancestor entry are durable. A failed/cancelled close never removes it.
-            marker.sync_all()?;
-            self.admission.sync_all()?;
-            self.active.sync_all()?;
-            self.key.sync_all()?;
-            self.seals.sync_all()?;
-            self.root.sync_all()?;
-            flock(&self.admission, FlockOperation::Unlock)?;
-            Ok(self)
-        }
-    }
-
-    async fn lock(file: &File, operation: FlockOperation) -> io::Result<()> {
-        tokio::time::timeout(LOCK_TIMEOUT, async {
-            loop {
-                match flock(file, operation) {
-                    Ok(()) => return Ok(()),
-                    Err(Errno::WOULDBLOCK | Errno::INTR) => tokio::time::sleep(POLL_INTERVAL).await,
-                    Err(error) => return Err(error.into()),
-                }
-            }
-        })
-        .await
-        .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "artifact disk lease lock deadline"))?
-    }
-
-    async fn open(root: File, key: [u8; 32], domain: Domain) -> Result<Opened> {
-        tokio::task::spawn_blocking(move || Opened::open(root, key, domain))
-            .await
-            .map_err(DiskError::from)?
-            .map_err(DiskError::from)
-    }
-
-    fn read_error(error: io::Error) -> DiskError {
-        match error.kind() {
-            io::ErrorKind::NotFound => DiskError::other("artifact inventory requires closed PUT and repair domains"),
-            io::ErrorKind::InvalidData | io::ErrorKind::NotADirectory => DiskError::FileCorrupt,
-            io::ErrorKind::TimedOut => DiskError::Timeout,
-            _ if error.raw_os_error() == Some(Errno::LOOP.raw_os_error()) => DiskError::FileCorrupt,
-            _ => DiskError::from(error),
-        }
-    }
-
-    fn existing_directory(parent: &File, name: &str, device: u64) -> io::Result<File> {
-        let file = File::from(openat(
-            parent,
-            name,
-            OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
-            Mode::empty(),
-        )?);
-        let metadata = file.metadata()?;
-        if metadata.dev() != device || metadata.mode() & 0o077 != 0 {
-            return Err(invalid());
-        }
-        Ok(file)
-    }
-
-    fn existing(root: File, key: [u8; 32], domain: Domain) -> io::Result<Opened> {
-        let device = root.metadata()?.dev();
-        let seals = existing_directory(&root, domain.directory(), device)?;
-        let name = hex_simd::encode_to_string(key, hex_simd::AsciiCase::Lower);
-        let key = existing_directory(&seals, &name, device)?;
-        let lock = |name| -> io::Result<File> {
-            let file = File::from(openat(
-                &key,
-                name,
-                OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC | OFlags::NONBLOCK,
-                Mode::empty(),
-            )?);
-            if regular(&file, device)?.len() != 0 {
-                return Err(invalid());
-            }
-            Ok(file)
-        };
-        let admission = lock("admission.lock")?;
-        let active = lock("active.lock")?;
-        let opened = Opened {
-            root,
-            seals,
-            key,
-            admission,
-            active,
-            domain,
-        };
-        if opened.closed_with_flags(OFlags::RDONLY)?.is_none() {
-            return Err(io::Error::from(io::ErrorKind::NotFound));
-        }
-        Ok(opened)
-    }
-
-    fn retained(opened: Opened) -> Arc<ArtifactLease> {
-        Arc::new(ArtifactLease {
-            _active: opened.active,
-            _key_directory: opened.key,
-            _seal_directory: opened.seals,
-            _root: opened.root,
-        })
-    }
-
-    pub(super) async fn drain_closed(
-        root: File,
-        key: [u8; 32],
-        deadline: tokio::time::Instant,
-    ) -> Result<ArtifactInventoryLease> {
-        let (put, repair) = tokio::task::spawn_blocking(move || {
-            let put = existing(root.try_clone()?, key, Domain::Put)?;
-            let repair = existing(root, key, Domain::Repair)?;
-            Ok::<_, io::Error>((put, repair))
-        })
-        .await
-        .map_err(DiskError::from)?
-        .map_err(read_error)?;
-        // Lock order: closed PUT active exclusive -> closed repair active
-        // exclusive. No admission lock is acquired and no seal is created.
-        tokio::time::timeout_at(deadline, async {
-            lock(&put.active, FlockOperation::NonBlockingLockExclusive).await?;
-            lock(&repair.active, FlockOperation::NonBlockingLockExclusive).await
-        })
-        .await
-        .map_err(|_| DiskError::Timeout)?
-        .map_err(read_error)?;
-        Ok(ArtifactInventoryLease {
-            _put: retained(put),
-            _repair: retained(repair),
-        })
-    }
-
-    pub(super) async fn acquire(root: File, key: [u8; 32], domain: Domain) -> Result<Arc<ArtifactLease>> {
-        let opened = open(root, key, domain).await?;
-        // Lock order: admission exclusive -> active shared. Contention sleeps
-        // asynchronously; no blocking executor thread waits for another owner.
-        lock(&opened.admission, FlockOperation::NonBlockingLockExclusive).await?;
-        tokio::task::spawn_blocking(move || opened.admit())
-            .await
-            .map_err(DiskError::from)?
-            .map_err(DiskError::from)
-    }
-
-    pub(super) async fn seal(root: File, key: [u8; 32], domain: Domain) -> Result<()> {
-        let opened = open(root, key, domain).await?;
-        lock(&opened.admission, FlockOperation::NonBlockingLockExclusive).await?;
-        let opened = tokio::task::spawn_blocking(move || opened.close())
-            .await
-            .map_err(DiskError::from)??;
-        // Closing is already durable. Cancellation or timeout here only loses
-        // the confirmation; admission stays permanently closed across reopen.
-        lock(&opened.active, FlockOperation::NonBlockingLockExclusive).await?;
-        Ok(())
-    }
-}
-
-#[cfg(all(test, unix))]
-mod tests {
-    use super::*;
-    use std::{path::Path, time::Duration};
-
-    const KEY: [u8; 32] = [7; 32];
-
-    fn root(path: &Path) -> File {
-        use rustix::fs::{Mode, OFlags, open};
-        File::from(
-            open(
-                path,
-                OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
-                Mode::empty(),
-            )
-            .expect("open pinned test root"),
-        )
-    }
-
-    fn directory(path: &Path) -> std::path::PathBuf {
-        path.join(Domain::Put.directory())
-            .join(hex_simd::encode_to_string(KEY, hex_simd::AsciiCase::Lower))
-    }
-
-    async fn closed(path: &Path) {
-        tokio::time::timeout(Duration::from_secs(5), async {
-            while !directory(path).join("closed").exists() {
-                tokio::time::sleep(Duration::from_millis(5)).await;
-            }
-        })
-        .await
-        .expect("sealer reaches its monotonic close point");
-    }
-
-    #[tokio::test]
-    async fn artifact_repair_closing_domain_is_independent_and_survives_reopen() {
-        let temp = tempfile::tempdir().expect("disk root");
-        seal(root(temp.path()), KEY).await.expect("PUT closes first");
-        let repair = acquire_in(root(temp.path()), KEY, Domain::Repair)
-            .await
-            .expect("PUT closure permits existing-version repair");
-        assert!(acquire(root(temp.path()), KEY).await.is_err());
-        drop(repair);
-        seal_in(root(temp.path()), KEY, Domain::Repair)
-            .await
-            .expect("repair closes independently");
-        assert!(acquire_in(root(temp.path()), KEY, Domain::Repair).await.is_err());
-        seal_in(root(temp.path()), KEY, Domain::Repair)
-            .await
-            .expect("reopened repair close is durable");
-
-        let other = [8; 32];
-        seal_in(root(temp.path()), other, Domain::Repair)
-            .await
-            .expect("close only repair admission");
-        drop(
-            acquire(root(temp.path()), other)
-                .await
-                .expect("repair closure permits PUT domain"),
-        );
-    }
-
-    #[tokio::test]
-    async fn artifact_repair_cancelled_waiter_keeps_physical_owner_and_durable_close() {
-        let temp = tempfile::tempdir().expect("disk root");
-        let writer = acquire_in(root(temp.path()), KEY, Domain::Repair)
-            .await
-            .expect("repair admitted");
-        let (entered, started) = tokio::sync::oneshot::channel();
-        let (release, waiting) = std::sync::mpsc::channel();
-        let owner = tokio::spawn(async move {
-            tokio::task::spawn_blocking(move || {
-                let _owner = writer;
-                let _ = entered.send(());
-                let _ = waiting.recv();
-            })
-            .await
-            .expect("repair worker");
-        });
-        started.await.expect("physical repair entered");
-        owner.abort();
-        assert!(owner.await.expect_err("cancel repair waiter").is_cancelled());
-        let sealer = tokio::spawn(seal_in(root(temp.path()), KEY, Domain::Repair));
-        let marker = temp
-            .path()
-            .join(Domain::Repair.directory())
-            .join(hex_simd::encode_to_string(KEY, hex_simd::AsciiCase::Lower))
-            .join("closed");
-        tokio::time::timeout(Duration::from_secs(5), async {
-            while !marker.exists() {
-                tokio::time::sleep(Duration::from_millis(5)).await;
-            }
-        })
-        .await
-        .expect("repair admission durably closes");
-        assert!(!sealer.is_finished(), "ARTIFACT_REPAIR_CANCELLED_PHYSICAL_TAIL");
-        sealer.abort();
-        assert!(sealer.await.expect_err("cancel repair close waiter").is_cancelled());
-        assert!(acquire_in(root(temp.path()), KEY, Domain::Repair).await.is_err());
-        let pending = seal_in(root(temp.path()), KEY, Domain::Repair);
-        tokio::pin!(pending);
-        assert!(
-            tokio::time::timeout(Duration::from_millis(50), &mut pending).await.is_err(),
-            "ARTIFACT_REPAIR_REOPEN_MUST_DRAIN_PHYSICAL_TAIL"
-        );
-        release.send(()).expect("release physical repair");
-        tokio::time::timeout(Duration::from_secs(5), &mut pending)
-            .await
-            .expect("physical drain bounded")
-            .expect("reopened close confirms repair drain");
-        assert!(acquire_in(root(temp.path()), KEY, Domain::Repair).await.is_err());
-    }
-
-    #[tokio::test]
-    async fn artifact_put_seal_waits_for_independent_openers_and_last_clone() {
-        let temp = tempfile::tempdir().expect("disk root");
-        let first = acquire(root(temp.path()), KEY).await.expect("first admitted writer");
-        let tail = first.clone();
-        let second = acquire(root(temp.path()), KEY).await.expect("independent shared writer");
-        let pending = seal(root(temp.path()), KEY);
-        tokio::pin!(pending);
-        assert!(
-            tokio::time::timeout(Duration::from_millis(50), &mut pending).await.is_err(),
-            "ARTIFACT_PUT_ACTIVE_WRITERS_DRAIN"
-        );
-        drop(first);
-        drop(second);
-        assert!(
-            tokio::time::timeout(Duration::from_millis(50), &mut pending).await.is_err(),
-            "ARTIFACT_PUT_PHYSICAL_CLONE_RETAINED"
-        );
-        assert!(acquire(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_CLOSE_REJECTS_ADMISSION");
-        drop(tail);
-        tokio::time::timeout(Duration::from_secs(5), &mut pending)
-            .await
-            .expect("physical tail drain bounded")
-            .expect("seal complete");
-        seal(root(temp.path()), KEY)
-            .await
-            .expect("independent reopen preserves closed state");
-        assert!(acquire(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_CLOSED_AFTER_REOPEN");
-    }
-
-    #[tokio::test]
-    async fn artifact_put_close_cancellation_leaves_durable_denial() {
-        let temp = tempfile::tempdir().expect("disk root");
-        let writer = acquire(root(temp.path()), KEY).await.expect("admitted writer");
-        let sealer = tokio::spawn(seal(root(temp.path()), KEY));
-        closed(temp.path()).await;
-        sealer.abort();
-        assert!(sealer.await.expect_err("cancel close waiter").is_cancelled());
-        assert!(
-            acquire(root(temp.path()), KEY).await.is_err(),
-            "ARTIFACT_PUT_CANCELLED_CLOSE_STAYS_CLOSED"
-        );
-        drop(writer);
-        seal(root(temp.path()), KEY).await.expect("reopen finishes physical drain");
-        assert!(acquire(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_CANCELLED_CLOSE_REOPEN");
-    }
-
-    #[tokio::test]
-    async fn artifact_put_physical_worker_retains_lease_after_waiter_cancellation() {
-        let temp = tempfile::tempdir().expect("disk root");
-        let writer = acquire(root(temp.path()), KEY).await.expect("physical writer admitted");
-        let (entered, started) = tokio::sync::oneshot::channel();
-        let (release, waiting) = std::sync::mpsc::channel();
-        let owner = tokio::spawn(async move {
-            tokio::task::spawn_blocking(move || {
-                let _physical_owner = writer;
-                let _ = entered.send(());
-                let _ = waiting.recv();
-            })
-            .await
-            .expect("physical worker");
-        });
-        started.await.expect("physical syscall entered");
-        owner.abort();
-        assert!(owner.await.expect_err("cancel async writer").is_cancelled());
-        let pending = seal(root(temp.path()), KEY);
-        tokio::pin!(pending);
-        assert!(
-            tokio::time::timeout(Duration::from_millis(50), &mut pending).await.is_err(),
-            "ARTIFACT_PUT_CANCELLED_PHYSICAL_TAIL"
-        );
-        release.send(()).expect("release physical worker");
-        tokio::time::timeout(Duration::from_secs(5), &mut pending)
-            .await
-            .expect("physical worker drains")
-            .expect("seal confirms real completion");
-    }
-
-    #[tokio::test]
-    async fn artifact_put_seal_waits_for_another_process() {
-        const CHILD_ROOT: &str = "RUSTFS_TEST_ARTIFACT_PUT_SEAL_ROOT";
-        if let Some(path) = std::env::var_os(CHILD_ROOT) {
-            seal(root(Path::new(&path)), KEY)
-                .await
-                .expect("child closes and drains parent writer");
-            assert!(acquire(root(Path::new(&path)), KEY).await.is_err(), "ARTIFACT_PUT_CHILD_CLOSED");
-            return;
-        }
-        struct Child(std::process::Child);
-        impl Drop for Child {
-            fn drop(&mut self) {
-                let _ = self.0.kill();
-                let _ = self.0.wait();
-            }
-        }
-        let temp = tempfile::tempdir().expect("disk root");
-        let writer = acquire(root(temp.path()), KEY).await.expect("parent owns physical write");
-        let mut child = Child(
-            std::process::Command::new(std::env::current_exe().expect("test executable"))
-                .args([
-                    "--exact",
-                    "disk::local::artifact_lease::tests::artifact_put_seal_waits_for_another_process",
-                ])
-                .env(CHILD_ROOT, temp.path())
-                .spawn()
-                .expect("start independent sealer"),
-        );
-        closed(temp.path()).await;
-        assert!(child.0.try_wait().expect("probe child").is_none(), "ARTIFACT_PUT_CROSS_PROCESS_DRAIN");
-        assert!(
-            acquire(root(temp.path()), KEY).await.is_err(),
-            "ARTIFACT_PUT_CROSS_PROCESS_ADMISSION_CLOSED"
-        );
-        drop(writer);
-        let status = tokio::time::timeout(Duration::from_secs(5), async {
-            loop {
-                if let Some(status) = child.0.try_wait().expect("child wait") {
-                    break status;
-                }
-                tokio::time::sleep(Duration::from_millis(10)).await;
-            }
-        })
-        .await
-        .expect("child physically drains");
-        assert!(status.success(), "ARTIFACT_PUT_CROSS_PROCESS_SEAL");
-        assert!(acquire(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_PROCESS_REOPEN_CLOSED");
-    }
-
-    #[tokio::test]
-    async fn artifact_put_lease_rejects_symlinks_at_every_seal_component() {
-        use std::os::unix::fs::{PermissionsExt, symlink};
-        for component in ["directory", "key", "admission.lock", "active.lock", "closed"] {
-            let temp = tempfile::tempdir().expect("disk root");
-            let outside = tempfile::tempdir().expect("symlink destination");
-            let sentinel = outside.path().join("untouched");
-            std::fs::write(&sentinel, b"unchanged").expect("sentinel");
-            let target = match component {
-                "directory" => temp.path().join(Domain::Put.directory()),
-                "key" => {
-                    std::fs::create_dir(temp.path().join(Domain::Put.directory())).expect("seal directory");
-                    std::fs::set_permissions(temp.path().join(Domain::Put.directory()), std::fs::Permissions::from_mode(0o700))
-                        .expect("private seal directory");
-                    directory(temp.path())
-                }
-                file => {
-                    drop(acquire(root(temp.path()), KEY).await.expect("initialize seal files"));
-                    let target = directory(temp.path()).join(file);
-                    if file != "closed" {
-                        std::fs::remove_file(&target).expect("replace fixture lock");
-                    }
-                    target
-                }
-            };
-            symlink(
-                if matches!(component, "directory" | "key") {
-                    outside.path()
-                } else {
-                    &sentinel
-                },
-                target,
-            )
-            .expect("symlink fixture");
-            assert!(
-                acquire(root(temp.path()), KEY).await.is_err(),
-                "ARTIFACT_PUT_SYMLINK_REJECTED: {component}"
-            );
-            assert!(
-                seal(root(temp.path()), KEY).await.is_err(),
-                "ARTIFACT_PUT_SYMLINK_SEAL_REJECTED: {component}"
-            );
-            assert_eq!(std::fs::read(&sentinel).expect("sentinel intact"), b"unchanged");
-        }
-    }
-
-    #[tokio::test]
-    async fn artifact_put_lease_rejects_corrupt_closed_marker() {
-        use std::os::unix::fs::PermissionsExt;
-        let temp = tempfile::tempdir().expect("disk root");
-        drop(acquire(root(temp.path()), KEY).await.expect("initialize seal files"));
-        let marker = directory(temp.path()).join("closed");
-        std::fs::write(&marker, b"partial").expect("corrupt marker fixture");
-        std::fs::set_permissions(&marker, std::fs::Permissions::from_mode(0o600)).expect("marker permissions");
-        assert!(acquire(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_CORRUPT_MARKER_FAILS_CLOSED");
-        assert!(seal(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_CORRUPT_MARKER_NOT_CONFIRMED");
-        assert_eq!(std::fs::read(marker).expect("corruption retained"), b"partial");
-    }
-
-    #[tokio::test]
-    async fn artifact_put_lease_stays_on_pinned_root_after_path_replacement() {
-        let temp = tempfile::tempdir().expect("disk root parent");
-        let current = temp.path().join("current");
-        let original = temp.path().join("original");
-        std::fs::create_dir(&current).expect("initial disk root");
-        let pin = root(&current);
-        drop(
-            acquire(pin.try_clone().expect("root pin clone"), KEY)
-                .await
-                .expect("initialize pinned seal"),
-        );
-        std::fs::rename(&current, &original).expect("replace configured pathname");
-        std::fs::create_dir(&current).expect("replacement disk root");
-        seal(pin, KEY).await.expect("close original disk through its descriptor");
-        assert!(directory(&original).join("closed").exists(), "ARTIFACT_PUT_ROOT_PINNED");
-        assert!(!current.join(Domain::Put.directory()).exists(), "ARTIFACT_PUT_REPLACEMENT_UNTOUCHED");
-        assert!(acquire(root(&original), KEY).await.is_err());
-    }
-}

crates/ecstore/src/disk/local/commit.rs

@@ -21,11 +21,10 @@ use super::run_destination_commit_directory_preparation;
 use super::should_fail_local_inline_rollback_hardlink;
 use super::{
     EVENT_DISK_LOCAL_ACCESS_FAILED, EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, EVENT_DISK_LOCAL_RENAME_REJECTED, LOG_COMPONENT_ECSTORE,
-    LOG_SUBSYSTEM_DISK_LOCAL, LocalDisk, SyncMode, effective_durability, has_inline_commit_test_failpoints,
-    inline_metadata_rollback_dir, observe_old_current, remove_dir_all_if_exists, remove_dst_base_before_commit,
-    remove_file_if_exists, rename_data_versions_signature, run_inline_preparation_before_backup,
-    should_fail_after_metadata_commit, should_fail_before_old_metadata_backup, should_fail_commit_rename,
-    should_remove_staged_meta_before_commit, skip_access_checks, tmp_staging_mutation_path,
+    LOG_SUBSYSTEM_DISK_LOCAL, LocalDisk, SyncMode, effective_durability, inline_metadata_rollback_dir, observe_old_current,
+    remove_dir_all_if_exists, remove_dst_base_before_commit, remove_file_if_exists, rename_data_versions_signature,
+    run_inline_preparation_before_backup, should_fail_after_metadata_commit, should_fail_before_old_metadata_backup,
+    should_fail_commit_rename, should_remove_staged_meta_before_commit, skip_access_checks, tmp_staging_mutation_path,
 };
 #[cfg(test)]
 use super::{run_inline_before_file_sync_admission, run_owned_file_write_before_open, run_rename_data_after_first_publication};
@@ -106,7 +105,16 @@ impl StagedRollbackBackup {
 
 impl RenameCommitContext<'_> {
     fn rollback_metadata_std(&self, rollback_data_dir: Option<Uuid>) -> std::io::Result<()> {
-        rollback_metadata_std(self.destination_metadata, rollback_data_dir)
+        if let Some(old_data_dir) = rollback_data_dir {
+            let Some(dst_parent) = self.destination_metadata.parent() else {
+                return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing object metadata parent"));
+            };
+            let backup_path = dst_parent.join(old_data_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP);
+            std::fs::rename(backup_path, self.destination_metadata)?;
+        } else {
+            remove_file_if_exists(self.destination_metadata)?;
+        }
+        Ok(())
     }
 
     fn rollback_non_inline_std(&self, rollback_data_dir: Option<Uuid>) -> std::io::Result<()> {
@@ -122,7 +130,16 @@ impl RenameCommitContext<'_> {
         rollback_data_dir: Option<Uuid>,
         local_backup: Option<&StagedRollbackBackup>,
     ) -> std::io::Result<()> {
-        rollback_inline_metadata_std(self.destination_metadata, rollback_data_dir, local_backup)
+        #[cfg(all(test, not(windows)))]
+        os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Rollback, self.destination_metadata);
+        if let Some(backup) = local_backup {
+            // The preceding commit used this staged directory on the same
+            // filesystem. Never unlink xl.meta before its atomic replacement.
+            std::fs::rename(backup.path(), self.destination_metadata)?;
+        } else {
+            self.rollback_metadata_std(rollback_data_dir)?;
+        }
+        Ok(())
     }
 
     async fn restore_published_data_source(&self) -> Result<()> {
@@ -190,106 +207,6 @@ pub(super) fn create_local_inline_rollback_backup(
     Ok(backup_path)
 }
 
-fn rollback_metadata_std(destination_metadata: &Path, rollback_data_dir: Option<Uuid>) -> std::io::Result<()> {
-    if let Some(old_data_dir) = rollback_data_dir {
-        let Some(dst_parent) = destination_metadata.parent() else {
-            return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing object metadata parent"));
-        };
-        let backup_path = dst_parent.join(old_data_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP);
-        std::fs::rename(backup_path, destination_metadata)?;
-    } else {
-        remove_file_if_exists(destination_metadata)?;
-    }
-    Ok(())
-}
-
-fn rollback_inline_metadata_std(
-    destination_metadata: &Path,
-    rollback_data_dir: Option<Uuid>,
-    local_backup: Option<&StagedRollbackBackup>,
-) -> std::io::Result<()> {
-    #[cfg(all(test, not(windows)))]
-    os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Rollback, destination_metadata);
-    if let Some(backup) = local_backup {
-        // The preceding commit used this staged directory on the same
-        // filesystem. Never unlink xl.meta before its atomic replacement.
-        std::fs::rename(backup.path(), destination_metadata)?;
-        Ok(())
-    } else {
-        rollback_metadata_std(destination_metadata, rollback_data_dir)
-    }
-}
-
-fn fsync_inline_commit_std(
-    dst: &Path,
-    dst_volume_dir: &Path,
-    destination_was_absent: bool,
-    local_backup: Option<&StagedRollbackBackup>,
-    commit_guard: &os::RenameCommitGuard,
-) -> std::io::Result<()> {
-    if let Some(dst_parent) = dst.parent() {
-        let fsync_started = rustfs_io_metrics::put_stage_timer();
-        if let Err(err) = os::fsync_commit_directory_limited_std(dst_parent, commit_guard) {
-            rustfs_io_metrics::record_put_object_stage_duration_from(
-                rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC,
-                fsync_started,
-            );
-            rollback_inline_metadata_std(dst, None, local_backup)?;
-            return Err(err);
-        }
-        rustfs_io_metrics::record_put_object_stage_duration_from(
-            rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC,
-            fsync_started,
-        );
-    }
-
-    if destination_was_absent {
-        let mut ancestor = dst.parent().and_then(Path::parent);
-        while let Some(ancestor_dir) = ancestor {
-            if !ancestor_dir.starts_with(dst_volume_dir) {
-                break;
-            }
-            let fsync_started = rustfs_io_metrics::put_stage_timer();
-            if let Err(err) = os::fsync_dir_limited_std(ancestor_dir) {
-                rustfs_io_metrics::record_put_object_stage_duration_from(
-                    rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC,
-                    fsync_started,
-                );
-                rollback_inline_metadata_std(dst, None, local_backup)?;
-                return Err(err);
-            }
-            rustfs_io_metrics::record_put_object_stage_duration_from(
-                rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC,
-                fsync_started,
-            );
-            if ancestor_dir == dst_volume_dir {
-                break;
-            }
-            ancestor = ancestor_dir.parent();
-        }
-    }
-    Ok(())
-}
-
-type PreparedInlineMetadata = (
-    Option<Uuid>,
-    Option<Uuid>,
-    Option<Vec<u8>>,
-    Option<crate::disk::OldCurrentSize>,
-    bool,
-    Option<FileInfo>,
-    Option<StagedRollbackBackup>,
-    bool,
-    os::PreparedRenameSource,
-);
-
-enum InlinePreparationOutcome {
-    Committed(RenameDataResp),
-    CommitErr(DiskError),
-    Prepared(PreparedInlineMetadata),
-}
-
-#[cfg(test)]
 pub(super) async fn lock_rename_commit_directories(
     source_parent: &Path,
     destination_parent: &Path,
@@ -327,74 +244,33 @@ pub(super) async fn lock_rename_commit_directories(
     result.map_err(to_file_error).map_err(DiskError::from)
 }
 
-async fn prepare_rename_commit_and_destination_metadata(
-    create_source_parent: bool,
-    source_parent: &Path,
-    destination_parent: &Path,
-    destination_metadata: &Path,
-    base_dir: &Path,
-    publication_root: &os::PublicationRoot,
+async fn prepare_rename_destination_metadata(
+    file_path: &Path,
+    rename_commit_guard: &os::RenameCommitGuard,
     mutation_lease: Arc<os::NamespaceMutationLease>,
-) -> Result<(os::RenameCommitGuard, Option<Bytes>)> {
+) -> Result<Option<Bytes>> {
     #[cfg(any(windows, target_os = "linux"))]
     let result = {
-        let source_parent = source_parent.to_path_buf();
-        let destination_parent = destination_parent.to_path_buf();
-        let destination_metadata = destination_metadata.to_path_buf();
-        let base_dir = base_dir.to_path_buf();
-        let publication_root = publication_root.clone();
+        let file_path = file_path.to_path_buf();
+        let rename_commit_guard = rename_commit_guard.clone();
         os::run_blocking_namespace_operation(mutation_lease, move || {
-            if create_source_parent {
-                #[cfg(test)]
-                super::volume_mutation_test_hooks::run(super::volume_mutation_test_hooks::Stage::Create, &source_parent);
-                std::fs::create_dir_all(&source_parent)?;
-            }
-            let guard = match os::prepare_rename_commit_guard(&source_parent, &destination_parent, &base_dir, &publication_root) {
-                Ok(guard) => guard,
-                Err(err) => {
-                    return Err(match std::fs::symlink_metadata(&base_dir) {
-                        Err(base_err) if base_err.kind() == ErrorKind::NotFound => base_err,
-                        _ => err,
-                    });
-                }
-            };
-            #[cfg(all(test, windows))]
-            run_destination_commit_directory_preparation(&destination_parent);
-            let data = os::read_destination_file_with_commit_guard(&destination_metadata, &guard)?;
-            Ok((guard, data.map(Bytes::from)))
+            os::read_destination_file_with_commit_guard(&file_path, &rename_commit_guard)
         })
         .await
     };
     #[cfg(not(any(windows, target_os = "linux")))]
-    let result = {
-        if create_source_parent {
-            let source_parent = source_parent.to_path_buf();
-            os::run_blocking_namespace_operation(mutation_lease, move || {
-                #[cfg(test)]
-                super::volume_mutation_test_hooks::run(super::volume_mutation_test_hooks::Stage::Create, &source_parent);
-                std::fs::create_dir_all(source_parent)
-            })
-            .await
-            .map_err(to_file_error)?;
-        }
-        let guard =
-            os::prepare_rename_commit_guard(source_parent, destination_parent, base_dir, publication_root).map_err(|err| {
-                match std::fs::symlink_metadata(base_dir) {
-                    Err(base_err) if base_err.kind() == ErrorKind::NotFound => base_err,
-                    _ => err,
-                }
-            });
-        match guard {
-            Ok(guard) => match fs::read(destination_metadata).await {
-                Ok(data) => Ok((guard, Some(Bytes::from(data)))),
-                Err(err) if err.kind() == ErrorKind::NotFound => Ok((guard, None)),
-                Err(err) => Err(err),
-            },
-            Err(err) => Err(err),
-        }
+    let _ = (rename_commit_guard, mutation_lease);
+    #[cfg(not(any(windows, target_os = "linux")))]
+    let result = match fs::read(file_path).await {
+        Ok(data) => Ok(Some(data)),
+        Err(err) if err.kind() == ErrorKind::NotFound => Ok(None),
+        Err(err) => Err(err),
     };
 
-    result.map_err(to_file_error).map_err(DiskError::from)
+    result
+        .map(|data| data.map(Bytes::from))
+        .map_err(to_file_error)
+        .map_err(DiskError::from)
 }
 
 /// Proof produced only when the local rename returns at an existing access
@@ -406,25 +282,24 @@ pub(in crate::disk) struct LocalRenamePreflightRejection(());
 pub(super) struct RenameDataState {
     namespace_owner: Option<Arc<dyn Send + Sync>>,
     preflight_rejection: Option<LocalRenamePreflightRejection>,
-    prepared: Option<Arc<os::NamespaceMutationLease>>,
-}
-
-pub(in crate::disk) struct PreparedRenameData {
-    file_info: FileInfo,
-    mutation_lease: Arc<os::NamespaceMutationLease>,
 }
 
 impl LocalDisk {
-    async fn prepare_rename_data_admission(
+    #[tracing::instrument(name = "rename_data", target = "rustfs_ecstore::disk::local", level = "trace", skip_all)]
+    pub(super) async fn rename_data_inner(
         &self,
+        src_volume: &str,
+        src_path: &str,
         fi: FileInfo,
-        source: (&str, &str),
-        destination: (&str, &str),
-        source_volume: Option<&str>,
-        namespace_owner: Option<Arc<dyn Send + Sync>>,
-    ) -> Result<PreparedRenameData> {
-        let (src_volume, src_path) = source;
-        let (dst_volume, dst_path) = destination;
+        dst_volume: &str,
+        dst_path: &str,
+        state: &mut RenameDataState,
+    ) -> Result<RenameDataResp> {
+        crate::hp_guard!("LocalDisk::rename_data");
+        #[cfg(feature = "e2e-test-hooks")]
+        if fi.is_healing() {
+            wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
+        }
         let mut fi = fi;
         // A non-force DeleteBucket must not remove a directory while a local
         // object commit is publishing into it. The peer's empty scan remains
@@ -459,32 +334,16 @@ impl LocalDisk {
         } else {
             None
         };
-        let root_destination_object_path = self.root.join(
-            destination_object_path
-                .strip_prefix(self.io_root())
-                .map_err(|_| DiskError::InvalidPath)?,
-        );
-        let metadata_lease = os::acquire_metadata_mutation_lease(&root_destination_object_path, namespace_owner).await;
-        let mutation_lease = if let Some(source_volume) = source_volume {
-            os::acquire_rename_data_mutation_lease_for_volumes(
-                &self.root,
-                source_volume,
-                dst_volume,
-                &destination_object_path,
-                staging_prefix.as_deref(),
-                Some(metadata_lease),
-            )
-            .await
-        } else {
-            os::acquire_rename_data_mutation_lease_with_owner(
-                &self.root,
-                dst_volume,
-                &destination_object_path,
-                staging_prefix.as_deref(),
-                Some(metadata_lease),
-            )
-            .await
-        };
+        let metadata_lease =
+            os::acquire_metadata_mutation_lease(&self.get_object_path(dst_volume, dst_path)?, state.namespace_owner.take()).await;
+        let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner(
+            &self.root,
+            dst_volume,
+            &destination_object_path,
+            staging_prefix.as_deref(),
+            Some(metadata_lease),
+        )
+        .await;
         if let Some(claim) = quota_fence_claim {
             mutation_lease.attach_external_guard(claim);
         }
@@ -492,99 +351,6 @@ impl LocalDisk {
             fi.erasure.index = 0;
         }
         fi.validate_for_metadata_read()?;
-        Ok(PreparedRenameData {
-            file_info: fi,
-            mutation_lease,
-        })
-    }
-
-    pub(in crate::disk) async fn begin_legacy_rename_data(
-        &self,
-        source: (&str, &str),
-        fi: &FileInfo,
-        destination: (&str, &str),
-        permit: crate::disk::io_admission::DiskIoPermit,
-        namespace_owner: Option<Arc<dyn Send + Sync>>,
-    ) -> Result<PreparedRenameData> {
-        // Capture the same permit, mount FD and listener-instance owner before
-        // quota -> metadata -> namespace -> sorted volume READ admission.
-        #[cfg(target_os = "linux")]
-        let owner: Arc<dyn Send + Sync> = Arc::new((permit, self.mount_lease.clone(), namespace_owner));
-        #[cfg(not(target_os = "linux"))]
-        let owner: Arc<dyn Send + Sync> = Arc::new((permit, namespace_owner));
-        let prepared = self
-            .prepare_rename_data_admission(fi.clone(), source, destination, Some(source.0), Some(owner))
-            .await?;
-        for volume in [Some(source.0), (source.0 != destination.0).then_some(destination.0)]
-            .into_iter()
-            .flatten()
-        {
-            if !skip_access_checks(volume) {
-                let path = self.io_get_bucket_path(volume)?;
-                let metadata = fs::symlink_metadata(path)
-                    .await
-                    .map_err(crate::disk::error_conv::to_volume_error)?;
-                if !metadata.is_dir() {
-                    return Err(DiskError::VolumeNotFound);
-                }
-            }
-        }
-        Ok(prepared)
-    }
-
-    pub(in crate::disk) async fn rename_data_prepared(
-        &self,
-        source: (&str, &str),
-        destination: (&str, &str),
-        prepared: PreparedRenameData,
-    ) -> Result<RenameDataResp> {
-        let PreparedRenameData {
-            file_info,
-            mutation_lease,
-        } = prepared;
-        let mut state = RenameDataState {
-            prepared: Some(mutation_lease),
-            ..Default::default()
-        };
-        self.rename_data_inner(source.0, source.1, file_info, destination.0, destination.1, &mut state)
-            .await
-    }
-
-    #[tracing::instrument(name = "rename_data", target = "rustfs_ecstore::disk::local", level = "trace", skip_all)]
-    pub(super) async fn rename_data_inner(
-        &self,
-        src_volume: &str,
-        src_path: &str,
-        fi: FileInfo,
-        dst_volume: &str,
-        dst_path: &str,
-        state: &mut RenameDataState,
-    ) -> Result<RenameDataResp> {
-        crate::hp_guard!("LocalDisk::rename_data");
-        #[cfg(feature = "e2e-test-hooks")]
-        if fi.is_healing() {
-            wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
-        }
-        let PreparedRenameData {
-            file_info: fi,
-            mutation_lease,
-        } = if let Some(mutation_lease) = state.prepared.take() {
-            PreparedRenameData {
-                file_info: fi,
-                mutation_lease,
-            }
-        } else {
-            self.prepare_rename_data_admission(
-                fi,
-                (src_volume, src_path),
-                (dst_volume, dst_path),
-                None,
-                state.namespace_owner.take(),
-            )
-            .await?
-        };
-        let no_inline = fi.data.is_none() && fi.size > 0;
-        let fi_healing = fi.is_healing();
         // Snapshot the destination part paths before `fi` is consumed below. These
         // are the descriptors a reader may hold for the version this call is about
         // to replace (backlog#1145); readers build the identical string in
@@ -638,23 +404,22 @@ impl LocalDisk {
             return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
         }
 
-        // Keep recovery authority separate from active upload metadata so a
-        // committed upload cannot reopen when set-level cleanup fails.
-        let preserve_upload_metadata = src_volume == super::super::RUSTFS_META_MULTIPART_BUCKET;
-        let source_metadata_name = if preserve_upload_metadata {
-            super::super::MULTIPART_COMMIT_METADATA_FILE
-        } else {
-            STORAGE_FORMAT_FILE
-        };
-        let src_file_path = self.io_get_object_path(src_volume, format!("{src_path}/{source_metadata_name}").as_str())?;
+        // xl.meta path
+        let src_file_path = self.io_get_object_path(src_volume, format!("{}/{}", src_path, STORAGE_FORMAT_FILE).as_str())?;
         let dst_file_path = self.io_get_object_open_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?;
 
-        // data_dir path (only used by the non-inline commit branch)
-        let has_data_dir_path = if no_inline && !fi.is_remote() {
-            if let Some(data_dir) = fi
-                .data_dir
-                .map(|dir| rustfs_utils::path::retain_slash(dir.to_string().as_str()))
-            {
+        // data_dir path
+        let has_data_dir_path = {
+            let has_data_dir = {
+                if !fi.is_remote() {
+                    fi.data_dir
+                        .map(|dir| rustfs_utils::path::retain_slash(dir.to_string().as_str()))
+                } else {
+                    None
+                }
+            };
+
+            if let Some(data_dir) = has_data_dir {
                 let src_data_path = self.io_get_object_path(
                     src_volume,
                     rustfs_utils::path::retain_slash(format!("{}/{}", src_path, data_dir).as_str()).as_str(),
@@ -668,8 +433,6 @@ impl LocalDisk {
             } else {
                 None
             }
-        } else {
-            None
         };
 
         check_path_length(src_file_path.to_string_lossy().to_string().as_str())?;
@@ -689,13 +452,14 @@ impl LocalDisk {
         let dst_file_parent = dst_file_path
             .parent()
             .ok_or_else(|| DiskError::other("missing object metadata parent"))?;
+        if !no_inline {
+            fs::create_dir_all(src_file_parent).await.map_err(to_file_error)?;
+        }
         // Acquire the common trees before reading destination metadata. On
         // Windows this pins the object directory identity across publication.
-        let (rename_commit_guard, has_dst_buf) = prepare_rename_commit_and_destination_metadata(
-            !no_inline,
+        let rename_commit_guard = lock_rename_commit_directories(
             src_file_parent,
             dst_file_parent,
-            &dst_file_path,
             &dst_volume_dir,
             &self.publication_root,
             mutation_lease.clone(),
@@ -710,35 +474,12 @@ impl LocalDisk {
             source_volume_dir: &src_volume_dir,
             data_paths: has_data_dir_path.as_ref(),
         };
-
-        if preserve_upload_metadata {
-            let source_object_path = self.get_object_path(src_volume, src_path)?;
-            let authority_owner: Arc<dyn Send + Sync> = if source_object_path == self.get_object_path(dst_volume, dst_path)? {
-                commit.mutation_lease.clone()
-            } else {
-                os::try_acquire_metadata_mutation_lease_with_owner(&source_object_path, commit.mutation_lease.clone())?
-            };
-            let authority = self.io_get_object_path(src_volume, &format!("{src_path}/{STORAGE_FORMAT_FILE}"))?;
-            let recovery = self.io_get_object_path(
-                src_volume,
-                &format!("{src_path}/{}/{STORAGE_FORMAT_FILE}", super::super::MULTIPART_RECOVERY_DIR),
-            )?;
-            check_path_length(recovery.to_string_lossy().as_ref())?;
-            // Both volume READs are already held. Neither source key may wait
-            // behind a competing owner that is queued behind a volume WRITE.
-            let recovery_lease = os::try_acquire_namespace_mutation_lease_with_owner(&recovery, authority_owner.clone())?;
-            os::rename_all_with_lease(&authority, &recovery, &src_volume_dir, commit.publication_root, recovery_lease).await?;
-            // This is system metadata authority, so use its existing strict
-            // durability policy even when the object payload is relaxed.
-            if effective_durability(src_volume).syncs_commit_metadata() {
-                os::fsync_dir_with_owner(recovery.parent().ok_or(DiskError::InvalidPath)?, Some(authority_owner.clone()))
-                    .await
-                    .map_err(to_file_error)?;
-                os::fsync_dir_with_owner(src_file_parent, Some(authority_owner.clone()))
-                    .await
-                    .map_err(to_file_error)?;
-            }
-        }
+        let has_dst_buf = prepare_rename_destination_metadata(
+            commit.destination_metadata,
+            &commit.directory_guard,
+            commit.mutation_lease.clone(),
+        )
+        .await?;
 
         if no_inline {
             // Non-inline: read xl.meta, parse, write, rename data dir, rename xl.meta
@@ -948,7 +689,7 @@ impl LocalDisk {
             if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref()) {
                 let backup_parent = dst_file_parent.join(old_data_dir.to_string());
                 #[cfg(not(windows))]
-                if let Err(err) = os::make_dir_all_with_owner(&backup_parent, &skip_parent, commit.mutation_lease.clone()).await {
+                if let Err(err) = os::make_dir_all(&backup_parent, &skip_parent).await {
                     commit.restore_published_data_source().await?;
                     return Err(err);
                 }
@@ -1127,8 +868,14 @@ impl LocalDisk {
             #[cfg(windows)]
             drop(commit.directory_guard);
 
-            if !preserve_upload_metadata && let Some(parent) = src_file_path.parent() {
-                let _ = std::fs::remove_dir(parent);
+            if let Some(src_file_path_parent) = src_file_path.parent() {
+                if src_volume != super::super::RUSTFS_META_MULTIPART_BUCKET {
+                    let _ = std::fs::remove_dir(src_file_path_parent);
+                } else {
+                    let _ = self
+                        .delete_file(&dst_volume_dir, &src_file_path_parent.to_path_buf(), true, false)
+                        .await;
+                }
             }
 
             // Heal reuses a version's `data_dir` and lands the rebuilt shard on
@@ -1161,12 +908,14 @@ impl LocalDisk {
             // allowing a retry to reuse the deterministic staging path too early.
             let src = src_file_path.clone();
             let dst = dst_file_path.clone();
+            let cleanup_path = if src_volume == super::super::RUSTFS_META_MULTIPART_BUCKET {
+                src_file_path.parent().map(|p| p.to_path_buf())
+            } else {
+                None
+            };
             let dst_path_for_failpoint = dst_path.to_string();
             let dst_volume_for_source = dst_volume.to_string();
             let dst_path_for_source = dst_path.to_string();
-            let dst_volume_dir_for_commit = dst_volume_dir.clone();
-            let publication_root_for_commit = self.publication_root.clone();
-            let cleanup_src_parent = !preserve_upload_metadata;
             #[cfg(windows)]
             let source_parent = src_file_parent.to_path_buf();
             let rename_commit_guard_for_preparation = commit.directory_guard.clone();
@@ -1183,7 +932,7 @@ impl LocalDisk {
             } else {
                 None
             };
-            let prepare_inline_metadata = move |is_cancelled: &dyn Fn() -> bool| {
+            let prepare_inline_metadata = move || {
                 let mut prepared_metadata_source =
                     os::create_prepared_rename_source_with_commit_guard(&src, &dst, &rename_commit_guard_for_preparation)?;
                 #[cfg(windows)]
@@ -1263,71 +1012,7 @@ impl LocalDisk {
                     staged_rollback_path = Some(StagedRollbackBackup(backup_path));
                 }
 
-                let destination_was_absent = has_dst_buf.is_none();
-                if !is_cancelled()
-                    && rollback_data_dir.is_none()
-                    && !os::dst_dir_fsync_group_commit_enabled()
-                    && !has_inline_commit_test_failpoints(&dst_path_for_failpoint)
-                {
-                    let rename_res = if should_fail_commit_rename(&dst_path_for_failpoint) {
-                        drop(prepared_metadata_source);
-                        Err(DiskError::other("test fail during metadata commit rename"))
-                    } else {
-                        os::rename_prepared_source_on_thread(
-                            prepared_metadata_source,
-                            &src,
-                            &dst,
-                            &dst_volume_dir_for_commit,
-                            &publication_root_for_commit,
-                            &rename_commit_guard_for_preparation,
-                        )
-                    };
-                    if let Err(err) = rename_res {
-                        if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
-                            let _ = remove_file_if_exists(backup_path);
-                        }
-                        return Ok(InlinePreparationOutcome::CommitErr(err));
-                    }
-
-                    if should_fail_after_metadata_commit(&dst_path_for_failpoint) {
-                        let err = rollback_inline_metadata_std(&dst, None, staged_rollback_path.as_ref())
-                            .err()
-                            .unwrap_or_else(|| std::io::Error::other("test fail after metadata commit"));
-                        return Ok(InlinePreparationOutcome::CommitErr(DiskError::from(err)));
-                    }
-
-                    if sync
-                        && let Err(err) = fsync_inline_commit_std(
-                            &dst,
-                            &dst_volume_dir_for_commit,
-                            destination_was_absent,
-                            staged_rollback_path.as_ref(),
-                            &rename_commit_guard_for_preparation,
-                        )
-                    {
-                        return Ok(InlinePreparationOutcome::CommitErr(DiskError::from(err)));
-                    }
-
-                    if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
-                        let _ = remove_file_if_exists(backup_path);
-                    }
-                    #[cfg(not(windows))]
-                    if cleanup_src_parent && let Some(parent) = src.parent() {
-                        let _ = std::fs::remove_dir(parent);
-                    }
-
-                    return Ok::<_, std::io::Error>(InlinePreparationOutcome::Committed(RenameDataResp {
-                        old_data_dir,
-                        rollback_data_dir,
-                        cleanup_data_dir: old_data_dir,
-                        sign: version_signature,
-                        old_current_size,
-                        old_current_source_checked,
-                        old_current_source,
-                    }));
-                }
-
-                Ok::<_, std::io::Error>(InlinePreparationOutcome::Prepared((
+                Ok::<_, std::io::Error>((
                     rollback_data_dir,
                     old_data_dir,
                     version_signature,
@@ -1335,22 +1020,19 @@ impl LocalDisk {
                     old_current_source_checked,
                     old_current_source,
                     staged_rollback_path,
-                    destination_was_absent,
+                    has_dst_buf.is_none(),
                     prepared_metadata_source,
-                )))
+                ))
             };
             let inline_preparation = if let Some(admission) = file_sync_admission.as_ref() {
-                os::run_cancellable_blocking_namespace_file_sync_operation(
-                    commit.mutation_lease.clone(),
-                    admission,
-                    prepare_inline_metadata,
-                )
-                .await
+                os::run_blocking_namespace_file_sync_operation(commit.mutation_lease.clone(), admission, prepare_inline_metadata)
+                    .await
             } else {
-                os::run_cancellable_blocking_namespace_operation(commit.mutation_lease.clone(), prepare_inline_metadata).await
+                os::run_blocking_namespace_operation(commit.mutation_lease.clone(), prepare_inline_metadata).await
             }
             .map_err(to_file_error)
             .map_err(DiskError::from);
+
             let (
                 rollback_data_dir,
                 cleanup_data_dir,
@@ -1362,27 +1044,13 @@ impl LocalDisk {
                 destination_was_absent,
                 prepared_metadata_source,
             ) = match inline_preparation {
-                Ok(InlinePreparationOutcome::Committed(resp)) => {
-                    drop(file_sync_admission.take());
-                    #[cfg(windows)]
-                    drop(commit.directory_guard);
-                    #[cfg(windows)]
-                    if cleanup_src_parent && let Some(parent) = src_file_path.parent() {
-                        let _ = std::fs::remove_dir(parent);
-                    }
-                    for part_path in &invalidate_part_paths {
-                        self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
-                    }
-                    return Ok(resp);
-                }
-                Ok(InlinePreparationOutcome::CommitErr(err)) | Err(err) => {
-                    drop(file_sync_admission.take());
+                Ok(prepared) => prepared,
+                Err(err) => {
                     for part_path in &invalidate_part_paths {
                         self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
                     }
                     return Err(err);
                 }
-                Ok(InlinePreparationOutcome::Prepared(prepared)) => prepared,
             };
 
             commit.directory_guard = remove_dst_base_before_commit(
@@ -1415,25 +1083,18 @@ impl LocalDisk {
                 let backup_path = dst_parent
                     .join(rollback_data_dir.to_string())
                     .join(STORAGE_FORMAT_FILE_BACKUP);
-                // Keep backup mutations serialized, but never wait for a second
-                // namespace while holding the volume READ: its owner can be
-                // queued behind a volume DELETE that is waiting for this commit.
+                // rename_all acquires the backup path's namespace lease. Do not
+                // hold a disk admission while acquiring another namespace lock.
                 drop(file_sync_admission.take());
-                let backup_result =
-                    match os::try_acquire_namespace_mutation_lease_with_owner(&backup_path, commit.mutation_lease.clone()) {
-                        Ok(lease) => {
-                            os::rename_all_with_lease(
-                                staged_backup,
-                                &backup_path,
-                                &dst_volume_dir,
-                                commit.publication_root,
-                                lease,
-                            )
-                            .await
-                        }
-                        Err(err) => Err(err),
-                    };
-                if let Err(err) = backup_result {
+                if let Err(err) = os::rename_all_with_owner(
+                    staged_backup,
+                    &backup_path,
+                    &dst_volume_dir,
+                    commit.publication_root,
+                    Some(commit.mutation_lease.clone()),
+                )
+                .await
+                {
                     let _ = remove_file_if_exists(staged_backup);
                     return Err(err);
                 }
@@ -1593,7 +1254,9 @@ impl LocalDisk {
             }
 
             // Cleanup
-            if cleanup_src_parent && let Some(parent) = src_file_path.parent() {
+            if let Some(ref cleanup) = cleanup_path {
+                let _ = self.delete_file(&dst_volume_dir, cleanup, true, false).await;
+            } else if let Some(parent) = src_file_path.parent() {
                 let _ = std::fs::remove_dir(parent);
             }
 

crates/ecstore/src/disk/local/read_path_tests.rs

@@ -1,202 +0,0 @@
-// Copyright 2026 RustFS Team
-// Licensed under the Apache License, Version 2.0.
-
-use super::*;
-use std::cell::RefCell;
-
-struct Observation {
-    rebuilds: usize,
-    normalizations: usize,
-    before_recheck: Option<Box<dyn FnOnce()>>,
-}
-
-thread_local! {
-    static OBSERVATION: RefCell<Option<Observation>> = const { RefCell::new(None) };
-}
-
-struct Observe;
-
-impl Observe {
-    fn new(before_recheck: Option<Box<dyn FnOnce()>>) -> Self {
-        OBSERVATION.with(|state| {
-            assert!(state.borrow().is_none(), "read-path observations must not nest");
-            *state.borrow_mut() = Some(Observation {
-                rebuilds: 0,
-                normalizations: 0,
-                before_recheck,
-            });
-        });
-        Self
-    }
-
-    fn normalizations(&self) -> usize {
-        OBSERVATION.with(|state| state.borrow().as_ref().expect("active observation").normalizations)
-    }
-
-    fn rebuilds(&self) -> usize {
-        OBSERVATION.with(|state| state.borrow().as_ref().expect("active observation").rebuilds)
-    }
-}
-
-impl Drop for Observe {
-    fn drop(&mut self) {
-        OBSERVATION.with(|state| *state.borrow_mut() = None);
-    }
-}
-
-pub(super) fn record_bucket_rebuild() {
-    OBSERVATION.with(|state| {
-        if let Some(observation) = state.borrow_mut().as_mut() {
-            observation.rebuilds += 1;
-        }
-    });
-}
-
-pub(super) fn record_normalization() {
-    OBSERVATION.with(|state| {
-        if let Some(observation) = state.borrow_mut().as_mut() {
-            observation.normalizations += 1;
-        }
-    });
-}
-
-pub(super) fn before_bucket_recheck() {
-    let hook = OBSERVATION.with(|state| state.borrow_mut().as_mut().and_then(|value| value.before_recheck.take()));
-    if let Some(hook) = hook {
-        hook();
-    }
-}
-
-async fn local_disk(root: &Path) -> LocalDisk {
-    let endpoint = Endpoint::try_from(root.to_string_lossy().as_ref()).expect("local endpoint");
-    let disk = LocalDisk::new(&endpoint, false).await.expect("local disk");
-    disk.make_volume("bucket").await.expect("bucket");
-    disk
-}
-
-#[tokio::test(flavor = "current_thread")]
-async fn checked_read_paths_reuse_bucket_construction() {
-    let root = tempfile::tempdir().expect("disk root");
-    let disk = local_disk(root.path()).await;
-    std::fs::create_dir_all(root.path().join("bucket/prefix/object")).expect("object directories");
-    for key in ["prefix/object/xl.meta", "", "prefix/../object/xl.meta"] {
-        let expected = build_local_disk_object_path(disk.io_root(), "bucket", key);
-        let observation = Observe::new(None);
-        let actual = disk.io_get_checked_read_paths("bucket", key).expect("checked read paths");
-        assert_eq!(actual, expected, "returned paths must retain their existing spelling");
-        assert_eq!(observation.rebuilds(), 0, "the validated tuple already owns its bucket path");
-        // path_join_buf already resolves dot components in object keys.
-        let expected_normalizations = if cfg!(windows) { 3 } else { 0 };
-        assert_eq!(observation.normalizations(), expected_normalizations);
-        #[cfg(not(target_os = "linux"))]
-        {
-            assert_eq!(
-                disk.io_get_bucket_and_object_open_path("bucket", key)
-                    .expect("read-version paths"),
-                expected
-            );
-            assert_eq!(observation.rebuilds(), 0, "read_version shares the same checked path core");
-            assert_eq!(observation.normalizations(), 2 * expected_normalizations);
-        }
-    }
-}
-
-#[tokio::test(flavor = "current_thread")]
-async fn checked_read_paths_preserve_containment_errors() {
-    let root = tempfile::tempdir().expect("disk root");
-    let disk = local_disk(root.path()).await;
-    for (bucket, key) in [("../outside", "object"), ("bucket", "../outside"), ("bucket", "/outside")] {
-        let old = disk
-            .io_get_object_path(bucket, key)
-            .and_then(|path| disk.io_get_bucket_path(bucket).map(|volume| (volume, path)));
-        let new = disk.io_get_checked_read_paths(bucket, key);
-        assert_eq!(new, old, "path and error behavior for {bucket}/{key}");
-    }
-}
-
-#[tokio::test(flavor = "current_thread")]
-async fn checked_read_paths_metadata_bytes_and_absence_errors() {
-    let root = tempfile::tempdir().expect("disk root");
-    let disk = local_disk(root.path()).await;
-    let object = root.path().join("bucket/prefix/object");
-    std::fs::create_dir_all(&object).expect("object");
-    let metadata = b"opaque metadata bytes passed through the read API";
-    std::fs::write(object.join(STORAGE_FORMAT_FILE), metadata).expect("metadata");
-    {
-        let observation = Observe::new(None);
-        let actual = disk
-            .read_metadata("bucket", "prefix/object/xl.meta")
-            .await
-            .expect("read metadata");
-        assert_eq!(actual.as_ref(), metadata);
-        assert_eq!(observation.rebuilds(), 0, "scanner metadata reads use the shared path core");
-        assert_eq!(observation.normalizations(), if cfg!(windows) { 3 } else { 0 });
-    }
-    assert!(matches!(
-        disk.read_metadata("bucket", "missing/xl.meta").await,
-        Err(DiskError::FileNotFound)
-    ));
-    assert!(matches!(
-        disk.read_metadata("missing", "object/xl.meta").await,
-        Err(DiskError::VolumeNotFound)
-    ));
-}
-
-#[cfg(unix)]
-#[tokio::test(flavor = "current_thread")]
-async fn checked_read_paths_recheck_bucket_after_object_validation() {
-    let root = tempfile::tempdir().expect("disk root");
-    let outside = tempfile::tempdir().expect("outside root");
-    let disk = local_disk(root.path()).await;
-    std::fs::create_dir_all(root.path().join("bucket/object")).expect("object");
-    std::fs::write(root.path().join("bucket/object/xl.meta"), b"inside").expect("inside metadata");
-    let bucket = root.path().join("bucket");
-    let saved = root.path().join("saved");
-    let target = outside.path().to_path_buf();
-    let _observation = Observe::new(Some(Box::new(move || {
-        std::fs::rename(&bucket, saved).expect("replace bucket after full-path validation");
-        std::os::unix::fs::symlink(target, bucket).expect("bucket replacement symlink");
-    })));
-    assert!(
-        matches!(disk.io_get_checked_read_paths("bucket", "object/xl.meta"), Err(DiskError::InvalidPath)),
-        "the second bucket check must remain effective before queuing native IO"
-    );
-}
-
-#[test]
-fn read_path_normalization_matches_component_normalizer() {
-    for value in [
-        "",
-        ".",
-        "..",
-        "./a",
-        "a/./b",
-        "a//b/",
-        "a/../b",
-        "../../a",
-        "/",
-        "//a///b/",
-        "/a/../b",
-        "/../../a",
-        r"C:\a\b",
-        r"C:\a\..\b",
-        r"\\server\share\a",
-        r"\\?\C:\a\.\b",
-    ] {
-        let path = Path::new(value);
-        let expected = normalize_path_components(path);
-        let observation = Observe::new(None);
-        let actual = normalize_path_cow(path);
-        assert_eq!(
-            actual.components().collect::<Vec<_>>(),
-            expected.components().collect::<Vec<_>>(),
-            "{value}"
-        );
-        let needs_buffer = cfg!(windows)
-            || path
-                .components()
-                .any(|component| matches!(component, std::path::Component::ParentDir | std::path::Component::CurDir));
-        assert_eq!(matches!(actual, std::borrow::Cow::Owned(_)), needs_buffer, "{value}");
-        assert_eq!(observation.normalizations(), usize::from(needs_buffer), "{value}");
-    }
-}

crates/ecstore/src/disk/mod.rs

@@ -27,16 +27,12 @@ mod io_admission;
 #[cfg(test)]
 pub(crate) mod lifecycle_test_io;
 pub mod local;
-pub use local::artifact_inventory::{ArtifactInventoryFrame, ArtifactInventoryVersion};
 pub mod os;
 
 #[cfg(test)]
 pub(crate) use io_admission::DiskIoClass;
-pub(crate) use io_admission::{
-    acquire_context_disk_io_permit, current_disk_io_context, current_resource_unit, with_optional_disk_io_context,
-    with_optional_resource_unit,
-};
-pub use io_admission::{with_background_disk_io, with_background_resource_unit};
+pub use io_admission::with_background_disk_io;
+pub(crate) use io_admission::{acquire_context_disk_io_permit, current_disk_io_context, with_optional_disk_io_context};
 
 pub const RUSTFS_META_BUCKET: &str = ".rustfs.sys";
 pub const MIGRATING_META_BUCKET: &str = ".minio.sys";
@@ -50,9 +46,6 @@ pub const FORMAT_CONFIG_FILE: &str = "format.json";
 pub const HEALING_MARKER_PATH: &str = "healing.bin";
 pub const STORAGE_FORMAT_FILE: &str = "xl.meta";
 pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp";
-/// Original upload authority after it stops admitting active part mutations.
-pub(crate) const MULTIPART_RECOVERY_DIR: &str = ".abort-authority";
-pub(crate) const MULTIPART_COMMIT_METADATA_FILE: &str = "xl.meta.complete";
 pub const PART_TRANSACTION_NEW_META: &str = "new.meta";
 pub const PART_TRANSACTION_OLD_META: &str = "old.meta";
 pub const PART_TRANSACTION_ROLLBACK: &str = "rollback";
@@ -74,10 +67,6 @@ use crate::cluster::rpc::RemoteDisk;
 use crate::cluster::rpc::build_internode_data_transport_from_env;
 use crate::disk::disk_store::DiskStoreRenameDataExt;
 use crate::disk::disk_store::LocalDiskWrapper;
-pub use crate::disk::disk_store::{
-    LegacyFileStream, LegacyFileWrite, LegacyMetadataWrite, LegacyPartMutation, LegacyPartOperation, LegacyPathDelete,
-    LegacyPathDeleteOperation, LegacyRenameData, LegacyRenameFile,
-};
 use crate::disk::health_state::RuntimeDriveHealthState;
 use crate::disk::local::ScanGuard;
 use bytes::Bytes;
@@ -90,7 +79,12 @@ use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
 use rustfs_madmin::info_commands::DiskMetrics;
 use rustfs_rio::ChunkReaderBox;
 use serde::{Deserialize, Serialize};
-use std::{fmt::Debug, path::PathBuf, sync::Arc, time::Duration};
+use std::{
+    fmt::Debug,
+    path::PathBuf,
+    sync::{Arc, atomic::AtomicBool},
+    time::Duration,
+};
 use time::OffsetDateTime;
 use tokio::io::{AsyncRead, AsyncWrite};
 use uuid::Uuid;
@@ -98,7 +92,6 @@ use uuid::Uuid;
 /// Independent admission and physical ownership for one disk rename.
 #[derive(Default)]
 pub(crate) struct RenameDataGuards {
-    pub(crate) artifact_put: Option<crate::bucket::artifact::ArtifactPutIdentity>,
     pub(crate) scanner_publication_lease_token: Option<Uuid>,
     pub(crate) external_guard: Option<Arc<dyn Send + Sync>>,
     pub(crate) namespace_owner: Option<Arc<dyn Send + Sync>>,
@@ -240,117 +233,6 @@ pub enum Disk {
 }
 
 impl Disk {
-    /// Capture local file-write admission before authoritative classification.
-    pub async fn begin_legacy_file_write(&self, volume: &str) -> Result<LegacyFileWrite> {
-        match self {
-            Self::Local(disk) => disk.begin_legacy_file_write(volume).await,
-            Self::Remote(_) => Err(DiskError::DiskNotFound),
-        }
-    }
-
-    pub(crate) async fn begin_legacy_file_write_with_namespace_owner(
-        &self,
-        volume: &str,
-        owner: Option<Arc<dyn Send + Sync>>,
-    ) -> Result<LegacyFileWrite> {
-        match self {
-            Self::Local(disk) => disk.begin_legacy_file_write_with_namespace_owner(volume, owner).await,
-            Self::Remote(_) => Err(DiskError::DiskNotFound),
-        }
-    }
-
-    /// Capture metadata publication admission before authoritative classification.
-    pub async fn begin_legacy_metadata_write(&self, volume: &str, path: &str) -> Result<LegacyMetadataWrite> {
-        match self {
-            Self::Local(disk) => disk.begin_legacy_metadata_write(volume, path).await,
-            Self::Remote(_) => Err(DiskError::DiskNotFound),
-        }
-    }
-
-    pub(crate) async fn begin_legacy_metadata_write_with_namespace_owner(
-        &self,
-        volume: &str,
-        path: &str,
-        namespace_owner: Option<Arc<dyn Send + Sync>>,
-    ) -> Result<LegacyMetadataWrite> {
-        match self {
-            Self::Local(disk) => {
-                disk.begin_legacy_metadata_write_with_namespace_owner(volume, path, namespace_owner)
-                    .await
-            }
-            Self::Remote(_) => Err(DiskError::DiskNotFound),
-        }
-    }
-
-    pub(crate) fn record_legacy_admission_failure(
-        &self,
-        operation: &'static str,
-        error: &DiskError,
-        started: std::time::Instant,
-    ) {
-        if let Self::Local(disk) = self {
-            disk.record_legacy_admission_failure(operation, error, started);
-        }
-    }
-
-    pub(crate) async fn begin_legacy_path_delete(
-        &self,
-        volume: &str,
-        operation: LegacyPathDeleteOperation,
-        owner: Option<Arc<dyn Send + Sync>>,
-        deadline: tokio::time::Instant,
-    ) -> Result<LegacyPathDelete> {
-        match self {
-            Self::Local(disk) => disk.begin_legacy_path_delete(volume, operation, owner, deadline).await,
-            Self::Remote(_) => Err(DiskError::DiskNotFound),
-        }
-    }
-
-    pub(crate) async fn begin_legacy_part_mutation(
-        &self,
-        source: Option<(&str, &str)>,
-        destination: (&str, &str),
-        namespace_owner: Option<Arc<dyn Send + Sync>>,
-        operation: LegacyPartOperation,
-    ) -> Result<LegacyPartMutation> {
-        match self {
-            Self::Local(disk) => {
-                disk.begin_legacy_part_mutation(source, destination, namespace_owner, operation)
-                    .await
-            }
-            Self::Remote(_) => Err(DiskError::DiskNotFound),
-        }
-    }
-
-    pub(crate) async fn begin_legacy_rename_file(
-        &self,
-        source: (&str, &str),
-        destination: (&str, &str),
-        durable: bool,
-        namespace_owner: Option<Arc<dyn Send + Sync>>,
-    ) -> Result<LegacyRenameFile> {
-        match self {
-            Self::Local(disk) => {
-                disk.begin_legacy_rename_file(source, destination, durable, namespace_owner)
-                    .await
-            }
-            Self::Remote(_) => Err(DiskError::DiskNotFound),
-        }
-    }
-
-    pub(crate) async fn begin_legacy_rename_data(
-        &self,
-        source: (&str, &str),
-        fi: &FileInfo,
-        destination: (&str, &str),
-        namespace_owner: Option<Arc<dyn Send + Sync>>,
-    ) -> Result<LegacyRenameData> {
-        match self {
-            Self::Local(disk) => disk.begin_legacy_rename_data(source, fi, destination, namespace_owner).await,
-            Self::Remote(_) => Err(DiskError::DiskNotFound),
-        }
-    }
-
     /// A read-only probe; the physical delete repeats the same leaf checks.
     pub async fn is_plain_orphan_data_dir(&self, volume: &str, path: &str) -> Result<bool> {
         match self {
@@ -1174,27 +1056,13 @@ impl Disk {
         dst_volume: &str,
         dst_path: &str,
     ) -> Result<RenameDataResp> {
-        let bucket_scope = crate::store::bucket_heal_scope_for_object(dst_volume, dst_path);
-        let repair_scope = crate::bucket::artifact::artifact_repair_scope(dst_volume, dst_path);
-        if let Some(scope) = &bucket_scope {
-            scope.check()?;
-        }
-        if let Some(scope) = &repair_scope {
-            scope.check().map_err(DiskError::other)?;
-        }
-        let repair_lease = if let (Self::Local(disk), Some(scope)) = (self, &repair_scope) {
-            if let Some(identity) = scope.receiver_identity() {
-                scope.check_version(fi.version_id).await.map_err(DiskError::other)?;
-                Some(disk.acquire_artifact_repair_lease(identity.receiver_key).await?)
-            } else {
-                None
-            }
-        } else {
-            None
+        let Some(scope) = crate::store::bucket_heal_scope_for_object(dst_volume, dst_path) else {
+            return self
+                .rename_data_borrowed_with_fence(src_volume, src_path, fi, dst_volume, dst_path, None)
+                .await;
         };
-        let owner: Option<Arc<dyn Send + Sync>> = (bucket_scope.is_some() || repair_scope.is_some())
-            .then(|| Arc::new((bucket_scope, repair_scope, repair_lease)) as Arc<dyn Send + Sync>);
-        self.rename_data_borrowed_with_fence_and_guard(src_volume, src_path, fi, dst_volume, dst_path, None, owner)
+        scope.check()?;
+        self.rename_data_borrowed_with_fence_and_guard(src_volume, src_path, fi, dst_volume, dst_path, None, Some(scope))
             .await
     }
 
@@ -1205,20 +1073,10 @@ impl Disk {
         fi: &FileInfo,
         dst_volume: &str,
         dst_path: &str,
-        mut guards: RenameDataGuards,
+        guards: RenameDataGuards,
     ) -> RenameDataObservation {
         match self {
             Disk::Local(local_disk) => {
-                if let Some(identity) = guards.artifact_put.take() {
-                    let lease = match local_disk.acquire_artifact_put_lease(identity.receiver_key).await {
-                        Ok(lease) => lease,
-                        Err(error) => return RenameDataObservation::unknown(Err(error)),
-                    };
-                    // Keep the existing timeout policy; physical workers own the
-                    // receiver lease even after the caller's disk deadline.
-                    guards.namespace_owner =
-                        Some(Arc::new((guards.external_guard.clone(), guards.namespace_owner.take(), lease)));
-                }
                 local_disk
                     .rename_data_observed_with_guards(src_volume, src_path, fi, dst_volume, dst_path, guards)
                     .await
@@ -1226,17 +1084,39 @@ impl Disk {
             Disk::Remote(remote_disk) => RenameDataObservation::unknown(
                 remote_disk
                     .rename_data_borrowed_with_fence(
-                        (src_volume, src_path),
+                        src_volume,
+                        src_path,
                         fi,
-                        (dst_volume, dst_path),
+                        dst_volume,
+                        dst_path,
                         guards.scanner_publication_lease_token,
-                        guards.artifact_put.map(|identity| identity.incarnation),
                     )
                     .await,
             ),
         }
     }
 
+    pub(crate) async fn rename_data_borrowed_with_fence(
+        &self,
+        src_volume: &str,
+        src_path: &str,
+        fi: &FileInfo,
+        dst_volume: &str,
+        dst_path: &str,
+        scanner_publication_lease_token: Option<Uuid>,
+    ) -> Result<RenameDataResp> {
+        self.rename_data_borrowed_with_fence_and_guard(
+            src_volume,
+            src_path,
+            fi,
+            dst_volume,
+            dst_path,
+            scanner_publication_lease_token,
+            None,
+        )
+        .await
+    }
+
     #[allow(clippy::too_many_arguments)]
     pub async fn rename_data_borrowed_with_fence_and_guard(
         &self,
@@ -1257,11 +1137,12 @@ impl Disk {
             Disk::Remote(remote_disk) => {
                 remote_disk
                     .rename_data_borrowed_with_fence(
-                        (src_volume, src_path),
+                        src_volume,
+                        src_path,
                         fi,
-                        (dst_volume, dst_path),
+                        dst_volume,
+                        dst_path,
                         scanner_publication_lease_token,
-                        None,
                     )
                     .await
             }
@@ -2026,6 +1907,12 @@ pub struct WalkDirOptions {
     // Override the remote stream stall timeout for long background walks.
     #[serde(default)]
     pub stall_timeout_ms: Option<u64>,
+
+    /// In-process completion state for bounded local walks. This is skipped
+    /// from RPC serialization; remote peers retain the legacy natural-EOF
+    /// behavior until they support an explicit capability.
+    #[serde(skip)]
+    pub producer_limit_reached: Option<Arc<AtomicBool>>,
 }
 
 impl WalkDirOptions {
@@ -2870,6 +2757,7 @@ mod tests {
             skip_total_timeout: false,
             timeout_ms: Some(10_000),
             stall_timeout_ms: Some(20_000),
+            producer_limit_reached: None,
         };
 
         assert_eq!(opts.bucket, "test-bucket");

crates/ecstore/src/ecstore_validation_blackbox.rs

@@ -153,13 +153,10 @@ async fn refresh_loss_lock_client_keeps_rejection_observable_for_late_waiters()
     );
 }
 
-/// Keep the real metadata owner and backing directories alive together.
-pub(crate) async fn make_local_set_disks(
-    drive_count: usize,
-    parity_count: usize,
-) -> (crate::set_disk::HermeticSetFixture, Arc<SetDisks>) {
-    let (fixture, _, set) = crate::set_disk::hermetic_set_disks_for_pool_with_default_parity(drive_count, 0, parity_count).await;
-    (fixture, set)
+/// Returns the backing [`tempfile::TempDir`]s alongside the set so callers keep
+/// them alive for the test's duration and the directories are removed on drop.
+pub(crate) async fn make_local_set_disks(drive_count: usize, parity_count: usize) -> (Vec<tempfile::TempDir>, Arc<SetDisks>) {
+    make_local_set_disks_with_ctx(drive_count, parity_count, crate::runtime::instance::bootstrap_ctx()).await
 }
 
 /// Like [`make_local_set_disks`], but binds the set to an explicit instance
@@ -759,9 +756,6 @@ async fn blackbox_range_read_restores_exact_slice_with_one_offline_disk() {
 #[tokio::test]
 async fn blackbox_delete_marker_hides_object_body_without_erasing_prior_version_metadata() {
     let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
-    let store = crate::bucket::metadata_sys::object_store_in(set_disks.instance_ctx())
-        .await
-        .expect("fixture metadata authority should belong to this set");
     let bucket = "bb-delete-marker-read-negative";
     let object = "object.bin";
     let opts = ObjectOptions {
@@ -771,26 +765,13 @@ async fn blackbox_delete_marker_hides_object_body_without_erasing_prior_version_
         ..Default::default()
     };
 
-    store
+    set_disks
         .make_bucket(bucket, &MakeBucketOptions::default())
         .await
         .expect("bucket should be created");
     let mut reader = PutObjReader::from_vec(b"body hidden by delete marker".to_vec());
     set_disks
-        .put_object(
-            bucket,
-            object,
-            &mut reader,
-            &ObjectOptions {
-                object_lock_config_snapshot: Some(
-                    store
-                        .object_lock_config_snapshot(bucket)
-                        .await
-                        .expect("PUT fixture should own the bucket lifecycle and Object Lock snapshot"),
-                ),
-                ..opts.clone()
-            },
-        )
+        .put_object(bucket, object, &mut reader, &opts)
         .await
         .expect("object should be written before delete marker");
 

crates/ecstore/src/erasure/codec/bridge.rs

@@ -15,7 +15,6 @@
 use crate::erasure::codec::workspace::RustfsCodecDecodeWorkspace;
 use crate::erasure::coding::Erasure;
 use rustfs_erasure_codec::galois_8::ReedSolomon;
-use smallvec::SmallVec;
 use std::io;
 use std::sync::{Arc, OnceLock};
 
@@ -214,7 +213,7 @@ impl RustfsCodecDecodeEngine {
         // All slots must be present here: the verification path reconstructs
         // missing parity alongside data (see `reconstruct_into`), so a `None`
         // slot is an internal invariant violation, not a degraded-read state.
-        let mut shard_refs: SmallVec<[&[u8]; 16]> = SmallVec::with_capacity(self.data_shards + self.parity_shards);
+        let mut shard_refs = Vec::with_capacity(self.data_shards + self.parity_shards);
         for (index, shard) in shards.iter().enumerate() {
             let shard = shard.as_ref().ok_or_else(|| {
                 io::Error::new(
@@ -424,30 +423,6 @@ mod tests {
         engine.reconstruct_into(shards, &mut workspace).map(|_| ())
     }
 
-    #[test]
-    fn reconstruct_verifies_sources_across_descriptor_boundary() {
-        let data = (0..1025)
-            .map(|index| u8::try_from((index * 29 + 17) % 251).expect("bounded byte"))
-            .collect::<Vec<_>>();
-        for (data_shards, parity_shards) in [(8, 8), (9, 8)] {
-            let erasure = Erasure::new(data_shards, parity_shards, 2048);
-            let expected = encoded_shards(&erasure, &data);
-            let engine = RustfsCodecDecodeEngine::new(&erasure).expect("decode engine");
-            let mut shards = expected.clone();
-            shards[0] = None;
-            shards[data_shards] = None;
-            reconstruct_with(&engine, &mut shards).expect("restore missing data and parity");
-            assert_eq!(shards, expected, "every restored byte must match at width {}", shards.len());
-
-            let mut corrupt = expected.clone();
-            corrupt[0] = None;
-            corrupt[data_shards] = None;
-            corrupt[data_shards + 1].as_mut().expect("surplus parity present")[0] ^= 1;
-            let error = reconstruct_with(&engine, &mut corrupt).expect_err("corrupt surplus source must fail closed");
-            assert_eq!(error.kind(), io::ErrorKind::InvalidData);
-        }
-    }
-
     #[test]
     fn data_shards_complete_ignores_parity_slots() {
         let shards = vec![Some(vec![1]), Some(vec![2]), None];

crates/ecstore/src/erasure/coding/erasure.rs

@@ -189,11 +189,12 @@ impl LegacyReedSolomonEncoder {
                 .is_some_and(|bytes| bytes <= LEGACY_REED_SOLOMON_CACHE_MAX_LOGICAL_SHARD_BYTES_PER_WORKSPACE)
     }
 
-    fn encode(&self, mut shards: SmallVec<[&mut [u8]; 16]>) -> io::Result<()> {
-        if shards.is_empty() {
+    fn encode(&self, shards: SmallVec<[&mut [u8]; 16]>) -> io::Result<()> {
+        let mut shards_vec: Vec<&mut [u8]> = shards.into_vec();
+        if shards_vec.is_empty() {
             return Ok(());
         }
-        let shard_len = shards[0].len();
+        let shard_len = shards_vec[0].len();
         let cached_encoder = self
             .encoder_cache
             .write()
@@ -213,7 +214,7 @@ impl LegacyReedSolomonEncoder {
                     .map_err(|e| io::Error::other(format!("Failed to create SIMD encoder: {e:?}")))?,
             }
         };
-        for (i, shard) in shards.iter().enumerate().take(self.data_shards) {
+        for (i, shard) in shards_vec.iter().enumerate().take(self.data_shards) {
             encoder
                 .add_original_shard(shard)
                 .map_err(|e| io::Error::other(format!("Failed to add shard {i}: {e:?}")))?;
@@ -222,8 +223,8 @@ impl LegacyReedSolomonEncoder {
             .encode()
             .map_err(|e| io::Error::other(format!("SIMD encoding failed: {e:?}")))?;
         for (i, recovery_shard) in result.recovery_iter().enumerate() {
-            if i + self.data_shards < shards.len() {
-                shards[i + self.data_shards].copy_from_slice(recovery_shard);
+            if i + self.data_shards < shards_vec.len() {
+                shards_vec[i + self.data_shards].copy_from_slice(recovery_shard);
             }
         }
         drop(result);
@@ -392,13 +393,14 @@ impl ReedSolomonEncoder {
     }
 
     /// Encode data shards with parity.
-    pub fn encode(&self, mut shards: SmallVec<[&mut [u8]; 16]>) -> io::Result<()> {
-        if shards.is_empty() {
+    pub fn encode(&self, shards: SmallVec<[&mut [u8]; 16]>) -> io::Result<()> {
+        let mut shards_vec: Vec<&mut [u8]> = shards.into_vec();
+        if shards_vec.is_empty() {
             return Ok(());
         }
 
         if let Some(ref rs) = self.encoder {
-            rs.encode(shards.as_mut_slice())
+            rs.encode(&mut shards_vec)
                 .map_err(|e| io::Error::other(format!("Reed-Solomon encode failed: {e:?}")))
         } else {
             Ok(())
@@ -1008,7 +1010,7 @@ impl Erasure {
             return Ok(true);
         }
 
-        let mut shard_refs: SmallVec<[&[u8]; 16]> = SmallVec::with_capacity(expected_shards);
+        let mut shard_refs = Vec::with_capacity(expected_shards);
         let mut shard_len = None;
         for (index, shard) in shards.iter().enumerate() {
             let shard = shard
@@ -1206,53 +1208,6 @@ mod tests {
     use std::task::{Context, Poll};
     use tokio::io::ReadBuf;
 
-    #[test]
-    fn encode_preserves_codec_bytes_across_descriptor_boundary() {
-        let data = (0..1025)
-            .map(|index| u8::try_from((index * 29 + 17) % 251).expect("bounded byte"))
-            .collect::<Vec<_>>();
-        for uses_legacy in [false, true] {
-            for (data_shards, parity_shards) in [(4, 2), (8, 8), (9, 8)] {
-                let erasure = Erasure::new_with_options(data_shards, parity_shards, 2048, uses_legacy);
-                let shards = erasure.encode_data(&data).expect("encode boundary layout");
-                let joined = shards
-                    .iter()
-                    .take(data_shards)
-                    .flat_map(|shard| shard.iter().copied())
-                    .collect::<Vec<_>>();
-                assert_eq!(&joined[..data.len()], &data);
-                assert!(joined[data.len()..].iter().all(|byte| *byte == 0));
-                if uses_legacy {
-                    let mut reference = reed_solomon_simd::ReedSolomonEncoder::new(data_shards, parity_shards, shards[0].len())
-                        .expect("independent legacy encoder");
-                    for shard in shards.iter().take(data_shards) {
-                        reference.add_original_shard(shard).expect("add original shard");
-                    }
-                    let result = reference.encode().expect("independent legacy parity");
-                    for (index, parity) in result.recovery_iter().enumerate() {
-                        assert_eq!(shards[data_shards + index].as_ref(), parity, "legacy parity {index}");
-                    }
-                } else {
-                    let reference = ReedSolomon::new(data_shards, parity_shards).expect("independent modern codec");
-                    let refs = shards.iter().map(Bytes::as_ref).collect::<Vec<_>>();
-                    assert!(reference.verify(&refs).expect("verify independently generated parity"));
-                }
-                let mut optional = optional_shards(&shards);
-                assert!(
-                    erasure
-                        .verify_data_and_parity(&optional)
-                        .expect("verify complete boundary layout")
-                );
-                optional[data_shards].as_mut().expect("parity present")[0] ^= 1;
-                assert!(
-                    !erasure
-                        .verify_data_and_parity(&optional)
-                        .expect("detect corrupt boundary parity")
-                );
-            }
-        }
-    }
-
     fn erasure_with_invalid_dimensions(data_shards: usize, parity_shards: usize, block_size: usize) -> Erasure {
         Erasure {
             data_shards,

crates/ecstore/src/lib.rs

@@ -55,8 +55,8 @@ mod set_disk;
 mod storage_api_contracts;
 mod store;
 
-pub use disk::{with_background_disk_io, with_background_resource_unit};
-pub use store::{BucketIncarnationFenceGuard, PoolMetaWriteGateStatus};
+pub use disk::with_background_disk_io;
+pub use store::PoolMetaWriteGateStatus;
 
 // pub mod checksum;
 mod event;

crates/ecstore/src/object_api/readers.rs

@@ -4473,7 +4473,3 @@ mod tests {
             .expect_err("a lying frame-layout marker must fail closed, not serve plaintext");
     }
 }
-
-#[cfg(test)]
-#[path = "readers/compression_index_compatibility_tests.rs"]
-mod compression_index_compatibility_tests;

crates/ecstore/src/object_api/readers/compression_index_compatibility_tests.rs

@@ -1,185 +0,0 @@
-use super::*;
-use rustfs_utils::http::metadata_compat::{SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, insert_str};
-use sha2::{Digest, Sha256};
-use std::io::Cursor;
-use tokio::io::AsyncReadExt;
-
-const STREAM: &[u8] = include_bytes!(concat!(
-    env!("CARGO_MANIFEST_DIR"),
-    "/tests/fixtures/compression-index/legacy-lz4-stream.bin"
-));
-const HISTORICAL_INDEX: &[u8] = include_bytes!(concat!(
-    env!("CARGO_MANIFEST_DIR"),
-    "/tests/fixtures/compression-index/pre-zigzag-index.bin"
-));
-#[cfg(not(feature = "rio-v2"))]
-const CURRENT_INDEX: &[u8] =
-    include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/compression-index/zigzag-index.bin"));
-const PLAINTEXT_LEN: usize = 3 * 1024 * 1024 + 123;
-const PLAINTEXT_SHA256: &str = "68f9edf6f9600395d6d2b5517d97570fecd2a0c6dbe6130617ef415a3b48d89c";
-
-fn fixture_info(index: &[u8]) -> ObjectInfo {
-    let mut metadata = HashMap::new();
-    insert_str(&mut metadata, SUFFIX_COMPRESSION, "lz4".to_owned());
-    insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, PLAINTEXT_LEN.to_string());
-    ObjectInfo {
-        size: i64::try_from(STREAM.len()).expect("fixed fixture size fits i64"),
-        parts: Arc::new(vec![ObjectPartInfo {
-            number: 1,
-            size: STREAM.len(),
-            actual_size: i64::try_from(PLAINTEXT_LEN).expect("fixed plaintext length fits i64"),
-            index: Some(Bytes::copy_from_slice(index)),
-            ..Default::default()
-        }]),
-        user_defined: Arc::new(metadata),
-        ..Default::default()
-    }
-}
-
-fn fixture_plaintext() -> Vec<u8> {
-    (0..PLAINTEXT_LEN)
-        .map(|offset| {
-            let chunk = offset / 1024;
-            let lane = offset % 1024;
-            if lane < 8 {
-                u64::try_from(chunk).expect("fixed chunk number fits u64").to_le_bytes()[lane]
-            } else {
-                b"abcdefghijklmnopqrstuvwxyz"[(offset + chunk) % 26]
-            }
-        })
-        .collect()
-}
-
-#[tokio::test]
-async fn historical_index_preserves_the_complete_compressed_body() {
-    let info = fixture_info(HISTORICAL_INDEX);
-    let (mut reader, storage_offset, _) = GetObjectReader::new(
-        Box::new(Cursor::new(STREAM.to_vec())),
-        None,
-        &info,
-        &ObjectOptions::default(),
-        &HeaderMap::new(),
-    )
-    .await
-    .expect("construct reader for fixed historical index");
-    let mut actual = Vec::new();
-    reader.read_to_end(&mut actual).await.expect("read complete historical body");
-    assert_eq!(storage_offset, 0);
-    assert_eq!(actual.len(), PLAINTEXT_LEN);
-    assert_eq!(
-        hex_simd::encode_to_string(Sha256::digest(&actual), hex_simd::AsciiCase::Lower),
-        PLAINTEXT_SHA256
-    );
-}
-
-#[tokio::test]
-async fn historical_index_preserves_ranges_across_compression_blocks() {
-    let info = fixture_info(HISTORICAL_INDEX);
-    let expected = fixture_plaintext();
-    for start in [7, 1024 * 1024 - 17, 2 * 1024 * 1024 + 31, PLAINTEXT_LEN - 97] {
-        let range = HTTPRangeSpec {
-            is_suffix_length: false,
-            start: i64::try_from(start).expect("fixed range start fits i64"),
-            end: i64::try_from(start + 63).expect("fixed range end fits i64"),
-        };
-        let plan = ReadPlan::build(Some(range.clone()), &info, &ObjectOptions::default(), &HeaderMap::new())
-            .await
-            .expect("plan historical compressed range");
-        let physical = plan.storage_offset;
-        let input = STREAM.get(physical..).expect("planned offset stays within the fixed stream");
-        let (mut reader, _, _) = GetObjectReader::new(
-            Box::new(Cursor::new(input.to_vec())),
-            Some(range),
-            &info,
-            &ObjectOptions::default(),
-            &HeaderMap::new(),
-        )
-        .await
-        .expect("construct historical range reader");
-        let mut actual = Vec::new();
-        reader.read_to_end(&mut actual).await.expect("read historical range");
-        assert_eq!(actual, expected[start..start + 64], "range starting at {start}");
-    }
-}
-
-#[cfg(not(feature = "rio-v2"))]
-#[tokio::test]
-async fn current_index_selects_a_fixed_compressed_block_for_a_range() {
-    let info = fixture_info(CURRENT_INDEX);
-    let start = 2 * 1024 * 1024 + 31;
-    let range = HTTPRangeSpec {
-        is_suffix_length: false,
-        start: i64::try_from(start).expect("fixed range start fits i64"),
-        end: i64::try_from(start + 63).expect("fixed range end fits i64"),
-    };
-    let plan = ReadPlan::build(Some(range.clone()), &info, &ObjectOptions::default(), &HeaderMap::new())
-        .await
-        .expect("plan range with fixed current index");
-    assert_eq!(plan.storage_offset, 34_663, "use the recorded second block boundary");
-    let physical = plan.storage_offset;
-    let (mut reader, _, _) = GetObjectReader::new(
-        Box::new(Cursor::new(STREAM[physical..].to_vec())),
-        Some(range),
-        &info,
-        &ObjectOptions::default(),
-        &HeaderMap::new(),
-    )
-    .await
-    .expect("construct indexed range reader");
-    let mut actual = Vec::new();
-    reader.read_to_end(&mut actual).await.expect("read indexed range");
-    assert_eq!(actual, fixture_plaintext()[start..start + 64]);
-}
-
-#[tokio::test]
-async fn truncated_index_falls_back_without_truncating_the_range() {
-    let info = fixture_info(&HISTORICAL_INDEX[..HISTORICAL_INDEX.len() - 3]);
-    let start = 2 * 1024 * 1024 + 31;
-    let range = HTTPRangeSpec {
-        is_suffix_length: false,
-        start: i64::try_from(start).expect("fixed range start fits i64"),
-        end: i64::try_from(start + 96).expect("fixed range end fits i64"),
-    };
-    let plan = ReadPlan::build(Some(range.clone()), &info, &ObjectOptions::default(), &HeaderMap::new())
-        .await
-        .expect("invalid index still permits a sequential range plan");
-    assert_eq!(plan.storage_offset, 0, "truncated index must not select a storage offset");
-    let (mut reader, _, _) = GetObjectReader::new(
-        Box::new(Cursor::new(STREAM.to_vec())),
-        Some(range),
-        &info,
-        &ObjectOptions::default(),
-        &HeaderMap::new(),
-    )
-    .await
-    .expect("construct sequential fallback reader");
-    let mut actual = Vec::new();
-    reader
-        .read_to_end(&mut actual)
-        .await
-        .expect("read range without the truncated index");
-    assert_eq!(actual, fixture_plaintext()[start..start + 97]);
-}
-
-#[tokio::test]
-async fn historical_index_fallback_does_not_hide_a_body_checksum_failure() {
-    let info = fixture_info(HISTORICAL_INDEX);
-    let mut corrupted = STREAM.to_vec();
-    corrupted[4] ^= 1; // Corrupt the first block's stored CRC, leaving its payload intact.
-    let (mut reader, _, _) = GetObjectReader::new(
-        Box::new(Cursor::new(corrupted)),
-        None,
-        &info,
-        &ObjectOptions::default(),
-        &HeaderMap::new(),
-    )
-    .await
-    .expect("construct reader before body checksum verification");
-    let mut actual = Vec::new();
-    let error = reader
-        .read_to_end(&mut actual)
-        .await
-        .expect_err("bad body CRC cannot become successful EOF");
-    assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
-    assert!(actual.len() < PLAINTEXT_LEN);
-}

crates/ecstore/src/object_api/types.rs

@@ -152,7 +152,7 @@ pub struct ObjectLockConfigSnapshot {
     config_revision: Option<OffsetDateTime>,
     state: crate::bucket::metadata_sys::ObjectLockConfigState,
     lifecycle_fence: NamespaceLockFence,
-    _lifecycle_guard: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
+    _lifecycle_guard: Option<rustfs_lock::NamespaceLockGuard>,
     metadata_transaction_guard: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
 }
 
@@ -210,33 +210,8 @@ impl ObjectLockConfigSnapshot {
             config_revision: Some(config_revision),
             state,
             lifecycle_fence,
-            _lifecycle_guard: Some(Arc::new(lifecycle_guard)),
-            metadata_transaction_guard: Some(Arc::new(metadata_transaction_guard)),
-        }
-    }
-
-    pub(crate) fn for_store_bucket_under_lifecycle_owner(
-        store_id: Uuid,
-        bucket: &str,
-        bucket_incarnation_id: Uuid,
-        config_revision: OffsetDateTime,
-        state: crate::bucket::metadata_sys::ObjectLockConfigState,
-        lifecycle_guard: Arc<rustfs_lock::NamespaceLockGuard>,
-        metadata_transaction_guard: Arc<rustfs_lock::NamespaceLockGuard>,
-    ) -> Self {
-        let mut lifecycle_fence = NamespaceLockFence::new();
-        if let Some(signal) = lifecycle_guard.lock_lost_signal() {
-            lifecycle_fence.add_signal(signal);
-        }
-        Self {
-            store_id: Some(store_id),
-            bucket: Some(bucket.to_owned()),
-            bucket_incarnation_id: Some(bucket_incarnation_id),
-            config_revision: Some(config_revision),
-            state,
-            lifecycle_fence,
             _lifecycle_guard: Some(lifecycle_guard),
-            metadata_transaction_guard: Some(metadata_transaction_guard),
+            metadata_transaction_guard: Some(Arc::new(metadata_transaction_guard)),
         }
     }
 
@@ -279,15 +254,6 @@ impl ObjectLockConfigSnapshot {
         &self.state
     }
 
-    pub(crate) fn is_unbound(&self) -> bool {
-        self.store_id.is_none()
-            && self.bucket.is_none()
-            && self.bucket_incarnation_id.is_none()
-            && self.config_revision.is_none()
-            && self._lifecycle_guard.is_none()
-            && self.metadata_transaction_guard.is_none()
-    }
-
     pub(crate) fn is_valid_for_destructive_put(&self, store_id: Uuid, bucket: &str, bucket_incarnation_id: Uuid) -> bool {
         self.store_id == Some(store_id)
             && self.bucket.as_deref() == Some(bucket)
@@ -300,11 +266,6 @@ impl ObjectLockConfigSnapshot {
                 .is_some_and(|guard| !guard.is_lock_lost())
     }
 
-    pub(crate) fn owns_bucket_lifecycle_fence(&self, store_id: Uuid, bucket: &str, bucket_incarnation_id: Uuid) -> bool {
-        self.is_valid_for_destructive_put(store_id, bucket, bucket_incarnation_id)
-            && self._lifecycle_guard.as_ref().is_some_and(|guard| !guard.is_lock_lost())
-    }
-
     /// Share the held transaction lock without queuing another reader behind
     /// a metadata writer that is itself waiting for this snapshot to drop.
     pub(crate) fn metadata_transaction_guard_for(
@@ -322,22 +283,6 @@ impl ObjectLockConfigSnapshot {
         self.metadata_transaction_guard.clone()
     }
 
-    /// Returns the snapshot's validated bucket incarnation ID only while both the
-    /// lifecycle read fence and the metadata-transaction guard remain active and
-    /// unreleased for `(store_id, bucket)`.
-    pub(crate) fn validated_bucket_incarnation_for(
-        &self,
-        store_id: Uuid,
-        bucket: &str,
-        expected_incarnation_id: Option<Uuid>,
-    ) -> Option<Uuid> {
-        let guard = self.metadata_transaction_guard_for(store_id, bucket, expected_incarnation_id)?;
-        if guard.is_released() {
-            return None;
-        }
-        self.bucket_incarnation_id
-    }
-
     pub(crate) fn add_lock_fences(&self, opts: &mut ObjectOptions) {
         opts.bucket_lifecycle_lock_fence
             .get_or_insert_with(NamespaceLockFence::new)
@@ -1001,12 +946,6 @@ pub struct ObjectOptions {
     /// A genuine target write guard; loss fences alone do not authorize a borrowed write.
     #[doc(hidden)]
     pub write_lock_context: Option<WriteLockContext>,
-    /// Ordinary ECStore PUT target bound to its bucket's immutable artifact namespace.
-    #[doc(hidden)]
-    pub artifact_put_target: Option<Arc<crate::bucket::artifact::ArtifactPutTarget>>,
-    /// Owned, non-serializable authority for one fully sealed managed version.
-    #[doc(hidden)]
-    pub artifact_delete_target: Option<Arc<crate::bucket::artifact::ArtifactDeleteTarget>>,
     /// In-process delete ownership for late filesystem completion. Never
     /// serialized or copied into object metadata.
     #[doc(hidden)]
@@ -1086,9 +1025,6 @@ pub struct ObjectOptions {
     /// Proves an upper layer holds the bucket lifecycle sentinel. A separate
     /// fence avoids recursively acquiring the read lock behind a queued writer.
     pub bucket_lifecycle_lock_fence: Option<NamespaceLockFence>,
-    /// Actual generation owner retained by storage commits and their physical tails.
-    #[doc(hidden)]
-    pub bucket_incarnation_fence: Option<Arc<crate::store::BucketIncarnationFenceGuard>>,
     /// Retains the actual lifecycle guard for an owned multipart commit. A
     /// lock-loss fence alone does not keep a cancelled caller's lock alive. The
     /// owner also retains the continuous incarnation-validation coverage.

crates/ecstore/src/object_api/write_commit_context.rs

@@ -71,7 +71,7 @@ impl WriteCommitGuard {
 
     /// Namespace locks key the disk identity of the name (see `new_ns_lock`),
     /// so a guard proves every spelling that resolves to the same file.
-    pub(crate) fn matches(&self, bucket: &str, object: &str) -> bool {
+    fn matches(&self, bucket: &str, object: &str) -> bool {
         self.bucket.as_ref() == bucket && self.object.as_ref() == object_name_disk_identity(object).as_ref()
     }
 }

crates/ecstore/src/runtime/instance.rs

@@ -110,82 +110,6 @@ impl Drop for NamespaceCommitGuard {
     }
 }
 
-/// A durable decommission pause as seen by this instance's worker watchdog.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub(crate) struct DecommissionPauseRecord {
-    pub(crate) reason: String,
-    /// The pool's settled progress when it paused; see
-    /// [`DecommissionCountLedger::progress`].
-    pub(crate) progress: u64,
-    /// Consecutive pauses on `reason` without progress in between.
-    pub(crate) repeats: u32,
-    /// The watchdog does not relaunch the pool before this instant.
-    pub(crate) relaunch_after: Instant,
-}
-
-/// What this instance counted for one decommission generation of a pool.
-///
-/// Progress counters persist only at checkpoints and a resumed pass relists
-/// the bucket, so a retained entry (one whose source stays) is seen again on
-/// every pass. The ledger keeps its counted versions so a relist in the same
-/// process does not count them twice.
-#[derive(Debug, Default)]
-pub(crate) struct DecommissionCountLedger {
-    generation: Option<time::OffsetDateTime>,
-    /// (bucket, object, version id, delete marker) of retained versions.
-    retained: HashSet<(String, String, Option<uuid::Uuid>, bool)>,
-    retained_key_bytes: usize,
-    /// Entries whose source was cleaned up. Only moving an entry off the pool
-    /// advances it; a relisted retained entry cannot.
-    cleaned_entries: u64,
-}
-
-impl DecommissionCountLedger {
-    const MAX_RETAINED_VERSIONS: usize = 1024;
-    const MAX_RETAINED_KEY_BYTES: usize = 1024 * 1024;
-
-    /// Whether this version is counted now: a retained version once, and a
-    /// cleaned-up version unless it was already counted while retained.
-    /// When the retained ledger is full, omit that progress count until source
-    /// cleanup. This bounds diagnostic memory without evicting deduplication
-    /// evidence or blocking data movement.
-    pub(crate) fn admit(&mut self, bucket: &str, object: &str, version: (Option<uuid::Uuid>, bool), cleaned: bool) -> bool {
-        let key_bytes = bucket.len().saturating_add(object.len());
-        if cleaned {
-            // Most successful entries never needed retained-version tracking.
-            if self.retained.is_empty() || key_bytes > Self::MAX_RETAINED_KEY_BYTES {
-                return true;
-            }
-            let key = (bucket.to_owned(), object.to_owned(), version.0, version.1);
-            if self.retained.remove(&key) {
-                self.retained_key_bytes -= key_bytes;
-                return false;
-            }
-            return true;
-        }
-        if self.retained.len() >= Self::MAX_RETAINED_VERSIONS
-            || key_bytes > Self::MAX_RETAINED_KEY_BYTES.saturating_sub(self.retained_key_bytes)
-        {
-            return false;
-        }
-        let key = (bucket.to_owned(), object.to_owned(), version.0, version.1);
-        if self.retained.insert(key) {
-            self.retained_key_bytes += key_bytes;
-            true
-        } else {
-            false
-        }
-    }
-
-    pub(crate) fn record_cleaned_entry(&mut self) {
-        self.cleaned_entries = self.cleaned_entries.saturating_add(1);
-    }
-
-    pub(crate) fn progress(&self) -> u64 {
-        self.cleaned_entries
-    }
-}
-
 /// Runtime state owned by a single `ECStore` instance.
 ///
 /// This is intentionally minimal in the first migration slice; subsequent
@@ -297,12 +221,6 @@ pub struct InstanceContext {
     /// start runs detached from its caller, so this keeps abandoned requests
     /// and their retries from stacking concurrent starts and preflight heals.
     decommission_start_in_flight: AtomicBool,
-    /// The last durable pause of each local decommission, keyed by pool
-    /// index. The worker watchdog backs off relaunching a pool that keeps
-    /// pausing on the same cause without progress.
-    decommission_pauses: std::sync::Mutex<HashMap<usize, DecommissionPauseRecord>>,
-    /// Per-pool counting ledger of the current decommission generation.
-    decommission_count_ledgers: std::sync::Mutex<HashMap<usize, DecommissionCountLedger>>,
     /// Remote scanner publication leases own a read guard until explicit
     /// release or bounded expiry.  Keeping the guard in storage-owned state
     /// makes a remote movement transition wait on the same fence as a local
@@ -377,8 +295,6 @@ impl InstanceContext {
             background_cancel_token: OnceLock::new(),
             data_movement_operation_gate: Arc::new(RwLock::new(())),
             decommission_start_in_flight: AtomicBool::new(false),
-            decommission_pauses: std::sync::Mutex::new(HashMap::new()),
-            decommission_count_ledgers: std::sync::Mutex::new(HashMap::new()),
             scanner_publication_leases: Arc::new(Mutex::new(HashMap::new())),
             data_movement_operation_epoch: AtomicU64::new(0),
             data_movement_operation_epoch_exhausted: AtomicBool::new(false),
@@ -464,52 +380,6 @@ impl InstanceContext {
         self.decommission_start_in_flight.load(Ordering::Acquire)
     }
 
-    /// Record a durable pause of `pool_index`; `update` derives the new record
-    /// from the previous one.
-    pub(crate) fn record_decommission_pause(
-        &self,
-        pool_index: usize,
-        update: impl FnOnce(Option<&DecommissionPauseRecord>) -> DecommissionPauseRecord,
-    ) -> DecommissionPauseRecord {
-        let mut pauses = self
-            .decommission_pauses
-            .lock()
-            .unwrap_or_else(std::sync::PoisonError::into_inner);
-        let record = update(pauses.get(&pool_index));
-        pauses.insert(pool_index, record.clone());
-        record
-    }
-
-    /// Run `update` on `pool_index`'s counting ledger for `generation`,
-    /// starting a fresh ledger when the generation changed.
-    pub(crate) fn with_decommission_count_ledger<T>(
-        &self,
-        pool_index: usize,
-        generation: time::OffsetDateTime,
-        update: impl FnOnce(&mut DecommissionCountLedger) -> T,
-    ) -> T {
-        let mut ledgers = self
-            .decommission_count_ledgers
-            .lock()
-            .unwrap_or_else(std::sync::PoisonError::into_inner);
-        let ledger = ledgers.entry(pool_index).or_default();
-        if ledger.generation != Some(generation) {
-            *ledger = DecommissionCountLedger {
-                generation: Some(generation),
-                ..Default::default()
-            };
-        }
-        update(ledger)
-    }
-
-    pub(crate) fn decommission_pause(&self, pool_index: usize) -> Option<DecommissionPauseRecord> {
-        self.decommission_pauses
-            .lock()
-            .unwrap_or_else(std::sync::PoisonError::into_inner)
-            .get(&pool_index)
-            .cloned()
-    }
-
     pub(crate) async fn install_scanner_publication_lease(
         &self,
         token: Uuid,
@@ -814,14 +684,6 @@ impl InstanceContext {
         self.tier_config_mgr.get_or_init(TierConfigMgr::new).clone()
     }
 
-    #[cfg(test)]
-    pub(crate) fn set_test_tier_config_mgr(&self, manager: Arc<RwLock<TierConfigMgr>>) {
-        assert!(
-            self.tier_config_mgr.set(manager).is_ok(),
-            "test tier manager must be installed before use"
-        );
-    }
-
     /// This instance's event notifier, materializing it on first access.
     /// Shared for the lifetime of the instance.
     pub fn event_notifier(&self) -> Arc<RwLock<EventNotifier>> {
@@ -1057,38 +919,6 @@ pub fn bootstrap_ctx() -> Arc<InstanceContext> {
 mod tests {
     use super::*;
 
-    #[test]
-    fn decommission_count_ledger_bounds_retained_entries() {
-        let mut ledger = super::DecommissionCountLedger::default();
-        for index in 0..1024 {
-            assert!(ledger.admit("bucket", &format!("key-{index}"), (None, false), false));
-        }
-        assert!(!ledger.admit("bucket", "overflow", (None, false), false));
-        assert_eq!(ledger.retained.len(), 1024);
-        // An uncounted retained version can still be counted after cleanup.
-        assert!(ledger.admit("bucket", "overflow", (None, false), true));
-        assert!(!ledger.admit("bucket", "key-0", (None, false), true));
-        assert!(ledger.admit("bucket", "next", (None, false), false));
-        assert!(!ledger.admit("bucket", "next", (None, false), false));
-        assert_eq!(ledger.retained.len(), 1024);
-    }
-
-    #[test]
-    fn decommission_count_ledger_bounds_retained_key_bytes() {
-        let mut ledger = super::DecommissionCountLedger::default();
-        let bucket = "b".repeat(63);
-        let object_prefix = "x".repeat(1020);
-        let mut admitted = 0;
-        for index in 0..1024 {
-            admitted += usize::from(ledger.admit(&bucket, &format!("{object_prefix}{index:04}"), (None, false), false));
-        }
-        assert!(admitted > 0 && admitted < 1024);
-        assert!(ledger.retained.iter().map(|(b, o, _, _)| b.len() + o.len()).sum::<usize>() <= 1024 * 1024);
-        // Removal releases the byte reservation as well as the entry slot.
-        assert!(!ledger.admit(&bucket, &format!("{object_prefix}0000"), (None, false), true));
-        assert!(ledger.admit(&bucket, &format!("{object_prefix}1024"), (None, false), false));
-    }
-
     #[test]
     fn namespace_commit_guards_are_instance_local_and_count_until_last_owner() {
         let first = Arc::new(InstanceContext::new());

crates/ecstore/src/services/rebalance/entry.rs

@@ -250,7 +250,7 @@ impl ECStore {
         }
 
         let bucket_incarnation_fence = match bucket_configs.bucket_incarnation_id {
-            Some(expected) => Some(Arc::new(self.acquire_bucket_incarnation_fence(&bucket, expected).await?)),
+            Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?),
             None => None,
         };
 
@@ -533,8 +533,9 @@ impl ECStore {
                         &cleanup_transferred_tier_versions,
                         data_movement::SourceCleanupBucketFence {
                             expected_incarnation_id: bucket_configs.bucket_incarnation_id,
-                            lifecycle_owner: bucket_incarnation_fence.as_ref(),
-                            lifecycle_guard: None,
+                            lifecycle_guard: bucket_incarnation_fence
+                                .as_ref()
+                                .and_then(|guard| guard.namespace_lock_guard()),
                             namespace_lock_lost_signal: lock_lost_signal.clone(),
                             ..Default::default()
                         },
@@ -1728,8 +1729,6 @@ mod tests {
         /// Both versions are on the target with their identities, the null
         /// version's content and SHA-256 are intact, and the source is gone.
         async fn assert_moved(&self) {
-            use crate::disk::DiskAPI as _;
-
             let target = self.store.pools[1]
                 .get_disks_by_key(self.object)
                 .load_file_info_versions_exact(self.bucket, self.object)
@@ -1770,16 +1769,6 @@ mod tests {
                 self.source_versions().await.is_none(),
                 "the source entry must be removed only after both versions moved"
             );
-            let source_disks = self.source_set.disk_inventory().await;
-            assert_eq!(source_disks.iter().flatten().count(), 2, "both source disks must be online");
-            for disk in source_disks.into_iter().flatten() {
-                assert!(
-                    !tokio::fs::try_exists(disk.path().join(self.bucket).join(self.object))
-                        .await
-                        .expect("inspect the physical rebalance source path"),
-                    "rebalance must remove source metadata and shards from every disk"
-                );
-            }
         }
     }
 
@@ -1793,141 +1782,6 @@ mod tests {
         fixture.assert_moved().await;
     }
 
-    /// An `ExpiredObjectAllVersions` expiry is a prefix delete that takes the
-    /// bucket lifecycle write lock, while the entry holds that lock for read
-    /// through its bucket incarnation fence. Applied inline, it waited out the
-    /// lock timeout, stalled every lifecycle read in the bucket behind it, and
-    /// failed the entry. The version moves instead and lifecycle expires it on
-    /// the target (rustfs/backlog#2873).
-    #[test]
-    #[serial_test::serial]
-    fn real_rebalance_moves_a_version_expired_by_a_delete_all_rule() {
-        run_large_stack_async_test("rebalance-delete-all-expiry", run_delete_all_expiry_case);
-    }
-
-    async fn run_delete_all_expiry_case() {
-        const REBALANCE_ID: &str = "rebalance-delete-all-expiry";
-        let (_temp_dirs, store, _peer) = crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None).await;
-        let bucket = "rebalance-delete-all";
-        let object = "expired.bin";
-        // An active run blocks the data-usage publication bucket creation needs.
-        Box::pin(store.make_bucket(
-            bucket,
-            &MakeBucketOptions {
-                versioning_enabled: true,
-                ..Default::default()
-            },
-        ))
-        .await
-        .expect("create bucket");
-        let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
-        let rebalance_meta = active_rebalance_meta(REBALANCE_ID);
-        rebalance_meta
-            .save(Arc::clone(&store.pools[0]))
-            .await
-            .expect("persist the active rebalance run");
-        *store.rebalance_meta.write().await = Some(rebalance_meta);
-        let source_set = store.pools[0].get_disks_by_key(object);
-        let payload = b"expired by an all-versions rule".to_vec();
-        let written = Box::pin(source_set.put_object(
-            bucket,
-            object,
-            &mut PutObjReader::from_vec(payload.clone()),
-            &ObjectOptions {
-                versioned: true,
-                mod_time: Some(OffsetDateTime::now_utc() - time::Duration::days(10)),
-                expected_bucket_incarnation_id: Some(incarnation),
-                ..Default::default()
-            },
-        ))
-        .await
-        .expect("seed the expired source version");
-        let lifecycle = BucketLifecycleConfiguration {
-            expiry_updated_at: None,
-            rules: vec![LifecycleRule {
-                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
-                expiration: Some(LifecycleExpiration {
-                    days: Some(1),
-                    expired_object_all_versions: Some(true),
-                    ..Default::default()
-                }),
-                abort_incomplete_multipart_upload: None,
-                del_marker_expiration: None,
-                filter: None,
-                id: Some("delete-all".to_string()),
-                noncurrent_version_expiration: None,
-                noncurrent_version_transitions: None,
-                prefix: None,
-                transitions: None,
-            }],
-        };
-        let entry = metacache_entry_from_source(&source_set, bucket, object).await;
-
-        let lock_timeout = crate::set_disk::get_lock_acquire_timeout();
-        let started = std::time::Instant::now();
-        let outcome = tokio::time::timeout(
-            lock_timeout,
-            Box::pin(Arc::clone(&store).rebalance_entry(
-                RebalanceEntryTarget {
-                    bucket: bucket.to_string(),
-                    pool_index: 0,
-                },
-                entry,
-                Arc::clone(&source_set),
-                Arc::new(RebalanceBucketConfigs {
-                    bucket_incarnation_id: Some(incarnation),
-                    lifecycle_config: Some(lifecycle),
-                    ..Default::default()
-                }),
-                Arc::from(REBALANCE_ID),
-                CancellationToken::new(),
-            )),
-        )
-        .await
-        .expect("the entry must not wait out the bucket lifecycle lock timeout")
-        .expect("rebalance entry");
-        let elapsed = started.elapsed();
-        assert_eq!(outcome, RebalanceEntryOutcome::Completed);
-        assert!(
-            elapsed < lock_timeout / 2,
-            "the entry must finish well under the {lock_timeout:?} lock timeout, took {elapsed:?}"
-        );
-
-        assert!(
-            source_set
-                .load_file_info_versions_exact(bucket, object)
-                .await
-                .expect("read source versions")
-                .is_none(),
-            "the source entry must be removed once the version moved"
-        );
-        let target = store.pools[1]
-            .get_disks_by_key(object)
-            .load_file_info_versions_exact(bucket, object)
-            .await
-            .expect("read target versions")
-            .expect("the version must reach the target pool");
-        assert_eq!(target.versions.len(), 1, "{:?}", target.versions);
-        assert_eq!(target.versions[0].version_id, written.version_id);
-        let mut reader = store.pools[1]
-            .get_object_reader(
-                bucket,
-                object,
-                None,
-                HeaderMap::new(),
-                &ObjectOptions {
-                    versioned: true,
-                    version_id: written.version_id.map(|version| version.to_string()),
-                    ..Default::default()
-                },
-            )
-            .await
-            .expect("the moved version should be readable");
-        let mut body = Vec::new();
-        reader.stream.read_to_end(&mut body).await.expect("drain the moved version");
-        assert_eq!(body, payload);
-    }
-
     /// Overwriting after versioning was enabled leaves a newer version above
     /// the null version. The old `None` read selected that newer version, the
     /// target accepted it as an equivalent copy, and cleanup then deleted the

crates/ecstore/src/services/rebalance/mod.rs

@@ -88,7 +88,6 @@ pub async fn test_store_with_persisted_rebalance_meta(
         pool_meta_save_gate: tokio::sync::Mutex::default(),
         ctx,
         bucket_fence_registry: std::sync::Arc::default(),
-        ordinary_classification: Default::default(),
     });
     (temp_dirs, store)
 }
@@ -148,19 +147,6 @@ pub(crate) async fn test_three_pool_stores_with_isolated_node_contexts(
     test_pool_stores_with_contexts(rebalance_meta, true, 3, 2).await
 }
 
-/// Four drives per set, so a set can hold metadata on fewer than its read
-/// quorum of drives while each drive stays online.
-#[cfg(all(test, feature = "test-util"))]
-pub(crate) async fn test_three_pool_stores_with_four_disk_sets_with_isolated_node_contexts(
-    rebalance_meta: Option<RebalanceMeta>,
-) -> (
-    Vec<tempfile::TempDir>,
-    std::sync::Arc<crate::store::ECStore>,
-    std::sync::Arc<crate::store::ECStore>,
-) {
-    test_pool_stores_with_contexts(rebalance_meta, true, 3, 4).await
-}
-
 #[cfg(all(test, feature = "test-util"))]
 pub(crate) async fn test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(
     rebalance_meta: Option<RebalanceMeta>,
@@ -267,7 +253,6 @@ async fn test_pool_stores_with_contexts(
             pool_meta_save_gate: tokio::sync::Mutex::new(pool_meta_write_state.independent_clone_for_test()),
             ctx: store_ctx,
             bucket_fence_registry: std::sync::Arc::default(),
-            ordinary_classification: Default::default(),
         })
     };
     let store = make_store(ctx);

crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs

@@ -3415,7 +3415,6 @@ fn test_store_with_rebalance_meta(meta: RebalanceMeta) -> Arc<crate::store::ECSt
         pool_meta_save_gate: tokio::sync::Mutex::default(),
         ctx: crate::runtime::instance::bootstrap_ctx(),
         bucket_fence_registry: std::sync::Arc::default(),
-        ordinary_classification: Default::default(),
     })
 }
 

crates/ecstore/src/services/rebalance/worker.rs

@@ -641,13 +641,7 @@ impl SetDisks {
                         state = "agreed_entry",
                         "Rebalance listing agreed entry"
                     );
-                    {
-                        let future = cb1(entry);
-                        Box::pin(async move {
-                            future.await;
-                            Ok(())
-                        })
-                    }
+                    Box::pin(cb1(entry))
                 })),
                 partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
                     let resolver = resolver.clone();

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -3053,7 +3053,7 @@ impl RenameRollbackReceipt {
 
 struct RenameRollbackOwnership {
     receipt: Option<RenameRollbackReceipt>,
-    namespace_commit_guard: Option<Arc<dyn Send + Sync>>,
+    namespace_commit_guard: Option<Arc<crate::runtime::instance::NamespaceCommitGuard>>,
 }
 
 async fn inspect_incomplete_rename_rollback(
@@ -3169,7 +3169,7 @@ async fn rollback_failed_rename(
                         old_data_dir: rollback_dir,
                         ..Default::default()
                     },
-                    disk_namespace_commit_guard,
+                    disk_namespace_commit_guard.map(|owner| owner as Arc<dyn Send + Sync>),
                 )
                 .await
             });
@@ -3281,11 +3281,10 @@ async fn record_rename_rollback_outcomes(
 /// quorum and optional scanner lease map together avoids widening either
 /// fanout helper's argument list while preserving the fence semantics.
 pub(in crate::set_disk) struct RenameDataFenceOptions<'a> {
-    artifact_put: Option<crate::bucket::artifact::ArtifactPutIdentity>,
     write_quorum: usize,
     scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
     scanner_publication_commit_scope: Option<crate::object_api::ScannerPublicationCommitScope>,
-    namespace_commit_guard: Option<Arc<dyn Send + Sync>>,
+    namespace_commit_guard: Option<Arc<crate::runtime::instance::NamespaceCommitGuard>>,
     rollback_receipt: Option<RenameRollbackReceipt>,
 }
 
@@ -3295,7 +3294,6 @@ impl<'a> RenameDataFenceOptions<'a> {
         scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
     ) -> Self {
         Self {
-            artifact_put: None,
             write_quorum,
             scanner_publication_lease_tokens,
             scanner_publication_commit_scope: None,
@@ -3304,14 +3302,6 @@ impl<'a> RenameDataFenceOptions<'a> {
         }
     }
 
-    pub(in crate::set_disk) fn with_artifact_put(
-        mut self,
-        identity: Option<crate::bucket::artifact::ArtifactPutIdentity>,
-    ) -> Self {
-        self.artifact_put = identity;
-        self
-    }
-
     pub(in crate::set_disk) fn with_rollback_receipt(mut self, receipt: RenameRollbackReceipt) -> Self {
         self.rollback_receipt = Some(receipt);
         self
@@ -3327,7 +3317,7 @@ impl<'a> RenameDataFenceOptions<'a> {
 
     pub(in crate::set_disk) fn with_namespace_commit_guard(
         mut self,
-        namespace_commit_guard: Option<Arc<dyn Send + Sync>>,
+        namespace_commit_guard: Option<Arc<crate::runtime::instance::NamespaceCommitGuard>>,
     ) -> Self {
         self.namespace_commit_guard = namespace_commit_guard;
         self
@@ -3780,7 +3770,6 @@ impl SetDisks {
         fence_options: RenameDataFenceOptions<'_>,
     ) -> disk::error::Result<RenameDataCommit> {
         let RenameDataFenceOptions {
-            artifact_put,
             write_quorum,
             scanner_publication_lease_tokens,
             scanner_publication_commit_scope: _scanner_publication_commit_scope,
@@ -3875,10 +3864,11 @@ impl SetDisks {
                                     &file_info,
                                     &dst_bucket,
                                     &dst_object,
-                                    crate::disk::RenameDataGuards {
-                                        artifact_put,
+                                    disk::RenameDataGuards {
                                         scanner_publication_lease_token,
-                                        namespace_owner: namespace_commit_guard.clone(),
+                                        namespace_owner: namespace_commit_guard
+                                            .clone()
+                                            .map(|owner| owner as Arc<dyn Send + Sync>),
                                         ..Default::default()
                                     },
                                 )
@@ -4187,7 +4177,6 @@ impl SetDisks {
             .await;
         }
         let RenameDataFenceOptions {
-            artifact_put,
             write_quorum,
             scanner_publication_lease_tokens,
             scanner_publication_commit_scope,
@@ -4308,10 +4297,11 @@ impl SetDisks {
                                     file_info,
                                     &dst_bucket,
                                     &dst_object,
-                                    crate::disk::RenameDataGuards {
-                                        artifact_put,
+                                    disk::RenameDataGuards {
                                         scanner_publication_lease_token,
-                                        namespace_owner: namespace_commit_guard.clone(),
+                                        namespace_owner: namespace_commit_guard
+                                            .clone()
+                                            .map(|owner| owner as Arc<dyn Send + Sync>),
                                         ..Default::default()
                                     },
                                 )
@@ -12825,118 +12815,6 @@ mod tests {
         assert!(missing.below_quorum, "missing disk slots should remain visible in the quorum lens");
     }
 
-    #[cfg(not(windows))]
-    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-    async fn legacy_path_delete_flow_prefix_delete_keeps_scanner_owner_until_drain() {
-        use crate::disk::os::{drain_namespace_for_test, prepared_publication_test_hooks as hooks};
-        use crate::object_api::{
-            ScannerPublicationCommitScope, ScannerPublicationCommitScopeGuard, ScannerPublicationCommitState,
-        };
-        use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_isolated;
-        use crate::storage_api_contracts::bucket::BucketOperations;
-
-        let (_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
-        let bucket = "path-delete-scanner";
-        crate::bucket::metadata_sys::object_store_in(&set.ctx)
-            .await
-            .expect("fixture object store")
-            .make_bucket(bucket, &Default::default())
-            .await
-            .expect("authoritative bucket creation");
-        let mut paths = Vec::new();
-        for disk in &disks {
-            disk.write_all(bucket, "prefix/part.1", Bytes::from_static(b"old shard"))
-                .await
-                .expect("seed prefix on every disk");
-            paths.push(
-                disk.get_object_path_for_io_if_local(bucket, "prefix")
-                    .expect("local disk")
-                    .expect("prefix path"),
-            );
-        }
-        let movement = Arc::new(tokio::sync::RwLock::new(()));
-        let scope = ScannerPublicationCommitScope::new_storage_owned(
-            7,
-            tokio::time::Instant::now() + Duration::from_secs(30),
-            Vec::new(),
-            movement.clone().read_owned().await,
-        );
-        scope.try_begin().expect("begin scanner publication");
-        let caller_guard = ScannerPublicationCommitScopeGuard::new(scope.clone());
-        let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
-        let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
-        let _hook = hooks::install_at(hooks::Stage::Rename, &paths[0], move || {
-            let _ = entered_tx.send(());
-            let _ = release_rx.recv();
-        });
-        let owner_set = set.clone();
-        let owner_scope = scope.clone();
-        let caller = tokio::spawn(async move {
-            owner_set
-                .delete_prefix_with_scanner_publication_lease(bucket, "prefix", None, Some(owner_scope))
-                .await
-        });
-        let entered = tokio::time::timeout(Duration::from_secs(5), entered_rx).await;
-        caller.abort();
-        let _ = caller.await;
-        drop(caller_guard);
-        let state_while_blocked = scope.state();
-        let movement_while_blocked = movement.clone().try_write_owned().is_ok();
-        let disk::Disk::Local(local) = disks[0].as_ref() else {
-            unreachable!("fixture contains local disks");
-        };
-        let local = local.get_disk();
-        let volume_read_held = crate::disk::os::disk_volume_mutation_lock(&local.root, bucket)
-            .try_write_owned()
-            .is_err();
-        let mut deletion = Box::pin(local.delete_volume(bucket, true));
-        let early_deletion = futures::poll!(deletion.as_mut());
-        let deletion_was_blocked = early_deletion.is_pending();
-        // Always release and drain physical work before reporting an ownership failure.
-        let _ = release_tx.send(());
-        let mut drained = Vec::new();
-        for path in &paths {
-            drained.push(tokio::time::timeout(Duration::from_secs(5), drain_namespace_for_test(path)).await);
-        }
-        let deletion_result = match early_deletion {
-            std::task::Poll::Ready(result) => result,
-            std::task::Poll::Pending => tokio::time::timeout(Duration::from_secs(5), deletion)
-                .await
-                .expect("volume deletion drains"),
-        };
-        let terminal = tokio::time::timeout(Duration::from_secs(5), scope.wait_for_completion()).await;
-        assert!(
-            entered.is_ok_and(|result| result.is_ok()),
-            "real prefix deletion must reach its filesystem worker"
-        );
-        assert!(drained.into_iter().all(|result| result.is_ok()), "every physical namespace must drain");
-        deletion_result.expect("volume deletion after physical drain");
-        assert!(
-            volume_read_held && deletion_was_blocked,
-            "volume deletion must wait for the physical prefix worker"
-        );
-        assert_eq!(state_while_blocked, ScannerPublicationCommitState::InFlight);
-        assert!(!movement_while_blocked, "scanner movement remains fenced after caller cancellation");
-        assert_eq!(terminal.expect("scanner owner completion"), ScannerPublicationCommitState::Committed);
-        assert!(scope.release_movement_permit().await, "terminal scanner owner releases movement");
-        assert!(movement.try_write_owned().is_ok());
-        for path in paths.iter().skip(1) {
-            assert!(!path.exists(), "successful prefix fanout removes every surviving volume's prefix");
-        }
-        disks[0].make_volume(bucket).await.expect("recreate deleted volume");
-        disks[0]
-            .write_all(bucket, "prefix/sentinel", Bytes::from_static(b"new generation"))
-            .await
-            .expect("new generation sentinel");
-        assert_eq!(
-            disks[0]
-                .read_all(bucket, "prefix/sentinel")
-                .await
-                .expect("sentinel survives old deletion"),
-            b"new generation".as_slice()
-        );
-    }
-
     #[tokio::test]
     async fn delete_prefix_succeeds_when_present_disks_reach_quorum() {
         let bucket = "delete-prefix-bucket";

crates/ecstore/src/set_disk/core/metadata_read.rs

@@ -67,7 +67,6 @@ pub(in crate::set_disk) struct MetadataReadResult {
     pub(in crate::set_disk) slots: Vec<MetadataDiskObservation>,
     pub(in crate::set_disk) diagnostics: MetadataFanoutDiagnostics,
     pub(in crate::set_disk) condition_stopped: bool,
-    pub(in crate::set_disk) verified_inline_body: Option<bytes::Bytes>,
 }
 
 impl MetadataReadResult {
@@ -635,7 +634,7 @@ pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason(
     parts_metadata: &[FileInfo],
     disks: &[Option<DiskStore>],
 ) -> Option<&'static str> {
-    verify_inline_body_from_observations(
+    inline_body_miss_reason_from_observations(
         bucket,
         object,
         candidate,
@@ -646,18 +645,17 @@ pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason(
         disks,
     )
     .await
-    .err()
 }
 
-async fn verify_inline_body_from_observations<'a>(
+async fn inline_body_miss_reason_from_observations<'a>(
     bucket: &str,
     object: &str,
     candidate: &FileInfo,
     observations: impl IntoIterator<Item = (usize, Option<&'a FileInfo>)>,
     disks: &[Option<DiskStore>],
-) -> Result<bytes::Bytes, &'static str> {
+) -> Option<&'static str> {
     if let Some(reason) = data_read_early_stop_inline_candidate_miss_reason(candidate) {
-        return Err(reason);
+        return Some(reason);
     }
 
     let Ok(erasure) = coding::Erasure::try_new_with_options(
@@ -666,21 +664,21 @@ async fn verify_inline_body_from_observations<'a>(
         candidate.erasure.block_size,
         candidate.uses_legacy_checksum,
     ) else {
-        return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY);
+        return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY);
     };
     let data_files =
         match collect_inline_data_shard_fileinfos_from_observations(observations, candidate, erasure.data_shards, |index| {
             disks.get(index).is_some_and(Option::is_some)
         }) {
             Ok(data_files) => data_files,
-            Err(reason) => return Err(reason),
+            Err(reason) => return Some(reason),
         };
 
     let Some(part) = candidate.parts.first() else {
-        return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE);
+        return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE);
     };
     let Ok(object_size) = usize::try_from(candidate.size) else {
-        return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
+        return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
     };
     let checksum_info = candidate.erasure.get_checksum_info(part.number);
     let checksum_algo = if candidate.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S {
@@ -700,14 +698,14 @@ async fn verify_inline_body_from_observations<'a>(
     let Ok(mut readers) =
         build_inline_bitrot_readers_from_refs(&data_files, bucket, object, read_length, shard_size, &checksum_algo, false).await
     else {
-        return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
+        return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
     };
 
     #[cfg(test)]
     crate::set_disk::disk_call_counters::record(object, crate::set_disk::disk_call_counters::KIND_BODY_DECODE, 0);
     match try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size).await {
-        Some(body) if body.len() == object_size => Ok(body),
-        _ => Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY),
+        Some(body) if body.len() == object_size => None,
+        _ => Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY),
     }
 }
 
@@ -1082,8 +1080,6 @@ impl SetDisks {
                     let response_start = observe.then(Instant::now);
                     let result = if let Some(disk) = disk {
                         Self::record_read_version_call(&object, disk_index);
-                        #[cfg(test)]
-                        Self::read_version_fanout_barrier(&object, disk_index).await;
                         if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) {
                             Self::record_metadata_slowtail_fault(&object, disk_index);
                             tokio::time::sleep(delay).await;
@@ -1152,7 +1148,6 @@ impl SetDisks {
             slots,
             diagnostics,
             condition_stopped: false,
-            verified_inline_body: None,
         })
     }
 
@@ -1373,7 +1368,6 @@ impl SetDisks {
                                 ),
                             )
                     });
-                let mut verified_inline_body = None;
                 let should_return_early = if condition_stopped {
                     true
                 } else if force_full_wait {
@@ -1383,7 +1377,7 @@ impl SetDisks {
                         Some(_candidate) if non_inline_candidate_eligible == Some(true) => {
                             accumulator.candidate_has_read_reserve()
                         }
-                        Some(candidate) => match verify_inline_body_from_observations(
+                        Some(candidate) => match inline_body_miss_reason_from_observations(
                             bucket.as_ref(),
                             object.as_ref(),
                             candidate,
@@ -1392,11 +1386,8 @@ impl SetDisks {
                         )
                         .await
                         {
-                            Ok(body) => {
-                                verified_inline_body = Some(body);
-                                true
-                            }
-                            Err(reason) => {
+                            None => true,
+                            Some(reason) => {
                                 final_miss_reason_override = Some(reason);
                                 if bounded_fanout
                                     && reason == GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD
@@ -1454,7 +1445,6 @@ impl SetDisks {
                         slots,
                         diagnostics,
                         condition_stopped,
-                        verified_inline_body,
                     });
                 }
             }
@@ -1537,7 +1527,6 @@ impl SetDisks {
             slots,
             diagnostics,
             condition_stopped: false,
-            verified_inline_body: None,
         })
     }
 

crates/ecstore/src/set_disk/mod.rs

@@ -906,8 +906,6 @@ pub(crate) use ops::object::body_cache_plaintext_len;
 #[cfg(all(test, feature = "test-util"))]
 pub(crate) use ops::object::cleanup_rejected_transition_upload_durably;
 #[cfg(test)]
-pub(crate) use ops::object::hermetic_set_disks_support::{HermeticSetFixture, hermetic_set_disks_for_pool_with_default_parity};
-#[cfg(test)]
 pub(crate) use ops::object::{DeleteCleanupBarrier, DeleteObjectCommitBarrier};
 #[cfg(any(test, feature = "test-util"))]
 pub use ops::object::{PutObjectCommitBarrier, PutObjectCommitPause};
@@ -941,7 +939,6 @@ struct OwnedGetObjectFileInfo {
     fi: FileInfo,
     parts_metadata: Vec<FileInfo>,
     online_disks: Vec<Option<DiskStore>>,
-    verified_inline_body: Option<Bytes>,
 }
 
 enum LateMetadataFanout {
@@ -958,7 +955,6 @@ impl GetObjectFileInfo {
                 fi,
                 parts_metadata,
                 online_disks,
-                verified_inline_body: None,
             }),
             shared: None,
             condition_stopped: false,
@@ -1021,27 +1017,6 @@ impl GetObjectFileInfo {
         self.late_metadata_fanout.is_some()
     }
 
-    fn set_verified_inline_body(&mut self, body: Option<Bytes>) {
-        if let Some(owned) = self.owned.as_mut() {
-            owned.verified_inline_body = body;
-        }
-    }
-
-    fn take_verified_inline_body(&mut self) -> Option<Bytes> {
-        self.owned.as_mut().and_then(|owned| owned.verified_inline_body.take())
-    }
-
-    fn into_object_info(self, bucket: &str, object: &str, versioned: bool) -> ObjectInfo {
-        match (self.owned, self.shared) {
-            (Some(snapshot), None) => ObjectInfo::from_owned_file_info(snapshot.fi, bucket, object, versioned),
-            (None, Some(entry)) => match Arc::try_unwrap(entry) {
-                Ok(entry) => ObjectInfo::from_owned_file_info(entry.fi, bucket, object, versioned),
-                Err(entry) => ObjectInfo::from_file_info(&entry.fi, bucket, object, versioned),
-            },
-            _ => unreachable!("GET metadata snapshot representation must be exclusive"),
-        }
-    }
-
     fn into_owned(self) -> (FileInfo, Vec<FileInfo>, Vec<Option<DiskStore>>) {
         let (fi, parts_metadata, online_disks, _) = self.into_owned_with_late_metadata_fanout();
         (fi, parts_metadata, online_disks)
@@ -1147,310 +1122,6 @@ mod prepared_get_object_metadata_tests {
     use http::HeaderMap;
     use tokio::io::AsyncReadExt;
 
-    const OBSERVATION_HOOK_BODY_LEN: usize = 1024 * 1024 + 1;
-
-    struct ObservationBodyHook {
-        object: &'static str,
-        calls: Arc<AtomicU64>,
-        body: bytes::Bytes,
-    }
-
-    #[async_trait::async_trait]
-    impl crate::object_api::GetObjectBodyCacheHook for ObservationBodyHook {
-        async fn lookup(&self, bucket: &str, object: &str, info: &ObjectInfo) -> Option<bytes::Bytes> {
-            if bucket == crate::disk::RUSTFS_META_BUCKET && object == self.object {
-                assert_eq!(info.size, i64::try_from(self.body.len()).expect("hook fixture size"));
-                self.calls.fetch_add(1, Ordering::SeqCst);
-                Some(self.body.clone())
-            } else {
-                None
-            }
-        }
-    }
-
-    struct ObservationHookGuard;
-
-    impl Drop for ObservationHookGuard {
-        fn drop(&mut self) {
-            crate::object_api::unregister_get_object_body_cache_hook();
-        }
-    }
-
-    async fn ordinary_observation_get(set: &SetDisks, object: &str) -> Vec<u8> {
-        let mut reader = set
-            .get_object_reader(crate::disk::RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
-            .await
-            .expect("ordinary GET positive control");
-        let mut bytes = Vec::new();
-        reader.stream.read_to_end(&mut bytes).await.expect("ordinary GET body");
-        bytes
-    }
-
-    fn observation_register_hook(object: &'static str, body: Vec<u8>) -> (ObservationHookGuard, Arc<AtomicU64>) {
-        let calls = Arc::new(AtomicU64::new(0));
-        crate::object_api::register_get_object_body_cache_hook(Arc::new(ObservationBodyHook {
-            object,
-            calls: calls.clone(),
-            body: bytes::Bytes::from(body),
-        }));
-        (ObservationHookGuard, calls)
-    }
-
-    #[tokio::test]
-    #[serial_test::serial(body_cache_hook)]
-    async fn locked_observation_uses_real_outer_guard() {
-        let object = "config/locked-observation-real.bin";
-        let payload = vec![b'p'; OBSERVATION_HOOK_BODY_LEN];
-        let (_dirs, set) = make_local_set_disks(4, 2).await;
-        let mut reader = PutObjReader::from_vec(payload.clone());
-        let written = set
-            .put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
-            .await
-            .expect("persist locked observation fixture");
-        let (_hook, calls) = observation_register_hook(object, vec![b'x'; payload.len()]);
-        assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; payload.len()]);
-        assert_eq!(calls.swap(0, Ordering::SeqCst), 1);
-        let namespace = set
-            .new_ns_lock(crate::disk::RUSTFS_META_BUCKET, object)
-            .await
-            .expect("namespace");
-        let guard = crate::object_api::WriteCommitGuard::acquire(&namespace, std::time::Duration::from_secs(5))
-            .await
-            .expect("matching real owner");
-        let mut contender = Box::pin(crate::object_api::WriteCommitGuard::acquire(
-            &namespace,
-            std::time::Duration::from_secs(30),
-        ));
-        assert!(futures::poll!(contender.as_mut()).is_pending(), "I06L_OWNER_RETAINED");
-        let result = tokio::time::timeout(
-            std::time::Duration::from_secs(10),
-            crate::config::com::read_config_observation_no_lock(set.clone(), object, payload.len(), &guard),
-        )
-        .await
-        .expect("locked observation hang watchdog");
-        assert!(result.is_ok(), "I06L_NO_NESTED_LOCK: {result:?}");
-        let (bytes, info) = result.expect("locked observation");
-        assert!(bytes == payload, "I06L_HOOK_BYPASS");
-        assert!(calls.load(Ordering::SeqCst) == 0, "I06L_HOOK_BYPASS");
-        assert_eq!(info.size, written.size);
-        assert_eq!(info.etag, written.etag);
-        assert_eq!(info.data_dir, written.data_dir);
-        assert_eq!(info.name, object);
-        assert!(futures::poll!(contender.as_mut()).is_pending(), "I06L_OWNER_RETAINED");
-        drop(guard);
-        let next = tokio::time::timeout(std::time::Duration::from_secs(10), contender)
-            .await
-            .expect("owner release watchdog")
-            .expect("competing owner acquired");
-        drop(next);
-        assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; payload.len()]);
-        assert_eq!(calls.load(Ordering::SeqCst), 1);
-    }
-
-    #[tokio::test]
-    #[serial_test::serial(body_cache_hook)]
-    async fn locked_observation_preserves_empty_missing_and_bound() {
-        let (_dirs, set) = make_local_set_disks(4, 2).await;
-        for (object, body) in [
-            ("config/locked-empty.bin", Vec::new()),
-            ("config/locked-bound.bin", b"raw".to_vec()),
-        ] {
-            let mut reader = PutObjReader::from_vec(body.clone());
-            let written = set
-                .put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
-                .await
-                .expect("persist boundary fixture");
-            let namespace = set
-                .new_ns_lock(crate::disk::RUSTFS_META_BUCKET, object)
-                .await
-                .expect("namespace");
-            let guard = crate::object_api::WriteCommitGuard::acquire(&namespace, std::time::Duration::from_secs(5))
-                .await
-                .expect("matching boundary owner");
-            let result = crate::config::com::read_config_observation_no_lock(set.clone(), object, body.len(), &guard).await;
-            assert!(result.is_ok(), "I06L_EMPTY_DISTINCT: {result:?}");
-            let (bytes, info) = result.expect("boundary observation");
-            assert_eq!(bytes, body);
-            assert_eq!(info.size, written.size);
-            if !body.is_empty() {
-                let result =
-                    crate::config::com::read_config_observation_no_lock(set.clone(), object, body.len() - 1, &guard).await;
-                assert!(matches!(result, Err(Error::CorruptedFormat)), "I06L_BOUND_ENFORCED");
-            }
-        }
-        let object = "config/locked-absent.bin";
-        let namespace = set
-            .new_ns_lock(crate::disk::RUSTFS_META_BUCKET, object)
-            .await
-            .expect("namespace");
-        let guard = crate::object_api::WriteCommitGuard::acquire(&namespace, std::time::Duration::from_secs(5))
-            .await
-            .expect("matching missing owner");
-        let result = crate::config::com::read_config_observation_no_lock(set, object, 0, &guard).await;
-        assert!(matches!(result, Err(Error::ConfigNotFound)), "I06L_EMPTY_DISTINCT");
-    }
-
-    #[tokio::test]
-    #[serial_test::serial(body_cache_hook)]
-    async fn observation_real_backend_boundaries() {
-        let object = "config/observation-real.bin";
-        let payload = b"persisted config";
-        let (_dirs, set) = make_local_set_disks(4, 2).await;
-        let mut reader = PutObjReader::from_vec(payload.to_vec());
-        let written = set
-            .put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
-            .await
-            .expect("persist real config fixture");
-        let result = crate::config::com::read_config_observation(set.clone(), object, payload.len()).await;
-        assert!(result.is_ok(), "I05_OBSERVATION_SUCCESS: {result:?}");
-        let (bytes, info) = result.expect("real observation success");
-        assert_eq!(bytes, payload);
-        assert_eq!(info.size, written.size);
-        assert_eq!(info.etag, written.etag);
-        assert_eq!(info.data_dir, written.data_dir);
-        assert_eq!(info.name, object);
-        let over = crate::config::com::read_config_observation(set.clone(), object, payload.len() - 1).await;
-        assert!(over.is_err(), "I05_BOUND_ENFORCED");
-        assert_eq!(over.expect_err("over limit must fail"), Error::CorruptedFormat, "I05_BOUND_ENFORCED");
-        let absent = crate::config::com::read_config_observation(set.clone(), "config/observation-absent.bin", 32).await;
-        assert_eq!(absent.expect_err("absent config"), Error::ConfigNotFound);
-        let empty_object = "config/observation-empty.bin";
-        let mut empty = PutObjReader::from_vec(Vec::new());
-        set.put_object(crate::disk::RUSTFS_META_BUCKET, empty_object, &mut empty, &ObjectOptions::default())
-            .await
-            .expect("persist empty config");
-        let empty = crate::config::com::read_config_observation(set, empty_object, 0).await;
-        assert!(empty.is_ok(), "existing empty config must succeed: {empty:?}");
-        let (bytes, info) = empty.expect("empty observation");
-        assert!(bytes.is_empty());
-        assert_eq!(info.size, 0);
-    }
-
-    #[tokio::test]
-    #[serial_test::serial(body_cache_hook)]
-    async fn observation_bypasses_registered_body_hook() {
-        let object = "config/observation-hook.bin";
-        let (_dirs, set) = make_local_set_disks(4, 2).await;
-        let mut reader = PutObjReader::from_vec(vec![b'p'; OBSERVATION_HOOK_BODY_LEN]);
-        set.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
-            .await
-            .expect("persist real config fixture");
-        let (_hook, calls) = observation_register_hook(object, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
-        assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
-        assert_eq!(calls.swap(0, Ordering::SeqCst), 1);
-        let result = crate::config::com::read_config_observation(set, object, OBSERVATION_HOOK_BODY_LEN).await;
-        assert!(result.is_ok(), "I05_HOOK_BYPASS: {result:?}");
-        assert_eq!(
-            result.expect("observed persisted body").0,
-            vec![b'p'; OBSERVATION_HOOK_BODY_LEN],
-            "I05_HOOK_BYPASS"
-        );
-        assert_eq!(calls.load(Ordering::SeqCst), 0, "I05_HOOK_BYPASS");
-    }
-
-    #[tokio::test]
-    #[serial_test::serial(body_cache_hook)]
-    async fn observation_hook_scope_restores_after_error() {
-        let object = "config/observation-hook-error.bin";
-        let (_dirs, set) = make_local_set_disks(4, 2).await;
-        let mut reader = PutObjReader::from_vec(vec![b'p'; OBSERVATION_HOOK_BODY_LEN]);
-        set.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
-            .await
-            .expect("persist real config fixture");
-        let (_hook, calls) = observation_register_hook(object, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
-        let result = crate::config::com::read_config_observation(set.clone(), object, OBSERVATION_HOOK_BODY_LEN - 1).await;
-        assert_eq!(result.expect_err("bounded observation error"), Error::CorruptedFormat);
-        assert_eq!(calls.load(Ordering::SeqCst), 0);
-        assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
-        assert_eq!(calls.load(Ordering::SeqCst), 1);
-    }
-
-    struct PendingObservationStore {
-        set: Arc<SetDisks>,
-        ready: std::sync::Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
-    }
-
-    impl std::fmt::Debug for PendingObservationStore {
-        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-            f.debug_struct("PendingObservationStore").finish()
-        }
-    }
-
-    #[async_trait::async_trait]
-    impl crate::storage_api_contracts::object::ObjectIO for PendingObservationStore {
-        type Error = Error;
-        type RangeSpec = HTTPRangeSpec;
-        type HeaderMap = HeaderMap;
-        type ObjectOptions = ObjectOptions;
-        type ObjectInfo = ObjectInfo;
-        type GetObjectReader = GetObjectReader;
-        type PutObjectReader = PutObjReader;
-
-        async fn get_object_reader(
-            &self,
-            bucket: &str,
-            object: &str,
-            range: Option<HTTPRangeSpec>,
-            headers: HeaderMap,
-            opts: &ObjectOptions,
-        ) -> Result<GetObjectReader> {
-            let mut reader = self.set.get_object_reader(bucket, object, range, headers, opts).await?;
-            let mut body = Vec::new();
-            reader.stream.read_to_end(&mut body).await.expect("pending producer real GET");
-            assert_eq!(
-                body,
-                vec![b'p'; OBSERVATION_HOOK_BODY_LEN],
-                "I05_HOOK_BYPASS: producer must enter its own scope"
-            );
-            self.ready
-                .lock()
-                .expect("pending signal")
-                .take()
-                .expect("one GET")
-                .send(())
-                .expect("producer entry signal");
-            std::future::pending::<Result<GetObjectReader>>().await
-        }
-
-        async fn put_object(
-            &self,
-            _bucket: &str,
-            _object: &str,
-            _data: &mut PutObjReader,
-            _opts: &ObjectOptions,
-        ) -> Result<ObjectInfo> {
-            panic!("observation must not PUT");
-        }
-    }
-
-    #[tokio::test]
-    #[serial_test::serial(body_cache_hook)]
-    async fn observation_hook_scope_restores_after_cancel() {
-        let object = "config/observation-hook-cancel.bin";
-        let (_dirs, set) = make_local_set_disks(4, 2).await;
-        let mut reader = PutObjReader::from_vec(vec![b'p'; OBSERVATION_HOOK_BODY_LEN]);
-        set.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
-            .await
-            .expect("persist real config fixture");
-        let (_hook, calls) = observation_register_hook(object, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
-        assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
-        assert_eq!(calls.swap(0, Ordering::SeqCst), 1);
-        let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
-        let adapter = Arc::new(PendingObservationStore {
-            set: set.clone(),
-            ready: std::sync::Mutex::new(Some(ready_tx)),
-        });
-        let mut scope = Box::pin(crate::config::com::read_config_observation(adapter, object, OBSERVATION_HOOK_BODY_LEN));
-        tokio::select! {
-            result = ready_rx => { result.expect("producer reached pending GET"); }
-            result = &mut scope => { panic!("producer must remain pending until cancelled: {result:?}"); }
-        }
-        assert_eq!(calls.load(Ordering::SeqCst), 0);
-        drop(scope);
-        assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
-        assert_eq!(calls.load(Ordering::SeqCst), 1);
-    }
-
     const READ_VERSION_BARRIER_GUARD: std::time::Duration = std::time::Duration::from_secs(10);
 
     fn object_with_initial_data_shards(bucket: &str, prefix: &str) -> String {
@@ -2535,7 +2206,7 @@ mod prepared_get_object_metadata_tests {
         let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled();
         rustfs_io_metrics::set_get_stage_metrics_enabled(true);
 
-        let (restored, object_size, calls_total, body_decodes) = metrics::with_local_recorder(&recorder, || {
+        let (restored, object_size, calls_total) = metrics::with_local_recorder(&recorder, || {
             runtime.block_on(async {
                 let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
                 let opts = ObjectOptions {
@@ -2589,12 +2260,7 @@ mod prepared_get_object_metadata_tests {
                             .await
                             .expect("inline GET body should stream");
 
-                        (
-                            restored,
-                            object_size,
-                            calls.total(disk_call_counters::KIND_READ_VERSION),
-                            calls.total(disk_call_counters::KIND_BODY_DECODE),
-                        )
+                        (restored, object_size, calls.total(disk_call_counters::KIND_READ_VERSION))
                     },
                 )
                 .await
@@ -2608,10 +2274,6 @@ mod prepared_get_object_metadata_tests {
             calls_total, 4,
             "default production inline GET should schedule the initial bounded quorum plus one hedge"
         );
-        assert_eq!(
-            body_decodes, 1,
-            "cold inline early-stop GET must reuse the quorum-verified inline body instead of decoding shards twice"
-        );
         assert_eq!(
             recorder.histogram_values(
                 "rustfs_io_get_object_metadata_fanout_scheduled",
@@ -2638,124 +2300,6 @@ mod prepared_get_object_metadata_tests {
         );
     }
 
-    #[test]
-    #[serial_test::serial(body_cache_hook)]
-    fn get_object_info_uses_early_stop_for_locked_or_cache_safe_reads_and_full_wait_for_unlocked() {
-        use crate::storage_api_contracts::object::ObjectOperations;
-
-        let runtime = tokio::runtime::Builder::new_current_thread()
-            .enable_all()
-            .build()
-            .expect("current-thread runtime should build");
-        let bucket = "get-object-info-early-stop-bucket";
-        let object = object_with_initial_data_shards(bucket, "get-object-info-early-stop-object");
-        let payload = b"head object early-stop payload".repeat(128);
-
-        runtime.block_on(async {
-            let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
-            let write_opts = ObjectOptions {
-                no_lock: true,
-                ..Default::default()
-            };
-
-            set_disks
-                .make_bucket(bucket, &MakeBucketOptions::default())
-                .await
-                .expect("bucket should be created");
-            let mut put_reader = PutObjReader::from_vec(payload.clone());
-            set_disks
-                .put_object(bucket, &object, &mut put_reader, &write_opts)
-                .await
-                .expect("inline object should be written");
-
-            temp_env::async_with_vars(
-                [
-                    ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", None::<&str>),
-                    ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", None::<&str>),
-                    ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>),
-                ],
-                async {
-                    let slow_parity_disk = bounded_initial_parity_disk_index(bucket, &object);
-                    let barrier =
-                        rename_fanout_barrier::arm(&object, slow_parity_disk, rename_fanout_barrier::PHASE_READ_VERSION);
-                    let calls = disk_call_counters::observe(&object);
-                    let set_disks_for_head = Arc::clone(&set_disks);
-                    let object_for_head = object.clone();
-                    let head_opts = ObjectOptions {
-                        no_lock: true,
-                        metadata_cache_safe: true,
-                        ..Default::default()
-                    };
-                    let mut head_task =
-                        tokio::spawn(
-                            async move { set_disks_for_head.get_object_info(bucket, &object_for_head, &head_opts).await },
-                        );
-
-                    tokio::time::timeout(READ_VERSION_BARRIER_GUARD, barrier.wait_until_paused())
-                        .await
-                        .expect("cache-safe HEAD should pause a slow parity metadata read");
-                    let info = tokio::time::timeout(READ_VERSION_BARRIER_GUARD, &mut head_task)
-                        .await
-                        .expect("cache-safe HEAD should return before the paused parity metadata response")
-                        .expect("HEAD task should not panic")
-                        .expect("HEAD should succeed");
-                    assert_eq!(info.size, payload.len() as i64);
-                    assert_eq!(
-                        calls.total(disk_call_counters::KIND_READ_VERSION),
-                        4,
-                        "cache-safe HEAD should schedule bounded quorum plus one hedge"
-                    );
-                    assert_eq!(
-                        calls.total(disk_call_counters::KIND_BODY_DECODE),
-                        1,
-                        "early-stop HEAD verifies inline quorum once"
-                    );
-
-                    // Second cache-safe HEAD should hit the QuorumSelected metadata cache entry (0 new disk reads).
-                    let cached_info = set_disks
-                        .get_object_info(bucket, &object, &ObjectOptions::default())
-                        .await
-                        .expect("cached HEAD with default locked options should succeed");
-                    assert_eq!(cached_info.size, payload.len() as i64);
-                    assert_eq!(
-                        calls.total(disk_call_counters::KIND_READ_VERSION),
-                        4,
-                        "repeat locked HEAD must hit the QuorumSelected metadata cache entry"
-                    );
-
-                    // Release the slow parity barrier so full-wait can complete all 4 disks.
-                    barrier.release();
-
-                    // Unlocked internal preflight (no_lock=true, metadata_cache_safe=false) must bypass cache and full-wait.
-                    let unlocked_info = set_disks
-                        .get_object_info(
-                            bucket,
-                            &object,
-                            &ObjectOptions {
-                                no_lock: true,
-                                metadata_cache_safe: false,
-                                ..Default::default()
-                            },
-                        )
-                        .await
-                        .expect("unlocked preflight get_object_info should succeed");
-                    assert_eq!(unlocked_info.size, payload.len() as i64);
-                    assert_eq!(
-                        calls.total(disk_call_counters::KIND_READ_VERSION),
-                        8,
-                        "unlocked preflight get_object_info must query all 4 disks"
-                    );
-                    assert_eq!(
-                        calls.total(disk_call_counters::KIND_BODY_DECODE),
-                        1,
-                        "unlocked preflight full-wait must not run early-stop inline body decode"
-                    );
-                },
-            )
-            .await;
-        });
-    }
-
     #[test]
     #[serial_test::serial(body_cache_hook)]
     fn prepared_metadata_uses_full_fanout_even_when_data_read_early_stop_is_enabled() {
@@ -13059,7 +12603,7 @@ mod tests {
         );
     }
 
-    #[tokio::test]
+    #[tokio::test(start_paused = true)]
     async fn test_get_disks_info_probes_drives_concurrently() {
         use crate::disk::disk_store::DISK_INFO_PROBE_DELAY_FOR_TEST;
 
@@ -13074,8 +12618,6 @@ mod tests {
             disks.push(Some(disk));
         }
 
-        // Complete real filesystem setup before measuring virtual probe delays.
-        tokio::time::pause();
         let probe_delay = std::time::Duration::from_secs(2);
         let started = tokio::time::Instant::now();
         let info = DISK_INFO_PROBE_DELAY_FOR_TEST
@@ -16922,14 +16464,7 @@ mod tests {
 
     #[tokio::test]
     async fn set_level_versioned_delete_marker_hides_object_without_corrupting_version_metadata() {
-        let (_fixture, _, set_disks) = hermetic_set_disks_for_pool_with_default_parity(4, 0, 2).await;
-        set_disks.set_test_storage_class_config(
-            storageclass::lookup_config_for_pools_without_env(&rustfs_config::server_config::KVS::new(), &[4])
-                .expect("test storage class should resolve for the local drive count"),
-        );
-        let store = crate::bucket::metadata_sys::object_store_in(&set_disks.ctx)
-            .await
-            .expect("fixture metadata authority should belong to this set");
+        let set_disks = make_local_bucket_test_set_disks_with_drive_count(4).await;
         let bucket = "bucket-versioned-delete";
         let object = "object.txt";
         let opts = ObjectOptions {
@@ -16938,26 +16473,13 @@ mod tests {
             ..Default::default()
         };
 
-        store
+        set_disks
             .make_bucket(bucket, &MakeBucketOptions::default())
             .await
             .expect("bucket should be created");
         let mut reader = PutObjReader::from_vec(b"versioned object body".to_vec());
         let written = set_disks
-            .put_object(
-                bucket,
-                object,
-                &mut reader,
-                &ObjectOptions {
-                    object_lock_config_snapshot: Some(
-                        store
-                            .object_lock_config_snapshot(bucket)
-                            .await
-                            .expect("PUT fixture should own the bucket lifecycle and Object Lock snapshot"),
-                    ),
-                    ..opts.clone()
-                },
-            )
+            .put_object(bucket, object, &mut reader, &opts)
             .await
             .expect("versioned object should be written");
         assert!(written.version_id.is_some());
@@ -17274,14 +16796,7 @@ mod tests {
     /// `BucketNotEmpty` for a bucket the client has already emptied.
     #[tokio::test]
     async fn set_level_explicit_null_version_delete_purges_the_null_delete_marker() {
-        let (_fixture, _, set_disks) = hermetic_set_disks_for_pool_with_default_parity(2, 0, 1).await;
-        set_disks.set_test_storage_class_config(
-            storageclass::lookup_config_for_pools_without_env(&rustfs_config::server_config::KVS::new(), &[2])
-                .expect("test storage class should resolve for the local drive count"),
-        );
-        let store = crate::bucket::metadata_sys::object_store_in(&set_disks.ctx)
-            .await
-            .expect("fixture metadata authority should belong to this set");
+        let set_disks = make_local_bucket_test_set_disks().await;
         let bucket = "bucket-null-marker-purge";
         let object = "object.txt";
         // The delete path reads versioned/suspended from the bucket-config
@@ -17303,26 +16818,13 @@ mod tests {
             ..Default::default()
         };
 
-        store
+        set_disks
             .make_bucket(bucket, &MakeBucketOptions::default())
             .await
             .expect("bucket should be created");
         let mut reader = PutObjReader::from_vec(b"suspended version body".to_vec());
         set_disks
-            .put_object(
-                bucket,
-                object,
-                &mut reader,
-                &ObjectOptions {
-                    object_lock_config_snapshot: Some(
-                        store
-                            .object_lock_config_snapshot(bucket)
-                            .await
-                            .expect("PUT fixture should own the bucket lifecycle and Object Lock snapshot"),
-                    ),
-                    ..opts.clone()
-                },
-            )
+            .put_object(bucket, object, &mut reader, &opts)
             .await
             .expect("suspended-version object should be written");
 
@@ -17347,20 +16849,7 @@ mod tests {
 
         let mut replacement_reader = PutObjReader::from_vec(b"replacement suspended version body".to_vec());
         set_disks
-            .put_object(
-                bucket,
-                object,
-                &mut replacement_reader,
-                &ObjectOptions {
-                    object_lock_config_snapshot: Some(
-                        store
-                            .object_lock_config_snapshot(bucket)
-                            .await
-                            .expect("PUT fixture should own the bucket lifecycle and Object Lock snapshot"),
-                    ),
-                    ..opts.clone()
-                },
-            )
+            .put_object(bucket, object, &mut replacement_reader, &opts)
             .await
             .expect("a suspended-version PUT should replace the first null marker");
         let replacement_marker = set_disks
@@ -17420,14 +16909,7 @@ mod tests {
 
     #[tokio::test]
     async fn set_level_version_suspended_delete_creates_null_delete_marker() {
-        let (_fixture, _, set_disks) = hermetic_set_disks_for_pool_with_default_parity(2, 0, 1).await;
-        set_disks.set_test_storage_class_config(
-            storageclass::lookup_config_for_pools_without_env(&rustfs_config::server_config::KVS::new(), &[2])
-                .expect("test storage class should resolve for the local drive count"),
-        );
-        let store = crate::bucket::metadata_sys::object_store_in(&set_disks.ctx)
-            .await
-            .expect("fixture metadata authority should belong to this set");
+        let set_disks = make_local_bucket_test_set_disks().await;
         let bucket = "bucket-version-suspended-delete";
         let object = "object.txt";
         let opts = ObjectOptions {
@@ -17437,26 +16919,13 @@ mod tests {
             ..Default::default()
         };
 
-        store
+        set_disks
             .make_bucket(bucket, &MakeBucketOptions::default())
             .await
             .expect("bucket should be created");
         let mut reader = PutObjReader::from_vec(b"suspended version body".to_vec());
         set_disks
-            .put_object(
-                bucket,
-                object,
-                &mut reader,
-                &ObjectOptions {
-                    object_lock_config_snapshot: Some(
-                        store
-                            .object_lock_config_snapshot(bucket)
-                            .await
-                            .expect("PUT fixture should own the bucket lifecycle and Object Lock snapshot"),
-                    ),
-                    ..opts.clone()
-                },
-            )
+            .put_object(bucket, object, &mut reader, &opts)
             .await
             .expect("suspended-version object should be written");
 

crates/ecstore/src/set_disk/ops/delete_recovery.rs

@@ -113,7 +113,6 @@ pub(crate) struct DeleteRecoveryContext<'a> {
     pub repair_marker_source: bool,
     pub namespace_owner: Option<Arc<dyn Send + Sync>>,
     pub recovery_ticket: Option<crate::store::delete_recovery_runtime::DeleteRecoveryTicket>,
-    pub published_hint: Option<(Uuid, &'a [Result<DeleteRecoveryInspection>])>,
     pub can_commit: &'a (dyn Fn() -> bool + Sync),
 }
 
@@ -126,7 +125,6 @@ impl<'a> DeleteRecoveryContext<'a> {
             repair_marker_source: false,
             namespace_owner: None,
             recovery_ticket: None,
-            published_hint: None,
             can_commit,
         }
     }
@@ -423,9 +421,7 @@ impl SetDisks {
         let cleanup = loop {
             #[cfg(test)]
             test_hooks::record_cleanup_attempt(bucket, object);
-            let mut attempt_context = context.clone();
-            attempt_context.published_hint = (attempt == 1).then_some((transaction, published.as_slice()));
-            match self.recover_delete_key(bucket, object, attempt_context).await {
+            match self.recover_delete_key(bucket, object, context.clone()).await {
                 // A stale settlement fingerprint only means the snapshot raced
                 // another writer; each attempt re-inspects every participant.
                 Err(DiskError::OutdatedXLMeta) if attempt < PUBLISHED_CLEANUP_ATTEMPTS && can_commit() => attempt += 1,
@@ -555,7 +551,6 @@ impl SetDisks {
             explicit_delete,
             repair_marker_source,
             namespace_owner,
-            published_hint,
             can_commit,
             recovery_ticket: _,
         } = context;
@@ -594,18 +589,7 @@ impl SetDisks {
         let mut all_states = BTreeMap::new();
         for transaction in transactions {
             let mut complete = Vec::with_capacity(disks.len());
-            let inspections = if let Some((published_tx, published_states)) = published_hint
-                && !repair_marker_source
-                && !dry_run
-                && published_tx == transaction
-                && published_states.len() == disks.len()
-                && published_states.iter().all(Result::is_ok)
-            {
-                published_states.to_vec()
-            } else {
-                inspect(&disks, bucket, object, transaction).await
-            };
-            for state in inspections {
+            for state in inspect(&disks, bucket, object, transaction).await {
                 match state {
                     Ok(state) => {
                         if let Some(journal) = &state.journal {
@@ -848,9 +832,6 @@ impl SetDisks {
                     bucket,
                     copies,
                     |copy| {
-                        if receipt(copy.state) && copy.commit_data_safe {
-                            return None;
-                        }
                         Some(
                             if copy.state == DeleteRecoveryState::Absent
                                 || (certified_same_transaction
@@ -876,10 +857,7 @@ impl SetDisks {
                         &disks,
                         bucket,
                         copies,
-                        |copy| {
-                            (copy.state != DeleteRecoveryState::CommitFinished || !copy.commit_data_safe)
-                                .then_some(DeleteRecoveryAction::FinishCommit)
-                        },
+                        |_| Some(DeleteRecoveryAction::FinishCommit),
                         can_commit,
                         namespace_owner.as_ref(),
                         &mut summary.removed_versions,
@@ -944,25 +922,13 @@ impl SetDisks {
             // GC is a key-wide barrier: every predecessor must have settled
             // before the last surviving commit witness can disappear.
             let mut settled = Vec::with_capacity(all_states.len());
-            if all_states.len() == 1 {
-                let copies = all_states.into_values().next().ok_or(DiskError::FileCorrupt)?;
-                if copies
-                    .iter()
-                    .any(|state| !matches!(state.state, DeleteRecoveryState::Absent | DeleteRecoveryState::CommitFinished))
-                {
+            for transaction in all_states.keys() {
+                let fresh = inspect(&disks, bucket, object, *transaction).await;
+                if fresh.iter().any(|state| !matches!(state, Ok(state) if matches!(state.state, DeleteRecoveryState::Absent | DeleteRecoveryState::CommitFinished))) {
                     summary.pending += 1;
                     return Ok(summary);
                 }
-                settled.push(copies);
-            } else {
-                for transaction in all_states.keys() {
-                    let fresh = inspect(&disks, bucket, object, *transaction).await;
-                    if fresh.iter().any(|state| !matches!(state, Ok(state) if matches!(state.state, DeleteRecoveryState::Absent | DeleteRecoveryState::CommitFinished))) {
-                        summary.pending += 1;
-                        return Ok(summary);
-                    }
-                    settled.push(fresh.into_iter().collect::<Result<Vec<_>>>()?);
-                }
+                settled.push(fresh.into_iter().collect::<Result<Vec<_>>>()?);
             }
             for (index, mut copies) in settled.into_iter().enumerate() {
                 if index > 0 {
@@ -1817,56 +1783,6 @@ mod tests {
             .expect("returning old replica must not leave bucket residue");
     }
 
-    #[tokio::test]
-    async fn delete_recovery_set_replays_partial_gc_with_empty_transaction_directory() {
-        let bucket = "recover-empty-gc";
-        let (dirs, store, disks) = fixture(bucket).await;
-        put(&store, bucket, &[24; 4096], false).await;
-        let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
-        for (index, disk) in disks.iter().enumerate() {
-            publish(disk, bucket, transaction).await;
-            let state = disk.inspect_delete_transaction(bucket, "object", transaction).await.unwrap();
-            let finished = disk
-                .settle_delete_transaction(bucket, "object", transaction, state.fingerprint, DeleteRecoveryAction::FinishCommit)
-                .await
-                .unwrap();
-            if index > 0 {
-                disk.settle_delete_transaction(
-                    bucket,
-                    "object",
-                    transaction,
-                    finished.fingerprint,
-                    DeleteRecoveryAction::FinalizeCommit,
-                )
-                .await
-                .unwrap();
-            }
-        }
-        // A crash can persist proof removal before the final empty rmdir.
-        let empty = dirs[3].path().join(bucket).join("object").join(transaction.to_string());
-        tokio::fs::create_dir_all(&empty).await.unwrap();
-        for _ in 0..2 {
-            store
-                .recover_delete_transactions(bucket, &HealOpts::default())
-                .await
-                .expect("the remaining receipt must settle every participant without a fresh delete");
-            for disk in &disks {
-                assert!(discover(disk, bucket, Some("object")).await.unwrap().is_empty());
-                assert!(
-                    disk.inspect_delete_transaction(bucket, "object", transaction)
-                        .await
-                        .unwrap()
-                        .current_digest
-                        .is_none()
-                );
-            }
-        }
-        store
-            .delete_bucket(bucket, &DeleteBucketOptions::default())
-            .await
-            .expect("ordinary bucket removal must succeed after replay");
-    }
-
     #[tokio::test]
     async fn delete_recovery_tracks_only_observed_removed_version() {
         let bucket = "recover-actual-removal";

crates/ecstore/src/set_disk/ops/heal/shard_integrity_rollout_tests.rs

@@ -15,9 +15,7 @@
 use crate::disk::{DiskAPI as _, ReadOptions};
 use crate::object_api::{ObjectOptions, PutObjReader, ShardIntegrityWriteMode};
 use crate::set_disk::CompletePart;
-use crate::set_disk::ops::object::hermetic_set_disks_support::{
-    hermetic_set_disks_for_pool_with_default_parity_isolated, lifecycle_owned_put_options,
-};
+use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity_isolated;
 use crate::storage_api_contracts::multipart::MultipartOperations as _;
 use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
 use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
@@ -41,19 +39,18 @@ async fn remote_metadata_heal_does_not_certify_unread_tier_payload() {
     for disk in &disks {
         disk.make_volume(bucket).await.expect("fixture bucket");
     }
-    let opts = lifecycle_owned_put_options(
-        &set,
+    set.put_object(
         bucket,
-        ObjectOptions {
+        object,
+        &mut PutObjReader::from_vec(vec![0x31; 1024 * 1024 + 17]),
+        &ObjectOptions {
             no_lock: true,
             shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected),
             ..Default::default()
         },
     )
-    .await;
-    set.put_object(bucket, object, &mut PutObjReader::from_vec(vec![0x31; 1024 * 1024 + 17]), &opts)
-        .await
-        .expect("protected source");
+    .await
+    .expect("protected source");
     for (disk, dir) in disks.iter().zip(&dirs) {
         let mut info = disk
             .read_version("", bucket, object, "", &ReadOptions::default())
@@ -97,15 +94,10 @@ async fn rollout_requires_both_flags_for_new_puts() {
     for disk in &disks {
         disk.make_volume(bucket).await.expect("fixture bucket");
     }
-    let opts = lifecycle_owned_put_options(
-        &set,
-        bucket,
-        ObjectOptions {
-            no_lock: true,
-            ..Default::default()
-        },
-    )
-    .await;
+    let opts = ObjectOptions {
+        no_lock: true,
+        ..Default::default()
+    };
 
     for (requested, confirmed, protected) in [
         (false, false, false),
@@ -167,17 +159,12 @@ async fn legacy_shard_repair_and_explicit_version_metadata_recovery_remain_avail
     for disk in &disks {
         disk.make_volume(bucket).await.expect("fixture bucket");
     }
-    let opts = lifecycle_owned_put_options(
-        &set,
-        bucket,
-        ObjectOptions {
-            no_lock: true,
-            versioned: true,
-            shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
-            ..Default::default()
-        },
-    )
-    .await;
+    let opts = ObjectOptions {
+        no_lock: true,
+        versioned: true,
+        shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
+        ..Default::default()
+    };
     let expected = vec![0x67; 1024 * 1024 + 37];
     for case in ["missing-part", "bad-bitrot", "missing-version-metadata"] {
         let object = case;
@@ -285,15 +272,10 @@ async fn multipart_mode_survives_rollout_switch_changes() {
     for disk in &disks {
         disk.make_volume(bucket).await.expect("fixture bucket");
     }
-    let opts = lifecycle_owned_put_options(
-        &set,
-        bucket,
-        ObjectOptions {
-            no_lock: true,
-            ..Default::default()
-        },
-    )
-    .await;
+    let opts = ObjectOptions {
+        no_lock: true,
+        ..Default::default()
+    };
     for protected in [false, true] {
         let object = format!("upload-{protected}");
         let upload =
@@ -372,16 +354,11 @@ async fn physical_copy_inherits_source_mode_instead_of_current_switch() {
         };
         let object = format!("source-{protected}");
         let expected = vec![0x39; 1024 * 1024 + 11];
-        let seed_opts = lifecycle_owned_put_options(
-            &set,
-            bucket,
-            ObjectOptions {
-                no_lock: true,
-                shard_integrity_write_mode: Some(mode),
-                ..Default::default()
-            },
-        )
-        .await;
+        let seed_opts = ObjectOptions {
+            no_lock: true,
+            shard_integrity_write_mode: Some(mode),
+            ..Default::default()
+        };
         let mut source = set
             .put_object(bucket, &object, &mut PutObjReader::from_vec(expected.clone()), &seed_opts)
             .await
@@ -389,8 +366,8 @@ async fn physical_copy_inherits_source_mode_instead_of_current_switch() {
         source.metadata_only = false;
         source.put_object_reader = Some(PutObjReader::from_vec(expected.clone()));
         let opts = ObjectOptions {
-            shard_integrity_write_mode: None,
-            ..seed_opts.clone()
+            no_lock: true,
+            ..Default::default()
         };
         let copied = temp_env::async_with_vars(
             rollout_vars(!protected, !protected),

crates/ecstore/src/set_disk/ops/heal_lock_wait_tests.rs

@@ -1,194 +0,0 @@
-// Copyright 2026 RustFS Team
-// SPDX-License-Identifier: Apache-2.0
-
-use super::*;
-use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_isolated;
-use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
-use crate::storage_api_contracts::heal::HealOperations as _;
-use metrics_util::debugging::{DebugValue, DebuggingRecorder};
-use std::collections::BTreeMap;
-use std::future::Future;
-use std::pin::Pin;
-use std::task::{Context, Poll};
-use std::time::{Duration, Instant};
-
-const METRIC: &str = "rustfs_heal_object_lock_wait_seconds";
-
-async fn captured<F: Future>(recorder: &DebuggingRecorder, future: F) -> F::Output {
-    let mut future = std::pin::pin!(future);
-    std::future::poll_fn(|cx| metrics::with_local_recorder(recorder, || future.as_mut().poll(cx))).await
-}
-
-fn poll_once<F: Future>(recorder: &DebuggingRecorder, future: Pin<&mut F>) -> Poll<F::Output> {
-    let mut cx = Context::from_waker(futures::task::noop_waker_ref());
-    metrics::with_local_recorder(recorder, || future.poll(&mut cx))
-}
-
-fn samples(recorder: &DebuggingRecorder) -> Vec<(BTreeMap<String, String>, f64)> {
-    recorder
-        .snapshotter()
-        .snapshot()
-        .into_vec()
-        .into_iter()
-        .filter(|(key, _, _, _)| key.key().name() == METRIC)
-        .flat_map(|(key, _, _, value)| {
-            let labels: BTreeMap<_, _> = key
-                .key()
-                .labels()
-                .map(|label| (label.key().to_owned(), label.value().to_owned()))
-                .collect();
-            let DebugValue::Histogram(values) = value else { panic!("wait must be a histogram") };
-            values.into_iter().map(move |value| (labels.clone(), value.0))
-        })
-        .collect()
-}
-
-fn one_sample(recorder: &DebuggingRecorder, op: &str, mode: &str, result: &str) -> f64 {
-    let rows = samples(recorder);
-    assert_eq!(rows.len(), 1, "one observation per actual acquisition attempt");
-    assert_eq!(
-        rows[0].0,
-        BTreeMap::from([
-            ("op".to_owned(), op.to_owned()),
-            ("mode".to_owned(), mode.to_owned()),
-            ("result".to_owned(), result.to_owned()),
-        ]),
-        "no object, bucket, version, or owner labels"
-    );
-    assert!(rows[0].1.is_finite() && rows[0].1 >= 0.0);
-    rows[0].1
-}
-
-#[tokio::test]
-async fn heal_object_lock_metrics_cover_paths_and_skip_borrowed_locks() {
-    let (_dirs, _disks, set) = hermetic_set_disks_isolated(4).await;
-    let bucket = "heal-lock-metric-paths";
-    set.make_bucket(bucket, &MakeBucketOptions::default()).await.expect("bucket");
-    let recorder = DebuggingRecorder::new();
-    for (op, kind) in [
-        ("read_repair_data", HealObjectLockKind::Read),
-        ("read_repair_commit", HealObjectLockKind::Write),
-        ("heal_object", HealObjectLockKind::Write),
-    ] {
-        let guard = captured(&recorder, set.acquire_heal_object_lock(bucket, "missing", "", kind, op))
-            .await
-            .expect("namespace lock");
-        one_sample(&recorder, op, kind.as_str(), "acquired");
-        metrics::with_local_recorder(&recorder, || drop(guard));
-        assert!(samples(&recorder).is_empty(), "hold time must not become another wait sample");
-    }
-    let opts = HealOpts {
-        dry_run: true,
-        ..Default::default()
-    };
-    captured(&recorder, set.heal_object(bucket, "missing", "", &opts))
-        .await
-        .expect("missing object report");
-    one_sample(&recorder, "heal_object", "write", "acquired");
-    captured(&recorder, set.check_abandoned_parts(bucket, "missing", &opts))
-        .await
-        .expect("abandoned parts check");
-    one_sample(&recorder, "check_abandoned_parts", "write", "acquired");
-    captured(&recorder, set.heal_object_dir(bucket, "missing-dir/", true, false))
-        .await
-        .expect("directory report");
-    one_sample(&recorder, "heal_object_dir", "write", "acquired");
-
-    let no_lock = HealOpts { no_lock: true, ..opts };
-    captured(&recorder, set.heal_object(bucket, "missing", "", &no_lock))
-        .await
-        .expect("borrowed-lock report");
-    captured(&recorder, set.check_abandoned_parts(bucket, "missing", &no_lock))
-        .await
-        .expect("borrowed-lock check");
-    assert!(samples(&recorder).is_empty(), "no_lock must not invent an acquisition");
-}
-
-#[tokio::test]
-async fn heal_object_lock_metrics_preserve_contention_and_cancellation() {
-    let (_dirs, _disks, set) = hermetic_set_disks_isolated(4).await;
-    let namespace = set.new_ns_lock("metric-contention", "object").await.expect("namespace");
-    let recorder = DebuggingRecorder::new();
-    for cancel in [false, true] {
-        let blocker = namespace.get_write_lock(Duration::from_secs(30)).await.expect("blocker");
-        let mut waiting = Box::pin(set.acquire_heal_object_lock(
-            "metric-contention",
-            "object",
-            "",
-            HealObjectLockKind::Read,
-            "read_repair_data",
-        ));
-        assert!(poll_once(&recorder, waiting.as_mut()).is_pending(), "real holder must block heal");
-        let pending_at = Instant::now();
-        assert!(samples(&recorder).is_empty(), "unfinished attempt has no completed sample");
-        if cancel {
-            let blocked_for = pending_at.elapsed();
-            metrics::with_local_recorder(&recorder, || drop(waiting));
-            assert!(one_sample(&recorder, "read_repair_data", "read", "cancelled") >= blocked_for.as_secs_f64());
-            let mut successor = Box::pin(namespace.get_read_lock(Duration::from_secs(30)));
-            assert!(
-                poll_once(&recorder, successor.as_mut()).is_pending(),
-                "cancelling heal must not release another holder"
-            );
-            drop(blocker);
-            drop(successor.await.expect("cancelled waiter must not leak ownership"));
-        } else {
-            let blocked_for = pending_at.elapsed();
-            drop(blocker);
-            let guard = captured(&recorder, waiting).await.expect("acquire after blocker releases");
-            assert!(one_sample(&recorder, "read_repair_data", "read", "acquired") >= blocked_for.as_secs_f64());
-            let mut successor = Box::pin(namespace.get_write_lock(Duration::from_secs(30)));
-            assert!(
-                poll_once(&recorder, successor.as_mut()).is_pending(),
-                "observation must preserve the returned guard"
-            );
-            metrics::with_local_recorder(&recorder, || drop(guard));
-            drop(successor.await.expect("release at the original guard drop"));
-        }
-        assert!(samples(&recorder).is_empty(), "one outcome per attempt");
-    }
-}
-
-#[tokio::test]
-#[serial_test::serial]
-async fn heal_object_lock_metrics_retain_failed_acquisition() {
-    temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
-        let (_dirs, _disks, set) = hermetic_set_disks_isolated(4).await;
-        let namespace = set.new_ns_lock("metric-timeout", "object").await.expect("namespace");
-        let blocker = namespace.get_write_lock(Duration::from_secs(30)).await.expect("blocker");
-        let recorder = DebuggingRecorder::new();
-        let result = captured(
-            &recorder,
-            set.acquire_heal_object_lock("metric-timeout", "object", "", HealObjectLockKind::Write, "heal_object"),
-        )
-        .await;
-        assert!(result.is_err(), "measurement must not turn a timeout into success");
-        one_sample(&recorder, "heal_object", "write", "failed");
-        drop(blocker);
-    })
-    .await;
-}
-
-#[tokio::test]
-async fn heal_object_lock_metrics_separate_disabled_and_unpolled_acquisitions() {
-    use rustfs_lock::{GlobalLockManager, NamespaceLock, NamespaceLockWrapper, ObjectKey};
-    let manager = Arc::new(GlobalLockManager::Disabled(rustfs_lock::fast_lock::DisabledLockManager::new()));
-    let namespace = NamespaceLockWrapper::new(
-        NamespaceLock::with_local_manager("disabled-test".to_owned(), manager),
-        ObjectKey::new("bucket", "object"),
-        "owner".to_owned(),
-    );
-    let recorder = DebuggingRecorder::new();
-    let unpolled =
-        measure_heal_object_lock("heal_object", HealObjectLockKind::Write, namespace.get_write_lock(Duration::from_secs(1)));
-    metrics::with_local_recorder(&recorder, || drop(unpolled));
-    assert!(samples(&recorder).is_empty(), "a never-polled future is not an attempted acquisition");
-    let guard = captured(
-        &recorder,
-        measure_heal_object_lock("heal_object", HealObjectLockKind::Write, namespace.get_write_lock(Duration::from_secs(1))),
-    )
-    .await
-    .expect("disabled guard");
-    assert!(matches!(&guard, rustfs_lock::NamespaceLockGuard::Fast(guard) if guard.is_disabled()));
-    one_sample(&recorder, "heal_object", "write", "disabled");
-}