rustfs/crates/ecstore/src/store/object.rs

// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use super::*;
use crate::bucket::lifecycle::{
    tier_delete_journal::{
        abort_prepared_tier_delete_journal_entry as abort_prepared_journal_entry_if_current, commit_tier_delete_journal_entry,
        enqueue_committed_tier_delete_journal_entry, persist_tier_delete_journal_entry,
        record_tier_delete_journal_backend_identity,
    },
    tier_sweeper::{
        Jentry, attach_tier_delete_source, transitioned_delete_journal_entry_for_source, transitioned_force_delete_journal_entry,
    },
};
use crate::bucket::metadata_sys::{
    acquire_bucket_metadata_transaction_read_lock_in, get_bucket_incarnation_id_in, get_cached_bucket_incarnation_id_in,
    get_object_lock_config_and_incarnation_from_disk_in,
};
use crate::bucket::object_lock::objectlock_sys::{
    check_object_lock_for_deletion_with_state, ensure_recursive_force_delete_allowed_for_state,
};
use crate::bucket::replication::ReplicationObjectBridge;
use crate::disk::OldCurrentSize;
use crate::object_api::{NamespaceLockFence, ObjectLockConfigSnapshot};
use crate::set_disk::{
    get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold,
    is_lock_optimization_enabled, is_object_lock_diag_enabled,
};
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
use rustfs_io_metrics::{
    record_object_lock_diag_acquire_duration, record_object_lock_diag_hold_duration, record_object_lock_diag_slow_acquire,
    record_object_lock_diag_slow_hold,
};
use std::{
    fmt,
    pin::Pin,
    task::{Context, Poll},
    time::{Duration, Instant},
};
use tokio::io::{AsyncRead, ReadBuf};

#[cfg(not(test))]
const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 1000;
#[cfg(test)]
const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 2;
const FORCE_DELETE_LIST_PAGE_SIZE: i32 = 1_000;

fn build_tier_delete_journal_entry(
    bucket: &str,
    object: &str,
    opts: &ObjectOptions,
    source: &ObjectInfo,
) -> Result<Option<Jentry>> {
    let version_id = opts.version_id.as_deref().map(Uuid::parse_str).transpose()?;
    let source_object = decode_dir_object(object);
    let Some(mut je) = (if opts.delete_prefix {
        transitioned_force_delete_journal_entry(&source.transitioned_object, source.transition_version_state).map(|mut je| {
            attach_tier_delete_source(&mut je, bucket, source_object.as_str(), source, opts.versioned, opts.version_suspended);
            je
        })
    } else {
        transitioned_delete_journal_entry_for_source(
            version_id,
            opts.versioned,
            opts.version_suspended,
            bucket,
            source_object.as_str(),
            source,
        )
    }) else {
        return Ok(None);
    };
    record_tier_delete_journal_backend_identity(&mut je, &source.user_defined).map_err(Error::other)?;
    Ok(Some(je))
}

async fn prepare_tier_delete_journal_entry(
    api: &Arc<ECStore>,
    bucket: &str,
    object: &str,
    opts: &ObjectOptions,
    source: &ObjectInfo,
) -> Result<Option<Jentry>> {
    let Some(je) = build_tier_delete_journal_entry(bucket, object, opts, source)? else {
        return Ok(None);
    };
    persist_tier_delete_journal_entry(Arc::clone(api), &je)
        .await
        .map_err(Error::other)?;
    Ok(Some(je))
}

async fn abort_prepared_tier_delete_journal_entry(api: &Arc<ECStore>, je: &Jentry) {
    if let Err(err) = abort_prepared_journal_entry_if_current(Arc::clone(api), je).await {
        warn!(
            object = %je.obj_name,
            tier = %je.tier_name,
            error = ?err,
            "failed to remove aborted tier delete journal"
        );
    }
}

async fn abort_prepared_tier_delete_journal_entries(api: &Arc<ECStore>, entries: &[Jentry]) {
    for entry in entries {
        abort_prepared_tier_delete_journal_entry(api, entry).await;
    }
}

async fn commit_prepared_tier_delete_journal_entry(api: &Arc<ECStore>, je: &Jentry) {
    if let Err(err) = commit_tier_delete_journal_entry(Arc::clone(api), je).await {
        warn!(
            object = %je.obj_name,
            tier = %je.tier_name,
            error = ?err,
            "tier delete committed locally but journal commit failed; recovery will retry"
        );
        return;
    }
    let mut committed = je.clone();
    committed.state = crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed;
    if let Err(err) = enqueue_committed_tier_delete_journal_entry(&committed).await {
        warn!(
            object = %je.obj_name,
            tier = %je.tier_name,
            error = ?err,
            "tier delete journal committed but could not be queued; recovery will retry"
        );
    }
}

async fn commit_prepared_tier_delete_journal_entries(api: &Arc<ECStore>, entries: &[Jentry]) {
    for entry in entries {
        commit_prepared_tier_delete_journal_entry(api, entry).await;
    }
}

async fn prepare_prefix_tier_delete_journal_entries(
    api: &Arc<ECStore>,
    bucket: &str,
    prefix: &str,
    opts: &ObjectOptions,
) -> Result<Vec<Jentry>> {
    let mut marker = None;
    let mut version_marker = None;
    let mut entries = Vec::new();

    loop {
        let page = Arc::clone(api)
            .list_object_versions_for_lifecycle(
                bucket,
                prefix,
                marker.clone(),
                version_marker.clone(),
                None,
                FORCE_DELETE_LIST_PAGE_SIZE,
            )
            .await?;

        for source in page.objects {
            if let Some(entry) = build_tier_delete_journal_entry(bucket, &source.name, opts, &source)? {
                entries.push(entry);
            }
        }

        if !page.is_truncated {
            break;
        }

        let next_marker = page
            .next_marker
            .ok_or_else(|| Error::other("truncated force delete listing has no next marker"))?;
        let next_version_marker = page.next_version_idmarker;
        if marker.as_deref() == Some(next_marker.as_str()) && version_marker == next_version_marker {
            return Err(Error::other("force delete listing marker did not advance"));
        }
        marker = Some(next_marker);
        version_marker = next_version_marker;
    }

    let mut persisted = Vec::with_capacity(entries.len());
    for entry in entries {
        if let Err(err) = persist_tier_delete_journal_entry(Arc::clone(api), &entry).await {
            abort_prepared_tier_delete_journal_entries(api, &persisted).await;
            return Err(Error::other(err));
        }
        persisted.push(entry);
    }
    Ok(persisted)
}
async fn delete_prefix_with_tier_delete_journal(
    store: &ECStore,
    bucket: &str,
    object: &str,
    opts: &ObjectOptions,
    tier_journal_api: Option<&Arc<ECStore>>,
) -> Result<()> {
    let journal_entry = if let Some(api) = tier_journal_api {
        Some(prepare_prefix_tier_delete_journal_entries(api, bucket, object, opts).await?)
    } else {
        None
    };

    let result = store.delete_prefix(bucket, object, opts).await;
    match result {
        Ok(()) => {
            if let (Some(api), Some(entries)) = (tier_journal_api, journal_entry.as_ref()) {
                commit_prepared_tier_delete_journal_entries(api, entries).await;
            }
            Ok(())
        }
        Err(err) => {
            if let (Some(api), Some(entries)) = (tier_journal_api, journal_entry.as_ref()) {
                abort_prepared_tier_delete_journal_entries(api, entries).await;
            }
            Err(err)
        }
    }
}

/// A GET whose object identity has been resolved while its namespace read lock
/// remains held, but whose body reader has not been constructed yet.
///
/// The application can evaluate request preconditions and cache coordination
/// against [`Self::object_info`] before consuming this value. Dropping it
/// releases the read lock without constructing a body reader.
pub struct PreparedGetObjectReader {
    pool: Arc<Sets>,
    bucket: String,
    object: String,
    range: Option<HTTPRangeSpec>,
    headers: HeaderMap,
    opts: ObjectOptions,
    metadata: crate::set_disk::PreparedGetObjectMetadata,
    read_lock_guard: Option<ObjectLockDiagGuard>,
}

impl PreparedGetObjectReader {
    /// Returns the fresh metadata snapshot protected by this prepared read.
    pub fn object_info(&self) -> &ObjectInfo {
        self.metadata.object_info()
    }

    /// Finishes a metadata-only decision without constructing a body reader.
    pub fn into_object_info(mut self) -> ObjectInfo {
        self.metadata.take_object_info()
    }

    /// Replaces the headers used when this prepared value constructs its body reader.
    #[must_use]
    pub fn with_headers(mut self, headers: HeaderMap) -> Self {
        self.headers = headers;
        self
    }

    /// Constructs the body reader while retaining the metadata snapshot's read
    /// lock, then transfers that lock to the returned stream as usual. The
    /// staged caller already performed the authoritative app-layer cache probe,
    /// so the nested set-disk reader must not probe the same hook again.
    pub async fn into_reader(self) -> Result<GetObjectReader> {
        let mut reader =
            crate::object_api::without_get_object_body_cache_hook(self.pool.get_object_reader_with_prepared_metadata(
                &self.bucket,
                &self.object,
                self.range,
                self.headers,
                &self.opts,
                self.metadata,
            ))
            .await?;
        reader.body_source = crate::object_api::GetObjectBodySource::HookMissed;
        Ok(ECStore::attach_read_lock_guard(reader, self.read_lock_guard))
    }
}

struct LockGuardedReader {
    inner: Box<dyn AsyncRead + Unpin + Send + Sync>,
    guard: Option<ObjectLockDiagGuard>,
}

impl AsyncRead for LockGuardedReader {
    fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
        let had_capacity = buf.remaining() > 0;
        let filled_before = buf.filled().len();
        let poll = Pin::new(&mut self.inner).poll_read(cx, buf);
        if had_capacity && matches!(poll, Poll::Ready(Ok(()))) && buf.filled().len() == filled_before {
            self.guard.take();
        }
        poll
    }
}

#[derive(Clone, Copy, Debug)]
enum ObjectLockDiagMode {
    Read,
    Write,
}

impl ObjectLockDiagMode {
    fn as_str(self) -> &'static str {
        match self {
            Self::Read => "read",
            Self::Write => "write",
        }
    }
}

impl fmt::Display for ObjectLockDiagMode {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.write_str(self.as_str())
    }
}

struct ObjectLockDiagGuard {
    guard: rustfs_lock::NamespaceLockGuard,
    enabled: bool,
    op: &'static str,
    bucket: Option<String>,
    object: Option<String>,
    owner: Option<String>,
    mode: ObjectLockDiagMode,
    acquired_at: Instant,
}

impl ObjectLockDiagGuard {
    fn new(
        guard: rustfs_lock::NamespaceLockGuard,
        enabled: bool,
        op: &'static str,
        bucket: Option<String>,
        object: Option<String>,
        owner: Option<String>,
        mode: ObjectLockDiagMode,
    ) -> Self {
        Self {
            guard,
            enabled,
            op,
            bucket,
            object,
            owner,
            mode,
            acquired_at: Instant::now(),
        }
    }

    fn lock_lost_signal(&self) -> Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>> {
        match &self.guard {
            rustfs_lock::NamespaceLockGuard::Standard(guard) => Some(guard.lock_lost()),
            rustfs_lock::NamespaceLockGuard::Fast(_) => None,
        }
    }
}

/// Opaque write-lock guard for the RestoreObject accept path; see
/// [`ECStore::acquire_restore_accept_guard`]. Deliberately not a general lock
/// API — it only exists so the accept path's restore-status compare-and-set
/// can span the ecstore/API layer boundary.
pub struct RestoreAcceptGuard(ObjectLockDiagGuard);

impl RestoreAcceptGuard {
    /// True when the underlying namespace lock was lost (e.g. heartbeat
    /// refresh lost quorum). Callers must check this before committing the
    /// restore-status write the guard exists to serialize.
    pub fn is_lock_lost(&self) -> bool {
        self.0.guard.is_lock_lost()
    }

    pub fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
        opts.ensure_namespace_lock_fence();
        if let Some(signal) = self.0.lock_lost_signal() {
            opts.add_namespace_lock_lost_signal(signal);
        }
    }
}

impl Drop for ObjectLockDiagGuard {
    fn drop(&mut self) {
        if !self.enabled || self.guard.is_released() {
            return;
        }

        let hold = self.acquired_at.elapsed();
        record_object_lock_diag_hold_duration(self.op, self.mode.as_str(), hold);
        let threshold = get_object_lock_diag_slow_hold_threshold();
        if hold >= threshold {
            record_object_lock_diag_slow_hold(self.op, self.mode.as_str());
            warn!(
                target: "rustfs_ecstore::object_lock_diag",
                op = self.op,
                bucket = %self.bucket.as_deref().unwrap_or_default(),
                object = %self.object.as_deref().unwrap_or_default(),
                mode = %self.mode,
                owner = %self.owner.as_deref().unwrap_or_default(),
                hold_ms = hold.as_millis(),
                threshold_ms = threshold.as_millis(),
                "object namespace lock held longer than threshold"
            );
        }
    }
}

fn log_object_lock_acquire_if_slow(
    op: &'static str,
    bucket: &str,
    object: &str,
    owner: Option<&str>,
    mode: ObjectLockDiagMode,
    elapsed: Duration,
    diag_enabled: bool,
) {
    if !diag_enabled {
        return;
    }

    let threshold = get_object_lock_diag_slow_acquire_threshold();
    record_object_lock_diag_acquire_duration(op, mode.as_str(), elapsed);
    if elapsed >= threshold {
        record_object_lock_diag_slow_acquire(op, mode.as_str());
        warn!(
            target: "rustfs_ecstore::object_lock_diag",
            op,
            bucket,
            object,
            mode = %mode,
            owner = owner.unwrap_or_default(),
            acquire_ms = elapsed.as_millis(),
            threshold_ms = threshold.as_millis(),
            "object namespace lock acquisition exceeded threshold"
        );
    }
}

fn select_data_movement_target_pool(
    existing_pool_idx: Result<usize>,
    src_pool_idx: usize,
    delete_marker: bool,
) -> Result<Option<usize>> {
    match existing_pool_idx {
        Ok(pool_idx) => {
            if delete_marker && pool_idx == src_pool_idx {
                Ok(None)
            } else {
                Ok(Some(pool_idx))
            }
        }
        Err(err) => {
            if is_err_read_quorum(&err) {
                return Err(StorageError::ErasureWriteQuorum);
            }
            if delete_marker && (is_err_object_not_found(&err) || is_err_version_not_found(&err)) {
                Ok(None)
            } else {
                Err(err)
            }
        }
    }
}

fn latest_object_access_delete_marker_error(
    bucket: &str,
    object: &str,
    info: &ObjectInfo,
    opts: &ObjectOptions,
) -> Option<Error> {
    if !info.delete_marker {
        return None;
    }

    Some(if opts.version_id.is_none() || opts.delete_marker {
        to_object_err(StorageError::FileNotFound, vec![bucket, object])
    } else {
        to_object_err(StorageError::MethodNotAllowed, vec![bucket, object])
    })
}

fn resolve_latest_object_access(
    bucket: &str,
    object: &str,
    info: ObjectInfo,
    idx: usize,
    opts: &ObjectOptions,
) -> Result<(ObjectInfo, usize)> {
    if let Some(err) = latest_object_access_delete_marker_error(bucket, object, &info, opts) {
        return Err(err);
    }

    Ok((info, idx))
}

fn should_create_delete_marker_for_missing_object(opts: &ObjectOptions) -> bool {
    opts.versioned && opts.version_id.is_none() && !opts.delete_marker && !opts.data_movement
}

#[cfg(test)]
struct DeleteAfterObjectLockSnapshotBarrierState {
    bucket: String,
    arrived: tokio::sync::Notify,
    release: tokio::sync::Notify,
}

#[cfg(test)]
pub(crate) struct DeleteAfterObjectLockSnapshotBarrier {
    state: Arc<DeleteAfterObjectLockSnapshotBarrierState>,
}

#[cfg(test)]
static DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER: std::sync::OnceLock<
    std::sync::Mutex<Option<Arc<DeleteAfterObjectLockSnapshotBarrierState>>>,
> = std::sync::OnceLock::new();

#[cfg(test)]
impl DeleteAfterObjectLockSnapshotBarrier {
    pub(crate) fn install(bucket: &str) -> Self {
        let state = Arc::new(DeleteAfterObjectLockSnapshotBarrierState {
            bucket: bucket.to_string(),
            arrived: tokio::sync::Notify::new(),
            release: tokio::sync::Notify::new(),
        });
        let mut slot = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER
            .get_or_init(|| std::sync::Mutex::new(None))
            .lock()
            .expect("delete snapshot barrier mutex should not poison");
        assert!(slot.is_none(), "delete snapshot barrier must not already be installed");
        *slot = Some(Arc::clone(&state));
        Self { state }
    }

    pub(crate) async fn wait_until_paused(&self) {
        self.state.arrived.notified().await;
    }

    pub(crate) fn release(&self) {
        self.state.release.notify_one();
    }
}

#[cfg(test)]
impl Drop for DeleteAfterObjectLockSnapshotBarrier {
    fn drop(&mut self) {
        self.state.release.notify_one();
        if let Some(slot) = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER.get() {
            let mut slot = slot.lock().expect("delete snapshot barrier mutex should not poison");
            if slot.as_ref().is_some_and(|installed| Arc::ptr_eq(installed, &self.state)) {
                *slot = None;
            }
        }
    }
}

#[cfg(test)]
async fn pause_delete_after_object_lock_snapshot(bucket: &str) {
    let state = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER
        .get_or_init(|| std::sync::Mutex::new(None))
        .lock()
        .expect("delete snapshot barrier mutex should not poison")
        .as_ref()
        .filter(|state| state.bucket == bucket)
        .cloned();
    if let Some(state) = state {
        state.arrived.notify_one();
        state.release.notified().await;
    }
}

/// Whether a delete-time lookup miss on a directory key should trigger an orphan
/// empty-directory tree purge (issue #4189).
///
/// The lookup surfaces *version*-not-found here, not object-not-found: `del_opts`
/// pins `version_id = Uuid::nil()` for directory keys, so a missing dir object fails
/// the specific-version lookup. Both misses must be accepted, otherwise the real
/// HTTP delete path (which always sets the nil version) never reaches the purge and
/// the ghost folder survives with a fake 204 — the exact #4189 symptom.
fn should_purge_orphan_dir_on_missing(err: &Error, object: &str) -> bool {
    (is_err_object_not_found(err) || is_err_version_not_found(err)) && rustfs_utils::path::is_dir_object(object)
}

fn version_aware_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions {
    let mut lookup_opts = opts.clone();
    lookup_opts.no_lock = no_lock;
    if lookup_opts.version_id.is_some() {
        lookup_opts.metadata_chg = true;
    }

    lookup_opts
}

fn data_movement_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions {
    let mut lookup_opts = version_aware_lookup_opts(opts, no_lock);
    lookup_opts.skip_decommissioned = true;
    lookup_opts.skip_rebalancing = true;

    lookup_opts
}

fn transition_restore_pool_opts(opts: &ObjectOptions) -> ObjectOptions {
    let mut lookup_opts = opts.clone();
    lookup_opts.skip_decommissioned = true;
    lookup_opts
}

fn effective_object_actual_size(info: &ObjectInfo) -> Option<i64> {
    info.get_actual_size().ok()
}

fn is_equivalent_data_movement_delete_marker(source: &ObjectInfo, target: &ObjectInfo) -> bool {
    is_data_movement_delete_marker(source)
        && is_data_movement_delete_marker(target)
        && source.version_id == target.version_id
        && source.mod_time == target.mod_time
        && source.user_defined == target.user_defined
        && source.user_tags == target.user_tags
        && source.replication_status_internal == target.replication_status_internal
        && source.replication_status == target.replication_status
        && source.version_purge_status_internal == target.version_purge_status_internal
        && source.version_purge_status == target.version_purge_status
}

fn is_data_movement_delete_marker(info: &ObjectInfo) -> bool {
    info.delete_marker
}

fn expected_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo) -> ObjectInfo {
    ObjectInfo::from_file_info(source, "", &source.name, source.version_id.is_some())
}

fn is_equivalent_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo, target: &ObjectInfo) -> bool {
    let expected = expected_data_movement_tiered_object(source);

    source.version_id == target.version_id
        && !target.delete_marker
        && source.size == target.size
        && source.get_etag() == target.etag
        && source.checksum == target.checksum
        && source.mod_time == target.mod_time
        && expected.user_defined == target.user_defined
        && expected.user_tags == target.user_tags
        && expected.expires == target.expires
        && expected.storage_class == target.storage_class
        && expected.replication_status_internal == target.replication_status_internal
        && expected.replication_status == target.replication_status
        && expected.version_purge_status_internal == target.version_purge_status_internal
        && expected.version_purge_status == target.version_purge_status
        && expected.transitioned_object.status == target.transitioned_object.status
        && expected.transitioned_object.name == target.transitioned_object.name
        && expected.transitioned_object.tier == target.transitioned_object.tier
        && expected.transitioned_object.version_id == target.transitioned_object.version_id
        && expected.transitioned_object.free_version == target.transitioned_object.free_version
        && effective_object_actual_size(target) == Some(source.size)
}

fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
    target_pool_idx != src_pool_idx
}

fn resolve_data_movement_resume_target_pool(
    selected_target_pool_idx: usize,
    resume_target_pool_idx: Option<usize>,
    src_pool_idx: usize,
) -> usize {
    if should_check_data_movement_resume_target(src_pool_idx, selected_target_pool_idx) {
        selected_target_pool_idx
    } else {
        resume_target_pool_idx.unwrap_or(selected_target_pool_idx)
    }
}

fn resolve_data_movement_delete_marker_resume_result(
    target_result: Result<Option<ObjectInfo>>,
    source: &ObjectInfo,
    src_pool_idx: usize,
    target_pool_idx: usize,
) -> Result<bool> {
    if !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx) {
        return Ok(false);
    }

    let Some(target) = target_result? else {
        return Ok(false);
    };

    Ok(is_equivalent_data_movement_delete_marker(source, &target))
}

fn resolve_data_movement_tiered_resume_result(
    target_result: Result<Option<ObjectInfo>>,
    source: &rustfs_filemeta::FileInfo,
    src_pool_idx: usize,
    target_pool_idx: usize,
) -> Result<bool> {
    if !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx) {
        return Ok(false);
    }

    let Some(target) = target_result? else {
        return Ok(false);
    };

    Ok(is_equivalent_data_movement_tiered_object(source, &target))
}

fn return_batch_delete_lock_error(objects: &[ObjectToDelete], err: Error) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
    let del_objects = objects
        .iter()
        .map(|object| DeletedObject {
            object_name: decode_dir_object(&object.object_name),
            version_id: object.version_id,
            ..Default::default()
        })
        .collect();
    let del_errs = objects.iter().map(|_| Some(err.clone())).collect();

    (del_objects, del_errs)
}

fn sorted_unique_delete_object_names(objects: &[ObjectToDelete]) -> Vec<&str> {
    let mut object_names: Vec<&str> = objects.iter().map(|object| object.object_name.as_str()).collect();
    object_names.sort_unstable();
    object_names.dedup();
    object_names
}

impl ECStore {
    /// Captures Object Lock state once for a batch of PUTs to the same bucket.
    /// `handle_put_object` only reuses the token for the same store, bucket,
    /// bucket incarnation, and Object Lock configuration revision.
    pub async fn object_lock_config_snapshot(&self, bucket: &str) -> Result<Arc<ObjectLockConfigSnapshot>> {
        check_valid_bucket_name(bucket)?;
        let lifecycle_guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
        let metadata_guard = acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
        let (state, bucket_incarnation_id, config_revision) =
            get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?;
        if lifecycle_guard.is_lock_lost() || metadata_guard.is_lock_lost() {
            return Err(Error::other("bucket lifecycle lock was lost while loading the Object Lock snapshot"));
        }
        Ok(Arc::new(ObjectLockConfigSnapshot::for_guarded_store_bucket(
            self.id,
            bucket,
            bucket_incarnation_id,
            config_revision,
            state,
            lifecycle_guard,
            metadata_guard,
        )))
    }

    async fn object_lock_config_snapshot_under_lifecycle_fence(
        &self,
        bucket: &str,
        lifecycle_fence: &NamespaceLockFence,
    ) -> Result<Arc<ObjectLockConfigSnapshot>> {
        if lifecycle_fence.is_lock_lost() {
            return Err(Error::other("bucket lifecycle lock was lost before loading the Object Lock snapshot"));
        }
        let metadata_guard = acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
        let (state, bucket_incarnation_id, config_revision) =
            get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?;
        if lifecycle_fence.is_lock_lost() || metadata_guard.is_lock_lost() {
            return Err(Error::other("bucket lock was lost while loading the Object Lock snapshot"));
        }
        Ok(Arc::new(ObjectLockConfigSnapshot::for_store_bucket_under_lifecycle_fence(
            self.id,
            bucket,
            bucket_incarnation_id,
            config_revision,
            state,
            lifecycle_fence.clone(),
            metadata_guard,
        )))
    }

    /// Resolves a GET's object identity without constructing its body reader.
    ///
    /// This is an additive two-stage counterpart to `get_object_reader`. The
    /// existing method remains the compatibility path for callers that do not
    /// need a pre-reader decision point.
    pub async fn prepare_get_object_reader(
        &self,
        bucket: &str,
        object: &str,
        range: Option<HTTPRangeSpec>,
        headers: HeaderMap,
        opts: &ObjectOptions,
    ) -> Result<PreparedGetObjectReader> {
        check_get_obj_args(bucket, object)?;

        let object = encode_dir_object(object);
        let mut opts = opts.clone();
        let read_lock_guard = self
            .acquire_object_read_lock_if_needed("prepare_get_object", bucket, &object, &mut opts)
            .await?;

        let (metadata, pool) = if self.single_pool() {
            let pool = Arc::clone(&self.pools[0]);
            let metadata = pool.prepare_get_object_reader_metadata(bucket, &object, &opts).await?;
            (metadata, pool)
        } else {
            let (_, pool_idx) = self
                .get_latest_accessible_object_info_with_idx(bucket, &object, &opts)
                .await?;
            let pool = self
                .pools
                .get(pool_idx)
                .cloned()
                .ok_or_else(|| Error::other(format!("resolved GET pool index {pool_idx} is out of bounds")))?;
            let metadata = pool.prepare_get_object_reader_metadata(bucket, &object, &opts).await?;
            (metadata, pool)
        };

        Ok(PreparedGetObjectReader {
            pool,
            bucket: bucket.to_owned(),
            object,
            range,
            headers,
            opts,
            metadata,
            read_lock_guard,
        })
    }

    fn map_namespace_lock_error(bucket: &str, object: &str, mode: &'static str, err: rustfs_lock::LockError) -> StorageError {
        match err {
            rustfs_lock::LockError::QuorumNotReached { required, achieved } => StorageError::NamespaceLockQuorumUnavailable {
                mode,
                bucket: bucket.to_string(),
                object: object.to_string(),
                required,
                achieved,
            },
            other => StorageError::Lock(other),
        }
    }

    async fn acquire_object_write_lock(&self, op: &'static str, bucket: &str, object: &str) -> Result<ObjectLockDiagGuard> {
        let diag_enabled = is_object_lock_diag_enabled();
        let ns_lock = self.handle_new_ns_lock(bucket, object).await?;
        let acquire_start = Instant::now();
        let guard = ns_lock
            .get_write_lock(get_lock_acquire_timeout())
            .await
            .map_err(|err| Self::map_namespace_lock_error(bucket, object, "write", err))?;
        let owner = diag_enabled.then(|| ns_lock.owner().to_string());
        log_object_lock_acquire_if_slow(
            op,
            bucket,
            object,
            owner.as_deref(),
            ObjectLockDiagMode::Write,
            acquire_start.elapsed(),
            diag_enabled,
        );

        Ok(ObjectLockDiagGuard::new(
            guard,
            diag_enabled,
            op,
            diag_enabled.then(|| bucket.to_string()),
            diag_enabled.then(|| object.to_string()),
            owner,
            ObjectLockDiagMode::Write,
        ))
    }

    async fn acquire_object_write_lock_if_needed(
        &self,
        op: &'static str,
        bucket: &str,
        object: &str,
        opts: &mut ObjectOptions,
    ) -> Result<Option<ObjectLockDiagGuard>> {
        if opts.no_lock {
            return Ok(None);
        }

        let guard = self.acquire_object_write_lock(op, bucket, object).await?;
        if let Some(signal) = guard.lock_lost_signal() {
            opts.add_namespace_lock_lost_signal(signal);
        }
        opts.ensure_namespace_lock_fence();
        opts.no_lock = true;

        Ok(Some(guard))
    }

    /// Serializes the RestoreObject accept path — the read of the current
    /// `x-amz-restore` status, the ongoing/already-restored decision, and the
    /// metadata write that flips `ongoing-request="true"` — against concurrent
    /// accepts of the same object, making that read-check-write an atomic
    /// compare-and-set (backlog#1304). While the guard is held the caller must
    /// pass `no_lock: true` on its reads/writes of this object, check
    /// [`RestoreAcceptGuard::is_lock_lost`] before the status write, and drop
    /// the guard before starting the tier copy-back so concurrent
    /// HEAD/`get_object_info` stay non-blocking during the restore.
    pub async fn acquire_restore_accept_guard(&self, bucket: &str, object: &str) -> Result<RestoreAcceptGuard> {
        let object = encode_dir_object(object);
        let guard = self
            .acquire_object_write_lock("restore_object_accept", bucket, &object)
            .await?;
        Ok(RestoreAcceptGuard(guard))
    }

    async fn acquire_delete_objects_write_locks(
        &self,
        bucket: &str,
        objects: &[ObjectToDelete],
        opts: &mut ObjectOptions,
    ) -> Result<Vec<ObjectLockDiagGuard>> {
        if opts.no_lock || objects.is_empty() {
            return Ok(Vec::new());
        }

        let object_names = sorted_unique_delete_object_names(objects);
        // Lock order: encoded object names are acquired in ascending order, then
        // the set-layer calls receive no_lock so they do not reacquire them.
        let mut guards = Vec::with_capacity(object_names.len());
        for object in object_names {
            guards.push(self.acquire_object_write_lock("delete_objects", bucket, object).await?);
        }
        opts.no_lock = true;
        for signal in guards.iter().filter_map(ObjectLockDiagGuard::lock_lost_signal) {
            opts.add_namespace_lock_lost_signal(signal);
        }
        opts.ensure_namespace_lock_fence();

        Ok(guards)
    }

    async fn acquire_object_read_lock_if_needed(
        &self,
        op: &'static str,
        bucket: &str,
        object: &str,
        opts: &mut ObjectOptions,
    ) -> Result<Option<ObjectLockDiagGuard>> {
        if opts.no_lock {
            return Ok(None);
        }

        let diag_enabled = is_object_lock_diag_enabled();
        let ns_lock = self.handle_new_ns_lock(bucket, object).await?;
        let acquire_start = Instant::now();
        let guard = ns_lock
            .get_read_lock(get_lock_acquire_timeout())
            .await
            .map_err(|err| Self::map_namespace_lock_error(bucket, object, "read", err))?;
        let owner = diag_enabled.then(|| ns_lock.owner().to_string());
        log_object_lock_acquire_if_slow(
            op,
            bucket,
            object,
            owner.as_deref(),
            ObjectLockDiagMode::Read,
            acquire_start.elapsed(),
            diag_enabled,
        );
        opts.no_lock = true;
        opts.metadata_cache_safe = true;

        Ok(Some(ObjectLockDiagGuard::new(
            guard,
            diag_enabled,
            op,
            diag_enabled.then(|| bucket.to_string()),
            diag_enabled.then(|| object.to_string()),
            owner,
            ObjectLockDiagMode::Read,
        )))
    }

    fn attach_read_lock_guard(mut reader: GetObjectReader, guard: Option<ObjectLockDiagGuard>) -> GetObjectReader {
        if is_lock_optimization_enabled() || reader.buffered_body.is_some() {
            return reader;
        }

        if let Some(guard) = guard {
            reader.stream = Box::new(LockGuardedReader {
                inner: reader.stream,
                guard: Some(guard),
            });
        }

        reader
    }

    async fn get_latest_accessible_object_info_with_idx(
        &self,
        bucket: &str,
        object: &str,
        opts: &ObjectOptions,
    ) -> Result<(ObjectInfo, usize)> {
        let (info, idx) = self.get_latest_object_info_with_idx(bucket, object, opts).await?;
        resolve_latest_object_access(bucket, object, info, idx, opts)
    }

    pub(super) async fn select_data_movement_pool_idx(
        &self,
        bucket: &str,
        object: &str,
        size: i64,
        opts: &ObjectOptions,
        no_lock: bool,
    ) -> Result<usize> {
        match self
            .get_pool_info_existing_with_opts(bucket, object, &data_movement_pool_lookup_opts(opts, no_lock))
            .await
        {
            Ok((pinfo, _)) => Ok(pinfo.index),
            Err(err) => {
                if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
                    return Err(err);
                }

                self.get_available_pool_idx(bucket, object, size).await.ok_or(Error::DiskFull)
            }
        }
    }

    async fn find_data_movement_target_info(
        &self,
        bucket: &str,
        object: &str,
        target_pool_idx: usize,
        opts: &ObjectOptions,
    ) -> Result<Option<ObjectInfo>> {
        let lookup_opts = version_aware_lookup_opts(opts, true);

        let Some(pool) = self.pools.get(target_pool_idx) else {
            return Err(Error::other(format!(
                "data movement resume target pool {target_pool_idx} is out of range for {bucket}/{object}"
            )));
        };

        match pool.get_object_info(bucket, object, &lookup_opts).await {
            Ok(info) => Ok(Some(info)),
            Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(None),
            Err(err) => Err(err),
        }
    }

    async fn has_equivalent_data_movement_delete_marker(
        &self,
        bucket: &str,
        object: &str,
        source: &ObjectInfo,
        opts: &ObjectOptions,
        target_pool_idx: usize,
    ) -> Result<bool> {
        resolve_data_movement_delete_marker_resume_result(
            self.find_data_movement_target_info(bucket, object, target_pool_idx, opts)
                .await,
            source,
            opts.src_pool_idx,
            target_pool_idx,
        )
    }

    async fn has_equivalent_data_movement_tiered_object(
        &self,
        bucket: &str,
        object: &str,
        source: &rustfs_filemeta::FileInfo,
        opts: &ObjectOptions,
        target_pool_idx: usize,
    ) -> Result<bool> {
        resolve_data_movement_tiered_resume_result(
            self.find_data_movement_target_info(bucket, object, target_pool_idx, opts)
                .await,
            source,
            opts.src_pool_idx,
            target_pool_idx,
        )
    }

    fn resolve_decommission_target_pool_idx_result(result: Result<usize>, bucket: &str, object: &str) -> Result<usize> {
        result.map_err(|err| Error::other(format!("failed to select decommission target pool for {bucket}/{object}: {err}")))
    }

    fn resolve_decommission_tiered_object_result(result: Result<()>, bucket: &str, object: &str) -> Result<()> {
        result.map_err(|err| Error::other(format!("failed to decommission tiered object for {bucket}/{object}: {err}")))
    }

    #[instrument(skip(self, fi, opts))]
    pub(crate) async fn decommission_tiered_object(
        &self,
        bucket: &str,
        object: &str,
        fi: &rustfs_filemeta::FileInfo,
        opts: &ObjectOptions,
    ) -> Result<()> {
        check_put_object_args(bucket, object)?;

        let object = encode_dir_object(object);

        if self.single_pool() {
            return Self::resolve_decommission_tiered_object_result(
                Err(Error::other("single pool deployments cannot decommission tiered objects")),
                bucket,
                &object,
            );
        }

        let idx = if opts.data_movement && opts.version_id.is_some() {
            Self::resolve_decommission_target_pool_idx_result(
                self.select_data_movement_pool_idx(bucket, &object, fi.size, opts, true).await,
                bucket,
                &object,
            )?
        } else {
            Self::resolve_decommission_target_pool_idx_result(
                self.get_pool_idx_no_lock(bucket, &object, fi.size).await,
                bucket,
                &object,
            )?
        };
        if opts.data_movement && idx == opts.src_pool_idx {
            let resume_target_pool_idx = self
                .get_available_pool_idx_excluding(bucket, &object, fi.size, opts.src_pool_idx)
                .await;
            let target_pool_idx = resolve_data_movement_resume_target_pool(idx, resume_target_pool_idx, opts.src_pool_idx);
            if self
                .has_equivalent_data_movement_tiered_object(bucket, &object, fi, opts, target_pool_idx)
                .await?
            {
                return Ok(());
            }

            return Err(StorageError::DataMovementOverwriteErr(
                bucket.to_owned(),
                object.to_owned(),
                opts.version_id.clone().unwrap_or_default(),
            ));
        }

        Self::resolve_decommission_tiered_object_result(
            self.pools[idx]
                .get_disks_by_key(&object)
                .decommission_tiered_object(bucket, &object, fi, opts)
                .await,
            bucket,
            &object,
        )
    }

    #[instrument(level = "debug", skip(self))]
    #[hotpath::measure(impl_type = "ECStore")]
    pub(super) async fn handle_get_object_reader(
        &self,
        bucket: &str,
        object: &str,
        range: Option<HTTPRangeSpec>,
        h: HeaderMap,
        opts: &ObjectOptions,
    ) -> Result<GetObjectReader> {
        check_get_obj_args(bucket, object)?;

        let object = encode_dir_object(object);
        let mut opts = opts.clone();
        let read_lock_guard = self
            .acquire_object_read_lock_if_needed("get_object", bucket, &object, &mut opts)
            .await?;

        let reader = if self.single_pool() {
            self.pools[0]
                .get_object_reader(bucket, object.as_str(), range, h, &opts)
                .await?
        } else {
            let (_, idx) = self
                .get_latest_accessible_object_info_with_idx(bucket, &object, &opts)
                .await?;
            self.pools[idx]
                .get_object_reader(bucket, object.as_str(), range, h, &opts)
                .await?
        };

        Ok(Self::attach_read_lock_guard(reader, read_lock_guard))
    }

    #[instrument(level = "debug", skip(self, data))]
    #[hotpath::measure(impl_type = "ECStore")]
    pub(super) async fn handle_put_object(
        &self,
        bucket: &str,
        object: &str,
        data: &mut PutObjReader,
        opts: &ObjectOptions,
    ) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
        check_put_object_args(bucket, object)?;

        let object = encode_dir_object(object);
        let mut opts = opts.clone();
        if !is_meta_bucketname(bucket) && opts.expected_bucket_incarnation_id.is_none() {
            opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(bucket).await?);
        }
        if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
            let expected_incarnation_id = opts
                .expected_bucket_incarnation_id
                .ok_or_else(|| Error::other("destructive PUT is missing its bucket incarnation"))?;
            if opts.object_lock_config_snapshot.is_none() {
                opts.object_lock_config_snapshot = Some(self.object_lock_config_snapshot(bucket).await?);
            }
            let snapshot = match opts.object_lock_config_snapshot.as_ref() {
                Some(snapshot) if snapshot.is_valid_for_destructive_put(self.id, bucket, expected_incarnation_id) => {
                    Arc::clone(snapshot)
                }
                _ => {
                    return Err(Error::other(
                        "Object Lock snapshot does not hold valid target bucket generation and configuration fences",
                    ));
                }
            };
            snapshot.add_lock_fences(&mut opts);
        }

        // Keep PUT atomic-read friendly: SetDisks takes the object write lock only
        // around precondition checks and the final rename/commit.
        if self.single_pool() {
            return self.pools[0]
                .put_object_with_old_current_size(bucket, object.as_str(), data, &opts)
                .await;
        }

        let idx = if opts.data_movement && opts.version_id.is_some() {
            self.select_data_movement_pool_idx(bucket, &object, data.size(), &opts, false)
                .await?
        } else if opts.no_lock {
            self.get_pool_idx_no_lock(bucket, &object, data.size()).await?
        } else {
            self.get_pool_idx(bucket, &object, data.size()).await?
        };

        if opts.data_movement && idx == opts.src_pool_idx {
            return Err(StorageError::DataMovementOverwriteErr(
                bucket.to_owned(),
                object.to_owned(),
                opts.version_id.clone().unwrap_or_default(),
            ));
        }

        self.pools[idx]
            .put_object_with_old_current_size(bucket, &object, data, &opts)
            .await
    }

    #[instrument(skip(self))]
    pub(super) async fn handle_get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
        check_object_args(bucket, object)?;

        let object = encode_dir_object(object);
        let mut opts = opts.clone();
        let _object_lock_guard = self
            .acquire_object_read_lock_if_needed("get_object_info", bucket, &object, &mut opts)
            .await?;

        let info = if self.single_pool() {
            self.pools[0].get_object_info(bucket, object.as_str(), &opts).await?
        } else {
            self.get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &opts)
                .await?
                .0
        };
        opts.precondition_check(&info)?;
        Ok(info)
    }

    #[instrument(skip(self))]
    #[allow(clippy::too_many_arguments)]
    pub(super) async fn handle_copy_object(
        &self,
        src_bucket: &str,
        src_object: &str,
        dst_bucket: &str,
        dst_object: &str,
        src_info: &mut ObjectInfo,
        src_opts: &ObjectOptions,
        dst_opts: &ObjectOptions,
    ) -> Result<ObjectInfo> {
        check_copy_obj_args(src_bucket, src_object)?;
        check_copy_obj_args(dst_bucket, dst_object)?;

        let src_object = encode_dir_object(src_object);
        let dst_object = encode_dir_object(dst_object);

        let cp_src_dst_same = path_join_buf(&[src_bucket, &src_object]) == path_join_buf(&[dst_bucket, &dst_object]);

        let mut dst_opts = dst_opts.clone();
        if !is_meta_bucketname(dst_bucket) && dst_opts.expected_bucket_incarnation_id.is_none() {
            dst_opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(dst_bucket).await?);
        }
        let _bucket_lifecycle_guard = if is_meta_bucketname(dst_bucket) || dst_opts.bucket_lifecycle_lock_fence.is_some() {
            None
        } else {
            Some(self.acquire_bucket_lifecycle_read_lock(dst_bucket).await?)
        };
        let current_bucket_incarnation_id = if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
            dst_opts.add_bucket_lifecycle_lock_guard(guard);
            let current_incarnation_id = get_bucket_incarnation_id_in(&self.ctx, dst_bucket).await?;
            if dst_opts
                .expected_bucket_incarnation_id
                .is_some_and(|expected| expected != current_incarnation_id)
            {
                return Err(StorageError::BucketNotFound(dst_bucket.to_string()));
            }
            Some(current_incarnation_id)
        } else {
            dst_opts.expected_bucket_incarnation_id
        };
        if dst_opts
            .bucket_lifecycle_lock_fence
            .as_ref()
            .is_some_and(NamespaceLockFence::is_lock_lost)
        {
            return Err(StorageError::NamespaceLockQuorumUnavailable {
                mode: "copy_object_bucket_generation",
                bucket: dst_bucket.to_string(),
                object: dst_object.clone(),
                required: 1,
                achieved: 0,
            });
        }
        if dst_opts.overwrites_existing_version() && !is_meta_bucketname(dst_bucket) {
            let incarnation_id =
                current_bucket_incarnation_id.ok_or_else(|| Error::other("copy is missing its bucket incarnation snapshot"))?;
            let lifecycle_fence = dst_opts
                .bucket_lifecycle_lock_fence
                .as_ref()
                .ok_or_else(|| Error::other("copy is missing its bucket lifecycle fence"))?;
            let snapshot = match dst_opts.object_lock_config_snapshot.as_ref() {
                Some(snapshot) => Arc::clone(snapshot),
                None => {
                    self.object_lock_config_snapshot_under_lifecycle_fence(dst_bucket, lifecycle_fence)
                        .await?
                }
            };
            if !snapshot.is_valid_for_destructive_put(self.id, dst_bucket, incarnation_id) {
                return Err(Error::other("copy Object Lock snapshot does not match the target bucket generation"));
            }
            snapshot.add_lock_fences(&mut dst_opts);
            dst_opts.object_lock_config_snapshot = Some(snapshot);
        }
        let _dst_lock_guard = if cp_src_dst_same && dst_opts.expected_current_version_id.is_none() {
            self.acquire_object_write_lock_if_needed("copy_object", dst_bucket, &dst_object, &mut dst_opts)
                .await?
        } else {
            None
        };

        if cp_src_dst_same {
            let pool_idx = self
                .get_pool_info_existing_with_opts(src_bucket, &src_object, &version_aware_lookup_opts(src_opts, true))
                .await?
                .0
                .index;

            if let (Some(src_vid), Some(dst_vid)) = (&src_opts.version_id, &dst_opts.version_id)
                && src_vid == dst_vid
            {
                return self.pools[pool_idx]
                    .copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &dst_opts)
                    .await;
            }

            if !dst_opts.versioned && src_opts.version_id.is_none() {
                if src_info.metadata_only {
                    // Zero-copy update: only xl.meta is rewritten, the data blocks stay as they
                    // are. The caller must therefore guarantee that the destination metadata
                    // still describes the stored bytes. In particular a copy that re-derives
                    // encryption material may NOT set metadata_only — that would leave a fresh
                    // DEK beside ciphertext sealed under the old one, permanently destroying the
                    // object. The S3 handler enforces this before calling in (see the
                    // metadata_only decision in rustfs/src/app/object_usecase.rs); the sibling
                    // versioned branch below resolves the same risk by rewriting through
                    // put_object (issue #4238).
                    return self.pools[pool_idx]
                        .copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &dst_opts)
                        .await;
                }
                // Transitioned object self-copy: restore from tier into the same pool.
                let put_opts = ObjectOptions {
                    user_defined: (*src_info.user_defined).clone(),
                    versioned: dst_opts.versioned,
                    version_id: dst_opts.version_id.clone(),
                    no_lock: dst_opts.no_lock,
                    mod_time: dst_opts.mod_time,
                    http_preconditions: dst_opts.http_preconditions.clone(),
                    expected_current_version_id: dst_opts.expected_current_version_id.clone(),
                    expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id,
                    namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
                    bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
                    object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
                    ..Default::default()
                };
                return if let Some(reader) = src_info.put_object_reader.as_mut() {
                    self.pools[pool_idx]
                        .put_object(dst_bucket, &dst_object, reader, &put_opts)
                        .await
                } else {
                    Err(StorageError::InvalidArgument(
                        src_bucket.to_owned(),
                        src_object.to_owned(),
                        "put_object_reader is none".to_owned(),
                    ))
                };
            }

            if dst_opts.versioned && src_opts.version_id != dst_opts.version_id {
                // Restoring a specific historical version onto the current key creates a NEW
                // version. When the caller supplies a reader (S3 CopyObject), write the fetched
                // bytes through put_object so any re-encryption/compression applied to the reader
                // stays consistent with the new version's metadata. Sharing the source data_dir via
                // a metadata-only version copy would corrupt SSE/compressed objects (issue #4238).
                if let Some(reader) = src_info.put_object_reader.as_mut() {
                    let put_opts = ObjectOptions {
                        user_defined: (*src_info.user_defined).clone(),
                        versioned: dst_opts.versioned,
                        version_id: dst_opts.version_id.clone(),
                        no_lock: dst_opts.no_lock,
                        mod_time: dst_opts.mod_time,
                        http_preconditions: dst_opts.http_preconditions.clone(),
                        expected_current_version_id: dst_opts.expected_current_version_id.clone(),
                        expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id,
                        namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
                        bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
                        object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
                        ..Default::default()
                    };
                    return self.pools[pool_idx]
                        .put_object(dst_bucket, &dst_object, reader, &put_opts)
                        .await;
                }
                src_info.version_only = true;
                return self.pools[pool_idx]
                    .copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &dst_opts)
                    .await;
            }
        }

        let pool_idx = if dst_opts.no_lock {
            self.get_pool_idx_no_lock(dst_bucket, &dst_object, src_info.size).await?
        } else {
            self.get_pool_idx(dst_bucket, &dst_object, src_info.size).await?
        };

        let put_opts = ObjectOptions {
            user_defined: (*src_info.user_defined).clone(),
            versioned: dst_opts.versioned,
            version_id: dst_opts.version_id.clone(),
            no_lock: dst_opts.no_lock,
            mod_time: dst_opts.mod_time,
            http_preconditions: dst_opts.http_preconditions.clone(),
            expected_current_version_id: dst_opts.expected_current_version_id.clone(),
            expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id,
            namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
            bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
            object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
            ..Default::default()
        };

        if let Some(put_object_reader) = src_info.put_object_reader.as_mut() {
            return self.pools[pool_idx]
                .put_object(dst_bucket, &dst_object, put_object_reader, &put_opts)
                .await;
        }

        Err(StorageError::InvalidArgument(
            src_bucket.to_owned(),
            src_object.to_owned(),
            "put_object_reader is none".to_owned(),
        ))
    }

    /// Best-effort purge of an orphan directory prefix — an on-disk tree of empty
    /// directories with no `xl.meta` anywhere (issue #4189). Orphan fragments can sit
    /// on any erasure set of any pool (they are left behind by whichever sets stored
    /// the now-deleted children), so every set is swept. Returns true when at least
    /// one set removed an orphan tree. Hard per-set failures are logged and skipped:
    /// the caller falls back to surfacing the original NotFound.
    async fn purge_orphan_dir_object(&self, bucket: &str, object: &str) -> bool {
        let prefix = decode_dir_object(object);
        let mut purged = false;
        for pool in self.pools.iter() {
            for set in pool.disk_set.iter() {
                match set.purge_orphan_dir_object(bucket, &prefix).await {
                    Ok(set_purged) => purged |= set_purged,
                    Err(err) => {
                        warn!(
                            bucket,
                            prefix,
                            pool_index = pool.pool_idx,
                            error = ?err,
                            "failed to purge orphan directory prefix"
                        );
                    }
                }
            }
        }
        purged
    }

    pub async fn delete_object_with_tier_delete_journal(
        self: &Arc<Self>,
        bucket: &str,
        object: &str,
        opts: ObjectOptions,
    ) -> Result<ObjectInfo> {
        let result = self
            .handle_delete_object_with_journal(bucket, object, opts, Some(Arc::clone(self)))
            .await;
        if result.is_ok() {
            list_objects::observe_list_objects_mutation(self, bucket).await;
        }
        result
    }

    pub async fn delete_objects_with_tier_delete_journal(
        self: &Arc<Self>,
        bucket: &str,
        objects: Vec<ObjectToDelete>,
        opts: ObjectOptions,
    ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
        let result = self
            .handle_delete_objects_with_journal(bucket, objects, opts, Some(Arc::clone(self)))
            .await;
        let success_count = result.1.iter().filter(|err| err.is_none()).count();
        if success_count > 0 {
            list_objects::observe_list_objects_mutations(self, bucket, success_count).await;
        }
        result
    }

    #[instrument(skip(self))]
    pub(super) async fn handle_delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
        self.handle_delete_object_with_journal(bucket, object, opts, None).await
    }

    pub(super) async fn handle_delete_object_with_journal(
        &self,
        bucket: &str,
        object: &str,
        opts: ObjectOptions,
        tier_journal_api: Option<Arc<ECStore>>,
    ) -> Result<ObjectInfo> {
        check_del_obj_args(bucket, object)?;

        let _bucket_lifecycle_guard = if is_meta_bucketname(bucket) {
            None
        } else if opts.delete_prefix {
            Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
        } else {
            Some(self.acquire_bucket_lifecycle_read_lock(bucket).await?)
        };
        let object = if opts.delete_prefix && !opts.delete_prefix_object {
            object.to_owned()
        } else {
            encode_dir_object(object)
        };
        let object = object.as_str();
        let mut opts = opts;
        opts.tier_delete_journal_api = tier_journal_api.clone();
        if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
            opts.add_bucket_lifecycle_lock_guard(guard);
        }

        if !is_meta_bucketname(bucket) {
            get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await?;
        }
        let _object_lock_metadata_guard = if !is_meta_bucketname(bucket) {
            Some(acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?)
        } else {
            None
        };
        if let Some(guard) = _object_lock_metadata_guard.as_ref() {
            opts.add_namespace_lock_guard(guard);
        }
        let current_bucket_incarnation_id = if _object_lock_metadata_guard.is_some() {
            let (state, incarnation_id, config_revision) =
                get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?;
            opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket(
                self.id,
                bucket,
                incarnation_id,
                config_revision,
                state,
            )));
            Some(incarnation_id)
        } else {
            None
        };
        if let (Some(expected), Some(current)) = (opts.expected_bucket_incarnation_id, current_bucket_incarnation_id)
            && expected != current
        {
            return Err(StorageError::BucketNotFound(bucket.to_string()));
        }
        #[cfg(test)]
        if current_bucket_incarnation_id.is_some() {
            pause_delete_after_object_lock_snapshot(bucket).await;
        }

        if opts.delete_prefix && !opts.delete_prefix_object {
            // Prefix deletes cover multiple object keys; an exact lock on the prefix string
            // would not protect child objects.
            if !is_meta_bucketname(bucket) {
                let state = opts
                    .object_lock_config_snapshot
                    .as_deref()
                    .ok_or_else(|| Error::other("recursive delete is missing its Object Lock configuration snapshot"))?
                    .state();
                ensure_recursive_force_delete_allowed_for_state(bucket, state)?;
                let bypass_governance = opts
                    .object_lock_delete
                    .as_ref()
                    .is_some_and(|delete_opts| delete_opts.bypass_governance);
                for pool in &self.pools {
                    for set in &pool.disk_set {
                        let mut marker = None;
                        let mut version_marker = None;
                        loop {
                            let page = set
                                .clone()
                                .inner_list_object_versions_for_recursive_delete(
                                    bucket,
                                    object,
                                    marker.clone(),
                                    version_marker.clone(),
                                    RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE,
                                )
                                .await?;
                            for object_info in &page.objects {
                                if check_object_lock_for_deletion_with_state(state, object_info, bypass_governance)?.is_some() {
                                    return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object_info.name.clone()));
                                }
                            }
                            if !page.is_truncated {
                                break;
                            }
                            let next_marker = page.next_marker.ok_or_else(|| {
                                Error::other("recursive delete version scan did not return a continuation marker")
                            })?;
                            if marker.as_ref() == Some(&next_marker) && version_marker == page.next_version_idmarker {
                                return Err(Error::other("recursive delete version scan did not advance"));
                            }
                            marker = Some(next_marker);
                            version_marker = page.next_version_idmarker;
                        }
                    }
                }
            }
            delete_prefix_with_tier_delete_journal(self, bucket, object, &opts, tier_journal_api.as_ref()).await?;
            return Ok(ObjectInfo::default());
        }

        let _object_lock_guard = if opts.expected_current_version_id.is_none() {
            self.acquire_object_write_lock_if_needed("delete_object", bucket, object, &mut opts)
                .await?
        } else {
            None
        };
        if opts.delete_prefix {
            delete_prefix_with_tier_delete_journal(self, bucket, object, &opts, tier_journal_api.as_ref()).await?;
            return Ok(ObjectInfo::default());
        }

        let gopts = version_aware_lookup_opts(&opts, true);

        if opts.data_movement {
            let existing_pool_info = self.get_pool_info_existing_with_opts(bucket, object, &gopts).await;
            let existing_pool_idx = existing_pool_info
                .as_ref()
                .map(|(pinfo, _)| pinfo.index)
                .map_err(Clone::clone);
            let selected_target_pool_idx =
                match select_data_movement_target_pool(existing_pool_idx, opts.src_pool_idx, opts.delete_marker)? {
                    Some(pool_idx) => pool_idx,
                    None => self.get_pool_idx_no_lock(bucket, object, 0).await?,
                };
            let resume_target_pool_idx = if selected_target_pool_idx == opts.src_pool_idx {
                self.get_available_pool_idx_excluding(bucket, object, 0, opts.src_pool_idx)
                    .await
            } else {
                None
            };
            let target_pool_idx =
                resolve_data_movement_resume_target_pool(selected_target_pool_idx, resume_target_pool_idx, opts.src_pool_idx);

            if !should_check_data_movement_resume_target(opts.src_pool_idx, target_pool_idx) {
                if let Ok((source_pool_info, _)) = existing_pool_info
                    && opts.delete_marker
                    && is_data_movement_delete_marker(&source_pool_info.object_info)
                    && self
                        .has_equivalent_data_movement_delete_marker(
                            bucket,
                            object,
                            &source_pool_info.object_info,
                            &opts,
                            target_pool_idx,
                        )
                        .await?
                {
                    let mut obj = source_pool_info.object_info;
                    obj.name = decode_dir_object(object);
                    return Ok(obj);
                }

                return Err(StorageError::DataMovementOverwriteErr(
                    bucket.to_owned(),
                    object.to_owned(),
                    opts.version_id.unwrap_or_default(),
                ));
            }

            let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, opts).await?;
            obj.name = decode_dir_object(obj.name.as_str());
            return Ok(obj);
        }

        // Determine which pool contains it
        let (mut pinfo, errs) = match self.get_pool_info_existing_with_opts(bucket, object, &gopts).await {
            Ok(res) => res,
            Err(err) if is_err_read_quorum(&err) => return Err(StorageError::ErasureWriteQuorum),
            Err(err) if is_err_object_not_found(&err) && should_create_delete_marker_for_missing_object(&opts) => {
                let target_pool_idx = self.get_pool_idx_no_lock(bucket, object, 0).await?;
                let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, opts).await?;
                obj.name = decode_dir_object(object);
                return Ok(obj);
            }
            Err(err) => {
                // A folder key (`prefix/`) with no object metadata may still exist on
                // disk as an orphan empty-directory tree (issue #4189): listings show
                // it as a common prefix, but no regular delete path can remove it.
                // Purge the orphan tree so folder deletes actually take effect.
                if should_purge_orphan_dir_on_missing(&err, object) && self.purge_orphan_dir_object(bucket, object).await {
                    return Ok(ObjectInfo {
                        bucket: bucket.to_owned(),
                        name: decode_dir_object(object),
                        ..Default::default()
                    });
                }
                return Err(err);
            }
        };

        if pinfo.object_info.delete_marker && opts.version_id.is_none() {
            pinfo.object_info.name = decode_dir_object(object);
            return Ok(pinfo.object_info);
        }

        if opts.data_movement && opts.src_pool_idx == pinfo.index {
            return Err(StorageError::DataMovementOverwriteErr(
                bucket.to_owned(),
                object.to_owned(),
                opts.version_id.unwrap_or_default(),
            ));
        }

        let journal_entry = if let Some(api) = tier_journal_api.as_ref() {
            prepare_tier_delete_journal_entry(api, bucket, object, &opts, &pinfo.object_info).await?
        } else {
            None
        };

        if !errs.is_empty() && !opts.versioned && !opts.version_suspended {
            let mut obj = match self.delete_object_from_all_pools(bucket, object, &opts, errs).await {
                Ok(obj) => obj,
                Err(err) => {
                    if let (Some(api), Some(je)) = (tier_journal_api.as_ref(), journal_entry.as_ref()) {
                        abort_prepared_tier_delete_journal_entry(api, je).await;
                    }
                    return Err(err);
                }
            };
            if let (Some(api), Some(je)) = (tier_journal_api.as_ref(), journal_entry.as_ref()) {
                commit_prepared_tier_delete_journal_entry(api, je).await;
            }
            obj.name = decode_dir_object(object);
            return Ok(obj);
        }

        for pool in self.pools.iter() {
            match pool.delete_object(bucket, object, opts.clone()).await {
                Ok(res) => {
                    if let (Some(api), Some(je)) = (tier_journal_api.as_ref(), journal_entry.as_ref()) {
                        commit_prepared_tier_delete_journal_entry(api, je).await;
                    }
                    let mut obj = res;
                    obj.name = decode_dir_object(object);
                    return Ok(obj);
                }
                Err(err) => {
                    if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
                        return Err(err);
                    }
                }
            }
        }

        if let (Some(api), Some(je)) = (tier_journal_api.as_ref(), journal_entry.as_ref()) {
            abort_prepared_tier_delete_journal_entry(api, je).await;
        }

        if let Some(ver) = opts.version_id {
            return Err(StorageError::VersionNotFound(bucket.to_owned(), object.to_owned(), ver));
        }

        Err(StorageError::ObjectNotFound(bucket.to_owned(), object.to_owned()))
    }

    #[instrument(skip(self, objects, opts))]
    pub(super) async fn handle_delete_objects(
        &self,
        bucket: &str,
        objects: Vec<ObjectToDelete>,
        opts: ObjectOptions,
    ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
        self.handle_delete_objects_with_journal(bucket, objects, opts, None).await
    }

    pub(super) async fn handle_delete_objects_with_journal(
        &self,
        bucket: &str,
        objects: Vec<ObjectToDelete>,
        opts: ObjectOptions,
        tier_journal_api: Option<Arc<ECStore>>,
    ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
        // encode object name
        let objects: Vec<ObjectToDelete> = objects
            .iter()
            .map(|v| {
                let mut v = v.clone();
                v.object_name = encode_dir_object(v.object_name.as_str());
                v
            })
            .collect();

        // Default return value
        let mut del_objects = vec![DeletedObject::default(); objects.len()];

        let mut del_errs = Vec::with_capacity(objects.len());
        for _ in 0..objects.len() {
            del_errs.push(None)
        }

        let mut opts = opts;
        opts.tier_delete_journal_api = tier_journal_api;
        let _bucket_lifecycle_guard = if is_meta_bucketname(bucket) {
            None
        } else {
            match self.acquire_bucket_lifecycle_read_lock(bucket).await {
                Ok(guard) => Some(guard),
                Err(err) => return return_batch_delete_lock_error(objects.as_slice(), err),
            }
        };
        if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
            opts.add_bucket_lifecycle_lock_guard(guard);
        }
        if opts.delete_replication_config_snapshot.is_none() {
            match ReplicationObjectBridge::delete_request_config_in(&self.ctx, bucket).await {
                Ok(snapshot) => opts.delete_replication_config_snapshot = Some(Arc::new(snapshot)),
                Err(err) => {
                    let message = err.to_string();
                    let errors = (0..objects.len()).map(|_| Some(Error::other(message.clone()))).collect();
                    return (del_objects, errors);
                }
            }
        }
        if !is_meta_bucketname(bucket)
            && let Err(err) = get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await
        {
            return return_batch_delete_lock_error(objects.as_slice(), err);
        }
        let _object_lock_metadata_guard = if is_meta_bucketname(bucket) {
            None
        } else {
            Some(match acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await {
                Ok(guard) => guard,
                Err(err) => return return_batch_delete_lock_error(objects.as_slice(), err),
            })
        };
        if let Some(guard) = _object_lock_metadata_guard.as_ref() {
            opts.add_namespace_lock_guard(guard);
        }
        let current_bucket_incarnation_id = if _object_lock_metadata_guard.is_some() {
            let (state, incarnation_id, config_revision) =
                match get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await {
                    Ok(snapshot) => snapshot,
                    Err(err) => return return_batch_delete_lock_error(objects.as_slice(), err),
                };
            opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket(
                self.id,
                bucket,
                incarnation_id,
                config_revision,
                state,
            )));
            Some(incarnation_id)
        } else {
            None
        };
        if let (Some(expected), Some(current)) = (opts.expected_bucket_incarnation_id, current_bucket_incarnation_id)
            && expected != current
        {
            return return_batch_delete_lock_error(objects.as_slice(), StorageError::BucketNotFound(bucket.to_string()));
        }
        #[cfg(test)]
        if current_bucket_incarnation_id.is_some() {
            pause_delete_after_object_lock_snapshot(bucket).await;
        }
        let _object_lock_guards = match self.acquire_delete_objects_write_locks(bucket, &objects, &mut opts).await {
            Ok(guards) => guards,
            Err(err) => return return_batch_delete_lock_error(objects.as_slice(), err),
        };

        let mut futures = Vec::with_capacity(self.pools.len());

        for pool in self.pools.iter() {
            futures.push(pool.delete_objects(bucket, objects.clone(), opts.clone()));
        }

        let results = join_all(futures).await;

        for idx in 0..del_objects.len() {
            for (dels, errs) in results.iter() {
                if errs[idx].is_none() && dels[idx].found {
                    del_errs[idx] = None;
                    del_objects[idx] = dels[idx].clone();
                    break;
                }

                if del_errs[idx].is_none() {
                    del_errs[idx] = errs[idx].clone();
                    del_objects[idx] = dels[idx].clone();
                }
            }
        }

        del_objects.iter_mut().for_each(|v| {
            v.object_name = decode_dir_object(&v.object_name);
        });

        (del_objects, del_errs)

        // let mut futures = Vec::with_capacity(objects.len());

        // for obj in objects.iter() {
        //     futures.push(async move {
        //         self.internal_get_pool_info_existing_with_opts(
        //             bucket,
        //             &obj.object_name,
        //             &ObjectOptions {
        //                 no_lock: true,
        //                 ..Default::default()
        //             },
        //         )
        //         .await
        //     });
        // }

        // let results = join_all(futures).await;

        // // let mut jhs = Vec::new();
        // // let semaphore = Arc::new(Semaphore::new(num_cpus::get()));
        // // let pools = Arc::new(self.pools.clone());

        // // for obj in objects.iter() {
        // //     let (semaphore, pools, bucket, object_name, opt) = (
        // //         semaphore.clone(),
        // //         pools.clone(),
        // //         bucket.to_string(),
        // //         obj.object_name.to_string(),
        // //         ObjectOptions::default(),
        // //     );

        // //     let jh = tokio::spawn(async move {
        // //         let _permit = semaphore.acquire().await.unwrap();
        // //         self.internal_get_pool_info_existing_with_opts(pools.as_ref(), &bucket, &object_name, &opt)
        // //             .await
        // //     });
        // //     jhs.push(jh);
        // // }
        // // let mut results = Vec::new();
        // // for jh in jhs {
        // //     results.push(jh.await.unwrap());
        // // }

        // // Record the mapping pool_idx -> object index
        // let mut pool_obj_idx_map = HashMap::new();
        // let mut orig_index_map = HashMap::new();

        // for (i, res) in results.into_iter().enumerate() {
        //     match res {
        //         Ok((pinfo, _)) => {
        //             if let Some(obj) = objects.get(i) {
        //                 if pinfo.object_info.delete_marker && obj.version_id.is_none() {
        //                     del_objects[i] = DeletedObject {
        //                         delete_marker: pinfo.object_info.delete_marker,
        //                         delete_marker_version_id: pinfo.object_info.version_id.map(|v| v.to_string()),
        //                         object_name: decode_dir_object(&pinfo.object_info.name),
        //                         delete_marker_mtime: pinfo.object_info.mod_time,
        //                         ..Default::default()
        //                     };
        //                     continue;
        //                 }

        //                 if !pool_obj_idx_map.contains_key(&pinfo.index) {
        //                     pool_obj_idx_map.insert(pinfo.index, vec![obj.clone()]);
        //                 } else if let Some(val) = pool_obj_idx_map.get_mut(&pinfo.index) {
        //                     val.push(obj.clone());
        //                 }

        //                 if !orig_index_map.contains_key(&pinfo.index) {
        //                     orig_index_map.insert(pinfo.index, vec![i]);
        //                 } else if let Some(val) = orig_index_map.get_mut(&pinfo.index) {
        //                     val.push(i);
        //                 }
        //             }
        //         }
        //         Err(e) => {
        //             if !is_err_object_not_found(&e) && is_err_version_not_found(&e) {
        //                 del_errs[i] = Some(e)
        //             }

        //             if let Some(obj) = objects.get(i) {
        //                 del_objects[i] = DeletedObject {
        //                     object_name: decode_dir_object(&obj.object_name),
        //                     version_id: obj.version_id.map(|v| v.to_string()),
        //                     ..Default::default()
        //                 }
        //             }
        //         }
        //     }
        // }

        // if !pool_obj_idx_map.is_empty() {
        //     for (i, sets) in self.pools.iter().enumerate() {
        //         // Retrieve the object index for a pool idx
        //         if let Some(objs) = pool_obj_idx_map.get(&i) {
        //             // Fetch the corresponding object (should never be None)
        //             // let objs: Vec<ObjectToDelete> = obj_idxs.iter().filter_map(|&idx| objects.get(idx).cloned()).collect();

        //             if objs.is_empty() {
        //                 continue;
        //             }

        //             let (pdel_objs, perrs) = sets.delete_objects(bucket, objs.clone(), opts.clone()).await?;

        //             // Insert simultaneously (should never be None)
        //             let org_indexes = orig_index_map.get(&i).unwrap();

        //             // perrs should follow the same order as obj_idxs
        //             for (i, err) in perrs.into_iter().enumerate() {
        //                 let obj_idx = org_indexes[i];

        //                 if err.is_some() {
        //                     del_errs[obj_idx] = err;
        //                 }

        //                 let mut dobj = pdel_objs.get(i).unwrap().clone();
        //                 dobj.object_name = decode_dir_object(&dobj.object_name);

        //                 del_objects[obj_idx] = dobj;
        //             }
        //         }
        //     }
        // }

        // Ok((del_objects, del_errs))
    }

    #[instrument(skip(self))]
    pub(super) async fn handle_add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
        let object = encode_dir_object(object);

        if self.single_pool() {
            let _ = self.pools[0].add_partial(bucket, object.as_str(), version_id).await;
            return Ok(());
        }

        let opts = ObjectOptions {
            version_id: Some(version_id.to_string()),
            ..Default::default()
        };
        let (_, idx) = self
            .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &opts)
            .await?;

        let _ = self.pools[idx].add_partial(bucket, object.as_str(), version_id).await;
        Ok(())
    }

    #[instrument(skip(self))]
    pub(super) async fn handle_transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
        let object = encode_dir_object(object);
        if self.single_pool() {
            return self.pools[0].transition_object(bucket, &object, opts).await;
        }

        let opts = transition_restore_pool_opts(opts);
        let (_, idx) = self
            .get_latest_accessible_object_info_with_idx(bucket, &object, &opts)
            .await?;

        self.pools[idx].transition_object(bucket, &object, &opts).await
    }

    #[instrument(skip(self))]
    pub(super) async fn handle_restore_transitioned_object(
        self: Arc<Self>,
        bucket: &str,
        object: &str,
        opts: &ObjectOptions,
    ) -> Result<()> {
        let object = encode_dir_object(object);
        let mut opts = transition_restore_pool_opts(opts);
        if !is_meta_bucketname(bucket) && opts.expected_bucket_incarnation_id.is_none() {
            opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(bucket).await?);
        }
        let bucket_lifecycle_guard = if is_meta_bucketname(bucket) {
            None
        } else {
            Some(self.acquire_bucket_lifecycle_read_lock(bucket).await?)
        };
        if let Some(guard) = bucket_lifecycle_guard.as_ref() {
            opts.add_bucket_lifecycle_lock_guard(guard);
        }
        if !is_meta_bucketname(bucket) {
            let current_incarnation_id = get_bucket_incarnation_id_in(&self.ctx, bucket).await?;
            if opts.expected_bucket_incarnation_id != Some(current_incarnation_id) {
                return Err(StorageError::BucketNotFound(bucket.to_string()));
            }
        }
        if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
            let expected_incarnation_id = opts
                .expected_bucket_incarnation_id
                .ok_or_else(|| Error::other("restore is missing its bucket incarnation snapshot"))?;
            let lifecycle_fence = opts
                .bucket_lifecycle_lock_fence
                .as_ref()
                .ok_or_else(|| Error::other("restore is missing its bucket lifecycle fence"))?;
            let snapshot = match opts.object_lock_config_snapshot.as_ref() {
                Some(snapshot) => Arc::clone(snapshot),
                None => {
                    self.object_lock_config_snapshot_under_lifecycle_fence(bucket, lifecycle_fence)
                        .await?
                }
            };
            if !snapshot.is_valid_for_destructive_put(self.id, bucket, expected_incarnation_id) {
                return Err(Error::other("restore Object Lock snapshot does not match the target bucket generation"));
            }
            snapshot.add_lock_fences(&mut opts);
            opts.object_lock_config_snapshot = Some(snapshot);
        }
        // Deliberately NOT holding the object write lock across the tier
        // copy-back (backlog#1304): non-SELECT restore-vs-restore is
        // serialized by the accept path's compare-and-set of the ongoing flag
        // (see acquire_restore_accept_guard), and while ongoing-request="true"
        // the restore header parses with no expiry, so DeleteRestoredAction
        // cannot fire mid-copy-back. Torn-write protection against concurrent
        // readers and writers stays with the inner put_object /
        // complete_multipart_upload commit phases, which take this object's
        // write lock themselves. A delete (user or lifecycle) landing between
        // the tier read and that commit can still be overwritten by the
        // commit — the same window MinIO accepts; a commit-time existence
        // re-check is tracked separately. Holding the lock here instead
        // (#4877) blocked HEAD/get_object_info for the whole copy-back and
        // self-deadlocked on the inner commits.
        if self.single_pool() {
            return self.pools[0]
                .clone()
                .restore_transitioned_object(bucket, &object, &opts)
                .await;
        }

        let (_, idx) = self
            .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &opts)
            .await?;

        self.pools[idx]
            .clone()
            .restore_transitioned_object(bucket, &object, &opts)
            .await
    }

    #[instrument(skip(self))]
    pub(super) async fn handle_put_object_metadata(
        &self,
        bucket: &str,
        object: &str,
        opts: &ObjectOptions,
    ) -> Result<ObjectInfo> {
        let object = encode_dir_object(object);
        let mut opts = opts.clone();
        opts.metadata_chg = true;
        let bucket_lifecycle_guard = if is_meta_bucketname(bucket) {
            None
        } else {
            let guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
            let current_incarnation_id = get_bucket_incarnation_id_in(&self.ctx, bucket).await?;
            if opts
                .expected_bucket_incarnation_id
                .is_some_and(|expected| expected != current_incarnation_id)
            {
                return Err(StorageError::BucketNotFound(bucket.to_string()));
            }
            opts.expected_bucket_incarnation_id = Some(current_incarnation_id);
            opts.add_bucket_lifecycle_lock_guard(&guard);
            if guard.is_lock_lost() {
                return Err(Error::other("bucket lifecycle lock was lost before the metadata update"));
            }
            Some(guard)
        };

        if self.single_pool() {
            return self.pools[0].put_object_metadata(bucket, object.as_str(), &opts).await;
        }

        let (_, idx) = self
            .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &opts)
            .await?;

        let result = self.pools[idx].put_object_metadata(bucket, object.as_str(), &opts).await;
        drop(bucket_lifecycle_guard);
        result
    }

    #[instrument(skip(self))]
    pub(super) async fn handle_get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<String> {
        let object = encode_dir_object(object);

        if self.single_pool() {
            return self.pools[0].get_object_tags(bucket, object.as_str(), opts).await;
        }

        let (oi, _) = self.get_latest_accessible_object_info_with_idx(bucket, &object, opts).await?;
        Ok((*oi.user_tags).clone())
    }

    #[instrument(level = "debug", skip(self))]
    pub(super) async fn handle_put_object_tags(
        &self,
        bucket: &str,
        object: &str,
        tags: &str,
        opts: &ObjectOptions,
    ) -> Result<ObjectInfo> {
        let object = encode_dir_object(object);

        if self.single_pool() {
            return self.pools[0].put_object_tags(bucket, object.as_str(), tags, opts).await;
        }

        let (_, idx) = self
            .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), opts)
            .await?;

        self.pools[idx].put_object_tags(bucket, object.as_str(), tags, opts).await
    }

    #[instrument(skip(self))]
    pub(super) async fn handle_delete_object_version(
        &self,
        bucket: &str,
        object: &str,
        fi: &FileInfo,
        force_del_marker: bool,
    ) -> Result<()> {
        check_del_obj_args(bucket, object)?;

        let object = encode_dir_object(object);

        if self.single_pool() {
            return self.pools[0]
                .delete_object_version(bucket, object.as_str(), fi, force_del_marker)
                .await;
        }
        Err(StorageError::NotImplemented)
    }

    #[instrument(skip(self))]
    pub(super) async fn handle_delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
        let object = encode_dir_object(object);

        if self.single_pool() {
            return self.pools[0].delete_object_tags(bucket, object.as_str(), opts).await;
        }

        let (_, idx) = self
            .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), opts)
            .await?;

        self.pools[idx].delete_object_tags(bucket, object.as_str(), opts).await
    }

    pub(super) async fn handle_verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
        let get_object_reader = <Self as crate::storage_api_contracts::object::ObjectIO>::get_object_reader(
            self,
            bucket,
            object,
            None,
            HeaderMap::new(),
            opts,
        )
        .await?;
        // Stream to sink to avoid loading entire object into memory during verification
        let mut reader = get_object_reader.stream;
        tokio::io::copy(&mut reader, &mut tokio::io::sink()).await?;
        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::bucket::lifecycle::core::TRANSITION_COMPLETE;
    use crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState;
    use crate::bucket::replication::{
        ReplicationState, ReplicationStatusType, VersionPurgeStatusType, replication_state_to_filemeta, replication_statuses_map,
        version_purge_statuses_map,
    };
    use crate::ecstore_validation_blackbox::make_local_set_disks;
    use crate::layout::{
        endpoints::{Endpoints, PoolEndpoints},
        format::FormatV3,
    };
    use crate::object_api::{
        GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectBodySource, clear_get_object_body_cache_hook,
        lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
    };
    use crate::set_disk::SetDisks;
    use crate::storage_api_contracts::bucket::MakeBucketOptions;
    use crate::storage_api_contracts::lifecycle::TransitionedObject;
    use bytes::Bytes;
    use std::io::Cursor;
    use std::sync::Arc;
    use std::sync::atomic::{AtomicUsize, Ordering};
    use tokio::io::AsyncReadExt;

    struct CountingMissHook {
        calls: AtomicUsize,
    }

    #[async_trait::async_trait]
    impl GetObjectBodyCacheHook for CountingMissHook {
        async fn lookup(&self, _bucket: &str, _object: &str, _info: &ObjectInfo) -> Option<Bytes> {
            self.calls.fetch_add(1, Ordering::Relaxed);
            None
        }
    }

    struct BodyCacheHookGuard;

    #[test]
    fn tier_delete_entry_is_prepared_and_bound_to_source_generation() {
        let identity = [9_u8; 32];
        let mut metadata = HashMap::new();
        rustfs_utils::http::metadata_compat::insert_str(
            &mut metadata,
            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
            rustfs_utils::crypto::hex(identity),
        );
        let version_id = Uuid::from_u128(1);
        let data_dir = Uuid::from_u128(2);
        let source = ObjectInfo {
            bucket: "bucket".to_string(),
            name: "object".to_string(),
            version_id: Some(version_id),
            data_dir: Some(data_dir),
            user_defined: Arc::new(metadata),
            transitioned_object: TransitionedObject {
                name: "remote/object".to_string(),
                version_id: "remote-version".to_string(),
                tier: "WARM".to_string(),
                status: TRANSITION_COMPLETE.to_string(),
                ..Default::default()
            },
            transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
            ..Default::default()
        };
        let entry = build_tier_delete_journal_entry(
            "bucket",
            "object",
            &ObjectOptions {
                version_id: Some(version_id.to_string()),
                versioned: true,
                ..Default::default()
            },
            &source,
        )
        .expect("transition source should produce a journal entry")
        .expect("completed transition should be journaled");

        assert_eq!(entry.state, TierDeleteJournalState::Prepared);
        assert_eq!(entry.backend_identity, Some(identity));
        let data_dir_string = data_dir.to_string();
        assert_eq!(
            entry.source.as_ref().and_then(|source| source.data_dir.as_deref()),
            Some(data_dir_string.as_str())
        );
    }

    impl Drop for BodyCacheHookGuard {
        fn drop(&mut self) {
            clear_get_object_body_cache_hook();
        }
    }

    #[test]
    fn delete_marker_data_movement_falls_back_when_only_source_pool_has_object() {
        let target = select_data_movement_target_pool(Ok(1), 1, true).unwrap();
        assert_eq!(target, None);
    }

    #[test]
    fn delete_marker_data_movement_falls_back_when_version_does_not_exist_yet() {
        let err = StorageError::ObjectNotFound("bucket".to_string(), "object".to_string());
        let target = select_data_movement_target_pool(Err(err), 1, true).unwrap();
        assert_eq!(target, None);
    }

    #[test]
    fn non_delete_marker_data_movement_keeps_existing_pool() {
        let target = select_data_movement_target_pool(Ok(0), 1, false).unwrap();
        assert_eq!(target, Some(0));
    }

    #[test]
    fn equivalent_data_movement_delete_marker_requires_same_version_and_mod_time() {
        let version_id = Uuid::nil();
        let mod_time = OffsetDateTime::UNIX_EPOCH;
        let source = ObjectInfo {
            version_id: Some(version_id),
            delete_marker: true,
            mod_time: Some(mod_time),
            ..Default::default()
        };
        let target = source.clone();

        assert!(is_equivalent_data_movement_delete_marker(&source, &target));

        let mismatched = ObjectInfo {
            mod_time: Some(mod_time + Duration::from_secs(1)),
            ..target
        };
        assert!(!is_equivalent_data_movement_delete_marker(&source, &mismatched));
    }

    #[test]
    fn equivalent_data_movement_delete_marker_rejects_metadata_and_replication_mismatch() {
        let version_id = Uuid::nil();
        let mod_time = OffsetDateTime::UNIX_EPOCH;
        let source = ObjectInfo {
            version_id: Some(version_id),
            delete_marker: true,
            mod_time: Some(mod_time),
            user_defined: Arc::new(HashMap::from([("x-amz-meta-source".to_string(), "true".to_string())])),
            replication_status_internal: Some("arn:minio:replication:target=COMPLETED;".to_string()),
            version_purge_status_internal: Some("arn:minio:replication:target=PENDING;".to_string()),
            ..Default::default()
        };

        let mut target = source.clone();
        target.user_defined = Arc::new(HashMap::from([("x-amz-meta-source".to_string(), "false".to_string())]));
        assert!(!is_equivalent_data_movement_delete_marker(&source, &target));

        let mut target = source.clone();
        target.replication_status_internal = Some("arn:minio:replication:target=FAILED;".to_string());
        assert!(!is_equivalent_data_movement_delete_marker(&source, &target));

        let mut target = source.clone();
        target.version_purge_status_internal = Some("arn:minio:replication:target=COMPLETE;".to_string());
        assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
    }

    #[test]
    fn equivalent_data_movement_delete_marker_rejects_live_object() {
        let source = ObjectInfo {
            delete_marker: true,
            ..Default::default()
        };
        let target = ObjectInfo {
            delete_marker: false,
            ..source.clone()
        };

        assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
    }

    #[test]
    fn data_movement_delete_marker_resume_accepts_equivalent_target() {
        let source = ObjectInfo {
            version_id: Some(Uuid::nil()),
            delete_marker: true,
            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
            ..Default::default()
        };

        let should_resume = resolve_data_movement_delete_marker_resume_result(Ok(Some(source.clone())), &source, 0, 1)
            .expect("equivalent delete marker target should be evaluated");

        assert!(should_resume);
    }

    #[test]
    fn data_movement_delete_marker_resume_rejects_source_pool_target() {
        let source = ObjectInfo {
            version_id: Some(Uuid::nil()),
            delete_marker: true,
            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
            ..Default::default()
        };

        let should_resume = resolve_data_movement_delete_marker_resume_result(Ok(Some(source.clone())), &source, 0, 0)
            .expect("source-pool target should be rejected before target lookup");

        assert!(!should_resume);
    }

    #[test]
    fn data_movement_resume_target_prefers_selected_non_source_pool() {
        let target_pool_idx = resolve_data_movement_resume_target_pool(2, Some(3), 1);
        assert_eq!(target_pool_idx, 2);
    }

    #[test]
    fn data_movement_resume_target_uses_resolved_non_source_pool_when_selected_is_source() {
        let target_pool_idx = resolve_data_movement_resume_target_pool(1, Some(3), 1);
        assert_eq!(target_pool_idx, 3);
        assert!(should_check_data_movement_resume_target(1, target_pool_idx));
    }

    #[test]
    fn data_movement_resume_target_keeps_source_when_no_other_pool_is_available() {
        let target_pool_idx = resolve_data_movement_resume_target_pool(1, None, 1);
        assert_eq!(target_pool_idx, 1);
    }

    #[test]
    fn data_movement_delete_marker_resume_propagates_target_lookup_error() {
        let source = ObjectInfo {
            delete_marker: true,
            ..Default::default()
        };
        let result = resolve_data_movement_delete_marker_resume_result(Err(Error::SlowDown), &source, 0, 1);

        assert!(matches!(result, Err(Error::SlowDown)));
    }

    fn tiered_equivalence_source() -> FileInfo {
        let version_id = Uuid::nil();
        let transition_version_id = Uuid::new_v4();
        let mod_time = OffsetDateTime::UNIX_EPOCH;

        FileInfo {
            version_id: Some(version_id),
            size: 1024,
            mod_time: Some(mod_time),
            checksum: Some(Bytes::from_static(b"checksum")),
            transition_status: TRANSITION_COMPLETE.to_string(),
            transitioned_objname: "remote/object".to_string(),
            transition_tier: "WARM".to_string(),
            transition_version_id: Some(transition_version_id),
            replication_state_internal: Some(replication_state_to_filemeta(&ReplicationState {
                replication_status_internal: Some("arn:minio:replication:target=COMPLETED;".to_string()),
                targets: replication_statuses_map("arn:minio:replication:target=COMPLETED;"),
                version_purge_status_internal: Some("arn:minio:replication:target=PENDING;".to_string()),
                purge_targets: version_purge_statuses_map("arn:minio:replication:target=PENDING;"),
                ..Default::default()
            })),
            metadata: HashMap::from([
                ("etag".to_string(), "etag-value".to_string()),
                ("x-amz-meta-key".to_string(), "metadata-value".to_string()),
                (rustfs_utils::http::AMZ_OBJECT_TAGGING.to_string(), "tag=value".to_string()),
                ("expires".to_string(), "1970-01-01T00:33:20Z".to_string()),
            ]),
            ..Default::default()
        }
    }

    fn tiered_equivalence_target(source: &FileInfo) -> ObjectInfo {
        ObjectInfo::from_file_info(source, "bucket", "object", source.version_id.is_some())
    }

    #[test]
    fn equivalent_data_movement_tiered_object_accepts_matching_persisted_metadata() {
        let source = tiered_equivalence_source();
        let target = tiered_equivalence_target(&source);

        assert!(is_equivalent_data_movement_tiered_object(&source, &target));
    }

    #[test]
    fn equivalent_data_movement_tiered_object_rejects_transition_mismatch() {
        let source = tiered_equivalence_source();
        let mut target = tiered_equivalence_target(&source);
        target.transitioned_object.name = "remote/target".to_string();

        assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
    }

    #[test]
    fn equivalent_data_movement_tiered_object_rejects_user_metadata_mismatch() {
        let source = tiered_equivalence_source();
        let mut target = tiered_equivalence_target(&source);
        target.user_defined = Arc::new(HashMap::from([("x-amz-meta-key".to_string(), "target-value".to_string())]));

        assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
    }

    #[test]
    fn equivalent_data_movement_tiered_object_rejects_tag_mismatch() {
        let source = tiered_equivalence_source();
        let mut target = tiered_equivalence_target(&source);
        target.user_tags = Arc::new("tag=target".to_string());

        assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
    }

    #[test]
    fn equivalent_data_movement_tiered_object_rejects_replication_mismatch() {
        let source = tiered_equivalence_source();
        let mut target = tiered_equivalence_target(&source);
        target.replication_status_internal = Some("arn:minio:replication:target=FAILED;".to_string());
        target.replication_status = ReplicationStatusType::Failed;

        assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
    }

    #[test]
    fn equivalent_data_movement_tiered_object_rejects_version_purge_mismatch() {
        let source = tiered_equivalence_source();
        let mut target = tiered_equivalence_target(&source);
        target.version_purge_status_internal = Some("arn:minio:replication:target=COMPLETE;".to_string());
        target.version_purge_status = VersionPurgeStatusType::Complete;

        assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
    }

    #[test]
    fn data_movement_tiered_resume_accepts_equivalent_target() {
        let source = tiered_equivalence_source();
        let target = tiered_equivalence_target(&source);

        let should_resume = resolve_data_movement_tiered_resume_result(Ok(Some(target)), &source, 0, 1)
            .expect("equivalent tiered target should be evaluated");

        assert!(should_resume);
    }

    #[test]
    fn data_movement_tiered_resume_rejects_source_pool_target() {
        let source = tiered_equivalence_source();
        let target = tiered_equivalence_target(&source);

        let should_resume = resolve_data_movement_tiered_resume_result(Ok(Some(target)), &source, 0, 0)
            .expect("source-pool target should be rejected before target lookup");

        assert!(!should_resume);
    }

    #[test]
    fn data_movement_tiered_resume_rejects_missing_target() {
        let source = FileInfo {
            version_id: Some(Uuid::nil()),
            size: 1024,
            ..Default::default()
        };

        let should_resume = resolve_data_movement_tiered_resume_result(Ok(None), &source, 0, 1)
            .expect("missing tiered target should be evaluated");

        assert!(!should_resume);
    }

    #[test]
    fn latest_object_access_delete_marker_error_returns_none_for_live_object() {
        let info = ObjectInfo::default();
        let opts = ObjectOptions::default();

        assert!(latest_object_access_delete_marker_error("bucket", "object", &info, &opts).is_none());
    }

    #[test]
    fn latest_object_access_delete_marker_error_returns_not_found_without_version_id() {
        let info = ObjectInfo {
            delete_marker: true,
            ..Default::default()
        };
        let opts = ObjectOptions::default();

        let err = latest_object_access_delete_marker_error("bucket", "object", &info, &opts)
            .expect("delete marker should stop latest-object reads");

        assert!(crate::error::is_err_object_not_found(&err));
    }

    #[test]
    fn latest_object_access_delete_marker_error_returns_method_not_allowed_for_version_read() {
        let info = ObjectInfo {
            delete_marker: true,
            ..Default::default()
        };
        let opts = ObjectOptions {
            version_id: Some("vid-1".to_string()),
            ..Default::default()
        };

        let err = latest_object_access_delete_marker_error("bucket", "object", &info, &opts)
            .expect("delete marker version reads should be rejected");

        assert!(matches!(err, Error::MethodNotAllowed));
    }

    #[test]
    fn latest_object_access_delete_marker_error_returns_not_found_for_delete_marker_lookup() {
        let info = ObjectInfo {
            delete_marker: true,
            ..Default::default()
        };
        let opts = ObjectOptions {
            version_id: Some("vid-1".to_string()),
            delete_marker: true,
            ..Default::default()
        };

        let err = latest_object_access_delete_marker_error("bucket", "object", &info, &opts)
            .expect("delete marker lookup should keep not-found semantics");

        assert!(crate::error::is_err_object_not_found(&err));
    }

    #[test]
    fn resolve_latest_object_access_returns_live_object_and_pool_idx() {
        let info = ObjectInfo::default();
        let opts = ObjectOptions::default();

        let (resolved, idx) = resolve_latest_object_access("bucket", "object", info, 7, &opts).unwrap();

        assert_eq!(idx, 7);
        assert!(!resolved.delete_marker);
    }

    #[test]
    fn resolve_latest_object_access_rejects_delete_marker_without_version_id() {
        let info = ObjectInfo {
            delete_marker: true,
            ..Default::default()
        };
        let opts = ObjectOptions::default();

        let err = resolve_latest_object_access("bucket", "object", info, 2, &opts).unwrap_err();

        assert!(crate::error::is_err_object_not_found(&err));
    }

    #[test]
    fn resolve_latest_object_access_rejects_delete_marker_version_read() {
        let info = ObjectInfo {
            delete_marker: true,
            ..Default::default()
        };
        let opts = ObjectOptions {
            version_id: Some("vid-1".to_string()),
            ..Default::default()
        };

        let err = resolve_latest_object_access("bucket", "object", info, 2, &opts).unwrap_err();

        assert!(matches!(err, Error::MethodNotAllowed));
    }

    #[test]
    fn should_create_delete_marker_for_missing_object_allows_latest_versioned_delete() {
        let opts = ObjectOptions {
            versioned: true,
            ..Default::default()
        };

        assert!(should_create_delete_marker_for_missing_object(&opts));
    }

    #[test]
    fn should_create_delete_marker_for_missing_object_rejects_specialized_deletes() {
        let version_delete = ObjectOptions {
            versioned: true,
            version_id: Some("vid-1".to_string()),
            ..Default::default()
        };
        let delete_marker_replication = ObjectOptions {
            versioned: true,
            delete_marker: true,
            ..Default::default()
        };
        let data_movement = ObjectOptions {
            versioned: true,
            data_movement: true,
            ..Default::default()
        };

        assert!(!should_create_delete_marker_for_missing_object(&version_delete));
        assert!(!should_create_delete_marker_for_missing_object(&delete_marker_replication));
        assert!(!should_create_delete_marker_for_missing_object(&data_movement));
    }

    // issue #4189 regression: `del_opts` pins `version_id = Uuid::nil()` on directory
    // keys, so deleting a ghost folder over HTTP fails the lookup with *version*-not-found
    // (not object-not-found). The orphan-purge guard must accept both misses, or the
    // ghost tree survives behind a fake 204 — the exact reported symptom.
    #[test]
    fn should_purge_orphan_dir_on_version_not_found_for_dir_key() {
        assert!(
            should_purge_orphan_dir_on_missing(&StorageError::FileVersionNotFound, "ghost/"),
            "the real HTTP delete path yields version-not-found on dir keys and must reach the purge"
        );
        assert!(
            should_purge_orphan_dir_on_missing(
                &StorageError::VersionNotFound("bucket".into(), "ghost/".into(), Uuid::nil().to_string()),
                "ghost/"
            ),
            "typed VersionNotFound on a dir key must also reach the purge"
        );
    }

    #[test]
    fn should_purge_orphan_dir_on_object_not_found_for_dir_key() {
        assert!(should_purge_orphan_dir_on_missing(&StorageError::FileNotFound, "ghost/"));
        assert!(should_purge_orphan_dir_on_missing(
            &StorageError::ObjectNotFound("bucket".into(), "ghost/".into()),
            "ghost/"
        ));
    }

    #[test]
    fn should_not_purge_orphan_dir_for_regular_key_or_other_errors() {
        // A regular (non-directory) key must never trigger a prefix purge, even on a miss.
        assert!(!should_purge_orphan_dir_on_missing(&StorageError::FileVersionNotFound, "regular.txt"));
        assert!(!should_purge_orphan_dir_on_missing(&StorageError::FileNotFound, "regular.txt"));
        // Non-miss errors (e.g. quorum failures) must not be masked by a purge attempt.
        assert!(!should_purge_orphan_dir_on_missing(&StorageError::ErasureReadQuorum, "ghost/"));
    }

    #[test]
    fn resolve_decommission_target_pool_idx_result_passthrough_ok() {
        let idx = ECStore::resolve_decommission_target_pool_idx_result(Ok(3), "bucket", "object").unwrap();

        assert_eq!(idx, 3);
    }

    #[test]
    fn resolve_decommission_target_pool_idx_result_wraps_error_context() {
        let err = ECStore::resolve_decommission_target_pool_idx_result(Err(Error::other("boom")), "bucket", "object")
            .expect_err("expected contextual error");
        let rendered = err.to_string();

        assert!(rendered.contains("failed to select decommission target pool"), "{rendered}");
        assert!(rendered.contains("bucket"), "{rendered}");
        assert!(rendered.contains("object"), "{rendered}");
        assert!(rendered.contains("boom"), "{rendered}");
    }

    #[test]
    fn resolve_decommission_tiered_object_result_passthrough_ok() {
        ECStore::resolve_decommission_tiered_object_result(Ok(()), "bucket", "object")
            .expect("successful decommission result should pass through");
    }

    #[test]
    fn resolve_decommission_tiered_object_result_wraps_error_context() {
        let err = ECStore::resolve_decommission_tiered_object_result(Err(Error::other("boom")), "bucket", "object")
            .expect_err("expected contextual error");
        let rendered = err.to_string();

        assert!(rendered.contains("failed to decommission tiered object"), "{rendered}");
        assert!(rendered.contains("bucket"), "{rendered}");
        assert!(rendered.contains("object"), "{rendered}");
        assert!(rendered.contains("boom"), "{rendered}");
    }

    #[test]
    fn version_aware_lookup_opts_enables_version_aware_lookup() {
        let opts = ObjectOptions {
            version_id: Some("vid-1".to_string()),
            ..Default::default()
        };

        let lookup_opts = version_aware_lookup_opts(&opts, true);

        assert!(lookup_opts.no_lock);
        assert!(lookup_opts.metadata_chg);
        assert_eq!(lookup_opts.version_id.as_deref(), Some("vid-1"));
    }

    #[test]
    fn version_aware_lookup_opts_keeps_latest_lookup_for_unversioned_requests() {
        let lookup_opts = version_aware_lookup_opts(&ObjectOptions::default(), true);

        assert!(lookup_opts.no_lock);
        assert!(!lookup_opts.metadata_chg);
        assert!(lookup_opts.version_id.is_none());
    }

    #[test]
    fn data_movement_pool_lookup_opts_enables_version_aware_lookup_and_skip_flags() {
        let opts = ObjectOptions {
            version_id: Some("vid-1".to_string()),
            ..Default::default()
        };

        let lookup_opts = data_movement_pool_lookup_opts(&opts, false);

        assert!(!lookup_opts.no_lock);
        assert!(lookup_opts.metadata_chg);
        assert!(lookup_opts.skip_decommissioned);
        assert!(lookup_opts.skip_rebalancing);
        assert_eq!(lookup_opts.version_id.as_deref(), Some("vid-1"));
    }

    #[test]
    fn data_movement_pool_lookup_opts_keeps_no_lock_for_tiered_moves() {
        let lookup_opts = data_movement_pool_lookup_opts(
            &ObjectOptions {
                version_id: Some("vid-1".to_string()),
                ..Default::default()
            },
            true,
        );

        assert!(lookup_opts.no_lock);
        assert!(lookup_opts.metadata_chg);
        assert!(lookup_opts.skip_decommissioned);
        assert!(lookup_opts.skip_rebalancing);
    }

    #[test]
    fn transition_restore_pool_opts_skips_decommissioned_and_preserves_locking() {
        let lookup_opts = transition_restore_pool_opts(&ObjectOptions {
            no_lock: false,
            skip_decommissioned: false,
            ..Default::default()
        });

        assert!(lookup_opts.skip_decommissioned);
        assert!(!lookup_opts.no_lock);
    }

    #[test]
    fn transition_restore_pool_opts_preserves_existing_no_lock() {
        let lookup_opts = transition_restore_pool_opts(&ObjectOptions {
            no_lock: true,
            ..Default::default()
        });

        assert!(lookup_opts.skip_decommissioned);
        assert!(lookup_opts.no_lock);
    }

    #[test]
    fn delete_objects_lock_names_are_sorted_and_unique() {
        let objects = vec![
            ObjectToDelete {
                object_name: "beta".to_string(),
                ..Default::default()
            },
            ObjectToDelete {
                object_name: "alpha".to_string(),
                ..Default::default()
            },
            ObjectToDelete {
                object_name: "beta".to_string(),
                ..Default::default()
            },
        ];

        assert_eq!(sorted_unique_delete_object_names(&objects), vec!["alpha", "beta"]);
    }

    async fn new_read_lock_test_store() -> ECStore {
        let format = FormatV3::new(1, 2);
        let endpoints = vec![
            Endpoint::try_from("http://127.0.0.1:9000/data0").expect("first endpoint should parse"),
            Endpoint::try_from("http://127.0.0.1:9001/data1").expect("second endpoint should parse"),
        ];
        let pool_endpoints = PoolEndpoints {
            legacy: false,
            set_count: 1,
            drives_per_set: 2,
            endpoints: Endpoints::from(endpoints),
            cmd_line: "read-lock-metadata-cache-safe-test".to_string(),
            platform: "test".to_string(),
        };
        let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints.clone()]);
        let sets = Sets::new(vec![None, None], &pool_endpoints, &format, 0, 1)
            .await
            .expect("test sets should be created with empty disks");

        ECStore {
            id: Uuid::new_v4(),
            disk_map: HashMap::new(),
            pools: vec![sets],
            peer_sys: S3PeerSys::new(&endpoint_pools),
            pool_meta: RwLock::new(PoolMeta::default()),
            rebalance_meta: RwLock::new(None),
            decommission_cancelers: RwLock::new(Vec::new()),
            start_gate: Mutex::new(()),
            pool_meta_save_gate: Mutex::new(()),
            ctx: crate::runtime::instance::bootstrap_ctx(),
        }
    }

    async fn new_prepared_reader_test_store(set_disks: &[Arc<SetDisks>]) -> ECStore {
        let mut pool_configs = Vec::with_capacity(set_disks.len());
        let mut pools = Vec::with_capacity(set_disks.len());

        for (pool_idx, set_disks) in set_disks.iter().enumerate() {
            let mut endpoints = Endpoints::from(set_disks.set_endpoints.clone());
            for endpoint in endpoints.as_mut() {
                endpoint.set_pool_index(pool_idx);
            }
            let pool_config = PoolEndpoints {
                legacy: false,
                set_count: 1,
                drives_per_set: set_disks.set_drive_count,
                endpoints,
                cmd_line: format!("prepared-reader-test-pool-{pool_idx}"),
                platform: "test".to_string(),
            };
            let disks = set_disks.disks.read().await.clone();
            let pool = Sets::new(disks, &pool_config, &set_disks.format, pool_idx, set_disks.default_parity_count)
                .await
                .expect("prepared-reader test pool should be created from local disks");
            pool_configs.push(pool_config);
            pools.push(pool);
        }

        let endpoint_pools = EndpointServerPools::from(pool_configs);
        ECStore {
            id: Uuid::new_v4(),
            disk_map: HashMap::new(),
            pools,
            peer_sys: S3PeerSys::new(&endpoint_pools),
            pool_meta: RwLock::new(PoolMeta::default()),
            rebalance_meta: RwLock::new(None),
            decommission_cancelers: RwLock::new(Vec::new()),
            start_gate: Mutex::new(()),
            pool_meta_save_gate: Mutex::new(()),
            ctx: crate::runtime::instance::bootstrap_ctx(),
        }
    }

    async fn assert_prepared_reader_blocks_writer(store: &ECStore, bucket: &str, object: &str) {
        let manager = Arc::clone(store.pools[0].disk_set[0].local_lock_manager_for_test());
        let lock = rustfs_lock::NamespaceLock::with_local_manager("prepared-reader-writer".to_string(), manager);
        let err = lock
            .get_write_lock(rustfs_lock::ObjectKey::new(bucket, object), "competing-writer", Duration::from_millis(50))
            .await
            .expect_err("prepared read lock should block the writer");
        assert!(matches!(err, rustfs_lock::LockError::Timeout { .. }));
    }

    async fn acquire_prepared_reader_writer(store: &ECStore, bucket: &str, object: &str) -> rustfs_lock::NamespaceLockGuard {
        let manager = Arc::clone(store.pools[0].disk_set[0].local_lock_manager_for_test());
        let lock = rustfs_lock::NamespaceLock::with_local_manager("prepared-reader-writer".to_string(), manager);
        lock.get_write_lock(rustfs_lock::ObjectKey::new(bucket, object), "competing-writer", Duration::from_secs(1))
            .await
            .expect("prepared read lock should have been released")
    }

    #[tokio::test]
    #[serial_test::serial(body_cache_hook)]
    async fn prepared_reader_uses_authoritative_hook_miss_once_and_streams_full_body() {
        let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
        let store = new_prepared_reader_test_store(&[set_disks]).await;
        let bucket = "prepared-reader-hook-miss";
        let object = "object.bin";
        let payload = b"prepared-reader-hook-miss-payload-".repeat(40_000);
        let opts = ObjectOptions {
            no_lock: true,
            ..Default::default()
        };

        store.pools[0]
            .make_bucket(bucket, &MakeBucketOptions::default())
            .await
            .expect("bucket should be created");
        let mut put_reader = PutObjReader::from_vec(payload.clone());
        store.pools[0]
            .put_object(bucket, object, &mut put_reader, &opts)
            .await
            .expect("object should be written");

        clear_get_object_body_cache_hook();
        let hook = Arc::new(CountingMissHook {
            calls: AtomicUsize::new(0),
        });
        register_get_object_body_cache_hook(Arc::clone(&hook) as Arc<dyn GetObjectBodyCacheHook>);
        let _hook_guard = BodyCacheHookGuard;

        let prepared = store
            .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
            .await
            .expect("prepared reader metadata should resolve");
        assert!(matches!(
            lookup_get_object_body_cache_hook(bucket, object, &None, &opts, prepared.object_info()).await,
            GetObjectBodyCacheHookLookup::Miss
        ));
        assert_eq!(hook.calls.load(Ordering::Relaxed), 1, "the authoritative probe should call the hook once");

        let mut reader = prepared.into_reader().await.expect("prepared body reader should open");
        assert_eq!(reader.body_source, GetObjectBodySource::HookMissed);
        let mut restored = Vec::new();
        reader
            .stream
            .read_to_end(&mut restored)
            .await
            .expect("prepared body should stream");

        assert_eq!(restored, payload);
        assert_eq!(hook.calls.load(Ordering::Relaxed), 1, "reader construction must not probe the hook again");
    }

    #[tokio::test]
    #[serial_test::serial(body_cache_hook)]
    async fn prepared_reader_holds_namespace_lock_until_eof_or_drop() {
        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))], async {
            let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
            let store = new_prepared_reader_test_store(&[set_disks]).await;
            let bucket = "prepared-reader-lock-lifetime";
            let object = "object.bin";
            let payload = b"prepared-reader-lock-lifetime-payload-".repeat(40_000);
            let put_opts = ObjectOptions {
                no_lock: true,
                ..Default::default()
            };

            store.pools[0]
                .make_bucket(bucket, &MakeBucketOptions::default())
                .await
                .expect("bucket should be created");
            let mut put_reader = PutObjReader::from_vec(payload.clone());
            store.pools[0]
                .put_object(bucket, object, &mut put_reader, &put_opts)
                .await
                .expect("object should be written");

            let prepared = store
                .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
                .await
                .expect("prepared reader metadata should resolve");
            assert!(prepared.read_lock_guard.is_some());
            assert_prepared_reader_blocks_writer(&store, bucket, object).await;

            let mut reader = prepared.into_reader().await.expect("prepared body reader should open");
            assert_prepared_reader_blocks_writer(&store, bucket, object).await;
            let mut restored = Vec::new();
            reader
                .stream
                .read_to_end(&mut restored)
                .await
                .expect("prepared body should stream");
            assert_eq!(restored, payload);
            drop(acquire_prepared_reader_writer(&store, bucket, object).await);

            let prepared = store
                .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
                .await
                .expect("second prepared reader metadata should resolve");
            let reader = prepared.into_reader().await.expect("second prepared body reader should open");
            assert_prepared_reader_blocks_writer(&store, bucket, object).await;
            drop(reader);
            drop(acquire_prepared_reader_writer(&store, bucket, object).await);
        })
        .await;
    }

    #[tokio::test]
    #[serial_test::serial(body_cache_hook)]
    async fn prepared_object_info_releases_namespace_lock_immediately() {
        let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
        let store = new_prepared_reader_test_store(&[set_disks]).await;
        let bucket = "prepared-object-info-lock-release";
        let object = "object.bin";
        let payload = b"prepared-object-info-lock-release".to_vec();
        let put_opts = ObjectOptions {
            no_lock: true,
            ..Default::default()
        };

        store.pools[0]
            .make_bucket(bucket, &MakeBucketOptions::default())
            .await
            .expect("bucket should be created");
        let mut put_reader = PutObjReader::from_vec(payload.clone());
        store.pools[0]
            .put_object(bucket, object, &mut put_reader, &put_opts)
            .await
            .expect("object should be written");

        let prepared = store
            .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
            .await
            .expect("prepared reader metadata should resolve");
        assert_prepared_reader_blocks_writer(&store, bucket, object).await;
        assert_eq!(prepared.into_object_info().size, payload.len() as i64);

        drop(acquire_prepared_reader_writer(&store, bucket, object).await);
    }

    #[tokio::test]
    #[serial_test::serial(body_cache_hook)]
    async fn prepared_reader_resolves_object_from_second_pool() {
        let (_first_dirs, first_set) = make_local_set_disks(4, 2).await;
        let (_second_dirs, second_set) = make_local_set_disks(4, 2).await;
        let store = new_prepared_reader_test_store(&[first_set, second_set]).await;
        let bucket = "prepared-reader-second-pool";
        let object = "object.bin";
        let payload = b"prepared-reader-second-pool-payload-".repeat(40_000);
        let opts = ObjectOptions {
            no_lock: true,
            ..Default::default()
        };

        for pool in &store.pools {
            pool.make_bucket(bucket, &MakeBucketOptions::default())
                .await
                .expect("bucket should be created in each pool");
        }
        let mut put_reader = PutObjReader::from_vec(payload.clone());
        store.pools[1]
            .put_object(bucket, object, &mut put_reader, &opts)
            .await
            .expect("object should be written only to the second pool");

        let prepared = store
            .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
            .await
            .expect("prepared reader should resolve the second-pool object");
        assert_eq!(prepared.object_info().size, payload.len() as i64);
        let mut reader = prepared.into_reader().await.expect("prepared body reader should open");
        let mut restored = Vec::new();
        reader
            .stream
            .read_to_end(&mut restored)
            .await
            .expect("prepared body should stream");
        assert_eq!(restored, payload);
    }

    // Phase 5 Slice 2 (backlog#939): the instance context flows down the whole
    // object graph — ECStore, its Sets, and their SetDisks must all carry the
    // same `Arc<InstanceContext>` in a single-instance deployment.
    #[tokio::test]
    async fn instance_context_flows_through_object_graph() {
        let store = new_read_lock_test_store().await;

        let sets = store.pools.first().expect("test store has one pool");
        assert!(
            std::sync::Arc::ptr_eq(&store.ctx, sets.instance_ctx()),
            "Sets must carry the store's instance context"
        );

        let set_disks = sets.disk_set.first().expect("pool has one set");
        assert!(
            std::sync::Arc::ptr_eq(sets.instance_ctx(), set_disks.instance_ctx()),
            "SetDisks must carry the Sets' instance context"
        );
    }

    // Phase 5 Slice 3 (backlog#939): a SetDisks sources its lock manager from
    // its instance context (not an independent process lookup), and in a
    // single-instance build that context aliases the process lock-manager
    // singleton — so the lock namespace is unchanged.
    #[tokio::test]
    async fn set_disks_lock_manager_comes_from_instance_context() {
        let store = new_read_lock_test_store().await;
        let set_disks = store.pools[0].disk_set.first().expect("pool has one set");

        assert!(
            std::sync::Arc::ptr_eq(set_disks.local_lock_manager_for_test(), &set_disks.instance_ctx().lock_manager()),
            "SetDisks lock manager must be sourced from its instance context"
        );
        assert!(
            std::sync::Arc::ptr_eq(set_disks.local_lock_manager_for_test(), &rustfs_lock::get_global_lock_manager()),
            "single-instance lock manager must alias the process singleton"
        );
    }

    #[tokio::test]
    #[serial_test::serial]
    async fn delete_objects_write_locks_cover_each_unique_object() {
        let store = new_read_lock_test_store().await;
        let objects = vec![
            ObjectToDelete {
                object_name: "beta".to_string(),
                ..Default::default()
            },
            ObjectToDelete {
                object_name: "alpha".to_string(),
                ..Default::default()
            },
            ObjectToDelete {
                object_name: "beta".to_string(),
                ..Default::default()
            },
        ];
        let mut opts = ObjectOptions::default();

        let guards = store
            .acquire_delete_objects_write_locks("bucket", &objects, &mut opts)
            .await
            .expect("delete object locks should be acquired");

        assert_eq!(guards.len(), 2, "duplicate object names should share one namespace lock");
        assert!(opts.no_lock, "set layer should not reacquire locks already held by ECStore");
        assert!(
            opts.namespace_lock_fence.is_some(),
            "set layer must receive the outer write-lock loss fence"
        );

        let alpha_lock = store
            .handle_new_ns_lock("bucket", "alpha")
            .await
            .expect("alpha namespace lock should be created");
        let err = alpha_lock
            .get_read_lock(Duration::from_millis(20))
            .await
            .expect_err("batch delete write guard should block alpha readers");
        assert!(matches!(err, rustfs_lock::LockError::Timeout { .. }));

        drop(guards);
        alpha_lock
            .get_read_lock(Duration::from_secs(1))
            .await
            .expect("alpha read lock should be available after dropping batch guards");
    }

    #[tokio::test]
    async fn acquired_read_lock_marks_metadata_cache_safe_for_set_layer() {
        let store = new_read_lock_test_store().await;
        let mut opts = ObjectOptions::default();

        let guard = store
            .acquire_object_read_lock_if_needed("get_object", "bucket", "object", &mut opts)
            .await
            .expect("read lock should be acquired");

        assert!(guard.is_some(), "read lock should be held by the outer store layer");
        assert!(opts.no_lock, "set layer should not reacquire the object lock");
        assert!(
            opts.metadata_cache_safe,
            "metadata cache is safe only because the outer store layer acquired the read lock"
        );
    }

    #[tokio::test]
    async fn prelocked_read_request_does_not_mark_metadata_cache_safe() {
        let store = new_read_lock_test_store().await;
        let mut opts = ObjectOptions {
            no_lock: true,
            ..Default::default()
        };

        let guard = store
            .acquire_object_read_lock_if_needed("get_object", "bucket", "object", &mut opts)
            .await
            .expect("prelocked read should not acquire another lock");

        assert!(guard.is_none(), "prelocked caller should keep lock ownership outside ECStore");
        assert!(
            !opts.metadata_cache_safe,
            "generic no_lock callers must stay ineligible for metadata cache unless explicitly marked safe"
        );
    }

    // NOTE: #4877's `restore_transitioned_object_waits_for_existing_reader`
    // was removed with the whole-copy-back write lock it asserted
    // (backlog#1304): restore entry no longer serializes on the object lock.
    // The replacement semantics — non-blocking reads during the copy-back and
    // fast rejection of a concurrent restore — are covered end-to-end by
    // `restore_object_usecase_reports_ongoing_conflict`
    // (rustfs/src/app/lifecycle_transition_api_test.rs), while the SetDisks
    // transition matrix covers the final local commit. Restore-vs-reader data
    // protection lives in the inner put_object/complete_multipart_upload locks.
    #[tokio::test]
    #[serial_test::serial]
    async fn restore_accept_guard_serializes_concurrent_accepts() {
        // backlog#1304: the accept guard is the compare-and-set boundary for
        // the restore ongoing flag — a second accept of the same object must
        // wait behind (here: time out on) the first.
        let store = Arc::new(new_read_lock_test_store().await);
        let _first = store
            .acquire_restore_accept_guard("bucket", "object")
            .await
            .expect("first accept guard should be acquired");

        let err = temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
            match store.acquire_restore_accept_guard("bucket", "object").await {
                Ok(_) => panic!("second accept guard must wait behind the first"),
                Err(err) => err,
            }
        })
        .await;

        assert!(matches!(err, StorageError::Lock(rustfs_lock::LockError::Timeout { .. })));
    }

    #[tokio::test]
    #[serial_test::serial]
    async fn reader_lock_is_held_when_optimization_is_disabled() {
        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))], async {
            let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
            let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
            let key = rustfs_lock::ObjectKey::new("bucket", "object");
            let read_guard = lock
                .get_read_lock(key.clone(), "reader", Duration::from_secs(1))
                .await
                .expect("read lock should be acquired");
            let read_guard = ObjectLockDiagGuard::new(
                read_guard,
                true,
                "test_get_object",
                Some("bucket".to_string()),
                Some("object".to_string()),
                Some("reader".to_string()),
                ObjectLockDiagMode::Read,
            );
            let reader = GetObjectReader {
                stream: Box::new(Cursor::new(Vec::<u8>::new())),
                object_info: ObjectInfo::default(),
                buffered_body: None,
                body_source: Default::default(),
            };

            let reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));

            lock.get_write_lock(key.clone(), "writer", Duration::from_millis(20))
                .await
                .expect_err("reader should hold the read lock");
            drop(reader);
            lock.get_write_lock(key, "writer", Duration::from_secs(1))
                .await
                .expect("dropping the reader should release the read lock");
        })
        .await;
    }

    #[tokio::test]
    #[serial_test::serial]
    async fn reader_lock_is_not_held_for_stream_when_optimization_is_enabled() {
        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true"))], async {
            let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
            let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
            let key = rustfs_lock::ObjectKey::new("bucket", "object");
            let read_guard = lock
                .get_read_lock(key.clone(), "reader", Duration::from_secs(1))
                .await
                .expect("read lock should be acquired");
            let read_guard = ObjectLockDiagGuard::new(
                read_guard,
                true,
                "test_get_object",
                Some("bucket".to_string()),
                Some("object".to_string()),
                Some("reader".to_string()),
                ObjectLockDiagMode::Read,
            );
            let reader = GetObjectReader {
                stream: Box::new(Cursor::new(vec![1, 2, 3])),
                object_info: ObjectInfo::default(),
                buffered_body: None,
                body_source: Default::default(),
            };

            let reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));

            lock.get_write_lock(key, "writer", Duration::from_secs(1))
                .await
                .expect("lock optimization should release the read lock before returning the stream");
            drop(reader);
        })
        .await;
    }

    #[tokio::test]
    #[serial_test::serial]
    async fn reader_lock_is_not_held_for_buffered_body_when_optimization_is_enabled() {
        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true"))], async {
            let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
            let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
            let key = rustfs_lock::ObjectKey::new("bucket", "object");
            let read_guard = lock
                .get_read_lock(key.clone(), "reader", Duration::from_secs(1))
                .await
                .expect("read lock should be acquired");
            let read_guard = ObjectLockDiagGuard::new(
                read_guard,
                true,
                "test_get_object",
                Some("bucket".to_string()),
                Some("object".to_string()),
                Some("reader".to_string()),
                ObjectLockDiagMode::Read,
            );
            let reader = GetObjectReader {
                stream: Box::new(Cursor::new(vec![1, 2, 3])),
                object_info: ObjectInfo::default(),
                buffered_body: Some(Bytes::from_static(b"123")),
                body_source: Default::default(),
            };

            let reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));

            lock.get_write_lock(key, "writer", Duration::from_secs(1))
                .await
                .expect("buffered reader should release the read lock immediately");
            drop(reader);
        })
        .await;
    }

    #[tokio::test]
    #[serial_test::serial]
    async fn reader_lock_is_released_after_stream_eof() {
        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))], async {
            let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
            let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
            let key = rustfs_lock::ObjectKey::new("bucket", "object");
            let read_guard = lock
                .get_read_lock(key.clone(), "reader", Duration::from_secs(1))
                .await
                .expect("read lock should be acquired");
            let read_guard = ObjectLockDiagGuard::new(
                read_guard,
                true,
                "test_get_object",
                Some("bucket".to_string()),
                Some("object".to_string()),
                Some("reader".to_string()),
                ObjectLockDiagMode::Read,
            );
            let reader = GetObjectReader {
                stream: Box::new(Cursor::new(vec![1, 2, 3])),
                object_info: ObjectInfo::default(),
                buffered_body: None,
                body_source: Default::default(),
            };

            let mut reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));
            let mut output = Vec::new();
            reader.stream.read_to_end(&mut output).await.expect("reader should reach EOF");
            assert_eq!(output, vec![1, 2, 3]);

            lock.get_write_lock(key, "writer", Duration::from_secs(1))
                .await
                .expect("EOF should release the read lock before the reader is dropped");
            drop(reader);
        })
        .await;
    }
}