rustfs/crates/ecstore/src/services/tier/tier.rs

// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]

use byteorder::{ByteOrder, LittleEndian};
use bytes::Bytes;
use futures::FutureExt;
use http::HeaderMap;
use http::status::StatusCode;
use lazy_static::lazy_static;
use rand::{Rng, RngExt};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::{
    collections::{HashMap, HashSet, hash_map::Entry},
    io::{self, Cursor},
    ops::Deref,
    panic::AssertUnwindSafe,
    sync::{
        Arc, LazyLock, Mutex, MutexGuard, RwLock as StdRwLock, Weak,
        atomic::{AtomicBool, AtomicUsize, Ordering},
    },
    time::Duration,
};
use time::OffsetDateTime;
use tokio::io::BufReader;
use tokio::{
    select,
    sync::RwLock,
    time::{Instant, interval, interval_at, timeout_at},
};
use tracing::{debug, error, info, warn};

use crate::client::{admin_handler_utils::AdminError, provider_versions::ProviderVersionCapabilities};
use crate::error::{Error, Result, StorageError};
use crate::services::tier::{
    tier_admin::TierCreds,
    tier_config::{TierConfig, TierType, TierWasabi},
    tier_handlers::{ERR_TIER_ALREADY_EXISTS, ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_NOT_FOUND, ERR_TIER_RESERVED_NAME},
    warm_backend::{TransitionCandidateProbe, WarmBackend, check_warm_backend, new_warm_backend},
};
use crate::storage_api_contracts::{
    bucket::BucketOperations,
    list::{ListOperations, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
    namespace::NamespaceLocking,
    object::{
        DeletedObject, EcstoreObjectIO, EcstoreObjectOperations, HTTPPreconditions, ObjectIO, ObjectOperations, ObjectToDelete,
    },
    range::HTTPRangeSpec,
};
use crate::{
    bucket::lifecycle::{
        tier_delete_journal::{TIER_DELETE_JOURNAL_PREFIX, decode_tier_delete_journal_entry},
        transition_transaction::{TRANSITION_TRANSACTION_RECORD_PREFIX, decode_transition_transaction_record},
    },
    cluster::rpc::peer_rest_client::{PeerRestClient, PeerTierMutationState},
    config::com::{CONFIG_PREFIX, read_config, read_config_with_metadata},
    disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET},
    layout::endpoints::EndpointServerPools,
    object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
    runtime::sources as runtime_sources,
    set_disk::get_lock_acquire_timeout,
    store::ECStore,
};
use rustfs_filemeta::FileInfo;
use rustfs_rio::HashReader;
use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
use s3s::S3ErrorCode;

use super::{
    tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_PERM_ERR},
    tier_mutation_intent::{
        TierMutationDigest, TierMutationIntent, TierMutationIntentKind, TierMutationIntentState, TierMutationIntentTarget,
        advance_tier_coordinator_mutation_intent_record_idempotent, delete_tier_coordinator_mutation_intent_record,
        delete_tier_mutation_intent_record, list_tier_coordinator_mutation_intent_records, list_tier_mutation_intent_records,
        save_tier_coordinator_mutation_intent_record_if_absent,
    },
    warm_backend::WarmBackendImpl,
};

const TIER_CFG_REFRESH: Duration = Duration::from_secs(15 * 60);
const TIER_OPERATION_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
const TIER_REMOTE_VALIDATION_TIMEOUT: Duration = Duration::from_secs(30);
const TIER_REFERENCE_PROOF_LIST_LIMIT: i32 = 1000;
const TIER_MUTATION_INTENT_RECOVERY_SCAN_LIMIT: usize = 1000;
const TIER_MUTATION_INTENT_TTL: Duration = Duration::from_secs(15 * 60);
const TIER_MUTATION_BLOCK_MESSAGE: &str = "Remote tier configuration is being replaced";

type TierReferenceProofWalkOptions = StorageWalkOptions<fn(&rustfs_filemeta::FileInfo) -> bool>;

fn delayed_tier_refresh_interval(period: Duration) -> tokio::time::Interval {
    interval_at(Instant::now() + period, period)
}

#[cfg(test)]
struct TierDriverBuildBarrier {
    tier_name: String,
    arrived: tokio::sync::Notify,
    release: tokio::sync::Semaphore,
}

#[cfg(test)]
static TIER_DRIVER_BUILD_BARRIER: LazyLock<Mutex<Option<Arc<TierDriverBuildBarrier>>>> = LazyLock::new(|| Mutex::new(None));

#[cfg(test)]
struct TierDriverBuildBarrierGuard;

#[cfg(test)]
impl Drop for TierDriverBuildBarrierGuard {
    fn drop(&mut self) {
        *lock_unpoisoned(&TIER_DRIVER_BUILD_BARRIER) = None;
    }
}

#[cfg(test)]
fn install_tier_driver_build_barrier(tier_name: &str) -> (Arc<TierDriverBuildBarrier>, TierDriverBuildBarrierGuard) {
    let barrier = Arc::new(TierDriverBuildBarrier {
        tier_name: tier_name.to_string(),
        arrived: tokio::sync::Notify::new(),
        release: tokio::sync::Semaphore::new(0),
    });
    *lock_unpoisoned(&TIER_DRIVER_BUILD_BARRIER) = Some(barrier.clone());
    (barrier, TierDriverBuildBarrierGuard)
}

async fn build_warm_backend(tier: &TierConfig, probe: bool) -> std::result::Result<WarmBackendImpl, AdminError> {
    #[cfg(test)]
    let test_barrier = { lock_unpoisoned(&TIER_DRIVER_BUILD_BARRIER).clone() };
    #[cfg(test)]
    if let Some(barrier) = test_barrier
        && barrier.tier_name == tier.name
    {
        barrier.arrived.notify_one();
        barrier
            .release
            .acquire()
            .await
            .expect("tier driver build test barrier should stay open")
            .forget();
    }
    new_warm_backend(tier, probe).await
}

const TIER_CONFIG_LEGACY_FILE: &str = "tier-config.json";
pub const TIER_CONFIG_FILE: &str = "tier-config.bin";
pub const TIER_CONFIG_FORMAT: u16 = 1;
pub const TIER_CONFIG_V1: u16 = 1;
pub const TIER_CONFIG_VERSION: u16 = 2;

const EXTERNAL_TIER_TYPE_UNSUPPORTED: i32 = 0;
const EXTERNAL_TIER_TYPE_S3: i32 = 1;
const EXTERNAL_TIER_TYPE_AZURE: i32 = 2;
const EXTERNAL_TIER_TYPE_GCS: i32 = 3;
const EXTERNAL_TIER_TYPE_MINIO: i32 = 4;
const EXTERNAL_TIER_VERSION_WASABI_V1: &str = "rustfs-wasabi-v1";
const EXTERNAL_TIER_WASABI_ENDPOINT_SENTINEL: &str = "rustfs-wasabi-v1:";
const TIER_BACKEND_IDENTITY_VERSION: u8 = 2;
const _TIER_CFG_REFRESH_AT_HDR: &str = "X-RustFS-TierCfg-RefreshedAt";

lazy_static! {
    pub static ref ERR_TIER_MISSING_CREDENTIALS: AdminError = AdminError {
        code: "XRustFSAdminTierMissingCredentials".to_string(),
        message: "Specified remote credentials are empty".to_string(),
        status_code: StatusCode::FORBIDDEN,
    };
    pub static ref ERR_TIER_BACKEND_IN_USE: AdminError = AdminError {
        code: "XRustFSAdminTierBackendInUse".to_string(),
        message: "Specified remote tier is already in use".to_string(),
        status_code: StatusCode::CONFLICT,
    };
    pub static ref ERR_TIER_TYPE_UNSUPPORTED: AdminError = AdminError {
        code: "XRustFSAdminTierTypeUnsupported".to_string(),
        message: "Specified tier type is unsupported".to_string(),
        status_code: StatusCode::BAD_REQUEST,
    };
    pub static ref ERR_TIER_BACKEND_NOT_EMPTY: AdminError = AdminError {
        code: "XRustFSAdminTierBackendNotEmpty".to_string(),
        message: "Specified remote backend is not empty".to_string(),
        status_code: StatusCode::BAD_REQUEST,
    };
    pub static ref ERR_TIER_INVALID_CONFIG: AdminError = AdminError {
        code: "XRustFSAdminTierInvalidConfig".to_string(),
        message: "Unable to setup remote tier, check tier configuration".to_string(),
        status_code: StatusCode::BAD_REQUEST,
    };
}

#[derive(Serialize, Deserialize)]
pub struct TierConfigMgr {
    #[serde(skip)]
    pub driver_cache: HashMap<String, WarmBackendImpl>,
    pub tiers: HashMap<String, TierConfig>,
    pub last_refreshed_at: OffsetDateTime,
}

type SharedWarmBackend = Arc<dyn WarmBackend + Send + Sync + 'static>;

#[derive(Default)]
struct TierDriverRuntime {
    generations: HashMap<String, Arc<TierDriverGeneration>>,
    draining: HashMap<String, u64>,
    prepared_mutation_blocks: HashMap<String, uuid::Uuid>,
    prepared_mutation_blocks_revision: u64,
    next_generation: u64,
    next_drain_epoch: u64,
    admin_updates: Arc<tokio::sync::Mutex<()>>,
}

struct TierDriverRegistryEntry {
    manager: Weak<RwLock<TierConfigMgr>>,
    runtime: Arc<Mutex<TierDriverRuntime>>,
}

struct TierPublishTransition {
    runtime: Arc<Mutex<TierDriverRuntime>>,
    tokens: Vec<(String, u64)>,
    revoked: HashMap<String, Arc<TierDriverGeneration>>,
    replaced_destinations: HashMap<String, TierConfig>,
    published: bool,
}

struct PreparedTierDriver {
    tier_name: String,
    tier_config: TierConfig,
    config_fingerprint: TierDriverFingerprint,
    backend_identity: TierDestinationId,
    exact_get_delete: bool,
    driver: SharedWarmBackend,
}

impl TierPublishTransition {
    async fn wait_for_active_leases(&self) -> std::result::Result<(), AdminError> {
        let deadline = Instant::now() + TIER_OPERATION_DRAIN_TIMEOUT;
        for generation in self.revoked.values() {
            if timeout_at(deadline, generation.wait_for_no_active_leases()).await.is_err() {
                let mut err = ERR_TIER_BACKEND_IN_USE.clone();
                err.message = "Timed out waiting for active remote tier operations to finish".to_string();
                return Err(err);
            }
        }
        Ok(())
    }

    async fn ensure_replaced_destinations_are_empty(&self) -> std::result::Result<(), AdminError> {
        ensure_replaced_destinations_are_empty(&self.replaced_destinations, &self.revoked).await
    }
}

async fn ensure_replaced_destinations_are_empty(
    replaced_destinations: &HashMap<String, TierConfig>,
    revoked: &HashMap<String, Arc<TierDriverGeneration>>,
) -> std::result::Result<(), AdminError> {
    let deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT;
    for (tier_name, old_config) in replaced_destinations {
        let prepared;
        let driver = match revoked.get(tier_name) {
            Some(generation) => generation.driver.as_ref(),
            None => {
                prepared = match timeout_at(deadline, build_warm_backend(old_config, false)).await {
                    Ok(result) => result?,
                    Err(_) => {
                        let mut err = ERR_TIER_BACKEND_IN_USE.clone();
                        err.message = "Timed out preparing the replaced remote tier backend".to_string();
                        return Err(err);
                    }
                };
                prepared.as_ref()
            }
        };
        match timeout_at(deadline, driver.in_use()).await {
            Ok(Ok(false)) => {}
            Ok(Ok(true)) => return Err(ERR_TIER_BACKEND_NOT_EMPTY.clone()),
            Ok(Err(err)) => {
                let mut admin_err = ERR_TIER_PERM_ERR.clone();
                admin_err.message.push('.');
                admin_err.message.push_str(&err.to_string());
                return Err(admin_err);
            }
            Err(_) => {
                let mut err = ERR_TIER_BACKEND_IN_USE.clone();
                err.message = "Timed out checking whether the replaced remote tier backend is empty".to_string();
                return Err(err);
            }
        }
    }
    Ok(())
}

impl Drop for TierPublishTransition {
    fn drop(&mut self) {
        let mut runtime = lock_unpoisoned(&self.runtime);
        for (tier_name, epoch) in &self.tokens {
            if runtime.draining.get(tier_name) == Some(epoch) {
                if !self.published
                    && !runtime.generations.contains_key(tier_name)
                    && let Some(generation) = self.revoked.get(tier_name)
                {
                    generation.accepting.store(true, Ordering::Release);
                    runtime.generations.insert(tier_name.clone(), generation.clone());
                }
                runtime.draining.remove(tier_name);
            }
        }
    }
}

static TIER_DRIVER_REGISTRY: LazyLock<StdRwLock<HashMap<usize, TierDriverRegistryEntry>>> =
    LazyLock::new(|| StdRwLock::new(HashMap::new()));

fn lock_unpoisoned<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
    mutex.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}

fn tier_manager_key(manager: &TierConfigMgr) -> usize {
    std::ptr::from_ref(manager).addr()
}

fn tier_driver_runtime(handle: &Arc<RwLock<TierConfigMgr>>, manager: &TierConfigMgr) -> Arc<Mutex<TierDriverRuntime>> {
    let key = tier_manager_key(manager);
    {
        let registry = TIER_DRIVER_REGISTRY.read().unwrap_or_else(std::sync::PoisonError::into_inner);
        if let Some(entry) = registry.get(&key)
            && entry
                .manager
                .upgrade()
                .is_some_and(|registered| Arc::ptr_eq(&registered, handle))
        {
            return entry.runtime.clone();
        }
    }
    #[cfg(test)]
    let _ = TIER_REGISTRY_MISS_SCAN_COUNT.try_with(|count| count.set(count.get() + 1));
    let mut registry = TIER_DRIVER_REGISTRY
        .write()
        .unwrap_or_else(std::sync::PoisonError::into_inner);
    if let Some(entry) = registry.get(&key)
        && entry
            .manager
            .upgrade()
            .is_some_and(|registered| Arc::ptr_eq(&registered, handle))
    {
        return entry.runtime.clone();
    }
    registry.retain(|_, entry| entry.manager.strong_count() > 0);
    let runtime = Arc::new(Mutex::new(TierDriverRuntime::default()));
    registry.insert(
        key,
        TierDriverRegistryEntry {
            manager: Arc::downgrade(handle),
            runtime: runtime.clone(),
        },
    );
    runtime
}

#[cfg(test)]
tokio::task_local! {
    static TIER_REGISTRY_MISS_SCAN_COUNT: std::cell::Cell<usize>;
}

fn registered_tier_driver_runtime(manager: &TierConfigMgr) -> Option<Arc<Mutex<TierDriverRuntime>>> {
    let key = tier_manager_key(manager);
    {
        let registry = TIER_DRIVER_REGISTRY.read().unwrap_or_else(std::sync::PoisonError::into_inner);
        let entry = registry.get(&key)?;
        if entry.manager.strong_count() > 0 {
            return Some(entry.runtime.clone());
        }
    }
    let mut registry = TIER_DRIVER_REGISTRY
        .write()
        .unwrap_or_else(std::sync::PoisonError::into_inner);
    if registry.get(&key).is_some_and(|entry| entry.manager.strong_count() == 0) {
        registry.remove(&key);
    }
    None
}

fn runtime_blocks_tier(runtime: &TierDriverRuntime, tier_name: &str) -> bool {
    runtime.draining.contains_key(tier_name) || runtime.prepared_mutation_blocks.contains_key(tier_name)
}

fn runtime_blocks_tier_transition(
    runtime: &TierDriverRuntime,
    tier_name: &str,
    allowed_mutation_blocks: Option<&HashSet<uuid::Uuid>>,
) -> bool {
    if runtime.draining.contains_key(tier_name) {
        return true;
    }
    runtime.prepared_mutation_blocks.get(tier_name).is_some_and(|mutation_id| {
        !allowed_mutation_blocks.is_some_and(|allowed_mutation_blocks| allowed_mutation_blocks.contains(mutation_id))
    })
}

fn runtime_has_mutation_block(runtime: &TierDriverRuntime) -> bool {
    !runtime.draining.is_empty() || !runtime.prepared_mutation_blocks.is_empty()
}

type TierDriverFingerprint = [u8; 32];
pub(crate) type TierDestinationId = [u8; 32];
pub(crate) type DriverRevision = u64;

fn tier_config_fingerprint(tier_name: &str, config: &TierConfig) -> io::Result<TierDriverFingerprint> {
    let external = to_external_tier_config(tier_name, config)?;
    let encoded = rmp_serde::to_vec_named(&external)
        .map_err(|err| io::Error::other(format!("serialize tier driver identity failed: {err}")))?;
    Ok(Sha256::digest(encoded).into())
}

fn tier_configs_match(tier_name: &str, current: &TierConfig, replacement: &TierConfig) -> bool {
    match (
        tier_config_fingerprint(tier_name, current),
        tier_config_fingerprint(tier_name, replacement),
    ) {
        (Ok(current), Ok(replacement)) => current == replacement,
        _ => false,
    }
}

#[derive(Debug)]
pub enum TierConfigUpdateError {
    Load(io::Error),
    Mutation(AdminError),
    Save(io::Error),
    Publish(AdminError),
}

enum TierCandidateMutation {
    Add(TierConfig, bool),
    Edit(String, TierCreds),
    Remove(String, bool),
    Clear(bool),
}

impl TierCandidateMutation {
    fn intent_kind(&self) -> TierMutationIntentKind {
        match self {
            Self::Add(_, _) => TierMutationIntentKind::Add,
            Self::Edit(_, _) => TierMutationIntentKind::Edit,
            Self::Remove(_, _) => TierMutationIntentKind::Remove,
            Self::Clear(_) => TierMutationIntentKind::Clear,
        }
    }

    fn explicit_tier_name(&self) -> Option<&str> {
        match self {
            Self::Add(config, _) => Some(&config.name),
            Self::Edit(tier_name, _) | Self::Remove(tier_name, _) => Some(tier_name),
            Self::Clear(_) => None,
        }
    }

    fn target_tiers(&self, manager: &TierConfigMgr, candidate: &TierConfigMgr) -> HashSet<String> {
        let mut targets = changed_tier_names(manager, candidate);
        match self {
            Self::Add(config, _) => {
                targets.insert(config.name.clone());
            }
            Self::Edit(tier_name, _) => {
                targets.insert(tier_name.clone());
            }
            Self::Remove(tier_name, _) => {
                if manager.tiers.contains_key(tier_name) || candidate.tiers.contains_key(tier_name) {
                    targets.insert(tier_name.clone());
                }
            }
            Self::Clear(_) => {
                targets.extend(manager.tiers.keys().chain(candidate.tiers.keys()).cloned());
            }
        }
        targets
    }

    #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
    fn affected_targets(
        &self,
        manager: &TierConfigMgr,
        candidate: &TierConfigMgr,
    ) -> std::result::Result<Vec<TierMutationIntentTarget>, AdminError> {
        let explicit_tier_name = self.explicit_tier_name();
        build_tier_mutation_affected_targets(
            self.intent_kind(),
            tier_mutation_proof_targets(self.intent_kind(), explicit_tier_name, manager, candidate),
            manager,
            candidate,
        )
    }

    async fn apply(self, candidate: &mut TierConfigMgr) -> std::result::Result<Option<String>, AdminError> {
        match self {
            Self::Add(config, force) => {
                let tier_name = config.name.clone();
                candidate.add(config, force).await?;
                Ok(Some(tier_name))
            }
            Self::Edit(tier_name, credentials) => {
                candidate.edit(&tier_name, credentials).await?;
                Ok(Some(tier_name))
            }
            Self::Remove(tier_name, force) => {
                candidate.remove(&tier_name, force).await?;
                Ok(None)
            }
            Self::Clear(force) => {
                candidate.clear_tier(force).await?;
                Ok(None)
            }
        }
    }
}

fn tier_mutation_proof_targets(
    kind: TierMutationIntentKind,
    explicit_tier_name: Option<&str>,
    current: &TierConfigMgr,
    candidate: &TierConfigMgr,
) -> HashSet<String> {
    let mut targets = HashSet::new();
    match kind {
        TierMutationIntentKind::Add | TierMutationIntentKind::Edit | TierMutationIntentKind::Remove => {
            if let Some(tier_name) = explicit_tier_name
                && (current.tiers.contains_key(tier_name) || candidate.tiers.contains_key(tier_name))
            {
                targets.insert(tier_name.to_string());
            }
        }
        TierMutationIntentKind::Clear => {
            targets.extend(current.tiers.keys().chain(candidate.tiers.keys()).cloned());
        }
    }
    targets
}

fn build_tier_mutation_affected_targets(
    kind: TierMutationIntentKind,
    target_tiers: HashSet<String>,
    current: &TierConfigMgr,
    candidate: &TierConfigMgr,
) -> std::result::Result<Vec<TierMutationIntentTarget>, AdminError> {
    let mut targets = target_tiers.into_iter().collect::<Vec<_>>();
    targets.sort();
    let mut out = Vec::with_capacity(targets.len());
    for tier_name in targets {
        let old_backend_identity = current
            .tiers
            .get(&tier_name)
            .map(tier_backend_identity)
            .transpose()
            .map_err(tier_backend_identity_admin_error)?;
        let new_backend_identity = candidate
            .tiers
            .get(&tier_name)
            .map(tier_backend_identity)
            .transpose()
            .map_err(tier_backend_identity_admin_error)?;
        if old_backend_identity.is_none() && new_backend_identity.is_none() {
            continue;
        }
        let target = TierMutationIntentTarget {
            tier_name,
            old_backend_identity,
            new_backend_identity,
        };
        validate_tier_mutation_target_shape(kind, &target)?;
        out.push(target);
    }
    Ok(out)
}

fn validate_tier_mutation_target_shape(
    kind: TierMutationIntentKind,
    target: &TierMutationIntentTarget,
) -> std::result::Result<(), AdminError> {
    let valid = match kind {
        TierMutationIntentKind::Add => target.old_backend_identity.is_none() && target.new_backend_identity.is_some(),
        TierMutationIntentKind::Edit => target.old_backend_identity.is_some() && target.new_backend_identity.is_some(),
        TierMutationIntentKind::Remove | TierMutationIntentKind::Clear => {
            target.old_backend_identity.is_some() && target.new_backend_identity.is_none()
        }
    };
    if valid {
        return Ok(());
    }
    let mut err = ERR_TIER_INVALID_CONFIG.clone();
    err.message = "Remote tier mutation target identity proof does not match mutation kind".to_string();
    Err(err)
}

fn tier_backend_identity_admin_error(err: io::Error) -> AdminError {
    let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
    admin_err.message = err.to_string();
    admin_err
}

trait TierReferenceProofStore:
    EcstoreObjectIO
    + BucketOperations<Error = Error>
    + ListOperations<
        Error = Error,
        ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>,
        ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>,
        ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>,
        WalkOptions = TierReferenceProofWalkOptions,
        WalkCancellation = tokio_util::sync::CancellationToken,
        WalkResultSender = tokio::sync::mpsc::Sender<StorageObjectInfoOrErr<ObjectInfo, Error>>,
    >
{
}

impl<T> TierReferenceProofStore for T where
    T: EcstoreObjectIO
        + BucketOperations<Error = Error>
        + ListOperations<
            Error = Error,
            ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>,
            ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>,
            ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>,
            WalkOptions = TierReferenceProofWalkOptions,
            WalkCancellation = tokio_util::sync::CancellationToken,
            WalkResultSender = tokio::sync::mpsc::Sender<StorageObjectInfoOrErr<ObjectInfo, Error>>,
        >
{
}

async fn ensure_no_authoritative_tier_object_references<S>(
    api: Arc<S>,
    affected_targets: &[TierMutationIntentTarget],
) -> std::result::Result<(), AdminError>
where
    S: TierReferenceProofStore,
{
    let targets = affected_targets
        .iter()
        .filter(|target| target.old_backend_identity.is_some())
        .cloned()
        .collect::<Vec<_>>();
    if targets.is_empty() {
        return Ok(());
    }
    ensure_no_authoritative_target_references(api, &targets).await
}

async fn ensure_no_authoritative_target_references<S>(
    api: Arc<S>,
    targets: &[TierMutationIntentTarget],
) -> std::result::Result<(), AdminError>
where
    S: TierReferenceProofStore,
{
    let buckets = api
        .list_bucket(&Default::default())
        .await
        .map_err(tier_reference_proof_admin_error)?;
    for bucket in buckets {
        let mut marker = None;
        let mut version_marker = None;
        loop {
            let page = api
                .clone()
                .list_object_versions(
                    &bucket.name,
                    "",
                    marker.take(),
                    version_marker.take(),
                    None,
                    TIER_REFERENCE_PROOF_LIST_LIMIT,
                )
                .await
                .map_err(tier_reference_proof_admin_error)?;
            for object in &page.objects {
                if tier_object_blocks_any_target_rebind(object, targets).map_err(tier_reference_proof_admin_error)? {
                    return Err(tier_reference_proof_in_use_error(&object.transitioned_object.tier, object));
                }
            }
            if !page.is_truncated {
                break;
            }
            marker = page.next_marker;
            version_marker = page.next_version_idmarker;
            if marker.is_none() {
                return Err(tier_reference_proof_admin_error(io::Error::other(
                    "tier reference proof listing is truncated without a next marker",
                )));
            }
        }
    }
    ensure_no_authoritative_persisted_references(api, targets).await
}

async fn ensure_no_authoritative_persisted_references<S>(
    api: Arc<S>,
    targets: &[TierMutationIntentTarget],
) -> std::result::Result<(), AdminError>
where
    S: TierReferenceProofStore,
{
    ensure_no_authoritative_persisted_references_with(api.clone(), TIER_DELETE_JOURNAL_PREFIX, |_object, data| {
        let journal = decode_tier_delete_journal_entry(data).map_err(io::Error::other)?;
        Ok((
            journal.tier_name.clone(),
            tier_persisted_reference_blocks_any_target(&journal.tier_name, journal.backend_identity, targets),
        ))
    })
    .await?;
    ensure_no_authoritative_persisted_references_with(api, TRANSITION_TRANSACTION_RECORD_PREFIX, |object, data| {
        let transaction = decode_transition_transaction_record(object, data).map_err(io::Error::other)?;
        Ok((
            transaction.tier_name.clone(),
            tier_persisted_reference_blocks_any_target(&transaction.tier_name, Some(transaction.backend_fingerprint), targets),
        ))
    })
    .await
}

async fn ensure_no_authoritative_persisted_references_with<S, F>(
    api: Arc<S>,
    prefix: &str,
    blocks_target: F,
) -> std::result::Result<(), AdminError>
where
    S: TierReferenceProofStore,
    F: Fn(&str, &[u8]) -> io::Result<(String, bool)>,
{
    let mut marker = None;
    loop {
        let page = api
            .clone()
            .list_objects_v2(
                RUSTFS_META_BUCKET,
                prefix,
                marker.take(),
                None,
                TIER_REFERENCE_PROOF_LIST_LIMIT,
                false,
                None,
                false,
            )
            .await
            .map_err(tier_reference_proof_admin_error)?;
        for object in &page.objects {
            let data = read_config(api.clone(), &object.name)
                .await
                .map_err(tier_reference_proof_admin_error)?;
            let (tier_name, blocks) = blocks_target(&object.name, &data).map_err(tier_reference_proof_admin_error)?;
            if blocks {
                return Err(tier_reference_proof_persisted_in_use_error(&tier_name, &object.name));
            }
        }
        if !page.is_truncated {
            return Ok(());
        }
        marker = page.next_continuation_token;
        if marker.is_none() {
            return Err(tier_reference_proof_admin_error(io::Error::other(
                "tier persisted reference proof listing is truncated without a next marker",
            )));
        }
    }
}

fn tier_object_blocks_any_target_rebind(object: &ObjectInfo, targets: &[TierMutationIntentTarget]) -> io::Result<bool> {
    if object.transitioned_object.status != rustfs_filemeta::TRANSITION_COMPLETE && !object.transitioned_object.free_version {
        return Ok(false);
    }
    if !targets
        .iter()
        .any(|target| target.tier_name == object.transitioned_object.tier)
    {
        return Ok(false);
    }
    let current_identity = tier_destination_id_from_metadata(&object.user_defined)?;
    Ok(tier_persisted_reference_blocks_any_target(
        &object.transitioned_object.tier,
        current_identity,
        targets,
    ))
}

fn tier_persisted_reference_blocks_any_target(
    tier_name: &str,
    backend_identity: Option<TierDestinationId>,
    targets: &[TierMutationIntentTarget],
) -> bool {
    targets
        .iter()
        .any(|target| tier_persisted_reference_blocks_target(tier_name, backend_identity, target))
}

#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
fn tier_object_blocks_target_rebind(object: &ObjectInfo, target: &TierMutationIntentTarget) -> io::Result<bool> {
    tier_object_blocks_any_target_rebind(object, std::slice::from_ref(target))
}

fn tier_persisted_reference_blocks_target(
    tier_name: &str,
    backend_identity: Option<TierDestinationId>,
    target: &TierMutationIntentTarget,
) -> bool {
    tier_name == target.tier_name
        && match target.new_backend_identity {
            None => true,
            Some(new_backend_identity) => backend_identity != Some(new_backend_identity),
        }
}

fn tier_reference_proof_in_use_error(tier_name: &str, object: &ObjectInfo) -> AdminError {
    let mut err = ERR_TIER_BACKEND_IN_USE.clone();
    err.message = format!(
        "Remote tier {tier_name} still has object references, for example {}/{}",
        object.bucket, object.name
    );
    err
}

fn tier_reference_proof_persisted_in_use_error(tier_name: &str, object: &str) -> AdminError {
    let mut err = ERR_TIER_BACKEND_IN_USE.clone();
    err.message = format!("Remote tier {tier_name} still has a persisted reference, for example {object}");
    err
}

fn tier_reference_proof_admin_error(err: impl std::fmt::Display) -> AdminError {
    let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
    admin_err.message = format!("Remote tier reference proof failed: {err}");
    admin_err
}

fn tier_mutation_replay_error(err: impl std::fmt::Display) -> io::Error {
    io::Error::other(format!("Remote tier mutation committed replay failed: {err}"))
}

fn tier_mutation_fanout_admin_error(action: &str, err: impl std::fmt::Display) -> AdminError {
    let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
    admin_err.message = format!("Remote tier mutation {action} failed: {err}");
    admin_err
}

fn ensure_complete_tier_mutation_commit_peer_set(peer_count: usize, remote_host_count: usize) -> io::Result<()> {
    if peer_count != remote_host_count {
        return Err(tier_mutation_replay_error(
            "cluster endpoint topology has remote hosts without peer commit clients",
        ));
    }
    Ok(())
}

#[async_trait::async_trait]
trait TierMutationPeer: Send + Sync {
    fn peer_label(&self) -> String;
    async fn prepare_tier_mutation(&self, mutation_id: uuid::Uuid, canonical_payload: Bytes) -> Result<PeerTierMutationState>;
    async fn commit_tier_mutation(&self, mutation_id: uuid::Uuid, canonical_payload: Bytes) -> Result<PeerTierMutationState>;
    async fn abort_tier_mutation(&self, mutation_id: uuid::Uuid) -> Result<PeerTierMutationState>;
}

#[async_trait::async_trait]
impl TierMutationPeer for PeerRestClient {
    fn peer_label(&self) -> String {
        self.grid_host.clone()
    }

    async fn prepare_tier_mutation(&self, mutation_id: uuid::Uuid, canonical_payload: Bytes) -> Result<PeerTierMutationState> {
        Ok(PeerRestClient::prepare_tier_mutation(self, mutation_id, canonical_payload)
            .await?
            .state)
    }

    async fn commit_tier_mutation(&self, mutation_id: uuid::Uuid, canonical_payload: Bytes) -> Result<PeerTierMutationState> {
        Ok(PeerRestClient::commit_tier_mutation(self, mutation_id, canonical_payload)
            .await?
            .state)
    }

    async fn abort_tier_mutation(&self, mutation_id: uuid::Uuid) -> Result<PeerTierMutationState> {
        Ok(PeerRestClient::abort_tier_mutation(self, mutation_id).await?.state)
    }
}

#[cfg(test)]
tokio::task_local! {
    static TIER_MUTATION_TEST_PEERS: Vec<Arc<dyn TierMutationPeer>>;
}

async fn remote_tier_mutation_peers() -> io::Result<Vec<Arc<dyn TierMutationPeer>>> {
    #[cfg(test)]
    if let Ok(peers) = TIER_MUTATION_TEST_PEERS.try_with(|peers| peers.clone()) {
        return Ok(peers);
    }
    if !runtime_sources::setup_is_dist_erasure().await {
        return Ok(Vec::new());
    }
    let Some(endpoints) = runtime_sources::endpoint_pools() else {
        return Err(tier_mutation_replay_error("cluster endpoint topology is not initialized"));
    };
    remote_tier_mutation_peers_from_topology(endpoints).await
}

async fn remote_tier_mutation_peers_from_topology(endpoints: EndpointServerPools) -> io::Result<Vec<Arc<dyn TierMutationPeer>>> {
    let (peers, _, remote_topology_hosts) = PeerRestClient::new_clients_with_topology(endpoints).await;
    let peers = peers
        .into_iter()
        .flatten()
        .map(|peer| Arc::new(peer) as Arc<dyn TierMutationPeer>)
        .collect::<Vec<_>>();
    ensure_complete_tier_mutation_commit_peer_set(peers.len(), remote_topology_hosts.len())?;
    Ok(peers)
}

async fn prepare_tier_mutation_peers(
    mutation_id: uuid::Uuid,
    peers: Vec<Arc<dyn TierMutationPeer>>,
    intent: &TierMutationIntent,
) -> std::result::Result<Vec<Arc<dyn TierMutationPeer>>, TierMutationPrepareFailure> {
    let payload = Bytes::from(intent.encode().map_err(|err| TierMutationPrepareFailure {
        error: tier_mutation_fanout_admin_error("prepare", err),
        prepared_peers: Vec::new(),
    })?);

    let mut prepared = Vec::with_capacity(peers.len());
    for peer in peers {
        let label = peer.peer_label();
        let result = peer.prepare_tier_mutation(mutation_id, payload.clone()).await;
        match result {
            Ok(PeerTierMutationState::Prepared) => prepared.push(peer),
            Ok(PeerTierMutationState::Committed) => {}
            Ok(state) => {
                return Err(TierMutationPrepareFailure {
                    error: tier_mutation_fanout_admin_error(
                        "prepare",
                        format!("peer {label} returned unexpected state {state:?}"),
                    ),
                    prepared_peers: prepared,
                });
            }
            Err(err) => {
                return Err(TierMutationPrepareFailure {
                    error: tier_mutation_fanout_admin_error("prepare", format!("peer {label}: {err}")),
                    prepared_peers: prepared,
                });
            }
        }
    }
    Ok(prepared)
}

struct TierMutationPrepareFailure {
    error: AdminError,
    prepared_peers: Vec<Arc<dyn TierMutationPeer>>,
}

async fn abort_tier_mutation_peers(mutation_id: uuid::Uuid, peers: Vec<Arc<dyn TierMutationPeer>>) -> io::Result<()> {
    let mut failures = Vec::new();
    for peer in peers {
        let label = peer.peer_label();
        let result = peer.abort_tier_mutation(mutation_id).await;
        match result {
            Ok(PeerTierMutationState::Aborted) => {}
            Ok(state) => {
                failures.push(format!("peer {label} returned unexpected abort state {state:?}"));
            }
            Err(err) => failures.push(format!("peer {label}: {err}")),
        }
    }
    if failures.is_empty() {
        Ok(())
    } else {
        Err(tier_mutation_replay_error(failures.join("; ")))
    }
}

async fn commit_tier_mutation_peers(
    mutation_id: uuid::Uuid,
    peers: Vec<Arc<dyn TierMutationPeer>>,
    committed_config_etag: &str,
) -> io::Result<()> {
    let payload = Bytes::copy_from_slice(committed_config_etag.as_bytes());
    for peer in peers {
        let label = peer.peer_label();
        let result = peer.commit_tier_mutation(mutation_id, payload.clone()).await;
        match result {
            Ok(PeerTierMutationState::Committed) => {}
            Ok(state) => {
                return Err(tier_mutation_replay_error(format!("peer {label} returned unexpected state {state:?}")));
            }
            Err(err) => return Err(tier_mutation_replay_error(format!("peer {label}: {err}"))),
        }
    }
    Ok(())
}

fn tier_config_candidate_digest(candidate: &TierConfigMgr) -> io::Result<TierMutationDigest> {
    let bytes = encode_external_tiering_config_blob(candidate)?;
    let mut digest = Sha256::new();
    digest.update(&bytes);
    Ok(digest.finalize().into())
}

fn tier_mutation_intent_expiry_unix_nanos() -> i64 {
    let now = OffsetDateTime::now_utc().unix_timestamp_nanos();
    let ttl = i128::try_from(TIER_MUTATION_INTENT_TTL.as_nanos()).unwrap_or(i128::MAX);
    i64::try_from(now.saturating_add(ttl)).unwrap_or(i64::MAX)
}

fn build_coordinator_tier_mutation_intent(
    kind: TierMutationIntentKind,
    old_config_etag: Option<String>,
    candidate: &TierConfigMgr,
    affected_targets: Vec<TierMutationIntentTarget>,
) -> io::Result<Option<TierMutationIntent>> {
    if affected_targets.is_empty() || (old_config_etag.is_none() && kind != TierMutationIntentKind::Add) {
        return Ok(None);
    }
    Ok(Some(TierMutationIntent {
        mutation_id: uuid::Uuid::new_v4(),
        revision: 1,
        kind,
        state: TierMutationIntentState::Prepared,
        old_config_etag,
        committed_config_etag: None,
        candidate_digest: tier_config_candidate_digest(candidate)?,
        affected_targets,
        expires_at_unix_nanos: tier_mutation_intent_expiry_unix_nanos(),
    }))
}

async fn save_coordinator_tier_mutation_intent<S>(api: Arc<S>, intent: Option<&TierMutationIntent>) -> io::Result<()>
where
    S: TierReferenceProofStore,
{
    if let Some(intent) = intent {
        save_tier_coordinator_mutation_intent_record_if_absent(api, intent)
            .await
            .map_err(io::Error::other)?;
    }
    Ok(())
}

async fn commit_coordinator_tier_mutation_intent<S>(
    api: Arc<S>,
    intent: Option<&TierMutationIntent>,
    committed_config_etag: &str,
) -> io::Result<()>
where
    S: TierReferenceProofStore,
{
    if let Some(intent) = intent {
        advance_tier_coordinator_mutation_intent_record_idempotent(
            api,
            intent.mutation_id,
            TierMutationIntentState::Committed,
            Some(committed_config_etag.to_string()),
        )
        .await
        .map_err(io::Error::other)?;
    }
    Ok(())
}

async fn abort_coordinator_tier_mutation_intent<S>(api: Arc<S>, intent: Option<&TierMutationIntent>)
where
    S: TierReferenceProofStore,
{
    let Some(intent) = intent else {
        return;
    };
    match advance_tier_coordinator_mutation_intent_record_idempotent(
        api,
        intent.mutation_id,
        TierMutationIntentState::Aborted,
        None,
    )
    .await
    {
        Ok(_) => {}
        Err(err) => {
            warn!(mutation_id = %intent.mutation_id, error = ?err, "failed to mark coordinator tier mutation intent aborted")
        }
    }
}

async fn apply_tier_candidate_mutation(
    mutation: TierCandidateMutation,
    candidate: &mut TierConfigMgr,
    deadline: Instant,
) -> std::result::Result<Option<String>, AdminError> {
    match timeout_at(deadline, mutation.apply(candidate)).await {
        Ok(result) => result,
        Err(_) => {
            let mut err = ERR_TIER_BACKEND_IN_USE.clone();
            err.message = "Timed out validating the remote tier mutation".to_string();
            Err(err)
        }
    }
}

fn changed_tier_names(current: &TierConfigMgr, replacement: &TierConfigMgr) -> HashSet<String> {
    let mut changed = HashSet::new();
    for (tier_name, current_config) in &current.tiers {
        if replacement
            .tiers
            .get(tier_name)
            .is_none_or(|replacement_config| !tier_configs_match(tier_name, current_config, replacement_config))
        {
            changed.insert(tier_name.clone());
        }
    }
    changed.extend(
        replacement
            .tiers
            .keys()
            .filter(|tier_name| !current.tiers.contains_key(*tier_name))
            .cloned(),
    );
    changed
}

fn replaced_tier_destinations(
    current: &TierConfigMgr,
    replacement: &TierConfigMgr,
) -> std::result::Result<HashMap<String, TierConfig>, AdminError> {
    let mut replaced = HashMap::new();
    for (tier_name, current_config) in &current.tiers {
        let changed = match replacement.tiers.get(tier_name) {
            Some(replacement_config) => {
                if matches!(current_config.tier_type, TierType::Wasabi)
                    != matches!(replacement_config.tier_type, TierType::Wasabi)
                {
                    let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
                    admin_err.message =
                        "Changing an existing remote tier to or from Wasabi is not supported; add a new tier instead".to_string();
                    return Err(admin_err);
                }
                let current_identity = tier_backend_identity(current_config).map_err(|err| {
                    let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
                    admin_err.message = err.to_string();
                    admin_err
                })?;
                let replacement_identity = tier_backend_identity(replacement_config).map_err(|err| {
                    let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
                    admin_err.message = err.to_string();
                    admin_err
                })?;
                current_identity != replacement_identity
            }
            None => true,
        };
        if changed {
            replaced.insert(tier_name.clone(), current_config.clone_with_credentials());
        }
    }
    Ok(replaced)
}

fn encode_tier_backend_identity(
    tier_type: &str,
    endpoint: &str,
    bucket: &str,
    prefix: &str,
    region: &str,
    routing_account: &str,
) -> io::Result<TierDestinationId> {
    let encoded = rmp_serde::to_vec(&(
        TIER_BACKEND_IDENTITY_VERSION,
        tier_type,
        endpoint,
        bucket,
        prefix,
        region,
        routing_account,
    ))
    .map_err(|err| io::Error::other(format!("serialize tier backend identity failed: {err}")))?;
    Ok(Sha256::digest(encoded).into())
}

fn tier_backend_identity(config: &TierConfig) -> io::Result<TierDestinationId> {
    if matches!(config.tier_type, TierType::Wasabi) {
        let wasabi = config
            .wasabi
            .as_ref()
            .ok_or_else(|| io::Error::other("tier backend identity payload is missing"))?;
        return encode_tier_backend_identity(
            "s3",
            &wasabi.canonical_endpoint()?,
            &wasabi.bucket,
            normalized_tier_prefix(&config.tier_type, &wasabi.prefix),
            &wasabi.region,
            "",
        );
    }

    let (tier_type, endpoint, bucket, prefix, region, routing_account) = match config.tier_type {
        TierType::S3 => config.s3.as_ref().map(|value| {
            (
                "s3",
                value.endpoint.as_str(),
                value.bucket.as_str(),
                value.prefix.as_str(),
                value.region.as_str(),
                "",
            )
        }),
        TierType::Wasabi => None,
        TierType::RustFS => config.rustfs.as_ref().map(|value| {
            (
                "rustfs",
                value.endpoint.as_str(),
                value.bucket.as_str(),
                value.prefix.as_str(),
                value.region.as_str(),
                "",
            )
        }),
        TierType::MinIO => config.minio.as_ref().map(|value| {
            (
                "minio",
                value.endpoint.as_str(),
                value.bucket.as_str(),
                value.prefix.as_str(),
                value.region.as_str(),
                "",
            )
        }),
        TierType::Aliyun => config.aliyun.as_ref().map(|value| {
            (
                "aliyun",
                value.endpoint.as_str(),
                value.bucket.as_str(),
                value.prefix.as_str(),
                value.region.as_str(),
                "",
            )
        }),
        TierType::Tencent => config.tencent.as_ref().map(|value| {
            (
                "tencent",
                value.endpoint.as_str(),
                value.bucket.as_str(),
                value.prefix.as_str(),
                value.region.as_str(),
                "",
            )
        }),
        TierType::Huaweicloud => config.huaweicloud.as_ref().map(|value| {
            (
                "huaweicloud",
                value.endpoint.as_str(),
                value.bucket.as_str(),
                value.prefix.as_str(),
                value.region.as_str(),
                "",
            )
        }),
        TierType::Azure => config.azure.as_ref().map(|value| {
            (
                "azure",
                value.endpoint.as_str(),
                value.bucket.as_str(),
                value.prefix.as_str(),
                value.region.as_str(),
                value.access_key.as_str(),
            )
        }),
        TierType::GCS => config.gcs.as_ref().map(|value| {
            (
                "gcs",
                value.endpoint.as_str(),
                value.bucket.as_str(),
                value.prefix.as_str(),
                value.region.as_str(),
                "",
            )
        }),
        TierType::R2 => config.r2.as_ref().map(|value| {
            (
                "r2",
                value.endpoint.as_str(),
                value.bucket.as_str(),
                value.prefix.as_str(),
                value.region.as_str(),
                "",
            )
        }),
        TierType::Unsupported => None,
    }
    .ok_or_else(|| io::Error::other("tier backend identity payload is missing"))?;
    let prefix = normalized_tier_prefix(&config.tier_type, prefix);
    encode_tier_backend_identity(tier_type, endpoint, bucket, prefix, region, routing_account)
}

pub(crate) fn tier_destination_id_from_metadata(metadata: &HashMap<String, String>) -> io::Result<Option<TierDestinationId>> {
    let Some(encoded) = rustfs_utils::http::metadata_compat::get_consistent_str(
        metadata,
        rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
    ) else {
        if rustfs_utils::http::metadata_compat::contains_key_str(
            metadata,
            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
        ) {
            return Err(io::Error::other(
                "transition tier backend identity compatibility keys conflict or are empty",
            ));
        }
        return Ok(None);
    };
    if encoded.len() != 64 {
        return Err(io::Error::other("transition tier backend identity has an invalid length"));
    }
    let mut identity = [0_u8; 32];
    for (index, byte) in identity.iter_mut().enumerate() {
        *byte = u8::from_str_radix(&encoded[index * 2..index * 2 + 2], 16)
            .map_err(|_| io::Error::other("transition tier backend identity is not valid hexadecimal"))?;
    }
    Ok(Some(identity))
}

fn normalized_tier_prefix<'a>(tier_type: &TierType, prefix: &'a str) -> &'a str {
    match tier_type {
        TierType::S3 | TierType::Wasabi => prefix.trim_matches('/'),
        TierType::RustFS
        | TierType::MinIO
        | TierType::Aliyun
        | TierType::Tencent
        | TierType::Huaweicloud
        | TierType::Azure
        | TierType::GCS
        | TierType::R2 => prefix.strip_suffix('/').unwrap_or(prefix),
        TierType::Unsupported => prefix,
    }
}

fn tier_exact_get_delete(config: &TierConfig) -> bool {
    ProviderVersionCapabilities::for_tier_type(&config.tier_type.as_lowercase()).exact_get_delete
}

struct TierDriverGeneration {
    tier_name: Arc<str>,
    tier_config: TierConfig,
    generation: DriverRevision,
    // Process-local only: this may reflect credential changes and must never be persisted or logged.
    config_fingerprint: TierDriverFingerprint,
    // Durable cleanup identity: routing fields only, with all credentials excluded.
    backend_identity: TierDestinationId,
    exact_get_delete: bool,
    driver: SharedWarmBackend,
    reconciler: tokio::sync::OnceCell<
        Option<Arc<dyn crate::services::tier::warm_backend::TransitionCandidateReconciler + Send + Sync + 'static>>,
    >,
    accepting: AtomicBool,
    active_leases: AtomicUsize,
    drained: tokio::sync::Notify,
}

struct SharedWarmBackendProxy(SharedWarmBackend);

#[async_trait::async_trait]
impl WarmBackend for SharedWarmBackendProxy {
    async fn validate(&self) -> io::Result<()> {
        self.0.validate().await
    }

    fn validate_remote_version_id(&self, remote_version_id: &str) -> io::Result<()> {
        self.0.validate_remote_version_id(remote_version_id)
    }

    async fn put(&self, object: &str, r: crate::client::transition_api::ReaderImpl, length: i64) -> io::Result<String> {
        self.0.put(object, r, length).await
    }

    async fn put_with_meta(
        &self,
        object: &str,
        r: crate::client::transition_api::ReaderImpl,
        length: i64,
        meta: HashMap<String, String>,
    ) -> io::Result<String> {
        self.0.put_with_meta(object, r, length, meta).await
    }

    async fn get(
        &self,
        object: &str,
        rv: &str,
        opts: crate::services::tier::warm_backend::WarmBackendGetOpts,
    ) -> io::Result<crate::client::transition_api::ReadCloser> {
        self.0.get(object, rv, opts).await
    }

    async fn remove(&self, object: &str, rv: &str) -> io::Result<()> {
        self.0.remove(object, rv).await
    }

    async fn remove_exact(&self, object: &str, rv: &str) -> io::Result<()> {
        self.0.remove_exact(object, rv).await
    }

    async fn probe_transition_candidate(&self, object: &str) -> io::Result<TransitionCandidateProbe> {
        self.0.probe_transition_candidate(object).await
    }

    async fn in_use(&self) -> io::Result<bool> {
        self.0.in_use().await
    }
}

pub(crate) struct TierOperationLease {
    inner: Arc<TierDriverGeneration>,
    runtime: Arc<Mutex<TierDriverRuntime>>,
}

impl TierOperationLease {
    fn try_new(
        inner: Arc<TierDriverGeneration>,
        runtime: Arc<Mutex<TierDriverRuntime>>,
    ) -> std::result::Result<Self, AdminError> {
        inner
            .active_leases
            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |active| active.checked_add(1))
            .map_err(|_| {
                let mut err = ERR_TIER_INVALID_CONFIG.clone();
                err.message = "Remote tier operation lease capacity exhausted".to_string();
                err
            })?;
        Ok(Self { inner, runtime })
    }

    pub(crate) fn try_clone(&self) -> std::result::Result<Self, AdminError> {
        Self::try_new(self.inner.clone(), self.runtime.clone())
    }
}

impl Drop for TierOperationLease {
    fn drop(&mut self) {
        let result = self
            .inner
            .active_leases
            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |active| active.checked_sub(1));
        match result {
            Ok(1) => self.inner.drained.notify_one(),
            Ok(_) => {}
            Err(_) => {
                error!(
                    tier = self.tier_name(),
                    tier_generation = self.generation(),
                    "tier operation lease counter underflow"
                );
            }
        }
    }
}

impl Deref for TierOperationLease {
    type Target = dyn WarmBackend + Send + Sync + 'static;

    fn deref(&self) -> &Self::Target {
        self.inner.driver.as_ref()
    }
}

impl TierOperationLease {
    pub(crate) fn tier_name(&self) -> &str {
        &self.inner.tier_name
    }

    pub(crate) fn generation(&self) -> DriverRevision {
        self.inner.generation
    }

    pub(crate) fn backend_identity(&self) -> TierDestinationId {
        self.inner.backend_identity
    }

    pub(crate) async fn probe_transition_candidate_for(
        &self,
        object: &str,
        transaction_id: uuid::Uuid,
    ) -> io::Result<TransitionCandidateProbe> {
        let Some(reconciler) = self
            .inner
            .reconciler
            .get_or_try_init(|| async {
                crate::services::tier::warm_backend::new_transition_candidate_reconciler(&self.inner.tier_config)
                    .await
                    .map(|reconciler| reconciler.map(Arc::from))
            })
            .await
            .map_err(|err| io::Error::other(err.message))?
        else {
            return self.inner.driver.probe_transition_candidate(object).await;
        };
        reconciler
            .probe_transition_candidate_for(
                object,
                crate::services::tier::warm_backend::TransitionCandidateIdentity {
                    transaction_id,
                    destination_id: self.backend_identity(),
                },
            )
            .await
    }

    pub(crate) fn validate_remote_version_id(&self, remote_version_id: &str) -> io::Result<()> {
        self.inner.driver.validate_remote_version_id(remote_version_id)?;
        if !remote_version_id.is_empty() && !self.inner.exact_get_delete {
            return Err(io::Error::new(
                io::ErrorKind::Unsupported,
                "remote tier does not support exact versioned GET and DELETE",
            ));
        }
        Ok(())
    }

    pub(crate) fn is_current_generation(&self) -> bool {
        lock_unpoisoned(&self.runtime)
            .generations
            .get(self.tier_name())
            .is_some_and(|current| {
                current.generation == self.generation()
                    && current.accepting.load(Ordering::Acquire)
                    && Arc::ptr_eq(current, &self.inner)
            })
    }

    #[cfg(test)]
    async fn is_current(&self, handle: &Arc<RwLock<TierConfigMgr>>) -> bool {
        let manager = handle.read().await;
        let Some(runtime) = registered_tier_driver_runtime(&manager) else {
            return false;
        };
        if !Arc::ptr_eq(&runtime, &self.runtime) {
            return false;
        }
        self.is_current_generation()
    }
}

impl TierDriverGeneration {
    async fn wait_for_no_active_leases(&self) {
        loop {
            let notified = self.drained.notified();
            if self.active_leases.load(Ordering::Acquire) == 0 {
                return;
            }
            notified.await;
        }
    }
}

#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierConfigMgr {
    #[serde(rename = "Tiers")]
    tiers: HashMap<String, ExternalTierConfig>,
}

#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierConfig {
    #[serde(rename = "Version")]
    version: String,
    #[serde(rename = "Type")]
    tier_type: i32,
    #[serde(rename = "Name")]
    name: String,
    #[serde(rename = "S3")]
    s3: Option<ExternalTierS3>,
    #[serde(rename = "Azure")]
    azure: Option<ExternalTierAzure>,
    #[serde(rename = "GCS")]
    gcs: Option<ExternalTierGcs>,
    #[serde(rename = "MinIO", alias = "Compatible")]
    compatible_backend: Option<ExternalTierCompatible>,
    #[serde(rename = "XTierType", skip_serializing_if = "Option::is_none")]
    tier_type_hint: Option<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierS3 {
    #[serde(rename = "Endpoint")]
    endpoint: String,
    #[serde(rename = "AccessKey")]
    access_key: String,
    #[serde(rename = "SecretKey")]
    secret_key: String,
    #[serde(rename = "Bucket")]
    bucket: String,
    #[serde(rename = "Prefix")]
    prefix: String,
    #[serde(rename = "Region")]
    region: String,
    #[serde(rename = "StorageClass")]
    storage_class: String,
    #[serde(rename = "AWSRole")]
    aws_role: bool,
    #[serde(rename = "AWSRoleWebIdentityTokenFile")]
    aws_role_web_identity_token_file: String,
    #[serde(rename = "AWSRoleARN")]
    aws_role_arn: String,
    #[serde(rename = "AWSRoleSessionName")]
    aws_role_session_name: String,
    #[serde(rename = "AWSRoleDurationSeconds")]
    aws_role_duration_seconds: i32,
}

#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalServicePrincipalAuth {
    #[serde(rename = "TenantID")]
    tenant_id: String,
    #[serde(rename = "ClientID")]
    client_id: String,
    #[serde(rename = "ClientSecret")]
    client_secret: String,
}

#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierAzure {
    #[serde(rename = "Endpoint")]
    endpoint: String,
    #[serde(rename = "AccountName")]
    account_name: String,
    #[serde(rename = "AccountKey")]
    account_key: String,
    #[serde(rename = "Bucket")]
    bucket: String,
    #[serde(rename = "Prefix")]
    prefix: String,
    #[serde(rename = "Region")]
    region: String,
    #[serde(rename = "StorageClass")]
    storage_class: String,
    #[serde(rename = "SPAuth")]
    sp_auth: ExternalServicePrincipalAuth,
}

#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierGcs {
    #[serde(rename = "Endpoint")]
    endpoint: String,
    #[serde(rename = "Creds")]
    creds: String,
    #[serde(rename = "Bucket")]
    bucket: String,
    #[serde(rename = "Prefix")]
    prefix: String,
    #[serde(rename = "Region")]
    region: String,
    #[serde(rename = "StorageClass")]
    storage_class: String,
}

#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierCompatible {
    #[serde(rename = "Endpoint")]
    endpoint: String,
    #[serde(rename = "AccessKey")]
    access_key: String,
    #[serde(rename = "SecretKey")]
    secret_key: String,
    #[serde(rename = "Bucket")]
    bucket: String,
    #[serde(rename = "Prefix")]
    prefix: String,
    #[serde(rename = "Region")]
    region: String,
}

fn tier_config_path(file: &str) -> String {
    format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, file)
}

fn tier_config_lock_path() -> String {
    format!("{}.lock", tier_config_path(TIER_CONFIG_FILE))
}

fn tier_hint_for_type(tier_type: TierType) -> Option<&'static str> {
    match tier_type {
        TierType::RustFS => Some("rustfs"),
        TierType::Wasabi => Some("wasabi"),
        TierType::Aliyun => Some("aliyun"),
        TierType::Tencent => Some("tencent"),
        TierType::Huaweicloud => Some("huaweicloud"),
        TierType::R2 => Some("r2"),
        _ => None,
    }
}

fn tier_type_from_hint(hint: Option<&str>) -> Option<TierType> {
    match hint {
        Some("rustfs") => Some(TierType::RustFS),
        Some("aliyun") => Some(TierType::Aliyun),
        Some("tencent") => Some(TierType::Tencent),
        Some("huaweicloud") => Some(TierType::Huaweicloud),
        Some("r2") => Some(TierType::R2),
        _ => None,
    }
}

fn external_tier_s3_from_internal(s3: &crate::services::tier::tier_config::TierS3) -> ExternalTierS3 {
    ExternalTierS3 {
        endpoint: s3.endpoint.clone(),
        access_key: s3.access_key.clone(),
        secret_key: s3.secret_key.clone(),
        bucket: s3.bucket.clone(),
        prefix: s3.prefix.clone(),
        region: s3.region.clone(),
        storage_class: s3.storage_class.clone(),
        aws_role: s3.aws_role,
        aws_role_web_identity_token_file: s3.aws_role_web_identity_token_file.clone(),
        aws_role_arn: s3.aws_role_arn.clone(),
        aws_role_session_name: s3.aws_role_session_name.clone(),
        aws_role_duration_seconds: s3.aws_role_duration_seconds,
    }
}

fn external_tier_s3_from_compatible_payload(
    endpoint: String,
    access_key: String,
    secret_key: String,
    bucket: String,
    prefix: String,
    region: String,
) -> ExternalTierS3 {
    ExternalTierS3 {
        endpoint,
        access_key,
        secret_key,
        bucket,
        prefix,
        region,
        storage_class: String::new(),
        aws_role: false,
        aws_role_web_identity_token_file: String::new(),
        aws_role_arn: String::new(),
        aws_role_session_name: String::new(),
        aws_role_duration_seconds: 0,
    }
}

fn external_tier_alias_from_compatible_payload(
    endpoint: String,
    access_key: String,
    secret_key: String,
    bucket: String,
    prefix: String,
    region: String,
) -> ExternalTierCompatible {
    ExternalTierCompatible {
        endpoint,
        access_key,
        secret_key,
        bucket,
        prefix,
        region,
    }
}

fn to_external_tier_config(name: &str, tier: &TierConfig) -> io::Result<ExternalTierConfig> {
    let mut out = ExternalTierConfig {
        version: if tier.version.is_empty() {
            "v1".to_string()
        } else {
            tier.version.clone()
        },
        name: if tier.name.is_empty() {
            name.to_string()
        } else {
            tier.name.clone()
        },
        ..Default::default()
    };

    match tier.tier_type {
        TierType::S3 => {
            let s3 = tier
                .s3
                .as_ref()
                .ok_or_else(|| io::Error::other("tier config missing s3 backend payload"))?;
            out.tier_type = EXTERNAL_TIER_TYPE_S3;
            out.s3 = Some(external_tier_s3_from_internal(s3));
        }
        TierType::Wasabi => {
            let backend = tier
                .wasabi
                .as_ref()
                .ok_or_else(|| io::Error::other("tier config missing Wasabi backend payload"))?;
            out.version = EXTERNAL_TIER_VERSION_WASABI_V1.to_string();
            out.tier_type = EXTERNAL_TIER_TYPE_S3;
            out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
            out.s3 = Some(external_tier_s3_from_compatible_payload(
                {
                    backend.canonical_endpoint()?;
                    EXTERNAL_TIER_WASABI_ENDPOINT_SENTINEL.to_string()
                },
                backend.access_key.clone(),
                backend.secret_key.clone(),
                backend.bucket.clone(),
                backend.prefix.clone(),
                backend.region.clone(),
            ));
        }
        TierType::Azure => {
            let az = tier
                .azure
                .as_ref()
                .ok_or_else(|| io::Error::other("tier config missing azure backend payload"))?;
            out.tier_type = EXTERNAL_TIER_TYPE_AZURE;
            out.azure = Some(ExternalTierAzure {
                endpoint: az.endpoint.clone(),
                account_name: az.access_key.clone(),
                account_key: az.secret_key.clone(),
                bucket: az.bucket.clone(),
                prefix: az.prefix.clone(),
                region: az.region.clone(),
                storage_class: az.storage_class.clone(),
                sp_auth: ExternalServicePrincipalAuth {
                    tenant_id: az.sp_auth.tenant_id.clone(),
                    client_id: az.sp_auth.client_id.clone(),
                    client_secret: az.sp_auth.client_secret.clone(),
                },
            });
        }
        TierType::GCS => {
            let gcs = tier
                .gcs
                .as_ref()
                .ok_or_else(|| io::Error::other("tier config missing gcs backend payload"))?;
            out.tier_type = EXTERNAL_TIER_TYPE_GCS;
            out.gcs = Some(ExternalTierGcs {
                endpoint: gcs.endpoint.clone(),
                creds: gcs.creds.clone(),
                bucket: gcs.bucket.clone(),
                prefix: gcs.prefix.clone(),
                region: gcs.region.clone(),
                storage_class: gcs.storage_class.clone(),
            });
        }
        TierType::MinIO => {
            let backend = tier
                .minio
                .as_ref()
                .ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
            out.tier_type = EXTERNAL_TIER_TYPE_MINIO;
            out.compatible_backend = Some(external_tier_alias_from_compatible_payload(
                backend.endpoint.clone(),
                backend.access_key.clone(),
                backend.secret_key.clone(),
                backend.bucket.clone(),
                backend.prefix.clone(),
                backend.region.clone(),
            ));
        }
        TierType::RustFS => {
            let backend = tier
                .rustfs
                .as_ref()
                .ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
            out.tier_type = EXTERNAL_TIER_TYPE_S3;
            out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
            out.s3 = Some(external_tier_s3_from_compatible_payload(
                backend.endpoint.clone(),
                backend.access_key.clone(),
                backend.secret_key.clone(),
                backend.bucket.clone(),
                backend.prefix.clone(),
                backend.region.clone(),
            ));
        }
        TierType::Aliyun => {
            let backend = tier
                .aliyun
                .as_ref()
                .ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
            out.tier_type = EXTERNAL_TIER_TYPE_S3;
            out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
            out.s3 = Some(external_tier_s3_from_compatible_payload(
                backend.endpoint.clone(),
                backend.access_key.clone(),
                backend.secret_key.clone(),
                backend.bucket.clone(),
                backend.prefix.clone(),
                backend.region.clone(),
            ));
        }
        TierType::Tencent => {
            let backend = tier
                .tencent
                .as_ref()
                .ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
            out.tier_type = EXTERNAL_TIER_TYPE_S3;
            out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
            out.s3 = Some(external_tier_s3_from_compatible_payload(
                backend.endpoint.clone(),
                backend.access_key.clone(),
                backend.secret_key.clone(),
                backend.bucket.clone(),
                backend.prefix.clone(),
                backend.region.clone(),
            ));
        }
        TierType::Huaweicloud => {
            let backend = tier
                .huaweicloud
                .as_ref()
                .ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
            out.tier_type = EXTERNAL_TIER_TYPE_S3;
            out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
            out.s3 = Some(external_tier_s3_from_compatible_payload(
                backend.endpoint.clone(),
                backend.access_key.clone(),
                backend.secret_key.clone(),
                backend.bucket.clone(),
                backend.prefix.clone(),
                backend.region.clone(),
            ));
        }
        TierType::R2 => {
            let backend = tier
                .r2
                .as_ref()
                .ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
            out.tier_type = EXTERNAL_TIER_TYPE_S3;
            out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
            out.s3 = Some(external_tier_s3_from_compatible_payload(
                backend.endpoint.clone(),
                backend.access_key.clone(),
                backend.secret_key.clone(),
                backend.bucket.clone(),
                backend.prefix.clone(),
                backend.region.clone(),
            ));
        }
        TierType::Unsupported => {
            out.tier_type = EXTERNAL_TIER_TYPE_UNSUPPORTED;
        }
    }
    Ok(out)
}

fn decode_legacy_s3_like(name: &str, ext: &ExternalTierConfig) -> io::Result<ExternalTierS3> {
    if let Some(s3) = ext.s3.as_ref() {
        return Ok(s3.clone());
    }
    if let Some(m) = ext.compatible_backend.as_ref() {
        return Ok(external_tier_s3_from_compatible_payload(
            m.endpoint.clone(),
            m.access_key.clone(),
            m.secret_key.clone(),
            m.bucket.clone(),
            m.prefix.clone(),
            m.region.clone(),
        ));
    }
    Err(io::Error::other(format!("tier config '{name}' missing compatible backend payload")))
}

fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Result<TierConfig> {
    let mut cfg = TierConfig {
        version: if ext.version.is_empty() {
            "v1".to_string()
        } else {
            ext.version.clone()
        },
        name: if ext.name.is_empty() { name } else { ext.name.clone() },
        ..Default::default()
    };

    let wasabi_hint = ext.tier_type_hint.as_deref() == Some("wasabi");
    let wasabi_version = ext.version == EXTERNAL_TIER_VERSION_WASABI_V1;
    if wasabi_hint && !wasabi_version {
        return Err(io::Error::other(format!(
            "tier config '{}' has inconsistent Wasabi type discriminators",
            cfg.name
        )));
    }
    if (wasabi_hint || wasabi_version) && ext.tier_type != EXTERNAL_TIER_TYPE_S3 {
        return Err(io::Error::other(format!(
            "tier config '{}' has inconsistent Wasabi type discriminators",
            cfg.name
        )));
    }
    if wasabi_version && ext.tier_type_hint.as_deref().is_some_and(|hint| hint != "wasabi") {
        return Err(io::Error::other(format!(
            "tier config '{}' has inconsistent Wasabi type discriminators",
            cfg.name
        )));
    }
    let tier_type = if wasabi_version {
        TierType::Wasabi
    } else {
        tier_type_from_hint(ext.tier_type_hint.as_deref()).unwrap_or_else(|| match ext.tier_type {
            EXTERNAL_TIER_TYPE_S3 => TierType::S3,
            EXTERNAL_TIER_TYPE_AZURE => TierType::Azure,
            EXTERNAL_TIER_TYPE_GCS => TierType::GCS,
            EXTERNAL_TIER_TYPE_MINIO => TierType::MinIO,
            _ => TierType::Unsupported,
        })
    };

    cfg.tier_type = tier_type.clone();

    match tier_type {
        TierType::S3 => {
            let s3 = ext
                .s3
                .as_ref()
                .ok_or_else(|| io::Error::other(format!("tier config '{}' missing s3 backend payload", cfg.name)))?;
            cfg.s3 = Some(crate::services::tier::tier_config::TierS3 {
                name: cfg.name.clone(),
                endpoint: s3.endpoint.clone(),
                access_key: s3.access_key.clone(),
                secret_key: s3.secret_key.clone(),
                bucket: s3.bucket.clone(),
                prefix: s3.prefix.clone(),
                region: s3.region.clone(),
                storage_class: s3.storage_class.clone(),
                aws_role: s3.aws_role,
                aws_role_web_identity_token_file: s3.aws_role_web_identity_token_file.clone(),
                aws_role_arn: s3.aws_role_arn.clone(),
                aws_role_session_name: s3.aws_role_session_name.clone(),
                aws_role_duration_seconds: s3.aws_role_duration_seconds,
            });
        }
        TierType::Wasabi => {
            let s3 = ext
                .s3
                .as_ref()
                .ok_or_else(|| io::Error::other(format!("tier config '{}' missing Wasabi S3 payload", cfg.name)))?;
            if !s3.storage_class.is_empty()
                || s3.aws_role
                || !s3.aws_role_web_identity_token_file.is_empty()
                || !s3.aws_role_arn.is_empty()
                || !s3.aws_role_session_name.is_empty()
                || s3.aws_role_duration_seconds != 0
            {
                return Err(io::Error::other(format!(
                    "tier config '{}' has unsupported Wasabi S3 credential fields",
                    cfg.name
                )));
            }
            if s3.endpoint != EXTERNAL_TIER_WASABI_ENDPOINT_SENTINEL {
                return Err(io::Error::other(format!(
                    "tier config '{}' has an invalid Wasabi compatibility endpoint",
                    cfg.name
                )));
            }
            let mut wasabi = TierWasabi {
                name: cfg.name.clone(),
                endpoint: String::new(),
                access_key: s3.access_key.clone(),
                secret_key: s3.secret_key.clone(),
                bucket: s3.bucket.clone(),
                prefix: s3.prefix.clone(),
                region: s3.region.clone(),
            };
            wasabi.normalize_endpoint()?;
            cfg.wasabi = Some(wasabi);
        }
        TierType::Azure => {
            let az = ext
                .azure
                .as_ref()
                .ok_or_else(|| io::Error::other(format!("tier config '{}' missing azure backend payload", cfg.name)))?;
            cfg.azure = Some(crate::services::tier::tier_config::TierAzure {
                name: cfg.name.clone(),
                endpoint: az.endpoint.clone(),
                access_key: az.account_name.clone(),
                secret_key: az.account_key.clone(),
                bucket: az.bucket.clone(),
                prefix: az.prefix.clone(),
                region: az.region.clone(),
                storage_class: az.storage_class.clone(),
                sp_auth: crate::services::tier::tier_config::ServicePrincipalAuth {
                    tenant_id: az.sp_auth.tenant_id.clone(),
                    client_id: az.sp_auth.client_id.clone(),
                    client_secret: az.sp_auth.client_secret.clone(),
                },
            });
        }
        TierType::GCS => {
            let gcs = ext
                .gcs
                .as_ref()
                .ok_or_else(|| io::Error::other(format!("tier config '{}' missing gcs backend payload", cfg.name)))?;
            cfg.gcs = Some(crate::services::tier::tier_config::TierGCS {
                name: cfg.name.clone(),
                endpoint: gcs.endpoint.clone(),
                creds: gcs.creds.clone(),
                bucket: gcs.bucket.clone(),
                prefix: gcs.prefix.clone(),
                region: gcs.region.clone(),
                storage_class: gcs.storage_class.clone(),
            });
        }
        TierType::MinIO => {
            let m = ext
                .compatible_backend
                .as_ref()
                .ok_or_else(|| io::Error::other(format!("tier config '{}' missing compatible backend payload", cfg.name)))?;
            cfg.minio = Some(crate::services::tier::tier_config::TierMinIO {
                name: cfg.name.clone(),
                endpoint: m.endpoint.clone(),
                access_key: m.access_key.clone(),
                secret_key: m.secret_key.clone(),
                bucket: m.bucket.clone(),
                prefix: m.prefix.clone(),
                region: m.region.clone(),
            });
        }
        TierType::RustFS => {
            let m = decode_legacy_s3_like(&cfg.name, &ext)?;
            cfg.rustfs = Some(crate::services::tier::tier_config::TierRustFS {
                name: cfg.name.clone(),
                endpoint: m.endpoint,
                access_key: m.access_key,
                secret_key: m.secret_key,
                bucket: m.bucket,
                prefix: m.prefix,
                region: m.region,
                storage_class: m.storage_class,
            });
        }
        TierType::Aliyun => {
            let m = decode_legacy_s3_like(&cfg.name, &ext)?;
            cfg.aliyun = Some(crate::services::tier::tier_config::TierAliyun {
                name: cfg.name.clone(),
                endpoint: m.endpoint,
                access_key: m.access_key,
                secret_key: m.secret_key,
                bucket: m.bucket,
                prefix: m.prefix,
                region: m.region,
            });
        }
        TierType::Tencent => {
            let m = decode_legacy_s3_like(&cfg.name, &ext)?;
            cfg.tencent = Some(crate::services::tier::tier_config::TierTencent {
                name: cfg.name.clone(),
                endpoint: m.endpoint,
                access_key: m.access_key,
                secret_key: m.secret_key,
                bucket: m.bucket,
                prefix: m.prefix,
                region: m.region,
            });
        }
        TierType::Huaweicloud => {
            let m = decode_legacy_s3_like(&cfg.name, &ext)?;
            cfg.huaweicloud = Some(crate::services::tier::tier_config::TierHuaweicloud {
                name: cfg.name.clone(),
                endpoint: m.endpoint,
                access_key: m.access_key,
                secret_key: m.secret_key,
                bucket: m.bucket,
                prefix: m.prefix,
                region: m.region,
            });
        }
        TierType::R2 => {
            let m = decode_legacy_s3_like(&cfg.name, &ext)?;
            cfg.r2 = Some(crate::services::tier::tier_config::TierR2 {
                name: cfg.name.clone(),
                endpoint: m.endpoint,
                access_key: m.access_key,
                secret_key: m.secret_key,
                bucket: m.bucket,
                prefix: m.prefix,
                region: m.region,
            });
        }
        TierType::Unsupported => {}
    }
    Ok(cfg)
}

fn encode_external_tiering_config_blob(cfg: &TierConfigMgr) -> io::Result<Bytes> {
    let mut tiers = HashMap::with_capacity(cfg.tiers.len());
    for (name, tier_cfg) in &cfg.tiers {
        tiers.insert(name.clone(), to_external_tier_config(name, tier_cfg)?);
    }
    let payload = rmp_serde::to_vec(&ExternalTierConfigMgr { tiers })
        .map_err(|err| io::Error::other(format!("serialize tier config payload failed: {err}")))?;
    let mut data = Vec::with_capacity(4 + payload.len());
    let mut format = [0u8; 2];
    LittleEndian::write_u16(&mut format, TIER_CONFIG_FORMAT);
    data.extend_from_slice(&format);
    let mut version = [0u8; 2];
    LittleEndian::write_u16(&mut version, TIER_CONFIG_VERSION);
    data.extend_from_slice(&version);
    data.extend_from_slice(&payload);
    Ok(Bytes::from(data))
}

fn decode_external_tiering_config_blob(data: &[u8]) -> io::Result<TierConfigMgr> {
    if data.len() <= 4 {
        return Err(io::Error::other("tierConfigInit: no data"));
    }
    let format = LittleEndian::read_u16(&data[0..2]);
    if format != TIER_CONFIG_FORMAT {
        return Err(io::Error::other(format!("tierConfigInit: unknown format: {format}")));
    }
    let version = LittleEndian::read_u16(&data[2..4]);
    if version != TIER_CONFIG_V1 && version != TIER_CONFIG_VERSION {
        return Err(io::Error::other(format!("tierConfigInit: unknown version: {version}")));
    }

    let external: ExternalTierConfigMgr =
        rmp_serde::from_slice(&data[4..]).map_err(|err| io::Error::other(format!("decode tier config payload failed: {err}")))?;
    let mut tiers = HashMap::with_capacity(external.tiers.len());
    for (name, ext_cfg) in external.tiers {
        tiers.insert(name.clone(), from_external_tier_config(name, ext_cfg)?);
    }
    Ok(TierConfigMgr {
        driver_cache: HashMap::new(),
        tiers,
        last_refreshed_at: OffsetDateTime::now_utc(),
    })
}

fn decode_tiering_config_blob(data: &[u8]) -> io::Result<TierConfigMgr> {
    if let Ok(cfg) = TierConfigMgr::unmarshal(data) {
        return Ok(cfg);
    }
    decode_external_tiering_config_blob(data)
}

impl TierConfigMgr {
    pub fn new() -> Arc<RwLock<Self>> {
        Arc::new(RwLock::new(Self {
            driver_cache: HashMap::new(),
            tiers: HashMap::new(),
            last_refreshed_at: OffsetDateTime::now_utc(),
        }))
    }

    pub fn unmarshal(data: &[u8]) -> std::result::Result<TierConfigMgr, std::io::Error> {
        let mut cfg: TierConfigMgr = serde_json::from_slice(data)?;
        for (name, tier) in &mut cfg.tiers {
            if matches!(&tier.tier_type, TierType::Wasabi) {
                let wasabi = tier
                    .wasabi
                    .as_mut()
                    .ok_or_else(|| io::Error::other(format!("tier config '{name}' missing Wasabi backend payload")))?;
                if wasabi.endpoint.is_empty() {
                    return Err(io::Error::other(format!("tier config '{name}' missing Wasabi endpoint")));
                }
                wasabi.normalize_endpoint()?;
            }
        }
        Ok(cfg)
    }

    pub fn marshal(&self) -> std::result::Result<Bytes, std::io::Error> {
        let data = serde_json::to_vec(&self)?;
        let mut data = Bytes::from(data);

        Ok(data)
    }

    pub fn refreshed_at(&self) -> OffsetDateTime {
        self.last_refreshed_at
    }

    pub fn is_tier_valid(&self, tier_name: &str) -> bool {
        let (_, valid) = self.is_tier_name_in_use(tier_name);
        valid
    }

    pub fn is_tier_name_in_use(&self, tier_name: &str) -> (TierType, bool) {
        if let Some(t) = self.tiers.get(tier_name) {
            return (t.tier_type.clone(), true);
        }
        (TierType::Unsupported, false)
    }

    pub async fn add(&mut self, mut tier_config: TierConfig, force: bool) -> std::result::Result<(), AdminError> {
        self.ensure_generation_is_idle(&tier_config.name)?;
        let tier_name = tier_config.name.clone();
        if tier_name != tier_name.to_uppercase() {
            return Err(ERR_TIER_NAME_NOT_UPPERCASE.clone());
        }

        // Reject AWS/MinIO reserved storage-class names as remote-tier names.
        // MinIO reserves STANDARD and REDUCED_REDUNDANCY (RRS); a tier must not
        // shadow them. The comparison is case-insensitive to match MinIO parity
        // and to stay robust regardless of the uppercase gate above.
        if tier_name.eq_ignore_ascii_case(crate::config::storageclass::STANDARD)
            || tier_name.eq_ignore_ascii_case(crate::config::storageclass::RRS)
        {
            return Err(ERR_TIER_RESERVED_NAME.clone());
        }

        let (_, b) = self.is_tier_name_in_use(&tier_name);
        if b {
            return Err(ERR_TIER_ALREADY_EXISTS.clone());
        }

        if matches!(&tier_config.tier_type, TierType::Wasabi)
            && let Some(wasabi) = tier_config.wasabi.as_mut()
        {
            wasabi.normalize_endpoint().map_err(|source| {
                let mut err = ERR_TIER_INVALID_CONFIG.clone();
                err.message = source.to_string();
                err
            })?;
        }

        let d = new_warm_backend(&tier_config, true).await?;

        if !force {
            let in_use = d.in_use().await;
            match in_use {
                Ok(b) => {
                    if b {
                        return Err(ERR_TIER_BACKEND_IN_USE.clone());
                    }
                }
                Err(err) => {
                    warn!("tier add failed, err: {:?}", err);
                    if err.to_string().contains("connect") {
                        return Err(ERR_TIER_CONNECT_ERR.clone());
                    } else if err.to_string().contains("authorization") {
                        return Err(ERR_TIER_INVALID_CREDENTIALS.clone());
                    } else if err.to_string().contains("bucket") {
                        return Err(ERR_TIER_BUCKET_NOT_FOUND.clone());
                    }
                    let mut e = ERR_TIER_PERM_ERR.clone();
                    e.message.push('.');
                    e.message.push_str(&err.to_string());
                    return Err(e);
                }
            }
        }

        self.tiers.insert(tier_name.clone(), tier_config);
        if let Err(err) = self.replace_driver(&tier_name, d) {
            self.tiers.remove(&tier_name);
            return Err(err);
        }
        Ok(())
    }

    pub async fn remove(&mut self, tier_name: &str, force: bool) -> std::result::Result<(), AdminError> {
        self.ensure_generation_is_idle(tier_name)?;
        let d = self.get_driver(tier_name).await;
        if let Err(err) = d {
            if err.code == ERR_TIER_NOT_FOUND.code {
                return Ok(());
            } else {
                return Err(err);
            }
        }
        if !force {
            if let Ok(driver) = d {
                match driver.in_use().await {
                    Err(err) => {
                        let mut e = ERR_TIER_PERM_ERR.clone();
                        e.message.push('.');
                        e.message.push_str(&err.to_string());
                        return Err(e);
                    }
                    Ok(in_use) if in_use => {
                        return Err(ERR_TIER_BACKEND_NOT_EMPTY.clone());
                    }
                    _ => {}
                }
            }
        }
        self.tiers.remove(tier_name);
        self.revoke_driver(tier_name);
        Ok(())
    }

    pub async fn verify(&mut self, tier_name: &str) -> std::result::Result<(), std::io::Error> {
        let d = match self.get_driver(tier_name).await {
            Ok(d) => d,
            Err(err) => {
                return Err(std::io::Error::other(err));
            }
        };
        if let Err(err) = check_warm_backend(Some(d)).await {
            return Err(std::io::Error::other(err));
        } else {
            return Ok(());
        }
    }

    pub fn empty(&self) -> bool {
        self.list_tiers().len() == 0
    }

    pub fn tier_type(&self, tier_name: &str) -> String {
        if let Some(cfg) = self.tiers.get(tier_name) {
            cfg.tier_type.as_lowercase()
        } else {
            "internal".to_string()
        }
    }

    pub fn list_tiers(&self) -> Vec<TierConfig> {
        let mut tier_cfgs = Vec::<TierConfig>::new();
        for (_, tier) in self.tiers.iter() {
            let tier = tier.clone();
            tier_cfgs.push(tier);
        }
        tier_cfgs
    }

    pub fn get(&self, tier_name: &str) -> Option<TierConfig> {
        for (tier_name2, tier) in self.tiers.iter() {
            if tier_name == tier_name2 {
                return Some(tier.clone());
            }
        }
        None
    }

    pub async fn edit(&mut self, tier_name: &str, creds: TierCreds) -> std::result::Result<(), AdminError> {
        self.ensure_generation_is_idle(tier_name)?;
        let (tier_type, exists) = self.is_tier_name_in_use(tier_name);
        if !exists {
            return Err(ERR_TIER_NOT_FOUND.clone());
        }

        let mut tier_config = self.tiers[tier_name].clone();
        match tier_type {
            TierType::S3 => {
                if let Some(s3) = tier_config.s3.as_mut() {
                    if creds.aws_role {
                        s3.aws_role = true
                    }
                    if creds.aws_role_web_identity_token_file != "" && creds.aws_role_arn != "" {
                        s3.aws_role_arn = creds.aws_role_arn;
                        s3.aws_role_web_identity_token_file = creds.aws_role_web_identity_token_file;
                    }
                    if creds.access_key != "" && creds.secret_key != "" {
                        s3.access_key = creds.access_key;
                        s3.secret_key = creds.secret_key;
                    }
                }
            }
            TierType::Wasabi => {
                if let Some(wasabi) = tier_config.wasabi.as_mut() {
                    if creds.access_key.is_empty() || creds.secret_key.is_empty() {
                        return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
                    }
                    wasabi.access_key = creds.access_key;
                    wasabi.secret_key = creds.secret_key;
                }
            }
            TierType::RustFS => {
                if let Some(rustfs) = tier_config.rustfs.as_mut() {
                    if creds.access_key == "" || creds.secret_key == "" {
                        return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
                    }
                    rustfs.access_key = creds.access_key;
                    rustfs.secret_key = creds.secret_key;
                }
            }
            TierType::MinIO => {
                if let Some(compatible_backend) = tier_config.minio.as_mut() {
                    if creds.access_key == "" || creds.secret_key == "" {
                        return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
                    }
                    compatible_backend.access_key = creds.access_key;
                    compatible_backend.secret_key = creds.secret_key;
                }
            }
            TierType::Aliyun => {
                if let Some(aliyun) = tier_config.aliyun.as_mut() {
                    if creds.access_key == "" || creds.secret_key == "" {
                        return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
                    }
                    aliyun.access_key = creds.access_key;
                    aliyun.secret_key = creds.secret_key;
                }
            }
            TierType::Tencent => {
                if let Some(tencent) = tier_config.tencent.as_mut() {
                    if creds.access_key == "" || creds.secret_key == "" {
                        return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
                    }
                    tencent.access_key = creds.access_key;
                    tencent.secret_key = creds.secret_key;
                }
            }
            TierType::Huaweicloud => {
                if let Some(huaweicloud) = tier_config.huaweicloud.as_mut() {
                    if creds.access_key == "" || creds.secret_key == "" {
                        return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
                    }
                    huaweicloud.access_key = creds.access_key;
                    huaweicloud.secret_key = creds.secret_key;
                }
            }
            TierType::Azure => {
                if let Some(azure) = tier_config.azure.as_mut() {
                    if creds.access_key == "" || creds.secret_key == "" {
                        return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
                    }
                    azure.access_key = creds.access_key;
                    azure.secret_key = creds.secret_key;
                }
            }
            TierType::GCS => {
                if let Some(gcs) = tier_config.gcs.as_mut() {
                    if creds.access_key == "" || creds.secret_key == "" {
                        return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
                    }
                    gcs.creds = creds.access_key; //creds.creds_json
                }
            }
            TierType::R2 => {
                if let Some(r2) = tier_config.r2.as_mut() {
                    if creds.access_key == "" || creds.secret_key == "" {
                        return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
                    }
                    r2.access_key = creds.access_key;
                    r2.secret_key = creds.secret_key;
                }
            }
            _ => (),
        }

        let d = new_warm_backend(&tier_config, true).await?;
        self.revoke_driver(tier_name);
        self.tiers.insert(tier_name.to_string(), tier_config);
        self.replace_driver(tier_name, d)?;
        Ok(())
    }

    pub async fn get_driver<'a>(&'a mut self, tier_name: &str) -> std::result::Result<&'a WarmBackendImpl, AdminError> {
        if registered_tier_driver_runtime(self).is_some_and(|runtime| runtime_blocks_tier(&lock_unpoisoned(&runtime), tier_name))
        {
            let mut err = ERR_TIER_INVALID_CONFIG.clone();
            err.message = TIER_MUTATION_BLOCK_MESSAGE.to_string();
            return Err(err);
        }
        // Return cached driver if present
        if self.driver_cache.contains_key(tier_name) {
            return Ok(self.driver_cache.get(tier_name).expect("Driver not found in cache"));
        }

        // Get tier configuration and create new driver
        let tier_config = self.tiers.get(tier_name).ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;

        let driver = new_warm_backend(tier_config, false).await?;

        self.replace_driver(tier_name, driver)?;
        Ok(self
            .driver_cache
            .get(tier_name)
            .expect("Driver not found in cache after insertion"))
    }

    pub(crate) async fn acquire_operation_lease(
        handle: &Arc<RwLock<Self>>,
        tier_name: &str,
    ) -> std::result::Result<TierOperationLease, AdminError> {
        {
            let manager = handle.read().await;
            let runtime = tier_driver_runtime(handle, &manager);
            let runtime_guard = lock_unpoisoned(&runtime);
            if runtime_blocks_tier(&runtime_guard, tier_name) {
                let mut err = ERR_TIER_INVALID_CONFIG.clone();
                err.message = TIER_MUTATION_BLOCK_MESSAGE.to_string();
                return Err(err);
            }
            if let Some(generation) = runtime_guard
                .generations
                .get(tier_name)
                .filter(|generation| generation.accepting.load(Ordering::Acquire))
                .cloned()
            {
                let lease = TierOperationLease::try_new(generation, runtime.clone());
                drop(runtime_guard);
                return lease;
            }
        }

        let (config, config_fingerprint) = {
            let mut manager = handle.write().await;
            let runtime = tier_driver_runtime(handle, &manager);
            if runtime_blocks_tier(&lock_unpoisoned(&runtime), tier_name) {
                let mut err = ERR_TIER_INVALID_CONFIG.clone();
                err.message = TIER_MUTATION_BLOCK_MESSAGE.to_string();
                return Err(err);
            }
            if let Some(driver) = manager.driver_cache.remove(tier_name) {
                manager.replace_driver(tier_name, driver)?;
                let runtime_guard = lock_unpoisoned(&runtime);
                let generation = runtime_guard
                    .generations
                    .get(tier_name)
                    .filter(|generation| generation.accepting.load(Ordering::Acquire))
                    .ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?
                    .clone();
                let lease = TierOperationLease::try_new(generation, runtime.clone());
                drop(runtime_guard);
                return lease;
            }
            let config = manager
                .tiers
                .get(tier_name)
                .ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?
                .clone_with_credentials();
            let fingerprint = tier_config_fingerprint(tier_name, &config).map_err(|err| {
                let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
                admin_err.message = err.to_string();
                admin_err
            })?;
            (config, fingerprint)
        };

        let driver = build_warm_backend(&config, false).await?;
        let mut manager = handle.write().await;
        let runtime = tier_driver_runtime(handle, &manager);
        let runtime_guard = lock_unpoisoned(&runtime);
        if runtime_blocks_tier(&runtime_guard, tier_name) {
            let mut err = ERR_TIER_INVALID_CONFIG.clone();
            err.message = TIER_MUTATION_BLOCK_MESSAGE.to_string();
            return Err(err);
        }
        if let Some(generation) = runtime_guard
            .generations
            .get(tier_name)
            .filter(|generation| {
                generation.config_fingerprint == config_fingerprint && generation.accepting.load(Ordering::Acquire)
            })
            .cloned()
        {
            let lease = TierOperationLease::try_new(generation, runtime.clone());
            drop(runtime_guard);
            return lease;
        }
        drop(runtime_guard);
        let current = manager.tiers.get(tier_name).ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
        if tier_config_fingerprint(tier_name, current).map_err(|err| {
            let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
            admin_err.message = err.to_string();
            admin_err
        })? != config_fingerprint
        {
            let mut err = ERR_TIER_INVALID_CONFIG.clone();
            err.message = "Remote tier configuration changed while its driver was being prepared".to_string();
            return Err(err);
        }
        manager.replace_driver(tier_name, driver)?;
        let runtime_guard = lock_unpoisoned(&runtime);
        let generation = runtime_guard
            .generations
            .get(tier_name)
            .filter(|generation| generation.accepting.load(Ordering::Acquire))
            .ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?
            .clone();
        let lease = TierOperationLease::try_new(generation, runtime.clone());
        drop(runtime_guard);
        lease
    }

    async fn admin_update_lock(handle: &Arc<RwLock<Self>>) -> tokio::sync::OwnedMutexGuard<()> {
        let update_lock = {
            let manager = handle.read().await;
            let runtime = tier_driver_runtime(handle, &manager);
            lock_unpoisoned(&runtime).admin_updates.clone()
        };
        update_lock.lock_owned().await
    }

    async fn acquire_tier_config_write_lock<S>(
        api: Arc<S>,
    ) -> std::result::Result<rustfs_lock::NamespaceLockGuard, TierConfigUpdateError>
    where
        S: NamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper> + 'static,
    {
        let config_file = tier_config_lock_path();
        let ns_lock = api
            .new_ns_lock(RUSTFS_META_BUCKET, &config_file)
            .await
            .map_err(|err| TierConfigUpdateError::Load(io::Error::other(err)))?;
        ns_lock
            .get_write_lock(get_lock_acquire_timeout())
            .await
            .map_err(|err| TierConfigUpdateError::Load(io::Error::other(err)))
    }

    async fn update_candidate_with_config_lock<S>(
        handle: &Arc<RwLock<Self>>,
        api: Arc<S>,
        mutation: TierCandidateMutation,
    ) -> std::result::Result<(), TierConfigUpdateError>
    where
        S: TierReferenceProofStore + NamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper> + 'static,
    {
        let config_lock = Self::acquire_tier_config_write_lock(api.clone()).await?;
        let update = Self::admin_update_lock(handle).await;
        let (candidate, version) = load_tier_config_for_update(api.clone())
            .await
            .map_err(TierConfigUpdateError::Load)?;
        Self::update_candidate_owned(handle, api, candidate, version, mutation, update, Some(config_lock)).await
    }

    #[cfg(test)]
    pub(crate) async fn publish_candidate(
        handle: &Arc<RwLock<Self>>,
        candidate: Self,
        driver_tier: Option<&str>,
    ) -> std::result::Result<(), AdminError> {
        let update = Self::admin_update_lock(handle).await;
        Self::publish_candidate_owned(handle, candidate, driver_tier.map(str::to_string), update).await
    }

    fn begin_publish_transition_with_allowed_mutation_blocks(
        handle: &Arc<RwLock<Self>>,
        manager: &mut Self,
        candidate: &Self,
        allowed_mutation_blocks: Option<&HashSet<uuid::Uuid>>,
    ) -> std::result::Result<TierPublishTransition, AdminError> {
        let changed = changed_tier_names(manager, candidate);
        let replaced_destinations = replaced_tier_destinations(manager, candidate)?;
        Self::begin_tier_transition_with_destinations(handle, manager, changed, replaced_destinations, allowed_mutation_blocks)
    }

    fn begin_tier_transition(
        handle: &Arc<RwLock<Self>>,
        manager: &mut Self,
        changed: HashSet<String>,
    ) -> std::result::Result<TierPublishTransition, AdminError> {
        Self::begin_tier_transition_with_destinations(handle, manager, changed, HashMap::new(), None)
    }

    fn begin_tier_transition_with_destinations(
        handle: &Arc<RwLock<Self>>,
        manager: &mut Self,
        changed: HashSet<String>,
        replaced_destinations: HashMap<String, TierConfig>,
        allowed_mutation_blocks: Option<&HashSet<uuid::Uuid>>,
    ) -> std::result::Result<TierPublishTransition, AdminError> {
        let runtime = tier_driver_runtime(handle, manager);
        for tier_name in replaced_destinations.keys() {
            if !lock_unpoisoned(&runtime).generations.contains_key(tier_name)
                && let Some(driver) = manager.driver_cache.remove(tier_name)
            {
                manager.replace_driver(tier_name, driver)?;
            }
        }
        Self::begin_tier_transition_in_runtime(runtime, changed, replaced_destinations, allowed_mutation_blocks)
    }

    fn begin_tier_transition_in_runtime(
        runtime: Arc<Mutex<TierDriverRuntime>>,
        changed: HashSet<String>,
        replaced_destinations: HashMap<String, TierConfig>,
        allowed_mutation_blocks: Option<&HashSet<uuid::Uuid>>,
    ) -> std::result::Result<TierPublishTransition, AdminError> {
        let count = u64::try_from(changed.len()).map_err(|_| {
            let mut err = ERR_TIER_INVALID_CONFIG.clone();
            err.message = "Remote tier replacement count exceeds the supported limit".to_string();
            err
        })?;
        let mut runtime_guard = lock_unpoisoned(&runtime);
        if changed
            .iter()
            .any(|tier_name| runtime_blocks_tier_transition(&runtime_guard, tier_name, allowed_mutation_blocks))
        {
            let mut err = ERR_TIER_INVALID_CONFIG.clone();
            err.message = "Remote tier configuration is already being replaced".to_string();
            return Err(err);
        }
        let final_epoch = runtime_guard.next_drain_epoch.checked_add(count).ok_or_else(|| {
            let mut err = ERR_TIER_INVALID_CONFIG.clone();
            err.message = "Remote tier replacement epoch exhausted".to_string();
            err
        })?;
        let mut next_epoch = runtime_guard.next_drain_epoch;
        let mut tokens = Vec::with_capacity(changed.len());
        let mut revoked = HashMap::with_capacity(changed.len());
        for tier_name in changed {
            next_epoch += 1;
            runtime_guard.draining.insert(tier_name.clone(), next_epoch);
            tokens.push((tier_name.clone(), next_epoch));
            if let Some(generation) = runtime_guard.generations.remove(&tier_name) {
                generation.accepting.store(false, Ordering::Release);
                revoked.insert(tier_name, generation);
            }
        }
        runtime_guard.next_drain_epoch = final_epoch;
        drop(runtime_guard);
        Ok(TierPublishTransition {
            runtime,
            tokens,
            revoked,
            replaced_destinations,
            published: false,
        })
    }

    async fn publish_candidate_inner_with_allowed_mutation_blocks(
        handle: &Arc<RwLock<Self>>,
        candidate: Self,
        driver_tier: Option<&str>,
        allowed_mutation_blocks: Option<&HashSet<uuid::Uuid>>,
    ) -> std::result::Result<(), AdminError> {
        let transition = {
            let mut manager = handle.write().await;
            Self::begin_publish_transition_with_allowed_mutation_blocks(
                handle,
                &mut manager,
                &candidate,
                allowed_mutation_blocks,
            )?
        };

        transition.wait_for_active_leases().await?;
        transition.ensure_replaced_destinations_are_empty().await?;
        Self::publish_candidate_after_drain(handle, candidate, driver_tier, transition).await
    }

    async fn publish_candidate_after_drain(
        handle: &Arc<RwLock<Self>>,
        mut candidate: Self,
        driver_tier: Option<&str>,
        mut transition: TierPublishTransition,
    ) -> std::result::Result<(), AdminError> {
        let prepared_driver =
            match driver_tier.and_then(|tier_name| candidate.driver_cache.remove(tier_name).map(|driver| (tier_name, driver))) {
                Some((tier_name, driver)) => {
                    let config = candidate.tiers.get(tier_name).ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
                    let config_fingerprint = tier_config_fingerprint(tier_name, config).map_err(|err| {
                        let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
                        admin_err.message = err.to_string();
                        admin_err
                    })?;
                    let backend_identity = tier_backend_identity(config).map_err(|err| {
                        let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
                        admin_err.message = err.to_string();
                        admin_err
                    })?;
                    let exact_get_delete = tier_exact_get_delete(config);
                    Some(PreparedTierDriver {
                        tier_name: tier_name.to_string(),
                        tier_config: config.clone(),
                        config_fingerprint,
                        backend_identity,
                        exact_get_delete,
                        driver: Arc::from(driver),
                    })
                }
                None => None,
            };
        // Lock order invariant: manager state before the per-manager driver runtime.
        let mut manager = handle.write().await;
        let mut runtime = lock_unpoisoned(&transition.runtime);
        if !transition
            .tokens
            .iter()
            .all(|(tier_name, epoch)| runtime.draining.get(tier_name) == Some(epoch))
        {
            let mut err = ERR_TIER_INVALID_CONFIG.clone();
            err.message = "Remote tier replacement ownership was lost before publish".to_string();
            return Err(err);
        }
        let prepared_generation = if let Some(prepared) = prepared_driver {
            let generation = runtime.next_generation.checked_add(1).ok_or_else(|| {
                let mut err = ERR_TIER_INVALID_CONFIG.clone();
                err.message = "Remote tier driver generation exhausted".to_string();
                err
            })?;
            let entry = Arc::new(TierDriverGeneration {
                tier_name: Arc::from(prepared.tier_name.as_str()),
                tier_config: prepared.tier_config.clone(),
                generation,
                config_fingerprint: prepared.config_fingerprint,
                backend_identity: prepared.backend_identity,
                exact_get_delete: prepared.exact_get_delete,
                driver: prepared.driver.clone(),
                reconciler: tokio::sync::OnceCell::new(),
                accepting: AtomicBool::new(true),
                active_leases: AtomicUsize::new(0),
                drained: tokio::sync::Notify::new(),
            });
            Some((prepared, generation, entry))
        } else {
            None
        };
        for (tier_name, _) in &transition.tokens {
            manager.driver_cache.remove(tier_name);
        }
        manager.tiers = candidate.tiers;
        manager.last_refreshed_at = OffsetDateTime::now_utc();
        if let Some((prepared, generation, entry)) = prepared_generation {
            runtime.generations.insert(prepared.tier_name.clone(), entry);
            runtime.next_generation = generation;
            manager
                .driver_cache
                .insert(prepared.tier_name, Box::new(SharedWarmBackendProxy(prepared.driver)));
        }
        drop(runtime);
        transition.published = true;
        Ok(())
    }

    fn publish_task_error(err: tokio::task::JoinError) -> AdminError {
        error!(error = ?err, "remote tier configuration publish task failed");
        let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
        admin_err.message = format!("Remote tier configuration publish task failed: {err}");
        admin_err
    }

    #[allow(dead_code, reason = "reached only through #[cfg(test)] helpers in this file (backlog#1823)")]
    async fn publish_candidate_owned(
        handle: &Arc<RwLock<Self>>,
        candidate: Self,
        driver_tier: Option<String>,
        update: tokio::sync::OwnedMutexGuard<()>,
    ) -> std::result::Result<(), AdminError> {
        Self::publish_candidate_owned_with_allowed_mutation_blocks(handle, candidate, driver_tier, update, HashSet::new()).await
    }

    async fn publish_candidate_owned_with_allowed_mutation_blocks(
        handle: &Arc<RwLock<Self>>,
        candidate: Self,
        driver_tier: Option<String>,
        update: tokio::sync::OwnedMutexGuard<()>,
        allowed_mutation_blocks: HashSet<uuid::Uuid>,
    ) -> std::result::Result<(), AdminError> {
        let handle = handle.clone();
        tokio::spawn(async move {
            match AssertUnwindSafe(async move {
                let _update = update;
                let allowed_mutation_blocks = (!allowed_mutation_blocks.is_empty()).then_some(&allowed_mutation_blocks);
                Self::publish_candidate_inner_with_allowed_mutation_blocks(
                    &handle,
                    candidate,
                    driver_tier.as_deref(),
                    allowed_mutation_blocks,
                )
                .await
            })
            .catch_unwind()
            .await
            {
                Ok(Ok(())) => Ok(()),
                Ok(Err(err)) => {
                    error!(error = ?err, "remote tier configuration publish failed");
                    Err(err)
                }
                Err(_) => {
                    error!("remote tier configuration publish panicked");
                    let mut err = ERR_TIER_INVALID_CONFIG.clone();
                    err.message = "Remote tier configuration publish panicked".to_string();
                    Err(err)
                }
            }
        })
        .await
        .map_err(Self::publish_task_error)?
    }

    async fn update_candidate_owned<S>(
        handle: &Arc<RwLock<Self>>,
        api: Arc<S>,
        mut candidate: Self,
        version: Option<String>,
        mutation: TierCandidateMutation,
        update: tokio::sync::OwnedMutexGuard<()>,
        config_lock: Option<rustfs_lock::NamespaceLockGuard>,
    ) -> std::result::Result<(), TierConfigUpdateError>
    where
        S: TierReferenceProofStore + 'static,
    {
        let handle = handle.clone();
        #[cfg(test)]
        let test_peers = TIER_MUTATION_TEST_PEERS.try_with(|peers| peers.clone()).ok();
        tokio::spawn(async move {
            let update_task = async move {
                match AssertUnwindSafe(async move {
                    let _config_lock = config_lock;
                    let _update = update;
                    let mutation_kind = mutation.intent_kind();
                    if version.is_none() && !candidate.tiers.is_empty() && mutation_kind != TierMutationIntentKind::Add {
                        return Err(TierConfigUpdateError::Load(io::Error::other(
                            "tier configuration mutation requires an existing config ETag",
                        )));
                    }
                    let explicit_tier_name = mutation.explicit_tier_name().map(str::to_string);
                    let current_for_targets = TierConfigMgr {
                        driver_cache: HashMap::new(),
                        tiers: candidate
                            .tiers
                            .iter()
                            .map(|(tier_name, config)| (tier_name.clone(), config.clone_with_credentials()))
                            .collect(),
                        last_refreshed_at: candidate.last_refreshed_at,
                    };
                    let transition = {
                        let mut manager = handle.write().await;
                        let target_tiers = mutation.target_tiers(&manager, &candidate);
                        Self::begin_tier_transition(&handle, &mut manager, target_tiers)
                            .map_err(TierConfigUpdateError::Publish)?
                    };
                    transition
                        .wait_for_active_leases()
                        .await
                        .map_err(TierConfigUpdateError::Publish)?;
                    let driver_tier =
                        apply_tier_candidate_mutation(mutation, &mut candidate, Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT)
                            .await
                            .map_err(TierConfigUpdateError::Mutation)?;
                    let proof_targets = tier_mutation_proof_targets(
                        mutation_kind,
                        explicit_tier_name.as_deref(),
                        &current_for_targets,
                        &candidate,
                    );
                    let affected_targets =
                        build_tier_mutation_affected_targets(mutation_kind, proof_targets, &current_for_targets, &candidate)
                            .map_err(TierConfigUpdateError::Publish)?;
                    ensure_no_authoritative_tier_object_references(api.clone(), &affected_targets)
                        .await
                        .map_err(TierConfigUpdateError::Publish)?;
                    let coordinator_intent =
                        build_coordinator_tier_mutation_intent(mutation_kind, version.clone(), &candidate, affected_targets)
                            .map_err(TierConfigUpdateError::Save)?;
                    save_coordinator_tier_mutation_intent(api.clone(), coordinator_intent.as_ref())
                        .await
                        .map_err(TierConfigUpdateError::Save)?;
                    let prepared_peers = if let Some(intent) = coordinator_intent.as_ref() {
                        let peers = remote_tier_mutation_peers()
                            .await
                            .map_err(|err| TierConfigUpdateError::Publish(tier_mutation_fanout_admin_error("prepare", err)))?;
                        if peers.is_empty() {
                            Vec::new()
                        } else {
                            match prepare_tier_mutation_peers(intent.mutation_id, peers, intent).await {
                                Ok(prepared) => prepared,
                                Err(failure) => {
                                    if abort_tier_mutation_peers(intent.mutation_id, failure.prepared_peers).await.is_ok() {
                                        abort_coordinator_tier_mutation_intent(api.clone(), coordinator_intent.as_ref()).await;
                                    } else {
                                        warn!(mutation_id = %intent.mutation_id, "prepared coordinator intent retained after peer abort failure");
                                    }
                                    return Err(TierConfigUpdateError::Publish(failure.error));
                                }
                            }
                        }
                    } else {
                        Vec::new()
                    };
                    let saved = match candidate
                        .save_tiering_config_if_current_with_info(api.clone(), version.as_deref())
                        .await
                    {
                        Ok(saved) => saved,
                        Err(err) => {
                            if let Some(intent) = coordinator_intent.as_ref()
                                && !prepared_peers.is_empty()
                            {
                                if abort_tier_mutation_peers(intent.mutation_id, prepared_peers).await.is_err() {
                                    warn!(mutation_id = %intent.mutation_id, "prepared coordinator intent retained after peer abort failure");
                                    return Err(TierConfigUpdateError::Save(err));
                                }
                            }
                            abort_coordinator_tier_mutation_intent(api.clone(), coordinator_intent.as_ref()).await;
                            return Err(TierConfigUpdateError::Save(err));
                        }
                    };
                    let committed_config_etag = saved
                        .etag
                        .filter(|etag| !etag.trim().is_empty())
                        .ok_or_else(|| io::Error::other("tier configuration object is missing an ETag after save"))
                        .map_err(TierConfigUpdateError::Save)?;
                    commit_coordinator_tier_mutation_intent(api, coordinator_intent.as_ref(), &committed_config_etag)
                        .await
                        .map_err(TierConfigUpdateError::Save)?;
                    if let Some(intent) = coordinator_intent.as_ref()
                        && !prepared_peers.is_empty()
                    {
                        commit_tier_mutation_peers(intent.mutation_id, prepared_peers, &committed_config_etag)
                            .await
                            .map_err(|err| TierConfigUpdateError::Publish(tier_mutation_fanout_admin_error("commit", err)))?;
                    }
                    Self::publish_candidate_after_drain(&handle, candidate, driver_tier.as_deref(), transition)
                        .await
                        .map_err(TierConfigUpdateError::Publish)
                })
                .catch_unwind()
                .await
                {
                    Ok(Ok(())) => Ok(()),
                    Ok(Err(err)) => {
                        error!(error = ?err, "remote tier configuration update failed");
                        Err(err)
                    }
                    Err(_) => {
                        error!("remote tier configuration update panicked");
                        let mut err = ERR_TIER_INVALID_CONFIG.clone();
                        err.message = "Remote tier configuration update panicked".to_string();
                        Err(TierConfigUpdateError::Publish(err))
                    }
                }
            };
            #[cfg(test)]
            {
                if let Some(peers) = test_peers {
                    TIER_MUTATION_TEST_PEERS.scope(peers, update_task).await
                } else {
                    update_task.await
                }
            }
            #[cfg(not(test))]
            {
                update_task.await
            }
        })
        .await
        .map_err(|err| TierConfigUpdateError::Publish(Self::publish_task_error(err)))?
    }

    pub async fn reload_handle(handle: &Arc<RwLock<Self>>, api: Arc<ECStore>) -> io::Result<()> {
        Self::reload_handle_with(handle, api).await
    }

    async fn reload_handle_with<S>(handle: &Arc<RwLock<Self>>, api: Arc<S>) -> io::Result<()>
    where
        S: EcstoreObjectIO
            + EcstoreObjectOperations
            + NamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
            + ListOperations<Error = Error, ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>>,
    {
        let mut peer_mutation_intents = Self::load_tier_mutation_intents(api.clone()).await?;
        let mut coordinator_mutation_intents = Self::load_coordinator_mutation_intents(api.clone()).await?;
        // Lock order matches update_candidate_with_config_lock: namespace tier-config lock before admin_updates.
        let config_lock = if peer_mutation_intents
            .iter()
            .any(|intent| intent.state == TierMutationIntentState::Committed)
            || !coordinator_mutation_intents.is_empty()
        {
            let guard = Self::acquire_tier_config_write_lock(api.clone())
                .await
                .map_err(|err| match err {
                    TierConfigUpdateError::Load(err) => err,
                    other => io::Error::other(format!("{other:?}")),
                })?;
            peer_mutation_intents = Self::load_tier_mutation_intents(api.clone()).await?;
            coordinator_mutation_intents = Self::load_coordinator_mutation_intents(api.clone()).await?;
            Some(guard)
        } else {
            None
        };
        let mut prepared_mutation_intents = peer_mutation_intents.clone();
        prepared_mutation_intents.extend(coordinator_mutation_intents.iter().cloned());
        Self::reconcile_prepared_mutation_intents(handle, &prepared_mutation_intents)
            .await
            .map_err(io::Error::other)?;
        Self::recover_prepared_coordinator_mutation_intents(api.clone(), &mut coordinator_mutation_intents).await?;
        let mut all_mutation_intents = peer_mutation_intents.clone();
        all_mutation_intents.extend(coordinator_mutation_intents.iter().cloned());
        Self::reconcile_committed_mutation_intent_blocks(handle, &all_mutation_intents).await?;
        Self::replay_committed_mutation_intents(&all_mutation_intents).await?;
        let allowed_mutation_blocks = all_mutation_intents
            .iter()
            .filter(|intent| matches!(intent.state, TierMutationIntentState::Prepared | TierMutationIntentState::Committed))
            .map(|intent| intent.mutation_id)
            .collect::<HashSet<_>>();
        let update = Self::admin_update_lock(handle).await;
        let candidate = load_tier_config(api.clone()).await?;
        Self::publish_candidate_owned_with_allowed_mutation_blocks(handle, candidate, None, update, allowed_mutation_blocks)
            .await
            .map_err(io::Error::other)?;
        Self::delete_replayed_committed_mutation_intents(api.clone(), &peer_mutation_intents).await?;
        Self::delete_finished_coordinator_mutation_intents(api.clone(), &coordinator_mutation_intents).await?;
        Self::reconcile_prepared_mutation_intents_from_store(handle, api).await?;
        drop(config_lock);
        Ok(())
    }

    async fn load_tier_mutation_intents<S>(api: Arc<S>) -> io::Result<Vec<TierMutationIntent>>
    where
        S: EcstoreObjectIO + ListOperations<Error = Error, ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>>,
    {
        let mut marker = None;
        let mut intents = Vec::new();
        loop {
            let scan = list_tier_mutation_intent_records(api.clone(), TIER_MUTATION_INTENT_RECOVERY_SCAN_LIMIT, marker)
                .await
                .map_err(io::Error::other)?;
            if scan.failed != 0 {
                return Err(io::Error::other(format!(
                    "failed to scan {} tier mutation intent record(s) during recovery",
                    scan.failed
                )));
            }
            intents.extend(scan.intents);
            if !scan.truncated {
                return Ok(intents);
            }
            let next_marker = scan.next_marker.ok_or_else(|| {
                io::Error::other("tier mutation intent recovery scan was truncated without a continuation marker")
            })?;
            marker = Some(next_marker);
        }
    }

    async fn load_prepared_coordinator_mutation_intents<S>(api: Arc<S>) -> io::Result<Vec<TierMutationIntent>>
    where
        S: EcstoreObjectIO + ListOperations<Error = Error, ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>>,
    {
        let mut intents = Self::load_coordinator_mutation_intents(api).await?;
        intents.retain(|intent| intent.state == TierMutationIntentState::Prepared);
        Ok(intents)
    }

    async fn load_coordinator_mutation_intents<S>(api: Arc<S>) -> io::Result<Vec<TierMutationIntent>>
    where
        S: EcstoreObjectIO + ListOperations<Error = Error, ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>>,
    {
        let mut marker = None;
        let mut intents = Vec::new();
        loop {
            let scan =
                list_tier_coordinator_mutation_intent_records(api.clone(), TIER_MUTATION_INTENT_RECOVERY_SCAN_LIMIT, marker)
                    .await
                    .map_err(io::Error::other)?;
            if scan.failed != 0 {
                return Err(io::Error::other(format!(
                    "failed to scan {} coordinator tier mutation intent record(s) during recovery",
                    scan.failed
                )));
            }
            intents.extend(scan.intents);
            if !scan.truncated {
                return Ok(intents);
            }
            let next_marker = scan.next_marker.ok_or_else(|| {
                io::Error::other("coordinator tier mutation intent recovery scan was truncated without a continuation marker")
            })?;
            marker = Some(next_marker);
        }
    }

    async fn recover_prepared_coordinator_mutation_intents<S>(api: Arc<S>, intents: &mut [TierMutationIntent]) -> io::Result<()>
    where
        S: EcstoreObjectIO,
    {
        if !intents.iter().any(|intent| intent.state == TierMutationIntentState::Prepared) {
            return Ok(());
        }
        let (candidate, config_etag) = load_tier_config_for_update(api.clone()).await?;
        let current_digest = tier_config_candidate_digest(&candidate)?;
        let now = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos()).unwrap_or(i64::MAX);
        let mut peers: Option<Vec<Arc<dyn TierMutationPeer>>> = None;
        for intent in intents
            .iter_mut()
            .filter(|intent| intent.state == TierMutationIntentState::Prepared)
        {
            if intent.candidate_digest == current_digest {
                let config_etag = config_etag
                    .as_ref()
                    .ok_or_else(|| io::Error::other("matching coordinator intent has no tier config ETag"))?;
                let (advanced, _) = advance_tier_coordinator_mutation_intent_record_idempotent(
                    api.clone(),
                    intent.mutation_id,
                    TierMutationIntentState::Committed,
                    Some(config_etag.clone()),
                )
                .await
                .map_err(io::Error::other)?;
                *intent = advanced;
                continue;
            }

            let peers = match &peers {
                Some(peers) => peers.clone(),
                None => {
                    let resolved = remote_tier_mutation_peers().await?;
                    peers = Some(resolved.clone());
                    resolved
                }
            };
            let recovery = if intent.expires_at_unix_nanos <= now {
                "expired"
            } else {
                "unmatched"
            };
            abort_tier_mutation_peers(intent.mutation_id, peers)
                .await
                .map_err(|err| io::Error::other(format!("{recovery} coordinator tier mutation recovery abort failed: {err}")))?;
            let (advanced, _) = advance_tier_coordinator_mutation_intent_record_idempotent(
                api.clone(),
                intent.mutation_id,
                TierMutationIntentState::Aborted,
                None,
            )
            .await
            .map_err(io::Error::other)?;
            *intent = advanced;
        }
        Ok(())
    }

    async fn replay_committed_mutation_intents(intents: &[TierMutationIntent]) -> io::Result<()> {
        if !intents
            .iter()
            .any(|intent| intent.state == TierMutationIntentState::Committed)
        {
            return Ok(());
        }
        let peers = remote_tier_mutation_peers().await?;
        if peers.is_empty() {
            return Ok(());
        }
        for intent in intents
            .iter()
            .filter(|intent| intent.state == TierMutationIntentState::Committed)
        {
            let committed_config_etag = intent
                .committed_config_etag
                .as_deref()
                .ok_or_else(|| io::Error::other("committed tier mutation intent is missing committed config etag"))?;
            commit_tier_mutation_peers(intent.mutation_id, peers.clone(), committed_config_etag).await?;
        }
        Ok(())
    }

    async fn delete_replayed_committed_mutation_intents<S>(api: Arc<S>, intents: &[TierMutationIntent]) -> io::Result<()>
    where
        S: EcstoreObjectOperations,
    {
        for intent in intents
            .iter()
            .filter(|intent| intent.state == TierMutationIntentState::Committed)
        {
            delete_tier_mutation_intent_record(api.clone(), intent.mutation_id)
                .await
                .map_err(tier_mutation_replay_error)?;
        }
        Ok(())
    }

    async fn delete_finished_coordinator_mutation_intents<S>(api: Arc<S>, intents: &[TierMutationIntent]) -> io::Result<()>
    where
        S: EcstoreObjectOperations,
    {
        for intent in intents
            .iter()
            .filter(|intent| matches!(intent.state, TierMutationIntentState::Committed | TierMutationIntentState::Aborted))
        {
            delete_tier_coordinator_mutation_intent_record(api.clone(), intent.mutation_id)
                .await
                .map_err(tier_mutation_replay_error)?;
        }
        Ok(())
    }

    async fn reconcile_prepared_mutation_intents(
        handle: &Arc<RwLock<Self>>,
        intents: &[TierMutationIntent],
    ) -> std::result::Result<(), AdminError> {
        Self::reconcile_prepared_mutation_intents_with_revision(handle, intents, None)
            .await
            .map(|_| ())
    }

    async fn reconcile_prepared_mutation_intents_from_store<S>(handle: &Arc<RwLock<Self>>, api: Arc<S>) -> io::Result<()>
    where
        S: EcstoreObjectIO + ListOperations<Error = Error, ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>>,
    {
        for _ in 0..3 {
            let prepared_blocks_revision = Self::prepared_mutation_blocks_revision(handle).await;
            let mut prepared_intents = Self::load_tier_mutation_intents(api.clone()).await?;
            prepared_intents.retain(|intent| intent.state == TierMutationIntentState::Prepared);
            prepared_intents.extend(Self::load_prepared_coordinator_mutation_intents(api.clone()).await?);
            if Self::reconcile_prepared_mutation_intents_with_revision(handle, &prepared_intents, Some(prepared_blocks_revision))
                .await
                .map_err(io::Error::other)?
            {
                return Ok(());
            }
        }
        Err(io::Error::other("tier mutation prepared block recovery changed repeatedly during reload"))
    }

    async fn prepared_mutation_blocks_revision(handle: &Arc<RwLock<Self>>) -> u64 {
        let manager = handle.read().await;
        let runtime = tier_driver_runtime(handle, &manager);
        lock_unpoisoned(&runtime).prepared_mutation_blocks_revision
    }

    async fn reconcile_prepared_mutation_intents_with_revision(
        handle: &Arc<RwLock<Self>>,
        intents: &[TierMutationIntent],
        base_revision: Option<u64>,
    ) -> std::result::Result<bool, AdminError> {
        let mut prepared_mutation_blocks = HashMap::new();
        for intent in intents {
            Self::collect_mutation_intent_block(&mut prepared_mutation_blocks, intent, TierMutationIntentState::Prepared)?;
        }
        let manager = handle.read().await;
        let runtime = tier_driver_runtime(handle, &manager);
        let mut runtime = lock_unpoisoned(&runtime);
        if base_revision.is_some_and(|revision| runtime.prepared_mutation_blocks_revision != revision) {
            return Ok(false);
        }
        let changed = runtime.prepared_mutation_blocks != prepared_mutation_blocks;
        runtime.prepared_mutation_blocks = prepared_mutation_blocks;
        if changed {
            runtime.prepared_mutation_blocks_revision = runtime.prepared_mutation_blocks_revision.saturating_add(1);
        }
        Ok(true)
    }

    async fn reconcile_committed_mutation_intent_blocks(
        handle: &Arc<RwLock<Self>>,
        intents: &[TierMutationIntent],
    ) -> io::Result<()> {
        let mut mutation_blocks = HashMap::new();
        for intent in intents {
            Self::collect_mutation_intent_block(&mut mutation_blocks, intent, TierMutationIntentState::Committed)
                .map_err(io::Error::other)?;
        }
        let manager = handle.read().await;
        let runtime = tier_driver_runtime(handle, &manager);
        let mut runtime = lock_unpoisoned(&runtime);
        runtime.prepared_mutation_blocks = mutation_blocks;
        Ok(())
    }

    pub(crate) async fn apply_prepared_mutation_intent_block(
        handle: &Arc<RwLock<Self>>,
        intent: &TierMutationIntent,
    ) -> std::result::Result<(), AdminError> {
        let manager = handle.read().await;
        let runtime = tier_driver_runtime(handle, &manager);
        let mut runtime = lock_unpoisoned(&runtime);
        let mut prepared_mutation_blocks = runtime.prepared_mutation_blocks.clone();
        Self::collect_mutation_intent_block(&mut prepared_mutation_blocks, intent, TierMutationIntentState::Prepared)?;
        let changed = prepared_mutation_blocks != runtime.prepared_mutation_blocks;
        runtime.prepared_mutation_blocks = prepared_mutation_blocks;
        if changed {
            runtime.prepared_mutation_blocks_revision = runtime.prepared_mutation_blocks_revision.saturating_add(1);
        }
        Ok(())
    }

    pub(crate) async fn clear_prepared_mutation_intent_block(handle: &Arc<RwLock<Self>>, mutation_id: uuid::Uuid) {
        let manager = handle.read().await;
        let Some(runtime) = registered_tier_driver_runtime(&manager) else {
            return;
        };
        let mut runtime = lock_unpoisoned(&runtime);
        let before = runtime.prepared_mutation_blocks.len();
        runtime
            .prepared_mutation_blocks
            .retain(|_, blocked_mutation_id| *blocked_mutation_id != mutation_id);
        if runtime.prepared_mutation_blocks.len() != before {
            runtime.prepared_mutation_blocks_revision = runtime.prepared_mutation_blocks_revision.saturating_add(1);
        }
    }

    fn collect_mutation_intent_block(
        prepared_mutation_blocks: &mut HashMap<String, uuid::Uuid>,
        intent: &TierMutationIntent,
        state: TierMutationIntentState,
    ) -> std::result::Result<(), AdminError> {
        if intent.state != state {
            return Ok(());
        }
        for target in &intent.affected_targets {
            match prepared_mutation_blocks.entry(target.tier_name.clone()) {
                Entry::Vacant(entry) => {
                    entry.insert(intent.mutation_id);
                }
                Entry::Occupied(entry) if *entry.get() == intent.mutation_id => {}
                Entry::Occupied(_) => {
                    let mut err = ERR_TIER_BACKEND_IN_USE.clone();
                    err.message = format!("Remote tier {} already has another prepared mutation", target.tier_name);
                    return Err(err);
                }
            }
        }
        Ok(())
    }

    pub async fn add_and_save(
        handle: &Arc<RwLock<Self>>,
        api: Arc<ECStore>,
        tier_config: TierConfig,
        force: bool,
    ) -> std::result::Result<(), TierConfigUpdateError> {
        Self::update_candidate_with_config_lock(handle, api, TierCandidateMutation::Add(tier_config, force)).await
    }

    pub async fn edit_and_save(
        handle: &Arc<RwLock<Self>>,
        api: Arc<ECStore>,
        tier_name: &str,
        credentials: TierCreds,
    ) -> std::result::Result<(), TierConfigUpdateError> {
        Self::update_candidate_with_config_lock(handle, api, TierCandidateMutation::Edit(tier_name.to_string(), credentials))
            .await
    }

    pub async fn remove_and_save(
        handle: &Arc<RwLock<Self>>,
        api: Arc<ECStore>,
        tier_name: &str,
        force: bool,
    ) -> std::result::Result<(), TierConfigUpdateError> {
        Self::update_candidate_with_config_lock(handle, api, TierCandidateMutation::Remove(tier_name.to_string(), force)).await
    }

    #[allow(dead_code, reason = "reached only through #[cfg(test)] helpers in this file (backlog#1823)")]
    async fn remove_and_save_with<S>(
        handle: &Arc<RwLock<Self>>,
        api: Arc<S>,
        tier_name: &str,
        force: bool,
    ) -> std::result::Result<(), TierConfigUpdateError>
    where
        S: TierReferenceProofStore + 'static,
    {
        let update = Self::admin_update_lock(handle).await;
        let (candidate, version) = load_tier_config_for_update(api.clone())
            .await
            .map_err(TierConfigUpdateError::Load)?;
        Self::update_candidate_owned(
            handle,
            api,
            candidate,
            version,
            TierCandidateMutation::Remove(tier_name.to_string(), force),
            update,
            None,
        )
        .await
    }

    pub async fn clear_and_save(
        handle: &Arc<RwLock<Self>>,
        api: Arc<ECStore>,
        force: bool,
    ) -> std::result::Result<(), TierConfigUpdateError> {
        Self::update_candidate_with_config_lock(handle, api, TierCandidateMutation::Clear(force)).await
    }

    #[allow(dead_code, reason = "reached only through #[cfg(test)] helpers in this file (backlog#1823)")]
    async fn clear_and_save_with<S>(
        handle: &Arc<RwLock<Self>>,
        api: Arc<S>,
        force: bool,
    ) -> std::result::Result<(), TierConfigUpdateError>
    where
        S: TierReferenceProofStore + 'static,
    {
        let update = Self::admin_update_lock(handle).await;
        let (candidate, version) = load_tier_config_for_update(api.clone())
            .await
            .map_err(TierConfigUpdateError::Load)?;
        Self::update_candidate_owned(handle, api, candidate, version, TierCandidateMutation::Clear(force), update, None).await
    }

    pub async fn verify_without_manager_lock(handle: &Arc<RwLock<Self>>, tier_name: &str) -> std::result::Result<(), io::Error> {
        let lease = Self::acquire_operation_lease(handle, tier_name)
            .await
            .map_err(io::Error::other)?;
        let driver: WarmBackendImpl = Box::new(SharedWarmBackendProxy(lease.inner.driver.clone()));
        check_warm_backend(Some(&driver)).await.map_err(io::Error::other)
    }

    pub(crate) async fn acquire_operation_lease_for_backend_identity(
        handle: &Arc<RwLock<Self>>,
        tier_name: &str,
        expected: TierDestinationId,
    ) -> std::result::Result<TierOperationLease, AdminError> {
        let lease = Self::acquire_operation_lease(handle, tier_name).await?;
        if lease.backend_identity() != expected {
            let mut err = ERR_TIER_INVALID_CONFIG.clone();
            err.message = "Remote tier backend identity no longer matches the recorded operation".to_string();
            return Err(err);
        }
        Ok(lease)
    }

    #[cfg(test)]
    #[allow(
        dead_code,
        reason = "lease accounting asserted by a bucket_lifecycle_ops test behind `--features test-util` (backlog#1823)"
    )]
    pub(crate) async fn active_operation_lease_count(handle: &Arc<RwLock<Self>>, tier_name: &str) -> usize {
        let manager = handle.read().await;
        let Some(runtime) = registered_tier_driver_runtime(&manager) else {
            return 0;
        };
        lock_unpoisoned(&runtime)
            .generations
            .get(tier_name)
            .map(|generation| generation.active_leases.load(Ordering::Acquire))
            .unwrap_or(0)
    }

    fn replace_driver(&mut self, tier_name: &str, driver: WarmBackendImpl) -> std::result::Result<(), AdminError> {
        let Some(runtime) = registered_tier_driver_runtime(self) else {
            self.driver_cache.insert(tier_name.to_string(), driver);
            return Ok(());
        };
        let config = self.tiers.get(tier_name).ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
        let config_fingerprint = tier_config_fingerprint(tier_name, config).map_err(|err| {
            let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
            admin_err.message = err.to_string();
            admin_err
        })?;
        let backend_identity = tier_backend_identity(config).map_err(|err| {
            let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
            admin_err.message = err.to_string();
            admin_err
        })?;
        let exact_get_delete = tier_exact_get_delete(config);
        let mut runtime_guard = lock_unpoisoned(&runtime);
        let generation = runtime_guard.next_generation.checked_add(1).ok_or_else(|| {
            let mut err = ERR_TIER_INVALID_CONFIG.clone();
            err.message = "Remote tier driver generation exhausted".to_string();
            err
        })?;
        let driver: SharedWarmBackend = Arc::from(driver);
        let entry = Arc::new(TierDriverGeneration {
            tier_name: Arc::from(tier_name),
            tier_config: config.clone(),
            generation,
            config_fingerprint,
            backend_identity,
            exact_get_delete,
            driver: driver.clone(),
            reconciler: tokio::sync::OnceCell::new(),
            accepting: AtomicBool::new(true),
            active_leases: AtomicUsize::new(0),
            drained: tokio::sync::Notify::new(),
        });

        if let Some(previous) = runtime_guard.generations.insert(tier_name.to_string(), entry) {
            previous.accepting.store(false, Ordering::Release);
        }
        runtime_guard.next_generation = generation;
        drop(runtime_guard);
        self.driver_cache
            .insert(tier_name.to_string(), Box::new(SharedWarmBackendProxy(driver)));
        Ok(())
    }

    #[cfg(feature = "test-util")]
    pub(crate) fn install_test_driver(
        &mut self,
        tier_name: &str,
        driver: WarmBackendImpl,
    ) -> std::result::Result<(), AdminError> {
        self.replace_driver(tier_name, driver)
    }

    fn revoke_driver(&mut self, tier_name: &str) -> Option<Arc<TierDriverGeneration>> {
        let generation =
            registered_tier_driver_runtime(self).and_then(|runtime| lock_unpoisoned(&runtime).generations.remove(tier_name));
        if let Some(generation) = generation.as_ref() {
            generation.accepting.store(false, Ordering::Release);
        }
        self.driver_cache.remove(tier_name);
        generation
    }

    fn ensure_generation_is_idle(&self, tier_name: &str) -> std::result::Result<(), AdminError> {
        let Some(runtime) = registered_tier_driver_runtime(self) else {
            return Ok(());
        };
        let runtime = lock_unpoisoned(&runtime);
        let blocked = runtime_blocks_tier(&runtime, tier_name)
            || runtime
                .generations
                .get(tier_name)
                .is_some_and(|generation| generation.active_leases.load(Ordering::Acquire) != 0);
        if blocked {
            return Err(ERR_TIER_BACKEND_IN_USE.clone());
        }
        Ok(())
    }

    fn ensure_generations_are_idle<'a>(
        &self,
        tier_names: impl IntoIterator<Item = &'a String>,
    ) -> std::result::Result<(), AdminError> {
        for tier_name in tier_names {
            self.ensure_generation_is_idle(tier_name)?;
        }
        Ok(())
    }

    fn revoke_all_drivers(&mut self) {
        if let Some(runtime) = registered_tier_driver_runtime(self) {
            let mut runtime = lock_unpoisoned(&runtime);
            for (_, generation) in runtime.generations.drain() {
                generation.accepting.store(false, Ordering::Release);
            }
        }
        self.driver_cache.clear();
    }

    fn retire_all_drivers(&mut self) {
        self.revoke_all_drivers();
    }

    pub async fn reload(&mut self, api: Arc<ECStore>) -> std::result::Result<(), std::io::Error> {
        let config = load_tier_config(api).await?;
        self.publish_legacy_reload(config).await?;
        Ok(())
    }

    pub(crate) async fn publish_legacy_reload(&mut self, config: Self) -> io::Result<()> {
        let changed = changed_tier_names(self, &config);
        let replaced_destinations = replaced_tier_destinations(self, &config).map_err(io::Error::other)?;
        if let Some(runtime) = registered_tier_driver_runtime(self) {
            for tier_name in replaced_destinations.keys() {
                if !lock_unpoisoned(&runtime).generations.contains_key(tier_name)
                    && let Some(driver) = self.driver_cache.remove(tier_name)
                {
                    self.replace_driver(tier_name, driver).map_err(io::Error::other)?;
                }
            }
            let mut transition = Self::begin_tier_transition_in_runtime(runtime, changed, replaced_destinations, None)
                .map_err(io::Error::other)?;
            transition.wait_for_active_leases().await.map_err(io::Error::other)?;
            transition
                .ensure_replaced_destinations_are_empty()
                .await
                .map_err(io::Error::other)?;
            let runtime = lock_unpoisoned(&transition.runtime);
            if !transition
                .tokens
                .iter()
                .all(|(tier_name, epoch)| runtime.draining.get(tier_name) == Some(epoch))
            {
                return Err(io::Error::other("remote tier replacement ownership was lost before reload publish"));
            }
            for (tier_name, _) in &transition.tokens {
                self.driver_cache.remove(tier_name);
            }
            self.tiers = config.tiers;
            drop(runtime);
            transition.published = true;
        } else {
            ensure_replaced_destinations_are_empty(&replaced_destinations, &HashMap::new())
                .await
                .map_err(io::Error::other)?;
            self.tiers = config.tiers;
        }
        self.last_refreshed_at = OffsetDateTime::now_utc();
        Ok(())
    }

    fn apply_reloaded_tiers(&mut self, tiers: HashMap<String, TierConfig>) -> std::result::Result<(), AdminError> {
        if registered_tier_driver_runtime(self).is_some_and(|runtime| runtime_has_mutation_block(&lock_unpoisoned(&runtime))) {
            return Err(ERR_TIER_BACKEND_IN_USE.clone());
        }
        let changed_or_removed = self
            .tiers
            .iter()
            .filter_map(|(tier_name, current)| {
                let unchanged = tiers
                    .get(tier_name)
                    .is_some_and(|replacement| tier_configs_match(tier_name, current, replacement));
                (!unchanged).then(|| tier_name.clone())
            })
            .collect::<Vec<_>>();
        self.ensure_generations_are_idle(&changed_or_removed)?;
        for tier_name in &changed_or_removed {
            self.revoke_driver(tier_name);
        }
        self.tiers = tiers;
        Ok(())
    }

    pub async fn rollback_after_failed_save(&mut self, api: Arc<ECStore>) -> io::Result<()> {
        match load_tier_config(api).await {
            Ok(config) => {
                self.apply_reloaded_tiers(config.tiers).map_err(io::Error::other)?;
                Ok(())
            }
            Err(err) => {
                if let Some(runtime) = registered_tier_driver_runtime(self) {
                    let runtime = lock_unpoisoned(&runtime);
                    if runtime_has_mutation_block(&runtime)
                        || runtime
                            .generations
                            .values()
                            .any(|generation| generation.active_leases.load(Ordering::Acquire) != 0)
                    {
                        return Err(io::Error::other(format!(
                            "failed to reload tier configuration for rollback while a tier generation is active: {err}"
                        )));
                    }
                }
                self.retire_all_drivers();
                self.tiers.clear();
                Err(err)
            }
        }
    }

    pub async fn clear_tier(&mut self, force: bool) -> std::result::Result<(), AdminError> {
        if registered_tier_driver_runtime(self).is_some_and(|runtime| runtime_has_mutation_block(&lock_unpoisoned(&runtime))) {
            return Err(ERR_TIER_BACKEND_IN_USE.clone());
        }
        self.ensure_generations_are_idle(self.tiers.keys())?;
        if !force {
            let tier_names = self.tiers.keys().cloned().collect::<Vec<_>>();
            for tier_name in tier_names {
                match self.get_driver(&tier_name).await?.in_use().await {
                    Ok(true) => return Err(ERR_TIER_BACKEND_NOT_EMPTY.clone()),
                    Ok(false) => {}
                    Err(err) => {
                        let mut admin_err = ERR_TIER_PERM_ERR.clone();
                        admin_err.message.push('.');
                        admin_err.message.push_str(&err.to_string());
                        return Err(admin_err);
                    }
                }
            }
        }
        self.tiers.clear();
        self.revoke_all_drivers();
        Ok(())
    }

    #[tracing::instrument(level = "debug", name = "tier_save", skip(self))]
    pub async fn save(&self) -> std::result::Result<(), std::io::Error> {
        let Some(api) = runtime_sources::object_store_handle() else {
            return Err(tier_config_not_initialized_error("save tiering config"));
        };

        self.save_tiering_config(api).await
    }

    pub async fn save_tiering_config<S>(&self, api: Arc<S>) -> std::result::Result<(), std::io::Error>
    where
        S: ObjectIO<
                Error = Error,
                RangeSpec = HTTPRangeSpec,
                HeaderMap = HeaderMap,
                ObjectOptions = ObjectOptions,
                ObjectInfo = ObjectInfo,
                GetObjectReader = GetObjectReader,
                PutObjectReader = PutObjReader,
            >,
    {
        let data = encode_external_tiering_config_blob(self)?;
        let config_file = tier_config_path(TIER_CONFIG_FILE);

        self.save_config(api, &config_file, data).await
    }

    #[allow(dead_code, reason = "reached only through #[cfg(test)] helpers in this file (backlog#1823)")]
    async fn save_tiering_config_if_current<S>(
        &self,
        api: Arc<S>,
        version: Option<&str>,
    ) -> std::result::Result<(), std::io::Error>
    where
        S: ObjectIO<
                Error = Error,
                RangeSpec = HTTPRangeSpec,
                HeaderMap = HeaderMap,
                ObjectOptions = ObjectOptions,
                ObjectInfo = ObjectInfo,
                GetObjectReader = GetObjectReader,
                PutObjectReader = PutObjReader,
            >,
    {
        self.save_tiering_config_if_current_with_info(api, version).await?;
        Ok(())
    }

    async fn save_tiering_config_if_current_with_info<S>(
        &self,
        api: Arc<S>,
        version: Option<&str>,
    ) -> std::result::Result<ObjectInfo, std::io::Error>
    where
        S: ObjectIO<
                Error = Error,
                RangeSpec = HTTPRangeSpec,
                HeaderMap = HeaderMap,
                ObjectOptions = ObjectOptions,
                ObjectInfo = ObjectInfo,
                GetObjectReader = GetObjectReader,
                PutObjectReader = PutObjReader,
            >,
    {
        let data = encode_external_tiering_config_blob(self)?;
        let config_file = tier_config_path(TIER_CONFIG_FILE);
        let http_preconditions = match version {
            Some(etag) => HTTPPreconditions {
                if_match: Some(etag.to_string()),
                ..Default::default()
            },
            None => HTTPPreconditions {
                if_none_match: Some("*".to_string()),
                ..Default::default()
            },
        };

        self.save_config_with_opts_info(
            api,
            &config_file,
            data,
            &ObjectOptions {
                max_parity: true,
                http_preconditions: Some(http_preconditions),
                ..Default::default()
            },
        )
        .await
    }

    pub async fn save_config<S>(&self, api: Arc<S>, file: &str, data: Bytes) -> std::result::Result<(), std::io::Error>
    where
        S: ObjectIO<
                Error = Error,
                RangeSpec = HTTPRangeSpec,
                HeaderMap = HeaderMap,
                ObjectOptions = ObjectOptions,
                ObjectInfo = ObjectInfo,
                GetObjectReader = GetObjectReader,
                PutObjectReader = PutObjReader,
            >,
    {
        self.save_config_with_opts(
            api,
            file,
            data,
            &ObjectOptions {
                max_parity: true,
                ..Default::default()
            },
        )
        .await
    }

    pub async fn save_config_with_opts<S>(
        &self,
        api: Arc<S>,
        file: &str,
        data: Bytes,
        opts: &ObjectOptions,
    ) -> std::result::Result<(), std::io::Error>
    where
        S: ObjectIO<
                Error = Error,
                RangeSpec = HTTPRangeSpec,
                HeaderMap = HeaderMap,
                ObjectOptions = ObjectOptions,
                ObjectInfo = ObjectInfo,
                GetObjectReader = GetObjectReader,
                PutObjectReader = PutObjReader,
            >,
    {
        self.save_config_with_opts_info(api, file, data, opts).await?;
        Ok(())
    }

    async fn save_config_with_opts_info<S>(
        &self,
        api: Arc<S>,
        file: &str,
        data: Bytes,
        opts: &ObjectOptions,
    ) -> std::result::Result<ObjectInfo, std::io::Error>
    where
        S: ObjectIO<
                Error = Error,
                RangeSpec = HTTPRangeSpec,
                HeaderMap = HeaderMap,
                ObjectOptions = ObjectOptions,
                ObjectInfo = ObjectInfo,
                GetObjectReader = GetObjectReader,
                PutObjectReader = PutObjReader,
            >,
    {
        debug!("save tier config:{}", file);
        let mut put_data = PutObjReader::from_vec(data.to_vec());
        api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts)
            .await
            .map_err(io::Error::other)
    }

    pub async fn refresh_tier_config(&mut self, api: Arc<ECStore>) {
        let r = rand::rng().random_range(0.0..1.0);
        let rand_interval = || Duration::from_secs((r * 60_f64).round() as u64);

        let mut t = interval(TIER_CFG_REFRESH + rand_interval());
        loop {
            select! {
                _ = t.tick() => {
                    if let Err(err) = self.reload(api.clone()).await {
                      info!("{}", err);
                    }
                }
            }
        }
    }

    pub(crate) async fn refresh_tier_config_handle(handle: Arc<RwLock<Self>>, api: Arc<ECStore>) {
        //let r = rand.New(rand.NewSource(time.Now().UnixNano()));
        let r = rand::rng().random_range(0.0..1.0);
        let rand_interval = || Duration::from_secs((r * 60_f64).round() as u64);

        let refresh_interval = TIER_CFG_REFRESH + rand_interval();
        let mut t = delayed_tier_refresh_interval(refresh_interval);
        loop {
            select! {
                _ = t.tick() => {
                    if let Err(err) = Self::reload_handle(&handle, api.clone()).await {
                      info!("{}", err);
                    }
                }
                else => ()
            }
            t.reset();
        }
    }

    pub async fn init(&mut self, api: Arc<ECStore>) -> Result<()> {
        self.reload(api).await?;
        //if globalIsDistErasure {
        //    self.refresh_tier_config(api).await;
        //}
        Ok(())
    }
}

async fn new_and_save_tiering_config<S>(api: Arc<S>) -> Result<TierConfigMgr>
where
    S: EcstoreObjectIO,
{
    let mut cfg = TierConfigMgr {
        driver_cache: HashMap::new(),
        tiers: HashMap::new(),
        last_refreshed_at: OffsetDateTime::now_utc(),
    };
    //lookup_configs(&mut cfg, api.clone()).await;
    cfg.save_tiering_config(api).await?;

    Ok(cfg)
}

#[tracing::instrument(level = "debug", name = "load_tier_config", skip(api))]
async fn load_tier_config<S>(api: Arc<S>) -> std::result::Result<TierConfigMgr, std::io::Error>
where
    S: EcstoreObjectIO,
{
    let config_file = tier_config_path(TIER_CONFIG_FILE);
    match read_config(api.clone(), config_file.as_str()).await {
        Ok(data) => decode_tiering_config_blob(&data),
        Err(err) if is_err_config_not_found(&err) => {
            let legacy_file = tier_config_path(TIER_CONFIG_LEGACY_FILE);
            match read_config(api.clone(), legacy_file.as_str()).await {
                Ok(data) => {
                    let cfg = TierConfigMgr::unmarshal(&data)?;
                    let normalized = encode_external_tiering_config_blob(&cfg)?;
                    let mut put_data = PutObjReader::from_vec(normalized.to_vec());
                    let _ = api
                        .put_object(
                            RUSTFS_META_BUCKET,
                            &config_file,
                            &mut put_data,
                            &ObjectOptions {
                                max_parity: true,
                                ..Default::default()
                            },
                        )
                        .await;
                    Ok(cfg)
                }
                Err(legacy_err) if is_err_config_not_found(&legacy_err) => {
                    warn!("config not found, start to init");
                    if runtime_sources::first_cluster_node_is_local().await {
                        new_and_save_tiering_config(api).await.map_err(io::Error::other)
                    } else {
                        Ok(TierConfigMgr {
                            driver_cache: HashMap::new(),
                            tiers: HashMap::new(),
                            last_refreshed_at: OffsetDateTime::now_utc(),
                        })
                    }
                }
                Err(legacy_err) => Err(io::Error::other(legacy_err)),
            }
        }
        Err(err) => {
            error!("read config err {:?}", &err);
            Err(io::Error::other(err))
        }
    }
}

async fn load_tier_config_for_update<S>(api: Arc<S>) -> std::result::Result<(TierConfigMgr, Option<String>), std::io::Error>
where
    S: EcstoreObjectIO,
{
    let config_file = tier_config_path(TIER_CONFIG_FILE);
    match read_config_with_metadata(api.clone(), &config_file, &ObjectOptions::default()).await {
        Ok((data, object_info)) => {
            let etag = object_info
                .etag
                .filter(|etag| !etag.trim().is_empty())
                .ok_or_else(|| io::Error::other("tier configuration object is missing an ETag"))?;
            Ok((decode_tiering_config_blob(&data)?, Some(etag)))
        }
        Err(err) if is_err_config_not_found(&err) => {
            let legacy_file = tier_config_path(TIER_CONFIG_LEGACY_FILE);
            match read_config(api, &legacy_file).await {
                Ok(data) => Ok((TierConfigMgr::unmarshal(&data)?, None)),
                Err(legacy_err) if is_err_config_not_found(&legacy_err) => Ok((
                    TierConfigMgr {
                        driver_cache: HashMap::new(),
                        tiers: HashMap::new(),
                        last_refreshed_at: OffsetDateTime::now_utc(),
                    },
                    None,
                )),
                Err(legacy_err) => Err(io::Error::other(legacy_err)),
            }
        }
        Err(err) => Err(io::Error::other(err)),
    }
}

pub(crate) async fn tier_config_etag_matches<S>(api: Arc<S>, expected: &str) -> io::Result<bool>
where
    S: EcstoreObjectIO,
{
    let (_, etag) = load_tier_config_for_update(api).await?;
    Ok(etag.as_deref() == Some(expected))
}

async fn read_tier_config_from_bucket<S>(
    api: Arc<S>,
    bucket: &str,
    path: &str,
    opts: &ObjectOptions,
) -> io::Result<Option<Vec<u8>>>
where
    S: EcstoreObjectIO,
{
    let mut rd = match api.get_object_reader(bucket, path, None, HeaderMap::new(), opts).await {
        Ok(v) => v,
        Err(err) if is_err_config_not_found(&err) => return Ok(None),
        Err(err) => return Err(io::Error::other(err)),
    };
    let data = rd.read_all().await.map_err(io::Error::other)?;
    if data.is_empty() {
        return Ok(None);
    }
    Ok(Some(data))
}

async fn write_tier_config_to_rustfs<S>(api: Arc<S>, path: &str, data: Bytes) -> io::Result<()>
where
    S: EcstoreObjectIO,
{
    let mut put_data = PutObjReader::from_vec(data.to_vec());
    api.put_object(
        RUSTFS_META_BUCKET,
        path,
        &mut put_data,
        &ObjectOptions {
            max_parity: true,
            ..Default::default()
        },
    )
    .await
    .map_err(io::Error::other)?;
    Ok(())
}

pub async fn try_migrate_tiering_config<S>(api: Arc<S>)
where
    S: ObjectIO<
            Error = Error,
            RangeSpec = HTTPRangeSpec,
            HeaderMap = HeaderMap,
            ObjectOptions = ObjectOptions,
            ObjectInfo = ObjectInfo,
            GetObjectReader = GetObjectReader,
            PutObjectReader = PutObjReader,
        > + ObjectOperations<
            Error = Error,
            ObjectInfo = ObjectInfo,
            ObjectOptions = ObjectOptions,
            FileInfo = FileInfo,
            ObjectToDelete = ObjectToDelete,
            DeletedObject = DeletedObject,
        >,
{
    let target_path = tier_config_path(TIER_CONFIG_FILE);
    if api
        .get_object_info(
            RUSTFS_META_BUCKET,
            &target_path,
            &ObjectOptions {
                no_lock: true,
                ..Default::default()
            },
        )
        .await
        .is_ok()
    {
        debug!("tier config already exists in RustFS metadata bucket, skip migration");
        return;
    }

    let opts = ObjectOptions {
        max_parity: true,
        no_lock: true,
        ..Default::default()
    };

    let legacy_path = tier_config_path(TIER_CONFIG_LEGACY_FILE);
    match read_tier_config_from_bucket(api.clone(), RUSTFS_META_BUCKET, &legacy_path, &opts).await {
        Ok(Some(data)) => match TierConfigMgr::unmarshal(&data)
            .and_then(|cfg| encode_external_tiering_config_blob(&cfg).map_err(io::Error::other))
        {
            Ok(out) => {
                if write_tier_config_to_rustfs(api.clone(), &target_path, out).await.is_ok() {
                    info!("Migrated tier config from legacy RustFS metadata format");
                    return;
                }
            }
            Err(err) => debug!(
                bucket = RUSTFS_META_BUCKET,
                path = %legacy_path,
                error = %err,
                "Skipping incompatible legacy tier config migration"
            ),
        },
        Ok(None) => {}
        Err(err) => debug!(
            bucket = RUSTFS_META_BUCKET,
            path = %legacy_path,
            error = %err,
            "Skipping legacy tier config migration after read failure"
        ),
    }

    match read_tier_config_from_bucket(api.clone(), MIGRATING_META_BUCKET, &target_path, &opts).await {
        Ok(Some(data)) => match decode_tiering_config_blob(&data).and_then(|cfg| encode_external_tiering_config_blob(&cfg)) {
            Ok(out) => {
                if write_tier_config_to_rustfs(api.clone(), &target_path, out).await.is_ok() {
                    info!("Migrated compatible tier config from migrating metadata bucket");
                }
            }
            Err(err) => debug!(
                bucket = MIGRATING_META_BUCKET,
                path = %target_path,
                error = %err,
                "Skipping incompatible migrating tier config"
            ),
        },
        Ok(None) => {}
        Err(err) => debug!(
            bucket = MIGRATING_META_BUCKET,
            path = %target_path,
            error = %err,
            "Skipping migrating tier config after read failure"
        ),
    }
}

pub fn is_err_config_not_found(err: &StorageError) -> bool {
    matches!(err, StorageError::ObjectNotFound(_, _) | StorageError::BucketNotFound(_)) || err == &StorageError::ConfigNotFound
}

fn tier_config_not_initialized_error(operation: &str) -> std::io::Error {
    std::io::Error::other(format!("failed to {operation}: object layer not initialized"))
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::layout::{
        endpoint::Endpoint,
        endpoints::{Endpoints, PoolEndpoints, SetupType},
    };

    struct SetupTypeGuard {
        previous: SetupType,
    }

    impl SetupTypeGuard {
        async fn switch_to(next: SetupType) -> Self {
            let previous = runtime_sources::current_setup_type().await;
            runtime_sources::set_setup_type(next).await;
            Self { previous }
        }
    }

    impl Drop for SetupTypeGuard {
        fn drop(&mut self) {
            let previous = self.previous.clone();
            let handle = tokio::runtime::Handle::current();
            tokio::task::block_in_place(|| {
                handle.block_on(async move {
                    runtime_sources::set_setup_type(previous).await;
                });
            });
        }
    }

    fn build_s3_tier(name: &str) -> TierConfig {
        TierConfig {
            version: "v1".to_string(),
            tier_type: TierType::S3,
            name: name.to_string(),
            s3: Some(crate::services::tier::tier_config::TierS3 {
                name: name.to_string(),
                endpoint: "https://example-s3.invalid".to_string(),
                access_key: "ak".to_string(),
                secret_key: "sk".to_string(),
                bucket: "bucket-a".to_string(),
                prefix: "prefix-a".to_string(),
                region: "us-east-1".to_string(),
                storage_class: "STANDARD".to_string(),
                aws_role: false,
                aws_role_web_identity_token_file: String::new(),
                aws_role_arn: String::new(),
                aws_role_session_name: String::new(),
                aws_role_duration_seconds: 0,
            }),
            ..Default::default()
        }
    }

    fn build_wasabi_tier(name: &str) -> TierConfig {
        TierConfig {
            version: "v1".to_string(),
            tier_type: TierType::Wasabi,
            name: name.to_string(),
            wasabi: Some(TierWasabi {
                name: name.to_string(),
                endpoint: "https://s3.ap-northeast-1.wasabisys.com".to_string(),
                access_key: "ak".to_string(),
                secret_key: "sk".to_string(),
                bucket: "wasabi-bucket".to_string(),
                prefix: "archive".to_string(),
                region: "ap-northeast-1".to_string(),
            }),
            ..Default::default()
        }
    }

    #[test]
    fn test_tiering_external_blob_roundtrip_for_standard_type() {
        let mut cfg = TierConfigMgr {
            driver_cache: HashMap::new(),
            tiers: HashMap::new(),
            last_refreshed_at: OffsetDateTime::now_utc(),
        };
        cfg.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A"));

        let bytes = encode_external_tiering_config_blob(&cfg).expect("encode should succeed");
        assert_eq!(&bytes[0..2], &TIER_CONFIG_FORMAT.to_le_bytes());
        assert_eq!(&bytes[2..4], &TIER_CONFIG_VERSION.to_le_bytes());

        let decoded = decode_external_tiering_config_blob(&bytes).expect("decode should succeed");
        let tier = decoded.tiers.get("COLD-A").expect("tier should exist");
        assert_eq!(tier.tier_type.as_lowercase(), "s3");
        assert_eq!(tier.s3.as_ref().expect("s3 should exist").endpoint, "https://example-s3.invalid");
    }

    #[test]
    fn test_tiering_external_blob_roundtrip_for_extended_type_hint() {
        let mut cfg = TierConfigMgr {
            driver_cache: HashMap::new(),
            tiers: HashMap::new(),
            last_refreshed_at: OffsetDateTime::now_utc(),
        };
        cfg.tiers.insert(
            "COLD-B".to_string(),
            TierConfig {
                version: "v1".to_string(),
                tier_type: TierType::RustFS,
                name: "COLD-B".to_string(),
                rustfs: Some(crate::services::tier::tier_config::TierRustFS {
                    name: "COLD-B".to_string(),
                    endpoint: "https://example-compat.invalid".to_string(),
                    access_key: "ak".to_string(),
                    secret_key: "sk".to_string(),
                    bucket: "bucket-b".to_string(),
                    prefix: "prefix-b".to_string(),
                    region: "us-east-1".to_string(),
                    storage_class: "STANDARD".to_string(),
                }),
                ..Default::default()
            },
        );

        let bytes = encode_external_tiering_config_blob(&cfg).expect("encode should succeed");
        let decoded = decode_external_tiering_config_blob(&bytes).expect("decode should succeed");
        let tier = decoded.tiers.get("COLD-B").expect("tier should exist");
        assert_eq!(tier.tier_type.as_lowercase(), "rustfs");
        assert_eq!(
            tier.rustfs.as_ref().expect("backend should exist").endpoint,
            "https://example-compat.invalid"
        );
    }

    #[tokio::test]
    async fn wasabi_external_blob_survives_old_node_rewrite_and_fences_old_drivers() {
        const WASABI_V2_GOLDEN: &[u8] = &[
            1, 0, 2, 0, 145, 129, 171, 67, 79, 76, 68, 45, 87, 65, 83, 65, 66, 73, 152, 176, 114, 117, 115, 116, 102, 115, 45,
            119, 97, 115, 97, 98, 105, 45, 118, 49, 1, 171, 67, 79, 76, 68, 45, 87, 65, 83, 65, 66, 73, 156, 177, 114, 117, 115,
            116, 102, 115, 45, 119, 97, 115, 97, 98, 105, 45, 118, 49, 58, 162, 97, 107, 162, 115, 107, 173, 119, 97, 115, 97,
            98, 105, 45, 98, 117, 99, 107, 101, 116, 167, 97, 114, 99, 104, 105, 118, 101, 174, 97, 112, 45, 110, 111, 114, 116,
            104, 101, 97, 115, 116, 45, 49, 160, 194, 160, 160, 160, 0, 192, 192, 192, 166, 119, 97, 115, 97, 98, 105,
        ];

        let mut cfg = empty_mgr();
        let mut tier = build_wasabi_tier("COLD-WASABI");
        tier.wasabi.as_mut().expect("Wasabi payload should exist").endpoint.clear();
        cfg.tiers.insert("COLD-WASABI".to_string(), tier);

        let bytes = encode_external_tiering_config_blob(&cfg).expect("Wasabi tier should encode");
        assert_eq!(bytes.as_ref(), WASABI_V2_GOLDEN, "Wasabi v2 bytes must remain stable");
        assert_eq!(&bytes[0..2], &TIER_CONFIG_FORMAT.to_le_bytes());
        assert_eq!(&bytes[2..4], &TIER_CONFIG_VERSION.to_le_bytes());
        let external: ExternalTierConfigMgr = rmp_serde::from_slice(&bytes[4..]).expect("external Wasabi payload should decode");
        let stored = external.tiers.get("COLD-WASABI").expect("stored Wasabi tier should exist");
        assert_eq!(stored.version, EXTERNAL_TIER_VERSION_WASABI_V1);
        assert_eq!(stored.tier_type, EXTERNAL_TIER_TYPE_S3);
        assert_eq!(stored.tier_type_hint.as_deref(), Some("wasabi"));
        assert!(stored.compatible_backend.is_none());
        let s3 = stored.s3.as_ref().expect("Wasabi should use the S3 payload");
        assert_eq!(s3.endpoint, EXTERNAL_TIER_WASABI_ENDPOINT_SENTINEL);
        assert_eq!(s3.bucket, "wasabi-bucket");
        assert_eq!(s3.prefix, "archive");
        assert_eq!(s3.region, "ap-northeast-1");
        assert!(s3.storage_class.is_empty());

        let decoded = decode_external_tiering_config_blob(&bytes).expect("Wasabi tier should decode");
        let wasabi = decoded.tiers["COLD-WASABI"]
            .wasabi
            .as_ref()
            .expect("Wasabi payload should survive roundtrip");
        assert_eq!(wasabi.endpoint, "https://s3.ap-northeast-1.wasabisys.com");
        assert_eq!(wasabi.secret_key, "sk");
        assert_eq!(decoded.tiers["COLD-WASABI"].version, EXTERNAL_TIER_VERSION_WASABI_V1);

        let encode_fixture = |external: &ExternalTierConfigMgr| {
            let payload = rmp_serde::to_vec(external).expect("Wasabi fixture should encode");
            let mut fixture = Vec::with_capacity(4 + payload.len());
            fixture.extend_from_slice(&TIER_CONFIG_FORMAT.to_le_bytes());
            fixture.extend_from_slice(&TIER_CONFIG_VERSION.to_le_bytes());
            fixture.extend_from_slice(&payload);
            fixture
        };

        // Current older nodes preserve the per-tier Version and S3 payload when
        // rewriting the binary document, but drop an unrecognized XTierType.
        let mut old_node_rewrite = external.clone();
        old_node_rewrite
            .tiers
            .get_mut("COLD-WASABI")
            .expect("stored Wasabi tier should exist")
            .tier_type_hint = None;
        let rewritten = encode_fixture(&old_node_rewrite);
        let decoded = decode_external_tiering_config_blob(&rewritten)
            .expect("the durable per-tier marker must restore Wasabi after an old-node rewrite");
        assert!(matches!(decoded.tiers["COLD-WASABI"].tier_type, TierType::Wasabi));

        let old_s3_payload = old_node_rewrite.tiers["COLD-WASABI"]
            .s3
            .as_ref()
            .expect("old nodes should retain the physical S3 payload");
        let old_s3 = crate::services::tier::tier_config::TierS3 {
            name: "COLD-WASABI".to_string(),
            endpoint: old_s3_payload.endpoint.clone(),
            access_key: old_s3_payload.access_key.clone(),
            secret_key: old_s3_payload.secret_key.clone(),
            bucket: old_s3_payload.bucket.clone(),
            prefix: old_s3_payload.prefix.clone(),
            region: old_s3_payload.region.clone(),
            ..Default::default()
        };
        let err = match crate::services::tier::warm_backend_s3::WarmBackendS3::new(&old_s3, "COLD-WASABI").await {
            Ok(_) => panic!("an older generic S3 driver must not operate a Wasabi compatibility envelope"),
            Err(err) => err,
        };
        assert!(err.to_string().contains("missing a host"), "{err}");

        let mut generic_s3 = old_node_rewrite.clone();
        generic_s3
            .tiers
            .get_mut("COLD-WASABI")
            .expect("stored Wasabi tier should exist")
            .version = "v1".to_string();
        let decoded = decode_external_tiering_config_blob(&encode_fixture(&generic_s3))
            .expect("a generic S3 tier without an explicit marker must stay generic");
        assert!(matches!(decoded.tiers["COLD-WASABI"].tier_type, TierType::S3));

        let mut hint_only = external.clone();
        hint_only
            .tiers
            .get_mut("COLD-WASABI")
            .expect("stored Wasabi tier should exist")
            .version = "v1".to_string();
        let err = expect_decode_err(&encode_fixture(&hint_only));
        assert!(err.to_string().contains("inconsistent Wasabi type discriminators"), "{err}");

        let mut wrong_type = old_node_rewrite.clone();
        wrong_type
            .tiers
            .get_mut("COLD-WASABI")
            .expect("stored Wasabi tier should exist")
            .tier_type = EXTERNAL_TIER_TYPE_MINIO;
        let err = expect_decode_err(&encode_fixture(&wrong_type));
        assert!(err.to_string().contains("inconsistent Wasabi type discriminators"), "{err}");

        let mut wrong_hint = external.clone();
        wrong_hint
            .tiers
            .get_mut("COLD-WASABI")
            .expect("stored Wasabi tier should exist")
            .tier_type_hint = Some("rustfs".to_string());
        let err = expect_decode_err(&encode_fixture(&wrong_hint));
        assert!(err.to_string().contains("inconsistent Wasabi type discriminators"), "{err}");

        let poison_fields: [(&str, fn(&mut ExternalTierS3)); 6] = [
            ("storage_class", |s3| s3.storage_class = "GLACIER".to_string()),
            ("aws_role", |s3| s3.aws_role = true),
            ("web_identity_token", |s3| s3.aws_role_web_identity_token_file = "/tmp/token".to_string()),
            ("role_arn", |s3| s3.aws_role_arn = "arn:aws:iam::1:role/test".to_string()),
            ("role_session", |s3| s3.aws_role_session_name = "session".to_string()),
            ("role_duration", |s3| s3.aws_role_duration_seconds = 900),
        ];
        for (field, poison) in poison_fields {
            let mut unsupported_credentials = external.clone();
            let s3 = unsupported_credentials
                .tiers
                .get_mut("COLD-WASABI")
                .expect("stored Wasabi tier should exist")
                .s3
                .as_mut()
                .expect("Wasabi S3 payload should exist");
            poison(s3);
            let err = expect_decode_err(&encode_fixture(&unsupported_credentials));
            assert!(
                err.to_string().contains("unsupported Wasabi S3 credential fields"),
                "field {field}: {err}"
            );
        }
    }

    #[test]
    fn legacy_extended_type_hint_keeps_precedence_over_numeric_type() {
        let mut external = ExternalTierConfigMgr::default();
        external.tiers.insert(
            "COLD-LEGACY".to_string(),
            ExternalTierConfig {
                version: "v1".to_string(),
                tier_type: EXTERNAL_TIER_TYPE_MINIO,
                name: "COLD-LEGACY".to_string(),
                tier_type_hint: Some("rustfs".to_string()),
                s3: Some(ExternalTierS3 {
                    endpoint: "https://rustfs.example.invalid".to_string(),
                    access_key: "ak".to_string(),
                    secret_key: "sk".to_string(),
                    bucket: "archive".to_string(),
                    prefix: "objects".to_string(),
                    region: "us-east-1".to_string(),
                    ..Default::default()
                }),
                compatible_backend: Some(ExternalTierCompatible {
                    endpoint: "https://minio.example.invalid".to_string(),
                    bucket: "different".to_string(),
                    ..Default::default()
                }),
                ..Default::default()
            },
        );
        let payload = rmp_serde::to_vec(&external).expect("legacy fixture should encode");
        let mut blob = Vec::with_capacity(4 + payload.len());
        blob.extend_from_slice(&TIER_CONFIG_FORMAT.to_le_bytes());
        blob.extend_from_slice(&TIER_CONFIG_VERSION.to_le_bytes());
        blob.extend_from_slice(&payload);

        let decoded = decode_external_tiering_config_blob(&blob).expect("legacy extended hint should decode as before");
        let rustfs = decoded.tiers["COLD-LEGACY"]
            .rustfs
            .as_ref()
            .expect("recognized legacy hint must override the numeric type");
        assert_eq!(rustfs.endpoint, "https://rustfs.example.invalid");
        assert_eq!(rustfs.bucket, "archive");
    }

    #[test]
    fn wasabi_external_blob_rejects_noncanonical_endpoint_and_unsafe_region() {
        let mut cfg = empty_mgr();
        cfg.tiers.insert("COLD-WASABI".to_string(), build_wasabi_tier("COLD-WASABI"));
        let bytes = encode_external_tiering_config_blob(&cfg).expect("Wasabi tier should encode");
        let base: ExternalTierConfigMgr = rmp_serde::from_slice(&bytes[4..]).expect("external Wasabi payload should decode");

        for (endpoint, region, expected) in [
            ("https://s3.example.invalid", "ap-northeast-1", "invalid Wasabi compatibility endpoint"),
            (EXTERNAL_TIER_WASABI_ENDPOINT_SENTINEL, "ap.northeast-1", "invalid Wasabi region"),
        ] {
            let mut external = base.clone();
            let s3 = external
                .tiers
                .get_mut("COLD-WASABI")
                .and_then(|tier| tier.s3.as_mut())
                .expect("stored Wasabi S3 payload should exist");
            s3.endpoint = endpoint.to_string();
            s3.region = region.to_string();
            let payload = rmp_serde::to_vec(&external).expect("corrupt fixture should encode");
            let mut corrupt = Vec::with_capacity(4 + payload.len());
            corrupt.extend_from_slice(&TIER_CONFIG_FORMAT.to_le_bytes());
            corrupt.extend_from_slice(&TIER_CONFIG_VERSION.to_le_bytes());
            corrupt.extend_from_slice(&payload);

            let err = expect_decode_err(&corrupt);
            assert!(err.to_string().contains(expected), "{err}");
        }

        let mut external = base;
        let stored = external
            .tiers
            .get_mut("COLD-WASABI")
            .expect("stored Wasabi tier should exist");
        let s3 = stored.s3.take().expect("stored Wasabi S3 payload should exist");
        stored.compatible_backend = Some(external_tier_alias_from_compatible_payload(
            s3.endpoint,
            s3.access_key,
            s3.secret_key,
            s3.bucket,
            s3.prefix,
            s3.region,
        ));
        let payload = rmp_serde::to_vec(&external).expect("wrong-payload fixture should encode");
        let mut corrupt = Vec::with_capacity(4 + payload.len());
        corrupt.extend_from_slice(&TIER_CONFIG_FORMAT.to_le_bytes());
        corrupt.extend_from_slice(&TIER_CONFIG_VERSION.to_le_bytes());
        corrupt.extend_from_slice(&payload);
        let err = expect_decode_err(&corrupt);
        assert!(err.to_string().contains("missing Wasabi S3 payload"), "{err}");
    }

    #[test]
    fn test_decode_tiering_config_blob_accepts_legacy_json() {
        let mut cfg = TierConfigMgr {
            driver_cache: HashMap::new(),
            tiers: HashMap::new(),
            last_refreshed_at: OffsetDateTime::now_utc(),
        };
        cfg.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A"));

        let data = serde_json::to_vec(&cfg).expect("legacy json should encode");
        let decoded = decode_tiering_config_blob(&data).expect("legacy json should decode");
        assert_eq!(
            decoded
                .tiers
                .get("COLD-A")
                .and_then(|tier| tier.s3.as_ref())
                .map(|s3| s3.bucket.as_str()),
            Some("bucket-a")
        );
    }

    #[test]
    fn wasabi_legacy_json_rejects_missing_payload_and_invalid_endpoints() {
        let mut cfg = empty_mgr();
        let mut tier = build_wasabi_tier("COLD-WASABI");
        tier.wasabi.as_mut().expect("Wasabi payload should exist").endpoint = "https://s3.example.invalid".to_string();
        cfg.tiers.insert("COLD-WASABI".to_string(), tier);

        let data = serde_json::to_vec(&cfg).expect("legacy JSON fixture should encode");
        let err = match TierConfigMgr::unmarshal(&data) {
            Ok(_) => panic!("legacy Wasabi endpoint mismatch must fail closed"),
            Err(err) => err,
        };
        assert!(err.to_string().contains("https://s3.ap-northeast-1.wasabisys.com"));

        cfg.tiers
            .get_mut("COLD-WASABI")
            .and_then(|tier| tier.wasabi.as_mut())
            .expect("Wasabi payload should exist")
            .endpoint
            .clear();
        let data = serde_json::to_vec(&cfg).expect("empty-endpoint fixture should encode");
        let err = match TierConfigMgr::unmarshal(&data) {
            Ok(_) => panic!("legacy Wasabi endpoint must not be empty"),
            Err(err) => err,
        };
        assert!(err.to_string().contains("missing Wasabi endpoint"), "{err}");

        cfg.tiers.get_mut("COLD-WASABI").expect("Wasabi tier should exist").wasabi = None;
        let data = serde_json::to_vec(&cfg).expect("missing-payload fixture should encode");
        let err = match TierConfigMgr::unmarshal(&data) {
            Ok(_) => panic!("legacy Wasabi payload must exist"),
            Err(err) => err,
        };
        assert!(err.to_string().contains("missing Wasabi backend payload"), "{err}");
    }

    #[test]
    fn test_tier_config_not_initialized_error_formats_operation_context() {
        let err = tier_config_not_initialized_error("save tiering config");
        let rendered = err.to_string();

        assert!(rendered.contains("failed to save tiering config"), "{rendered}");
        assert!(rendered.contains("object layer not initialized"), "{rendered}");
    }

    // ---------------------------------------------------------------------
    // State-machine coverage (ilm-4): add / edit / remove / verify.
    //
    // These tests must not reach a real remote tier. Two techniques keep them
    // hermetic:
    //   * error paths that return *before* `new_warm_backend` constructs a
    //     client (name validation, duplicate detection, unsupported type,
    //     missing backend payload, missing credentials);
    //   * a `MockWarmBackend` injected directly into `driver_cache`, so
    //     `get_driver` returns it from cache without any network I/O, which
    //     lets us drive `remove`/`verify` through every branch.
    // ---------------------------------------------------------------------

    use crate::client::transition_api::{ReadCloser, ReaderImpl};
    use crate::services::tier::warm_backend::{WarmBackend, WarmBackendGetOpts};

    fn empty_mgr() -> TierConfigMgr {
        TierConfigMgr {
            driver_cache: HashMap::new(),
            tiers: HashMap::new(),
            last_refreshed_at: OffsetDateTime::now_utc(),
        }
    }

    fn build_rustfs_tier(name: &str) -> TierConfig {
        TierConfig {
            version: "v1".to_string(),
            tier_type: TierType::RustFS,
            name: name.to_string(),
            rustfs: Some(crate::services::tier::tier_config::TierRustFS {
                name: name.to_string(),
                endpoint: "https://example-compat.invalid".to_string(),
                access_key: "ak".to_string(),
                secret_key: "sk".to_string(),
                bucket: "bucket-r".to_string(),
                prefix: "prefix-r".to_string(),
                region: "us-east-1".to_string(),
                storage_class: "STANDARD".to_string(),
            }),
            ..Default::default()
        }
    }

    #[derive(Debug)]
    struct LockingTierConfigStore {
        locks: Mutex<Vec<(String, String)>>,
        put_after_lock: AtomicBool,
        lock_manager: Arc<rustfs_lock::GlobalLockManager>,
    }

    impl LockingTierConfigStore {
        fn new() -> Self {
            Self {
                locks: Mutex::new(Vec::new()),
                put_after_lock: AtomicBool::new(false),
                lock_manager: Arc::new(rustfs_lock::GlobalLockManager::new()),
            }
        }

        fn lock_events(&self) -> Vec<(String, String)> {
            self.locks
                .lock()
                .expect("tier config lock recorder should not poison")
                .clone()
        }
    }

    #[async_trait::async_trait]
    impl ObjectIO for LockingTierConfigStore {
        type Error = Error;
        type RangeSpec = HTTPRangeSpec;
        type HeaderMap = HeaderMap;
        type ObjectOptions = ObjectOptions;
        type ObjectInfo = ObjectInfo;
        type GetObjectReader = GetObjectReader;
        type PutObjectReader = PutObjReader;

        async fn get_object_reader(
            &self,
            bucket: &str,
            object: &str,
            _range: Option<HTTPRangeSpec>,
            _h: HeaderMap,
            _opts: &ObjectOptions,
        ) -> Result<GetObjectReader> {
            Err(Error::ObjectNotFound(bucket.to_string(), object.to_string()))
        }

        async fn put_object(
            &self,
            bucket: &str,
            object: &str,
            _data: &mut PutObjReader,
            opts: &ObjectOptions,
        ) -> Result<ObjectInfo> {
            let expected_lock = (RUSTFS_META_BUCKET.to_string(), tier_config_lock_path());
            let saw_expected_lock = self
                .locks
                .lock()
                .expect("tier config lock recorder should not poison")
                .contains(&expected_lock);
            self.put_after_lock.store(saw_expected_lock, Ordering::SeqCst);
            assert!(opts.max_parity, "tier config save must retain max parity");
            assert!(
                opts.http_preconditions
                    .as_ref()
                    .and_then(|preconditions| preconditions.if_none_match.as_deref())
                    == Some("*"),
                "initial tier config save must retain if-none-match protection"
            );
            Ok(ObjectInfo {
                bucket: bucket.to_string(),
                name: object.to_string(),
                etag: Some("saved-etag".to_string()),
                ..Default::default()
            })
        }
    }

    #[async_trait::async_trait]
    impl BucketOperations for LockingTierConfigStore {
        type Error = Error;

        async fn make_bucket(
            &self,
            _bucket: &str,
            _opts: &crate::storage_api_contracts::bucket::MakeBucketOptions,
        ) -> Result<()> {
            Err(Error::NotImplemented)
        }

        async fn get_bucket_info(
            &self,
            _bucket: &str,
            _opts: &crate::storage_api_contracts::bucket::BucketOptions,
        ) -> Result<crate::storage_api_contracts::bucket::BucketInfo> {
            Err(Error::NotImplemented)
        }

        async fn list_bucket(
            &self,
            _opts: &crate::storage_api_contracts::bucket::BucketOptions,
        ) -> Result<Vec<crate::storage_api_contracts::bucket::BucketInfo>> {
            Ok(Vec::new())
        }

        async fn delete_bucket(
            &self,
            _bucket: &str,
            _opts: &crate::storage_api_contracts::bucket::DeleteBucketOptions,
        ) -> Result<()> {
            Err(Error::NotImplemented)
        }
    }

    #[async_trait::async_trait]
    impl ListOperations for LockingTierConfigStore {
        type Error = Error;
        type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
        type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
        type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
        type WalkOptions = TierReferenceProofWalkOptions;
        type WalkCancellation = tokio_util::sync::CancellationToken;
        type WalkResultSender = tokio::sync::mpsc::Sender<StorageObjectInfoOrErr<ObjectInfo, Error>>;

        async fn list_objects_v2(
            self: Arc<Self>,
            _bucket: &str,
            _prefix: &str,
            _continuation_token: Option<String>,
            _delimiter: Option<String>,
            _max_keys: i32,
            _fetch_owner: bool,
            _start_after: Option<String>,
            _incl_deleted: bool,
        ) -> Result<Self::ListObjectsV2Info> {
            Ok(StorageListObjectsV2Info::default())
        }

        async fn list_object_versions(
            self: Arc<Self>,
            _bucket: &str,
            _prefix: &str,
            _marker: Option<String>,
            _version_marker: Option<String>,
            _delimiter: Option<String>,
            _max_keys: i32,
        ) -> Result<Self::ListObjectVersionsInfo> {
            Ok(StorageListObjectVersionsInfo::default())
        }

        async fn walk(
            self: Arc<Self>,
            _rx: Self::WalkCancellation,
            _bucket: &str,
            _prefix: &str,
            _result: Self::WalkResultSender,
            _opts: Self::WalkOptions,
        ) -> Result<()> {
            Err(Error::NotImplemented)
        }
    }

    #[async_trait::async_trait]
    impl NamespaceLocking for LockingTierConfigStore {
        type Error = Error;
        type NamespaceLock = rustfs_lock::NamespaceLockWrapper;

        async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<rustfs_lock::NamespaceLockWrapper> {
            self.locks
                .lock()
                .expect("tier config lock recorder should not poison")
                .push((bucket.to_string(), object.to_string()));
            Ok(rustfs_lock::NamespaceLockWrapper::new(
                rustfs_lock::NamespaceLock::with_local_manager("tier-config-test".to_string(), self.lock_manager.clone()),
                rustfs_lock::ObjectKey {
                    bucket: Arc::from(bucket),
                    object: Arc::from(object),
                    version: None,
                },
                "tier-config-test-owner".to_string(),
            ))
        }
    }

    #[tokio::test]
    async fn tier_config_update_path_acquires_meta_namespace_sidecar_lock_before_save() {
        let manager = TierConfigMgr::new();
        let store = Arc::new(LockingTierConfigStore::new());

        TierConfigMgr::update_candidate_with_config_lock(&manager, store.clone(), TierCandidateMutation::Clear(true))
            .await
            .expect("empty clear should still acquire and save through the coordinator lock");

        assert_eq!(store.lock_events(), vec![(RUSTFS_META_BUCKET.to_string(), tier_config_lock_path())]);
        assert!(
            store.put_after_lock.load(Ordering::SeqCst),
            "tier config save must happen after the coordinator namespace lock is acquired"
        );
    }

    #[test]
    fn tier_mutation_targets_prove_remove_and_rebind_authoritative_identity() {
        let mut current = empty_mgr();
        current.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));

        let mut rebound = empty_mgr();
        let mut replacement = build_rustfs_tier("COLD-A");
        replacement.rustfs.as_mut().expect("replacement payload should exist").prefix = "new-prefix".to_string();
        rebound.tiers.insert("COLD-A".to_string(), replacement);

        let edit_targets = TierCandidateMutation::Edit("COLD-A".to_string(), TierCreds::default())
            .affected_targets(&current, &rebound)
            .expect("rebind target proof should build");
        assert_eq!(edit_targets.len(), 1);
        assert_eq!(edit_targets[0].tier_name, "COLD-A");
        assert!(edit_targets[0].old_backend_identity.is_some());
        assert!(edit_targets[0].new_backend_identity.is_some());
        assert_ne!(edit_targets[0].old_backend_identity, edit_targets[0].new_backend_identity);

        let removed = empty_mgr();
        let remove_targets = TierCandidateMutation::Remove("COLD-A".to_string(), true)
            .affected_targets(&current, &removed)
            .expect("remove target proof should build");
        assert_eq!(remove_targets.len(), 1);
        assert_eq!(remove_targets[0].tier_name, "COLD-A");
        assert!(remove_targets[0].old_backend_identity.is_some());
        assert!(remove_targets[0].new_backend_identity.is_none());

        let noop_targets = TierCandidateMutation::Remove("MISSING".to_string(), true)
            .affected_targets(&current, &current)
            .expect("idempotent remove of a missing tier should not require an identity proof");
        assert!(noop_targets.is_empty());
    }

    /// A fully offline `WarmBackend` used to exercise the driver-facing
    /// branches of `remove`/`verify` without touching a remote tier.
    struct MockWarmBackend {
        /// `Some(b)` -> `in_use()` returns `Ok(b)`; `None` -> returns `Err`.
        in_use_value: Option<bool>,
        /// When false, put/get/remove all fail (drives `verify` error paths).
        healthy: bool,
    }

    #[async_trait::async_trait]
    impl WarmBackend for MockWarmBackend {
        async fn validate(&self) -> std::result::Result<(), std::io::Error> {
            if self.healthy {
                Ok(())
            } else {
                Err(std::io::Error::other("mock validation failed"))
            }
        }

        fn validate_remote_version_id(&self, remote_version_id: &str) -> std::result::Result<(), std::io::Error> {
            if remote_version_id == "unsupported-version" {
                Err(std::io::Error::other("mock remote version rejected"))
            } else {
                Ok(())
            }
        }

        async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> std::result::Result<String, std::io::Error> {
            if self.healthy {
                Ok("mock-version".to_string())
            } else {
                Err(std::io::Error::other("mock put failed"))
            }
        }

        async fn put_with_meta(
            &self,
            object: &str,
            r: ReaderImpl,
            length: i64,
            _meta: HashMap<String, String>,
        ) -> std::result::Result<String, std::io::Error> {
            self.put(object, r, length).await
        }

        async fn get(
            &self,
            _object: &str,
            _rv: &str,
            _opts: WarmBackendGetOpts,
        ) -> std::result::Result<ReadCloser, std::io::Error> {
            if self.healthy {
                Ok(BufReader::new(Cursor::new(b"RustFS".to_vec())))
            } else {
                Err(std::io::Error::other("mock get failed"))
            }
        }

        async fn remove(&self, _object: &str, _rv: &str) -> std::result::Result<(), std::io::Error> {
            if self.healthy {
                Ok(())
            } else {
                Err(std::io::Error::other("mock remove failed"))
            }
        }

        async fn remove_exact(&self, object: &str, rv: &str) -> std::result::Result<(), std::io::Error> {
            if rv == "exact-only" {
                return Err(std::io::Error::other("mock exact remove forwarded"));
            }
            self.remove(object, rv).await
        }

        async fn in_use(&self) -> std::result::Result<bool, std::io::Error> {
            match self.in_use_value {
                Some(b) => Ok(b),
                None => Err(std::io::Error::other("mock in_use failed")),
            }
        }
    }

    fn inject_mock(mgr: &mut TierConfigMgr, name: &str, tier: TierConfig, mock: MockWarmBackend) {
        mgr.tiers.insert(name.to_string(), tier);
        mgr.driver_cache.insert(name.to_string(), Box::new(mock));
    }

    // ---- add ------------------------------------------------------------

    #[tokio::test]
    async fn test_add_rejects_non_uppercase_name() {
        let mut mgr = empty_mgr();
        let err = mgr
            .add(build_s3_tier("cold-a"), true)
            .await
            .expect_err("lowercase tier name must be rejected");
        assert_eq!(err.code, ERR_TIER_NAME_NOT_UPPERCASE.code);
        assert!(mgr.tiers.is_empty(), "rejected tier must not be persisted");
    }

    #[tokio::test]
    async fn test_add_rejects_duplicate_name() {
        let mut mgr = empty_mgr();
        mgr.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A"));

        let err = mgr
            .add(build_s3_tier("COLD-A"), true)
            .await
            .expect_err("duplicate tier name must be rejected");
        assert_eq!(err.code, ERR_TIER_ALREADY_EXISTS.code);
    }

    #[tokio::test]
    async fn test_add_rejects_unsupported_tier_type() {
        let mut mgr = empty_mgr();
        // `Unsupported` is rejected by `new_warm_backend` before any network I/O.
        let tier = TierConfig {
            version: "v1".to_string(),
            tier_type: TierType::Unsupported,
            name: "COLD-U".to_string(),
            ..Default::default()
        };
        let err = mgr.add(tier, true).await.expect_err("unsupported tier type must be rejected");
        assert_eq!(err.code, ERR_TIER_TYPE_UNSUPPORTED.code);
        assert!(mgr.tiers.is_empty());
    }

    #[tokio::test]
    async fn test_add_rejects_missing_backend_payload() {
        let mut mgr = empty_mgr();
        // Declares S3 type but omits the S3 payload; rejected before client build.
        let tier = TierConfig {
            version: "v1".to_string(),
            tier_type: TierType::S3,
            name: "COLD-A".to_string(),
            s3: None,
            ..Default::default()
        };
        let err = mgr
            .add(tier, true)
            .await
            .expect_err("missing backend payload must be rejected");
        assert_eq!(err.code, "XRustFSAdminTierInvalidConfig");
        assert!(err.message.contains("S3 tier configuration not found"), "{}", err.message);
    }

    #[tokio::test]
    async fn test_add_rejects_custom_wasabi_endpoint_before_backend_setup() {
        let mut mgr = empty_mgr();
        let mut tier = build_wasabi_tier("COLD-WASABI");
        tier.wasabi.as_mut().expect("Wasabi payload should exist").endpoint = "https://s3.example.invalid".to_string();

        let err = mgr
            .add(tier, true)
            .await
            .expect_err("custom Wasabi endpoint must fail before backend setup");
        assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
        assert_eq!(err.message, "Wasabi endpoint must be https://s3.ap-northeast-1.wasabisys.com");
        assert!(mgr.tiers.is_empty());
    }

    #[tokio::test]
    async fn test_add_rejects_reserved_names() {
        // Supersedes the former `test_add_does_not_reserve_standard_name_regression_anchor`
        // (added by PR #4713 to pin the pre-fix behavior). RustFS now rejects the
        // AWS/MinIO reserved storage-class names STANDARD and REDUCED_REDUNDANCY
        // (RRS) as remote-tier names, matching MinIO parity.
        //
        // `tier_type` is deliberately `Unsupported`: before the guard existed,
        // "STANDARD" reached `new_warm_backend` and failed with
        // `ERR_TIER_TYPE_UNSUPPORTED`. Asserting `ERR_TIER_RESERVED_NAME` here
        // proves the reserved-name guard fires *before* the type/backend check.
        for reserved in [crate::config::storageclass::STANDARD, crate::config::storageclass::RRS] {
            let mut mgr = empty_mgr();
            let tier = TierConfig {
                version: "v1".to_string(),
                tier_type: TierType::Unsupported,
                name: reserved.to_string(),
                ..Default::default()
            };
            let err = mgr.add(tier, true).await.expect_err("reserved tier name must be rejected");
            assert_eq!(
                err.code, ERR_TIER_RESERVED_NAME.code,
                "reserved tier name {reserved} must be rejected before the type check"
            );
            assert!(mgr.tiers.is_empty(), "rejected reserved tier {reserved} must not be persisted");
        }
    }

    // ---- edit -----------------------------------------------------------

    #[tokio::test]
    async fn test_edit_rejects_unknown_tier() {
        let mut mgr = empty_mgr();
        let err = mgr
            .edit("NOPE", TierCreds::default())
            .await
            .expect_err("editing an unknown tier must fail");
        assert_eq!(err.code, ERR_TIER_NOT_FOUND.code);
    }

    #[tokio::test]
    async fn test_edit_rejects_missing_credentials_for_rustfs() {
        let mut mgr = empty_mgr();
        mgr.tiers.insert("COLD-R".to_string(), build_rustfs_tier("COLD-R"));

        // Empty access/secret keys => rejected before any driver rebuild.
        let err = mgr
            .edit("COLD-R", TierCreds::default())
            .await
            .expect_err("empty credentials must be rejected");
        assert_eq!(err.code, ERR_TIER_MISSING_CREDENTIALS.code);
    }

    #[tokio::test]
    async fn test_edit_rejects_missing_credentials_for_wasabi() {
        let mut mgr = empty_mgr();
        mgr.tiers.insert("COLD-WASABI".to_string(), build_wasabi_tier("COLD-WASABI"));

        let err = mgr
            .edit("COLD-WASABI", TierCreds::default())
            .await
            .expect_err("empty Wasabi credentials must be rejected before backend setup");
        assert_eq!(err.code, ERR_TIER_MISSING_CREDENTIALS.code);
    }

    #[tokio::test]
    async fn test_edit_rejects_missing_credentials_for_minio() {
        let mut mgr = empty_mgr();
        let tier = TierConfig {
            version: "v1".to_string(),
            tier_type: TierType::MinIO,
            name: "COLD-M".to_string(),
            minio: Some(crate::services::tier::tier_config::TierMinIO {
                name: "COLD-M".to_string(),
                endpoint: "https://example-compat.invalid".to_string(),
                access_key: "ak".to_string(),
                secret_key: "sk".to_string(),
                bucket: "bucket-m".to_string(),
                prefix: String::new(),
                region: String::new(),
            }),
            ..Default::default()
        };
        mgr.tiers.insert("COLD-M".to_string(), tier);

        let err = mgr
            .edit("COLD-M", TierCreds::default())
            .await
            .expect_err("empty credentials must be rejected");
        assert_eq!(err.code, ERR_TIER_MISSING_CREDENTIALS.code);
    }

    // ---- remove ---------------------------------------------------------

    #[tokio::test]
    async fn test_remove_unknown_tier_is_idempotent() {
        let mut mgr = empty_mgr();
        // Unknown tier: get_driver returns NotFound, which `remove` swallows.
        mgr.remove("NOPE", false).await.expect("removing an unknown tier is a no-op");
    }

    #[tokio::test]
    async fn test_remove_rejects_backend_in_use() {
        let mut mgr = empty_mgr();
        inject_mock(
            &mut mgr,
            "COLD-A",
            build_s3_tier("COLD-A"),
            MockWarmBackend {
                in_use_value: Some(true),
                healthy: true,
            },
        );

        let err = mgr
            .remove("COLD-A", false)
            .await
            .expect_err("in-use backend must not be removed");
        assert_eq!(err.code, ERR_TIER_BACKEND_NOT_EMPTY.code);
        assert!(mgr.tiers.contains_key("COLD-A"), "tier must survive a rejected remove");
        assert!(mgr.driver_cache.contains_key("COLD-A"));
    }

    #[tokio::test]
    async fn test_remove_succeeds_when_backend_empty() {
        let mut mgr = empty_mgr();
        inject_mock(
            &mut mgr,
            "COLD-A",
            build_s3_tier("COLD-A"),
            MockWarmBackend {
                in_use_value: Some(false),
                healthy: true,
            },
        );

        mgr.remove("COLD-A", false).await.expect("empty backend can be removed");
        assert!(!mgr.tiers.contains_key("COLD-A"));
        assert!(!mgr.driver_cache.contains_key("COLD-A"));
    }

    #[tokio::test]
    async fn test_remove_force_skips_in_use_probe() {
        let mut mgr = empty_mgr();
        // in_use_value: None would error if probed; force=true must skip it.
        inject_mock(
            &mut mgr,
            "COLD-A",
            build_s3_tier("COLD-A"),
            MockWarmBackend {
                in_use_value: None,
                healthy: true,
            },
        );

        mgr.remove("COLD-A", true).await.expect("force remove must not probe in_use");
        assert!(!mgr.tiers.contains_key("COLD-A"));
        assert!(!mgr.driver_cache.contains_key("COLD-A"));
    }

    #[tokio::test]
    async fn persisted_remove_preserves_force_semantics() {
        let mut candidate = empty_mgr();
        inject_mock(
            &mut candidate,
            "COLD-A",
            build_s3_tier("COLD-A"),
            MockWarmBackend {
                in_use_value: None,
                healthy: true,
            },
        );
        let mutation = TierCandidateMutation::Remove("COLD-A".to_string(), true);
        mutation
            .apply(&mut candidate)
            .await
            .expect("force remove must skip in-use probing");
        assert!(!candidate.tiers.contains_key("COLD-A"));
    }

    #[tokio::test]
    async fn persisted_remove_nonforce_preserves_backend_in_use_error() {
        let mut candidate = empty_mgr();
        inject_mock(
            &mut candidate,
            "COLD-A",
            build_s3_tier("COLD-A"),
            MockWarmBackend {
                in_use_value: Some(true),
                healthy: true,
            },
        );
        let err = TierCandidateMutation::Remove("COLD-A".to_string(), false)
            .apply(&mut candidate)
            .await
            .expect_err("non-force remove must reject an in-use backend");
        assert_eq!(err.code, ERR_TIER_BACKEND_NOT_EMPTY.code);
        assert!(candidate.tiers.contains_key("COLD-A"));
    }

    #[tokio::test]
    async fn clear_preserves_force_and_nonforce_semantics() {
        let mut forced = empty_mgr();
        inject_mock(
            &mut forced,
            "COLD-A",
            build_s3_tier("COLD-A"),
            MockWarmBackend {
                in_use_value: None,
                healthy: true,
            },
        );
        forced.clear_tier(true).await.expect("force clear must skip in-use probing");
        assert!(forced.tiers.is_empty());

        let mut guarded = empty_mgr();
        inject_mock(
            &mut guarded,
            "COLD-A",
            build_s3_tier("COLD-A"),
            MockWarmBackend {
                in_use_value: Some(true),
                healthy: true,
            },
        );
        let err = guarded
            .clear_tier(false)
            .await
            .expect_err("non-force clear must reject an in-use backend");
        assert_eq!(err.code, ERR_TIER_BACKEND_NOT_EMPTY.code);
        assert!(guarded.tiers.contains_key("COLD-A"));
    }

    #[tokio::test]
    async fn test_remove_reports_probe_error() {
        let mut mgr = empty_mgr();
        inject_mock(
            &mut mgr,
            "COLD-A",
            build_s3_tier("COLD-A"),
            MockWarmBackend {
                in_use_value: None,
                healthy: true,
            },
        );

        let err = mgr
            .remove("COLD-A", false)
            .await
            .expect_err("a failed in_use probe must surface as an error");
        assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
        assert!(mgr.tiers.contains_key("COLD-A"), "tier must survive a failed probe");
    }

    // ---- verify ---------------------------------------------------------

    #[tokio::test]
    async fn test_verify_unknown_tier_errors() {
        let mut mgr = empty_mgr();
        let err = mgr.verify("NOPE").await.expect_err("verifying an unknown tier must fail");
        assert!(err.to_string().contains("not found"), "{err}");
    }

    #[tokio::test]
    async fn test_verify_succeeds_with_healthy_backend() {
        let mut mgr = empty_mgr();
        inject_mock(
            &mut mgr,
            "COLD-A",
            build_s3_tier("COLD-A"),
            MockWarmBackend {
                in_use_value: Some(false),
                healthy: true,
            },
        );

        mgr.verify("COLD-A").await.expect("healthy backend must verify");
    }

    #[tokio::test]
    async fn test_verify_reports_backend_failure() {
        let mut mgr = empty_mgr();
        inject_mock(
            &mut mgr,
            "COLD-A",
            build_s3_tier("COLD-A"),
            MockWarmBackend {
                in_use_value: Some(false),
                healthy: false,
            },
        );

        mgr.verify("COLD-A")
            .await
            .expect_err("an unhealthy backend must fail verification");
    }

    #[tokio::test]
    async fn shared_backend_proxy_forwards_validation_hooks() {
        let unhealthy: SharedWarmBackend = Arc::new(MockWarmBackend {
            in_use_value: Some(false),
            healthy: false,
        });
        let proxy = SharedWarmBackendProxy(unhealthy);
        let err = proxy.validate().await.expect_err("proxy must forward backend validation");
        assert_eq!(err.to_string(), "mock validation failed");

        let healthy: SharedWarmBackend = Arc::new(MockWarmBackend {
            in_use_value: Some(false),
            healthy: true,
        });
        let proxy = SharedWarmBackendProxy(healthy);
        let err = proxy
            .validate_remote_version_id("unsupported-version")
            .expect_err("proxy must forward remote version validation");
        assert_eq!(err.to_string(), "mock remote version rejected");
        let err = proxy
            .remove_exact("remote-object", "exact-only")
            .await
            .expect_err("proxy must forward exact-version cleanup");
        assert_eq!(err.to_string(), "mock exact remove forwarded");
    }

    // ---- pure query helpers --------------------------------------------

    #[test]
    fn test_query_helpers_reflect_membership() {
        let mut mgr = empty_mgr();
        assert!(mgr.empty());
        assert!(!mgr.is_tier_valid("COLD-A"));
        assert_eq!(mgr.tier_type("COLD-A"), "internal");
        assert!(mgr.get("COLD-A").is_none());

        mgr.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A"));

        assert!(!mgr.empty());
        assert!(mgr.is_tier_valid("COLD-A"));
        let (ty, in_use) = mgr.is_tier_name_in_use("COLD-A");
        assert!(in_use);
        assert_eq!(ty.as_lowercase(), "s3");
        assert_eq!(mgr.tier_type("COLD-A"), "s3");
        assert_eq!(mgr.list_tiers().len(), 1);
        assert!(mgr.get("COLD-A").is_some());
    }

    // ---- persistence: encode/decode and legacy migration ---------------

    #[test]
    fn test_marshal_unmarshal_json_roundtrip_preserves_tier() {
        let mut mgr = empty_mgr();
        mgr.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A"));

        let bytes = mgr.marshal().expect("marshal should succeed");
        let decoded = TierConfigMgr::unmarshal(&bytes).expect("unmarshal should succeed");

        let tier = decoded.tiers.get("COLD-A").expect("tier survives json roundtrip");
        assert_eq!(tier.tier_type.as_lowercase(), "s3");
        let s3 = tier.s3.as_ref().expect("s3 payload survives");
        assert_eq!(s3.bucket, "bucket-a");
        // secret_key is a serialized field (unlike Clone, marshal does not redact).
        assert_eq!(s3.secret_key, "sk");
    }

    #[test]
    fn test_external_blob_roundtrip_azure_maps_account_fields() {
        let mut mgr = empty_mgr();
        mgr.tiers.insert(
            "COLD-AZ".to_string(),
            TierConfig {
                version: "v1".to_string(),
                tier_type: TierType::Azure,
                name: "COLD-AZ".to_string(),
                azure: Some(crate::services::tier::tier_config::TierAzure {
                    name: "COLD-AZ".to_string(),
                    endpoint: "https://example-azure.invalid".to_string(),
                    access_key: "account-name".to_string(),
                    secret_key: "account-key".to_string(),
                    bucket: "container".to_string(),
                    prefix: "p".to_string(),
                    region: "eastus".to_string(),
                    storage_class: "HOT".to_string(),
                    sp_auth: crate::services::tier::tier_config::ServicePrincipalAuth {
                        tenant_id: "tenant".to_string(),
                        client_id: "client".to_string(),
                        client_secret: "secret".to_string(),
                    },
                }),
                ..Default::default()
            },
        );

        let bytes = encode_external_tiering_config_blob(&mgr).expect("encode azure tier");
        let decoded = decode_external_tiering_config_blob(&bytes).expect("decode azure tier");
        let tier = decoded.tiers.get("COLD-AZ").expect("azure tier survives roundtrip");
        assert_eq!(tier.tier_type.as_lowercase(), "azure");
        let az = tier.azure.as_ref().expect("azure payload survives");
        // AccountName/AccountKey in the on-disk format map back to access/secret.
        assert_eq!(az.access_key, "account-name");
        assert_eq!(az.secret_key, "account-key");
        assert_eq!(az.sp_auth.tenant_id, "tenant");
        assert_eq!(az.sp_auth.client_secret, "secret");
    }

    #[test]
    fn test_external_blob_roundtrip_gcs_preserves_creds() {
        let mut mgr = empty_mgr();
        mgr.tiers.insert(
            "COLD-G".to_string(),
            TierConfig {
                version: "v1".to_string(),
                tier_type: TierType::GCS,
                name: "COLD-G".to_string(),
                gcs: Some(crate::services::tier::tier_config::TierGCS {
                    name: "COLD-G".to_string(),
                    endpoint: "https://storage.googleapis.com".to_string(),
                    creds: "service-account-json".to_string(),
                    bucket: "gbucket".to_string(),
                    prefix: "gp".to_string(),
                    region: "us".to_string(),
                    storage_class: "NEARLINE".to_string(),
                }),
                ..Default::default()
            },
        );

        let bytes = encode_external_tiering_config_blob(&mgr).expect("encode gcs tier");
        let decoded = decode_external_tiering_config_blob(&bytes).expect("decode gcs tier");
        let tier = decoded.tiers.get("COLD-G").expect("gcs tier survives roundtrip");
        assert_eq!(tier.tier_type.as_lowercase(), "gcs");
        assert_eq!(tier.gcs.as_ref().expect("gcs payload survives").creds, "service-account-json");
    }

    // `TierConfigMgr` intentionally does not derive `Debug` (it holds live
    // driver handles), so `Result::expect_err` cannot be used on decode
    // results. Extract the error explicitly instead.
    fn expect_decode_err(data: &[u8]) -> std::io::Error {
        match decode_external_tiering_config_blob(data) {
            Ok(_) => panic!("expected decode to fail"),
            Err(err) => err,
        }
    }

    #[test]
    fn test_external_blob_rejects_truncated_data() {
        let err = expect_decode_err(&[0u8; 3]);
        assert!(err.to_string().contains("no data"), "{err}");
    }

    #[test]
    fn test_external_blob_rejects_unknown_format() {
        // Valid length, but a format word the decoder does not recognise.
        let mut data = Vec::new();
        data.extend_from_slice(&99u16.to_le_bytes());
        data.extend_from_slice(&TIER_CONFIG_VERSION.to_le_bytes());
        data.extend_from_slice(&[0u8; 8]);
        let err = expect_decode_err(&data);
        assert!(err.to_string().contains("unknown format"), "{err}");
    }

    #[test]
    fn test_external_blob_rejects_unknown_version() {
        let mut data = Vec::new();
        data.extend_from_slice(&TIER_CONFIG_FORMAT.to_le_bytes());
        data.extend_from_slice(&99u16.to_le_bytes());
        data.extend_from_slice(&[0u8; 8]);
        let err = expect_decode_err(&data);
        assert!(err.to_string().contains("unknown version"), "{err}");
    }

    #[test]
    fn test_external_blob_ignores_unknown_fields_at_all_legacy_layers() {
        let payload = serde_json::json!({
            "Tiers": {
                "COLD-A": {
                    "Version": "v1",
                    "Type": EXTERNAL_TIER_TYPE_S3,
                    "Name": "COLD-A",
                    "S3": {
                        "Endpoint": "https://example.invalid",
                        "AccessKey": "ak",
                        "SecretKey": "sk",
                        "Bucket": "bucket",
                        "Prefix": "objects",
                        "UnknownS3": true
                    },
                    "Azure": {
                        "SPAuth": { "UnknownSPAuth": true },
                        "UnknownAzure": true
                    },
                    "GCS": { "UnknownGCS": true },
                    "MinIO": { "UnknownMinIO": true },
                    "UnknownTier": true
                }
            },
            "UnknownManager": true
        });
        let encoded = rmp_serde::to_vec(&payload).expect("test payload should encode");
        let mut data = Vec::with_capacity(4 + encoded.len());
        data.extend_from_slice(&TIER_CONFIG_FORMAT.to_le_bytes());
        data.extend_from_slice(&TIER_CONFIG_VERSION.to_le_bytes());
        data.extend_from_slice(&encoded);

        let decoded = decode_external_tiering_config_blob(&data).expect("legacy unknown fields must remain forward-compatible");
        let s3 = decoded.tiers["COLD-A"].s3.as_ref().expect("s3 payload should decode");
        assert_eq!(s3.prefix, "objects");
    }

    #[test]
    fn test_external_blob_decodes_minio_compatible_fixture() {
        let payload = serde_json::json!({
            "Tiers": {
                "COLD-M": {
                    "Version": "v1",
                    "Type": EXTERNAL_TIER_TYPE_MINIO,
                    "Name": "COLD-M",
                    "MinIO": {
                        "Endpoint": "https://minio.example.invalid",
                        "AccessKey": "ak",
                        "SecretKey": "sk",
                        "Bucket": "archive",
                        "Prefix": "objects/",
                        "Region": "us-east-1"
                    }
                }
            }
        });
        let encoded = rmp_serde::to_vec(&payload).expect("fixture payload should encode");
        let mut data = Vec::with_capacity(4 + encoded.len());
        data.extend_from_slice(&TIER_CONFIG_FORMAT.to_le_bytes());
        data.extend_from_slice(&TIER_CONFIG_VERSION.to_le_bytes());
        data.extend_from_slice(&encoded);

        let decoded = decode_external_tiering_config_blob(&data).expect("MinIO-compatible fixture must decode");
        let minio = decoded.tiers["COLD-M"].minio.as_ref().expect("MinIO payload should decode");
        assert_eq!(minio.endpoint, "https://minio.example.invalid");
        assert_eq!(minio.bucket, "archive");
        assert_eq!(minio.prefix, "objects/");
    }

    #[test]
    fn test_external_blob_accepts_legacy_v1_version_word() {
        // The decoder must keep accepting the historical v1 version word, not
        // just the current TIER_CONFIG_VERSION, so old on-disk configs load.
        let mut mgr = empty_mgr();
        mgr.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A"));
        let current = encode_external_tiering_config_blob(&mgr).expect("encode");

        // Rewrite the version word (bytes 2..4) from VERSION to V1.
        let mut legacy = current.to_vec();
        legacy[2..4].copy_from_slice(&TIER_CONFIG_V1.to_le_bytes());

        let decoded = decode_external_tiering_config_blob(&legacy).expect("v1 version word must decode");
        assert!(decoded.tiers.contains_key("COLD-A"));
    }

    #[test]
    fn test_encode_external_blob_errors_on_missing_payload() {
        let mut mgr = empty_mgr();
        mgr.tiers.insert(
            "COLD-A".to_string(),
            TierConfig {
                version: "v1".to_string(),
                tier_type: TierType::S3,
                name: "COLD-A".to_string(),
                s3: None,
                ..Default::default()
            },
        );
        let err = encode_external_tiering_config_blob(&mgr).expect_err("missing payload must fail encode");
        assert!(err.to_string().contains("missing s3 backend payload"), "{err}");
    }

    #[derive(Clone)]
    struct LeaseTestBackend {
        id: &'static str,
        calls: Arc<std::sync::Mutex<Vec<&'static str>>>,
        remove_started: Option<Arc<tokio::sync::Notify>>,
        remove_release: Option<Arc<tokio::sync::Semaphore>>,
        backend_in_use: bool,
        pending_in_use: bool,
        panic_in_use: bool,
    }

    impl LeaseTestBackend {
        fn ready(id: &'static str) -> Self {
            Self {
                id,
                calls: Arc::new(std::sync::Mutex::new(Vec::new())),
                remove_started: None,
                remove_release: None,
                backend_in_use: false,
                pending_in_use: false,
                panic_in_use: false,
            }
        }

        fn blocking(id: &'static str) -> Self {
            Self {
                remove_started: Some(Arc::new(tokio::sync::Notify::new())),
                remove_release: Some(Arc::new(tokio::sync::Semaphore::new(0))),
                ..Self::ready(id)
            }
        }

        fn calls(&self) -> Vec<&'static str> {
            self.calls.lock().expect("lease test call log should not poison").clone()
        }

        fn panicking_in_use(id: &'static str) -> Self {
            Self {
                panic_in_use: true,
                ..Self::ready(id)
            }
        }

        fn in_use(id: &'static str) -> Self {
            Self {
                backend_in_use: true,
                ..Self::ready(id)
            }
        }

        fn pending_in_use(id: &'static str) -> Self {
            Self {
                pending_in_use: true,
                ..Self::ready(id)
            }
        }
    }

    #[async_trait::async_trait]
    impl WarmBackend for LeaseTestBackend {
        async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> io::Result<String> {
            Ok(self.id.to_string())
        }

        async fn put_with_meta(
            &self,
            object: &str,
            r: ReaderImpl,
            length: i64,
            _meta: HashMap<String, String>,
        ) -> io::Result<String> {
            self.put(object, r, length).await
        }

        async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> io::Result<ReadCloser> {
            Ok(BufReader::new(Cursor::new(Vec::new())))
        }

        async fn remove(&self, _object: &str, _rv: &str) -> io::Result<()> {
            self.calls
                .lock()
                .expect("lease test call log should not poison")
                .push(self.id);
            if let Some(started) = &self.remove_started {
                started.notify_one();
            }
            if let Some(release) = &self.remove_release {
                release
                    .acquire()
                    .await
                    .expect("lease test release semaphore should stay open")
                    .forget();
            }
            Ok(())
        }

        async fn in_use(&self) -> io::Result<bool> {
            if self.panic_in_use {
                panic!("simulated tier backend in-use panic");
            }
            if self.pending_in_use {
                std::future::pending::<()>().await;
            }
            Ok(self.backend_in_use)
        }
    }

    fn install_lease_backend(manager: &mut TierConfigMgr, tier_name: &str, backend: LeaseTestBackend) {
        manager.tiers.insert(tier_name.to_string(), build_rustfs_tier(tier_name));
        manager
            .replace_driver(tier_name, Box::new(backend))
            .expect("test driver generation should install");
    }

    fn prepared_remove_intent(tier_name: &str, mutation_id: uuid::Uuid) -> TierMutationIntent {
        TierMutationIntent {
            mutation_id,
            revision: 1,
            kind: TierMutationIntentKind::Remove,
            state: TierMutationIntentState::Prepared,
            old_config_etag: Some("etag-old".to_string()),
            committed_config_etag: None,
            candidate_digest: [1; 32],
            affected_targets: vec![TierMutationIntentTarget {
                tier_name: tier_name.to_string(),
                old_backend_identity: Some([2; 32]),
                new_backend_identity: None,
            }],
            expires_at_unix_nanos: 1,
        }
    }

    fn committed_remove_intent(tier_name: &str, mutation_id: uuid::Uuid, etag: &str) -> TierMutationIntent {
        let mut intent = prepared_remove_intent(tier_name, mutation_id);
        intent
            .advance(TierMutationIntentState::Committed, Some(etag.to_string()))
            .expect("fixture intent should commit");
        intent
    }

    struct FakeTierMutationPeer {
        label: &'static str,
        calls: Arc<Mutex<Vec<String>>>,
        prepare: std::result::Result<PeerTierMutationState, &'static str>,
        commit: std::result::Result<PeerTierMutationState, &'static str>,
        abort: std::result::Result<PeerTierMutationState, &'static str>,
    }

    impl FakeTierMutationPeer {
        fn boxed(
            label: &'static str,
            calls: Arc<Mutex<Vec<String>>>,
            commit: std::result::Result<PeerTierMutationState, &'static str>,
        ) -> Arc<dyn TierMutationPeer> {
            Self::boxed_with_prepare_commit(label, calls, Ok(PeerTierMutationState::Prepared), commit)
        }

        fn boxed_with_prepare_commit(
            label: &'static str,
            calls: Arc<Mutex<Vec<String>>>,
            prepare: std::result::Result<PeerTierMutationState, &'static str>,
            commit: std::result::Result<PeerTierMutationState, &'static str>,
        ) -> Arc<dyn TierMutationPeer> {
            Arc::new(Self {
                label,
                calls,
                prepare,
                commit,
                abort: Ok(PeerTierMutationState::Aborted),
            })
        }

        fn record(&self, call: String) {
            lock_unpoisoned(&self.calls).push(call);
        }
    }

    #[async_trait::async_trait]
    impl TierMutationPeer for FakeTierMutationPeer {
        fn peer_label(&self) -> String {
            self.label.to_string()
        }

        async fn prepare_tier_mutation(
            &self,
            mutation_id: uuid::Uuid,
            canonical_payload: Bytes,
        ) -> Result<PeerTierMutationState> {
            self.record(format!("{}:prepare:{}:{}", self.label, mutation_id, canonical_payload.len()));
            self.prepare.map_err(Error::other)
        }

        async fn commit_tier_mutation(&self, mutation_id: uuid::Uuid, canonical_payload: Bytes) -> Result<PeerTierMutationState> {
            self.record(format!(
                "{}:commit:{}:{}",
                self.label,
                mutation_id,
                String::from_utf8_lossy(canonical_payload.as_ref())
            ));
            self.commit.map_err(Error::other)
        }

        async fn abort_tier_mutation(&self, mutation_id: uuid::Uuid) -> Result<PeerTierMutationState> {
            self.record(format!("{}:abort:{}", self.label, mutation_id));
            self.abort.map_err(Error::other)
        }
    }

    struct ConcurrencyTrackingTierMutationPeer {
        label: &'static str,
        calls: Arc<Mutex<Vec<String>>>,
        active: Arc<AtomicUsize>,
        max_active: Arc<AtomicUsize>,
    }

    impl ConcurrencyTrackingTierMutationPeer {
        fn boxed(
            label: &'static str,
            calls: Arc<Mutex<Vec<String>>>,
            active: Arc<AtomicUsize>,
            max_active: Arc<AtomicUsize>,
        ) -> Arc<dyn TierMutationPeer> {
            Arc::new(Self {
                label,
                calls,
                active,
                max_active,
            })
        }

        async fn track(&self, action: &str) {
            let current = self.active.fetch_add(1, Ordering::SeqCst) + 1;
            self.max_active.fetch_max(current, Ordering::SeqCst);
            tokio::time::sleep(Duration::from_millis(20)).await;
            self.active.fetch_sub(1, Ordering::SeqCst);
            lock_unpoisoned(&self.calls).push(format!("{}:{action}", self.label));
        }
    }

    #[async_trait::async_trait]
    impl TierMutationPeer for ConcurrencyTrackingTierMutationPeer {
        fn peer_label(&self) -> String {
            self.label.to_string()
        }

        async fn prepare_tier_mutation(
            &self,
            _mutation_id: uuid::Uuid,
            _canonical_payload: Bytes,
        ) -> Result<PeerTierMutationState> {
            self.track("prepare").await;
            Ok(PeerTierMutationState::Prepared)
        }

        async fn commit_tier_mutation(
            &self,
            _mutation_id: uuid::Uuid,
            _canonical_payload: Bytes,
        ) -> Result<PeerTierMutationState> {
            self.track("commit").await;
            Ok(PeerTierMutationState::Committed)
        }

        async fn abort_tier_mutation(&self, _mutation_id: uuid::Uuid) -> Result<PeerTierMutationState> {
            self.track("abort").await;
            Ok(PeerTierMutationState::Aborted)
        }
    }

    #[tokio::test]
    async fn prepare_tier_mutation_peers_serializes_peer_prepare_writes() {
        let mutation_id = uuid::Uuid::from_u128(34);
        let intent = prepared_remove_intent("COLD-A", mutation_id);
        let calls = Arc::new(Mutex::new(Vec::new()));
        let active = Arc::new(AtomicUsize::new(0));
        let max_active = Arc::new(AtomicUsize::new(0));

        let prepared = prepare_tier_mutation_peers(
            mutation_id,
            vec![
                ConcurrencyTrackingTierMutationPeer::boxed("peer-a", calls.clone(), active.clone(), max_active.clone()),
                ConcurrencyTrackingTierMutationPeer::boxed("peer-b", calls.clone(), active.clone(), max_active.clone()),
                ConcurrencyTrackingTierMutationPeer::boxed("peer-c", calls.clone(), active.clone(), max_active.clone()),
            ],
            &intent,
        )
        .await
        .unwrap_or_else(|failure| panic!("successful prepare fanout should prepare every peer: {}", failure.error.message));

        assert_eq!(prepared.len(), 3);
        assert_eq!(
            max_active.load(Ordering::SeqCst),
            1,
            "peer prepare fanout must not write the same intent concurrently"
        );
        assert_eq!(
            lock_unpoisoned(&calls).as_slice(),
            &["peer-a:prepare", "peer-b:prepare", "peer-c:prepare"]
        );
    }

    #[tokio::test]
    async fn prepare_tier_mutation_peers_accepts_replayed_committed_peer() {
        let mutation_id = uuid::Uuid::from_u128(36);
        let intent = prepared_remove_intent("COLD-A", mutation_id);
        let calls = Arc::new(Mutex::new(Vec::new()));
        let peers = vec![
            FakeTierMutationPeer::boxed_with_prepare_commit(
                "peer-a",
                calls.clone(),
                Ok(PeerTierMutationState::Committed),
                Ok(PeerTierMutationState::Committed),
            ),
            ConcurrencyTrackingTierMutationPeer::boxed(
                "peer-b",
                calls.clone(),
                Arc::new(AtomicUsize::new(0)),
                Arc::new(AtomicUsize::new(0)),
            ),
        ];

        let prepared = match prepare_tier_mutation_peers(mutation_id, peers, &intent).await {
            Ok(prepared) => prepared,
            Err(_) => panic!("replayed committed peer should not fail the prepare fanout"),
        };

        assert_eq!(prepared.len(), 1);
        let calls = lock_unpoisoned(&calls);
        assert_eq!(calls.len(), 2);
        assert!(
            calls[0].starts_with("peer-a:prepare:"),
            "replayed peer must be prepared before the remaining peer"
        );
        assert_eq!(calls[1], "peer-b:prepare");
    }

    #[tokio::test]
    async fn commit_tier_mutation_peers_serializes_shared_record_writes() {
        let mutation_id = uuid::Uuid::from_u128(35);
        let calls = Arc::new(Mutex::new(Vec::new()));
        let active = Arc::new(AtomicUsize::new(0));
        let max_active = Arc::new(AtomicUsize::new(0));

        commit_tier_mutation_peers(
            mutation_id,
            vec![
                ConcurrencyTrackingTierMutationPeer::boxed("peer-a", calls.clone(), active.clone(), max_active.clone()),
                ConcurrencyTrackingTierMutationPeer::boxed("peer-b", calls.clone(), active.clone(), max_active.clone()),
                ConcurrencyTrackingTierMutationPeer::boxed("peer-c", calls.clone(), active.clone(), max_active.clone()),
            ],
            "etag-new",
        )
        .await
        .expect("successful commit fanout should commit every peer");

        assert_eq!(
            max_active.load(Ordering::SeqCst),
            1,
            "peer commit fanout must not write the same intent concurrently"
        );
        assert_eq!(lock_unpoisoned(&calls).as_slice(), &["peer-a:commit", "peer-b:commit", "peer-c:commit"]);
    }

    #[tokio::test]
    async fn abort_tier_mutation_peers_serializes_shared_record_writes() {
        let mutation_id = uuid::Uuid::from_u128(36);
        let calls = Arc::new(Mutex::new(Vec::new()));
        let active = Arc::new(AtomicUsize::new(0));
        let max_active = Arc::new(AtomicUsize::new(0));

        abort_tier_mutation_peers(
            mutation_id,
            vec![
                ConcurrencyTrackingTierMutationPeer::boxed("peer-a", calls.clone(), active.clone(), max_active.clone()),
                ConcurrencyTrackingTierMutationPeer::boxed("peer-b", calls.clone(), active.clone(), max_active.clone()),
                ConcurrencyTrackingTierMutationPeer::boxed("peer-c", calls.clone(), active.clone(), max_active.clone()),
            ],
        )
        .await
        .expect("successful abort fanout should abort every peer");

        assert_eq!(
            max_active.load(Ordering::SeqCst),
            1,
            "peer abort fanout must not write the same intent concurrently"
        );
        assert_eq!(lock_unpoisoned(&calls).as_slice(), &["peer-a:abort", "peer-b:abort", "peer-c:abort"]);
    }

    #[tokio::test]
    async fn committed_mutation_recovery_replays_peer_commit() {
        let mutation_id = uuid::Uuid::from_u128(13);
        let intent = committed_remove_intent("COLD-A", mutation_id, "etag-new");
        let calls = Arc::new(Mutex::new(Vec::new()));

        TIER_MUTATION_TEST_PEERS
            .scope(
                vec![FakeTierMutationPeer::boxed(
                    "peer-a",
                    calls.clone(),
                    Ok(PeerTierMutationState::Committed),
                )],
                async {
                    TierConfigMgr::replay_committed_mutation_intents(&[intent])
                        .await
                        .expect("committed recovery should replay peer commit");
                },
            )
            .await;

        assert_eq!(lock_unpoisoned(&calls).as_slice(), &[format!("peer-a:commit:{mutation_id}:etag-new")]);
    }

    #[tokio::test]
    async fn committed_mutation_recovery_fails_closed_on_peer_commit_error() {
        let mutation_id = uuid::Uuid::from_u128(14);
        let intent = committed_remove_intent("COLD-A", mutation_id, "etag-new");
        let calls = Arc::new(Mutex::new(Vec::new()));

        let err = TIER_MUTATION_TEST_PEERS
            .scope(vec![FakeTierMutationPeer::boxed("peer-a", calls.clone(), Err("network down"))], async {
                TierConfigMgr::replay_committed_mutation_intents(&[intent]).await
            })
            .await
            .expect_err("committed recovery must fail closed when peer commit cannot replay");

        assert!(err.to_string().contains("network down"), "{err}");
        assert_eq!(lock_unpoisoned(&calls).as_slice(), &[format!("peer-a:commit:{mutation_id}:etag-new")]);
    }

    #[tokio::test]
    async fn committed_replay_mixed_version_peer_matrix_fails_before_local_publish() {
        for (case, peer_error, expected_error) in [
            (
                "old_unimplemented",
                "peer tier mutation commit RPC failed: status: Unimplemented, message: \"old peer has no tier mutation control service\"",
                "old peer",
            ),
            (
                "deadline_exceeded",
                "peer tier mutation commit RPC failed: status: DeadlineExceeded, message: \"peer tier mutation control timed out\"",
                "timed out",
            ),
            (
                "unavailable",
                "peer tier mutation commit RPC failed: status: Unavailable, message: \"peer tier mutation control unavailable\"",
                "unavailable",
            ),
        ] {
            let store = Arc::new(CasConfigStore::default());
            let mut persisted = empty_mgr();
            persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
            persisted
                .save_tiering_config_if_current(store.clone(), None)
                .await
                .expect("tier config fixture should persist");

            let mutation_id = uuid::Uuid::from_u128(match case {
                "old_unimplemented" => 31,
                "deadline_exceeded" => 32,
                "unavailable" => 33,
                _ => unreachable!("matrix cases are enumerated above"),
            });
            let intent = committed_remove_intent("COLD-A", mutation_id, "etag-new");
            crate::services::tier::tier_mutation_intent::save_tier_mutation_intent_record(store.clone(), &intent)
                .await
                .expect("committed intent fixture should persist");
            let writes_after_fixture = store.next_etag.load(Ordering::SeqCst);
            let calls = Arc::new(Mutex::new(Vec::new()));
            let handle = TierConfigMgr::new();

            let err = TIER_MUTATION_TEST_PEERS
                .scope(vec![FakeTierMutationPeer::boxed("peer-a", calls.clone(), Err(peer_error))], async {
                    TierConfigMgr::reload_handle_with(&handle, store.clone()).await
                })
                .await
                .expect_err("mixed-version peer failure must abort committed replay");

            assert!(err.to_string().contains(expected_error), "{case} returned {err}");
            assert_eq!(lock_unpoisoned(&calls).as_slice(), &[format!("peer-a:commit:{mutation_id}:etag-new")]);
            assert!(
                !handle.read().await.tiers.contains_key("COLD-A"),
                "{case} must not publish the local tier config after peer replay fails"
            );
            assert_eq!(
                store.next_etag.load(Ordering::SeqCst),
                writes_after_fixture,
                "{case} must not perform a second tier-config CAS write after replay fails"
            );
            let reloaded = TierConfigMgr::load_tier_mutation_intents(store)
                .await
                .expect("intent scan should succeed after failed mixed-version replay");
            assert!(
                reloaded.iter().any(|intent| intent.mutation_id == mutation_id),
                "{case} must keep the committed intent for retry"
            );
            let blocked = match TierConfigMgr::acquire_operation_lease(&handle, "COLD-A").await {
                Ok(_) => panic!("{case} must keep committed replay blocking old tier operations"),
                Err(err) => err,
            };
            assert!(blocked.message.contains("being replaced"), "{case} returned {blocked}");
        }
    }

    #[tokio::test]
    async fn intent_advance_treats_matching_committed_cas_race_as_idempotent() {
        let store = Arc::new(CasConfigStore::default());
        let mutation_id = uuid::Uuid::from_u128(34);
        let prepared = prepared_remove_intent("COLD-A", mutation_id);
        crate::services::tier::tier_mutation_intent::save_tier_mutation_intent_record(store.clone(), &prepared)
            .await
            .expect("prepared intent fixture should persist");

        let committed = committed_remove_intent("COLD-A", mutation_id, "etag-new");
        let object = crate::services::tier::tier_mutation_intent::tier_mutation_intent_record_object_name(mutation_id)
            .expect("record object should build");
        store
            .rewrite_on_next_if_match(object, committed.encode().expect("committed intent fixture should encode"))
            .await;

        let (observed, applied) = crate::services::tier::tier_mutation_intent::advance_tier_mutation_intent_record_idempotent(
            store,
            mutation_id,
            TierMutationIntentState::Committed,
            Some("etag-new".to_string()),
        )
        .await
        .expect("same-state CAS race should be idempotent");

        assert!(!applied);
        assert_eq!(observed.state, TierMutationIntentState::Committed);
        assert_eq!(observed.committed_config_etag.as_deref(), Some("etag-new"));
    }

    #[tokio::test]
    async fn intent_advance_reconciles_matching_commit_after_final_cas_race() {
        let store = Arc::new(CasConfigStore::default());
        let mutation_id = uuid::Uuid::from_u128(37);
        let prepared = prepared_remove_intent("COLD-A", mutation_id);
        crate::services::tier::tier_mutation_intent::save_tier_mutation_intent_record(store.clone(), &prepared)
            .await
            .expect("prepared intent fixture should persist");

        let committed = committed_remove_intent("COLD-A", mutation_id, "etag-new");
        let object = crate::services::tier::tier_mutation_intent::tier_mutation_intent_record_object_name(mutation_id)
            .expect("record object should build");
        store
            .rewrite_on_next_if_match(object.clone(), prepared.encode().expect("prepared intent fixture should encode"))
            .await;
        store
            .rewrite_on_next_if_match(object.clone(), prepared.encode().expect("prepared intent fixture should encode"))
            .await;
        store
            .rewrite_on_next_if_match(object, committed.encode().expect("committed intent fixture should encode"))
            .await;

        let (observed, applied) = crate::services::tier::tier_mutation_intent::advance_tier_mutation_intent_record_idempotent(
            store,
            mutation_id,
            TierMutationIntentState::Committed,
            Some("etag-new".to_string()),
        )
        .await
        .expect("matching commit after the final CAS race should be idempotent");

        assert!(!applied);
        assert_eq!(observed.state, TierMutationIntentState::Committed);
        assert_eq!(observed.committed_config_etag.as_deref(), Some("etag-new"));
    }

    #[tokio::test]
    async fn intent_advance_rejects_unresolved_final_cas_race() {
        let store = Arc::new(CasConfigStore::default());
        let mutation_id = uuid::Uuid::from_u128(38);
        let prepared = prepared_remove_intent("COLD-A", mutation_id);
        crate::services::tier::tier_mutation_intent::save_tier_mutation_intent_record(store.clone(), &prepared)
            .await
            .expect("prepared intent fixture should persist");

        let object = crate::services::tier::tier_mutation_intent::tier_mutation_intent_record_object_name(mutation_id)
            .expect("record object should build");
        const CAS_ATTEMPTS_UNDER_TEST: usize = 3;
        for _ in 0..CAS_ATTEMPTS_UNDER_TEST {
            store
                .rewrite_on_next_if_match(object.clone(), prepared.encode().expect("prepared intent fixture should encode"))
                .await;
        }

        let err = crate::services::tier::tier_mutation_intent::advance_tier_mutation_intent_record_idempotent(
            store,
            mutation_id,
            TierMutationIntentState::Committed,
            Some("etag-new".to_string()),
        )
        .await
        .expect_err("a still-prepared intent after the final CAS race must fail closed");

        assert!(matches!(err, Error::PreconditionFailed));
    }

    #[tokio::test]
    async fn committed_mutation_recovery_requires_every_peer_to_commit() {
        let mutation_id = uuid::Uuid::from_u128(17);
        let intent = committed_remove_intent("COLD-A", mutation_id, "etag-new");
        let calls = Arc::new(Mutex::new(Vec::new()));

        let err = TIER_MUTATION_TEST_PEERS
            .scope(
                vec![
                    FakeTierMutationPeer::boxed("peer-a", calls.clone(), Ok(PeerTierMutationState::Committed)),
                    FakeTierMutationPeer::boxed("peer-b", calls.clone(), Ok(PeerTierMutationState::Prepared)),
                ],
                async { TierConfigMgr::replay_committed_mutation_intents(&[intent]).await },
            )
            .await
            .expect_err("committed recovery must fail unless every peer reports committed");

        assert!(err.to_string().contains("unexpected state"), "{err}");
        assert_eq!(
            lock_unpoisoned(&calls).as_slice(),
            &[
                format!("peer-a:commit:{mutation_id}:etag-new"),
                format!("peer-b:commit:{mutation_id}:etag-new")
            ]
        );
    }

    #[test]
    fn committed_replay_rejects_partial_peer_discovery() {
        ensure_complete_tier_mutation_commit_peer_set(0, 0).expect("local-only topology has no remote peers");
        ensure_complete_tier_mutation_commit_peer_set(2, 2).expect("two remote hosts should require two peers");
        let err = ensure_complete_tier_mutation_commit_peer_set(1, 2).expect_err("missing one expected peer must fail closed");
        assert!(err.to_string().contains("without peer commit clients"), "{err}");
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial_test::serial]
    async fn tier_mutation_peer_composition_preserves_unresolved_topology_slots() {
        let mut endpoints = Vec::new();
        for disk_index in 0..4 {
            let mut endpoint = Endpoint::try_from(format!("http://rustfs-{disk_index}.invalid:9000/data{disk_index}").as_str())
                .expect("unresolved topology endpoint should parse without DNS");
            endpoint.is_local = disk_index == 0;
            endpoint.set_pool_index(0);
            endpoint.set_set_index(0);
            endpoint.set_disk_index(disk_index);
            endpoints.push(endpoint);
        }
        let topology = EndpointServerPools::from(vec![PoolEndpoints {
            legacy: false,
            set_count: 1,
            drives_per_set: 4,
            endpoints: Endpoints::from(endpoints),
            cmd_line: "unresolved-tier-mutation-topology".to_string(),
            platform: "test".to_string(),
        }]);
        let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;

        let peers = remote_tier_mutation_peers_from_topology(topology)
            .await
            .expect("every unresolved remote topology slot should retain a tier mutation client");
        assert_eq!(
            peers.iter().map(|peer| peer.peer_label()).collect::<Vec<_>>(),
            vec![
                "http://rustfs-1.invalid:9000".to_string(),
                "http://rustfs-2.invalid:9000".to_string(),
                "http://rustfs-3.invalid:9000".to_string(),
            ]
        );
    }

    #[tokio::test]
    async fn coordinator_fanout_prepare_failure_aborts_prepared_peers_without_cas() {
        let manager = TierConfigMgr::new();
        let store = Arc::new(CasConfigStore::default());
        let mut persisted = empty_mgr();
        persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        persisted
            .save_tiering_config_if_current(store.clone(), None)
            .await
            .expect("tier config fixture should persist");
        let (mut candidate, version) = load_tier_config_for_update(store.clone())
            .await
            .expect("tier config fixture should reload");
        let original_version = version.clone();
        candidate
            .driver_cache
            .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("candidate")));
        let update = TierConfigMgr::admin_update_lock(&manager).await;
        let calls = Arc::new(Mutex::new(Vec::new()));

        let err = TIER_MUTATION_TEST_PEERS
            .scope(
                vec![
                    FakeTierMutationPeer::boxed_with_prepare_commit(
                        "peer-a",
                        calls.clone(),
                        Ok(PeerTierMutationState::Prepared),
                        Ok(PeerTierMutationState::Committed),
                    ),
                    FakeTierMutationPeer::boxed_with_prepare_commit(
                        "peer-b",
                        calls.clone(),
                        Err("prepare unavailable"),
                        Ok(PeerTierMutationState::Committed),
                    ),
                ],
                async {
                    TierConfigMgr::update_candidate_owned(
                        &manager,
                        store.clone(),
                        candidate,
                        version,
                        TierCandidateMutation::Remove("COLD-A".to_string(), true),
                        update,
                        None,
                    )
                    .await
                },
            )
            .await
            .expect_err("partial prepare failure must fail before config CAS");

        let TierConfigUpdateError::Publish(err) = err else {
            panic!("prepare fanout failure should be reported as publish failure");
        };
        assert!(err.message.contains("peer-b"), "{}", err.message);
        let calls = lock_unpoisoned(&calls).clone();
        assert!(calls.iter().any(|call| call.starts_with("peer-a:prepare:")), "{calls:?}");
        assert!(calls.iter().any(|call| call.starts_with("peer-b:prepare:")), "{calls:?}");
        assert!(calls.iter().any(|call| call.starts_with("peer-a:abort:")), "{calls:?}");
        assert!(!calls.iter().any(|call| call.contains(":commit:")), "{calls:?}");
        let (persisted, current_version) = load_tier_config_for_update(store.clone())
            .await
            .expect("tier config should remain readable");
        assert_eq!(current_version, original_version, "prepare failure must not CAS the tier config object");
        assert!(persisted.tiers.contains_key("COLD-A"));
        let intents = TierConfigMgr::load_coordinator_mutation_intents(store)
            .await
            .expect("coordinator intent scan should succeed");
        assert_eq!(intents.len(), 1);
        assert_eq!(intents[0].state, TierMutationIntentState::Aborted);
    }

    #[tokio::test]
    async fn coordinator_prepare_abort_failure_retains_intent_until_reload_converges() {
        let manager = TierConfigMgr::new();
        let store = Arc::new(CasConfigStore::default());
        let mut persisted = empty_mgr();
        persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        persisted
            .save_tiering_config_if_current(store.clone(), None)
            .await
            .expect("tier config fixture should persist");
        let (mut candidate, version) = load_tier_config_for_update(store.clone())
            .await
            .expect("tier config fixture should reload");
        candidate
            .driver_cache
            .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("candidate")));
        let update = TierConfigMgr::admin_update_lock(&manager).await;
        let calls = Arc::new(Mutex::new(Vec::new()));

        let err = TIER_MUTATION_TEST_PEERS
            .scope(
                vec![
                    Arc::new(FakeTierMutationPeer {
                        label: "peer-a",
                        calls: calls.clone(),
                        prepare: Ok(PeerTierMutationState::Prepared),
                        commit: Ok(PeerTierMutationState::Committed),
                        abort: Err("abort unavailable"),
                    }) as Arc<dyn TierMutationPeer>,
                    FakeTierMutationPeer::boxed_with_prepare_commit(
                        "peer-b",
                        calls.clone(),
                        Err("prepare unavailable"),
                        Ok(PeerTierMutationState::Committed),
                    ),
                ],
                async {
                    TierConfigMgr::update_candidate_owned(
                        &manager,
                        store.clone(),
                        candidate,
                        version,
                        TierCandidateMutation::Remove("COLD-A".to_string(), true),
                        update,
                        None,
                    )
                    .await
                },
            )
            .await
            .expect_err("failed abort must retain durable coordinator recovery evidence");
        assert!(matches!(err, TierConfigUpdateError::Publish(_)));

        let intents = TierConfigMgr::load_coordinator_mutation_intents(store.clone())
            .await
            .expect("coordinator intent scan should retain abort recovery evidence");
        assert_eq!(intents.len(), 1);
        assert_eq!(intents[0].state, TierMutationIntentState::Prepared);
        let mutation_id = intents[0].mutation_id;

        TIER_MUTATION_TEST_PEERS
            .scope(
                vec![
                    FakeTierMutationPeer::boxed("peer-a", calls.clone(), Ok(PeerTierMutationState::Committed)),
                    FakeTierMutationPeer::boxed("peer-b", calls.clone(), Ok(PeerTierMutationState::Committed)),
                ],
                async {
                    TierConfigMgr::reload_handle_with(&manager, store.clone())
                        .await
                        .expect("reload should retry the durable coordinator abort");
                },
            )
            .await;

        let calls = lock_unpoisoned(&calls).clone();
        assert!(
            calls
                .iter()
                .filter(|call| *call == &format!("peer-a:abort:{mutation_id}"))
                .count()
                >= 2,
            "{calls:?}"
        );
        assert!(calls.iter().any(|call| call == &format!("peer-b:abort:{mutation_id}")), "{calls:?}");
        let intents = TierConfigMgr::load_coordinator_mutation_intents(store)
            .await
            .expect("coordinator intent scan should succeed after recovery");
        assert!(intents.is_empty(), "successful abort recovery should remove the coordinator intent");
    }

    #[tokio::test]
    async fn coordinator_fanout_commit_failure_keeps_committed_intent_for_reload_retry() {
        let manager = TierConfigMgr::new();
        {
            let mut guard = manager.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
        }
        let store = Arc::new(CasConfigStore::default());
        let mut persisted = empty_mgr();
        persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        persisted
            .save_tiering_config_if_current(store.clone(), None)
            .await
            .expect("tier config fixture should persist");
        let (mut candidate, version) = load_tier_config_for_update(store.clone())
            .await
            .expect("tier config fixture should reload");
        candidate
            .driver_cache
            .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("candidate")));
        let update = TierConfigMgr::admin_update_lock(&manager).await;
        let calls = Arc::new(Mutex::new(Vec::new()));

        let err = TIER_MUTATION_TEST_PEERS
            .scope(
                vec![FakeTierMutationPeer::boxed_with_prepare_commit(
                    "peer-a",
                    calls.clone(),
                    Ok(PeerTierMutationState::Prepared),
                    Err("commit unavailable"),
                )],
                async {
                    TierConfigMgr::update_candidate_owned(
                        &manager,
                        store.clone(),
                        candidate,
                        version,
                        TierCandidateMutation::Remove("COLD-A".to_string(), true),
                        update,
                        None,
                    )
                    .await
                },
            )
            .await
            .expect_err("commit peer failure must fail closed before local publish");

        let TierConfigUpdateError::Publish(err) = err else {
            panic!("commit fanout failure should be reported as publish failure");
        };
        assert!(err.message.contains("commit unavailable"), "{}", err.message);
        assert!(
            manager.read().await.tiers.contains_key("COLD-A"),
            "failed peer commit must not publish the new generation locally"
        );
        let intents = TierConfigMgr::load_coordinator_mutation_intents(store.clone())
            .await
            .expect("coordinator intent scan should succeed after commit failure");
        assert_eq!(intents.len(), 1);
        assert_eq!(intents[0].state, TierMutationIntentState::Committed);
        let committed_id = intents[0].mutation_id;
        let committed_etag = intents[0]
            .committed_config_etag
            .clone()
            .expect("committed coordinator intent should carry the saved tier config ETag");

        TIER_MUTATION_TEST_PEERS
            .scope(
                vec![FakeTierMutationPeer::boxed(
                    "peer-a",
                    calls.clone(),
                    Ok(PeerTierMutationState::Committed),
                )],
                async {
                    TierConfigMgr::reload_handle_with(&manager, store.clone())
                        .await
                        .expect("reload should retry committed coordinator peer fanout and publish");
                },
            )
            .await;

        assert!(
            lock_unpoisoned(&calls)
                .iter()
                .any(|call| call == &format!("peer-a:commit:{committed_id}:{committed_etag}")),
            "reload should retry the committed coordinator peer commit: {:?}",
            lock_unpoisoned(&calls)
        );
        assert!(
            !manager.read().await.tiers.contains_key("COLD-A"),
            "reload retry should publish the committed tier removal"
        );
        let intents = TierConfigMgr::load_coordinator_mutation_intents(store)
            .await
            .expect("coordinator intent scan should succeed after retry cleanup");
        assert!(intents.iter().all(|intent| intent.mutation_id != committed_id));
    }

    #[tokio::test]
    async fn coordinator_fanout_success_commits_peers_before_publish() {
        let manager = TierConfigMgr::new();
        {
            let mut guard = manager.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
        }
        let store = Arc::new(CasConfigStore::default());
        let mut persisted = empty_mgr();
        persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        persisted
            .save_tiering_config_if_current(store.clone(), None)
            .await
            .expect("tier config fixture should persist");
        let (mut candidate, version) = load_tier_config_for_update(store.clone())
            .await
            .expect("tier config fixture should reload");
        candidate
            .driver_cache
            .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("candidate")));
        let update = TierConfigMgr::admin_update_lock(&manager).await;
        let calls = Arc::new(Mutex::new(Vec::new()));

        TIER_MUTATION_TEST_PEERS
            .scope(
                vec![FakeTierMutationPeer::boxed(
                    "peer-a",
                    calls.clone(),
                    Ok(PeerTierMutationState::Committed),
                )],
                async {
                    TierConfigMgr::update_candidate_owned(
                        &manager,
                        store.clone(),
                        candidate,
                        version,
                        TierCandidateMutation::Remove("COLD-A".to_string(), true),
                        update,
                        None,
                    )
                    .await
                    .expect("successful peer fanout should publish the tier mutation");
                },
            )
            .await;

        let calls = lock_unpoisoned(&calls).clone();
        let prepare_index = calls
            .iter()
            .position(|call| call.starts_with("peer-a:prepare:"))
            .expect("successful update should prepare the peer");
        let commit_index = calls
            .iter()
            .position(|call| call.contains(":commit:"))
            .expect("successful update should commit the peer");
        assert!(prepare_index < commit_index, "{calls:?}");
        let saved_etag = load_tier_config_for_update(store)
            .await
            .expect("saved tier config should reload")
            .1
            .expect("saved tier config should have an ETag");
        assert!(calls.iter().any(|call| call.ends_with(&format!(":{saved_etag}"))), "{calls:?}");
        assert!(
            !manager.read().await.tiers.contains_key("COLD-A"),
            "local manager should publish only after peer commit succeeds"
        );
    }

    #[tokio::test]
    async fn coordinator_fanout_commits_prepared_peers_after_replayed_committed_prepare() {
        let manager = TierConfigMgr::new();
        {
            let mut guard = manager.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
        }
        let store = Arc::new(CasConfigStore::default());
        let mut persisted = empty_mgr();
        persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        persisted
            .save_tiering_config_if_current(store.clone(), None)
            .await
            .expect("tier config fixture should persist");
        let (mut candidate, version) = load_tier_config_for_update(store.clone())
            .await
            .expect("tier config fixture should reload");
        candidate
            .driver_cache
            .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("candidate")));
        let update = TierConfigMgr::admin_update_lock(&manager).await;
        let calls = Arc::new(Mutex::new(Vec::new()));

        TIER_MUTATION_TEST_PEERS
            .scope(
                vec![
                    FakeTierMutationPeer::boxed_with_prepare_commit(
                        "peer-a",
                        calls.clone(),
                        Ok(PeerTierMutationState::Committed),
                        Ok(PeerTierMutationState::Committed),
                    ),
                    FakeTierMutationPeer::boxed_with_prepare_commit(
                        "peer-b",
                        calls.clone(),
                        Ok(PeerTierMutationState::Prepared),
                        Ok(PeerTierMutationState::Committed),
                    ),
                ],
                async {
                    TierConfigMgr::update_candidate_owned(
                        &manager,
                        store.clone(),
                        candidate,
                        version,
                        TierCandidateMutation::Remove("COLD-A".to_string(), true),
                        update,
                        None,
                    )
                    .await
                    .expect("committed replay plus partial prepare should publish after prepared peers commit");
                },
            )
            .await;

        let calls = lock_unpoisoned(&calls).clone();
        let peer_a_prepare_call = calls
            .iter()
            .find(|call| call.starts_with("peer-a:prepare:"))
            .expect("replayed peer should still receive prepare");
        let peer_b_prepare = calls
            .iter()
            .position(|call| call.starts_with("peer-b:prepare:"))
            .expect("remaining peer should prepare the mutation");
        let peer_b_commit = calls
            .iter()
            .position(|call| call.starts_with("peer-b:commit:"))
            .expect("prepared peer should receive commit");
        assert!(peer_b_prepare < peer_b_commit, "{calls:?}");
        let peer_b_prepare_id = calls[peer_b_prepare]
            .split(':')
            .nth(2)
            .expect("prepare call should record the mutation id");
        let peer_b_commit_id = calls[peer_b_commit]
            .split(':')
            .nth(2)
            .expect("commit call should record the mutation id");
        let peer_a_prepare_id = peer_a_prepare_call
            .split(':')
            .nth(2)
            .expect("replayed prepare call should record the mutation id");
        assert_eq!(peer_b_commit_id, peer_b_prepare_id, "{calls:?}");
        assert_eq!(peer_a_prepare_id, peer_b_prepare_id, "{calls:?}");
        assert!(
            !calls.iter().any(|call| call.starts_with("peer-a:commit:")),
            "already-committed prepare replay should not receive a duplicate commit: {calls:?}"
        );
        let saved_etag = load_tier_config_for_update(store.clone())
            .await
            .expect("saved tier config should reload")
            .1
            .expect("saved tier config should have an ETag");
        assert!(
            calls
                .iter()
                .any(|call| call.starts_with("peer-b:commit:") && call.ends_with(&format!(":{saved_etag}"))),
            "prepared peer commit should carry the saved tier config ETag: {calls:?}"
        );
        assert!(
            !manager.read().await.tiers.contains_key("COLD-A"),
            "local manager should publish after the partially prepared peer commits"
        );
    }

    #[tokio::test]
    async fn coordinator_fanout_converges_four_hot_reporter_committed_prepare_replay() {
        let manager = TierConfigMgr::new();
        let store = Arc::new(CasConfigStore::default());
        empty_mgr()
            .save_tiering_config_if_current(store.clone(), None)
            .await
            .expect("empty tier config fixture should persist");
        let (candidate, version) = load_tier_config_for_update(store.clone())
            .await
            .expect("tier config fixture should reload");
        let update = TierConfigMgr::admin_update_lock(&manager).await;
        let calls = Arc::new(Mutex::new(Vec::new()));

        TIER_MUTATION_TEST_PEERS
            .scope(
                vec![
                    FakeTierMutationPeer::boxed_with_prepare_commit(
                        "peer-a",
                        calls.clone(),
                        Ok(PeerTierMutationState::Committed),
                        Ok(PeerTierMutationState::Committed),
                    ),
                    FakeTierMutationPeer::boxed_with_prepare_commit(
                        "peer-b",
                        calls.clone(),
                        Ok(PeerTierMutationState::Prepared),
                        Ok(PeerTierMutationState::Committed),
                    ),
                    FakeTierMutationPeer::boxed_with_prepare_commit(
                        "peer-c",
                        calls.clone(),
                        Ok(PeerTierMutationState::Prepared),
                        Ok(PeerTierMutationState::Committed),
                    ),
                ],
                async {
                    TierConfigMgr::update_candidate_owned(
                        &manager,
                        store.clone(),
                        candidate,
                        version,
                        TierCandidateMutation::Add(build_rustfs_tier("COLD-A"), true),
                        update,
                        None,
                    )
                    .await
                    .expect("four-hot AddTier reporter replay should publish after prepared peers commit");
                },
            )
            .await;

        let calls = lock_unpoisoned(&calls).clone();
        let reporter_prepare_call = calls
            .iter()
            .find(|call| call.starts_with("peer-a:prepare:"))
            .expect("reporter peer should receive prepare replay");
        let reporter_mutation_id = reporter_prepare_call
            .split(':')
            .nth(2)
            .expect("reporter prepare call should record the mutation id");
        assert!(
            !calls.iter().any(|call| call.starts_with("peer-a:commit:")),
            "already-committed reporter must not receive a duplicate commit: {calls:?}"
        );

        let saved_etag = load_tier_config_for_update(store)
            .await
            .expect("saved tier config should reload")
            .1
            .expect("saved tier config should have an ETag");
        for peer in ["peer-b", "peer-c"] {
            let prepare_index = calls
                .iter()
                .position(|call| call.starts_with(&format!("{peer}:prepare:")))
                .expect("prepared peer should receive prepare");
            let commit_index = calls
                .iter()
                .position(|call| call.starts_with(&format!("{peer}:commit:")))
                .expect("prepared peer should receive commit");
            assert!(prepare_index < commit_index, "{peer} commit must follow prepare: {calls:?}");
            let prepared_mutation_id = calls[prepare_index]
                .split(':')
                .nth(2)
                .expect("prepare call should record the mutation id");
            let committed_mutation_id = calls[commit_index]
                .split(':')
                .nth(2)
                .expect("commit call should record the mutation id");
            assert_eq!(
                prepared_mutation_id, reporter_mutation_id,
                "{peer} prepare should share the reporter mutation id"
            );
            assert_eq!(
                committed_mutation_id, reporter_mutation_id,
                "{peer} commit should share the reporter mutation id"
            );
            assert!(
                calls[commit_index].ends_with(&format!(":{saved_etag}")),
                "{peer} commit should carry the saved tier config ETag: {calls:?}"
            );
        }
        assert!(
            manager.read().await.tiers.contains_key("COLD-A"),
            "local manager should publish the AddTier candidate after peer convergence"
        );
    }

    #[tokio::test]
    async fn reload_handle_replays_committed_mutation_and_removes_intent_record() {
        let store = Arc::new(CasConfigStore::default());
        let mut persisted = empty_mgr();
        persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        persisted
            .save_tiering_config_if_current(store.clone(), None)
            .await
            .expect("tier config fixture should persist");

        let mutation_id = uuid::Uuid::from_u128(15);
        let intent = committed_remove_intent("COLD-A", mutation_id, "etag-new");
        crate::services::tier::tier_mutation_intent::save_tier_mutation_intent_record(store.clone(), &intent)
            .await
            .expect("committed intent fixture should persist");
        let calls = Arc::new(Mutex::new(Vec::new()));
        let handle = TierConfigMgr::new();

        TIER_MUTATION_TEST_PEERS
            .scope(
                vec![FakeTierMutationPeer::boxed(
                    "peer-a",
                    calls.clone(),
                    Ok(PeerTierMutationState::Committed),
                )],
                async {
                    TierConfigMgr::reload_handle_with(&handle, store.clone())
                        .await
                        .expect("reload should replay committed intent before publishing");
                },
            )
            .await;

        assert_eq!(lock_unpoisoned(&calls).as_slice(), &[format!("peer-a:commit:{mutation_id}:etag-new")]);
        assert!(handle.read().await.tiers.contains_key("COLD-A"));
        let reloaded = TierConfigMgr::load_tier_mutation_intents(store)
            .await
            .expect("intent scan should succeed after cleanup");
        assert!(
            reloaded.iter().all(|intent| intent.mutation_id != mutation_id),
            "committed intent should be removed after successful replay"
        );
    }

    #[tokio::test]
    async fn reload_handle_replays_committed_and_restores_prepared_blocks() {
        let store = Arc::new(CasConfigStore::default());
        let mut persisted = empty_mgr();
        persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        persisted.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B"));
        persisted
            .save_tiering_config_if_current(store.clone(), None)
            .await
            .expect("tier config fixture should persist");

        let committed_id = uuid::Uuid::from_u128(18);
        let committed = committed_remove_intent("COLD-A", committed_id, "etag-new");
        crate::services::tier::tier_mutation_intent::save_tier_mutation_intent_record(store.clone(), &committed)
            .await
            .expect("committed intent fixture should persist");
        let prepared_id = uuid::Uuid::from_u128(19);
        let prepared = prepared_remove_intent("COLD-B", prepared_id);
        crate::services::tier::tier_mutation_intent::save_tier_mutation_intent_record(store.clone(), &prepared)
            .await
            .expect("prepared intent fixture should persist");
        let calls = Arc::new(Mutex::new(Vec::new()));
        let handle = TierConfigMgr::new();

        TIER_MUTATION_TEST_PEERS
            .scope(
                vec![FakeTierMutationPeer::boxed(
                    "peer-a",
                    calls.clone(),
                    Ok(PeerTierMutationState::Committed),
                )],
                async {
                    TierConfigMgr::reload_handle_with(&handle, store.clone())
                        .await
                        .expect("reload should replay committed intent and restore prepared blocks");
                },
            )
            .await;

        assert_eq!(lock_unpoisoned(&calls).as_slice(), &[format!("peer-a:commit:{committed_id}:etag-new")]);
        let err = match TierConfigMgr::acquire_operation_lease(&handle, "COLD-B").await {
            Ok(_) => panic!("prepared intent must still block operation leases after reload"),
            Err(err) => err,
        };
        assert!(err.message.contains("being replaced"), "{err}");
        let reloaded = TierConfigMgr::load_tier_mutation_intents(store)
            .await
            .expect("intent scan should succeed after mixed reload");
        assert!(reloaded.iter().all(|intent| intent.mutation_id != committed_id));
        assert!(reloaded.iter().any(|intent| intent.mutation_id == prepared_id));
    }

    #[tokio::test]
    async fn reload_handle_keeps_committed_block_until_publish_swaps_generation() {
        let store = Arc::new(CasConfigStore::default());
        empty_mgr()
            .save_tiering_config_if_current(store.clone(), None)
            .await
            .expect("empty tier config fixture should persist");

        let mutation_id = uuid::Uuid::from_u128(21);
        let intent = committed_remove_intent("COLD-A", mutation_id, "etag-new");
        crate::services::tier::tier_mutation_intent::save_tier_mutation_intent_record(store.clone(), &intent)
            .await
            .expect("committed intent fixture should persist");
        let calls = Arc::new(Mutex::new(Vec::new()));
        let handle = TierConfigMgr::new();
        {
            let mut guard = handle.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
        }
        let old = TierConfigMgr::acquire_operation_lease(&handle, "COLD-A")
            .await
            .expect("old generation lease should be available before recovery");

        TIER_MUTATION_TEST_PEERS
            .scope(
                vec![FakeTierMutationPeer::boxed(
                    "peer-a",
                    calls.clone(),
                    Ok(PeerTierMutationState::Committed),
                )],
                async {
                    let reload_handle = handle.clone();
                    let reload_store = store.clone();
                    let reload = async move { TierConfigMgr::reload_handle_with(&reload_handle, reload_store).await };
                    let probe_handle = handle.clone();
                    let probe = async move {
                        tokio::time::timeout(Duration::from_secs(1), async {
                            while old.inner.accepting.load(Ordering::Acquire) {
                                tokio::task::yield_now().await;
                            }
                        })
                        .await
                        .expect("reload publish should revoke the old generation before waiting");
                        {
                            let guard = probe_handle.read().await;
                            let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
                            assert_eq!(
                                lock_unpoisoned(&runtime).prepared_mutation_blocks.get("COLD-A"),
                                Some(&mutation_id),
                                "committed replay block must remain until the local publish completes"
                            );
                        }
                        let blocked = match TierConfigMgr::acquire_operation_lease(&probe_handle, "COLD-A").await {
                            Ok(_) => panic!("committed replay must block new old-generation leases while publish waits"),
                            Err(err) => err,
                        };
                        assert!(blocked.message.contains("being replaced"), "{blocked}");
                        drop(old);
                    };
                    let (reload_result, ()) = tokio::join!(reload, probe);
                    reload_result.expect("reload should complete after the old lease drains");
                },
            )
            .await;

        assert_eq!(lock_unpoisoned(&calls).as_slice(), &[format!("peer-a:commit:{mutation_id}:etag-new")]);
        assert!(
            !handle.read().await.tiers.contains_key("COLD-A"),
            "reloaded manager should publish the committed removal after the old lease drains"
        );
        let reloaded = TierConfigMgr::load_tier_mutation_intents(store)
            .await
            .expect("intent scan should succeed after committed removal reload");
        assert!(reloaded.iter().all(|intent| intent.mutation_id != mutation_id));
    }

    #[tokio::test(start_paused = true)]
    async fn reload_handle_keeps_committed_intent_when_local_publish_fails() {
        let store = Arc::new(CasConfigStore::default());
        empty_mgr()
            .save_tiering_config_if_current(store.clone(), None)
            .await
            .expect("empty tier config fixture should persist");

        let mutation_id = uuid::Uuid::from_u128(22);
        let intent = committed_remove_intent("COLD-A", mutation_id, "etag-new");
        crate::services::tier::tier_mutation_intent::save_tier_mutation_intent_record(store.clone(), &intent)
            .await
            .expect("committed intent fixture should persist");
        let calls = Arc::new(Mutex::new(Vec::new()));
        let handle = TierConfigMgr::new();
        {
            let mut guard = handle.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
        }
        let old = TierConfigMgr::acquire_operation_lease(&handle, "COLD-A")
            .await
            .expect("old generation lease should be available before recovery");

        TIER_MUTATION_TEST_PEERS
            .scope(
                vec![FakeTierMutationPeer::boxed(
                    "peer-a",
                    calls.clone(),
                    Ok(PeerTierMutationState::Committed),
                )],
                async {
                    let reload_handle = handle.clone();
                    let reload_store = store.clone();
                    let reload = async move { TierConfigMgr::reload_handle_with(&reload_handle, reload_store).await };
                    let probe = async {
                        while old.inner.accepting.load(Ordering::Acquire) {
                            tokio::task::yield_now().await;
                        }
                        tokio::time::advance(TIER_OPERATION_DRAIN_TIMEOUT).await;
                    };
                    let (reload_result, ()) = tokio::join!(reload, probe);
                    let err = reload_result.expect_err("reload must fail while the old lease refuses to drain");
                    assert!(
                        err.to_string()
                            .contains("Timed out waiting for active remote tier operations"),
                        "{err}"
                    );
                },
            )
            .await;

        assert_eq!(lock_unpoisoned(&calls).as_slice(), &[format!("peer-a:commit:{mutation_id}:etag-new")]);
        let reloaded = TierConfigMgr::load_tier_mutation_intents(store)
            .await
            .expect("intent scan should succeed after failed local publish");
        assert!(
            reloaded.iter().any(|intent| intent.mutation_id == mutation_id),
            "failed local publish must keep the committed intent for retry"
        );
        {
            let guard = handle.read().await;
            let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
            assert_eq!(lock_unpoisoned(&runtime).prepared_mutation_blocks.get("COLD-A"), Some(&mutation_id));
        }
        let blocked = match TierConfigMgr::acquire_operation_lease(&handle, "COLD-A").await {
            Ok(_) => panic!("failed local publish must keep blocking old-generation leases"),
            Err(err) => err,
        };
        assert!(blocked.message.contains("being replaced"), "{blocked}");
        drop(old);
    }

    #[tokio::test]
    async fn committed_replay_claim_prevents_duplicate_reload_fanout() {
        let store = Arc::new(CasConfigStore::default());
        let mut persisted = empty_mgr();
        persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        persisted
            .save_tiering_config_if_current(store.clone(), None)
            .await
            .expect("tier config fixture should persist");

        let mutation_id = uuid::Uuid::from_u128(20);
        let intent = committed_remove_intent("COLD-A", mutation_id, "etag-new");
        crate::services::tier::tier_mutation_intent::save_tier_mutation_intent_record(store.clone(), &intent)
            .await
            .expect("committed intent fixture should persist");
        let calls = Arc::new(Mutex::new(Vec::new()));
        let left = TierConfigMgr::new();
        let right = TierConfigMgr::new();

        TIER_MUTATION_TEST_PEERS
            .scope(
                vec![FakeTierMutationPeer::boxed(
                    "peer-a",
                    calls.clone(),
                    Ok(PeerTierMutationState::Committed),
                )],
                async {
                    let (left_result, right_result) = tokio::join!(
                        TierConfigMgr::reload_handle_with(&left, store.clone()),
                        TierConfigMgr::reload_handle_with(&right, store.clone())
                    );
                    left_result.expect("first reload should complete");
                    right_result.expect("second reload should observe the cleaned committed intent");
                },
            )
            .await;

        assert_eq!(
            lock_unpoisoned(&calls).as_slice(),
            &[format!("peer-a:commit:{mutation_id}:etag-new")],
            "only the claimed reload should fan out committed replay"
        );
    }

    #[tokio::test]
    async fn coordinator_mutation_recovery_scan_blocks_new_operation_lease() {
        let store = Arc::new(CasConfigStore::default());
        let mutation_id = uuid::Uuid::from_u128(23);
        let intent = prepared_remove_intent("COLD-A", mutation_id);
        save_tier_coordinator_mutation_intent_record_if_absent(store.clone(), &intent)
            .await
            .expect("coordinator intent fixture should persist");

        let loaded_prepared = TierConfigMgr::load_prepared_coordinator_mutation_intents(store.clone())
            .await
            .expect("coordinator intent scan should load prepared record");
        assert_eq!(loaded_prepared.len(), 1);
        assert_eq!(loaded_prepared[0].mutation_id, mutation_id);

        let manager = TierConfigMgr::new();
        TierConfigMgr::reconcile_prepared_mutation_intents_from_store(&manager, store)
            .await
            .expect("coordinator prepared recovery should reconcile");
        let err = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await {
            Ok(_) => panic!("coordinator prepared intent must block new operation leases"),
            Err(err) => err,
        };
        assert!(err.message.contains("being replaced"), "{err}");
    }

    #[tokio::test]
    async fn coordinator_mutation_recovery_commits_saved_config_digest() {
        let store = Arc::new(CasConfigStore::default());
        let mut current = empty_mgr();
        current.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        current
            .save_tiering_config_if_current(store.clone(), None)
            .await
            .expect("current tier config fixture should persist");
        let (_, old_etag) = load_tier_config_for_update(store.clone())
            .await
            .expect("current tier config fixture should have metadata");

        let candidate = empty_mgr();
        let mut intent = build_coordinator_tier_mutation_intent(
            TierMutationIntentKind::Remove,
            old_etag.clone(),
            &candidate,
            build_tier_mutation_affected_targets(
                TierMutationIntentKind::Remove,
                HashSet::from(["COLD-A".to_string()]),
                &current,
                &candidate,
            )
            .expect("remove fixture should produce an affected target"),
        )
        .expect("coordinator intent fixture should build")
        .expect("remove fixture should require a coordinator intent");
        intent.expires_at_unix_nanos = 1;
        save_tier_coordinator_mutation_intent_record_if_absent(store.clone(), &intent)
            .await
            .expect("coordinator intent fixture should persist");
        let mut aborted = prepared_remove_intent("COLD-Z", uuid::Uuid::from_u128(26));
        aborted
            .advance(TierMutationIntentState::Aborted, None)
            .expect("aborted coordinator fixture should advance");
        save_tier_coordinator_mutation_intent_record_if_absent(store.clone(), &aborted)
            .await
            .expect("aborted coordinator fixture should persist");
        candidate
            .save_tiering_config_if_current(store.clone(), old_etag.as_deref())
            .await
            .expect("candidate fixture should persist before coordinator commit");

        let handle = TierConfigMgr::new();
        TierConfigMgr::reload_handle_with(&handle, store.clone())
            .await
            .expect("reload should commit coordinator intent whose candidate digest is already saved");

        assert!(
            !handle.read().await.tiers.contains_key("COLD-A"),
            "reload should publish the already saved coordinator candidate"
        );
        let reloaded = TierConfigMgr::load_coordinator_mutation_intents(store)
            .await
            .expect("coordinator intent scan should succeed after recovery cleanup");
        assert!(
            reloaded.iter().all(|record| record.mutation_id != intent.mutation_id),
            "committed coordinator intent should be removed after local publish"
        );
        assert!(
            reloaded.iter().all(|record| record.mutation_id != aborted.mutation_id),
            "aborted coordinator intent should be removed after local publish"
        );
    }

    #[tokio::test]
    async fn coordinator_mutation_recovery_aborts_expired_unmatched_intent() {
        let store = Arc::new(CasConfigStore::default());
        let mut persisted = empty_mgr();
        persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        persisted
            .save_tiering_config_if_current(store.clone(), None)
            .await
            .expect("tier config fixture should persist");
        let mutation_id = uuid::Uuid::from_u128(27);
        save_tier_coordinator_mutation_intent_record_if_absent(store.clone(), &prepared_remove_intent("COLD-A", mutation_id))
            .await
            .expect("expired unmatched coordinator intent should persist");
        let calls = Arc::new(Mutex::new(Vec::new()));
        let handle = TierConfigMgr::new();

        let err = TIER_MUTATION_TEST_PEERS
            .scope(
                vec![Arc::new(FakeTierMutationPeer {
                    label: "peer-a",
                    calls: calls.clone(),
                    prepare: Ok(PeerTierMutationState::Prepared),
                    commit: Ok(PeerTierMutationState::Committed),
                    abort: Err("abort unavailable"),
                }) as Arc<dyn TierMutationPeer>],
                async { TierConfigMgr::reload_handle_with(&handle, store.clone()).await },
            )
            .await
            .expect_err("failed recovery abort must fail closed");
        assert!(
            err.to_string()
                .contains("expired coordinator tier mutation recovery abort failed"),
            "{err}"
        );
        assert!(
            TierConfigMgr::acquire_operation_lease(&handle, "COLD-A").await.is_err(),
            "failed recovery abort must retain the prepared runtime block"
        );
        let intents = TierConfigMgr::load_coordinator_mutation_intents(store.clone())
            .await
            .expect("coordinator intent scan should retain failed abort recovery evidence");
        assert_eq!(intents[0].state, TierMutationIntentState::Prepared);

        TIER_MUTATION_TEST_PEERS
            .scope(
                vec![FakeTierMutationPeer::boxed(
                    "peer-a",
                    calls.clone(),
                    Ok(PeerTierMutationState::Committed),
                )],
                async {
                    TierConfigMgr::reload_handle_with(&handle, store.clone())
                        .await
                        .expect("expired unmatched coordinator intent should converge by aborting peers");
                },
            )
            .await;

        assert_eq!(
            lock_unpoisoned(&calls).as_slice(),
            &[format!("peer-a:abort:{mutation_id}"), format!("peer-a:abort:{mutation_id}")]
        );
        assert!(handle.read().await.tiers.contains_key("COLD-A"));
        let intents = TierConfigMgr::load_coordinator_mutation_intents(store)
            .await
            .expect("coordinator intent scan should succeed after expired recovery");
        assert!(
            intents.is_empty(),
            "expired unmatched coordinator intent should be removed after abort convergence"
        );
    }

    #[tokio::test]
    async fn prepared_mutation_recovery_retries_when_runtime_blocks_change() {
        let manager = TierConfigMgr::new();
        let first = prepared_remove_intent("COLD-A", uuid::Uuid::from_u128(24));
        let second = prepared_remove_intent("COLD-B", uuid::Uuid::from_u128(25));

        TierConfigMgr::reconcile_prepared_mutation_intents(&manager, std::slice::from_ref(&first))
            .await
            .expect("initial prepared block should reconcile");
        let base_revision = TierConfigMgr::prepared_mutation_blocks_revision(&manager).await;
        TierConfigMgr::apply_prepared_mutation_intent_block(&manager, &second)
            .await
            .expect("concurrent peer prepare should update runtime blocks");

        assert!(
            !TierConfigMgr::reconcile_prepared_mutation_intents_with_revision(
                &manager,
                std::slice::from_ref(&first),
                Some(base_revision),
            )
            .await
            .expect("stale revision should be detected without overwriting")
        );
        {
            let guard = manager.read().await;
            let runtime = registered_tier_driver_runtime(&guard).expect("runtime should be registered");
            let blocks = &lock_unpoisoned(&runtime).prepared_mutation_blocks;
            assert_eq!(blocks.get("COLD-A"), Some(&first.mutation_id));
            assert_eq!(blocks.get("COLD-B"), Some(&second.mutation_id));
        }

        let retry_revision = TierConfigMgr::prepared_mutation_blocks_revision(&manager).await;
        assert!(
            TierConfigMgr::reconcile_prepared_mutation_intents_with_revision(
                &manager,
                &[first.clone(), second.clone()],
                Some(retry_revision),
            )
            .await
            .expect("fresh revision should reconcile")
        );
    }

    #[tokio::test]
    async fn reload_handle_fails_closed_when_committed_replay_fails() {
        let store = Arc::new(CasConfigStore::default());
        let mut persisted = empty_mgr();
        persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        persisted
            .save_tiering_config_if_current(store.clone(), None)
            .await
            .expect("tier config fixture should persist");

        let mutation_id = uuid::Uuid::from_u128(16);
        let intent = committed_remove_intent("COLD-A", mutation_id, "etag-new");
        crate::services::tier::tier_mutation_intent::save_tier_mutation_intent_record(store.clone(), &intent)
            .await
            .expect("committed intent fixture should persist");
        let calls = Arc::new(Mutex::new(Vec::new()));
        let handle = TierConfigMgr::new();

        let err = TIER_MUTATION_TEST_PEERS
            .scope(vec![FakeTierMutationPeer::boxed("peer-a", calls.clone(), Err("network down"))], async {
                TierConfigMgr::reload_handle_with(&handle, store.clone()).await
            })
            .await
            .expect_err("reload must fail closed when committed replay fails");

        assert!(err.to_string().contains("network down"), "{err}");
        assert_eq!(lock_unpoisoned(&calls).as_slice(), &[format!("peer-a:commit:{mutation_id}:etag-new")]);
        assert!(
            !handle.read().await.tiers.contains_key("COLD-A"),
            "local manager must not publish persisted config before peer replay succeeds"
        );
        let blocked = match TierConfigMgr::acquire_operation_lease(&handle, "COLD-A").await {
            Ok(_) => panic!("failed committed replay must block old tier operations"),
            Err(err) => err,
        };
        assert!(blocked.message.contains("being replaced"), "{blocked}");
        let reloaded = TierConfigMgr::load_tier_mutation_intents(store)
            .await
            .expect("intent scan should succeed after failed replay");
        assert!(
            reloaded.iter().any(|intent| intent.mutation_id == mutation_id),
            "failed committed replay must keep the durable intent for retry"
        );
    }

    #[tokio::test]
    async fn committed_missing_intent_requires_matching_tier_config_etag() {
        let store = Arc::new(CasConfigStore::default());
        let mut persisted = empty_mgr();
        persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        persisted
            .save_tiering_config_if_current(store.clone(), None)
            .await
            .expect("tier config fixture should persist");
        let (_, committed_etag) = load_tier_config_for_update(store.clone())
            .await
            .expect("persisted tier config should reload");
        let committed_etag = committed_etag.expect("persisted tier config should carry an ETag");

        assert!(
            tier_config_etag_matches(store.clone(), &committed_etag)
                .await
                .expect("matching ETag proof should load")
        );
        assert!(
            !tier_config_etag_matches(store, "other-etag")
                .await
                .expect("mismatched ETag proof should load"),
            "missing peer intent must not be terminal unless the committed config ETag matches"
        );
    }

    #[tokio::test]
    async fn prepared_mutation_recovery_blocks_new_operation_lease() {
        let manager = TierConfigMgr::new();
        manager
            .write()
            .await
            .tiers
            .insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        let mutation_id = uuid::Uuid::from_u128(1);
        TierConfigMgr::reconcile_prepared_mutation_intents(&manager, &[prepared_remove_intent("COLD-A", mutation_id)])
            .await
            .expect("prepared recovery block should apply");

        let err = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await {
            Ok(_) => panic!("prepared mutation must block new operation leases"),
            Err(err) => err,
        };
        assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
        assert_eq!(err.message, TIER_MUTATION_BLOCK_MESSAGE);
        let guard = manager.read().await;
        let runtime = registered_tier_driver_runtime(&guard).expect("runtime should be registered by recovery block");
        assert_eq!(lock_unpoisoned(&runtime).prepared_mutation_blocks.get("COLD-A"), Some(&mutation_id));
    }

    #[tokio::test]
    async fn prepared_mutation_recovery_blocks_admin_publish() {
        let manager = TierConfigMgr::new();
        manager
            .write()
            .await
            .tiers
            .insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        TierConfigMgr::reconcile_prepared_mutation_intents(
            &manager,
            &[prepared_remove_intent("COLD-A", uuid::Uuid::from_u128(1))],
        )
        .await
        .expect("prepared recovery block should apply");
        let candidate = empty_mgr();

        let err = TierConfigMgr::publish_candidate(&manager, candidate, None)
            .await
            .expect_err("prepared mutation must block conflicting admin publishes");
        assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
        assert!(manager.read().await.tiers.contains_key("COLD-A"));
    }

    #[tokio::test]
    async fn prepared_mutation_recovery_is_idempotent_and_rejects_conflicts() {
        let manager = TierConfigMgr::new();
        let first = prepared_remove_intent("COLD-A", uuid::Uuid::from_u128(1));
        TierConfigMgr::reconcile_prepared_mutation_intents(&manager, &[first.clone(), first])
            .await
            .expect("same prepared mutation should be idempotent");

        let first = prepared_remove_intent("COLD-A", uuid::Uuid::from_u128(1));
        let second = prepared_remove_intent("COLD-A", uuid::Uuid::from_u128(2));
        let err = TierConfigMgr::reconcile_prepared_mutation_intents(&manager, &[first, second])
            .await
            .expect_err("a second prepared mutation for the same tier must fail closed");
        assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
    }

    #[tokio::test]
    async fn prepared_mutation_recovery_clears_resolved_intents() {
        let manager = TierConfigMgr::new();
        manager
            .write()
            .await
            .tiers
            .insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        TierConfigMgr::reconcile_prepared_mutation_intents(
            &manager,
            &[prepared_remove_intent("COLD-A", uuid::Uuid::from_u128(1))],
        )
        .await
        .expect("prepared recovery block should apply");
        TierConfigMgr::reconcile_prepared_mutation_intents(&manager, &[])
            .await
            .expect("resolved recovery scan should clear stale blocks");

        let guard = manager.read().await;
        let runtime = registered_tier_driver_runtime(&guard).expect("runtime should stay registered");
        assert!(lock_unpoisoned(&runtime).prepared_mutation_blocks.is_empty());
    }

    #[tokio::test]
    async fn reload_handle_clears_resolved_prepared_intent_from_store_scan() {
        let store = Arc::new(CasConfigStore::default());
        let mut persisted = empty_mgr();
        persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        persisted
            .save_tiering_config_if_current(store.clone(), None)
            .await
            .expect("tier config fixture should persist");

        let mutation_id = uuid::Uuid::from_u128(23);
        crate::services::tier::tier_mutation_intent::save_tier_mutation_intent_record(
            store.clone(),
            &prepared_remove_intent("COLD-A", mutation_id),
        )
        .await
        .expect("prepared intent fixture should persist");
        let handle = TierConfigMgr::new();

        TierConfigMgr::reload_handle_with(&handle, store.clone())
            .await
            .expect("reload should restore prepared intent from store");
        let blocked = match TierConfigMgr::acquire_operation_lease(&handle, "COLD-A").await {
            Ok(_) => panic!("prepared intent restored from store must block new operation leases"),
            Err(err) => err,
        };
        assert_eq!(blocked.message, TIER_MUTATION_BLOCK_MESSAGE);

        crate::services::tier::tier_mutation_intent::delete_tier_mutation_intent_record(store.clone(), mutation_id)
            .await
            .expect("prepared intent fixture should delete");
        TierConfigMgr::reload_handle_with(&handle, store)
            .await
            .expect("reload should clear resolved prepared intent blocks");
        let guard = handle.read().await;
        let runtime = registered_tier_driver_runtime(&guard).expect("runtime should stay registered");
        assert!(lock_unpoisoned(&runtime).prepared_mutation_blocks.is_empty());
    }

    #[tokio::test]
    async fn registry_steady_state_lookup_does_not_scan_misses() {
        let manager = TierConfigMgr::new();
        TIER_REGISTRY_MISS_SCAN_COUNT
            .scope(std::cell::Cell::new(0), async {
                let guard = manager.read().await;
                let first = tier_driver_runtime(&manager, &guard);
                assert_eq!(TIER_REGISTRY_MISS_SCAN_COUNT.with(std::cell::Cell::get), 1);
                let second = tier_driver_runtime(&manager, &guard);
                assert!(Arc::ptr_eq(&first, &second));
                assert_eq!(TIER_REGISTRY_MISS_SCAN_COUNT.with(std::cell::Cell::get), 1);
            })
            .await;
    }

    #[tokio::test]
    #[serial_test::serial]
    async fn cold_driver_build_does_not_hold_manager_write_lock() {
        let cold_tier = "COLD-BUILD-LOCK";
        let manager = TierConfigMgr::new();
        {
            let mut guard = manager.write().await;
            guard.tiers.insert(cold_tier.to_string(), build_rustfs_tier(cold_tier));
            install_lease_backend(&mut guard, "COLD-B", LeaseTestBackend::ready("b"));
        }
        let (barrier, _barrier_guard) = install_tier_driver_build_barrier(cold_tier);
        let build_manager = manager.clone();
        let build = tokio::spawn(async move { TierConfigMgr::acquire_operation_lease(&build_manager, cold_tier).await });
        barrier.arrived.notified().await;

        tokio::time::timeout(Duration::from_millis(100), manager.read())
            .await
            .expect("cold driver construction must not block manager readers");
        let tier_b = tokio::time::timeout(Duration::from_millis(100), TierConfigMgr::acquire_operation_lease(&manager, "COLD-B"))
            .await
            .expect("cold tier A construction must not block tier B")
            .expect("tier B lease should remain available");
        drop(tier_b);

        barrier.release.add_permits(1);
        build
            .await
            .expect("cold driver build task should join")
            .expect("cold driver should finish after the test barrier releases");
    }

    #[tokio::test(start_paused = true)]
    #[serial_test::serial]
    async fn cold_reload_driver_build_timeout_restores_config() {
        let cold_tier = "COLD-BUILD-TIMEOUT";
        let manager = TierConfigMgr::new();
        manager
            .write()
            .await
            .tiers
            .insert(cold_tier.to_string(), build_rustfs_tier(cold_tier));
        let mut candidate = empty_mgr();
        let mut replacement = build_rustfs_tier(cold_tier);
        replacement.rustfs.as_mut().expect("replacement payload should exist").prefix = "new-prefix".to_string();
        candidate.tiers.insert(cold_tier.to_string(), replacement);
        let (_barrier, _barrier_guard) = install_tier_driver_build_barrier(cold_tier);

        let err = TierConfigMgr::publish_candidate(&manager, candidate, None)
            .await
            .expect_err("stalled cold backend construction must time out");

        assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
        assert!(manager.read().await.tiers.contains_key(cold_tier));
        let guard = manager.read().await;
        let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
        assert!(lock_unpoisoned(&runtime).draining.is_empty());
    }

    #[test]
    fn internal_tier_snapshot_preserves_credentials() {
        let mut config = build_rustfs_tier("COLD-A");
        let rustfs = config.rustfs.as_mut().expect("rustfs payload should exist");
        rustfs.access_key = "access-key".to_string();
        rustfs.secret_key = "secret-key".to_string();

        let snapshot = config.clone_with_credentials();

        let rustfs = snapshot.rustfs.expect("snapshot payload should exist");
        assert_eq!(rustfs.access_key, "access-key");
        assert_eq!(rustfs.secret_key, "secret-key");
    }

    #[tokio::test(start_paused = true)]
    async fn remote_mutation_validation_timeout_restores_candidate() {
        let mut candidate = empty_mgr();
        candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        candidate
            .driver_cache
            .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::pending_in_use("pending")));

        let err = apply_tier_candidate_mutation(
            TierCandidateMutation::Remove("COLD-A".to_string(), false),
            &mut candidate,
            Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT,
        )
        .await
        .expect_err("unbounded backend validation must time out");

        assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
        assert!(candidate.tiers.contains_key("COLD-A"));
    }

    #[tokio::test]
    async fn cold_cached_reload_rejects_in_use_route_change_and_removal() {
        for remove in [false, true] {
            let manager = TierConfigMgr::new();
            {
                let mut guard = manager.write().await;
                guard.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
                guard
                    .driver_cache
                    .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::in_use("cold")));
            }
            let mut candidate = empty_mgr();
            if !remove {
                let mut replacement = build_rustfs_tier("COLD-A");
                replacement.rustfs.as_mut().expect("replacement payload should exist").prefix = "new-prefix".to_string();
                candidate.tiers.insert("COLD-A".to_string(), replacement);
            }

            let err = TierConfigMgr::publish_candidate(&manager, candidate, None)
                .await
                .expect_err("an in-use cold-cache destination must not be rebound or removed");

            assert_eq!(err.code, ERR_TIER_BACKEND_NOT_EMPTY.code);
            assert!(manager.read().await.tiers.contains_key("COLD-A"));
            let restored = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
                .await
                .expect("failed reload must restore the old cold-cache generation");
            assert!(restored.is_current(&manager).await);
        }
    }

    #[tokio::test]
    async fn empty_backend_route_change_and_credential_rotation_can_publish() {
        let manager = TierConfigMgr::new();
        {
            let mut guard = manager.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("empty"));
        }
        let mut route_candidate = empty_mgr();
        let mut replacement = build_rustfs_tier("COLD-A");
        replacement.rustfs.as_mut().expect("replacement payload should exist").prefix = "new-prefix".to_string();
        route_candidate.tiers.insert("COLD-A".to_string(), replacement);
        TierConfigMgr::publish_candidate(&manager, route_candidate, None)
            .await
            .expect("an empty backend may be rebound");

        {
            let mut guard = manager.write().await;
            guard
                .driver_cache
                .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::panicking_in_use("credentials")));
        }
        let mut credential_candidate = empty_mgr();
        let mut rotated = manager.read().await.tiers["COLD-A"].clone_with_credentials();
        rotated.rustfs.as_mut().expect("rotated payload should exist").secret_key = "rotated".to_string();
        credential_candidate.tiers.insert("COLD-A".to_string(), rotated);
        TierConfigMgr::publish_candidate(&manager, credential_candidate, None)
            .await
            .expect("credential-only rotation must not inspect backend occupancy");
    }

    #[tokio::test]
    async fn slow_tier_operation_does_not_block_another_tier() {
        let slow = LeaseTestBackend::blocking("slow");
        let fast = LeaseTestBackend::ready("fast");
        let manager = Arc::new(RwLock::new(empty_mgr()));
        {
            let mut manager = manager.write().await;
            install_lease_backend(&mut manager, "COLD-A", slow.clone());
            install_lease_backend(&mut manager, "COLD-B", fast.clone());
        }
        let slow_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
            .await
            .expect("slow lease should be available");
        let fast_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-B")
            .await
            .expect("fast lease should be available");
        let slow_task = tokio::spawn(async move { slow_lease.remove("object", "").await });
        slow.remove_started
            .as_ref()
            .expect("slow backend should expose start notification")
            .notified()
            .await;

        let manager_read = tokio::time::timeout(Duration::from_millis(100), manager.read())
            .await
            .expect("manager reads must not wait for tier A");
        assert!(manager_read.is_tier_valid("COLD-B"));
        drop(manager_read);

        tokio::time::timeout(Duration::from_millis(100), fast_lease.remove("object", ""))
            .await
            .expect("tier B must not wait for tier A")
            .expect("fast remove should succeed");
        assert_eq!(fast.calls(), vec!["fast"]);

        slow.remove_release
            .as_ref()
            .expect("slow backend should expose release semaphore")
            .add_permits(1);
        slow_task
            .await
            .expect("slow task should join")
            .expect("slow remove should succeed");
    }

    #[tokio::test]
    async fn slow_admin_verify_does_not_hold_manager_lock() {
        let manager = TierConfigMgr::new();
        let backend = LeaseTestBackend::blocking("verify");
        let started = backend
            .remove_started
            .as_ref()
            .expect("verify remove notifier should exist")
            .clone();
        let release = backend
            .remove_release
            .as_ref()
            .expect("verify remove release should exist")
            .clone();
        {
            let mut guard = manager.write().await;
            install_lease_backend(&mut guard, "COLD-A", backend);
        }

        let verify_manager = manager.clone();
        let verify = tokio::spawn(async move { TierConfigMgr::verify_without_manager_lock(&verify_manager, "COLD-A").await });
        started.notified().await;
        tokio::time::timeout(Duration::from_millis(100), manager.read())
            .await
            .expect("slow verify must not hold the manager lock");
        release.add_permits(1);
        verify.await.expect("verify task should join").expect("verify should finish");
    }

    #[tokio::test]
    async fn same_name_replacement_drains_old_generation_and_routes_new_work_to_new_driver() {
        let old = LeaseTestBackend::ready("old");
        let new = LeaseTestBackend::ready("new");
        let manager = Arc::new(RwLock::new(empty_mgr()));
        {
            let mut guard = manager.write().await;
            install_lease_backend(&mut guard, "COLD-A", old.clone());
        }
        let old_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
            .await
            .expect("old generation lease should be available");
        {
            manager
                .write()
                .await
                .replace_driver("COLD-A", Box::new(new.clone()))
                .expect("replacement generation should install");
        }
        let new_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
            .await
            .expect("new generation lease should be available");

        assert!(!old_lease.is_current(&manager).await);
        assert!(new_lease.is_current(&manager).await);
        assert!(new_lease.generation() > old_lease.generation());
        old_lease
            .remove("old-object", "")
            .await
            .expect("draining lease should remain usable");
        new_lease
            .remove("new-object", "")
            .await
            .expect("new lease should use replacement driver");
        assert_eq!(old.calls(), vec!["old"]);
        assert_eq!(new.calls(), vec!["new"]);
    }

    #[tokio::test]
    async fn replacement_waits_for_inflight_operation_lease() {
        let manager = TierConfigMgr::new();
        {
            let mut guard = manager.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
        }
        let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
            .await
            .expect("old lease should be available");
        let mut candidate = empty_mgr();
        let mut config = build_rustfs_tier("COLD-A");
        config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://replacement.invalid".to_string();
        candidate.tiers.insert("COLD-A".to_string(), config);
        candidate
            .driver_cache
            .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("new")));
        let publish_manager = manager.clone();
        let publish =
            tokio::spawn(async move { TierConfigMgr::publish_candidate(&publish_manager, candidate, Some("COLD-A")).await });

        tokio::time::timeout(Duration::from_secs(1), async {
            while old.is_current(&manager).await {
                tokio::task::yield_now().await;
            }
        })
        .await
        .expect("replacement should revoke the old generation before waiting");
        assert!(!publish.is_finished(), "replacement must wait for the old operation lease");
        drop(old);
        publish
            .await
            .expect("publish task should join")
            .expect("replacement should publish after the lease drains");
    }

    #[tokio::test(start_paused = true)]
    async fn replacement_drain_timeout_restores_generation_and_allows_retry() {
        let manager = TierConfigMgr::new();
        {
            let mut guard = manager.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
        }
        let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
            .await
            .expect("old lease should be available");
        let mut candidate = empty_mgr();
        let mut config = build_rustfs_tier("COLD-A");
        config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://timeout.invalid".to_string();
        candidate.tiers.insert("COLD-A".to_string(), config);
        candidate
            .driver_cache
            .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("timed-out")));
        let publish_manager = manager.clone();
        let publish =
            tokio::spawn(async move { TierConfigMgr::publish_candidate(&publish_manager, candidate, Some("COLD-A")).await });
        while old.inner.accepting.load(Ordering::Acquire) {
            tokio::task::yield_now().await;
        }

        tokio::time::advance(TIER_OPERATION_DRAIN_TIMEOUT).await;
        let err = publish
            .await
            .expect("publish task should join")
            .expect_err("replacement must time out while the old lease remains active");
        assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
        assert!(old.is_current(&manager).await, "timeout rollback must restore the old generation");
        {
            let guard = manager.read().await;
            let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
            assert!(!lock_unpoisoned(&runtime).draining.contains_key("COLD-A"));
        }
        drop(old);

        let mut retry = empty_mgr();
        let mut retry_config = build_rustfs_tier("COLD-A");
        retry_config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://retry.invalid".to_string();
        retry.tiers.insert("COLD-A".to_string(), retry_config);
        retry
            .driver_cache
            .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("retry")));
        TierConfigMgr::publish_candidate(&manager, retry, Some("COLD-A"))
            .await
            .expect("a later replacement must succeed after timeout rollback");
    }

    #[tokio::test]
    async fn generation_prepare_failure_restores_old_manager_and_generation() {
        let manager = TierConfigMgr::new();
        {
            let mut guard = manager.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
        }
        let lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
            .await
            .expect("old generation should initialize");
        drop(lease);
        {
            let guard = manager.read().await;
            let runtime = registered_tier_driver_runtime(&guard).expect("runtime should be registered");
            lock_unpoisoned(&runtime).next_generation = u64::MAX;
        }
        let mut candidate = empty_mgr();
        let mut config = build_rustfs_tier("COLD-A");
        config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://overflow.invalid".to_string();
        candidate.tiers.insert("COLD-A".to_string(), config);
        candidate
            .driver_cache
            .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("new")));

        let err = TierConfigMgr::publish_candidate(&manager, candidate, Some("COLD-A"))
            .await
            .expect_err("generation exhaustion must fail before manager state changes");
        assert!(err.message.contains("generation exhausted"));
        assert_eq!(
            manager.read().await.tiers["COLD-A"]
                .rustfs
                .as_ref()
                .expect("old config should remain")
                .endpoint,
            "https://example-compat.invalid"
        );
        let restored = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
            .await
            .expect("prepare failure must restore the old generation");
        assert!(restored.is_current(&manager).await);
        let guard = manager.read().await;
        let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
        assert!(!lock_unpoisoned(&runtime).draining.contains_key("COLD-A"));
    }

    #[tokio::test]
    async fn legacy_reload_does_not_block_revoked_operation_completion() {
        let manager = TierConfigMgr::new();
        {
            let mut guard = manager.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
        }
        let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
            .await
            .expect("old lease should be available");
        let mut candidate = empty_mgr();
        let mut config = build_rustfs_tier("COLD-A");
        config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://legacy-reload.invalid".to_string();
        candidate.tiers.insert("COLD-A".to_string(), config);
        let reload_manager = manager.clone();
        let reload = tokio::spawn(async move { reload_manager.write().await.publish_legacy_reload(candidate).await });
        while old.inner.accepting.load(Ordering::Acquire) {
            tokio::task::yield_now().await;
        }
        assert!(
            manager.try_read().is_err(),
            "legacy reload must still hold the manager write guard while draining"
        );
        assert!(!reload.is_finished(), "legacy reload must wait for the active generation");
        let operation = tokio::spawn(async move {
            assert!(!old.is_current_generation(), "the revoked operation must observe the generation fence");
            drop(old);
        });
        tokio::time::timeout(Duration::from_secs(1), operation)
            .await
            .expect("the revoked operation must finish without waiting for the manager guard")
            .expect("the revoked operation task should join");
        reload
            .await
            .expect("legacy reload task should join")
            .expect("legacy reload should publish after the lease drains");
        assert_eq!(
            manager.read().await.tiers["COLD-A"]
                .rustfs
                .as_ref()
                .expect("reloaded tier should exist")
                .endpoint,
            "https://legacy-reload.invalid"
        );
    }

    #[tokio::test]
    async fn reload_publish_waits_for_inflight_operation_lease() {
        let manager = TierConfigMgr::new();
        {
            let mut guard = manager.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
        }
        let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
            .await
            .expect("old lease should be available");
        let mut replacement = build_rustfs_tier("COLD-A");
        replacement.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://reload.invalid".to_string();
        let mut candidate = empty_mgr();
        candidate.tiers.insert("COLD-A".to_string(), replacement);
        let reload_manager = manager.clone();
        let reload = tokio::spawn(async move { TierConfigMgr::publish_candidate(&reload_manager, candidate, None).await });
        tokio::time::timeout(Duration::from_secs(1), async {
            while old.is_current(&manager).await {
                tokio::task::yield_now().await;
            }
        })
        .await
        .expect("reload publish should revoke before waiting");
        assert!(!reload.is_finished(), "reload must wait for the old operation lease");
        drop(old);
        reload
            .await
            .expect("reload task should join")
            .expect("reload should publish after the lease drains");
    }

    #[tokio::test]
    async fn cancelled_publish_caller_does_not_leave_tier_draining() {
        let manager = TierConfigMgr::new();
        {
            let mut guard = manager.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
        }
        let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
            .await
            .expect("old lease should be available");
        let mut candidate = empty_mgr();
        let mut config = build_rustfs_tier("COLD-A");
        config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://owned.invalid".to_string();
        candidate.tiers.insert("COLD-A".to_string(), config);
        candidate
            .driver_cache
            .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("new")));
        let update = TierConfigMgr::admin_update_lock(&manager).await;
        let publish_manager = manager.clone();
        let caller = tokio::spawn(async move {
            TierConfigMgr::publish_candidate_owned(&publish_manager, candidate, Some("COLD-A".to_string()), update).await
        });
        tokio::time::timeout(Duration::from_secs(1), async {
            while old.is_current(&manager).await {
                tokio::task::yield_now().await;
            }
        })
        .await
        .expect("owned publish should revoke before caller cancellation");
        caller.abort();

        let mut second = empty_mgr();
        let mut second_config = build_rustfs_tier("COLD-A");
        second_config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://second.invalid".to_string();
        second.tiers.insert("COLD-A".to_string(), second_config);
        second
            .driver_cache
            .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("second")));
        let second_manager = manager.clone();
        let second_publish =
            tokio::spawn(async move { TierConfigMgr::publish_candidate(&second_manager, second, Some("COLD-A")).await });
        tokio::task::yield_now().await;
        assert!(!second_publish.is_finished(), "a later update must remain behind the detached publish");

        drop(old);
        second_publish
            .await
            .expect("second publish task should join")
            .expect("second publish should follow the detached publish");
        let endpoint = manager
            .read()
            .await
            .tiers
            .get("COLD-A")
            .and_then(|tier| tier.rustfs.as_ref())
            .expect("second tier should be published")
            .endpoint
            .clone();
        assert_eq!(endpoint, "https://second.invalid");
    }

    #[tokio::test]
    async fn slow_tier_replacement_does_not_block_other_tiers_or_manager_reads() {
        let manager = TierConfigMgr::new();
        {
            let mut guard = manager.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("a"));
            install_lease_backend(&mut guard, "COLD-B", LeaseTestBackend::ready("b"));
        }
        let old_a = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
            .await
            .expect("tier A lease should be available");
        let old_b = TierConfigMgr::acquire_operation_lease(&manager, "COLD-B")
            .await
            .expect("tier B lease should be available");
        let mut candidate = empty_mgr();
        candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        candidate.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B"));
        candidate
            .tiers
            .get_mut("COLD-A")
            .and_then(|tier| tier.rustfs.as_mut())
            .expect("tier A payload should exist")
            .endpoint = "https://replacement.invalid".to_string();
        candidate
            .driver_cache
            .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("new-a")));
        let publish_manager = manager.clone();
        let publish =
            tokio::spawn(async move { TierConfigMgr::publish_candidate(&publish_manager, candidate, Some("COLD-A")).await });

        tokio::time::timeout(Duration::from_secs(1), async {
            while old_a.is_current(&manager).await {
                tokio::task::yield_now().await;
            }
        })
        .await
        .expect("tier A should enter draining");
        {
            let guard = manager.read().await;
            let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
            assert_eq!(
                lock_unpoisoned(&runtime).draining.keys().cloned().collect::<Vec<_>>(),
                vec!["COLD-A".to_string()]
            );
        }
        tokio::time::timeout(Duration::from_secs(1), manager.read())
            .await
            .expect("manager reads must not wait for tier A leases");
        let next_b = tokio::time::timeout(Duration::from_secs(1), TierConfigMgr::acquire_operation_lease(&manager, "COLD-B"))
            .await
            .expect("tier B lease acquisition must not wait for tier A")
            .expect("tier B lease should remain available");
        assert_eq!(next_b.generation(), old_b.generation());
        assert!(!publish.is_finished(), "tier A replacement should still wait for its lease");

        drop(old_a);
        publish
            .await
            .expect("publish task should join")
            .expect("tier A replacement should publish after its lease drains");
    }

    #[tokio::test]
    async fn direct_mutation_fails_closed_while_generation_is_active() {
        let manager = TierConfigMgr::new();
        {
            let mut guard = manager.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
        }
        let lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
            .await
            .expect("lease should be available");
        let err = manager
            .write()
            .await
            .remove("COLD-A", true)
            .await
            .expect_err("direct removal must not wait for an active generation");
        assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
        assert!(lease.is_current(&manager).await);
    }

    #[tokio::test]
    async fn steady_lease_acquisition_is_concurrent_across_tiers() {
        let manager = TierConfigMgr::new();
        {
            let mut guard = manager.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("a"));
            install_lease_backend(&mut guard, "COLD-B", LeaseTestBackend::ready("b"));
        }
        let mut tasks = Vec::new();
        for index in 0..200 {
            let manager = manager.clone();
            tasks.push(tokio::spawn(async move {
                let tier = if index % 2 == 0 { "COLD-A" } else { "COLD-B" };
                TierConfigMgr::acquire_operation_lease(&manager, tier).await
            }));
        }
        tokio::time::timeout(Duration::from_secs(1), async {
            for task in tasks {
                task.await
                    .expect("lease task should join")
                    .expect("published generation should remain available");
            }
        })
        .await
        .expect("steady lease acquisition should not serialize on manager writes or config hashing");
    }

    #[tokio::test]
    async fn cancelled_operation_releases_generation_lease() {
        let backend = LeaseTestBackend::blocking("cancelled");
        let manager = Arc::new(RwLock::new(empty_mgr()));
        {
            let mut guard = manager.write().await;
            install_lease_backend(&mut guard, "COLD-A", backend.clone());
        }
        let lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
            .await
            .expect("lease should be available");
        let task = tokio::spawn(async move { lease.remove("object", "").await });
        backend
            .remove_started
            .as_ref()
            .expect("blocking backend should expose start notification")
            .notified()
            .await;
        task.abort();
        let _ = task.await;

        let manager = manager.read().await;
        let runtime = registered_tier_driver_runtime(&manager).expect("runtime sidecar should be registered");
        drop(manager);
        let runtime = lock_unpoisoned(&runtime);
        let generation = runtime.generations.get("COLD-A").expect("generation should remain installed");
        assert_eq!(generation.active_leases.load(Ordering::Acquire), 0);
    }

    #[tokio::test]
    async fn tier_generations_are_isolated_between_runtime_instances() {
        let manager_a = TierConfigMgr::new();
        let manager_b = TierConfigMgr::new();
        {
            let mut guard = manager_a.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("a"));
        }
        {
            let mut guard = manager_b.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("b"));
        }
        let lease_a = TierConfigMgr::acquire_operation_lease(&manager_a, "COLD-A")
            .await
            .expect("instance A lease should be available");
        let lease_b = TierConfigMgr::acquire_operation_lease(&manager_b, "COLD-A")
            .await
            .expect("instance B lease should be available");

        assert!(lease_a.is_current(&manager_a).await);
        assert!(lease_b.is_current(&manager_b).await);
        assert!(!lease_a.is_current(&manager_b).await);
        assert!(!lease_b.is_current(&manager_a).await);
    }

    #[tokio::test]
    async fn lease_acquisition_has_constant_generation_state() {
        let manager = Arc::new(RwLock::new(empty_mgr()));
        {
            let mut guard = manager.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("bounded"));
        }
        for _ in 0..10_000 {
            drop(
                TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
                    .await
                    .expect("lease should be available"),
            );
        }

        let manager = manager.read().await;
        let runtime = registered_tier_driver_runtime(&manager).expect("runtime sidecar should be registered");
        drop(manager);
        let runtime = lock_unpoisoned(&runtime);
        assert_eq!(runtime.generations.len(), 1);
        assert_eq!(runtime.next_generation, 1);
        assert_eq!(
            runtime
                .generations
                .get("COLD-A")
                .expect("generation should remain installed")
                .active_leases
                .load(Ordering::Acquire),
            0
        );
    }

    #[tokio::test]
    async fn unrelated_reload_keeps_active_generation_current() {
        let manager = Arc::new(RwLock::new(empty_mgr()));
        {
            let mut manager = manager.write().await;
            install_lease_backend(&mut manager, "COLD-A", LeaseTestBackend::ready("a"));
            manager.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B"));
        }
        let lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
            .await
            .expect("lease should be available");
        let generation = lease.generation();
        let mut reloaded = HashMap::from([
            ("COLD-A".to_string(), build_rustfs_tier("COLD-A")),
            ("COLD-B".to_string(), build_rustfs_tier("COLD-B")),
            ("COLD-C".to_string(), build_rustfs_tier("COLD-C")),
        ]);
        reloaded
            .get_mut("COLD-B")
            .and_then(|tier| tier.rustfs.as_mut())
            .expect("tier B payload should exist")
            .access_key = "rotated".to_string();
        manager
            .write()
            .await
            .apply_reloaded_tiers(reloaded)
            .expect("unrelated reload should apply");

        assert!(lease.is_current(&manager).await);
        let next = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
            .await
            .expect("tier A should remain available");
        assert_eq!(next.generation(), generation);
    }

    #[tokio::test]
    async fn backend_identity_survives_credentials_rotation_but_rejects_route_change() {
        let manager = Arc::new(RwLock::new(empty_mgr()));
        {
            let mut guard = manager.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
        }
        let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
            .await
            .expect("old lease should be available");
        let identity = old.backend_identity();

        let mut rotated = build_rustfs_tier("COLD-A");
        let rotated_config = rotated.rustfs.as_mut().expect("rustfs payload should exist");
        rotated_config.access_key = "rotated-access".to_string();
        rotated_config.secret_key = "rotated-secret".to_string();
        manager.write().await.tiers.insert("COLD-A".to_string(), rotated);
        manager
            .write()
            .await
            .replace_driver("COLD-A", Box::new(LeaseTestBackend::ready("rotated")))
            .expect("rotated driver should install");
        let rotated = TierConfigMgr::acquire_operation_lease_for_backend_identity(&manager, "COLD-A", identity)
            .await
            .expect("credential rotation should preserve backend identity");
        assert_ne!(rotated.generation(), old.generation());
        assert_eq!(rotated.backend_identity(), identity);

        let mut moved = build_rustfs_tier("COLD-A");
        moved.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://moved.invalid".to_string();
        manager.write().await.tiers.insert("COLD-A".to_string(), moved);
        manager
            .write()
            .await
            .replace_driver("COLD-A", Box::new(LeaseTestBackend::ready("moved")))
            .expect("moved driver should install");
        let err = match TierConfigMgr::acquire_operation_lease_for_backend_identity(&manager, "COLD-A", identity).await {
            Ok(_) => panic!("route change must fail closed"),
            Err(err) => err,
        };
        assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
    }

    fn build_azure_tier(account_name: &str) -> TierConfig {
        TierConfig {
            version: "v1".to_string(),
            tier_type: TierType::Azure,
            name: "COLD-AZURE".to_string(),
            azure: Some(crate::services::tier::tier_config::TierAzure {
                endpoint: "https://blob.example.invalid".to_string(),
                access_key: account_name.to_string(),
                secret_key: "account-key".to_string(),
                bucket: "archive".to_string(),
                prefix: "objects/".to_string(),
                region: "us-east-1".to_string(),
                ..Default::default()
            }),
            ..Default::default()
        }
    }

    #[tokio::test]
    async fn operation_lease_rejects_nonempty_versions_without_exact_get_delete() {
        let manager = TierConfigMgr::new();
        let azure_tier = build_azure_tier("account-a");
        let azure_tier_name = azure_tier.name.clone();
        let exact_tier = build_rustfs_tier("COLD-EXACT");
        let exact_tier_name = exact_tier.name.clone();
        {
            let mut guard = manager.write().await;
            guard.tiers.insert(azure_tier_name.clone(), azure_tier);
            guard
                .replace_driver(&azure_tier_name, Box::new(LeaseTestBackend::ready("azure")))
                .expect("test Azure tier driver should install");
            guard.tiers.insert(exact_tier_name.clone(), exact_tier);
            guard
                .replace_driver(&exact_tier_name, Box::new(LeaseTestBackend::ready("exact")))
                .expect("test exact tier driver should install");
        }

        let lease = TierConfigMgr::acquire_operation_lease(&manager, &azure_tier_name)
            .await
            .expect("tier operation lease should resolve");
        let err = lease
            .validate_remote_version_id("provider-version")
            .expect_err("a provider without exact GET and DELETE must not commit a versioned transition");

        assert_eq!(err.kind(), io::ErrorKind::Unsupported);
        lease
            .validate_remote_version_id("")
            .expect("unversioned remote state remains supported");

        TierConfigMgr::acquire_operation_lease(&manager, &exact_tier_name)
            .await
            .expect("exact tier operation lease should resolve")
            .validate_remote_version_id("provider-version")
            .expect("providers declaring exact GET and DELETE must retain versioned transition support");
    }

    #[test]
    fn azure_account_name_is_part_of_backend_identity() {
        let account_a = build_azure_tier("account-a");
        let mut rotated_key = build_azure_tier("account-a");
        rotated_key.azure.as_mut().expect("azure payload should exist").secret_key = "rotated-key".to_string();
        let account_b = build_azure_tier("account-b");

        assert_eq!(
            tier_backend_identity(&account_a).expect("account A identity should encode"),
            tier_backend_identity(&rotated_key).expect("rotated key identity should encode"),
            "Azure account-key rotation must preserve destination identity"
        );
        assert_ne!(
            tier_backend_identity(&account_a).expect("account A identity should encode"),
            tier_backend_identity(&account_b).expect("account B identity should encode"),
            "Azure account-name changes must fence cleanup from the new account"
        );
    }

    #[test]
    fn wasabi_backend_identity_uses_its_canonical_physical_destination() {
        let mut wasabi = build_wasabi_tier("COLD-WASABI");
        let wasabi_config = wasabi.wasabi.as_mut().expect("Wasabi payload should exist");
        wasabi_config.endpoint = "https://s3.nl-1.wasabisys.com".to_string();
        wasabi_config.prefix = "/archive//".to_string();
        wasabi_config.region = "eu-central-1".to_string();

        let mut physical_s3 = build_s3_tier("COLD-WASABI");
        let s3 = physical_s3.s3.as_mut().expect("S3 payload should exist");
        s3.endpoint = "https://s3.eu-central-1.wasabisys.com".to_string();
        s3.bucket = wasabi_config.bucket.clone();
        s3.prefix = "archive".to_string();
        s3.region = wasabi_config.region.clone();

        assert_eq!(
            tier_backend_identity(&wasabi).expect("Wasabi identity should encode"),
            tier_backend_identity(&physical_s3).expect("physical S3 identity should encode")
        );
    }

    #[test]
    fn in_place_wasabi_type_changes_are_rejected() {
        let mut wasabi = build_wasabi_tier("COLD-WASABI");
        wasabi.wasabi.as_mut().expect("Wasabi payload should exist").region = "us-east-1".to_string();
        let mut s3 = build_s3_tier("COLD-WASABI");
        let s3_config = s3.s3.as_mut().expect("S3 payload should exist");
        s3_config.endpoint = "https://s3.wasabisys.com".to_string();
        s3_config.bucket = "wasabi-bucket".to_string();
        s3_config.prefix = "archive".to_string();
        s3_config.region = "us-east-1".to_string();

        for (current_tier, replacement_tier) in [(s3.clone_with_credentials(), wasabi.clone_with_credentials()), (wasabi, s3)] {
            let mut current = empty_mgr();
            current.tiers.insert("COLD-WASABI".to_string(), current_tier);
            let mut replacement = empty_mgr();
            replacement.tiers.insert("COLD-WASABI".to_string(), replacement_tier);

            let err = replaced_tier_destinations(&current, &replacement)
                .expect_err("in-place type changes must not bypass mixed-version Wasabi fencing");
            assert!(err.message.contains("add a new tier instead"), "{}", err.message);
        }
    }

    #[test]
    fn backend_identity_v2_has_stable_fixtures() {
        assert_eq!(
            tier_backend_identity(&build_rustfs_tier("COLD-A")).expect("identity should encode"),
            [
                112, 73, 111, 29, 53, 43, 207, 7, 170, 12, 99, 116, 213, 135, 173, 227, 6, 220, 179, 135, 232, 83, 25, 13, 128,
                98, 254, 103, 132, 128, 229, 97,
            ]
        );
        assert_eq!(
            tier_backend_identity(&build_azure_tier("account-a")).expect("Azure identity should encode"),
            [
                57, 188, 231, 23, 184, 188, 233, 228, 118, 24, 158, 152, 178, 169, 2, 146, 149, 152, 104, 0, 146, 196, 61, 145,
                18, 1, 188, 20, 36, 215, 100, 125,
            ]
        );
    }

    #[test]
    fn destination_identity_prefix_normalization_matches_driver_matrix() {
        assert_eq!(normalized_tier_prefix(&TierType::S3, "/foo//"), "foo");
        assert_eq!(normalized_tier_prefix(&TierType::Wasabi, "/foo//"), "foo");
        for tier_type in [
            TierType::RustFS,
            TierType::MinIO,
            TierType::Aliyun,
            TierType::Tencent,
            TierType::Huaweicloud,
            TierType::Azure,
            TierType::GCS,
            TierType::R2,
        ] {
            assert_eq!(normalized_tier_prefix(&tier_type, "foo/"), "foo");
            assert_eq!(normalized_tier_prefix(&tier_type, "foo//"), "foo/");
        }
    }

    #[tokio::test]
    async fn direct_rollback_fails_closed_until_unpublished_generation_is_idle() {
        let manager = Arc::new(RwLock::new(empty_mgr()));
        {
            let mut guard = manager.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("persisted"));
        }
        let persisted = HashMap::from([("COLD-A".to_string(), build_rustfs_tier("COLD-A"))]);

        let mut unpublished_config = build_rustfs_tier("COLD-A");
        unpublished_config
            .rustfs
            .as_mut()
            .expect("rustfs payload should exist")
            .endpoint = "https://unpublished.invalid".to_string();
        {
            let mut manager = manager.write().await;
            manager.tiers.insert("COLD-A".to_string(), unpublished_config);
            manager
                .replace_driver("COLD-A", Box::new(LeaseTestBackend::ready("unpublished")))
                .expect("unpublished driver should install before simulated save failure");
        }
        let unpublished = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
            .await
            .expect("unpublished lease should be visible before rollback");

        let err = manager
            .write()
            .await
            .apply_reloaded_tiers(persisted.clone())
            .expect_err("direct rollback must not wait for an active generation");
        assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
        assert!(unpublished.is_current(&manager).await);
        drop(unpublished);
        manager
            .write()
            .await
            .apply_reloaded_tiers(persisted)
            .expect("rollback should apply once the generation is idle");

        let manager_guard = manager.read().await;
        assert_eq!(
            manager_guard
                .tiers
                .get("COLD-A")
                .and_then(|tier| tier.rustfs.as_ref())
                .expect("persisted tier should be restored")
                .endpoint,
            "https://example-compat.invalid"
        );
        let runtime = registered_tier_driver_runtime(&manager_guard).expect("runtime sidecar should remain registered");
        assert!(lock_unpoisoned(&runtime).generations.get("COLD-A").is_none());
    }

    #[derive(Debug)]
    struct CasConfigStore {
        objects: tokio::sync::Mutex<HashMap<String, (Vec<u8>, String)>>,
        legacy_state: tokio::sync::Mutex<Option<Vec<u8>>>,
        if_match_race_rewrite: tokio::sync::Mutex<std::collections::VecDeque<(String, Vec<u8>)>>,
        next_etag: AtomicUsize,
        fail_put: AtomicBool,
        truncate_reference_page_without_marker: AtomicBool,
        lock_manager: Arc<rustfs_lock::GlobalLockManager>,
        lock_requests: Mutex<Vec<(String, String)>>,
        listed_versions: Mutex<Vec<ObjectInfo>>,
    }

    impl Default for CasConfigStore {
        fn default() -> Self {
            Self {
                objects: tokio::sync::Mutex::new(HashMap::new()),
                legacy_state: tokio::sync::Mutex::new(None),
                if_match_race_rewrite: tokio::sync::Mutex::new(std::collections::VecDeque::new()),
                next_etag: AtomicUsize::new(0),
                fail_put: AtomicBool::new(false),
                truncate_reference_page_without_marker: AtomicBool::new(false),
                lock_manager: Arc::new(rustfs_lock::GlobalLockManager::new()),
                lock_requests: Mutex::new(Vec::new()),
                listed_versions: Mutex::new(Vec::new()),
            }
        }
    }

    impl CasConfigStore {
        fn add_listed_version(&self, object: ObjectInfo) {
            self.listed_versions
                .lock()
                .expect("tier reference fixture should not poison")
                .push(object);
        }

        async fn insert_config_object(&self, object: String, data: Vec<u8>) {
            self.objects
                .lock()
                .await
                .insert(object, (data, "reference-proof-etag".to_string()));
        }

        async fn rewrite_on_next_if_match(&self, object: String, data: Vec<u8>) {
            self.if_match_race_rewrite.lock().await.push_back((object, data));
        }

        fn omit_truncated_reference_marker(&self) {
            self.truncate_reference_page_without_marker.store(true, Ordering::SeqCst);
        }
    }

    #[async_trait::async_trait]
    impl ObjectIO for CasConfigStore {
        type Error = Error;
        type RangeSpec = HTTPRangeSpec;
        type HeaderMap = HeaderMap;
        type ObjectOptions = ObjectOptions;
        type ObjectInfo = ObjectInfo;
        type GetObjectReader = GetObjectReader;
        type PutObjectReader = PutObjReader;

        async fn get_object_reader(
            &self,
            bucket: &str,
            object: &str,
            _range: Option<Self::RangeSpec>,
            _headers: Self::HeaderMap,
            _opts: &Self::ObjectOptions,
        ) -> Result<Self::GetObjectReader> {
            let legacy_config_path = tier_config_path(TIER_CONFIG_LEGACY_FILE);
            let (data, etag) = if object == legacy_config_path {
                let state = self.legacy_state.lock().await;
                let data = state.as_ref().ok_or(Error::ConfigNotFound)?;
                (data.clone(), "legacy-config-etag".to_string())
            } else {
                let objects = self.objects.lock().await;
                let (data, etag) = objects.get(object).ok_or(Error::ConfigNotFound)?;
                (data.clone(), etag.clone())
            };
            Ok(GetObjectReader {
                stream: Box::new(Cursor::new(data.clone())),
                object_info: ObjectInfo {
                    bucket: bucket.to_string(),
                    name: object.to_string(),
                    size: data.len() as i64,
                    actual_size: data.len() as i64,
                    etag: Some(etag),
                    ..Default::default()
                },
                buffered_body: None,
                body_source: Default::default(),
            })
        }

        async fn put_object(
            &self,
            bucket: &str,
            object: &str,
            data: &mut Self::PutObjectReader,
            opts: &Self::ObjectOptions,
        ) -> Result<Self::ObjectInfo> {
            if self.fail_put.load(Ordering::SeqCst) {
                return Err(Error::other("injected tier config save failure"));
            }
            let mut payload = Vec::new();
            tokio::io::AsyncReadExt::read_to_end(&mut data.stream, &mut payload).await?;
            let race_rewrite = if opts
                .http_preconditions
                .as_ref()
                .and_then(HTTPPreconditions::if_match_value)
                .is_some()
            {
                self.if_match_race_rewrite.lock().await.pop_front()
            } else {
                None
            };
            let mut objects = self.objects.lock().await;
            if let Some((target, data)) = race_rewrite
                && target == object
            {
                let etag = format!("etag-{}", self.next_etag.fetch_add(1, Ordering::SeqCst) + 1);
                objects.insert(object.to_string(), (data, etag));
            }
            match objects.get(object) {
                Some((_, etag)) => opts.precondition_check(&ObjectInfo {
                    etag: Some(etag.clone()),
                    ..Default::default()
                })?,
                None => {
                    if opts
                        .http_preconditions
                        .as_ref()
                        .and_then(HTTPPreconditions::if_match_value)
                        .is_some()
                    {
                        return Err(Error::ObjectNotFound(bucket.to_string(), object.to_string()));
                    }
                }
            }
            let etag = format!("etag-{}", self.next_etag.fetch_add(1, Ordering::SeqCst) + 1);
            objects.insert(object.to_string(), (payload.clone(), etag.clone()));
            Ok(ObjectInfo {
                bucket: bucket.to_string(),
                name: object.to_string(),
                size: payload.len() as i64,
                actual_size: payload.len() as i64,
                etag: Some(etag),
                ..Default::default()
            })
        }
    }

    #[async_trait::async_trait]
    impl ObjectOperations for CasConfigStore {
        type Error = Error;
        type ObjectInfo = ObjectInfo;
        type ObjectOptions = ObjectOptions;
        type FileInfo = FileInfo;
        type ObjectToDelete = ObjectToDelete;
        type DeletedObject = DeletedObject;

        async fn get_object_info(&self, bucket: &str, object: &str, _opts: &Self::ObjectOptions) -> Result<Self::ObjectInfo> {
            let objects = self.objects.lock().await;
            let (data, etag) = objects
                .get(object)
                .ok_or(Error::ObjectNotFound(bucket.to_string(), object.to_string()))?;
            Ok(ObjectInfo {
                bucket: bucket.to_string(),
                name: object.to_string(),
                size: data.len() as i64,
                actual_size: data.len() as i64,
                etag: Some(etag.clone()),
                ..Default::default()
            })
        }

        async fn verify_object_integrity(&self, _bucket: &str, _object: &str, _opts: &Self::ObjectOptions) -> Result<()> {
            Err(Error::NotImplemented)
        }

        async fn copy_object(
            &self,
            _src_bucket: &str,
            _src_object: &str,
            _dst_bucket: &str,
            _dst_object: &str,
            _src_info: &mut Self::ObjectInfo,
            _src_opts: &Self::ObjectOptions,
            _dst_opts: &Self::ObjectOptions,
        ) -> Result<Self::ObjectInfo> {
            Err(Error::NotImplemented)
        }

        async fn delete_object_version(
            &self,
            _bucket: &str,
            _object: &str,
            _fi: &Self::FileInfo,
            _force_del_marker: bool,
        ) -> Result<()> {
            Err(Error::NotImplemented)
        }

        async fn delete_object(&self, bucket: &str, object: &str, _opts: Self::ObjectOptions) -> Result<Self::ObjectInfo> {
            let mut objects = self.objects.lock().await;
            let (data, etag) = objects
                .remove(object)
                .ok_or_else(|| Error::ObjectNotFound(bucket.to_string(), object.to_string()))?;
            Ok(ObjectInfo {
                bucket: bucket.to_string(),
                name: object.to_string(),
                size: data.len() as i64,
                actual_size: data.len() as i64,
                etag: Some(etag),
                ..Default::default()
            })
        }

        async fn delete_objects(
            &self,
            _bucket: &str,
            _objects: Vec<Self::ObjectToDelete>,
            _opts: Self::ObjectOptions,
        ) -> (Vec<Self::DeletedObject>, Vec<Option<Self::Error>>) {
            (Vec::new(), vec![Some(Error::NotImplemented)])
        }

        async fn put_object_metadata(
            &self,
            _bucket: &str,
            _object: &str,
            _opts: &Self::ObjectOptions,
        ) -> Result<Self::ObjectInfo> {
            Err(Error::NotImplemented)
        }

        async fn get_object_tags(&self, _bucket: &str, _object: &str, _opts: &Self::ObjectOptions) -> Result<String> {
            Err(Error::NotImplemented)
        }

        async fn put_object_tags(
            &self,
            _bucket: &str,
            _object: &str,
            _tags: &str,
            _opts: &Self::ObjectOptions,
        ) -> Result<Self::ObjectInfo> {
            Err(Error::NotImplemented)
        }

        async fn delete_object_tags(
            &self,
            _bucket: &str,
            _object: &str,
            _opts: &Self::ObjectOptions,
        ) -> Result<Self::ObjectInfo> {
            Err(Error::NotImplemented)
        }

        async fn add_partial(&self, _bucket: &str, _object: &str, _version_id: &str) -> Result<()> {
            Err(Error::NotImplemented)
        }

        async fn transition_object(&self, _bucket: &str, _object: &str, _opts: &Self::ObjectOptions) -> Result<()> {
            Err(Error::NotImplemented)
        }

        async fn restore_transitioned_object(
            self: Arc<Self>,
            _bucket: &str,
            _object: &str,
            _opts: &Self::ObjectOptions,
        ) -> Result<()> {
            Err(Error::NotImplemented)
        }
    }

    #[async_trait::async_trait]
    impl BucketOperations for CasConfigStore {
        type Error = Error;

        async fn make_bucket(
            &self,
            _bucket: &str,
            _opts: &crate::storage_api_contracts::bucket::MakeBucketOptions,
        ) -> Result<()> {
            Err(Error::NotImplemented)
        }

        async fn get_bucket_info(
            &self,
            _bucket: &str,
            _opts: &crate::storage_api_contracts::bucket::BucketOptions,
        ) -> Result<crate::storage_api_contracts::bucket::BucketInfo> {
            Err(Error::NotImplemented)
        }

        async fn list_bucket(
            &self,
            _opts: &crate::storage_api_contracts::bucket::BucketOptions,
        ) -> Result<Vec<crate::storage_api_contracts::bucket::BucketInfo>> {
            let mut seen = HashSet::new();
            let mut buckets = Vec::new();
            for object in self
                .listed_versions
                .lock()
                .expect("tier reference fixture should not poison")
                .iter()
            {
                if seen.insert(object.bucket.clone()) {
                    buckets.push(crate::storage_api_contracts::bucket::BucketInfo {
                        name: object.bucket.clone(),
                        ..Default::default()
                    });
                }
            }
            Ok(buckets)
        }

        async fn delete_bucket(
            &self,
            _bucket: &str,
            _opts: &crate::storage_api_contracts::bucket::DeleteBucketOptions,
        ) -> Result<()> {
            Err(Error::NotImplemented)
        }
    }

    #[async_trait::async_trait]
    impl ListOperations for CasConfigStore {
        type Error = Error;
        type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
        type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
        type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
        type WalkOptions = TierReferenceProofWalkOptions;
        type WalkCancellation = tokio_util::sync::CancellationToken;
        type WalkResultSender = tokio::sync::mpsc::Sender<StorageObjectInfoOrErr<ObjectInfo, Error>>;

        async fn list_objects_v2(
            self: Arc<Self>,
            bucket: &str,
            prefix: &str,
            continuation_token: Option<String>,
            _delimiter: Option<String>,
            max_keys: i32,
            _fetch_owner: bool,
            _start_after: Option<String>,
            _incl_deleted: bool,
        ) -> Result<Self::ListObjectsV2Info> {
            let mut objects = if bucket == RUSTFS_META_BUCKET {
                self.objects
                    .lock()
                    .await
                    .keys()
                    .filter(|object| object.starts_with(prefix))
                    .map(|object| ObjectInfo {
                        bucket: bucket.to_string(),
                        name: object.clone(),
                        ..Default::default()
                    })
                    .collect::<Vec<_>>()
            } else {
                Vec::new()
            };
            objects.sort_by(|left, right| left.name.cmp(&right.name));
            if let Some(marker) = continuation_token {
                objects.retain(|object| object.name > marker);
            }
            let limit = usize::try_from(max_keys).unwrap_or(0);
            let is_truncated = objects.len() > limit;
            if is_truncated {
                objects.truncate(limit);
            }
            let next_continuation_token = is_truncated
                .then(|| objects.last().map(|object| object.name.clone()))
                .flatten();
            Ok(StorageListObjectsV2Info {
                objects,
                is_truncated,
                next_continuation_token,
                ..Default::default()
            })
        }

        async fn list_object_versions(
            self: Arc<Self>,
            bucket: &str,
            prefix: &str,
            marker: Option<String>,
            version_marker: Option<String>,
            _delimiter: Option<String>,
            max_keys: i32,
        ) -> Result<Self::ListObjectVersionsInfo> {
            let mut objects: Vec<_> = self
                .listed_versions
                .lock()
                .expect("tier reference fixture should not poison")
                .iter()
                .filter(|object| object.bucket == bucket && object.name.starts_with(prefix))
                .cloned()
                .collect();
            objects.sort_by(|left, right| tier_test_object_marker(left).cmp(&tier_test_object_marker(right)));
            if marker.is_some() || version_marker.is_some() {
                let marker = (marker.unwrap_or_default(), version_marker.unwrap_or_default());
                objects.retain(|object| tier_test_object_marker(object) > marker);
            }
            let limit = match usize::try_from(max_keys) {
                Ok(limit) => limit,
                Err(_) => 0,
            };
            let is_truncated = objects.len() > limit;
            if is_truncated {
                objects.truncate(limit);
            }
            let (mut next_marker, next_version_idmarker) = if is_truncated {
                objects
                    .last()
                    .map(|object| (Some(object.name.clone()), object.version_id.map(|version| version.to_string())))
                    .unwrap_or((None, None))
            } else {
                (None, None)
            };
            if is_truncated && self.truncate_reference_page_without_marker.load(Ordering::SeqCst) {
                next_marker = None;
            }
            Ok(StorageListObjectVersionsInfo {
                is_truncated,
                next_marker,
                next_version_idmarker,
                objects,
                prefixes: Vec::new(),
            })
        }

        async fn walk(
            self: Arc<Self>,
            _rx: Self::WalkCancellation,
            _bucket: &str,
            _prefix: &str,
            _result: Self::WalkResultSender,
            _opts: Self::WalkOptions,
        ) -> Result<()> {
            Err(Error::NotImplemented)
        }
    }

    fn tier_test_object_marker(object: &ObjectInfo) -> (String, String) {
        (
            object.name.clone(),
            object.version_id.map(|version| version.to_string()).unwrap_or_default(),
        )
    }

    fn transitioned_tier_object(
        bucket: &str,
        object: &str,
        tier_name: &str,
        backend_identity: Option<TierDestinationId>,
    ) -> ObjectInfo {
        let mut user_defined = HashMap::new();
        if let Some(identity) = backend_identity {
            rustfs_utils::http::metadata_compat::insert_str(
                &mut user_defined,
                rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
                rustfs_utils::crypto::hex(identity),
            );
        }
        ObjectInfo {
            bucket: bucket.to_string(),
            name: object.to_string(),
            user_defined: Arc::new(user_defined),
            transitioned_object: crate::storage_api_contracts::lifecycle::TransitionedObject {
                name: object.to_string(),
                tier: tier_name.to_string(),
                status: rustfs_filemeta::TRANSITION_COMPLETE.to_string(),
                ..Default::default()
            },
            ..Default::default()
        }
    }

    #[async_trait::async_trait]
    impl NamespaceLocking for CasConfigStore {
        type Error = Error;
        type NamespaceLock = rustfs_lock::NamespaceLockWrapper;

        async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<Self::NamespaceLock> {
            self.lock_requests
                .lock()
                .expect("tier config lock request log should not poison")
                .push((bucket.to_string(), object.to_string()));
            let lock =
                rustfs_lock::NamespaceLock::with_local_manager("tier-config-update-test".to_string(), self.lock_manager.clone());
            Ok(rustfs_lock::NamespaceLockWrapper::new(
                lock,
                rustfs_lock::ObjectKey::new(bucket.to_string(), object.to_string()),
                "tier-config-update-test-owner".to_string(),
            ))
        }
    }

    #[tokio::test]
    async fn remove_and_clear_full_update_paths_preserve_force() {
        let remove_store = Arc::new(CasConfigStore::default());
        let mut persisted = empty_mgr();
        persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        persisted
            .save_tiering_config_if_current(remove_store.clone(), None)
            .await
            .expect("remove fixture should persist");
        let remove_manager = TierConfigMgr::new();
        {
            let mut guard = remove_manager.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::in_use("remove"));
        }
        TierConfigMgr::remove_and_save_with(&remove_manager, remove_store.clone(), "COLD-A", true)
            .await
            .expect("forced remove must complete through load, mutate, save, and publish");
        assert!(!remove_manager.read().await.tiers.contains_key("COLD-A"));
        assert!(
            load_tier_config_for_update(remove_store)
                .await
                .expect("removed config should reload")
                .0
                .tiers
                .is_empty(),
            "forced remove must persist the empty candidate"
        );

        let clear_store = Arc::new(CasConfigStore::default());
        persisted
            .save_tiering_config_if_current(clear_store.clone(), None)
            .await
            .expect("clear fixture should persist");
        let clear_manager = TierConfigMgr::new();
        {
            let mut guard = clear_manager.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::in_use("clear"));
        }
        TierConfigMgr::clear_and_save_with(&clear_manager, clear_store.clone(), true)
            .await
            .expect("forced clear must complete through load, mutate, save, and publish");
        assert!(clear_manager.read().await.tiers.is_empty());
        assert!(
            load_tier_config_for_update(clear_store)
                .await
                .expect("cleared config should reload")
                .0
                .tiers
                .is_empty(),
            "forced clear must persist the empty candidate"
        );
    }

    #[tokio::test]
    async fn zero_reference_proof_blocks_remove_before_config_save() {
        let store = Arc::new(CasConfigStore::default());
        let tier = build_rustfs_tier("COLD-A");
        let identity = tier_backend_identity(&tier).expect("test tier identity should encode");
        let mut persisted = empty_mgr();
        persisted.tiers.insert("COLD-A".to_string(), tier.clone_with_credentials());
        persisted
            .save_tiering_config_if_current(store.clone(), None)
            .await
            .expect("reference proof fixture should persist");
        store.add_listed_version(transitioned_tier_object("photos", "2026/a.jpg", "COLD-A", Some(identity)));

        let manager = TierConfigMgr::new();
        manager.write().await.tiers.insert("COLD-A".to_string(), tier);
        let err = TierConfigMgr::remove_and_save_with(&manager, store.clone(), "COLD-A", true)
            .await
            .expect_err("authoritative object reference must block tier removal");

        match err {
            TierConfigUpdateError::Publish(err) => {
                assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
                assert!(err.message.contains("photos/2026/a.jpg"), "{}", err.message);
            }
            other => panic!("reference proof failures must be surfaced as publish errors: {other:?}"),
        }
        assert!(manager.read().await.tiers.contains_key("COLD-A"));
        assert!(
            load_tier_config_for_update(store)
                .await
                .expect("blocked config should still reload")
                .0
                .tiers
                .contains_key("COLD-A"),
            "blocked removal must not persist the empty candidate"
        );
    }

    #[tokio::test]
    async fn zero_reference_proof_blocks_clear_before_config_save() {
        let store = Arc::new(CasConfigStore::default());
        let tier_a = build_rustfs_tier("COLD-A");
        let tier_b = build_rustfs_tier("COLD-B");
        let identity_b = tier_backend_identity(&tier_b).expect("test tier identity should encode");
        let mut persisted = empty_mgr();
        persisted.tiers.insert("COLD-A".to_string(), tier_a.clone_with_credentials());
        persisted.tiers.insert("COLD-B".to_string(), tier_b.clone_with_credentials());
        persisted
            .save_tiering_config_if_current(store.clone(), None)
            .await
            .expect("reference proof fixture should persist");
        store.add_listed_version(transitioned_tier_object("photos", "2026/b.jpg", "COLD-B", Some(identity_b)));

        let manager = TierConfigMgr::new();
        {
            let mut guard = manager.write().await;
            guard.tiers.insert("COLD-A".to_string(), tier_a);
            guard.tiers.insert("COLD-B".to_string(), tier_b);
        }
        let err = TierConfigMgr::clear_and_save_with(&manager, store.clone(), true)
            .await
            .expect_err("authoritative object reference must block tier clear");

        match err {
            TierConfigUpdateError::Publish(err) => {
                assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
                assert!(err.message.contains("photos/2026/b.jpg"), "{}", err.message);
            }
            other => panic!("reference proof failures must be surfaced as publish errors: {other:?}"),
        }
        let guard = manager.read().await;
        assert!(guard.tiers.contains_key("COLD-A"));
        assert!(guard.tiers.contains_key("COLD-B"));
        drop(guard);
        let reloaded = load_tier_config_for_update(store)
            .await
            .expect("blocked config should still reload")
            .0;
        assert!(reloaded.tiers.contains_key("COLD-A"));
        assert!(reloaded.tiers.contains_key("COLD-B"));
    }

    #[tokio::test]
    async fn zero_reference_proof_rebind_allows_only_new_destination_references() {
        let current = build_rustfs_tier("COLD-A");
        let current_identity = tier_backend_identity(&current).expect("current identity should encode");
        let mut replacement = build_rustfs_tier("COLD-A");
        replacement.rustfs.as_mut().expect("replacement payload should exist").prefix = "new-prefix".to_string();
        let replacement_identity = tier_backend_identity(&replacement).expect("replacement identity should encode");
        assert_ne!(current_identity, replacement_identity);
        let target = TierMutationIntentTarget {
            tier_name: "COLD-A".to_string(),
            old_backend_identity: Some(current_identity),
            new_backend_identity: Some(replacement_identity),
        };

        let store = Arc::new(CasConfigStore::default());
        store.add_listed_version(transitioned_tier_object(
            "photos",
            "2026/new-destination.jpg",
            "COLD-A",
            Some(replacement_identity),
        ));
        ensure_no_authoritative_tier_object_references(store.clone(), std::slice::from_ref(&target))
            .await
            .expect("references already written with the new destination identity should not block rebind");

        store.add_listed_version(transitioned_tier_object(
            "photos",
            "2026/old-destination.jpg",
            "COLD-A",
            Some(current_identity),
        ));
        let err = ensure_no_authoritative_tier_object_references(store, &[target])
            .await
            .expect_err("references to the old destination identity must block rebind");
        assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
        assert!(err.message.contains("photos/2026/old-destination.jpg"), "{}", err.message);
    }

    #[tokio::test]
    async fn zero_reference_proof_blocks_persisted_journal_transaction_and_free_version_references() {
        let current = build_rustfs_tier("COLD-A");
        let current_identity = tier_backend_identity(&current).expect("current identity should encode");
        let replacement = build_azure_tier("account-a");
        let replacement_identity = tier_backend_identity(&replacement).expect("replacement identity should encode");
        let target = TierMutationIntentTarget {
            tier_name: "COLD-A".to_string(),
            old_backend_identity: Some(current_identity),
            new_backend_identity: Some(replacement_identity),
        };

        let journal_store = Arc::new(CasConfigStore::default());
        let journal = crate::bucket::lifecycle::tier_sweeper::Jentry {
            obj_name: "remote/journal-object".to_string(),
            version_id: "v1".to_string(),
            tier_name: "COLD-A".to_string(),
            backend_identity: Some(current_identity),
            version_id_exact: true,
            version_state: rustfs_filemeta::TransitionVersionState::Exact,
            state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
            source: None,
        };
        journal_store
            .insert_config_object(
                crate::bucket::lifecycle::tier_delete_journal::tier_delete_journal_object_name(&journal),
                crate::bucket::lifecycle::tier_delete_journal::encode_tier_delete_journal_entry(&journal)
                    .expect("journal record should encode"),
            )
            .await;
        let err = ensure_no_authoritative_tier_object_references(journal_store, std::slice::from_ref(&target))
            .await
            .expect_err("unfinished delete journal for the old backend must block rebind");
        assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
        assert!(err.message.contains(TIER_DELETE_JOURNAL_PREFIX), "{}", err.message);

        let transaction_store = Arc::new(CasConfigStore::default());
        let transaction = crate::bucket::lifecycle::transition_transaction::TransitionTransaction::new(
            crate::bucket::lifecycle::transition_transaction::TransitionTransactionInit {
                deployment_id: uuid::Uuid::new_v4(),
                transaction_id: uuid::Uuid::new_v4(),
                owner_epoch: uuid::Uuid::new_v4(),
                write_id: uuid::Uuid::new_v4(),
                source: crate::bucket::lifecycle::transition_transaction::TransitionSourceIdentity {
                    bucket: "photos".to_string(),
                    object: "2026/transaction.jpg".to_string(),
                    version_id: None,
                    data_dir: uuid::Uuid::new_v4(),
                    mod_time_unix_nanos: 1,
                    size: 1,
                    etag: "etag".to_string(),
                    version_mode: crate::bucket::lifecycle::transition_transaction::TransitionSourceVersionMode::Unversioned,
                },
                tier_name: "COLD-A".to_string(),
                backend_fingerprint: current_identity,
                not_after_unix_nanos: 2,
            },
        )
        .expect("transaction record should initialize");
        transaction_store
            .insert_config_object(
                crate::bucket::lifecycle::transition_transaction::transition_transaction_record_object_name(
                    transaction.transaction_id,
                )
                .expect("transaction record path should build"),
                transaction.encode().expect("transaction record should encode"),
            )
            .await;
        let err = ensure_no_authoritative_tier_object_references(transaction_store, std::slice::from_ref(&target))
            .await
            .expect_err("unfinished transition transaction for the old backend must block rebind");
        assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
        assert!(err.message.contains(TRANSITION_TRANSACTION_RECORD_PREFIX), "{}", err.message);

        let free_version_store = Arc::new(CasConfigStore::default());
        let mut free_version = transitioned_tier_object("photos", "2026/free-version.jpg", "COLD-A", Some(current_identity));
        free_version.transitioned_object.status = "pending".to_string();
        free_version.transitioned_object.free_version = true;
        ensure_no_authoritative_tier_object_references(free_version_store.clone(), std::slice::from_ref(&target))
            .await
            .expect("empty free-version fixture should permit rebind");
        free_version_store.add_listed_version(free_version);
        let err = ensure_no_authoritative_tier_object_references(free_version_store, &[target])
            .await
            .expect_err("recoverable free version for the old backend must block rebind");
        assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
        assert!(err.message.contains("photos/2026/free-version.jpg"), "{}", err.message);
    }

    #[tokio::test]
    async fn zero_reference_proof_fails_closed_when_version_listing_marker_is_missing() {
        let current = build_rustfs_tier("COLD-A");
        let current_identity = tier_backend_identity(&current).expect("current identity should encode");
        let target = TierMutationIntentTarget {
            tier_name: "COLD-A".to_string(),
            old_backend_identity: Some(current_identity),
            new_backend_identity: None,
        };
        let store = Arc::new(CasConfigStore::default());
        for index in 0..=TIER_REFERENCE_PROOF_LIST_LIMIT {
            store.add_listed_version(ObjectInfo {
                bucket: "photos".to_string(),
                name: format!("safe/{index:04}.jpg"),
                ..Default::default()
            });
        }
        store.omit_truncated_reference_marker();

        let err = ensure_no_authoritative_tier_object_references(store, &[target])
            .await
            .expect_err("truncated authoritative reference scan without a marker must fail closed");
        assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
        assert!(
            err.message.contains("truncated without a next marker"),
            "unexpected reference proof error: {}",
            err.message
        );
    }

    #[test]
    fn zero_reference_proof_rebind_identity_boundaries_fail_closed() {
        let tier = build_rustfs_tier("COLD-A");
        let current_identity = tier_backend_identity(&tier).expect("current identity should encode");
        let replacement_identity =
            tier_backend_identity(&build_azure_tier("account-a")).expect("replacement identity should encode");
        let target = TierMutationIntentTarget {
            tier_name: "COLD-A".to_string(),
            old_backend_identity: Some(current_identity),
            new_backend_identity: Some(current_identity),
        };
        let object = transitioned_tier_object("photos", "2026/a.jpg", "COLD-A", Some(current_identity));
        assert!(!tier_object_blocks_target_rebind(&object, &target).expect("matching identity should parse"));

        let target = TierMutationIntentTarget {
            tier_name: "COLD-A".to_string(),
            old_backend_identity: Some(current_identity),
            new_backend_identity: Some(replacement_identity),
        };
        assert!(tier_object_blocks_target_rebind(&object, &target).expect("mismatched identity should parse"));

        let object_without_identity = transitioned_tier_object("photos", "2026/b.jpg", "COLD-A", None);
        assert!(
            tier_object_blocks_target_rebind(&object_without_identity, &target)
                .expect("missing identity means unknown reference owner")
        );

        let mut pending = transitioned_tier_object("photos", "2026/c.jpg", "COLD-A", Some(current_identity));
        pending.transitioned_object.status = "pending".to_string();
        assert!(!tier_object_blocks_target_rebind(&pending, &target).expect("non-complete status should not block"));
    }

    #[tokio::test]
    async fn mutation_target_proof_uses_persisted_config_snapshot_not_stale_manager() {
        let manager = TierConfigMgr::new();
        let store = Arc::new(CasConfigStore::default());
        let mut candidate = empty_mgr();
        candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        candidate.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B"));
        candidate
            .save_tiering_config_if_current(store.clone(), None)
            .await
            .expect("stale-manager fixture should persist");
        let (candidate, version) = load_tier_config_for_update(store.clone())
            .await
            .expect("stale-manager fixture should reload with an ETag");
        let update = TierConfigMgr::admin_update_lock(&manager).await;

        TierConfigMgr::update_candidate_owned(
            &manager,
            store.clone(),
            candidate,
            version,
            TierCandidateMutation::Remove("COLD-A".to_string(), true),
            update,
            None,
        )
        .await
        .expect("authoritative proof should use the loaded config even when the local manager is stale");
        assert!(!manager.read().await.tiers.contains_key("COLD-A"));
        assert!(manager.read().await.tiers.contains_key("COLD-B"));
        let reloaded = load_tier_config_for_update(store)
            .await
            .expect("removed config should reload")
            .0;
        assert!(!reloaded.tiers.contains_key("COLD-A"));
        assert!(reloaded.tiers.contains_key("COLD-B"));
    }

    #[tokio::test]
    async fn legacy_config_without_etag_blocks_non_add_mutations_before_save() {
        for (case, mutation) in [
            (
                "edit",
                TierCandidateMutation::Edit(
                    "COLD-A".to_string(),
                    TierCreds {
                        access_key: "rotated-access".to_string(),
                        secret_key: "rotated-secret".to_string(),
                        ..Default::default()
                    },
                ),
            ),
            ("remove", TierCandidateMutation::Remove("COLD-A".to_string(), true)),
            ("clear", TierCandidateMutation::Clear(true)),
        ] {
            let manager = TierConfigMgr::new();
            let store = Arc::new(CasConfigStore::default());
            let mut legacy = empty_mgr();
            legacy.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
            let legacy_data = legacy.marshal().expect("legacy tier config should encode");
            *store.legacy_state.lock().await = Some(legacy_data.to_vec());

            let (loaded, version) = load_tier_config_for_update(store.clone())
                .await
                .expect("legacy tier config should load for update");
            assert!(loaded.tiers.contains_key("COLD-A"), "{case} fixture should load legacy config");
            assert_eq!(version, None, "{case} fixture must represent a legacy config without current ETag");

            let err = TierConfigMgr::update_candidate_with_config_lock(&manager, store.clone(), mutation)
                .await
                .expect_err("non-add legacy config mutations must fail closed before save");

            match err {
                TierConfigUpdateError::Load(err) => {
                    assert!(
                        err.to_string().contains("requires an existing config ETag"),
                        "{case} returned unexpected load error: {err}"
                    );
                }
                other => panic!("{case} should fail before mutation/save, got {other:?}"),
            }
            assert!(
                !store.objects.lock().await.contains_key(&tier_config_path(TIER_CONFIG_FILE)),
                "{case} must not create the durable binary tier config after a failed legacy mutation"
            );
            assert!(
                manager.read().await.tiers.is_empty(),
                "{case} must not publish a failed legacy mutation into the live manager"
            );
        }
    }

    #[test]
    fn add_target_proof_ignores_unchanged_persisted_tiers_when_manager_is_stale() {
        let mut current = empty_mgr();
        current.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));

        let mut candidate = empty_mgr();
        candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        candidate.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B"));

        let targets = TierCandidateMutation::Add(build_rustfs_tier("COLD-B"), true)
            .affected_targets(&current, &candidate)
            .expect("add proof should ignore unchanged durable tiers");
        assert_eq!(targets.len(), 1);
        assert_eq!(targets[0].tier_name, "COLD-B");
        assert!(targets[0].old_backend_identity.is_none());
        assert!(targets[0].new_backend_identity.is_some());
    }

    #[tokio::test]
    async fn update_with_config_lock_add_uses_loaded_snapshot_for_target_proof() {
        let manager = TierConfigMgr::new();
        let store = Arc::new(CasConfigStore::default());
        let mut persisted = empty_mgr();
        persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        persisted
            .save_tiering_config_if_current(store.clone(), None)
            .await
            .expect("add fixture should persist");

        TierConfigMgr::update_candidate_with_config_lock(
            &manager,
            store.clone(),
            TierCandidateMutation::Add(build_rustfs_tier("COLD-B"), true),
        )
        .await
        .expect("add proof must ignore unchanged durable tiers missing from the stale manager");

        let reloaded = load_tier_config_for_update(store)
            .await
            .expect("updated config should reload")
            .0;
        assert!(reloaded.tiers.contains_key("COLD-A"));
        assert!(reloaded.tiers.contains_key("COLD-B"));
    }

    #[tokio::test]
    async fn failed_owned_update_restores_generation_and_reports_save_error() {
        let manager = TierConfigMgr::new();
        {
            let mut guard = manager.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
        }
        let store = Arc::new(CasConfigStore::default());
        let mut candidate = empty_mgr();
        candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        candidate
            .save_tiering_config_if_current(store.clone(), None)
            .await
            .expect("save-failure fixture should persist");
        let (_, version) = load_tier_config_for_update(store.clone())
            .await
            .expect("save-failure fixture should reload with an ETag");
        store.fail_put.store(true, Ordering::SeqCst);
        candidate
            .driver_cache
            .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("candidate")));
        let update = TierConfigMgr::admin_update_lock(&manager).await;

        let err = TierConfigMgr::update_candidate_owned(
            &manager,
            store.clone(),
            candidate,
            version.clone(),
            TierCandidateMutation::Remove("COLD-A".to_string(), true),
            update,
            None,
        )
        .await
        .expect_err("save failure must be observable to the admin caller");
        assert!(matches!(err, TierConfigUpdateError::Save(_)));
        assert!(manager.read().await.tiers.contains_key("COLD-A"));
        let restored = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
            .await
            .expect("the old generation must be restored after save failure");
        assert!(restored.is_current(&manager).await);
        drop(restored);
        let guard = manager.read().await;
        let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
        assert!(!lock_unpoisoned(&runtime).draining.contains_key("COLD-A"));
        drop(guard);

        store.fail_put.store(false, Ordering::SeqCst);
        let mut retry = empty_mgr();
        retry.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        retry
            .driver_cache
            .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("retry")));
        let update = TierConfigMgr::admin_update_lock(&manager).await;
        TierConfigMgr::update_candidate_owned(
            &manager,
            store,
            retry,
            version,
            TierCandidateMutation::Remove("COLD-A".to_string(), true),
            update,
            None,
        )
        .await
        .expect("a later update must succeed after save failure recovery");
        assert!(!manager.read().await.tiers.contains_key("COLD-A"));
    }

    #[tokio::test]
    async fn cancelled_owned_update_finishes_after_lease_drain() {
        let manager = TierConfigMgr::new();
        {
            let mut guard = manager.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
        }
        let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
            .await
            .expect("old generation lease should be available");
        let store = Arc::new(CasConfigStore::default());
        let mut candidate = empty_mgr();
        candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        candidate
            .save_tiering_config_if_current(store.clone(), None)
            .await
            .expect("cancelled update fixture should persist");
        let (_, version) = load_tier_config_for_update(store.clone())
            .await
            .expect("cancelled update fixture should reload with an ETag");
        candidate
            .driver_cache
            .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("candidate")));
        let update = TierConfigMgr::admin_update_lock(&manager).await;
        let update_manager = manager.clone();
        let update_store = store.clone();
        let caller = tokio::spawn(async move {
            TierConfigMgr::update_candidate_owned(
                &update_manager,
                update_store,
                candidate,
                version,
                TierCandidateMutation::Remove("COLD-A".to_string(), true),
                update,
                None,
            )
            .await
        });
        tokio::time::timeout(Duration::from_secs(1), async {
            while old.is_current(&manager).await {
                tokio::task::yield_now().await;
            }
        })
        .await
        .expect("owned update should revoke before caller cancellation");
        caller.abort();
        drop(old);

        tokio::time::timeout(Duration::from_secs(1), async {
            while manager.read().await.tiers.contains_key("COLD-A") {
                tokio::task::yield_now().await;
            }
        })
        .await
        .expect("detached update must finish after its operation lease drains");
        let guard = manager.read().await;
        let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
        assert!(!lock_unpoisoned(&runtime).draining.contains_key("COLD-A"));
        assert!(!store.objects.lock().await.is_empty(), "detached update must persist its result");
    }

    #[tokio::test]
    async fn config_write_lock_survives_cancelled_wrapped_update_until_detached_publish_finishes() {
        let manager = TierConfigMgr::new();
        {
            let mut guard = manager.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
        }
        let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
            .await
            .expect("old generation lease should be available");
        let store = Arc::new(CasConfigStore::default());
        let mut candidate = empty_mgr();
        candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        candidate
            .save_tiering_config_if_current(store.clone(), None)
            .await
            .expect("tier config fixture should persist");
        let update_manager = manager.clone();
        let update_store = store.clone();
        let caller = tokio::spawn(async move {
            TierConfigMgr::update_candidate_with_config_lock(
                &update_manager,
                update_store,
                TierCandidateMutation::Remove("COLD-A".to_string(), true),
            )
            .await
        });
        tokio::time::timeout(Duration::from_secs(1), async {
            while old.is_current(&manager).await {
                tokio::task::yield_now().await;
            }
        })
        .await
        .expect("owned update should revoke before caller cancellation");
        caller.abort();

        let config_file = tier_config_lock_path();
        let competing_lock = store
            .new_ns_lock(RUSTFS_META_BUCKET, &config_file)
            .await
            .expect("competing tier config lock should be created");
        let err = competing_lock
            .get_write_lock(Duration::from_millis(20))
            .await
            .expect_err("detached update must keep the config write lock until it finishes");
        assert!(matches!(err, rustfs_lock::LockError::Timeout { .. }));

        drop(old);
        tokio::time::timeout(Duration::from_secs(1), async {
            while manager.read().await.tiers.contains_key("COLD-A") {
                tokio::task::yield_now().await;
            }
        })
        .await
        .expect("detached update must finish after its operation lease drains");
        competing_lock
            .get_write_lock(Duration::from_secs(1))
            .await
            .expect("tier config lock should release after detached update finishes");
    }

    #[tokio::test]
    #[serial_test::serial]
    async fn tier_config_update_waits_for_config_namespace_lock_before_loading() {
        let manager = TierConfigMgr::new();
        let store = Arc::new(CasConfigStore::default());
        let config_file = tier_config_lock_path();
        let outer_lock = store
            .new_ns_lock(RUSTFS_META_BUCKET, &config_file)
            .await
            .expect("outer tier config lock should be created");
        let _outer_guard = outer_lock
            .get_write_lock(Duration::from_secs(1))
            .await
            .expect("outer tier config lock should be acquired");

        let result = temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
            TierConfigMgr::update_candidate_with_config_lock(&manager, store.clone(), TierCandidateMutation::Clear(true)).await
        })
        .await;
        let TierConfigUpdateError::Load(err) = result.expect_err("config mutation must wait behind the config namespace lock")
        else {
            panic!("config lock contention should fail before candidate load or mutation");
        };
        let rendered = err.to_string();
        assert!(rendered.to_ascii_lowercase().contains("timeout"), "{rendered}");
        assert!(rendered.contains(&config_file), "{rendered}");
        assert_eq!(
            store
                .lock_requests
                .lock()
                .expect("tier config lock request log should not poison")
                .as_slice(),
            &[
                (RUSTFS_META_BUCKET.to_string(), config_file.clone()),
                (RUSTFS_META_BUCKET.to_string(), config_file),
            ]
        );
    }

    #[tokio::test]
    async fn panicked_owned_update_restores_generation_and_reports_error() {
        let manager = TierConfigMgr::new();
        {
            let mut guard = manager.write().await;
            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
        }
        let store = Arc::new(CasConfigStore::default());
        let mut candidate = empty_mgr();
        candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        candidate
            .save_tiering_config_if_current(store.clone(), None)
            .await
            .expect("panic fixture should persist");
        let (_, version) = load_tier_config_for_update(store.clone())
            .await
            .expect("panic fixture should reload with an ETag");
        candidate
            .driver_cache
            .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::panicking_in_use("panic")));
        let update = TierConfigMgr::admin_update_lock(&manager).await;

        let err = TierConfigMgr::update_candidate_owned(
            &manager,
            store,
            candidate,
            version,
            TierCandidateMutation::Remove("COLD-A".to_string(), false),
            update,
            None,
        )
        .await
        .expect_err("mutation panic must be observable to the admin caller");
        let TierConfigUpdateError::Publish(err) = err else {
            panic!("mutation panic should be reported as a publish failure");
        };
        assert!(err.message.contains("panicked"));
        assert!(manager.read().await.tiers.contains_key("COLD-A"));
        let restored = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
            .await
            .expect("the old generation must be restored after panic");
        assert!(restored.is_current(&manager).await);
        drop(restored);
        let guard = manager.read().await;
        let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
        assert!(!lock_unpoisoned(&runtime).draining.contains_key("COLD-A"));
    }

    #[tokio::test]
    async fn stale_tier_config_etag_allows_only_one_candidate_to_publish() {
        let store = Arc::new(CasConfigStore::default());
        empty_mgr()
            .save_tiering_config_if_current(store.clone(), None)
            .await
            .expect("initial conditional create should succeed");

        let (mut candidate_a, version_a) = load_tier_config_for_update(store.clone())
            .await
            .expect("first node should load config revision");
        let (mut candidate_b, version_b) = load_tier_config_for_update(store.clone())
            .await
            .expect("second node should load the same config revision");
        assert_eq!(version_a, version_b);
        candidate_a.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
        candidate_b.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B"));

        let save_a = candidate_a.save_tiering_config_if_current(store.clone(), version_a.as_deref());
        let save_b = candidate_b.save_tiering_config_if_current(store, version_b.as_deref());
        let (result_a, result_b) = tokio::join!(save_a, save_b);
        assert_ne!(result_a.is_ok(), result_b.is_ok(), "exactly one stale-ETag writer must win");

        let manager_a = TierConfigMgr::new();
        let manager_b = TierConfigMgr::new();
        if result_a.is_ok() {
            TierConfigMgr::publish_candidate(&manager_a, candidate_a, None)
                .await
                .expect("winning node should publish");
        }
        if result_b.is_ok() {
            TierConfigMgr::publish_candidate(&manager_b, candidate_b, None)
                .await
                .expect("winning node should publish");
        }
        assert_ne!(manager_a.read().await.empty(), manager_b.read().await.empty());
    }

    #[test]
    fn legacy_refresh_method_signature_remains_callable() {
        async fn call_legacy_refresh(manager: &mut TierConfigMgr, api: Arc<ECStore>) {
            manager.refresh_tier_config(api).await;
        }

        let _ = call_legacy_refresh;
    }

    #[tokio::test(start_paused = true)]
    async fn periodic_refresh_timer_does_not_tick_on_startup() {
        let period = Duration::from_secs(60);
        let mut timer = delayed_tier_refresh_interval(period);
        assert!(
            tokio::time::timeout(Duration::ZERO, timer.tick()).await.is_err(),
            "periodic reload must not duplicate the startup reload"
        );
        tokio::time::advance(period).await;
        tokio::time::timeout(Duration::ZERO, timer.tick())
            .await
            .expect("the first periodic reload should become ready after one interval");
    }
}