gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-20 19:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
a8e7cce5e13ef5f5a937c8d9fe19212be4bea1a3
rustfs/crates/ecstore/src/services/tier/tier.rs
T
houseme d5f8c6c044 test(tier): cover zero reference proof matrix (#5100) 
Co-authored-by: heihutu <heihutu@gmail.com>
2026-07-21 17:17:38 +00:00

6994 lines
271 KiB
Rust
Raw Blame History

// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use byteorder::{ByteOrder, LittleEndian};
use bytes::Bytes;
use futures::FutureExt;
use http::HeaderMap;
use http::status::StatusCode;
use lazy_static::lazy_static;
use rand::{Rng, RngExt};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::{
collections::{HashMap, HashSet, hash_map::Entry},
io::{self, Cursor},
ops::Deref,
panic::AssertUnwindSafe,
sync::{
Arc, LazyLock, Mutex, MutexGuard, RwLock as StdRwLock, Weak,
atomic::{AtomicBool, AtomicUsize, Ordering},
},
time::Duration,
};
use time::OffsetDateTime;
use tokio::io::BufReader;
use tokio::{
select,
sync::RwLock,
time::{Instant, interval, interval_at, timeout_at},
};
use tracing::{debug, error, info, warn};
use crate::client::admin_handler_utils::AdminError;
use crate::error::{Error, Result, StorageError};
use crate::services::tier::{
tier_admin::TierCreds,
tier_config::{TierConfig, TierType, TierWasabi},
tier_handlers::{ERR_TIER_ALREADY_EXISTS, ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_NOT_FOUND, ERR_TIER_RESERVED_NAME},
warm_backend::{WarmBackend, check_warm_backend, new_warm_backend},
};
use crate::storage_api_contracts::{
bucket::BucketOperations,
list::{ListOperations, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
namespace::NamespaceLocking,
object::{
DeletedObject, EcstoreObjectIO, EcstoreObjectOperations, HTTPPreconditions, ObjectIO, ObjectOperations, ObjectToDelete,
},
range::HTTPRangeSpec,
};
use crate::{
config::com::{CONFIG_PREFIX, read_config, read_config_with_metadata},
disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET},
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
runtime::sources as runtime_sources,
set_disk::get_lock_acquire_timeout,
store::ECStore,
};
use rustfs_filemeta::FileInfo;
use rustfs_rio::HashReader;
use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
use s3s::S3ErrorCode;
use super::{
tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_PERM_ERR},
tier_mutation_intent::{
TierMutationIntent, TierMutationIntentKind, TierMutationIntentState, TierMutationIntentTarget,
list_tier_mutation_intent_records,
},
warm_backend::WarmBackendImpl,
};
const TIER_CFG_REFRESH: Duration = Duration::from_secs(15 * 60);
const TIER_OPERATION_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
const TIER_REMOTE_VALIDATION_TIMEOUT: Duration = Duration::from_secs(30);
const TIER_REFERENCE_PROOF_LIST_LIMIT: i32 = 1000;
const TIER_MUTATION_INTENT_RECOVERY_SCAN_LIMIT: usize = 1000;
const TIER_MUTATION_BLOCK_MESSAGE: &str = "Remote tier configuration is being replaced";
type TierReferenceProofWalkOptions = StorageWalkOptions<fn(&rustfs_filemeta::FileInfo) -> bool>;
fn delayed_tier_refresh_interval(period: Duration) -> tokio::time::Interval {
interval_at(Instant::now() + period, period)
}
#[cfg(test)]
struct TierDriverBuildBarrier {
tier_name: String,
arrived: tokio::sync::Notify,
release: tokio::sync::Semaphore,
}
#[cfg(test)]
static TIER_DRIVER_BUILD_BARRIER: LazyLock<Mutex<Option<Arc<TierDriverBuildBarrier>>>> = LazyLock::new(|| Mutex::new(None));
#[cfg(test)]
struct TierDriverBuildBarrierGuard;
#[cfg(test)]
impl Drop for TierDriverBuildBarrierGuard {
fn drop(&mut self) {
*lock_unpoisoned(&TIER_DRIVER_BUILD_BARRIER) = None;
}
}
#[cfg(test)]
fn install_tier_driver_build_barrier(tier_name: &str) -> (Arc<TierDriverBuildBarrier>, TierDriverBuildBarrierGuard) {
let barrier = Arc::new(TierDriverBuildBarrier {
tier_name: tier_name.to_string(),
arrived: tokio::sync::Notify::new(),
release: tokio::sync::Semaphore::new(0),
});
*lock_unpoisoned(&TIER_DRIVER_BUILD_BARRIER) = Some(barrier.clone());
(barrier, TierDriverBuildBarrierGuard)
}
async fn build_warm_backend(tier: &TierConfig, probe: bool) -> std::result::Result<WarmBackendImpl, AdminError> {
#[cfg(test)]
let test_barrier = { lock_unpoisoned(&TIER_DRIVER_BUILD_BARRIER).clone() };
#[cfg(test)]
if let Some(barrier) = test_barrier
&& barrier.tier_name == tier.name
{
barrier.arrived.notify_one();
barrier
.release
.acquire()
.await
.expect("tier driver build test barrier should stay open")
.forget();
}
new_warm_backend(tier, probe).await
}
const TIER_CONFIG_LEGACY_FILE: &str = "tier-config.json";
pub const TIER_CONFIG_FILE: &str = "tier-config.bin";
pub const TIER_CONFIG_FORMAT: u16 = 1;
pub const TIER_CONFIG_V1: u16 = 1;
pub const TIER_CONFIG_VERSION: u16 = 2;
const EXTERNAL_TIER_TYPE_UNSUPPORTED: i32 = 0;
const EXTERNAL_TIER_TYPE_S3: i32 = 1;
const EXTERNAL_TIER_TYPE_AZURE: i32 = 2;
const EXTERNAL_TIER_TYPE_GCS: i32 = 3;
const EXTERNAL_TIER_TYPE_MINIO: i32 = 4;
const EXTERNAL_TIER_VERSION_WASABI_V1: &str = "rustfs-wasabi-v1";
const EXTERNAL_TIER_WASABI_ENDPOINT_SENTINEL: &str = "rustfs-wasabi-v1:";
const TIER_BACKEND_IDENTITY_VERSION: u8 = 2;
const _TIER_CFG_REFRESH_AT_HDR: &str = "X-RustFS-TierCfg-RefreshedAt";
lazy_static! {
pub static ref ERR_TIER_MISSING_CREDENTIALS: AdminError = AdminError {
code: "XRustFSAdminTierMissingCredentials".to_string(),
message: "Specified remote credentials are empty".to_string(),
status_code: StatusCode::FORBIDDEN,
};
pub static ref ERR_TIER_BACKEND_IN_USE: AdminError = AdminError {
code: "XRustFSAdminTierBackendInUse".to_string(),
message: "Specified remote tier is already in use".to_string(),
status_code: StatusCode::CONFLICT,
};
pub static ref ERR_TIER_TYPE_UNSUPPORTED: AdminError = AdminError {
code: "XRustFSAdminTierTypeUnsupported".to_string(),
message: "Specified tier type is unsupported".to_string(),
status_code: StatusCode::BAD_REQUEST,
};
pub static ref ERR_TIER_BACKEND_NOT_EMPTY: AdminError = AdminError {
code: "XRustFSAdminTierBackendNotEmpty".to_string(),
message: "Specified remote backend is not empty".to_string(),
status_code: StatusCode::BAD_REQUEST,
};
pub static ref ERR_TIER_INVALID_CONFIG: AdminError = AdminError {
code: "XRustFSAdminTierInvalidConfig".to_string(),
message: "Unable to setup remote tier, check tier configuration".to_string(),
status_code: StatusCode::BAD_REQUEST,
};
}
#[derive(Serialize, Deserialize)]
pub struct TierConfigMgr {
#[serde(skip)]
pub driver_cache: HashMap<String, WarmBackendImpl>,
pub tiers: HashMap<String, TierConfig>,
pub last_refreshed_at: OffsetDateTime,
}
type SharedWarmBackend = Arc<dyn WarmBackend + Send + Sync + 'static>;
#[derive(Default)]
struct TierDriverRuntime {
generations: HashMap<String, Arc<TierDriverGeneration>>,
draining: HashMap<String, u64>,
prepared_mutation_blocks: HashMap<String, uuid::Uuid>,
next_generation: u64,
next_drain_epoch: u64,
admin_updates: Arc<tokio::sync::Mutex<()>>,
}
struct TierDriverRegistryEntry {
manager: Weak<RwLock<TierConfigMgr>>,
runtime: Arc<Mutex<TierDriverRuntime>>,
}
struct TierPublishTransition {
runtime: Arc<Mutex<TierDriverRuntime>>,
tokens: Vec<(String, u64)>,
revoked: HashMap<String, Arc<TierDriverGeneration>>,
replaced_destinations: HashMap<String, TierConfig>,
published: bool,
}
struct PreparedTierDriver {
tier_name: String,
config_fingerprint: TierDriverFingerprint,
backend_identity: TierDestinationId,
driver: SharedWarmBackend,
}
impl TierPublishTransition {
async fn wait_for_active_leases(&self) -> std::result::Result<(), AdminError> {
let deadline = Instant::now() + TIER_OPERATION_DRAIN_TIMEOUT;
for generation in self.revoked.values() {
if timeout_at(deadline, generation.wait_for_no_active_leases()).await.is_err() {
let mut err = ERR_TIER_BACKEND_IN_USE.clone();
err.message = "Timed out waiting for active remote tier operations to finish".to_string();
return Err(err);
}
}
Ok(())
}
async fn ensure_replaced_destinations_are_empty(&self) -> std::result::Result<(), AdminError> {
ensure_replaced_destinations_are_empty(&self.replaced_destinations, &self.revoked).await
}
}
async fn ensure_replaced_destinations_are_empty(
replaced_destinations: &HashMap<String, TierConfig>,
revoked: &HashMap<String, Arc<TierDriverGeneration>>,
) -> std::result::Result<(), AdminError> {
let deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT;
for (tier_name, old_config) in replaced_destinations {
let prepared;
let driver = match revoked.get(tier_name) {
Some(generation) => generation.driver.as_ref(),
None => {
prepared = match timeout_at(deadline, build_warm_backend(old_config, false)).await {
Ok(result) => result?,
Err(_) => {
let mut err = ERR_TIER_BACKEND_IN_USE.clone();
err.message = "Timed out preparing the replaced remote tier backend".to_string();
return Err(err);
}
};
prepared.as_ref()
}
};
match timeout_at(deadline, driver.in_use()).await {
Ok(Ok(false)) => {}
Ok(Ok(true)) => return Err(ERR_TIER_BACKEND_NOT_EMPTY.clone()),
Ok(Err(err)) => {
let mut admin_err = ERR_TIER_PERM_ERR.clone();
admin_err.message.push('.');
admin_err.message.push_str(&err.to_string());
return Err(admin_err);
}
Err(_) => {
let mut err = ERR_TIER_BACKEND_IN_USE.clone();
err.message = "Timed out checking whether the replaced remote tier backend is empty".to_string();
return Err(err);
}
}
}
Ok(())
}
impl Drop for TierPublishTransition {
fn drop(&mut self) {
let mut runtime = lock_unpoisoned(&self.runtime);
for (tier_name, epoch) in &self.tokens {
if runtime.draining.get(tier_name) == Some(epoch) {
if !self.published
&& !runtime.generations.contains_key(tier_name)
&& let Some(generation) = self.revoked.get(tier_name)
{
generation.accepting.store(true, Ordering::Release);
runtime.generations.insert(tier_name.clone(), generation.clone());
}
runtime.draining.remove(tier_name);
}
}
}
}
static TIER_DRIVER_REGISTRY: LazyLock<StdRwLock<HashMap<usize, TierDriverRegistryEntry>>> =
LazyLock::new(|| StdRwLock::new(HashMap::new()));
fn lock_unpoisoned<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
mutex.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}
fn tier_manager_key(manager: &TierConfigMgr) -> usize {
std::ptr::from_ref(manager).addr()
}
fn tier_driver_runtime(handle: &Arc<RwLock<TierConfigMgr>>, manager: &TierConfigMgr) -> Arc<Mutex<TierDriverRuntime>> {
let key = tier_manager_key(manager);
{
let registry = TIER_DRIVER_REGISTRY.read().unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(entry) = registry.get(&key)
&& entry
.manager
.upgrade()
.is_some_and(|registered| Arc::ptr_eq(&registered, handle))
{
return entry.runtime.clone();
}
}
#[cfg(test)]
let _ = TIER_REGISTRY_MISS_SCAN_COUNT.try_with(|count| count.set(count.get() + 1));
let mut registry = TIER_DRIVER_REGISTRY
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(entry) = registry.get(&key)
&& entry
.manager
.upgrade()
.is_some_and(|registered| Arc::ptr_eq(&registered, handle))
{
return entry.runtime.clone();
}
registry.retain(|_, entry| entry.manager.strong_count() > 0);
let runtime = Arc::new(Mutex::new(TierDriverRuntime::default()));
registry.insert(
key,
TierDriverRegistryEntry {
manager: Arc::downgrade(handle),
runtime: runtime.clone(),
},
);
runtime
}
#[cfg(test)]
tokio::task_local! {
static TIER_REGISTRY_MISS_SCAN_COUNT: std::cell::Cell<usize>;
}
fn registered_tier_driver_runtime(manager: &TierConfigMgr) -> Option<Arc<Mutex<TierDriverRuntime>>> {
let key = tier_manager_key(manager);
{
let registry = TIER_DRIVER_REGISTRY.read().unwrap_or_else(std::sync::PoisonError::into_inner);
let entry = registry.get(&key)?;
if entry.manager.strong_count() > 0 {
return Some(entry.runtime.clone());
}
}
let mut registry = TIER_DRIVER_REGISTRY
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if registry.get(&key).is_some_and(|entry| entry.manager.strong_count() == 0) {
registry.remove(&key);
}
None
}
fn runtime_blocks_tier(runtime: &TierDriverRuntime, tier_name: &str) -> bool {
runtime.draining.contains_key(tier_name) || runtime.prepared_mutation_blocks.contains_key(tier_name)
}
fn runtime_has_mutation_block(runtime: &TierDriverRuntime) -> bool {
!runtime.draining.is_empty() || !runtime.prepared_mutation_blocks.is_empty()
}
type TierDriverFingerprint = [u8; 32];
pub(crate) type TierDestinationId = [u8; 32];
pub(crate) type DriverRevision = u64;
fn tier_config_fingerprint(tier_name: &str, config: &TierConfig) -> io::Result<TierDriverFingerprint> {
let external = to_external_tier_config(tier_name, config)?;
let encoded = rmp_serde::to_vec_named(&external)
.map_err(|err| io::Error::other(format!("serialize tier driver identity failed: {err}")))?;
Ok(Sha256::digest(encoded).into())
}
fn tier_configs_match(tier_name: &str, current: &TierConfig, replacement: &TierConfig) -> bool {
match (
tier_config_fingerprint(tier_name, current),
tier_config_fingerprint(tier_name, replacement),
) {
(Ok(current), Ok(replacement)) => current == replacement,
_ => false,
}
}
#[derive(Debug)]
pub enum TierConfigUpdateError {
Load(io::Error),
Mutation(AdminError),
Save(io::Error),
Publish(AdminError),
}
enum TierCandidateMutation {
Add(TierConfig, bool),
Edit(String, TierCreds),
Remove(String, bool),
Clear(bool),
}
impl TierCandidateMutation {
fn intent_kind(&self) -> TierMutationIntentKind {
match self {
Self::Add(_, _) => TierMutationIntentKind::Add,
Self::Edit(_, _) => TierMutationIntentKind::Edit,
Self::Remove(_, _) => TierMutationIntentKind::Remove,
Self::Clear(_) => TierMutationIntentKind::Clear,
}
}
fn explicit_tier_name(&self) -> Option<&str> {
match self {
Self::Add(config, _) => Some(&config.name),
Self::Edit(tier_name, _) | Self::Remove(tier_name, _) => Some(tier_name),
Self::Clear(_) => None,
}
}
fn target_tiers(&self, manager: &TierConfigMgr, candidate: &TierConfigMgr) -> HashSet<String> {
let mut targets = changed_tier_names(manager, candidate);
match self {
Self::Add(config, _) => {
targets.insert(config.name.clone());
}
Self::Edit(tier_name, _) => {
targets.insert(tier_name.clone());
}
Self::Remove(tier_name, _) => {
if manager.tiers.contains_key(tier_name) || candidate.tiers.contains_key(tier_name) {
targets.insert(tier_name.clone());
}
}
Self::Clear(_) => {
targets.extend(manager.tiers.keys().chain(candidate.tiers.keys()).cloned());
}
}
targets
}
fn affected_targets(
&self,
manager: &TierConfigMgr,
candidate: &TierConfigMgr,
) -> std::result::Result<Vec<TierMutationIntentTarget>, AdminError> {
let explicit_tier_name = self.explicit_tier_name();
build_tier_mutation_affected_targets(
self.intent_kind(),
tier_mutation_proof_targets(self.intent_kind(), explicit_tier_name, manager, candidate),
manager,
candidate,
)
}
async fn apply(self, candidate: &mut TierConfigMgr) -> std::result::Result<Option<String>, AdminError> {
match self {
Self::Add(config, force) => {
let tier_name = config.name.clone();
candidate.add(config, force).await?;
Ok(Some(tier_name))
}
Self::Edit(tier_name, credentials) => {
candidate.edit(&tier_name, credentials).await?;
Ok(Some(tier_name))
}
Self::Remove(tier_name, force) => {
candidate.remove(&tier_name, force).await?;
Ok(None)
}
Self::Clear(force) => {
candidate.clear_tier(force).await?;
Ok(None)
}
}
}
}
fn tier_mutation_proof_targets(
kind: TierMutationIntentKind,
explicit_tier_name: Option<&str>,
current: &TierConfigMgr,
candidate: &TierConfigMgr,
) -> HashSet<String> {
let mut targets = HashSet::new();
match kind {
TierMutationIntentKind::Add | TierMutationIntentKind::Edit | TierMutationIntentKind::Remove => {
if let Some(tier_name) = explicit_tier_name
&& (current.tiers.contains_key(tier_name) || candidate.tiers.contains_key(tier_name))
{
targets.insert(tier_name.to_string());
}
}
TierMutationIntentKind::Clear => {
targets.extend(current.tiers.keys().chain(candidate.tiers.keys()).cloned());
}
}
targets
}
fn build_tier_mutation_affected_targets(
kind: TierMutationIntentKind,
target_tiers: HashSet<String>,
current: &TierConfigMgr,
candidate: &TierConfigMgr,
) -> std::result::Result<Vec<TierMutationIntentTarget>, AdminError> {
let mut targets = target_tiers.into_iter().collect::<Vec<_>>();
targets.sort();
let mut out = Vec::with_capacity(targets.len());
for tier_name in targets {
let old_backend_identity = current
.tiers
.get(&tier_name)
.map(tier_backend_identity)
.transpose()
.map_err(tier_backend_identity_admin_error)?;
let new_backend_identity = candidate
.tiers
.get(&tier_name)
.map(tier_backend_identity)
.transpose()
.map_err(tier_backend_identity_admin_error)?;
if old_backend_identity.is_none() && new_backend_identity.is_none() {
continue;
}
let target = TierMutationIntentTarget {
tier_name,
old_backend_identity,
new_backend_identity,
};
validate_tier_mutation_target_shape(kind, &target)?;
out.push(target);
}
Ok(out)
}
fn validate_tier_mutation_target_shape(
kind: TierMutationIntentKind,
target: &TierMutationIntentTarget,
) -> std::result::Result<(), AdminError> {
let valid = match kind {
TierMutationIntentKind::Add => target.old_backend_identity.is_none() && target.new_backend_identity.is_some(),
TierMutationIntentKind::Edit => target.old_backend_identity.is_some() && target.new_backend_identity.is_some(),
TierMutationIntentKind::Remove | TierMutationIntentKind::Clear => {
target.old_backend_identity.is_some() && target.new_backend_identity.is_none()
}
};
if valid {
return Ok(());
}
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier mutation target identity proof does not match mutation kind".to_string();
Err(err)
}
fn tier_backend_identity_admin_error(err: io::Error) -> AdminError {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
}
trait TierReferenceProofStore:
EcstoreObjectIO
+ BucketOperations<Error = Error>
+ ListOperations<
Error = Error,
ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>,
ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>,
ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>,
WalkOptions = TierReferenceProofWalkOptions,
WalkCancellation = tokio_util::sync::CancellationToken,
WalkResultSender = tokio::sync::mpsc::Sender<StorageObjectInfoOrErr<ObjectInfo, Error>>,
>
{
}
impl<T> TierReferenceProofStore for T where
T: EcstoreObjectIO
+ BucketOperations<Error = Error>
+ ListOperations<
Error = Error,
ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>,
ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>,
ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>,
WalkOptions = TierReferenceProofWalkOptions,
WalkCancellation = tokio_util::sync::CancellationToken,
WalkResultSender = tokio::sync::mpsc::Sender<StorageObjectInfoOrErr<ObjectInfo, Error>>,
>
{
}
async fn ensure_no_authoritative_tier_object_references<S>(
api: Arc<S>,
affected_targets: &[TierMutationIntentTarget],
) -> std::result::Result<(), AdminError>
where
S: TierReferenceProofStore,
{
for target in affected_targets {
if target.old_backend_identity.is_some() {
ensure_no_authoritative_target_references(api.clone(), target).await?;
}
}
Ok(())
}
async fn ensure_no_authoritative_target_references<S>(
api: Arc<S>,
target: &TierMutationIntentTarget,
) -> std::result::Result<(), AdminError>
where
S: TierReferenceProofStore,
{
let buckets = api
.list_bucket(&Default::default())
.await
.map_err(tier_reference_proof_admin_error)?;
for bucket in buckets {
let mut marker = None;
let mut version_marker = None;
loop {
let page = api
.clone()
.list_object_versions(
&bucket.name,
"",
marker.take(),
version_marker.take(),
None,
TIER_REFERENCE_PROOF_LIST_LIMIT,
)
.await
.map_err(tier_reference_proof_admin_error)?;
for object in &page.objects {
if tier_object_blocks_target_rebind(object, target).map_err(tier_reference_proof_admin_error)? {
return Err(tier_reference_proof_in_use_error(&target.tier_name, object));
}
}
if !page.is_truncated {
break;
}
marker = page.next_marker;
version_marker = page.next_version_idmarker;
if marker.is_none() {
return Err(tier_reference_proof_admin_error(io::Error::other(
"tier reference proof listing is truncated without a next marker",
)));
}
}
}
Ok(())
}
fn tier_object_blocks_target_rebind(object: &ObjectInfo, target: &TierMutationIntentTarget) -> io::Result<bool> {
if object.transitioned_object.status != rustfs_filemeta::TRANSITION_COMPLETE
|| object.transitioned_object.tier != target.tier_name
{
return Ok(false);
}
let current_identity = tier_destination_id_from_metadata(&object.user_defined)?;
Ok(match (&current_identity, &target.new_backend_identity) {
(_, None) => true,
(Some(current), Some(new)) => current != new,
(None, Some(_)) => true,
})
}
fn tier_reference_proof_in_use_error(tier_name: &str, object: &ObjectInfo) -> AdminError {
let mut err = ERR_TIER_BACKEND_IN_USE.clone();
err.message = format!(
"Remote tier {tier_name} still has object references, for example {}/{}",
object.bucket, object.name
);
err
}
fn tier_reference_proof_admin_error(err: impl std::fmt::Display) -> AdminError {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = format!("Remote tier reference proof failed: {err}");
admin_err
}
async fn apply_tier_candidate_mutation(
mutation: TierCandidateMutation,
candidate: &mut TierConfigMgr,
deadline: Instant,
) -> std::result::Result<Option<String>, AdminError> {
match timeout_at(deadline, mutation.apply(candidate)).await {
Ok(result) => result,
Err(_) => {
let mut err = ERR_TIER_BACKEND_IN_USE.clone();
err.message = "Timed out validating the remote tier mutation".to_string();
Err(err)
}
}
}
fn changed_tier_names(current: &TierConfigMgr, replacement: &TierConfigMgr) -> HashSet<String> {
let mut changed = HashSet::new();
for (tier_name, current_config) in &current.tiers {
if replacement
.tiers
.get(tier_name)
.is_none_or(|replacement_config| !tier_configs_match(tier_name, current_config, replacement_config))
{
changed.insert(tier_name.clone());
}
}
changed.extend(
replacement
.tiers
.keys()
.filter(|tier_name| !current.tiers.contains_key(*tier_name))
.cloned(),
);
changed
}
fn replaced_tier_destinations(
current: &TierConfigMgr,
replacement: &TierConfigMgr,
) -> std::result::Result<HashMap<String, TierConfig>, AdminError> {
let mut replaced = HashMap::new();
for (tier_name, current_config) in &current.tiers {
let changed = match replacement.tiers.get(tier_name) {
Some(replacement_config) => {
if matches!(current_config.tier_type, TierType::Wasabi)
!= matches!(replacement_config.tier_type, TierType::Wasabi)
{
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message =
"Changing an existing remote tier to or from Wasabi is not supported; add a new tier instead".to_string();
return Err(admin_err);
}
let current_identity = tier_backend_identity(current_config).map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})?;
let replacement_identity = tier_backend_identity(replacement_config).map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})?;
current_identity != replacement_identity
}
None => true,
};
if changed {
replaced.insert(tier_name.clone(), current_config.clone_with_credentials());
}
}
Ok(replaced)
}
fn encode_tier_backend_identity(
tier_type: &str,
endpoint: &str,
bucket: &str,
prefix: &str,
region: &str,
routing_account: &str,
) -> io::Result<TierDestinationId> {
let encoded = rmp_serde::to_vec(&(
TIER_BACKEND_IDENTITY_VERSION,
tier_type,
endpoint,
bucket,
prefix,
region,
routing_account,
))
.map_err(|err| io::Error::other(format!("serialize tier backend identity failed: {err}")))?;
Ok(Sha256::digest(encoded).into())
}
fn tier_backend_identity(config: &TierConfig) -> io::Result<TierDestinationId> {
if matches!(config.tier_type, TierType::Wasabi) {
let wasabi = config
.wasabi
.as_ref()
.ok_or_else(|| io::Error::other("tier backend identity payload is missing"))?;
return encode_tier_backend_identity(
"s3",
&wasabi.canonical_endpoint()?,
&wasabi.bucket,
normalized_tier_prefix(&config.tier_type, &wasabi.prefix),
&wasabi.region,
"",
);
}
let (tier_type, endpoint, bucket, prefix, region, routing_account) = match config.tier_type {
TierType::S3 => config.s3.as_ref().map(|value| {
(
"s3",
value.endpoint.as_str(),
value.bucket.as_str(),
value.prefix.as_str(),
value.region.as_str(),
"",
)
}),
TierType::Wasabi => None,
TierType::RustFS => config.rustfs.as_ref().map(|value| {
(
"rustfs",
value.endpoint.as_str(),
value.bucket.as_str(),
value.prefix.as_str(),
value.region.as_str(),
"",
)
}),
TierType::MinIO => config.minio.as_ref().map(|value| {
(
"minio",
value.endpoint.as_str(),
value.bucket.as_str(),
value.prefix.as_str(),
value.region.as_str(),
"",
)
}),
TierType::Aliyun => config.aliyun.as_ref().map(|value| {
(
"aliyun",
value.endpoint.as_str(),
value.bucket.as_str(),
value.prefix.as_str(),
value.region.as_str(),
"",
)
}),
TierType::Tencent => config.tencent.as_ref().map(|value| {
(
"tencent",
value.endpoint.as_str(),
value.bucket.as_str(),
value.prefix.as_str(),
value.region.as_str(),
"",
)
}),
TierType::Huaweicloud => config.huaweicloud.as_ref().map(|value| {
(
"huaweicloud",
value.endpoint.as_str(),
value.bucket.as_str(),
value.prefix.as_str(),
value.region.as_str(),
"",
)
}),
TierType::Azure => config.azure.as_ref().map(|value| {
(
"azure",
value.endpoint.as_str(),
value.bucket.as_str(),
value.prefix.as_str(),
value.region.as_str(),
value.access_key.as_str(),
)
}),
TierType::GCS => config.gcs.as_ref().map(|value| {
(
"gcs",
value.endpoint.as_str(),
value.bucket.as_str(),
value.prefix.as_str(),
value.region.as_str(),
"",
)
}),
TierType::R2 => config.r2.as_ref().map(|value| {
(
"r2",
value.endpoint.as_str(),
value.bucket.as_str(),
value.prefix.as_str(),
value.region.as_str(),
"",
)
}),
TierType::Unsupported => None,
}
.ok_or_else(|| io::Error::other("tier backend identity payload is missing"))?;
let prefix = normalized_tier_prefix(&config.tier_type, prefix);
encode_tier_backend_identity(tier_type, endpoint, bucket, prefix, region, routing_account)
}
pub(crate) fn tier_destination_id_from_metadata(metadata: &HashMap<String, String>) -> io::Result<Option<TierDestinationId>> {
let Some(encoded) = rustfs_utils::http::metadata_compat::get_consistent_str(
metadata,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
) else {
if rustfs_utils::http::metadata_compat::contains_key_str(
metadata,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
) {
return Err(io::Error::other(
"transition tier backend identity compatibility keys conflict or are empty",
));
}
return Ok(None);
};
if encoded.len() != 64 {
return Err(io::Error::other("transition tier backend identity has an invalid length"));
}
let mut identity = [0_u8; 32];
for (index, byte) in identity.iter_mut().enumerate() {
*byte = u8::from_str_radix(&encoded[index * 2..index * 2 + 2], 16)
.map_err(|_| io::Error::other("transition tier backend identity is not valid hexadecimal"))?;
}
Ok(Some(identity))
}
fn normalized_tier_prefix<'a>(tier_type: &TierType, prefix: &'a str) -> &'a str {
match tier_type {
TierType::S3 | TierType::Wasabi => prefix.trim_matches('/'),
TierType::RustFS
| TierType::MinIO
| TierType::Aliyun
| TierType::Tencent
| TierType::Huaweicloud
| TierType::Azure
| TierType::GCS
| TierType::R2 => prefix.strip_suffix('/').unwrap_or(prefix),
TierType::Unsupported => prefix,
}
}
struct TierDriverGeneration {
tier_name: Arc<str>,
generation: DriverRevision,
// Process-local only: this may reflect credential changes and must never be persisted or logged.
config_fingerprint: TierDriverFingerprint,
// Durable cleanup identity: routing fields only, with all credentials excluded.
backend_identity: TierDestinationId,
driver: SharedWarmBackend,
accepting: AtomicBool,
active_leases: AtomicUsize,
drained: tokio::sync::Notify,
}
struct SharedWarmBackendProxy(SharedWarmBackend);
#[async_trait::async_trait]
impl WarmBackend for SharedWarmBackendProxy {
async fn validate(&self) -> io::Result<()> {
self.0.validate().await
}
fn validate_remote_version_id(&self, remote_version_id: &str) -> io::Result<()> {
self.0.validate_remote_version_id(remote_version_id)
}
async fn put(&self, object: &str, r: crate::client::transition_api::ReaderImpl, length: i64) -> io::Result<String> {
self.0.put(object, r, length).await
}
async fn put_with_meta(
&self,
object: &str,
r: crate::client::transition_api::ReaderImpl,
length: i64,
meta: HashMap<String, String>,
) -> io::Result<String> {
self.0.put_with_meta(object, r, length, meta).await
}
async fn get(
&self,
object: &str,
rv: &str,
opts: crate::services::tier::warm_backend::WarmBackendGetOpts,
) -> io::Result<crate::client::transition_api::ReadCloser> {
self.0.get(object, rv, opts).await
}
async fn remove(&self, object: &str, rv: &str) -> io::Result<()> {
self.0.remove(object, rv).await
}
async fn remove_exact(&self, object: &str, rv: &str) -> io::Result<()> {
self.0.remove_exact(object, rv).await
}
async fn in_use(&self) -> io::Result<bool> {
self.0.in_use().await
}
}
pub(crate) struct TierOperationLease {
inner: Arc<TierDriverGeneration>,
runtime: Arc<Mutex<TierDriverRuntime>>,
}
impl TierOperationLease {
fn try_new(
inner: Arc<TierDriverGeneration>,
runtime: Arc<Mutex<TierDriverRuntime>>,
) -> std::result::Result<Self, AdminError> {
inner
.active_leases
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |active| active.checked_add(1))
.map_err(|_| {
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier operation lease capacity exhausted".to_string();
err
})?;
Ok(Self { inner, runtime })
}
pub(crate) fn try_clone(&self) -> std::result::Result<Self, AdminError> {
Self::try_new(self.inner.clone(), self.runtime.clone())
}
}
impl Drop for TierOperationLease {
fn drop(&mut self) {
let result = self
.inner
.active_leases
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |active| active.checked_sub(1));
match result {
Ok(1) => self.inner.drained.notify_one(),
Ok(_) => {}
Err(_) => {
error!(
tier = self.tier_name(),
tier_generation = self.generation(),
"tier operation lease counter underflow"
);
}
}
}
}
impl Deref for TierOperationLease {
type Target = dyn WarmBackend + Send + Sync + 'static;
fn deref(&self) -> &Self::Target {
self.inner.driver.as_ref()
}
}
impl TierOperationLease {
pub(crate) fn tier_name(&self) -> &str {
&self.inner.tier_name
}
pub(crate) fn generation(&self) -> DriverRevision {
self.inner.generation
}
pub(crate) fn backend_identity(&self) -> TierDestinationId {
self.inner.backend_identity
}
pub(crate) fn is_current_generation(&self) -> bool {
lock_unpoisoned(&self.runtime)
.generations
.get(self.tier_name())
.is_some_and(|current| {
current.generation == self.generation()
&& current.accepting.load(Ordering::Acquire)
&& Arc::ptr_eq(current, &self.inner)
})
}
#[cfg(test)]
async fn is_current(&self, handle: &Arc<RwLock<TierConfigMgr>>) -> bool {
let manager = handle.read().await;
let Some(runtime) = registered_tier_driver_runtime(&manager) else {
return false;
};
if !Arc::ptr_eq(&runtime, &self.runtime) {
return false;
}
self.is_current_generation()
}
}
impl TierDriverGeneration {
async fn wait_for_no_active_leases(&self) {
loop {
let notified = self.drained.notified();
if self.active_leases.load(Ordering::Acquire) == 0 {
return;
}
notified.await;
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierConfigMgr {
#[serde(rename = "Tiers")]
tiers: HashMap<String, ExternalTierConfig>,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierConfig {
#[serde(rename = "Version")]
version: String,
#[serde(rename = "Type")]
tier_type: i32,
#[serde(rename = "Name")]
name: String,
#[serde(rename = "S3")]
s3: Option<ExternalTierS3>,
#[serde(rename = "Azure")]
azure: Option<ExternalTierAzure>,
#[serde(rename = "GCS")]
gcs: Option<ExternalTierGcs>,
#[serde(rename = "MinIO", alias = "Compatible")]
compatible_backend: Option<ExternalTierCompatible>,
#[serde(rename = "XTierType", skip_serializing_if = "Option::is_none")]
tier_type_hint: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierS3 {
#[serde(rename = "Endpoint")]
endpoint: String,
#[serde(rename = "AccessKey")]
access_key: String,
#[serde(rename = "SecretKey")]
secret_key: String,
#[serde(rename = "Bucket")]
bucket: String,
#[serde(rename = "Prefix")]
prefix: String,
#[serde(rename = "Region")]
region: String,
#[serde(rename = "StorageClass")]
storage_class: String,
#[serde(rename = "AWSRole")]
aws_role: bool,
#[serde(rename = "AWSRoleWebIdentityTokenFile")]
aws_role_web_identity_token_file: String,
#[serde(rename = "AWSRoleARN")]
aws_role_arn: String,
#[serde(rename = "AWSRoleSessionName")]
aws_role_session_name: String,
#[serde(rename = "AWSRoleDurationSeconds")]
aws_role_duration_seconds: i32,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalServicePrincipalAuth {
#[serde(rename = "TenantID")]
tenant_id: String,
#[serde(rename = "ClientID")]
client_id: String,
#[serde(rename = "ClientSecret")]
client_secret: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierAzure {
#[serde(rename = "Endpoint")]
endpoint: String,
#[serde(rename = "AccountName")]
account_name: String,
#[serde(rename = "AccountKey")]
account_key: String,
#[serde(rename = "Bucket")]
bucket: String,
#[serde(rename = "Prefix")]
prefix: String,
#[serde(rename = "Region")]
region: String,
#[serde(rename = "StorageClass")]
storage_class: String,
#[serde(rename = "SPAuth")]
sp_auth: ExternalServicePrincipalAuth,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierGcs {
#[serde(rename = "Endpoint")]
endpoint: String,
#[serde(rename = "Creds")]
creds: String,
#[serde(rename = "Bucket")]
bucket: String,
#[serde(rename = "Prefix")]
prefix: String,
#[serde(rename = "Region")]
region: String,
#[serde(rename = "StorageClass")]
storage_class: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierCompatible {
#[serde(rename = "Endpoint")]
endpoint: String,
#[serde(rename = "AccessKey")]
access_key: String,
#[serde(rename = "SecretKey")]
secret_key: String,
#[serde(rename = "Bucket")]
bucket: String,
#[serde(rename = "Prefix")]
prefix: String,
#[serde(rename = "Region")]
region: String,
}
fn tier_config_path(file: &str) -> String {
format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, file)
}
fn tier_config_lock_path() -> String {
format!("{}.lock", tier_config_path(TIER_CONFIG_FILE))
}
fn tier_hint_for_type(tier_type: TierType) -> Option<&'static str> {
match tier_type {
TierType::RustFS => Some("rustfs"),
TierType::Wasabi => Some("wasabi"),
TierType::Aliyun => Some("aliyun"),
TierType::Tencent => Some("tencent"),
TierType::Huaweicloud => Some("huaweicloud"),
TierType::R2 => Some("r2"),
_ => None,
}
}
fn tier_type_from_hint(hint: Option<&str>) -> Option<TierType> {
match hint {
Some("rustfs") => Some(TierType::RustFS),
Some("aliyun") => Some(TierType::Aliyun),
Some("tencent") => Some(TierType::Tencent),
Some("huaweicloud") => Some(TierType::Huaweicloud),
Some("r2") => Some(TierType::R2),
_ => None,
}
}
fn external_tier_s3_from_internal(s3: &crate::services::tier::tier_config::TierS3) -> ExternalTierS3 {
ExternalTierS3 {
endpoint: s3.endpoint.clone(),
access_key: s3.access_key.clone(),
secret_key: s3.secret_key.clone(),
bucket: s3.bucket.clone(),
prefix: s3.prefix.clone(),
region: s3.region.clone(),
storage_class: s3.storage_class.clone(),
aws_role: s3.aws_role,
aws_role_web_identity_token_file: s3.aws_role_web_identity_token_file.clone(),
aws_role_arn: s3.aws_role_arn.clone(),
aws_role_session_name: s3.aws_role_session_name.clone(),
aws_role_duration_seconds: s3.aws_role_duration_seconds,
}
}
fn external_tier_s3_from_compatible_payload(
endpoint: String,
access_key: String,
secret_key: String,
bucket: String,
prefix: String,
region: String,
) -> ExternalTierS3 {
ExternalTierS3 {
endpoint,
access_key,
secret_key,
bucket,
prefix,
region,
storage_class: String::new(),
aws_role: false,
aws_role_web_identity_token_file: String::new(),
aws_role_arn: String::new(),
aws_role_session_name: String::new(),
aws_role_duration_seconds: 0,
}
}
fn external_tier_alias_from_compatible_payload(
endpoint: String,
access_key: String,
secret_key: String,
bucket: String,
prefix: String,
region: String,
) -> ExternalTierCompatible {
ExternalTierCompatible {
endpoint,
access_key,
secret_key,
bucket,
prefix,
region,
}
}
fn to_external_tier_config(name: &str, tier: &TierConfig) -> io::Result<ExternalTierConfig> {
let mut out = ExternalTierConfig {
version: if tier.version.is_empty() {
"v1".to_string()
} else {
tier.version.clone()
},
name: if tier.name.is_empty() {
name.to_string()
} else {
tier.name.clone()
},
..Default::default()
};
match tier.tier_type {
TierType::S3 => {
let s3 = tier
.s3
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing s3 backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_S3;
out.s3 = Some(external_tier_s3_from_internal(s3));
}
TierType::Wasabi => {
let backend = tier
.wasabi
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing Wasabi backend payload"))?;
out.version = EXTERNAL_TIER_VERSION_WASABI_V1.to_string();
out.tier_type = EXTERNAL_TIER_TYPE_S3;
out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
out.s3 = Some(external_tier_s3_from_compatible_payload(
{
backend.canonical_endpoint()?;
EXTERNAL_TIER_WASABI_ENDPOINT_SENTINEL.to_string()
},
backend.access_key.clone(),
backend.secret_key.clone(),
backend.bucket.clone(),
backend.prefix.clone(),
backend.region.clone(),
));
}
TierType::Azure => {
let az = tier
.azure
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing azure backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_AZURE;
out.azure = Some(ExternalTierAzure {
endpoint: az.endpoint.clone(),
account_name: az.access_key.clone(),
account_key: az.secret_key.clone(),
bucket: az.bucket.clone(),
prefix: az.prefix.clone(),
region: az.region.clone(),
storage_class: az.storage_class.clone(),
sp_auth: ExternalServicePrincipalAuth {
tenant_id: az.sp_auth.tenant_id.clone(),
client_id: az.sp_auth.client_id.clone(),
client_secret: az.sp_auth.client_secret.clone(),
},
});
}
TierType::GCS => {
let gcs = tier
.gcs
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing gcs backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_GCS;
out.gcs = Some(ExternalTierGcs {
endpoint: gcs.endpoint.clone(),
creds: gcs.creds.clone(),
bucket: gcs.bucket.clone(),
prefix: gcs.prefix.clone(),
region: gcs.region.clone(),
storage_class: gcs.storage_class.clone(),
});
}
TierType::MinIO => {
let backend = tier
.minio
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_MINIO;
out.compatible_backend = Some(external_tier_alias_from_compatible_payload(
backend.endpoint.clone(),
backend.access_key.clone(),
backend.secret_key.clone(),
backend.bucket.clone(),
backend.prefix.clone(),
backend.region.clone(),
));
}
TierType::RustFS => {
let backend = tier
.rustfs
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_S3;
out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
out.s3 = Some(external_tier_s3_from_compatible_payload(
backend.endpoint.clone(),
backend.access_key.clone(),
backend.secret_key.clone(),
backend.bucket.clone(),
backend.prefix.clone(),
backend.region.clone(),
));
}
TierType::Aliyun => {
let backend = tier
.aliyun
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_S3;
out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
out.s3 = Some(external_tier_s3_from_compatible_payload(
backend.endpoint.clone(),
backend.access_key.clone(),
backend.secret_key.clone(),
backend.bucket.clone(),
backend.prefix.clone(),
backend.region.clone(),
));
}
TierType::Tencent => {
let backend = tier
.tencent
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_S3;
out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
out.s3 = Some(external_tier_s3_from_compatible_payload(
backend.endpoint.clone(),
backend.access_key.clone(),
backend.secret_key.clone(),
backend.bucket.clone(),
backend.prefix.clone(),
backend.region.clone(),
));
}
TierType::Huaweicloud => {
let backend = tier
.huaweicloud
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_S3;
out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
out.s3 = Some(external_tier_s3_from_compatible_payload(
backend.endpoint.clone(),
backend.access_key.clone(),
backend.secret_key.clone(),
backend.bucket.clone(),
backend.prefix.clone(),
backend.region.clone(),
));
}
TierType::R2 => {
let backend = tier
.r2
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_S3;
out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
out.s3 = Some(external_tier_s3_from_compatible_payload(
backend.endpoint.clone(),
backend.access_key.clone(),
backend.secret_key.clone(),
backend.bucket.clone(),
backend.prefix.clone(),
backend.region.clone(),
));
}
TierType::Unsupported => {
out.tier_type = EXTERNAL_TIER_TYPE_UNSUPPORTED;
}
}
Ok(out)
}
fn decode_legacy_s3_like(name: &str, ext: &ExternalTierConfig) -> io::Result<ExternalTierS3> {
if let Some(s3) = ext.s3.as_ref() {
return Ok(s3.clone());
}
if let Some(m) = ext.compatible_backend.as_ref() {
return Ok(external_tier_s3_from_compatible_payload(
m.endpoint.clone(),
m.access_key.clone(),
m.secret_key.clone(),
m.bucket.clone(),
m.prefix.clone(),
m.region.clone(),
));
}
Err(io::Error::other(format!("tier config '{name}' missing compatible backend payload")))
}
fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Result<TierConfig> {
let mut cfg = TierConfig {
version: if ext.version.is_empty() {
"v1".to_string()
} else {
ext.version.clone()
},
name: if ext.name.is_empty() { name } else { ext.name.clone() },
..Default::default()
};
let wasabi_hint = ext.tier_type_hint.as_deref() == Some("wasabi");
let wasabi_version = ext.version == EXTERNAL_TIER_VERSION_WASABI_V1;
if wasabi_hint && !wasabi_version {
return Err(io::Error::other(format!(
"tier config '{}' has inconsistent Wasabi type discriminators",
cfg.name
)));
}
if (wasabi_hint || wasabi_version) && ext.tier_type != EXTERNAL_TIER_TYPE_S3 {
return Err(io::Error::other(format!(
"tier config '{}' has inconsistent Wasabi type discriminators",
cfg.name
)));
}
if wasabi_version && ext.tier_type_hint.as_deref().is_some_and(|hint| hint != "wasabi") {
return Err(io::Error::other(format!(
"tier config '{}' has inconsistent Wasabi type discriminators",
cfg.name
)));
}
let tier_type = if wasabi_version {
TierType::Wasabi
} else {
tier_type_from_hint(ext.tier_type_hint.as_deref()).unwrap_or_else(|| match ext.tier_type {
EXTERNAL_TIER_TYPE_S3 => TierType::S3,
EXTERNAL_TIER_TYPE_AZURE => TierType::Azure,
EXTERNAL_TIER_TYPE_GCS => TierType::GCS,
EXTERNAL_TIER_TYPE_MINIO => TierType::MinIO,
_ => TierType::Unsupported,
})
};
cfg.tier_type = tier_type.clone();
match tier_type {
TierType::S3 => {
let s3 = ext
.s3
.as_ref()
.ok_or_else(|| io::Error::other(format!("tier config '{}' missing s3 backend payload", cfg.name)))?;
cfg.s3 = Some(crate::services::tier::tier_config::TierS3 {
name: cfg.name.clone(),
endpoint: s3.endpoint.clone(),
access_key: s3.access_key.clone(),
secret_key: s3.secret_key.clone(),
bucket: s3.bucket.clone(),
prefix: s3.prefix.clone(),
region: s3.region.clone(),
storage_class: s3.storage_class.clone(),
aws_role: s3.aws_role,
aws_role_web_identity_token_file: s3.aws_role_web_identity_token_file.clone(),
aws_role_arn: s3.aws_role_arn.clone(),
aws_role_session_name: s3.aws_role_session_name.clone(),
aws_role_duration_seconds: s3.aws_role_duration_seconds,
});
}
TierType::Wasabi => {
let s3 = ext
.s3
.as_ref()
.ok_or_else(|| io::Error::other(format!("tier config '{}' missing Wasabi S3 payload", cfg.name)))?;
if !s3.storage_class.is_empty()
|| s3.aws_role
|| !s3.aws_role_web_identity_token_file.is_empty()
|| !s3.aws_role_arn.is_empty()
|| !s3.aws_role_session_name.is_empty()
|| s3.aws_role_duration_seconds != 0
{
return Err(io::Error::other(format!(
"tier config '{}' has unsupported Wasabi S3 credential fields",
cfg.name
)));
}
if s3.endpoint != EXTERNAL_TIER_WASABI_ENDPOINT_SENTINEL {
return Err(io::Error::other(format!(
"tier config '{}' has an invalid Wasabi compatibility endpoint",
cfg.name
)));
}
let mut wasabi = TierWasabi {
name: cfg.name.clone(),
endpoint: String::new(),
access_key: s3.access_key.clone(),
secret_key: s3.secret_key.clone(),
bucket: s3.bucket.clone(),
prefix: s3.prefix.clone(),
region: s3.region.clone(),
};
wasabi.normalize_endpoint()?;
cfg.wasabi = Some(wasabi);
}
TierType::Azure => {
let az = ext
.azure
.as_ref()
.ok_or_else(|| io::Error::other(format!("tier config '{}' missing azure backend payload", cfg.name)))?;
cfg.azure = Some(crate::services::tier::tier_config::TierAzure {
name: cfg.name.clone(),
endpoint: az.endpoint.clone(),
access_key: az.account_name.clone(),
secret_key: az.account_key.clone(),
bucket: az.bucket.clone(),
prefix: az.prefix.clone(),
region: az.region.clone(),
storage_class: az.storage_class.clone(),
sp_auth: crate::services::tier::tier_config::ServicePrincipalAuth {
tenant_id: az.sp_auth.tenant_id.clone(),
client_id: az.sp_auth.client_id.clone(),
client_secret: az.sp_auth.client_secret.clone(),
},
});
}
TierType::GCS => {
let gcs = ext
.gcs
.as_ref()
.ok_or_else(|| io::Error::other(format!("tier config '{}' missing gcs backend payload", cfg.name)))?;
cfg.gcs = Some(crate::services::tier::tier_config::TierGCS {
name: cfg.name.clone(),
endpoint: gcs.endpoint.clone(),
creds: gcs.creds.clone(),
bucket: gcs.bucket.clone(),
prefix: gcs.prefix.clone(),
region: gcs.region.clone(),
storage_class: gcs.storage_class.clone(),
});
}
TierType::MinIO => {
let m = ext
.compatible_backend
.as_ref()
.ok_or_else(|| io::Error::other(format!("tier config '{}' missing compatible backend payload", cfg.name)))?;
cfg.minio = Some(crate::services::tier::tier_config::TierMinIO {
name: cfg.name.clone(),
endpoint: m.endpoint.clone(),
access_key: m.access_key.clone(),
secret_key: m.secret_key.clone(),
bucket: m.bucket.clone(),
prefix: m.prefix.clone(),
region: m.region.clone(),
});
}
TierType::RustFS => {
let m = decode_legacy_s3_like(&cfg.name, &ext)?;
cfg.rustfs = Some(crate::services::tier::tier_config::TierRustFS {
name: cfg.name.clone(),
endpoint: m.endpoint,
access_key: m.access_key,
secret_key: m.secret_key,
bucket: m.bucket,
prefix: m.prefix,
region: m.region,
storage_class: m.storage_class,
});
}
TierType::Aliyun => {
let m = decode_legacy_s3_like(&cfg.name, &ext)?;
cfg.aliyun = Some(crate::services::tier::tier_config::TierAliyun {
name: cfg.name.clone(),
endpoint: m.endpoint,
access_key: m.access_key,
secret_key: m.secret_key,
bucket: m.bucket,
prefix: m.prefix,
region: m.region,
});
}
TierType::Tencent => {
let m = decode_legacy_s3_like(&cfg.name, &ext)?;
cfg.tencent = Some(crate::services::tier::tier_config::TierTencent {
name: cfg.name.clone(),
endpoint: m.endpoint,
access_key: m.access_key,
secret_key: m.secret_key,
bucket: m.bucket,
prefix: m.prefix,
region: m.region,
});
}
TierType::Huaweicloud => {
let m = decode_legacy_s3_like(&cfg.name, &ext)?;
cfg.huaweicloud = Some(crate::services::tier::tier_config::TierHuaweicloud {
name: cfg.name.clone(),
endpoint: m.endpoint,
access_key: m.access_key,
secret_key: m.secret_key,
bucket: m.bucket,
prefix: m.prefix,
region: m.region,
});
}
TierType::R2 => {
let m = decode_legacy_s3_like(&cfg.name, &ext)?;
cfg.r2 = Some(crate::services::tier::tier_config::TierR2 {
name: cfg.name.clone(),
endpoint: m.endpoint,
access_key: m.access_key,
secret_key: m.secret_key,
bucket: m.bucket,
prefix: m.prefix,
region: m.region,
});
}
TierType::Unsupported => {}
}
Ok(cfg)
}
fn encode_external_tiering_config_blob(cfg: &TierConfigMgr) -> io::Result<Bytes> {
let mut tiers = HashMap::with_capacity(cfg.tiers.len());
for (name, tier_cfg) in &cfg.tiers {
tiers.insert(name.clone(), to_external_tier_config(name, tier_cfg)?);
}
let payload = rmp_serde::to_vec(&ExternalTierConfigMgr { tiers })
.map_err(|err| io::Error::other(format!("serialize tier config payload failed: {err}")))?;
let mut data = Vec::with_capacity(4 + payload.len());
let mut format = [0u8; 2];
LittleEndian::write_u16(&mut format, TIER_CONFIG_FORMAT);
data.extend_from_slice(&format);
let mut version = [0u8; 2];
LittleEndian::write_u16(&mut version, TIER_CONFIG_VERSION);
data.extend_from_slice(&version);
data.extend_from_slice(&payload);
Ok(Bytes::from(data))
}
fn decode_external_tiering_config_blob(data: &[u8]) -> io::Result<TierConfigMgr> {
if data.len() <= 4 {
return Err(io::Error::other("tierConfigInit: no data"));
}
let format = LittleEndian::read_u16(&data[0..2]);
if format != TIER_CONFIG_FORMAT {
return Err(io::Error::other(format!("tierConfigInit: unknown format: {format}")));
}
let version = LittleEndian::read_u16(&data[2..4]);
if version != TIER_CONFIG_V1 && version != TIER_CONFIG_VERSION {
return Err(io::Error::other(format!("tierConfigInit: unknown version: {version}")));
}
let external: ExternalTierConfigMgr =
rmp_serde::from_slice(&data[4..]).map_err(|err| io::Error::other(format!("decode tier config payload failed: {err}")))?;
let mut tiers = HashMap::with_capacity(external.tiers.len());
for (name, ext_cfg) in external.tiers {
tiers.insert(name.clone(), from_external_tier_config(name, ext_cfg)?);
}
Ok(TierConfigMgr {
driver_cache: HashMap::new(),
tiers,
last_refreshed_at: OffsetDateTime::now_utc(),
})
}
fn decode_tiering_config_blob(data: &[u8]) -> io::Result<TierConfigMgr> {
if let Ok(cfg) = TierConfigMgr::unmarshal(data) {
return Ok(cfg);
}
decode_external_tiering_config_blob(data)
}
impl TierConfigMgr {
pub fn new() -> Arc<RwLock<Self>> {
Arc::new(RwLock::new(Self {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
}))
}
pub fn unmarshal(data: &[u8]) -> std::result::Result<TierConfigMgr, std::io::Error> {
let mut cfg: TierConfigMgr = serde_json::from_slice(data)?;
for (name, tier) in &mut cfg.tiers {
if matches!(&tier.tier_type, TierType::Wasabi) {
let wasabi = tier
.wasabi
.as_mut()
.ok_or_else(|| io::Error::other(format!("tier config '{name}' missing Wasabi backend payload")))?;
if wasabi.endpoint.is_empty() {
return Err(io::Error::other(format!("tier config '{name}' missing Wasabi endpoint")));
}
wasabi.normalize_endpoint()?;
}
}
Ok(cfg)
}
pub fn marshal(&self) -> std::result::Result<Bytes, std::io::Error> {
let data = serde_json::to_vec(&self)?;
let mut data = Bytes::from(data);
Ok(data)
}
pub fn refreshed_at(&self) -> OffsetDateTime {
self.last_refreshed_at
}
pub fn is_tier_valid(&self, tier_name: &str) -> bool {
let (_, valid) = self.is_tier_name_in_use(tier_name);
valid
}
pub fn is_tier_name_in_use(&self, tier_name: &str) -> (TierType, bool) {
if let Some(t) = self.tiers.get(tier_name) {
return (t.tier_type.clone(), true);
}
(TierType::Unsupported, false)
}
pub async fn add(&mut self, mut tier_config: TierConfig, force: bool) -> std::result::Result<(), AdminError> {
self.ensure_generation_is_idle(&tier_config.name)?;
let tier_name = tier_config.name.clone();
if tier_name != tier_name.to_uppercase() {
return Err(ERR_TIER_NAME_NOT_UPPERCASE.clone());
}
// Reject AWS/MinIO reserved storage-class names as remote-tier names.
// MinIO reserves STANDARD and REDUCED_REDUNDANCY (RRS); a tier must not
// shadow them. The comparison is case-insensitive to match MinIO parity
// and to stay robust regardless of the uppercase gate above.
if tier_name.eq_ignore_ascii_case(crate::config::storageclass::STANDARD)
|| tier_name.eq_ignore_ascii_case(crate::config::storageclass::RRS)
{
return Err(ERR_TIER_RESERVED_NAME.clone());
}
let (_, b) = self.is_tier_name_in_use(&tier_name);
if b {
return Err(ERR_TIER_ALREADY_EXISTS.clone());
}
if matches!(&tier_config.tier_type, TierType::Wasabi)
&& let Some(wasabi) = tier_config.wasabi.as_mut()
{
wasabi.normalize_endpoint().map_err(|source| {
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = source.to_string();
err
})?;
}
let d = new_warm_backend(&tier_config, true).await?;
if !force {
let in_use = d.in_use().await;
match in_use {
Ok(b) => {
if b {
return Err(ERR_TIER_BACKEND_IN_USE.clone());
}
}
Err(err) => {
warn!("tier add failed, err: {:?}", err);
if err.to_string().contains("connect") {
return Err(ERR_TIER_CONNECT_ERR.clone());
} else if err.to_string().contains("authorization") {
return Err(ERR_TIER_INVALID_CREDENTIALS.clone());
} else if err.to_string().contains("bucket") {
return Err(ERR_TIER_BUCKET_NOT_FOUND.clone());
}
let mut e = ERR_TIER_PERM_ERR.clone();
e.message.push('.');
e.message.push_str(&err.to_string());
return Err(e);
}
}
}
self.tiers.insert(tier_name.clone(), tier_config);
if let Err(err) = self.replace_driver(&tier_name, d) {
self.tiers.remove(&tier_name);
return Err(err);
}
Ok(())
}
pub async fn remove(&mut self, tier_name: &str, force: bool) -> std::result::Result<(), AdminError> {
self.ensure_generation_is_idle(tier_name)?;
let d = self.get_driver(tier_name).await;
if let Err(err) = d {
if err.code == ERR_TIER_NOT_FOUND.code {
return Ok(());
} else {
return Err(err);
}
}
if !force {
if let Ok(driver) = d {
match driver.in_use().await {
Err(err) => {
let mut e = ERR_TIER_PERM_ERR.clone();
e.message.push('.');
e.message.push_str(&err.to_string());
return Err(e);
}
Ok(in_use) if in_use => {
return Err(ERR_TIER_BACKEND_NOT_EMPTY.clone());
}
_ => {}
}
}
}
self.tiers.remove(tier_name);
self.revoke_driver(tier_name);
Ok(())
}
pub async fn verify(&mut self, tier_name: &str) -> std::result::Result<(), std::io::Error> {
let d = match self.get_driver(tier_name).await {
Ok(d) => d,
Err(err) => {
return Err(std::io::Error::other(err));
}
};
if let Err(err) = check_warm_backend(Some(d)).await {
return Err(std::io::Error::other(err));
} else {
return Ok(());
}
}
pub fn empty(&self) -> bool {
self.list_tiers().len() == 0
}
pub fn tier_type(&self, tier_name: &str) -> String {
if let Some(cfg) = self.tiers.get(tier_name) {
cfg.tier_type.as_lowercase()
} else {
"internal".to_string()
}
}
pub fn list_tiers(&self) -> Vec<TierConfig> {
let mut tier_cfgs = Vec::<TierConfig>::new();
for (_, tier) in self.tiers.iter() {
let tier = tier.clone();
tier_cfgs.push(tier);
}
tier_cfgs
}
pub fn get(&self, tier_name: &str) -> Option<TierConfig> {
for (tier_name2, tier) in self.tiers.iter() {
if tier_name == tier_name2 {
return Some(tier.clone());
}
}
None
}
pub async fn edit(&mut self, tier_name: &str, creds: TierCreds) -> std::result::Result<(), AdminError> {
self.ensure_generation_is_idle(tier_name)?;
let (tier_type, exists) = self.is_tier_name_in_use(tier_name);
if !exists {
return Err(ERR_TIER_NOT_FOUND.clone());
}
let mut tier_config = self.tiers[tier_name].clone();
match tier_type {
TierType::S3 => {
if let Some(s3) = tier_config.s3.as_mut() {
if creds.aws_role {
s3.aws_role = true
}
if creds.aws_role_web_identity_token_file != "" && creds.aws_role_arn != "" {
s3.aws_role_arn = creds.aws_role_arn;
s3.aws_role_web_identity_token_file = creds.aws_role_web_identity_token_file;
}
if creds.access_key != "" && creds.secret_key != "" {
s3.access_key = creds.access_key;
s3.secret_key = creds.secret_key;
}
}
}
TierType::Wasabi => {
if let Some(wasabi) = tier_config.wasabi.as_mut() {
if creds.access_key.is_empty() || creds.secret_key.is_empty() {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
wasabi.access_key = creds.access_key;
wasabi.secret_key = creds.secret_key;
}
}
TierType::RustFS => {
if let Some(rustfs) = tier_config.rustfs.as_mut() {
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
rustfs.access_key = creds.access_key;
rustfs.secret_key = creds.secret_key;
}
}
TierType::MinIO => {
if let Some(compatible_backend) = tier_config.minio.as_mut() {
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
compatible_backend.access_key = creds.access_key;
compatible_backend.secret_key = creds.secret_key;
}
}
TierType::Aliyun => {
if let Some(aliyun) = tier_config.aliyun.as_mut() {
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
aliyun.access_key = creds.access_key;
aliyun.secret_key = creds.secret_key;
}
}
TierType::Tencent => {
if let Some(tencent) = tier_config.tencent.as_mut() {
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
tencent.access_key = creds.access_key;
tencent.secret_key = creds.secret_key;
}
}
TierType::Huaweicloud => {
if let Some(huaweicloud) = tier_config.huaweicloud.as_mut() {
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
huaweicloud.access_key = creds.access_key;
huaweicloud.secret_key = creds.secret_key;
}
}
TierType::Azure => {
if let Some(azure) = tier_config.azure.as_mut() {
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
azure.access_key = creds.access_key;
azure.secret_key = creds.secret_key;
}
}
TierType::GCS => {
if let Some(gcs) = tier_config.gcs.as_mut() {
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
gcs.creds = creds.access_key; //creds.creds_json
}
}
TierType::R2 => {
if let Some(r2) = tier_config.r2.as_mut() {
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
r2.access_key = creds.access_key;
r2.secret_key = creds.secret_key;
}
}
_ => (),
}
let d = new_warm_backend(&tier_config, true).await?;
self.revoke_driver(tier_name);
self.tiers.insert(tier_name.to_string(), tier_config);
self.replace_driver(tier_name, d)?;
Ok(())
}
pub async fn get_driver<'a>(&'a mut self, tier_name: &str) -> std::result::Result<&'a WarmBackendImpl, AdminError> {
if registered_tier_driver_runtime(self).is_some_and(|runtime| runtime_blocks_tier(&lock_unpoisoned(&runtime), tier_name))
{
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = TIER_MUTATION_BLOCK_MESSAGE.to_string();
return Err(err);
}
// Return cached driver if present
if self.driver_cache.contains_key(tier_name) {
return Ok(self.driver_cache.get(tier_name).expect("Driver not found in cache"));
}
// Get tier configuration and create new driver
let tier_config = self.tiers.get(tier_name).ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
let driver = new_warm_backend(tier_config, false).await?;
self.replace_driver(tier_name, driver)?;
Ok(self
.driver_cache
.get(tier_name)
.expect("Driver not found in cache after insertion"))
}
pub(crate) async fn acquire_operation_lease(
handle: &Arc<RwLock<Self>>,
tier_name: &str,
) -> std::result::Result<TierOperationLease, AdminError> {
{
let manager = handle.read().await;
let runtime = tier_driver_runtime(handle, &manager);
let runtime_guard = lock_unpoisoned(&runtime);
if runtime_blocks_tier(&runtime_guard, tier_name) {
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = TIER_MUTATION_BLOCK_MESSAGE.to_string();
return Err(err);
}
if let Some(generation) = runtime_guard
.generations
.get(tier_name)
.filter(|generation| generation.accepting.load(Ordering::Acquire))
.cloned()
{
let lease = TierOperationLease::try_new(generation, runtime.clone());
drop(runtime_guard);
return lease;
}
}
let (config, config_fingerprint) = {
let mut manager = handle.write().await;
let runtime = tier_driver_runtime(handle, &manager);
if runtime_blocks_tier(&lock_unpoisoned(&runtime), tier_name) {
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = TIER_MUTATION_BLOCK_MESSAGE.to_string();
return Err(err);
}
if let Some(driver) = manager.driver_cache.remove(tier_name) {
manager.replace_driver(tier_name, driver)?;
let runtime_guard = lock_unpoisoned(&runtime);
let generation = runtime_guard
.generations
.get(tier_name)
.filter(|generation| generation.accepting.load(Ordering::Acquire))
.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?
.clone();
let lease = TierOperationLease::try_new(generation, runtime.clone());
drop(runtime_guard);
return lease;
}
let config = manager
.tiers
.get(tier_name)
.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?
.clone_with_credentials();
let fingerprint = tier_config_fingerprint(tier_name, &config).map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})?;
(config, fingerprint)
};
let driver = build_warm_backend(&config, false).await?;
let mut manager = handle.write().await;
let runtime = tier_driver_runtime(handle, &manager);
let runtime_guard = lock_unpoisoned(&runtime);
if runtime_blocks_tier(&runtime_guard, tier_name) {
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = TIER_MUTATION_BLOCK_MESSAGE.to_string();
return Err(err);
}
if let Some(generation) = runtime_guard
.generations
.get(tier_name)
.filter(|generation| {
generation.config_fingerprint == config_fingerprint && generation.accepting.load(Ordering::Acquire)
})
.cloned()
{
let lease = TierOperationLease::try_new(generation, runtime.clone());
drop(runtime_guard);
return lease;
}
drop(runtime_guard);
let current = manager.tiers.get(tier_name).ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
if tier_config_fingerprint(tier_name, current).map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})? != config_fingerprint
{
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier configuration changed while its driver was being prepared".to_string();
return Err(err);
}
manager.replace_driver(tier_name, driver)?;
let runtime_guard = lock_unpoisoned(&runtime);
let generation = runtime_guard
.generations
.get(tier_name)
.filter(|generation| generation.accepting.load(Ordering::Acquire))
.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?
.clone();
let lease = TierOperationLease::try_new(generation, runtime.clone());
drop(runtime_guard);
lease
}
async fn admin_update_lock(handle: &Arc<RwLock<Self>>) -> tokio::sync::OwnedMutexGuard<()> {
let update_lock = {
let manager = handle.read().await;
let runtime = tier_driver_runtime(handle, &manager);
lock_unpoisoned(&runtime).admin_updates.clone()
};
update_lock.lock_owned().await
}
async fn acquire_tier_config_write_lock<S>(
api: Arc<S>,
) -> std::result::Result<rustfs_lock::NamespaceLockGuard, TierConfigUpdateError>
where
S: NamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper> + 'static,
{
let config_file = tier_config_lock_path();
let ns_lock = api
.new_ns_lock(RUSTFS_META_BUCKET, &config_file)
.await
.map_err(|err| TierConfigUpdateError::Load(io::Error::other(err)))?;
ns_lock
.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(|err| TierConfigUpdateError::Load(io::Error::other(err)))
}
async fn update_candidate_with_config_lock<S>(
handle: &Arc<RwLock<Self>>,
api: Arc<S>,
mutation: TierCandidateMutation,
) -> std::result::Result<(), TierConfigUpdateError>
where
S: TierReferenceProofStore + NamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper> + 'static,
{
let config_lock = Self::acquire_tier_config_write_lock(api.clone()).await?;
let update = Self::admin_update_lock(handle).await;
let (candidate, version) = load_tier_config_for_update(api.clone())
.await
.map_err(TierConfigUpdateError::Load)?;
Self::update_candidate_owned(handle, api, candidate, version, mutation, update, Some(config_lock)).await
}
#[cfg(test)]
pub(crate) async fn publish_candidate(
handle: &Arc<RwLock<Self>>,
candidate: Self,
driver_tier: Option<&str>,
) -> std::result::Result<(), AdminError> {
let update = Self::admin_update_lock(handle).await;
Self::publish_candidate_owned(handle, candidate, driver_tier.map(str::to_string), update).await
}
fn begin_publish_transition(
handle: &Arc<RwLock<Self>>,
manager: &mut Self,
candidate: &Self,
) -> std::result::Result<TierPublishTransition, AdminError> {
let changed = changed_tier_names(manager, candidate);
let replaced_destinations = replaced_tier_destinations(manager, candidate)?;
Self::begin_tier_transition_with_destinations(handle, manager, changed, replaced_destinations)
}
fn begin_tier_transition(
handle: &Arc<RwLock<Self>>,
manager: &mut Self,
changed: HashSet<String>,
) -> std::result::Result<TierPublishTransition, AdminError> {
Self::begin_tier_transition_with_destinations(handle, manager, changed, HashMap::new())
}
fn begin_tier_transition_with_destinations(
handle: &Arc<RwLock<Self>>,
manager: &mut Self,
changed: HashSet<String>,
replaced_destinations: HashMap<String, TierConfig>,
) -> std::result::Result<TierPublishTransition, AdminError> {
let runtime = tier_driver_runtime(handle, manager);
for tier_name in replaced_destinations.keys() {
if !lock_unpoisoned(&runtime).generations.contains_key(tier_name)
&& let Some(driver) = manager.driver_cache.remove(tier_name)
{
manager.replace_driver(tier_name, driver)?;
}
}
Self::begin_tier_transition_in_runtime(runtime, changed, replaced_destinations)
}
fn begin_tier_transition_in_runtime(
runtime: Arc<Mutex<TierDriverRuntime>>,
changed: HashSet<String>,
replaced_destinations: HashMap<String, TierConfig>,
) -> std::result::Result<TierPublishTransition, AdminError> {
let count = u64::try_from(changed.len()).map_err(|_| {
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier replacement count exceeds the supported limit".to_string();
err
})?;
let mut runtime_guard = lock_unpoisoned(&runtime);
if changed.iter().any(|tier_name| runtime_blocks_tier(&runtime_guard, tier_name)) {
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier configuration is already being replaced".to_string();
return Err(err);
}
let final_epoch = runtime_guard.next_drain_epoch.checked_add(count).ok_or_else(|| {
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier replacement epoch exhausted".to_string();
err
})?;
let mut next_epoch = runtime_guard.next_drain_epoch;
let mut tokens = Vec::with_capacity(changed.len());
let mut revoked = HashMap::with_capacity(changed.len());
for tier_name in changed {
next_epoch += 1;
runtime_guard.draining.insert(tier_name.clone(), next_epoch);
tokens.push((tier_name.clone(), next_epoch));
if let Some(generation) = runtime_guard.generations.remove(&tier_name) {
generation.accepting.store(false, Ordering::Release);
revoked.insert(tier_name, generation);
}
}
runtime_guard.next_drain_epoch = final_epoch;
drop(runtime_guard);
Ok(TierPublishTransition {
runtime,
tokens,
revoked,
replaced_destinations,
published: false,
})
}
async fn publish_candidate_inner(
handle: &Arc<RwLock<Self>>,
candidate: Self,
driver_tier: Option<&str>,
) -> std::result::Result<(), AdminError> {
let transition = {
let mut manager = handle.write().await;
Self::begin_publish_transition(handle, &mut manager, &candidate)?
};
transition.wait_for_active_leases().await?;
transition.ensure_replaced_destinations_are_empty().await?;
Self::publish_candidate_after_drain(handle, candidate, driver_tier, transition).await
}
async fn publish_candidate_after_drain(
handle: &Arc<RwLock<Self>>,
mut candidate: Self,
driver_tier: Option<&str>,
mut transition: TierPublishTransition,
) -> std::result::Result<(), AdminError> {
let prepared_driver =
match driver_tier.and_then(|tier_name| candidate.driver_cache.remove(tier_name).map(|driver| (tier_name, driver))) {
Some((tier_name, driver)) => {
let config = candidate.tiers.get(tier_name).ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
let config_fingerprint = tier_config_fingerprint(tier_name, config).map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})?;
let backend_identity = tier_backend_identity(config).map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})?;
Some(PreparedTierDriver {
tier_name: tier_name.to_string(),
config_fingerprint,
backend_identity,
driver: Arc::from(driver),
})
}
None => None,
};
// Lock order invariant: manager state before the per-manager driver runtime.
let mut manager = handle.write().await;
let mut runtime = lock_unpoisoned(&transition.runtime);
if !transition
.tokens
.iter()
.all(|(tier_name, epoch)| runtime.draining.get(tier_name) == Some(epoch))
{
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier replacement ownership was lost before publish".to_string();
return Err(err);
}
let prepared_generation = if let Some(prepared) = prepared_driver {
let generation = runtime.next_generation.checked_add(1).ok_or_else(|| {
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier driver generation exhausted".to_string();
err
})?;
let entry = Arc::new(TierDriverGeneration {
tier_name: Arc::from(prepared.tier_name.as_str()),
generation,
config_fingerprint: prepared.config_fingerprint,
backend_identity: prepared.backend_identity,
driver: prepared.driver.clone(),
accepting: AtomicBool::new(true),
active_leases: AtomicUsize::new(0),
drained: tokio::sync::Notify::new(),
});
Some((prepared, generation, entry))
} else {
None
};
for (tier_name, _) in &transition.tokens {
manager.driver_cache.remove(tier_name);
}
manager.tiers = candidate.tiers;
manager.last_refreshed_at = OffsetDateTime::now_utc();
if let Some((prepared, generation, entry)) = prepared_generation {
runtime.generations.insert(prepared.tier_name.clone(), entry);
runtime.next_generation = generation;
manager
.driver_cache
.insert(prepared.tier_name, Box::new(SharedWarmBackendProxy(prepared.driver)));
}
drop(runtime);
transition.published = true;
Ok(())
}
fn publish_task_error(err: tokio::task::JoinError) -> AdminError {
error!(error = ?err, "remote tier configuration publish task failed");
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = format!("Remote tier configuration publish task failed: {err}");
admin_err
}
async fn publish_candidate_owned(
handle: &Arc<RwLock<Self>>,
candidate: Self,
driver_tier: Option<String>,
update: tokio::sync::OwnedMutexGuard<()>,
) -> std::result::Result<(), AdminError> {
let handle = handle.clone();
tokio::spawn(async move {
match AssertUnwindSafe(async move {
let _update = update;
Self::publish_candidate_inner(&handle, candidate, driver_tier.as_deref()).await
})
.catch_unwind()
.await
{
Ok(Ok(())) => Ok(()),
Ok(Err(err)) => {
error!(error = ?err, "remote tier configuration publish failed");
Err(err)
}
Err(_) => {
error!("remote tier configuration publish panicked");
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier configuration publish panicked".to_string();
Err(err)
}
}
})
.await
.map_err(Self::publish_task_error)?
}
async fn update_candidate_owned<S>(
handle: &Arc<RwLock<Self>>,
api: Arc<S>,
mut candidate: Self,
version: Option<String>,
mutation: TierCandidateMutation,
update: tokio::sync::OwnedMutexGuard<()>,
config_lock: Option<rustfs_lock::NamespaceLockGuard>,
) -> std::result::Result<(), TierConfigUpdateError>
where
S: TierReferenceProofStore + 'static,
{
let handle = handle.clone();
tokio::spawn(async move {
match AssertUnwindSafe(async move {
let _config_lock = config_lock;
let _update = update;
let mutation_kind = mutation.intent_kind();
let explicit_tier_name = mutation.explicit_tier_name().map(str::to_string);
let current_for_targets = TierConfigMgr {
driver_cache: HashMap::new(),
tiers: candidate
.tiers
.iter()
.map(|(tier_name, config)| (tier_name.clone(), config.clone_with_credentials()))
.collect(),
last_refreshed_at: candidate.last_refreshed_at,
};
let transition = {
let mut manager = handle.write().await;
let target_tiers = mutation.target_tiers(&manager, &candidate);
Self::begin_tier_transition(&handle, &mut manager, target_tiers).map_err(TierConfigUpdateError::Publish)?
};
transition
.wait_for_active_leases()
.await
.map_err(TierConfigUpdateError::Publish)?;
let driver_tier =
apply_tier_candidate_mutation(mutation, &mut candidate, Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT)
.await
.map_err(TierConfigUpdateError::Mutation)?;
let proof_targets =
tier_mutation_proof_targets(mutation_kind, explicit_tier_name.as_deref(), &current_for_targets, &candidate);
let affected_targets =
build_tier_mutation_affected_targets(mutation_kind, proof_targets, &current_for_targets, &candidate)
.map_err(TierConfigUpdateError::Publish)?;
ensure_no_authoritative_tier_object_references(api.clone(), &affected_targets)
.await
.map_err(TierConfigUpdateError::Publish)?;
candidate
.save_tiering_config_if_current(api, version.as_deref())
.await
.map_err(TierConfigUpdateError::Save)?;
Self::publish_candidate_after_drain(&handle, candidate, driver_tier.as_deref(), transition)
.await
.map_err(TierConfigUpdateError::Publish)
})
.catch_unwind()
.await
{
Ok(Ok(())) => Ok(()),
Ok(Err(err)) => {
error!(error = ?err, "remote tier configuration update failed");
Err(err)
}
Err(_) => {
error!("remote tier configuration update panicked");
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier configuration update panicked".to_string();
Err(TierConfigUpdateError::Publish(err))
}
}
})
.await
.map_err(|err| TierConfigUpdateError::Publish(Self::publish_task_error(err)))?
}
pub async fn reload_handle(handle: &Arc<RwLock<Self>>, api: Arc<ECStore>) -> io::Result<()> {
let prepared_intents = Self::load_prepared_mutation_intents(api.clone()).await?;
let update = Self::admin_update_lock(handle).await;
let candidate = load_tier_config(api).await?;
Self::publish_candidate_owned(handle, candidate, None, update)
.await
.map_err(io::Error::other)?;
Self::reconcile_prepared_mutation_intents(handle, &prepared_intents)
.await
.map_err(io::Error::other)
}
async fn load_prepared_mutation_intents(api: Arc<ECStore>) -> io::Result<Vec<TierMutationIntent>> {
let mut marker = None;
let mut prepared = Vec::new();
loop {
let scan = list_tier_mutation_intent_records(api.clone(), TIER_MUTATION_INTENT_RECOVERY_SCAN_LIMIT, marker)
.await
.map_err(io::Error::other)?;
if scan.failed != 0 {
return Err(io::Error::other(format!(
"failed to scan {} tier mutation intent record(s) during recovery",
scan.failed
)));
}
prepared.extend(
scan.intents
.into_iter()
.filter(|intent| intent.state == TierMutationIntentState::Prepared),
);
if !scan.truncated {
return Ok(prepared);
}
let next_marker = scan.next_marker.ok_or_else(|| {
io::Error::other("tier mutation intent recovery scan was truncated without a continuation marker")
})?;
marker = Some(next_marker);
}
}
async fn reconcile_prepared_mutation_intents(
handle: &Arc<RwLock<Self>>,
intents: &[TierMutationIntent],
) -> std::result::Result<(), AdminError> {
let mut prepared_mutation_blocks = HashMap::new();
for intent in intents {
Self::collect_prepared_mutation_intent_block(&mut prepared_mutation_blocks, intent)?;
}
let manager = handle.read().await;
let runtime = tier_driver_runtime(handle, &manager);
let mut runtime = lock_unpoisoned(&runtime);
runtime.prepared_mutation_blocks = prepared_mutation_blocks;
Ok(())
}
pub(crate) async fn apply_prepared_mutation_intent_block(
handle: &Arc<RwLock<Self>>,
intent: &TierMutationIntent,
) -> std::result::Result<(), AdminError> {
let manager = handle.read().await;
let runtime = tier_driver_runtime(handle, &manager);
let mut runtime = lock_unpoisoned(&runtime);
let mut prepared_mutation_blocks = runtime.prepared_mutation_blocks.clone();
Self::collect_prepared_mutation_intent_block(&mut prepared_mutation_blocks, intent)?;
runtime.prepared_mutation_blocks = prepared_mutation_blocks;
Ok(())
}
pub(crate) async fn clear_prepared_mutation_intent_block(handle: &Arc<RwLock<Self>>, mutation_id: uuid::Uuid) {
let manager = handle.read().await;
let Some(runtime) = registered_tier_driver_runtime(&manager) else {
return;
};
lock_unpoisoned(&runtime)
.prepared_mutation_blocks
.retain(|_, blocked_mutation_id| *blocked_mutation_id != mutation_id);
}
fn collect_prepared_mutation_intent_block(
prepared_mutation_blocks: &mut HashMap<String, uuid::Uuid>,
intent: &TierMutationIntent,
) -> std::result::Result<(), AdminError> {
if intent.state != TierMutationIntentState::Prepared {
return Ok(());
}
for target in &intent.affected_targets {
match prepared_mutation_blocks.entry(target.tier_name.clone()) {
Entry::Vacant(entry) => {
entry.insert(intent.mutation_id);
}
Entry::Occupied(entry) if *entry.get() == intent.mutation_id => {}
Entry::Occupied(_) => {
let mut err = ERR_TIER_BACKEND_IN_USE.clone();
err.message = format!("Remote tier {} already has another prepared mutation", target.tier_name);
return Err(err);
}
}
}
Ok(())
}
pub async fn add_and_save(
handle: &Arc<RwLock<Self>>,
api: Arc<ECStore>,
tier_config: TierConfig,
force: bool,
) -> std::result::Result<(), TierConfigUpdateError> {
Self::update_candidate_with_config_lock(handle, api, TierCandidateMutation::Add(tier_config, force)).await
}
pub async fn edit_and_save(
handle: &Arc<RwLock<Self>>,
api: Arc<ECStore>,
tier_name: &str,
credentials: TierCreds,
) -> std::result::Result<(), TierConfigUpdateError> {
Self::update_candidate_with_config_lock(handle, api, TierCandidateMutation::Edit(tier_name.to_string(), credentials))
.await
}
pub async fn remove_and_save(
handle: &Arc<RwLock<Self>>,
api: Arc<ECStore>,
tier_name: &str,
force: bool,
) -> std::result::Result<(), TierConfigUpdateError> {
Self::update_candidate_with_config_lock(handle, api, TierCandidateMutation::Remove(tier_name.to_string(), force)).await
}
async fn remove_and_save_with<S>(
handle: &Arc<RwLock<Self>>,
api: Arc<S>,
tier_name: &str,
force: bool,
) -> std::result::Result<(), TierConfigUpdateError>
where
S: TierReferenceProofStore + 'static,
{
let update = Self::admin_update_lock(handle).await;
let (candidate, version) = load_tier_config_for_update(api.clone())
.await
.map_err(TierConfigUpdateError::Load)?;
Self::update_candidate_owned(
handle,
api,
candidate,
version,
TierCandidateMutation::Remove(tier_name.to_string(), force),
update,
None,
)
.await
}
pub async fn clear_and_save(
handle: &Arc<RwLock<Self>>,
api: Arc<ECStore>,
force: bool,
) -> std::result::Result<(), TierConfigUpdateError> {
Self::update_candidate_with_config_lock(handle, api, TierCandidateMutation::Clear(force)).await
}
async fn clear_and_save_with<S>(
handle: &Arc<RwLock<Self>>,
api: Arc<S>,
force: bool,
) -> std::result::Result<(), TierConfigUpdateError>
where
S: TierReferenceProofStore + 'static,
{
let update = Self::admin_update_lock(handle).await;
let (candidate, version) = load_tier_config_for_update(api.clone())
.await
.map_err(TierConfigUpdateError::Load)?;
Self::update_candidate_owned(handle, api, candidate, version, TierCandidateMutation::Clear(force), update, None).await
}
pub async fn verify_without_manager_lock(handle: &Arc<RwLock<Self>>, tier_name: &str) -> std::result::Result<(), io::Error> {
let lease = Self::acquire_operation_lease(handle, tier_name)
.await
.map_err(io::Error::other)?;
let driver: WarmBackendImpl = Box::new(SharedWarmBackendProxy(lease.inner.driver.clone()));
check_warm_backend(Some(&driver)).await.map_err(io::Error::other)
}
pub(crate) async fn acquire_operation_lease_for_backend_identity(
handle: &Arc<RwLock<Self>>,
tier_name: &str,
expected: TierDestinationId,
) -> std::result::Result<TierOperationLease, AdminError> {
let lease = Self::acquire_operation_lease(handle, tier_name).await?;
if lease.backend_identity() != expected {
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier backend identity no longer matches the recorded operation".to_string();
return Err(err);
}
Ok(lease)
}
#[cfg(test)]
pub(crate) async fn active_operation_lease_count(handle: &Arc<RwLock<Self>>, tier_name: &str) -> usize {
let manager = handle.read().await;
let Some(runtime) = registered_tier_driver_runtime(&manager) else {
return 0;
};
lock_unpoisoned(&runtime)
.generations
.get(tier_name)
.map(|generation| generation.active_leases.load(Ordering::Acquire))
.unwrap_or(0)
}
fn replace_driver(&mut self, tier_name: &str, driver: WarmBackendImpl) -> std::result::Result<(), AdminError> {
let Some(runtime) = registered_tier_driver_runtime(self) else {
self.driver_cache.insert(tier_name.to_string(), driver);
return Ok(());
};
let config_fingerprint = self
.tiers
.get(tier_name)
.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())
.and_then(|config| {
tier_config_fingerprint(tier_name, config).map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})
})?;
let backend_identity = self
.tiers
.get(tier_name)
.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())
.and_then(|config| {
tier_backend_identity(config).map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})
})?;
let mut runtime_guard = lock_unpoisoned(&runtime);
let generation = runtime_guard.next_generation.checked_add(1).ok_or_else(|| {
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier driver generation exhausted".to_string();
err
})?;
let driver: SharedWarmBackend = Arc::from(driver);
let entry = Arc::new(TierDriverGeneration {
tier_name: Arc::from(tier_name),
generation,
config_fingerprint,
backend_identity,
driver: driver.clone(),
accepting: AtomicBool::new(true),
active_leases: AtomicUsize::new(0),
drained: tokio::sync::Notify::new(),
});
if let Some(previous) = runtime_guard.generations.insert(tier_name.to_string(), entry) {
previous.accepting.store(false, Ordering::Release);
}
runtime_guard.next_generation = generation;
drop(runtime_guard);
self.driver_cache
.insert(tier_name.to_string(), Box::new(SharedWarmBackendProxy(driver)));
Ok(())
}
#[cfg(feature = "test-util")]
pub(crate) fn install_test_driver(
&mut self,
tier_name: &str,
driver: WarmBackendImpl,
) -> std::result::Result<(), AdminError> {
self.replace_driver(tier_name, driver)
}
fn revoke_driver(&mut self, tier_name: &str) -> Option<Arc<TierDriverGeneration>> {
let generation =
registered_tier_driver_runtime(self).and_then(|runtime| lock_unpoisoned(&runtime).generations.remove(tier_name));
if let Some(generation) = generation.as_ref() {
generation.accepting.store(false, Ordering::Release);
}
self.driver_cache.remove(tier_name);
generation
}
fn ensure_generation_is_idle(&self, tier_name: &str) -> std::result::Result<(), AdminError> {
let Some(runtime) = registered_tier_driver_runtime(self) else {
return Ok(());
};
let runtime = lock_unpoisoned(&runtime);
let blocked = runtime_blocks_tier(&runtime, tier_name)
|| runtime
.generations
.get(tier_name)
.is_some_and(|generation| generation.active_leases.load(Ordering::Acquire) != 0);
if blocked {
return Err(ERR_TIER_BACKEND_IN_USE.clone());
}
Ok(())
}
fn ensure_generations_are_idle<'a>(
&self,
tier_names: impl IntoIterator<Item = &'a String>,
) -> std::result::Result<(), AdminError> {
for tier_name in tier_names {
self.ensure_generation_is_idle(tier_name)?;
}
Ok(())
}
fn retire_driver(&mut self, tier_name: &str) {
self.revoke_driver(tier_name);
}
fn revoke_all_drivers(&mut self) {
if let Some(runtime) = registered_tier_driver_runtime(self) {
let mut runtime = lock_unpoisoned(&runtime);
for (_, generation) in runtime.generations.drain() {
generation.accepting.store(false, Ordering::Release);
}
}
self.driver_cache.clear();
}
fn retire_all_drivers(&mut self) {
self.revoke_all_drivers();
}
pub async fn reload(&mut self, api: Arc<ECStore>) -> std::result::Result<(), std::io::Error> {
let config = load_tier_config(api).await?;
self.publish_legacy_reload(config).await?;
Ok(())
}
pub(crate) async fn publish_legacy_reload(&mut self, config: Self) -> io::Result<()> {
let changed = changed_tier_names(self, &config);
let replaced_destinations = replaced_tier_destinations(self, &config).map_err(io::Error::other)?;
if let Some(runtime) = registered_tier_driver_runtime(self) {
for tier_name in replaced_destinations.keys() {
if !lock_unpoisoned(&runtime).generations.contains_key(tier_name)
&& let Some(driver) = self.driver_cache.remove(tier_name)
{
self.replace_driver(tier_name, driver).map_err(io::Error::other)?;
}
}
let mut transition =
Self::begin_tier_transition_in_runtime(runtime, changed, replaced_destinations).map_err(io::Error::other)?;
transition.wait_for_active_leases().await.map_err(io::Error::other)?;
transition
.ensure_replaced_destinations_are_empty()
.await
.map_err(io::Error::other)?;
let runtime = lock_unpoisoned(&transition.runtime);
if !transition
.tokens
.iter()
.all(|(tier_name, epoch)| runtime.draining.get(tier_name) == Some(epoch))
{
return Err(io::Error::other("remote tier replacement ownership was lost before reload publish"));
}
for (tier_name, _) in &transition.tokens {
self.driver_cache.remove(tier_name);
}
self.tiers = config.tiers;
drop(runtime);
transition.published = true;
} else {
ensure_replaced_destinations_are_empty(&replaced_destinations, &HashMap::new())
.await
.map_err(io::Error::other)?;
self.tiers = config.tiers;
}
self.last_refreshed_at = OffsetDateTime::now_utc();
Ok(())
}
fn apply_reloaded_tiers(&mut self, tiers: HashMap<String, TierConfig>) -> std::result::Result<(), AdminError> {
if registered_tier_driver_runtime(self).is_some_and(|runtime| runtime_has_mutation_block(&lock_unpoisoned(&runtime))) {
return Err(ERR_TIER_BACKEND_IN_USE.clone());
}
let changed_or_removed = self
.tiers
.iter()
.filter_map(|(tier_name, current)| {
let unchanged = tiers
.get(tier_name)
.is_some_and(|replacement| tier_configs_match(tier_name, current, replacement));
(!unchanged).then(|| tier_name.clone())
})
.collect::<Vec<_>>();
self.ensure_generations_are_idle(&changed_or_removed)?;
for tier_name in &changed_or_removed {
self.revoke_driver(tier_name);
}
self.tiers = tiers;
Ok(())
}
pub async fn rollback_after_failed_save(&mut self, api: Arc<ECStore>) -> io::Result<()> {
match load_tier_config(api).await {
Ok(config) => {
self.apply_reloaded_tiers(config.tiers).map_err(io::Error::other)?;
Ok(())
}
Err(err) => {
if let Some(runtime) = registered_tier_driver_runtime(self) {
let runtime = lock_unpoisoned(&runtime);
if runtime_has_mutation_block(&runtime)
|| runtime
.generations
.values()
.any(|generation| generation.active_leases.load(Ordering::Acquire) != 0)
{
return Err(io::Error::other(format!(
"failed to reload tier configuration for rollback while a tier generation is active: {err}"
)));
}
}
self.retire_all_drivers();
self.tiers.clear();
Err(err)
}
}
}
pub async fn clear_tier(&mut self, force: bool) -> std::result::Result<(), AdminError> {
if registered_tier_driver_runtime(self).is_some_and(|runtime| runtime_has_mutation_block(&lock_unpoisoned(&runtime))) {
return Err(ERR_TIER_BACKEND_IN_USE.clone());
}
self.ensure_generations_are_idle(self.tiers.keys())?;
if !force {
let tier_names = self.tiers.keys().cloned().collect::<Vec<_>>();
for tier_name in tier_names {
match self.get_driver(&tier_name).await?.in_use().await {
Ok(true) => return Err(ERR_TIER_BACKEND_NOT_EMPTY.clone()),
Ok(false) => {}
Err(err) => {
let mut admin_err = ERR_TIER_PERM_ERR.clone();
admin_err.message.push('.');
admin_err.message.push_str(&err.to_string());
return Err(admin_err);
}
}
}
}
self.tiers.clear();
self.revoke_all_drivers();
Ok(())
}
#[tracing::instrument(level = "debug", name = "tier_save", skip(self))]
pub async fn save(&self) -> std::result::Result<(), std::io::Error> {
let Some(api) = runtime_sources::object_store_handle() else {
return Err(tier_config_not_initialized_error("save tiering config"));
};
self.save_tiering_config(api).await
}
pub async fn save_tiering_config<S>(&self, api: Arc<S>) -> std::result::Result<(), std::io::Error>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let data = encode_external_tiering_config_blob(self)?;
let config_file = tier_config_path(TIER_CONFIG_FILE);
self.save_config(api, &config_file, data).await
}
async fn save_tiering_config_if_current<S>(
&self,
api: Arc<S>,
version: Option<&str>,
) -> std::result::Result<(), std::io::Error>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let data = encode_external_tiering_config_blob(self)?;
let config_file = tier_config_path(TIER_CONFIG_FILE);
let http_preconditions = match version {
Some(etag) => HTTPPreconditions {
if_match: Some(etag.to_string()),
..Default::default()
},
None => HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
},
};
self.save_config_with_opts(
api,
&config_file,
data,
&ObjectOptions {
max_parity: true,
http_preconditions: Some(http_preconditions),
..Default::default()
},
)
.await
}
pub async fn save_config<S>(&self, api: Arc<S>, file: &str, data: Bytes) -> std::result::Result<(), std::io::Error>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
self.save_config_with_opts(
api,
file,
data,
&ObjectOptions {
max_parity: true,
..Default::default()
},
)
.await
}
pub async fn save_config_with_opts<S>(
&self,
api: Arc<S>,
file: &str,
data: Bytes,
opts: &ObjectOptions,
) -> std::result::Result<(), std::io::Error>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
debug!("save tier config:{}", file);
let mut put_data = PutObjReader::from_vec(data.to_vec());
let _ = api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await?;
Ok(())
}
pub async fn refresh_tier_config(&mut self, api: Arc<ECStore>) {
let r = rand::rng().random_range(0.0..1.0);
let rand_interval = || Duration::from_secs((r * 60_f64).round() as u64);
let mut t = interval(TIER_CFG_REFRESH + rand_interval());
loop {
select! {
_ = t.tick() => {
if let Err(err) = self.reload(api.clone()).await {
info!("{}", err);
}
}
}
}
}
pub(crate) async fn refresh_tier_config_handle(handle: Arc<RwLock<Self>>, api: Arc<ECStore>) {
//let r = rand.New(rand.NewSource(time.Now().UnixNano()));
let r = rand::rng().random_range(0.0..1.0);
let rand_interval = || Duration::from_secs((r * 60_f64).round() as u64);
let refresh_interval = TIER_CFG_REFRESH + rand_interval();
let mut t = delayed_tier_refresh_interval(refresh_interval);
loop {
select! {
_ = t.tick() => {
if let Err(err) = Self::reload_handle(&handle, api.clone()).await {
info!("{}", err);
}
}
else => ()
}
t.reset();
}
}
pub async fn init(&mut self, api: Arc<ECStore>) -> Result<()> {
self.reload(api).await?;
//if globalIsDistErasure {
// self.refresh_tier_config(api).await;
//}
Ok(())
}
}
async fn new_and_save_tiering_config<S>(api: Arc<S>) -> Result<TierConfigMgr>
where
S: EcstoreObjectIO,
{
let mut cfg = TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
};
//lookup_configs(&mut cfg, api.clone()).await;
cfg.save_tiering_config(api).await?;
Ok(cfg)
}
#[tracing::instrument(level = "debug", name = "load_tier_config", skip(api))]
async fn load_tier_config(api: Arc<ECStore>) -> std::result::Result<TierConfigMgr, std::io::Error> {
let config_file = tier_config_path(TIER_CONFIG_FILE);
match read_config(api.clone(), config_file.as_str()).await {
Ok(data) => decode_tiering_config_blob(&data),
Err(err) if is_err_config_not_found(&err) => {
let legacy_file = tier_config_path(TIER_CONFIG_LEGACY_FILE);
match read_config(api.clone(), legacy_file.as_str()).await {
Ok(data) => {
let cfg = TierConfigMgr::unmarshal(&data)?;
let normalized = encode_external_tiering_config_blob(&cfg)?;
let mut put_data = PutObjReader::from_vec(normalized.to_vec());
let _ = api
.put_object(
RUSTFS_META_BUCKET,
&config_file,
&mut put_data,
&ObjectOptions {
max_parity: true,
..Default::default()
},
)
.await;
Ok(cfg)
}
Err(legacy_err) if is_err_config_not_found(&legacy_err) => {
warn!("config not found, start to init");
if runtime_sources::first_cluster_node_is_local().await {
new_and_save_tiering_config(api).await.map_err(io::Error::other)
} else {
Ok(TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
})
}
}
Err(legacy_err) => Err(io::Error::other(legacy_err)),
}
}
Err(err) => {
error!("read config err {:?}", &err);
Err(io::Error::other(err))
}
}
}
async fn load_tier_config_for_update<S>(api: Arc<S>) -> std::result::Result<(TierConfigMgr, Option<String>), std::io::Error>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let config_file = tier_config_path(TIER_CONFIG_FILE);
match read_config_with_metadata(api.clone(), &config_file, &ObjectOptions::default()).await {
Ok((data, object_info)) => {
let etag = object_info
.etag
.filter(|etag| !etag.trim().is_empty())
.ok_or_else(|| io::Error::other("tier configuration object is missing an ETag"))?;
Ok((decode_tiering_config_blob(&data)?, Some(etag)))
}
Err(err) if is_err_config_not_found(&err) => {
let legacy_file = tier_config_path(TIER_CONFIG_LEGACY_FILE);
match read_config(api, &legacy_file).await {
Ok(data) => Ok((TierConfigMgr::unmarshal(&data)?, None)),
Err(legacy_err) if is_err_config_not_found(&legacy_err) => Ok((
TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
},
None,
)),
Err(legacy_err) => Err(io::Error::other(legacy_err)),
}
}
Err(err) => Err(io::Error::other(err)),
}
}
async fn read_tier_config_from_bucket<S>(
api: Arc<S>,
bucket: &str,
path: &str,
opts: &ObjectOptions,
) -> io::Result<Option<Vec<u8>>>
where
S: EcstoreObjectIO,
{
let mut rd = match api.get_object_reader(bucket, path, None, HeaderMap::new(), opts).await {
Ok(v) => v,
Err(err) if is_err_config_not_found(&err) => return Ok(None),
Err(err) => return Err(io::Error::other(err)),
};
let data = rd.read_all().await.map_err(io::Error::other)?;
if data.is_empty() {
return Ok(None);
}
Ok(Some(data))
}
async fn write_tier_config_to_rustfs<S>(api: Arc<S>, path: &str, data: Bytes) -> io::Result<()>
where
S: EcstoreObjectIO,
{
let mut put_data = PutObjReader::from_vec(data.to_vec());
api.put_object(
RUSTFS_META_BUCKET,
path,
&mut put_data,
&ObjectOptions {
max_parity: true,
..Default::default()
},
)
.await
.map_err(io::Error::other)?;
Ok(())
}
pub async fn try_migrate_tiering_config<S>(api: Arc<S>)
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
>,
{
let target_path = tier_config_path(TIER_CONFIG_FILE);
if api
.get_object_info(
RUSTFS_META_BUCKET,
&target_path,
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await
.is_ok()
{
debug!("tier config already exists in RustFS metadata bucket, skip migration");
return;
}
let opts = ObjectOptions {
max_parity: true,
no_lock: true,
..Default::default()
};
let legacy_path = tier_config_path(TIER_CONFIG_LEGACY_FILE);
match read_tier_config_from_bucket(api.clone(), RUSTFS_META_BUCKET, &legacy_path, &opts).await {
Ok(Some(data)) => match TierConfigMgr::unmarshal(&data)
.and_then(|cfg| encode_external_tiering_config_blob(&cfg).map_err(io::Error::other))
{
Ok(out) => {
if write_tier_config_to_rustfs(api.clone(), &target_path, out).await.is_ok() {
info!("Migrated tier config from legacy RustFS metadata format");
return;
}
}
Err(err) => debug!(
bucket = RUSTFS_META_BUCKET,
path = %legacy_path,
error = %err,
"Skipping incompatible legacy tier config migration"
),
},
Ok(None) => {}
Err(err) => debug!(
bucket = RUSTFS_META_BUCKET,
path = %legacy_path,
error = %err,
"Skipping legacy tier config migration after read failure"
),
}
match read_tier_config_from_bucket(api.clone(), MIGRATING_META_BUCKET, &target_path, &opts).await {
Ok(Some(data)) => match decode_tiering_config_blob(&data).and_then(|cfg| encode_external_tiering_config_blob(&cfg)) {
Ok(out) => {
if write_tier_config_to_rustfs(api.clone(), &target_path, out).await.is_ok() {
info!("Migrated compatible tier config from migrating metadata bucket");
}
}
Err(err) => debug!(
bucket = MIGRATING_META_BUCKET,
path = %target_path,
error = %err,
"Skipping incompatible migrating tier config"
),
},
Ok(None) => {}
Err(err) => debug!(
bucket = MIGRATING_META_BUCKET,
path = %target_path,
error = %err,
"Skipping migrating tier config after read failure"
),
}
}
pub fn is_err_config_not_found(err: &StorageError) -> bool {
matches!(err, StorageError::ObjectNotFound(_, _) | StorageError::BucketNotFound(_)) || err == &StorageError::ConfigNotFound
}
fn tier_config_not_initialized_error(operation: &str) -> std::io::Error {
std::io::Error::other(format!("failed to {operation}: object layer not initialized"))
}
#[cfg(test)]
mod tests {
use super::*;
fn build_s3_tier(name: &str) -> TierConfig {
TierConfig {
version: "v1".to_string(),
tier_type: TierType::S3,
name: name.to_string(),
s3: Some(crate::services::tier::tier_config::TierS3 {
name: name.to_string(),
endpoint: "https://example-s3.invalid".to_string(),
access_key: "ak".to_string(),
secret_key: "sk".to_string(),
bucket: "bucket-a".to_string(),
prefix: "prefix-a".to_string(),
region: "us-east-1".to_string(),
storage_class: "STANDARD".to_string(),
aws_role: false,
aws_role_web_identity_token_file: String::new(),
aws_role_arn: String::new(),
aws_role_session_name: String::new(),
aws_role_duration_seconds: 0,
}),
..Default::default()
}
}
fn build_wasabi_tier(name: &str) -> TierConfig {
TierConfig {
version: "v1".to_string(),
tier_type: TierType::Wasabi,
name: name.to_string(),
wasabi: Some(TierWasabi {
name: name.to_string(),
endpoint: "https://s3.ap-northeast-1.wasabisys.com".to_string(),
access_key: "ak".to_string(),
secret_key: "sk".to_string(),
bucket: "wasabi-bucket".to_string(),
prefix: "archive".to_string(),
region: "ap-northeast-1".to_string(),
}),
..Default::default()
}
}
#[test]
fn test_tiering_external_blob_roundtrip_for_standard_type() {
let mut cfg = TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
};
cfg.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A"));
let bytes = encode_external_tiering_config_blob(&cfg).expect("encode should succeed");
assert_eq!(&bytes[0..2], &TIER_CONFIG_FORMAT.to_le_bytes());
assert_eq!(&bytes[2..4], &TIER_CONFIG_VERSION.to_le_bytes());
let decoded = decode_external_tiering_config_blob(&bytes).expect("decode should succeed");
let tier = decoded.tiers.get("COLD-A").expect("tier should exist");
assert_eq!(tier.tier_type.as_lowercase(), "s3");
assert_eq!(tier.s3.as_ref().expect("s3 should exist").endpoint, "https://example-s3.invalid");
}
#[test]
fn test_tiering_external_blob_roundtrip_for_extended_type_hint() {
let mut cfg = TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
};
cfg.tiers.insert(
"COLD-B".to_string(),
TierConfig {
version: "v1".to_string(),
tier_type: TierType::RustFS,
name: "COLD-B".to_string(),
rustfs: Some(crate::services::tier::tier_config::TierRustFS {
name: "COLD-B".to_string(),
endpoint: "https://example-compat.invalid".to_string(),
access_key: "ak".to_string(),
secret_key: "sk".to_string(),
bucket: "bucket-b".to_string(),
prefix: "prefix-b".to_string(),
region: "us-east-1".to_string(),
storage_class: "STANDARD".to_string(),
}),
..Default::default()
},
);
let bytes = encode_external_tiering_config_blob(&cfg).expect("encode should succeed");
let decoded = decode_external_tiering_config_blob(&bytes).expect("decode should succeed");
let tier = decoded.tiers.get("COLD-B").expect("tier should exist");
assert_eq!(tier.tier_type.as_lowercase(), "rustfs");
assert_eq!(
tier.rustfs.as_ref().expect("backend should exist").endpoint,
"https://example-compat.invalid"
);
}
#[tokio::test]
async fn wasabi_external_blob_survives_old_node_rewrite_and_fences_old_drivers() {
const WASABI_V2_GOLDEN: &[u8] = &[
1, 0, 2, 0, 145, 129, 171, 67, 79, 76, 68, 45, 87, 65, 83, 65, 66, 73, 152, 176, 114, 117, 115, 116, 102, 115, 45,
119, 97, 115, 97, 98, 105, 45, 118, 49, 1, 171, 67, 79, 76, 68, 45, 87, 65, 83, 65, 66, 73, 156, 177, 114, 117, 115,
116, 102, 115, 45, 119, 97, 115, 97, 98, 105, 45, 118, 49, 58, 162, 97, 107, 162, 115, 107, 173, 119, 97, 115, 97,
98, 105, 45, 98, 117, 99, 107, 101, 116, 167, 97, 114, 99, 104, 105, 118, 101, 174, 97, 112, 45, 110, 111, 114, 116,
104, 101, 97, 115, 116, 45, 49, 160, 194, 160, 160, 160, 0, 192, 192, 192, 166, 119, 97, 115, 97, 98, 105,
];
let mut cfg = empty_mgr();
let mut tier = build_wasabi_tier("COLD-WASABI");
tier.wasabi.as_mut().expect("Wasabi payload should exist").endpoint.clear();
cfg.tiers.insert("COLD-WASABI".to_string(), tier);
let bytes = encode_external_tiering_config_blob(&cfg).expect("Wasabi tier should encode");
assert_eq!(bytes.as_ref(), WASABI_V2_GOLDEN, "Wasabi v2 bytes must remain stable");
assert_eq!(&bytes[0..2], &TIER_CONFIG_FORMAT.to_le_bytes());
assert_eq!(&bytes[2..4], &TIER_CONFIG_VERSION.to_le_bytes());
let external: ExternalTierConfigMgr = rmp_serde::from_slice(&bytes[4..]).expect("external Wasabi payload should decode");
let stored = external.tiers.get("COLD-WASABI").expect("stored Wasabi tier should exist");
assert_eq!(stored.version, EXTERNAL_TIER_VERSION_WASABI_V1);
assert_eq!(stored.tier_type, EXTERNAL_TIER_TYPE_S3);
assert_eq!(stored.tier_type_hint.as_deref(), Some("wasabi"));
assert!(stored.compatible_backend.is_none());
let s3 = stored.s3.as_ref().expect("Wasabi should use the S3 payload");
assert_eq!(s3.endpoint, EXTERNAL_TIER_WASABI_ENDPOINT_SENTINEL);
assert_eq!(s3.bucket, "wasabi-bucket");
assert_eq!(s3.prefix, "archive");
assert_eq!(s3.region, "ap-northeast-1");
assert!(s3.storage_class.is_empty());
let decoded = decode_external_tiering_config_blob(&bytes).expect("Wasabi tier should decode");
let wasabi = decoded.tiers["COLD-WASABI"]
.wasabi
.as_ref()
.expect("Wasabi payload should survive roundtrip");
assert_eq!(wasabi.endpoint, "https://s3.ap-northeast-1.wasabisys.com");
assert_eq!(wasabi.secret_key, "sk");
assert_eq!(decoded.tiers["COLD-WASABI"].version, EXTERNAL_TIER_VERSION_WASABI_V1);
let encode_fixture = |external: &ExternalTierConfigMgr| {
let payload = rmp_serde::to_vec(external).expect("Wasabi fixture should encode");
let mut fixture = Vec::with_capacity(4 + payload.len());
fixture.extend_from_slice(&TIER_CONFIG_FORMAT.to_le_bytes());
fixture.extend_from_slice(&TIER_CONFIG_VERSION.to_le_bytes());
fixture.extend_from_slice(&payload);
fixture
};
// Current older nodes preserve the per-tier Version and S3 payload when
// rewriting the binary document, but drop an unrecognized XTierType.
let mut old_node_rewrite = external.clone();
old_node_rewrite
.tiers
.get_mut("COLD-WASABI")
.expect("stored Wasabi tier should exist")
.tier_type_hint = None;
let rewritten = encode_fixture(&old_node_rewrite);
let decoded = decode_external_tiering_config_blob(&rewritten)
.expect("the durable per-tier marker must restore Wasabi after an old-node rewrite");
assert!(matches!(decoded.tiers["COLD-WASABI"].tier_type, TierType::Wasabi));
let old_s3_payload = old_node_rewrite.tiers["COLD-WASABI"]
.s3
.as_ref()
.expect("old nodes should retain the physical S3 payload");
let old_s3 = crate::services::tier::tier_config::TierS3 {
name: "COLD-WASABI".to_string(),
endpoint: old_s3_payload.endpoint.clone(),
access_key: old_s3_payload.access_key.clone(),
secret_key: old_s3_payload.secret_key.clone(),
bucket: old_s3_payload.bucket.clone(),
prefix: old_s3_payload.prefix.clone(),
region: old_s3_payload.region.clone(),
..Default::default()
};
let err = match crate::services::tier::warm_backend_s3::WarmBackendS3::new(&old_s3, "COLD-WASABI").await {
Ok(_) => panic!("an older generic S3 driver must not operate a Wasabi compatibility envelope"),
Err(err) => err,
};
assert!(err.to_string().contains("missing a host"), "{err}");
let mut generic_s3 = old_node_rewrite.clone();
generic_s3
.tiers
.get_mut("COLD-WASABI")
.expect("stored Wasabi tier should exist")
.version = "v1".to_string();
let decoded = decode_external_tiering_config_blob(&encode_fixture(&generic_s3))
.expect("a generic S3 tier without an explicit marker must stay generic");
assert!(matches!(decoded.tiers["COLD-WASABI"].tier_type, TierType::S3));
let mut hint_only = external.clone();
hint_only
.tiers
.get_mut("COLD-WASABI")
.expect("stored Wasabi tier should exist")
.version = "v1".to_string();
let err = expect_decode_err(&encode_fixture(&hint_only));
assert!(err.to_string().contains("inconsistent Wasabi type discriminators"), "{err}");
let mut wrong_type = old_node_rewrite.clone();
wrong_type
.tiers
.get_mut("COLD-WASABI")
.expect("stored Wasabi tier should exist")
.tier_type = EXTERNAL_TIER_TYPE_MINIO;
let err = expect_decode_err(&encode_fixture(&wrong_type));
assert!(err.to_string().contains("inconsistent Wasabi type discriminators"), "{err}");
let mut wrong_hint = external.clone();
wrong_hint
.tiers
.get_mut("COLD-WASABI")
.expect("stored Wasabi tier should exist")
.tier_type_hint = Some("rustfs".to_string());
let err = expect_decode_err(&encode_fixture(&wrong_hint));
assert!(err.to_string().contains("inconsistent Wasabi type discriminators"), "{err}");
let poison_fields: [(&str, fn(&mut ExternalTierS3)); 6] = [
("storage_class", |s3| s3.storage_class = "GLACIER".to_string()),
("aws_role", |s3| s3.aws_role = true),
("web_identity_token", |s3| s3.aws_role_web_identity_token_file = "/tmp/token".to_string()),
("role_arn", |s3| s3.aws_role_arn = "arn:aws:iam::1:role/test".to_string()),
("role_session", |s3| s3.aws_role_session_name = "session".to_string()),
("role_duration", |s3| s3.aws_role_duration_seconds = 900),
];
for (field, poison) in poison_fields {
let mut unsupported_credentials = external.clone();
let s3 = unsupported_credentials
.tiers
.get_mut("COLD-WASABI")
.expect("stored Wasabi tier should exist")
.s3
.as_mut()
.expect("Wasabi S3 payload should exist");
poison(s3);
let err = expect_decode_err(&encode_fixture(&unsupported_credentials));
assert!(
err.to_string().contains("unsupported Wasabi S3 credential fields"),
"field {field}: {err}"
);
}
}
#[test]
fn legacy_extended_type_hint_keeps_precedence_over_numeric_type() {
let mut external = ExternalTierConfigMgr::default();
external.tiers.insert(
"COLD-LEGACY".to_string(),
ExternalTierConfig {
version: "v1".to_string(),
tier_type: EXTERNAL_TIER_TYPE_MINIO,
name: "COLD-LEGACY".to_string(),
tier_type_hint: Some("rustfs".to_string()),
s3: Some(ExternalTierS3 {
endpoint: "https://rustfs.example.invalid".to_string(),
access_key: "ak".to_string(),
secret_key: "sk".to_string(),
bucket: "archive".to_string(),
prefix: "objects".to_string(),
region: "us-east-1".to_string(),
..Default::default()
}),
compatible_backend: Some(ExternalTierCompatible {
endpoint: "https://minio.example.invalid".to_string(),
bucket: "different".to_string(),
..Default::default()
}),
..Default::default()
},
);
let payload = rmp_serde::to_vec(&external).expect("legacy fixture should encode");
let mut blob = Vec::with_capacity(4 + payload.len());
blob.extend_from_slice(&TIER_CONFIG_FORMAT.to_le_bytes());
blob.extend_from_slice(&TIER_CONFIG_VERSION.to_le_bytes());
blob.extend_from_slice(&payload);
let decoded = decode_external_tiering_config_blob(&blob).expect("legacy extended hint should decode as before");
let rustfs = decoded.tiers["COLD-LEGACY"]
.rustfs
.as_ref()
.expect("recognized legacy hint must override the numeric type");
assert_eq!(rustfs.endpoint, "https://rustfs.example.invalid");
assert_eq!(rustfs.bucket, "archive");
}
#[test]
fn wasabi_external_blob_rejects_noncanonical_endpoint_and_unsafe_region() {
let mut cfg = empty_mgr();
cfg.tiers.insert("COLD-WASABI".to_string(), build_wasabi_tier("COLD-WASABI"));
let bytes = encode_external_tiering_config_blob(&cfg).expect("Wasabi tier should encode");
let base: ExternalTierConfigMgr = rmp_serde::from_slice(&bytes[4..]).expect("external Wasabi payload should decode");
for (endpoint, region, expected) in [
("https://s3.example.invalid", "ap-northeast-1", "invalid Wasabi compatibility endpoint"),
(EXTERNAL_TIER_WASABI_ENDPOINT_SENTINEL, "ap.northeast-1", "invalid Wasabi region"),
] {
let mut external = base.clone();
let s3 = external
.tiers
.get_mut("COLD-WASABI")
.and_then(|tier| tier.s3.as_mut())
.expect("stored Wasabi S3 payload should exist");
s3.endpoint = endpoint.to_string();
s3.region = region.to_string();
let payload = rmp_serde::to_vec(&external).expect("corrupt fixture should encode");
let mut corrupt = Vec::with_capacity(4 + payload.len());
corrupt.extend_from_slice(&TIER_CONFIG_FORMAT.to_le_bytes());
corrupt.extend_from_slice(&TIER_CONFIG_VERSION.to_le_bytes());
corrupt.extend_from_slice(&payload);
let err = expect_decode_err(&corrupt);
assert!(err.to_string().contains(expected), "{err}");
}
let mut external = base;
let stored = external
.tiers
.get_mut("COLD-WASABI")
.expect("stored Wasabi tier should exist");
let s3 = stored.s3.take().expect("stored Wasabi S3 payload should exist");
stored.compatible_backend = Some(external_tier_alias_from_compatible_payload(
s3.endpoint,
s3.access_key,
s3.secret_key,
s3.bucket,
s3.prefix,
s3.region,
));
let payload = rmp_serde::to_vec(&external).expect("wrong-payload fixture should encode");
let mut corrupt = Vec::with_capacity(4 + payload.len());
corrupt.extend_from_slice(&TIER_CONFIG_FORMAT.to_le_bytes());
corrupt.extend_from_slice(&TIER_CONFIG_VERSION.to_le_bytes());
corrupt.extend_from_slice(&payload);
let err = expect_decode_err(&corrupt);
assert!(err.to_string().contains("missing Wasabi S3 payload"), "{err}");
}
#[test]
fn test_decode_tiering_config_blob_accepts_legacy_json() {
let mut cfg = TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
};
cfg.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A"));
let data = serde_json::to_vec(&cfg).expect("legacy json should encode");
let decoded = decode_tiering_config_blob(&data).expect("legacy json should decode");
assert_eq!(
decoded
.tiers
.get("COLD-A")
.and_then(|tier| tier.s3.as_ref())
.map(|s3| s3.bucket.as_str()),
Some("bucket-a")
);
}
#[test]
fn wasabi_legacy_json_rejects_missing_payload_and_invalid_endpoints() {
let mut cfg = empty_mgr();
let mut tier = build_wasabi_tier("COLD-WASABI");
tier.wasabi.as_mut().expect("Wasabi payload should exist").endpoint = "https://s3.example.invalid".to_string();
cfg.tiers.insert("COLD-WASABI".to_string(), tier);
let data = serde_json::to_vec(&cfg).expect("legacy JSON fixture should encode");
let err = match TierConfigMgr::unmarshal(&data) {
Ok(_) => panic!("legacy Wasabi endpoint mismatch must fail closed"),
Err(err) => err,
};
assert!(err.to_string().contains("https://s3.ap-northeast-1.wasabisys.com"));
cfg.tiers
.get_mut("COLD-WASABI")
.and_then(|tier| tier.wasabi.as_mut())
.expect("Wasabi payload should exist")
.endpoint
.clear();
let data = serde_json::to_vec(&cfg).expect("empty-endpoint fixture should encode");
let err = match TierConfigMgr::unmarshal(&data) {
Ok(_) => panic!("legacy Wasabi endpoint must not be empty"),
Err(err) => err,
};
assert!(err.to_string().contains("missing Wasabi endpoint"), "{err}");
cfg.tiers.get_mut("COLD-WASABI").expect("Wasabi tier should exist").wasabi = None;
let data = serde_json::to_vec(&cfg).expect("missing-payload fixture should encode");
let err = match TierConfigMgr::unmarshal(&data) {
Ok(_) => panic!("legacy Wasabi payload must exist"),
Err(err) => err,
};
assert!(err.to_string().contains("missing Wasabi backend payload"), "{err}");
}
#[test]
fn test_tier_config_not_initialized_error_formats_operation_context() {
let err = tier_config_not_initialized_error("save tiering config");
let rendered = err.to_string();
assert!(rendered.contains("failed to save tiering config"), "{rendered}");
assert!(rendered.contains("object layer not initialized"), "{rendered}");
}
// ---------------------------------------------------------------------
// State-machine coverage (ilm-4): add / edit / remove / verify.
//
// These tests must not reach a real remote tier. Two techniques keep them
// hermetic:
// * error paths that return *before* `new_warm_backend` constructs a
// client (name validation, duplicate detection, unsupported type,
// missing backend payload, missing credentials);
// * a `MockWarmBackend` injected directly into `driver_cache`, so
// `get_driver` returns it from cache without any network I/O, which
// lets us drive `remove`/`verify` through every branch.
// ---------------------------------------------------------------------
use crate::client::transition_api::{ReadCloser, ReaderImpl};
use crate::services::tier::warm_backend::{WarmBackend, WarmBackendGetOpts};
fn empty_mgr() -> TierConfigMgr {
TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
}
}
fn build_rustfs_tier(name: &str) -> TierConfig {
TierConfig {
version: "v1".to_string(),
tier_type: TierType::RustFS,
name: name.to_string(),
rustfs: Some(crate::services::tier::tier_config::TierRustFS {
name: name.to_string(),
endpoint: "https://example-compat.invalid".to_string(),
access_key: "ak".to_string(),
secret_key: "sk".to_string(),
bucket: "bucket-r".to_string(),
prefix: "prefix-r".to_string(),
region: "us-east-1".to_string(),
storage_class: "STANDARD".to_string(),
}),
..Default::default()
}
}
#[derive(Debug)]
struct LockingTierConfigStore {
locks: Mutex<Vec<(String, String)>>,
put_after_lock: AtomicBool,
lock_manager: Arc<rustfs_lock::GlobalLockManager>,
}
impl LockingTierConfigStore {
fn new() -> Self {
Self {
locks: Mutex::new(Vec::new()),
put_after_lock: AtomicBool::new(false),
lock_manager: Arc::new(rustfs_lock::GlobalLockManager::new()),
}
}
fn lock_events(&self) -> Vec<(String, String)> {
self.locks
.lock()
.expect("tier config lock recorder should not poison")
.clone()
}
}
#[async_trait::async_trait]
impl ObjectIO for LockingTierConfigStore {
type Error = Error;
type RangeSpec = HTTPRangeSpec;
type HeaderMap = HeaderMap;
type ObjectOptions = ObjectOptions;
type ObjectInfo = ObjectInfo;
type GetObjectReader = GetObjectReader;
type PutObjectReader = PutObjReader;
async fn get_object_reader(
&self,
bucket: &str,
object: &str,
_range: Option<HTTPRangeSpec>,
_h: HeaderMap,
_opts: &ObjectOptions,
) -> Result<GetObjectReader> {
Err(Error::ObjectNotFound(bucket.to_string(), object.to_string()))
}
async fn put_object(
&self,
bucket: &str,
object: &str,
_data: &mut PutObjReader,
opts: &ObjectOptions,
) -> Result<ObjectInfo> {
let expected_lock = (RUSTFS_META_BUCKET.to_string(), tier_config_lock_path());
let saw_expected_lock = self
.locks
.lock()
.expect("tier config lock recorder should not poison")
.contains(&expected_lock);
self.put_after_lock.store(saw_expected_lock, Ordering::SeqCst);
assert!(opts.max_parity, "tier config save must retain max parity");
assert!(
opts.http_preconditions
.as_ref()
.and_then(|preconditions| preconditions.if_none_match.as_deref())
== Some("*"),
"initial tier config save must retain if-none-match protection"
);
Ok(ObjectInfo {
bucket: bucket.to_string(),
name: object.to_string(),
etag: Some("saved-etag".to_string()),
..Default::default()
})
}
}
#[async_trait::async_trait]
impl BucketOperations for LockingTierConfigStore {
type Error = Error;
async fn make_bucket(
&self,
_bucket: &str,
_opts: &crate::storage_api_contracts::bucket::MakeBucketOptions,
) -> Result<()> {
Err(Error::NotImplemented)
}
async fn get_bucket_info(
&self,
_bucket: &str,
_opts: &crate::storage_api_contracts::bucket::BucketOptions,
) -> Result<crate::storage_api_contracts::bucket::BucketInfo> {
Err(Error::NotImplemented)
}
async fn list_bucket(
&self,
_opts: &crate::storage_api_contracts::bucket::BucketOptions,
) -> Result<Vec<crate::storage_api_contracts::bucket::BucketInfo>> {
Ok(Vec::new())
}
async fn delete_bucket(
&self,
_bucket: &str,
_opts: &crate::storage_api_contracts::bucket::DeleteBucketOptions,
) -> Result<()> {
Err(Error::NotImplemented)
}
}
#[async_trait::async_trait]
impl ListOperations for LockingTierConfigStore {
type Error = Error;
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = TierReferenceProofWalkOptions;
type WalkCancellation = tokio_util::sync::CancellationToken;
type WalkResultSender = tokio::sync::mpsc::Sender<StorageObjectInfoOrErr<ObjectInfo, Error>>;
async fn list_objects_v2(
self: Arc<Self>,
_bucket: &str,
_prefix: &str,
_continuation_token: Option<String>,
_delimiter: Option<String>,
_max_keys: i32,
_fetch_owner: bool,
_start_after: Option<String>,
_incl_deleted: bool,
) -> Result<Self::ListObjectsV2Info> {
Ok(StorageListObjectsV2Info::default())
}
async fn list_object_versions(
self: Arc<Self>,
_bucket: &str,
_prefix: &str,
_marker: Option<String>,
_version_marker: Option<String>,
_delimiter: Option<String>,
_max_keys: i32,
) -> Result<Self::ListObjectVersionsInfo> {
Ok(StorageListObjectVersionsInfo::default())
}
async fn walk(
self: Arc<Self>,
_rx: Self::WalkCancellation,
_bucket: &str,
_prefix: &str,
_result: Self::WalkResultSender,
_opts: Self::WalkOptions,
) -> Result<()> {
Err(Error::NotImplemented)
}
}
#[async_trait::async_trait]
impl NamespaceLocking for LockingTierConfigStore {
type Error = Error;
type NamespaceLock = rustfs_lock::NamespaceLockWrapper;
async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<rustfs_lock::NamespaceLockWrapper> {
self.locks
.lock()
.expect("tier config lock recorder should not poison")
.push((bucket.to_string(), object.to_string()));
Ok(rustfs_lock::NamespaceLockWrapper::new(
rustfs_lock::NamespaceLock::with_local_manager("tier-config-test".to_string(), self.lock_manager.clone()),
rustfs_lock::ObjectKey {
bucket: Arc::from(bucket),
object: Arc::from(object),
version: None,
},
"tier-config-test-owner".to_string(),
))
}
}
#[tokio::test]
async fn tier_config_update_path_acquires_meta_namespace_sidecar_lock_before_save() {
let manager = TierConfigMgr::new();
let store = Arc::new(LockingTierConfigStore::new());
TierConfigMgr::update_candidate_with_config_lock(&manager, store.clone(), TierCandidateMutation::Clear(true))
.await
.expect("empty clear should still acquire and save through the coordinator lock");
assert_eq!(store.lock_events(), vec![(RUSTFS_META_BUCKET.to_string(), tier_config_lock_path())]);
assert!(
store.put_after_lock.load(Ordering::SeqCst),
"tier config save must happen after the coordinator namespace lock is acquired"
);
}
#[test]
fn tier_mutation_targets_prove_remove_and_rebind_authoritative_identity() {
let mut current = empty_mgr();
current.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
let mut rebound = empty_mgr();
let mut replacement = build_rustfs_tier("COLD-A");
replacement.rustfs.as_mut().expect("replacement payload should exist").prefix = "new-prefix".to_string();
rebound.tiers.insert("COLD-A".to_string(), replacement);
let edit_targets = TierCandidateMutation::Edit("COLD-A".to_string(), TierCreds::default())
.affected_targets(&current, &rebound)
.expect("rebind target proof should build");
assert_eq!(edit_targets.len(), 1);
assert_eq!(edit_targets[0].tier_name, "COLD-A");
assert!(edit_targets[0].old_backend_identity.is_some());
assert!(edit_targets[0].new_backend_identity.is_some());
assert_ne!(edit_targets[0].old_backend_identity, edit_targets[0].new_backend_identity);
let removed = empty_mgr();
let remove_targets = TierCandidateMutation::Remove("COLD-A".to_string(), true)
.affected_targets(&current, &removed)
.expect("remove target proof should build");
assert_eq!(remove_targets.len(), 1);
assert_eq!(remove_targets[0].tier_name, "COLD-A");
assert!(remove_targets[0].old_backend_identity.is_some());
assert!(remove_targets[0].new_backend_identity.is_none());
let noop_targets = TierCandidateMutation::Remove("MISSING".to_string(), true)
.affected_targets(&current, &current)
.expect("idempotent remove of a missing tier should not require an identity proof");
assert!(noop_targets.is_empty());
}
/// A fully offline `WarmBackend` used to exercise the driver-facing
/// branches of `remove`/`verify` without touching a remote tier.
struct MockWarmBackend {
/// `Some(b)` -> `in_use()` returns `Ok(b)`; `None` -> returns `Err`.
in_use_value: Option<bool>,
/// When false, put/get/remove all fail (drives `verify` error paths).
healthy: bool,
}
#[async_trait::async_trait]
impl WarmBackend for MockWarmBackend {
async fn validate(&self) -> std::result::Result<(), std::io::Error> {
if self.healthy {
Ok(())
} else {
Err(std::io::Error::other("mock validation failed"))
}
}
fn validate_remote_version_id(&self, remote_version_id: &str) -> std::result::Result<(), std::io::Error> {
if remote_version_id == "unsupported-version" {
Err(std::io::Error::other("mock remote version rejected"))
} else {
Ok(())
}
}
async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> std::result::Result<String, std::io::Error> {
if self.healthy {
Ok("mock-version".to_string())
} else {
Err(std::io::Error::other("mock put failed"))
}
}
async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
_meta: HashMap<String, String>,
) -> std::result::Result<String, std::io::Error> {
self.put(object, r, length).await
}
async fn get(
&self,
_object: &str,
_rv: &str,
_opts: WarmBackendGetOpts,
) -> std::result::Result<ReadCloser, std::io::Error> {
if self.healthy {
Ok(BufReader::new(Cursor::new(b"RustFS".to_vec())))
} else {
Err(std::io::Error::other("mock get failed"))
}
}
async fn remove(&self, _object: &str, _rv: &str) -> std::result::Result<(), std::io::Error> {
if self.healthy {
Ok(())
} else {
Err(std::io::Error::other("mock remove failed"))
}
}
async fn remove_exact(&self, object: &str, rv: &str) -> std::result::Result<(), std::io::Error> {
if rv == "exact-only" {
return Err(std::io::Error::other("mock exact remove forwarded"));
}
self.remove(object, rv).await
}
async fn in_use(&self) -> std::result::Result<bool, std::io::Error> {
match self.in_use_value {
Some(b) => Ok(b),
None => Err(std::io::Error::other("mock in_use failed")),
}
}
}
fn inject_mock(mgr: &mut TierConfigMgr, name: &str, tier: TierConfig, mock: MockWarmBackend) {
mgr.tiers.insert(name.to_string(), tier);
mgr.driver_cache.insert(name.to_string(), Box::new(mock));
}
// ---- add ------------------------------------------------------------
#[tokio::test]
async fn test_add_rejects_non_uppercase_name() {
let mut mgr = empty_mgr();
let err = mgr
.add(build_s3_tier("cold-a"), true)
.await
.expect_err("lowercase tier name must be rejected");
assert_eq!(err.code, ERR_TIER_NAME_NOT_UPPERCASE.code);
assert!(mgr.tiers.is_empty(), "rejected tier must not be persisted");
}
#[tokio::test]
async fn test_add_rejects_duplicate_name() {
let mut mgr = empty_mgr();
mgr.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A"));
let err = mgr
.add(build_s3_tier("COLD-A"), true)
.await
.expect_err("duplicate tier name must be rejected");
assert_eq!(err.code, ERR_TIER_ALREADY_EXISTS.code);
}
#[tokio::test]
async fn test_add_rejects_unsupported_tier_type() {
let mut mgr = empty_mgr();
// `Unsupported` is rejected by `new_warm_backend` before any network I/O.
let tier = TierConfig {
version: "v1".to_string(),
tier_type: TierType::Unsupported,
name: "COLD-U".to_string(),
..Default::default()
};
let err = mgr.add(tier, true).await.expect_err("unsupported tier type must be rejected");
assert_eq!(err.code, ERR_TIER_TYPE_UNSUPPORTED.code);
assert!(mgr.tiers.is_empty());
}
#[tokio::test]
async fn test_add_rejects_missing_backend_payload() {
let mut mgr = empty_mgr();
// Declares S3 type but omits the S3 payload; rejected before client build.
let tier = TierConfig {
version: "v1".to_string(),
tier_type: TierType::S3,
name: "COLD-A".to_string(),
s3: None,
..Default::default()
};
let err = mgr
.add(tier, true)
.await
.expect_err("missing backend payload must be rejected");
assert_eq!(err.code, "XRustFSAdminTierInvalidConfig");
assert!(err.message.contains("S3 tier configuration not found"), "{}", err.message);
}
#[tokio::test]
async fn test_add_rejects_custom_wasabi_endpoint_before_backend_setup() {
let mut mgr = empty_mgr();
let mut tier = build_wasabi_tier("COLD-WASABI");
tier.wasabi.as_mut().expect("Wasabi payload should exist").endpoint = "https://s3.example.invalid".to_string();
let err = mgr
.add(tier, true)
.await
.expect_err("custom Wasabi endpoint must fail before backend setup");
assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
assert_eq!(err.message, "Wasabi endpoint must be https://s3.ap-northeast-1.wasabisys.com");
assert!(mgr.tiers.is_empty());
}
#[tokio::test]
async fn test_add_rejects_reserved_names() {
// Supersedes the former `test_add_does_not_reserve_standard_name_regression_anchor`
// (added by PR #4713 to pin the pre-fix behavior). RustFS now rejects the
// AWS/MinIO reserved storage-class names STANDARD and REDUCED_REDUNDANCY
// (RRS) as remote-tier names, matching MinIO parity.
//
// `tier_type` is deliberately `Unsupported`: before the guard existed,
// "STANDARD" reached `new_warm_backend` and failed with
// `ERR_TIER_TYPE_UNSUPPORTED`. Asserting `ERR_TIER_RESERVED_NAME` here
// proves the reserved-name guard fires *before* the type/backend check.
for reserved in [crate::config::storageclass::STANDARD, crate::config::storageclass::RRS] {
let mut mgr = empty_mgr();
let tier = TierConfig {
version: "v1".to_string(),
tier_type: TierType::Unsupported,
name: reserved.to_string(),
..Default::default()
};
let err = mgr.add(tier, true).await.expect_err("reserved tier name must be rejected");
assert_eq!(
err.code, ERR_TIER_RESERVED_NAME.code,
"reserved tier name {reserved} must be rejected before the type check"
);
assert!(mgr.tiers.is_empty(), "rejected reserved tier {reserved} must not be persisted");
}
}
// ---- edit -----------------------------------------------------------
#[tokio::test]
async fn test_edit_rejects_unknown_tier() {
let mut mgr = empty_mgr();
let err = mgr
.edit("NOPE", TierCreds::default())
.await
.expect_err("editing an unknown tier must fail");
assert_eq!(err.code, ERR_TIER_NOT_FOUND.code);
}
#[tokio::test]
async fn test_edit_rejects_missing_credentials_for_rustfs() {
let mut mgr = empty_mgr();
mgr.tiers.insert("COLD-R".to_string(), build_rustfs_tier("COLD-R"));
// Empty access/secret keys => rejected before any driver rebuild.
let err = mgr
.edit("COLD-R", TierCreds::default())
.await
.expect_err("empty credentials must be rejected");
assert_eq!(err.code, ERR_TIER_MISSING_CREDENTIALS.code);
}
#[tokio::test]
async fn test_edit_rejects_missing_credentials_for_wasabi() {
let mut mgr = empty_mgr();
mgr.tiers.insert("COLD-WASABI".to_string(), build_wasabi_tier("COLD-WASABI"));
let err = mgr
.edit("COLD-WASABI", TierCreds::default())
.await
.expect_err("empty Wasabi credentials must be rejected before backend setup");
assert_eq!(err.code, ERR_TIER_MISSING_CREDENTIALS.code);
}
#[tokio::test]
async fn test_edit_rejects_missing_credentials_for_minio() {
let mut mgr = empty_mgr();
let tier = TierConfig {
version: "v1".to_string(),
tier_type: TierType::MinIO,
name: "COLD-M".to_string(),
minio: Some(crate::services::tier::tier_config::TierMinIO {
name: "COLD-M".to_string(),
endpoint: "https://example-compat.invalid".to_string(),
access_key: "ak".to_string(),
secret_key: "sk".to_string(),
bucket: "bucket-m".to_string(),
prefix: String::new(),
region: String::new(),
}),
..Default::default()
};
mgr.tiers.insert("COLD-M".to_string(), tier);
let err = mgr
.edit("COLD-M", TierCreds::default())
.await
.expect_err("empty credentials must be rejected");
assert_eq!(err.code, ERR_TIER_MISSING_CREDENTIALS.code);
}
// ---- remove ---------------------------------------------------------
#[tokio::test]
async fn test_remove_unknown_tier_is_idempotent() {
let mut mgr = empty_mgr();
// Unknown tier: get_driver returns NotFound, which `remove` swallows.
mgr.remove("NOPE", false).await.expect("removing an unknown tier is a no-op");
}
#[tokio::test]
async fn test_remove_rejects_backend_in_use() {
let mut mgr = empty_mgr();
inject_mock(
&mut mgr,
"COLD-A",
build_s3_tier("COLD-A"),
MockWarmBackend {
in_use_value: Some(true),
healthy: true,
},
);
let err = mgr
.remove("COLD-A", false)
.await
.expect_err("in-use backend must not be removed");
assert_eq!(err.code, ERR_TIER_BACKEND_NOT_EMPTY.code);
assert!(mgr.tiers.contains_key("COLD-A"), "tier must survive a rejected remove");
assert!(mgr.driver_cache.contains_key("COLD-A"));
}
#[tokio::test]
async fn test_remove_succeeds_when_backend_empty() {
let mut mgr = empty_mgr();
inject_mock(
&mut mgr,
"COLD-A",
build_s3_tier("COLD-A"),
MockWarmBackend {
in_use_value: Some(false),
healthy: true,
},
);
mgr.remove("COLD-A", false).await.expect("empty backend can be removed");
assert!(!mgr.tiers.contains_key("COLD-A"));
assert!(!mgr.driver_cache.contains_key("COLD-A"));
}
#[tokio::test]
async fn test_remove_force_skips_in_use_probe() {
let mut mgr = empty_mgr();
// in_use_value: None would error if probed; force=true must skip it.
inject_mock(
&mut mgr,
"COLD-A",
build_s3_tier("COLD-A"),
MockWarmBackend {
in_use_value: None,
healthy: true,
},
);
mgr.remove("COLD-A", true).await.expect("force remove must not probe in_use");
assert!(!mgr.tiers.contains_key("COLD-A"));
assert!(!mgr.driver_cache.contains_key("COLD-A"));
}
#[tokio::test]
async fn persisted_remove_preserves_force_semantics() {
let mut candidate = empty_mgr();
inject_mock(
&mut candidate,
"COLD-A",
build_s3_tier("COLD-A"),
MockWarmBackend {
in_use_value: None,
healthy: true,
},
);
let mutation = TierCandidateMutation::Remove("COLD-A".to_string(), true);
mutation
.apply(&mut candidate)
.await
.expect("force remove must skip in-use probing");
assert!(!candidate.tiers.contains_key("COLD-A"));
}
#[tokio::test]
async fn persisted_remove_nonforce_preserves_backend_in_use_error() {
let mut candidate = empty_mgr();
inject_mock(
&mut candidate,
"COLD-A",
build_s3_tier("COLD-A"),
MockWarmBackend {
in_use_value: Some(true),
healthy: true,
},
);
let err = TierCandidateMutation::Remove("COLD-A".to_string(), false)
.apply(&mut candidate)
.await
.expect_err("non-force remove must reject an in-use backend");
assert_eq!(err.code, ERR_TIER_BACKEND_NOT_EMPTY.code);
assert!(candidate.tiers.contains_key("COLD-A"));
}
#[tokio::test]
async fn clear_preserves_force_and_nonforce_semantics() {
let mut forced = empty_mgr();
inject_mock(
&mut forced,
"COLD-A",
build_s3_tier("COLD-A"),
MockWarmBackend {
in_use_value: None,
healthy: true,
},
);
forced.clear_tier(true).await.expect("force clear must skip in-use probing");
assert!(forced.tiers.is_empty());
let mut guarded = empty_mgr();
inject_mock(
&mut guarded,
"COLD-A",
build_s3_tier("COLD-A"),
MockWarmBackend {
in_use_value: Some(true),
healthy: true,
},
);
let err = guarded
.clear_tier(false)
.await
.expect_err("non-force clear must reject an in-use backend");
assert_eq!(err.code, ERR_TIER_BACKEND_NOT_EMPTY.code);
assert!(guarded.tiers.contains_key("COLD-A"));
}
#[tokio::test]
async fn test_remove_reports_probe_error() {
let mut mgr = empty_mgr();
inject_mock(
&mut mgr,
"COLD-A",
build_s3_tier("COLD-A"),
MockWarmBackend {
in_use_value: None,
healthy: true,
},
);
let err = mgr
.remove("COLD-A", false)
.await
.expect_err("a failed in_use probe must surface as an error");
assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
assert!(mgr.tiers.contains_key("COLD-A"), "tier must survive a failed probe");
}
// ---- verify ---------------------------------------------------------
#[tokio::test]
async fn test_verify_unknown_tier_errors() {
let mut mgr = empty_mgr();
let err = mgr.verify("NOPE").await.expect_err("verifying an unknown tier must fail");
assert!(err.to_string().contains("not found"), "{err}");
}
#[tokio::test]
async fn test_verify_succeeds_with_healthy_backend() {
let mut mgr = empty_mgr();
inject_mock(
&mut mgr,
"COLD-A",
build_s3_tier("COLD-A"),
MockWarmBackend {
in_use_value: Some(false),
healthy: true,
},
);
mgr.verify("COLD-A").await.expect("healthy backend must verify");
}
#[tokio::test]
async fn test_verify_reports_backend_failure() {
let mut mgr = empty_mgr();
inject_mock(
&mut mgr,
"COLD-A",
build_s3_tier("COLD-A"),
MockWarmBackend {
in_use_value: Some(false),
healthy: false,
},
);
mgr.verify("COLD-A")
.await
.expect_err("an unhealthy backend must fail verification");
}
#[tokio::test]
async fn shared_backend_proxy_forwards_validation_hooks() {
let unhealthy: SharedWarmBackend = Arc::new(MockWarmBackend {
in_use_value: Some(false),
healthy: false,
});
let proxy = SharedWarmBackendProxy(unhealthy);
let err = proxy.validate().await.expect_err("proxy must forward backend validation");
assert_eq!(err.to_string(), "mock validation failed");
let healthy: SharedWarmBackend = Arc::new(MockWarmBackend {
in_use_value: Some(false),
healthy: true,
});
let proxy = SharedWarmBackendProxy(healthy);
let err = proxy
.validate_remote_version_id("unsupported-version")
.expect_err("proxy must forward remote version validation");
assert_eq!(err.to_string(), "mock remote version rejected");
let err = proxy
.remove_exact("remote-object", "exact-only")
.await
.expect_err("proxy must forward exact-version cleanup");
assert_eq!(err.to_string(), "mock exact remove forwarded");
}
// ---- pure query helpers --------------------------------------------
#[test]
fn test_query_helpers_reflect_membership() {
let mut mgr = empty_mgr();
assert!(mgr.empty());
assert!(!mgr.is_tier_valid("COLD-A"));
assert_eq!(mgr.tier_type("COLD-A"), "internal");
assert!(mgr.get("COLD-A").is_none());
mgr.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A"));
assert!(!mgr.empty());
assert!(mgr.is_tier_valid("COLD-A"));
let (ty, in_use) = mgr.is_tier_name_in_use("COLD-A");
assert!(in_use);
assert_eq!(ty.as_lowercase(), "s3");
assert_eq!(mgr.tier_type("COLD-A"), "s3");
assert_eq!(mgr.list_tiers().len(), 1);
assert!(mgr.get("COLD-A").is_some());
}
// ---- persistence: encode/decode and legacy migration ---------------
#[test]
fn test_marshal_unmarshal_json_roundtrip_preserves_tier() {
let mut mgr = empty_mgr();
mgr.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A"));
let bytes = mgr.marshal().expect("marshal should succeed");
let decoded = TierConfigMgr::unmarshal(&bytes).expect("unmarshal should succeed");
let tier = decoded.tiers.get("COLD-A").expect("tier survives json roundtrip");
assert_eq!(tier.tier_type.as_lowercase(), "s3");
let s3 = tier.s3.as_ref().expect("s3 payload survives");
assert_eq!(s3.bucket, "bucket-a");
// secret_key is a serialized field (unlike Clone, marshal does not redact).
assert_eq!(s3.secret_key, "sk");
}
#[test]
fn test_external_blob_roundtrip_azure_maps_account_fields() {
let mut mgr = empty_mgr();
mgr.tiers.insert(
"COLD-AZ".to_string(),
TierConfig {
version: "v1".to_string(),
tier_type: TierType::Azure,
name: "COLD-AZ".to_string(),
azure: Some(crate::services::tier::tier_config::TierAzure {
name: "COLD-AZ".to_string(),
endpoint: "https://example-azure.invalid".to_string(),
access_key: "account-name".to_string(),
secret_key: "account-key".to_string(),
bucket: "container".to_string(),
prefix: "p".to_string(),
region: "eastus".to_string(),
storage_class: "HOT".to_string(),
sp_auth: crate::services::tier::tier_config::ServicePrincipalAuth {
tenant_id: "tenant".to_string(),
client_id: "client".to_string(),
client_secret: "secret".to_string(),
},
}),
..Default::default()
},
);
let bytes = encode_external_tiering_config_blob(&mgr).expect("encode azure tier");
let decoded = decode_external_tiering_config_blob(&bytes).expect("decode azure tier");
let tier = decoded.tiers.get("COLD-AZ").expect("azure tier survives roundtrip");
assert_eq!(tier.tier_type.as_lowercase(), "azure");
let az = tier.azure.as_ref().expect("azure payload survives");
// AccountName/AccountKey in the on-disk format map back to access/secret.
assert_eq!(az.access_key, "account-name");
assert_eq!(az.secret_key, "account-key");
assert_eq!(az.sp_auth.tenant_id, "tenant");
assert_eq!(az.sp_auth.client_secret, "secret");
}
#[test]
fn test_external_blob_roundtrip_gcs_preserves_creds() {
let mut mgr = empty_mgr();
mgr.tiers.insert(
"COLD-G".to_string(),
TierConfig {
version: "v1".to_string(),
tier_type: TierType::GCS,
name: "COLD-G".to_string(),
gcs: Some(crate::services::tier::tier_config::TierGCS {
name: "COLD-G".to_string(),
endpoint: "https://storage.googleapis.com".to_string(),
creds: "service-account-json".to_string(),
bucket: "gbucket".to_string(),
prefix: "gp".to_string(),
region: "us".to_string(),
storage_class: "NEARLINE".to_string(),
}),
..Default::default()
},
);
let bytes = encode_external_tiering_config_blob(&mgr).expect("encode gcs tier");
let decoded = decode_external_tiering_config_blob(&bytes).expect("decode gcs tier");
let tier = decoded.tiers.get("COLD-G").expect("gcs tier survives roundtrip");
assert_eq!(tier.tier_type.as_lowercase(), "gcs");
assert_eq!(tier.gcs.as_ref().expect("gcs payload survives").creds, "service-account-json");
}
// `TierConfigMgr` intentionally does not derive `Debug` (it holds live
// driver handles), so `Result::expect_err` cannot be used on decode
// results. Extract the error explicitly instead.
fn expect_decode_err(data: &[u8]) -> std::io::Error {
match decode_external_tiering_config_blob(data) {
Ok(_) => panic!("expected decode to fail"),
Err(err) => err,
}
}
#[test]
fn test_external_blob_rejects_truncated_data() {
let err = expect_decode_err(&[0u8; 3]);
assert!(err.to_string().contains("no data"), "{err}");
}
#[test]
fn test_external_blob_rejects_unknown_format() {
// Valid length, but a format word the decoder does not recognise.
let mut data = Vec::new();
data.extend_from_slice(&99u16.to_le_bytes());
data.extend_from_slice(&TIER_CONFIG_VERSION.to_le_bytes());
data.extend_from_slice(&[0u8; 8]);
let err = expect_decode_err(&data);
assert!(err.to_string().contains("unknown format"), "{err}");
}
#[test]
fn test_external_blob_rejects_unknown_version() {
let mut data = Vec::new();
data.extend_from_slice(&TIER_CONFIG_FORMAT.to_le_bytes());
data.extend_from_slice(&99u16.to_le_bytes());
data.extend_from_slice(&[0u8; 8]);
let err = expect_decode_err(&data);
assert!(err.to_string().contains("unknown version"), "{err}");
}
#[test]
fn test_external_blob_ignores_unknown_fields_at_all_legacy_layers() {
let payload = serde_json::json!({
"Tiers": {
"COLD-A": {
"Version": "v1",
"Type": EXTERNAL_TIER_TYPE_S3,
"Name": "COLD-A",
"S3": {
"Endpoint": "https://example.invalid",
"AccessKey": "ak",
"SecretKey": "sk",
"Bucket": "bucket",
"Prefix": "objects",
"UnknownS3": true
},
"Azure": {
"SPAuth": { "UnknownSPAuth": true },
"UnknownAzure": true
},
"GCS": { "UnknownGCS": true },
"MinIO": { "UnknownMinIO": true },
"UnknownTier": true
}
},
"UnknownManager": true
});
let encoded = rmp_serde::to_vec(&payload).expect("test payload should encode");
let mut data = Vec::with_capacity(4 + encoded.len());
data.extend_from_slice(&TIER_CONFIG_FORMAT.to_le_bytes());
data.extend_from_slice(&TIER_CONFIG_VERSION.to_le_bytes());
data.extend_from_slice(&encoded);
let decoded = decode_external_tiering_config_blob(&data).expect("legacy unknown fields must remain forward-compatible");
let s3 = decoded.tiers["COLD-A"].s3.as_ref().expect("s3 payload should decode");
assert_eq!(s3.prefix, "objects");
}
#[test]
fn test_external_blob_decodes_minio_compatible_fixture() {
let payload = serde_json::json!({
"Tiers": {
"COLD-M": {
"Version": "v1",
"Type": EXTERNAL_TIER_TYPE_MINIO,
"Name": "COLD-M",
"MinIO": {
"Endpoint": "https://minio.example.invalid",
"AccessKey": "ak",
"SecretKey": "sk",
"Bucket": "archive",
"Prefix": "objects/",
"Region": "us-east-1"
}
}
}
});
let encoded = rmp_serde::to_vec(&payload).expect("fixture payload should encode");
let mut data = Vec::with_capacity(4 + encoded.len());
data.extend_from_slice(&TIER_CONFIG_FORMAT.to_le_bytes());
data.extend_from_slice(&TIER_CONFIG_VERSION.to_le_bytes());
data.extend_from_slice(&encoded);
let decoded = decode_external_tiering_config_blob(&data).expect("MinIO-compatible fixture must decode");
let minio = decoded.tiers["COLD-M"].minio.as_ref().expect("MinIO payload should decode");
assert_eq!(minio.endpoint, "https://minio.example.invalid");
assert_eq!(minio.bucket, "archive");
assert_eq!(minio.prefix, "objects/");
}
#[test]
fn test_external_blob_accepts_legacy_v1_version_word() {
// The decoder must keep accepting the historical v1 version word, not
// just the current TIER_CONFIG_VERSION, so old on-disk configs load.
let mut mgr = empty_mgr();
mgr.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A"));
let current = encode_external_tiering_config_blob(&mgr).expect("encode");
// Rewrite the version word (bytes 2..4) from VERSION to V1.
let mut legacy = current.to_vec();
legacy[2..4].copy_from_slice(&TIER_CONFIG_V1.to_le_bytes());
let decoded = decode_external_tiering_config_blob(&legacy).expect("v1 version word must decode");
assert!(decoded.tiers.contains_key("COLD-A"));
}
#[test]
fn test_encode_external_blob_errors_on_missing_payload() {
let mut mgr = empty_mgr();
mgr.tiers.insert(
"COLD-A".to_string(),
TierConfig {
version: "v1".to_string(),
tier_type: TierType::S3,
name: "COLD-A".to_string(),
s3: None,
..Default::default()
},
);
let err = encode_external_tiering_config_blob(&mgr).expect_err("missing payload must fail encode");
assert!(err.to_string().contains("missing s3 backend payload"), "{err}");
}
#[derive(Clone)]
struct LeaseTestBackend {
id: &'static str,
calls: Arc<std::sync::Mutex<Vec<&'static str>>>,
remove_started: Option<Arc<tokio::sync::Notify>>,
remove_release: Option<Arc<tokio::sync::Semaphore>>,
backend_in_use: bool,
pending_in_use: bool,
panic_in_use: bool,
}
impl LeaseTestBackend {
fn ready(id: &'static str) -> Self {
Self {
id,
calls: Arc::new(std::sync::Mutex::new(Vec::new())),
remove_started: None,
remove_release: None,
backend_in_use: false,
pending_in_use: false,
panic_in_use: false,
}
}
fn blocking(id: &'static str) -> Self {
Self {
remove_started: Some(Arc::new(tokio::sync::Notify::new())),
remove_release: Some(Arc::new(tokio::sync::Semaphore::new(0))),
..Self::ready(id)
}
}
fn calls(&self) -> Vec<&'static str> {
self.calls.lock().expect("lease test call log should not poison").clone()
}
fn panicking_in_use(id: &'static str) -> Self {
Self {
panic_in_use: true,
..Self::ready(id)
}
}
fn in_use(id: &'static str) -> Self {
Self {
backend_in_use: true,
..Self::ready(id)
}
}
fn pending_in_use(id: &'static str) -> Self {
Self {
pending_in_use: true,
..Self::ready(id)
}
}
}
#[async_trait::async_trait]
impl WarmBackend for LeaseTestBackend {
async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> io::Result<String> {
Ok(self.id.to_string())
}
async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
_meta: HashMap<String, String>,
) -> io::Result<String> {
self.put(object, r, length).await
}
async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> io::Result<ReadCloser> {
Ok(BufReader::new(Cursor::new(Vec::new())))
}
async fn remove(&self, _object: &str, _rv: &str) -> io::Result<()> {
self.calls
.lock()
.expect("lease test call log should not poison")
.push(self.id);
if let Some(started) = &self.remove_started {
started.notify_one();
}
if let Some(release) = &self.remove_release {
release
.acquire()
.await
.expect("lease test release semaphore should stay open")
.forget();
}
Ok(())
}
async fn in_use(&self) -> io::Result<bool> {
if self.panic_in_use {
panic!("simulated tier backend in-use panic");
}
if self.pending_in_use {
std::future::pending::<()>().await;
}
Ok(self.backend_in_use)
}
}
fn install_lease_backend(manager: &mut TierConfigMgr, tier_name: &str, backend: LeaseTestBackend) {
manager.tiers.insert(tier_name.to_string(), build_rustfs_tier(tier_name));
manager
.replace_driver(tier_name, Box::new(backend))
.expect("test driver generation should install");
}
fn prepared_remove_intent(tier_name: &str, mutation_id: uuid::Uuid) -> TierMutationIntent {
TierMutationIntent {
mutation_id,
revision: 1,
kind: TierMutationIntentKind::Remove,
state: TierMutationIntentState::Prepared,
old_config_etag: Some("etag-old".to_string()),
committed_config_etag: None,
candidate_digest: [1; 32],
affected_targets: vec![TierMutationIntentTarget {
tier_name: tier_name.to_string(),
old_backend_identity: Some([2; 32]),
new_backend_identity: None,
}],
expires_at_unix_nanos: 1,
}
}
#[tokio::test]
async fn prepared_mutation_recovery_blocks_new_operation_lease() {
let manager = TierConfigMgr::new();
manager
.write()
.await
.tiers
.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
let mutation_id = uuid::Uuid::from_u128(1);
TierConfigMgr::reconcile_prepared_mutation_intents(&manager, &[prepared_remove_intent("COLD-A", mutation_id)])
.await
.expect("prepared recovery block should apply");
let err = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await {
Ok(_) => panic!("prepared mutation must block new operation leases"),
Err(err) => err,
};
assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
assert_eq!(err.message, TIER_MUTATION_BLOCK_MESSAGE);
let guard = manager.read().await;
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should be registered by recovery block");
assert_eq!(lock_unpoisoned(&runtime).prepared_mutation_blocks.get("COLD-A"), Some(&mutation_id));
}
#[tokio::test]
async fn prepared_mutation_recovery_blocks_admin_publish() {
let manager = TierConfigMgr::new();
manager
.write()
.await
.tiers
.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
TierConfigMgr::reconcile_prepared_mutation_intents(
&manager,
&[prepared_remove_intent("COLD-A", uuid::Uuid::from_u128(1))],
)
.await
.expect("prepared recovery block should apply");
let candidate = empty_mgr();
let err = TierConfigMgr::publish_candidate(&manager, candidate, None)
.await
.expect_err("prepared mutation must block conflicting admin publishes");
assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
assert!(manager.read().await.tiers.contains_key("COLD-A"));
}
#[tokio::test]
async fn prepared_mutation_recovery_is_idempotent_and_rejects_conflicts() {
let manager = TierConfigMgr::new();
let first = prepared_remove_intent("COLD-A", uuid::Uuid::from_u128(1));
TierConfigMgr::reconcile_prepared_mutation_intents(&manager, &[first.clone(), first])
.await
.expect("same prepared mutation should be idempotent");
let first = prepared_remove_intent("COLD-A", uuid::Uuid::from_u128(1));
let second = prepared_remove_intent("COLD-A", uuid::Uuid::from_u128(2));
let err = TierConfigMgr::reconcile_prepared_mutation_intents(&manager, &[first, second])
.await
.expect_err("a second prepared mutation for the same tier must fail closed");
assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
}
#[tokio::test]
async fn prepared_mutation_recovery_clears_resolved_intents() {
let manager = TierConfigMgr::new();
manager
.write()
.await
.tiers
.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
TierConfigMgr::reconcile_prepared_mutation_intents(
&manager,
&[prepared_remove_intent("COLD-A", uuid::Uuid::from_u128(1))],
)
.await
.expect("prepared recovery block should apply");
TierConfigMgr::reconcile_prepared_mutation_intents(&manager, &[])
.await
.expect("resolved recovery scan should clear stale blocks");
let guard = manager.read().await;
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should stay registered");
assert!(lock_unpoisoned(&runtime).prepared_mutation_blocks.is_empty());
}
#[tokio::test]
async fn registry_steady_state_lookup_does_not_scan_misses() {
let manager = TierConfigMgr::new();
TIER_REGISTRY_MISS_SCAN_COUNT
.scope(std::cell::Cell::new(0), async {
let guard = manager.read().await;
let first = tier_driver_runtime(&manager, &guard);
assert_eq!(TIER_REGISTRY_MISS_SCAN_COUNT.with(std::cell::Cell::get), 1);
let second = tier_driver_runtime(&manager, &guard);
assert!(Arc::ptr_eq(&first, &second));
assert_eq!(TIER_REGISTRY_MISS_SCAN_COUNT.with(std::cell::Cell::get), 1);
})
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn cold_driver_build_does_not_hold_manager_write_lock() {
let cold_tier = "COLD-BUILD-LOCK";
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
guard.tiers.insert(cold_tier.to_string(), build_rustfs_tier(cold_tier));
install_lease_backend(&mut guard, "COLD-B", LeaseTestBackend::ready("b"));
}
let (barrier, _barrier_guard) = install_tier_driver_build_barrier(cold_tier);
let build_manager = manager.clone();
let build = tokio::spawn(async move { TierConfigMgr::acquire_operation_lease(&build_manager, cold_tier).await });
barrier.arrived.notified().await;
tokio::time::timeout(Duration::from_millis(100), manager.read())
.await
.expect("cold driver construction must not block manager readers");
let tier_b = tokio::time::timeout(Duration::from_millis(100), TierConfigMgr::acquire_operation_lease(&manager, "COLD-B"))
.await
.expect("cold tier A construction must not block tier B")
.expect("tier B lease should remain available");
drop(tier_b);
barrier.release.add_permits(1);
build
.await
.expect("cold driver build task should join")
.expect("cold driver should finish after the test barrier releases");
}
#[tokio::test(start_paused = true)]
#[serial_test::serial]
async fn cold_reload_driver_build_timeout_restores_config() {
let cold_tier = "COLD-BUILD-TIMEOUT";
let manager = TierConfigMgr::new();
manager
.write()
.await
.tiers
.insert(cold_tier.to_string(), build_rustfs_tier(cold_tier));
let mut candidate = empty_mgr();
let mut replacement = build_rustfs_tier(cold_tier);
replacement.rustfs.as_mut().expect("replacement payload should exist").prefix = "new-prefix".to_string();
candidate.tiers.insert(cold_tier.to_string(), replacement);
let (_barrier, _barrier_guard) = install_tier_driver_build_barrier(cold_tier);
let err = TierConfigMgr::publish_candidate(&manager, candidate, None)
.await
.expect_err("stalled cold backend construction must time out");
assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
assert!(manager.read().await.tiers.contains_key(cold_tier));
let guard = manager.read().await;
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
assert!(lock_unpoisoned(&runtime).draining.is_empty());
}
#[test]
fn internal_tier_snapshot_preserves_credentials() {
let mut config = build_rustfs_tier("COLD-A");
let rustfs = config.rustfs.as_mut().expect("rustfs payload should exist");
rustfs.access_key = "access-key".to_string();
rustfs.secret_key = "secret-key".to_string();
let snapshot = config.clone_with_credentials();
let rustfs = snapshot.rustfs.expect("snapshot payload should exist");
assert_eq!(rustfs.access_key, "access-key");
assert_eq!(rustfs.secret_key, "secret-key");
}
#[tokio::test(start_paused = true)]
async fn remote_mutation_validation_timeout_restores_candidate() {
let mut candidate = empty_mgr();
candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
candidate
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::pending_in_use("pending")));
let err = apply_tier_candidate_mutation(
TierCandidateMutation::Remove("COLD-A".to_string(), false),
&mut candidate,
Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT,
)
.await
.expect_err("unbounded backend validation must time out");
assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
assert!(candidate.tiers.contains_key("COLD-A"));
}
#[tokio::test]
async fn cold_cached_reload_rejects_in_use_route_change_and_removal() {
for remove in [false, true] {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
guard.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
guard
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::in_use("cold")));
}
let mut candidate = empty_mgr();
if !remove {
let mut replacement = build_rustfs_tier("COLD-A");
replacement.rustfs.as_mut().expect("replacement payload should exist").prefix = "new-prefix".to_string();
candidate.tiers.insert("COLD-A".to_string(), replacement);
}
let err = TierConfigMgr::publish_candidate(&manager, candidate, None)
.await
.expect_err("an in-use cold-cache destination must not be rebound or removed");
assert_eq!(err.code, ERR_TIER_BACKEND_NOT_EMPTY.code);
assert!(manager.read().await.tiers.contains_key("COLD-A"));
let restored = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("failed reload must restore the old cold-cache generation");
assert!(restored.is_current(&manager).await);
}
}
#[tokio::test]
async fn empty_backend_route_change_and_credential_rotation_can_publish() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("empty"));
}
let mut route_candidate = empty_mgr();
let mut replacement = build_rustfs_tier("COLD-A");
replacement.rustfs.as_mut().expect("replacement payload should exist").prefix = "new-prefix".to_string();
route_candidate.tiers.insert("COLD-A".to_string(), replacement);
TierConfigMgr::publish_candidate(&manager, route_candidate, None)
.await
.expect("an empty backend may be rebound");
{
let mut guard = manager.write().await;
guard
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::panicking_in_use("credentials")));
}
let mut credential_candidate = empty_mgr();
let mut rotated = manager.read().await.tiers["COLD-A"].clone_with_credentials();
rotated.rustfs.as_mut().expect("rotated payload should exist").secret_key = "rotated".to_string();
credential_candidate.tiers.insert("COLD-A".to_string(), rotated);
TierConfigMgr::publish_candidate(&manager, credential_candidate, None)
.await
.expect("credential-only rotation must not inspect backend occupancy");
}
#[tokio::test]
async fn slow_tier_operation_does_not_block_another_tier() {
let slow = LeaseTestBackend::blocking("slow");
let fast = LeaseTestBackend::ready("fast");
let manager = Arc::new(RwLock::new(empty_mgr()));
{
let mut manager = manager.write().await;
install_lease_backend(&mut manager, "COLD-A", slow.clone());
install_lease_backend(&mut manager, "COLD-B", fast.clone());
}
let slow_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("slow lease should be available");
let fast_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-B")
.await
.expect("fast lease should be available");
let slow_task = tokio::spawn(async move { slow_lease.remove("object", "").await });
slow.remove_started
.as_ref()
.expect("slow backend should expose start notification")
.notified()
.await;
let manager_read = tokio::time::timeout(Duration::from_millis(100), manager.read())
.await
.expect("manager reads must not wait for tier A");
assert!(manager_read.is_tier_valid("COLD-B"));
drop(manager_read);
tokio::time::timeout(Duration::from_millis(100), fast_lease.remove("object", ""))
.await
.expect("tier B must not wait for tier A")
.expect("fast remove should succeed");
assert_eq!(fast.calls(), vec!["fast"]);
slow.remove_release
.as_ref()
.expect("slow backend should expose release semaphore")
.add_permits(1);
slow_task
.await
.expect("slow task should join")
.expect("slow remove should succeed");
}
#[tokio::test]
async fn slow_admin_verify_does_not_hold_manager_lock() {
let manager = TierConfigMgr::new();
let backend = LeaseTestBackend::blocking("verify");
let started = backend
.remove_started
.as_ref()
.expect("verify remove notifier should exist")
.clone();
let release = backend
.remove_release
.as_ref()
.expect("verify remove release should exist")
.clone();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", backend);
}
let verify_manager = manager.clone();
let verify = tokio::spawn(async move { TierConfigMgr::verify_without_manager_lock(&verify_manager, "COLD-A").await });
started.notified().await;
tokio::time::timeout(Duration::from_millis(100), manager.read())
.await
.expect("slow verify must not hold the manager lock");
release.add_permits(1);
verify.await.expect("verify task should join").expect("verify should finish");
}
#[tokio::test]
async fn same_name_replacement_drains_old_generation_and_routes_new_work_to_new_driver() {
let old = LeaseTestBackend::ready("old");
let new = LeaseTestBackend::ready("new");
let manager = Arc::new(RwLock::new(empty_mgr()));
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", old.clone());
}
let old_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("old generation lease should be available");
{
manager
.write()
.await
.replace_driver("COLD-A", Box::new(new.clone()))
.expect("replacement generation should install");
}
let new_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("new generation lease should be available");
assert!(!old_lease.is_current(&manager).await);
assert!(new_lease.is_current(&manager).await);
assert!(new_lease.generation() > old_lease.generation());
old_lease
.remove("old-object", "")
.await
.expect("draining lease should remain usable");
new_lease
.remove("new-object", "")
.await
.expect("new lease should use replacement driver");
assert_eq!(old.calls(), vec!["old"]);
assert_eq!(new.calls(), vec!["new"]);
}
#[tokio::test]
async fn replacement_waits_for_inflight_operation_lease() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("old lease should be available");
let mut candidate = empty_mgr();
let mut config = build_rustfs_tier("COLD-A");
config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://replacement.invalid".to_string();
candidate.tiers.insert("COLD-A".to_string(), config);
candidate
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("new")));
let publish_manager = manager.clone();
let publish =
tokio::spawn(async move { TierConfigMgr::publish_candidate(&publish_manager, candidate, Some("COLD-A")).await });
tokio::time::timeout(Duration::from_secs(1), async {
while old.is_current(&manager).await {
tokio::task::yield_now().await;
}
})
.await
.expect("replacement should revoke the old generation before waiting");
assert!(!publish.is_finished(), "replacement must wait for the old operation lease");
drop(old);
publish
.await
.expect("publish task should join")
.expect("replacement should publish after the lease drains");
}
#[tokio::test(start_paused = true)]
async fn replacement_drain_timeout_restores_generation_and_allows_retry() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("old lease should be available");
let mut candidate = empty_mgr();
let mut config = build_rustfs_tier("COLD-A");
config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://timeout.invalid".to_string();
candidate.tiers.insert("COLD-A".to_string(), config);
candidate
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("timed-out")));
let publish_manager = manager.clone();
let publish =
tokio::spawn(async move { TierConfigMgr::publish_candidate(&publish_manager, candidate, Some("COLD-A")).await });
while old.inner.accepting.load(Ordering::Acquire) {
tokio::task::yield_now().await;
}
tokio::time::advance(TIER_OPERATION_DRAIN_TIMEOUT).await;
let err = publish
.await
.expect("publish task should join")
.expect_err("replacement must time out while the old lease remains active");
assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
assert!(old.is_current(&manager).await, "timeout rollback must restore the old generation");
{
let guard = manager.read().await;
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
assert!(!lock_unpoisoned(&runtime).draining.contains_key("COLD-A"));
}
drop(old);
let mut retry = empty_mgr();
let mut retry_config = build_rustfs_tier("COLD-A");
retry_config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://retry.invalid".to_string();
retry.tiers.insert("COLD-A".to_string(), retry_config);
retry
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("retry")));
TierConfigMgr::publish_candidate(&manager, retry, Some("COLD-A"))
.await
.expect("a later replacement must succeed after timeout rollback");
}
#[tokio::test]
async fn generation_prepare_failure_restores_old_manager_and_generation() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("old generation should initialize");
drop(lease);
{
let guard = manager.read().await;
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should be registered");
lock_unpoisoned(&runtime).next_generation = u64::MAX;
}
let mut candidate = empty_mgr();
let mut config = build_rustfs_tier("COLD-A");
config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://overflow.invalid".to_string();
candidate.tiers.insert("COLD-A".to_string(), config);
candidate
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("new")));
let err = TierConfigMgr::publish_candidate(&manager, candidate, Some("COLD-A"))
.await
.expect_err("generation exhaustion must fail before manager state changes");
assert!(err.message.contains("generation exhausted"));
assert_eq!(
manager.read().await.tiers["COLD-A"]
.rustfs
.as_ref()
.expect("old config should remain")
.endpoint,
"https://example-compat.invalid"
);
let restored = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("prepare failure must restore the old generation");
assert!(restored.is_current(&manager).await);
let guard = manager.read().await;
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
assert!(!lock_unpoisoned(&runtime).draining.contains_key("COLD-A"));
}
#[tokio::test]
async fn legacy_reload_does_not_block_revoked_operation_completion() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("old lease should be available");
let mut candidate = empty_mgr();
let mut config = build_rustfs_tier("COLD-A");
config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://legacy-reload.invalid".to_string();
candidate.tiers.insert("COLD-A".to_string(), config);
let reload_manager = manager.clone();
let reload = tokio::spawn(async move { reload_manager.write().await.publish_legacy_reload(candidate).await });
while old.inner.accepting.load(Ordering::Acquire) {
tokio::task::yield_now().await;
}
assert!(
manager.try_read().is_err(),
"legacy reload must still hold the manager write guard while draining"
);
assert!(!reload.is_finished(), "legacy reload must wait for the active generation");
let operation = tokio::spawn(async move {
assert!(!old.is_current_generation(), "the revoked operation must observe the generation fence");
drop(old);
});
tokio::time::timeout(Duration::from_secs(1), operation)
.await
.expect("the revoked operation must finish without waiting for the manager guard")
.expect("the revoked operation task should join");
reload
.await
.expect("legacy reload task should join")
.expect("legacy reload should publish after the lease drains");
assert_eq!(
manager.read().await.tiers["COLD-A"]
.rustfs
.as_ref()
.expect("reloaded tier should exist")
.endpoint,
"https://legacy-reload.invalid"
);
}
#[tokio::test]
async fn reload_publish_waits_for_inflight_operation_lease() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("old lease should be available");
let mut replacement = build_rustfs_tier("COLD-A");
replacement.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://reload.invalid".to_string();
let mut candidate = empty_mgr();
candidate.tiers.insert("COLD-A".to_string(), replacement);
let reload_manager = manager.clone();
let reload = tokio::spawn(async move { TierConfigMgr::publish_candidate(&reload_manager, candidate, None).await });
tokio::time::timeout(Duration::from_secs(1), async {
while old.is_current(&manager).await {
tokio::task::yield_now().await;
}
})
.await
.expect("reload publish should revoke before waiting");
assert!(!reload.is_finished(), "reload must wait for the old operation lease");
drop(old);
reload
.await
.expect("reload task should join")
.expect("reload should publish after the lease drains");
}
#[tokio::test]
async fn cancelled_publish_caller_does_not_leave_tier_draining() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("old lease should be available");
let mut candidate = empty_mgr();
let mut config = build_rustfs_tier("COLD-A");
config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://owned.invalid".to_string();
candidate.tiers.insert("COLD-A".to_string(), config);
candidate
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("new")));
let update = TierConfigMgr::admin_update_lock(&manager).await;
let publish_manager = manager.clone();
let caller = tokio::spawn(async move {
TierConfigMgr::publish_candidate_owned(&publish_manager, candidate, Some("COLD-A".to_string()), update).await
});
tokio::time::timeout(Duration::from_secs(1), async {
while old.is_current(&manager).await {
tokio::task::yield_now().await;
}
})
.await
.expect("owned publish should revoke before caller cancellation");
caller.abort();
let mut second = empty_mgr();
let mut second_config = build_rustfs_tier("COLD-A");
second_config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://second.invalid".to_string();
second.tiers.insert("COLD-A".to_string(), second_config);
second
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("second")));
let second_manager = manager.clone();
let second_publish =
tokio::spawn(async move { TierConfigMgr::publish_candidate(&second_manager, second, Some("COLD-A")).await });
tokio::task::yield_now().await;
assert!(!second_publish.is_finished(), "a later update must remain behind the detached publish");
drop(old);
second_publish
.await
.expect("second publish task should join")
.expect("second publish should follow the detached publish");
let endpoint = manager
.read()
.await
.tiers
.get("COLD-A")
.and_then(|tier| tier.rustfs.as_ref())
.expect("second tier should be published")
.endpoint
.clone();
assert_eq!(endpoint, "https://second.invalid");
}
#[tokio::test]
async fn slow_tier_replacement_does_not_block_other_tiers_or_manager_reads() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("a"));
install_lease_backend(&mut guard, "COLD-B", LeaseTestBackend::ready("b"));
}
let old_a = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("tier A lease should be available");
let old_b = TierConfigMgr::acquire_operation_lease(&manager, "COLD-B")
.await
.expect("tier B lease should be available");
let mut candidate = empty_mgr();
candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
candidate.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B"));
candidate
.tiers
.get_mut("COLD-A")
.and_then(|tier| tier.rustfs.as_mut())
.expect("tier A payload should exist")
.endpoint = "https://replacement.invalid".to_string();
candidate
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("new-a")));
let publish_manager = manager.clone();
let publish =
tokio::spawn(async move { TierConfigMgr::publish_candidate(&publish_manager, candidate, Some("COLD-A")).await });
tokio::time::timeout(Duration::from_secs(1), async {
while old_a.is_current(&manager).await {
tokio::task::yield_now().await;
}
})
.await
.expect("tier A should enter draining");
{
let guard = manager.read().await;
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
assert_eq!(
lock_unpoisoned(&runtime).draining.keys().cloned().collect::<Vec<_>>(),
vec!["COLD-A".to_string()]
);
}
tokio::time::timeout(Duration::from_secs(1), manager.read())
.await
.expect("manager reads must not wait for tier A leases");
let next_b = tokio::time::timeout(Duration::from_secs(1), TierConfigMgr::acquire_operation_lease(&manager, "COLD-B"))
.await
.expect("tier B lease acquisition must not wait for tier A")
.expect("tier B lease should remain available");
assert_eq!(next_b.generation(), old_b.generation());
assert!(!publish.is_finished(), "tier A replacement should still wait for its lease");
drop(old_a);
publish
.await
.expect("publish task should join")
.expect("tier A replacement should publish after its lease drains");
}
#[tokio::test]
async fn direct_mutation_fails_closed_while_generation_is_active() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("lease should be available");
let err = manager
.write()
.await
.remove("COLD-A", true)
.await
.expect_err("direct removal must not wait for an active generation");
assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
assert!(lease.is_current(&manager).await);
}
#[tokio::test]
async fn steady_lease_acquisition_is_concurrent_across_tiers() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("a"));
install_lease_backend(&mut guard, "COLD-B", LeaseTestBackend::ready("b"));
}
let mut tasks = Vec::new();
for index in 0..200 {
let manager = manager.clone();
tasks.push(tokio::spawn(async move {
let tier = if index % 2 == 0 { "COLD-A" } else { "COLD-B" };
TierConfigMgr::acquire_operation_lease(&manager, tier).await
}));
}
tokio::time::timeout(Duration::from_secs(1), async {
for task in tasks {
task.await
.expect("lease task should join")
.expect("published generation should remain available");
}
})
.await
.expect("steady lease acquisition should not serialize on manager writes or config hashing");
}
#[tokio::test]
async fn cancelled_operation_releases_generation_lease() {
let backend = LeaseTestBackend::blocking("cancelled");
let manager = Arc::new(RwLock::new(empty_mgr()));
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", backend.clone());
}
let lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("lease should be available");
let task = tokio::spawn(async move { lease.remove("object", "").await });
backend
.remove_started
.as_ref()
.expect("blocking backend should expose start notification")
.notified()
.await;
task.abort();
let _ = task.await;
let manager = manager.read().await;
let runtime = registered_tier_driver_runtime(&manager).expect("runtime sidecar should be registered");
drop(manager);
let runtime = lock_unpoisoned(&runtime);
let generation = runtime.generations.get("COLD-A").expect("generation should remain installed");
assert_eq!(generation.active_leases.load(Ordering::Acquire), 0);
}
#[tokio::test]
async fn tier_generations_are_isolated_between_runtime_instances() {
let manager_a = TierConfigMgr::new();
let manager_b = TierConfigMgr::new();
{
let mut guard = manager_a.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("a"));
}
{
let mut guard = manager_b.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("b"));
}
let lease_a = TierConfigMgr::acquire_operation_lease(&manager_a, "COLD-A")
.await
.expect("instance A lease should be available");
let lease_b = TierConfigMgr::acquire_operation_lease(&manager_b, "COLD-A")
.await
.expect("instance B lease should be available");
assert!(lease_a.is_current(&manager_a).await);
assert!(lease_b.is_current(&manager_b).await);
assert!(!lease_a.is_current(&manager_b).await);
assert!(!lease_b.is_current(&manager_a).await);
}
#[tokio::test]
async fn lease_acquisition_has_constant_generation_state() {
let manager = Arc::new(RwLock::new(empty_mgr()));
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("bounded"));
}
for _ in 0..10_000 {
drop(
TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("lease should be available"),
);
}
let manager = manager.read().await;
let runtime = registered_tier_driver_runtime(&manager).expect("runtime sidecar should be registered");
drop(manager);
let runtime = lock_unpoisoned(&runtime);
assert_eq!(runtime.generations.len(), 1);
assert_eq!(runtime.next_generation, 1);
assert_eq!(
runtime
.generations
.get("COLD-A")
.expect("generation should remain installed")
.active_leases
.load(Ordering::Acquire),
0
);
}
#[tokio::test]
async fn unrelated_reload_keeps_active_generation_current() {
let manager = Arc::new(RwLock::new(empty_mgr()));
{
let mut manager = manager.write().await;
install_lease_backend(&mut manager, "COLD-A", LeaseTestBackend::ready("a"));
manager.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B"));
}
let lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("lease should be available");
let generation = lease.generation();
let mut reloaded = HashMap::from([
("COLD-A".to_string(), build_rustfs_tier("COLD-A")),
("COLD-B".to_string(), build_rustfs_tier("COLD-B")),
("COLD-C".to_string(), build_rustfs_tier("COLD-C")),
]);
reloaded
.get_mut("COLD-B")
.and_then(|tier| tier.rustfs.as_mut())
.expect("tier B payload should exist")
.access_key = "rotated".to_string();
manager
.write()
.await
.apply_reloaded_tiers(reloaded)
.expect("unrelated reload should apply");
assert!(lease.is_current(&manager).await);
let next = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("tier A should remain available");
assert_eq!(next.generation(), generation);
}
#[tokio::test]
async fn backend_identity_survives_credentials_rotation_but_rejects_route_change() {
let manager = Arc::new(RwLock::new(empty_mgr()));
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("old lease should be available");
let identity = old.backend_identity();
let mut rotated = build_rustfs_tier("COLD-A");
let rotated_config = rotated.rustfs.as_mut().expect("rustfs payload should exist");
rotated_config.access_key = "rotated-access".to_string();
rotated_config.secret_key = "rotated-secret".to_string();
manager.write().await.tiers.insert("COLD-A".to_string(), rotated);
manager
.write()
.await
.replace_driver("COLD-A", Box::new(LeaseTestBackend::ready("rotated")))
.expect("rotated driver should install");
let rotated = TierConfigMgr::acquire_operation_lease_for_backend_identity(&manager, "COLD-A", identity)
.await
.expect("credential rotation should preserve backend identity");
assert_ne!(rotated.generation(), old.generation());
assert_eq!(rotated.backend_identity(), identity);
let mut moved = build_rustfs_tier("COLD-A");
moved.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://moved.invalid".to_string();
manager.write().await.tiers.insert("COLD-A".to_string(), moved);
manager
.write()
.await
.replace_driver("COLD-A", Box::new(LeaseTestBackend::ready("moved")))
.expect("moved driver should install");
let err = match TierConfigMgr::acquire_operation_lease_for_backend_identity(&manager, "COLD-A", identity).await {
Ok(_) => panic!("route change must fail closed"),
Err(err) => err,
};
assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
}
fn build_azure_tier(account_name: &str) -> TierConfig {
TierConfig {
version: "v1".to_string(),
tier_type: TierType::Azure,
name: "COLD-AZURE".to_string(),
azure: Some(crate::services::tier::tier_config::TierAzure {
endpoint: "https://blob.example.invalid".to_string(),
access_key: account_name.to_string(),
secret_key: "account-key".to_string(),
bucket: "archive".to_string(),
prefix: "objects/".to_string(),
region: "us-east-1".to_string(),
..Default::default()
}),
..Default::default()
}
}
#[test]
fn azure_account_name_is_part_of_backend_identity() {
let account_a = build_azure_tier("account-a");
let mut rotated_key = build_azure_tier("account-a");
rotated_key.azure.as_mut().expect("azure payload should exist").secret_key = "rotated-key".to_string();
let account_b = build_azure_tier("account-b");
assert_eq!(
tier_backend_identity(&account_a).expect("account A identity should encode"),
tier_backend_identity(&rotated_key).expect("rotated key identity should encode"),
"Azure account-key rotation must preserve destination identity"
);
assert_ne!(
tier_backend_identity(&account_a).expect("account A identity should encode"),
tier_backend_identity(&account_b).expect("account B identity should encode"),
"Azure account-name changes must fence cleanup from the new account"
);
}
#[test]
fn wasabi_backend_identity_uses_its_canonical_physical_destination() {
let mut wasabi = build_wasabi_tier("COLD-WASABI");
let wasabi_config = wasabi.wasabi.as_mut().expect("Wasabi payload should exist");
wasabi_config.endpoint = "https://s3.nl-1.wasabisys.com".to_string();
wasabi_config.prefix = "/archive//".to_string();
wasabi_config.region = "eu-central-1".to_string();
let mut physical_s3 = build_s3_tier("COLD-WASABI");
let s3 = physical_s3.s3.as_mut().expect("S3 payload should exist");
s3.endpoint = "https://s3.eu-central-1.wasabisys.com".to_string();
s3.bucket = wasabi_config.bucket.clone();
s3.prefix = "archive".to_string();
s3.region = wasabi_config.region.clone();
assert_eq!(
tier_backend_identity(&wasabi).expect("Wasabi identity should encode"),
tier_backend_identity(&physical_s3).expect("physical S3 identity should encode")
);
}
#[test]
fn in_place_wasabi_type_changes_are_rejected() {
let mut wasabi = build_wasabi_tier("COLD-WASABI");
wasabi.wasabi.as_mut().expect("Wasabi payload should exist").region = "us-east-1".to_string();
let mut s3 = build_s3_tier("COLD-WASABI");
let s3_config = s3.s3.as_mut().expect("S3 payload should exist");
s3_config.endpoint = "https://s3.wasabisys.com".to_string();
s3_config.bucket = "wasabi-bucket".to_string();
s3_config.prefix = "archive".to_string();
s3_config.region = "us-east-1".to_string();
for (current_tier, replacement_tier) in [(s3.clone_with_credentials(), wasabi.clone_with_credentials()), (wasabi, s3)] {
let mut current = empty_mgr();
current.tiers.insert("COLD-WASABI".to_string(), current_tier);
let mut replacement = empty_mgr();
replacement.tiers.insert("COLD-WASABI".to_string(), replacement_tier);
let err = replaced_tier_destinations(&current, &replacement)
.expect_err("in-place type changes must not bypass mixed-version Wasabi fencing");
assert!(err.message.contains("add a new tier instead"), "{}", err.message);
}
}
#[test]
fn backend_identity_v2_has_stable_fixtures() {
assert_eq!(
tier_backend_identity(&build_rustfs_tier("COLD-A")).expect("identity should encode"),
[
112, 73, 111, 29, 53, 43, 207, 7, 170, 12, 99, 116, 213, 135, 173, 227, 6, 220, 179, 135, 232, 83, 25, 13, 128,
98, 254, 103, 132, 128, 229, 97,
]
);
assert_eq!(
tier_backend_identity(&build_azure_tier("account-a")).expect("Azure identity should encode"),
[
57, 188, 231, 23, 184, 188, 233, 228, 118, 24, 158, 152, 178, 169, 2, 146, 149, 152, 104, 0, 146, 196, 61, 145,
18, 1, 188, 20, 36, 215, 100, 125,
]
);
}
#[test]
fn destination_identity_prefix_normalization_matches_driver_matrix() {
assert_eq!(normalized_tier_prefix(&TierType::S3, "/foo//"), "foo");
assert_eq!(normalized_tier_prefix(&TierType::Wasabi, "/foo//"), "foo");
for tier_type in [
TierType::RustFS,
TierType::MinIO,
TierType::Aliyun,
TierType::Tencent,
TierType::Huaweicloud,
TierType::Azure,
TierType::GCS,
TierType::R2,
] {
assert_eq!(normalized_tier_prefix(&tier_type, "foo/"), "foo");
assert_eq!(normalized_tier_prefix(&tier_type, "foo//"), "foo/");
}
}
#[tokio::test]
async fn direct_rollback_fails_closed_until_unpublished_generation_is_idle() {
let manager = Arc::new(RwLock::new(empty_mgr()));
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("persisted"));
}
let persisted = HashMap::from([("COLD-A".to_string(), build_rustfs_tier("COLD-A"))]);
let mut unpublished_config = build_rustfs_tier("COLD-A");
unpublished_config
.rustfs
.as_mut()
.expect("rustfs payload should exist")
.endpoint = "https://unpublished.invalid".to_string();
{
let mut manager = manager.write().await;
manager.tiers.insert("COLD-A".to_string(), unpublished_config);
manager
.replace_driver("COLD-A", Box::new(LeaseTestBackend::ready("unpublished")))
.expect("unpublished driver should install before simulated save failure");
}
let unpublished = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("unpublished lease should be visible before rollback");
let err = manager
.write()
.await
.apply_reloaded_tiers(persisted.clone())
.expect_err("direct rollback must not wait for an active generation");
assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
assert!(unpublished.is_current(&manager).await);
drop(unpublished);
manager
.write()
.await
.apply_reloaded_tiers(persisted)
.expect("rollback should apply once the generation is idle");
let manager_guard = manager.read().await;
assert_eq!(
manager_guard
.tiers
.get("COLD-A")
.and_then(|tier| tier.rustfs.as_ref())
.expect("persisted tier should be restored")
.endpoint,
"https://example-compat.invalid"
);
let runtime = registered_tier_driver_runtime(&manager_guard).expect("runtime sidecar should remain registered");
assert!(lock_unpoisoned(&runtime).generations.get("COLD-A").is_none());
}
#[derive(Debug)]
struct CasConfigStore {
state: tokio::sync::Mutex<Option<(Vec<u8>, String)>>,
next_etag: AtomicUsize,
fail_put: AtomicBool,
truncate_reference_page_without_marker: AtomicBool,
lock_manager: Arc<rustfs_lock::GlobalLockManager>,
lock_requests: Mutex<Vec<(String, String)>>,
listed_versions: Mutex<Vec<ObjectInfo>>,
}
impl Default for CasConfigStore {
fn default() -> Self {
Self {
state: tokio::sync::Mutex::new(None),
next_etag: AtomicUsize::new(0),
fail_put: AtomicBool::new(false),
truncate_reference_page_without_marker: AtomicBool::new(false),
lock_manager: Arc::new(rustfs_lock::GlobalLockManager::new()),
lock_requests: Mutex::new(Vec::new()),
listed_versions: Mutex::new(Vec::new()),
}
}
}
impl CasConfigStore {
fn add_listed_version(&self, object: ObjectInfo) {
self.listed_versions
.lock()
.expect("tier reference fixture should not poison")
.push(object);
}
fn omit_truncated_reference_marker(&self) {
self.truncate_reference_page_without_marker.store(true, Ordering::SeqCst);
}
}
#[async_trait::async_trait]
impl ObjectIO for CasConfigStore {
type Error = Error;
type RangeSpec = HTTPRangeSpec;
type HeaderMap = HeaderMap;
type ObjectOptions = ObjectOptions;
type ObjectInfo = ObjectInfo;
type GetObjectReader = GetObjectReader;
type PutObjectReader = PutObjReader;
async fn get_object_reader(
&self,
bucket: &str,
object: &str,
_range: Option<Self::RangeSpec>,
_headers: Self::HeaderMap,
_opts: &Self::ObjectOptions,
) -> Result<Self::GetObjectReader> {
let state = self.state.lock().await;
let (data, etag) = state.as_ref().ok_or(Error::ConfigNotFound)?;
Ok(GetObjectReader {
stream: Box::new(Cursor::new(data.clone())),
object_info: ObjectInfo {
bucket: bucket.to_string(),
name: object.to_string(),
size: data.len() as i64,
actual_size: data.len() as i64,
etag: Some(etag.clone()),
..Default::default()
},
buffered_body: None,
body_source: Default::default(),
})
}
async fn put_object(
&self,
bucket: &str,
object: &str,
data: &mut Self::PutObjectReader,
opts: &Self::ObjectOptions,
) -> Result<Self::ObjectInfo> {
if self.fail_put.load(Ordering::SeqCst) {
return Err(Error::other("injected tier config save failure"));
}
let mut payload = Vec::new();
tokio::io::AsyncReadExt::read_to_end(&mut data.stream, &mut payload).await?;
let mut state = self.state.lock().await;
match state.as_ref() {
Some((_, etag)) => opts.precondition_check(&ObjectInfo {
etag: Some(etag.clone()),
..Default::default()
})?,
None => {
if opts
.http_preconditions
.as_ref()
.and_then(HTTPPreconditions::if_match_value)
.is_some()
{
return Err(Error::ObjectNotFound(bucket.to_string(), object.to_string()));
}
}
}
let etag = format!("etag-{}", self.next_etag.fetch_add(1, Ordering::SeqCst) + 1);
*state = Some((payload.clone(), etag.clone()));
Ok(ObjectInfo {
bucket: bucket.to_string(),
name: object.to_string(),
size: payload.len() as i64,
actual_size: payload.len() as i64,
etag: Some(etag),
..Default::default()
})
}
}
#[async_trait::async_trait]
impl BucketOperations for CasConfigStore {
type Error = Error;
async fn make_bucket(
&self,
_bucket: &str,
_opts: &crate::storage_api_contracts::bucket::MakeBucketOptions,
) -> Result<()> {
Err(Error::NotImplemented)
}
async fn get_bucket_info(
&self,
_bucket: &str,
_opts: &crate::storage_api_contracts::bucket::BucketOptions,
) -> Result<crate::storage_api_contracts::bucket::BucketInfo> {
Err(Error::NotImplemented)
}
async fn list_bucket(
&self,
_opts: &crate::storage_api_contracts::bucket::BucketOptions,
) -> Result<Vec<crate::storage_api_contracts::bucket::BucketInfo>> {
let mut seen = HashSet::new();
let mut buckets = Vec::new();
for object in self
.listed_versions
.lock()
.expect("tier reference fixture should not poison")
.iter()
{
if seen.insert(object.bucket.clone()) {
buckets.push(crate::storage_api_contracts::bucket::BucketInfo {
name: object.bucket.clone(),
..Default::default()
});
}
}
Ok(buckets)
}
async fn delete_bucket(
&self,
_bucket: &str,
_opts: &crate::storage_api_contracts::bucket::DeleteBucketOptions,
) -> Result<()> {
Err(Error::NotImplemented)
}
}
#[async_trait::async_trait]
impl ListOperations for CasConfigStore {
type Error = Error;
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = TierReferenceProofWalkOptions;
type WalkCancellation = tokio_util::sync::CancellationToken;
type WalkResultSender = tokio::sync::mpsc::Sender<StorageObjectInfoOrErr<ObjectInfo, Error>>;
async fn list_objects_v2(
self: Arc<Self>,
_bucket: &str,
_prefix: &str,
_continuation_token: Option<String>,
_delimiter: Option<String>,
_max_keys: i32,
_fetch_owner: bool,
_start_after: Option<String>,
_incl_deleted: bool,
) -> Result<Self::ListObjectsV2Info> {
Ok(StorageListObjectsV2Info::default())
}
async fn list_object_versions(
self: Arc<Self>,
bucket: &str,
prefix: &str,
marker: Option<String>,
version_marker: Option<String>,
_delimiter: Option<String>,
max_keys: i32,
) -> Result<Self::ListObjectVersionsInfo> {
let mut objects: Vec<_> = self
.listed_versions
.lock()
.expect("tier reference fixture should not poison")
.iter()
.filter(|object| object.bucket == bucket && object.name.starts_with(prefix))
.cloned()
.collect();
objects.sort_by(|left, right| tier_test_object_marker(left).cmp(&tier_test_object_marker(right)));
if marker.is_some() || version_marker.is_some() {
let marker = (marker.unwrap_or_default(), version_marker.unwrap_or_default());
objects.retain(|object| tier_test_object_marker(object) > marker);
}
let limit = match usize::try_from(max_keys) {
Ok(limit) => limit,
Err(_) => 0,
};
let is_truncated = objects.len() > limit;
if is_truncated {
objects.truncate(limit);
}
let (mut next_marker, next_version_idmarker) = if is_truncated {
objects
.last()
.map(|object| (Some(object.name.clone()), object.version_id.map(|version| version.to_string())))
.unwrap_or((None, None))
} else {
(None, None)
};
if is_truncated && self.truncate_reference_page_without_marker.load(Ordering::SeqCst) {
next_marker = None;
}
Ok(StorageListObjectVersionsInfo {
is_truncated,
next_marker,
next_version_idmarker,
objects,
prefixes: Vec::new(),
})
}
async fn walk(
self: Arc<Self>,
_rx: Self::WalkCancellation,
_bucket: &str,
_prefix: &str,
_result: Self::WalkResultSender,
_opts: Self::WalkOptions,
) -> Result<()> {
Err(Error::NotImplemented)
}
}
fn tier_test_object_marker(object: &ObjectInfo) -> (String, String) {
(
object.name.clone(),
object.version_id.map(|version| version.to_string()).unwrap_or_default(),
)
}
fn transitioned_tier_object(
bucket: &str,
object: &str,
tier_name: &str,
backend_identity: Option<TierDestinationId>,
) -> ObjectInfo {
let mut user_defined = HashMap::new();
if let Some(identity) = backend_identity {
rustfs_utils::http::metadata_compat::insert_str(
&mut user_defined,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
rustfs_utils::crypto::hex(identity),
);
}
ObjectInfo {
bucket: bucket.to_string(),
name: object.to_string(),
user_defined: Arc::new(user_defined),
transitioned_object: crate::storage_api_contracts::lifecycle::TransitionedObject {
name: object.to_string(),
tier: tier_name.to_string(),
status: rustfs_filemeta::TRANSITION_COMPLETE.to_string(),
..Default::default()
},
..Default::default()
}
}
#[async_trait::async_trait]
impl NamespaceLocking for CasConfigStore {
type Error = Error;
type NamespaceLock = rustfs_lock::NamespaceLockWrapper;
async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<Self::NamespaceLock> {
self.lock_requests
.lock()
.expect("tier config lock request log should not poison")
.push((bucket.to_string(), object.to_string()));
let lock =
rustfs_lock::NamespaceLock::with_local_manager("tier-config-update-test".to_string(), self.lock_manager.clone());
Ok(rustfs_lock::NamespaceLockWrapper::new(
lock,
rustfs_lock::ObjectKey::new(bucket.to_string(), object.to_string()),
"tier-config-update-test-owner".to_string(),
))
}
}
#[tokio::test]
async fn remove_and_clear_full_update_paths_preserve_force() {
let remove_store = Arc::new(CasConfigStore::default());
let mut persisted = empty_mgr();
persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
persisted
.save_tiering_config_if_current(remove_store.clone(), None)
.await
.expect("remove fixture should persist");
let remove_manager = TierConfigMgr::new();
{
let mut guard = remove_manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::in_use("remove"));
}
TierConfigMgr::remove_and_save_with(&remove_manager, remove_store.clone(), "COLD-A", true)
.await
.expect("forced remove must complete through load, mutate, save, and publish");
assert!(!remove_manager.read().await.tiers.contains_key("COLD-A"));
assert!(
load_tier_config_for_update(remove_store)
.await
.expect("removed config should reload")
.0
.tiers
.is_empty(),
"forced remove must persist the empty candidate"
);
let clear_store = Arc::new(CasConfigStore::default());
persisted
.save_tiering_config_if_current(clear_store.clone(), None)
.await
.expect("clear fixture should persist");
let clear_manager = TierConfigMgr::new();
{
let mut guard = clear_manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::in_use("clear"));
}
TierConfigMgr::clear_and_save_with(&clear_manager, clear_store.clone(), true)
.await
.expect("forced clear must complete through load, mutate, save, and publish");
assert!(clear_manager.read().await.tiers.is_empty());
assert!(
load_tier_config_for_update(clear_store)
.await
.expect("cleared config should reload")
.0
.tiers
.is_empty(),
"forced clear must persist the empty candidate"
);
}
#[tokio::test]
async fn zero_reference_proof_blocks_remove_before_config_save() {
let store = Arc::new(CasConfigStore::default());
let tier = build_rustfs_tier("COLD-A");
let identity = tier_backend_identity(&tier).expect("test tier identity should encode");
let mut persisted = empty_mgr();
persisted.tiers.insert("COLD-A".to_string(), tier.clone_with_credentials());
persisted
.save_tiering_config_if_current(store.clone(), None)
.await
.expect("reference proof fixture should persist");
store.add_listed_version(transitioned_tier_object("photos", "2026/a.jpg", "COLD-A", Some(identity)));
let manager = TierConfigMgr::new();
manager.write().await.tiers.insert("COLD-A".to_string(), tier);
let err = TierConfigMgr::remove_and_save_with(&manager, store.clone(), "COLD-A", true)
.await
.expect_err("authoritative object reference must block tier removal");
match err {
TierConfigUpdateError::Publish(err) => {
assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
assert!(err.message.contains("photos/2026/a.jpg"), "{}", err.message);
}
other => panic!("reference proof failures must be surfaced as publish errors: {other:?}"),
}
assert!(manager.read().await.tiers.contains_key("COLD-A"));
assert!(
load_tier_config_for_update(store)
.await
.expect("blocked config should still reload")
.0
.tiers
.contains_key("COLD-A"),
"blocked removal must not persist the empty candidate"
);
}
#[tokio::test]
async fn zero_reference_proof_blocks_clear_before_config_save() {
let store = Arc::new(CasConfigStore::default());
let tier_a = build_rustfs_tier("COLD-A");
let tier_b = build_rustfs_tier("COLD-B");
let identity_b = tier_backend_identity(&tier_b).expect("test tier identity should encode");
let mut persisted = empty_mgr();
persisted.tiers.insert("COLD-A".to_string(), tier_a.clone_with_credentials());
persisted.tiers.insert("COLD-B".to_string(), tier_b.clone_with_credentials());
persisted
.save_tiering_config_if_current(store.clone(), None)
.await
.expect("reference proof fixture should persist");
store.add_listed_version(transitioned_tier_object("photos", "2026/b.jpg", "COLD-B", Some(identity_b)));
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
guard.tiers.insert("COLD-A".to_string(), tier_a);
guard.tiers.insert("COLD-B".to_string(), tier_b);
}
let err = TierConfigMgr::clear_and_save_with(&manager, store.clone(), true)
.await
.expect_err("authoritative object reference must block tier clear");
match err {
TierConfigUpdateError::Publish(err) => {
assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
assert!(err.message.contains("photos/2026/b.jpg"), "{}", err.message);
}
other => panic!("reference proof failures must be surfaced as publish errors: {other:?}"),
}
let guard = manager.read().await;
assert!(guard.tiers.contains_key("COLD-A"));
assert!(guard.tiers.contains_key("COLD-B"));
drop(guard);
let reloaded = load_tier_config_for_update(store)
.await
.expect("blocked config should still reload")
.0;
assert!(reloaded.tiers.contains_key("COLD-A"));
assert!(reloaded.tiers.contains_key("COLD-B"));
}
#[tokio::test]
async fn zero_reference_proof_rebind_allows_only_new_destination_references() {
let current = build_rustfs_tier("COLD-A");
let current_identity = tier_backend_identity(&current).expect("current identity should encode");
let mut replacement = build_rustfs_tier("COLD-A");
replacement.rustfs.as_mut().expect("replacement payload should exist").prefix = "new-prefix".to_string();
let replacement_identity = tier_backend_identity(&replacement).expect("replacement identity should encode");
assert_ne!(current_identity, replacement_identity);
let target = TierMutationIntentTarget {
tier_name: "COLD-A".to_string(),
old_backend_identity: Some(current_identity),
new_backend_identity: Some(replacement_identity),
};
let store = Arc::new(CasConfigStore::default());
store.add_listed_version(transitioned_tier_object(
"photos",
"2026/new-destination.jpg",
"COLD-A",
Some(replacement_identity),
));
ensure_no_authoritative_tier_object_references(store.clone(), std::slice::from_ref(&target))
.await
.expect("references already written with the new destination identity should not block rebind");
store.add_listed_version(transitioned_tier_object(
"photos",
"2026/old-destination.jpg",
"COLD-A",
Some(current_identity),
));
let err = ensure_no_authoritative_tier_object_references(store, &[target])
.await
.expect_err("references to the old destination identity must block rebind");
assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
assert!(err.message.contains("photos/2026/old-destination.jpg"), "{}", err.message);
}
#[tokio::test]
async fn zero_reference_proof_fails_closed_when_version_listing_marker_is_missing() {
let current = build_rustfs_tier("COLD-A");
let current_identity = tier_backend_identity(&current).expect("current identity should encode");
let target = TierMutationIntentTarget {
tier_name: "COLD-A".to_string(),
old_backend_identity: Some(current_identity),
new_backend_identity: None,
};
let store = Arc::new(CasConfigStore::default());
for index in 0..=TIER_REFERENCE_PROOF_LIST_LIMIT {
store.add_listed_version(ObjectInfo {
bucket: "photos".to_string(),
name: format!("safe/{index:04}.jpg"),
..Default::default()
});
}
store.omit_truncated_reference_marker();
let err = ensure_no_authoritative_tier_object_references(store, &[target])
.await
.expect_err("truncated authoritative reference scan without a marker must fail closed");
assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
assert!(
err.message.contains("truncated without a next marker"),
"unexpected reference proof error: {}",
err.message
);
}
#[test]
fn zero_reference_proof_rebind_identity_boundaries_fail_closed() {
let tier = build_rustfs_tier("COLD-A");
let current_identity = tier_backend_identity(&tier).expect("current identity should encode");
let replacement_identity =
tier_backend_identity(&build_azure_tier("account-a")).expect("replacement identity should encode");
let target = TierMutationIntentTarget {
tier_name: "COLD-A".to_string(),
old_backend_identity: Some(current_identity),
new_backend_identity: Some(current_identity),
};
let object = transitioned_tier_object("photos", "2026/a.jpg", "COLD-A", Some(current_identity));
assert!(!tier_object_blocks_target_rebind(&object, &target).expect("matching identity should parse"));
let target = TierMutationIntentTarget {
tier_name: "COLD-A".to_string(),
old_backend_identity: Some(current_identity),
new_backend_identity: Some(replacement_identity),
};
assert!(tier_object_blocks_target_rebind(&object, &target).expect("mismatched identity should parse"));
let object_without_identity = transitioned_tier_object("photos", "2026/b.jpg", "COLD-A", None);
assert!(
tier_object_blocks_target_rebind(&object_without_identity, &target)
.expect("missing identity means unknown reference owner")
);
let mut pending = transitioned_tier_object("photos", "2026/c.jpg", "COLD-A", Some(current_identity));
pending.transitioned_object.status = "pending".to_string();
assert!(!tier_object_blocks_target_rebind(&pending, &target).expect("non-complete status should not block"));
}
#[tokio::test]
async fn mutation_target_proof_uses_persisted_config_snapshot_not_stale_manager() {
let manager = TierConfigMgr::new();
let store = Arc::new(CasConfigStore::default());
let mut candidate = empty_mgr();
candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
candidate.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B"));
let update = TierConfigMgr::admin_update_lock(&manager).await;
TierConfigMgr::update_candidate_owned(
&manager,
store.clone(),
candidate,
None,
TierCandidateMutation::Remove("COLD-A".to_string(), true),
update,
None,
)
.await
.expect("authoritative proof should use the loaded config even when the local manager is stale");
assert!(!manager.read().await.tiers.contains_key("COLD-A"));
assert!(manager.read().await.tiers.contains_key("COLD-B"));
let reloaded = load_tier_config_for_update(store)
.await
.expect("removed config should reload")
.0;
assert!(!reloaded.tiers.contains_key("COLD-A"));
assert!(reloaded.tiers.contains_key("COLD-B"));
}
#[test]
fn add_target_proof_ignores_unchanged_persisted_tiers_when_manager_is_stale() {
let mut current = empty_mgr();
current.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
let mut candidate = empty_mgr();
candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
candidate.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B"));
let targets = TierCandidateMutation::Add(build_rustfs_tier("COLD-B"), true)
.affected_targets(&current, &candidate)
.expect("add proof should ignore unchanged durable tiers");
assert_eq!(targets.len(), 1);
assert_eq!(targets[0].tier_name, "COLD-B");
assert!(targets[0].old_backend_identity.is_none());
assert!(targets[0].new_backend_identity.is_some());
}
#[tokio::test]
async fn update_with_config_lock_add_uses_loaded_snapshot_for_target_proof() {
let manager = TierConfigMgr::new();
let store = Arc::new(CasConfigStore::default());
let mut persisted = empty_mgr();
persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
persisted
.save_tiering_config_if_current(store.clone(), None)
.await
.expect("add fixture should persist");
TierConfigMgr::update_candidate_with_config_lock(
&manager,
store.clone(),
TierCandidateMutation::Add(build_rustfs_tier("COLD-B"), true),
)
.await
.expect("add proof must ignore unchanged durable tiers missing from the stale manager");
let reloaded = load_tier_config_for_update(store)
.await
.expect("updated config should reload")
.0;
assert!(reloaded.tiers.contains_key("COLD-A"));
assert!(reloaded.tiers.contains_key("COLD-B"));
}
#[tokio::test]
async fn failed_owned_update_restores_generation_and_reports_save_error() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let store = Arc::new(CasConfigStore::default());
store.fail_put.store(true, Ordering::SeqCst);
let mut candidate = empty_mgr();
candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
candidate
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("candidate")));
let update = TierConfigMgr::admin_update_lock(&manager).await;
let err = TierConfigMgr::update_candidate_owned(
&manager,
store.clone(),
candidate,
None,
TierCandidateMutation::Remove("COLD-A".to_string(), true),
update,
None,
)
.await
.expect_err("save failure must be observable to the admin caller");
assert!(matches!(err, TierConfigUpdateError::Save(_)));
assert!(manager.read().await.tiers.contains_key("COLD-A"));
let restored = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("the old generation must be restored after save failure");
assert!(restored.is_current(&manager).await);
drop(restored);
let guard = manager.read().await;
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
assert!(!lock_unpoisoned(&runtime).draining.contains_key("COLD-A"));
drop(guard);
store.fail_put.store(false, Ordering::SeqCst);
let mut retry = empty_mgr();
retry.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
retry
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("retry")));
let update = TierConfigMgr::admin_update_lock(&manager).await;
TierConfigMgr::update_candidate_owned(
&manager,
store,
retry,
None,
TierCandidateMutation::Remove("COLD-A".to_string(), true),
update,
None,
)
.await
.expect("a later update must succeed after save failure recovery");
assert!(!manager.read().await.tiers.contains_key("COLD-A"));
}
#[tokio::test]
async fn cancelled_owned_update_finishes_after_lease_drain() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("old generation lease should be available");
let store = Arc::new(CasConfigStore::default());
let mut candidate = empty_mgr();
candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
candidate
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("candidate")));
let update = TierConfigMgr::admin_update_lock(&manager).await;
let update_manager = manager.clone();
let update_store = store.clone();
let caller = tokio::spawn(async move {
TierConfigMgr::update_candidate_owned(
&update_manager,
update_store,
candidate,
None,
TierCandidateMutation::Remove("COLD-A".to_string(), true),
update,
None,
)
.await
});
tokio::time::timeout(Duration::from_secs(1), async {
while old.is_current(&manager).await {
tokio::task::yield_now().await;
}
})
.await
.expect("owned update should revoke before caller cancellation");
caller.abort();
drop(old);
tokio::time::timeout(Duration::from_secs(1), async {
while manager.read().await.tiers.contains_key("COLD-A") {
tokio::task::yield_now().await;
}
})
.await
.expect("detached update must finish after its operation lease drains");
let guard = manager.read().await;
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
assert!(!lock_unpoisoned(&runtime).draining.contains_key("COLD-A"));
assert!(store.state.lock().await.is_some(), "detached update must persist its result");
}
#[tokio::test]
async fn config_write_lock_survives_cancelled_wrapped_update_until_detached_publish_finishes() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("old generation lease should be available");
let store = Arc::new(CasConfigStore::default());
let mut candidate = empty_mgr();
candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
candidate
.save_tiering_config_if_current(store.clone(), None)
.await
.expect("tier config fixture should persist");
let update_manager = manager.clone();
let update_store = store.clone();
let caller = tokio::spawn(async move {
TierConfigMgr::update_candidate_with_config_lock(
&update_manager,
update_store,
TierCandidateMutation::Remove("COLD-A".to_string(), true),
)
.await
});
tokio::time::timeout(Duration::from_secs(1), async {
while old.is_current(&manager).await {
tokio::task::yield_now().await;
}
})
.await
.expect("owned update should revoke before caller cancellation");
caller.abort();
let config_file = tier_config_lock_path();
let competing_lock = store
.new_ns_lock(RUSTFS_META_BUCKET, &config_file)
.await
.expect("competing tier config lock should be created");
let err = competing_lock
.get_write_lock(Duration::from_millis(20))
.await
.expect_err("detached update must keep the config write lock until it finishes");
assert!(matches!(err, rustfs_lock::LockError::Timeout { .. }));
drop(old);
tokio::time::timeout(Duration::from_secs(1), async {
while manager.read().await.tiers.contains_key("COLD-A") {
tokio::task::yield_now().await;
}
})
.await
.expect("detached update must finish after its operation lease drains");
competing_lock
.get_write_lock(Duration::from_secs(1))
.await
.expect("tier config lock should release after detached update finishes");
}
#[tokio::test]
#[serial_test::serial]
async fn tier_config_update_waits_for_config_namespace_lock_before_loading() {
let manager = TierConfigMgr::new();
let store = Arc::new(CasConfigStore::default());
let config_file = tier_config_lock_path();
let outer_lock = store
.new_ns_lock(RUSTFS_META_BUCKET, &config_file)
.await
.expect("outer tier config lock should be created");
let _outer_guard = outer_lock
.get_write_lock(Duration::from_secs(1))
.await
.expect("outer tier config lock should be acquired");
let result = temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
TierConfigMgr::update_candidate_with_config_lock(&manager, store.clone(), TierCandidateMutation::Clear(true)).await
})
.await;
let TierConfigUpdateError::Load(err) = result.expect_err("config mutation must wait behind the config namespace lock")
else {
panic!("config lock contention should fail before candidate load or mutation");
};
let rendered = err.to_string();
assert!(rendered.to_ascii_lowercase().contains("timeout"), "{rendered}");
assert!(rendered.contains(&config_file), "{rendered}");
assert_eq!(
store
.lock_requests
.lock()
.expect("tier config lock request log should not poison")
.as_slice(),
&[
(RUSTFS_META_BUCKET.to_string(), config_file.clone()),
(RUSTFS_META_BUCKET.to_string(), config_file),
]
);
}
#[tokio::test]
async fn panicked_owned_update_restores_generation_and_reports_error() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let store = Arc::new(CasConfigStore::default());
let mut candidate = empty_mgr();
candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
candidate
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::panicking_in_use("panic")));
let update = TierConfigMgr::admin_update_lock(&manager).await;
let err = TierConfigMgr::update_candidate_owned(
&manager,
store,
candidate,
None,
TierCandidateMutation::Remove("COLD-A".to_string(), false),
update,
None,
)
.await
.expect_err("mutation panic must be observable to the admin caller");
let TierConfigUpdateError::Publish(err) = err else {
panic!("mutation panic should be reported as a publish failure");
};
assert!(err.message.contains("panicked"));
assert!(manager.read().await.tiers.contains_key("COLD-A"));
let restored = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("the old generation must be restored after panic");
assert!(restored.is_current(&manager).await);
drop(restored);
let guard = manager.read().await;
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
assert!(!lock_unpoisoned(&runtime).draining.contains_key("COLD-A"));
}
#[tokio::test]
async fn stale_tier_config_etag_allows_only_one_candidate_to_publish() {
let store = Arc::new(CasConfigStore::default());
empty_mgr()
.save_tiering_config_if_current(store.clone(), None)
.await
.expect("initial conditional create should succeed");
let (mut candidate_a, version_a) = load_tier_config_for_update(store.clone())
.await
.expect("first node should load config revision");
let (mut candidate_b, version_b) = load_tier_config_for_update(store.clone())
.await
.expect("second node should load the same config revision");
assert_eq!(version_a, version_b);
candidate_a.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
candidate_b.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B"));
let save_a = candidate_a.save_tiering_config_if_current(store.clone(), version_a.as_deref());
let save_b = candidate_b.save_tiering_config_if_current(store, version_b.as_deref());
let (result_a, result_b) = tokio::join!(save_a, save_b);
assert_ne!(result_a.is_ok(), result_b.is_ok(), "exactly one stale-ETag writer must win");
let manager_a = TierConfigMgr::new();
let manager_b = TierConfigMgr::new();
if result_a.is_ok() {
TierConfigMgr::publish_candidate(&manager_a, candidate_a, None)
.await
.expect("winning node should publish");
}
if result_b.is_ok() {
TierConfigMgr::publish_candidate(&manager_b, candidate_b, None)
.await
.expect("winning node should publish");
}
assert_ne!(manager_a.read().await.empty(), manager_b.read().await.empty());
}
#[test]
fn legacy_refresh_method_signature_remains_callable() {
async fn call_legacy_refresh(manager: &mut TierConfigMgr, api: Arc<ECStore>) {
manager.refresh_tier_config(api).await;
}
let _ = call_legacy_refresh;
}
#[tokio::test(start_paused = true)]
async fn periodic_refresh_timer_does_not_tick_on_startup() {
let period = Duration::from_secs(60);
let mut timer = delayed_tier_refresh_interval(period);
assert!(
tokio::time::timeout(Duration::ZERO, timer.tick()).await.is_err(),
"periodic reload must not duplicate the startup reload"
);
tokio::time::advance(period).await;
tokio::time::timeout(Duration::ZERO, timer.tick())
.await
.expect("the first periodic reload should become ready after one interval");
}
}
Reference in New Issue View Git Blame Copy Permalink