gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 08:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
21049401fa84542b2696d609381e4c4564e7ddbc
rustfs/crates/ecstore/src/services/tier/tier.rs
T
houseme 21049401fa fix(ilm): harden tier transition failure boundaries (#5031) 
* fix(tier): fence generation-scoped operations

Refs rustfs/backlog#1354

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(ilm): verify transition upload streams

Refs rustfs/backlog#1353

Co-Authored-By: heihutu <heihutu@gmail.com>

* test(ecstore): expand transition fault matrix

Refs rustfs/backlog#1355

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>
2026-07-19 10:48:32 +00:00

5093 lines
193 KiB
Rust
Raw Blame History

// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use byteorder::{ByteOrder, LittleEndian};
use bytes::Bytes;
use futures::FutureExt;
use http::HeaderMap;
use http::status::StatusCode;
use lazy_static::lazy_static;
use rand::{Rng, RngExt};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::{
collections::{HashMap, HashSet, hash_map::Entry},
io::{self, Cursor},
ops::Deref,
panic::AssertUnwindSafe,
sync::{
Arc, LazyLock, Mutex, MutexGuard, RwLock as StdRwLock, Weak,
atomic::{AtomicBool, AtomicUsize, Ordering},
},
time::Duration,
};
use time::OffsetDateTime;
use tokio::io::BufReader;
use tokio::{
select,
sync::RwLock,
time::{Instant, interval, interval_at, timeout_at},
};
use tracing::{debug, error, info, warn};
use crate::client::admin_handler_utils::AdminError;
use crate::error::{Error, Result, StorageError};
use crate::services::tier::{
tier_admin::TierCreds,
tier_config::{TierConfig, TierType},
tier_handlers::{ERR_TIER_ALREADY_EXISTS, ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_NOT_FOUND, ERR_TIER_RESERVED_NAME},
warm_backend::{WarmBackend, check_warm_backend, new_warm_backend},
};
use crate::storage_api_contracts::{
object::{
DeletedObject, EcstoreObjectIO, EcstoreObjectOperations, HTTPPreconditions, ObjectIO, ObjectOperations, ObjectToDelete,
},
range::HTTPRangeSpec,
};
use crate::{
config::com::{CONFIG_PREFIX, read_config, read_config_with_metadata},
disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET},
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
runtime::sources as runtime_sources,
store::ECStore,
};
use rustfs_filemeta::FileInfo;
use rustfs_rio::HashReader;
use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
use s3s::S3ErrorCode;
use super::{
tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_PERM_ERR},
warm_backend::WarmBackendImpl,
};
const TIER_CFG_REFRESH: Duration = Duration::from_secs(15 * 60);
const TIER_OPERATION_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
const TIER_REMOTE_VALIDATION_TIMEOUT: Duration = Duration::from_secs(30);
fn delayed_tier_refresh_interval(period: Duration) -> tokio::time::Interval {
interval_at(Instant::now() + period, period)
}
#[cfg(test)]
struct TierDriverBuildBarrier {
tier_name: String,
arrived: tokio::sync::Notify,
release: tokio::sync::Semaphore,
}
#[cfg(test)]
static TIER_DRIVER_BUILD_BARRIER: LazyLock<Mutex<Option<Arc<TierDriverBuildBarrier>>>> = LazyLock::new(|| Mutex::new(None));
#[cfg(test)]
struct TierDriverBuildBarrierGuard;
#[cfg(test)]
impl Drop for TierDriverBuildBarrierGuard {
fn drop(&mut self) {
*lock_unpoisoned(&TIER_DRIVER_BUILD_BARRIER) = None;
}
}
#[cfg(test)]
fn install_tier_driver_build_barrier(tier_name: &str) -> (Arc<TierDriverBuildBarrier>, TierDriverBuildBarrierGuard) {
let barrier = Arc::new(TierDriverBuildBarrier {
tier_name: tier_name.to_string(),
arrived: tokio::sync::Notify::new(),
release: tokio::sync::Semaphore::new(0),
});
*lock_unpoisoned(&TIER_DRIVER_BUILD_BARRIER) = Some(barrier.clone());
(barrier, TierDriverBuildBarrierGuard)
}
async fn build_warm_backend(tier: &TierConfig, probe: bool) -> std::result::Result<WarmBackendImpl, AdminError> {
#[cfg(test)]
let test_barrier = { lock_unpoisoned(&TIER_DRIVER_BUILD_BARRIER).clone() };
#[cfg(test)]
if let Some(barrier) = test_barrier
&& barrier.tier_name == tier.name
{
barrier.arrived.notify_one();
barrier
.release
.acquire()
.await
.expect("tier driver build test barrier should stay open")
.forget();
}
new_warm_backend(tier, probe).await
}
const TIER_CONFIG_LEGACY_FILE: &str = "tier-config.json";
pub const TIER_CONFIG_FILE: &str = "tier-config.bin";
pub const TIER_CONFIG_FORMAT: u16 = 1;
pub const TIER_CONFIG_V1: u16 = 1;
pub const TIER_CONFIG_VERSION: u16 = 2;
const EXTERNAL_TIER_TYPE_UNSUPPORTED: i32 = 0;
const EXTERNAL_TIER_TYPE_S3: i32 = 1;
const EXTERNAL_TIER_TYPE_AZURE: i32 = 2;
const EXTERNAL_TIER_TYPE_GCS: i32 = 3;
const EXTERNAL_TIER_TYPE_MINIO: i32 = 4;
const TIER_BACKEND_IDENTITY_VERSION: u8 = 2;
const _TIER_CFG_REFRESH_AT_HDR: &str = "X-RustFS-TierCfg-RefreshedAt";
lazy_static! {
pub static ref ERR_TIER_MISSING_CREDENTIALS: AdminError = AdminError {
code: "XRustFSAdminTierMissingCredentials".to_string(),
message: "Specified remote credentials are empty".to_string(),
status_code: StatusCode::FORBIDDEN,
};
pub static ref ERR_TIER_BACKEND_IN_USE: AdminError = AdminError {
code: "XRustFSAdminTierBackendInUse".to_string(),
message: "Specified remote tier is already in use".to_string(),
status_code: StatusCode::CONFLICT,
};
pub static ref ERR_TIER_TYPE_UNSUPPORTED: AdminError = AdminError {
code: "XRustFSAdminTierTypeUnsupported".to_string(),
message: "Specified tier type is unsupported".to_string(),
status_code: StatusCode::BAD_REQUEST,
};
pub static ref ERR_TIER_BACKEND_NOT_EMPTY: AdminError = AdminError {
code: "XRustFSAdminTierBackendNotEmpty".to_string(),
message: "Specified remote backend is not empty".to_string(),
status_code: StatusCode::BAD_REQUEST,
};
pub static ref ERR_TIER_INVALID_CONFIG: AdminError = AdminError {
code: "XRustFSAdminTierInvalidConfig".to_string(),
message: "Unable to setup remote tier, check tier configuration".to_string(),
status_code: StatusCode::BAD_REQUEST,
};
}
#[derive(Serialize, Deserialize)]
pub struct TierConfigMgr {
#[serde(skip)]
pub driver_cache: HashMap<String, WarmBackendImpl>,
pub tiers: HashMap<String, TierConfig>,
pub last_refreshed_at: OffsetDateTime,
}
type SharedWarmBackend = Arc<dyn WarmBackend + Send + Sync + 'static>;
#[derive(Default)]
struct TierDriverRuntime {
generations: HashMap<String, Arc<TierDriverGeneration>>,
draining: HashMap<String, u64>,
next_generation: u64,
next_drain_epoch: u64,
admin_updates: Arc<tokio::sync::Mutex<()>>,
}
struct TierDriverRegistryEntry {
manager: Weak<RwLock<TierConfigMgr>>,
runtime: Arc<Mutex<TierDriverRuntime>>,
}
struct TierPublishTransition {
runtime: Arc<Mutex<TierDriverRuntime>>,
tokens: Vec<(String, u64)>,
revoked: HashMap<String, Arc<TierDriverGeneration>>,
replaced_destinations: HashMap<String, TierConfig>,
published: bool,
}
struct PreparedTierDriver {
tier_name: String,
config_fingerprint: TierDriverFingerprint,
backend_identity: TierDestinationId,
driver: SharedWarmBackend,
}
impl TierPublishTransition {
async fn wait_for_active_leases(&self) -> std::result::Result<(), AdminError> {
let deadline = Instant::now() + TIER_OPERATION_DRAIN_TIMEOUT;
for generation in self.revoked.values() {
if timeout_at(deadline, generation.wait_for_no_active_leases()).await.is_err() {
let mut err = ERR_TIER_BACKEND_IN_USE.clone();
err.message = "Timed out waiting for active remote tier operations to finish".to_string();
return Err(err);
}
}
Ok(())
}
async fn ensure_replaced_destinations_are_empty(&self) -> std::result::Result<(), AdminError> {
ensure_replaced_destinations_are_empty(&self.replaced_destinations, &self.revoked).await
}
}
async fn ensure_replaced_destinations_are_empty(
replaced_destinations: &HashMap<String, TierConfig>,
revoked: &HashMap<String, Arc<TierDriverGeneration>>,
) -> std::result::Result<(), AdminError> {
let deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT;
for (tier_name, old_config) in replaced_destinations {
let prepared;
let driver = match revoked.get(tier_name) {
Some(generation) => generation.driver.as_ref(),
None => {
prepared = match timeout_at(deadline, build_warm_backend(old_config, false)).await {
Ok(result) => result?,
Err(_) => {
let mut err = ERR_TIER_BACKEND_IN_USE.clone();
err.message = "Timed out preparing the replaced remote tier backend".to_string();
return Err(err);
}
};
prepared.as_ref()
}
};
match timeout_at(deadline, driver.in_use()).await {
Ok(Ok(false)) => {}
Ok(Ok(true)) => return Err(ERR_TIER_BACKEND_NOT_EMPTY.clone()),
Ok(Err(err)) => {
let mut admin_err = ERR_TIER_PERM_ERR.clone();
admin_err.message.push('.');
admin_err.message.push_str(&err.to_string());
return Err(admin_err);
}
Err(_) => {
let mut err = ERR_TIER_BACKEND_IN_USE.clone();
err.message = "Timed out checking whether the replaced remote tier backend is empty".to_string();
return Err(err);
}
}
}
Ok(())
}
impl Drop for TierPublishTransition {
fn drop(&mut self) {
let mut runtime = lock_unpoisoned(&self.runtime);
for (tier_name, epoch) in &self.tokens {
if runtime.draining.get(tier_name) == Some(epoch) {
if !self.published
&& !runtime.generations.contains_key(tier_name)
&& let Some(generation) = self.revoked.get(tier_name)
{
generation.accepting.store(true, Ordering::Release);
runtime.generations.insert(tier_name.clone(), generation.clone());
}
runtime.draining.remove(tier_name);
}
}
}
}
static TIER_DRIVER_REGISTRY: LazyLock<StdRwLock<HashMap<usize, TierDriverRegistryEntry>>> =
LazyLock::new(|| StdRwLock::new(HashMap::new()));
fn lock_unpoisoned<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
mutex.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}
fn tier_manager_key(manager: &TierConfigMgr) -> usize {
std::ptr::from_ref(manager).addr()
}
fn tier_driver_runtime(handle: &Arc<RwLock<TierConfigMgr>>, manager: &TierConfigMgr) -> Arc<Mutex<TierDriverRuntime>> {
let key = tier_manager_key(manager);
{
let registry = TIER_DRIVER_REGISTRY.read().unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(entry) = registry.get(&key)
&& entry
.manager
.upgrade()
.is_some_and(|registered| Arc::ptr_eq(&registered, handle))
{
return entry.runtime.clone();
}
}
#[cfg(test)]
let _ = TIER_REGISTRY_MISS_SCAN_COUNT.try_with(|count| count.set(count.get() + 1));
let mut registry = TIER_DRIVER_REGISTRY
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(entry) = registry.get(&key)
&& entry
.manager
.upgrade()
.is_some_and(|registered| Arc::ptr_eq(&registered, handle))
{
return entry.runtime.clone();
}
registry.retain(|_, entry| entry.manager.strong_count() > 0);
let runtime = Arc::new(Mutex::new(TierDriverRuntime::default()));
registry.insert(
key,
TierDriverRegistryEntry {
manager: Arc::downgrade(handle),
runtime: runtime.clone(),
},
);
runtime
}
#[cfg(test)]
tokio::task_local! {
static TIER_REGISTRY_MISS_SCAN_COUNT: std::cell::Cell<usize>;
}
fn registered_tier_driver_runtime(manager: &TierConfigMgr) -> Option<Arc<Mutex<TierDriverRuntime>>> {
let key = tier_manager_key(manager);
{
let registry = TIER_DRIVER_REGISTRY.read().unwrap_or_else(std::sync::PoisonError::into_inner);
let entry = registry.get(&key)?;
if entry.manager.strong_count() > 0 {
return Some(entry.runtime.clone());
}
}
let mut registry = TIER_DRIVER_REGISTRY
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if registry.get(&key).is_some_and(|entry| entry.manager.strong_count() == 0) {
registry.remove(&key);
}
None
}
type TierDriverFingerprint = [u8; 32];
pub(crate) type TierDestinationId = [u8; 32];
pub(crate) type DriverRevision = u64;
fn tier_config_fingerprint(tier_name: &str, config: &TierConfig) -> io::Result<TierDriverFingerprint> {
let external = to_external_tier_config(tier_name, config)?;
let encoded = rmp_serde::to_vec_named(&external)
.map_err(|err| io::Error::other(format!("serialize tier driver identity failed: {err}")))?;
Ok(Sha256::digest(encoded).into())
}
fn tier_configs_match(tier_name: &str, current: &TierConfig, replacement: &TierConfig) -> bool {
match (
tier_config_fingerprint(tier_name, current),
tier_config_fingerprint(tier_name, replacement),
) {
(Ok(current), Ok(replacement)) => current == replacement,
_ => false,
}
}
#[derive(Debug)]
pub enum TierConfigUpdateError {
Load(io::Error),
Mutation(AdminError),
Save(io::Error),
Publish(AdminError),
}
enum TierCandidateMutation {
Add(TierConfig, bool),
Edit(String, TierCreds),
Remove(String, bool),
Clear(bool),
}
impl TierCandidateMutation {
fn target_tiers(&self, manager: &TierConfigMgr, candidate: &TierConfigMgr) -> HashSet<String> {
let mut targets = changed_tier_names(manager, candidate);
match self {
Self::Add(config, _) => {
targets.insert(config.name.clone());
}
Self::Edit(tier_name, _) | Self::Remove(tier_name, _) => {
targets.insert(tier_name.clone());
}
Self::Clear(_) => {
targets.extend(manager.tiers.keys().chain(candidate.tiers.keys()).cloned());
}
}
targets
}
async fn apply(self, candidate: &mut TierConfigMgr) -> std::result::Result<Option<String>, AdminError> {
match self {
Self::Add(config, force) => {
let tier_name = config.name.clone();
candidate.add(config, force).await?;
Ok(Some(tier_name))
}
Self::Edit(tier_name, credentials) => {
candidate.edit(&tier_name, credentials).await?;
Ok(Some(tier_name))
}
Self::Remove(tier_name, force) => {
candidate.remove(&tier_name, force).await?;
Ok(None)
}
Self::Clear(force) => {
candidate.clear_tier(force).await?;
Ok(None)
}
}
}
}
async fn apply_tier_candidate_mutation(
mutation: TierCandidateMutation,
candidate: &mut TierConfigMgr,
deadline: Instant,
) -> std::result::Result<Option<String>, AdminError> {
match timeout_at(deadline, mutation.apply(candidate)).await {
Ok(result) => result,
Err(_) => {
let mut err = ERR_TIER_BACKEND_IN_USE.clone();
err.message = "Timed out validating the remote tier mutation".to_string();
Err(err)
}
}
}
fn changed_tier_names(current: &TierConfigMgr, replacement: &TierConfigMgr) -> HashSet<String> {
let mut changed = HashSet::new();
for (tier_name, current_config) in &current.tiers {
if replacement
.tiers
.get(tier_name)
.is_none_or(|replacement_config| !tier_configs_match(tier_name, current_config, replacement_config))
{
changed.insert(tier_name.clone());
}
}
changed.extend(
replacement
.tiers
.keys()
.filter(|tier_name| !current.tiers.contains_key(*tier_name))
.cloned(),
);
changed
}
fn replaced_tier_destinations(
current: &TierConfigMgr,
replacement: &TierConfigMgr,
) -> std::result::Result<HashMap<String, TierConfig>, AdminError> {
let mut replaced = HashMap::new();
for (tier_name, current_config) in &current.tiers {
let changed = match replacement.tiers.get(tier_name) {
Some(replacement_config) => {
let current_identity = tier_backend_identity(current_config).map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})?;
let replacement_identity = tier_backend_identity(replacement_config).map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})?;
current_identity != replacement_identity
}
None => true,
};
if changed {
replaced.insert(tier_name.clone(), current_config.clone_with_credentials());
}
}
Ok(replaced)
}
fn tier_backend_identity(config: &TierConfig) -> io::Result<TierDestinationId> {
let (tier_type, endpoint, bucket, prefix, region, routing_account) = match config.tier_type {
TierType::S3 => config.s3.as_ref().map(|value| {
(
"s3",
value.endpoint.as_str(),
value.bucket.as_str(),
value.prefix.as_str(),
value.region.as_str(),
"",
)
}),
TierType::RustFS => config.rustfs.as_ref().map(|value| {
(
"rustfs",
value.endpoint.as_str(),
value.bucket.as_str(),
value.prefix.as_str(),
value.region.as_str(),
"",
)
}),
TierType::MinIO => config.minio.as_ref().map(|value| {
(
"minio",
value.endpoint.as_str(),
value.bucket.as_str(),
value.prefix.as_str(),
value.region.as_str(),
"",
)
}),
TierType::Aliyun => config.aliyun.as_ref().map(|value| {
(
"aliyun",
value.endpoint.as_str(),
value.bucket.as_str(),
value.prefix.as_str(),
value.region.as_str(),
"",
)
}),
TierType::Tencent => config.tencent.as_ref().map(|value| {
(
"tencent",
value.endpoint.as_str(),
value.bucket.as_str(),
value.prefix.as_str(),
value.region.as_str(),
"",
)
}),
TierType::Huaweicloud => config.huaweicloud.as_ref().map(|value| {
(
"huaweicloud",
value.endpoint.as_str(),
value.bucket.as_str(),
value.prefix.as_str(),
value.region.as_str(),
"",
)
}),
TierType::Azure => config.azure.as_ref().map(|value| {
(
"azure",
value.endpoint.as_str(),
value.bucket.as_str(),
value.prefix.as_str(),
value.region.as_str(),
value.access_key.as_str(),
)
}),
TierType::GCS => config.gcs.as_ref().map(|value| {
(
"gcs",
value.endpoint.as_str(),
value.bucket.as_str(),
value.prefix.as_str(),
value.region.as_str(),
"",
)
}),
TierType::R2 => config.r2.as_ref().map(|value| {
(
"r2",
value.endpoint.as_str(),
value.bucket.as_str(),
value.prefix.as_str(),
value.region.as_str(),
"",
)
}),
TierType::Unsupported => None,
}
.ok_or_else(|| io::Error::other("tier backend identity payload is missing"))?;
let prefix = normalized_tier_prefix(&config.tier_type, prefix);
let encoded = rmp_serde::to_vec(&(
TIER_BACKEND_IDENTITY_VERSION,
tier_type,
endpoint,
bucket,
prefix,
region,
routing_account,
))
.map_err(|err| io::Error::other(format!("serialize tier backend identity failed: {err}")))?;
Ok(Sha256::digest(encoded).into())
}
pub(crate) fn tier_destination_id_from_metadata(metadata: &HashMap<String, String>) -> io::Result<Option<TierDestinationId>> {
let Some(encoded) = rustfs_utils::http::metadata_compat::get_consistent_str(
metadata,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
) else {
if rustfs_utils::http::metadata_compat::contains_key_str(
metadata,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
) {
return Err(io::Error::other(
"transition tier backend identity compatibility keys conflict or are empty",
));
}
return Ok(None);
};
if encoded.len() != 64 {
return Err(io::Error::other("transition tier backend identity has an invalid length"));
}
let mut identity = [0_u8; 32];
for (index, byte) in identity.iter_mut().enumerate() {
*byte = u8::from_str_radix(&encoded[index * 2..index * 2 + 2], 16)
.map_err(|_| io::Error::other("transition tier backend identity is not valid hexadecimal"))?;
}
Ok(Some(identity))
}
fn normalized_tier_prefix<'a>(tier_type: &TierType, prefix: &'a str) -> &'a str {
match tier_type {
TierType::S3 => prefix.trim_matches('/'),
TierType::RustFS
| TierType::MinIO
| TierType::Aliyun
| TierType::Tencent
| TierType::Huaweicloud
| TierType::Azure
| TierType::GCS
| TierType::R2 => prefix.strip_suffix('/').unwrap_or(prefix),
TierType::Unsupported => prefix,
}
}
struct TierDriverGeneration {
tier_name: Arc<str>,
generation: DriverRevision,
// Process-local only: this may reflect credential changes and must never be persisted or logged.
config_fingerprint: TierDriverFingerprint,
// Durable cleanup identity: routing fields only, with all credentials excluded.
backend_identity: TierDestinationId,
driver: SharedWarmBackend,
accepting: AtomicBool,
active_leases: AtomicUsize,
drained: tokio::sync::Notify,
}
struct SharedWarmBackendProxy(SharedWarmBackend);
#[async_trait::async_trait]
impl WarmBackend for SharedWarmBackendProxy {
async fn put(&self, object: &str, r: crate::client::transition_api::ReaderImpl, length: i64) -> io::Result<String> {
self.0.put(object, r, length).await
}
async fn put_with_meta(
&self,
object: &str,
r: crate::client::transition_api::ReaderImpl,
length: i64,
meta: HashMap<String, String>,
) -> io::Result<String> {
self.0.put_with_meta(object, r, length, meta).await
}
async fn get(
&self,
object: &str,
rv: &str,
opts: crate::services::tier::warm_backend::WarmBackendGetOpts,
) -> io::Result<crate::client::transition_api::ReadCloser> {
self.0.get(object, rv, opts).await
}
async fn remove(&self, object: &str, rv: &str) -> io::Result<()> {
self.0.remove(object, rv).await
}
async fn in_use(&self) -> io::Result<bool> {
self.0.in_use().await
}
}
pub(crate) struct TierOperationLease {
inner: Arc<TierDriverGeneration>,
runtime: Arc<Mutex<TierDriverRuntime>>,
}
impl TierOperationLease {
fn try_new(
inner: Arc<TierDriverGeneration>,
runtime: Arc<Mutex<TierDriverRuntime>>,
) -> std::result::Result<Self, AdminError> {
inner
.active_leases
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |active| active.checked_add(1))
.map_err(|_| {
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier operation lease capacity exhausted".to_string();
err
})?;
Ok(Self { inner, runtime })
}
pub(crate) fn try_clone(&self) -> std::result::Result<Self, AdminError> {
Self::try_new(self.inner.clone(), self.runtime.clone())
}
}
impl Drop for TierOperationLease {
fn drop(&mut self) {
let result = self
.inner
.active_leases
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |active| active.checked_sub(1));
match result {
Ok(1) => self.inner.drained.notify_one(),
Ok(_) => {}
Err(_) => {
error!(
tier = self.tier_name(),
tier_generation = self.generation(),
"tier operation lease counter underflow"
);
}
}
}
}
impl Deref for TierOperationLease {
type Target = dyn WarmBackend + Send + Sync + 'static;
fn deref(&self) -> &Self::Target {
self.inner.driver.as_ref()
}
}
impl TierOperationLease {
pub(crate) fn tier_name(&self) -> &str {
&self.inner.tier_name
}
pub(crate) fn generation(&self) -> DriverRevision {
self.inner.generation
}
pub(crate) fn backend_identity(&self) -> TierDestinationId {
self.inner.backend_identity
}
pub(crate) fn is_current_generation(&self) -> bool {
lock_unpoisoned(&self.runtime)
.generations
.get(self.tier_name())
.is_some_and(|current| {
current.generation == self.generation()
&& current.accepting.load(Ordering::Acquire)
&& Arc::ptr_eq(current, &self.inner)
})
}
#[cfg(test)]
async fn is_current(&self, handle: &Arc<RwLock<TierConfigMgr>>) -> bool {
let manager = handle.read().await;
let Some(runtime) = registered_tier_driver_runtime(&manager) else {
return false;
};
if !Arc::ptr_eq(&runtime, &self.runtime) {
return false;
}
self.is_current_generation()
}
}
impl TierDriverGeneration {
async fn wait_for_no_active_leases(&self) {
loop {
let notified = self.drained.notified();
if self.active_leases.load(Ordering::Acquire) == 0 {
return;
}
notified.await;
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierConfigMgr {
#[serde(rename = "Tiers")]
tiers: HashMap<String, ExternalTierConfig>,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierConfig {
#[serde(rename = "Version")]
version: String,
#[serde(rename = "Type")]
tier_type: i32,
#[serde(rename = "Name")]
name: String,
#[serde(rename = "S3")]
s3: Option<ExternalTierS3>,
#[serde(rename = "Azure")]
azure: Option<ExternalTierAzure>,
#[serde(rename = "GCS")]
gcs: Option<ExternalTierGcs>,
#[serde(rename = "MinIO", alias = "Compatible")]
compatible_backend: Option<ExternalTierCompatible>,
#[serde(rename = "XTierType", skip_serializing_if = "Option::is_none")]
tier_type_hint: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierS3 {
#[serde(rename = "Endpoint")]
endpoint: String,
#[serde(rename = "AccessKey")]
access_key: String,
#[serde(rename = "SecretKey")]
secret_key: String,
#[serde(rename = "Bucket")]
bucket: String,
#[serde(rename = "Prefix")]
prefix: String,
#[serde(rename = "Region")]
region: String,
#[serde(rename = "StorageClass")]
storage_class: String,
#[serde(rename = "AWSRole")]
aws_role: bool,
#[serde(rename = "AWSRoleWebIdentityTokenFile")]
aws_role_web_identity_token_file: String,
#[serde(rename = "AWSRoleARN")]
aws_role_arn: String,
#[serde(rename = "AWSRoleSessionName")]
aws_role_session_name: String,
#[serde(rename = "AWSRoleDurationSeconds")]
aws_role_duration_seconds: i32,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalServicePrincipalAuth {
#[serde(rename = "TenantID")]
tenant_id: String,
#[serde(rename = "ClientID")]
client_id: String,
#[serde(rename = "ClientSecret")]
client_secret: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierAzure {
#[serde(rename = "Endpoint")]
endpoint: String,
#[serde(rename = "AccountName")]
account_name: String,
#[serde(rename = "AccountKey")]
account_key: String,
#[serde(rename = "Bucket")]
bucket: String,
#[serde(rename = "Prefix")]
prefix: String,
#[serde(rename = "Region")]
region: String,
#[serde(rename = "StorageClass")]
storage_class: String,
#[serde(rename = "SPAuth")]
sp_auth: ExternalServicePrincipalAuth,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierGcs {
#[serde(rename = "Endpoint")]
endpoint: String,
#[serde(rename = "Creds")]
creds: String,
#[serde(rename = "Bucket")]
bucket: String,
#[serde(rename = "Prefix")]
prefix: String,
#[serde(rename = "Region")]
region: String,
#[serde(rename = "StorageClass")]
storage_class: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierCompatible {
#[serde(rename = "Endpoint")]
endpoint: String,
#[serde(rename = "AccessKey")]
access_key: String,
#[serde(rename = "SecretKey")]
secret_key: String,
#[serde(rename = "Bucket")]
bucket: String,
#[serde(rename = "Prefix")]
prefix: String,
#[serde(rename = "Region")]
region: String,
}
fn tier_config_path(file: &str) -> String {
format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, file)
}
fn tier_hint_for_type(tier_type: TierType) -> Option<&'static str> {
match tier_type {
TierType::RustFS => Some("rustfs"),
TierType::Aliyun => Some("aliyun"),
TierType::Tencent => Some("tencent"),
TierType::Huaweicloud => Some("huaweicloud"),
TierType::R2 => Some("r2"),
_ => None,
}
}
fn tier_type_from_hint(hint: Option<&str>) -> Option<TierType> {
match hint {
Some("rustfs") => Some(TierType::RustFS),
Some("aliyun") => Some(TierType::Aliyun),
Some("tencent") => Some(TierType::Tencent),
Some("huaweicloud") => Some(TierType::Huaweicloud),
Some("r2") => Some(TierType::R2),
_ => None,
}
}
fn external_tier_s3_from_internal(s3: &crate::services::tier::tier_config::TierS3) -> ExternalTierS3 {
ExternalTierS3 {
endpoint: s3.endpoint.clone(),
access_key: s3.access_key.clone(),
secret_key: s3.secret_key.clone(),
bucket: s3.bucket.clone(),
prefix: s3.prefix.clone(),
region: s3.region.clone(),
storage_class: s3.storage_class.clone(),
aws_role: s3.aws_role,
aws_role_web_identity_token_file: s3.aws_role_web_identity_token_file.clone(),
aws_role_arn: s3.aws_role_arn.clone(),
aws_role_session_name: s3.aws_role_session_name.clone(),
aws_role_duration_seconds: s3.aws_role_duration_seconds,
}
}
fn external_tier_s3_from_compatible_payload(
endpoint: String,
access_key: String,
secret_key: String,
bucket: String,
prefix: String,
region: String,
) -> ExternalTierS3 {
ExternalTierS3 {
endpoint,
access_key,
secret_key,
bucket,
prefix,
region,
storage_class: String::new(),
aws_role: false,
aws_role_web_identity_token_file: String::new(),
aws_role_arn: String::new(),
aws_role_session_name: String::new(),
aws_role_duration_seconds: 0,
}
}
fn external_tier_alias_from_compatible_payload(
endpoint: String,
access_key: String,
secret_key: String,
bucket: String,
prefix: String,
region: String,
) -> ExternalTierCompatible {
ExternalTierCompatible {
endpoint,
access_key,
secret_key,
bucket,
prefix,
region,
}
}
fn to_external_tier_config(name: &str, tier: &TierConfig) -> io::Result<ExternalTierConfig> {
let mut out = ExternalTierConfig {
version: if tier.version.is_empty() {
"v1".to_string()
} else {
tier.version.clone()
},
name: if tier.name.is_empty() {
name.to_string()
} else {
tier.name.clone()
},
..Default::default()
};
match tier.tier_type {
TierType::S3 => {
let s3 = tier
.s3
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing s3 backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_S3;
out.s3 = Some(external_tier_s3_from_internal(s3));
}
TierType::Azure => {
let az = tier
.azure
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing azure backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_AZURE;
out.azure = Some(ExternalTierAzure {
endpoint: az.endpoint.clone(),
account_name: az.access_key.clone(),
account_key: az.secret_key.clone(),
bucket: az.bucket.clone(),
prefix: az.prefix.clone(),
region: az.region.clone(),
storage_class: az.storage_class.clone(),
sp_auth: ExternalServicePrincipalAuth {
tenant_id: az.sp_auth.tenant_id.clone(),
client_id: az.sp_auth.client_id.clone(),
client_secret: az.sp_auth.client_secret.clone(),
},
});
}
TierType::GCS => {
let gcs = tier
.gcs
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing gcs backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_GCS;
out.gcs = Some(ExternalTierGcs {
endpoint: gcs.endpoint.clone(),
creds: gcs.creds.clone(),
bucket: gcs.bucket.clone(),
prefix: gcs.prefix.clone(),
region: gcs.region.clone(),
storage_class: gcs.storage_class.clone(),
});
}
TierType::MinIO => {
let backend = tier
.minio
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_MINIO;
out.compatible_backend = Some(external_tier_alias_from_compatible_payload(
backend.endpoint.clone(),
backend.access_key.clone(),
backend.secret_key.clone(),
backend.bucket.clone(),
backend.prefix.clone(),
backend.region.clone(),
));
}
TierType::RustFS => {
let backend = tier
.rustfs
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_S3;
out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
out.s3 = Some(external_tier_s3_from_compatible_payload(
backend.endpoint.clone(),
backend.access_key.clone(),
backend.secret_key.clone(),
backend.bucket.clone(),
backend.prefix.clone(),
backend.region.clone(),
));
}
TierType::Aliyun => {
let backend = tier
.aliyun
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_S3;
out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
out.s3 = Some(external_tier_s3_from_compatible_payload(
backend.endpoint.clone(),
backend.access_key.clone(),
backend.secret_key.clone(),
backend.bucket.clone(),
backend.prefix.clone(),
backend.region.clone(),
));
}
TierType::Tencent => {
let backend = tier
.tencent
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_S3;
out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
out.s3 = Some(external_tier_s3_from_compatible_payload(
backend.endpoint.clone(),
backend.access_key.clone(),
backend.secret_key.clone(),
backend.bucket.clone(),
backend.prefix.clone(),
backend.region.clone(),
));
}
TierType::Huaweicloud => {
let backend = tier
.huaweicloud
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_S3;
out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
out.s3 = Some(external_tier_s3_from_compatible_payload(
backend.endpoint.clone(),
backend.access_key.clone(),
backend.secret_key.clone(),
backend.bucket.clone(),
backend.prefix.clone(),
backend.region.clone(),
));
}
TierType::R2 => {
let backend = tier
.r2
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_S3;
out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
out.s3 = Some(external_tier_s3_from_compatible_payload(
backend.endpoint.clone(),
backend.access_key.clone(),
backend.secret_key.clone(),
backend.bucket.clone(),
backend.prefix.clone(),
backend.region.clone(),
));
}
TierType::Unsupported => {
out.tier_type = EXTERNAL_TIER_TYPE_UNSUPPORTED;
}
}
Ok(out)
}
fn decode_legacy_s3_like(name: &str, ext: &ExternalTierConfig) -> io::Result<ExternalTierS3> {
if let Some(s3) = ext.s3.as_ref() {
return Ok(s3.clone());
}
if let Some(m) = ext.compatible_backend.as_ref() {
return Ok(external_tier_s3_from_compatible_payload(
m.endpoint.clone(),
m.access_key.clone(),
m.secret_key.clone(),
m.bucket.clone(),
m.prefix.clone(),
m.region.clone(),
));
}
Err(io::Error::other(format!("tier config '{name}' missing compatible backend payload")))
}
fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Result<TierConfig> {
let mut cfg = TierConfig {
version: if ext.version.is_empty() {
"v1".to_string()
} else {
ext.version.clone()
},
name: if ext.name.is_empty() { name } else { ext.name.clone() },
..Default::default()
};
let hinted = tier_type_from_hint(ext.tier_type_hint.as_deref());
let tier_type = if let Some(h) = hinted {
h
} else {
match ext.tier_type {
EXTERNAL_TIER_TYPE_S3 => TierType::S3,
EXTERNAL_TIER_TYPE_AZURE => TierType::Azure,
EXTERNAL_TIER_TYPE_GCS => TierType::GCS,
EXTERNAL_TIER_TYPE_MINIO => TierType::MinIO,
_ => TierType::Unsupported,
}
};
cfg.tier_type = tier_type.clone();
match tier_type {
TierType::S3 => {
let s3 = ext
.s3
.as_ref()
.ok_or_else(|| io::Error::other(format!("tier config '{}' missing s3 backend payload", cfg.name)))?;
cfg.s3 = Some(crate::services::tier::tier_config::TierS3 {
name: cfg.name.clone(),
endpoint: s3.endpoint.clone(),
access_key: s3.access_key.clone(),
secret_key: s3.secret_key.clone(),
bucket: s3.bucket.clone(),
prefix: s3.prefix.clone(),
region: s3.region.clone(),
storage_class: s3.storage_class.clone(),
aws_role: s3.aws_role,
aws_role_web_identity_token_file: s3.aws_role_web_identity_token_file.clone(),
aws_role_arn: s3.aws_role_arn.clone(),
aws_role_session_name: s3.aws_role_session_name.clone(),
aws_role_duration_seconds: s3.aws_role_duration_seconds,
});
}
TierType::Azure => {
let az = ext
.azure
.as_ref()
.ok_or_else(|| io::Error::other(format!("tier config '{}' missing azure backend payload", cfg.name)))?;
cfg.azure = Some(crate::services::tier::tier_config::TierAzure {
name: cfg.name.clone(),
endpoint: az.endpoint.clone(),
access_key: az.account_name.clone(),
secret_key: az.account_key.clone(),
bucket: az.bucket.clone(),
prefix: az.prefix.clone(),
region: az.region.clone(),
storage_class: az.storage_class.clone(),
sp_auth: crate::services::tier::tier_config::ServicePrincipalAuth {
tenant_id: az.sp_auth.tenant_id.clone(),
client_id: az.sp_auth.client_id.clone(),
client_secret: az.sp_auth.client_secret.clone(),
},
});
}
TierType::GCS => {
let gcs = ext
.gcs
.as_ref()
.ok_or_else(|| io::Error::other(format!("tier config '{}' missing gcs backend payload", cfg.name)))?;
cfg.gcs = Some(crate::services::tier::tier_config::TierGCS {
name: cfg.name.clone(),
endpoint: gcs.endpoint.clone(),
creds: gcs.creds.clone(),
bucket: gcs.bucket.clone(),
prefix: gcs.prefix.clone(),
region: gcs.region.clone(),
storage_class: gcs.storage_class.clone(),
});
}
TierType::MinIO => {
let m = ext
.compatible_backend
.as_ref()
.ok_or_else(|| io::Error::other(format!("tier config '{}' missing compatible backend payload", cfg.name)))?;
cfg.minio = Some(crate::services::tier::tier_config::TierMinIO {
name: cfg.name.clone(),
endpoint: m.endpoint.clone(),
access_key: m.access_key.clone(),
secret_key: m.secret_key.clone(),
bucket: m.bucket.clone(),
prefix: m.prefix.clone(),
region: m.region.clone(),
});
}
TierType::RustFS => {
let m = decode_legacy_s3_like(&cfg.name, &ext)?;
cfg.rustfs = Some(crate::services::tier::tier_config::TierRustFS {
name: cfg.name.clone(),
endpoint: m.endpoint,
access_key: m.access_key,
secret_key: m.secret_key,
bucket: m.bucket,
prefix: m.prefix,
region: m.region,
storage_class: m.storage_class,
});
}
TierType::Aliyun => {
let m = decode_legacy_s3_like(&cfg.name, &ext)?;
cfg.aliyun = Some(crate::services::tier::tier_config::TierAliyun {
name: cfg.name.clone(),
endpoint: m.endpoint,
access_key: m.access_key,
secret_key: m.secret_key,
bucket: m.bucket,
prefix: m.prefix,
region: m.region,
});
}
TierType::Tencent => {
let m = decode_legacy_s3_like(&cfg.name, &ext)?;
cfg.tencent = Some(crate::services::tier::tier_config::TierTencent {
name: cfg.name.clone(),
endpoint: m.endpoint,
access_key: m.access_key,
secret_key: m.secret_key,
bucket: m.bucket,
prefix: m.prefix,
region: m.region,
});
}
TierType::Huaweicloud => {
let m = decode_legacy_s3_like(&cfg.name, &ext)?;
cfg.huaweicloud = Some(crate::services::tier::tier_config::TierHuaweicloud {
name: cfg.name.clone(),
endpoint: m.endpoint,
access_key: m.access_key,
secret_key: m.secret_key,
bucket: m.bucket,
prefix: m.prefix,
region: m.region,
});
}
TierType::R2 => {
let m = decode_legacy_s3_like(&cfg.name, &ext)?;
cfg.r2 = Some(crate::services::tier::tier_config::TierR2 {
name: cfg.name.clone(),
endpoint: m.endpoint,
access_key: m.access_key,
secret_key: m.secret_key,
bucket: m.bucket,
prefix: m.prefix,
region: m.region,
});
}
TierType::Unsupported => {}
}
Ok(cfg)
}
fn encode_external_tiering_config_blob(cfg: &TierConfigMgr) -> io::Result<Bytes> {
let mut tiers = HashMap::with_capacity(cfg.tiers.len());
for (name, tier_cfg) in &cfg.tiers {
tiers.insert(name.clone(), to_external_tier_config(name, tier_cfg)?);
}
let payload = rmp_serde::to_vec(&ExternalTierConfigMgr { tiers })
.map_err(|err| io::Error::other(format!("serialize tier config payload failed: {err}")))?;
let mut data = Vec::with_capacity(4 + payload.len());
let mut format = [0u8; 2];
LittleEndian::write_u16(&mut format, TIER_CONFIG_FORMAT);
data.extend_from_slice(&format);
let mut version = [0u8; 2];
LittleEndian::write_u16(&mut version, TIER_CONFIG_VERSION);
data.extend_from_slice(&version);
data.extend_from_slice(&payload);
Ok(Bytes::from(data))
}
fn decode_external_tiering_config_blob(data: &[u8]) -> io::Result<TierConfigMgr> {
if data.len() <= 4 {
return Err(io::Error::other("tierConfigInit: no data"));
}
let format = LittleEndian::read_u16(&data[0..2]);
if format != TIER_CONFIG_FORMAT {
return Err(io::Error::other(format!("tierConfigInit: unknown format: {format}")));
}
let version = LittleEndian::read_u16(&data[2..4]);
if version != TIER_CONFIG_V1 && version != TIER_CONFIG_VERSION {
return Err(io::Error::other(format!("tierConfigInit: unknown version: {version}")));
}
let external: ExternalTierConfigMgr =
rmp_serde::from_slice(&data[4..]).map_err(|err| io::Error::other(format!("decode tier config payload failed: {err}")))?;
let mut tiers = HashMap::with_capacity(external.tiers.len());
for (name, ext_cfg) in external.tiers {
tiers.insert(name.clone(), from_external_tier_config(name, ext_cfg)?);
}
Ok(TierConfigMgr {
driver_cache: HashMap::new(),
tiers,
last_refreshed_at: OffsetDateTime::now_utc(),
})
}
fn decode_tiering_config_blob(data: &[u8]) -> io::Result<TierConfigMgr> {
if let Ok(cfg) = TierConfigMgr::unmarshal(data) {
return Ok(cfg);
}
decode_external_tiering_config_blob(data)
}
impl TierConfigMgr {
pub fn new() -> Arc<RwLock<Self>> {
Arc::new(RwLock::new(Self {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
}))
}
pub fn unmarshal(data: &[u8]) -> std::result::Result<TierConfigMgr, std::io::Error> {
let cfg: TierConfigMgr = serde_json::from_slice(data)?;
Ok(cfg)
}
pub fn marshal(&self) -> std::result::Result<Bytes, std::io::Error> {
let data = serde_json::to_vec(&self)?;
let mut data = Bytes::from(data);
Ok(data)
}
pub fn refreshed_at(&self) -> OffsetDateTime {
self.last_refreshed_at
}
pub fn is_tier_valid(&self, tier_name: &str) -> bool {
let (_, valid) = self.is_tier_name_in_use(tier_name);
valid
}
pub fn is_tier_name_in_use(&self, tier_name: &str) -> (TierType, bool) {
if let Some(t) = self.tiers.get(tier_name) {
return (t.tier_type.clone(), true);
}
(TierType::Unsupported, false)
}
pub async fn add(&mut self, tier_config: TierConfig, force: bool) -> std::result::Result<(), AdminError> {
self.ensure_generation_is_idle(&tier_config.name)?;
let tier_name = tier_config.name.clone();
if tier_name != tier_name.to_uppercase() {
return Err(ERR_TIER_NAME_NOT_UPPERCASE.clone());
}
// Reject AWS/MinIO reserved storage-class names as remote-tier names.
// MinIO reserves STANDARD and REDUCED_REDUNDANCY (RRS); a tier must not
// shadow them. The comparison is case-insensitive to match MinIO parity
// and to stay robust regardless of the uppercase gate above.
if tier_name.eq_ignore_ascii_case(crate::config::storageclass::STANDARD)
|| tier_name.eq_ignore_ascii_case(crate::config::storageclass::RRS)
{
return Err(ERR_TIER_RESERVED_NAME.clone());
}
let (_, b) = self.is_tier_name_in_use(&tier_name);
if b {
return Err(ERR_TIER_ALREADY_EXISTS.clone());
}
let d = new_warm_backend(&tier_config, true).await?;
if !force {
let in_use = d.in_use().await;
match in_use {
Ok(b) => {
if b {
return Err(ERR_TIER_BACKEND_IN_USE.clone());
}
}
Err(err) => {
warn!("tier add failed, err: {:?}", err);
if err.to_string().contains("connect") {
return Err(ERR_TIER_CONNECT_ERR.clone());
} else if err.to_string().contains("authorization") {
return Err(ERR_TIER_INVALID_CREDENTIALS.clone());
} else if err.to_string().contains("bucket") {
return Err(ERR_TIER_BUCKET_NOT_FOUND.clone());
}
let mut e = ERR_TIER_PERM_ERR.clone();
e.message.push('.');
e.message.push_str(&err.to_string());
return Err(e);
}
}
}
self.tiers.insert(tier_name.clone(), tier_config);
if let Err(err) = self.replace_driver(&tier_name, d) {
self.tiers.remove(&tier_name);
return Err(err);
}
Ok(())
}
pub async fn remove(&mut self, tier_name: &str, force: bool) -> std::result::Result<(), AdminError> {
self.ensure_generation_is_idle(tier_name)?;
let d = self.get_driver(tier_name).await;
if let Err(err) = d {
if err.code == ERR_TIER_NOT_FOUND.code {
return Ok(());
} else {
return Err(err);
}
}
if !force {
if let Ok(driver) = d {
match driver.in_use().await {
Err(err) => {
let mut e = ERR_TIER_PERM_ERR.clone();
e.message.push('.');
e.message.push_str(&err.to_string());
return Err(e);
}
Ok(in_use) if in_use => {
return Err(ERR_TIER_BACKEND_NOT_EMPTY.clone());
}
_ => {}
}
}
}
self.tiers.remove(tier_name);
self.revoke_driver(tier_name);
Ok(())
}
pub async fn verify(&mut self, tier_name: &str) -> std::result::Result<(), std::io::Error> {
let d = match self.get_driver(tier_name).await {
Ok(d) => d,
Err(err) => {
return Err(std::io::Error::other(err));
}
};
if let Err(err) = check_warm_backend(Some(d)).await {
return Err(std::io::Error::other(err));
} else {
return Ok(());
}
}
pub fn empty(&self) -> bool {
self.list_tiers().len() == 0
}
pub fn tier_type(&self, tier_name: &str) -> String {
if let Some(cfg) = self.tiers.get(tier_name) {
cfg.tier_type.as_lowercase()
} else {
"internal".to_string()
}
}
pub fn list_tiers(&self) -> Vec<TierConfig> {
let mut tier_cfgs = Vec::<TierConfig>::new();
for (_, tier) in self.tiers.iter() {
let tier = tier.clone();
tier_cfgs.push(tier);
}
tier_cfgs
}
pub fn get(&self, tier_name: &str) -> Option<TierConfig> {
for (tier_name2, tier) in self.tiers.iter() {
if tier_name == tier_name2 {
return Some(tier.clone());
}
}
None
}
pub async fn edit(&mut self, tier_name: &str, creds: TierCreds) -> std::result::Result<(), AdminError> {
self.ensure_generation_is_idle(tier_name)?;
let (tier_type, exists) = self.is_tier_name_in_use(tier_name);
if !exists {
return Err(ERR_TIER_NOT_FOUND.clone());
}
let mut tier_config = self.tiers[tier_name].clone();
match tier_type {
TierType::S3 => {
if let Some(s3) = tier_config.s3.as_mut() {
if creds.aws_role {
s3.aws_role = true
}
if creds.aws_role_web_identity_token_file != "" && creds.aws_role_arn != "" {
s3.aws_role_arn = creds.aws_role_arn;
s3.aws_role_web_identity_token_file = creds.aws_role_web_identity_token_file;
}
if creds.access_key != "" && creds.secret_key != "" {
s3.access_key = creds.access_key;
s3.secret_key = creds.secret_key;
}
}
}
TierType::RustFS => {
if let Some(rustfs) = tier_config.rustfs.as_mut() {
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
rustfs.access_key = creds.access_key;
rustfs.secret_key = creds.secret_key;
}
}
TierType::MinIO => {
if let Some(compatible_backend) = tier_config.minio.as_mut() {
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
compatible_backend.access_key = creds.access_key;
compatible_backend.secret_key = creds.secret_key;
}
}
TierType::Aliyun => {
if let Some(aliyun) = tier_config.aliyun.as_mut() {
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
aliyun.access_key = creds.access_key;
aliyun.secret_key = creds.secret_key;
}
}
TierType::Tencent => {
if let Some(tencent) = tier_config.tencent.as_mut() {
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
tencent.access_key = creds.access_key;
tencent.secret_key = creds.secret_key;
}
}
TierType::Huaweicloud => {
if let Some(huaweicloud) = tier_config.huaweicloud.as_mut() {
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
huaweicloud.access_key = creds.access_key;
huaweicloud.secret_key = creds.secret_key;
}
}
TierType::Azure => {
if let Some(azure) = tier_config.azure.as_mut() {
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
azure.access_key = creds.access_key;
azure.secret_key = creds.secret_key;
}
}
TierType::GCS => {
if let Some(gcs) = tier_config.gcs.as_mut() {
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
gcs.creds = creds.access_key; //creds.creds_json
}
}
TierType::R2 => {
if let Some(r2) = tier_config.r2.as_mut() {
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
r2.access_key = creds.access_key;
r2.secret_key = creds.secret_key;
}
}
_ => (),
}
let d = new_warm_backend(&tier_config, true).await?;
self.revoke_driver(tier_name);
self.tiers.insert(tier_name.to_string(), tier_config);
self.replace_driver(tier_name, d)?;
Ok(())
}
pub async fn get_driver<'a>(&'a mut self, tier_name: &str) -> std::result::Result<&'a WarmBackendImpl, AdminError> {
if registered_tier_driver_runtime(self).is_some_and(|runtime| lock_unpoisoned(&runtime).draining.contains_key(tier_name))
{
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier configuration is being replaced".to_string();
return Err(err);
}
// Return cached driver if present
if self.driver_cache.contains_key(tier_name) {
return Ok(self.driver_cache.get(tier_name).expect("Driver not found in cache"));
}
// Get tier configuration and create new driver
let tier_config = self.tiers.get(tier_name).ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
let driver = new_warm_backend(tier_config, false).await?;
self.replace_driver(tier_name, driver)?;
Ok(self
.driver_cache
.get(tier_name)
.expect("Driver not found in cache after insertion"))
}
pub(crate) async fn acquire_operation_lease(
handle: &Arc<RwLock<Self>>,
tier_name: &str,
) -> std::result::Result<TierOperationLease, AdminError> {
{
let manager = handle.read().await;
let runtime = tier_driver_runtime(handle, &manager);
let runtime_guard = lock_unpoisoned(&runtime);
if runtime_guard.draining.contains_key(tier_name) {
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier configuration is being replaced".to_string();
return Err(err);
}
if let Some(generation) = runtime_guard
.generations
.get(tier_name)
.filter(|generation| generation.accepting.load(Ordering::Acquire))
.cloned()
{
let lease = TierOperationLease::try_new(generation, runtime.clone());
drop(runtime_guard);
return lease;
}
}
let (config, config_fingerprint) = {
let mut manager = handle.write().await;
let runtime = tier_driver_runtime(handle, &manager);
if lock_unpoisoned(&runtime).draining.contains_key(tier_name) {
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier configuration is being replaced".to_string();
return Err(err);
}
if let Some(driver) = manager.driver_cache.remove(tier_name) {
manager.replace_driver(tier_name, driver)?;
let runtime_guard = lock_unpoisoned(&runtime);
let generation = runtime_guard
.generations
.get(tier_name)
.filter(|generation| generation.accepting.load(Ordering::Acquire))
.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?
.clone();
let lease = TierOperationLease::try_new(generation, runtime.clone());
drop(runtime_guard);
return lease;
}
let config = manager
.tiers
.get(tier_name)
.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?
.clone_with_credentials();
let fingerprint = tier_config_fingerprint(tier_name, &config).map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})?;
(config, fingerprint)
};
let driver = build_warm_backend(&config, false).await?;
let mut manager = handle.write().await;
let runtime = tier_driver_runtime(handle, &manager);
let runtime_guard = lock_unpoisoned(&runtime);
if runtime_guard.draining.contains_key(tier_name) {
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier configuration is being replaced".to_string();
return Err(err);
}
if let Some(generation) = runtime_guard
.generations
.get(tier_name)
.filter(|generation| {
generation.config_fingerprint == config_fingerprint && generation.accepting.load(Ordering::Acquire)
})
.cloned()
{
let lease = TierOperationLease::try_new(generation, runtime.clone());
drop(runtime_guard);
return lease;
}
drop(runtime_guard);
let current = manager.tiers.get(tier_name).ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
if tier_config_fingerprint(tier_name, current).map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})? != config_fingerprint
{
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier configuration changed while its driver was being prepared".to_string();
return Err(err);
}
manager.replace_driver(tier_name, driver)?;
let runtime_guard = lock_unpoisoned(&runtime);
let generation = runtime_guard
.generations
.get(tier_name)
.filter(|generation| generation.accepting.load(Ordering::Acquire))
.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?
.clone();
let lease = TierOperationLease::try_new(generation, runtime.clone());
drop(runtime_guard);
lease
}
async fn admin_update_lock(handle: &Arc<RwLock<Self>>) -> tokio::sync::OwnedMutexGuard<()> {
let update_lock = {
let manager = handle.read().await;
let runtime = tier_driver_runtime(handle, &manager);
lock_unpoisoned(&runtime).admin_updates.clone()
};
update_lock.lock_owned().await
}
#[cfg(test)]
pub(crate) async fn publish_candidate(
handle: &Arc<RwLock<Self>>,
candidate: Self,
driver_tier: Option<&str>,
) -> std::result::Result<(), AdminError> {
let update = Self::admin_update_lock(handle).await;
Self::publish_candidate_owned(handle, candidate, driver_tier.map(str::to_string), update).await
}
fn begin_publish_transition(
handle: &Arc<RwLock<Self>>,
manager: &mut Self,
candidate: &Self,
) -> std::result::Result<TierPublishTransition, AdminError> {
let changed = changed_tier_names(manager, candidate);
let replaced_destinations = replaced_tier_destinations(manager, candidate)?;
Self::begin_tier_transition_with_destinations(handle, manager, changed, replaced_destinations)
}
fn begin_tier_transition(
handle: &Arc<RwLock<Self>>,
manager: &mut Self,
changed: HashSet<String>,
) -> std::result::Result<TierPublishTransition, AdminError> {
Self::begin_tier_transition_with_destinations(handle, manager, changed, HashMap::new())
}
fn begin_tier_transition_with_destinations(
handle: &Arc<RwLock<Self>>,
manager: &mut Self,
changed: HashSet<String>,
replaced_destinations: HashMap<String, TierConfig>,
) -> std::result::Result<TierPublishTransition, AdminError> {
let runtime = tier_driver_runtime(handle, manager);
for tier_name in replaced_destinations.keys() {
if !lock_unpoisoned(&runtime).generations.contains_key(tier_name)
&& let Some(driver) = manager.driver_cache.remove(tier_name)
{
manager.replace_driver(tier_name, driver)?;
}
}
Self::begin_tier_transition_in_runtime(runtime, changed, replaced_destinations)
}
fn begin_tier_transition_in_runtime(
runtime: Arc<Mutex<TierDriverRuntime>>,
changed: HashSet<String>,
replaced_destinations: HashMap<String, TierConfig>,
) -> std::result::Result<TierPublishTransition, AdminError> {
let count = u64::try_from(changed.len()).map_err(|_| {
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier replacement count exceeds the supported limit".to_string();
err
})?;
let mut runtime_guard = lock_unpoisoned(&runtime);
if changed.iter().any(|tier_name| runtime_guard.draining.contains_key(tier_name)) {
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier configuration is already being replaced".to_string();
return Err(err);
}
let final_epoch = runtime_guard.next_drain_epoch.checked_add(count).ok_or_else(|| {
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier replacement epoch exhausted".to_string();
err
})?;
let mut next_epoch = runtime_guard.next_drain_epoch;
let mut tokens = Vec::with_capacity(changed.len());
let mut revoked = HashMap::with_capacity(changed.len());
for tier_name in changed {
next_epoch += 1;
runtime_guard.draining.insert(tier_name.clone(), next_epoch);
tokens.push((tier_name.clone(), next_epoch));
if let Some(generation) = runtime_guard.generations.remove(&tier_name) {
generation.accepting.store(false, Ordering::Release);
revoked.insert(tier_name, generation);
}
}
runtime_guard.next_drain_epoch = final_epoch;
drop(runtime_guard);
Ok(TierPublishTransition {
runtime,
tokens,
revoked,
replaced_destinations,
published: false,
})
}
async fn publish_candidate_inner(
handle: &Arc<RwLock<Self>>,
candidate: Self,
driver_tier: Option<&str>,
) -> std::result::Result<(), AdminError> {
let transition = {
let mut manager = handle.write().await;
Self::begin_publish_transition(handle, &mut manager, &candidate)?
};
transition.wait_for_active_leases().await?;
transition.ensure_replaced_destinations_are_empty().await?;
Self::publish_candidate_after_drain(handle, candidate, driver_tier, transition).await
}
async fn publish_candidate_after_drain(
handle: &Arc<RwLock<Self>>,
mut candidate: Self,
driver_tier: Option<&str>,
mut transition: TierPublishTransition,
) -> std::result::Result<(), AdminError> {
let prepared_driver =
match driver_tier.and_then(|tier_name| candidate.driver_cache.remove(tier_name).map(|driver| (tier_name, driver))) {
Some((tier_name, driver)) => {
let config = candidate.tiers.get(tier_name).ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
let config_fingerprint = tier_config_fingerprint(tier_name, config).map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})?;
let backend_identity = tier_backend_identity(config).map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})?;
Some(PreparedTierDriver {
tier_name: tier_name.to_string(),
config_fingerprint,
backend_identity,
driver: Arc::from(driver),
})
}
None => None,
};
// Lock order invariant: manager state before the per-manager driver runtime.
let mut manager = handle.write().await;
let mut runtime = lock_unpoisoned(&transition.runtime);
if !transition
.tokens
.iter()
.all(|(tier_name, epoch)| runtime.draining.get(tier_name) == Some(epoch))
{
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier replacement ownership was lost before publish".to_string();
return Err(err);
}
let prepared_generation = if let Some(prepared) = prepared_driver {
let generation = runtime.next_generation.checked_add(1).ok_or_else(|| {
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier driver generation exhausted".to_string();
err
})?;
let entry = Arc::new(TierDriverGeneration {
tier_name: Arc::from(prepared.tier_name.as_str()),
generation,
config_fingerprint: prepared.config_fingerprint,
backend_identity: prepared.backend_identity,
driver: prepared.driver.clone(),
accepting: AtomicBool::new(true),
active_leases: AtomicUsize::new(0),
drained: tokio::sync::Notify::new(),
});
Some((prepared, generation, entry))
} else {
None
};
for (tier_name, _) in &transition.tokens {
manager.driver_cache.remove(tier_name);
}
manager.tiers = candidate.tiers;
manager.last_refreshed_at = OffsetDateTime::now_utc();
if let Some((prepared, generation, entry)) = prepared_generation {
runtime.generations.insert(prepared.tier_name.clone(), entry);
runtime.next_generation = generation;
manager
.driver_cache
.insert(prepared.tier_name, Box::new(SharedWarmBackendProxy(prepared.driver)));
}
drop(runtime);
transition.published = true;
Ok(())
}
fn publish_task_error(err: tokio::task::JoinError) -> AdminError {
error!(error = ?err, "remote tier configuration publish task failed");
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = format!("Remote tier configuration publish task failed: {err}");
admin_err
}
async fn publish_candidate_owned(
handle: &Arc<RwLock<Self>>,
candidate: Self,
driver_tier: Option<String>,
update: tokio::sync::OwnedMutexGuard<()>,
) -> std::result::Result<(), AdminError> {
let handle = handle.clone();
tokio::spawn(async move {
match AssertUnwindSafe(async move {
let _update = update;
Self::publish_candidate_inner(&handle, candidate, driver_tier.as_deref()).await
})
.catch_unwind()
.await
{
Ok(Ok(())) => Ok(()),
Ok(Err(err)) => {
error!(error = ?err, "remote tier configuration publish failed");
Err(err)
}
Err(_) => {
error!("remote tier configuration publish panicked");
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier configuration publish panicked".to_string();
Err(err)
}
}
})
.await
.map_err(Self::publish_task_error)?
}
async fn update_candidate_owned<S>(
handle: &Arc<RwLock<Self>>,
api: Arc<S>,
mut candidate: Self,
version: Option<String>,
mutation: TierCandidateMutation,
update: tokio::sync::OwnedMutexGuard<()>,
) -> std::result::Result<(), TierConfigUpdateError>
where
S: EcstoreObjectIO + 'static,
{
let handle = handle.clone();
tokio::spawn(async move {
match AssertUnwindSafe(async move {
let _update = update;
let transition = {
let mut manager = handle.write().await;
let target_tiers = mutation.target_tiers(&manager, &candidate);
Self::begin_tier_transition(&handle, &mut manager, target_tiers).map_err(TierConfigUpdateError::Publish)?
};
transition
.wait_for_active_leases()
.await
.map_err(TierConfigUpdateError::Publish)?;
let driver_tier =
apply_tier_candidate_mutation(mutation, &mut candidate, Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT)
.await
.map_err(TierConfigUpdateError::Mutation)?;
candidate
.save_tiering_config_if_current(api, version.as_deref())
.await
.map_err(TierConfigUpdateError::Save)?;
Self::publish_candidate_after_drain(&handle, candidate, driver_tier.as_deref(), transition)
.await
.map_err(TierConfigUpdateError::Publish)
})
.catch_unwind()
.await
{
Ok(Ok(())) => Ok(()),
Ok(Err(err)) => {
error!(error = ?err, "remote tier configuration update failed");
Err(err)
}
Err(_) => {
error!("remote tier configuration update panicked");
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier configuration update panicked".to_string();
Err(TierConfigUpdateError::Publish(err))
}
}
})
.await
.map_err(|err| TierConfigUpdateError::Publish(Self::publish_task_error(err)))?
}
pub async fn reload_handle(handle: &Arc<RwLock<Self>>, api: Arc<ECStore>) -> io::Result<()> {
let update = Self::admin_update_lock(handle).await;
let candidate = load_tier_config(api).await?;
Self::publish_candidate_owned(handle, candidate, None, update)
.await
.map_err(io::Error::other)
}
pub async fn add_and_save(
handle: &Arc<RwLock<Self>>,
api: Arc<ECStore>,
tier_config: TierConfig,
force: bool,
) -> std::result::Result<(), TierConfigUpdateError> {
let update = Self::admin_update_lock(handle).await;
let (candidate, version) = load_tier_config_for_update(api.clone())
.await
.map_err(TierConfigUpdateError::Load)?;
Self::update_candidate_owned(handle, api, candidate, version, TierCandidateMutation::Add(tier_config, force), update)
.await
}
pub async fn edit_and_save(
handle: &Arc<RwLock<Self>>,
api: Arc<ECStore>,
tier_name: &str,
credentials: TierCreds,
) -> std::result::Result<(), TierConfigUpdateError> {
let update = Self::admin_update_lock(handle).await;
let (candidate, version) = load_tier_config_for_update(api.clone())
.await
.map_err(TierConfigUpdateError::Load)?;
Self::update_candidate_owned(
handle,
api,
candidate,
version,
TierCandidateMutation::Edit(tier_name.to_string(), credentials),
update,
)
.await
}
pub async fn remove_and_save(
handle: &Arc<RwLock<Self>>,
api: Arc<ECStore>,
tier_name: &str,
force: bool,
) -> std::result::Result<(), TierConfigUpdateError> {
Self::remove_and_save_with(handle, api, tier_name, force).await
}
async fn remove_and_save_with<S>(
handle: &Arc<RwLock<Self>>,
api: Arc<S>,
tier_name: &str,
force: bool,
) -> std::result::Result<(), TierConfigUpdateError>
where
S: EcstoreObjectIO + 'static,
{
let update = Self::admin_update_lock(handle).await;
let (candidate, version) = load_tier_config_for_update(api.clone())
.await
.map_err(TierConfigUpdateError::Load)?;
Self::update_candidate_owned(
handle,
api,
candidate,
version,
TierCandidateMutation::Remove(tier_name.to_string(), force),
update,
)
.await
}
pub async fn clear_and_save(
handle: &Arc<RwLock<Self>>,
api: Arc<ECStore>,
force: bool,
) -> std::result::Result<(), TierConfigUpdateError> {
Self::clear_and_save_with(handle, api, force).await
}
async fn clear_and_save_with<S>(
handle: &Arc<RwLock<Self>>,
api: Arc<S>,
force: bool,
) -> std::result::Result<(), TierConfigUpdateError>
where
S: EcstoreObjectIO + 'static,
{
let update = Self::admin_update_lock(handle).await;
let (candidate, version) = load_tier_config_for_update(api.clone())
.await
.map_err(TierConfigUpdateError::Load)?;
Self::update_candidate_owned(handle, api, candidate, version, TierCandidateMutation::Clear(force), update).await
}
pub async fn verify_without_manager_lock(handle: &Arc<RwLock<Self>>, tier_name: &str) -> std::result::Result<(), io::Error> {
let lease = Self::acquire_operation_lease(handle, tier_name)
.await
.map_err(io::Error::other)?;
let driver: WarmBackendImpl = Box::new(SharedWarmBackendProxy(lease.inner.driver.clone()));
check_warm_backend(Some(&driver)).await.map_err(io::Error::other)
}
pub(crate) async fn acquire_operation_lease_for_backend_identity(
handle: &Arc<RwLock<Self>>,
tier_name: &str,
expected: TierDestinationId,
) -> std::result::Result<TierOperationLease, AdminError> {
let lease = Self::acquire_operation_lease(handle, tier_name).await?;
if lease.backend_identity() != expected {
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier backend identity no longer matches the recorded operation".to_string();
return Err(err);
}
Ok(lease)
}
fn replace_driver(&mut self, tier_name: &str, driver: WarmBackendImpl) -> std::result::Result<(), AdminError> {
let Some(runtime) = registered_tier_driver_runtime(self) else {
self.driver_cache.insert(tier_name.to_string(), driver);
return Ok(());
};
let config_fingerprint = self
.tiers
.get(tier_name)
.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())
.and_then(|config| {
tier_config_fingerprint(tier_name, config).map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})
})?;
let backend_identity = self
.tiers
.get(tier_name)
.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())
.and_then(|config| {
tier_backend_identity(config).map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})
})?;
let mut runtime_guard = lock_unpoisoned(&runtime);
let generation = runtime_guard.next_generation.checked_add(1).ok_or_else(|| {
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier driver generation exhausted".to_string();
err
})?;
let driver: SharedWarmBackend = Arc::from(driver);
let entry = Arc::new(TierDriverGeneration {
tier_name: Arc::from(tier_name),
generation,
config_fingerprint,
backend_identity,
driver: driver.clone(),
accepting: AtomicBool::new(true),
active_leases: AtomicUsize::new(0),
drained: tokio::sync::Notify::new(),
});
if let Some(previous) = runtime_guard.generations.insert(tier_name.to_string(), entry) {
previous.accepting.store(false, Ordering::Release);
}
runtime_guard.next_generation = generation;
drop(runtime_guard);
self.driver_cache
.insert(tier_name.to_string(), Box::new(SharedWarmBackendProxy(driver)));
Ok(())
}
#[cfg(feature = "test-util")]
pub(crate) fn install_test_driver(
&mut self,
tier_name: &str,
driver: WarmBackendImpl,
) -> std::result::Result<(), AdminError> {
self.replace_driver(tier_name, driver)
}
fn revoke_driver(&mut self, tier_name: &str) -> Option<Arc<TierDriverGeneration>> {
let generation =
registered_tier_driver_runtime(self).and_then(|runtime| lock_unpoisoned(&runtime).generations.remove(tier_name));
if let Some(generation) = generation.as_ref() {
generation.accepting.store(false, Ordering::Release);
}
self.driver_cache.remove(tier_name);
generation
}
fn ensure_generation_is_idle(&self, tier_name: &str) -> std::result::Result<(), AdminError> {
let Some(runtime) = registered_tier_driver_runtime(self) else {
return Ok(());
};
let runtime = lock_unpoisoned(&runtime);
let blocked = runtime.draining.contains_key(tier_name)
|| runtime
.generations
.get(tier_name)
.is_some_and(|generation| generation.active_leases.load(Ordering::Acquire) != 0);
if blocked {
return Err(ERR_TIER_BACKEND_IN_USE.clone());
}
Ok(())
}
fn ensure_generations_are_idle<'a>(
&self,
tier_names: impl IntoIterator<Item = &'a String>,
) -> std::result::Result<(), AdminError> {
for tier_name in tier_names {
self.ensure_generation_is_idle(tier_name)?;
}
Ok(())
}
fn retire_driver(&mut self, tier_name: &str) {
self.revoke_driver(tier_name);
}
fn revoke_all_drivers(&mut self) {
if let Some(runtime) = registered_tier_driver_runtime(self) {
let mut runtime = lock_unpoisoned(&runtime);
for (_, generation) in runtime.generations.drain() {
generation.accepting.store(false, Ordering::Release);
}
}
self.driver_cache.clear();
}
fn retire_all_drivers(&mut self) {
self.revoke_all_drivers();
}
pub async fn reload(&mut self, api: Arc<ECStore>) -> std::result::Result<(), std::io::Error> {
let config = load_tier_config(api).await?;
self.publish_legacy_reload(config).await?;
Ok(())
}
pub(crate) async fn publish_legacy_reload(&mut self, config: Self) -> io::Result<()> {
let changed = changed_tier_names(self, &config);
let replaced_destinations = replaced_tier_destinations(self, &config).map_err(io::Error::other)?;
if let Some(runtime) = registered_tier_driver_runtime(self) {
for tier_name in replaced_destinations.keys() {
if !lock_unpoisoned(&runtime).generations.contains_key(tier_name)
&& let Some(driver) = self.driver_cache.remove(tier_name)
{
self.replace_driver(tier_name, driver).map_err(io::Error::other)?;
}
}
let mut transition =
Self::begin_tier_transition_in_runtime(runtime, changed, replaced_destinations).map_err(io::Error::other)?;
transition.wait_for_active_leases().await.map_err(io::Error::other)?;
transition
.ensure_replaced_destinations_are_empty()
.await
.map_err(io::Error::other)?;
let runtime = lock_unpoisoned(&transition.runtime);
if !transition
.tokens
.iter()
.all(|(tier_name, epoch)| runtime.draining.get(tier_name) == Some(epoch))
{
return Err(io::Error::other("remote tier replacement ownership was lost before reload publish"));
}
for (tier_name, _) in &transition.tokens {
self.driver_cache.remove(tier_name);
}
self.tiers = config.tiers;
drop(runtime);
transition.published = true;
} else {
ensure_replaced_destinations_are_empty(&replaced_destinations, &HashMap::new())
.await
.map_err(io::Error::other)?;
self.tiers = config.tiers;
}
self.last_refreshed_at = OffsetDateTime::now_utc();
Ok(())
}
fn apply_reloaded_tiers(&mut self, tiers: HashMap<String, TierConfig>) -> std::result::Result<(), AdminError> {
if registered_tier_driver_runtime(self).is_some_and(|runtime| !lock_unpoisoned(&runtime).draining.is_empty()) {
return Err(ERR_TIER_BACKEND_IN_USE.clone());
}
let changed_or_removed = self
.tiers
.iter()
.filter_map(|(tier_name, current)| {
let unchanged = tiers
.get(tier_name)
.is_some_and(|replacement| tier_configs_match(tier_name, current, replacement));
(!unchanged).then(|| tier_name.clone())
})
.collect::<Vec<_>>();
self.ensure_generations_are_idle(&changed_or_removed)?;
for tier_name in &changed_or_removed {
self.revoke_driver(tier_name);
}
self.tiers = tiers;
Ok(())
}
pub async fn rollback_after_failed_save(&mut self, api: Arc<ECStore>) -> io::Result<()> {
match load_tier_config(api).await {
Ok(config) => {
self.apply_reloaded_tiers(config.tiers).map_err(io::Error::other)?;
Ok(())
}
Err(err) => {
if let Some(runtime) = registered_tier_driver_runtime(self) {
let runtime = lock_unpoisoned(&runtime);
if !runtime.draining.is_empty()
|| runtime
.generations
.values()
.any(|generation| generation.active_leases.load(Ordering::Acquire) != 0)
{
return Err(io::Error::other(format!(
"failed to reload tier configuration for rollback while a tier generation is active: {err}"
)));
}
}
self.retire_all_drivers();
self.tiers.clear();
Err(err)
}
}
}
pub async fn clear_tier(&mut self, force: bool) -> std::result::Result<(), AdminError> {
if registered_tier_driver_runtime(self).is_some_and(|runtime| !lock_unpoisoned(&runtime).draining.is_empty()) {
return Err(ERR_TIER_BACKEND_IN_USE.clone());
}
self.ensure_generations_are_idle(self.tiers.keys())?;
if !force {
let tier_names = self.tiers.keys().cloned().collect::<Vec<_>>();
for tier_name in tier_names {
match self.get_driver(&tier_name).await?.in_use().await {
Ok(true) => return Err(ERR_TIER_BACKEND_NOT_EMPTY.clone()),
Ok(false) => {}
Err(err) => {
let mut admin_err = ERR_TIER_PERM_ERR.clone();
admin_err.message.push('.');
admin_err.message.push_str(&err.to_string());
return Err(admin_err);
}
}
}
}
self.tiers.clear();
self.revoke_all_drivers();
Ok(())
}
#[tracing::instrument(level = "debug", name = "tier_save", skip(self))]
pub async fn save(&self) -> std::result::Result<(), std::io::Error> {
let Some(api) = runtime_sources::object_store_handle() else {
return Err(tier_config_not_initialized_error("save tiering config"));
};
self.save_tiering_config(api).await
}
pub async fn save_tiering_config<S>(&self, api: Arc<S>) -> std::result::Result<(), std::io::Error>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let data = encode_external_tiering_config_blob(self)?;
let config_file = tier_config_path(TIER_CONFIG_FILE);
self.save_config(api, &config_file, data).await
}
async fn save_tiering_config_if_current<S>(
&self,
api: Arc<S>,
version: Option<&str>,
) -> std::result::Result<(), std::io::Error>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let data = encode_external_tiering_config_blob(self)?;
let config_file = tier_config_path(TIER_CONFIG_FILE);
let http_preconditions = match version {
Some(etag) => HTTPPreconditions {
if_match: Some(etag.to_string()),
..Default::default()
},
None => HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
},
};
self.save_config_with_opts(
api,
&config_file,
data,
&ObjectOptions {
max_parity: true,
http_preconditions: Some(http_preconditions),
..Default::default()
},
)
.await
}
pub async fn save_config<S>(&self, api: Arc<S>, file: &str, data: Bytes) -> std::result::Result<(), std::io::Error>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
self.save_config_with_opts(
api,
file,
data,
&ObjectOptions {
max_parity: true,
..Default::default()
},
)
.await
}
pub async fn save_config_with_opts<S>(
&self,
api: Arc<S>,
file: &str,
data: Bytes,
opts: &ObjectOptions,
) -> std::result::Result<(), std::io::Error>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
debug!("save tier config:{}", file);
let mut put_data = PutObjReader::from_vec(data.to_vec());
let _ = api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await?;
Ok(())
}
pub async fn refresh_tier_config(&mut self, api: Arc<ECStore>) {
let r = rand::rng().random_range(0.0..1.0);
let rand_interval = || Duration::from_secs((r * 60_f64).round() as u64);
let mut t = interval(TIER_CFG_REFRESH + rand_interval());
loop {
select! {
_ = t.tick() => {
if let Err(err) = self.reload(api.clone()).await {
info!("{}", err);
}
}
}
}
}
pub(crate) async fn refresh_tier_config_handle(handle: Arc<RwLock<Self>>, api: Arc<ECStore>) {
//let r = rand.New(rand.NewSource(time.Now().UnixNano()));
let r = rand::rng().random_range(0.0..1.0);
let rand_interval = || Duration::from_secs((r * 60_f64).round() as u64);
let refresh_interval = TIER_CFG_REFRESH + rand_interval();
let mut t = delayed_tier_refresh_interval(refresh_interval);
loop {
select! {
_ = t.tick() => {
if let Err(err) = Self::reload_handle(&handle, api.clone()).await {
info!("{}", err);
}
}
else => ()
}
t.reset();
}
}
pub async fn init(&mut self, api: Arc<ECStore>) -> Result<()> {
self.reload(api).await?;
//if globalIsDistErasure {
// self.refresh_tier_config(api).await;
//}
Ok(())
}
}
async fn new_and_save_tiering_config<S>(api: Arc<S>) -> Result<TierConfigMgr>
where
S: EcstoreObjectIO,
{
let mut cfg = TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
};
//lookup_configs(&mut cfg, api.clone()).await;
cfg.save_tiering_config(api).await?;
Ok(cfg)
}
#[tracing::instrument(level = "debug", name = "load_tier_config", skip(api))]
async fn load_tier_config(api: Arc<ECStore>) -> std::result::Result<TierConfigMgr, std::io::Error> {
let config_file = tier_config_path(TIER_CONFIG_FILE);
match read_config(api.clone(), config_file.as_str()).await {
Ok(data) => decode_tiering_config_blob(&data),
Err(err) if is_err_config_not_found(&err) => {
let legacy_file = tier_config_path(TIER_CONFIG_LEGACY_FILE);
match read_config(api.clone(), legacy_file.as_str()).await {
Ok(data) => {
let cfg = TierConfigMgr::unmarshal(&data)?;
let normalized = encode_external_tiering_config_blob(&cfg)?;
let mut put_data = PutObjReader::from_vec(normalized.to_vec());
let _ = api
.put_object(
RUSTFS_META_BUCKET,
&config_file,
&mut put_data,
&ObjectOptions {
max_parity: true,
..Default::default()
},
)
.await;
Ok(cfg)
}
Err(legacy_err) if is_err_config_not_found(&legacy_err) => {
warn!("config not found, start to init");
if runtime_sources::first_cluster_node_is_local().await {
new_and_save_tiering_config(api).await.map_err(io::Error::other)
} else {
Ok(TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
})
}
}
Err(legacy_err) => Err(io::Error::other(legacy_err)),
}
}
Err(err) => {
error!("read config err {:?}", &err);
Err(io::Error::other(err))
}
}
}
async fn load_tier_config_for_update<S>(api: Arc<S>) -> std::result::Result<(TierConfigMgr, Option<String>), std::io::Error>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let config_file = tier_config_path(TIER_CONFIG_FILE);
match read_config_with_metadata(api.clone(), &config_file, &ObjectOptions::default()).await {
Ok((data, object_info)) => {
let etag = object_info
.etag
.filter(|etag| !etag.trim().is_empty())
.ok_or_else(|| io::Error::other("tier configuration object is missing an ETag"))?;
Ok((decode_tiering_config_blob(&data)?, Some(etag)))
}
Err(err) if is_err_config_not_found(&err) => {
let legacy_file = tier_config_path(TIER_CONFIG_LEGACY_FILE);
match read_config(api, &legacy_file).await {
Ok(data) => Ok((TierConfigMgr::unmarshal(&data)?, None)),
Err(legacy_err) if is_err_config_not_found(&legacy_err) => Ok((
TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
},
None,
)),
Err(legacy_err) => Err(io::Error::other(legacy_err)),
}
}
Err(err) => Err(io::Error::other(err)),
}
}
async fn read_tier_config_from_bucket<S>(
api: Arc<S>,
bucket: &str,
path: &str,
opts: &ObjectOptions,
) -> io::Result<Option<Vec<u8>>>
where
S: EcstoreObjectIO,
{
let mut rd = match api.get_object_reader(bucket, path, None, HeaderMap::new(), opts).await {
Ok(v) => v,
Err(err) if is_err_config_not_found(&err) => return Ok(None),
Err(err) => return Err(io::Error::other(err)),
};
let data = rd.read_all().await.map_err(io::Error::other)?;
if data.is_empty() {
return Ok(None);
}
Ok(Some(data))
}
async fn write_tier_config_to_rustfs<S>(api: Arc<S>, path: &str, data: Bytes) -> io::Result<()>
where
S: EcstoreObjectIO,
{
let mut put_data = PutObjReader::from_vec(data.to_vec());
api.put_object(
RUSTFS_META_BUCKET,
path,
&mut put_data,
&ObjectOptions {
max_parity: true,
..Default::default()
},
)
.await
.map_err(io::Error::other)?;
Ok(())
}
pub async fn try_migrate_tiering_config<S>(api: Arc<S>)
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
>,
{
let target_path = tier_config_path(TIER_CONFIG_FILE);
if api
.get_object_info(
RUSTFS_META_BUCKET,
&target_path,
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await
.is_ok()
{
debug!("tier config already exists in RustFS metadata bucket, skip migration");
return;
}
let opts = ObjectOptions {
max_parity: true,
no_lock: true,
..Default::default()
};
let legacy_path = tier_config_path(TIER_CONFIG_LEGACY_FILE);
match read_tier_config_from_bucket(api.clone(), RUSTFS_META_BUCKET, &legacy_path, &opts).await {
Ok(Some(data)) => match TierConfigMgr::unmarshal(&data)
.and_then(|cfg| encode_external_tiering_config_blob(&cfg).map_err(io::Error::other))
{
Ok(out) => {
if write_tier_config_to_rustfs(api.clone(), &target_path, out).await.is_ok() {
info!("Migrated tier config from legacy RustFS metadata format");
return;
}
}
Err(err) => debug!(
bucket = RUSTFS_META_BUCKET,
path = %legacy_path,
error = %err,
"Skipping incompatible legacy tier config migration"
),
},
Ok(None) => {}
Err(err) => debug!(
bucket = RUSTFS_META_BUCKET,
path = %legacy_path,
error = %err,
"Skipping legacy tier config migration after read failure"
),
}
match read_tier_config_from_bucket(api.clone(), MIGRATING_META_BUCKET, &target_path, &opts).await {
Ok(Some(data)) => match decode_tiering_config_blob(&data).and_then(|cfg| encode_external_tiering_config_blob(&cfg)) {
Ok(out) => {
if write_tier_config_to_rustfs(api.clone(), &target_path, out).await.is_ok() {
info!("Migrated compatible tier config from migrating metadata bucket");
}
}
Err(err) => debug!(
bucket = MIGRATING_META_BUCKET,
path = %target_path,
error = %err,
"Skipping incompatible migrating tier config"
),
},
Ok(None) => {}
Err(err) => debug!(
bucket = MIGRATING_META_BUCKET,
path = %target_path,
error = %err,
"Skipping migrating tier config after read failure"
),
}
}
pub fn is_err_config_not_found(err: &StorageError) -> bool {
matches!(err, StorageError::ObjectNotFound(_, _) | StorageError::BucketNotFound(_)) || err == &StorageError::ConfigNotFound
}
fn tier_config_not_initialized_error(operation: &str) -> std::io::Error {
std::io::Error::other(format!("failed to {operation}: object layer not initialized"))
}
#[cfg(test)]
mod tests {
use super::*;
fn build_s3_tier(name: &str) -> TierConfig {
TierConfig {
version: "v1".to_string(),
tier_type: TierType::S3,
name: name.to_string(),
s3: Some(crate::services::tier::tier_config::TierS3 {
name: name.to_string(),
endpoint: "https://example-s3.invalid".to_string(),
access_key: "ak".to_string(),
secret_key: "sk".to_string(),
bucket: "bucket-a".to_string(),
prefix: "prefix-a".to_string(),
region: "us-east-1".to_string(),
storage_class: "STANDARD".to_string(),
aws_role: false,
aws_role_web_identity_token_file: String::new(),
aws_role_arn: String::new(),
aws_role_session_name: String::new(),
aws_role_duration_seconds: 0,
}),
..Default::default()
}
}
#[test]
fn test_tiering_external_blob_roundtrip_for_standard_type() {
let mut cfg = TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
};
cfg.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A"));
let bytes = encode_external_tiering_config_blob(&cfg).expect("encode should succeed");
assert_eq!(&bytes[0..2], &TIER_CONFIG_FORMAT.to_le_bytes());
assert_eq!(&bytes[2..4], &TIER_CONFIG_VERSION.to_le_bytes());
let decoded = decode_external_tiering_config_blob(&bytes).expect("decode should succeed");
let tier = decoded.tiers.get("COLD-A").expect("tier should exist");
assert_eq!(tier.tier_type.as_lowercase(), "s3");
assert_eq!(tier.s3.as_ref().expect("s3 should exist").endpoint, "https://example-s3.invalid");
}
#[test]
fn test_tiering_external_blob_roundtrip_for_extended_type_hint() {
let mut cfg = TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
};
cfg.tiers.insert(
"COLD-B".to_string(),
TierConfig {
version: "v1".to_string(),
tier_type: TierType::RustFS,
name: "COLD-B".to_string(),
rustfs: Some(crate::services::tier::tier_config::TierRustFS {
name: "COLD-B".to_string(),
endpoint: "https://example-compat.invalid".to_string(),
access_key: "ak".to_string(),
secret_key: "sk".to_string(),
bucket: "bucket-b".to_string(),
prefix: "prefix-b".to_string(),
region: "us-east-1".to_string(),
storage_class: "STANDARD".to_string(),
}),
..Default::default()
},
);
let bytes = encode_external_tiering_config_blob(&cfg).expect("encode should succeed");
let decoded = decode_external_tiering_config_blob(&bytes).expect("decode should succeed");
let tier = decoded.tiers.get("COLD-B").expect("tier should exist");
assert_eq!(tier.tier_type.as_lowercase(), "rustfs");
assert_eq!(
tier.rustfs.as_ref().expect("backend should exist").endpoint,
"https://example-compat.invalid"
);
}
#[test]
fn test_decode_tiering_config_blob_accepts_legacy_json() {
let mut cfg = TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
};
cfg.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A"));
let data = serde_json::to_vec(&cfg).expect("legacy json should encode");
let decoded = decode_tiering_config_blob(&data).expect("legacy json should decode");
assert_eq!(
decoded
.tiers
.get("COLD-A")
.and_then(|tier| tier.s3.as_ref())
.map(|s3| s3.bucket.as_str()),
Some("bucket-a")
);
}
#[test]
fn test_tier_config_not_initialized_error_formats_operation_context() {
let err = tier_config_not_initialized_error("save tiering config");
let rendered = err.to_string();
assert!(rendered.contains("failed to save tiering config"), "{rendered}");
assert!(rendered.contains("object layer not initialized"), "{rendered}");
}
// ---------------------------------------------------------------------
// State-machine coverage (ilm-4): add / edit / remove / verify.
//
// These tests must not reach a real remote tier. Two techniques keep them
// hermetic:
// * error paths that return *before* `new_warm_backend` constructs a
// client (name validation, duplicate detection, unsupported type,
// missing backend payload, missing credentials);
// * a `MockWarmBackend` injected directly into `driver_cache`, so
// `get_driver` returns it from cache without any network I/O, which
// lets us drive `remove`/`verify` through every branch.
// ---------------------------------------------------------------------
use crate::client::transition_api::{ReadCloser, ReaderImpl};
use crate::services::tier::warm_backend::{WarmBackend, WarmBackendGetOpts};
fn empty_mgr() -> TierConfigMgr {
TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
}
}
fn build_rustfs_tier(name: &str) -> TierConfig {
TierConfig {
version: "v1".to_string(),
tier_type: TierType::RustFS,
name: name.to_string(),
rustfs: Some(crate::services::tier::tier_config::TierRustFS {
name: name.to_string(),
endpoint: "https://example-compat.invalid".to_string(),
access_key: "ak".to_string(),
secret_key: "sk".to_string(),
bucket: "bucket-r".to_string(),
prefix: "prefix-r".to_string(),
region: "us-east-1".to_string(),
storage_class: "STANDARD".to_string(),
}),
..Default::default()
}
}
/// A fully offline `WarmBackend` used to exercise the driver-facing
/// branches of `remove`/`verify` without touching a remote tier.
struct MockWarmBackend {
/// `Some(b)` -> `in_use()` returns `Ok(b)`; `None` -> returns `Err`.
in_use_value: Option<bool>,
/// When false, put/get/remove all fail (drives `verify` error paths).
healthy: bool,
}
#[async_trait::async_trait]
impl WarmBackend for MockWarmBackend {
async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> std::result::Result<String, std::io::Error> {
if self.healthy {
Ok("mock-version".to_string())
} else {
Err(std::io::Error::other("mock put failed"))
}
}
async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
_meta: HashMap<String, String>,
) -> std::result::Result<String, std::io::Error> {
self.put(object, r, length).await
}
async fn get(
&self,
_object: &str,
_rv: &str,
_opts: WarmBackendGetOpts,
) -> std::result::Result<ReadCloser, std::io::Error> {
if self.healthy {
Ok(BufReader::new(Cursor::new(b"RustFS".to_vec())))
} else {
Err(std::io::Error::other("mock get failed"))
}
}
async fn remove(&self, _object: &str, _rv: &str) -> std::result::Result<(), std::io::Error> {
if self.healthy {
Ok(())
} else {
Err(std::io::Error::other("mock remove failed"))
}
}
async fn in_use(&self) -> std::result::Result<bool, std::io::Error> {
match self.in_use_value {
Some(b) => Ok(b),
None => Err(std::io::Error::other("mock in_use failed")),
}
}
}
fn inject_mock(mgr: &mut TierConfigMgr, name: &str, tier: TierConfig, mock: MockWarmBackend) {
mgr.tiers.insert(name.to_string(), tier);
mgr.driver_cache.insert(name.to_string(), Box::new(mock));
}
// ---- add ------------------------------------------------------------
#[tokio::test]
async fn test_add_rejects_non_uppercase_name() {
let mut mgr = empty_mgr();
let err = mgr
.add(build_s3_tier("cold-a"), true)
.await
.expect_err("lowercase tier name must be rejected");
assert_eq!(err.code, ERR_TIER_NAME_NOT_UPPERCASE.code);
assert!(mgr.tiers.is_empty(), "rejected tier must not be persisted");
}
#[tokio::test]
async fn test_add_rejects_duplicate_name() {
let mut mgr = empty_mgr();
mgr.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A"));
let err = mgr
.add(build_s3_tier("COLD-A"), true)
.await
.expect_err("duplicate tier name must be rejected");
assert_eq!(err.code, ERR_TIER_ALREADY_EXISTS.code);
}
#[tokio::test]
async fn test_add_rejects_unsupported_tier_type() {
let mut mgr = empty_mgr();
// `Unsupported` is rejected by `new_warm_backend` before any network I/O.
let tier = TierConfig {
version: "v1".to_string(),
tier_type: TierType::Unsupported,
name: "COLD-U".to_string(),
..Default::default()
};
let err = mgr.add(tier, true).await.expect_err("unsupported tier type must be rejected");
assert_eq!(err.code, ERR_TIER_TYPE_UNSUPPORTED.code);
assert!(mgr.tiers.is_empty());
}
#[tokio::test]
async fn test_add_rejects_missing_backend_payload() {
let mut mgr = empty_mgr();
// Declares S3 type but omits the S3 payload; rejected before client build.
let tier = TierConfig {
version: "v1".to_string(),
tier_type: TierType::S3,
name: "COLD-A".to_string(),
s3: None,
..Default::default()
};
let err = mgr
.add(tier, true)
.await
.expect_err("missing backend payload must be rejected");
assert_eq!(err.code, "XRustFSAdminTierInvalidConfig");
assert!(err.message.contains("S3 tier configuration not found"), "{}", err.message);
}
#[tokio::test]
async fn test_add_rejects_reserved_names() {
// Supersedes the former `test_add_does_not_reserve_standard_name_regression_anchor`
// (added by PR #4713 to pin the pre-fix behavior). RustFS now rejects the
// AWS/MinIO reserved storage-class names STANDARD and REDUCED_REDUNDANCY
// (RRS) as remote-tier names, matching MinIO parity.
//
// `tier_type` is deliberately `Unsupported`: before the guard existed,
// "STANDARD" reached `new_warm_backend` and failed with
// `ERR_TIER_TYPE_UNSUPPORTED`. Asserting `ERR_TIER_RESERVED_NAME` here
// proves the reserved-name guard fires *before* the type/backend check.
for reserved in [crate::config::storageclass::STANDARD, crate::config::storageclass::RRS] {
let mut mgr = empty_mgr();
let tier = TierConfig {
version: "v1".to_string(),
tier_type: TierType::Unsupported,
name: reserved.to_string(),
..Default::default()
};
let err = mgr.add(tier, true).await.expect_err("reserved tier name must be rejected");
assert_eq!(
err.code, ERR_TIER_RESERVED_NAME.code,
"reserved tier name {reserved} must be rejected before the type check"
);
assert!(mgr.tiers.is_empty(), "rejected reserved tier {reserved} must not be persisted");
}
}
// ---- edit -----------------------------------------------------------
#[tokio::test]
async fn test_edit_rejects_unknown_tier() {
let mut mgr = empty_mgr();
let err = mgr
.edit("NOPE", TierCreds::default())
.await
.expect_err("editing an unknown tier must fail");
assert_eq!(err.code, ERR_TIER_NOT_FOUND.code);
}
#[tokio::test]
async fn test_edit_rejects_missing_credentials_for_rustfs() {
let mut mgr = empty_mgr();
mgr.tiers.insert("COLD-R".to_string(), build_rustfs_tier("COLD-R"));
// Empty access/secret keys => rejected before any driver rebuild.
let err = mgr
.edit("COLD-R", TierCreds::default())
.await
.expect_err("empty credentials must be rejected");
assert_eq!(err.code, ERR_TIER_MISSING_CREDENTIALS.code);
}
#[tokio::test]
async fn test_edit_rejects_missing_credentials_for_minio() {
let mut mgr = empty_mgr();
let tier = TierConfig {
version: "v1".to_string(),
tier_type: TierType::MinIO,
name: "COLD-M".to_string(),
minio: Some(crate::services::tier::tier_config::TierMinIO {
name: "COLD-M".to_string(),
endpoint: "https://example-compat.invalid".to_string(),
access_key: "ak".to_string(),
secret_key: "sk".to_string(),
bucket: "bucket-m".to_string(),
prefix: String::new(),
region: String::new(),
}),
..Default::default()
};
mgr.tiers.insert("COLD-M".to_string(), tier);
let err = mgr
.edit("COLD-M", TierCreds::default())
.await
.expect_err("empty credentials must be rejected");
assert_eq!(err.code, ERR_TIER_MISSING_CREDENTIALS.code);
}
// ---- remove ---------------------------------------------------------
#[tokio::test]
async fn test_remove_unknown_tier_is_idempotent() {
let mut mgr = empty_mgr();
// Unknown tier: get_driver returns NotFound, which `remove` swallows.
mgr.remove("NOPE", false).await.expect("removing an unknown tier is a no-op");
}
#[tokio::test]
async fn test_remove_rejects_backend_in_use() {
let mut mgr = empty_mgr();
inject_mock(
&mut mgr,
"COLD-A",
build_s3_tier("COLD-A"),
MockWarmBackend {
in_use_value: Some(true),
healthy: true,
},
);
let err = mgr
.remove("COLD-A", false)
.await
.expect_err("in-use backend must not be removed");
assert_eq!(err.code, ERR_TIER_BACKEND_NOT_EMPTY.code);
assert!(mgr.tiers.contains_key("COLD-A"), "tier must survive a rejected remove");
assert!(mgr.driver_cache.contains_key("COLD-A"));
}
#[tokio::test]
async fn test_remove_succeeds_when_backend_empty() {
let mut mgr = empty_mgr();
inject_mock(
&mut mgr,
"COLD-A",
build_s3_tier("COLD-A"),
MockWarmBackend {
in_use_value: Some(false),
healthy: true,
},
);
mgr.remove("COLD-A", false).await.expect("empty backend can be removed");
assert!(!mgr.tiers.contains_key("COLD-A"));
assert!(!mgr.driver_cache.contains_key("COLD-A"));
}
#[tokio::test]
async fn test_remove_force_skips_in_use_probe() {
let mut mgr = empty_mgr();
// in_use_value: None would error if probed; force=true must skip it.
inject_mock(
&mut mgr,
"COLD-A",
build_s3_tier("COLD-A"),
MockWarmBackend {
in_use_value: None,
healthy: true,
},
);
mgr.remove("COLD-A", true).await.expect("force remove must not probe in_use");
assert!(!mgr.tiers.contains_key("COLD-A"));
assert!(!mgr.driver_cache.contains_key("COLD-A"));
}
#[tokio::test]
async fn persisted_remove_preserves_force_semantics() {
let mut candidate = empty_mgr();
inject_mock(
&mut candidate,
"COLD-A",
build_s3_tier("COLD-A"),
MockWarmBackend {
in_use_value: None,
healthy: true,
},
);
let mutation = TierCandidateMutation::Remove("COLD-A".to_string(), true);
mutation
.apply(&mut candidate)
.await
.expect("force remove must skip in-use probing");
assert!(!candidate.tiers.contains_key("COLD-A"));
}
#[tokio::test]
async fn persisted_remove_nonforce_preserves_backend_in_use_error() {
let mut candidate = empty_mgr();
inject_mock(
&mut candidate,
"COLD-A",
build_s3_tier("COLD-A"),
MockWarmBackend {
in_use_value: Some(true),
healthy: true,
},
);
let err = TierCandidateMutation::Remove("COLD-A".to_string(), false)
.apply(&mut candidate)
.await
.expect_err("non-force remove must reject an in-use backend");
assert_eq!(err.code, ERR_TIER_BACKEND_NOT_EMPTY.code);
assert!(candidate.tiers.contains_key("COLD-A"));
}
#[tokio::test]
async fn clear_preserves_force_and_nonforce_semantics() {
let mut forced = empty_mgr();
inject_mock(
&mut forced,
"COLD-A",
build_s3_tier("COLD-A"),
MockWarmBackend {
in_use_value: None,
healthy: true,
},
);
forced.clear_tier(true).await.expect("force clear must skip in-use probing");
assert!(forced.tiers.is_empty());
let mut guarded = empty_mgr();
inject_mock(
&mut guarded,
"COLD-A",
build_s3_tier("COLD-A"),
MockWarmBackend {
in_use_value: Some(true),
healthy: true,
},
);
let err = guarded
.clear_tier(false)
.await
.expect_err("non-force clear must reject an in-use backend");
assert_eq!(err.code, ERR_TIER_BACKEND_NOT_EMPTY.code);
assert!(guarded.tiers.contains_key("COLD-A"));
}
#[tokio::test]
async fn test_remove_reports_probe_error() {
let mut mgr = empty_mgr();
inject_mock(
&mut mgr,
"COLD-A",
build_s3_tier("COLD-A"),
MockWarmBackend {
in_use_value: None,
healthy: true,
},
);
let err = mgr
.remove("COLD-A", false)
.await
.expect_err("a failed in_use probe must surface as an error");
assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
assert!(mgr.tiers.contains_key("COLD-A"), "tier must survive a failed probe");
}
// ---- verify ---------------------------------------------------------
#[tokio::test]
async fn test_verify_unknown_tier_errors() {
let mut mgr = empty_mgr();
let err = mgr.verify("NOPE").await.expect_err("verifying an unknown tier must fail");
assert!(err.to_string().contains("not found"), "{err}");
}
#[tokio::test]
async fn test_verify_succeeds_with_healthy_backend() {
let mut mgr = empty_mgr();
inject_mock(
&mut mgr,
"COLD-A",
build_s3_tier("COLD-A"),
MockWarmBackend {
in_use_value: Some(false),
healthy: true,
},
);
mgr.verify("COLD-A").await.expect("healthy backend must verify");
}
#[tokio::test]
async fn test_verify_reports_backend_failure() {
let mut mgr = empty_mgr();
inject_mock(
&mut mgr,
"COLD-A",
build_s3_tier("COLD-A"),
MockWarmBackend {
in_use_value: Some(false),
healthy: false,
},
);
mgr.verify("COLD-A")
.await
.expect_err("an unhealthy backend must fail verification");
}
// ---- pure query helpers --------------------------------------------
#[test]
fn test_query_helpers_reflect_membership() {
let mut mgr = empty_mgr();
assert!(mgr.empty());
assert!(!mgr.is_tier_valid("COLD-A"));
assert_eq!(mgr.tier_type("COLD-A"), "internal");
assert!(mgr.get("COLD-A").is_none());
mgr.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A"));
assert!(!mgr.empty());
assert!(mgr.is_tier_valid("COLD-A"));
let (ty, in_use) = mgr.is_tier_name_in_use("COLD-A");
assert!(in_use);
assert_eq!(ty.as_lowercase(), "s3");
assert_eq!(mgr.tier_type("COLD-A"), "s3");
assert_eq!(mgr.list_tiers().len(), 1);
assert!(mgr.get("COLD-A").is_some());
}
// ---- persistence: encode/decode and legacy migration ---------------
#[test]
fn test_marshal_unmarshal_json_roundtrip_preserves_tier() {
let mut mgr = empty_mgr();
mgr.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A"));
let bytes = mgr.marshal().expect("marshal should succeed");
let decoded = TierConfigMgr::unmarshal(&bytes).expect("unmarshal should succeed");
let tier = decoded.tiers.get("COLD-A").expect("tier survives json roundtrip");
assert_eq!(tier.tier_type.as_lowercase(), "s3");
let s3 = tier.s3.as_ref().expect("s3 payload survives");
assert_eq!(s3.bucket, "bucket-a");
// secret_key is a serialized field (unlike Clone, marshal does not redact).
assert_eq!(s3.secret_key, "sk");
}
#[test]
fn test_external_blob_roundtrip_azure_maps_account_fields() {
let mut mgr = empty_mgr();
mgr.tiers.insert(
"COLD-AZ".to_string(),
TierConfig {
version: "v1".to_string(),
tier_type: TierType::Azure,
name: "COLD-AZ".to_string(),
azure: Some(crate::services::tier::tier_config::TierAzure {
name: "COLD-AZ".to_string(),
endpoint: "https://example-azure.invalid".to_string(),
access_key: "account-name".to_string(),
secret_key: "account-key".to_string(),
bucket: "container".to_string(),
prefix: "p".to_string(),
region: "eastus".to_string(),
storage_class: "HOT".to_string(),
sp_auth: crate::services::tier::tier_config::ServicePrincipalAuth {
tenant_id: "tenant".to_string(),
client_id: "client".to_string(),
client_secret: "secret".to_string(),
},
}),
..Default::default()
},
);
let bytes = encode_external_tiering_config_blob(&mgr).expect("encode azure tier");
let decoded = decode_external_tiering_config_blob(&bytes).expect("decode azure tier");
let tier = decoded.tiers.get("COLD-AZ").expect("azure tier survives roundtrip");
assert_eq!(tier.tier_type.as_lowercase(), "azure");
let az = tier.azure.as_ref().expect("azure payload survives");
// AccountName/AccountKey in the on-disk format map back to access/secret.
assert_eq!(az.access_key, "account-name");
assert_eq!(az.secret_key, "account-key");
assert_eq!(az.sp_auth.tenant_id, "tenant");
assert_eq!(az.sp_auth.client_secret, "secret");
}
#[test]
fn test_external_blob_roundtrip_gcs_preserves_creds() {
let mut mgr = empty_mgr();
mgr.tiers.insert(
"COLD-G".to_string(),
TierConfig {
version: "v1".to_string(),
tier_type: TierType::GCS,
name: "COLD-G".to_string(),
gcs: Some(crate::services::tier::tier_config::TierGCS {
name: "COLD-G".to_string(),
endpoint: "https://storage.googleapis.com".to_string(),
creds: "service-account-json".to_string(),
bucket: "gbucket".to_string(),
prefix: "gp".to_string(),
region: "us".to_string(),
storage_class: "NEARLINE".to_string(),
}),
..Default::default()
},
);
let bytes = encode_external_tiering_config_blob(&mgr).expect("encode gcs tier");
let decoded = decode_external_tiering_config_blob(&bytes).expect("decode gcs tier");
let tier = decoded.tiers.get("COLD-G").expect("gcs tier survives roundtrip");
assert_eq!(tier.tier_type.as_lowercase(), "gcs");
assert_eq!(tier.gcs.as_ref().expect("gcs payload survives").creds, "service-account-json");
}
// `TierConfigMgr` intentionally does not derive `Debug` (it holds live
// driver handles), so `Result::expect_err` cannot be used on decode
// results. Extract the error explicitly instead.
fn expect_decode_err(data: &[u8]) -> std::io::Error {
match decode_external_tiering_config_blob(data) {
Ok(_) => panic!("expected decode to fail"),
Err(err) => err,
}
}
#[test]
fn test_external_blob_rejects_truncated_data() {
let err = expect_decode_err(&[0u8; 3]);
assert!(err.to_string().contains("no data"), "{err}");
}
#[test]
fn test_external_blob_rejects_unknown_format() {
// Valid length, but a format word the decoder does not recognise.
let mut data = Vec::new();
data.extend_from_slice(&99u16.to_le_bytes());
data.extend_from_slice(&TIER_CONFIG_VERSION.to_le_bytes());
data.extend_from_slice(&[0u8; 8]);
let err = expect_decode_err(&data);
assert!(err.to_string().contains("unknown format"), "{err}");
}
#[test]
fn test_external_blob_rejects_unknown_version() {
let mut data = Vec::new();
data.extend_from_slice(&TIER_CONFIG_FORMAT.to_le_bytes());
data.extend_from_slice(&99u16.to_le_bytes());
data.extend_from_slice(&[0u8; 8]);
let err = expect_decode_err(&data);
assert!(err.to_string().contains("unknown version"), "{err}");
}
#[test]
fn test_external_blob_ignores_unknown_fields_at_all_legacy_layers() {
let payload = serde_json::json!({
"Tiers": {
"COLD-A": {
"Version": "v1",
"Type": EXTERNAL_TIER_TYPE_S3,
"Name": "COLD-A",
"S3": {
"Endpoint": "https://example.invalid",
"AccessKey": "ak",
"SecretKey": "sk",
"Bucket": "bucket",
"Prefix": "objects",
"UnknownS3": true
},
"Azure": {
"SPAuth": { "UnknownSPAuth": true },
"UnknownAzure": true
},
"GCS": { "UnknownGCS": true },
"MinIO": { "UnknownMinIO": true },
"UnknownTier": true
}
},
"UnknownManager": true
});
let encoded = rmp_serde::to_vec(&payload).expect("test payload should encode");
let mut data = Vec::with_capacity(4 + encoded.len());
data.extend_from_slice(&TIER_CONFIG_FORMAT.to_le_bytes());
data.extend_from_slice(&TIER_CONFIG_VERSION.to_le_bytes());
data.extend_from_slice(&encoded);
let decoded = decode_external_tiering_config_blob(&data).expect("legacy unknown fields must remain forward-compatible");
let s3 = decoded.tiers["COLD-A"].s3.as_ref().expect("s3 payload should decode");
assert_eq!(s3.prefix, "objects");
}
#[test]
fn test_external_blob_decodes_minio_compatible_fixture() {
let payload = serde_json::json!({
"Tiers": {
"COLD-M": {
"Version": "v1",
"Type": EXTERNAL_TIER_TYPE_MINIO,
"Name": "COLD-M",
"MinIO": {
"Endpoint": "https://minio.example.invalid",
"AccessKey": "ak",
"SecretKey": "sk",
"Bucket": "archive",
"Prefix": "objects/",
"Region": "us-east-1"
}
}
}
});
let encoded = rmp_serde::to_vec(&payload).expect("fixture payload should encode");
let mut data = Vec::with_capacity(4 + encoded.len());
data.extend_from_slice(&TIER_CONFIG_FORMAT.to_le_bytes());
data.extend_from_slice(&TIER_CONFIG_VERSION.to_le_bytes());
data.extend_from_slice(&encoded);
let decoded = decode_external_tiering_config_blob(&data).expect("MinIO-compatible fixture must decode");
let minio = decoded.tiers["COLD-M"].minio.as_ref().expect("MinIO payload should decode");
assert_eq!(minio.endpoint, "https://minio.example.invalid");
assert_eq!(minio.bucket, "archive");
assert_eq!(minio.prefix, "objects/");
}
#[test]
fn test_external_blob_accepts_legacy_v1_version_word() {
// The decoder must keep accepting the historical v1 version word, not
// just the current TIER_CONFIG_VERSION, so old on-disk configs load.
let mut mgr = empty_mgr();
mgr.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A"));
let current = encode_external_tiering_config_blob(&mgr).expect("encode");
// Rewrite the version word (bytes 2..4) from VERSION to V1.
let mut legacy = current.to_vec();
legacy[2..4].copy_from_slice(&TIER_CONFIG_V1.to_le_bytes());
let decoded = decode_external_tiering_config_blob(&legacy).expect("v1 version word must decode");
assert!(decoded.tiers.contains_key("COLD-A"));
}
#[test]
fn test_encode_external_blob_errors_on_missing_payload() {
let mut mgr = empty_mgr();
mgr.tiers.insert(
"COLD-A".to_string(),
TierConfig {
version: "v1".to_string(),
tier_type: TierType::S3,
name: "COLD-A".to_string(),
s3: None,
..Default::default()
},
);
let err = encode_external_tiering_config_blob(&mgr).expect_err("missing payload must fail encode");
assert!(err.to_string().contains("missing s3 backend payload"), "{err}");
}
#[derive(Clone)]
struct LeaseTestBackend {
id: &'static str,
calls: Arc<std::sync::Mutex<Vec<&'static str>>>,
remove_started: Option<Arc<tokio::sync::Notify>>,
remove_release: Option<Arc<tokio::sync::Semaphore>>,
backend_in_use: bool,
pending_in_use: bool,
panic_in_use: bool,
}
impl LeaseTestBackend {
fn ready(id: &'static str) -> Self {
Self {
id,
calls: Arc::new(std::sync::Mutex::new(Vec::new())),
remove_started: None,
remove_release: None,
backend_in_use: false,
pending_in_use: false,
panic_in_use: false,
}
}
fn blocking(id: &'static str) -> Self {
Self {
remove_started: Some(Arc::new(tokio::sync::Notify::new())),
remove_release: Some(Arc::new(tokio::sync::Semaphore::new(0))),
..Self::ready(id)
}
}
fn calls(&self) -> Vec<&'static str> {
self.calls.lock().expect("lease test call log should not poison").clone()
}
fn panicking_in_use(id: &'static str) -> Self {
Self {
panic_in_use: true,
..Self::ready(id)
}
}
fn in_use(id: &'static str) -> Self {
Self {
backend_in_use: true,
..Self::ready(id)
}
}
fn pending_in_use(id: &'static str) -> Self {
Self {
pending_in_use: true,
..Self::ready(id)
}
}
}
#[async_trait::async_trait]
impl WarmBackend for LeaseTestBackend {
async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> io::Result<String> {
Ok(self.id.to_string())
}
async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
_meta: HashMap<String, String>,
) -> io::Result<String> {
self.put(object, r, length).await
}
async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> io::Result<ReadCloser> {
Ok(BufReader::new(Cursor::new(Vec::new())))
}
async fn remove(&self, _object: &str, _rv: &str) -> io::Result<()> {
self.calls
.lock()
.expect("lease test call log should not poison")
.push(self.id);
if let Some(started) = &self.remove_started {
started.notify_one();
}
if let Some(release) = &self.remove_release {
release
.acquire()
.await
.expect("lease test release semaphore should stay open")
.forget();
}
Ok(())
}
async fn in_use(&self) -> io::Result<bool> {
if self.panic_in_use {
panic!("simulated tier backend in-use panic");
}
if self.pending_in_use {
std::future::pending::<()>().await;
}
Ok(self.backend_in_use)
}
}
fn install_lease_backend(manager: &mut TierConfigMgr, tier_name: &str, backend: LeaseTestBackend) {
manager.tiers.insert(tier_name.to_string(), build_rustfs_tier(tier_name));
manager
.replace_driver(tier_name, Box::new(backend))
.expect("test driver generation should install");
}
#[tokio::test]
async fn registry_steady_state_lookup_does_not_scan_misses() {
let manager = TierConfigMgr::new();
TIER_REGISTRY_MISS_SCAN_COUNT
.scope(std::cell::Cell::new(0), async {
let guard = manager.read().await;
let first = tier_driver_runtime(&manager, &guard);
assert_eq!(TIER_REGISTRY_MISS_SCAN_COUNT.with(std::cell::Cell::get), 1);
let second = tier_driver_runtime(&manager, &guard);
assert!(Arc::ptr_eq(&first, &second));
assert_eq!(TIER_REGISTRY_MISS_SCAN_COUNT.with(std::cell::Cell::get), 1);
})
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn cold_driver_build_does_not_hold_manager_write_lock() {
let cold_tier = "COLD-BUILD-LOCK";
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
guard.tiers.insert(cold_tier.to_string(), build_rustfs_tier(cold_tier));
install_lease_backend(&mut guard, "COLD-B", LeaseTestBackend::ready("b"));
}
let (barrier, _barrier_guard) = install_tier_driver_build_barrier(cold_tier);
let build_manager = manager.clone();
let build = tokio::spawn(async move { TierConfigMgr::acquire_operation_lease(&build_manager, cold_tier).await });
barrier.arrived.notified().await;
tokio::time::timeout(Duration::from_millis(100), manager.read())
.await
.expect("cold driver construction must not block manager readers");
let tier_b = tokio::time::timeout(Duration::from_millis(100), TierConfigMgr::acquire_operation_lease(&manager, "COLD-B"))
.await
.expect("cold tier A construction must not block tier B")
.expect("tier B lease should remain available");
drop(tier_b);
barrier.release.add_permits(1);
build
.await
.expect("cold driver build task should join")
.expect("cold driver should finish after the test barrier releases");
}
#[tokio::test(start_paused = true)]
#[serial_test::serial]
async fn cold_reload_driver_build_timeout_restores_config() {
let cold_tier = "COLD-BUILD-TIMEOUT";
let manager = TierConfigMgr::new();
manager
.write()
.await
.tiers
.insert(cold_tier.to_string(), build_rustfs_tier(cold_tier));
let mut candidate = empty_mgr();
let mut replacement = build_rustfs_tier(cold_tier);
replacement.rustfs.as_mut().expect("replacement payload should exist").prefix = "new-prefix".to_string();
candidate.tiers.insert(cold_tier.to_string(), replacement);
let (_barrier, _barrier_guard) = install_tier_driver_build_barrier(cold_tier);
let err = TierConfigMgr::publish_candidate(&manager, candidate, None)
.await
.expect_err("stalled cold backend construction must time out");
assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
assert!(manager.read().await.tiers.contains_key(cold_tier));
let guard = manager.read().await;
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
assert!(lock_unpoisoned(&runtime).draining.is_empty());
}
#[test]
fn internal_tier_snapshot_preserves_credentials() {
let mut config = build_rustfs_tier("COLD-A");
let rustfs = config.rustfs.as_mut().expect("rustfs payload should exist");
rustfs.access_key = "access-key".to_string();
rustfs.secret_key = "secret-key".to_string();
let snapshot = config.clone_with_credentials();
let rustfs = snapshot.rustfs.expect("snapshot payload should exist");
assert_eq!(rustfs.access_key, "access-key");
assert_eq!(rustfs.secret_key, "secret-key");
}
#[tokio::test(start_paused = true)]
async fn remote_mutation_validation_timeout_restores_candidate() {
let mut candidate = empty_mgr();
candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
candidate
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::pending_in_use("pending")));
let err = apply_tier_candidate_mutation(
TierCandidateMutation::Remove("COLD-A".to_string(), false),
&mut candidate,
Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT,
)
.await
.expect_err("unbounded backend validation must time out");
assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
assert!(candidate.tiers.contains_key("COLD-A"));
}
#[tokio::test]
async fn cold_cached_reload_rejects_in_use_route_change_and_removal() {
for remove in [false, true] {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
guard.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
guard
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::in_use("cold")));
}
let mut candidate = empty_mgr();
if !remove {
let mut replacement = build_rustfs_tier("COLD-A");
replacement.rustfs.as_mut().expect("replacement payload should exist").prefix = "new-prefix".to_string();
candidate.tiers.insert("COLD-A".to_string(), replacement);
}
let err = TierConfigMgr::publish_candidate(&manager, candidate, None)
.await
.expect_err("an in-use cold-cache destination must not be rebound or removed");
assert_eq!(err.code, ERR_TIER_BACKEND_NOT_EMPTY.code);
assert!(manager.read().await.tiers.contains_key("COLD-A"));
let restored = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("failed reload must restore the old cold-cache generation");
assert!(restored.is_current(&manager).await);
}
}
#[tokio::test]
async fn empty_backend_route_change_and_credential_rotation_can_publish() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("empty"));
}
let mut route_candidate = empty_mgr();
let mut replacement = build_rustfs_tier("COLD-A");
replacement.rustfs.as_mut().expect("replacement payload should exist").prefix = "new-prefix".to_string();
route_candidate.tiers.insert("COLD-A".to_string(), replacement);
TierConfigMgr::publish_candidate(&manager, route_candidate, None)
.await
.expect("an empty backend may be rebound");
{
let mut guard = manager.write().await;
guard
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::panicking_in_use("credentials")));
}
let mut credential_candidate = empty_mgr();
let mut rotated = manager.read().await.tiers["COLD-A"].clone_with_credentials();
rotated.rustfs.as_mut().expect("rotated payload should exist").secret_key = "rotated".to_string();
credential_candidate.tiers.insert("COLD-A".to_string(), rotated);
TierConfigMgr::publish_candidate(&manager, credential_candidate, None)
.await
.expect("credential-only rotation must not inspect backend occupancy");
}
#[tokio::test]
async fn slow_tier_operation_does_not_block_another_tier() {
let slow = LeaseTestBackend::blocking("slow");
let fast = LeaseTestBackend::ready("fast");
let manager = Arc::new(RwLock::new(empty_mgr()));
{
let mut manager = manager.write().await;
install_lease_backend(&mut manager, "COLD-A", slow.clone());
install_lease_backend(&mut manager, "COLD-B", fast.clone());
}
let slow_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("slow lease should be available");
let fast_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-B")
.await
.expect("fast lease should be available");
let slow_task = tokio::spawn(async move { slow_lease.remove("object", "").await });
slow.remove_started
.as_ref()
.expect("slow backend should expose start notification")
.notified()
.await;
let manager_read = tokio::time::timeout(Duration::from_millis(100), manager.read())
.await
.expect("manager reads must not wait for tier A");
assert!(manager_read.is_tier_valid("COLD-B"));
drop(manager_read);
tokio::time::timeout(Duration::from_millis(100), fast_lease.remove("object", ""))
.await
.expect("tier B must not wait for tier A")
.expect("fast remove should succeed");
assert_eq!(fast.calls(), vec!["fast"]);
slow.remove_release
.as_ref()
.expect("slow backend should expose release semaphore")
.add_permits(1);
slow_task
.await
.expect("slow task should join")
.expect("slow remove should succeed");
}
#[tokio::test]
async fn slow_admin_verify_does_not_hold_manager_lock() {
let manager = TierConfigMgr::new();
let backend = LeaseTestBackend::blocking("verify");
let started = backend
.remove_started
.as_ref()
.expect("verify remove notifier should exist")
.clone();
let release = backend
.remove_release
.as_ref()
.expect("verify remove release should exist")
.clone();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", backend);
}
let verify_manager = manager.clone();
let verify = tokio::spawn(async move { TierConfigMgr::verify_without_manager_lock(&verify_manager, "COLD-A").await });
started.notified().await;
tokio::time::timeout(Duration::from_millis(100), manager.read())
.await
.expect("slow verify must not hold the manager lock");
release.add_permits(1);
verify.await.expect("verify task should join").expect("verify should finish");
}
#[tokio::test]
async fn same_name_replacement_drains_old_generation_and_routes_new_work_to_new_driver() {
let old = LeaseTestBackend::ready("old");
let new = LeaseTestBackend::ready("new");
let manager = Arc::new(RwLock::new(empty_mgr()));
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", old.clone());
}
let old_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("old generation lease should be available");
{
manager
.write()
.await
.replace_driver("COLD-A", Box::new(new.clone()))
.expect("replacement generation should install");
}
let new_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("new generation lease should be available");
assert!(!old_lease.is_current(&manager).await);
assert!(new_lease.is_current(&manager).await);
assert!(new_lease.generation() > old_lease.generation());
old_lease
.remove("old-object", "")
.await
.expect("draining lease should remain usable");
new_lease
.remove("new-object", "")
.await
.expect("new lease should use replacement driver");
assert_eq!(old.calls(), vec!["old"]);
assert_eq!(new.calls(), vec!["new"]);
}
#[tokio::test]
async fn replacement_waits_for_inflight_operation_lease() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("old lease should be available");
let mut candidate = empty_mgr();
let mut config = build_rustfs_tier("COLD-A");
config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://replacement.invalid".to_string();
candidate.tiers.insert("COLD-A".to_string(), config);
candidate
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("new")));
let publish_manager = manager.clone();
let publish =
tokio::spawn(async move { TierConfigMgr::publish_candidate(&publish_manager, candidate, Some("COLD-A")).await });
tokio::time::timeout(Duration::from_secs(1), async {
while old.is_current(&manager).await {
tokio::task::yield_now().await;
}
})
.await
.expect("replacement should revoke the old generation before waiting");
assert!(!publish.is_finished(), "replacement must wait for the old operation lease");
drop(old);
publish
.await
.expect("publish task should join")
.expect("replacement should publish after the lease drains");
}
#[tokio::test(start_paused = true)]
async fn replacement_drain_timeout_restores_generation_and_allows_retry() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("old lease should be available");
let mut candidate = empty_mgr();
let mut config = build_rustfs_tier("COLD-A");
config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://timeout.invalid".to_string();
candidate.tiers.insert("COLD-A".to_string(), config);
candidate
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("timed-out")));
let publish_manager = manager.clone();
let publish =
tokio::spawn(async move { TierConfigMgr::publish_candidate(&publish_manager, candidate, Some("COLD-A")).await });
while old.inner.accepting.load(Ordering::Acquire) {
tokio::task::yield_now().await;
}
tokio::time::advance(TIER_OPERATION_DRAIN_TIMEOUT).await;
let err = publish
.await
.expect("publish task should join")
.expect_err("replacement must time out while the old lease remains active");
assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
assert!(old.is_current(&manager).await, "timeout rollback must restore the old generation");
{
let guard = manager.read().await;
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
assert!(!lock_unpoisoned(&runtime).draining.contains_key("COLD-A"));
}
drop(old);
let mut retry = empty_mgr();
let mut retry_config = build_rustfs_tier("COLD-A");
retry_config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://retry.invalid".to_string();
retry.tiers.insert("COLD-A".to_string(), retry_config);
retry
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("retry")));
TierConfigMgr::publish_candidate(&manager, retry, Some("COLD-A"))
.await
.expect("a later replacement must succeed after timeout rollback");
}
#[tokio::test]
async fn generation_prepare_failure_restores_old_manager_and_generation() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("old generation should initialize");
drop(lease);
{
let guard = manager.read().await;
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should be registered");
lock_unpoisoned(&runtime).next_generation = u64::MAX;
}
let mut candidate = empty_mgr();
let mut config = build_rustfs_tier("COLD-A");
config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://overflow.invalid".to_string();
candidate.tiers.insert("COLD-A".to_string(), config);
candidate
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("new")));
let err = TierConfigMgr::publish_candidate(&manager, candidate, Some("COLD-A"))
.await
.expect_err("generation exhaustion must fail before manager state changes");
assert!(err.message.contains("generation exhausted"));
assert_eq!(
manager.read().await.tiers["COLD-A"]
.rustfs
.as_ref()
.expect("old config should remain")
.endpoint,
"https://example-compat.invalid"
);
let restored = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("prepare failure must restore the old generation");
assert!(restored.is_current(&manager).await);
let guard = manager.read().await;
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
assert!(!lock_unpoisoned(&runtime).draining.contains_key("COLD-A"));
}
#[tokio::test]
async fn legacy_reload_does_not_block_revoked_operation_completion() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("old lease should be available");
let mut candidate = empty_mgr();
let mut config = build_rustfs_tier("COLD-A");
config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://legacy-reload.invalid".to_string();
candidate.tiers.insert("COLD-A".to_string(), config);
let reload_manager = manager.clone();
let reload = tokio::spawn(async move { reload_manager.write().await.publish_legacy_reload(candidate).await });
while old.inner.accepting.load(Ordering::Acquire) {
tokio::task::yield_now().await;
}
assert!(
manager.try_read().is_err(),
"legacy reload must still hold the manager write guard while draining"
);
assert!(!reload.is_finished(), "legacy reload must wait for the active generation");
let operation = tokio::spawn(async move {
assert!(!old.is_current_generation(), "the revoked operation must observe the generation fence");
drop(old);
});
tokio::time::timeout(Duration::from_secs(1), operation)
.await
.expect("the revoked operation must finish without waiting for the manager guard")
.expect("the revoked operation task should join");
reload
.await
.expect("legacy reload task should join")
.expect("legacy reload should publish after the lease drains");
assert_eq!(
manager.read().await.tiers["COLD-A"]
.rustfs
.as_ref()
.expect("reloaded tier should exist")
.endpoint,
"https://legacy-reload.invalid"
);
}
#[tokio::test]
async fn reload_publish_waits_for_inflight_operation_lease() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("old lease should be available");
let mut replacement = build_rustfs_tier("COLD-A");
replacement.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://reload.invalid".to_string();
let mut candidate = empty_mgr();
candidate.tiers.insert("COLD-A".to_string(), replacement);
let reload_manager = manager.clone();
let reload = tokio::spawn(async move { TierConfigMgr::publish_candidate(&reload_manager, candidate, None).await });
tokio::time::timeout(Duration::from_secs(1), async {
while old.is_current(&manager).await {
tokio::task::yield_now().await;
}
})
.await
.expect("reload publish should revoke before waiting");
assert!(!reload.is_finished(), "reload must wait for the old operation lease");
drop(old);
reload
.await
.expect("reload task should join")
.expect("reload should publish after the lease drains");
}
#[tokio::test]
async fn cancelled_publish_caller_does_not_leave_tier_draining() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("old lease should be available");
let mut candidate = empty_mgr();
let mut config = build_rustfs_tier("COLD-A");
config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://owned.invalid".to_string();
candidate.tiers.insert("COLD-A".to_string(), config);
candidate
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("new")));
let update = TierConfigMgr::admin_update_lock(&manager).await;
let publish_manager = manager.clone();
let caller = tokio::spawn(async move {
TierConfigMgr::publish_candidate_owned(&publish_manager, candidate, Some("COLD-A".to_string()), update).await
});
tokio::time::timeout(Duration::from_secs(1), async {
while old.is_current(&manager).await {
tokio::task::yield_now().await;
}
})
.await
.expect("owned publish should revoke before caller cancellation");
caller.abort();
let mut second = empty_mgr();
let mut second_config = build_rustfs_tier("COLD-A");
second_config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://second.invalid".to_string();
second.tiers.insert("COLD-A".to_string(), second_config);
second
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("second")));
let second_manager = manager.clone();
let second_publish =
tokio::spawn(async move { TierConfigMgr::publish_candidate(&second_manager, second, Some("COLD-A")).await });
tokio::task::yield_now().await;
assert!(!second_publish.is_finished(), "a later update must remain behind the detached publish");
drop(old);
second_publish
.await
.expect("second publish task should join")
.expect("second publish should follow the detached publish");
let endpoint = manager
.read()
.await
.tiers
.get("COLD-A")
.and_then(|tier| tier.rustfs.as_ref())
.expect("second tier should be published")
.endpoint
.clone();
assert_eq!(endpoint, "https://second.invalid");
}
#[tokio::test]
async fn slow_tier_replacement_does_not_block_other_tiers_or_manager_reads() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("a"));
install_lease_backend(&mut guard, "COLD-B", LeaseTestBackend::ready("b"));
}
let old_a = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("tier A lease should be available");
let old_b = TierConfigMgr::acquire_operation_lease(&manager, "COLD-B")
.await
.expect("tier B lease should be available");
let mut candidate = empty_mgr();
candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
candidate.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B"));
candidate
.tiers
.get_mut("COLD-A")
.and_then(|tier| tier.rustfs.as_mut())
.expect("tier A payload should exist")
.endpoint = "https://replacement.invalid".to_string();
candidate
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("new-a")));
let publish_manager = manager.clone();
let publish =
tokio::spawn(async move { TierConfigMgr::publish_candidate(&publish_manager, candidate, Some("COLD-A")).await });
tokio::time::timeout(Duration::from_secs(1), async {
while old_a.is_current(&manager).await {
tokio::task::yield_now().await;
}
})
.await
.expect("tier A should enter draining");
{
let guard = manager.read().await;
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
assert_eq!(
lock_unpoisoned(&runtime).draining.keys().cloned().collect::<Vec<_>>(),
vec!["COLD-A".to_string()]
);
}
tokio::time::timeout(Duration::from_secs(1), manager.read())
.await
.expect("manager reads must not wait for tier A leases");
let next_b = tokio::time::timeout(Duration::from_secs(1), TierConfigMgr::acquire_operation_lease(&manager, "COLD-B"))
.await
.expect("tier B lease acquisition must not wait for tier A")
.expect("tier B lease should remain available");
assert_eq!(next_b.generation(), old_b.generation());
assert!(!publish.is_finished(), "tier A replacement should still wait for its lease");
drop(old_a);
publish
.await
.expect("publish task should join")
.expect("tier A replacement should publish after its lease drains");
}
#[tokio::test]
async fn direct_mutation_fails_closed_while_generation_is_active() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("lease should be available");
let err = manager
.write()
.await
.remove("COLD-A", true)
.await
.expect_err("direct removal must not wait for an active generation");
assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
assert!(lease.is_current(&manager).await);
}
#[tokio::test]
async fn steady_lease_acquisition_is_concurrent_across_tiers() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("a"));
install_lease_backend(&mut guard, "COLD-B", LeaseTestBackend::ready("b"));
}
let mut tasks = Vec::new();
for index in 0..200 {
let manager = manager.clone();
tasks.push(tokio::spawn(async move {
let tier = if index % 2 == 0 { "COLD-A" } else { "COLD-B" };
TierConfigMgr::acquire_operation_lease(&manager, tier).await
}));
}
tokio::time::timeout(Duration::from_secs(1), async {
for task in tasks {
task.await
.expect("lease task should join")
.expect("published generation should remain available");
}
})
.await
.expect("steady lease acquisition should not serialize on manager writes or config hashing");
}
#[tokio::test]
async fn cancelled_operation_releases_generation_lease() {
let backend = LeaseTestBackend::blocking("cancelled");
let manager = Arc::new(RwLock::new(empty_mgr()));
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", backend.clone());
}
let lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("lease should be available");
let task = tokio::spawn(async move { lease.remove("object", "").await });
backend
.remove_started
.as_ref()
.expect("blocking backend should expose start notification")
.notified()
.await;
task.abort();
let _ = task.await;
let manager = manager.read().await;
let runtime = registered_tier_driver_runtime(&manager).expect("runtime sidecar should be registered");
drop(manager);
let runtime = lock_unpoisoned(&runtime);
let generation = runtime.generations.get("COLD-A").expect("generation should remain installed");
assert_eq!(generation.active_leases.load(Ordering::Acquire), 0);
}
#[tokio::test]
async fn tier_generations_are_isolated_between_runtime_instances() {
let manager_a = TierConfigMgr::new();
let manager_b = TierConfigMgr::new();
{
let mut guard = manager_a.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("a"));
}
{
let mut guard = manager_b.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("b"));
}
let lease_a = TierConfigMgr::acquire_operation_lease(&manager_a, "COLD-A")
.await
.expect("instance A lease should be available");
let lease_b = TierConfigMgr::acquire_operation_lease(&manager_b, "COLD-A")
.await
.expect("instance B lease should be available");
assert!(lease_a.is_current(&manager_a).await);
assert!(lease_b.is_current(&manager_b).await);
assert!(!lease_a.is_current(&manager_b).await);
assert!(!lease_b.is_current(&manager_a).await);
}
#[tokio::test]
async fn lease_acquisition_has_constant_generation_state() {
let manager = Arc::new(RwLock::new(empty_mgr()));
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("bounded"));
}
for _ in 0..10_000 {
drop(
TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("lease should be available"),
);
}
let manager = manager.read().await;
let runtime = registered_tier_driver_runtime(&manager).expect("runtime sidecar should be registered");
drop(manager);
let runtime = lock_unpoisoned(&runtime);
assert_eq!(runtime.generations.len(), 1);
assert_eq!(runtime.next_generation, 1);
assert_eq!(
runtime
.generations
.get("COLD-A")
.expect("generation should remain installed")
.active_leases
.load(Ordering::Acquire),
0
);
}
#[tokio::test]
async fn unrelated_reload_keeps_active_generation_current() {
let manager = Arc::new(RwLock::new(empty_mgr()));
{
let mut manager = manager.write().await;
install_lease_backend(&mut manager, "COLD-A", LeaseTestBackend::ready("a"));
manager.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B"));
}
let lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("lease should be available");
let generation = lease.generation();
let mut reloaded = HashMap::from([
("COLD-A".to_string(), build_rustfs_tier("COLD-A")),
("COLD-B".to_string(), build_rustfs_tier("COLD-B")),
("COLD-C".to_string(), build_rustfs_tier("COLD-C")),
]);
reloaded
.get_mut("COLD-B")
.and_then(|tier| tier.rustfs.as_mut())
.expect("tier B payload should exist")
.access_key = "rotated".to_string();
manager
.write()
.await
.apply_reloaded_tiers(reloaded)
.expect("unrelated reload should apply");
assert!(lease.is_current(&manager).await);
let next = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("tier A should remain available");
assert_eq!(next.generation(), generation);
}
#[tokio::test]
async fn backend_identity_survives_credentials_rotation_but_rejects_route_change() {
let manager = Arc::new(RwLock::new(empty_mgr()));
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("old lease should be available");
let identity = old.backend_identity();
let mut rotated = build_rustfs_tier("COLD-A");
let rotated_config = rotated.rustfs.as_mut().expect("rustfs payload should exist");
rotated_config.access_key = "rotated-access".to_string();
rotated_config.secret_key = "rotated-secret".to_string();
manager.write().await.tiers.insert("COLD-A".to_string(), rotated);
manager
.write()
.await
.replace_driver("COLD-A", Box::new(LeaseTestBackend::ready("rotated")))
.expect("rotated driver should install");
let rotated = TierConfigMgr::acquire_operation_lease_for_backend_identity(&manager, "COLD-A", identity)
.await
.expect("credential rotation should preserve backend identity");
assert_ne!(rotated.generation(), old.generation());
assert_eq!(rotated.backend_identity(), identity);
let mut moved = build_rustfs_tier("COLD-A");
moved.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://moved.invalid".to_string();
manager.write().await.tiers.insert("COLD-A".to_string(), moved);
manager
.write()
.await
.replace_driver("COLD-A", Box::new(LeaseTestBackend::ready("moved")))
.expect("moved driver should install");
let err = match TierConfigMgr::acquire_operation_lease_for_backend_identity(&manager, "COLD-A", identity).await {
Ok(_) => panic!("route change must fail closed"),
Err(err) => err,
};
assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
}
fn build_azure_tier(account_name: &str) -> TierConfig {
TierConfig {
version: "v1".to_string(),
tier_type: TierType::Azure,
name: "COLD-AZURE".to_string(),
azure: Some(crate::services::tier::tier_config::TierAzure {
endpoint: "https://blob.example.invalid".to_string(),
access_key: account_name.to_string(),
secret_key: "account-key".to_string(),
bucket: "archive".to_string(),
prefix: "objects/".to_string(),
region: "us-east-1".to_string(),
..Default::default()
}),
..Default::default()
}
}
#[test]
fn azure_account_name_is_part_of_backend_identity() {
let account_a = build_azure_tier("account-a");
let mut rotated_key = build_azure_tier("account-a");
rotated_key.azure.as_mut().expect("azure payload should exist").secret_key = "rotated-key".to_string();
let account_b = build_azure_tier("account-b");
assert_eq!(
tier_backend_identity(&account_a).expect("account A identity should encode"),
tier_backend_identity(&rotated_key).expect("rotated key identity should encode"),
"Azure account-key rotation must preserve destination identity"
);
assert_ne!(
tier_backend_identity(&account_a).expect("account A identity should encode"),
tier_backend_identity(&account_b).expect("account B identity should encode"),
"Azure account-name changes must fence cleanup from the new account"
);
}
#[test]
fn backend_identity_v2_has_stable_fixtures() {
assert_eq!(
tier_backend_identity(&build_rustfs_tier("COLD-A")).expect("identity should encode"),
[
112, 73, 111, 29, 53, 43, 207, 7, 170, 12, 99, 116, 213, 135, 173, 227, 6, 220, 179, 135, 232, 83, 25, 13, 128,
98, 254, 103, 132, 128, 229, 97,
]
);
assert_eq!(
tier_backend_identity(&build_azure_tier("account-a")).expect("Azure identity should encode"),
[
57, 188, 231, 23, 184, 188, 233, 228, 118, 24, 158, 152, 178, 169, 2, 146, 149, 152, 104, 0, 146, 196, 61, 145,
18, 1, 188, 20, 36, 215, 100, 125,
]
);
}
#[test]
fn destination_identity_prefix_normalization_matches_driver_matrix() {
assert_eq!(normalized_tier_prefix(&TierType::S3, "/foo//"), "foo");
for tier_type in [
TierType::RustFS,
TierType::MinIO,
TierType::Aliyun,
TierType::Tencent,
TierType::Huaweicloud,
TierType::Azure,
TierType::GCS,
TierType::R2,
] {
assert_eq!(normalized_tier_prefix(&tier_type, "foo/"), "foo");
assert_eq!(normalized_tier_prefix(&tier_type, "foo//"), "foo/");
}
}
#[tokio::test]
async fn direct_rollback_fails_closed_until_unpublished_generation_is_idle() {
let manager = Arc::new(RwLock::new(empty_mgr()));
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("persisted"));
}
let persisted = HashMap::from([("COLD-A".to_string(), build_rustfs_tier("COLD-A"))]);
let mut unpublished_config = build_rustfs_tier("COLD-A");
unpublished_config
.rustfs
.as_mut()
.expect("rustfs payload should exist")
.endpoint = "https://unpublished.invalid".to_string();
{
let mut manager = manager.write().await;
manager.tiers.insert("COLD-A".to_string(), unpublished_config);
manager
.replace_driver("COLD-A", Box::new(LeaseTestBackend::ready("unpublished")))
.expect("unpublished driver should install before simulated save failure");
}
let unpublished = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("unpublished lease should be visible before rollback");
let err = manager
.write()
.await
.apply_reloaded_tiers(persisted.clone())
.expect_err("direct rollback must not wait for an active generation");
assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
assert!(unpublished.is_current(&manager).await);
drop(unpublished);
manager
.write()
.await
.apply_reloaded_tiers(persisted)
.expect("rollback should apply once the generation is idle");
let manager_guard = manager.read().await;
assert_eq!(
manager_guard
.tiers
.get("COLD-A")
.and_then(|tier| tier.rustfs.as_ref())
.expect("persisted tier should be restored")
.endpoint,
"https://example-compat.invalid"
);
let runtime = registered_tier_driver_runtime(&manager_guard).expect("runtime sidecar should remain registered");
assert!(lock_unpoisoned(&runtime).generations.get("COLD-A").is_none());
}
#[derive(Debug, Default)]
struct CasConfigStore {
state: tokio::sync::Mutex<Option<(Vec<u8>, String)>>,
next_etag: AtomicUsize,
fail_put: AtomicBool,
}
#[async_trait::async_trait]
impl ObjectIO for CasConfigStore {
type Error = Error;
type RangeSpec = HTTPRangeSpec;
type HeaderMap = HeaderMap;
type ObjectOptions = ObjectOptions;
type ObjectInfo = ObjectInfo;
type GetObjectReader = GetObjectReader;
type PutObjectReader = PutObjReader;
async fn get_object_reader(
&self,
bucket: &str,
object: &str,
_range: Option<Self::RangeSpec>,
_headers: Self::HeaderMap,
_opts: &Self::ObjectOptions,
) -> Result<Self::GetObjectReader> {
let state = self.state.lock().await;
let (data, etag) = state.as_ref().ok_or(Error::ConfigNotFound)?;
Ok(GetObjectReader {
stream: Box::new(Cursor::new(data.clone())),
object_info: ObjectInfo {
bucket: bucket.to_string(),
name: object.to_string(),
size: data.len() as i64,
actual_size: data.len() as i64,
etag: Some(etag.clone()),
..Default::default()
},
buffered_body: None,
body_source: Default::default(),
})
}
async fn put_object(
&self,
bucket: &str,
object: &str,
data: &mut Self::PutObjectReader,
opts: &Self::ObjectOptions,
) -> Result<Self::ObjectInfo> {
if self.fail_put.load(Ordering::SeqCst) {
return Err(Error::other("injected tier config save failure"));
}
let mut payload = Vec::new();
tokio::io::AsyncReadExt::read_to_end(&mut data.stream, &mut payload).await?;
let mut state = self.state.lock().await;
match state.as_ref() {
Some((_, etag)) => opts.precondition_check(&ObjectInfo {
etag: Some(etag.clone()),
..Default::default()
})?,
None => {
if opts
.http_preconditions
.as_ref()
.and_then(HTTPPreconditions::if_match_value)
.is_some()
{
return Err(Error::ObjectNotFound(bucket.to_string(), object.to_string()));
}
}
}
let etag = format!("etag-{}", self.next_etag.fetch_add(1, Ordering::SeqCst) + 1);
*state = Some((payload.clone(), etag.clone()));
Ok(ObjectInfo {
bucket: bucket.to_string(),
name: object.to_string(),
size: payload.len() as i64,
actual_size: payload.len() as i64,
etag: Some(etag),
..Default::default()
})
}
}
#[tokio::test]
async fn remove_and_clear_full_update_paths_preserve_force() {
let remove_store = Arc::new(CasConfigStore::default());
let mut persisted = empty_mgr();
persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
persisted
.save_tiering_config_if_current(remove_store.clone(), None)
.await
.expect("remove fixture should persist");
let remove_manager = TierConfigMgr::new();
{
let mut guard = remove_manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::in_use("remove"));
}
TierConfigMgr::remove_and_save_with(&remove_manager, remove_store.clone(), "COLD-A", true)
.await
.expect("forced remove must complete through load, mutate, save, and publish");
assert!(!remove_manager.read().await.tiers.contains_key("COLD-A"));
assert!(
load_tier_config_for_update(remove_store)
.await
.expect("removed config should reload")
.0
.tiers
.is_empty(),
"forced remove must persist the empty candidate"
);
let clear_store = Arc::new(CasConfigStore::default());
persisted
.save_tiering_config_if_current(clear_store.clone(), None)
.await
.expect("clear fixture should persist");
let clear_manager = TierConfigMgr::new();
{
let mut guard = clear_manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::in_use("clear"));
}
TierConfigMgr::clear_and_save_with(&clear_manager, clear_store.clone(), true)
.await
.expect("forced clear must complete through load, mutate, save, and publish");
assert!(clear_manager.read().await.tiers.is_empty());
assert!(
load_tier_config_for_update(clear_store)
.await
.expect("cleared config should reload")
.0
.tiers
.is_empty(),
"forced clear must persist the empty candidate"
);
}
#[tokio::test]
async fn failed_owned_update_restores_generation_and_reports_save_error() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let store = Arc::new(CasConfigStore::default());
store.fail_put.store(true, Ordering::SeqCst);
let mut candidate = empty_mgr();
candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
candidate
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("candidate")));
let update = TierConfigMgr::admin_update_lock(&manager).await;
let err = TierConfigMgr::update_candidate_owned(
&manager,
store.clone(),
candidate,
None,
TierCandidateMutation::Remove("COLD-A".to_string(), true),
update,
)
.await
.expect_err("save failure must be observable to the admin caller");
assert!(matches!(err, TierConfigUpdateError::Save(_)));
assert!(manager.read().await.tiers.contains_key("COLD-A"));
let restored = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("the old generation must be restored after save failure");
assert!(restored.is_current(&manager).await);
drop(restored);
let guard = manager.read().await;
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
assert!(!lock_unpoisoned(&runtime).draining.contains_key("COLD-A"));
drop(guard);
store.fail_put.store(false, Ordering::SeqCst);
let mut retry = empty_mgr();
retry.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
retry
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("retry")));
let update = TierConfigMgr::admin_update_lock(&manager).await;
TierConfigMgr::update_candidate_owned(
&manager,
store,
retry,
None,
TierCandidateMutation::Remove("COLD-A".to_string(), true),
update,
)
.await
.expect("a later update must succeed after save failure recovery");
assert!(!manager.read().await.tiers.contains_key("COLD-A"));
}
#[tokio::test]
async fn cancelled_owned_update_finishes_after_lease_drain() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("old generation lease should be available");
let store = Arc::new(CasConfigStore::default());
let mut candidate = empty_mgr();
candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
candidate
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("candidate")));
let update = TierConfigMgr::admin_update_lock(&manager).await;
let update_manager = manager.clone();
let update_store = store.clone();
let caller = tokio::spawn(async move {
TierConfigMgr::update_candidate_owned(
&update_manager,
update_store,
candidate,
None,
TierCandidateMutation::Remove("COLD-A".to_string(), true),
update,
)
.await
});
tokio::time::timeout(Duration::from_secs(1), async {
while old.is_current(&manager).await {
tokio::task::yield_now().await;
}
})
.await
.expect("owned update should revoke before caller cancellation");
caller.abort();
drop(old);
tokio::time::timeout(Duration::from_secs(1), async {
while manager.read().await.tiers.contains_key("COLD-A") {
tokio::task::yield_now().await;
}
})
.await
.expect("detached update must finish after its operation lease drains");
let guard = manager.read().await;
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
assert!(!lock_unpoisoned(&runtime).draining.contains_key("COLD-A"));
assert!(store.state.lock().await.is_some(), "detached update must persist its result");
}
#[tokio::test]
async fn panicked_owned_update_restores_generation_and_reports_error() {
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let store = Arc::new(CasConfigStore::default());
let mut candidate = empty_mgr();
candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
candidate
.driver_cache
.insert("COLD-A".to_string(), Box::new(LeaseTestBackend::panicking_in_use("panic")));
let update = TierConfigMgr::admin_update_lock(&manager).await;
let err = TierConfigMgr::update_candidate_owned(
&manager,
store,
candidate,
None,
TierCandidateMutation::Remove("COLD-A".to_string(), false),
update,
)
.await
.expect_err("mutation panic must be observable to the admin caller");
let TierConfigUpdateError::Publish(err) = err else {
panic!("mutation panic should be reported as a publish failure");
};
assert!(err.message.contains("panicked"));
assert!(manager.read().await.tiers.contains_key("COLD-A"));
let restored = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("the old generation must be restored after panic");
assert!(restored.is_current(&manager).await);
drop(restored);
let guard = manager.read().await;
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
assert!(!lock_unpoisoned(&runtime).draining.contains_key("COLD-A"));
}
#[tokio::test]
async fn stale_tier_config_etag_allows_only_one_candidate_to_publish() {
let store = Arc::new(CasConfigStore::default());
empty_mgr()
.save_tiering_config_if_current(store.clone(), None)
.await
.expect("initial conditional create should succeed");
let (mut candidate_a, version_a) = load_tier_config_for_update(store.clone())
.await
.expect("first node should load config revision");
let (mut candidate_b, version_b) = load_tier_config_for_update(store.clone())
.await
.expect("second node should load the same config revision");
assert_eq!(version_a, version_b);
candidate_a.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
candidate_b.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B"));
let save_a = candidate_a.save_tiering_config_if_current(store.clone(), version_a.as_deref());
let save_b = candidate_b.save_tiering_config_if_current(store, version_b.as_deref());
let (result_a, result_b) = tokio::join!(save_a, save_b);
assert_ne!(result_a.is_ok(), result_b.is_ok(), "exactly one stale-ETag writer must win");
let manager_a = TierConfigMgr::new();
let manager_b = TierConfigMgr::new();
if result_a.is_ok() {
TierConfigMgr::publish_candidate(&manager_a, candidate_a, None)
.await
.expect("winning node should publish");
}
if result_b.is_ok() {
TierConfigMgr::publish_candidate(&manager_b, candidate_b, None)
.await
.expect("winning node should publish");
}
assert_ne!(manager_a.read().await.empty(), manager_b.read().await.empty());
}
#[test]
fn legacy_refresh_method_signature_remains_callable() {
async fn call_legacy_refresh(manager: &mut TierConfigMgr, api: Arc<ECStore>) {
manager.refresh_tier_config(api).await;
}
let _ = call_legacy_refresh;
}
#[tokio::test(start_paused = true)]
async fn periodic_refresh_timer_does_not_tick_on_startup() {
let period = Duration::from_secs(60);
let mut timer = delayed_tier_refresh_interval(period);
assert!(
tokio::time::timeout(Duration::ZERO, timer.tick()).await.is_err(),
"periodic reload must not duplicate the startup reload"
);
tokio::time::advance(period).await;
tokio::time::timeout(Duration::ZERO, timer.tick())
.await
.expect("the first periodic reload should become ready after one interval");
}
}
Reference in New Issue View Git Blame Copy Permalink