From 21049401fa84542b2696d609381e4c4564e7ddbc Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 19 Jul 2026 18:48:32 +0800 Subject: [PATCH] fix(ilm): harden tier transition failure boundaries (#5031) * fix(tier): fence generation-scoped operations Refs rustfs/backlog#1354 Co-Authored-By: heihutu * fix(ilm): verify transition upload streams Refs rustfs/backlog#1353 Co-Authored-By: heihutu * test(ecstore): expand transition fault matrix Refs rustfs/backlog#1355 Co-Authored-By: heihutu --------- Co-authored-by: heihutu --- crates/ecstore/src/api/mod.rs | 6 +- .../bucket/lifecycle/bucket_lifecycle_ops.rs | 403 ++- .../bucket/lifecycle/tier_delete_journal.rs | 132 +- .../src/bucket/lifecycle/tier_sweeper.rs | 91 +- crates/ecstore/src/runtime/sources.rs | 7 +- crates/ecstore/src/services/tier/test_util.rs | 39 +- crates/ecstore/src/services/tier/tier.rs | 3142 ++++++++++++++++- .../ecstore/src/services/tier/tier_config.rs | 17 + .../ecstore/src/services/tier/warm_backend.rs | 8 + crates/ecstore/src/set_disk/mod.rs | 4 +- crates/ecstore/src/set_disk/ops/multipart.rs | 68 + crates/ecstore/src/set_disk/ops/object.rs | 1186 ++++++- crates/ecstore/src/set_disk/read.rs | 144 +- .../src/set_disk/transition_matrix_tests.rs | 104 + crates/ecstore/src/store/mod.rs | 2 +- crates/filemeta/src/filemeta/version.rs | 123 +- crates/utils/src/http/metadata_compat.rs | 65 + rustfs/src/admin/handlers/tier.rs | 399 ++- rustfs/src/admin/storage_api.rs | 3 +- .../src/app/lifecycle_transition_api_test.rs | 3 - rustfs/src/app/mod.rs | 2 +- rustfs/src/app/object_usecase.rs | 97 +- rustfs/src/app/storage_api.rs | 7 + rustfs/src/storage/storage_api.rs | 6 +- ...alidate_object_data_cache_cold_stampede.sh | 173 +- 25 files changed, 5808 insertions(+), 423 deletions(-) create mode 100644 crates/ecstore/src/set_disk/transition_matrix_tests.rs diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index d51fbaa8e..ca1a1b439 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -67,7 +67,9 @@ pub mod bucket { } pub mod tier_delete_journal { - pub use crate::bucket::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry; + pub use crate::bucket::lifecycle::tier_delete_journal::{ + persist_tier_delete_journal_entry, record_tier_delete_journal_backend_identity, + }; } pub mod tier_last_day_stats { @@ -408,7 +410,7 @@ pub mod tier { pub use crate::services::tier::tier::{ ERR_TIER_BACKEND_IN_USE, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_INVALID_CONFIG, ERR_TIER_MISSING_CREDENTIALS, ERR_TIER_TYPE_UNSUPPORTED, TIER_CONFIG_FILE, TIER_CONFIG_FORMAT, TIER_CONFIG_V1, TIER_CONFIG_VERSION, TierConfigMgr, - is_err_config_not_found, try_migrate_tiering_config, + TierConfigUpdateError, is_err_config_not_found, try_migrate_tiering_config, }; } diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index a04140810..f06e33cd3 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -29,7 +29,7 @@ use crate::bucket::lifecycle::replication_sink::{ use crate::bucket::lifecycle::tier_delete_journal::{process_tier_delete_journal_entry, run_tier_delete_journal_recovery_loop}; use crate::bucket::lifecycle::tier_free_version_recovery::{DEFAULT_FREE_VERSION_RECOVERY_LIMIT, recover_tier_free_versions}; use crate::bucket::lifecycle::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats}; -use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_idempotent}; +use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_idempotent_with_manager_and_identity}; use crate::bucket::versioning_sys::BucketVersioningSys; use crate::client::object_api_utils::new_getobjectreader; use crate::disk::error::DiskError; @@ -38,7 +38,10 @@ use crate::error::Error; use crate::error::StorageError; use crate::error::{error_resp_to_object_err, is_err_object_not_found, is_err_version_not_found, is_network_or_host_down}; use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions}; -use crate::services::tier::warm_backend::WarmBackendGetOpts; +use crate::services::tier::{ + tier::{TierConfigMgr, tier_destination_id_from_metadata}, + warm_backend::WarmBackendGetOpts, +}; use crate::set_disk::{MAX_PARTS_COUNT, RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, SetDisks}; use crate::storage_api_contracts::{ lifecycle::ExpirationOptions, @@ -402,6 +405,36 @@ impl ExpiryOp for FreeVersionTask { } } +async fn delete_free_version_remote_object( + oi: &ObjectInfo, + tier_config_mgr: &Arc>, +) -> Result<(), std::io::Error> { + let identity = tier_destination_id_from_metadata(&oi.user_defined)? + .ok_or_else(|| std::io::Error::other("tier free-version has no durable backend identity"))?; + delete_object_from_remote_tier_idempotent_with_manager_and_identity( + &oi.transitioned_object.name, + &oi.transitioned_object.version_id, + &oi.transitioned_object.tier, + identity, + tier_config_mgr, + ) + .await?; + Ok(()) +} + +async fn delete_free_version_remote_object_then( + oi: &ObjectInfo, + tier_config_mgr: &Arc>, + delete_local: F, +) -> Result +where + F: FnOnce() -> Fut, + Fut: std::future::Future, +{ + delete_free_version_remote_object(oi, tier_config_mgr).await?; + Ok(delete_local().await) +} + struct NewerNoncurrentTask { bucket: String, versions: Vec, @@ -680,13 +713,60 @@ impl ExpiryState { else if v.as_any().is::() { let v = v.as_any().downcast_ref::().expect("FreeVersionTask downcast failed"); let oi = v.0.clone(); - if let Err(err) = delete_object_from_remote_tier_idempotent( - &oi.transitioned_object.name, - &oi.transitioned_object.version_id, - &oi.transitioned_object.tier, - ) - .await - { + let cleanup = delete_free_version_remote_object_then(&oi, &api.tier_config_mgr(), || async { + let mut fi = FileInfo { + name: oi.name.clone(), + version_id: oi.version_id, + deleted: true, + ..Default::default() + }; + fi.set_tier_free_version(); + + let mut deleted_locally = false; + for pool in api.pools.iter() { + let set = pool.get_disks_by_key(&oi.name); + match set.delete_object_version(&oi.bucket, &oi.name, &fi, false).await { + Ok(()) => { + deleted_locally = true; + break; + } + Err(err) if is_err_version_not_found(&err) || is_err_object_not_found(&err) => continue, + Err(err) => { + debug!( + event = EVENT_LIFECYCLE_WORKER_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + bucket = %oi.bucket, + object = %oi.name, + remote_object = %oi.transitioned_object.name, + remote_version_id = %oi.transitioned_object.version_id, + tier = %oi.transitioned_object.tier, + error = ?err, + reason = "local_free_version_delete_failed", + "Lifecycle worker failed local free-version cleanup" + ); + break; + } + } + } + + if !deleted_locally { + debug!( + event = EVENT_LIFECYCLE_WORKER_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + bucket = %oi.bucket, + object = %oi.name, + remote_object = %oi.transitioned_object.name, + remote_version_id = %oi.transitioned_object.version_id, + tier = %oi.transitioned_object.tier, + reason = "local_free_version_missing", + "Lifecycle worker could not find transitioned free version locally" + ); + } + }) + .await; + if let Err(err) = cleanup { debug!( bucket = %oi.bucket, object = %oi.name, @@ -702,57 +782,6 @@ impl ExpiryState { ); continue; } - - let mut fi = FileInfo { - name: oi.name.clone(), - version_id: oi.version_id, - deleted: true, - ..Default::default() - }; - fi.set_tier_free_version(); - - let mut deleted_locally = false; - for pool in api.pools.iter() { - let set = pool.get_disks_by_key(&oi.name); - match set.delete_object_version(&oi.bucket, &oi.name, &fi, false).await { - Ok(()) => { - deleted_locally = true; - break; - } - Err(err) if is_err_version_not_found(&err) || is_err_object_not_found(&err) => continue, - Err(err) => { - debug!( - event = EVENT_LIFECYCLE_WORKER_STATE, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_LIFECYCLE, - bucket = %oi.bucket, - object = %oi.name, - remote_object = %oi.transitioned_object.name, - remote_version_id = %oi.transitioned_object.version_id, - tier = %oi.transitioned_object.tier, - error = ?err, - reason = "local_free_version_delete_failed", - "Lifecycle worker failed local free-version cleanup" - ); - break; - } - } - } - - if !deleted_locally { - debug!( - event = EVENT_LIFECYCLE_WORKER_STATE, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_LIFECYCLE, - bucket = %oi.bucket, - object = %oi.name, - remote_object = %oi.transitioned_object.name, - remote_version_id = %oi.transitioned_object.version_id, - tier = %oi.transitioned_object.tier, - reason = "local_free_version_missing", - "Lifecycle worker could not find transitioned free version locally" - ); - } } else { //info!("Invalid work type - {:?}", v); @@ -2444,8 +2473,27 @@ pub async fn get_transitioned_object_reader( opts: &ObjectOptions, ) -> Result { let tier_config_mgr = runtime_sources::tier_config_mgr_handle(); - let mut tier_config_mgr = tier_config_mgr.write().await; - let tgt_client = match tier_config_mgr.get_driver(&oi.transitioned_object.tier).await { + get_transitioned_object_reader_with_tier_manager(bucket, object, rs, h, oi, opts, &tier_config_mgr).await +} + +pub(crate) async fn get_transitioned_object_reader_with_tier_manager( + bucket: &str, + object: &str, + rs: &Option, + h: &HeaderMap, + oi: &ObjectInfo, + opts: &ObjectOptions, + tier_config_mgr: &Arc>, +) -> Result { + let expected_identity = tier_destination_id_from_metadata(&oi.user_defined)?; + let lease = match expected_identity { + Some(identity) => { + TierConfigMgr::acquire_operation_lease_for_backend_identity(tier_config_mgr, &oi.transitioned_object.tier, identity) + .await + } + None => TierConfigMgr::acquire_operation_lease(tier_config_mgr, &oi.transitioned_object.tier).await, + }; + let tgt_client = match lease { Ok(d) => d, Err(err) => return Err(std::io::Error::other(err)), }; @@ -3081,6 +3129,8 @@ mod tests { select_restore_s3_location, should_defer_date_expiry_for_recent_config_update, should_reuse_lifecycle_delete_replication_state, transitioned_cleanup_tuple, transitioned_object_delete_opts, }; + #[cfg(feature = "test-util")] + use super::{delete_free_version_remote_object_then, get_transitioned_object_reader_with_tier_manager}; use crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc; use crate::bucket::lifecycle::replication_sink::{ ReplicateDecision, ReplicateTargetDecision, ReplicationStatusType, VersionPurgeStatusType, @@ -3121,6 +3171,233 @@ mod tests { use tokio_util::sync::CancellationToken; use uuid::Uuid; + #[cfg(feature = "test-util")] + #[tokio::test] + async fn free_version_remote_delete_requires_persisted_destination_identity() { + let manager = crate::services::tier::tier::TierConfigMgr::new(); + let old_backend = crate::services::tier::test_util::register_mock_tier(&manager, "WARM").await; + let mut conflicting_sys = HashMap::new(); + rustfs_utils::http::metadata_compat::insert_bytes( + &mut conflicting_sys, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_STATUS, + crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.as_bytes().to_vec(), + ); + conflicting_sys.insert( + format!( + "{}{}", + rustfs_utils::http::metadata_compat::RUSTFS_INTERNAL_PREFIX, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID + ), + b"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef".to_vec(), + ); + conflicting_sys.insert( + format!( + "{}{}", + rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID + ), + b"abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789".to_vec(), + ); + let conflicting_meta = rustfs_filemeta::MetaObject { + meta_sys: conflicting_sys, + ..Default::default() + }; + let mut free_version_info = rustfs_filemeta::FileInfo::new("object", 2, 2); + free_version_info.set_tier_free_version_id(&Uuid::new_v4().to_string()); + assert_eq!( + conflicting_meta + .init_free_version(&free_version_info) + .expect_err("conflicting persisted identities must not create an executable free-version"), + rustfs_filemeta::Error::FileCorrupt + ); + assert_eq!(old_backend.remove_count().await, 0); + + let old_identity = crate::services::tier::tier::TierConfigMgr::acquire_operation_lease(&manager, "WARM") + .await + .expect("old tier lease should be available") + .backend_identity(); + let mut oi = ObjectInfo::default(); + oi.transitioned_object.tier = "WARM".to_string(); + oi.transitioned_object.name = "remote/object".to_string(); + oi.transitioned_object.version_id = "remote-version".to_string(); + let local_delete_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + + let legacy_err = delete_free_version_remote_object_then(&oi, &manager, { + let local_delete_calls = Arc::clone(&local_delete_calls); + move || async move { + local_delete_calls.fetch_add(1, Ordering::Relaxed); + } + }) + .await + .expect_err("legacy free-version without identity must be retained"); + assert!(legacy_err.to_string().contains("no durable backend identity")); + assert_eq!(local_delete_calls.load(Ordering::Relaxed), 0); + + let mut invalid_metadata = HashMap::new(); + rustfs_utils::http::metadata_compat::insert_str( + &mut invalid_metadata, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + "not-a-backend-identity".to_string(), + ); + oi.user_defined = Arc::new(invalid_metadata); + let invalid_err = delete_free_version_remote_object_then(&oi, &manager, { + let local_delete_calls = Arc::clone(&local_delete_calls); + move || async move { + local_delete_calls.fetch_add(1, Ordering::Relaxed); + } + }) + .await + .expect_err("free-version with an invalid identity must be retained"); + assert!(invalid_err.to_string().contains("invalid length")); + assert_eq!(local_delete_calls.load(Ordering::Relaxed), 0); + + let mut metadata = HashMap::new(); + rustfs_utils::http::metadata_compat::insert_str( + &mut metadata, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + rustfs_utils::crypto::hex(old_identity), + ); + oi.user_defined = Arc::new(metadata.clone()); + delete_free_version_remote_object_then(&oi, &manager, { + let local_delete_calls = Arc::clone(&local_delete_calls); + move || async move { + local_delete_calls.fetch_add(1, Ordering::Relaxed); + } + }) + .await + .expect("matching destination identity should allow idempotent remote cleanup"); + assert_eq!(old_backend.remove_count().await, 1); + assert_eq!(local_delete_calls.load(Ordering::Relaxed), 1); + + let mut single_prefix_metadata = HashMap::new(); + single_prefix_metadata.insert( + format!( + "{}{}", + rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID + ), + rustfs_utils::crypto::hex(old_identity), + ); + oi.user_defined = Arc::new(single_prefix_metadata); + delete_free_version_remote_object_then(&oi, &manager, { + let local_delete_calls = Arc::clone(&local_delete_calls); + move || async move { + local_delete_calls.fetch_add(1, Ordering::Relaxed); + } + }) + .await + .expect("single-prefix legacy identity should remain compatible"); + assert_eq!(old_backend.remove_count().await, 2); + assert_eq!(local_delete_calls.load(Ordering::Relaxed), 2); + + let new_backend = crate::services::tier::test_util::register_mock_tier(&manager, "WARM").await; + let new_identity = crate::services::tier::tier::TierConfigMgr::acquire_operation_lease(&manager, "WARM") + .await + .expect("rebound tier lease should be available") + .backend_identity(); + + let mut conflicting_metadata = HashMap::from([( + rustfs_utils::http::metadata_compat::internal_key_rustfs( + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + ), + rustfs_utils::crypto::hex(new_identity), + )]); + conflicting_metadata.insert( + format!( + "{}{}", + rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID + ), + rustfs_utils::crypto::hex(old_identity), + ); + oi.user_defined = Arc::new(conflicting_metadata); + let conflict_err = delete_free_version_remote_object_then(&oi, &manager, { + let local_delete_calls = Arc::clone(&local_delete_calls); + move || async move { + local_delete_calls.fetch_add(1, Ordering::Relaxed); + } + }) + .await + .expect_err("conflicting compatibility identities must retain the free-version"); + assert!(conflict_err.to_string().contains("compatibility keys conflict")); + assert_eq!(new_backend.remove_count().await, 0); + assert_eq!(local_delete_calls.load(Ordering::Relaxed), 2); + + oi.user_defined = Arc::new(metadata); + let rebound_err = delete_free_version_remote_object_then(&oi, &manager, { + let local_delete_calls = Arc::clone(&local_delete_calls); + move || async move { + local_delete_calls.fetch_add(1, Ordering::Relaxed); + } + }) + .await + .expect_err("same-name tier rebind must retain the old free-version"); + assert!(rebound_err.to_string().contains("identity no longer matches")); + assert_eq!(new_backend.remove_count().await, 0); + assert_eq!(local_delete_calls.load(Ordering::Relaxed), 2); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + async fn transitioned_get_rejects_same_name_rebind_before_remote_io() { + let manager = crate::services::tier::tier::TierConfigMgr::new(); + crate::services::tier::test_util::register_mock_tier(&manager, "WARM").await; + let old_identity = crate::services::tier::tier::TierConfigMgr::acquire_operation_lease(&manager, "WARM") + .await + .expect("old tier lease should be available") + .backend_identity(); + let new_backend = crate::services::tier::test_util::register_mock_tier(&manager, "WARM").await; + + let mut metadata = HashMap::new(); + rustfs_utils::http::metadata_compat::insert_str( + &mut metadata, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + rustfs_utils::crypto::hex(old_identity), + ); + let mut oi = ObjectInfo { + user_defined: Arc::new(metadata), + ..Default::default() + }; + oi.transitioned_object.tier = "WARM".to_string(); + oi.transitioned_object.name = "remote/object".to_string(); + oi.transitioned_object.version_id = "remote-version".to_string(); + + let err = match get_transitioned_object_reader_with_tier_manager( + "bucket", + "object", + &None, + &http::HeaderMap::new(), + &oi, + &ObjectOptions::default(), + &manager, + ) + .await + { + Ok(_) => panic!("identity-bound GET must reject a same-name tier rebind"), + Err(err) => err, + }; + assert!(err.to_string().contains("identity no longer matches")); + assert_eq!(new_backend.get_count().await, 0); + + oi.user_defined = Arc::new(HashMap::new()); + let err = match get_transitioned_object_reader_with_tier_manager( + "bucket", + "object", + &None, + &http::HeaderMap::new(), + &oi, + &ObjectOptions::default(), + &manager, + ) + .await + { + Ok(_) => panic!("missing remote legacy object should return an error"), + Err(err) => err, + }; + assert!(!err.to_string().is_empty()); + assert_eq!(new_backend.get_count().await, 1); + } + /// Pins the expiry-event routing for transitioned objects /// (rustfs/backlog#1302): restore-expiry events must set /// `transition.expire_restored` (strip-restored-copy semantics, never a @@ -3192,6 +3469,7 @@ mod tests { obj_name: "remote/object".to_string(), version_id: "remote-version".to_string(), tier_name: "WARM".to_string(), + backend_identity: Some([1; 32]), }; let err = state @@ -3279,6 +3557,7 @@ mod tests { obj_name: "remote/object".to_string(), version_id: "remote-version".to_string(), tier_name: "WARM".to_string(), + backend_identity: Some([1; 32]), }; state diff --git a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs index 6a760a223..1c6778880 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs @@ -20,10 +20,11 @@ use tokio_util::sync::CancellationToken; use tracing::{debug, warn}; use crate::bucket::lifecycle::config_boundary; -use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_idempotent}; +use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_idempotent_with_manager_and_identity}; use crate::disk::RUSTFS_META_BUCKET; use crate::error::{Error, Result}; use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}; +use crate::services::tier::tier::tier_destination_id_from_metadata; use crate::storage_api_contracts::{ list::ListOperations as _, object::{DeletedObject, ObjectIO, ObjectOperations, ObjectToDelete}, @@ -37,7 +38,7 @@ const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle"; const EVENT_LIFECYCLE_TIER_DELETE_JOURNAL: &str = "lifecycle_tier_delete_journal"; pub const DEFAULT_TIER_DELETE_JOURNAL_RECOVERY_LIMIT: usize = 1_000; -const TIER_DELETE_JOURNAL_VERSION: u8 = 1; +const TIER_DELETE_JOURNAL_VERSION: u8 = 2; const TIER_DELETE_JOURNAL_PREFIX: &str = "ilm/tier-delete-journal/"; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -47,22 +48,26 @@ struct PersistedTierDeleteJournalEntry { obj_name: String, version_id: String, tier_name: String, + #[serde(default)] + backend_identity: Option<[u8; 32]>, } impl PersistedTierDeleteJournalEntry { fn from_jentry(je: &Jentry) -> Self { Self { - version: TIER_DELETE_JOURNAL_VERSION, + version: if je.backend_identity.is_some() { + TIER_DELETE_JOURNAL_VERSION + } else { + 1 + }, obj_name: je.obj_name.clone(), version_id: je.version_id.clone(), tier_name: je.tier_name.clone(), + backend_identity: je.backend_identity, } } fn into_jentry(self) -> Result { - if self.version != TIER_DELETE_JOURNAL_VERSION { - return Err(Error::other(format!("unsupported tier delete journal version {}", self.version))); - } // Empty `version_id` is a legal sentinel for objects transitioned to an // unversioned remote tier (see CLAUDE.md: a tier version of `None`/`""` // means the tier bucket is unversioned, so the remote delete is issued @@ -71,10 +76,19 @@ impl PersistedTierDeleteJournalEntry { if self.obj_name.is_empty() || self.tier_name.is_empty() { return Err(Error::other("tier delete journal entry is incomplete")); } + let backend_identity = match self.version { + 1 => None, + TIER_DELETE_JOURNAL_VERSION => Some( + self.backend_identity + .ok_or_else(|| Error::other("tier delete journal v2 entry is missing its backend identity"))?, + ), + version => return Err(Error::other(format!("unsupported tier delete journal version {version}"))), + }; Ok(Jentry { obj_name: self.obj_name, version_id: self.version_id, tier_name: self.tier_name, + backend_identity, }) } } @@ -95,6 +109,10 @@ pub(crate) fn tier_delete_journal_object_name(je: &Jentry) -> String { hasher.update(je.obj_name.as_bytes()); hasher.update([0]); hasher.update(je.version_id.as_bytes()); + if let Some(backend_identity) = je.backend_identity { + hasher.update([0]); + hasher.update(backend_identity); + } format!( "{TIER_DELETE_JOURNAL_PREFIX}{}.json", rustfs_utils::crypto::hex(hasher.finalize().as_slice()) @@ -112,6 +130,16 @@ fn encode_tier_delete_journal_entry(je: &Jentry) -> Result> { .map_err(|err| Error::other(format!("encode tier delete journal failed: {err}"))) } +pub fn record_tier_delete_journal_backend_identity( + je: &mut Jentry, + metadata: &std::collections::HashMap, +) -> std::io::Result<()> { + if let Some(identity) = tier_destination_id_from_metadata(metadata)? { + je.backend_identity = Some(identity); + } + Ok(()) +} + pub async fn persist_tier_delete_journal_entry(api: Arc, je: &Jentry) -> std::io::Result<()> where S: ObjectIO< @@ -148,7 +176,17 @@ where } pub async fn process_tier_delete_journal_entry(api: Arc, je: &Jentry) -> std::io::Result<()> { - delete_object_from_remote_tier_idempotent(&je.obj_name, &je.version_id, &je.tier_name).await?; + let backend_identity = je + .backend_identity + .ok_or_else(|| std::io::Error::other("legacy tier delete journal has no durable backend identity"))?; + delete_object_from_remote_tier_idempotent_with_manager_and_identity( + &je.obj_name, + &je.version_id, + &je.tier_name, + backend_identity, + &api.tier_config_mgr(), + ) + .await?; remove_tier_delete_journal_entry(api, je).await } @@ -218,6 +256,21 @@ pub async fn recover_tier_delete_journal_entries( } }; + if je.backend_identity.is_none() { + stats.failed += 1; + warn!( + event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + journal_object = %object.name, + remote_object = %je.obj_name, + remote_version_id = %je.version_id, + tier = %je.tier_name, + "Legacy tier delete journal entry has no durable backend identity and will be retained" + ); + continue; + } + match process_tier_delete_journal_entry(api.clone(), &je).await { Ok(()) => stats.deleted += 1, Err(err) => { @@ -281,7 +334,10 @@ pub async fn run_tier_delete_journal_recovery_loop(api: Arc, cancel_tok #[cfg(test)] mod tests { - use super::{decode_tier_delete_journal_entry, encode_tier_delete_journal_entry, tier_delete_journal_object_name}; + use super::{ + decode_tier_delete_journal_entry, encode_tier_delete_journal_entry, record_tier_delete_journal_backend_identity, + tier_delete_journal_object_name, + }; use crate::bucket::lifecycle::tier_sweeper::Jentry; fn journal_entry() -> Jentry { @@ -289,6 +345,7 @@ mod tests { obj_name: "remote/object".to_string(), version_id: "remote-version".to_string(), tier_name: "WARM".to_string(), + backend_identity: Some([7; 32]), } } @@ -302,6 +359,7 @@ mod tests { assert_eq!(decoded.obj_name, je.obj_name); assert_eq!(decoded.version_id, je.version_id); assert_eq!(decoded.tier_name, je.tier_name); + assert_eq!(decoded.backend_identity, je.backend_identity); } #[test] @@ -317,6 +375,63 @@ mod tests { assert!(!first.contains("remote/object")); } + #[test] + fn tier_delete_journal_paths_separate_legacy_and_backend_identities() { + let mut legacy = journal_entry(); + legacy.backend_identity = None; + let mut backend_a = journal_entry(); + backend_a.backend_identity = Some([1; 32]); + let mut backend_b = journal_entry(); + backend_b.backend_identity = Some([2; 32]); + + assert_eq!( + tier_delete_journal_object_name(&legacy), + "ilm/tier-delete-journal/5ba6a7eb6338412b771613a6845a42ae5b8e26b5d201323eb01b38c5b42ff300.json" + ); + assert_ne!(tier_delete_journal_object_name(&legacy), tier_delete_journal_object_name(&backend_a)); + assert_ne!(tier_delete_journal_object_name(&backend_a), tier_delete_journal_object_name(&backend_b)); + } + + #[test] + fn tier_delete_journal_v2_requires_backend_identity() { + let payload = br#"{"version":2,"obj_name":"remote/object","version_id":"v1","tier_name":"WARM"}"#; + + let err = decode_tier_delete_journal_entry(payload).expect_err("v2 entry without identity must fail closed"); + + assert!(err.to_string().contains("backend identity")); + } + + #[test] + fn tier_delete_journal_uses_persisted_transition_destination_identity() { + let mut je = journal_entry(); + je.backend_identity = None; + let identity = [9_u8; 32]; + let mut metadata = std::collections::HashMap::new(); + rustfs_utils::http::metadata_compat::insert_str( + &mut metadata, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + rustfs_utils::crypto::hex(identity), + ); + + record_tier_delete_journal_backend_identity(&mut je, &metadata).expect("persisted transition identity should decode"); + let encoded = encode_tier_delete_journal_entry(&je).expect("identity-bound journal should encode"); + let decoded = decode_tier_delete_journal_entry(&encoded).expect("identity-bound journal should decode"); + + assert_eq!(decoded.backend_identity, Some(identity)); + } + + #[test] + fn tier_delete_journal_without_transition_identity_stays_legacy() { + let mut je = journal_entry(); + je.backend_identity = None; + + let encoded = encode_tier_delete_journal_entry(&je).expect("legacy journal should remain encodable"); + let persisted: serde_json::Value = serde_json::from_slice(&encoded).expect("journal JSON should decode"); + + assert_eq!(persisted["version"], 1); + assert!(persisted["backend_identity"].is_null()); + } + #[test] fn tier_delete_journal_rejects_incomplete_entry() { let payload = br#"{"version":1,"obj_name":"","version_id":"v1","tier_name":"WARM"}"#; @@ -339,6 +454,7 @@ mod tests { assert_eq!(decoded.obj_name, "remote/object"); assert!(decoded.version_id.is_empty()); assert_eq!(decoded.tier_name, "WARM"); + assert_eq!(decoded.backend_identity, None); } #[test] diff --git a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs index 338b904cf..3fe5acc48 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs @@ -23,6 +23,7 @@ use crate::bucket::lifecycle::bucket_lifecycle_ops::ExpiryOp; use crate::bucket::lifecycle::lifecycle::{self, ObjectOpts}; use crate::bucket::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry; use crate::client::signer_error::error_chain_contains_signer_header_marker; +use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease}; use crate::storage_api_contracts::lifecycle::TransitionedObject; use crate::store::ECStore; use rustfs_utils::get_env_usize; @@ -247,6 +248,7 @@ impl ObjSweeper { obj_name: self.remote_object.clone(), version_id: self.transition_version_id.clone(), tier_name: self.transition_tier.clone(), + backend_identity: None, }); } None @@ -281,6 +283,7 @@ pub struct Jentry { pub(crate) obj_name: String, pub(crate) version_id: String, pub(crate) tier_name: String, + pub(crate) backend_identity: Option, } impl ExpiryOp for Jentry { @@ -312,6 +315,27 @@ async fn delete_object_from_remote_tier_raw(obj_name: &str, rv_id: &str, tier_na return result; } + let tier_config_mgr = runtime_sources::tier_config_mgr_handle(); + delete_object_from_remote_tier_raw_with_manager(obj_name, rv_id, tier_name, &tier_config_mgr).await +} + +async fn delete_object_from_remote_tier_raw_with_manager( + obj_name: &str, + rv_id: &str, + tier_name: &str, + tier_config_mgr: &Arc>, +) -> Result<(), std::io::Error> { + let lease = TierConfigMgr::acquire_operation_lease(&tier_config_mgr, tier_name) + .await + .map_err(std::io::Error::other)?; + delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, &lease).await +} + +async fn delete_object_from_remote_tier_raw_with_lease( + obj_name: &str, + rv_id: &str, + lease: &TierOperationLease, +) -> Result<(), std::io::Error> { if remote_delete_breaker_is_open(Instant::now()).await { metrics::counter!(METRIC_DELETE_REMOTE_BREAKER_TOTAL).increment(1); return Err(std::io::Error::other(ERR_REMOTE_DELETE_BREAKER_OPEN)); @@ -323,13 +347,7 @@ async fn delete_object_from_remote_tier_raw(obj_name: &str, rv_id: &str, tier_na .map_err(|_| std::io::Error::other(ERR_REMOTE_DELETE_LIMITER_CLOSED))?; let _inflight = RemoteDeleteInflightGuard::new(); - let tier_config_mgr = runtime_sources::tier_config_mgr_handle(); - let mut config_mgr = tier_config_mgr.write().await; - let w = match config_mgr.get_driver(tier_name).await { - Ok(w) => w, - Err(e) => return Err(std::io::Error::other(e)), - }; - w.remove(obj_name, rv_id).await + lease.remove(obj_name, rv_id).await } #[cfg(test)] @@ -364,6 +382,36 @@ pub async fn delete_object_from_remote_tier_idempotent( } } +pub(crate) async fn delete_object_from_remote_tier_idempotent_with_manager_and_identity( + obj_name: &str, + rv_id: &str, + tier_name: &str, + backend_identity: TierDestinationId, + tier_config_mgr: &Arc>, +) -> Result { + let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(tier_config_mgr, tier_name, backend_identity) + .await + .map_err(std::io::Error::other)?; + delete_object_from_remote_tier_with_lease_idempotent(obj_name, rv_id, &lease).await +} + +pub(crate) async fn delete_object_from_remote_tier_with_lease_idempotent( + obj_name: &str, + rv_id: &str, + lease: &TierOperationLease, +) -> Result { + match delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, lease).await { + Ok(()) => Ok(RemoteTierDeleteOutcome::Deleted), + Err(err) if is_remote_tier_not_found_error(&err) => Ok(RemoteTierDeleteOutcome::AlreadyRemoved), + Err(err) => { + if should_record_remote_delete_failure(&err) { + record_remote_delete_failure(&err, Instant::now()).await; + } + Err(err) + } + } +} + pub(crate) fn is_remote_tier_not_found_error(err: &std::io::Error) -> bool { let message = err.to_string(); message.contains("NoSuchKey") @@ -401,6 +449,7 @@ pub fn transitioned_force_delete_journal_entry(transitioned: &TransitionedObject obj_name: transitioned.name.clone(), version_id: transitioned.version_id.clone(), tier_name: transitioned.tier.clone(), + backend_identity: None, }) } @@ -410,7 +459,8 @@ mod test { use super::{ ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, REMOTE_TIER_DELETE_TEST_HOOK, RemoteDeleteBreaker, - RemoteTierDeleteOutcome, delete_object_from_remote_tier_idempotent, is_remote_tier_not_found_error, + RemoteTierDeleteOutcome, delete_object_from_remote_tier_idempotent, + delete_object_from_remote_tier_idempotent_with_manager_and_identity, is_remote_tier_not_found_error, is_signer_header_error, should_record_remote_delete_failure, }; use std::io::{Error, ErrorKind}; @@ -506,6 +556,31 @@ mod test { assert!(err.to_string().contains("driver not found")); } + #[cfg(feature = "test-util")] + #[tokio::test] + async fn journal_delete_rejects_backend_identity_mismatch() { + let manager = crate::services::tier::tier::TierConfigMgr::new(); + crate::services::tier::test_util::register_mock_tier(&manager, "WARM").await; + let lease = crate::services::tier::tier::TierConfigMgr::acquire_operation_lease(&manager, "WARM") + .await + .expect("test tier lease should be available"); + let mut mismatched = lease.backend_identity(); + mismatched[0] ^= 1; + drop(lease); + + let err = delete_object_from_remote_tier_idempotent_with_manager_and_identity( + "remote/object", + "remote-version", + "WARM", + mismatched, + &manager, + ) + .await + .expect_err("journal recovery must fail closed when the tier name was rebound"); + + assert!(err.to_string().contains("identity no longer matches")); + } + #[test] fn breaker_opens_at_threshold_and_recovers_after_window() { let mut breaker = RemoteDeleteBreaker::new(3, Duration::from_secs(30)); diff --git a/crates/ecstore/src/runtime/sources.rs b/crates/ecstore/src/runtime/sources.rs index a63a92f4d..78ac568fb 100644 --- a/crates/ecstore/src/runtime/sources.rs +++ b/crates/ecstore/src/runtime/sources.rs @@ -552,7 +552,12 @@ pub(crate) async fn initialize_local_disk_maps( } pub(crate) async fn init_tier_config_mgr(store: Arc) -> Result<()> { - get_global_tier_config_mgr().write().await.init(store).await + let handle = get_global_tier_config_mgr(); + TierConfigMgr::reload_handle(&handle, store.clone()).await?; + if setup_is_dist_erasure().await { + tokio::spawn(TierConfigMgr::refresh_tier_config_handle(handle, store)); + } + Ok(()) } #[cfg(test)] diff --git a/crates/ecstore/src/services/tier/test_util.rs b/crates/ecstore/src/services/tier/test_util.rs index 84d7d4b6e..1273e9c37 100644 --- a/crates/ecstore/src/services/tier/test_util.rs +++ b/crates/ecstore/src/services/tier/test_util.rs @@ -140,6 +140,8 @@ pub struct MockStoredObject { struct MockWarmBackendInner { objects: Mutex>, faults: Mutex, + put_read_limit: Mutex>, + put_remote_version: Mutex>, op_log: Mutex>, put_versions: Mutex>, remove_versions: Mutex>, @@ -232,6 +234,18 @@ impl MockWarmBackend { *self.inner.faults.lock().await = FaultConfig::default(); } + /// Limit how many body bytes a successful mock PUT consumes. `None` drains + /// the complete body. This models a backend that incorrectly accepts a + /// truncated stream while still returning success. + pub async fn set_put_read_limit(&self, limit: Option) { + *self.inner.put_read_limit.lock().await = limit; + } + + /// Override the remote version returned by subsequent successful PUTs. + pub async fn set_put_remote_version(&self, remote_version: Option) { + *self.inner.put_remote_version.lock().await = remote_version; + } + async fn precondition(&self) -> Result<(), std::io::Error> { let (latency, error) = { let faults = self.inner.faults.lock().await; @@ -379,7 +393,13 @@ impl MockWarmBackend { // ---- internal helpers ----------------------------------------------- async fn put_bytes(&self, object: &str, bytes: Vec, metadata: HashMap) -> String { - let remote_version_id = Uuid::new_v4().to_string(); + let remote_version_id = self + .inner + .put_remote_version + .lock() + .await + .clone() + .unwrap_or_else(|| Uuid::new_v4().to_string()); self.inner.objects.lock().await.insert( object.to_string(), MockStoredObject { @@ -392,11 +412,18 @@ impl MockWarmBackend { } async fn read_bytes(&self, reader: ReaderImpl) -> Result, std::io::Error> { + let limit = *self.inner.put_read_limit.lock().await; match reader { - ReaderImpl::Body(bytes) => Ok(bytes.to_vec()), + ReaderImpl::Body(bytes) => Ok(bytes.slice(..limit.unwrap_or(bytes.len()).min(bytes.len())).to_vec()), ReaderImpl::ObjectBody(mut reader) => { let mut buf = Vec::new(); - reader.stream.read_to_end(&mut buf).await?; + if let Some(limit) = limit { + let limit = + u64::try_from(limit).map_err(|_| std::io::Error::other("mock PUT read limit exceeds u64::MAX"))?; + reader.stream.take(limit).read_to_end(&mut buf).await?; + } else { + reader.stream.read_to_end(&mut buf).await?; + } Ok(buf) } } @@ -519,7 +546,7 @@ impl WarmBackend for MockWarmBackend { async fn in_use(&self) -> Result { self.precondition().await?; self.record(MockWarmOp::InUse).await; - Ok(false) + Ok(!self.inner.objects.lock().await.is_empty()) } } @@ -558,7 +585,9 @@ pub async fn register_mock_tier_backend(handle: &Arc>, tie ..Default::default() }, ); - tier_config_mgr.driver_cache.insert(tier_name.to_string(), Box::new(backend)); + tier_config_mgr + .install_test_driver(tier_name, Box::new(backend)) + .expect("mock tier driver should install"); } /// The transition-state tuple read from an on-disk `xl.meta`, plus the object's diff --git a/crates/ecstore/src/services/tier/tier.rs b/crates/ecstore/src/services/tier/tier.rs index f85de9bc5..317e1af65 100644 --- a/crates/ecstore/src/services/tier/tier.rs +++ b/crates/ecstore/src/services/tier/tier.rs @@ -20,20 +20,31 @@ use byteorder::{ByteOrder, LittleEndian}; use bytes::Bytes; +use futures::FutureExt; use http::HeaderMap; use http::status::StatusCode; use lazy_static::lazy_static; use rand::{Rng, RngExt}; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use std::{ - collections::{HashMap, hash_map::Entry}, + collections::{HashMap, HashSet, hash_map::Entry}, io::{self, Cursor}, - sync::Arc, + ops::Deref, + panic::AssertUnwindSafe, + sync::{ + Arc, LazyLock, Mutex, MutexGuard, RwLock as StdRwLock, Weak, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, time::Duration, }; use time::OffsetDateTime; use tokio::io::BufReader; -use tokio::{select, sync::RwLock, time::interval}; +use tokio::{ + select, + sync::RwLock, + time::{Instant, interval, interval_at, timeout_at}, +}; use tracing::{debug, error, info, warn}; use crate::client::admin_handler_utils::AdminError; @@ -42,14 +53,16 @@ use crate::services::tier::{ tier_admin::TierCreds, tier_config::{TierConfig, TierType}, tier_handlers::{ERR_TIER_ALREADY_EXISTS, ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_NOT_FOUND, ERR_TIER_RESERVED_NAME}, - warm_backend::{check_warm_backend, new_warm_backend}, + warm_backend::{WarmBackend, check_warm_backend, new_warm_backend}, }; use crate::storage_api_contracts::{ - object::{DeletedObject, EcstoreObjectIO, EcstoreObjectOperations, ObjectIO, ObjectOperations, ObjectToDelete}, + object::{ + DeletedObject, EcstoreObjectIO, EcstoreObjectOperations, HTTPPreconditions, ObjectIO, ObjectOperations, ObjectToDelete, + }, range::HTTPRangeSpec, }; use crate::{ - config::com::{CONFIG_PREFIX, read_config}, + config::com::{CONFIG_PREFIX, read_config, read_config_with_metadata}, disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET}, object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}, runtime::sources as runtime_sources, @@ -66,6 +79,61 @@ use super::{ }; const TIER_CFG_REFRESH: Duration = Duration::from_secs(15 * 60); +const TIER_OPERATION_DRAIN_TIMEOUT: Duration = Duration::from_secs(30); +const TIER_REMOTE_VALIDATION_TIMEOUT: Duration = Duration::from_secs(30); + +fn delayed_tier_refresh_interval(period: Duration) -> tokio::time::Interval { + interval_at(Instant::now() + period, period) +} + +#[cfg(test)] +struct TierDriverBuildBarrier { + tier_name: String, + arrived: tokio::sync::Notify, + release: tokio::sync::Semaphore, +} + +#[cfg(test)] +static TIER_DRIVER_BUILD_BARRIER: LazyLock>>> = LazyLock::new(|| Mutex::new(None)); + +#[cfg(test)] +struct TierDriverBuildBarrierGuard; + +#[cfg(test)] +impl Drop for TierDriverBuildBarrierGuard { + fn drop(&mut self) { + *lock_unpoisoned(&TIER_DRIVER_BUILD_BARRIER) = None; + } +} + +#[cfg(test)] +fn install_tier_driver_build_barrier(tier_name: &str) -> (Arc, TierDriverBuildBarrierGuard) { + let barrier = Arc::new(TierDriverBuildBarrier { + tier_name: tier_name.to_string(), + arrived: tokio::sync::Notify::new(), + release: tokio::sync::Semaphore::new(0), + }); + *lock_unpoisoned(&TIER_DRIVER_BUILD_BARRIER) = Some(barrier.clone()); + (barrier, TierDriverBuildBarrierGuard) +} + +async fn build_warm_backend(tier: &TierConfig, probe: bool) -> std::result::Result { + #[cfg(test)] + let test_barrier = { lock_unpoisoned(&TIER_DRIVER_BUILD_BARRIER).clone() }; + #[cfg(test)] + if let Some(barrier) = test_barrier + && barrier.tier_name == tier.name + { + barrier.arrived.notify_one(); + barrier + .release + .acquire() + .await + .expect("tier driver build test barrier should stay open") + .forget(); + } + new_warm_backend(tier, probe).await +} const TIER_CONFIG_LEGACY_FILE: &str = "tier-config.json"; pub const TIER_CONFIG_FILE: &str = "tier-config.bin"; @@ -78,7 +146,7 @@ const EXTERNAL_TIER_TYPE_S3: i32 = 1; const EXTERNAL_TIER_TYPE_AZURE: i32 = 2; const EXTERNAL_TIER_TYPE_GCS: i32 = 3; const EXTERNAL_TIER_TYPE_MINIO: i32 = 4; - +const TIER_BACKEND_IDENTITY_VERSION: u8 = 2; const _TIER_CFG_REFRESH_AT_HDR: &str = "X-RustFS-TierCfg-RefreshedAt"; lazy_static! { @@ -117,6 +185,628 @@ pub struct TierConfigMgr { pub last_refreshed_at: OffsetDateTime, } +type SharedWarmBackend = Arc; + +#[derive(Default)] +struct TierDriverRuntime { + generations: HashMap>, + draining: HashMap, + next_generation: u64, + next_drain_epoch: u64, + admin_updates: Arc>, +} + +struct TierDriverRegistryEntry { + manager: Weak>, + runtime: Arc>, +} + +struct TierPublishTransition { + runtime: Arc>, + tokens: Vec<(String, u64)>, + revoked: HashMap>, + replaced_destinations: HashMap, + published: bool, +} + +struct PreparedTierDriver { + tier_name: String, + config_fingerprint: TierDriverFingerprint, + backend_identity: TierDestinationId, + driver: SharedWarmBackend, +} + +impl TierPublishTransition { + async fn wait_for_active_leases(&self) -> std::result::Result<(), AdminError> { + let deadline = Instant::now() + TIER_OPERATION_DRAIN_TIMEOUT; + for generation in self.revoked.values() { + if timeout_at(deadline, generation.wait_for_no_active_leases()).await.is_err() { + let mut err = ERR_TIER_BACKEND_IN_USE.clone(); + err.message = "Timed out waiting for active remote tier operations to finish".to_string(); + return Err(err); + } + } + Ok(()) + } + + async fn ensure_replaced_destinations_are_empty(&self) -> std::result::Result<(), AdminError> { + ensure_replaced_destinations_are_empty(&self.replaced_destinations, &self.revoked).await + } +} + +async fn ensure_replaced_destinations_are_empty( + replaced_destinations: &HashMap, + revoked: &HashMap>, +) -> std::result::Result<(), AdminError> { + let deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT; + for (tier_name, old_config) in replaced_destinations { + let prepared; + let driver = match revoked.get(tier_name) { + Some(generation) => generation.driver.as_ref(), + None => { + prepared = match timeout_at(deadline, build_warm_backend(old_config, false)).await { + Ok(result) => result?, + Err(_) => { + let mut err = ERR_TIER_BACKEND_IN_USE.clone(); + err.message = "Timed out preparing the replaced remote tier backend".to_string(); + return Err(err); + } + }; + prepared.as_ref() + } + }; + match timeout_at(deadline, driver.in_use()).await { + Ok(Ok(false)) => {} + Ok(Ok(true)) => return Err(ERR_TIER_BACKEND_NOT_EMPTY.clone()), + Ok(Err(err)) => { + let mut admin_err = ERR_TIER_PERM_ERR.clone(); + admin_err.message.push('.'); + admin_err.message.push_str(&err.to_string()); + return Err(admin_err); + } + Err(_) => { + let mut err = ERR_TIER_BACKEND_IN_USE.clone(); + err.message = "Timed out checking whether the replaced remote tier backend is empty".to_string(); + return Err(err); + } + } + } + Ok(()) +} + +impl Drop for TierPublishTransition { + fn drop(&mut self) { + let mut runtime = lock_unpoisoned(&self.runtime); + for (tier_name, epoch) in &self.tokens { + if runtime.draining.get(tier_name) == Some(epoch) { + if !self.published + && !runtime.generations.contains_key(tier_name) + && let Some(generation) = self.revoked.get(tier_name) + { + generation.accepting.store(true, Ordering::Release); + runtime.generations.insert(tier_name.clone(), generation.clone()); + } + runtime.draining.remove(tier_name); + } + } + } +} + +static TIER_DRIVER_REGISTRY: LazyLock>> = + LazyLock::new(|| StdRwLock::new(HashMap::new())); + +fn lock_unpoisoned(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} + +fn tier_manager_key(manager: &TierConfigMgr) -> usize { + std::ptr::from_ref(manager).addr() +} + +fn tier_driver_runtime(handle: &Arc>, manager: &TierConfigMgr) -> Arc> { + let key = tier_manager_key(manager); + { + let registry = TIER_DRIVER_REGISTRY.read().unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(entry) = registry.get(&key) + && entry + .manager + .upgrade() + .is_some_and(|registered| Arc::ptr_eq(®istered, handle)) + { + return entry.runtime.clone(); + } + } + #[cfg(test)] + let _ = TIER_REGISTRY_MISS_SCAN_COUNT.try_with(|count| count.set(count.get() + 1)); + let mut registry = TIER_DRIVER_REGISTRY + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(entry) = registry.get(&key) + && entry + .manager + .upgrade() + .is_some_and(|registered| Arc::ptr_eq(®istered, handle)) + { + return entry.runtime.clone(); + } + registry.retain(|_, entry| entry.manager.strong_count() > 0); + let runtime = Arc::new(Mutex::new(TierDriverRuntime::default())); + registry.insert( + key, + TierDriverRegistryEntry { + manager: Arc::downgrade(handle), + runtime: runtime.clone(), + }, + ); + runtime +} + +#[cfg(test)] +tokio::task_local! { + static TIER_REGISTRY_MISS_SCAN_COUNT: std::cell::Cell; +} + +fn registered_tier_driver_runtime(manager: &TierConfigMgr) -> Option>> { + let key = tier_manager_key(manager); + { + let registry = TIER_DRIVER_REGISTRY.read().unwrap_or_else(std::sync::PoisonError::into_inner); + let entry = registry.get(&key)?; + if entry.manager.strong_count() > 0 { + return Some(entry.runtime.clone()); + } + } + let mut registry = TIER_DRIVER_REGISTRY + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if registry.get(&key).is_some_and(|entry| entry.manager.strong_count() == 0) { + registry.remove(&key); + } + None +} + +type TierDriverFingerprint = [u8; 32]; +pub(crate) type TierDestinationId = [u8; 32]; +pub(crate) type DriverRevision = u64; + +fn tier_config_fingerprint(tier_name: &str, config: &TierConfig) -> io::Result { + let external = to_external_tier_config(tier_name, config)?; + let encoded = rmp_serde::to_vec_named(&external) + .map_err(|err| io::Error::other(format!("serialize tier driver identity failed: {err}")))?; + Ok(Sha256::digest(encoded).into()) +} + +fn tier_configs_match(tier_name: &str, current: &TierConfig, replacement: &TierConfig) -> bool { + match ( + tier_config_fingerprint(tier_name, current), + tier_config_fingerprint(tier_name, replacement), + ) { + (Ok(current), Ok(replacement)) => current == replacement, + _ => false, + } +} + +#[derive(Debug)] +pub enum TierConfigUpdateError { + Load(io::Error), + Mutation(AdminError), + Save(io::Error), + Publish(AdminError), +} + +enum TierCandidateMutation { + Add(TierConfig, bool), + Edit(String, TierCreds), + Remove(String, bool), + Clear(bool), +} + +impl TierCandidateMutation { + fn target_tiers(&self, manager: &TierConfigMgr, candidate: &TierConfigMgr) -> HashSet { + let mut targets = changed_tier_names(manager, candidate); + match self { + Self::Add(config, _) => { + targets.insert(config.name.clone()); + } + Self::Edit(tier_name, _) | Self::Remove(tier_name, _) => { + targets.insert(tier_name.clone()); + } + Self::Clear(_) => { + targets.extend(manager.tiers.keys().chain(candidate.tiers.keys()).cloned()); + } + } + targets + } + + async fn apply(self, candidate: &mut TierConfigMgr) -> std::result::Result, AdminError> { + match self { + Self::Add(config, force) => { + let tier_name = config.name.clone(); + candidate.add(config, force).await?; + Ok(Some(tier_name)) + } + Self::Edit(tier_name, credentials) => { + candidate.edit(&tier_name, credentials).await?; + Ok(Some(tier_name)) + } + Self::Remove(tier_name, force) => { + candidate.remove(&tier_name, force).await?; + Ok(None) + } + Self::Clear(force) => { + candidate.clear_tier(force).await?; + Ok(None) + } + } + } +} + +async fn apply_tier_candidate_mutation( + mutation: TierCandidateMutation, + candidate: &mut TierConfigMgr, + deadline: Instant, +) -> std::result::Result, AdminError> { + match timeout_at(deadline, mutation.apply(candidate)).await { + Ok(result) => result, + Err(_) => { + let mut err = ERR_TIER_BACKEND_IN_USE.clone(); + err.message = "Timed out validating the remote tier mutation".to_string(); + Err(err) + } + } +} + +fn changed_tier_names(current: &TierConfigMgr, replacement: &TierConfigMgr) -> HashSet { + let mut changed = HashSet::new(); + for (tier_name, current_config) in ¤t.tiers { + if replacement + .tiers + .get(tier_name) + .is_none_or(|replacement_config| !tier_configs_match(tier_name, current_config, replacement_config)) + { + changed.insert(tier_name.clone()); + } + } + changed.extend( + replacement + .tiers + .keys() + .filter(|tier_name| !current.tiers.contains_key(*tier_name)) + .cloned(), + ); + changed +} + +fn replaced_tier_destinations( + current: &TierConfigMgr, + replacement: &TierConfigMgr, +) -> std::result::Result, AdminError> { + let mut replaced = HashMap::new(); + for (tier_name, current_config) in ¤t.tiers { + let changed = match replacement.tiers.get(tier_name) { + Some(replacement_config) => { + let current_identity = tier_backend_identity(current_config).map_err(|err| { + let mut admin_err = ERR_TIER_INVALID_CONFIG.clone(); + admin_err.message = err.to_string(); + admin_err + })?; + let replacement_identity = tier_backend_identity(replacement_config).map_err(|err| { + let mut admin_err = ERR_TIER_INVALID_CONFIG.clone(); + admin_err.message = err.to_string(); + admin_err + })?; + current_identity != replacement_identity + } + None => true, + }; + if changed { + replaced.insert(tier_name.clone(), current_config.clone_with_credentials()); + } + } + Ok(replaced) +} + +fn tier_backend_identity(config: &TierConfig) -> io::Result { + let (tier_type, endpoint, bucket, prefix, region, routing_account) = match config.tier_type { + TierType::S3 => config.s3.as_ref().map(|value| { + ( + "s3", + value.endpoint.as_str(), + value.bucket.as_str(), + value.prefix.as_str(), + value.region.as_str(), + "", + ) + }), + TierType::RustFS => config.rustfs.as_ref().map(|value| { + ( + "rustfs", + value.endpoint.as_str(), + value.bucket.as_str(), + value.prefix.as_str(), + value.region.as_str(), + "", + ) + }), + TierType::MinIO => config.minio.as_ref().map(|value| { + ( + "minio", + value.endpoint.as_str(), + value.bucket.as_str(), + value.prefix.as_str(), + value.region.as_str(), + "", + ) + }), + TierType::Aliyun => config.aliyun.as_ref().map(|value| { + ( + "aliyun", + value.endpoint.as_str(), + value.bucket.as_str(), + value.prefix.as_str(), + value.region.as_str(), + "", + ) + }), + TierType::Tencent => config.tencent.as_ref().map(|value| { + ( + "tencent", + value.endpoint.as_str(), + value.bucket.as_str(), + value.prefix.as_str(), + value.region.as_str(), + "", + ) + }), + TierType::Huaweicloud => config.huaweicloud.as_ref().map(|value| { + ( + "huaweicloud", + value.endpoint.as_str(), + value.bucket.as_str(), + value.prefix.as_str(), + value.region.as_str(), + "", + ) + }), + TierType::Azure => config.azure.as_ref().map(|value| { + ( + "azure", + value.endpoint.as_str(), + value.bucket.as_str(), + value.prefix.as_str(), + value.region.as_str(), + value.access_key.as_str(), + ) + }), + TierType::GCS => config.gcs.as_ref().map(|value| { + ( + "gcs", + value.endpoint.as_str(), + value.bucket.as_str(), + value.prefix.as_str(), + value.region.as_str(), + "", + ) + }), + TierType::R2 => config.r2.as_ref().map(|value| { + ( + "r2", + value.endpoint.as_str(), + value.bucket.as_str(), + value.prefix.as_str(), + value.region.as_str(), + "", + ) + }), + TierType::Unsupported => None, + } + .ok_or_else(|| io::Error::other("tier backend identity payload is missing"))?; + let prefix = normalized_tier_prefix(&config.tier_type, prefix); + let encoded = rmp_serde::to_vec(&( + TIER_BACKEND_IDENTITY_VERSION, + tier_type, + endpoint, + bucket, + prefix, + region, + routing_account, + )) + .map_err(|err| io::Error::other(format!("serialize tier backend identity failed: {err}")))?; + Ok(Sha256::digest(encoded).into()) +} + +pub(crate) fn tier_destination_id_from_metadata(metadata: &HashMap) -> io::Result> { + let Some(encoded) = rustfs_utils::http::metadata_compat::get_consistent_str( + metadata, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + ) else { + if rustfs_utils::http::metadata_compat::contains_key_str( + metadata, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + ) { + return Err(io::Error::other( + "transition tier backend identity compatibility keys conflict or are empty", + )); + } + return Ok(None); + }; + if encoded.len() != 64 { + return Err(io::Error::other("transition tier backend identity has an invalid length")); + } + let mut identity = [0_u8; 32]; + for (index, byte) in identity.iter_mut().enumerate() { + *byte = u8::from_str_radix(&encoded[index * 2..index * 2 + 2], 16) + .map_err(|_| io::Error::other("transition tier backend identity is not valid hexadecimal"))?; + } + Ok(Some(identity)) +} + +fn normalized_tier_prefix<'a>(tier_type: &TierType, prefix: &'a str) -> &'a str { + match tier_type { + TierType::S3 => prefix.trim_matches('/'), + TierType::RustFS + | TierType::MinIO + | TierType::Aliyun + | TierType::Tencent + | TierType::Huaweicloud + | TierType::Azure + | TierType::GCS + | TierType::R2 => prefix.strip_suffix('/').unwrap_or(prefix), + TierType::Unsupported => prefix, + } +} + +struct TierDriverGeneration { + tier_name: Arc, + generation: DriverRevision, + // Process-local only: this may reflect credential changes and must never be persisted or logged. + config_fingerprint: TierDriverFingerprint, + // Durable cleanup identity: routing fields only, with all credentials excluded. + backend_identity: TierDestinationId, + driver: SharedWarmBackend, + accepting: AtomicBool, + active_leases: AtomicUsize, + drained: tokio::sync::Notify, +} + +struct SharedWarmBackendProxy(SharedWarmBackend); + +#[async_trait::async_trait] +impl WarmBackend for SharedWarmBackendProxy { + async fn put(&self, object: &str, r: crate::client::transition_api::ReaderImpl, length: i64) -> io::Result { + self.0.put(object, r, length).await + } + + async fn put_with_meta( + &self, + object: &str, + r: crate::client::transition_api::ReaderImpl, + length: i64, + meta: HashMap, + ) -> io::Result { + self.0.put_with_meta(object, r, length, meta).await + } + + async fn get( + &self, + object: &str, + rv: &str, + opts: crate::services::tier::warm_backend::WarmBackendGetOpts, + ) -> io::Result { + self.0.get(object, rv, opts).await + } + + async fn remove(&self, object: &str, rv: &str) -> io::Result<()> { + self.0.remove(object, rv).await + } + + async fn in_use(&self) -> io::Result { + self.0.in_use().await + } +} + +pub(crate) struct TierOperationLease { + inner: Arc, + runtime: Arc>, +} + +impl TierOperationLease { + fn try_new( + inner: Arc, + runtime: Arc>, + ) -> std::result::Result { + inner + .active_leases + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |active| active.checked_add(1)) + .map_err(|_| { + let mut err = ERR_TIER_INVALID_CONFIG.clone(); + err.message = "Remote tier operation lease capacity exhausted".to_string(); + err + })?; + Ok(Self { inner, runtime }) + } + + pub(crate) fn try_clone(&self) -> std::result::Result { + Self::try_new(self.inner.clone(), self.runtime.clone()) + } +} + +impl Drop for TierOperationLease { + fn drop(&mut self) { + let result = self + .inner + .active_leases + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |active| active.checked_sub(1)); + match result { + Ok(1) => self.inner.drained.notify_one(), + Ok(_) => {} + Err(_) => { + error!( + tier = self.tier_name(), + tier_generation = self.generation(), + "tier operation lease counter underflow" + ); + } + } + } +} + +impl Deref for TierOperationLease { + type Target = dyn WarmBackend + Send + Sync + 'static; + + fn deref(&self) -> &Self::Target { + self.inner.driver.as_ref() + } +} + +impl TierOperationLease { + pub(crate) fn tier_name(&self) -> &str { + &self.inner.tier_name + } + + pub(crate) fn generation(&self) -> DriverRevision { + self.inner.generation + } + + pub(crate) fn backend_identity(&self) -> TierDestinationId { + self.inner.backend_identity + } + + pub(crate) fn is_current_generation(&self) -> bool { + lock_unpoisoned(&self.runtime) + .generations + .get(self.tier_name()) + .is_some_and(|current| { + current.generation == self.generation() + && current.accepting.load(Ordering::Acquire) + && Arc::ptr_eq(current, &self.inner) + }) + } + + #[cfg(test)] + async fn is_current(&self, handle: &Arc>) -> bool { + let manager = handle.read().await; + let Some(runtime) = registered_tier_driver_runtime(&manager) else { + return false; + }; + if !Arc::ptr_eq(&runtime, &self.runtime) { + return false; + } + self.is_current_generation() + } +} + +impl TierDriverGeneration { + async fn wait_for_no_active_leases(&self) { + loop { + let notified = self.drained.notified(); + if self.active_leases.load(Ordering::Acquire) == 0 { + return; + } + notified.await; + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize, Default)] #[serde(default)] struct ExternalTierConfigMgr { @@ -758,8 +1448,9 @@ impl TierConfigMgr { } pub async fn add(&mut self, tier_config: TierConfig, force: bool) -> std::result::Result<(), AdminError> { - let tier_name = &tier_config.name; - if tier_name != tier_name.to_uppercase().as_str() { + self.ensure_generation_is_idle(&tier_config.name)?; + let tier_name = tier_config.name.clone(); + if tier_name != tier_name.to_uppercase() { return Err(ERR_TIER_NAME_NOT_UPPERCASE.clone()); } @@ -773,7 +1464,7 @@ impl TierConfigMgr { return Err(ERR_TIER_RESERVED_NAME.clone()); } - let (_, b) = self.is_tier_name_in_use(tier_name); + let (_, b) = self.is_tier_name_in_use(&tier_name); if b { return Err(ERR_TIER_ALREADY_EXISTS.clone()); } @@ -805,13 +1496,16 @@ impl TierConfigMgr { } } - self.driver_cache.insert(tier_name.to_string(), d); - self.tiers.insert(tier_name.to_string(), tier_config); - + self.tiers.insert(tier_name.clone(), tier_config); + if let Err(err) = self.replace_driver(&tier_name, d) { + self.tiers.remove(&tier_name); + return Err(err); + } Ok(()) } pub async fn remove(&mut self, tier_name: &str, force: bool) -> std::result::Result<(), AdminError> { + self.ensure_generation_is_idle(tier_name)?; let d = self.get_driver(tier_name).await; if let Err(err) = d { if err.code == ERR_TIER_NOT_FOUND.code { @@ -837,7 +1531,7 @@ impl TierConfigMgr { } } self.tiers.remove(tier_name); - self.driver_cache.remove(tier_name); + self.revoke_driver(tier_name); Ok(()) } @@ -886,6 +1580,7 @@ impl TierConfigMgr { } pub async fn edit(&mut self, tier_name: &str, creds: TierCreds) -> std::result::Result<(), AdminError> { + self.ensure_generation_is_idle(tier_name)?; let (tier_type, exists) = self.is_tier_name_in_use(tier_name); if !exists { return Err(ERR_TIER_NOT_FOUND.clone()); @@ -983,12 +1678,19 @@ impl TierConfigMgr { } let d = new_warm_backend(&tier_config, true).await?; + self.revoke_driver(tier_name); self.tiers.insert(tier_name.to_string(), tier_config); - self.driver_cache.insert(tier_name.to_string(), d); + self.replace_driver(tier_name, d)?; Ok(()) } pub async fn get_driver<'a>(&'a mut self, tier_name: &str) -> std::result::Result<&'a WarmBackendImpl, AdminError> { + if registered_tier_driver_runtime(self).is_some_and(|runtime| lock_unpoisoned(&runtime).draining.contains_key(tier_name)) + { + let mut err = ERR_TIER_INVALID_CONFIG.clone(); + err.message = "Remote tier configuration is being replaced".to_string(); + return Err(err); + } // Return cached driver if present if self.driver_cache.contains_key(tier_name) { return Ok(self.driver_cache.get(tier_name).expect("Driver not found in cache")); @@ -999,39 +1701,762 @@ impl TierConfigMgr { let driver = new_warm_backend(tier_config, false).await?; - // Insert and return reference - self.driver_cache.insert(tier_name.to_string(), driver); + self.replace_driver(tier_name, driver)?; Ok(self .driver_cache .get(tier_name) .expect("Driver not found in cache after insertion")) } - pub async fn reload(&mut self, api: Arc) -> std::result::Result<(), std::io::Error> { - let new_config = load_tier_config(api).await; + pub(crate) async fn acquire_operation_lease( + handle: &Arc>, + tier_name: &str, + ) -> std::result::Result { + { + let manager = handle.read().await; + let runtime = tier_driver_runtime(handle, &manager); + let runtime_guard = lock_unpoisoned(&runtime); + if runtime_guard.draining.contains_key(tier_name) { + let mut err = ERR_TIER_INVALID_CONFIG.clone(); + err.message = "Remote tier configuration is being replaced".to_string(); + return Err(err); + } + if let Some(generation) = runtime_guard + .generations + .get(tier_name) + .filter(|generation| generation.accepting.load(Ordering::Acquire)) + .cloned() + { + let lease = TierOperationLease::try_new(generation, runtime.clone()); + drop(runtime_guard); + return lease; + } + } - match &new_config { - Ok(_c) => {} - Err(err) => { - return Err(std::io::Error::other(err.to_string())); + let (config, config_fingerprint) = { + let mut manager = handle.write().await; + let runtime = tier_driver_runtime(handle, &manager); + if lock_unpoisoned(&runtime).draining.contains_key(tier_name) { + let mut err = ERR_TIER_INVALID_CONFIG.clone(); + err.message = "Remote tier configuration is being replaced".to_string(); + return Err(err); + } + if let Some(driver) = manager.driver_cache.remove(tier_name) { + manager.replace_driver(tier_name, driver)?; + let runtime_guard = lock_unpoisoned(&runtime); + let generation = runtime_guard + .generations + .get(tier_name) + .filter(|generation| generation.accepting.load(Ordering::Acquire)) + .ok_or_else(|| ERR_TIER_NOT_FOUND.clone())? + .clone(); + let lease = TierOperationLease::try_new(generation, runtime.clone()); + drop(runtime_guard); + return lease; + } + let config = manager + .tiers + .get(tier_name) + .ok_or_else(|| ERR_TIER_NOT_FOUND.clone())? + .clone_with_credentials(); + let fingerprint = tier_config_fingerprint(tier_name, &config).map_err(|err| { + let mut admin_err = ERR_TIER_INVALID_CONFIG.clone(); + admin_err.message = err.to_string(); + admin_err + })?; + (config, fingerprint) + }; + + let driver = build_warm_backend(&config, false).await?; + let mut manager = handle.write().await; + let runtime = tier_driver_runtime(handle, &manager); + let runtime_guard = lock_unpoisoned(&runtime); + if runtime_guard.draining.contains_key(tier_name) { + let mut err = ERR_TIER_INVALID_CONFIG.clone(); + err.message = "Remote tier configuration is being replaced".to_string(); + return Err(err); + } + if let Some(generation) = runtime_guard + .generations + .get(tier_name) + .filter(|generation| { + generation.config_fingerprint == config_fingerprint && generation.accepting.load(Ordering::Acquire) + }) + .cloned() + { + let lease = TierOperationLease::try_new(generation, runtime.clone()); + drop(runtime_guard); + return lease; + } + drop(runtime_guard); + let current = manager.tiers.get(tier_name).ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?; + if tier_config_fingerprint(tier_name, current).map_err(|err| { + let mut admin_err = ERR_TIER_INVALID_CONFIG.clone(); + admin_err.message = err.to_string(); + admin_err + })? != config_fingerprint + { + let mut err = ERR_TIER_INVALID_CONFIG.clone(); + err.message = "Remote tier configuration changed while its driver was being prepared".to_string(); + return Err(err); + } + manager.replace_driver(tier_name, driver)?; + let runtime_guard = lock_unpoisoned(&runtime); + let generation = runtime_guard + .generations + .get(tier_name) + .filter(|generation| generation.accepting.load(Ordering::Acquire)) + .ok_or_else(|| ERR_TIER_NOT_FOUND.clone())? + .clone(); + let lease = TierOperationLease::try_new(generation, runtime.clone()); + drop(runtime_guard); + lease + } + + async fn admin_update_lock(handle: &Arc>) -> tokio::sync::OwnedMutexGuard<()> { + let update_lock = { + let manager = handle.read().await; + let runtime = tier_driver_runtime(handle, &manager); + lock_unpoisoned(&runtime).admin_updates.clone() + }; + update_lock.lock_owned().await + } + + #[cfg(test)] + pub(crate) async fn publish_candidate( + handle: &Arc>, + candidate: Self, + driver_tier: Option<&str>, + ) -> std::result::Result<(), AdminError> { + let update = Self::admin_update_lock(handle).await; + Self::publish_candidate_owned(handle, candidate, driver_tier.map(str::to_string), update).await + } + + fn begin_publish_transition( + handle: &Arc>, + manager: &mut Self, + candidate: &Self, + ) -> std::result::Result { + let changed = changed_tier_names(manager, candidate); + let replaced_destinations = replaced_tier_destinations(manager, candidate)?; + Self::begin_tier_transition_with_destinations(handle, manager, changed, replaced_destinations) + } + + fn begin_tier_transition( + handle: &Arc>, + manager: &mut Self, + changed: HashSet, + ) -> std::result::Result { + Self::begin_tier_transition_with_destinations(handle, manager, changed, HashMap::new()) + } + + fn begin_tier_transition_with_destinations( + handle: &Arc>, + manager: &mut Self, + changed: HashSet, + replaced_destinations: HashMap, + ) -> std::result::Result { + let runtime = tier_driver_runtime(handle, manager); + for tier_name in replaced_destinations.keys() { + if !lock_unpoisoned(&runtime).generations.contains_key(tier_name) + && let Some(driver) = manager.driver_cache.remove(tier_name) + { + manager.replace_driver(tier_name, driver)?; + } + } + Self::begin_tier_transition_in_runtime(runtime, changed, replaced_destinations) + } + + fn begin_tier_transition_in_runtime( + runtime: Arc>, + changed: HashSet, + replaced_destinations: HashMap, + ) -> std::result::Result { + let count = u64::try_from(changed.len()).map_err(|_| { + let mut err = ERR_TIER_INVALID_CONFIG.clone(); + err.message = "Remote tier replacement count exceeds the supported limit".to_string(); + err + })?; + let mut runtime_guard = lock_unpoisoned(&runtime); + if changed.iter().any(|tier_name| runtime_guard.draining.contains_key(tier_name)) { + let mut err = ERR_TIER_INVALID_CONFIG.clone(); + err.message = "Remote tier configuration is already being replaced".to_string(); + return Err(err); + } + let final_epoch = runtime_guard.next_drain_epoch.checked_add(count).ok_or_else(|| { + let mut err = ERR_TIER_INVALID_CONFIG.clone(); + err.message = "Remote tier replacement epoch exhausted".to_string(); + err + })?; + let mut next_epoch = runtime_guard.next_drain_epoch; + let mut tokens = Vec::with_capacity(changed.len()); + let mut revoked = HashMap::with_capacity(changed.len()); + for tier_name in changed { + next_epoch += 1; + runtime_guard.draining.insert(tier_name.clone(), next_epoch); + tokens.push((tier_name.clone(), next_epoch)); + if let Some(generation) = runtime_guard.generations.remove(&tier_name) { + generation.accepting.store(false, Ordering::Release); + revoked.insert(tier_name, generation); + } + } + runtime_guard.next_drain_epoch = final_epoch; + drop(runtime_guard); + Ok(TierPublishTransition { + runtime, + tokens, + revoked, + replaced_destinations, + published: false, + }) + } + + async fn publish_candidate_inner( + handle: &Arc>, + candidate: Self, + driver_tier: Option<&str>, + ) -> std::result::Result<(), AdminError> { + let transition = { + let mut manager = handle.write().await; + Self::begin_publish_transition(handle, &mut manager, &candidate)? + }; + + transition.wait_for_active_leases().await?; + transition.ensure_replaced_destinations_are_empty().await?; + Self::publish_candidate_after_drain(handle, candidate, driver_tier, transition).await + } + + async fn publish_candidate_after_drain( + handle: &Arc>, + mut candidate: Self, + driver_tier: Option<&str>, + mut transition: TierPublishTransition, + ) -> std::result::Result<(), AdminError> { + let prepared_driver = + match driver_tier.and_then(|tier_name| candidate.driver_cache.remove(tier_name).map(|driver| (tier_name, driver))) { + Some((tier_name, driver)) => { + let config = candidate.tiers.get(tier_name).ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?; + let config_fingerprint = tier_config_fingerprint(tier_name, config).map_err(|err| { + let mut admin_err = ERR_TIER_INVALID_CONFIG.clone(); + admin_err.message = err.to_string(); + admin_err + })?; + let backend_identity = tier_backend_identity(config).map_err(|err| { + let mut admin_err = ERR_TIER_INVALID_CONFIG.clone(); + admin_err.message = err.to_string(); + admin_err + })?; + Some(PreparedTierDriver { + tier_name: tier_name.to_string(), + config_fingerprint, + backend_identity, + driver: Arc::from(driver), + }) + } + None => None, + }; + // Lock order invariant: manager state before the per-manager driver runtime. + let mut manager = handle.write().await; + let mut runtime = lock_unpoisoned(&transition.runtime); + if !transition + .tokens + .iter() + .all(|(tier_name, epoch)| runtime.draining.get(tier_name) == Some(epoch)) + { + let mut err = ERR_TIER_INVALID_CONFIG.clone(); + err.message = "Remote tier replacement ownership was lost before publish".to_string(); + return Err(err); + } + let prepared_generation = if let Some(prepared) = prepared_driver { + let generation = runtime.next_generation.checked_add(1).ok_or_else(|| { + let mut err = ERR_TIER_INVALID_CONFIG.clone(); + err.message = "Remote tier driver generation exhausted".to_string(); + err + })?; + let entry = Arc::new(TierDriverGeneration { + tier_name: Arc::from(prepared.tier_name.as_str()), + generation, + config_fingerprint: prepared.config_fingerprint, + backend_identity: prepared.backend_identity, + driver: prepared.driver.clone(), + accepting: AtomicBool::new(true), + active_leases: AtomicUsize::new(0), + drained: tokio::sync::Notify::new(), + }); + Some((prepared, generation, entry)) + } else { + None + }; + for (tier_name, _) in &transition.tokens { + manager.driver_cache.remove(tier_name); + } + manager.tiers = candidate.tiers; + manager.last_refreshed_at = OffsetDateTime::now_utc(); + if let Some((prepared, generation, entry)) = prepared_generation { + runtime.generations.insert(prepared.tier_name.clone(), entry); + runtime.next_generation = generation; + manager + .driver_cache + .insert(prepared.tier_name, Box::new(SharedWarmBackendProxy(prepared.driver))); + } + drop(runtime); + transition.published = true; + Ok(()) + } + + fn publish_task_error(err: tokio::task::JoinError) -> AdminError { + error!(error = ?err, "remote tier configuration publish task failed"); + let mut admin_err = ERR_TIER_INVALID_CONFIG.clone(); + admin_err.message = format!("Remote tier configuration publish task failed: {err}"); + admin_err + } + + async fn publish_candidate_owned( + handle: &Arc>, + candidate: Self, + driver_tier: Option, + update: tokio::sync::OwnedMutexGuard<()>, + ) -> std::result::Result<(), AdminError> { + let handle = handle.clone(); + tokio::spawn(async move { + match AssertUnwindSafe(async move { + let _update = update; + Self::publish_candidate_inner(&handle, candidate, driver_tier.as_deref()).await + }) + .catch_unwind() + .await + { + Ok(Ok(())) => Ok(()), + Ok(Err(err)) => { + error!(error = ?err, "remote tier configuration publish failed"); + Err(err) + } + Err(_) => { + error!("remote tier configuration publish panicked"); + let mut err = ERR_TIER_INVALID_CONFIG.clone(); + err.message = "Remote tier configuration publish panicked".to_string(); + Err(err) + } + } + }) + .await + .map_err(Self::publish_task_error)? + } + + async fn update_candidate_owned( + handle: &Arc>, + api: Arc, + mut candidate: Self, + version: Option, + mutation: TierCandidateMutation, + update: tokio::sync::OwnedMutexGuard<()>, + ) -> std::result::Result<(), TierConfigUpdateError> + where + S: EcstoreObjectIO + 'static, + { + let handle = handle.clone(); + tokio::spawn(async move { + match AssertUnwindSafe(async move { + let _update = update; + let transition = { + let mut manager = handle.write().await; + let target_tiers = mutation.target_tiers(&manager, &candidate); + Self::begin_tier_transition(&handle, &mut manager, target_tiers).map_err(TierConfigUpdateError::Publish)? + }; + transition + .wait_for_active_leases() + .await + .map_err(TierConfigUpdateError::Publish)?; + let driver_tier = + apply_tier_candidate_mutation(mutation, &mut candidate, Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT) + .await + .map_err(TierConfigUpdateError::Mutation)?; + candidate + .save_tiering_config_if_current(api, version.as_deref()) + .await + .map_err(TierConfigUpdateError::Save)?; + Self::publish_candidate_after_drain(&handle, candidate, driver_tier.as_deref(), transition) + .await + .map_err(TierConfigUpdateError::Publish) + }) + .catch_unwind() + .await + { + Ok(Ok(())) => Ok(()), + Ok(Err(err)) => { + error!(error = ?err, "remote tier configuration update failed"); + Err(err) + } + Err(_) => { + error!("remote tier configuration update panicked"); + let mut err = ERR_TIER_INVALID_CONFIG.clone(); + err.message = "Remote tier configuration update panicked".to_string(); + Err(TierConfigUpdateError::Publish(err)) + } + } + }) + .await + .map_err(|err| TierConfigUpdateError::Publish(Self::publish_task_error(err)))? + } + + pub async fn reload_handle(handle: &Arc>, api: Arc) -> io::Result<()> { + let update = Self::admin_update_lock(handle).await; + let candidate = load_tier_config(api).await?; + Self::publish_candidate_owned(handle, candidate, None, update) + .await + .map_err(io::Error::other) + } + + pub async fn add_and_save( + handle: &Arc>, + api: Arc, + tier_config: TierConfig, + force: bool, + ) -> std::result::Result<(), TierConfigUpdateError> { + let update = Self::admin_update_lock(handle).await; + let (candidate, version) = load_tier_config_for_update(api.clone()) + .await + .map_err(TierConfigUpdateError::Load)?; + Self::update_candidate_owned(handle, api, candidate, version, TierCandidateMutation::Add(tier_config, force), update) + .await + } + + pub async fn edit_and_save( + handle: &Arc>, + api: Arc, + tier_name: &str, + credentials: TierCreds, + ) -> std::result::Result<(), TierConfigUpdateError> { + let update = Self::admin_update_lock(handle).await; + let (candidate, version) = load_tier_config_for_update(api.clone()) + .await + .map_err(TierConfigUpdateError::Load)?; + Self::update_candidate_owned( + handle, + api, + candidate, + version, + TierCandidateMutation::Edit(tier_name.to_string(), credentials), + update, + ) + .await + } + + pub async fn remove_and_save( + handle: &Arc>, + api: Arc, + tier_name: &str, + force: bool, + ) -> std::result::Result<(), TierConfigUpdateError> { + Self::remove_and_save_with(handle, api, tier_name, force).await + } + + async fn remove_and_save_with( + handle: &Arc>, + api: Arc, + tier_name: &str, + force: bool, + ) -> std::result::Result<(), TierConfigUpdateError> + where + S: EcstoreObjectIO + 'static, + { + let update = Self::admin_update_lock(handle).await; + let (candidate, version) = load_tier_config_for_update(api.clone()) + .await + .map_err(TierConfigUpdateError::Load)?; + Self::update_candidate_owned( + handle, + api, + candidate, + version, + TierCandidateMutation::Remove(tier_name.to_string(), force), + update, + ) + .await + } + + pub async fn clear_and_save( + handle: &Arc>, + api: Arc, + force: bool, + ) -> std::result::Result<(), TierConfigUpdateError> { + Self::clear_and_save_with(handle, api, force).await + } + + async fn clear_and_save_with( + handle: &Arc>, + api: Arc, + force: bool, + ) -> std::result::Result<(), TierConfigUpdateError> + where + S: EcstoreObjectIO + 'static, + { + let update = Self::admin_update_lock(handle).await; + let (candidate, version) = load_tier_config_for_update(api.clone()) + .await + .map_err(TierConfigUpdateError::Load)?; + Self::update_candidate_owned(handle, api, candidate, version, TierCandidateMutation::Clear(force), update).await + } + + pub async fn verify_without_manager_lock(handle: &Arc>, tier_name: &str) -> std::result::Result<(), io::Error> { + let lease = Self::acquire_operation_lease(handle, tier_name) + .await + .map_err(io::Error::other)?; + let driver: WarmBackendImpl = Box::new(SharedWarmBackendProxy(lease.inner.driver.clone())); + check_warm_backend(Some(&driver)).await.map_err(io::Error::other) + } + + pub(crate) async fn acquire_operation_lease_for_backend_identity( + handle: &Arc>, + tier_name: &str, + expected: TierDestinationId, + ) -> std::result::Result { + let lease = Self::acquire_operation_lease(handle, tier_name).await?; + if lease.backend_identity() != expected { + let mut err = ERR_TIER_INVALID_CONFIG.clone(); + err.message = "Remote tier backend identity no longer matches the recorded operation".to_string(); + return Err(err); + } + Ok(lease) + } + + fn replace_driver(&mut self, tier_name: &str, driver: WarmBackendImpl) -> std::result::Result<(), AdminError> { + let Some(runtime) = registered_tier_driver_runtime(self) else { + self.driver_cache.insert(tier_name.to_string(), driver); + return Ok(()); + }; + let config_fingerprint = self + .tiers + .get(tier_name) + .ok_or_else(|| ERR_TIER_NOT_FOUND.clone()) + .and_then(|config| { + tier_config_fingerprint(tier_name, config).map_err(|err| { + let mut admin_err = ERR_TIER_INVALID_CONFIG.clone(); + admin_err.message = err.to_string(); + admin_err + }) + })?; + let backend_identity = self + .tiers + .get(tier_name) + .ok_or_else(|| ERR_TIER_NOT_FOUND.clone()) + .and_then(|config| { + tier_backend_identity(config).map_err(|err| { + let mut admin_err = ERR_TIER_INVALID_CONFIG.clone(); + admin_err.message = err.to_string(); + admin_err + }) + })?; + let mut runtime_guard = lock_unpoisoned(&runtime); + let generation = runtime_guard.next_generation.checked_add(1).ok_or_else(|| { + let mut err = ERR_TIER_INVALID_CONFIG.clone(); + err.message = "Remote tier driver generation exhausted".to_string(); + err + })?; + let driver: SharedWarmBackend = Arc::from(driver); + let entry = Arc::new(TierDriverGeneration { + tier_name: Arc::from(tier_name), + generation, + config_fingerprint, + backend_identity, + driver: driver.clone(), + accepting: AtomicBool::new(true), + active_leases: AtomicUsize::new(0), + drained: tokio::sync::Notify::new(), + }); + + if let Some(previous) = runtime_guard.generations.insert(tier_name.to_string(), entry) { + previous.accepting.store(false, Ordering::Release); + } + runtime_guard.next_generation = generation; + drop(runtime_guard); + self.driver_cache + .insert(tier_name.to_string(), Box::new(SharedWarmBackendProxy(driver))); + Ok(()) + } + + #[cfg(feature = "test-util")] + pub(crate) fn install_test_driver( + &mut self, + tier_name: &str, + driver: WarmBackendImpl, + ) -> std::result::Result<(), AdminError> { + self.replace_driver(tier_name, driver) + } + + fn revoke_driver(&mut self, tier_name: &str) -> Option> { + let generation = + registered_tier_driver_runtime(self).and_then(|runtime| lock_unpoisoned(&runtime).generations.remove(tier_name)); + if let Some(generation) = generation.as_ref() { + generation.accepting.store(false, Ordering::Release); + } + self.driver_cache.remove(tier_name); + generation + } + + fn ensure_generation_is_idle(&self, tier_name: &str) -> std::result::Result<(), AdminError> { + let Some(runtime) = registered_tier_driver_runtime(self) else { + return Ok(()); + }; + let runtime = lock_unpoisoned(&runtime); + let blocked = runtime.draining.contains_key(tier_name) + || runtime + .generations + .get(tier_name) + .is_some_and(|generation| generation.active_leases.load(Ordering::Acquire) != 0); + if blocked { + return Err(ERR_TIER_BACKEND_IN_USE.clone()); + } + Ok(()) + } + + fn ensure_generations_are_idle<'a>( + &self, + tier_names: impl IntoIterator, + ) -> std::result::Result<(), AdminError> { + for tier_name in tier_names { + self.ensure_generation_is_idle(tier_name)?; + } + Ok(()) + } + + fn retire_driver(&mut self, tier_name: &str) { + self.revoke_driver(tier_name); + } + + fn revoke_all_drivers(&mut self) { + if let Some(runtime) = registered_tier_driver_runtime(self) { + let mut runtime = lock_unpoisoned(&runtime); + for (_, generation) in runtime.generations.drain() { + generation.accepting.store(false, Ordering::Release); } } self.driver_cache.clear(); - self.tiers.clear(); - if let Ok(config) = new_config { - for (tier, cfg) in config.tiers { - self.tiers.insert(tier, cfg); + } + + fn retire_all_drivers(&mut self) { + self.revoke_all_drivers(); + } + + pub async fn reload(&mut self, api: Arc) -> std::result::Result<(), std::io::Error> { + let config = load_tier_config(api).await?; + self.publish_legacy_reload(config).await?; + Ok(()) + } + + pub(crate) async fn publish_legacy_reload(&mut self, config: Self) -> io::Result<()> { + let changed = changed_tier_names(self, &config); + let replaced_destinations = replaced_tier_destinations(self, &config).map_err(io::Error::other)?; + if let Some(runtime) = registered_tier_driver_runtime(self) { + for tier_name in replaced_destinations.keys() { + if !lock_unpoisoned(&runtime).generations.contains_key(tier_name) + && let Some(driver) = self.driver_cache.remove(tier_name) + { + self.replace_driver(tier_name, driver).map_err(io::Error::other)?; + } } + let mut transition = + Self::begin_tier_transition_in_runtime(runtime, changed, replaced_destinations).map_err(io::Error::other)?; + transition.wait_for_active_leases().await.map_err(io::Error::other)?; + transition + .ensure_replaced_destinations_are_empty() + .await + .map_err(io::Error::other)?; + let runtime = lock_unpoisoned(&transition.runtime); + if !transition + .tokens + .iter() + .all(|(tier_name, epoch)| runtime.draining.get(tier_name) == Some(epoch)) + { + return Err(io::Error::other("remote tier replacement ownership was lost before reload publish")); + } + for (tier_name, _) in &transition.tokens { + self.driver_cache.remove(tier_name); + } + self.tiers = config.tiers; + drop(runtime); + transition.published = true; } else { - return Err(std::io::Error::other("Failed to load tier configuration")); + ensure_replaced_destinations_are_empty(&replaced_destinations, &HashMap::new()) + .await + .map_err(io::Error::other)?; + self.tiers = config.tiers; } self.last_refreshed_at = OffsetDateTime::now_utc(); Ok(()) } + fn apply_reloaded_tiers(&mut self, tiers: HashMap) -> std::result::Result<(), AdminError> { + if registered_tier_driver_runtime(self).is_some_and(|runtime| !lock_unpoisoned(&runtime).draining.is_empty()) { + return Err(ERR_TIER_BACKEND_IN_USE.clone()); + } + let changed_or_removed = self + .tiers + .iter() + .filter_map(|(tier_name, current)| { + let unchanged = tiers + .get(tier_name) + .is_some_and(|replacement| tier_configs_match(tier_name, current, replacement)); + (!unchanged).then(|| tier_name.clone()) + }) + .collect::>(); + self.ensure_generations_are_idle(&changed_or_removed)?; + for tier_name in &changed_or_removed { + self.revoke_driver(tier_name); + } + self.tiers = tiers; + Ok(()) + } + + pub async fn rollback_after_failed_save(&mut self, api: Arc) -> io::Result<()> { + match load_tier_config(api).await { + Ok(config) => { + self.apply_reloaded_tiers(config.tiers).map_err(io::Error::other)?; + Ok(()) + } + Err(err) => { + if let Some(runtime) = registered_tier_driver_runtime(self) { + let runtime = lock_unpoisoned(&runtime); + if !runtime.draining.is_empty() + || runtime + .generations + .values() + .any(|generation| generation.active_leases.load(Ordering::Acquire) != 0) + { + return Err(io::Error::other(format!( + "failed to reload tier configuration for rollback while a tier generation is active: {err}" + ))); + } + } + self.retire_all_drivers(); + self.tiers.clear(); + Err(err) + } + } + } + pub async fn clear_tier(&mut self, force: bool) -> std::result::Result<(), AdminError> { + if registered_tier_driver_runtime(self).is_some_and(|runtime| !lock_unpoisoned(&runtime).draining.is_empty()) { + return Err(ERR_TIER_BACKEND_IN_USE.clone()); + } + self.ensure_generations_are_idle(self.tiers.keys())?; + if !force { + let tier_names = self.tiers.keys().cloned().collect::>(); + for tier_name in tier_names { + match self.get_driver(&tier_name).await?.in_use().await { + Ok(true) => return Err(ERR_TIER_BACKEND_NOT_EMPTY.clone()), + Ok(false) => {} + Err(err) => { + let mut admin_err = ERR_TIER_PERM_ERR.clone(); + admin_err.message.push('.'); + admin_err.message.push_str(&err.to_string()); + return Err(admin_err); + } + } + } + } self.tiers.clear(); - self.driver_cache.clear(); + self.revoke_all_drivers(); Ok(()) } @@ -1062,6 +2487,48 @@ impl TierConfigMgr { self.save_config(api, &config_file, data).await } + async fn save_tiering_config_if_current( + &self, + api: Arc, + version: Option<&str>, + ) -> std::result::Result<(), std::io::Error> + where + S: ObjectIO< + Error = Error, + RangeSpec = HTTPRangeSpec, + HeaderMap = HeaderMap, + ObjectOptions = ObjectOptions, + ObjectInfo = ObjectInfo, + GetObjectReader = GetObjectReader, + PutObjectReader = PutObjReader, + >, + { + let data = encode_external_tiering_config_blob(self)?; + let config_file = tier_config_path(TIER_CONFIG_FILE); + let http_preconditions = match version { + Some(etag) => HTTPPreconditions { + if_match: Some(etag.to_string()), + ..Default::default() + }, + None => HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }, + }; + + self.save_config_with_opts( + api, + &config_file, + data, + &ObjectOptions { + max_parity: true, + http_preconditions: Some(http_preconditions), + ..Default::default() + }, + ) + .await + } + pub async fn save_config(&self, api: Arc, file: &str, data: Bytes) -> std::result::Result<(), std::io::Error> where S: ObjectIO< @@ -1111,9 +2578,7 @@ impl TierConfigMgr { } pub async fn refresh_tier_config(&mut self, api: Arc) { - //let r = rand.New(rand.NewSource(time.Now().UnixNano())); - let mut rng = rand::rng(); - let r = rng.random_range(0.0..1.0); + let r = rand::rng().random_range(0.0..1.0); let rand_interval = || Duration::from_secs((r * 60_f64).round() as u64); let mut t = interval(TIER_CFG_REFRESH + rand_interval()); @@ -1124,6 +2589,24 @@ impl TierConfigMgr { info!("{}", err); } } + } + } + } + + pub(crate) async fn refresh_tier_config_handle(handle: Arc>, api: Arc) { + //let r = rand.New(rand.NewSource(time.Now().UnixNano())); + let r = rand::rng().random_range(0.0..1.0); + let rand_interval = || Duration::from_secs((r * 60_f64).round() as u64); + + let refresh_interval = TIER_CFG_REFRESH + rand_interval(); + let mut t = delayed_tier_refresh_interval(refresh_interval); + loop { + select! { + _ = t.tick() => { + if let Err(err) = Self::reload_handle(&handle, api.clone()).await { + info!("{}", err); + } + } else => () } t.reset(); @@ -1201,6 +2684,46 @@ async fn load_tier_config(api: Arc) -> std::result::Result(api: Arc) -> std::result::Result<(TierConfigMgr, Option), std::io::Error> +where + S: ObjectIO< + Error = Error, + RangeSpec = HTTPRangeSpec, + HeaderMap = HeaderMap, + ObjectOptions = ObjectOptions, + ObjectInfo = ObjectInfo, + GetObjectReader = GetObjectReader, + PutObjectReader = PutObjReader, + >, +{ + let config_file = tier_config_path(TIER_CONFIG_FILE); + match read_config_with_metadata(api.clone(), &config_file, &ObjectOptions::default()).await { + Ok((data, object_info)) => { + let etag = object_info + .etag + .filter(|etag| !etag.trim().is_empty()) + .ok_or_else(|| io::Error::other("tier configuration object is missing an ETag"))?; + Ok((decode_tiering_config_blob(&data)?, Some(etag))) + } + Err(err) if is_err_config_not_found(&err) => { + let legacy_file = tier_config_path(TIER_CONFIG_LEGACY_FILE); + match read_config(api, &legacy_file).await { + Ok(data) => Ok((TierConfigMgr::unmarshal(&data)?, None)), + Err(legacy_err) if is_err_config_not_found(&legacy_err) => Ok(( + TierConfigMgr { + driver_cache: HashMap::new(), + tiers: HashMap::new(), + last_refreshed_at: OffsetDateTime::now_utc(), + }, + None, + )), + Err(legacy_err) => Err(io::Error::other(legacy_err)), + } + } + Err(err) => Err(io::Error::other(err)), + } +} + async fn read_tier_config_from_bucket( api: Arc, bucket: &str, @@ -1769,6 +3292,79 @@ mod tests { assert!(!mgr.driver_cache.contains_key("COLD-A")); } + #[tokio::test] + async fn persisted_remove_preserves_force_semantics() { + let mut candidate = empty_mgr(); + inject_mock( + &mut candidate, + "COLD-A", + build_s3_tier("COLD-A"), + MockWarmBackend { + in_use_value: None, + healthy: true, + }, + ); + let mutation = TierCandidateMutation::Remove("COLD-A".to_string(), true); + mutation + .apply(&mut candidate) + .await + .expect("force remove must skip in-use probing"); + assert!(!candidate.tiers.contains_key("COLD-A")); + } + + #[tokio::test] + async fn persisted_remove_nonforce_preserves_backend_in_use_error() { + let mut candidate = empty_mgr(); + inject_mock( + &mut candidate, + "COLD-A", + build_s3_tier("COLD-A"), + MockWarmBackend { + in_use_value: Some(true), + healthy: true, + }, + ); + let err = TierCandidateMutation::Remove("COLD-A".to_string(), false) + .apply(&mut candidate) + .await + .expect_err("non-force remove must reject an in-use backend"); + assert_eq!(err.code, ERR_TIER_BACKEND_NOT_EMPTY.code); + assert!(candidate.tiers.contains_key("COLD-A")); + } + + #[tokio::test] + async fn clear_preserves_force_and_nonforce_semantics() { + let mut forced = empty_mgr(); + inject_mock( + &mut forced, + "COLD-A", + build_s3_tier("COLD-A"), + MockWarmBackend { + in_use_value: None, + healthy: true, + }, + ); + forced.clear_tier(true).await.expect("force clear must skip in-use probing"); + assert!(forced.tiers.is_empty()); + + let mut guarded = empty_mgr(); + inject_mock( + &mut guarded, + "COLD-A", + build_s3_tier("COLD-A"), + MockWarmBackend { + in_use_value: Some(true), + healthy: true, + }, + ); + let err = guarded + .clear_tier(false) + .await + .expect_err("non-force clear must reject an in-use backend"); + assert_eq!(err.code, ERR_TIER_BACKEND_NOT_EMPTY.code); + assert!(guarded.tiers.contains_key("COLD-A")); + } + #[tokio::test] async fn test_remove_reports_probe_error() { let mut mgr = empty_mgr(); @@ -1979,6 +3575,76 @@ mod tests { assert!(err.to_string().contains("unknown version"), "{err}"); } + #[test] + fn test_external_blob_ignores_unknown_fields_at_all_legacy_layers() { + let payload = serde_json::json!({ + "Tiers": { + "COLD-A": { + "Version": "v1", + "Type": EXTERNAL_TIER_TYPE_S3, + "Name": "COLD-A", + "S3": { + "Endpoint": "https://example.invalid", + "AccessKey": "ak", + "SecretKey": "sk", + "Bucket": "bucket", + "Prefix": "objects", + "UnknownS3": true + }, + "Azure": { + "SPAuth": { "UnknownSPAuth": true }, + "UnknownAzure": true + }, + "GCS": { "UnknownGCS": true }, + "MinIO": { "UnknownMinIO": true }, + "UnknownTier": true + } + }, + "UnknownManager": true + }); + let encoded = rmp_serde::to_vec(&payload).expect("test payload should encode"); + let mut data = Vec::with_capacity(4 + encoded.len()); + data.extend_from_slice(&TIER_CONFIG_FORMAT.to_le_bytes()); + data.extend_from_slice(&TIER_CONFIG_VERSION.to_le_bytes()); + data.extend_from_slice(&encoded); + + let decoded = decode_external_tiering_config_blob(&data).expect("legacy unknown fields must remain forward-compatible"); + let s3 = decoded.tiers["COLD-A"].s3.as_ref().expect("s3 payload should decode"); + assert_eq!(s3.prefix, "objects"); + } + + #[test] + fn test_external_blob_decodes_minio_compatible_fixture() { + let payload = serde_json::json!({ + "Tiers": { + "COLD-M": { + "Version": "v1", + "Type": EXTERNAL_TIER_TYPE_MINIO, + "Name": "COLD-M", + "MinIO": { + "Endpoint": "https://minio.example.invalid", + "AccessKey": "ak", + "SecretKey": "sk", + "Bucket": "archive", + "Prefix": "objects/", + "Region": "us-east-1" + } + } + } + }); + let encoded = rmp_serde::to_vec(&payload).expect("fixture payload should encode"); + let mut data = Vec::with_capacity(4 + encoded.len()); + data.extend_from_slice(&TIER_CONFIG_FORMAT.to_le_bytes()); + data.extend_from_slice(&TIER_CONFIG_VERSION.to_le_bytes()); + data.extend_from_slice(&encoded); + + let decoded = decode_external_tiering_config_blob(&data).expect("MinIO-compatible fixture must decode"); + let minio = decoded.tiers["COLD-M"].minio.as_ref().expect("MinIO payload should decode"); + assert_eq!(minio.endpoint, "https://minio.example.invalid"); + assert_eq!(minio.bucket, "archive"); + assert_eq!(minio.prefix, "objects/"); + } + #[test] fn test_external_blob_accepts_legacy_v1_version_word() { // The decoder must keep accepting the historical v1 version word, not @@ -2011,4 +3677,1416 @@ mod tests { let err = encode_external_tiering_config_blob(&mgr).expect_err("missing payload must fail encode"); assert!(err.to_string().contains("missing s3 backend payload"), "{err}"); } + + #[derive(Clone)] + struct LeaseTestBackend { + id: &'static str, + calls: Arc>>, + remove_started: Option>, + remove_release: Option>, + backend_in_use: bool, + pending_in_use: bool, + panic_in_use: bool, + } + + impl LeaseTestBackend { + fn ready(id: &'static str) -> Self { + Self { + id, + calls: Arc::new(std::sync::Mutex::new(Vec::new())), + remove_started: None, + remove_release: None, + backend_in_use: false, + pending_in_use: false, + panic_in_use: false, + } + } + + fn blocking(id: &'static str) -> Self { + Self { + remove_started: Some(Arc::new(tokio::sync::Notify::new())), + remove_release: Some(Arc::new(tokio::sync::Semaphore::new(0))), + ..Self::ready(id) + } + } + + fn calls(&self) -> Vec<&'static str> { + self.calls.lock().expect("lease test call log should not poison").clone() + } + + fn panicking_in_use(id: &'static str) -> Self { + Self { + panic_in_use: true, + ..Self::ready(id) + } + } + + fn in_use(id: &'static str) -> Self { + Self { + backend_in_use: true, + ..Self::ready(id) + } + } + + fn pending_in_use(id: &'static str) -> Self { + Self { + pending_in_use: true, + ..Self::ready(id) + } + } + } + + #[async_trait::async_trait] + impl WarmBackend for LeaseTestBackend { + async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> io::Result { + Ok(self.id.to_string()) + } + + async fn put_with_meta( + &self, + object: &str, + r: ReaderImpl, + length: i64, + _meta: HashMap, + ) -> io::Result { + self.put(object, r, length).await + } + + async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> io::Result { + Ok(BufReader::new(Cursor::new(Vec::new()))) + } + + async fn remove(&self, _object: &str, _rv: &str) -> io::Result<()> { + self.calls + .lock() + .expect("lease test call log should not poison") + .push(self.id); + if let Some(started) = &self.remove_started { + started.notify_one(); + } + if let Some(release) = &self.remove_release { + release + .acquire() + .await + .expect("lease test release semaphore should stay open") + .forget(); + } + Ok(()) + } + + async fn in_use(&self) -> io::Result { + if self.panic_in_use { + panic!("simulated tier backend in-use panic"); + } + if self.pending_in_use { + std::future::pending::<()>().await; + } + Ok(self.backend_in_use) + } + } + + fn install_lease_backend(manager: &mut TierConfigMgr, tier_name: &str, backend: LeaseTestBackend) { + manager.tiers.insert(tier_name.to_string(), build_rustfs_tier(tier_name)); + manager + .replace_driver(tier_name, Box::new(backend)) + .expect("test driver generation should install"); + } + + #[tokio::test] + async fn registry_steady_state_lookup_does_not_scan_misses() { + let manager = TierConfigMgr::new(); + TIER_REGISTRY_MISS_SCAN_COUNT + .scope(std::cell::Cell::new(0), async { + let guard = manager.read().await; + let first = tier_driver_runtime(&manager, &guard); + assert_eq!(TIER_REGISTRY_MISS_SCAN_COUNT.with(std::cell::Cell::get), 1); + let second = tier_driver_runtime(&manager, &guard); + assert!(Arc::ptr_eq(&first, &second)); + assert_eq!(TIER_REGISTRY_MISS_SCAN_COUNT.with(std::cell::Cell::get), 1); + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn cold_driver_build_does_not_hold_manager_write_lock() { + let cold_tier = "COLD-BUILD-LOCK"; + let manager = TierConfigMgr::new(); + { + let mut guard = manager.write().await; + guard.tiers.insert(cold_tier.to_string(), build_rustfs_tier(cold_tier)); + install_lease_backend(&mut guard, "COLD-B", LeaseTestBackend::ready("b")); + } + let (barrier, _barrier_guard) = install_tier_driver_build_barrier(cold_tier); + let build_manager = manager.clone(); + let build = tokio::spawn(async move { TierConfigMgr::acquire_operation_lease(&build_manager, cold_tier).await }); + barrier.arrived.notified().await; + + tokio::time::timeout(Duration::from_millis(100), manager.read()) + .await + .expect("cold driver construction must not block manager readers"); + let tier_b = tokio::time::timeout(Duration::from_millis(100), TierConfigMgr::acquire_operation_lease(&manager, "COLD-B")) + .await + .expect("cold tier A construction must not block tier B") + .expect("tier B lease should remain available"); + drop(tier_b); + + barrier.release.add_permits(1); + build + .await + .expect("cold driver build task should join") + .expect("cold driver should finish after the test barrier releases"); + } + + #[tokio::test(start_paused = true)] + #[serial_test::serial] + async fn cold_reload_driver_build_timeout_restores_config() { + let cold_tier = "COLD-BUILD-TIMEOUT"; + let manager = TierConfigMgr::new(); + manager + .write() + .await + .tiers + .insert(cold_tier.to_string(), build_rustfs_tier(cold_tier)); + let mut candidate = empty_mgr(); + let mut replacement = build_rustfs_tier(cold_tier); + replacement.rustfs.as_mut().expect("replacement payload should exist").prefix = "new-prefix".to_string(); + candidate.tiers.insert(cold_tier.to_string(), replacement); + let (_barrier, _barrier_guard) = install_tier_driver_build_barrier(cold_tier); + + let err = TierConfigMgr::publish_candidate(&manager, candidate, None) + .await + .expect_err("stalled cold backend construction must time out"); + + assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code); + assert!(manager.read().await.tiers.contains_key(cold_tier)); + let guard = manager.read().await; + let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered"); + assert!(lock_unpoisoned(&runtime).draining.is_empty()); + } + + #[test] + fn internal_tier_snapshot_preserves_credentials() { + let mut config = build_rustfs_tier("COLD-A"); + let rustfs = config.rustfs.as_mut().expect("rustfs payload should exist"); + rustfs.access_key = "access-key".to_string(); + rustfs.secret_key = "secret-key".to_string(); + + let snapshot = config.clone_with_credentials(); + + let rustfs = snapshot.rustfs.expect("snapshot payload should exist"); + assert_eq!(rustfs.access_key, "access-key"); + assert_eq!(rustfs.secret_key, "secret-key"); + } + + #[tokio::test(start_paused = true)] + async fn remote_mutation_validation_timeout_restores_candidate() { + let mut candidate = empty_mgr(); + candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + candidate + .driver_cache + .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::pending_in_use("pending"))); + + let err = apply_tier_candidate_mutation( + TierCandidateMutation::Remove("COLD-A".to_string(), false), + &mut candidate, + Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT, + ) + .await + .expect_err("unbounded backend validation must time out"); + + assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code); + assert!(candidate.tiers.contains_key("COLD-A")); + } + + #[tokio::test] + async fn cold_cached_reload_rejects_in_use_route_change_and_removal() { + for remove in [false, true] { + let manager = TierConfigMgr::new(); + { + let mut guard = manager.write().await; + guard.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + guard + .driver_cache + .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::in_use("cold"))); + } + let mut candidate = empty_mgr(); + if !remove { + let mut replacement = build_rustfs_tier("COLD-A"); + replacement.rustfs.as_mut().expect("replacement payload should exist").prefix = "new-prefix".to_string(); + candidate.tiers.insert("COLD-A".to_string(), replacement); + } + + let err = TierConfigMgr::publish_candidate(&manager, candidate, None) + .await + .expect_err("an in-use cold-cache destination must not be rebound or removed"); + + assert_eq!(err.code, ERR_TIER_BACKEND_NOT_EMPTY.code); + assert!(manager.read().await.tiers.contains_key("COLD-A")); + let restored = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("failed reload must restore the old cold-cache generation"); + assert!(restored.is_current(&manager).await); + } + } + + #[tokio::test] + async fn empty_backend_route_change_and_credential_rotation_can_publish() { + let manager = TierConfigMgr::new(); + { + let mut guard = manager.write().await; + install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("empty")); + } + let mut route_candidate = empty_mgr(); + let mut replacement = build_rustfs_tier("COLD-A"); + replacement.rustfs.as_mut().expect("replacement payload should exist").prefix = "new-prefix".to_string(); + route_candidate.tiers.insert("COLD-A".to_string(), replacement); + TierConfigMgr::publish_candidate(&manager, route_candidate, None) + .await + .expect("an empty backend may be rebound"); + + { + let mut guard = manager.write().await; + guard + .driver_cache + .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::panicking_in_use("credentials"))); + } + let mut credential_candidate = empty_mgr(); + let mut rotated = manager.read().await.tiers["COLD-A"].clone_with_credentials(); + rotated.rustfs.as_mut().expect("rotated payload should exist").secret_key = "rotated".to_string(); + credential_candidate.tiers.insert("COLD-A".to_string(), rotated); + TierConfigMgr::publish_candidate(&manager, credential_candidate, None) + .await + .expect("credential-only rotation must not inspect backend occupancy"); + } + + #[tokio::test] + async fn slow_tier_operation_does_not_block_another_tier() { + let slow = LeaseTestBackend::blocking("slow"); + let fast = LeaseTestBackend::ready("fast"); + let manager = Arc::new(RwLock::new(empty_mgr())); + { + let mut manager = manager.write().await; + install_lease_backend(&mut manager, "COLD-A", slow.clone()); + install_lease_backend(&mut manager, "COLD-B", fast.clone()); + } + let slow_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("slow lease should be available"); + let fast_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-B") + .await + .expect("fast lease should be available"); + let slow_task = tokio::spawn(async move { slow_lease.remove("object", "").await }); + slow.remove_started + .as_ref() + .expect("slow backend should expose start notification") + .notified() + .await; + + let manager_read = tokio::time::timeout(Duration::from_millis(100), manager.read()) + .await + .expect("manager reads must not wait for tier A"); + assert!(manager_read.is_tier_valid("COLD-B")); + drop(manager_read); + + tokio::time::timeout(Duration::from_millis(100), fast_lease.remove("object", "")) + .await + .expect("tier B must not wait for tier A") + .expect("fast remove should succeed"); + assert_eq!(fast.calls(), vec!["fast"]); + + slow.remove_release + .as_ref() + .expect("slow backend should expose release semaphore") + .add_permits(1); + slow_task + .await + .expect("slow task should join") + .expect("slow remove should succeed"); + } + + #[tokio::test] + async fn slow_admin_verify_does_not_hold_manager_lock() { + let manager = TierConfigMgr::new(); + let backend = LeaseTestBackend::blocking("verify"); + let started = backend + .remove_started + .as_ref() + .expect("verify remove notifier should exist") + .clone(); + let release = backend + .remove_release + .as_ref() + .expect("verify remove release should exist") + .clone(); + { + let mut guard = manager.write().await; + install_lease_backend(&mut guard, "COLD-A", backend); + } + + let verify_manager = manager.clone(); + let verify = tokio::spawn(async move { TierConfigMgr::verify_without_manager_lock(&verify_manager, "COLD-A").await }); + started.notified().await; + tokio::time::timeout(Duration::from_millis(100), manager.read()) + .await + .expect("slow verify must not hold the manager lock"); + release.add_permits(1); + verify.await.expect("verify task should join").expect("verify should finish"); + } + + #[tokio::test] + async fn same_name_replacement_drains_old_generation_and_routes_new_work_to_new_driver() { + let old = LeaseTestBackend::ready("old"); + let new = LeaseTestBackend::ready("new"); + let manager = Arc::new(RwLock::new(empty_mgr())); + { + let mut guard = manager.write().await; + install_lease_backend(&mut guard, "COLD-A", old.clone()); + } + let old_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("old generation lease should be available"); + { + manager + .write() + .await + .replace_driver("COLD-A", Box::new(new.clone())) + .expect("replacement generation should install"); + } + let new_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("new generation lease should be available"); + + assert!(!old_lease.is_current(&manager).await); + assert!(new_lease.is_current(&manager).await); + assert!(new_lease.generation() > old_lease.generation()); + old_lease + .remove("old-object", "") + .await + .expect("draining lease should remain usable"); + new_lease + .remove("new-object", "") + .await + .expect("new lease should use replacement driver"); + assert_eq!(old.calls(), vec!["old"]); + assert_eq!(new.calls(), vec!["new"]); + } + + #[tokio::test] + async fn replacement_waits_for_inflight_operation_lease() { + let manager = TierConfigMgr::new(); + { + let mut guard = manager.write().await; + install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old")); + } + let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("old lease should be available"); + let mut candidate = empty_mgr(); + let mut config = build_rustfs_tier("COLD-A"); + config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://replacement.invalid".to_string(); + candidate.tiers.insert("COLD-A".to_string(), config); + candidate + .driver_cache + .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("new"))); + let publish_manager = manager.clone(); + let publish = + tokio::spawn(async move { TierConfigMgr::publish_candidate(&publish_manager, candidate, Some("COLD-A")).await }); + + tokio::time::timeout(Duration::from_secs(1), async { + while old.is_current(&manager).await { + tokio::task::yield_now().await; + } + }) + .await + .expect("replacement should revoke the old generation before waiting"); + assert!(!publish.is_finished(), "replacement must wait for the old operation lease"); + drop(old); + publish + .await + .expect("publish task should join") + .expect("replacement should publish after the lease drains"); + } + + #[tokio::test(start_paused = true)] + async fn replacement_drain_timeout_restores_generation_and_allows_retry() { + let manager = TierConfigMgr::new(); + { + let mut guard = manager.write().await; + install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old")); + } + let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("old lease should be available"); + let mut candidate = empty_mgr(); + let mut config = build_rustfs_tier("COLD-A"); + config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://timeout.invalid".to_string(); + candidate.tiers.insert("COLD-A".to_string(), config); + candidate + .driver_cache + .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("timed-out"))); + let publish_manager = manager.clone(); + let publish = + tokio::spawn(async move { TierConfigMgr::publish_candidate(&publish_manager, candidate, Some("COLD-A")).await }); + while old.inner.accepting.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + + tokio::time::advance(TIER_OPERATION_DRAIN_TIMEOUT).await; + let err = publish + .await + .expect("publish task should join") + .expect_err("replacement must time out while the old lease remains active"); + assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code); + assert!(old.is_current(&manager).await, "timeout rollback must restore the old generation"); + { + let guard = manager.read().await; + let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered"); + assert!(!lock_unpoisoned(&runtime).draining.contains_key("COLD-A")); + } + drop(old); + + let mut retry = empty_mgr(); + let mut retry_config = build_rustfs_tier("COLD-A"); + retry_config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://retry.invalid".to_string(); + retry.tiers.insert("COLD-A".to_string(), retry_config); + retry + .driver_cache + .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("retry"))); + TierConfigMgr::publish_candidate(&manager, retry, Some("COLD-A")) + .await + .expect("a later replacement must succeed after timeout rollback"); + } + + #[tokio::test] + async fn generation_prepare_failure_restores_old_manager_and_generation() { + let manager = TierConfigMgr::new(); + { + let mut guard = manager.write().await; + install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old")); + } + let lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("old generation should initialize"); + drop(lease); + { + let guard = manager.read().await; + let runtime = registered_tier_driver_runtime(&guard).expect("runtime should be registered"); + lock_unpoisoned(&runtime).next_generation = u64::MAX; + } + let mut candidate = empty_mgr(); + let mut config = build_rustfs_tier("COLD-A"); + config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://overflow.invalid".to_string(); + candidate.tiers.insert("COLD-A".to_string(), config); + candidate + .driver_cache + .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("new"))); + + let err = TierConfigMgr::publish_candidate(&manager, candidate, Some("COLD-A")) + .await + .expect_err("generation exhaustion must fail before manager state changes"); + assert!(err.message.contains("generation exhausted")); + assert_eq!( + manager.read().await.tiers["COLD-A"] + .rustfs + .as_ref() + .expect("old config should remain") + .endpoint, + "https://example-compat.invalid" + ); + let restored = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("prepare failure must restore the old generation"); + assert!(restored.is_current(&manager).await); + let guard = manager.read().await; + let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered"); + assert!(!lock_unpoisoned(&runtime).draining.contains_key("COLD-A")); + } + + #[tokio::test] + async fn legacy_reload_does_not_block_revoked_operation_completion() { + let manager = TierConfigMgr::new(); + { + let mut guard = manager.write().await; + install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old")); + } + let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("old lease should be available"); + let mut candidate = empty_mgr(); + let mut config = build_rustfs_tier("COLD-A"); + config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://legacy-reload.invalid".to_string(); + candidate.tiers.insert("COLD-A".to_string(), config); + let reload_manager = manager.clone(); + let reload = tokio::spawn(async move { reload_manager.write().await.publish_legacy_reload(candidate).await }); + while old.inner.accepting.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + assert!( + manager.try_read().is_err(), + "legacy reload must still hold the manager write guard while draining" + ); + assert!(!reload.is_finished(), "legacy reload must wait for the active generation"); + let operation = tokio::spawn(async move { + assert!(!old.is_current_generation(), "the revoked operation must observe the generation fence"); + drop(old); + }); + tokio::time::timeout(Duration::from_secs(1), operation) + .await + .expect("the revoked operation must finish without waiting for the manager guard") + .expect("the revoked operation task should join"); + reload + .await + .expect("legacy reload task should join") + .expect("legacy reload should publish after the lease drains"); + assert_eq!( + manager.read().await.tiers["COLD-A"] + .rustfs + .as_ref() + .expect("reloaded tier should exist") + .endpoint, + "https://legacy-reload.invalid" + ); + } + + #[tokio::test] + async fn reload_publish_waits_for_inflight_operation_lease() { + let manager = TierConfigMgr::new(); + { + let mut guard = manager.write().await; + install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old")); + } + let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("old lease should be available"); + let mut replacement = build_rustfs_tier("COLD-A"); + replacement.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://reload.invalid".to_string(); + let mut candidate = empty_mgr(); + candidate.tiers.insert("COLD-A".to_string(), replacement); + let reload_manager = manager.clone(); + let reload = tokio::spawn(async move { TierConfigMgr::publish_candidate(&reload_manager, candidate, None).await }); + tokio::time::timeout(Duration::from_secs(1), async { + while old.is_current(&manager).await { + tokio::task::yield_now().await; + } + }) + .await + .expect("reload publish should revoke before waiting"); + assert!(!reload.is_finished(), "reload must wait for the old operation lease"); + drop(old); + reload + .await + .expect("reload task should join") + .expect("reload should publish after the lease drains"); + } + + #[tokio::test] + async fn cancelled_publish_caller_does_not_leave_tier_draining() { + let manager = TierConfigMgr::new(); + { + let mut guard = manager.write().await; + install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old")); + } + let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("old lease should be available"); + let mut candidate = empty_mgr(); + let mut config = build_rustfs_tier("COLD-A"); + config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://owned.invalid".to_string(); + candidate.tiers.insert("COLD-A".to_string(), config); + candidate + .driver_cache + .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("new"))); + let update = TierConfigMgr::admin_update_lock(&manager).await; + let publish_manager = manager.clone(); + let caller = tokio::spawn(async move { + TierConfigMgr::publish_candidate_owned(&publish_manager, candidate, Some("COLD-A".to_string()), update).await + }); + tokio::time::timeout(Duration::from_secs(1), async { + while old.is_current(&manager).await { + tokio::task::yield_now().await; + } + }) + .await + .expect("owned publish should revoke before caller cancellation"); + caller.abort(); + + let mut second = empty_mgr(); + let mut second_config = build_rustfs_tier("COLD-A"); + second_config.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://second.invalid".to_string(); + second.tiers.insert("COLD-A".to_string(), second_config); + second + .driver_cache + .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("second"))); + let second_manager = manager.clone(); + let second_publish = + tokio::spawn(async move { TierConfigMgr::publish_candidate(&second_manager, second, Some("COLD-A")).await }); + tokio::task::yield_now().await; + assert!(!second_publish.is_finished(), "a later update must remain behind the detached publish"); + + drop(old); + second_publish + .await + .expect("second publish task should join") + .expect("second publish should follow the detached publish"); + let endpoint = manager + .read() + .await + .tiers + .get("COLD-A") + .and_then(|tier| tier.rustfs.as_ref()) + .expect("second tier should be published") + .endpoint + .clone(); + assert_eq!(endpoint, "https://second.invalid"); + } + + #[tokio::test] + async fn slow_tier_replacement_does_not_block_other_tiers_or_manager_reads() { + let manager = TierConfigMgr::new(); + { + let mut guard = manager.write().await; + install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("a")); + install_lease_backend(&mut guard, "COLD-B", LeaseTestBackend::ready("b")); + } + let old_a = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("tier A lease should be available"); + let old_b = TierConfigMgr::acquire_operation_lease(&manager, "COLD-B") + .await + .expect("tier B lease should be available"); + let mut candidate = empty_mgr(); + candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + candidate.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B")); + candidate + .tiers + .get_mut("COLD-A") + .and_then(|tier| tier.rustfs.as_mut()) + .expect("tier A payload should exist") + .endpoint = "https://replacement.invalid".to_string(); + candidate + .driver_cache + .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("new-a"))); + let publish_manager = manager.clone(); + let publish = + tokio::spawn(async move { TierConfigMgr::publish_candidate(&publish_manager, candidate, Some("COLD-A")).await }); + + tokio::time::timeout(Duration::from_secs(1), async { + while old_a.is_current(&manager).await { + tokio::task::yield_now().await; + } + }) + .await + .expect("tier A should enter draining"); + { + let guard = manager.read().await; + let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered"); + assert_eq!( + lock_unpoisoned(&runtime).draining.keys().cloned().collect::>(), + vec!["COLD-A".to_string()] + ); + } + tokio::time::timeout(Duration::from_secs(1), manager.read()) + .await + .expect("manager reads must not wait for tier A leases"); + let next_b = tokio::time::timeout(Duration::from_secs(1), TierConfigMgr::acquire_operation_lease(&manager, "COLD-B")) + .await + .expect("tier B lease acquisition must not wait for tier A") + .expect("tier B lease should remain available"); + assert_eq!(next_b.generation(), old_b.generation()); + assert!(!publish.is_finished(), "tier A replacement should still wait for its lease"); + + drop(old_a); + publish + .await + .expect("publish task should join") + .expect("tier A replacement should publish after its lease drains"); + } + + #[tokio::test] + async fn direct_mutation_fails_closed_while_generation_is_active() { + let manager = TierConfigMgr::new(); + { + let mut guard = manager.write().await; + install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old")); + } + let lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("lease should be available"); + let err = manager + .write() + .await + .remove("COLD-A", true) + .await + .expect_err("direct removal must not wait for an active generation"); + assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code); + assert!(lease.is_current(&manager).await); + } + + #[tokio::test] + async fn steady_lease_acquisition_is_concurrent_across_tiers() { + let manager = TierConfigMgr::new(); + { + let mut guard = manager.write().await; + install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("a")); + install_lease_backend(&mut guard, "COLD-B", LeaseTestBackend::ready("b")); + } + let mut tasks = Vec::new(); + for index in 0..200 { + let manager = manager.clone(); + tasks.push(tokio::spawn(async move { + let tier = if index % 2 == 0 { "COLD-A" } else { "COLD-B" }; + TierConfigMgr::acquire_operation_lease(&manager, tier).await + })); + } + tokio::time::timeout(Duration::from_secs(1), async { + for task in tasks { + task.await + .expect("lease task should join") + .expect("published generation should remain available"); + } + }) + .await + .expect("steady lease acquisition should not serialize on manager writes or config hashing"); + } + + #[tokio::test] + async fn cancelled_operation_releases_generation_lease() { + let backend = LeaseTestBackend::blocking("cancelled"); + let manager = Arc::new(RwLock::new(empty_mgr())); + { + let mut guard = manager.write().await; + install_lease_backend(&mut guard, "COLD-A", backend.clone()); + } + let lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("lease should be available"); + let task = tokio::spawn(async move { lease.remove("object", "").await }); + backend + .remove_started + .as_ref() + .expect("blocking backend should expose start notification") + .notified() + .await; + task.abort(); + let _ = task.await; + + let manager = manager.read().await; + let runtime = registered_tier_driver_runtime(&manager).expect("runtime sidecar should be registered"); + drop(manager); + let runtime = lock_unpoisoned(&runtime); + let generation = runtime.generations.get("COLD-A").expect("generation should remain installed"); + assert_eq!(generation.active_leases.load(Ordering::Acquire), 0); + } + + #[tokio::test] + async fn tier_generations_are_isolated_between_runtime_instances() { + let manager_a = TierConfigMgr::new(); + let manager_b = TierConfigMgr::new(); + { + let mut guard = manager_a.write().await; + install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("a")); + } + { + let mut guard = manager_b.write().await; + install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("b")); + } + let lease_a = TierConfigMgr::acquire_operation_lease(&manager_a, "COLD-A") + .await + .expect("instance A lease should be available"); + let lease_b = TierConfigMgr::acquire_operation_lease(&manager_b, "COLD-A") + .await + .expect("instance B lease should be available"); + + assert!(lease_a.is_current(&manager_a).await); + assert!(lease_b.is_current(&manager_b).await); + assert!(!lease_a.is_current(&manager_b).await); + assert!(!lease_b.is_current(&manager_a).await); + } + + #[tokio::test] + async fn lease_acquisition_has_constant_generation_state() { + let manager = Arc::new(RwLock::new(empty_mgr())); + { + let mut guard = manager.write().await; + install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("bounded")); + } + for _ in 0..10_000 { + drop( + TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("lease should be available"), + ); + } + + let manager = manager.read().await; + let runtime = registered_tier_driver_runtime(&manager).expect("runtime sidecar should be registered"); + drop(manager); + let runtime = lock_unpoisoned(&runtime); + assert_eq!(runtime.generations.len(), 1); + assert_eq!(runtime.next_generation, 1); + assert_eq!( + runtime + .generations + .get("COLD-A") + .expect("generation should remain installed") + .active_leases + .load(Ordering::Acquire), + 0 + ); + } + + #[tokio::test] + async fn unrelated_reload_keeps_active_generation_current() { + let manager = Arc::new(RwLock::new(empty_mgr())); + { + let mut manager = manager.write().await; + install_lease_backend(&mut manager, "COLD-A", LeaseTestBackend::ready("a")); + manager.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B")); + } + let lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("lease should be available"); + let generation = lease.generation(); + let mut reloaded = HashMap::from([ + ("COLD-A".to_string(), build_rustfs_tier("COLD-A")), + ("COLD-B".to_string(), build_rustfs_tier("COLD-B")), + ("COLD-C".to_string(), build_rustfs_tier("COLD-C")), + ]); + reloaded + .get_mut("COLD-B") + .and_then(|tier| tier.rustfs.as_mut()) + .expect("tier B payload should exist") + .access_key = "rotated".to_string(); + manager + .write() + .await + .apply_reloaded_tiers(reloaded) + .expect("unrelated reload should apply"); + + assert!(lease.is_current(&manager).await); + let next = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("tier A should remain available"); + assert_eq!(next.generation(), generation); + } + + #[tokio::test] + async fn backend_identity_survives_credentials_rotation_but_rejects_route_change() { + let manager = Arc::new(RwLock::new(empty_mgr())); + { + let mut guard = manager.write().await; + install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old")); + } + let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("old lease should be available"); + let identity = old.backend_identity(); + + let mut rotated = build_rustfs_tier("COLD-A"); + let rotated_config = rotated.rustfs.as_mut().expect("rustfs payload should exist"); + rotated_config.access_key = "rotated-access".to_string(); + rotated_config.secret_key = "rotated-secret".to_string(); + manager.write().await.tiers.insert("COLD-A".to_string(), rotated); + manager + .write() + .await + .replace_driver("COLD-A", Box::new(LeaseTestBackend::ready("rotated"))) + .expect("rotated driver should install"); + let rotated = TierConfigMgr::acquire_operation_lease_for_backend_identity(&manager, "COLD-A", identity) + .await + .expect("credential rotation should preserve backend identity"); + assert_ne!(rotated.generation(), old.generation()); + assert_eq!(rotated.backend_identity(), identity); + + let mut moved = build_rustfs_tier("COLD-A"); + moved.rustfs.as_mut().expect("rustfs payload should exist").endpoint = "https://moved.invalid".to_string(); + manager.write().await.tiers.insert("COLD-A".to_string(), moved); + manager + .write() + .await + .replace_driver("COLD-A", Box::new(LeaseTestBackend::ready("moved"))) + .expect("moved driver should install"); + let err = match TierConfigMgr::acquire_operation_lease_for_backend_identity(&manager, "COLD-A", identity).await { + Ok(_) => panic!("route change must fail closed"), + Err(err) => err, + }; + assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code); + } + + fn build_azure_tier(account_name: &str) -> TierConfig { + TierConfig { + version: "v1".to_string(), + tier_type: TierType::Azure, + name: "COLD-AZURE".to_string(), + azure: Some(crate::services::tier::tier_config::TierAzure { + endpoint: "https://blob.example.invalid".to_string(), + access_key: account_name.to_string(), + secret_key: "account-key".to_string(), + bucket: "archive".to_string(), + prefix: "objects/".to_string(), + region: "us-east-1".to_string(), + ..Default::default() + }), + ..Default::default() + } + } + + #[test] + fn azure_account_name_is_part_of_backend_identity() { + let account_a = build_azure_tier("account-a"); + let mut rotated_key = build_azure_tier("account-a"); + rotated_key.azure.as_mut().expect("azure payload should exist").secret_key = "rotated-key".to_string(); + let account_b = build_azure_tier("account-b"); + + assert_eq!( + tier_backend_identity(&account_a).expect("account A identity should encode"), + tier_backend_identity(&rotated_key).expect("rotated key identity should encode"), + "Azure account-key rotation must preserve destination identity" + ); + assert_ne!( + tier_backend_identity(&account_a).expect("account A identity should encode"), + tier_backend_identity(&account_b).expect("account B identity should encode"), + "Azure account-name changes must fence cleanup from the new account" + ); + } + + #[test] + fn backend_identity_v2_has_stable_fixtures() { + assert_eq!( + tier_backend_identity(&build_rustfs_tier("COLD-A")).expect("identity should encode"), + [ + 112, 73, 111, 29, 53, 43, 207, 7, 170, 12, 99, 116, 213, 135, 173, 227, 6, 220, 179, 135, 232, 83, 25, 13, 128, + 98, 254, 103, 132, 128, 229, 97, + ] + ); + assert_eq!( + tier_backend_identity(&build_azure_tier("account-a")).expect("Azure identity should encode"), + [ + 57, 188, 231, 23, 184, 188, 233, 228, 118, 24, 158, 152, 178, 169, 2, 146, 149, 152, 104, 0, 146, 196, 61, 145, + 18, 1, 188, 20, 36, 215, 100, 125, + ] + ); + } + + #[test] + fn destination_identity_prefix_normalization_matches_driver_matrix() { + assert_eq!(normalized_tier_prefix(&TierType::S3, "/foo//"), "foo"); + for tier_type in [ + TierType::RustFS, + TierType::MinIO, + TierType::Aliyun, + TierType::Tencent, + TierType::Huaweicloud, + TierType::Azure, + TierType::GCS, + TierType::R2, + ] { + assert_eq!(normalized_tier_prefix(&tier_type, "foo/"), "foo"); + assert_eq!(normalized_tier_prefix(&tier_type, "foo//"), "foo/"); + } + } + + #[tokio::test] + async fn direct_rollback_fails_closed_until_unpublished_generation_is_idle() { + let manager = Arc::new(RwLock::new(empty_mgr())); + { + let mut guard = manager.write().await; + install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("persisted")); + } + let persisted = HashMap::from([("COLD-A".to_string(), build_rustfs_tier("COLD-A"))]); + + let mut unpublished_config = build_rustfs_tier("COLD-A"); + unpublished_config + .rustfs + .as_mut() + .expect("rustfs payload should exist") + .endpoint = "https://unpublished.invalid".to_string(); + { + let mut manager = manager.write().await; + manager.tiers.insert("COLD-A".to_string(), unpublished_config); + manager + .replace_driver("COLD-A", Box::new(LeaseTestBackend::ready("unpublished"))) + .expect("unpublished driver should install before simulated save failure"); + } + let unpublished = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("unpublished lease should be visible before rollback"); + + let err = manager + .write() + .await + .apply_reloaded_tiers(persisted.clone()) + .expect_err("direct rollback must not wait for an active generation"); + assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code); + assert!(unpublished.is_current(&manager).await); + drop(unpublished); + manager + .write() + .await + .apply_reloaded_tiers(persisted) + .expect("rollback should apply once the generation is idle"); + + let manager_guard = manager.read().await; + assert_eq!( + manager_guard + .tiers + .get("COLD-A") + .and_then(|tier| tier.rustfs.as_ref()) + .expect("persisted tier should be restored") + .endpoint, + "https://example-compat.invalid" + ); + let runtime = registered_tier_driver_runtime(&manager_guard).expect("runtime sidecar should remain registered"); + assert!(lock_unpoisoned(&runtime).generations.get("COLD-A").is_none()); + } + + #[derive(Debug, Default)] + struct CasConfigStore { + state: tokio::sync::Mutex, String)>>, + next_etag: AtomicUsize, + fail_put: AtomicBool, + } + + #[async_trait::async_trait] + impl ObjectIO for CasConfigStore { + type Error = Error; + type RangeSpec = HTTPRangeSpec; + type HeaderMap = HeaderMap; + type ObjectOptions = ObjectOptions; + type ObjectInfo = ObjectInfo; + type GetObjectReader = GetObjectReader; + type PutObjectReader = PutObjReader; + + async fn get_object_reader( + &self, + bucket: &str, + object: &str, + _range: Option, + _headers: Self::HeaderMap, + _opts: &Self::ObjectOptions, + ) -> Result { + let state = self.state.lock().await; + let (data, etag) = state.as_ref().ok_or(Error::ConfigNotFound)?; + Ok(GetObjectReader { + stream: Box::new(Cursor::new(data.clone())), + object_info: ObjectInfo { + bucket: bucket.to_string(), + name: object.to_string(), + size: data.len() as i64, + actual_size: data.len() as i64, + etag: Some(etag.clone()), + ..Default::default() + }, + buffered_body: None, + body_source: Default::default(), + }) + } + + async fn put_object( + &self, + bucket: &str, + object: &str, + data: &mut Self::PutObjectReader, + opts: &Self::ObjectOptions, + ) -> Result { + if self.fail_put.load(Ordering::SeqCst) { + return Err(Error::other("injected tier config save failure")); + } + let mut payload = Vec::new(); + tokio::io::AsyncReadExt::read_to_end(&mut data.stream, &mut payload).await?; + let mut state = self.state.lock().await; + match state.as_ref() { + Some((_, etag)) => opts.precondition_check(&ObjectInfo { + etag: Some(etag.clone()), + ..Default::default() + })?, + None => { + if opts + .http_preconditions + .as_ref() + .and_then(HTTPPreconditions::if_match_value) + .is_some() + { + return Err(Error::ObjectNotFound(bucket.to_string(), object.to_string())); + } + } + } + let etag = format!("etag-{}", self.next_etag.fetch_add(1, Ordering::SeqCst) + 1); + *state = Some((payload.clone(), etag.clone())); + Ok(ObjectInfo { + bucket: bucket.to_string(), + name: object.to_string(), + size: payload.len() as i64, + actual_size: payload.len() as i64, + etag: Some(etag), + ..Default::default() + }) + } + } + + #[tokio::test] + async fn remove_and_clear_full_update_paths_preserve_force() { + let remove_store = Arc::new(CasConfigStore::default()); + let mut persisted = empty_mgr(); + persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + persisted + .save_tiering_config_if_current(remove_store.clone(), None) + .await + .expect("remove fixture should persist"); + let remove_manager = TierConfigMgr::new(); + { + let mut guard = remove_manager.write().await; + install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::in_use("remove")); + } + TierConfigMgr::remove_and_save_with(&remove_manager, remove_store.clone(), "COLD-A", true) + .await + .expect("forced remove must complete through load, mutate, save, and publish"); + assert!(!remove_manager.read().await.tiers.contains_key("COLD-A")); + assert!( + load_tier_config_for_update(remove_store) + .await + .expect("removed config should reload") + .0 + .tiers + .is_empty(), + "forced remove must persist the empty candidate" + ); + + let clear_store = Arc::new(CasConfigStore::default()); + persisted + .save_tiering_config_if_current(clear_store.clone(), None) + .await + .expect("clear fixture should persist"); + let clear_manager = TierConfigMgr::new(); + { + let mut guard = clear_manager.write().await; + install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::in_use("clear")); + } + TierConfigMgr::clear_and_save_with(&clear_manager, clear_store.clone(), true) + .await + .expect("forced clear must complete through load, mutate, save, and publish"); + assert!(clear_manager.read().await.tiers.is_empty()); + assert!( + load_tier_config_for_update(clear_store) + .await + .expect("cleared config should reload") + .0 + .tiers + .is_empty(), + "forced clear must persist the empty candidate" + ); + } + + #[tokio::test] + async fn failed_owned_update_restores_generation_and_reports_save_error() { + let manager = TierConfigMgr::new(); + { + let mut guard = manager.write().await; + install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old")); + } + let store = Arc::new(CasConfigStore::default()); + store.fail_put.store(true, Ordering::SeqCst); + let mut candidate = empty_mgr(); + candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + candidate + .driver_cache + .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("candidate"))); + let update = TierConfigMgr::admin_update_lock(&manager).await; + + let err = TierConfigMgr::update_candidate_owned( + &manager, + store.clone(), + candidate, + None, + TierCandidateMutation::Remove("COLD-A".to_string(), true), + update, + ) + .await + .expect_err("save failure must be observable to the admin caller"); + assert!(matches!(err, TierConfigUpdateError::Save(_))); + assert!(manager.read().await.tiers.contains_key("COLD-A")); + let restored = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("the old generation must be restored after save failure"); + assert!(restored.is_current(&manager).await); + drop(restored); + let guard = manager.read().await; + let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered"); + assert!(!lock_unpoisoned(&runtime).draining.contains_key("COLD-A")); + drop(guard); + + store.fail_put.store(false, Ordering::SeqCst); + let mut retry = empty_mgr(); + retry.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + retry + .driver_cache + .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("retry"))); + let update = TierConfigMgr::admin_update_lock(&manager).await; + TierConfigMgr::update_candidate_owned( + &manager, + store, + retry, + None, + TierCandidateMutation::Remove("COLD-A".to_string(), true), + update, + ) + .await + .expect("a later update must succeed after save failure recovery"); + assert!(!manager.read().await.tiers.contains_key("COLD-A")); + } + + #[tokio::test] + async fn cancelled_owned_update_finishes_after_lease_drain() { + let manager = TierConfigMgr::new(); + { + let mut guard = manager.write().await; + install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old")); + } + let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("old generation lease should be available"); + let store = Arc::new(CasConfigStore::default()); + let mut candidate = empty_mgr(); + candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + candidate + .driver_cache + .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::ready("candidate"))); + let update = TierConfigMgr::admin_update_lock(&manager).await; + let update_manager = manager.clone(); + let update_store = store.clone(); + let caller = tokio::spawn(async move { + TierConfigMgr::update_candidate_owned( + &update_manager, + update_store, + candidate, + None, + TierCandidateMutation::Remove("COLD-A".to_string(), true), + update, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(1), async { + while old.is_current(&manager).await { + tokio::task::yield_now().await; + } + }) + .await + .expect("owned update should revoke before caller cancellation"); + caller.abort(); + drop(old); + + tokio::time::timeout(Duration::from_secs(1), async { + while manager.read().await.tiers.contains_key("COLD-A") { + tokio::task::yield_now().await; + } + }) + .await + .expect("detached update must finish after its operation lease drains"); + let guard = manager.read().await; + let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered"); + assert!(!lock_unpoisoned(&runtime).draining.contains_key("COLD-A")); + assert!(store.state.lock().await.is_some(), "detached update must persist its result"); + } + + #[tokio::test] + async fn panicked_owned_update_restores_generation_and_reports_error() { + let manager = TierConfigMgr::new(); + { + let mut guard = manager.write().await; + install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old")); + } + let store = Arc::new(CasConfigStore::default()); + let mut candidate = empty_mgr(); + candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + candidate + .driver_cache + .insert("COLD-A".to_string(), Box::new(LeaseTestBackend::panicking_in_use("panic"))); + let update = TierConfigMgr::admin_update_lock(&manager).await; + + let err = TierConfigMgr::update_candidate_owned( + &manager, + store, + candidate, + None, + TierCandidateMutation::Remove("COLD-A".to_string(), false), + update, + ) + .await + .expect_err("mutation panic must be observable to the admin caller"); + let TierConfigUpdateError::Publish(err) = err else { + panic!("mutation panic should be reported as a publish failure"); + }; + assert!(err.message.contains("panicked")); + assert!(manager.read().await.tiers.contains_key("COLD-A")); + let restored = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("the old generation must be restored after panic"); + assert!(restored.is_current(&manager).await); + drop(restored); + let guard = manager.read().await; + let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered"); + assert!(!lock_unpoisoned(&runtime).draining.contains_key("COLD-A")); + } + + #[tokio::test] + async fn stale_tier_config_etag_allows_only_one_candidate_to_publish() { + let store = Arc::new(CasConfigStore::default()); + empty_mgr() + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("initial conditional create should succeed"); + + let (mut candidate_a, version_a) = load_tier_config_for_update(store.clone()) + .await + .expect("first node should load config revision"); + let (mut candidate_b, version_b) = load_tier_config_for_update(store.clone()) + .await + .expect("second node should load the same config revision"); + assert_eq!(version_a, version_b); + candidate_a.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + candidate_b.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B")); + + let save_a = candidate_a.save_tiering_config_if_current(store.clone(), version_a.as_deref()); + let save_b = candidate_b.save_tiering_config_if_current(store, version_b.as_deref()); + let (result_a, result_b) = tokio::join!(save_a, save_b); + assert_ne!(result_a.is_ok(), result_b.is_ok(), "exactly one stale-ETag writer must win"); + + let manager_a = TierConfigMgr::new(); + let manager_b = TierConfigMgr::new(); + if result_a.is_ok() { + TierConfigMgr::publish_candidate(&manager_a, candidate_a, None) + .await + .expect("winning node should publish"); + } + if result_b.is_ok() { + TierConfigMgr::publish_candidate(&manager_b, candidate_b, None) + .await + .expect("winning node should publish"); + } + assert_ne!(manager_a.read().await.empty(), manager_b.read().await.empty()); + } + + #[test] + fn legacy_refresh_method_signature_remains_callable() { + async fn call_legacy_refresh(manager: &mut TierConfigMgr, api: Arc) { + manager.refresh_tier_config(api).await; + } + + let _ = call_legacy_refresh; + } + + #[tokio::test(start_paused = true)] + async fn periodic_refresh_timer_does_not_tick_on_startup() { + let period = Duration::from_secs(60); + let mut timer = delayed_tier_refresh_interval(period); + assert!( + tokio::time::timeout(Duration::ZERO, timer.tick()).await.is_err(), + "periodic reload must not duplicate the startup reload" + ); + tokio::time::advance(period).await; + tokio::time::timeout(Duration::ZERO, timer.tick()) + .await + .expect("the first periodic reload should become ready after one interval"); + } } diff --git a/crates/ecstore/src/services/tier/tier_config.rs b/crates/ecstore/src/services/tier/tier_config.rs index b1f0cd1eb..5d5717cb8 100644 --- a/crates/ecstore/src/services/tier/tier_config.rs +++ b/crates/ecstore/src/services/tier/tier_config.rs @@ -238,6 +238,23 @@ impl Clone for TierConfig { #[allow(dead_code)] impl TierConfig { + pub(crate) fn clone_with_credentials(&self) -> Self { + Self { + version: self.version.clone(), + tier_type: self.tier_type.clone(), + name: self.name.clone(), + s3: self.s3.clone(), + aliyun: self.aliyun.clone(), + tencent: self.tencent.clone(), + huaweicloud: self.huaweicloud.clone(), + azure: self.azure.clone(), + gcs: self.gcs.clone(), + r2: self.r2.clone(), + rustfs: self.rustfs.clone(), + minio: self.minio.clone(), + } + } + fn endpoint(&self) -> String { match self.tier_type { TierType::S3 => self.s3.as_ref().map(|s| s.endpoint.clone()).unwrap_or_default(), diff --git a/crates/ecstore/src/services/tier/warm_backend.rs b/crates/ecstore/src/services/tier/warm_backend.rs index 0dccc6f0b..51a2c7c77 100644 --- a/crates/ecstore/src/services/tier/warm_backend.rs +++ b/crates/ecstore/src/services/tier/warm_backend.rs @@ -65,7 +65,15 @@ pub struct WarmBackendGetOpts { #[async_trait::async_trait] pub trait WarmBackend { + /// Return `Ok` only after the backend has consumed the complete declared + /// body and its storage service has acknowledged the PUT. The built-in S3 + /// family uses the transition client's declared-length request plus + /// Content-MD5 for multipart parts, while GCS materializes the body before + /// awaiting its buffered write response. Test backends may deliberately + /// violate this contract to exercise transition compensation. async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result; + /// The same completion contract as [`WarmBackend::put`] applies when + /// metadata is attached. async fn put_with_meta( &self, object: &str, diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 3484d4368..80bbc82e0 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -97,7 +97,7 @@ use crate::storage_api_contracts::{ use crate::store::utils::is_reserved_or_invalid_bucket; use crate::{ bucket::lifecycle::bucket_lifecycle_ops::{ - LifecycleOps, gen_transition_objname, get_transitioned_object_reader, put_restore_opts, + LifecycleOps, gen_transition_objname, get_transitioned_object_reader_with_tier_manager, put_restore_opts, }, cache_value::metacache_set::{ListPathRawOptions, list_path_raw}, config::storageclass, @@ -628,6 +628,8 @@ pub(crate) use ops::object::body_cache_plaintext_len; mod read; mod replication; pub(crate) mod shard_source; +#[cfg(all(test, feature = "test-util"))] +mod transition_matrix_tests; pub use ops::heal_walk::HealWalkVersion; diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index 52612d149..f281549d8 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -1607,6 +1607,74 @@ mod tests { .await } + #[tokio::test] + #[serial(metadata_cache_invalidation_probe)] + async fn complete_multipart_generation_retires_cached_snapshot() { + use crate::storage_api_contracts::multipart::MultipartOperations as _; + use crate::storage_api_contracts::object::ObjectIO as _; + + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "multipart-metadata-generation-bucket"; + let object = "object"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + let mut initial_reader = PutObjReader::from_vec(b"old multipart body".to_vec()); + set_disks + .put_object(bucket, object, &mut initial_reader, &ObjectOptions::default()) + .await + .expect("initial object should be written"); + set_disks + .get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false) + .await + .expect("initial metadata should resolve"); + let generation = set_disks + .get_object_metadata_cache_generation(bucket, object) + .expect("metadata cache generation should be active"); + let retired_key = GetObjectMetadataCacheKey::new(bucket, object, generation); + assert!(set_disks.get_object_metadata_cache.get(&retired_key).await.is_some()); + + let upload = set_disks + .new_multipart_upload(bucket, object, &ObjectOptions::default()) + .await + .expect("multipart upload should be created"); + let payload = vec![9u8; 4096]; + let payload_len = i64::try_from(payload.len()).expect("test payload length should fit i64"); + let mut part_reader = PutObjReader::new( + HashReader::from_stream(Cursor::new(payload), payload_len, payload_len, None, None, false) + .expect("part hash reader should be created"), + ); + let part = set_disks + .put_object_part(bucket, object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default()) + .await + .expect("multipart part should be written"); + + let invalidations = MetadataCacheInvalidationProbe::install(bucket, object); + set_disks + .clone() + .complete_multipart_upload( + bucket, + object, + &upload.upload_id, + vec![CompletePart { + part_num: part.part_num, + etag: part.etag, + ..Default::default() + }], + &ObjectOptions::default(), + ) + .await + .expect("multipart completion should succeed"); + + assert_eq!( + invalidations.count(), + 2, + "multipart completion must invalidate before mutation and after commit" + ); + set_disks.get_object_metadata_cache.run_pending_tasks().await; + assert!(set_disks.get_object_metadata_cache.get(&retired_key).await.is_none()); + } + #[tokio::test(flavor = "multi_thread")] #[serial] async fn multipart_upload_read_lock_waits_for_upload_writer() { diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index d0d912250..e35c8904a 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -21,9 +21,12 @@ use super::super::*; -use crate::bucket::lifecycle::tier_sweeper::delete_object_from_remote_tier_idempotent; +use crate::bucket::lifecycle::tier_sweeper::{RemoteTierDeleteOutcome, delete_object_from_remote_tier_with_lease_idempotent}; use crate::disk::OldCurrentSize; use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed}; +use crate::services::tier::tier::{TierConfigMgr, TierOperationLease}; +use futures::FutureExt as _; +use std::future::Future; fn erasure_from_file_info(fi: &FileInfo, uses_legacy: bool) -> Result { coding::Erasure::try_new_with_options(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size, uses_legacy) @@ -401,7 +404,16 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { if object_info.parts.len() == 1 { opts.part_number = Some(1); } - let gr = get_transitioned_object_reader(bucket, object, &range, &h, &object_info, &opts).await?; + let gr = get_transitioned_object_reader_with_tier_manager( + bucket, + object, + &range, + &h, + &object_info, + &opts, + &self.ctx.tier_config_mgr(), + ) + .await?; return Ok(finish_set_disk_read_lock( gr, read_lock_guard.take(), @@ -1242,41 +1254,241 @@ impl SetDisks { } } -async fn cleanup_uncommitted_transition_upload(tier: &str, object: &str, remote_version: &str) { - if let Err(err) = delete_object_from_remote_tier_idempotent(object, remote_version, tier).await { - warn!( - tier, - object, - remote_version, - error = ?err, - "failed to clean uncommitted transition upload" - ); +struct TransitionUploadReader { + inner: R, + consumed: Arc, +} + +impl TransitionUploadReader { + fn new(inner: R, consumed: Arc) -> Self { + Self { inner, consumed } } } -struct TransitionUploadCleanup { - tier: String, +impl AsyncRead for TransitionUploadReader { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + let before = buf.filled().len(); + match Pin::new(&mut self.inner).poll_read(cx, buf) { + Poll::Ready(Ok(())) => { + let read = buf.filled().len() - before; + let read = + u64::try_from(read).map_err(|_| std::io::Error::other("transition upload read count exceeds u64::MAX"))?; + self.consumed + .fetch_update(Ordering::Release, Ordering::Relaxed, |consumed| consumed.checked_add(read)) + .map_err(|_| std::io::Error::other("transition upload read count overflow"))?; + Poll::Ready(Ok(())) + } + other => other, + } + } +} + +struct TransitionUploadWriter { + inner: W, + produced: u64, +} + +impl TransitionUploadWriter { + fn new(inner: W) -> Self { + Self { inner, produced: 0 } + } + + fn produced(&self) -> u64 { + self.produced + } +} + +impl AsyncWrite for TransitionUploadWriter { + fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { + match Pin::new(&mut self.inner).poll_write(cx, buf) { + Poll::Ready(Ok(written)) => { + let written_u64 = u64::try_from(written) + .map_err(|_| std::io::Error::other("transition upload write count exceeds u64::MAX"))?; + self.produced = self + .produced + .checked_add(written_u64) + .ok_or_else(|| std::io::Error::other("transition upload write count overflow"))?; + Poll::Ready(Ok(written)) + } + other => other, + } + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_shutdown(cx) + } +} + +#[derive(Debug)] +pub(crate) struct TransitionUploadFailure { + pub(crate) error: StorageError, + pub(crate) candidate: Option, +} + +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct TransitionUploadCompletion { + pub(crate) candidate: TransitionUploadCandidate, + pub(crate) produced: u64, + pub(crate) consumed: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct TransitionUploadCandidate { + remote_version: TransitionUploadRemoteVersion, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum TransitionUploadRemoteVersion { + KnownExact(String), + KnownUnversioned(String), +} + +impl TransitionUploadCandidate { + pub(crate) fn from_put_response(remote_version: String) -> Self { + let remote_version = + if remote_version.is_empty() || Uuid::parse_str(&remote_version).is_ok_and(|version_id| version_id.is_nil()) { + TransitionUploadRemoteVersion::KnownUnversioned(remote_version) + } else { + TransitionUploadRemoteVersion::KnownExact(remote_version) + }; + Self { remote_version } + } + + pub(crate) fn remote_version(&self) -> &str { + match &self.remote_version { + TransitionUploadRemoteVersion::KnownExact(remote_version) + | TransitionUploadRemoteVersion::KnownUnversioned(remote_version) => remote_version, + } + } + + pub(crate) fn cleanup_version(&self) -> &str { + match &self.remote_version { + TransitionUploadRemoteVersion::KnownExact(remote_version) => remote_version, + TransitionUploadRemoteVersion::KnownUnversioned(_) => "", + } + } +} + +pub(crate) async fn complete_transition_upload( + remote_upload: Remote, + producer: Producer, + expected_size: u64, + consumed: Arc, +) -> std::result::Result +where + Remote: Future>, + Producer: Future>, +{ + let producer = std::panic::AssertUnwindSafe(producer).catch_unwind(); + let (remote_result, producer_result) = tokio::join!(remote_upload, producer); + let remote_version = match remote_result { + Ok(remote_version) => remote_version, + Err(remote_error) => { + let error = match producer_result { + Ok(Err(StorageError::Io(producer_error))) if producer_error.kind() == std::io::ErrorKind::BrokenPipe => { + StorageError::Io(remote_error) + } + Ok(Err(producer_error)) => producer_error, + Err(_) => StorageError::Unexpected, + Ok(Ok(_)) => StorageError::Io(remote_error), + }; + return Err(TransitionUploadFailure { error, candidate: None }); + } + }; + let candidate = TransitionUploadCandidate::from_put_response(remote_version); + let produced = match producer_result { + Ok(Ok(produced)) => produced, + Ok(Err(error)) => { + return Err(TransitionUploadFailure { + error, + candidate: Some(candidate), + }); + } + Err(_) => { + return Err(TransitionUploadFailure { + error: StorageError::Unexpected, + candidate: Some(candidate), + }); + } + }; + let consumed = consumed.load(Ordering::Acquire); + if produced != expected_size || consumed != expected_size { + let error = if produced < expected_size || consumed < expected_size { + StorageError::LessData + } else { + StorageError::MoreData + }; + return Err(TransitionUploadFailure { + error, + candidate: Some(candidate), + }); + } + Ok(TransitionUploadCompletion { + candidate, + produced, + consumed, + }) +} + +pub(crate) async fn cleanup_uncommitted_transition_upload( + lease: &TierOperationLease, + object: &str, + candidate: &TransitionUploadCandidate, +) -> std::io::Result { + delete_object_from_remote_tier_with_lease_idempotent(object, candidate.cleanup_version(), lease).await +} + +fn log_transition_upload_cleanup_failure( + lease: &TierOperationLease, + object: &str, + candidate: &TransitionUploadCandidate, + err: &std::io::Error, +) { + warn!( + tier = lease.tier_name(), + tier_generation = lease.generation(), + object, + remote_version = candidate.cleanup_version(), + error = ?err, + "failed to clean uncommitted transition upload" + ); +} + +pub(crate) struct TransitionUploadCleanup { + lease: TierOperationLease, object: String, - remote_version: String, + candidate: TransitionUploadCandidate, armed: bool, } impl TransitionUploadCleanup { - fn new(tier: &str, object: &str, remote_version: &str) -> Self { + pub(crate) fn new(lease: TierOperationLease, object: &str, candidate: TransitionUploadCandidate) -> Self { Self { - tier: tier.to_string(), + lease, object: object.to_string(), - remote_version: remote_version.to_string(), + candidate, armed: true, } } - async fn cleanup(&mut self) { - cleanup_uncommitted_transition_upload(&self.tier, &self.object, &self.remote_version).await; - self.armed = false; + pub(crate) async fn cleanup(&mut self) -> std::io::Result { + match cleanup_uncommitted_transition_upload(&self.lease, &self.object, &self.candidate).await { + Ok(outcome) => { + self.armed = false; + Ok(outcome) + } + Err(err) => { + log_transition_upload_cleanup_failure(&self.lease, &self.object, &self.candidate, &err); + Err(err) + } + } } - fn disarm(&mut self) { + pub(crate) fn disarm(&mut self) { self.armed = false; } } @@ -1286,21 +1498,43 @@ impl Drop for TransitionUploadCleanup { if !self.armed { return; } - let tier = self.tier.clone(); + let lease = match self.lease.try_clone() { + Ok(lease) => lease, + Err(err) => { + warn!( + tier = self.lease.tier_name(), + tier_generation = self.lease.generation(), + object = self.object, + error = ?err, + "unable to retain tier lease for cancelled transition cleanup" + ); + return; + } + }; let object = self.object.clone(); - let remote_version = self.remote_version.clone(); + let candidate = self.candidate.clone(); if let Ok(handle) = tokio::runtime::Handle::try_current() { handle.spawn(async move { - cleanup_uncommitted_transition_upload(&tier, &object, &remote_version).await; + if let Err(err) = cleanup_uncommitted_transition_upload(&lease, &object, &candidate).await { + log_transition_upload_cleanup_failure(&lease, &object, &candidate, &err); + } }); } } } +#[cfg(test)] +#[derive(Clone, Copy, PartialEq, Eq)] +enum TransitionCommitPause { + BeforeLeaseCheck, + AfterLeaseCheck, +} + #[cfg(test)] struct TransitionCommitBarrierState { bucket: String, object: String, + pause: TransitionCommitPause, arrived: tokio::sync::Notify, release: tokio::sync::Notify, } @@ -1317,9 +1551,18 @@ static TRANSITION_COMMIT_BARRIER: std::sync::OnceLock Self { + Self::install_at(bucket, object, TransitionCommitPause::BeforeLeaseCheck) + } + + fn install_after_lease_check(bucket: &str, object: &str) -> Self { + Self::install_at(bucket, object, TransitionCommitPause::AfterLeaseCheck) + } + + fn install_at(bucket: &str, object: &str, pause: TransitionCommitPause) -> Self { let state = Arc::new(TransitionCommitBarrierState { bucket: bucket.to_string(), object: object.to_string(), + pause, arrived: tokio::sync::Notify::new(), release: tokio::sync::Notify::new(), }); @@ -1359,13 +1602,13 @@ impl Drop for TransitionCommitBarrier { } #[cfg(test)] -async fn pause_transition_before_local_commit(bucket: &str, object: &str) { +async fn pause_transition_commit(bucket: &str, object: &str, pause: TransitionCommitPause) { let barrier = TRANSITION_COMMIT_BARRIER .get_or_init(|| std::sync::Mutex::new(None)) .lock() .expect("transition commit barrier mutex should not poison") .as_ref() - .filter(|barrier| barrier.bucket == bucket && barrier.object == object) + .filter(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.pause == pause) .cloned(); if let Some(barrier) = barrier { barrier.arrived.notify_one(); @@ -1380,13 +1623,166 @@ fn parse_transition_version_id(remote_version: &str) -> std::result::Result) -> &str { - version_id.map(|_| remote_version).unwrap_or("") +#[cfg(test)] +mod transition_upload_completion_tests { + use super::*; + + fn consumed(bytes: u64) -> Arc { + Arc::new(AtomicU64::new(bytes)) + } + + #[tokio::test] + async fn rejects_source_errors_at_first_middle_and_last_chunk() { + let remote_version = Uuid::nil().to_string(); + for consumed_bytes in [0, 512, 1023] { + let result = complete_transition_upload( + std::future::ready(Ok(remote_version.clone())), + std::future::ready(Err(StorageError::FileCorrupt)), + 1024, + consumed(consumed_bytes), + ) + .await; + let failure = result.expect_err("a source read error must fail the upload completion protocol"); + assert!(matches!(failure.error, StorageError::FileCorrupt)); + assert_eq!( + failure.candidate.as_ref().map(TransitionUploadCandidate::remote_version), + Some(remote_version.as_str()) + ); + } + } + + #[tokio::test] + async fn rejects_partial_body_accepted_by_remote() { + let failure = complete_transition_upload( + std::future::ready(Ok(Uuid::new_v4().to_string())), + std::future::ready(Ok(1024)), + 1024, + consumed(511), + ) + .await + .expect_err("remote success must not hide a partially consumed body"); + assert!(matches!(failure.error, StorageError::LessData)); + assert!(failure.candidate.is_some()); + } + + #[tokio::test] + async fn maps_source_panic_cancel_and_early_close_to_failures() { + let panic_failure = complete_transition_upload( + std::future::ready(Ok(Uuid::new_v4().to_string())), + async { + panic!("injected transition producer panic"); + #[allow(unreachable_code)] + Ok(0) + }, + 1, + consumed(0), + ) + .await + .expect_err("a producer panic must not enter local commit"); + assert!(matches!(panic_failure.error, StorageError::Unexpected)); + + let cancelled = complete_transition_upload( + std::future::ready(Ok(Uuid::new_v4().to_string())), + std::future::ready(Err(StorageError::OperationCanceled)), + 1, + consumed(0), + ) + .await + .expect_err("a cancelled producer must not enter local commit"); + assert!(matches!(cancelled.error, StorageError::OperationCanceled)); + + let early_close = complete_transition_upload( + std::future::ready(Ok(Uuid::new_v4().to_string())), + std::future::ready(Err(StorageError::Io(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "remote reader closed early", + )))), + 1024, + consumed(16), + ) + .await + .expect_err("an early remote close must not enter local commit"); + assert!(matches!(early_close.error, StorageError::Io(ref err) if err.kind() == std::io::ErrorKind::BrokenPipe)); + } + + #[tokio::test] + async fn rejects_declared_size_mismatches_and_accepts_zero_size() { + let shorter = complete_transition_upload( + std::future::ready(Ok(Uuid::new_v4().to_string())), + std::future::ready(Ok(511)), + 512, + consumed(511), + ) + .await + .expect_err("a short source must fail the declared-size check"); + assert!(matches!(shorter.error, StorageError::LessData)); + + let longer = complete_transition_upload( + std::future::ready(Ok(Uuid::new_v4().to_string())), + std::future::ready(Ok(513)), + 512, + consumed(513), + ) + .await + .expect_err("an oversized source must fail the declared-size check"); + assert!(matches!(longer.error, StorageError::MoreData)); + + let exact_remote_version = Uuid::new_v4().to_string(); + let exact = complete_transition_upload( + std::future::ready(Ok(exact_remote_version.clone())), + std::future::ready(Ok(512)), + 512, + consumed(512), + ) + .await + .expect("an exact producer and consumer byte count must complete"); + assert_eq!(exact.candidate.remote_version(), exact_remote_version); + assert_eq!((exact.produced, exact.consumed), (512, 512)); + + let remote_version = Uuid::nil().to_string(); + let result = + complete_transition_upload(std::future::ready(Ok(remote_version.clone())), std::future::ready(Ok(0)), 0, consumed(0)) + .await + .expect("an empty source and empty remote body must complete"); + assert_eq!(result.candidate.remote_version(), remote_version); + assert_eq!((result.produced, result.consumed), (0, 0)); + } + + #[tokio::test] + async fn preserves_remote_error_when_commit_status_is_unknown() { + let failure = complete_transition_upload( + std::future::ready(Err(std::io::Error::new(std::io::ErrorKind::ConnectionReset, "remote response was lost"))), + std::future::ready(Err(StorageError::Io(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "consumer disappeared", + )))), + 1024, + consumed(0), + ) + .await + .expect_err("an unknown remote commit result must fail closed"); + assert!(matches!(failure.error, StorageError::Io(ref err) if err.kind() == std::io::ErrorKind::ConnectionReset)); + assert!(failure.candidate.is_none(), "unknown remote versions must never enter precise cleanup"); + + let source_failure = complete_transition_upload( + std::future::ready(Err(std::io::Error::new( + std::io::ErrorKind::ConnectionReset, + "remote rejected the truncated body", + ))), + std::future::ready(Err(StorageError::FileCorrupt)), + 1024, + consumed(128), + ) + .await + .expect_err("a source integrity error must survive a concurrent remote failure"); + assert!(matches!(source_failure.error, StorageError::FileCorrupt)); + assert!(source_failure.candidate.is_none()); + } } #[cfg(test)] mod transition_version_id_tests { - use super::{parse_transition_version_id, transition_cleanup_remote_version}; + use super::{TransitionUploadCandidate, parse_transition_version_id}; use uuid::Uuid; #[test] @@ -1396,7 +1792,11 @@ mod transition_version_id_tests { parse_transition_version_id(&Uuid::nil().to_string()).expect("nil remote version should be valid"), None ); - assert_eq!(transition_cleanup_remote_version(&Uuid::nil().to_string(), None), ""); + assert_eq!( + TransitionUploadCandidate::from_put_response(Uuid::nil().to_string()).cleanup_version(), + "" + ); + assert_eq!(TransitionUploadCandidate::from_put_response(String::new()).cleanup_version(), ""); } #[test] @@ -1407,9 +1807,13 @@ mod transition_version_id_tests { Some(version_id) ); assert_eq!( - transition_cleanup_remote_version(&version_id.to_string(), Some(version_id)), + TransitionUploadCandidate::from_put_response(version_id.to_string()).cleanup_version(), version_id.to_string() ); + assert_eq!( + TransitionUploadCandidate::from_put_response("opaque-version-token".to_string()).cleanup_version(), + "opaque-version-token" + ); assert!(parse_transition_version_id("not-a-uuid").is_err()); } } @@ -2391,9 +2795,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { #[tracing::instrument(level = "debug", skip(self))] async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { - let tier_config_mgr = runtime_sources::tier_config_mgr_handle(); - let mut tier_config_mgr = tier_config_mgr.write().await; - let tgt_client = match tier_config_mgr.get_driver(&opts.transition.tier).await { + let tier_config_mgr = self.ctx.tier_config_mgr(); + let tgt_client = match TierConfigMgr::acquire_operation_lease(&tier_config_mgr, &opts.transition.tier).await { Ok(client) => client, Err(err) => { return Err(Error::other(format!("remote tier error: {err}"))); @@ -2481,9 +2884,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } } - let (pr, mut pw) = tokio::io::duplex(fi.erasure.block_size); + let expected_size = u64::try_from(fi.size).map_err(|_| StorageError::FileCorrupt)?; + let (pr, pw) = tokio::io::duplex(fi.erasure.block_size); + let consumed = Arc::new(AtomicU64::new(0)); let reader = ReaderImpl::ObjectBody(GetObjectReader { - stream: Box::new(pr), + stream: Box::new(TransitionUploadReader::new(pr, Arc::clone(&consumed))), object_info: oi, buffered_body: None, body_source: GetObjectBodySource::Unprobed, @@ -2496,13 +2901,14 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let pool_index = self.pool_index; let skip_verify = opts.skip_verify_bitrot; let metrics_size_bucket = rustfs_io_metrics::get_object_size_bucket(cloned_fi.size); - tokio::spawn(async move { - if let Err(e) = Self::get_object_with_fileinfo( + let producer = async move { + let mut writer = TransitionUploadWriter::new(pw); + Self::get_object_with_fileinfo( &cloned_bucket, &cloned_object, 0, cloned_fi.size, - &mut pw, + &mut writer, cloned_fi, meta_arr, &online_disks, @@ -2514,28 +2920,38 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, metrics_size_bucket, ) - .await - { - error!("get_object_with_fileinfo err {:?}", e); - }; - }); + .await?; + writer.shutdown().await?; + Ok(writer.produced()) + }; - let rv = tgt_client.put_with_meta(&dest_obj, reader, fi.size, transition_meta).await; - if let Err(err) = rv { - return Err(StorageError::Io(err)); - } - let rv = rv?; - drop(tier_config_mgr); + let rv = complete_transition_upload( + tgt_client.put_with_meta(&dest_obj, reader, fi.size, transition_meta), + producer, + expected_size, + consumed, + ) + .await; + let candidate = match rv { + Ok(completion) => completion.candidate, + Err(failure) => { + if let Some(candidate) = failure.candidate { + let mut upload_cleanup = TransitionUploadCleanup::new(tgt_client, &dest_obj, candidate); + let _cleanup_result = upload_cleanup.cleanup().await; + } + return Err(failure.error); + } + }; - let transition_version_id = match parse_transition_version_id(&rv) { + let transition_version_id = match parse_transition_version_id(candidate.remote_version()) { Ok(version_id) => version_id, Err(err) => { - cleanup_uncommitted_transition_upload(&opts.transition.tier, &dest_obj, &rv).await; + let mut upload_cleanup = TransitionUploadCleanup::new(tgt_client, &dest_obj, candidate); + let _cleanup_result = upload_cleanup.cleanup().await; return Err(err.into()); } }; - let cleanup_remote_version = transition_cleanup_remote_version(&rv, transition_version_id); - let mut upload_cleanup = TransitionUploadCleanup::new(&opts.transition.tier, &dest_obj, cleanup_remote_version); + let mut upload_cleanup = TransitionUploadCleanup::new(tgt_client, &dest_obj, candidate); let mut commit_opts = opts.clone(); commit_opts.no_lock = true; @@ -2546,7 +2962,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { match self.acquire_write_lock_diag("transition_object_commit", bucket, object).await { Ok(guard) => Some(guard), Err(err) => { - upload_cleanup.cleanup().await; + let _cleanup_result = upload_cleanup.cleanup().await; return Err(err); } } @@ -2557,7 +2973,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { Ok(current) => current, Err(err) => { drop(transition_lock_guard); - upload_cleanup.cleanup().await; + let _cleanup_result = upload_cleanup.cleanup().await; return Err(err); } }; @@ -2569,7 +2985,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { if current_fi.transition_status == TRANSITION_COMPLETE || !source_matches { let already_transitioned = current_fi.transition_status == TRANSITION_COMPLETE; drop(transition_lock_guard); - upload_cleanup.cleanup().await; + let _cleanup_result = upload_cleanup.cleanup().await; if already_transitioned { return Ok(()); } @@ -2580,12 +2996,17 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { current_fi.transitioned_objname = dest_obj; current_fi.transition_tier = opts.transition.tier.clone(); current_fi.transition_version_id = transition_version_id; + rustfs_utils::http::metadata_compat::insert_str( + &mut current_fi.metadata, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + rustfs_utils::crypto::hex(upload_cleanup.lease.backend_identity()), + ); fi = current_fi; let event_name = EventName::LifecycleTransition.as_str(); if transition_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) { drop(transition_lock_guard); - upload_cleanup.cleanup().await; + let _cleanup_result = upload_cleanup.cleanup().await; return Err(StorageError::NamespaceLockQuorumUnavailable { mode: "transition_object_commit", bucket: bucket.to_string(), @@ -2595,7 +3016,14 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { }); } #[cfg(test)] - pause_transition_before_local_commit(bucket, object).await; + pause_transition_commit(bucket, object, TransitionCommitPause::BeforeLeaseCheck).await; + if !upload_cleanup.lease.is_current_generation() { + drop(transition_lock_guard); + let _cleanup_result = upload_cleanup.cleanup().await; + return Err(Error::other("remote tier configuration changed during transition")); + } + #[cfg(test)] + pause_transition_commit(bucket, object, TransitionCommitPause::AfterLeaseCheck).await; upload_cleanup.disarm(); if let Err(err) = self.delete_object_version(bucket, object, &fi, false).await { warn!( @@ -2687,7 +3115,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let mut opts = opts.clone(); opts.part_number = Some(1); let rs: Option = None; - let gr = get_transitioned_object_reader(bucket, object, &rs, &HeaderMap::new(), &oi, &opts).await; + let gr = get_transitioned_object_reader_with_tier_manager( + bucket, + object, + &rs, + &HeaderMap::new(), + &oi, + &opts, + &self_.ctx.tier_config_mgr(), + ) + .await; if let Err(err) = gr { return set_restore_header_fn(&mut oi, Some(to_object_err(err.into(), vec![bucket, object]))).await; } @@ -2754,7 +3191,17 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { .await; } }; - let gr = match get_transitioned_object_reader(bucket, object, &rs, &HeaderMap::new(), &oi, &part_opts).await { + let gr = match get_transitioned_object_reader_with_tier_manager( + bucket, + object, + &rs, + &HeaderMap::new(), + &oi, + &part_opts, + &self_.ctx.tier_config_mgr(), + ) + .await + { Ok(reader) => reader, Err(err) => { return set_restore_header_fn(&mut oi, Some(StorageError::Io(err))).await; @@ -3018,13 +3465,134 @@ pub(in crate::set_disk::ops) mod hermetic_set_disks_support { } } +#[cfg(test)] +mod metadata_mutation_generation_tests { + use super::hermetic_set_disks_support::hermetic_set_disks; + use super::*; + use crate::disk::DiskAPI as _; + use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; + + async fn put_and_prime( + set_disks: &Arc, + bucket: &str, + object: &str, + payload: &[u8], + ) -> (ObjectInfo, GetObjectMetadataCacheKey) { + let mut reader = PutObjReader::from_vec(payload.to_vec()); + let info = set_disks + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("test object should be written"); + set_disks + .get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false) + .await + .expect("test object metadata should resolve"); + let generation = set_disks + .get_object_metadata_cache_generation(bucket, object) + .expect("metadata cache generation should be active"); + let key = GetObjectMetadataCacheKey::new(bucket, object, generation); + assert!( + set_disks.get_object_metadata_cache.get(&key).await.is_some(), + "metadata priming should publish the current generation" + ); + (info, key) + } + + async fn assert_retired(set_disks: &SetDisks, key: &GetObjectMetadataCacheKey) { + set_disks.get_object_metadata_cache.run_pending_tasks().await; + assert!( + set_disks.get_object_metadata_cache.get(key).await.is_none(), + "the mutation must physically retire the prior metadata generation" + ); + } + + #[tokio::test] + #[serial_test::serial(metadata_cache_invalidation_probe)] + async fn metadata_semantic_mutation_generation_matrix_retires_cached_snapshot() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "metadata-mutation-generation-bucket"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let put_object = "put-object"; + let (_, put_key) = put_and_prime(&set_disks, bucket, put_object, b"initial PUT body").await; + let put_probe = MetadataCacheInvalidationProbe::install(bucket, put_object); + let mut replacement = PutObjReader::from_vec(b"replacement PUT body".to_vec()); + set_disks + .put_object(bucket, put_object, &mut replacement, &ObjectOptions::default()) + .await + .expect("replacement PUT should succeed"); + assert_eq!(put_probe.count(), 2, "PUT must invalidate before mutation and after commit"); + assert_retired(&set_disks, &put_key).await; + drop(put_probe); + + let delete_object = "delete-object"; + let (_, delete_key) = put_and_prime(&set_disks, bucket, delete_object, b"DELETE body").await; + let delete_probe = MetadataCacheInvalidationProbe::install(bucket, delete_object); + set_disks + .delete_object(bucket, delete_object, ObjectOptions::default()) + .await + .expect("DELETE should succeed"); + assert_eq!(delete_probe.count(), 2, "DELETE must invalidate before mutation and after commit"); + assert_retired(&set_disks, &delete_key).await; + drop(delete_probe); + + let copy_object = "copy-object"; + let (mut copy_info, copy_key) = put_and_prime(&set_disks, bucket, copy_object, b"COPY body").await; + copy_info.metadata_only = true; + Arc::make_mut(&mut copy_info.user_defined).insert("x-amz-meta-copy".to_string(), "updated".to_string()); + let copy_probe = MetadataCacheInvalidationProbe::install(bucket, copy_object); + set_disks + .copy_object( + bucket, + copy_object, + bucket, + copy_object, + &mut copy_info, + &ObjectOptions::default(), + &ObjectOptions::default(), + ) + .await + .expect("metadata COPY should succeed"); + assert_eq!( + copy_probe.count(), + 4, + "metadata COPY must retain both outer and update-object-meta fences" + ); + assert_retired(&set_disks, ©_key).await; + drop(copy_probe); + + let metadata_object = "metadata-object"; + let (_, metadata_key) = put_and_prime(&set_disks, bucket, metadata_object, b"metadata body").await; + let mut metadata = HashMap::new(); + metadata.insert("x-amz-meta-updated".to_string(), "true".to_string()); + let metadata_opts = ObjectOptions { + eval_metadata: Some(metadata), + ..Default::default() + }; + let metadata_probe = MetadataCacheInvalidationProbe::install(bucket, metadata_object); + set_disks + .put_object_metadata(bucket, metadata_object, &metadata_opts) + .await + .expect("metadata PUT should succeed"); + assert_eq!( + metadata_probe.count(), + 4, + "metadata PUT must retain both outer and update-object-meta fences" + ); + assert_retired(&set_disks, &metadata_key).await; + } +} + #[cfg(all(test, feature = "test-util"))] mod transition_commit_failure_tests { use super::hermetic_set_disks_support::hermetic_set_disks; use super::*; use crate::bucket::lifecycle::lifecycle::{TRANSITION_PENDING, TransitionOptions}; use crate::disk::DiskAPI as _; - use crate::services::tier::test_util::register_mock_tier; + use crate::services::tier::test_util::{MockWarmBackend, register_mock_tier}; + use crate::services::tier::tier::TierConfigMgr; use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; use http::HeaderMap; use tokio::io::AsyncReadExt; @@ -3137,6 +3705,500 @@ mod transition_commit_failure_tests { .expect("the local source body should drain"); assert_eq!(restored, payload); } + + #[tokio::test] + #[serial_test::serial] + async fn production_transition_replacement_revokes_commit_and_cleans_with_old_driver() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "transition-generation-fence-bucket"; + let object = "object.bin"; + let payload = b"generation replacement must fence the local transition commit".repeat(1024); + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + let mut reader = PutObjReader::from_vec(payload); + let original = set_disks + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("source object should be written"); + + let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let manager = runtime_sources::global_tier_config_mgr(); + let old_backend = register_mock_tier(&manager, &tier_name).await; + let opts = ObjectOptions { + no_lock: true, + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.clone(), + etag: original.etag.clone().unwrap_or_default(), + ..Default::default() + }, + version_id: original.version_id.map(|version| version.to_string()), + mod_time: original.mod_time, + ..Default::default() + }; + + let barrier = TransitionCommitBarrier::install(bucket, object); + let transition_set = Arc::clone(&set_disks); + let transition = tokio::spawn(async move { transition_set.transition_object(bucket, object, &opts).await }); + barrier.wait_until_paused().await; + assert_eq!(old_backend.put_count().await, 1, "remote candidate must exist before replacement"); + + let replacement_backend = MockWarmBackend::new(); + let replacement_manager = Arc::new(RwLock::new(TierConfigMgr { + driver_cache: HashMap::new(), + tiers: manager.read().await.tiers.clone(), + last_refreshed_at: OffsetDateTime::now_utc(), + })); + { + let mut replacement = replacement_manager.write().await; + replacement + .tiers + .get_mut(&tier_name) + .and_then(|tier| tier.minio.as_mut()) + .expect("replacement tier should exist") + .prefix = "replacement/".to_string(); + replacement + .install_test_driver(&tier_name, Box::new(replacement_backend)) + .expect("replacement driver should install"); + } + let replacement = match Arc::try_unwrap(replacement_manager) { + Ok(manager) => manager.into_inner(), + Err(_) => panic!("replacement manager should have one owner"), + }; + let publish_handle = manager.clone(); + let publish_tier = tier_name.clone(); + let publish = + tokio::spawn( + async move { TierConfigMgr::publish_candidate(&publish_handle, replacement, Some(&publish_tier)).await }, + ); + + tokio::time::timeout(Duration::from_secs(5), async { + loop { + match TierConfigMgr::acquire_operation_lease(&manager, &tier_name).await { + Err(err) if err.message.contains("being replaced") => break, + Ok(lease) => drop(lease), + Err(err) => panic!("unexpected lease error while replacement drains: {err}"), + } + tokio::task::yield_now().await; + } + }) + .await + .expect("replacement should revoke the in-flight generation"); + assert!(!publish.is_finished(), "replacement must wait for the production transition lease"); + + barrier.release(); + transition + .await + .expect("transition task should join") + .expect_err("revoked generation must not commit tier-name metadata"); + publish + .await + .expect("replacement task should join") + .expect("replacement should finish after old cleanup releases its lease"); + assert_eq!( + old_backend.remove_count().await, + 1, + "cancelled transition must clean up with the old driver" + ); + assert_eq!(old_backend.object_count().await, 0); + } + + #[tokio::test] + #[serial_test::serial] + async fn legacy_reload_rejects_route_change_after_local_transition_commit() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "transition-post-check-fence-bucket"; + let object = "object.bin"; + let payload = b"the operation lease must cover the local transition commit".repeat(1024); + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + let mut reader = PutObjReader::from_vec(payload.clone()); + let original = set_disks + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("source object should be written"); + + let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let manager = runtime_sources::global_tier_config_mgr(); + let old_backend = register_mock_tier(&manager, &tier_name).await; + let old_prefix = manager.read().await.tiers[&tier_name] + .minio + .as_ref() + .expect("old tier route should exist") + .prefix + .clone(); + let old_identity = TierConfigMgr::acquire_operation_lease(&manager, &tier_name) + .await + .expect("old tier identity should be available") + .backend_identity(); + let opts = ObjectOptions { + no_lock: true, + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.clone(), + etag: original.etag.clone().unwrap_or_default(), + ..Default::default() + }, + version_id: original.version_id.map(|version| version.to_string()), + mod_time: original.mod_time, + ..Default::default() + }; + + let barrier = TransitionCommitBarrier::install_after_lease_check(bucket, object); + let transition_set = Arc::clone(&set_disks); + let transition = tokio::spawn(async move { transition_set.transition_object(bucket, object, &opts).await }); + barrier.wait_until_paused().await; + assert_eq!(old_backend.put_count().await, 1, "remote candidate must exist before replacement"); + + let replacement_manager = Arc::new(RwLock::new(TierConfigMgr { + driver_cache: HashMap::new(), + tiers: manager.read().await.tiers.clone(), + last_refreshed_at: OffsetDateTime::now_utc(), + })); + { + let mut replacement = replacement_manager.write().await; + replacement + .tiers + .get_mut(&tier_name) + .and_then(|tier| tier.minio.as_mut()) + .expect("replacement tier should exist") + .prefix = "replacement/".to_string(); + } + let replacement = match Arc::try_unwrap(replacement_manager) { + Ok(manager) => manager.into_inner(), + Err(_) => panic!("replacement manager should have one owner"), + }; + let publish_handle = manager.clone(); + let publish = tokio::spawn(async move { publish_handle.write().await.publish_legacy_reload(replacement).await }); + tokio::time::timeout(Duration::from_secs(5), async { + while manager.try_read().is_ok() { + tokio::task::yield_now().await; + } + }) + .await + .expect("legacy reload should hold the manager guard while the checked generation drains"); + assert!( + !publish.is_finished(), + "replacement must wait until the local transition commit releases its lease" + ); + + barrier.release(); + transition + .await + .expect("transition task should join") + .expect("a transition linearized before replacement should commit"); + publish + .await + .expect("replacement task should join") + .expect_err("replacement must not rebind a tier name referenced by the committed transition"); + assert_eq!(old_backend.remove_count().await, 0, "committed transition data must not be cleaned up"); + assert_eq!(old_backend.object_count().await, 1); + let transitioned = set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("transitioned metadata should resolve"); + let expected_identity = rustfs_utils::crypto::hex(old_identity); + assert_eq!( + rustfs_utils::http::metadata_compat::get_str( + &transitioned.user_defined, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + ) + .as_deref(), + Some(expected_identity.as_str()) + ); + assert_eq!( + manager + .read() + .await + .tiers + .get(&tier_name) + .and_then(|tier| tier.minio.as_ref()) + .expect("old tier route should remain configured") + .prefix, + old_prefix + ); + + let mut restored = Vec::new(); + set_disks + .get_object_reader( + bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("the transitioned object must remain routed through the old generation") + .stream + .read_to_end(&mut restored) + .await + .expect("the transitioned object body should drain"); + assert_eq!(restored, payload); + } +} + +#[cfg(all(test, feature = "test-util"))] +mod transition_upload_integrity_tests { + use super::hermetic_set_disks_support::hermetic_set_disks; + use super::*; + use crate::bucket::lifecycle::lifecycle::{TRANSITION_PENDING, TransitionOptions}; + use crate::disk::DiskAPI as _; + use crate::services::tier::test_util::register_mock_tier; + use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; + use http::HeaderMap; + + async fn assert_local_source_intact(set_disks: &Arc, bucket: &str, object: &str, payload: &[u8]) { + let mut restored = Vec::new(); + set_disks + .get_object_reader( + bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("failed transition must leave the local source readable") + .stream + .read_to_end(&mut restored) + .await + .expect("local source should drain after failed transition"); + assert_eq!(restored, payload); + let (fi, _, _) = set_disks + .get_object_fileinfo( + bucket, + object, + &ObjectOptions { + no_lock: true, + metadata_cache_safe: false, + ..Default::default() + }, + true, + false, + ) + .await + .expect("local source metadata should remain available"); + assert_ne!(fi.transition_status, TRANSITION_COMPLETE); + } + + async fn write_source( + set_disks: &Arc, + disk_stores: &[DiskStore], + bucket: &str, + object: &str, + payload: &[u8], + ) -> ObjectInfo { + for disk in disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + let mut reader = PutObjReader::from_vec(payload.to_vec()); + set_disks + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("source object should be written") + } + + fn transition_options(original: &ObjectInfo, tier_name: String) -> ObjectOptions { + ObjectOptions { + no_lock: true, + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name, + etag: original.etag.clone().unwrap_or_default(), + ..Default::default() + }, + version_id: original.version_id.map(|version| version.to_string()), + mod_time: original.mod_time, + ..Default::default() + } + } + + #[tokio::test] + #[serial_test::serial] + async fn partial_remote_acceptance_cleans_exact_candidate_and_preserves_source() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "transition-partial-accept-bucket"; + let object = "object.bin"; + let payload = vec![0x5a; 2 * 1024 * 1024]; + let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await; + let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let remote_version = Uuid::nil().to_string(); + let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await; + backend.set_put_read_limit(Some(4096)).await; + backend.set_put_remote_version(Some(remote_version.clone())).await; + + let error = set_disks + .transition_object(bucket, object, &transition_options(&original, tier_name)) + .await + .expect_err("accepting only a prefix must fail transition completion"); + assert!(matches!(error, StorageError::Io(_) | StorageError::LessData)); + let removed_versions = backend.remove_versions().await; + assert_eq!(removed_versions.len(), 1); + assert_eq!( + removed_versions[0].1, "", + "the nil UUID response is the backend's unversioned sentinel and must not become an S3 versionId" + ); + assert_eq!(backend.object_count().await, 0); + assert_local_source_intact(&set_disks, bucket, object, &payload).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn opaque_remote_version_is_cleaned_before_parse_failure() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "transition-unknown-version-bucket"; + let object = "object.bin"; + let payload = b"unknown remote version must retain local data".repeat(1024); + let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await; + let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await; + backend.set_put_remote_version(Some("opaque-version-token".to_string())).await; + + set_disks + .transition_object(bucket, object, &transition_options(&original, tier_name)) + .await + .expect_err("an unparseable remote version must fail closed"); + let removed_versions = backend.remove_versions().await; + assert_eq!(removed_versions.len(), 1); + assert_eq!(removed_versions[0].1, "opaque-version-token"); + assert_eq!(backend.object_count().await, 0); + assert_local_source_intact(&set_disks, bucket, object, &payload).await; + } +} + +#[cfg(all(test, feature = "test-util"))] +mod transition_source_identity_matrix_tests { + use super::hermetic_set_disks_support::hermetic_set_disks; + use super::*; + use crate::bucket::lifecycle::lifecycle::{TRANSITION_PENDING, TransitionOptions}; + use crate::disk::DiskAPI as _; + use crate::services::tier::test_util::register_mock_tier; + use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; + + #[tokio::test] + #[serial_test::serial] + async fn transition_source_identity_field_matrix_rejects_single_field_drift() { + #[derive(Clone, Copy, Debug)] + enum IdentityField { + VersionId, + DataDir, + ModTime, + Size, + Etag, + } + + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "transition-identity-matrix-bucket"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await; + + for (index, field) in [ + IdentityField::VersionId, + IdentityField::DataDir, + IdentityField::ModTime, + IdentityField::Size, + IdentityField::Etag, + ] + .into_iter() + .enumerate() + { + let object = format!("identity-{index}.bin"); + let payload = vec![u8::try_from(index + 1).expect("matrix index should fit u8"); 1024 * 1024]; + let mut reader = PutObjReader::from_vec(payload); + let original = set_disks + .put_object(bucket, &object, &mut reader, &ObjectOptions::default()) + .await + .expect("source object should be written"); + let (source, _, _) = set_disks + .get_object_fileinfo(bucket, &object, &ObjectOptions::default(), true, false) + .await + .expect("source metadata should resolve"); + let opts = ObjectOptions { + no_lock: true, + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.clone(), + etag: original.etag.clone().unwrap_or_default(), + ..Default::default() + }, + version_id: original.version_id.map(|version| version.to_string()), + mod_time: original.mod_time, + ..Default::default() + }; + let put_barrier = backend.arm_put_barrier().await; + let transition_set = Arc::clone(&set_disks); + let transition_object = object.clone(); + let transition = + tokio::spawn(async move { transition_set.transition_object(bucket, &transition_object, &opts).await }); + put_barrier.wait_until_paused().await; + + let mut changed = source.clone(); + match field { + IdentityField::VersionId => changed.version_id = Some(Uuid::new_v4()), + IdentityField::DataDir => changed.data_dir = Some(Uuid::new_v4()), + IdentityField::ModTime => { + changed.mod_time = changed.mod_time.map(|value| value + time::Duration::nanoseconds(1)); + } + IdentityField::Size => changed.size += 1, + IdentityField::Etag => { + changed.metadata.insert("etag".to_string(), format!("changed-{index}")); + } + } + for disk in &disk_stores { + disk.write_metadata("", bucket, &object, changed.clone()) + .await + .expect("single-field metadata drift should be written"); + } + put_barrier.release(); + + transition + .await + .expect("transition task should not panic") + .expect_err("transition must reject a source whose identity changed after upload"); + let expected_attempts = index + 1; + assert_eq!(backend.put_count().await, expected_attempts); + assert_eq!(backend.remove_count().await, expected_attempts); + assert_eq!( + backend.remove_versions().await, + backend.put_versions().await, + "rejected identity drift must remove the exact uploaded version" + ); + + match field { + IdentityField::VersionId => assert_ne!(source.version_id, changed.version_id), + IdentityField::DataDir => assert_ne!(source.data_dir, changed.data_dir), + IdentityField::ModTime => assert_ne!(source.mod_time, changed.mod_time), + IdentityField::Size => assert_ne!(source.size, changed.size), + IdentityField::Etag => assert_ne!(get_raw_etag(&source.metadata), get_raw_etag(&changed.metadata)), + } + if !matches!(field, IdentityField::VersionId) { + assert_eq!(source.version_id, changed.version_id); + } + if !matches!(field, IdentityField::DataDir) { + assert_eq!(source.data_dir, changed.data_dir); + } + if !matches!(field, IdentityField::ModTime) { + assert_eq!(source.mod_time, changed.mod_time); + } + if !matches!(field, IdentityField::Size) { + assert_eq!(source.size, changed.size); + } + if !matches!(field, IdentityField::Etag) { + assert_eq!(get_raw_etag(&source.metadata), get_raw_etag(&changed.metadata)); + } + } + } } #[cfg(test)] diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index 0ac667c68..880e1421d 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -2540,20 +2540,27 @@ mod metadata_cache_tests { } #[tokio::test] - async fn get_object_metadata_cache_invalidation_removes_object_entry() { + async fn metadata_cache_per_key_invalidation_physically_reclaims_retired_generation() { let set = new_metadata_cache_test_set().await; let fi = valid_test_fileinfo("object"); - let generation = set.get_object_metadata_cache_generation("bucket", "object"); - set.cache_get_object_fileinfo(("bucket", "object"), generation, &fi, std::slice::from_ref(&fi), &[], 0) + let generation = set + .get_object_metadata_cache_generation("bucket", "object") + .expect("metadata cache generation should be active"); + let retired_key = GetObjectMetadataCacheKey::new("bucket", "object", generation); + set.cache_get_object_fileinfo(("bucket", "object"), Some(generation), &fi, std::slice::from_ref(&fi), &[], 0) .await; + set.get_object_metadata_cache.run_pending_tasks().await; assert!(set.cached_get_object_fileinfo("bucket", "object").await.is_some()); + assert_eq!(set.get_object_metadata_cache.entry_count(), 1); set.invalidate_get_object_metadata_cache("bucket", "object").await; + set.get_object_metadata_cache.run_pending_tasks().await; assert!( - set.cached_get_object_fileinfo("bucket", "object").await.is_none(), - "explicit invalidation must remove the cached object metadata" + set.get_object_metadata_cache.get(&retired_key).await.is_none(), + "per-key invalidation must physically remove the retired generation" ); + assert_eq!(set.get_object_metadata_cache.entry_count(), 0); } #[tokio::test] @@ -2704,6 +2711,77 @@ mod metadata_cache_tests { ); } + #[tokio::test] + async fn metadata_cache_generation_isolated_between_set_instances() { + let first = new_metadata_cache_test_set().await; + let second = new_metadata_cache_test_set().await; + let fi = valid_test_fileinfo("object"); + let first_generation = first.get_object_metadata_cache_generation("bucket", "object"); + let second_generation = second.get_object_metadata_cache_generation("bucket", "object"); + first + .cache_get_object_fileinfo(("bucket", "object"), first_generation, &fi, std::slice::from_ref(&fi), &[], 0) + .await; + second + .cache_get_object_fileinfo(("bucket", "object"), second_generation, &fi, std::slice::from_ref(&fi), &[], 0) + .await; + + first.invalidate_get_object_metadata_cache("bucket", "object").await; + + assert!(first.cached_get_object_fileinfo("bucket", "object").await.is_none()); + assert!(second.cached_get_object_fileinfo("bucket", "object").await.is_some()); + assert_eq!(second.get_object_metadata_cache_generation("bucket", "object"), second_generation); + } + + #[tokio::test] + async fn metadata_cache_cached_hash_collision_preserves_full_identity() { + let set = new_metadata_cache_test_set().await; + let generation = set + .get_object_metadata_cache_generation("bucket-a", "object-a") + .expect("metadata cache generation should be active"); + let first_key = GetObjectMetadataCacheKey::new("bucket-a", "object-a", generation); + let second_key = GetObjectMetadataCacheKey { + bucket: Arc::from("bucket-b"), + object: Arc::from("object-b"), + generation: generation.value, + hash: generation.hash, + }; + let first_fi = valid_test_fileinfo("object-a"); + let second_fi = valid_test_fileinfo("object-b"); + let entry = |fi: FileInfo| { + Arc::new(GetObjectMetadataCacheEntry { + created_at: Instant::now(), + parts_metadata: vec![fi.clone()], + fi, + online_disks: Vec::new(), + read_quorum: 0, + }) + }; + + set.get_object_metadata_cache.insert(first_key.clone(), entry(first_fi)).await; + set.get_object_metadata_cache + .insert(second_key.clone(), entry(second_fi)) + .await; + + assert_eq!( + set.get_object_metadata_cache + .get(&first_key) + .await + .expect("first colliding entry should remain addressable") + .fi + .name, + "object-a" + ); + assert_eq!( + set.get_object_metadata_cache + .get(&second_key) + .await + .expect("second colliding entry should remain addressable") + .fi + .name, + "object-b" + ); + } + #[tokio::test] async fn metadata_cache_generation_overflow_fails_closed() { let set = new_metadata_cache_test_set().await; @@ -2743,6 +2821,62 @@ mod metadata_cache_tests { assert!(set.cached_get_object_fileinfo("bucket", "object").await.is_none()); } + #[tokio::test] + async fn metadata_cache_invalidate_all_physically_reclaims_retired_generations() { + let set = new_metadata_cache_test_set().await; + let mut retired_keys = Vec::new(); + for object in ["object-a", "object-b", "object-c"] { + let fi = valid_test_fileinfo(object); + let generation = set + .get_object_metadata_cache_generation("bucket", object) + .expect("metadata cache generation should be active"); + retired_keys.push(GetObjectMetadataCacheKey::new("bucket", object, generation)); + set.cache_get_object_fileinfo(("bucket", object), Some(generation), &fi, std::slice::from_ref(&fi), &[], 0) + .await; + } + set.get_object_metadata_cache.run_pending_tasks().await; + assert_eq!(set.get_object_metadata_cache.entry_count(), 3); + + set.invalidate_all_get_object_metadata_cache(); + set.get_object_metadata_cache.run_pending_tasks().await; + + for key in retired_keys { + assert!( + set.get_object_metadata_cache.get(&key).await.is_none(), + "invalidate-all must physically remove every retired generation" + ); + } + assert_eq!(set.get_object_metadata_cache.entry_count(), 0); + } + + #[tokio::test] + async fn metadata_cache_invalidate_all_at_max_fails_closed_and_clears_entries() { + let set = new_metadata_cache_test_set().await; + let fi = valid_test_fileinfo("object"); + let generation = set + .get_object_metadata_cache_generation("bucket", "object") + .expect("metadata cache generation should be active"); + let retired_key = GetObjectMetadataCacheKey::new("bucket", "object", generation); + set.cache_get_object_fileinfo(("bucket", "object"), Some(generation), &fi, std::slice::from_ref(&fi), &[], 0) + .await; + for fence in set.get_object_metadata_cache_generations.iter() { + fence.store(u64::MAX, Ordering::Release); + } + + set.invalidate_all_get_object_metadata_cache(); + set.get_object_metadata_cache.run_pending_tasks().await; + + assert!( + set.get_object_metadata_cache_generations + .iter() + .all(|fence| fence.load(Ordering::Acquire) == u64::MAX), + "invalidate-all must not wrap a saturated fence" + ); + assert_eq!(set.get_object_metadata_cache_generation("bucket", "object"), None); + assert!(set.get_object_metadata_cache.get(&retired_key).await.is_none()); + assert_eq!(set.get_object_metadata_cache.entry_count(), 0); + } + #[tokio::test] async fn get_object_metadata_cache_prunes_when_capacity_is_reached() { // moka handles capacity eviction automatically via the configured max_capacity. diff --git a/crates/ecstore/src/set_disk/transition_matrix_tests.rs b/crates/ecstore/src/set_disk/transition_matrix_tests.rs new file mode 100644 index 000000000..d2ae52486 --- /dev/null +++ b/crates/ecstore/src/set_disk/transition_matrix_tests.rs @@ -0,0 +1,104 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use crate::bucket::lifecycle::lifecycle::{TRANSITION_PENDING, TransitionOptions}; +use crate::ecstore_validation_blackbox::make_local_set_disks; +use crate::services::tier::test_util::register_mock_tier; +use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; +use tokio::io::AsyncReadExt; + +async fn prime_metadata_generation(set_disks: &SetDisks, bucket: &str, object: &str) -> GetObjectMetadataCacheKey { + set_disks + .get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false) + .await + .expect("object metadata should resolve"); + let generation = set_disks + .get_object_metadata_cache_generation(bucket, object) + .expect("metadata generation should be active"); + let key = GetObjectMetadataCacheKey::new(bucket, object, generation); + assert!( + set_disks.get_object_metadata_cache.get(&key).await.is_some(), + "metadata read should publish the generation under test" + ); + key +} + +async fn assert_generation_reclaimed(set_disks: &SetDisks, key: &GetObjectMetadataCacheKey) { + set_disks.get_object_metadata_cache.run_pending_tasks().await; + assert!( + set_disks.get_object_metadata_cache.get(key).await.is_none(), + "metadata mutation must physically reclaim the prior generation" + ); +} + +#[tokio::test] +#[serial_test::serial] +async fn transition_and_restore_reclaim_prior_metadata_generations() { + let (_dirs, set_disks) = make_local_set_disks(4, 2).await; + let bucket = "transition-restore-generation-bucket"; + let object = "object.bin"; + let payload = vec![0x5au8; 1024 * 1024]; + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + let mut reader = PutObjReader::from_vec(payload.clone()); + let original = set_disks + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("source object should be written"); + let source_generation = prime_metadata_generation(&set_disks, bucket, object).await; + + let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let backend = register_mock_tier(&set_disks.instance_ctx().tier_config_mgr(), &tier_name).await; + let transition_opts = ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name, + etag: original.etag.clone().expect("source ETag should be present"), + ..Default::default() + }, + version_id: original.version_id.map(|version| version.to_string()), + mod_time: original.mod_time, + ..Default::default() + }; + set_disks + .transition_object(bucket, object, &transition_opts) + .await + .expect("transition should succeed"); + assert_generation_reclaimed(&set_disks, &source_generation).await; + + let transitioned_generation = prime_metadata_generation(&set_disks, bucket, object).await; + let mut restore_opts = ObjectOptions::default(); + restore_opts.transition.restore_request.days = Some(1); + Arc::clone(&set_disks) + .restore_transitioned_object(bucket, object, &restore_opts) + .await + .expect("restore should succeed"); + assert_generation_reclaimed(&set_disks, &transitioned_generation).await; + assert_eq!(backend.get_count().await, 1, "restore should read the remote candidate exactly once"); + + let mut restored = Vec::new(); + set_disks + .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("restored object should be readable") + .stream + .read_to_end(&mut restored) + .await + .expect("restored body should drain"); + assert_eq!(restored, payload); + assert_eq!(backend.get_count().await, 1, "restored GET should use the local copy"); +} diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index 4c00a6420..9e3b2b244 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -264,7 +264,7 @@ impl ECStore { /// Get the tier config manager pub fn tier_config_mgr(&self) -> Arc> { - runtime_sources::global_tier_config_mgr() + self.ctx.tier_config_mgr() } /// Get the server configuration diff --git a/crates/filemeta/src/filemeta/version.rs b/crates/filemeta/src/filemeta/version.rs index 5025d39f2..d42da89ae 100644 --- a/crates/filemeta/src/filemeta/version.rs +++ b/crates/filemeta/src/filemeta/version.rs @@ -30,8 +30,9 @@ use crate::ChecksumInfo; use rustfs_utils::HashAlgorithm; use rustfs_utils::http::{ SUFFIX_CRC, SUFFIX_FREE_VERSION, SUFFIX_INLINE_DATA, SUFFIX_PURGESTATUS, SUFFIX_TIER_FV_ID, SUFFIX_TIER_FV_MARKER, - SUFFIX_TRANSITION_STATUS, SUFFIX_TRANSITION_TIER, SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, - contains_key_bytes, get_bytes, has_internal_suffix, insert_bytes, is_internal_key, remove_bytes, strip_internal_prefix, + SUFFIX_TRANSITION_STATUS, SUFFIX_TRANSITION_TIER, SUFFIX_TRANSITION_TIER_DESTINATION_ID, SUFFIX_TRANSITIONED_OBJECTNAME, + SUFFIX_TRANSITIONED_VERSION_ID, contains_key_bytes, get_bytes, get_consistent_bytes, get_str, has_internal_suffix, + insert_bytes, is_internal_key, remove_bytes, strip_internal_prefix, }; const MSGPACK_EXT8: u8 = 0xc7; @@ -2403,6 +2404,9 @@ impl MetaObject { ); } insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER, fi.transition_tier.as_bytes().to_vec()); + if let Some(destination_id) = get_str(&fi.metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID) { + insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID, destination_id.into_bytes()); + } } pub fn remove_restore_hdrs(&mut self) { @@ -2467,6 +2471,16 @@ impl MetaObject { insert_bytes(&mut delete_marker.meta_sys, suffix, v); } } + if contains_key_bytes(&self.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID) { + let destination_id = get_consistent_bytes(&self.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID) + .filter(|value| value.len() == 64 && value.iter().all(u8::is_ascii_hexdigit)) + .ok_or(Error::FileCorrupt)?; + insert_bytes( + &mut delete_marker.meta_sys, + SUFFIX_TRANSITION_TIER_DESTINATION_ID, + destination_id.to_vec(), + ); + } return Ok((free_entry, true)); } Ok((FileMetaVersion::default(), false)) @@ -4204,6 +4218,111 @@ mod tests { assert!(matches!(err, Error::UuidParse(_))); } + #[test] + fn meta_object_init_free_version_preserves_transition_destination_identity() { + let mut sys = HashMap::new(); + insert_bytes(&mut sys, SUFFIX_TRANSITION_STATUS, TRANSITION_COMPLETE.as_bytes().to_vec()); + let identity = b"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef".to_vec(); + insert_bytes(&mut sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID, identity.clone()); + let obj = make_meta_object_with_sys(sys); + let mut fi = FileInfo::new("object", 2, 2); + fi.set_tier_free_version_id(&Uuid::new_v4().to_string()); + + let (free_version, created) = obj + .init_free_version(&fi) + .expect("free-version initialization should succeed"); + let meta_sys = &free_version + .delete_marker + .expect("free-version should be a delete marker") + .meta_sys; + + assert!(created); + assert_eq!(get_bytes(meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID), Some(identity)); + assert!(meta_sys.contains_key(&format!( + "{}{}", + rustfs_utils::http::metadata_compat::RUSTFS_INTERNAL_PREFIX, + SUFFIX_TRANSITION_TIER_DESTINATION_ID + ))); + assert!(meta_sys.contains_key(&format!( + "{}{}", + rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX, + SUFFIX_TRANSITION_TIER_DESTINATION_ID + ))); + } + + #[test] + fn meta_object_init_free_version_accepts_single_prefix_transition_destination_identity() { + let mut sys = HashMap::new(); + insert_bytes(&mut sys, SUFFIX_TRANSITION_STATUS, TRANSITION_COMPLETE.as_bytes().to_vec()); + let identity = b"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef".to_vec(); + sys.insert( + format!( + "{}{}", + rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX, + SUFFIX_TRANSITION_TIER_DESTINATION_ID + ), + identity.clone(), + ); + let obj = make_meta_object_with_sys(sys); + let mut fi = FileInfo::new("object", 2, 2); + fi.set_tier_free_version_id(&Uuid::new_v4().to_string()); + + let (free_version, created) = obj + .init_free_version(&fi) + .expect("single-prefix legacy destination identity should remain compatible"); + let meta_sys = &free_version + .delete_marker + .expect("free-version should be a delete marker") + .meta_sys; + + assert!(created); + assert_eq!( + get_consistent_bytes(meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID), + Some(identity.as_slice()) + ); + } + + #[test] + fn meta_object_init_free_version_rejects_conflicting_or_invalid_transition_destination_identity() { + let valid_identity = b"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef".to_vec(); + for (rustfs_identity, minio_identity) in [ + ( + valid_identity, + b"abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789".to_vec(), + ), + (b"not-hex".to_vec(), b"not-hex".to_vec()), + (Vec::new(), Vec::new()), + ] { + let mut sys = HashMap::new(); + insert_bytes(&mut sys, SUFFIX_TRANSITION_STATUS, TRANSITION_COMPLETE.as_bytes().to_vec()); + sys.insert( + format!( + "{}{}", + rustfs_utils::http::metadata_compat::RUSTFS_INTERNAL_PREFIX, + SUFFIX_TRANSITION_TIER_DESTINATION_ID + ), + rustfs_identity, + ); + sys.insert( + format!( + "{}{}", + rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX, + SUFFIX_TRANSITION_TIER_DESTINATION_ID + ), + minio_identity, + ); + let obj = make_meta_object_with_sys(sys); + let mut fi = FileInfo::new("object", 2, 2); + fi.set_tier_free_version_id(&Uuid::new_v4().to_string()); + + assert_eq!( + obj.init_free_version(&fi) + .expect_err("unsafe destination identity must fail closed"), + Error::FileCorrupt + ); + } + } + #[test] fn delete_marker_decode_skips_unknown_fields_for_forward_compat() { // A newer writer emits the three known fields plus an extra one. Decoding diff --git a/crates/utils/src/http/metadata_compat.rs b/crates/utils/src/http/metadata_compat.rs index ba4334411..404df9d16 100644 --- a/crates/utils/src/http/metadata_compat.rs +++ b/crates/utils/src/http/metadata_compat.rs @@ -38,6 +38,7 @@ pub const SUFFIX_TRANSITION_STATUS: &str = "transition-status"; pub const SUFFIX_TRANSITIONED_OBJECTNAME: &str = "transitioned-object"; pub const SUFFIX_TRANSITIONED_VERSION_ID: &str = "transitioned-versionID"; pub const SUFFIX_TRANSITION_TIER: &str = "transition-tier"; +pub const SUFFIX_TRANSITION_TIER_DESTINATION_ID: &str = "transition-tier-destination-id"; pub const SUFFIX_FREE_VERSION: &str = "free-version"; pub const SUFFIX_PURGESTATUS: &str = "purgestatus"; pub const SUFFIX_REPLICA_STATUS: &str = "replica-status"; @@ -173,6 +174,28 @@ pub fn get_str(map: &HashMap, suffix: &str) -> Option { .map(|(_, value)| value.clone()) } +fn get_consistent_value<'a, V: AsRef<[u8]>>(map: &'a HashMap, suffix: &str) -> Option<&'a V> { + let (rustfs_key, minio_key) = both_keys(suffix); + let mut value = None; + for (key, candidate) in map { + if !key.eq_ignore_ascii_case(&rustfs_key) && !key.eq_ignore_ascii_case(&minio_key) { + continue; + } + if candidate.as_ref().is_empty() || value.is_some_and(|current: &V| current.as_ref() != candidate.as_ref()) { + return None; + } + value = Some(candidate); + } + value +} + +/// Returns a non-empty value when every compatibility key present for `suffix` agrees. +/// A single RustFS or MinIO key is accepted for backward compatibility; conflicting or empty +/// values return `None` so callers at destructive boundaries can fail closed. +pub fn get_consistent_str<'a>(map: &'a HashMap, suffix: &str) -> Option<&'a str> { + get_consistent_value(map, suffix).map(String::as_str) +} + pub fn contains_key_str(map: &HashMap, suffix: &str) -> bool { if with_internal_key(RUSTFS_INTERNAL_PREFIX, suffix, |k1| map.contains_key(k1)) { return true; @@ -206,6 +229,11 @@ pub fn get_bytes(map: &HashMap>, suffix: &str) -> Option .or_else(|| with_internal_key(MINIO_INTERNAL_PREFIX, suffix, |k2| map.get(k2).cloned())) } +/// Byte-valued counterpart of [`get_consistent_str`]. +pub fn get_consistent_bytes<'a>(map: &'a HashMap>, suffix: &str) -> Option<&'a [u8]> { + get_consistent_value(map, suffix).map(Vec::as_slice) +} + pub fn contains_key_bytes(map: &HashMap>, suffix: &str) -> bool { with_internal_key(RUSTFS_INTERNAL_PREFIX, suffix, |k1| map.contains_key(k1)) || with_internal_key(MINIO_INTERNAL_PREFIX, suffix, |k2| map.contains_key(k2)) @@ -271,6 +299,43 @@ mod tests { assert_eq!(get_str(&metadata, SUFFIX_TRANSITION_TIER).as_deref(), Some("rustfs-tier")); } + #[test] + fn test_consistent_str_accepts_single_or_matching_values_and_rejects_conflicts() { + let rustfs_key = internal_key_rustfs(SUFFIX_TRANSITION_TIER_DESTINATION_ID); + let minio_key = format!("{MINIO_INTERNAL_PREFIX}{SUFFIX_TRANSITION_TIER_DESTINATION_ID}"); + let mut metadata = HashMap::from([(rustfs_key, "identity-a".to_string())]); + assert_eq!(get_consistent_str(&metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID), Some("identity-a")); + + metadata.insert(minio_key, "identity-a".to_string()); + assert_eq!(get_consistent_str(&metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID), Some("identity-a")); + + metadata.insert( + format!("{MINIO_INTERNAL_PREFIX}{SUFFIX_TRANSITION_TIER_DESTINATION_ID}"), + "identity-b".to_string(), + ); + assert_eq!(get_consistent_str(&metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID), None); + } + + #[test] + fn test_consistent_bytes_accepts_single_or_matching_values_and_rejects_conflicts() { + let rustfs_key = internal_key_rustfs(SUFFIX_TRANSITION_TIER_DESTINATION_ID); + let minio_key = format!("{MINIO_INTERNAL_PREFIX}{SUFFIX_TRANSITION_TIER_DESTINATION_ID}"); + let mut metadata = HashMap::from([(rustfs_key, b"identity-a".to_vec())]); + assert_eq!( + get_consistent_bytes(&metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID), + Some(b"identity-a".as_slice()) + ); + + metadata.insert(minio_key.clone(), b"identity-a".to_vec()); + assert_eq!( + get_consistent_bytes(&metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID), + Some(b"identity-a".as_slice()) + ); + + metadata.insert(minio_key, b"identity-b".to_vec()); + assert_eq!(get_consistent_bytes(&metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID), None); + } + #[test] fn test_bytes_lookup_falls_back_to_minio_key() { let mut meta_sys = diff --git a/rustfs/src/admin/handlers/tier.rs b/rustfs/src/admin/handlers/tier.rs index d285dbf4d..396d5bdf6 100644 --- a/rustfs/src/admin/handlers/tier.rs +++ b/rustfs/src/admin/handlers/tier.rs @@ -14,10 +14,11 @@ #![allow(unused_variables, unused_mut, unused_must_use)] use crate::admin::runtime_sources::object_store_from_extensions; +use crate::admin::storage_api::runtime_sources::TierConfigMgr; use crate::admin::storage_api::tier::{ AdminError, DailyAllTierStats, ERR_TIER_ALREADY_EXISTS, ERR_TIER_BACKEND_IN_USE, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_MISSING_CREDENTIALS, ERR_TIER_NAME_NOT_UPPERCASE, - ERR_TIER_NOT_FOUND, ERR_TIER_RESERVED_NAME, TierConfig, TierCreds, TierType, + ERR_TIER_NOT_FOUND, ERR_TIER_RESERVED_NAME, TierConfig, TierConfigUpdateError, TierCreds, TierType, }; use crate::{ admin::runtime_sources::{current_daily_tier_stats, current_notification_system, current_tier_config_handle}, @@ -29,8 +30,7 @@ use crate::{ server::{ADMIN_PREFIX, RemoteAddr}, storage::request_context::spawn_traced, }; -use http::Uri; -use http::{HeaderMap, StatusCode}; +use http::{HeaderMap, StatusCode, Uri}; use hyper::Method; use matchit::Params; use percent_encoding::percent_decode_str; @@ -100,6 +100,52 @@ fn spawn_transition_tier_config_propagation(action: &'static str) { } } +fn tier_mutation_error( + update_error: TierConfigUpdateError, + action: &'static str, + failure_code: &'static str, +) -> Result { + match update_error { + TierConfigUpdateError::Load(err) => { + warn!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action, + result = "reload_failed", + error = ?err, + "admin tier state" + ); + Err(S3Error::with_message( + S3ErrorCode::Custom(failure_code.into()), + format!("tier reload failed. {err}"), + )) + } + TierConfigUpdateError::Save(err) => { + warn!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action, + result = "save_failed", + error = ?err, + "admin tier state" + ); + Err(S3Error::with_message(S3ErrorCode::Custom(failure_code.into()), "tier save failed")) + } + TierConfigUpdateError::Mutation(err) | TierConfigUpdateError::Publish(err) => Ok(err), + } +} + +fn tier_backend_in_use_response(err: &AdminError) -> Option { + (err.code == ERR_TIER_BACKEND_IN_USE.code) + .then(|| S3Error::with_message(S3ErrorCode::Custom("TierNameBackendInUse".into()), "tier backend is not empty")) +} + +fn clear_tier_error_response(err: &AdminError) -> S3Error { + S3Error::with_message(S3ErrorCode::Custom("TierClearFailed".into()), format!("tier clear failed. {err}")) +} + fn resolve_tier_name(uri: &Uri, params: &Params<'_, '_>) -> S3Result { if let Some(tier) = params.get("tier") { let decoded = percent_decode_str(tier) @@ -315,87 +361,58 @@ impl Operation for AddTier { return Err(s3_error!(InternalError, "object store is not initialized")); }; - { - let tier_config_mgr_handle = current_tier_config_handle(); - let mut tier_config_mgr = tier_config_mgr_handle.write().await; - if let Err(err) = tier_config_mgr.reload(store).await { + let tier_config_mgr_handle = current_tier_config_handle(); + if let Err(update_err) = TierConfigMgr::add_and_save(&tier_config_mgr_handle, store, args, force).await { + let err = tier_mutation_error(update_err, "add_tier", "TierAddFailed")?; + return if err.code == ERR_TIER_RESERVED_NAME.code { warn!( event = EVENT_ADMIN_TIER_STATE, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_TIER, action = "add_tier", - result = "reload_failed", + tier_name = %tier_name_for_log, + result = "reserved_name_rejected", + "admin tier state" + ); + Err(s3_error!(InvalidRequest, "Cannot use reserved tier name")) + } else if err.code == ERR_TIER_ALREADY_EXISTS.code { + Err(S3Error::with_message( + S3ErrorCode::Custom("TierNameAlreadyExist".into()), + "tier name already exists", + )) + } else if err.code == ERR_TIER_NAME_NOT_UPPERCASE.code { + Err(S3Error::with_message( + S3ErrorCode::Custom("TierNameNotUppercase".into()), + "tier name must be uppercase", + )) + } else if err.code == ERR_TIER_BACKEND_IN_USE.code { + Err(S3Error::with_message( + S3ErrorCode::Custom("TierNameBackendInUse!".into()), + "tier backend is already in use", + )) + } else if err.code == ERR_TIER_CONNECT_ERR.code { + Err(S3Error::with_message( + S3ErrorCode::Custom("TierConnectError".into()), + "tier connectivity check failed", + )) + } else if err.code == ERR_TIER_INVALID_CREDENTIALS.code { + Err(S3Error::with_message(S3ErrorCode::Custom(err.code.clone().into()), err.message)) + } else { + warn!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action = "add_tier", + tier_name = %tier_name_for_log, + result = "add_failed", error = ?err, "admin tier state" ); - return Err(S3Error::with_message( + Err(S3Error::with_message( S3ErrorCode::Custom("TierAddFailed".into()), - format!("tier reload failed. {err}"), - )); - } - if let Err(err) = tier_config_mgr.add(args, force).await { - return if err.code == ERR_TIER_RESERVED_NAME.code { - warn!( - event = EVENT_ADMIN_TIER_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_TIER, - action = "add_tier", - tier_name = %tier_name_for_log, - result = "reserved_name_rejected", - "admin tier state" - ); - Err(s3_error!(InvalidRequest, "Cannot use reserved tier name")) - } else if err.code == ERR_TIER_ALREADY_EXISTS.code { - Err(S3Error::with_message( - S3ErrorCode::Custom("TierNameAlreadyExist".into()), - "tier name already exists", - )) - } else if err.code == ERR_TIER_NAME_NOT_UPPERCASE.code { - Err(S3Error::with_message( - S3ErrorCode::Custom("TierNameNotUppercase".into()), - "tier name must be uppercase", - )) - } else if err.code == ERR_TIER_BACKEND_IN_USE.code { - Err(S3Error::with_message( - S3ErrorCode::Custom("TierNameBackendInUse!".into()), - "tier backend is already in use", - )) - } else if err.code == ERR_TIER_CONNECT_ERR.code { - Err(S3Error::with_message( - S3ErrorCode::Custom("TierConnectError".into()), - "tier connectivity check failed", - )) - } else if err.code == ERR_TIER_INVALID_CREDENTIALS.code { - Err(S3Error::with_message(S3ErrorCode::Custom(err.code.clone().into()), err.message)) - } else { - warn!( - event = EVENT_ADMIN_TIER_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_TIER, - action = "add_tier", - tier_name = %tier_name_for_log, - result = "add_failed", - error = ?err, - "admin tier state" - ); - Err(S3Error::with_message( - S3ErrorCode::Custom("TierAddFailed".into()), - format!("tier add failed. {err}"), - )) - }; - } - if let Err(e) = tier_config_mgr.save().await { - warn!( - event = EVENT_ADMIN_TIER_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_TIER, - action = "add_tier", - result = "save_failed", - error = ?e, - "admin tier state" - ); - return Err(S3Error::with_message(S3ErrorCode::Custom("TierAddFailed".into()), "tier save failed")); - } + format!("tier add failed. {err}"), + )) + }; } spawn_transition_tier_config_propagation("add"); @@ -473,61 +490,32 @@ impl Operation for EditTier { return Err(s3_error!(InternalError, "object store is not initialized")); }; - { - let tier_config_mgr_handle = current_tier_config_handle(); - let mut tier_config_mgr = tier_config_mgr_handle.write().await; - if let Err(err) = tier_config_mgr.reload(store).await { + let tier_config_mgr_handle = current_tier_config_handle(); + if let Err(update_err) = TierConfigMgr::edit_and_save(&tier_config_mgr_handle, store, &tier_name, creds).await { + let err = tier_mutation_error(update_err, "edit_tier", "TierEditFailed")?; + return if err.code == ERR_TIER_NOT_FOUND.code { + Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found")) + } else if err.code == ERR_TIER_MISSING_CREDENTIALS.code { + Err(S3Error::with_message( + S3ErrorCode::Custom("TierMissingCredentials".into()), + "tier credentials are required", + )) + } else { warn!( event = EVENT_ADMIN_TIER_STATE, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_TIER, action = "edit_tier", - result = "reload_failed", + tier_name = %tier_name, + result = "edit_failed", error = ?err, "admin tier state" ); - return Err(S3Error::with_message( + Err(S3Error::with_message( S3ErrorCode::Custom("TierEditFailed".into()), - format!("tier reload failed. {err}"), - )); - } - if let Err(err) = tier_config_mgr.edit(&tier_name, creds).await { - return if err.code == ERR_TIER_NOT_FOUND.code { - Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found")) - } else if err.code == ERR_TIER_MISSING_CREDENTIALS.code { - Err(S3Error::with_message( - S3ErrorCode::Custom("TierMissingCredentials".into()), - "tier credentials are required", - )) - } else { - warn!( - event = EVENT_ADMIN_TIER_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_TIER, - action = "edit_tier", - tier_name = %tier_name, - result = "edit_failed", - error = ?err, - "admin tier state" - ); - Err(S3Error::with_message( - S3ErrorCode::Custom("TierEditFailed".into()), - format!("tier edit failed. {err}"), - )) - }; - } - if let Err(e) = tier_config_mgr.save().await { - warn!( - event = EVENT_ADMIN_TIER_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_TIER, - action = "edit_tier", - result = "save_failed", - error = ?e, - "admin tier state" - ); - return Err(S3Error::with_message(S3ErrorCode::Custom("TierEditFailed".into()), "tier save failed")); - } + format!("tier edit failed. {err}"), + )) + }; } spawn_transition_tier_config_propagation("edit"); @@ -643,62 +631,34 @@ impl Operation for RemoveTier { return Err(s3_error!(InternalError, "object store is not initialized")); }; - { - let tier_config_mgr_handle = current_tier_config_handle(); - let mut tier_config_mgr = tier_config_mgr_handle.write().await; - if let Err(err) = tier_config_mgr.reload(store).await { + let tier_config_mgr_handle = current_tier_config_handle(); + if let Err(update_err) = TierConfigMgr::remove_and_save(&tier_config_mgr_handle, store, &tier_name, force).await { + let err = tier_mutation_error(update_err, "remove_tier", "TierRemoveFailed")?; + return if err.code == ERR_TIER_NOT_FOUND.code { + Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found")) + } else if err.code == ERR_TIER_BACKEND_NOT_EMPTY.code { + Err(S3Error::with_message( + S3ErrorCode::Custom("TierNameBackendInUse".into()), + "tier backend is not empty", + )) + } else if let Some(response) = tier_backend_in_use_response(&err) { + Err(response) + } else { warn!( event = EVENT_ADMIN_TIER_STATE, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_TIER, action = "remove_tier", - result = "reload_failed", + tier_name = %tier_name, + result = "remove_failed", error = ?err, "admin tier state" ); - return Err(S3Error::with_message( + Err(S3Error::with_message( S3ErrorCode::Custom("TierRemoveFailed".into()), - format!("tier reload failed. {err}"), - )); - } - if let Err(err) = tier_config_mgr.remove(&tier_name, force).await { - return if err.code == ERR_TIER_NOT_FOUND.code { - Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found")) - } else if err.code == ERR_TIER_BACKEND_NOT_EMPTY.code { - Err(S3Error::with_message( - S3ErrorCode::Custom("TierNameBackendInUse".into()), - "tier backend is not empty", - )) - } else { - warn!( - event = EVENT_ADMIN_TIER_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_TIER, - action = "remove_tier", - tier_name = %tier_name, - result = "remove_failed", - error = ?err, - "admin tier state" - ); - Err(S3Error::with_message( - S3ErrorCode::Custom("TierRemoveFailed".into()), - format!("tier remove failed. {err}"), - )) - }; - } - - if let Err(e) = tier_config_mgr.save().await { - warn!( - event = EVENT_ADMIN_TIER_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_TIER, - action = "remove_tier", - result = "save_failed", - error = ?e, - "admin tier state" - ); - return Err(S3Error::with_message(S3ErrorCode::Custom("TierRemoveFailed".into()), "tier save failed")); - } + format!("tier remove failed. {err}"), + )) + }; } spawn_transition_tier_config_propagation("remove"); @@ -733,8 +693,9 @@ impl Operation for VerifyTier { let tier = resolve_tier_name(&req.uri, ¶ms)?; let tier_config_mgr_handle = current_tier_config_handle(); - let mut tier_config_mgr = tier_config_mgr_handle.write().await; - tier_config_mgr.verify(&tier).await.map_err(map_tier_verify_error)?; + TierConfigMgr::verify_without_manager_lock(&tier_config_mgr_handle, &tier) + .await + .map_err(map_tier_verify_error)?; let mut header = HeaderMap::new(); header.insert(CONTENT_TYPE, "application/json".parse().expect("valid header value")); @@ -921,10 +882,42 @@ impl Operation for ClearTier { return Err(s3_error!(InvalidRequest, "invalid clear-tier confirmation token")); }; + let Some(store) = object_store_from_extensions(&req.extensions) else { + return Err(s3_error!(InternalError, "object store is not initialized")); + }; + let tier_config_mgr_handle = current_tier_config_handle(); - let mut tier_config_mgr = tier_config_mgr_handle.write().await; - //tier_config_mgr.reload(api); - if let Err(err) = tier_config_mgr.clear_tier(force).await { + if let Err(update_err) = TierConfigMgr::clear_and_save(&tier_config_mgr_handle, store, force).await { + let err = match update_err { + TierConfigUpdateError::Load(err) => { + warn!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action = "clear_tier", + result = "reload_failed", + error = ?err, + "admin tier state" + ); + return Err(S3Error::with_message( + S3ErrorCode::Custom("TierClearFailed".into()), + format!("tier clear failed. {err}"), + )); + } + TierConfigUpdateError::Save(err) => { + warn!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action = "clear_tier", + result = "save_failed", + error = ?err, + "admin tier state" + ); + return Err(S3Error::with_message(S3ErrorCode::Custom("TierEditFailed".into()), "tier save failed")); + } + TierConfigUpdateError::Mutation(err) | TierConfigUpdateError::Publish(err) => err, + }; warn!( event = EVENT_ADMIN_TIER_STATE, component = LOG_COMPONENT_ADMIN, @@ -934,22 +927,7 @@ impl Operation for ClearTier { error = ?err, "admin tier state" ); - return Err(S3Error::with_message( - S3ErrorCode::Custom("TierClearFailed".into()), - format!("tier clear failed. {err}"), - )); - } - if let Err(e) = tier_config_mgr.save().await { - warn!( - event = EVENT_ADMIN_TIER_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_TIER, - action = "clear_tier", - result = "save_failed", - error = ?e, - "admin tier state" - ); - return Err(S3Error::with_message(S3ErrorCode::Custom("TierEditFailed".into()), "tier save failed")); + return Err(clear_tier_error_response(&err)); } let mut header = HeaderMap::new(); @@ -1106,6 +1084,49 @@ mod tests { assert_eq!(mapped.message(), Some("tier verification failed. backend unavailable")); } + #[test] + fn tier_mutation_error_preserves_reload_and_save_responses() { + let reload = tier_mutation_error( + TierConfigUpdateError::Load(std::io::Error::other("read failed")), + "add_tier", + "TierAddFailed", + ) + .expect_err("reload error should map to an S3 response"); + assert_eq!(reload.code(), &S3ErrorCode::Custom("TierAddFailed".into())); + assert_eq!(reload.message(), Some("tier reload failed. read failed")); + + let save = tier_mutation_error( + TierConfigUpdateError::Save(std::io::Error::other("conditional write failed")), + "edit_tier", + "TierEditFailed", + ) + .expect_err("save error should map to an S3 response"); + assert_eq!(save.code(), &S3ErrorCode::Custom("TierEditFailed".into())); + assert_eq!(save.message(), Some("tier save failed")); + } + + #[test] + fn clear_preserves_legacy_backend_not_empty_error_code() { + let response = clear_tier_error_response(&ERR_TIER_BACKEND_NOT_EMPTY); + assert_eq!(response.code(), &S3ErrorCode::Custom("TierClearFailed".into())); + assert!( + response + .message() + .is_some_and(|message| message.starts_with("tier clear failed.")) + ); + + let response = clear_tier_error_response(&ERR_TIER_BACKEND_IN_USE); + assert_eq!(response.code(), &S3ErrorCode::Custom("TierClearFailed".into())); + assert!( + response + .message() + .is_some_and(|message| message.starts_with("tier clear failed.")) + ); + + assert!(tier_backend_in_use_response(&ERR_TIER_BACKEND_NOT_EMPTY).is_none()); + assert!(tier_backend_in_use_response(&ERR_TIER_NOT_FOUND).is_none()); + } + #[test] fn parse_clear_tier_query_rejects_unknown_duplicate_and_invalid_force() { for raw in [ diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index 2de3f05bf..0027cdbd5 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -110,6 +110,7 @@ pub(crate) type Result = core::result::Result; pub(crate) type TierConfig = ecstore_tier::tier_config::TierConfig; pub(crate) type TierCreds = ecstore_tier::tier_admin::TierCreds; pub(crate) type TierType = ecstore_tier::tier_config::TierType; +pub(crate) type TierConfigUpdateError = crate::storage::storage_api::TierConfigUpdateError; pub(crate) mod runtime_sources { pub(crate) type DailyAllTierStats = super::DailyAllTierStats; @@ -575,6 +576,6 @@ pub(crate) mod tier { pub(crate) use super::{ AdminError, DailyAllTierStats, ERR_TIER_ALREADY_EXISTS, ERR_TIER_BACKEND_IN_USE, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_MISSING_CREDENTIALS, ERR_TIER_NAME_NOT_UPPERCASE, - ERR_TIER_NOT_FOUND, ERR_TIER_RESERVED_NAME, TierConfig, TierCreds, TierType, + ERR_TIER_NOT_FOUND, ERR_TIER_RESERVED_NAME, TierConfig, TierConfigUpdateError, TierCreds, TierType, }; } diff --git a/rustfs/src/app/lifecycle_transition_api_test.rs b/rustfs/src/app/lifecycle_transition_api_test.rs index c38e0e0b6..3cad45679 100644 --- a/rustfs/src/app/lifecycle_transition_api_test.rs +++ b/rustfs/src/app/lifecycle_transition_api_test.rs @@ -866,9 +866,6 @@ async fn get_transitioned_object_uses_remote_codec_fallback_path() { .collect(); create_test_bucket(&ecstore, bucket.as_str()).await; - set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name) - .await - .expect("Failed to set lifecycle configuration"); let uploaded = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await; let transition_opts = ObjectOptions { diff --git a/rustfs/src/app/mod.rs b/rustfs/src/app/mod.rs index c0ee9f0fe..7d3e05ab7 100644 --- a/rustfs/src/app/mod.rs +++ b/rustfs/src/app/mod.rs @@ -33,7 +33,7 @@ mod data_usage_snapshot_gating_test; #[cfg(test)] mod delete_objects_stat_gating_test; #[cfg(test)] -mod gating_test_env; +pub(crate) mod gating_test_env; #[cfg(test)] mod lifecycle_transition_api_test; #[cfg(test)] diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index de6ccd611..3d3537595 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -757,10 +757,11 @@ async fn enqueue_transitioned_delete_cleanup( &existing.transitioned_object, ) }; - let Some(je) = je else { + let Some(mut je) = je else { return Ok(()); }; + tier_delete_journal::record_tier_delete_journal_backend_identity(&mut je, &existing.user_defined)?; tier_delete_journal::persist_tier_delete_journal_entry(store, &je).await?; let expiry_state = current_expiry_state_handle(); @@ -9250,6 +9251,100 @@ mod tests { (store, context) } + #[tokio::test] + #[serial_test::serial] + async fn transitioned_delete_cleanup_persists_identity_bound_and_legacy_journals() { + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + if current_app_context().is_none() { + crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await; + } + let identity = [11_u8; 32]; + let mut metadata = HashMap::new(); + rustfs_utils::http::metadata_compat::insert_str( + &mut metadata, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + rustfs_utils::crypto::hex(identity), + ); + let mut current = ObjectInfo { + user_defined: Arc::new(metadata), + ..Default::default() + }; + current.transitioned_object.status = lifecycle::TRANSITION_COMPLETE.to_string(); + current.transitioned_object.tier = "WARM".to_string(); + current.transitioned_object.name = "remote/identity-bound".to_string(); + current.transitioned_object.version_id = "remote-version".to_string(); + + let journal_name = |remote_object: &str, backend_identity: Option<[u8; 32]>| { + use sha2::{Digest, Sha256}; + + let mut hasher = Sha256::new(); + hasher.update(b"WARM"); + hasher.update([0]); + hasher.update(remote_object.as_bytes()); + hasher.update([0]); + hasher.update(b"remote-version"); + if let Some(backend_identity) = backend_identity { + hasher.update([0]); + hasher.update(backend_identity); + } + format!("ilm/tier-delete-journal/{}.json", rustfs_utils::crypto::hex(hasher.finalize().as_slice())) + }; + + enqueue_transitioned_delete_cleanup(store.clone(), "bucket", "identity-bound", &ObjectOptions::default(), Some(¤t)) + .await + .expect("normal transitioned delete should persist an identity-bound journal"); + let mut identity_bound = store + .get_object_reader( + ".rustfs.sys", + &journal_name("remote/identity-bound", Some(identity)), + None, + http::HeaderMap::new(), + &ObjectOptions::default(), + ) + .await + .expect("identity-bound journal should be readable"); + let mut identity_bound_data = Vec::new(); + tokio::io::AsyncReadExt::read_to_end(&mut identity_bound.stream, &mut identity_bound_data) + .await + .expect("identity-bound journal body should be readable"); + let identity_bound: serde_json::Value = + serde_json::from_slice(&identity_bound_data).expect("identity-bound journal should decode as JSON"); + assert_eq!(identity_bound["version"], serde_json::json!(2)); + assert_eq!(identity_bound["backend_identity"], serde_json::json!(identity)); + + current.user_defined = Arc::new(HashMap::new()); + current.transitioned_object.name = "remote/legacy".to_string(); + enqueue_transitioned_delete_cleanup( + store.clone(), + "bucket", + "legacy", + &ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + Some(¤t), + ) + .await + .expect("legacy force-delete cleanup should persist a fail-closed v1 journal"); + let mut legacy = store + .get_object_reader( + ".rustfs.sys", + &journal_name("remote/legacy", None), + None, + http::HeaderMap::new(), + &ObjectOptions::default(), + ) + .await + .expect("legacy journal should be readable"); + let mut legacy_data = Vec::new(); + tokio::io::AsyncReadExt::read_to_end(&mut legacy.stream, &mut legacy_data) + .await + .expect("legacy journal body should be readable"); + let legacy: serde_json::Value = serde_json::from_slice(&legacy_data).expect("legacy journal should decode as JSON"); + assert_eq!(legacy["version"], serde_json::json!(1)); + assert_eq!(legacy["backend_identity"], serde_json::Value::Null); + } + async fn put_real_cold_fill_object(store: &Arc, bucket: &str, object: &str, body: &[u8]) -> ObjectInfo { let mut reader = PutObjReader::from_vec(body.to_vec()); store diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index 8f3bb651b..d3b0b426d 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -390,6 +390,13 @@ pub(crate) mod bucket { pub(crate) mod tier_delete_journal { use std::sync::Arc; + pub(crate) fn record_tier_delete_journal_backend_identity( + je: &mut super::tier_sweeper::Jentry, + metadata: &std::collections::HashMap, + ) -> std::io::Result<()> { + crate::storage::storage_api::ecstore_bucket::lifecycle::tier_delete_journal::record_tier_delete_journal_backend_identity(je, metadata) + } + pub(crate) async fn persist_tier_delete_journal_entry( api: Arc, je: &super::tier_sweeper::Jentry, diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index d4a04959f..b23da2ec8 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -492,7 +492,7 @@ pub(crate) mod ecstore_storage { } pub(crate) mod ecstore_tier { - pub(crate) use rustfs_ecstore::api::tier::tier::TierConfigMgr; + pub(crate) use rustfs_ecstore::api::tier::tier::{TierConfigMgr, TierConfigUpdateError}; pub(crate) use rustfs_ecstore::api::tier::{tier, tier_admin, tier_config, tier_handlers}; // Shared lifecycle/tier test utilities behind ecstore's `test-util` feature // (rustfs/backlog#1148 ilm-6). Only linked into test builds. @@ -577,6 +577,7 @@ pub(crate) type ReplicationStatusType = ecstore_bucket::replication::Replication pub(crate) type ReplicationStats = StorageReplicationStatsHandle; pub(crate) type StorageError = ecstore_error::StorageError; pub(crate) type TierConfigMgr = ecstore_tier::TierConfigMgr; +pub(crate) type TierConfigUpdateError = ecstore_tier::TierConfigUpdateError; pub(crate) use ecstore_disk::validate_batch_read_version_item_count; pub(crate) type TransitionState = ecstore_bucket::lifecycle::bucket_lifecycle_ops::TransitionState; pub(crate) type Error = ecstore_error::Error; @@ -1438,7 +1439,8 @@ pub(crate) fn topology_snapshot_from_endpoint_pools_with_capabilities( } pub(crate) async fn reload_transition_tier_config(api: Arc) -> std::io::Result<()> { - ecstore_runtime::global_tier_config_mgr().write().await.reload(api).await + let handle = api.tier_config_mgr(); + TierConfigMgr::reload_handle(&handle, api).await } pub(crate) async fn all_local_disk_path() -> Vec { diff --git a/scripts/validate_object_data_cache_cold_stampede.sh b/scripts/validate_object_data_cache_cold_stampede.sh index 9cd51acb3..90ed379a5 100755 --- a/scripts/validate_object_data_cache_cold_stampede.sh +++ b/scripts/validate_object_data_cache_cold_stampede.sh @@ -16,8 +16,8 @@ readonly REQUESTS=2000 readonly KEY_MATRIX="1 4 32" readonly REQUIRED_MEMORY_BYTES=$((28 * 1024 * 1024 * 1024)) readonly OBJECT_SIZE=$((384 * 1024 * 1024)) -readonly READER_BYTES_QUERY='sum(rustfs_io_get_object_reader_bytes_total) or vector(0)' -readonly FOLLOWER_PERMIT_QUERY='sum(rustfs_object_data_cache_cold_fill_follower_disk_permits)' +readonly READER_BYTES_METRIC='rustfs_io_get_object_reader_bytes_total' +readonly FOLLOWER_PERMIT_METRIC='rustfs_object_data_cache_cold_fill_follower_disk_permits' RUN=0 SELF_TEST=0 @@ -35,6 +35,7 @@ RESET_COMMAND="" CACHE_OFF_COMMAND="" CACHE_ON_COMMAND="" SERVER_PID="" +SERVER_PID_COMMAND="" CGROUP_PATH="" SAMPLE_INTERVAL="0.25" METRICS_SETTLE_SECONDS="5" @@ -49,7 +50,8 @@ Usage: --expected-sha256 HEX --prometheus-query-url URL \ --prometheus-scrape-seconds SECONDS \ --reset-command COMMAND --cache-off-command COMMAND \ - --cache-on-command COMMAND --server-pid PID [--cgroup-path PATH] [options] + --cache-on-command COMMAND \ + (--server-pid PID | --server-pid-command COMMAND) [--cgroup-path PATH] [options] Required run contract: * N is fixed at 2000; K is fixed at 1, 4, and 32; each K runs >= 3 rounds. @@ -59,17 +61,17 @@ Required run contract: * The RustFS process must be isolated from unrelated traffic and run in a dedicated cgroup v2 with memory.max exactly 28 GiB. * The built-in first-party reader-byte counter and follower-permit gauge must - each return exactly one Prometheus vector sample. - * Cache switch commands must return only after the mode is effective and must - keep the same RustFS PID inside the dedicated cgroup. + each return one value vector and one raw-scrape timestamp vector. + * Cache switch commands must return only after the mode is effective. A dynamic + PID resolver may observe a restarted RustFS process, but every process must + remain inside the same dedicated cgroup. * Every response must match SHA-256, ETag, Content-Length, and its per-key stable header contract. Date, x-amz-id-2, x-amz-request-id, x-minio-request-id, x-rustfs-request-id, Connection, Keep-Alive, and Transfer-Encoding are explicitly excluded as volatile/transport headers. -Built-in PromQL: - reader bytes: sum(rustfs_io_get_object_reader_bytes_total) or vector(0) - follower permits: sum(rustfs_object_data_cache_cold_fill_follower_disk_permits) +Built-in PromQL observes each metric value together with max(timestamp(metric)); +the HTTP API evaluation timestamp is never treated as a scrape timestamp. Options: --self-test Validate the follower sample append/check chain locally @@ -79,6 +81,7 @@ Options: --prometheus-scrape-seconds N Configured scrape interval; required --metrics-settle-seconds N Wait for final Prometheus scrape (default: 5) --load-timeout-seconds N Per-round load timeout (default: 1800) + --server-pid-command COMMAND Resolve the live RustFS PID after every operator command --out-dir PATH Artifact directory (default: temporary) --strict Missing prerequisite is FAIL instead of SKIP -h, --help Show this help @@ -135,15 +138,76 @@ run_operator_command() { bash -c "$command" } +parse_server_pid() { + local output=$1 + [[ $output =~ ^[[:space:]]*([0-9]+)[[:space:]]*$ ]] || return 1 + printf '%s\n' "${BASH_REMATCH[1]}" +} + +resolve_server_pid() { + local output + if [[ -n $SERVER_PID_COMMAND ]]; then + output=$(env -u AWS_ACCESS_KEY_ID -u AWS_SECRET_ACCESS_KEY -u AWS_SESSION_TOKEN bash -c "$SERVER_PID_COMMAND") || return 1 + parse_server_pid "$output" + else + parse_server_pid "$SERVER_PID" + fi +} + +parse_prometheus_observation() { + python3 -c ' +import json, math, sys +doc = json.load(sys.stdin) +if doc.get("status") != "success": + raise SystemExit("Prometheus query was not successful") +result = doc.get("data", {}).get("result", []) +if len(result) != 2: + raise SystemExit(f"expected value and sample_timestamp vectors, got {len(result)}") +observations = {} +for item in result: + kind = item.get("metric", {}).get("__rustfs_probe_kind") + pair = item.get("value") + if kind not in ("value", "sample_timestamp") or not isinstance(pair, list) or len(pair) != 2: + raise SystemExit("invalid Prometheus observation shape") + evaluation_timestamp = float(pair[0]) + observed_value = float(pair[1]) + if not math.isfinite(evaluation_timestamp) or not math.isfinite(observed_value): + raise SystemExit("Prometheus observation is not finite") + if kind in observations: + raise SystemExit(f"duplicate Prometheus observation kind: {kind}") + observations[kind] = observed_value +if set(observations) != {"value", "sample_timestamp"}: + raise SystemExit("Prometheus observation is incomplete") +if observations["value"] < 0 or observations["sample_timestamp"] <= 0: + raise SystemExit("Prometheus observation is outside the accepted range") +print(format(observations["sample_timestamp"], ".17g"), format(observations["value"], ".17g")) +' +} + follower_samples_self_test() ( command -v python3 >/dev/null 2>&1 || fail "python3 is required for --self-test" - local temp_dir samples_file ready_file result sample + local temp_dir samples_file ready_file result sample stale_response fresh_response temp_dir=$(mktemp -d "${TMPDIR:-/tmp}/rustfs-follower-samples.XXXXXX") trap 'rm -rf -- "$temp_dir"' EXIT samples_file="$temp_dir/follower.samples" ready_file="$temp_dir/ready" printf '999\n' >"$ready_file" + stale_response='{"status":"success","data":{"result":[{"metric":{"__rustfs_probe_kind":"value"},"value":[2000,"0"]},{"metric":{"__rustfs_probe_kind":"sample_timestamp"},"value":[2000,"900"]}]}}' + sample=$(parse_prometheus_observation <<<"$stale_response") || fail "self-test could not parse stale Prometheus observation" + [[ $sample == '900 0' ]] || fail "Prometheus evaluation timestamp was mistaken for scrape timestamp: $sample" + fresh_response='{"status":"success","data":{"result":[{"metric":{"__rustfs_probe_kind":"sample_timestamp"},"value":[2000,"1000"]},{"metric":{"__rustfs_probe_kind":"value"},"value":[2000,"0"]}]}}' + sample=$(parse_prometheus_observation <<<"$fresh_response") || fail "self-test could not parse fresh Prometheus observation" + [[ $sample == '1000 0' ]] || fail "Prometheus scrape timestamp was not preserved: $sample" + + [[ $(parse_server_pid '101') == 101 ]] || fail "single-line PID output was not accepted" + if parse_server_pid '101 102' >/dev/null 2>&1; then + fail "multi-value PID output unexpectedly passed" + fi + if parse_server_pid $'101\n102' >/dev/null 2>&1; then + fail "multi-line PID output unexpectedly passed" + fi + : >"$samples_file" for sample in '1000 0' '1000 0' '1000 0'; do append_follower_sample "$samples_file" "$sample" || fail "self-test could not append duplicate follower sample" @@ -197,6 +261,7 @@ while (($#)); do --cache-off-command) need_value "$@"; CACHE_OFF_COMMAND=$2; shift ;; --cache-on-command) need_value "$@"; CACHE_ON_COMMAND=$2; shift ;; --server-pid) need_value "$@"; SERVER_PID=$2; shift ;; + --server-pid-command) need_value "$@"; SERVER_PID_COMMAND=$2; shift ;; --cgroup-path) need_value "$@"; CGROUP_PATH=$2; shift ;; --rounds) need_value "$@"; ROUNDS=$2; shift ;; --sample-interval) need_value "$@"; SAMPLE_INTERVAL=$2; shift ;; @@ -251,7 +316,13 @@ import sys raise SystemExit(0 if float(sys.argv[1]) >= float(sys.argv[2]) else 1) PY -[[ $SERVER_PID =~ ^[0-9]+$ ]] || skip_or_fail "--server-pid is required to bind OOM evidence to RustFS" +if [[ -n $SERVER_PID && -n $SERVER_PID_COMMAND ]]; then + fail "--server-pid and --server-pid-command are mutually exclusive" +fi +if [[ -z $SERVER_PID && -z $SERVER_PID_COMMAND ]]; then + skip_or_fail "--server-pid or --server-pid-command is required to bind OOM evidence to RustFS" +fi +SERVER_PID=$(resolve_server_pid) || skip_or_fail "could not resolve exactly one RustFS PID" [[ -r /proc/$SERVER_PID/cgroup ]] || skip_or_fail "cannot read /proc/$SERVER_PID/cgroup" if [[ -z $CGROUP_PATH ]]; then cgroup_relative=$(awk -F: '$1 == "0" { print $3; exit }' "/proc/$SERVER_PID/cgroup") @@ -272,6 +343,18 @@ server_in_cgroup() { } server_in_cgroup || skip_or_fail "RustFS PID $SERVER_PID is not in $CGROUP_PATH" +refresh_server_pid() { + local previous_pid=$SERVER_PID + local resolved + resolved=$(resolve_server_pid) || fail "could not resolve exactly one live RustFS PID" + [[ -r /proc/$resolved/cgroup ]] || fail "cannot read /proc/$resolved/cgroup" + SERVER_PID=$resolved + server_in_cgroup || fail "RustFS PID $SERVER_PID is not in $CGROUP_PATH" + if [[ $previous_pid != "$SERVER_PID" ]] && grep -Fxq -- "$previous_pid" "$CGROUP_PATH/cgroup.procs"; then + fail "replaced RustFS PID $previous_pid is still running in $CGROUP_PATH" + fi +} + nofile_limit=$(ulimit -n) [[ $nofile_limit =~ ^[0-9]+$ ]] || skip_or_fail "unable to determine the open-file limit" ((nofile_limit >= REQUESTS + 256)) || skip_or_fail "open-file limit must be at least $((REQUESTS + 256)), got $nofile_limit" @@ -294,31 +377,39 @@ cleanup_load() { trap cleanup_load EXIT prometheus_sample() { - local query=$1 + local metric=$1 + local query local response + query="label_replace(sum($metric), \"__rustfs_probe_kind\", \"value\", \"__name__\", \".*\") or label_replace(max(timestamp($metric)), \"__rustfs_probe_kind\", \"sample_timestamp\", \"__name__\", \".*\")" response=$(curl --fail --silent --show-error --get \ --data-urlencode "query=$query" "$PROMETHEUS_QUERY_URL") || return 1 - python3 -c ' -import json, math, sys -doc = json.load(sys.stdin) -if doc.get("status") != "success": - raise SystemExit("Prometheus query was not successful") -result = doc.get("data", {}).get("result", []) -if len(result) != 1 or "value" not in result[0]: - raise SystemExit(f"expected exactly one vector sample, got {len(result)}") -timestamp = float(result[0]["value"][0]) -value = float(result[0]["value"][1]) -if not math.isfinite(timestamp) or not math.isfinite(value) or value < 0: - raise SystemExit(f"invalid metric value: {value}") -print(format(timestamp, ".17g"), format(value, ".17g")) -' <<<"$response" + parse_prometheus_observation <<<"$response" } -prometheus_value() { +prometheus_value_after() { + local metric=$1 + local minimum_epoch=$2 local timestamp value - read -r timestamp value < <(prometheus_sample "$1") || return 1 - [[ -n $timestamp && -n $value ]] || return 1 - printf '%s\n' "$value" + local attempts + attempts=$(python3 - "$METRICS_SETTLE_SECONDS" "$PROMETHEUS_SCRAPE_SECONDS" "$SAMPLE_INTERVAL" <<'PY' +import math, sys +settle, scrape, poll = map(float, sys.argv[1:]) +print(max(1, math.ceil((settle + 2 * scrape) / poll))) +PY + ) || return 1 + for ((attempt = 0; attempt < attempts; attempt++)); do + if read -r timestamp value < <(prometheus_sample "$metric") \ + && python3 - "$timestamp" "$minimum_epoch" <<'PY' +import sys +raise SystemExit(0 if float(sys.argv[1]) >= float(sys.argv[2]) else 1) +PY + then + printf '%s\n' "$value" + return 0 + fi + sleep "$SAMPLE_INTERVAL" + done + return 1 } cgroup_event() { @@ -565,6 +656,9 @@ switch_cache_mode() { local key_count=$4 run_operator_command "$mode" "$round" "$key_count" "$command" \ || fail "cache-$mode command failed for round=$round K=$key_count" + if [[ -n $SERVER_PID_COMMAND ]]; then + refresh_server_pid + fi server_in_cgroup || fail "cache-$mode command moved RustFS PID $SERVER_PID out of $CGROUP_PATH" } @@ -574,6 +668,9 @@ reset_cache() { local key_count=$3 run_operator_command "$mode" "$round" "$key_count" "$RESET_COMMAND" \ || fail "reset command failed for mode=$mode round=$round K=$key_count" + if [[ -n $SERVER_PID_COMMAND ]]; then + refresh_server_pid + fi server_in_cgroup || fail "reset command moved RustFS PID $SERVER_PID out of $CGROUP_PATH" } @@ -594,14 +691,15 @@ for ((round = 1; round <= ROUNDS; round++)); do printf 'INFO: K=%d round=%d: switching cache off for one request per key baseline\n' "$key_count" "$round" switch_cache_mode off "$CACHE_OFF_COMMAND" "$round" "$key_count" reset_cache off "$round" "$key_count" - baseline_reader_before=$(prometheus_value "$READER_BYTES_QUERY") || fail "reader query failed before cache-off K=$key_count round=$round" + baseline_start_epoch=$(python3 -c 'import time; print(format(time.time(), ".17g"))') + baseline_reader_before=$(prometheus_value_after "$READER_BYTES_METRIC" "$baseline_start_epoch") || fail "fresh reader query failed before cache-off K=$key_count round=$round" baseline_oom_before=$(cgroup_event oom) || fail "cannot read oom before baseline" baseline_oom_kill_before=$(cgroup_event oom_kill) || fail "cannot read oom_kill before baseline" baseline_peak_before=$(cat "$CGROUP_PATH/memory.peak" 2>/dev/null || printf 'NA') run_load "$key_count" "$key_count" "$baseline_summary" "$baseline_ready" - sleep "$METRICS_SETTLE_SECONDS" - baseline_reader_after=$(prometheus_value "$READER_BYTES_QUERY") || fail "reader query failed after cache-off K=$key_count round=$round" + baseline_finished_epoch=$(python3 -c 'import time; print(format(time.time(), ".17g"))') + baseline_reader_after=$(prometheus_value_after "$READER_BYTES_METRIC" "$baseline_finished_epoch") || fail "fresh reader query failed after cache-off K=$key_count round=$round" baseline_reader_delta=$(numeric_positive_delta "$baseline_reader_before" "$baseline_reader_after") \ || fail "cache-off reader baseline is invalid for K=$key_count round=$round" baseline_oom_after=$(cgroup_event oom) || fail "cannot read oom after baseline" @@ -629,7 +727,8 @@ for ((round = 1; round <= ROUNDS; round++)); do switch_cache_mode on "$CACHE_ON_COMMAND" "$round" "$key_count" reset_cache on "$round" "$key_count" - reader_before=$(prometheus_value "$READER_BYTES_QUERY") || fail "reader query failed before K=$key_count round=$round" + round_start_epoch=$(python3 -c 'import time; print(format(time.time(), ".17g"))') + reader_before=$(prometheus_value_after "$READER_BYTES_METRIC" "$round_start_epoch") || fail "fresh reader query failed before K=$key_count round=$round" oom_before=$(cgroup_event oom) || fail "cannot read oom before round" oom_kill_before=$(cgroup_event oom_kill) || fail "cannot read oom_kill before round" memory_peak_before=$(cat "$CGROUP_PATH/memory.peak" 2>/dev/null || printf 'NA') @@ -639,7 +738,7 @@ for ((round = 1; round <= ROUNDS; round++)); do LOAD_PID=$load_pid while kill -0 "$load_pid" 2>/dev/null; do if [[ -e $ready_file ]]; then - if sample=$(prometheus_sample "$FOLLOWER_PERMIT_QUERY" 2>/dev/null); then + if sample=$(prometheus_sample "$FOLLOWER_PERMIT_METRIC" 2>/dev/null); then append_follower_sample "$follower_file" "$sample" || fail "invalid follower sample for K=$key_count round=$round" fi fi @@ -648,8 +747,8 @@ for ((round = 1; round <= ROUNDS; round++)); do wait "$load_pid" || fail "load or SHA-256 validation failed for K=$key_count round=$round (see $summary_file)" LOAD_PID="" - sleep "$METRICS_SETTLE_SECONDS" - reader_after=$(prometheus_value "$READER_BYTES_QUERY") || fail "reader query failed after K=$key_count round=$round" + round_finished_epoch=$(python3 -c 'import time; print(format(time.time(), ".17g"))') + reader_after=$(prometheus_value_after "$READER_BYTES_METRIC" "$round_finished_epoch") || fail "fresh reader query failed after K=$key_count round=$round" expected_reader_bytes=$(((baseline_reader_delta * 90) / 100)) reader_limit=$(((baseline_reader_delta * 110 + 99) / 100)) reader_delta=$(numeric_delta_check "$reader_before" "$reader_after" \