diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
index 326b798ee..abdca65c1 100644
--- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
@@ -4994,6 +4994,9 @@ pub async fn apply_expiry_on_transitioned_object(
src: &LcEventSrc,
bucket_incarnation_id: Uuid,
) -> bool {
+ if lc_event.action.delete_all() {
+ return apply_expiry_on_non_transitioned_objects(api, oi, lc_event, src, bucket_incarnation_id).await;
+ }
let time_ilm = Metrics::time_ilm(lc_event.action);
if let Err(_err) = expire_transitioned_object(api, oi, lc_event, src, bucket_incarnation_id).await {
return false;
@@ -5047,13 +5050,24 @@ pub async fn apply_expiry_on_non_transitioned_objects(
if lc_event.action.delete_all() {
opts.delete_prefix = true;
opts.delete_prefix_object = true;
+ opts.lifecycle_delete_all = Some(crate::object_api::LifecycleDeleteAllRequest {
+ version_id: oi.version_id.filter(|version_id| !version_id.is_nil()),
+ delete_marker: oi.delete_marker,
+ action: lc_event.action,
+ rule_id: lc_event.rule_id.clone(),
+ phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
+ });
+ opts.ensure_lifecycle_delete_all_journal();
}
let time_ilm = Metrics::time_ilm(lc_event.action);
//debug!("lc_event.action: {:?}", lc_event.action);
debug!("expiry_on_non_transitioned_objects opts: {:?}", opts);
- let mut dobj = match api.delete_object(&oi.bucket, &encode_dir_object(&oi.name), opts).await {
+ let mut dobj = match api
+ .delete_object_with_tier_delete_journal(&oi.bucket, &encode_dir_object(&oi.name), opts)
+ .await
+ {
Ok(dobj) => dobj,
Err(e) => {
error!(
@@ -5283,7 +5297,7 @@ mod tests {
};
use crate::bucket::lifecycle::tier_last_day_stats::LastDayTierStats;
use crate::bucket::lifecycle::tier_sweeper::Jentry;
- use crate::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
+ use crate::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_VERSIONING_CONFIG};
use crate::bucket::metadata_sys;
#[cfg(feature = "test-util")]
use crate::client::transition_api::ReaderImpl;
@@ -5304,6 +5318,7 @@ mod tests {
use crate::storage_api_contracts::{
bucket::{BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
lifecycle::ExpirationOptions,
+ list::ListOperations as _,
multipart::MultipartOperations as _,
object::{ObjectIO as _, ObjectOperations as _},
};
@@ -10917,6 +10932,199 @@ mod tests {
);
}
+ #[tokio::test]
+ #[serial]
+ async fn queued_delete_all_rechecks_a_same_id_rule_moved_into_the_future() {
+ let (_disk_paths, ecstore) = setup_test_env().await;
+ let bucket = format!("stale-delete-all-rule-{}", Uuid::new_v4().simple());
+ let object = "object";
+ create_test_bucket(&ecstore, &bucket).await;
+ metadata_sys::update(
+ &bucket,
+ BUCKET_VERSIONING_CONFIG,
+ b"Enabled".to_vec(),
+ )
+ .await
+ .expect("bucket versioning should be enabled");
+ let lifecycle_xml = |days| {
+ format!(
+ r#"
+
+ delete-marker-history
+ Enabled
+
+ {days}
+
+"#
+ )
+ };
+ metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml(1).into_bytes())
+ .await
+ .expect("initial lifecycle rule should be stored");
+
+ let old_time = OffsetDateTime::now_utc() - time::Duration::days(3);
+ let mut reader = PutObjReader::from_vec(b"old version".to_vec());
+ ecstore
+ .put_object(
+ &bucket,
+ object,
+ &mut reader,
+ &ObjectOptions {
+ versioned: true,
+ mod_time: Some(old_time - time::Duration::hours(1)),
+ ..Default::default()
+ },
+ )
+ .await
+ .expect("old version should be stored");
+ let marker = ecstore
+ .delete_object(
+ &bucket,
+ object,
+ ObjectOptions {
+ versioned: true,
+ mod_time: Some(old_time),
+ ..Default::default()
+ },
+ )
+ .await
+ .expect("delete marker should be created");
+
+ let queued_event = crate::bucket::lifecycle::lifecycle::Event {
+ action: IlmAction::DelMarkerDeleteAllVersionsAction,
+ rule_id: "delete-marker-history".to_string(),
+ ..Default::default()
+ };
+ metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml(30).into_bytes())
+ .await
+ .expect("updated lifecycle rule should be stored");
+ let incarnation = ecstore
+ .bucket_incarnation_id_from_disk(&bucket)
+ .await
+ .expect("bucket incarnation should be available");
+
+ let deleted = super::apply_expiry_on_non_transitioned_objects(
+ ecstore.clone(),
+ &marker,
+ &queued_event,
+ &LcEventSrc::Scanner,
+ incarnation,
+ )
+ .await;
+ assert!(!deleted, "the stale queued rule must be rejected");
+ let versions = ecstore
+ .clone()
+ .list_object_versions(&bucket, object, None, None, None, 10)
+ .await
+ .expect("remaining versions should be listable");
+ assert_eq!(versions.objects.iter().filter(|version| version.name == object).count(), 2);
+
+ metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml(1).into_bytes())
+ .await
+ .expect("due lifecycle rule should be restored");
+ let deleted = super::apply_expiry_on_non_transitioned_objects(
+ ecstore.clone(),
+ &marker,
+ &queued_event,
+ &LcEventSrc::Scanner,
+ incarnation,
+ )
+ .await;
+ assert!(deleted, "the current due rule should purge marker and history");
+ let versions = ecstore
+ .clone()
+ .list_object_versions(&bucket, object, None, None, None, 10)
+ .await
+ .expect("purged versions should be listable");
+ assert_eq!(versions.objects.iter().filter(|version| version.name == object).count(), 0);
+ }
+
+ #[tokio::test]
+ #[serial]
+ async fn queued_expired_object_all_versions_purges_history_through_transitioned_dispatch() {
+ let (_disk_paths, ecstore) = setup_test_env().await;
+ let bucket = format!("expired-all-versions-{}", Uuid::new_v4().simple());
+ let object = "object";
+ create_test_bucket(&ecstore, &bucket).await;
+ metadata_sys::update(
+ &bucket,
+ BUCKET_VERSIONING_CONFIG,
+ b"Enabled".to_vec(),
+ )
+ .await
+ .expect("bucket versioning should be enabled");
+ metadata_sys::update(
+ &bucket,
+ BUCKET_LIFECYCLE_CONFIG,
+ br#"
+
+ delete-all-versions
+ Enabled
+
+ 1true
+
+"#
+ .to_vec(),
+ )
+ .await
+ .expect("delete-all lifecycle rule should be stored");
+
+ let old_time = OffsetDateTime::now_utc() - time::Duration::days(3);
+ let mut old_reader = PutObjReader::from_vec(b"old version".to_vec());
+ ecstore
+ .put_object(
+ &bucket,
+ object,
+ &mut old_reader,
+ &ObjectOptions {
+ versioned: true,
+ mod_time: Some(old_time - time::Duration::hours(1)),
+ ..Default::default()
+ },
+ )
+ .await
+ .expect("old version should be stored");
+ let mut current_reader = PutObjReader::from_vec(b"current version".to_vec());
+ let mut current = ecstore
+ .put_object(
+ &bucket,
+ object,
+ &mut current_reader,
+ &ObjectOptions {
+ versioned: true,
+ mod_time: Some(old_time),
+ ..Default::default()
+ },
+ )
+ .await
+ .expect("current version should be stored");
+ current.transitioned_object.status = crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string();
+
+ let incarnation = ecstore
+ .bucket_incarnation_id_from_disk(&bucket)
+ .await
+ .expect("bucket incarnation should be available");
+ let deleted = super::apply_expiry_on_transitioned_object(
+ ecstore.clone(),
+ ¤t,
+ &crate::bucket::lifecycle::lifecycle::Event {
+ action: IlmAction::DeleteAllVersionsAction,
+ rule_id: "delete-all-versions".to_string(),
+ ..Default::default()
+ },
+ &LcEventSrc::Scanner,
+ incarnation,
+ )
+ .await;
+
+ assert!(deleted, "delete-all must not degrade to transitioned single-version expiry");
+ let versions = ecstore
+ .list_object_versions(&bucket, object, None, None, None, 10)
+ .await
+ .expect("purged versions should be listable");
+ assert_eq!(versions.objects.iter().filter(|version| version.name == object).count(), 0);
+ }
+
#[tokio::test]
async fn existing_object_lifecycle_skips_current_expiration_for_explicit_legal_hold() {
let lc = latest_expiration_lifecycle();
diff --git a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs
index ee05be7c7..6308b3767 100644
--- a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs
+++ b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs
@@ -435,12 +435,88 @@ async fn process_committed_tier_delete_journal_entry(api: Arc, je: &Jen
remove_tier_delete_journal_entry(api, je).await
}
-async fn reconcile_prepared_tier_delete_journal_entry(api: Arc, je: &Jentry) -> std::io::Result<()> {
- let (data, metadata) =
- config_boundary::read_config_with_metadata(api.clone(), &tier_delete_journal_object_name(je), &ObjectOptions::default())
+fn object_info_references_tier_delete(info: &ObjectInfo, je: &Jentry) -> std::io::Result {
+ if info.transitioned_object.status != rustfs_filemeta::TRANSITION_COMPLETE
+ || info.transitioned_object.name != je.obj_name
+ || info.transitioned_object.tier != je.tier_name
+ {
+ return Ok(false);
+ }
+ let source_backend_identity = tier_destination_id_from_metadata(&info.user_defined)?;
+ if source_backend_identity.is_some() && source_backend_identity != je.backend_identity {
+ return Ok(false);
+ }
+ if !je.version_id_exact {
+ return Ok(true);
+ }
+ Ok(match info.transition_version_state {
+ rustfs_filemeta::TransitionVersionState::Unknown => true,
+ rustfs_filemeta::TransitionVersionState::KnownDisabled => false,
+ rustfs_filemeta::TransitionVersionState::SuspendedNull | rustfs_filemeta::TransitionVersionState::Exact => {
+ info.transitioned_object.version_id == je.version_id
+ }
+ })
+}
+
+async fn prepared_tier_delete_has_live_source(
+ api: &ECStore,
+ source: &TierDeleteSourceIdentity,
+ je: &Jentry,
+) -> std::io::Result<(bool, Vec)> {
+ let lock_object = rustfs_utils::path::encode_dir_object(&source.object);
+ let mut lock_opts = ObjectOptions::default();
+ let read_guards = api
+ .acquire_all_object_read_locks("tier_delete_journal_recovery", &source.bucket, &lock_object, &mut lock_opts)
+ .await
+ .map_err(std::io::Error::other)?;
+ if api.ctx.lock_manager().is_disabled() {
+ return Err(std::io::Error::new(
+ std::io::ErrorKind::WouldBlock,
+ "tier delete journal recovery requires namespace locking",
+ ));
+ }
+ let mut has_live_source = false;
+ for pool in &api.pools {
+ let set = pool.get_disks_by_key(&lock_object);
+ let Some(versions) = set
+ .load_file_info_versions_exact(&source.bucket, &source.object)
.await
- .map_err(std::io::Error::other)?;
+ .map_err(std::io::Error::other)?
+ else {
+ continue;
+ };
+ for version in versions.versions.iter().filter(|version| !version.tier_free_version()) {
+ let info = ObjectInfo::from_file_info(version, &source.bucket, &source.object, source.versioned);
+ if object_info_references_tier_delete(&info, je)? {
+ has_live_source = true;
+ break;
+ }
+ }
+ if has_live_source {
+ break;
+ }
+ }
+ if read_guards.iter().any(crate::store::ObjectLockDiagGuard::is_lock_lost) {
+ return Err(std::io::Error::new(
+ std::io::ErrorKind::WouldBlock,
+ "tier delete journal recovery object read lock was lost",
+ ));
+ }
+ Ok((has_live_source, read_guards))
+}
+
+async fn reconcile_prepared_tier_delete_journal_entry(api: Arc, je: &Jentry) -> std::io::Result<()> {
+ let journal_name = tier_delete_journal_object_name(je);
+ let (data, metadata) = config_boundary::read_config_with_metadata(api.clone(), &journal_name, &ObjectOptions::default())
+ .await
+ .map_err(std::io::Error::other)?;
let current = decode_tier_delete_journal_entry(&data).map_err(std::io::Error::other)?;
+ if tier_delete_journal_object_name(¤t) != journal_name {
+ return Err(std::io::Error::new(
+ std::io::ErrorKind::InvalidData,
+ "prepared tier delete journal content does not match its object name",
+ ));
+ }
if current.state != TierDeleteJournalState::Prepared {
return Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
@@ -453,16 +529,21 @@ async fn reconcile_prepared_tier_delete_journal_entry(api: Arc, je: &Je
"prepared tier delete journal has no entity tag",
));
};
- let source = je
+ let source = current
.source
.as_ref()
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "prepared tier delete journal has no source"))?;
- match api
- .get_object_info(&source.bucket, &source.object, &source.lookup_options())
- .await
- {
- Ok(info) if source.matches(&info) => {
- match config_boundary::delete_config_if_match(api, &tier_delete_journal_object_name(¤t), &etag).await {
+ match prepared_tier_delete_has_live_source(&api, source, ¤t).await {
+ Ok((true, read_guards)) => {
+ if read_guards.iter().any(crate::store::ObjectLockDiagGuard::is_lock_lost) {
+ return Err(std::io::Error::new(
+ std::io::ErrorKind::WouldBlock,
+ "tier delete journal recovery object read lock was lost before abort",
+ ));
+ }
+ let result = config_boundary::delete_config_if_match(api, &tier_delete_journal_object_name(¤t), &etag).await;
+ drop(read_guards);
+ match result {
Ok(()) => Ok(()),
Err(Error::PreconditionFailed) => Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
@@ -471,17 +552,32 @@ async fn reconcile_prepared_tier_delete_journal_entry(api: Arc, je: &Je
Err(err) => Err(std::io::Error::other(err)),
}
}
- Ok(_info) if source.has_stable_identity() => {
- commit_prepared_tier_delete_journal_entry_if_current(api, current, etag).await
+ Ok((false, read_guards)) if source.has_stable_identity() => {
+ if read_guards.iter().any(crate::store::ObjectLockDiagGuard::is_lock_lost) {
+ return Err(std::io::Error::new(
+ std::io::ErrorKind::WouldBlock,
+ "tier delete journal recovery object read lock was lost before commit",
+ ));
+ }
+ let mut commit_opts = ObjectOptions::default();
+ for signal in read_guards
+ .iter()
+ .filter_map(crate::store::ObjectLockDiagGuard::lock_lost_signal)
+ {
+ commit_opts.add_namespace_lock_lost_signal(signal);
+ }
+ let committed =
+ commit_prepared_tier_delete_journal_entry_if_current(api.clone(), current, etag, &commit_opts).await?;
+ // Keep namespace locks only through the journal CAS. Remote-tier IO
+ // must not block writers for the object during recovery.
+ drop(read_guards);
+ process_committed_tier_delete_journal_entry(api, &committed).await
}
- Ok(_) => Err(std::io::Error::new(
+ Ok((false, _read_guards)) => Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"prepared tier delete journal source identity is not sufficient to confirm deletion",
)),
- Err(Error::ObjectNotFound(_, _)) | Err(Error::FileNotFound) | Err(Error::FileVersionNotFound) => {
- commit_prepared_tier_delete_journal_entry_if_current(api, current, etag).await
- }
- Err(err) => Err(std::io::Error::other(err)),
+ Err(err) => Err(err),
}
}
@@ -489,7 +585,8 @@ async fn commit_prepared_tier_delete_journal_entry_if_current(
api: Arc,
mut committed: Jentry,
etag: String,
-) -> std::io::Result<()> {
+ lock_opts: &ObjectOptions,
+) -> std::io::Result {
committed.state = TierDeleteJournalState::Committed;
let data = encode_tier_delete_journal_entry(&committed).map_err(std::io::Error::other)?;
match config_boundary::save_config_with_opts(
@@ -502,12 +599,13 @@ async fn commit_prepared_tier_delete_journal_entry_if_current(
if_match: Some(etag),
..Default::default()
}),
+ namespace_lock_fence: lock_opts.namespace_lock_fence.clone(),
..Default::default()
},
)
.await
{
- Ok(()) => process_committed_tier_delete_journal_entry(api, &committed).await,
+ Ok(()) => Ok(committed),
Err(Error::PreconditionFailed) => Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"prepared tier delete journal changed before commit",
@@ -582,6 +680,18 @@ pub async fn recover_tier_delete_journal_entries(
}
};
+ if tier_delete_journal_object_name(&je) != object.name {
+ stats.failed += 1;
+ warn!(
+ event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
+ component = LOG_COMPONENT_ECSTORE,
+ subsystem = LOG_SUBSYSTEM_LIFECYCLE,
+ journal_object = %object.name,
+ "Tier delete journal content does not match its object name and will be retained"
+ );
+ continue;
+ }
+
if je.backend_identity.is_none() {
stats.failed += 1;
warn!(
@@ -699,16 +809,14 @@ where
mod tests {
use super::{
TIER_DELETE_JOURNAL_EXACT_VERSION, TIER_DELETE_JOURNAL_STATE_VERSION, await_tier_delete_journal_recovery,
- decode_tier_delete_journal_entry, encode_tier_delete_journal_entry, record_tier_delete_journal_backend_identity,
- tier_delete_journal_object_name,
+ decode_tier_delete_journal_entry, encode_tier_delete_journal_entry, object_info_references_tier_delete,
+ record_tier_delete_journal_backend_identity, tier_delete_journal_object_name,
};
use crate::bucket::lifecycle::tier_sweeper::{Jentry, TierDeleteJournalState, TierDeleteSourceIdentity};
use crate::error::Result;
use crate::object_api::ObjectInfo;
use std::time::Duration;
- use time::OffsetDateTime;
use tokio_util::sync::CancellationToken;
- use uuid::Uuid;
fn journal_entry() -> Jentry {
Jentry {
@@ -738,6 +846,16 @@ mod tests {
assert_eq!(decoded.version_state, je.version_state);
}
+ #[test]
+ fn tier_delete_journal_object_name_binds_persisted_content() {
+ let original = journal_entry();
+ let original_name = tier_delete_journal_object_name(&original);
+ let mut replaced = original;
+ replaced.obj_name = "remote/replaced".to_string();
+
+ assert_ne!(tier_delete_journal_object_name(&replaced), original_name);
+ }
+
#[test]
fn tier_delete_transaction_roundtrips_prepared_source_identity() {
let mut je = journal_entry();
@@ -765,26 +883,35 @@ mod tests {
}
#[test]
- fn tier_delete_source_identity_rejects_recreated_object() {
- let version_id = Uuid::from_u128(1);
- let data_dir = Uuid::from_u128(2);
- let mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1);
- let info = ObjectInfo {
- bucket: "bucket".to_string(),
- name: "object".to_string(),
- version_id: Some(version_id),
- data_dir: Some(data_dir),
- mod_time: Some(mod_time),
+ fn prepared_recovery_blocks_any_live_reference_to_the_remote_version() {
+ let je = journal_entry();
+ let mut metadata = std::collections::HashMap::new();
+ rustfs_utils::http::metadata_compat::insert_str(
+ &mut metadata,
+ rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+ rustfs_utils::crypto::hex(je.backend_identity.expect("test journal should bind a backend")),
+ );
+ let mut info = ObjectInfo {
+ user_defined: std::sync::Arc::new(metadata),
+ transitioned_object: crate::storage_api_contracts::lifecycle::TransitionedObject {
+ name: je.obj_name.clone(),
+ version_id: je.version_id.clone(),
+ tier: je.tier_name.clone(),
+ status: rustfs_filemeta::TRANSITION_COMPLETE.to_string(),
+ ..Default::default()
+ },
+ transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
..Default::default()
};
- let source = TierDeleteSourceIdentity::from_object_info("bucket", "object", &info, true, false);
- assert!(source.matches(&info));
- let recreated = ObjectInfo {
- data_dir: Some(Uuid::from_u128(3)),
- ..info
- };
- assert!(!source.matches(&recreated));
+ assert!(object_info_references_tier_delete(&info, &je).expect("matching reference should be valid"));
+ info.transitioned_object.version_id = "other-version".to_string();
+ assert!(!object_info_references_tier_delete(&info, &je).expect("different exact version should be valid"));
+ info.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown;
+ assert!(
+ object_info_references_tier_delete(&info, &je).expect("legacy unknown reference should fail closed"),
+ "an unknown live source may still reference the journaled remote version"
+ );
}
#[test]
diff --git a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs
index 0bde1a23c..b2786fb18 100644
--- a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs
+++ b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs
@@ -334,32 +334,6 @@ impl TierDeleteSourceIdentity {
}
}
- pub(crate) fn lookup_options(&self) -> crate::object_api::ObjectOptions {
- crate::object_api::ObjectOptions {
- version_id: self.version_id.clone(),
- versioned: self.versioned,
- version_suspended: self.version_suspended,
- ..Default::default()
- }
- }
-
- pub(crate) fn matches(&self, info: &ObjectInfo) -> bool {
- if self.bucket != info.bucket {
- return false;
- }
- if let Some(version_id) = &self.version_id {
- return info.version_id.map(|id| id.to_string()).as_deref() == Some(version_id.as_str())
- && self.data_dir == info.data_dir.map(|id| id.to_string());
- }
- if self.data_dir.is_some() {
- return self.data_dir == info.data_dir.map(|id| id.to_string());
- }
- self.etag.is_some()
- && self.etag == info.etag
- && self.mod_time.is_some()
- && self.mod_time == info.mod_time.map(|time| time.to_string())
- }
-
pub(crate) fn has_stable_identity(&self) -> bool {
self.version_id.is_some() || self.data_dir.is_some() || (self.etag.is_some() && self.mod_time.is_some())
}
diff --git a/crates/ecstore/src/bucket/replication/replication_lifecycle_bridge.rs b/crates/ecstore/src/bucket/replication/replication_lifecycle_bridge.rs
index c35bf99c1..a89e89519 100644
--- a/crates/ecstore/src/bucket/replication/replication_lifecycle_bridge.rs
+++ b/crates/ecstore/src/bucket/replication/replication_lifecycle_bridge.rs
@@ -23,6 +23,8 @@ use super::replication_queue_boundary::DeletedObjectReplicationInfo;
use super::replication_storage_boundary::{
DeletedObject, ObjectInfo, ObjectOptions, ObjectToDelete, deleted_object_for_replication,
};
+#[cfg(test)]
+use std::sync::Mutex;
#[allow(
dead_code,
@@ -32,6 +34,9 @@ pub(crate) type ReplicationLifecycleConfig = ReplicationConfig;
pub(crate) struct ReplicationLifecycleBridge;
+#[cfg(test)]
+static SCHEDULED_DELETE_OBJECTS: Mutex> = Mutex::new(Vec::new());
+
impl ReplicationLifecycleBridge {
#[allow(
dead_code,
@@ -85,6 +90,13 @@ impl ReplicationLifecycleBridge {
}
pub(crate) async fn schedule_delete(bucket: String, delete_object: DeletedObject) {
+ #[cfg(test)]
+ {
+ SCHEDULED_DELETE_OBJECTS
+ .lock()
+ .expect("scheduled delete test hook lock should not poison")
+ .push(delete_object.clone());
+ }
super::replication_pool::schedule_replication_delete(DeletedObjectReplicationInfo {
delete_object: deleted_object_for_replication(delete_object),
bucket,
@@ -93,6 +105,15 @@ impl ReplicationLifecycleBridge {
})
.await;
}
+
+ #[cfg(test)]
+ pub(crate) fn take_scheduled_deletes_for_test() -> Vec {
+ std::mem::take(
+ &mut *SCHEDULED_DELETE_OBJECTS
+ .lock()
+ .expect("scheduled delete test hook lock should not poison"),
+ )
+ }
}
#[cfg(test)]
diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs
index 99ec6c83e..1b97a6df1 100644
--- a/crates/ecstore/src/object_api/types.rs
+++ b/crates/ecstore/src/object_api/types.rs
@@ -218,6 +218,63 @@ pub struct QuotaAdmission {
quota_limit: u64,
}
+#[doc(hidden)]
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct LifecycleDeleteAllRequest {
+ pub(crate) version_id: Option,
+ pub(crate) delete_marker: bool,
+ pub(crate) action: rustfs_common::metrics::IlmAction,
+ pub(crate) rule_id: String,
+ pub(crate) phase: LifecycleDeleteAllPhase,
+}
+
+#[doc(hidden)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum LifecycleDeleteAllPhase {
+ Preflight,
+ History,
+ FinalPreflight,
+ Trigger,
+}
+
+#[doc(hidden)]
+#[derive(Default)]
+pub struct LifecycleDeleteAllJournalState {
+ prepared: HashMap,
+ mutation_started: bool,
+}
+
+impl Debug for LifecycleDeleteAllJournalState {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("LifecycleDeleteAllJournalState")
+ .field("prepared_count", &self.prepared.len())
+ .field("mutation_started", &self.mutation_started)
+ .finish()
+ }
+}
+
+impl LifecycleDeleteAllJournalState {
+ pub(crate) fn contains(&self, name: &str) -> bool {
+ self.prepared.contains_key(name)
+ }
+
+ pub(crate) fn insert(&mut self, name: String, entry: crate::bucket::lifecycle::tier_sweeper::Jentry) {
+ self.prepared.insert(name, entry);
+ }
+
+ pub(crate) fn prepared_entries(&self) -> Vec {
+ self.prepared.values().cloned().collect()
+ }
+
+ pub(crate) fn mark_mutation_started(&mut self) {
+ self.mutation_started = true;
+ }
+
+ pub(crate) fn mutation_started(&self) -> bool {
+ self.mutation_started
+ }
+}
+
impl QuotaAdmission {
pub(crate) fn current_usage(self) -> u64 {
self.current_usage
@@ -242,6 +299,11 @@ pub struct ObjectOptions {
pub delete_prefix: bool,
pub delete_prefix_object: bool,
pub version_id: Option,
+ /// Lifecycle-only staged purge request checked under the object write lock.
+ #[doc(hidden)]
+ pub lifecycle_delete_all: Option,
+ #[doc(hidden)]
+ pub lifecycle_delete_all_journal: Option>>,
/// RustFS-only compare-and-set condition checked under the object write lock.
pub expected_current_version_id: Option,
/// Persisted bucket incarnation observed before authorization.
@@ -349,6 +411,15 @@ impl ObjectOptions {
self.namespace_lock_fence.get_or_insert_with(NamespaceLockFence::new);
}
+ pub(crate) fn ensure_lifecycle_delete_all_journal(&mut self) {
+ self.lifecycle_delete_all_journal
+ .get_or_insert_with(|| Arc::new(parking_lot::Mutex::new(LifecycleDeleteAllJournalState::default())));
+ }
+
+ pub(crate) fn lifecycle_delete_all_journal(&self) -> Option<&Arc>> {
+ self.lifecycle_delete_all_journal.as_ref()
+ }
+
pub fn add_namespace_lock_guard(&mut self, guard: &rustfs_lock::NamespaceLockGuard) {
if let Some(signal) = guard.lock_lost_signal() {
self.add_namespace_lock_lost_signal(signal);
@@ -1890,4 +1961,13 @@ mod tests {
assert!(default_cloned.user_tags.is_empty());
assert!(default_cloned.parts.is_empty());
}
+
+ #[test]
+ fn object_options_default_does_not_allocate_lifecycle_delete_all_journal() {
+ let mut opts = ObjectOptions::default();
+
+ assert!(opts.lifecycle_delete_all_journal().is_none());
+ opts.ensure_lifecycle_delete_all_journal();
+ assert!(opts.lifecycle_delete_all_journal().is_some());
+ }
}
diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs
index 5a56ea36e..d4c535461 100644
--- a/crates/ecstore/src/set_disk/mod.rs
+++ b/crates/ecstore/src/set_disk/mod.rs
@@ -4578,10 +4578,15 @@ impl SetDisks {
)?;
let fi = build_tiered_decommission_file_info(bucket, object, fi, layout);
let write_quorum = layout.write_quorum;
- if opts
- .bucket_lifecycle_lock_fence
- .as_ref()
- .is_some_and(NamespaceLockFence::is_lock_lost)
+ if _lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
+ || opts
+ .namespace_lock_fence
+ .as_ref()
+ .is_some_and(NamespaceLockFence::is_lock_lost)
+ || opts
+ .bucket_lifecycle_lock_fence
+ .as_ref()
+ .is_some_and(NamespaceLockFence::is_lock_lost)
|| bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
{
return Err(StorageError::NamespaceLockQuorumUnavailable {
diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs
index 929ac4d5c..5430b5d1d 100644
--- a/crates/ecstore/src/set_disk/ops/object.rs
+++ b/crates/ecstore/src/set_disk/ops/object.rs
@@ -27,12 +27,12 @@ use crate::set_disk::read::GetObjectDownstreamWriter;
use crate::bucket::lifecycle::{
tier_delete_journal::{
enqueue_committed_tier_delete_journal_entry, persist_tier_delete_journal_entry,
- record_tier_delete_journal_backend_identity, remove_tier_delete_journal_entry,
+ record_tier_delete_journal_backend_identity, remove_tier_delete_journal_entry, tier_delete_journal_object_name,
},
tier_sweeper::{
- Jentry, RemoteTierDeleteOutcome, TierDeleteJournalState,
+ Jentry, RemoteTierDeleteOutcome, TierDeleteJournalState, attach_tier_delete_source,
delete_confirmed_transition_candidate_exact_with_lease_idempotent, delete_object_from_remote_tier_with_lease_idempotent,
- transitioned_delete_journal_entry_for_source,
+ transitioned_delete_journal_entry_for_source, transitioned_force_delete_journal_entry,
},
transition_transaction::{
TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction,
@@ -42,7 +42,8 @@ use crate::bucket::lifecycle::{
};
use crate::bucket::quota::reservation;
use crate::bucket::replication::{
- DeleteReplicationConfigSnapshot, VersionPurgeStatusType, replication_state_to_filemeta, version_purge_status_to_filemeta,
+ DeleteReplicationConfigSnapshot, ReplicationLifecycleBridge, ReplicationStatusType, VersionPurgeStatusType,
+ replication_state_to_filemeta, replication_status_from_filemeta, version_purge_status_to_filemeta,
};
use crate::diagnostics::get::GetObjectFailureReason;
use crate::disk::{DataDirDeleteStatus, OldCurrentSize};
@@ -97,6 +98,636 @@ fn duration_millis_f64(duration: std::time::Duration) -> f64 {
duration.as_secs_f64() * 1000.0
}
+struct LifecycleDeleteAllPlan<'a> {
+ history: Vec<&'a FileInfo>,
+ trigger: Option<&'a FileInfo>,
+}
+
+impl<'a> LifecycleDeleteAllPlan<'a> {
+ fn trigger_only(&self) -> Result> {
+ if !self.history.is_empty() {
+ return Err(StorageError::PreconditionFailed);
+ }
+ Ok(self.trigger)
+ }
+}
+
+fn lifecycle_delete_all_plan<'a>(
+ versions: &'a rustfs_filemeta::FileInfoVersions,
+ trigger: &crate::object_api::LifecycleDeleteAllRequest,
+) -> Result> {
+ let normalized_version_id = |version: &FileInfo| version.version_id.filter(|version_id| !version_id.is_nil());
+ let mut history = Vec::with_capacity(versions.versions.len());
+ let mut trigger_version = None;
+ for (ordinary_index, version) in versions
+ .versions
+ .iter()
+ .filter(|version| !version.tier_free_version())
+ .enumerate()
+ {
+ if matches!(
+ replication_status_from_filemeta(version.replication_status()),
+ ReplicationStatusType::Pending | ReplicationStatusType::Failed
+ ) || version.version_purge_status().is_pending()
+ {
+ return Err(StorageError::PreconditionFailed);
+ }
+ if normalized_version_id(version) == trigger.version_id {
+ if trigger_version.is_some() || ordinary_index != 0 || version.deleted != trigger.delete_marker {
+ return Err(StorageError::PreconditionFailed);
+ }
+ trigger_version = Some(version);
+ } else {
+ history.push(version);
+ }
+ }
+ Ok(LifecycleDeleteAllPlan {
+ history,
+ trigger: trigger_version,
+ })
+}
+
+fn lifecycle_delete_all_tier_journal_entry(
+ bucket: &str,
+ object: &str,
+ version: &FileInfo,
+ opts: &ObjectOptions,
+) -> Result> {
+ if version.transition_status != rustfs_filemeta::TRANSITION_COMPLETE {
+ return Ok(None);
+ }
+ if version.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown {
+ return Err(StorageError::PreconditionFailed);
+ }
+
+ let logical_object = decode_dir_object(object);
+ let mut source = ObjectInfo::from_file_info(version, bucket, object, true);
+ source.version_id = source.version_id.filter(|version_id| !version_id.is_nil());
+ let mut entry = transitioned_force_delete_journal_entry(&source.transitioned_object, source.transition_version_state)
+ .ok_or(StorageError::PreconditionFailed)?;
+ attach_tier_delete_source(&mut entry, bucket, &logical_object, &source, opts.versioned, opts.version_suspended);
+ record_tier_delete_journal_backend_identity(&mut entry, &source.user_defined).map_err(Error::other)?;
+ let name = tier_delete_journal_object_name(&entry);
+ Ok(Some((name, entry)))
+}
+
+fn lifecycle_delete_all_replication_delete(
+ bucket: &str,
+ object: &str,
+ version: &FileInfo,
+ opts: &ObjectOptions,
+) -> Result> {
+ let snapshot = opts
+ .delete_replication_config_snapshot
+ .as_deref()
+ .ok_or(StorageError::PreconditionFailed)?;
+ let logical_object = decode_dir_object(object);
+ if !snapshot.has_active_rule(&logical_object) {
+ return Ok(None);
+ }
+
+ let versioned = opts.versioned || opts.version_suspended;
+ let source = ObjectInfo::from_file_info(version, bucket, object, versioned);
+ let Some(version_id) = source.version_id else {
+ return Err(StorageError::PreconditionFailed);
+ };
+ let delete_opts = ObjectOptions {
+ version_id: Some(version_id.to_string()),
+ versioned: opts.versioned,
+ version_suspended: opts.version_suspended,
+ replication_request: opts.replication_request,
+ no_lock: true,
+ ..Default::default()
+ };
+ let object_to_delete = ObjectToDelete {
+ object_name: logical_object.clone(),
+ version_id: Some(version_id),
+ ..Default::default()
+ };
+ let decision = ReplicationObjectBridge::check_delete_with_snapshot(&object_to_delete, &source, &delete_opts, false, snapshot);
+ if !decision.replicate_any() {
+ return Ok(None);
+ }
+
+ let replication_state = ReplicationLifecycleBridge::version_delete_replication_state(&decision);
+ let deleted_object = if source.delete_marker {
+ DeletedObject {
+ delete_marker: true,
+ delete_marker_version_id: Some(version_id),
+ delete_marker_mtime: source.mod_time,
+ object_name: logical_object,
+ replication_state: Some(replication_state_to_filemeta(&replication_state)),
+ ..Default::default()
+ }
+ } else {
+ DeletedObject {
+ object_name: logical_object,
+ version_id: Some(version_id),
+ replication_state: Some(replication_state_to_filemeta(&replication_state)),
+ ..Default::default()
+ }
+ };
+ Ok(Some((replication_state, deleted_object)))
+}
+
+async fn prepare_lifecycle_delete_all_tier_journals(
+ bucket: &str,
+ object: &str,
+ plan: &LifecycleDeleteAllPlan<'_>,
+ opts: &ObjectOptions,
+) -> Result<()> {
+ let Some(api) = opts.tier_delete_journal_api.as_ref() else {
+ return Ok(());
+ };
+ let journal = opts.lifecycle_delete_all_journal().ok_or(StorageError::PreconditionFailed)?;
+ for version in plan.history.iter().copied().chain(plan.trigger) {
+ let Some((name, entry)) = lifecycle_delete_all_tier_journal_entry(bucket, object, version, opts)? else {
+ continue;
+ };
+ if journal.lock().contains(&name) {
+ continue;
+ }
+ persist_tier_delete_journal_entry(Arc::clone(api), &entry)
+ .await
+ .map_err(Error::other)?;
+ journal.lock().insert(name, entry);
+ }
+ Ok(())
+}
+
+#[cfg(test)]
+mod lifecycle_delete_all_plan_tests {
+ use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
+ use super::*;
+ use crate::bucket::replication::ReplicationState;
+
+ fn delete_opts(request: crate::object_api::LifecycleDeleteAllRequest) -> ObjectOptions {
+ let mut opts = ObjectOptions {
+ delete_prefix: true,
+ delete_prefix_object: true,
+ versioned: true,
+ lifecycle_delete_all: Some(request),
+ delete_replication_config_snapshot: Some(Arc::new(DeleteReplicationConfigSnapshot::default())),
+ object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
+ ..Default::default()
+ };
+ opts.ensure_lifecycle_delete_all_journal();
+ opts
+ }
+
+ fn trigger(version_id: Uuid) -> crate::object_api::LifecycleDeleteAllRequest {
+ crate::object_api::LifecycleDeleteAllRequest {
+ version_id: Some(version_id),
+ delete_marker: true,
+ action: rustfs_common::metrics::IlmAction::DelMarkerDeleteAllVersionsAction,
+ rule_id: "rule".to_string(),
+ phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
+ }
+ }
+
+ #[test]
+ fn orders_history_before_trigger_and_excludes_tier_free_versions() {
+ let trigger_id = Uuid::new_v4();
+ let old_id = Uuid::new_v4();
+ let mut free = FileInfo::default();
+ free.set_tier_free_version();
+ free.version_id = Some(Uuid::new_v4());
+ let versions = rustfs_filemeta::FileInfoVersions {
+ versions: vec![
+ FileInfo {
+ version_id: Some(trigger_id),
+ deleted: true,
+ ..Default::default()
+ },
+ free,
+ FileInfo {
+ version_id: Some(old_id),
+ ..Default::default()
+ },
+ ],
+ ..Default::default()
+ };
+ let trigger = trigger(trigger_id);
+
+ let plan = lifecycle_delete_all_plan(&versions, &trigger).expect("valid lifecycle plan");
+ assert_eq!(
+ plan.history.iter().map(|version| version.version_id).collect::>(),
+ vec![Some(old_id)]
+ );
+ assert_eq!(plan.trigger.and_then(|version| version.version_id), Some(trigger_id));
+ assert!(plan.history.iter().all(|version| !version.tier_free_version()));
+ }
+
+ #[test]
+ fn rejects_a_noncurrent_trigger_copy() {
+ let trigger_id = Uuid::new_v4();
+ let versions = rustfs_filemeta::FileInfoVersions {
+ versions: vec![
+ FileInfo {
+ version_id: Some(Uuid::new_v4()),
+ deleted: false,
+ ..Default::default()
+ },
+ FileInfo {
+ version_id: Some(trigger_id),
+ deleted: true,
+ ..Default::default()
+ },
+ ],
+ ..Default::default()
+ };
+ let trigger = trigger(trigger_id);
+
+ assert!(matches!(
+ lifecycle_delete_all_plan(&versions, &trigger),
+ Err(StorageError::PreconditionFailed)
+ ));
+ }
+
+ #[test]
+ fn allows_a_history_only_pool_without_the_trigger() {
+ let old_id = Uuid::new_v4();
+ let versions = rustfs_filemeta::FileInfoVersions {
+ versions: vec![FileInfo {
+ version_id: Some(old_id),
+ ..Default::default()
+ }],
+ ..Default::default()
+ };
+
+ let plan = lifecycle_delete_all_plan(&versions, &trigger(Uuid::new_v4())).expect("history-only pool should plan");
+ assert_eq!(
+ plan.history.iter().map(|version| version.version_id).collect::>(),
+ vec![Some(old_id)]
+ );
+ assert!(plan.trigger.is_none());
+ }
+
+ #[test]
+ fn rejects_duplicate_trigger_versions() {
+ let trigger_id = Uuid::new_v4();
+ let versions = rustfs_filemeta::FileInfoVersions {
+ versions: vec![
+ FileInfo {
+ version_id: Some(trigger_id),
+ deleted: true,
+ ..Default::default()
+ },
+ FileInfo {
+ version_id: Some(trigger_id),
+ deleted: true,
+ ..Default::default()
+ },
+ ],
+ ..Default::default()
+ };
+
+ assert!(matches!(
+ lifecycle_delete_all_plan(&versions, &trigger(trigger_id)),
+ Err(StorageError::PreconditionFailed)
+ ));
+ }
+
+ #[test]
+ fn rejects_nonterminal_replication_and_purge_states_before_building_a_delete_plan() {
+ let trigger_id = Uuid::new_v4();
+ for (replication_status, purge_status) in [
+ (Some("PENDING"), None),
+ (Some("FAILED"), None),
+ (None, Some("PENDING")),
+ (None, Some("FAILED")),
+ ] {
+ let versions = rustfs_filemeta::FileInfoVersions {
+ versions: vec![
+ FileInfo {
+ version_id: Some(trigger_id),
+ deleted: true,
+ ..Default::default()
+ },
+ FileInfo {
+ version_id: Some(Uuid::new_v4()),
+ replication_state_internal: Some(replication_state_to_filemeta(&ReplicationState {
+ replication_status_internal: replication_status.map(str::to_string),
+ version_purge_status_internal: purge_status.map(str::to_string),
+ ..Default::default()
+ })),
+ ..Default::default()
+ },
+ ],
+ ..Default::default()
+ };
+
+ assert!(matches!(
+ lifecycle_delete_all_plan(&versions, &trigger(trigger_id)),
+ Err(StorageError::PreconditionFailed)
+ ));
+ }
+ }
+
+ #[test]
+ fn final_preflight_and_trigger_reject_remaining_history() {
+ let trigger_id = Uuid::new_v4();
+ let current = FileInfo {
+ version_id: Some(trigger_id),
+ deleted: true,
+ ..Default::default()
+ };
+ let history = FileInfo {
+ version_id: Some(Uuid::new_v4()),
+ ..Default::default()
+ };
+
+ let request = trigger(trigger_id);
+ let with_history = rustfs_filemeta::FileInfoVersions {
+ versions: vec![current.clone(), history.clone()],
+ ..Default::default()
+ };
+ assert!(matches!(
+ lifecycle_delete_all_plan(&with_history, &request).and_then(|plan| plan.trigger_only()),
+ Err(StorageError::PreconditionFailed)
+ ));
+
+ let trigger_only_versions = rustfs_filemeta::FileInfoVersions {
+ versions: vec![current],
+ ..Default::default()
+ };
+ let trigger_only = lifecycle_delete_all_plan(&trigger_only_versions, &request)
+ .and_then(|plan| plan.trigger_only())
+ .expect("trigger-only phase should proceed")
+ .expect("trigger should remain");
+ assert_eq!(trigger_only.version_id, Some(trigger_id));
+
+ let history_only_versions = rustfs_filemeta::FileInfoVersions {
+ versions: vec![history],
+ ..Default::default()
+ };
+ assert!(matches!(
+ lifecycle_delete_all_plan(&history_only_versions, &request).and_then(|plan| plan.trigger_only()),
+ Err(StorageError::PreconditionFailed)
+ ));
+ }
+
+ #[test]
+ fn null_trigger_matches_none_and_nil_version_ids() {
+ let request = crate::object_api::LifecycleDeleteAllRequest {
+ version_id: None,
+ delete_marker: false,
+ action: rustfs_common::metrics::IlmAction::DeleteAllVersionsAction,
+ rule_id: "rule".to_string(),
+ phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
+ };
+ for version_id in [None, Some(Uuid::nil())] {
+ let versions = rustfs_filemeta::FileInfoVersions {
+ versions: vec![FileInfo {
+ version_id,
+ ..Default::default()
+ }],
+ ..Default::default()
+ };
+
+ let plan = lifecycle_delete_all_plan(&versions, &request).expect("null trigger should match its persisted form");
+ assert!(plan.history.is_empty());
+ assert!(plan.trigger.is_some());
+ }
+ }
+
+ #[test]
+ fn tier_journal_coverage_is_source_exact_and_rejects_legacy_unknown_state() {
+ let identity = [7_u8; 32];
+ let version_id = Uuid::from_u128(1);
+ let mut metadata = HashMap::new();
+ rustfs_utils::http::metadata_compat::insert_str(
+ &mut metadata,
+ rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+ rustfs_utils::crypto::hex(identity),
+ );
+ let transitioned = |data_dir| FileInfo {
+ version_id: Some(version_id),
+ data_dir: Some(data_dir),
+ metadata: metadata.clone(),
+ transition_status: rustfs_filemeta::TRANSITION_COMPLETE.to_string(),
+ transition_tier: "WARM".to_string(),
+ transitioned_objname: "remote/object".to_string(),
+ transition_version: Some("remote-version".to_string()),
+ transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
+ ..Default::default()
+ };
+ let opts = ObjectOptions {
+ versioned: true,
+ ..Default::default()
+ };
+
+ let (first_name, _) =
+ lifecycle_delete_all_tier_journal_entry("bucket", "object", &transitioned(Uuid::from_u128(2)), &opts)
+ .expect("exact transitioned source should be journalable")
+ .expect("completed transition should require a journal");
+ let (second_name, _) =
+ lifecycle_delete_all_tier_journal_entry("bucket", "object", &transitioned(Uuid::from_u128(3)), &opts)
+ .expect("second pool source should be journalable")
+ .expect("completed transition should require a journal");
+ assert_ne!(first_name, second_name, "each pool-local source needs independent coverage");
+
+ let mut unknown = transitioned(Uuid::from_u128(4));
+ unknown.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown;
+ assert!(matches!(
+ lifecycle_delete_all_tier_journal_entry("bucket", "object", &unknown, &opts),
+ Err(StorageError::PreconditionFailed)
+ ));
+ }
+
+ #[tokio::test]
+ async fn staged_delete_removes_history_before_the_trigger() {
+ let (temp_dirs, disks, set) = hermetic_set_disks(4).await;
+ let bucket = "lifecycle-delete-all-staged";
+ let object = "object";
+ for disk in &disks {
+ disk.make_volume(bucket).await.expect("bucket volume should be created");
+ }
+
+ for body in [b"old".as_slice(), b"new".as_slice()] {
+ let mut reader = PutObjReader::from_vec(body.to_vec());
+ set.put_object(
+ bucket,
+ object,
+ &mut reader,
+ &ObjectOptions {
+ versioned: true,
+ ..Default::default()
+ },
+ )
+ .await
+ .expect("versioned object should be stored");
+ }
+ let marker = set
+ .delete_object(
+ bucket,
+ object,
+ ObjectOptions {
+ versioned: true,
+ ..Default::default()
+ },
+ )
+ .await
+ .expect("delete marker should be created");
+ let marker_id = marker.version_id.expect("delete marker should have a version id");
+ let transitioned_id = Uuid::new_v4();
+ let remote_version_id = Uuid::new_v4();
+ let free_version_id = Uuid::new_v4();
+ for temp_dir in &temp_dirs {
+ let meta_path = temp_dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE);
+ let encoded = tokio::fs::read(&meta_path).await.expect("xl.meta should be readable");
+ let mut metadata = FileMeta::load(&encoded).expect("xl.meta should decode");
+ metadata
+ .add_version(FileInfo {
+ volume: bucket.to_string(),
+ name: object.to_string(),
+ version_id: Some(transitioned_id),
+ transition_status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(),
+ transitioned_objname: "remote/lifecycle-delete-all-staged/object".to_string(),
+ transition_version_id: Some(remote_version_id),
+ transition_version: Some(remote_version_id.to_string()),
+ transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
+ transition_tier: "WARM".to_string(),
+ mod_time: Some(OffsetDateTime::now_utc() - time::Duration::days(10)),
+ ..Default::default()
+ })
+ .expect("transitioned version should be added");
+ let mut transition_delete = FileInfo {
+ volume: bucket.to_string(),
+ name: object.to_string(),
+ version_id: Some(transitioned_id),
+ mod_time: Some(OffsetDateTime::now_utc()),
+ ..Default::default()
+ };
+ transition_delete.set_tier_free_version_id(&free_version_id.to_string());
+ metadata
+ .delete_version(&transition_delete)
+ .expect("transitioned version should become a free-version record");
+ tokio::fs::write(&meta_path, metadata.marshal_msg().expect("xl.meta should encode"))
+ .await
+ .expect("xl.meta should be rewritten");
+ }
+
+ let mut request = trigger(marker_id);
+ request.phase = crate::object_api::LifecycleDeleteAllPhase::Trigger;
+ let err = set
+ .delete_object(bucket, object, delete_opts(request.clone()))
+ .await
+ .expect_err("trigger must remain while historical versions exist");
+ assert_eq!(err, StorageError::PreconditionFailed);
+
+ request.phase = crate::object_api::LifecycleDeleteAllPhase::Preflight;
+ set.delete_object(bucket, object, delete_opts(request.clone()))
+ .await
+ .expect("preflight should pass");
+ request.phase = crate::object_api::LifecycleDeleteAllPhase::History;
+ set.delete_object(bucket, object, delete_opts(request.clone()))
+ .await
+ .expect("history phase should delete old versions");
+ let after_history = set
+ .load_file_info_versions_exact(bucket, object)
+ .await
+ .expect("metadata should load")
+ .expect("trigger metadata should remain");
+ let ordinary: Vec<_> = after_history
+ .versions
+ .iter()
+ .filter(|version| !version.tier_free_version())
+ .collect();
+ assert_eq!(ordinary.len(), 1);
+ assert_eq!(ordinary[0].version_id, Some(marker_id));
+
+ request.phase = crate::object_api::LifecycleDeleteAllPhase::Trigger;
+ set.delete_object(bucket, object, delete_opts(request))
+ .await
+ .expect("trigger phase should delete the final marker");
+ for temp_dir in &temp_dirs {
+ let meta_path = temp_dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE);
+ let encoded = tokio::fs::read(&meta_path).await.expect("free-version xl.meta should remain");
+ let remaining = FileMeta::load(&encoded)
+ .expect("free-version xl.meta should decode")
+ .get_all_file_info_versions(bucket, object, true)
+ .expect("free-version metadata should remain readable");
+ let free_versions: Vec<_> = remaining
+ .versions
+ .iter()
+ .filter(|version| version.tier_free_version())
+ .collect();
+ assert_eq!(free_versions.len(), 1);
+ assert_eq!(free_versions[0].version_id, Some(free_version_id));
+ assert_eq!(free_versions[0].transition_tier, "WARM");
+ assert_eq!(free_versions[0].transitioned_objname, "remote/lifecycle-delete-all-staged/object");
+ assert_eq!(free_versions[0].transition_version_id, Some(remote_version_id));
+ }
+ }
+
+ #[tokio::test]
+ async fn stale_trigger_after_a_new_put_is_rejected_without_writes() {
+ let (_temp_dirs, disks, set) = hermetic_set_disks(4).await;
+ let bucket = "lifecycle-delete-all-stale-trigger";
+ let object = "object";
+ for disk in &disks {
+ disk.make_volume(bucket).await.expect("bucket volume should be created");
+ }
+ let mut reader = PutObjReader::from_vec(b"old".to_vec());
+ set.put_object(
+ bucket,
+ object,
+ &mut reader,
+ &ObjectOptions {
+ versioned: true,
+ ..Default::default()
+ },
+ )
+ .await
+ .expect("old version should be stored");
+ let marker = set
+ .delete_object(
+ bucket,
+ object,
+ ObjectOptions {
+ versioned: true,
+ ..Default::default()
+ },
+ )
+ .await
+ .expect("delete marker should be created");
+ let marker_id = marker.version_id.expect("delete marker should have a version id");
+ let mut replacement = PutObjReader::from_vec(b"replacement".to_vec());
+ set.put_object(
+ bucket,
+ object,
+ &mut replacement,
+ &ObjectOptions {
+ versioned: true,
+ ..Default::default()
+ },
+ )
+ .await
+ .expect("new current version should be stored");
+ let before = set
+ .load_file_info_versions_exact(bucket, object)
+ .await
+ .expect("metadata should load")
+ .expect("versions should exist");
+
+ let err = set
+ .delete_object(bucket, object, delete_opts(trigger(marker_id)))
+ .await
+ .expect_err("stale marker must not authorize a purge");
+ assert_eq!(err, StorageError::PreconditionFailed);
+ let after = set
+ .load_file_info_versions_exact(bucket, object)
+ .await
+ .expect("metadata should load after rejection")
+ .expect("versions should remain");
+ assert_eq!(
+ after.versions.iter().map(|version| version.version_id).collect::>(),
+ before.versions.iter().map(|version| version.version_id).collect::>()
+ );
+ }
+}
+
pub(in crate::set_disk::ops) fn assign_object_transaction_epoch(
shuffle_disks: &[Option],
parts_metadatas: &mut [FileInfo],
@@ -5579,7 +6210,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
.as_ref()
.is_some_and(|delete_opts| delete_opts.bypass_governance);
if let Some(object_lock_config) = object_lock_config.as_ref() {
- for version in versions.versions.iter().chain(versions.free_versions.iter()) {
+ for version in versions
+ .versions
+ .iter()
+ .chain(versions.free_versions.iter().filter(|_| opts.lifecycle_delete_all.is_none()))
+ {
let object_info = ObjectInfo::from_file_info(version, bucket, object, true);
if check_object_lock_for_deletion_with_state(object_lock_config, &object_info, bypass_governance)?
.is_some()
@@ -5588,26 +6223,76 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
}
}
- for version in &versions.versions {
- ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
- let mut delete_request = FileInfo {
- name: object.to_string(),
- version_id: version.version_id,
- ..Default::default()
+ if let Some(trigger) = opts.lifecycle_delete_all.as_ref() {
+ let plan = lifecycle_delete_all_plan(&versions, trigger)?;
+ if trigger.phase == crate::object_api::LifecycleDeleteAllPhase::Preflight {
+ prepare_lifecycle_delete_all_tier_journals(bucket, object, &plan, &opts).await?;
+ return Ok(ObjectInfo::default());
+ }
+ if trigger.phase == crate::object_api::LifecycleDeleteAllPhase::FinalPreflight {
+ return Ok(plan
+ .trigger_only()?
+ .map(|version| ObjectInfo::from_file_info(version, bucket, object, true))
+ .unwrap_or_default());
+ }
+ let plan = match trigger.phase {
+ crate::object_api::LifecycleDeleteAllPhase::History => plan.history,
+ crate::object_api::LifecycleDeleteAllPhase::Trigger => plan.trigger_only()?.into_iter().collect(),
+ crate::object_api::LifecycleDeleteAllPhase::Preflight
+ | crate::object_api::LifecycleDeleteAllPhase::FinalPreflight => {
+ return Err(StorageError::PreconditionFailed);
+ }
};
- delete_request.set_tier_free_version_id(&Uuid::new_v4().to_string());
- self.delete_object_version(bucket, object, &delete_request, false).await?;
- }
- for version in &versions.free_versions {
- ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
- let mut delete_request = FileInfo {
- name: object.to_string(),
- version_id: version.version_id,
- deleted: true,
- ..Default::default()
- };
- delete_request.set_tier_free_version();
- self.delete_object_version(bucket, object, &delete_request, false).await?;
+ for version in plan {
+ ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
+ let replication_delete = lifecycle_delete_all_replication_delete(bucket, object, version, &opts)?;
+ let mut delete_request = FileInfo {
+ name: object.to_string(),
+ version_id: version.version_id,
+ replication_state_internal: replication_delete
+ .as_ref()
+ .map(|(state, _)| replication_state_to_filemeta(state)),
+ ..Default::default()
+ };
+ delete_request.set_tier_free_version_id(&Uuid::new_v4().to_string());
+ if opts.tier_delete_journal_api.is_some()
+ && version.transition_status == rustfs_filemeta::TRANSITION_COMPLETE
+ {
+ let (name, _entry) = lifecycle_delete_all_tier_journal_entry(bucket, object, version, &opts)?
+ .ok_or(StorageError::PreconditionFailed)?;
+ let journal = opts.lifecycle_delete_all_journal().ok_or(StorageError::PreconditionFailed)?;
+ if !journal.lock().contains(&name) {
+ return Err(StorageError::PreconditionFailed);
+ }
+ delete_request.set_skip_tier_free_version();
+ }
+ self.delete_object_version(bucket, object, &delete_request, false).await?;
+ if let Some((_, deleted_object)) = replication_delete {
+ ReplicationLifecycleBridge::schedule_delete(bucket.to_string(), deleted_object).await;
+ }
+ }
+ } else {
+ for version in &versions.versions {
+ ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
+ let mut delete_request = FileInfo {
+ name: object.to_string(),
+ version_id: version.version_id,
+ ..Default::default()
+ };
+ delete_request.set_tier_free_version_id(&Uuid::new_v4().to_string());
+ self.delete_object_version(bucket, object, &delete_request, false).await?;
+ }
+ for version in &versions.free_versions {
+ ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
+ let mut delete_request = FileInfo {
+ name: object.to_string(),
+ version_id: version.version_id,
+ deleted: true,
+ ..Default::default()
+ };
+ delete_request.set_tier_free_version();
+ self.delete_object_version(bucket, object, &delete_request, false).await?;
+ }
}
}
self.invalidate_get_object_metadata_cache(bucket, object).await;
@@ -10079,6 +10764,65 @@ mod transition_upload_integrity_tests {
assert!(stored.transition_version.is_none());
}
+ #[tokio::test]
+ async fn data_movement_tiered_metadata_rejects_lost_outer_namespace_fence() {
+ let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+ let bucket = "tiered-data-movement-lost-outer-fence";
+ let object = "object.bin";
+ for disk in &disk_stores {
+ disk.make_volume(bucket).await.expect("bucket volume should be created");
+ }
+ let version_id = Uuid::new_v4();
+ let source = FileInfo {
+ volume: bucket.to_string(),
+ name: object.to_string(),
+ version_id: Some(version_id),
+ mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND),
+ size: 12,
+ parts: vec![ObjectPartInfo {
+ number: 1,
+ size: 12,
+ actual_size: 12,
+ etag: "part-etag".to_string(),
+ ..Default::default()
+ }],
+ transition_status: TRANSITION_COMPLETE.to_string(),
+ transition_tier: "WARM".to_string(),
+ transitioned_objname: "remote/object".to_string(),
+ transition_version: Some("remote-version".to_string()),
+ transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
+ fresh: true,
+ ..Default::default()
+ };
+ let err = set_disks
+ .decommission_tiered_object(
+ bucket,
+ object,
+ &source,
+ &ObjectOptions {
+ no_lock: true,
+ namespace_lock_fence: Some(NamespaceLockFence::lost_for_test()),
+ versioned: true,
+ version_id: Some(version_id.to_string()),
+ mod_time: source.mod_time,
+ data_movement: true,
+ ..Default::default()
+ },
+ )
+ .await
+ .expect_err("a lost outer object fence must block target metadata publication");
+
+ assert!(matches!(err, StorageError::NamespaceLockQuorumUnavailable { .. }));
+ assert!(
+ set_disks
+ .load_file_info_versions_exact(bucket, object)
+ .await
+ .expect("target metadata lookup should succeed")
+ .is_none(),
+ "lost outer fence must not publish a target version"
+ );
+ }
+
fn transition_options(original: &ObjectInfo, tier_name: String) -> ObjectOptions {
ObjectOptions {
no_lock: true,
@@ -13295,6 +14039,7 @@ mod delete_objects_lock_gating_tests {
use super::hermetic_set_disks_support::hermetic_set_disks_with_lockers_and_ctx;
use super::*;
use crate::disk::DiskAPI as _;
+ use serial_test::serial;
async fn put_plain_object(set_disks: &Arc, bucket: &str, object: &str) {
let mut reader = PutObjReader::from_vec(vec![3u8; 1024]);
@@ -13528,6 +14273,113 @@ mod delete_objects_lock_gating_tests {
assert!(state.replicate_decision_str.contains(arn));
}
+ #[tokio::test]
+ #[serial]
+ async fn lifecycle_delete_all_history_records_exact_replication_purge() {
+ use s3s::dto::{
+ BucketVersioningStatus, DeleteReplication, DeleteReplicationStatus, Destination, ReplicationConfiguration,
+ ReplicationRule, ReplicationRuleStatus, VersioningConfiguration,
+ };
+
+ let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+ let bucket = "lifecycle-delete-all-replication";
+ let object = "object";
+ let arn = "arn:rustfs:replication:us-east-1:target:bucket";
+ for disk in &disk_stores {
+ disk.make_volume(bucket).await.expect("bucket volume should be created");
+ }
+
+ let mut first_reader = PutObjReader::from_vec(b"first".to_vec());
+ let first = set_disks
+ .put_object(
+ bucket,
+ object,
+ &mut first_reader,
+ &ObjectOptions {
+ versioned: true,
+ ..Default::default()
+ },
+ )
+ .await
+ .expect("first version should be written");
+ let first_version_id = first.version_id.expect("first PUT should return a version id");
+ let mut trigger_reader = PutObjReader::from_vec(b"trigger".to_vec());
+ let trigger = set_disks
+ .put_object(
+ bucket,
+ object,
+ &mut trigger_reader,
+ &ObjectOptions {
+ versioned: true,
+ ..Default::default()
+ },
+ )
+ .await
+ .expect("trigger version should be written");
+ let trigger_version_id = trigger.version_id.expect("trigger PUT should return a version id");
+
+ let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
+ VersioningConfiguration {
+ status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
+ ..Default::default()
+ },
+ Some(ReplicationConfiguration {
+ role: String::new(),
+ rules: vec![ReplicationRule {
+ delete_marker_replication: None,
+ delete_replication: Some(DeleteReplication {
+ status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
+ }),
+ destination: Destination {
+ bucket: arn.to_string(),
+ ..Default::default()
+ },
+ existing_object_replication: None,
+ filter: None,
+ id: Some("delete-all-purge".to_string()),
+ prefix: Some(String::new()),
+ priority: Some(1),
+ source_selection_criteria: None,
+ status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
+ }],
+ }),
+ ));
+ let mut opts = ObjectOptions {
+ delete_prefix: true,
+ delete_prefix_object: true,
+ versioned: true,
+ lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
+ version_id: Some(trigger_version_id),
+ delete_marker: false,
+ action: rustfs_common::metrics::IlmAction::DeleteAllVersionsAction,
+ rule_id: "rule".to_string(),
+ phase: crate::object_api::LifecycleDeleteAllPhase::History,
+ }),
+ delete_replication_config_snapshot: Some(snapshot),
+ object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
+ ..Default::default()
+ };
+ opts.ensure_lifecycle_delete_all_journal();
+ let _ = crate::bucket::replication::ReplicationLifecycleBridge::take_scheduled_deletes_for_test();
+
+ set_disks
+ .delete_object(bucket, object, opts.clone())
+ .await
+ .expect("history phase should delete the old version");
+
+ let scheduled = crate::bucket::replication::ReplicationLifecycleBridge::take_scheduled_deletes_for_test();
+ assert_eq!(scheduled.len(), 1);
+ assert_eq!(scheduled[0].object_name, object);
+ assert_eq!(scheduled[0].version_id, Some(first_version_id));
+ assert!(!scheduled[0].delete_marker);
+ let state = scheduled[0]
+ .replication_state
+ .as_ref()
+ .expect("delete-all history purge should carry replication state");
+ assert_eq!(state.version_purge_status_internal.as_deref(), Some(format!("{arn}=PENDING;").as_str()));
+ assert!(state.replicate_decision_str.contains(arn));
+ }
+
#[tokio::test]
async fn synthetic_directory_delete_uses_decoded_prefix_and_marker_switch() {
use s3s::dto::{
diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs
index 9b4a484c1..e0452c5bd 100644
--- a/crates/ecstore/src/store/init.rs
+++ b/crates/ecstore/src/store/init.rs
@@ -558,8 +558,11 @@ mod tests {
lifecycle::{TRANSITION_PENDING, TransitionOptions},
tier_delete_journal::{
TIER_DELETE_JOURNAL_PREFIX, persist_tier_delete_journal_entry, recover_tier_delete_journal_entries,
+ tier_delete_journal_object_name,
+ },
+ tier_sweeper::{
+ Jentry, TierDeleteJournalState, TierDeleteSourceIdentity, transitioned_delete_journal_entry_for_source,
},
- tier_sweeper::Jentry,
transition_transaction::{
TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionOperatorError,
TransitionOperatorProbe, TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode,
@@ -569,9 +572,10 @@ mod tests {
recover_transition_transaction_records, save_transition_transaction_record,
},
},
+ bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_VERSIONING_CONFIG},
client::transition_api::ReaderImpl,
config::com,
- disk::RUSTFS_META_BUCKET,
+ disk::{RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE},
runtime::{global::set_object_store_resolver, sources as runtime_sources},
services::tier::{
test_util::{MockWarmBackend, MockWarmOp, TransitionCleanupStoreBarrier, register_mock_tier},
@@ -608,6 +612,8 @@ mod tests {
use rustfs_config::server_config::KVS;
use rustfs_filemeta::ObjectPartInfo;
#[cfg(feature = "test-util")]
+ use rustfs_filemeta::{FileInfo, FileMeta};
+ #[cfg(feature = "test-util")]
use rustfs_protos::{TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase};
use rustfs_rio::{Checksum, ChecksumType};
use rustfs_utils::{
@@ -1114,21 +1120,41 @@ mod tests {
Arc,
CancellationToken,
) {
- let mut pools = Vec::with_capacity(pool_drive_counts.len());
- for (pool_index, &drives_per_set) in pool_drive_counts.iter().enumerate() {
- let mut endpoints = Vec::with_capacity(drives_per_set);
- for disk_index in 0..drives_per_set {
- let path = temp_dir.join(format!("pool{pool_index}/disk{disk_index}"));
- tokio::fs::create_dir_all(&path).await.expect("create disk dir");
- let mut endpoint = Endpoint::try_from(path.to_str().expect("disk path should be utf-8")).expect("local endpoint");
- endpoint.set_pool_index(pool_index);
- endpoint.set_set_index(0);
- endpoint.set_disk_index(disk_index);
- endpoints.push(endpoint);
+ let pool_layouts = pool_drive_counts
+ .iter()
+ .map(|&drives_per_set| (1, drives_per_set))
+ .collect::>();
+ build_isolated_test_store_with_layout(temp_dir, cmd_line, &pool_layouts, shutdown).await
+ }
+
+ async fn build_isolated_test_store_with_layout(
+ temp_dir: &std::path::Path,
+ cmd_line: &str,
+ pool_layouts: &[(usize, usize)],
+ shutdown: CancellationToken,
+ ) -> (
+ Arc,
+ Arc,
+ CancellationToken,
+ ) {
+ let mut pools = Vec::with_capacity(pool_layouts.len());
+ for (pool_index, &(set_count, drives_per_set)) in pool_layouts.iter().enumerate() {
+ let mut endpoints = Vec::with_capacity(set_count * drives_per_set);
+ for set_index in 0..set_count {
+ for disk_index in 0..drives_per_set {
+ let path = temp_dir.join(format!("pool{pool_index}/set{set_index}/disk{disk_index}"));
+ tokio::fs::create_dir_all(&path).await.expect("create disk dir");
+ let mut endpoint =
+ Endpoint::try_from(path.to_str().expect("disk path should be utf-8")).expect("local endpoint");
+ endpoint.set_pool_index(pool_index);
+ endpoint.set_set_index(set_index);
+ endpoint.set_disk_index(disk_index);
+ endpoints.push(endpoint);
+ }
}
pools.push(PoolEndpoints {
legacy: false,
- set_count: 1,
+ set_count,
drives_per_set,
endpoints: Endpoints::from(endpoints),
cmd_line: format!("{cmd_line}-pool-{pool_index}"),
@@ -2726,6 +2752,67 @@ mod tests {
.expect_err("suspended delete must remove the requested UUID version");
}
+ #[cfg(feature = "test-util")]
+ #[tokio::test]
+ #[serial_test::serial(storage_class_env)]
+ async fn tiered_data_movement_rejects_a_stale_source_snapshot_before_target_write() {
+ let temp_dir = tempfile::tempdir().expect("create stale-source data movement store dir");
+ let (_ctx, store, _shutdown) =
+ without_storage_class_env(build_isolated_test_store(temp_dir.path(), "stale-tier-source", &[4, 4])).await;
+ crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+ let bucket = "stale-tier-source-bucket";
+ let object = "stale-tier-source-object";
+ store
+ .make_bucket(bucket, &MakeBucketOptions::default())
+ .await
+ .expect("bucket should be created");
+ let incarnation = store
+ .bucket_incarnation_id(bucket)
+ .await
+ .expect("bucket incarnation should exist");
+ let version_id = uuid::Uuid::new_v4();
+ let stale = FileInfo {
+ volume: bucket.to_string(),
+ name: object.to_string(),
+ version_id: Some(version_id),
+ transition_status: rustfs_filemeta::TRANSITION_COMPLETE.to_string(),
+ transitioned_objname: "remote/stale-source".to_string(),
+ transition_tier: "STALE-TIER".to_string(),
+ transition_version_id: Some(uuid::Uuid::new_v4()),
+ transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
+ data_dir: Some(uuid::Uuid::new_v4()),
+ mod_time: Some(OffsetDateTime::UNIX_EPOCH),
+ size: 1,
+ metadata: HashMap::new(),
+ ..Default::default()
+ };
+ let err = store
+ .decommission_tiered_object(
+ bucket,
+ object,
+ &stale,
+ &ObjectOptions {
+ versioned: true,
+ version_id: Some(version_id.to_string()),
+ src_pool_idx: 0,
+ data_movement: true,
+ expected_bucket_incarnation_id: Some(incarnation),
+ ..Default::default()
+ },
+ )
+ .await
+ .expect_err("a source removed after queue capture must fail closed");
+ assert!(matches!(err, Error::ObjectNotFound(_, _) | Error::FileNotFound));
+ let target_err = store.pools[1]
+ .get_object_info(bucket, object, &ObjectOptions::default())
+ .await
+ .expect_err("stale source rejection must not write target metadata");
+ assert!(matches!(
+ target_err,
+ StorageError::ObjectNotFound(_, _) | StorageError::VersionNotFound(_, _, _)
+ ));
+ }
+
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn data_movement_put_conflict_validates_only_selected_target_pool() {
@@ -4468,6 +4555,653 @@ mod tests {
shutdown_b.cancel();
}
+ #[cfg(feature = "test-util")]
+ #[tokio::test]
+ #[serial_test::serial(storage_class_env)]
+ async fn prepared_tier_delete_recovery_finds_directory_source_on_encoded_set() {
+ let temp_dir = tempfile::tempdir().expect("create temp store dir");
+ let shutdown = CancellationToken::new();
+ let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
+ temp_dir.path(),
+ "prepared-directory-recovery",
+ &[(2, 4)],
+ shutdown,
+ ))
+ .await;
+ crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+
+ let pool = &store.pools[0];
+ let object = (0..10_000)
+ .map(|index| format!("directory-{index}/"))
+ .find(|candidate| {
+ let encoded = rustfs_utils::path::encode_dir_object(candidate);
+ !Arc::ptr_eq(&pool.get_disks_by_key(candidate), &pool.get_disks_by_key(&encoded))
+ })
+ .expect("test topology should have a directory key whose encoded form hashes to another set");
+ let encoded = rustfs_utils::path::encode_dir_object(&object);
+ assert!(!Arc::ptr_eq(&pool.get_disks_by_key(&object), &pool.get_disks_by_key(&encoded)));
+
+ let tier_name = "PREPAREDDIRECTORY";
+ let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
+ let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
+ .await
+ .expect("tier lease should resolve")
+ .backend_identity();
+ let bucket = "prepared-directory-recovery-bucket";
+ store
+ .make_bucket(bucket, &MakeBucketOptions::default())
+ .await
+ .expect("bucket should be created");
+ let mut reader = PutObjReader::from_vec(b"directory source".to_vec());
+ let original = store
+ .put_object(bucket, &object, &mut reader, &ObjectOptions::default())
+ .await
+ .expect("directory source should be written");
+ store
+ .transition_object(
+ bucket,
+ &object,
+ &ObjectOptions {
+ transition: TransitionOptions {
+ status: TRANSITION_PENDING.to_string(),
+ tier: tier_name.to_string(),
+ etag: original.etag.clone().expect("source should have an ETag"),
+ ..Default::default()
+ },
+ mod_time: original.mod_time,
+ ..Default::default()
+ },
+ )
+ .await
+ .expect("directory transition should commit");
+ let committed = store
+ .get_object_info(
+ bucket,
+ &object,
+ &ObjectOptions {
+ no_lock: true,
+ metadata_cache_safe: false,
+ ..Default::default()
+ },
+ )
+ .await
+ .expect("transitioned directory source should be readable");
+ let mut entry = transitioned_delete_journal_entry_for_source(None, false, false, bucket, &object, &committed)
+ .expect("transitioned source should produce a prepared journal entry");
+ entry.backend_identity = Some(backend_identity);
+ persist_tier_delete_journal_entry(store.clone(), &entry)
+ .await
+ .expect("prepared journal should persist");
+
+ let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
+ .await
+ .expect("prepared recovery should complete");
+
+ assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 1, 0));
+ assert_eq!(tier_delete_journal_count(store).await, 0);
+ assert_eq!(backend.remove_count().await, 0);
+ assert_eq!(backend.object_count().await, 1, "live directory source must retain its remote object");
+ }
+
+ #[cfg(feature = "test-util")]
+ #[tokio::test]
+ #[serial_test::serial(storage_class_env)]
+ async fn prepared_tier_delete_recovery_checks_later_pool_then_commits_after_source_removal() {
+ let temp_dir = tempfile::tempdir().expect("create cross-pool recovery store dir");
+ let (ctx, store, _shutdown) =
+ without_storage_class_env(build_isolated_test_store(temp_dir.path(), "prepared-cross-pool-recovery", &[4, 4])).await;
+ crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+ let tier_name = "PREPAREDCROSSPOOL";
+ let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
+ let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
+ .await
+ .expect("tier lease should resolve")
+ .backend_identity();
+ let bucket = "prepared-cross-pool-recovery-bucket";
+ let object = "object";
+ store
+ .make_bucket(bucket, &MakeBucketOptions::default())
+ .await
+ .expect("bucket should be created");
+ let mut reader = PutObjReader::from_vec(b"later pool source".to_vec());
+ let original = store.pools[1]
+ .put_object(bucket, object, &mut reader, &ObjectOptions::default())
+ .await
+ .expect("source should be written only to the later pool");
+ store.pools[1]
+ .transition_object(
+ bucket,
+ object,
+ &ObjectOptions {
+ transition: TransitionOptions {
+ status: TRANSITION_PENDING.to_string(),
+ tier: tier_name.to_string(),
+ etag: original.etag.clone().expect("source should have an ETag"),
+ ..Default::default()
+ },
+ mod_time: original.mod_time,
+ ..Default::default()
+ },
+ )
+ .await
+ .expect("later-pool transition should commit");
+ let committed = store.pools[1]
+ .get_object_info(bucket, object, &ObjectOptions::default())
+ .await
+ .expect("transitioned source should be readable");
+ let mut entry = transitioned_delete_journal_entry_for_source(None, false, false, bucket, object, &committed)
+ .expect("transitioned source should produce a prepared journal");
+ entry.backend_identity = Some(backend_identity);
+ persist_tier_delete_journal_entry(store.clone(), &entry)
+ .await
+ .expect("prepared journal should persist");
+
+ let retained = recover_tier_delete_journal_entries(store.clone(), 100, None)
+ .await
+ .expect("recovery should scan the later pool");
+ assert_eq!((retained.scanned, retained.deleted, retained.failed), (1, 1, 0));
+ assert_eq!(backend.remove_count().await, 0);
+ assert_eq!(backend.object_count().await, 1);
+ assert_eq!(
+ tier_delete_journal_count(store.clone()).await,
+ 0,
+ "live source should abort its prepared journal"
+ );
+
+ store.pools[1]
+ .delete_object(
+ bucket,
+ object,
+ ObjectOptions {
+ version_id: committed.version_id.map(|version| version.to_string()),
+ expiration: crate::storage_api_contracts::lifecycle::ExpirationOptions { expire: true },
+ ..Default::default()
+ },
+ )
+ .await
+ .expect("source version should be removed before recovery retry");
+ persist_tier_delete_journal_entry(store.clone(), &entry)
+ .await
+ .expect("prepared journal should persist for the absent source");
+
+ let deleted = recover_tier_delete_journal_entries(store.clone(), 100, None)
+ .await
+ .expect("recovery should commit an absent stable source");
+ assert_eq!((deleted.scanned, deleted.deleted, deleted.failed), (1, 1, 0));
+ assert_eq!(tier_delete_journal_count(store).await, 0);
+ assert_eq!(backend.remove_count().await, 1);
+ assert_eq!(backend.object_count().await, 0);
+ }
+
+ #[cfg(feature = "test-util")]
+ #[tokio::test]
+ #[serial_test::serial(storage_class_env)]
+ async fn prepared_tier_delete_recovery_retains_journal_on_source_metadata_error() {
+ let temp_dir = tempfile::tempdir().expect("create metadata-error recovery store dir");
+ let (ctx, store, _shutdown) =
+ without_storage_class_env(build_isolated_test_store(temp_dir.path(), "prepared-metadata-error-recovery", &[4])).await;
+ crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+ let tier_name = "PREPAREDMETADATAERROR";
+ let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
+ let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
+ .await
+ .expect("tier lease should resolve")
+ .backend_identity();
+ let bucket = "prepared-metadata-error-recovery-bucket";
+ let object = "object";
+ store
+ .make_bucket(bucket, &MakeBucketOptions::default())
+ .await
+ .expect("bucket should be created");
+ let mut reader = PutObjReader::from_vec(b"source with unreadable metadata".to_vec());
+ let original = store
+ .put_object(bucket, object, &mut reader, &ObjectOptions::default())
+ .await
+ .expect("source should be written");
+ store
+ .transition_object(
+ bucket,
+ object,
+ &ObjectOptions {
+ transition: TransitionOptions {
+ status: TRANSITION_PENDING.to_string(),
+ tier: tier_name.to_string(),
+ etag: original.etag.clone().expect("source should have an ETag"),
+ ..Default::default()
+ },
+ mod_time: original.mod_time,
+ ..Default::default()
+ },
+ )
+ .await
+ .expect("source transition should commit");
+ let committed = store
+ .get_object_info(bucket, object, &ObjectOptions::default())
+ .await
+ .expect("transitioned source should be readable");
+ let mut entry = transitioned_delete_journal_entry_for_source(None, false, false, bucket, object, &committed)
+ .expect("transitioned source should produce a prepared journal");
+ entry.backend_identity = Some(backend_identity);
+ persist_tier_delete_journal_entry(store.clone(), &entry)
+ .await
+ .expect("prepared journal should persist");
+
+ for disk_index in 0..4 {
+ let metadata_path = temp_dir
+ .path()
+ .join(format!("pool0/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
+ tokio::fs::write(metadata_path, b"not-xl-meta")
+ .await
+ .expect("source metadata should be corrupted");
+ }
+
+ let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
+ .await
+ .expect("recovery scan should complete despite the entry failure");
+
+ assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1));
+ assert_eq!(tier_delete_journal_count(store).await, 1, "journal must remain prepared for retry");
+ assert_eq!(backend.remove_count().await, 0, "unreadable source metadata must block remote deletion");
+ assert_eq!(backend.object_count().await, 1, "remote source must remain intact");
+ }
+
+ #[cfg(feature = "test-util")]
+ #[tokio::test]
+ #[serial_test::serial(storage_class_env)]
+ async fn tier_delete_recovery_retains_content_that_does_not_match_its_object_name() {
+ let temp_dir = tempfile::tempdir().expect("create mismatched journal store dir");
+ let (ctx, store, _shutdown) =
+ without_storage_class_env(build_isolated_test_store(temp_dir.path(), "mismatched-tier-journal", &[4])).await;
+ crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+ let tier_name = "MISMATCHEDJOURNAL";
+ let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
+ let lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
+ .await
+ .expect("tier lease should resolve");
+ let remote_version = uuid::Uuid::new_v4().to_string();
+ backend.set_put_remote_version(Some(remote_version.clone())).await;
+ lease
+ .put("remote/original", ReaderImpl::Body(bytes::Bytes::from_static(b"remote body")), 11)
+ .await
+ .expect("remote body should be seeded");
+ let entry = Jentry {
+ obj_name: "remote/original".to_string(),
+ version_id: remote_version,
+ tier_name: tier_name.to_string(),
+ backend_identity: Some(lease.backend_identity()),
+ version_id_exact: true,
+ version_state: rustfs_filemeta::TransitionVersionState::Exact,
+ state: TierDeleteJournalState::Prepared,
+ source: Some(TierDeleteSourceIdentity {
+ bucket: "absent-source-bucket".to_string(),
+ object: "absent-source-object".to_string(),
+ version_id: Some(uuid::Uuid::new_v4().to_string()),
+ versioned: true,
+ version_suspended: false,
+ data_dir: Some(uuid::Uuid::new_v4().to_string()),
+ etag: Some("etag".to_string()),
+ mod_time: Some(OffsetDateTime::UNIX_EPOCH.to_string()),
+ }),
+ };
+ persist_tier_delete_journal_entry(store.clone(), &entry)
+ .await
+ .expect("prepared journal should persist");
+ let journal_name = tier_delete_journal_object_name(&entry);
+ let data = com::read_config(store.clone(), &journal_name)
+ .await
+ .expect("prepared journal should be readable");
+ let mut value: serde_json::Value = serde_json::from_slice(&data).expect("journal should contain JSON");
+ value["obj_name"] = serde_json::json!("remote/replaced");
+ com::save_config(
+ store.clone(),
+ &journal_name,
+ serde_json::to_vec(&value).expect("mismatched journal should encode"),
+ )
+ .await
+ .expect("mismatched journal content should be written under the original name");
+
+ let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
+ .await
+ .expect("recovery scan should complete");
+
+ assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1));
+ assert_eq!(tier_delete_journal_count(store).await, 1, "mismatched journal must be retained");
+ assert_eq!(backend.remove_count().await, 0);
+ assert_eq!(backend.object_count().await, 1);
+ }
+
+ #[cfg(feature = "test-util")]
+ #[test]
+ #[serial_test::serial(storage_class_env)]
+ fn transitioned_history_expiry_journals_real_source_without_free_version() {
+ std::thread::Builder::new()
+ .name("transitioned-delete-all-test".to_string())
+ .stack_size(32 * 1024 * 1024)
+ .spawn(|| {
+ let runtime = tokio::runtime::Builder::new_multi_thread()
+ .enable_all()
+ .worker_threads(2)
+ .build()
+ .expect("test runtime should build");
+ runtime.block_on(async {
+ let temp_dir = tempfile::tempdir().expect("create transitioned delete-all store dir");
+ let (ctx, store, _shutdown) =
+ without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transitioned-delete-all", &[4]))
+ .await;
+ crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+ let tier_name = "DELETEALLTRANSITIONED";
+ let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
+ let bucket = "transitioned-delete-all-bucket";
+ let object = "object";
+ store
+ .make_bucket(bucket, &MakeBucketOptions::default())
+ .await
+ .expect("bucket should be created");
+ crate::bucket::metadata_sys::update(
+ bucket,
+ BUCKET_VERSIONING_CONFIG,
+ b"Enabled".to_vec(),
+ )
+ .await
+ .expect("bucket versioning should be enabled");
+ crate::bucket::metadata_sys::update(
+ bucket,
+ BUCKET_LIFECYCLE_CONFIG,
+ br#"
+
+ delete-all-versions
+ Enabled
+
+ 1true
+
+"#
+ .to_vec(),
+ )
+ .await
+ .expect("delete-all lifecycle should be configured");
+
+ let old_time = OffsetDateTime::now_utc() - time::Duration::days(3);
+ let mut history_reader = PutObjReader::from_vec(b"transitioned history".to_vec());
+ let history = store
+ .put_object(
+ bucket,
+ object,
+ &mut history_reader,
+ &ObjectOptions {
+ versioned: true,
+ mod_time: Some(old_time - time::Duration::hours(1)),
+ ..Default::default()
+ },
+ )
+ .await
+ .expect("historical version should be written");
+ let mut current_reader = PutObjReader::from_vec(b"current version".to_vec());
+ let current = store
+ .put_object(
+ bucket,
+ object,
+ &mut current_reader,
+ &ObjectOptions {
+ versioned: true,
+ mod_time: Some(old_time),
+ ..Default::default()
+ },
+ )
+ .await
+ .expect("current version should be written");
+ store
+ .transition_object(
+ bucket,
+ object,
+ &ObjectOptions {
+ versioned: true,
+ version_id: history.version_id.map(|version_id| version_id.to_string()),
+ transition: TransitionOptions {
+ status: TRANSITION_PENDING.to_string(),
+ tier: tier_name.to_string(),
+ etag: history.etag.clone().expect("history should have an ETag"),
+ ..Default::default()
+ },
+ mod_time: history.mod_time,
+ ..Default::default()
+ },
+ )
+ .await
+ .expect("historical version should transition");
+ assert_eq!(backend.object_count().await, 1);
+ let transitioned_remote_versions = backend.put_versions().await;
+ assert_eq!(transitioned_remote_versions.len(), 1);
+
+ let incarnation = store
+ .bucket_incarnation_id_from_disk(bucket)
+ .await
+ .expect("bucket incarnation should be available");
+ for disk_index in 0..4 {
+ let metadata_path = temp_dir
+ .path()
+ .join(format!("pool0/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
+ let encoded = tokio::fs::read(&metadata_path)
+ .await
+ .expect("transition metadata should be readable");
+ let mut metadata = FileMeta::load(&encoded).expect("transition metadata should decode");
+ let mut transitioned = metadata
+ .get_all_file_info_versions(bucket, object, true)
+ .expect("transitioned versions should decode")
+ .versions
+ .into_iter()
+ .find(|version| version.version_id == history.version_id)
+ .expect("transitioned history should exist");
+ transitioned.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown;
+ metadata
+ .add_version(transitioned)
+ .expect("unknown state should replace the transitioned version");
+ tokio::fs::write(
+ &metadata_path,
+ metadata.marshal_msg().expect("unknown transition metadata should encode"),
+ )
+ .await
+ .expect("unknown transition metadata should be written");
+ }
+ let lifecycle_event = crate::bucket::lifecycle::lifecycle::Event {
+ action: rustfs_common::metrics::IlmAction::DeleteAllVersionsAction,
+ rule_id: "delete-all-versions".to_string(),
+ ..Default::default()
+ };
+ let rejected = crate::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects(
+ store.clone(),
+ ¤t,
+ &lifecycle_event,
+ &crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner,
+ incarnation,
+ )
+ .await;
+ assert!(!rejected, "legacy unknown transition identity must fail before local mutation");
+ assert_eq!(tier_delete_journal_count(store.clone()).await, 0);
+ assert_eq!(backend.remove_count().await, 0);
+ let retained = store.pools[0].disk_set[0]
+ .load_file_info_versions_exact(bucket, object)
+ .await
+ .expect("rejected delete-all metadata should remain readable")
+ .expect("rejected delete-all should retain both versions");
+ assert_eq!(
+ retained
+ .versions
+ .iter()
+ .filter(|version| !version.tier_free_version())
+ .count(),
+ 2
+ );
+
+ for disk_index in 0..4 {
+ let metadata_path = temp_dir
+ .path()
+ .join(format!("pool0/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
+ let encoded = tokio::fs::read(&metadata_path)
+ .await
+ .expect("unknown transition metadata should be readable");
+ let mut metadata = FileMeta::load(&encoded).expect("unknown transition metadata should decode");
+ let mut transitioned = metadata
+ .get_all_file_info_versions(bucket, object, true)
+ .expect("unknown transition versions should decode")
+ .versions
+ .into_iter()
+ .find(|version| version.version_id == history.version_id)
+ .expect("unknown transitioned history should exist");
+ transitioned.transition_version_state = rustfs_filemeta::TransitionVersionState::Exact;
+ metadata
+ .add_version(transitioned)
+ .expect("exact state should replace the transitioned version");
+ tokio::fs::write(
+ &metadata_path,
+ metadata.marshal_msg().expect("exact transition metadata should encode"),
+ )
+ .await
+ .expect("exact transition metadata should be written");
+ }
+ let applied = crate::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects(
+ store.clone(),
+ ¤t,
+ &lifecycle_event,
+ &crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner,
+ incarnation,
+ )
+ .await;
+
+ assert!(applied, "delete-all should remove current and transitioned history");
+ let versions = store.pools[0].disk_set[0]
+ .load_file_info_versions_exact(bucket, object)
+ .await
+ .expect("remaining exact metadata should be readable");
+ assert!(versions.is_none(), "delete-all must not leave a tier free-version");
+ assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
+ assert_eq!(backend.object_count().await, 1, "remote deletion must remain journal-driven");
+
+ let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
+ .await
+ .expect("committed journal should recover");
+ assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 1, 0));
+ assert_eq!(tier_delete_journal_count(store).await, 0);
+ assert_eq!(backend.object_count().await, 0);
+ assert_eq!(backend.exact_remove_count(), 1);
+ assert_eq!(backend.remove_versions().await, transitioned_remote_versions);
+ });
+ })
+ .expect("test thread should spawn")
+ .join()
+ .expect("test thread should complete");
+ }
+
+ #[cfg(feature = "test-util")]
+ #[tokio::test]
+ #[serial_test::serial(storage_class_env)]
+ async fn prepared_tier_delete_recovery_requires_namespace_locking() {
+ temp_env::async_with_vars([("RUSTFS_LOCK_ENABLED", Some("false"))], async {
+ let temp_dir = tempfile::tempdir().expect("create lock-disabled store dir");
+ let (ctx, store, _shutdown) =
+ without_storage_class_env(build_isolated_test_store(temp_dir.path(), "prepared-recovery-lock-disabled", &[4]))
+ .await;
+ crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+ assert!(ctx.lock_manager().is_disabled());
+
+ let tier_name = "PREPAREDLOCKDISABLED";
+ let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
+ let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
+ .await
+ .expect("tier lease should resolve")
+ .backend_identity();
+ let entry = Jentry {
+ obj_name: "remote/lock-disabled".to_string(),
+ version_id: uuid::Uuid::new_v4().to_string(),
+ tier_name: tier_name.to_string(),
+ backend_identity: Some(backend_identity),
+ version_id_exact: true,
+ version_state: rustfs_filemeta::TransitionVersionState::Exact,
+ state: TierDeleteJournalState::Prepared,
+ source: Some(TierDeleteSourceIdentity {
+ bucket: "absent-source-bucket".to_string(),
+ object: "absent-source-object".to_string(),
+ version_id: Some(uuid::Uuid::new_v4().to_string()),
+ versioned: true,
+ version_suspended: false,
+ data_dir: Some(uuid::Uuid::new_v4().to_string()),
+ etag: Some("etag".to_string()),
+ mod_time: Some(OffsetDateTime::UNIX_EPOCH.to_string()),
+ }),
+ };
+ persist_tier_delete_journal_entry(store.clone(), &entry)
+ .await
+ .expect("prepared journal should persist");
+
+ let stats = recover_tier_delete_journal_entries(store.clone(), 100, None)
+ .await
+ .expect("recovery scan should complete");
+
+ assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 0, 1));
+ assert_eq!(tier_delete_journal_count(store).await, 1, "journal must remain prepared for retry");
+ assert_eq!(backend.remove_count().await, 0, "lock-disabled recovery must not delete remotely");
+ })
+ .await;
+ }
+
+ #[cfg(feature = "test-util")]
+ #[tokio::test]
+ #[serial_test::serial(storage_class_env)]
+ async fn lifecycle_delete_all_requires_namespace_locking_before_mutation() {
+ temp_env::async_with_vars([("RUSTFS_LOCK_ENABLED", Some("false"))], async {
+ let temp_dir = tempfile::tempdir().expect("create lock-disabled delete-all store dir");
+ let (ctx, store, _shutdown) =
+ without_storage_class_env(build_isolated_test_store(temp_dir.path(), "delete-all-lock-disabled", &[4])).await;
+ crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+ assert!(ctx.lock_manager().is_disabled());
+ let bucket = "delete-all-lock-disabled-bucket";
+ let object = "object";
+ store
+ .make_bucket(bucket, &MakeBucketOptions::default())
+ .await
+ .expect("bucket should be created");
+ let mut reader = PutObjReader::from_vec(b"must survive".to_vec());
+ let no_lock_opts = ObjectOptions {
+ no_lock: true,
+ ..Default::default()
+ };
+ let original = store.pools[0]
+ .put_object(bucket, object, &mut reader, &no_lock_opts)
+ .await
+ .expect("source should be written");
+
+ let mut delete_opts = ObjectOptions {
+ delete_prefix: true,
+ delete_prefix_object: true,
+ lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
+ version_id: original.version_id,
+ delete_marker: false,
+ action: rustfs_common::metrics::IlmAction::DeleteAllVersionsAction,
+ rule_id: "rule".to_string(),
+ phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
+ }),
+ delete_replication_config_snapshot: Some(Arc::new(
+ crate::bucket::replication::DeleteReplicationConfigSnapshot::default(),
+ )),
+ ..Default::default()
+ };
+ delete_opts.ensure_lifecycle_delete_all_journal();
+
+ let err = store
+ .delete_object_with_tier_delete_journal(bucket, object, delete_opts)
+ .await
+ .expect_err("delete-all must reject disabled namespace locking");
+
+ assert!(err.to_string().contains("requires namespace locking"));
+ let retained = store.pools[0]
+ .get_object_info(bucket, object, &no_lock_opts)
+ .await
+ .expect("rejected delete-all must retain the source");
+ assert_eq!(retained.etag, original.etag);
+ assert_eq!(tier_delete_journal_count(store).await, 0, "rejected delete-all must not prepare journals");
+ })
+ .await;
+ }
+
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs
index 8d02ef441..eb3b6ef1c 100644
--- a/crates/ecstore/src/store/mod.rs
+++ b/crates/ecstore/src/store/mod.rs
@@ -151,6 +151,7 @@ pub(crate) mod init_format;
pub(crate) mod list_objects;
mod multipart;
mod object;
+pub(crate) use object::ObjectLockDiagGuard;
pub use object::{
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
SnapshotConsistencyError,
diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs
index 0ba89a7ec..312a25662 100644
--- a/crates/ecstore/src/store/object.rs
+++ b/crates/ecstore/src/store/object.rs
@@ -14,6 +14,8 @@
use super::*;
use crate::bucket::lifecycle::{
+ bucket_lifecycle_ops::eval_action_from_lifecycle,
+ get_expiry_configs,
tier_delete_journal::{
abort_prepared_tier_delete_journal_entry as abort_prepared_journal_entry_if_current, commit_tier_delete_journal_entry,
enqueue_committed_tier_delete_journal_entry, persist_tier_delete_journal_entry,
@@ -211,7 +213,8 @@ async fn delete_prefix_with_tier_delete_journal(
opts: &ObjectOptions,
tier_journal_api: Option<&Arc>,
) -> Result<()> {
- let journal_entry = if let Some(api) = tier_journal_api {
+ let lifecycle_delete_all = opts.lifecycle_delete_all.is_some();
+ let journal_entry = if !lifecycle_delete_all && let Some(api) = tier_journal_api {
Some(prepare_prefix_tier_delete_journal_entries(api, bucket, object, opts).await?)
} else {
None
@@ -220,14 +223,34 @@ async fn delete_prefix_with_tier_delete_journal(
let result = store.delete_prefix(bucket, object, opts).await;
match result {
Ok(()) => {
- if let (Some(api), Some(entries)) = (tier_journal_api, journal_entry.as_ref()) {
+ let lifecycle_entries = if lifecycle_delete_all {
+ opts.lifecycle_delete_all_journal()
+ .ok_or(StorageError::PreconditionFailed)?
+ .lock()
+ .prepared_entries()
+ } else {
+ Vec::new()
+ };
+ let entries = journal_entry.as_deref().unwrap_or(&lifecycle_entries);
+ if let Some(api) = tier_journal_api {
commit_prepared_tier_delete_journal_entries(api, entries).await;
}
Ok(())
}
Err(err) => {
- if let (Some(api), Some(entries)) = (tier_journal_api, journal_entry.as_ref()) {
- abort_prepared_tier_delete_journal_entries(api, entries).await;
+ if let Some(api) = tier_journal_api {
+ if lifecycle_delete_all {
+ let (abort, entries) = {
+ let journal = opts.lifecycle_delete_all_journal().ok_or(StorageError::PreconditionFailed)?;
+ let state = journal.lock();
+ (!state.mutation_started(), state.prepared_entries())
+ };
+ if abort {
+ abort_prepared_tier_delete_journal_entries(api, &entries).await;
+ }
+ } else if let Some(entries) = journal_entry.as_ref() {
+ abort_prepared_tier_delete_journal_entries(api, entries).await;
+ }
}
Err(err)
}
@@ -327,7 +350,7 @@ impl fmt::Display for ObjectLockDiagMode {
}
}
-struct ObjectLockDiagGuard {
+pub(crate) struct ObjectLockDiagGuard {
guard: rustfs_lock::NamespaceLockGuard,
enabled: bool,
op: &'static str,
@@ -360,14 +383,14 @@ impl ObjectLockDiagGuard {
}
}
- fn lock_lost_signal(&self) -> Option> {
+ pub(crate) fn lock_lost_signal(&self) -> Option> {
match &self.guard {
rustfs_lock::NamespaceLockGuard::Standard(guard) => Some(guard.lock_lost()),
rustfs_lock::NamespaceLockGuard::Fast(_) => None,
}
}
- fn is_lock_lost(&self) -> bool {
+ pub(crate) fn is_lock_lost(&self) -> bool {
self.guard.is_lock_lost()
}
}
@@ -1109,6 +1132,26 @@ fn is_equivalent_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo,
&& source_actual_size == target_actual_size
}
+fn tiered_data_movement_source_matches(
+ expected: &rustfs_filemeta::FileInfo,
+ current: &rustfs_filemeta::FileInfo,
+) -> Result {
+ let expected_backend = crate::services::tier::tier::tier_destination_id_from_metadata(&expected.metadata)?;
+ let current_backend = crate::services::tier::tier::tier_destination_id_from_metadata(¤t.metadata)?;
+ Ok(expected.version_id == current.version_id
+ && expected.data_dir == current.data_dir
+ && expected.mod_time == current.mod_time
+ && expected.size == current.size
+ && expected.get_etag() == current.get_etag()
+ && expected.transition_status == current.transition_status
+ && expected.transitioned_objname == current.transitioned_objname
+ && expected.transition_tier == current.transition_tier
+ && expected.transition_version_id == current.transition_version_id
+ && expected.transition_version == current.transition_version
+ && expected.transition_version_state == current.transition_version_state
+ && expected_backend == current_backend)
+}
+
fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
target_pool_idx != src_pool_idx
}
@@ -1247,7 +1290,9 @@ impl ECStore {
let mut opts = opts.clone();
opts.no_lock = false;
opts.metadata_cache_safe = false;
- let read_lock_guards = self.acquire_select_object_read_locks(bucket, &object, &mut opts).await?;
+ let read_lock_guards = self
+ .acquire_all_object_read_locks("select_object", bucket, &object, &mut opts)
+ .await?;
if self.ctx.lock_manager().is_disabled() {
return Err(SnapshotConsistencyError::LockingDisabled.into());
}
@@ -1488,8 +1533,9 @@ impl ECStore {
)))
}
- async fn acquire_select_object_read_locks(
+ pub(crate) async fn acquire_all_object_read_locks(
&self,
+ op: &'static str,
bucket: &str,
object: &str,
opts: &mut ObjectOptions,
@@ -1501,10 +1547,7 @@ impl ECStore {
// for each object's hashed set. DELETE and same-key CopyObject use the
// fixed domain, while PUT commits and data movement use the hashed set.
let distributed = self.ctx.is_dist_erasure().await;
- if let Some(guard) = self
- .acquire_object_read_lock_if_needed("select_object", bucket, object, opts)
- .await?
- {
+ if let Some(guard) = self.acquire_object_read_lock_if_needed(op, bucket, object, opts).await? {
guards.push(guard);
}
let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
@@ -1527,7 +1570,7 @@ impl ECStore {
.map_err(|err| Self::map_namespace_lock_error(bucket, object, "read", err))?;
let owner = diag_enabled.then(|| ns_lock.owner().to_string());
log_object_lock_acquire_if_slow(
- "select_object",
+ op,
bucket,
object,
owner.as_deref(),
@@ -1538,7 +1581,7 @@ impl ECStore {
guards.push(ObjectLockDiagGuard::new(
guard,
diag_enabled,
- "select_object",
+ op,
diag_enabled.then(|| bucket.to_string()),
diag_enabled.then(|| object.to_string()),
owner,
@@ -1549,6 +1592,77 @@ impl ECStore {
Ok(guards)
}
+ async fn acquire_data_movement_object_write_locks(
+ &self,
+ bucket: &str,
+ object: &str,
+ source_pool_idx: usize,
+ target_pool_idx: usize,
+ opts: &mut ObjectOptions,
+ ) -> Result> {
+ if self.ctx.lock_manager().is_disabled() {
+ return Err(Error::other("tiered data movement requires namespace locking"));
+ }
+ let distributed = self.ctx.is_dist_erasure().await;
+ let diag_enabled = is_object_lock_diag_enabled();
+ let mut pool_indices = [source_pool_idx, target_pool_idx];
+ pool_indices.sort_unstable();
+ let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
+ let mut locked_sets = vec![fixed_set];
+ let mut guards = Vec::with_capacity(3);
+
+ // Lock order matches journal recovery: fixed store domain first, then
+ // hashed domains by ascending pool index. This also serializes source
+ // revalidation and target publication against ordinary object deletes.
+ guards.push(self.acquire_object_write_lock("tiered_data_movement", bucket, object).await?);
+ for pool_idx in pool_indices {
+ let pool = self
+ .pools
+ .get(pool_idx)
+ .ok_or_else(|| Error::other(format!("invalid tiered data movement pool {pool_idx}")))?;
+ let set = pool.get_disks_by_key(object);
+ let lock_domain_already_held = !distributed
+ || locked_sets.iter().any(|locked_set: &Arc| {
+ same_distributed_lock_domain(&locked_set.lockers, &set.lockers)
+ });
+ if lock_domain_already_held {
+ continue;
+ }
+ let ns_lock = set.new_ns_lock(bucket, object).await?;
+ let acquire_start = Instant::now();
+ let guard = ns_lock
+ .get_write_lock(get_lock_acquire_timeout())
+ .await
+ .map_err(|err| Self::map_namespace_lock_error(bucket, object, "write", err))?;
+ let owner = diag_enabled.then(|| ns_lock.owner().to_string());
+ log_object_lock_acquire_if_slow(
+ "tiered_data_movement",
+ bucket,
+ object,
+ owner.as_deref(),
+ ObjectLockDiagMode::Write,
+ acquire_start.elapsed(),
+ diag_enabled,
+ );
+ guards.push(ObjectLockDiagGuard::new(
+ guard,
+ diag_enabled,
+ "tiered_data_movement",
+ diag_enabled.then(|| bucket.to_string()),
+ diag_enabled.then(|| object.to_string()),
+ owner,
+ ObjectLockDiagMode::Write,
+ ));
+ locked_sets.push(set);
+ }
+ opts.no_lock = true;
+ for signal in guards.iter().filter_map(ObjectLockDiagGuard::lock_lost_signal) {
+ opts.add_namespace_lock_lost_signal(signal);
+ }
+ opts.ensure_namespace_lock_fence();
+ Ok(guards)
+ }
+
fn attach_read_lock_guard(mut reader: GetObjectReader, guard: Option) -> GetObjectReader {
if is_lock_optimization_enabled() || reader.buffered_body.is_some() {
return reader;
@@ -1686,13 +1800,8 @@ impl ECStore {
Some(guard)
};
- let mut fi = fi.clone();
- if opts.data_movement {
- crate::data_movement::prepare_tiered_data_movement_file_info(&mut fi)?;
- }
-
- let object = encode_dir_object(object);
-
+ let logical_object = object;
+ let object = encode_dir_object(logical_object);
if self.single_pool() {
return Self::resolve_decommission_tiered_object_result(
Err(Error::other("single pool deployments cannot decommission tiered objects")),
@@ -1715,6 +1824,33 @@ impl ECStore {
&object,
)?
};
+ let _object_guards = self
+ .acquire_data_movement_object_write_locks(bucket, &object, opts.src_pool_idx, idx, &mut opts)
+ .await?;
+ let source_pool = self
+ .pools
+ .get(opts.src_pool_idx)
+ .ok_or_else(|| Error::other(format!("invalid tiered data movement source pool {}", opts.src_pool_idx)))?;
+ let source_versions = source_pool
+ .get_disks_by_key(&object)
+ .load_file_info_versions_exact(bucket, logical_object)
+ .await?;
+ let current_source = source_versions
+ .as_ref()
+ .and_then(|versions| {
+ versions
+ .versions
+ .iter()
+ .find(|current| current.version_id == fi.version_id && !current.tier_free_version())
+ })
+ .ok_or_else(|| to_object_err(StorageError::FileNotFound, vec![bucket, object.as_str()]))?;
+ if !tiered_data_movement_source_matches(fi, current_source)? {
+ return Err(to_object_err(StorageError::FileNotFound, vec![bucket, object.as_str()]));
+ }
+ let mut fi = current_source.clone();
+ if opts.data_movement {
+ crate::data_movement::prepare_tiered_data_movement_file_info(&mut fi)?;
+ }
if opts.data_movement && idx == opts.src_pool_idx {
let resume_target_pool_idx = self
.get_available_pool_idx_excluding(bucket, &object, fi.size, opts.src_pool_idx)
@@ -2190,6 +2326,10 @@ impl ECStore {
) -> Result {
check_del_obj_args(bucket, object)?;
+ if opts.lifecycle_delete_all.is_some() && self.ctx.lock_manager().is_disabled() {
+ return Err(Error::other("lifecycle delete-all requires namespace locking"));
+ }
+
let _bucket_lifecycle_guard = if is_meta_bucketname(bucket) {
None
} else if opts.delete_prefix {
@@ -2204,6 +2344,11 @@ impl ECStore {
};
let object = object.as_str();
let mut opts = opts;
+ let delete_all_configs = if opts.lifecycle_delete_all.is_some() {
+ Some(get_expiry_configs(self, bucket).await?)
+ } else {
+ None
+ };
opts.tier_delete_journal_api = tier_journal_api.clone();
if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
opts.add_bucket_lifecycle_lock_guard(guard);
@@ -2301,6 +2446,34 @@ impl ECStore {
} else {
None
};
+ if let Some(trigger) = opts.lifecycle_delete_all.as_ref() {
+ let configs = delete_all_configs.as_ref().ok_or(StorageError::PreconditionFailed)?;
+ let expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
+ if configs.table_bucket_enabled || configs.bucket_incarnation_id != expected_bucket_incarnation_id {
+ return Err(StorageError::PreconditionFailed);
+ }
+ let lifecycle = configs.lifecycle.as_ref().ok_or(StorageError::PreconditionFailed)?;
+ let (mut current, _) = self
+ .get_latest_object_info_with_idx(
+ bucket,
+ object,
+ &ObjectOptions {
+ no_lock: true,
+ metadata_cache_safe: false,
+ ..Default::default()
+ },
+ )
+ .await?;
+ let current_version_id = current.version_id.filter(|version_id| !version_id.is_nil());
+ if current_version_id != trigger.version_id || current.delete_marker != trigger.delete_marker {
+ return Err(StorageError::PreconditionFailed);
+ }
+ current.name = decode_dir_object(¤t.name);
+ let current_event = eval_action_from_lifecycle(lifecycle, configs.object_lock.as_deref(), ¤t).await;
+ if current_event.action != trigger.action || current_event.rule_id != trigger.rule_id {
+ return Err(StorageError::PreconditionFailed);
+ }
+ }
if opts.delete_prefix {
delete_prefix_with_tier_delete_journal(self, bucket, object, &opts, tier_journal_api.as_ref()).await?;
return Ok(ObjectInfo::default());
@@ -3828,6 +4001,24 @@ mod tests {
assert!(is_equivalent_data_movement_tiered_object(&source, &target));
}
+ #[test]
+ fn tiered_data_movement_source_match_rejects_transition_identity_changes() {
+ let source = tiered_equivalence_source();
+ assert!(tiered_data_movement_source_matches(&source, &source).expect("matching source metadata should parse"));
+
+ let mut changed_remote = source.clone();
+ changed_remote.transitioned_objname = "remote/replaced".to_string();
+ assert!(!tiered_data_movement_source_matches(&source, &changed_remote).expect("changed remote metadata should parse"));
+
+ let mut changed_backend = source.clone();
+ rustfs_utils::http::metadata_compat::insert_str(
+ &mut changed_backend.metadata,
+ rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+ rustfs_utils::crypto::hex([9; 32]),
+ );
+ assert!(!tiered_data_movement_source_matches(&source, &changed_backend).expect("backend metadata should parse"));
+ }
+
#[test]
fn equivalent_data_movement_tiered_object_uses_logical_compressed_and_encrypted_sizes() {
let mut compressed = tiered_equivalence_source();
diff --git a/crates/ecstore/src/store/rebalance.rs b/crates/ecstore/src/store/rebalance.rs
index 75e08919f..79e3abf91 100644
--- a/crates/ecstore/src/store/rebalance.rs
+++ b/crates/ecstore/src/store/rebalance.rs
@@ -202,10 +202,76 @@ impl ECStore {
}
pub(super) async fn delete_prefix(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
+ if opts.lifecycle_delete_all.is_some() {
+ let mut preflight_opts = opts.clone();
+ preflight_opts
+ .lifecycle_delete_all
+ .as_mut()
+ .ok_or(StorageError::PreconditionFailed)?
+ .phase = crate::object_api::LifecycleDeleteAllPhase::Preflight;
+ for pool in &self.pools {
+ #[cfg(test)]
+ lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::Preflight, pool.pool_idx)?;
+ pool.delete_object(bucket, object, preflight_opts.clone()).await?;
+ }
+
+ opts.lifecycle_delete_all_journal()
+ .ok_or(StorageError::PreconditionFailed)?
+ .lock()
+ .mark_mutation_started();
+ let mut non_trigger_opts = opts.clone();
+ non_trigger_opts
+ .lifecycle_delete_all
+ .as_mut()
+ .ok_or(StorageError::PreconditionFailed)?
+ .phase = crate::object_api::LifecycleDeleteAllPhase::History;
+ for pool in &self.pools {
+ #[cfg(test)]
+ lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::History, pool.pool_idx)?;
+ let mut pool_opts = non_trigger_opts.clone();
+ pool_opts.delete_prefix = true;
+ pool.delete_object(bucket, object, pool_opts).await?;
+ }
+
+ let mut final_preflight_opts = opts.clone();
+ final_preflight_opts
+ .lifecycle_delete_all
+ .as_mut()
+ .ok_or(StorageError::PreconditionFailed)?
+ .phase = crate::object_api::LifecycleDeleteAllPhase::FinalPreflight;
+ let mut trigger_pools = Vec::new();
+ for (pool_index, pool) in self.pools.iter().enumerate() {
+ #[cfg(test)]
+ lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::FinalPreflight, pool.pool_idx)?;
+ let result = pool.delete_object(bucket, object, final_preflight_opts.clone()).await?;
+ if !result.name.is_empty() {
+ trigger_pools.push(pool_index);
+ }
+ }
+ if trigger_pools.is_empty() {
+ return Err(StorageError::PreconditionFailed);
+ }
+
+ let mut trigger_opts = opts.clone();
+ trigger_opts
+ .lifecycle_delete_all
+ .as_mut()
+ .ok_or(StorageError::PreconditionFailed)?
+ .phase = crate::object_api::LifecycleDeleteAllPhase::Trigger;
+ for pool_index in trigger_pools {
+ #[cfg(test)]
+ lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::Trigger, pool_index)?;
+ let mut pool_opts = trigger_opts.clone();
+ pool_opts.delete_prefix = true;
+ self.pools[pool_index].delete_object(bucket, object, pool_opts).await?;
+ }
+ return Ok(());
+ }
+
let mut first_error = None;
let mut first_volume_error = None;
let mut has_success = false;
- for pool in self.pools.iter() {
+ for pool in &self.pools {
let mut opts = opts.clone();
opts.delete_prefix = true;
match pool.delete_object(bucket, object, opts).await {
@@ -774,6 +840,22 @@ impl ECStore {
}
}
+#[cfg(test)]
+static LIFECYCLE_DELETE_ALL_TEST_FAILURE: std::sync::Mutex> =
+ std::sync::Mutex::new(None);
+
+#[cfg(test)]
+fn lifecycle_delete_all_test_failure(phase: crate::object_api::LifecycleDeleteAllPhase, pool_index: usize) -> Result<()> {
+ if LIFECYCLE_DELETE_ALL_TEST_FAILURE
+ .lock()
+ .expect("lifecycle delete-all failure hook should not poison")
+ .is_some_and(|failure| failure == (phase, pool_index))
+ {
+ return Err(StorageError::PreconditionFailed);
+ }
+ Ok(())
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -781,23 +863,28 @@ mod tests {
use crate::disk::error::DiskError;
use crate::layout::endpoint::Endpoint;
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
+ use crate::object_api::ObjectLockConfigSnapshot;
use crate::storage_api_contracts::bucket::MakeBucketOptions;
+ use crate::storage_api_contracts::object::ObjectIO as _;
use arc_swap::ArcSwap;
use rustfs_config::server_config::KVS;
+ use rustfs_filemeta::FileInfo;
use std::sync::Arc;
use tokio_util::sync::CancellationToken;
- #[tokio::test]
- async fn delete_prefix_attempts_later_pools_after_an_earlier_pool_error() {
- let temp_dir = tempfile::tempdir().expect("multi-pool delete test directory should be created");
- let mut pools = Vec::with_capacity(2);
- for (pool_index, drives_per_set) in [2, 4].into_iter().enumerate() {
+ async fn setup_multi_pool_test_store(
+ name: &str,
+ drives_per_pool: &[usize],
+ ) -> (tempfile::TempDir, Arc, CancellationToken) {
+ let temp_dir = tempfile::tempdir().expect("multi-pool test directory should be created");
+ let mut pools = Vec::with_capacity(drives_per_pool.len());
+ for (pool_index, drives_per_set) in drives_per_pool.iter().copied().enumerate() {
let mut endpoints = Vec::with_capacity(drives_per_set);
for disk_index in 0..drives_per_set {
let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
tokio::fs::create_dir_all(&disk_path)
.await
- .expect("multi-pool delete test disk should be created");
+ .expect("multi-pool test disk should be created");
let mut endpoint =
Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
endpoint.set_pool_index(pool_index);
@@ -810,7 +897,7 @@ mod tests {
set_count: 1,
drives_per_set,
endpoints: Endpoints::from(endpoints),
- cmd_line: format!("delete-prefix-pool-{pool_index}"),
+ cmd_line: format!("{name}-pool-{pool_index}"),
platform: "test".to_string(),
});
}
@@ -830,6 +917,92 @@ mod tests {
.await
.expect("multi-pool store should initialize");
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+ (temp_dir, store, shutdown)
+ }
+
+ struct LifecycleDeleteAllFailureGuard;
+
+ impl Drop for LifecycleDeleteAllFailureGuard {
+ fn drop(&mut self) {
+ *LIFECYCLE_DELETE_ALL_TEST_FAILURE
+ .lock()
+ .expect("lifecycle delete-all failure hook should not poison") = None;
+ }
+ }
+
+ async fn seed_multi_pool_delete_all(store: &Arc, bucket: &str, object: &str) -> ObjectOptions {
+ let trigger_id = Uuid::new_v4();
+ for (pool_index, pool) in store.pools.iter().enumerate() {
+ let mut history_reader = PutObjReader::from_vec(format!("{object}-history-{pool_index}").into_bytes());
+ pool.put_object(
+ bucket,
+ object,
+ &mut history_reader,
+ &ObjectOptions {
+ versioned: true,
+ version_id: Some(Uuid::new_v4().to_string()),
+ mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
+ ..Default::default()
+ },
+ )
+ .await
+ .expect("history should be stored");
+ let mut trigger_reader = PutObjReader::from_vec(format!("{object}-trigger-{pool_index}").into_bytes());
+ pool.put_object(
+ bucket,
+ object,
+ &mut trigger_reader,
+ &ObjectOptions {
+ versioned: true,
+ version_id: Some(trigger_id.to_string()),
+ mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)),
+ ..Default::default()
+ },
+ )
+ .await
+ .expect("shared trigger should be stored");
+ }
+ let mut opts = ObjectOptions {
+ delete_prefix: true,
+ delete_prefix_object: true,
+ versioned: true,
+ lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
+ version_id: Some(trigger_id),
+ delete_marker: false,
+ action: rustfs_common::metrics::IlmAction::DeleteAllVersionsAction,
+ rule_id: "rule".to_string(),
+ phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
+ }),
+ object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
+ crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
+ ))),
+ delete_replication_config_snapshot: Some(Arc::new(
+ crate::bucket::replication::DeleteReplicationConfigSnapshot::default(),
+ )),
+ ..Default::default()
+ };
+ opts.ensure_lifecycle_delete_all_journal();
+ opts
+ }
+
+ async fn ordinary_version_count(store: &ECStore, pool_index: usize, bucket: &str, object: &str) -> usize {
+ store.pools[pool_index].disk_set[0]
+ .load_file_info_versions_exact(bucket, object)
+ .await
+ .expect("pool metadata should load")
+ .map(|versions| {
+ versions
+ .versions
+ .iter()
+ .filter(|version| !version.tier_free_version())
+ .count()
+ })
+ .unwrap_or_default()
+ }
+
+ #[tokio::test]
+ async fn delete_prefix_attempts_later_pools_after_an_earlier_pool_error() {
+ let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("delete-prefix", &[2, 4]).await;
let bucket = format!("delete-prefix-{}", Uuid::new_v4().simple());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
@@ -921,6 +1094,171 @@ mod tests {
shutdown.cancel();
}
+ #[tokio::test]
+ #[serial_test::serial]
+ async fn lifecycle_delete_all_history_failure_preserves_trigger_and_retry_converges() {
+ let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("lifecycle-delete-all", &[4, 4]).await;
+ let bucket = format!("lifecycle-delete-all-{}", Uuid::new_v4().simple());
+ let object = "object";
+ store
+ .make_bucket(&bucket, &MakeBucketOptions::default())
+ .await
+ .expect("bucket should be created in both pools");
+
+ for pool_index in 0..2 {
+ let mut reader = PutObjReader::from_vec(format!("pool-{pool_index}-history").into_bytes());
+ store.pools[pool_index]
+ .put_object(
+ &bucket,
+ object,
+ &mut reader,
+ &ObjectOptions {
+ versioned: true,
+ ..Default::default()
+ },
+ )
+ .await
+ .expect("historical version should be stored");
+ }
+ let marker = store.pools[0]
+ .delete_object(
+ &bucket,
+ object,
+ ObjectOptions {
+ versioned: true,
+ ..Default::default()
+ },
+ )
+ .await
+ .expect("trigger marker should be stored in the first pool");
+ let marker_id = marker.version_id.expect("trigger marker should have a version id");
+ let mut opts = ObjectOptions {
+ delete_prefix: true,
+ delete_prefix_object: true,
+ versioned: true,
+ lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
+ version_id: Some(marker_id),
+ delete_marker: true,
+ action: rustfs_common::metrics::IlmAction::DelMarkerDeleteAllVersionsAction,
+ rule_id: "rule".to_string(),
+ phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
+ }),
+ object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
+ crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
+ ))),
+ delete_replication_config_snapshot: Some(Arc::new(
+ crate::bucket::replication::DeleteReplicationConfigSnapshot::default(),
+ )),
+ ..Default::default()
+ };
+ opts.ensure_lifecycle_delete_all_journal();
+
+ let _failure_guard = LifecycleDeleteAllFailureGuard;
+ *LIFECYCLE_DELETE_ALL_TEST_FAILURE
+ .lock()
+ .expect("lifecycle delete-all failure hook should not poison") =
+ Some((crate::object_api::LifecycleDeleteAllPhase::History, 1));
+ let err = store
+ .delete_prefix(&bucket, object, &opts)
+ .await
+ .expect_err("a later pool history failure must stop before trigger deletion");
+ assert_eq!(err, StorageError::PreconditionFailed);
+ assert!(
+ opts.lifecycle_delete_all_journal()
+ .expect("delete-all journal should be initialized")
+ .lock()
+ .mutation_started()
+ );
+
+ let first_pool = store.pools[0].disk_set[0]
+ .load_file_info_versions_exact(&bucket, object)
+ .await
+ .expect("first pool metadata should load")
+ .expect("the trigger should remain");
+ let first_pool_ordinary: Vec<&FileInfo> = first_pool
+ .versions
+ .iter()
+ .filter(|version| !version.tier_free_version())
+ .collect();
+ assert_eq!(first_pool_ordinary.len(), 1);
+ assert_eq!(first_pool_ordinary[0].version_id, Some(marker_id));
+ assert!(first_pool_ordinary[0].deleted);
+
+ *LIFECYCLE_DELETE_ALL_TEST_FAILURE
+ .lock()
+ .expect("lifecycle delete-all failure hook should not poison") = None;
+ store
+ .delete_prefix(&bucket, object, &opts)
+ .await
+ .expect("retry should delete remaining history and its trigger owner");
+ for pool in &store.pools {
+ assert!(
+ pool.disk_set[0]
+ .load_file_info_versions_exact(&bucket, object)
+ .await
+ .expect("pool metadata should load after retry")
+ .is_none(),
+ "all ordinary versions should be removed after retry"
+ );
+ }
+
+ shutdown.cancel();
+ }
+
+ #[tokio::test]
+ #[serial_test::serial]
+ async fn lifecycle_delete_all_phase_failures_preserve_barriers_and_retry() {
+ let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("lifecycle-delete-all-phases", &[4, 4]).await;
+ let bucket = format!("lifecycle-delete-all-phases-{}", Uuid::new_v4().simple());
+ store
+ .make_bucket(&bucket, &MakeBucketOptions::default())
+ .await
+ .expect("bucket should be created in both pools");
+ let _failure_guard = LifecycleDeleteAllFailureGuard;
+
+ for (object, phase, expected_counts, mutation_started) in [
+ ("preflight-failure", crate::object_api::LifecycleDeleteAllPhase::Preflight, [2, 2], false),
+ (
+ "final-preflight-failure",
+ crate::object_api::LifecycleDeleteAllPhase::FinalPreflight,
+ [1, 1],
+ true,
+ ),
+ ("trigger-failure", crate::object_api::LifecycleDeleteAllPhase::Trigger, [0, 1], true),
+ ] {
+ let opts = seed_multi_pool_delete_all(&store, &bucket, object).await;
+ *LIFECYCLE_DELETE_ALL_TEST_FAILURE
+ .lock()
+ .expect("lifecycle delete-all failure hook should not poison") = Some((phase, 1));
+ let err = store
+ .delete_prefix(&bucket, object, &opts)
+ .await
+ .expect_err("injected phase failure should stop the transaction");
+ assert_eq!(err, StorageError::PreconditionFailed);
+ assert_eq!(
+ opts.lifecycle_delete_all_journal()
+ .expect("delete-all journal should be initialized")
+ .lock()
+ .mutation_started(),
+ mutation_started
+ );
+ assert_eq!(ordinary_version_count(&store, 0, &bucket, object).await, expected_counts[0]);
+ assert_eq!(ordinary_version_count(&store, 1, &bucket, object).await, expected_counts[1]);
+
+ *LIFECYCLE_DELETE_ALL_TEST_FAILURE
+ .lock()
+ .expect("lifecycle delete-all failure hook should not poison") = None;
+ store
+ .delete_prefix(&bucket, object, &opts)
+ .await
+ .expect("retry should converge after the injected failure is removed");
+ assert_eq!(ordinary_version_count(&store, 0, &bucket, object).await, 0);
+ assert_eq!(ordinary_version_count(&store, 1, &bucket, object).await, 0);
+ }
+
+ shutdown.cancel();
+ }
+
fn assert_backend_layout_empty(info: &rustfs_madmin::BackendInfo) {
assert!(info.standard_sc_parities.is_empty());
assert!(info.standard_sc_data.is_empty());
diff --git a/crates/lifecycle/src/core.rs b/crates/lifecycle/src/core.rs
index 0fe52b9df..4d1f5298d 100644
--- a/crates/lifecycle/src/core.rs
+++ b/crates/lifecycle/src/core.rs
@@ -50,6 +50,9 @@ const ERR_LIFECYCLE_INVALID_ABORT_INCOMPLETE_MPU_DAYS: &str =
const ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT: &str = "Expiration.Date must be at midnight UTC";
const ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_DELETE_MARKER: &str =
"ExpiredObjectDeleteMarker cannot be specified with Days or Date";
+const ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_ALL_VERSIONS: &str =
+ "Days must be a positive integer and Date must not be specified inside Expiration with ExpiredObjectAllVersions";
+const ERR_LIFECYCLE_INVALID_DEL_MARKER_EXPIRATION_DAYS: &str = "Days must be a positive integer with DelMarkerExpiration";
const ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG: &str = "Rule ID must be at most 255 characters";
const ERR_LIFECYCLE_INVALID_RULE_STATUS: &str = "Rule status must be either Enabled or Disabled";
const ERR_LIFECYCLE_DEL_MARKER_WITH_TAGS: &str = "Rule with DelMarkerExpiration cannot have tags based filtering";
@@ -155,6 +158,13 @@ impl RuleValidate for LifecycleRule {
{
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_DELETE_MARKER));
}
+ if self
+ .del_marker_expiration
+ .as_ref()
+ .is_some_and(|expiration| expiration.days.is_none_or(|days| days < 1))
+ {
+ return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_DEL_MARKER_EXPIRATION_DAYS));
+ }
// Rule must have at least one action
let has_expiration = self.expiration.is_some();
let has_transition = self.transitions.as_ref().is_some_and(|t| !t.is_empty());
@@ -291,6 +301,14 @@ impl Lifecycle for BucketLifecycleConfiguration {
{
return true;
}
+ if rule
+ .del_marker_expiration
+ .as_ref()
+ .and_then(|expiration| expiration.days)
+ .is_some_and(|days| days > 0)
+ {
+ return true;
+ }
if let Some(rule_expiration) = &rule.expiration {
if let Some(date1) = rule_expiration.date.clone()
&& OffsetDateTime::from(date1).unix_timestamp() < OffsetDateTime::now_utc().unix_timestamp()
@@ -337,6 +355,11 @@ impl Lifecycle for BucketLifecycleConfiguration {
}
if let Some(expiration) = &r.expiration {
+ if expiration.expired_object_all_versions.is_some()
+ && (expiration.days.is_none_or(|days| days < 1) || expiration.date.is_some())
+ {
+ return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_ALL_VERSIONS));
+ }
if let Some(expiration_date) = &expiration.date {
let date = OffsetDateTime::from(expiration_date.clone());
if date.hour() != 0 || date.minute() != 0 || date.second() != 0 || date.nanosecond() != 0 {
@@ -563,25 +586,25 @@ impl Lifecycle for BucketLifecycleConfiguration {
});
}
}
- // DelMarkerExpiration: expire delete marker after N days from mod_time
- if obj.delete_marker
- && let Some(ref dme) = rule.del_marker_expiration
- && let Some(days) = dme.days
- && days > 0
- {
- let due = expected_expiry_time(mod_time, days);
- if now.unix_timestamp() >= due.unix_timestamp() {
- events.push(Event {
- action: IlmAction::DelMarkerDeleteAllVersionsAction,
- rule_id: rule.id.clone().unwrap_or_default(),
- due: Some(due),
- noncurrent_days: 0,
- newer_noncurrent_versions: 0,
- storage_class: "".into(),
- });
- }
- continue;
+ }
+
+ if obj.is_latest
+ && obj.delete_marker
+ && let Some(days) = rule.del_marker_expiration.as_ref().and_then(|expiration| expiration.days)
+ && days > 0
+ {
+ let due = expected_expiry_time(mod_time, days);
+ if now.unix_timestamp() >= due.unix_timestamp() {
+ events.push(Event {
+ action: IlmAction::DelMarkerDeleteAllVersionsAction,
+ rule_id: rule.id.clone().unwrap_or_default(),
+ due: Some(due),
+ noncurrent_days: 0,
+ newer_noncurrent_versions: 0,
+ storage_class: "".into(),
+ });
}
+ continue;
}
if !obj.is_latest
@@ -1105,6 +1128,25 @@ mod tests {
});
}
+ fn enabled_rule(
+ expiration: Option,
+ del_marker_expiration: Option,
+ id: Option<&str>,
+ ) -> LifecycleRule {
+ LifecycleRule {
+ status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+ expiration,
+ abort_incomplete_multipart_upload: None,
+ del_marker_expiration,
+ filter: None,
+ id: id.map(str::to_string),
+ noncurrent_version_expiration: None,
+ noncurrent_version_transitions: None,
+ prefix: None,
+ transitions: None,
+ }
+ }
+
#[test]
fn eval_inner_reports_invalid_mod_time_without_expiring_object() {
let lifecycle = BucketLifecycleConfiguration {
@@ -1344,6 +1386,18 @@ mod tests {
assert!(lc.has_active_rules("test/"));
}
+ #[test]
+ fn has_active_rules_requires_valid_del_marker_expiration_days() {
+ let lifecycle = |days| BucketLifecycleConfiguration {
+ expiry_updated_at: None,
+ rules: vec![enabled_rule(None, Some(s3s::dto::DelMarkerExpiration { days }), None)],
+ };
+
+ assert!(lifecycle(Some(1)).has_active_rules(""));
+ assert!(!lifecycle(Some(0)).has_active_rules(""));
+ assert!(!lifecycle(None).has_active_rules(""));
+ }
+
#[tokio::test]
async fn validate_rejects_zero_noncurrent_expiration_days() {
// S3 compatibility: NoncurrentVersionExpiration.NoncurrentDays must be a positive
@@ -3182,33 +3236,68 @@ mod tests {
}
#[tokio::test]
- async fn validate_rejects_zero_day_del_marker_expiration_on_locked_bucket() {
+ async fn validate_rejects_invalid_del_marker_expiration_days_even_with_another_action() {
+ for days in [None, Some(0), Some(-1)] {
+ let lc = BucketLifecycleConfiguration {
+ expiry_updated_at: None,
+ rules: vec![LifecycleRule {
+ status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+ expiration: Some(LifecycleExpiration {
+ days: Some(30),
+ ..Default::default()
+ }),
+ abort_incomplete_multipart_upload: None,
+ del_marker_expiration: Some(s3s::dto::DelMarkerExpiration { days }),
+ filter: None,
+ id: Some("test-rule".to_string()),
+ noncurrent_version_expiration: None,
+ noncurrent_version_transitions: None,
+ prefix: None,
+ transitions: None,
+ }],
+ };
+
+ let err = lc.validate(&ObjectLockConfiguration::default()).await.unwrap_err();
+ assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_DEL_MARKER_EXPIRATION_DAYS);
+ }
+ }
+
+ #[tokio::test]
+ #[serial]
+ async fn del_marker_expiration_deletes_marker_and_older_versions_when_due() {
+ let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("fixed timestamp should be valid");
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
- rules: vec![LifecycleRule {
- status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
- expiration: Some(LifecycleExpiration {
- days: Some(30),
- ..Default::default()
- }),
- abort_incomplete_multipart_upload: None,
- del_marker_expiration: Some(s3s::dto::DelMarkerExpiration { days: Some(0) }),
- filter: None,
- id: Some("test-rule".to_string()),
- noncurrent_version_expiration: None,
- noncurrent_version_transitions: None,
- prefix: None,
- transitions: None,
- }],
+ rules: vec![enabled_rule(
+ None,
+ Some(s3s::dto::DelMarkerExpiration { days: Some(3) }),
+ Some("delete-marker-history"),
+ )],
};
-
- let locked_config = ObjectLockConfiguration {
- object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
+ let marker_with_history = ObjectOpts {
+ name: "obj".to_string(),
+ mod_time: Some(base_time),
+ is_latest: true,
+ delete_marker: true,
+ num_versions: 3,
+ version_id: Some(Uuid::new_v4()),
..Default::default()
};
+ let due = expected_expiry_time(base_time, 3);
- let err = lc.validate(&locked_config).await.unwrap_err();
- assert_eq!(err.to_string(), ERR_LIFECYCLE_BUCKET_LOCKED);
+ let before_due = lc.eval_inner(&marker_with_history, due - Duration::seconds(1), 0).await;
+ assert_eq!(before_due.action, IlmAction::NoneAction);
+
+ let at_due = lc.eval_inner(&marker_with_history, due, 0).await;
+ assert_eq!(at_due.action, IlmAction::DelMarkerDeleteAllVersionsAction);
+ assert_eq!(at_due.rule_id, "delete-marker-history");
+ assert_eq!(at_due.due, Some(due));
+
+ let current_data = ObjectOpts {
+ delete_marker: false,
+ ..marker_with_history
+ };
+ assert_eq!(lc.eval_inner(¤t_data, due, 0).await.action, IlmAction::NoneAction);
}
// --- TASK-003 tests: Round up to next UTC processing boundary ---
@@ -3737,6 +3826,52 @@ mod tests {
.expect("ExpiredObjectAllVersions should be allowed on unlocked bucket");
}
+ #[tokio::test]
+ async fn validate_rejects_expired_object_all_versions_without_days_or_with_date() {
+ let expiration_date = datetime!(2025-01-01 00:00:00 UTC);
+ let invalid_expirations = [
+ LifecycleExpiration {
+ expired_object_all_versions: Some(true),
+ ..Default::default()
+ },
+ LifecycleExpiration {
+ date: Some(expiration_date.into()),
+ days: Some(1),
+ expired_object_all_versions: Some(true),
+ ..Default::default()
+ },
+ ];
+
+ for expiration in invalid_expirations {
+ let lc = BucketLifecycleConfiguration {
+ expiry_updated_at: None,
+ rules: vec![enabled_rule(Some(expiration), None, None)],
+ };
+
+ let err = lc.validate(&ObjectLockConfiguration::default()).await.unwrap_err();
+ assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_ALL_VERSIONS);
+ }
+ }
+
+ #[tokio::test]
+ async fn validate_rejects_false_expired_object_all_versions_with_date() {
+ let lc = BucketLifecycleConfiguration {
+ expiry_updated_at: None,
+ rules: vec![enabled_rule(
+ Some(LifecycleExpiration {
+ date: Some(datetime!(2025-01-01 00:00:00 UTC).into()),
+ expired_object_all_versions: Some(false),
+ ..Default::default()
+ }),
+ None,
+ None,
+ )],
+ };
+
+ let err = lc.validate(&ObjectLockConfiguration::default()).await.unwrap_err();
+ assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_ALL_VERSIONS);
+ }
+
#[tokio::test]
#[serial]
async fn eval_inner_triggers_delete_all_versions_when_expired_object_all_versions_set() {
@@ -3776,6 +3911,36 @@ mod tests {
assert_eq!(event.rule_id, "all-versions-rule");
}
+ #[tokio::test]
+ #[serial]
+ async fn expired_object_all_versions_does_not_apply_to_current_delete_marker() {
+ let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("fixed timestamp should be valid");
+ let lc = BucketLifecycleConfiguration {
+ expiry_updated_at: None,
+ rules: vec![enabled_rule(
+ Some(LifecycleExpiration {
+ days: Some(1),
+ expired_object_all_versions: Some(true),
+ ..Default::default()
+ }),
+ None,
+ Some("all-versions-rule"),
+ )],
+ };
+ let marker_with_history = ObjectOpts {
+ name: "obj".to_string(),
+ mod_time: Some(base_time),
+ is_latest: true,
+ delete_marker: true,
+ num_versions: 2,
+ version_id: Some(Uuid::new_v4()),
+ ..Default::default()
+ };
+
+ let event = lc.eval_inner(&marker_with_history, base_time + Duration::days(2), 0).await;
+ assert_eq!(event.action, IlmAction::NoneAction);
+ }
+
#[tokio::test]
#[serial]
async fn eval_inner_uses_delete_action_when_all_versions_not_set() {
diff --git a/crates/scanner/tests/lifecycle_integration_test.rs b/crates/scanner/tests/lifecycle_integration_test.rs
index 71fe961d2..12d928513 100644
--- a/crates/scanner/tests/lifecycle_integration_test.rs
+++ b/crates/scanner/tests/lifecycle_integration_test.rs
@@ -1058,318 +1058,330 @@ mod serial_tests {
}
}
- #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+ #[test]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
- async fn test_transition_and_restore_flows() {
- async move {
- let (disk_paths, ecstore) = setup_test_env().await;
+ fn test_transition_and_restore_flows() {
+ std::thread::Builder::new()
+ .name("scanner-transition-restore-flows".to_string())
+ .stack_size(32 * 1024 * 1024)
+ .spawn(|| {
+ let runtime = tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .expect("transition and restore test runtime should build");
- let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
- let backend = register_mock_tier(&tier_name).await;
+ runtime.block_on(test_transition_and_restore_flows_inner());
+ })
+ .expect("transition and restore test thread should spawn")
+ .join()
+ .expect("transition and restore test thread should finish");
+ }
- let put_bucket = format!("test-immediate-put-{}", &Uuid::new_v4().simple().to_string()[..8]);
- let put_object = "test/object.txt";
- let put_payload = b"Hello, immediate transition!";
+ async fn test_transition_and_restore_flows_inner() {
+ let (disk_paths, ecstore) = setup_test_env().await;
- create_test_bucket(&ecstore, put_bucket.as_str()).await;
- set_bucket_lifecycle_transition_with_tier(put_bucket.as_str(), &tier_name)
+ let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
+ let backend = register_mock_tier(&tier_name).await;
+
+ let put_bucket = format!("test-immediate-put-{}", &Uuid::new_v4().simple().to_string()[..8]);
+ let put_object = "test/object.txt";
+ let put_payload = b"Hello, immediate transition!";
+
+ create_test_bucket(&ecstore, put_bucket.as_str()).await;
+ set_bucket_lifecycle_transition_with_tier(put_bucket.as_str(), &tier_name)
+ .await
+ .expect("Failed to set lifecycle configuration");
+
+ let mut reader = PutObjReader::from_vec(put_payload.to_vec());
+ let mut metadata = HashMap::new();
+ metadata.insert("content-type".to_string(), "text/plain".to_string());
+ ecstore
+ .put_object(
+ put_bucket.as_str(),
+ put_object,
+ &mut reader,
+ &ObjectOptions {
+ user_defined: metadata,
+ ..Default::default()
+ },
+ )
+ .await
+ .expect("Failed to upload transition metadata test object");
+
+ enqueue_transition_for_existing_objects(ecstore.clone(), put_bucket.as_str())
+ .await
+ .expect("Failed to enqueue transitioned put object");
+
+ let put_info = wait_for_transition(&ecstore, put_bucket.as_str(), put_object, TRANSITION_WAIT_TIMEOUT)
+ .await
+ .expect("object should transition after enqueueing existing objects");
+
+ assert_eq!(put_info.transitioned_object.status, "complete");
+ assert_eq!(put_info.transitioned_object.tier, tier_name);
+ assert!(backend.contains(&put_info.transitioned_object.name).await);
+ {
+ let transitioned = backend
+ .stored(&put_info.transitioned_object.name)
.await
- .expect("Failed to set lifecycle configuration");
-
- let mut reader = PutObjReader::from_vec(put_payload.to_vec());
- let mut metadata = HashMap::new();
- metadata.insert("content-type".to_string(), "text/plain".to_string());
- ecstore
- .put_object(
- put_bucket.as_str(),
- put_object,
- &mut reader,
- &ObjectOptions {
- user_defined: metadata,
- ..Default::default()
- },
- )
- .await
- .expect("Failed to upload transition metadata test object");
-
- enqueue_transition_for_existing_objects(ecstore.clone(), put_bucket.as_str())
- .await
- .expect("Failed to enqueue transitioned put object");
-
- let put_info = wait_for_transition(&ecstore, put_bucket.as_str(), put_object, TRANSITION_WAIT_TIMEOUT)
- .await
- .expect("object should transition after enqueueing existing objects");
-
- assert_eq!(put_info.transitioned_object.status, "complete");
- assert_eq!(put_info.transitioned_object.tier, tier_name);
- assert!(backend.contains(&put_info.transitioned_object.name).await);
- {
- let transitioned = backend
- .stored(&put_info.transitioned_object.name)
- .await
- .expect("transitioned object should be present in mock backend");
- assert_eq!(transitioned.metadata.get("content-type"), Some(&"text/plain".to_string()));
- assert!(
- !transitioned.metadata.contains_key("x-amz-replication-status"),
- "transitioned objects must not inherit replication status defaults"
- );
- assert!(
- !transitioned.metadata.contains_key("x-amz-object-lock-legal-hold"),
- "transitioned objects must not invent object lock headers"
- );
- }
-
- // Cross-shard xl.meta transition assertion helper (rustfs/backlog#1148 ilm-6):
- // every disk must agree on the transition tuple for the object.
- let put_meta = assert_transition_meta_consistent(&disk_paths, put_bucket.as_str(), put_object).await;
- assert_eq!(put_meta.status, "complete");
- assert_eq!(put_meta.tier, tier_name);
-
- let multipart_bucket = format!("test-immediate-mpu-{}", &Uuid::new_v4().simple().to_string()[..8]);
- let multipart_object = "test/multipart.txt";
-
- create_test_bucket(&ecstore, multipart_bucket.as_str()).await;
- set_bucket_lifecycle_transition_with_tier(multipart_bucket.as_str(), &tier_name)
- .await
- .expect("Failed to set lifecycle configuration");
-
- let upload = ecstore
- .new_multipart_upload(multipart_bucket.as_str(), multipart_object, &ObjectOptions::default())
- .await
- .expect("Failed to create multipart upload");
-
- let part_data = b"multipart immediate transition";
- let mut reader = PutObjReader::from_vec(part_data.to_vec());
- let part = ecstore
- .put_object_part(
- multipart_bucket.as_str(),
- multipart_object,
- &upload.upload_id,
- 1,
- &mut reader,
- &ObjectOptions::default(),
- )
- .await
- .expect("Failed to upload multipart part");
-
- ecstore
- .clone()
- .complete_multipart_upload(
- multipart_bucket.as_str(),
- multipart_object,
- &upload.upload_id,
- vec![CompletePart {
- part_num: 1,
- etag: part.etag.clone(),
- ..Default::default()
- }],
- &ObjectOptions::default(),
- )
- .await
- .expect("Failed to complete multipart upload");
-
- enqueue_transition_for_existing_objects(ecstore.clone(), multipart_bucket.as_str())
- .await
- .expect("Failed to enqueue transitioned multipart object");
-
- let multipart_info =
- wait_for_transition(&ecstore, multipart_bucket.as_str(), multipart_object, TRANSITION_WAIT_TIMEOUT)
- .await
- .expect("object should transition after enqueueing existing objects");
-
- assert_eq!(multipart_info.transitioned_object.status, "complete");
- assert_eq!(multipart_info.transitioned_object.tier, tier_name);
- assert!(backend.contains(&multipart_info.transitioned_object.name).await);
-
- let src_bucket = format!("test-immediate-copy-src-{}", &Uuid::new_v4().simple().to_string()[..8]);
- let dst_bucket = format!("test-immediate-copy-dst-{}", &Uuid::new_v4().simple().to_string()[..8]);
- let src_object = "test/source.txt";
- let dst_object = "test/copied.txt";
- let payload = b"copy object immediate transition";
-
- create_test_bucket(&ecstore, src_bucket.as_str()).await;
- create_test_bucket(&ecstore, dst_bucket.as_str()).await;
- set_bucket_lifecycle_transition_with_tier(dst_bucket.as_str(), &tier_name)
- .await
- .expect("Failed to set destination lifecycle configuration");
-
- upload_test_object(&ecstore, src_bucket.as_str(), src_object, payload).await;
-
- let mut src_info = ecstore
- .get_object_info(src_bucket.as_str(), src_object, &ObjectOptions::default())
- .await
- .expect("Failed to load source object info");
- src_info.put_object_reader = Some(PutObjReader::from_vec(payload.to_vec()));
-
- ecstore
- .copy_object(
- src_bucket.as_str(),
- src_object,
- dst_bucket.as_str(),
- dst_object,
- &mut src_info,
- &ObjectOptions::default(),
- &ObjectOptions::default(),
- )
- .await
- .expect("Failed to copy object");
-
- enqueue_transition_for_existing_objects(ecstore.clone(), dst_bucket.as_str())
- .await
- .expect("Failed to enqueue transitioned copied object");
-
- let copy_info = wait_for_transition(&ecstore, dst_bucket.as_str(), dst_object, TRANSITION_WAIT_TIMEOUT)
- .await
- .expect("copied object should transition after enqueueing existing objects");
-
- assert_eq!(copy_info.transitioned_object.status, "complete");
- assert_eq!(copy_info.transitioned_object.tier, tier_name);
- assert!(backend.contains(©_info.transitioned_object.name).await);
-
- let bucket_name = format!("test-lifecycle-update-{}", &Uuid::new_v4().simple().to_string()[..8]);
- let object_name = "test/existing.txt";
- let payload = b"existing object before lifecycle";
-
- create_test_bucket(&ecstore, bucket_name.as_str()).await;
- upload_test_object(&ecstore, bucket_name.as_str(), object_name, payload).await;
-
- set_bucket_lifecycle_transition_with_tier(bucket_name.as_str(), &tier_name)
- .await
- .expect("Failed to set lifecycle configuration");
-
- enqueue_transition_for_existing_objects(ecstore.clone(), bucket_name.as_str())
- .await
- .expect("Failed to enqueue transition for existing objects");
-
- let info = wait_for_transition(&ecstore, bucket_name.as_str(), object_name, TRANSITION_WAIT_TIMEOUT)
- .await
- .expect("existing object should transition after lifecycle update");
-
- assert_eq!(info.transitioned_object.status, "complete");
- assert_eq!(info.transitioned_object.tier, tier_name);
- assert!(backend.contains(&info.transitioned_object.name).await);
-
- let bucket_name = format!("test-restore-mpu-{}", &Uuid::new_v4().simple().to_string()[..8]);
- let object_name = "test/restore.txt";
- let part1 = vec![b'a'; 5 * 1024 * 1024];
- let part2 = b"restored-tail".to_vec();
- let expected = [part1.clone(), part2.clone()].concat();
-
- create_test_bucket(&ecstore, bucket_name.as_str()).await;
- set_bucket_lifecycle_transition_with_tier(bucket_name.as_str(), &tier_name)
- .await
- .expect("Failed to set lifecycle configuration");
-
- let upload = ecstore
- .new_multipart_upload(bucket_name.as_str(), object_name, &ObjectOptions::default())
- .await
- .expect("Failed to create multipart upload");
-
- let mut part1_reader = PutObjReader::from_vec(part1);
- let uploaded_part1 = ecstore
- .put_object_part(
- bucket_name.as_str(),
- object_name,
- &upload.upload_id,
- 1,
- &mut part1_reader,
- &ObjectOptions::default(),
- )
- .await
- .expect("Failed to upload first multipart part");
-
- let mut part2_reader = PutObjReader::from_vec(part2);
- let uploaded_part2 = ecstore
- .put_object_part(
- bucket_name.as_str(),
- object_name,
- &upload.upload_id,
- 2,
- &mut part2_reader,
- &ObjectOptions::default(),
- )
- .await
- .expect("Failed to upload second multipart part");
-
- ecstore
- .clone()
- .complete_multipart_upload(
- bucket_name.as_str(),
- object_name,
- &upload.upload_id,
- vec![
- CompletePart {
- part_num: 1,
- etag: uploaded_part1.etag.clone(),
- ..Default::default()
- },
- CompletePart {
- part_num: 2,
- etag: uploaded_part2.etag.clone(),
- ..Default::default()
- },
- ],
- &ObjectOptions::default(),
- )
- .await
- .expect("Failed to complete multipart upload");
-
- enqueue_transition_for_existing_objects(ecstore.clone(), bucket_name.as_str())
- .await
- .expect("Failed to enqueue transitioned restore object");
-
- let transitioned = wait_for_transition(&ecstore, bucket_name.as_str(), object_name, TRANSITION_WAIT_TIMEOUT)
- .await
- .expect("multipart object should transition after enqueueing existing objects");
- assert_eq!(transitioned.parts.len(), 2);
-
- ecstore
- .clone()
- .restore_transitioned_object(
- bucket_name.as_str(),
- object_name,
- &ObjectOptions {
- transition: TransitionOptions {
- restore_request: RestoreRequest {
- days: Some(1),
- description: None,
- glacier_job_parameters: None,
- output_location: None,
- select_parameters: None,
- tier: None,
- type_: None,
- },
- ..Default::default()
- },
- ..Default::default()
- },
- )
- .await
- .expect("Failed to restore transitioned multipart object");
-
- let restored = ecstore
- .get_object_info(bucket_name.as_str(), object_name, &ObjectOptions::default())
- .await
- .expect("Failed to load restored object info");
- assert_eq!(restored.parts.len(), 2);
- assert!(restored.restore_expires.is_some());
- assert!(!restored.restore_ongoing);
-
- let mut reader = ecstore
- .get_object_reader(bucket_name.as_str(), object_name, None, http::HeaderMap::new(), &ObjectOptions::default())
- .await
- .expect("Failed to read restored object");
- let mut data = Vec::new();
- reader
- .stream
- .read_to_end(&mut data)
- .await
- .expect("Failed to consume restored object stream");
- assert_eq!(data, expected);
+ .expect("transitioned object should be present in mock backend");
+ assert_eq!(transitioned.metadata.get("content-type"), Some(&"text/plain".to_string()));
+ assert!(
+ !transitioned.metadata.contains_key("x-amz-replication-status"),
+ "transitioned objects must not inherit replication status defaults"
+ );
+ assert!(
+ !transitioned.metadata.contains_key("x-amz-object-lock-legal-hold"),
+ "transitioned objects must not invent object lock headers"
+ );
}
- .boxed_local()
- .await;
+
+ // Cross-shard xl.meta transition assertion helper (rustfs/backlog#1148 ilm-6):
+ // every disk must agree on the transition tuple for the object.
+ let put_meta = assert_transition_meta_consistent(&disk_paths, put_bucket.as_str(), put_object).await;
+ assert_eq!(put_meta.status, "complete");
+ assert_eq!(put_meta.tier, tier_name);
+
+ let multipart_bucket = format!("test-immediate-mpu-{}", &Uuid::new_v4().simple().to_string()[..8]);
+ let multipart_object = "test/multipart.txt";
+
+ create_test_bucket(&ecstore, multipart_bucket.as_str()).await;
+ set_bucket_lifecycle_transition_with_tier(multipart_bucket.as_str(), &tier_name)
+ .await
+ .expect("Failed to set lifecycle configuration");
+
+ let upload = ecstore
+ .new_multipart_upload(multipart_bucket.as_str(), multipart_object, &ObjectOptions::default())
+ .await
+ .expect("Failed to create multipart upload");
+
+ let part_data = b"multipart immediate transition";
+ let mut reader = PutObjReader::from_vec(part_data.to_vec());
+ let part = ecstore
+ .put_object_part(
+ multipart_bucket.as_str(),
+ multipart_object,
+ &upload.upload_id,
+ 1,
+ &mut reader,
+ &ObjectOptions::default(),
+ )
+ .await
+ .expect("Failed to upload multipart part");
+
+ ecstore
+ .clone()
+ .complete_multipart_upload(
+ multipart_bucket.as_str(),
+ multipart_object,
+ &upload.upload_id,
+ vec![CompletePart {
+ part_num: 1,
+ etag: part.etag.clone(),
+ ..Default::default()
+ }],
+ &ObjectOptions::default(),
+ )
+ .await
+ .expect("Failed to complete multipart upload");
+
+ enqueue_transition_for_existing_objects(ecstore.clone(), multipart_bucket.as_str())
+ .await
+ .expect("Failed to enqueue transitioned multipart object");
+
+ let multipart_info = wait_for_transition(&ecstore, multipart_bucket.as_str(), multipart_object, TRANSITION_WAIT_TIMEOUT)
+ .await
+ .expect("object should transition after enqueueing existing objects");
+
+ assert_eq!(multipart_info.transitioned_object.status, "complete");
+ assert_eq!(multipart_info.transitioned_object.tier, tier_name);
+ assert!(backend.contains(&multipart_info.transitioned_object.name).await);
+
+ let src_bucket = format!("test-immediate-copy-src-{}", &Uuid::new_v4().simple().to_string()[..8]);
+ let dst_bucket = format!("test-immediate-copy-dst-{}", &Uuid::new_v4().simple().to_string()[..8]);
+ let src_object = "test/source.txt";
+ let dst_object = "test/copied.txt";
+ let payload = b"copy object immediate transition";
+
+ create_test_bucket(&ecstore, src_bucket.as_str()).await;
+ create_test_bucket(&ecstore, dst_bucket.as_str()).await;
+ set_bucket_lifecycle_transition_with_tier(dst_bucket.as_str(), &tier_name)
+ .await
+ .expect("Failed to set destination lifecycle configuration");
+
+ upload_test_object(&ecstore, src_bucket.as_str(), src_object, payload).await;
+
+ let mut src_info = ecstore
+ .get_object_info(src_bucket.as_str(), src_object, &ObjectOptions::default())
+ .await
+ .expect("Failed to load source object info");
+ src_info.put_object_reader = Some(PutObjReader::from_vec(payload.to_vec()));
+
+ ecstore
+ .copy_object(
+ src_bucket.as_str(),
+ src_object,
+ dst_bucket.as_str(),
+ dst_object,
+ &mut src_info,
+ &ObjectOptions::default(),
+ &ObjectOptions::default(),
+ )
+ .await
+ .expect("Failed to copy object");
+
+ enqueue_transition_for_existing_objects(ecstore.clone(), dst_bucket.as_str())
+ .await
+ .expect("Failed to enqueue transitioned copied object");
+
+ let copy_info = wait_for_transition(&ecstore, dst_bucket.as_str(), dst_object, TRANSITION_WAIT_TIMEOUT)
+ .await
+ .expect("copied object should transition after enqueueing existing objects");
+
+ assert_eq!(copy_info.transitioned_object.status, "complete");
+ assert_eq!(copy_info.transitioned_object.tier, tier_name);
+ assert!(backend.contains(©_info.transitioned_object.name).await);
+
+ let bucket_name = format!("test-lifecycle-update-{}", &Uuid::new_v4().simple().to_string()[..8]);
+ let object_name = "test/existing.txt";
+ let payload = b"existing object before lifecycle";
+
+ create_test_bucket(&ecstore, bucket_name.as_str()).await;
+ upload_test_object(&ecstore, bucket_name.as_str(), object_name, payload).await;
+
+ set_bucket_lifecycle_transition_with_tier(bucket_name.as_str(), &tier_name)
+ .await
+ .expect("Failed to set lifecycle configuration");
+
+ enqueue_transition_for_existing_objects(ecstore.clone(), bucket_name.as_str())
+ .await
+ .expect("Failed to enqueue transition for existing objects");
+
+ let info = wait_for_transition(&ecstore, bucket_name.as_str(), object_name, TRANSITION_WAIT_TIMEOUT)
+ .await
+ .expect("existing object should transition after lifecycle update");
+
+ assert_eq!(info.transitioned_object.status, "complete");
+ assert_eq!(info.transitioned_object.tier, tier_name);
+ assert!(backend.contains(&info.transitioned_object.name).await);
+
+ let bucket_name = format!("test-restore-mpu-{}", &Uuid::new_v4().simple().to_string()[..8]);
+ let object_name = "test/restore.txt";
+ let part1 = vec![b'a'; 5 * 1024 * 1024];
+ let part2 = b"restored-tail".to_vec();
+ let expected = [part1.clone(), part2.clone()].concat();
+
+ create_test_bucket(&ecstore, bucket_name.as_str()).await;
+ set_bucket_lifecycle_transition_with_tier(bucket_name.as_str(), &tier_name)
+ .await
+ .expect("Failed to set lifecycle configuration");
+
+ let upload = ecstore
+ .new_multipart_upload(bucket_name.as_str(), object_name, &ObjectOptions::default())
+ .await
+ .expect("Failed to create multipart upload");
+
+ let mut part1_reader = PutObjReader::from_vec(part1);
+ let uploaded_part1 = ecstore
+ .put_object_part(
+ bucket_name.as_str(),
+ object_name,
+ &upload.upload_id,
+ 1,
+ &mut part1_reader,
+ &ObjectOptions::default(),
+ )
+ .await
+ .expect("Failed to upload first multipart part");
+
+ let mut part2_reader = PutObjReader::from_vec(part2);
+ let uploaded_part2 = ecstore
+ .put_object_part(
+ bucket_name.as_str(),
+ object_name,
+ &upload.upload_id,
+ 2,
+ &mut part2_reader,
+ &ObjectOptions::default(),
+ )
+ .await
+ .expect("Failed to upload second multipart part");
+
+ ecstore
+ .clone()
+ .complete_multipart_upload(
+ bucket_name.as_str(),
+ object_name,
+ &upload.upload_id,
+ vec![
+ CompletePart {
+ part_num: 1,
+ etag: uploaded_part1.etag.clone(),
+ ..Default::default()
+ },
+ CompletePart {
+ part_num: 2,
+ etag: uploaded_part2.etag.clone(),
+ ..Default::default()
+ },
+ ],
+ &ObjectOptions::default(),
+ )
+ .await
+ .expect("Failed to complete multipart upload");
+
+ enqueue_transition_for_existing_objects(ecstore.clone(), bucket_name.as_str())
+ .await
+ .expect("Failed to enqueue transitioned restore object");
+
+ let transitioned = wait_for_transition(&ecstore, bucket_name.as_str(), object_name, TRANSITION_WAIT_TIMEOUT)
+ .await
+ .expect("multipart object should transition after enqueueing existing objects");
+ assert_eq!(transitioned.parts.len(), 2);
+
+ ecstore
+ .clone()
+ .restore_transitioned_object(
+ bucket_name.as_str(),
+ object_name,
+ &ObjectOptions {
+ transition: TransitionOptions {
+ restore_request: RestoreRequest {
+ days: Some(1),
+ description: None,
+ glacier_job_parameters: None,
+ output_location: None,
+ select_parameters: None,
+ tier: None,
+ type_: None,
+ },
+ ..Default::default()
+ },
+ ..Default::default()
+ },
+ )
+ .await
+ .expect("Failed to restore transitioned multipart object");
+
+ let restored = ecstore
+ .get_object_info(bucket_name.as_str(), object_name, &ObjectOptions::default())
+ .await
+ .expect("Failed to load restored object info");
+ assert_eq!(restored.parts.len(), 2);
+ assert!(restored.restore_expires.is_some());
+ assert!(!restored.restore_ongoing);
+
+ let mut reader = ecstore
+ .get_object_reader(bucket_name.as_str(), object_name, None, http::HeaderMap::new(), &ObjectOptions::default())
+ .await
+ .expect("Failed to read restored object");
+ let mut data = Vec::new();
+ reader
+ .stream
+ .read_to_end(&mut data)
+ .await
+ .expect("Failed to consume restored object stream");
+ assert_eq!(data, expected);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
diff --git a/rustfs/src/app/lifecycle_transition_api_test.rs b/rustfs/src/app/lifecycle_transition_api_test.rs
index bdf50a5e3..5fb48e3e8 100644
--- a/rustfs/src/app/lifecycle_transition_api_test.rs
+++ b/rustfs/src/app/lifecycle_transition_api_test.rs
@@ -1162,55 +1162,71 @@ async fn complete_multipart_upload_transitions_immediately_via_usecase() {
assert!(backend.contains(&info.transitioned_object.name).await);
}
-#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[test]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
-async fn get_transitioned_object_uses_remote_codec_fallback_path() {
- with_get_codec_streaming_remote_probe_env(|| async {
- let (_disk_paths, ecstore) = setup_test_env().await;
+fn get_transitioned_object_uses_remote_codec_fallback_path() {
+ // CI's default test stack overflows this deep async path; run it on a dedicated large-stack thread.
+ std::thread::Builder::new()
+ .name("lifecycle-transition-remote-codec-fallback".to_string())
+ .stack_size(16 * 1024 * 1024)
+ .spawn(|| {
+ let runtime = tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .expect("remote codec fallback test runtime should build");
- let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
- let backend = register_mock_tier(&tier_name).await;
+ runtime.block_on(async {
+ with_get_codec_streaming_remote_probe_env(|| async {
+ let (_disk_paths, ecstore) = setup_test_env().await;
- let bucket = format!("test-api-get-remote-{}", &Uuid::new_v4().simple().to_string()[..8]);
- let object = "test/remote-codec-fallback.txt";
- let payload: Vec = (0..(1024 * 1024))
- .map(|index| u8::try_from(index % 251).expect("payload byte fits in u8"))
- .collect();
+ let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
+ let backend = register_mock_tier(&tier_name).await;
- create_test_bucket(&ecstore, bucket.as_str()).await;
+ let bucket = format!("test-api-get-remote-{}", &Uuid::new_v4().simple().to_string()[..8]);
+ let object = "test/remote-codec-fallback.txt";
+ let payload: Vec = (0..(1024 * 1024))
+ .map(|index| u8::try_from(index % 251).expect("payload byte fits in u8"))
+ .collect();
- let uploaded = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
- let transition_opts = ObjectOptions {
- transition: lifecycle::lifecycle_contract::TransitionOptions {
- status: lifecycle::lifecycle_contract::TRANSITION_PENDING.to_string(),
- tier: tier_name.clone(),
- etag: uploaded.etag.clone().unwrap_or_default(),
- ..Default::default()
- },
- version_id: uploaded.version_id.map(|version| version.to_string()),
- versioned: true,
- mod_time: uploaded.mod_time,
- ..Default::default()
- };
- ecstore
- .transition_object(bucket.as_str(), object, &transition_opts)
- .await
- .expect("Failed to transition object directly");
+ create_test_bucket(&ecstore, bucket.as_str()).await;
- let transitioned = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
- .await
- .expect("object should transition before remote fallback GET");
+ let uploaded = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
+ let transition_opts = ObjectOptions {
+ transition: lifecycle::lifecycle_contract::TransitionOptions {
+ status: lifecycle::lifecycle_contract::TRANSITION_PENDING.to_string(),
+ tier: tier_name.clone(),
+ etag: uploaded.etag.clone().unwrap_or_default(),
+ ..Default::default()
+ },
+ version_id: uploaded.version_id.map(|version| version.to_string()),
+ versioned: true,
+ mod_time: uploaded.mod_time,
+ ..Default::default()
+ };
+ ecstore
+ .transition_object(bucket.as_str(), object, &transition_opts)
+ .await
+ .expect("Failed to transition object directly");
- assert_eq!(transitioned.transitioned_object.status, "complete");
- assert_eq!(transitioned.transitioned_object.tier, tier_name);
- assert!(!transitioned.transitioned_object.name.is_empty());
- assert!(backend.contains(&transitioned.transitioned_object.name).await);
+ let transitioned = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
+ .await
+ .expect("object should transition before remote fallback GET");
- let actual = read_object_bytes(&ecstore, bucket.as_str(), object).await;
- assert_eq!(actual, payload);
- })
- .await;
+ assert_eq!(transitioned.transitioned_object.status, "complete");
+ assert_eq!(transitioned.transitioned_object.tier, tier_name);
+ assert!(!transitioned.transitioned_object.name.is_empty());
+ assert!(backend.contains(&transitioned.transitioned_object.name).await);
+
+ let actual = read_object_bytes(&ecstore, bucket.as_str(), object).await;
+ assert_eq!(actual, payload);
+ })
+ .await;
+ });
+ })
+ .expect("remote codec fallback test thread should spawn")
+ .join()
+ .expect("remote codec fallback test thread should not panic");
}
/// Regression test for rustfs/rustfs#4827: a duplicate transition task that runs
@@ -2059,9 +2075,7 @@ async fn put_bucket_lifecycle_configuration_rejects_zero_day_del_marker_expirati
assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument);
let message = err.message().unwrap_or_default();
assert!(
- message.contains(
- "ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket"
- ),
+ message.contains("Days must be a positive integer with DelMarkerExpiration"),
"unexpected error message: {message}"
);
}
diff --git a/rustfs/tests/embedded_select_snapshot_test.rs b/rustfs/tests/embedded_select_snapshot_test.rs
index ea15dac6d..16b174d55 100644
--- a/rustfs/tests/embedded_select_snapshot_test.rs
+++ b/rustfs/tests/embedded_select_snapshot_test.rs
@@ -150,7 +150,7 @@ async fn pending_overwrite_keeps_select_on_one_generation(
barrier.wait_until_paused().await;
let response = start_select(client, bucket).await;
- barrier.release_and_wait_until_namespace_pending().await;
+ barrier.release();
let output = collect_select(response).await;
assert_eq!(