mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-30 00:47:13 +00:00
fix(ecstore): migrate tier free versions during decommission (#6393)
This commit is contained in:
@@ -582,6 +582,8 @@ mod tests {
|
||||
wait_for_local_decommission_resume_delay,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::disk::DiskAPI;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::{
|
||||
bucket::lifecycle::{
|
||||
DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord,
|
||||
@@ -1530,6 +1532,77 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn seed_transitioned_free_version(
|
||||
ctx: &Arc<crate::runtime::instance::InstanceContext>,
|
||||
store: &Arc<crate::store::ECStore>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> (uuid::Uuid, uuid::Uuid) {
|
||||
let tier_name = format!("DECOMFREE{}", uuid::Uuid::new_v4().simple());
|
||||
register_mock_tier(&ctx.tier_config_mgr(), &tier_name).await;
|
||||
|
||||
let mut reader = PutObjReader::from_vec(b"transitioned source bytes".to_vec());
|
||||
let source = store.pools[0]
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write transitioned decommission source");
|
||||
let source_version = source.version_id.expect("transitioned source must be versioned");
|
||||
store.pools[0]
|
||||
.transition_object(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(source_version.to_string()),
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name,
|
||||
etag: source.etag.clone().expect("transitioned source must have an ETag"),
|
||||
..Default::default()
|
||||
},
|
||||
mod_time: source.mod_time,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("transition source before decommission");
|
||||
store.pools[0]
|
||||
.delete_object(
|
||||
bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(source_version.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("delete transitioned source version");
|
||||
|
||||
let versions = store.pools[0]
|
||||
.get_disks_by_key(object)
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("source versions should decode after transition delete")
|
||||
.expect("source free version should remain after transition delete");
|
||||
let free_version = versions
|
||||
.versions
|
||||
.iter()
|
||||
.find(|version| version.tier_free_version())
|
||||
.and_then(|version| version.version_id)
|
||||
.expect("transition delete should create a free version");
|
||||
(source_version, free_version)
|
||||
}
|
||||
|
||||
async fn write_decommission_test_multipart_source(
|
||||
store: &Arc<crate::store::ECStore>,
|
||||
pool_idx: usize,
|
||||
@@ -4771,6 +4844,383 @@ mod tests {
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn decommission_entry_skips_cleanup_only_marker_when_free_version_is_present() {
|
||||
let temp_dir = tempfile::tempdir().expect("create free-version decommission store dir");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-free-marker", &[4, 4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("decom-free-marker-{}", uuid::Uuid::new_v4());
|
||||
let object = "free-marker-object";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create free-version decommission bucket");
|
||||
let (_, free_version) = seed_transitioned_free_version(&ctx, &store, &bucket, object).await;
|
||||
let source_free = store.pools[0]
|
||||
.get_disks_by_key(object)
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
.expect("source free-version metadata should decode")
|
||||
.and_then(|versions| {
|
||||
versions
|
||||
.versions
|
||||
.into_iter()
|
||||
.find(|version| version.version_id == Some(free_version) && version.tier_free_version())
|
||||
})
|
||||
.expect("source free-version identity should be present before decommission");
|
||||
let mut target_reader = PutObjReader::from_vec(b"target ordinary bytes".to_vec());
|
||||
let target_w = store.pools[1]
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut target_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write unrelated target version");
|
||||
let target_w_version = target_w.version_id.expect("target version should have an id");
|
||||
let marker = store.pools[0]
|
||||
.delete_object(
|
||||
&bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write cleanup-only delete marker");
|
||||
assert!(marker.delete_marker);
|
||||
|
||||
mark_test_pool_decommissioning(&store, 0).await;
|
||||
let source_set = store.pools[0].get_disks_by_key(object);
|
||||
store
|
||||
.decommission_entry_for_test_with_bucket_incarnation(
|
||||
0,
|
||||
MetaCacheEntry {
|
||||
name: object.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
bucket.clone(),
|
||||
source_set.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("real decommission entry should migrate the free version");
|
||||
|
||||
let target_versions = store.pools[1]
|
||||
.get_disks_by_key(object)
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
.expect("target versions should decode")
|
||||
.expect("target free version should be present");
|
||||
assert!(
|
||||
target_versions
|
||||
.versions
|
||||
.iter()
|
||||
.any(|version| { version.version_id == Some(free_version) && version.tier_free_version() })
|
||||
);
|
||||
let migrated_free = target_versions
|
||||
.versions
|
||||
.iter()
|
||||
.find(|version| version.version_id == Some(free_version) && version.tier_free_version())
|
||||
.expect("migrated free-version identity should remain readable from target disks");
|
||||
assert!(
|
||||
crate::store::tiered_data_movement_source_matches(&source_free, migrated_free)
|
||||
.expect("migrated free-version identity should decode")
|
||||
);
|
||||
let retained_w = target_versions
|
||||
.versions
|
||||
.iter()
|
||||
.find(|version| version.version_id == Some(target_w_version))
|
||||
.expect("unrelated target version should remain");
|
||||
assert!(!retained_w.deleted && !retained_w.tier_free_version());
|
||||
assert_eq!(retained_w.size, target_w.size);
|
||||
assert_eq!(retained_w.get_etag(), target_w.etag);
|
||||
assert!(
|
||||
target_versions
|
||||
.versions
|
||||
.iter()
|
||||
.all(|version| { version.tier_free_version() || !version.deleted })
|
||||
);
|
||||
assert!(
|
||||
source_set
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
.expect("source versions should be readable after cleanup")
|
||||
.is_none(),
|
||||
"successful free-version migration should permit source cleanup"
|
||||
);
|
||||
let (heal_versions, _, _) = store
|
||||
.heal_walk_versions_page(1, 0, &bucket, "", None, 2, 16, true)
|
||||
.await
|
||||
.expect("heal walk should decode the migrated free version");
|
||||
let free_version_string = free_version.to_string();
|
||||
let healed_free = heal_versions
|
||||
.iter()
|
||||
.find(|version| version.version_id.as_deref() == Some(free_version_string.as_str()))
|
||||
.expect("heal walk should surface the migrated free version");
|
||||
let healed_info = healed_free
|
||||
.lifecycle_object_info
|
||||
.as_ref()
|
||||
.expect("heal walk should retain lifecycle identity for the migrated free version");
|
||||
assert!(healed_info.transitioned_object.free_version);
|
||||
assert_eq!(healed_info.transitioned_object.tier, source_free.transition_tier);
|
||||
assert_eq!(healed_info.transitioned_object.name, source_free.transitioned_objname);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn decommission_entry_allows_free_version_consumed_before_source_lock() {
|
||||
let temp_dir = tempfile::tempdir().expect("create consumed free-version store dir");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-free-consumed", &[4, 4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("decom-free-consumed-{}", uuid::Uuid::new_v4());
|
||||
let object = "free-consumed-object";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create consumed free-version bucket");
|
||||
let (_, free_version) = seed_transitioned_free_version(&ctx, &store, &bucket, object).await;
|
||||
|
||||
mark_test_pool_decommissioning(&store, 0).await;
|
||||
let source_set = store.pools[0].get_disks_by_key(object);
|
||||
let barrier = crate::store::object::DecommissionFreeVersionSourceRaceBarrier::install(&bucket, object);
|
||||
let decommission = tokio::spawn({
|
||||
let store = store.clone();
|
||||
let bucket = bucket.clone();
|
||||
let source_set = source_set.clone();
|
||||
async move {
|
||||
store
|
||||
.decommission_entry_for_test_with_bucket_incarnation(
|
||||
0,
|
||||
MetaCacheEntry {
|
||||
name: object.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
bucket,
|
||||
source_set,
|
||||
)
|
||||
.await
|
||||
}
|
||||
});
|
||||
|
||||
barrier.wait_until_paused().await;
|
||||
store.pools[0]
|
||||
.delete_object(
|
||||
&bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(free_version.to_string()),
|
||||
incl_free_versions: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("lifecycle should consume the source free version before decommission locks it");
|
||||
assert!(
|
||||
source_set
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
.expect("consumed source metadata should remain readable")
|
||||
.is_none(),
|
||||
"the lifecycle delete should remove the source free version"
|
||||
);
|
||||
|
||||
barrier.release();
|
||||
decommission
|
||||
.await
|
||||
.expect("decommission task should join")
|
||||
.expect("a concurrently consumed free version should not fail source cleanup");
|
||||
assert!(
|
||||
store.pools[1]
|
||||
.get_disks_by_key(object)
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
.expect("target metadata should remain readable")
|
||||
.is_none(),
|
||||
"an already consumed free version should not be recreated on the target"
|
||||
);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn decommission_entry_rejects_subquorum_free_version_conflict_and_retains_source() {
|
||||
let temp_dir = tempfile::tempdir().expect("create sub-quorum free-version store dir");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-free-conflict", &[4, 4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("decom-free-conflict-{}", uuid::Uuid::new_v4());
|
||||
let object = "free-conflict-object";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create sub-quorum conflict bucket");
|
||||
let (_, free_version) = seed_transitioned_free_version(&ctx, &store, &bucket, object).await;
|
||||
let source_free = store.pools[0]
|
||||
.get_disks_by_key(object)
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
.expect("source free version should decode before crash replay setup")
|
||||
.and_then(|versions| {
|
||||
versions
|
||||
.versions
|
||||
.into_iter()
|
||||
.find(|version| version.version_id == Some(free_version))
|
||||
})
|
||||
.expect("source free version should be available for crash replay setup");
|
||||
|
||||
let mut target_reader = PutObjReader::from_vec(b"target ordinary bytes".to_vec());
|
||||
let target = store.pools[1]
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut target_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed ordinary target version");
|
||||
let target_version = target.version_id.expect("target version must have an ID");
|
||||
let target_disks = store.pools[1].get_disks_by_key(object).disks.read().await.clone();
|
||||
for disk in target_disks.iter().skip(1) {
|
||||
disk.as_ref()
|
||||
.expect("target crash replay quorum disk should be online")
|
||||
.write_metadata("", &bucket, object, source_free.clone())
|
||||
.await
|
||||
.expect("seed an equivalent free version on the target quorum");
|
||||
}
|
||||
let conflict_path = temp_dir
|
||||
.path()
|
||||
.join(format!("pool1/set0/disk0/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
|
||||
let encoded = tokio::fs::read(&conflict_path)
|
||||
.await
|
||||
.expect("target metadata should be readable");
|
||||
let mut metadata = FileMeta::load(&encoded).expect("target metadata should decode");
|
||||
let target_index = metadata
|
||||
.versions
|
||||
.iter()
|
||||
.position(|version| version.header.version_id == Some(target_version))
|
||||
.expect("target version should be present on the conflict disk");
|
||||
let mut target_meta = metadata.versions[target_index]
|
||||
.parse_version_meta()
|
||||
.expect("target version metadata should decode");
|
||||
target_meta
|
||||
.object
|
||||
.as_mut()
|
||||
.expect("target conflict must remain an ordinary object")
|
||||
.version_id = Some(free_version);
|
||||
metadata.versions[target_index] = target_meta.try_into().expect("conflict metadata should encode");
|
||||
let expected_conflict_meta = metadata.versions[target_index].meta.clone();
|
||||
let expected_conflict = metadata.versions[target_index]
|
||||
.into_fileinfo(&bucket, object, true)
|
||||
.expect("conflict metadata should decode as an ordinary object");
|
||||
let duplicate_free: rustfs_filemeta::FileMetaShallowVersion = rustfs_filemeta::FileMetaVersion::from(source_free.clone())
|
||||
.try_into()
|
||||
.expect("duplicate free metadata should encode");
|
||||
metadata.versions.insert(target_index, duplicate_free);
|
||||
tokio::fs::write(&conflict_path, metadata.marshal_msg().expect("conflict metadata should encode"))
|
||||
.await
|
||||
.expect("write sub-quorum conflict metadata");
|
||||
|
||||
mark_test_pool_decommissioning(&store, 0).await;
|
||||
let source_set = store.pools[0].get_disks_by_key(object);
|
||||
store
|
||||
.decommission_entry_for_test_with_bucket_incarnation(
|
||||
0,
|
||||
MetaCacheEntry {
|
||||
name: object.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
bucket.clone(),
|
||||
source_set.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("conflicted decommission entry should retain the source and retry later");
|
||||
|
||||
let source_versions = source_set
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
.expect("retained source versions should decode")
|
||||
.expect("source free version should be retained after conflict");
|
||||
assert!(
|
||||
source_versions
|
||||
.versions
|
||||
.iter()
|
||||
.any(|version| { version.version_id == Some(free_version) && version.tier_free_version() })
|
||||
);
|
||||
let post_encoded = tokio::fs::read(&conflict_path)
|
||||
.await
|
||||
.expect("conflict metadata should remain readable");
|
||||
let post_metadata = FileMeta::load(&post_encoded).expect("post-conflict metadata should decode");
|
||||
let same_id = post_metadata
|
||||
.versions
|
||||
.iter()
|
||||
.filter(|version| version.header.version_id == Some(free_version))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(same_id.len(), 2, "conflict metadata should retain both same-ID records");
|
||||
assert_eq!(same_id.iter().filter(|version| version.header.free_version()).count(), 1);
|
||||
assert_eq!(same_id.iter().filter(|version| !version.header.free_version()).count(), 1);
|
||||
let post_conflict = same_id
|
||||
.into_iter()
|
||||
.find(|version| !version.header.free_version())
|
||||
.expect("ordinary conflict version must remain addressable by the source ID");
|
||||
let post_conflict_info = post_conflict
|
||||
.into_fileinfo(&bucket, object, true)
|
||||
.expect("post-conflict ordinary metadata should decode");
|
||||
assert!(!post_conflict_info.deleted && !post_conflict_info.tier_free_version());
|
||||
assert_eq!(post_conflict.meta, expected_conflict_meta);
|
||||
assert_eq!(post_conflict_info.size, expected_conflict.size);
|
||||
assert_eq!(post_conflict_info.data_dir, expected_conflict.data_dir);
|
||||
assert_eq!(post_conflict_info.metadata, expected_conflict.metadata);
|
||||
assert_eq!(post_conflict_info.get_etag(), expected_conflict.get_etag());
|
||||
|
||||
for disk_index in 0..4 {
|
||||
let target_path = temp_dir
|
||||
.path()
|
||||
.join(format!("pool1/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
|
||||
let target_encoded = tokio::fs::read(&target_path)
|
||||
.await
|
||||
.expect("target metadata should remain readable");
|
||||
let target_meta = FileMeta::load(&target_encoded).expect("target metadata should decode");
|
||||
let same_id = target_meta
|
||||
.versions
|
||||
.iter()
|
||||
.filter(|version| version.header.version_id == Some(free_version))
|
||||
.collect::<Vec<_>>();
|
||||
if disk_index == 0 {
|
||||
assert_eq!(same_id.len(), 2);
|
||||
assert!(same_id[0].header.free_version());
|
||||
assert!(!same_id[1].header.free_version());
|
||||
} else {
|
||||
assert_eq!(same_id.len(), 1);
|
||||
assert!(same_id[0].header.free_version());
|
||||
}
|
||||
}
|
||||
let sweep_err = store
|
||||
.check_after_decommission_for_test(0)
|
||||
.await
|
||||
.expect_err("final sweep must report the retained free version");
|
||||
assert!(
|
||||
sweep_err.to_string().contains("version(s) were found"),
|
||||
"unexpected final sweep error: {sweep_err}"
|
||||
);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn versioned_batch_delete_marker_skips_decommission_source() {
|
||||
|
||||
@@ -151,7 +151,7 @@ pub(crate) mod init_format;
|
||||
pub(crate) mod list_objects;
|
||||
mod multipart;
|
||||
mod object;
|
||||
pub(crate) use object::{ObjectLockDiagGuard, SourceCleanupMutationFence};
|
||||
pub(crate) use object::{ObjectLockDiagGuard, SourceCleanupMutationFence, tiered_data_movement_source_matches};
|
||||
pub use object::{
|
||||
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
|
||||
SnapshotConsistencyError,
|
||||
|
||||
@@ -490,6 +490,82 @@ fn decommission_mutation_fence_for_test(
|
||||
.map(|hook| hook.fence.clone())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct DecommissionFreeVersionSourceRaceState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
arrived: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct DecommissionFreeVersionSourceRaceBarrier {
|
||||
state: Arc<DecommissionFreeVersionSourceRaceState>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static DECOMMISSION_FREE_VERSION_SOURCE_RACE_BARRIER: std::sync::OnceLock<
|
||||
std::sync::Mutex<Option<Arc<DecommissionFreeVersionSourceRaceState>>>,
|
||||
> = std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
impl DecommissionFreeVersionSourceRaceBarrier {
|
||||
pub(crate) fn install(bucket: &str, object: &str) -> Self {
|
||||
let state = Arc::new(DecommissionFreeVersionSourceRaceState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Notify::new(),
|
||||
});
|
||||
let mut slot = DECOMMISSION_FREE_VERSION_SOURCE_RACE_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("decommission free-version source race barrier should not poison");
|
||||
assert!(slot.is_none(), "decommission free-version source race barrier must be unique");
|
||||
*slot = Some(Arc::clone(&state));
|
||||
Self { state }
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_until_paused(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
||||
.await
|
||||
.expect("decommission should pause before acquiring the free-version source lock");
|
||||
}
|
||||
|
||||
pub(crate) fn release(&self) {
|
||||
self.state.release.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for DecommissionFreeVersionSourceRaceBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
let mut slot = DECOMMISSION_FREE_VERSION_SOURCE_RACE_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("decommission free-version source race barrier should not poison");
|
||||
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
|
||||
*slot = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_decommission_free_version_before_source_lock(bucket: &str, object: &str) {
|
||||
let state = DECOMMISSION_FREE_VERSION_SOURCE_RACE_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("decommission free-version source race barrier should not poison")
|
||||
.as_ref()
|
||||
.filter(|state| state.bucket == bucket && state.object == object)
|
||||
.cloned();
|
||||
if let Some(state) = state {
|
||||
state.arrived.notify_one();
|
||||
state.release.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct SourceCleanupMutationFence {
|
||||
guard: ObjectLockDiagGuard,
|
||||
source_lock_covered: bool,
|
||||
@@ -1495,13 +1571,15 @@ fn is_equivalent_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo,
|
||||
&& source_actual_size == target_actual_size
|
||||
}
|
||||
|
||||
fn tiered_data_movement_source_matches(
|
||||
pub(crate) fn tiered_data_movement_source_matches(
|
||||
expected: &rustfs_filemeta::FileInfo,
|
||||
current: &rustfs_filemeta::FileInfo,
|
||||
) -> Result<bool> {
|
||||
let expected_backend = crate::services::tier::tier::tier_destination_id_from_metadata(&expected.metadata)?;
|
||||
let current_backend = crate::services::tier::tier::tier_destination_id_from_metadata(¤t.metadata)?;
|
||||
Ok(expected.version_id == current.version_id
|
||||
&& expected.deleted == current.deleted
|
||||
&& expected.tier_free_version() == current.tier_free_version()
|
||||
&& expected.data_dir == current.data_dir
|
||||
&& expected.mod_time == current.mod_time
|
||||
&& expected.size == current.size
|
||||
@@ -1515,6 +1593,14 @@ fn tiered_data_movement_source_matches(
|
||||
&& expected_backend == current_backend)
|
||||
}
|
||||
|
||||
fn decommission_free_version_overwrite_error(bucket: &str, object: &str, version_id: Option<Uuid>) -> Error {
|
||||
StorageError::DataMovementOverwriteErr(
|
||||
bucket.to_owned(),
|
||||
object.to_owned(),
|
||||
version_id.map(|id| id.to_string()).unwrap_or_default(),
|
||||
)
|
||||
}
|
||||
|
||||
fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
|
||||
target_pool_idx != src_pool_idx
|
||||
}
|
||||
@@ -2222,6 +2308,23 @@ impl ECStore {
|
||||
)
|
||||
}
|
||||
|
||||
async fn has_equivalent_data_movement_tier_free_version(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
source: &rustfs_filemeta::FileInfo,
|
||||
opts: &ObjectOptions,
|
||||
target_pool_idx: usize,
|
||||
) -> Result<bool> {
|
||||
let pool = self
|
||||
.pools
|
||||
.get(target_pool_idx)
|
||||
.ok_or_else(|| Error::other(format!("invalid tiered data movement target pool {target_pool_idx}")))?;
|
||||
pool.get_disks_by_key(object)
|
||||
.has_decommission_tier_free_version_write_quorum(bucket, object, source, opts)
|
||||
.await
|
||||
}
|
||||
|
||||
fn resolve_decommission_target_pool_idx_result(result: Result<usize>, bucket: &str, object: &str) -> Result<usize> {
|
||||
result.map_err(|err| Error::other(format!("failed to select decommission target pool for {bucket}/{object}: {err}")))
|
||||
}
|
||||
@@ -2241,6 +2344,10 @@ impl ECStore {
|
||||
check_put_object_args(bucket, object)?;
|
||||
|
||||
let mut opts = opts.clone();
|
||||
let is_free_version = fi.tier_free_version();
|
||||
if is_free_version {
|
||||
opts.incl_free_versions = true;
|
||||
}
|
||||
let bucket_incarnation_fence = if is_meta_bucketname(bucket) {
|
||||
None
|
||||
} else {
|
||||
@@ -2278,6 +2385,10 @@ impl ECStore {
|
||||
&object,
|
||||
)?
|
||||
};
|
||||
#[cfg(test)]
|
||||
if is_free_version {
|
||||
pause_decommission_free_version_before_source_lock(bucket, logical_object).await;
|
||||
}
|
||||
let _object_guards = self
|
||||
.acquire_data_movement_object_write_locks(bucket, &object, opts.src_pool_idx, idx, &mut opts)
|
||||
.await?;
|
||||
@@ -2295,7 +2406,7 @@ impl ECStore {
|
||||
versions
|
||||
.versions
|
||||
.iter()
|
||||
.find(|current| current.version_id == fi.version_id && !current.tier_free_version())
|
||||
.find(|current| current.version_id == fi.version_id && current.tier_free_version() == is_free_version)
|
||||
})
|
||||
.ok_or_else(|| to_object_err(StorageError::FileNotFound, vec![bucket, object.as_str()]))?;
|
||||
if !tiered_data_movement_source_matches(fi, current_source)? {
|
||||
@@ -2310,24 +2421,34 @@ impl ECStore {
|
||||
.get_available_pool_idx_excluding(bucket, &object, fi.size, opts.src_pool_idx)
|
||||
.await;
|
||||
let target_pool_idx = resolve_data_movement_resume_target_pool(idx, resume_target_pool_idx, opts.src_pool_idx);
|
||||
if self
|
||||
.has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, target_pool_idx)
|
||||
.await?
|
||||
{
|
||||
if is_free_version && target_pool_idx == opts.src_pool_idx {
|
||||
return Err(Error::DiskFull);
|
||||
}
|
||||
let equivalent = if is_free_version {
|
||||
self.has_equivalent_data_movement_tier_free_version(bucket, &object, &fi, &opts, target_pool_idx)
|
||||
.await?
|
||||
} else {
|
||||
self.has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, target_pool_idx)
|
||||
.await?
|
||||
};
|
||||
if equivalent {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
return Err(StorageError::DataMovementOverwriteErr(
|
||||
bucket.to_owned(),
|
||||
object.to_owned(),
|
||||
opts.version_id.clone().unwrap_or_default(),
|
||||
));
|
||||
return Err(decommission_free_version_overwrite_error(bucket, &object, fi.version_id));
|
||||
}
|
||||
|
||||
let result = self.pools[idx]
|
||||
.get_disks_by_key(&object)
|
||||
.decommission_tiered_object(bucket, &object, &fi, &opts)
|
||||
.await;
|
||||
let result = if is_free_version {
|
||||
self.pools[idx]
|
||||
.get_disks_by_key(&object)
|
||||
.decommission_tier_free_version(bucket, &object, &fi, &opts)
|
||||
.await
|
||||
} else {
|
||||
self.pools[idx]
|
||||
.get_disks_by_key(&object)
|
||||
.decommission_tiered_object(bucket, &object, &fi, &opts)
|
||||
.await
|
||||
};
|
||||
if matches!(result, Err(Error::PreconditionFailed)) {
|
||||
if self
|
||||
.has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, idx)
|
||||
|
||||
Reference in New Issue
Block a user