gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-13 16:46:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(rebalance): converge multipart data movement retries (#6057)

Browse Source 
* fix(rebalance): converge multipart data movement retries

* fix(rebalance): harden multipart retry replacement

* fix(rebalance): isolate internal multipart uploads

* test(ecstore): adapt metadata mutation fixtures

* fix(rebalance): preserve transition metadata semantics

* refactor(ecstore): reuse internal metadata matcher

* Revert "refactor(ecstore): reuse internal metadata matcher"

This reverts commit c87ca0328f.

* refactor(rebalance): reuse data movement log constants

* fix(rebalance): isolate migration-owned state

* fix(rebalance): preserve pre-gate retry compatibility
This commit is contained in:
cxymds
2026-08-13 14:12:26 +08:00
committed by GitHub
parent 11eecdc888
commit e11fcfbd08
36 changed files with 6725 additions and 845 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+363 -120
View File
@@ -2871,11 +2871,32 @@ fn stale_upload_default_due(initiated: OffsetDateTime, default_expiry: StdDurati
}
async fn stale_upload_current_size(set: &Arc<SetDisks>, metadata: &HashMap<String, String>, upload_dir: &str) -> Option<usize> {
stale_upload_current_size_with_opts(set, metadata, upload_dir, false).await
}
async fn stale_upload_current_size_with_opts(
set: &Arc<SetDisks>,
metadata: &HashMap<String, String>,
upload_dir: &str,
no_lock: bool,
) -> Option<usize> {
let bucket = metadata.get(RUSTFS_MULTIPART_BUCKET_KEY)?;
let object = metadata.get(RUSTFS_MULTIPART_OBJECT_KEY)?;
let upload_id = encode_stale_upload_id(upload_dir);
let data_movement = rustfs_utils::http::contains_key_str(metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD);
let parts = set
.list_object_parts(bucket, object, &upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
.list_object_parts(
bucket,
object,
&upload_id,
None,
MAX_PARTS_COUNT,
&ObjectOptions {
data_movement,
no_lock,
..Default::default()
},
)
.await
.ok()?;
@@ -2893,7 +2914,12 @@ async fn stale_upload_lifecycle_due(
metadata: &HashMap<String, String>,
initiated: OffsetDateTime,
upload_dir: &str,
no_lock: bool,
) -> Option<OffsetDateTime> {
if rustfs_utils::http::contains_key_str(metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD) {
return None;
}
let bucket = metadata.get(RUSTFS_MULTIPART_BUCKET_KEY)?;
let object = metadata.get(RUSTFS_MULTIPART_OBJECT_KEY)?;
@@ -2906,7 +2932,9 @@ async fn stale_upload_lifecycle_due(
name: object.clone(),
user_tags: metadata.get(AMZ_OBJECT_TAGGING).cloned().unwrap_or_default(),
mod_time: Some(initiated),
size: stale_upload_current_size(set, metadata, upload_dir).await.unwrap_or_default(),
size: stale_upload_current_size_with_opts(set, metadata, upload_dir, no_lock)
.await
.unwrap_or_default(),
is_latest: true,
delete_marker: false,
user_defined: metadata.clone(),
@@ -2934,6 +2962,7 @@ async fn read_stale_multipart_candidate(
FileInfoOpts {
data: false,
include_free_versions: false,
include_part_checksums: false,
},
) {
Ok(file_info) => (Some(file_info.metadata), file_info.mod_time),
@@ -2973,36 +3002,30 @@ fn merge_stale_multipart_candidate(
}
}
fn is_multipart_sha_dir(path: &str) -> bool {
path.len() == 64 && path.bytes().all(|byte| byte.is_ascii_hexdigit())
}
fn multipart_sha_path(root: &str, entry: &str) -> Option<String> {
let sha_dir = entry.trim_end_matches('/');
is_multipart_sha_dir(sha_dir).then(|| {
if root.is_empty() {
sha_dir.to_string()
} else {
format!("{root}/{sha_dir}")
}
})
}
async fn cleanup_empty_multipart_sha_dirs_on_local_disks(set: &Arc<SetDisks>) {
for disk in set.get_local_disks().await.into_iter().flatten() {
if !disk.is_online().await {
continue;
}
let sha_dirs = match disk
.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, "", -1)
.await
{
Ok(entries) => entries,
Err(err) => {
if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound {
debug!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
error = ?err,
reason = "multipart_root_list_failed",
"Skipped empty multipart sha cleanup"
);
}
continue;
}
};
for sha_dir in sha_dirs {
let sha_dir = sha_dir.trim_end_matches('/').to_string();
let upload_dirs = match disk
.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, &sha_dir, -1)
for root in ["", crate::set_disk::DATA_MOVEMENT_MULTIPART_PREFIX] {
let sha_dirs = match disk
.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, root, -1)
.await
{
Ok(entries) => entries,
@@ -3012,9 +3035,8 @@ async fn cleanup_empty_multipart_sha_dirs_on_local_disks(set: &Arc<SetDisks>) {
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
sha_dir = %sha_dir,
error = ?err,
reason = "multipart_sha_dir_list_failed",
reason = "multipart_root_list_failed",
"Skipped empty multipart sha cleanup"
);
}
@@ -3022,25 +3044,48 @@ async fn cleanup_empty_multipart_sha_dirs_on_local_disks(set: &Arc<SetDisks>) {
}
};
if !upload_dirs.is_empty() {
continue;
}
for sha_dir in sha_dirs.into_iter().filter_map(|entry| multipart_sha_path(root, &entry)) {
let upload_dirs = match disk
.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, &sha_dir, -1)
.await
{
Ok(entries) => entries,
Err(err) => {
if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound {
debug!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
sha_dir = %sha_dir,
error = ?err,
reason = "multipart_sha_dir_list_failed",
"Skipped empty multipart sha cleanup"
);
}
continue;
}
};
if let Err(err) = disk
.delete(RUSTFS_META_MULTIPART_BUCKET, &sha_dir, DeleteOptions::default())
.await
&& err != DiskError::FileNotFound
&& err != DiskError::VolumeNotFound
{
debug!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
sha_dir = %sha_dir,
error = ?err,
reason = "multipart_sha_dir_remove_failed",
"Failed to remove empty multipart sha dir"
);
if !upload_dirs.is_empty() {
continue;
}
if let Err(err) = disk
.delete(RUSTFS_META_MULTIPART_BUCKET, &sha_dir, DeleteOptions::default())
.await
&& err != DiskError::FileNotFound
&& err != DiskError::VolumeNotFound
{
debug!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
sha_dir = %sha_dir,
error = ?err,
reason = "multipart_sha_dir_remove_failed",
"Failed to remove empty multipart sha dir"
);
}
}
}
}
@@ -3058,30 +3103,9 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
continue;
}
let sha_dirs = match disk
.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, "", -1)
.await
{
Ok(entries) => entries,
Err(err) => {
if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound {
debug!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
error = ?err,
reason = "multipart_root_list_failed",
"Skipped stale multipart cleanup"
);
}
continue;
}
};
for sha_dir in sha_dirs {
let sha_dir = sha_dir.trim_end_matches('/').to_string();
let upload_dirs = match disk
.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, &sha_dir, -1)
for root in ["", crate::set_disk::DATA_MOVEMENT_MULTIPART_PREFIX] {
let sha_dirs = match disk
.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, root, -1)
.await
{
Ok(entries) => entries,
@@ -3091,9 +3115,8 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
sha_dir = %sha_dir,
error = ?err,
reason = "multipart_sha_dir_list_failed",
reason = "multipart_root_list_failed",
"Skipped stale multipart cleanup"
);
}
@@ -3101,39 +3124,62 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
}
};
for upload_dir in upload_dirs {
let upload_dir = upload_dir.trim_end_matches('/').to_string();
let candidate_path = format!("{sha_dir}/{upload_dir}");
if candidates
.get(&candidate_path)
.is_some_and(|existing: &StaleMultipartUploadCandidate| existing.metadata.is_some())
for sha_dir in sha_dirs.into_iter().filter_map(|entry| multipart_sha_path(root, &entry)) {
let upload_dirs = match disk
.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, &sha_dir, -1)
.await
{
continue;
}
let candidate = match read_stale_multipart_candidate(disk.as_ref(), &sha_dir, &upload_dir).await {
Ok(candidate) => candidate,
Ok(entries) => entries,
Err(err) => {
if err != DiskError::FileNotFound {
if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound {
debug!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
path = %candidate_path,
sha_dir = %sha_dir,
error = ?err,
reason = "multipart_metadata_read_failed",
"Multipart metadata unavailable during stale cleanup"
reason = "multipart_sha_dir_list_failed",
"Skipped stale multipart cleanup"
);
}
let initiated = initiated_from_upload_dir(&upload_dir, None);
StaleMultipartUploadCandidate {
path: candidate_path,
initiated,
metadata: None,
}
continue;
}
};
merge_stale_multipart_candidate(&mut candidates, candidate);
for upload_dir in upload_dirs {
let upload_dir = upload_dir.trim_end_matches('/').to_string();
let candidate_path = format!("{sha_dir}/{upload_dir}");
if candidates
.get(&candidate_path)
.is_some_and(|existing: &StaleMultipartUploadCandidate| existing.metadata.is_some())
{
continue;
}
let candidate = match read_stale_multipart_candidate(disk.as_ref(), &sha_dir, &upload_dir).await {
Ok(candidate) => candidate,
Err(err) => {
if err != DiskError::FileNotFound {
debug!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
path = %candidate_path,
error = ?err,
reason = "multipart_metadata_read_failed",
"Multipart metadata unavailable during stale cleanup"
);
}
let initiated = initiated_from_upload_dir(&upload_dir, None);
StaleMultipartUploadCandidate {
path: candidate_path,
initiated,
metadata: None,
}
}
};
merge_stale_multipart_candidate(&mut candidates, candidate);
}
}
}
}
@@ -3142,7 +3188,7 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
let upload_dir = candidate.path.rsplit('/').next().unwrap_or_default().to_string();
let mut due = stale_upload_default_due(candidate.initiated, default_expiry);
if let Some(metadata) = candidate.metadata.as_ref()
&& let Some(lifecycle_due) = stale_upload_lifecycle_due(set, metadata, candidate.initiated, &upload_dir).await
&& let Some(lifecycle_due) = stale_upload_lifecycle_due(set, metadata, candidate.initiated, &upload_dir, false).await
&& lifecycle_due < due
{
due = lifecycle_due;
@@ -3152,34 +3198,49 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
continue;
}
match set.delete_all(RUSTFS_META_MULTIPART_BUCKET, &candidate.path).await {
let cleanup_guard = match set.lock_stale_multipart_cleanup(&candidate.path).await {
Ok(guard) => guard,
Err(err) => {
debug!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
path = %candidate.path,
error = ?err,
reason = "multipart_cleanup_lock_or_recheck_failed",
"Skipped stale multipart cleanup"
);
continue;
}
};
let current_metadata = cleanup_guard.file_info().metadata.clone();
let current_initiated = initiated_from_upload_dir(&upload_dir, cleanup_guard.file_info().mod_time);
let mut current_due = stale_upload_default_due(current_initiated, default_expiry);
if let Some(lifecycle_due) =
stale_upload_lifecycle_due(set, &current_metadata, current_initiated, &upload_dir, true).await
&& lifecycle_due < current_due
{
current_due = lifecycle_due;
}
if now < current_due || cleanup_guard.is_lock_lost() {
continue;
}
match cleanup_guard.delete(set).await {
Ok(()) => {
deleted += 1;
let upload_id = encode_stale_upload_id(&upload_dir);
if let Some(metadata) = candidate.metadata.as_ref() {
debug!(
bucket = metadata.get(RUSTFS_MULTIPART_BUCKET_KEY).cloned().unwrap_or_default(),
object = metadata.get(RUSTFS_MULTIPART_OBJECT_KEY).cloned().unwrap_or_default(),
upload_id = %upload_id,
due = ?due,
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "removed",
"Removed stale multipart upload"
);
} else {
debug!(
path = %candidate.path,
upload_id = %upload_id,
due = ?due,
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "removed",
"Removed stale multipart upload"
);
}
debug!(
bucket = current_metadata.get(RUSTFS_MULTIPART_BUCKET_KEY).cloned().unwrap_or_default(),
object = current_metadata.get(RUSTFS_MULTIPART_OBJECT_KEY).cloned().unwrap_or_default(),
upload_id = %upload_id,
due = ?current_due,
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "removed",
"Removed stale multipart upload"
);
}
Err(err) => debug!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
@@ -5162,6 +5223,7 @@ mod tests {
use crate::services::tier::tier::TierConfigMgr;
#[cfg(feature = "test-util")]
use crate::services::tier::warm_backend::WarmBackend as _;
use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause};
use crate::set_disk::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY};
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
use crate::storage_api_contracts::{
@@ -11926,6 +11988,135 @@ mod tests {
assert!(is_err_invalid_upload_id(&err));
}
#[tokio::test]
#[serial]
async fn stale_multipart_cleanup_handles_data_movement_namespace() {
let (_paths, ecstore) = setup_test_env().await;
let bucket = format!("stale-data-movement-{}", Uuid::new_v4().simple());
create_test_bucket(&ecstore, &bucket).await;
let create_upload = |object: &'static str, mod_time| {
let ecstore = ecstore.clone();
let bucket = bucket.clone();
async move {
let mut metadata = HashMap::new();
rustfs_utils::http::insert_str(
&mut metadata,
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
"cleanup-test".to_string(),
);
ecstore
.new_multipart_upload(
&bucket,
object,
&ObjectOptions {
data_movement: true,
mod_time: Some(mod_time),
user_defined: metadata,
..Default::default()
},
)
.await
.expect("data movement multipart upload should be created")
.upload_id
}
};
let stale_object = "stale-internal.bin";
let active_object = "active-internal.bin";
let now = OffsetDateTime::now_utc();
let stale_upload_id = create_upload(stale_object, now - time::Duration::hours(30)).await;
let active_upload_id = create_upload(active_object, now).await;
let deleted = cleanup_stale_multipart_uploads_once_at(ecstore.clone(), now, StdDuration::from_secs(24 * 60 * 60)).await;
assert!(deleted >= 1, "expected stale data movement upload to be removed");
let internal_opts = ObjectOptions {
data_movement: true,
..Default::default()
};
let stale_err = ecstore
.get_multipart_info(&bucket, stale_object, &stale_upload_id, &internal_opts)
.await
.expect_err("stale data movement upload should be removed");
assert!(is_err_invalid_upload_id(&stale_err));
ecstore
.get_multipart_info(&bucket, active_object, &active_upload_id, &internal_opts)
.await
.expect("active data movement upload should remain available");
}
#[tokio::test]
#[serial]
async fn stale_multipart_cleanup_waits_for_data_movement_part_commit() {
let (_paths, ecstore) = setup_test_env().await;
let bucket = format!("stale-data-movement-lock-{}", Uuid::new_v4().simple());
let object = "stale-internal.bin";
create_test_bucket(&ecstore, &bucket).await;
let mut metadata = HashMap::new();
rustfs_utils::http::insert_str(
&mut metadata,
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
"cleanup-lock-test".to_string(),
);
let opts = ObjectOptions {
data_movement: true,
mod_time: Some(OffsetDateTime::now_utc() - time::Duration::hours(30)),
user_defined: metadata,
..Default::default()
};
let upload = ecstore
.new_multipart_upload(&bucket, object, &opts)
.await
.expect("data movement multipart upload should be created");
let barrier = MultipartCommitBarrier::install(bucket.as_str(), object, MultipartCommitPause::PutPartAfterRename);
let put_store = ecstore.clone();
let put_bucket = bucket.clone();
let upload_id = upload.upload_id.clone();
let put_task = tokio::spawn(async move {
let mut data = PutObjReader::from_vec(vec![1, 2, 3, 4]);
put_store
.put_object_part(
&put_bucket,
object,
&upload_id,
1,
&mut data,
&ObjectOptions {
data_movement: true,
..Default::default()
},
)
.await
});
barrier.wait_until_paused().await;
let cleanup_store = ecstore.clone();
let mut cleanup_task = tokio::spawn(async move {
cleanup_stale_multipart_uploads_once_at(
cleanup_store,
OffsetDateTime::now_utc(),
StdDuration::from_secs(24 * 60 * 60),
)
.await
});
assert!(
tokio::time::timeout(StdDuration::from_millis(200), &mut cleanup_task)
.await
.is_err(),
"stale cleanup must wait for the in-flight part commit upload lock"
);
barrier.release();
put_task
.await
.expect("part upload task should join")
.expect("part upload should commit before stale cleanup");
let deleted = cleanup_task.await.expect("stale cleanup task should join");
assert!(deleted >= 1, "stale cleanup should proceed after the part commit releases its lock");
}
#[tokio::test]
#[serial]
async fn stale_multipart_cleanup_applies_abort_incomplete_lifecycle_before_default_expiry() {
@@ -12000,6 +12191,58 @@ mod tests {
assert!(is_err_invalid_upload_id(&err));
}
#[tokio::test]
#[serial]
async fn stale_multipart_cleanup_excludes_data_movement_from_abort_lifecycle() {
let (_paths, ecstore) = setup_test_env().await;
let bucket = format!("stale-internal-lifecycle-{}", Uuid::new_v4().simple());
let object = "logs/internal/object.bin";
create_test_bucket(&ecstore, &bucket).await;
set_abort_incomplete_lifecycle(&bucket, "logs/", 0).await;
let initiated = OffsetDateTime::now_utc() - time::Duration::minutes(5);
let mut metadata = HashMap::new();
rustfs_utils::http::insert_str(
&mut metadata,
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
"lifecycle-exclusion-test".to_string(),
);
let upload = ecstore
.new_multipart_upload(
&bucket,
object,
&ObjectOptions {
data_movement: true,
mod_time: Some(initiated),
user_defined: metadata,
..Default::default()
},
)
.await
.expect("data movement multipart upload should be created");
let deleted = cleanup_stale_multipart_uploads_once_at(
ecstore.clone(),
OffsetDateTime::now_utc(),
StdDuration::from_secs(7 * 24 * 60 * 60),
)
.await;
assert_eq!(deleted, 0, "bucket lifecycle must not remove active data movement uploads");
ecstore
.get_multipart_info(
&bucket,
object,
&upload.upload_id,
&ObjectOptions {
data_movement: true,
..Default::default()
},
)
.await
.expect("active data movement upload should remain available");
}
#[tokio::test]
#[serial]
async fn stale_multipart_cleanup_applies_abort_lifecycle_with_size_filter() {
crates/ecstore/src/core/pools.rs
+76 -9
View File
@@ -2266,15 +2266,19 @@ fn decommission_delete_marker_opts(
version: &rustfs_filemeta::FileInfo,
version_id: Option<String>,
src_pool_idx: usize,
expected_bucket_incarnation_id: Option<uuid::Uuid>,
) -> ObjectOptions {
let version_suspended = version.version_id.is_none() && version_id.is_none();
ObjectOptions {
versioned: true,
version_id,
versioned: !version_suspended,
version_suspended,
version_id: version_id.or_else(|| version_suspended.then(|| uuid::Uuid::nil().to_string())),
mod_time: version.mod_time,
src_pool_idx,
data_movement: true,
delete_marker: true,
skip_decommissioned: true,
expected_bucket_incarnation_id,
delete_replication: version
.replication_state_internal
.as_ref()
@@ -2299,6 +2303,7 @@ fn decommission_remote_tiered_opts(
version: &rustfs_filemeta::FileInfo,
version_id: Option<String>,
src_pool_idx: usize,
expected_bucket_incarnation_id: Option<uuid::Uuid>,
) -> ObjectOptions {
ObjectOptions {
versioned: version_id.is_some(),
@@ -2307,6 +2312,9 @@ fn decommission_remote_tiered_opts(
user_defined: version.metadata.clone(),
src_pool_idx,
data_movement: true,
include_part_checksums: true,
http_preconditions: Some(crate::data_movement::data_movement_target_precondition()),
expected_bucket_incarnation_id,
..Default::default()
}
}
@@ -2805,6 +2813,7 @@ impl ECStore {
lifecycle_config: Option<BucketLifecycleConfiguration>,
object_lock_config: Option<ObjectLockConfiguration>,
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
expected_bucket_incarnation_id: Option<uuid::Uuid>,
) -> Result<()> {
debug!(
event = EVENT_DECOMMISSION_ENTRY,
@@ -2834,6 +2843,11 @@ impl ECStore {
}
decommission_cancel_signal_result(rx.is_cancelled())?;
let bucket_incarnation_fence = match expected_bucket_incarnation_id {
Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?),
None => None,
};
let mut fivs = load_decommission_entry_exact_versions(&set, &entry, &bucket, "file_info_versions").await?;
fivs.versions
@@ -2894,7 +2908,7 @@ impl ECStore {
.delete_object(
bucket.as_str(),
&version.name,
decommission_delete_marker_opts(version, version_id.clone(), idx),
decommission_delete_marker_opts(version, version_id.clone(), idx, expected_bucket_incarnation_id),
)
.await
{
@@ -2984,7 +2998,7 @@ impl ECStore {
bucket.as_str(),
&version.name,
version,
&decommission_remote_tiered_opts(version, version_id.clone(), idx),
&decommission_remote_tiered_opts(version, version_id.clone(), idx, expected_bucket_incarnation_id),
)
.await
{
@@ -3056,7 +3070,11 @@ impl ECStore {
)
.await?;
if let Err(err) = self.clone().decommission_object(idx, bucket, rd).await {
if let Err(err) = self
.clone()
.decommission_object(idx, bucket, rd, expected_bucket_incarnation_id)
.await
{
if is_decommission_copy_cleanup_safe_error(&err) {
ignore = true;
cleanup_ignored = true;
@@ -3133,6 +3151,9 @@ impl ECStore {
}
if should_cleanup_decommission_source_entry(decommissioned, fivs.versions.len(), expired) {
if bucket_incarnation_fence.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
return Err(Error::other("decommission bucket incarnation fence was lost before source cleanup"));
}
decommission_cancel_signal_result(rx.is_cancelled())?;
self.save_decommission_entry_progress_stage(
@@ -3157,6 +3178,12 @@ impl ECStore {
entry.name.as_str(),
&fivs,
&cleanup_preflight_allowed_missing,
data_movement::SourceCleanupBucketFence {
expected_incarnation_id: expected_bucket_incarnation_id,
lifecycle_guard: bucket_incarnation_fence
.as_ref()
.and_then(|guard| guard.namespace_lock_guard()),
},
"decommission",
)
.await
@@ -3268,6 +3295,11 @@ impl ECStore {
let mut lifecycle_config = None;
let mut object_lock_config = None;
let mut replication_config = None;
let expected_bucket_incarnation_id = if bi.name == RUSTFS_META_BUCKET {
None
} else {
Some(self.bucket_incarnation_id_from_disk(&bi.name).await?)
};
if bi.name != RUSTFS_META_BUCKET {
let _ = resolve_decommission_optional_bucket_config_result(
@@ -3321,6 +3353,7 @@ impl ECStore {
let lifecycle_config = lifecycle_config.clone();
let object_lock_config = object_lock_config.clone();
let replication_config = replication_config.clone();
let expected_bucket_incarnation_id = expected_bucket_incarnation_id;
let entry_error = entry_error.clone();
let callback_rx = callback_rx.clone();
@@ -3383,6 +3416,7 @@ impl ECStore {
lifecycle_config,
object_lock_config,
replication_config,
expected_bucket_incarnation_id,
)
.await
{
@@ -4168,10 +4202,24 @@ impl ECStore {
}
#[tracing::instrument(skip(self, rd))]
async fn decommission_object(self: Arc<Self>, pool_idx: usize, bucket: String, rd: GetObjectReader) -> Result<()> {
async fn decommission_object(
self: Arc<Self>,
pool_idx: usize,
bucket: String,
rd: GetObjectReader,
expected_bucket_incarnation_id: Option<uuid::Uuid>,
) -> Result<()> {
warn!("decommission_object: start {} {}", &bucket, &rd.object_info.name);
let object_name = rd.object_info.name.clone();
let result = data_movement::migrate_object(self, pool_idx, bucket.clone(), rd, "decommission_object").await;
let result = data_movement::migrate_object(
self,
pool_idx,
bucket.clone(),
rd,
expected_bucket_incarnation_id,
"decommission_object",
)
.await;
if result.is_ok() {
warn!("decommission_object: migrated {} {}", &bucket, &object_name);
}
@@ -4347,7 +4395,8 @@ mod tests {
..Default::default()
};
let opts = decommission_delete_marker_opts(&version, Some("version-id".to_string()), 7);
let incarnation = uuid::Uuid::new_v4();
let opts = decommission_delete_marker_opts(&version, Some("version-id".to_string()), 7, Some(incarnation));
let replication = opts.delete_replication.expect("replication state should be preserved");
assert!(opts.versioned);
@@ -4357,11 +4406,25 @@ mod tests {
assert_eq!(opts.src_pool_idx, 7);
assert_eq!(opts.version_id.as_deref(), Some("version-id"));
assert_eq!(opts.mod_time, Some(mod_time));
assert_eq!(opts.expected_bucket_incarnation_id, Some(incarnation));
assert_eq!(replication.replica_status, ReplicationStatusType::Replica);
assert!(replication.delete_marker);
assert_eq!(replication.replicate_decision_str, "existing");
}
#[test]
fn decommission_delete_marker_opts_preserves_suspended_null_version() {
let version = rustfs_filemeta::FileInfo {
deleted: true,
..Default::default()
};
let opts = decommission_delete_marker_opts(&version, None, 7, None);
assert!(!opts.versioned);
assert!(opts.version_suspended);
assert_eq!(opts.version_id.as_deref(), Some(uuid::Uuid::nil().to_string().as_str()));
}
#[test]
fn test_decommission_object_migration_read_opts_are_raw_data_movement() {
let opts = decommission_object_migration_read_opts(Some("vid-1".to_string()));
@@ -4383,7 +4446,8 @@ mod tests {
..Default::default()
};
let opts = decommission_remote_tiered_opts(&version, Some("version-id".to_string()), 9);
let incarnation = uuid::Uuid::new_v4();
let opts = decommission_remote_tiered_opts(&version, Some("version-id".to_string()), 9, Some(incarnation));
assert!(opts.versioned);
assert!(opts.data_movement);
@@ -4391,6 +4455,9 @@ mod tests {
assert_eq!(opts.version_id.as_deref(), Some("version-id"));
assert_eq!(opts.mod_time, Some(mod_time));
assert_eq!(opts.user_defined.get("x-amz-meta-key").map(String::as_str), Some("value"));
assert!(opts.include_part_checksums);
assert!(opts.http_preconditions.is_some());
assert_eq!(opts.expected_bucket_incarnation_id, Some(incarnation));
}
#[test]
crates/ecstore/src/data_movement/mod.rs
+1555 -252
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/diagnostics/get.rs
+2
View File
@@ -137,6 +137,7 @@ pub(crate) const GET_METADATA_CACHE_REASON_NO_LOCK: &str = "no_lock";
pub(crate) const GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED: &str = "not_found_or_expired";
pub(crate) const GET_METADATA_CACHE_REASON_NOT_READ_DATA: &str = "not_read_data";
pub(crate) const GET_METADATA_CACHE_REASON_PART_NUMBER: &str = "part_number";
pub(crate) const GET_METADATA_CACHE_REASON_PART_CHECKSUMS: &str = "part_checksums";
pub(crate) const GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ: &str = "raw_data_movement_read";
pub(crate) const GET_METADATA_CACHE_REASON_STALE_PUBLICATION: &str = "stale_publication";
pub(crate) const GET_METADATA_CACHE_REASON_USABLE: &str = "usable";
@@ -480,6 +481,7 @@ mod tests {
assert_eq!(GET_METADATA_CACHE_REASON_NO_LOCK, "no_lock");
assert_eq!(GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED, "not_found_or_expired");
assert_eq!(GET_METADATA_CACHE_REASON_NOT_READ_DATA, "not_read_data");
assert_eq!(GET_METADATA_CACHE_REASON_PART_CHECKSUMS, "part_checksums");
assert_eq!(GET_METADATA_CACHE_REASON_PART_NUMBER, "part_number");
assert_eq!(GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, "raw_data_movement_read");
assert_eq!(GET_METADATA_CACHE_REASON_STALE_PUBLICATION, "stale_publication");
crates/ecstore/src/disk/local.rs
+1
View File
@@ -9875,6 +9875,7 @@ impl DiskAPI for LocalDisk {
FileInfoOpts {
data: read_data,
include_free_versions: opts.incl_free_versions,
include_part_checksums: false,
},
)?;
crates/ecstore/src/object_api/types.rs
+3
View File
@@ -260,6 +260,9 @@ pub struct ObjectOptions {
pub data_movement: bool,
pub raw_data_movement_read: bool,
/// Materialize the data-movement per-part checksum sidecar for APIs that
/// return part checksums. Ordinary object reads leave it encoded.
pub include_part_checksums: bool,
pub src_pool_idx: usize,
pub user_defined: HashMap<String, String>,
pub preserve_etag: Option<String>,
crates/ecstore/src/services/rebalance/entry.rs
+31 -5
View File
@@ -16,7 +16,7 @@ use super::meta::{
clone_arc_by_index, ensure_valid_rebalance_pool_index, invalid_rebalance_pool_index_error,
rebalance_metadata_not_initialized_error, should_ignore_rebalance_data_usage_cache,
};
use super::migration::migrate_entry_version;
use super::migration::{RebalanceMigrationBackend, migrate_entry_version};
use super::worker::{
RebalanceEntryCleanupResult, RebalanceEntryTask, load_rebalance_bucket_configs, rebalance_max_attempts,
resolve_rebalance_bucket_error, resolve_rebalance_entry_cleanup_delete_result, resolve_rebalance_file_info_versions_result,
@@ -144,6 +144,11 @@ impl ECStore {
return Ok(RebalanceEntryOutcome::Completed);
}
let bucket_incarnation_fence = match bucket_configs.bucket_incarnation_id {
Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?),
None => None,
};
let mut fivs =
resolve_rebalance_file_info_versions_result(entry.file_info_versions(&bucket), bucket.as_str(), entry.name.as_str())?;
@@ -203,9 +208,14 @@ impl ECStore {
}
let version_id = version.version_id.map(|v| v.to_string());
let expected_bucket_incarnation_id = bucket_configs.bucket_incarnation_id;
let mut transfer = |src_pool_idx: usize, bucket: String, rd: GetObjectReader| {
let store = self.clone();
async move { store.rebalance_object(src_pool_idx, bucket, rd).await }
async move {
store
.rebalance_object(src_pool_idx, bucket, rd, expected_bucket_incarnation_id)
.await
}
};
// Route delete-marker migration through the store layer so it lands on the
// cross-pool target (excluding the source pool), not back onto the source set.
@@ -214,11 +224,12 @@ impl ECStore {
async move { store.delete_object(&bucket, &object, opts).await }
};
let result = migrate_entry_version(
set.as_ref(),
&RebalanceMigrationBackend::new(set.as_ref(), self.as_ref()),
bucket.clone(),
pool_index,
version,
version_id.clone(),
expected_bucket_incarnation_id,
rebalance_max_attempts(),
should_ignore_rebalance_data_usage_cache(bucket.as_str()),
&mut transfer,
@@ -303,6 +314,9 @@ impl ECStore {
}
if should_cleanup_rebalance_source_entry(rebalanced, fivs.versions.len(), expired) {
if bucket_incarnation_fence.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
return Err(Error::other("rebalance bucket incarnation fence was lost before source cleanup"));
}
let cleanup_result = self
.finish_rebalance_entry_after_cleanup(
pool_index,
@@ -315,6 +329,12 @@ impl ECStore {
entry.name.as_str(),
&fivs,
&cleanup_preflight_allowed_missing,
data_movement::SourceCleanupBucketFence {
expected_incarnation_id: bucket_configs.bucket_incarnation_id,
lifecycle_guard: bucket_incarnation_fence
.as_ref()
.and_then(|guard| guard.namespace_lock_guard()),
},
"rebalance",
),
)
@@ -389,8 +409,14 @@ impl ECStore {
}
#[tracing::instrument(skip(self, rd))]
async fn rebalance_object(self: Arc<Self>, pool_idx: usize, bucket: String, rd: GetObjectReader) -> Result<()> {
data_movement::migrate_object(self, pool_idx, bucket, rd, "rebalance_object").await
async fn rebalance_object(
self: Arc<Self>,
pool_idx: usize,
bucket: String,
rd: GetObjectReader,
expected_bucket_incarnation_id: Option<uuid::Uuid>,
) -> Result<()> {
data_movement::migrate_object(self, pool_idx, bucket, rd, expected_bucket_incarnation_id, "rebalance_object").await
}
async fn update_rebalance_last_error(&self, pool_idx: usize, message: String) -> Result<()> {
crates/ecstore/src/services/rebalance/migration.rs
+92 -15
View File
@@ -5,6 +5,7 @@ use crate::error::{Error, Result, is_err_object_not_found, is_err_version_not_fo
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use crate::set_disk::SetDisks;
use crate::storage_api_contracts::{object::ObjectIO, range::HTTPRangeSpec};
use crate::store::ECStore;
use http::HeaderMap;
use rustfs_filemeta::FileInfo;
use rustfs_utils::path::encode_dir_object;
@@ -21,15 +22,23 @@ pub(crate) struct MigrationVersionResult {
pub error: Option<Error>,
}
pub(super) fn rebalance_delete_marker_opts(version: &FileInfo, version_id: Option<String>, src_pool_idx: usize) -> ObjectOptions {
pub(super) fn rebalance_delete_marker_opts(
version: &FileInfo,
version_id: Option<String>,
src_pool_idx: usize,
expected_bucket_incarnation_id: Option<uuid::Uuid>,
) -> ObjectOptions {
let version_suspended = version.version_id.is_none() && version_id.is_none();
ObjectOptions {
versioned: true,
version_id,
versioned: !version_suspended,
version_suspended,
version_id: version_id.or_else(|| version_suspended.then(|| uuid::Uuid::nil().to_string())),
mod_time: version.mod_time,
src_pool_idx,
data_movement: true,
delete_marker: true,
skip_decommissioned: true,
expected_bucket_incarnation_id,
delete_replication: version
.replication_state_internal
.as_ref()
@@ -38,7 +47,12 @@ pub(super) fn rebalance_delete_marker_opts(version: &FileInfo, version_id: Optio
}
}
fn rebalance_remote_tiered_opts(version: &FileInfo, version_id: Option<String>, src_pool_idx: usize) -> ObjectOptions {
fn rebalance_remote_tiered_opts(
version: &FileInfo,
version_id: Option<String>,
src_pool_idx: usize,
expected_bucket_incarnation_id: Option<uuid::Uuid>,
) -> ObjectOptions {
ObjectOptions {
versioned: version_id.is_some(),
version_id,
@@ -46,6 +60,21 @@ fn rebalance_remote_tiered_opts(version: &FileInfo, version_id: Option<String>,
user_defined: version.metadata.clone(),
src_pool_idx,
data_movement: true,
include_part_checksums: true,
http_preconditions: Some(crate::data_movement::data_movement_target_precondition()),
expected_bucket_incarnation_id,
..Default::default()
}
}
pub(super) fn rebalance_object_migration_read_opts(version_id: Option<String>) -> ObjectOptions {
ObjectOptions {
version_id,
no_lock: true,
data_movement: true,
raw_data_movement_read: true,
skip_decommissioned: true,
skip_rebalancing: true,
..Default::default()
}
}
@@ -70,8 +99,19 @@ pub(crate) trait MigrationBackend: Send + Sync {
) -> Result<()>;
}
pub(crate) struct RebalanceMigrationBackend<'a> {
source: &'a SetDisks,
store: &'a ECStore,
}
impl<'a> RebalanceMigrationBackend<'a> {
pub(crate) fn new(source: &'a SetDisks, store: &'a ECStore) -> Self {
Self { source, store }
}
}
#[async_trait::async_trait]
impl MigrationBackend for SetDisks {
impl MigrationBackend for RebalanceMigrationBackend<'_> {
async fn get_object_reader_for_migration(
&self,
bucket: &str,
@@ -80,7 +120,7 @@ impl MigrationBackend for SetDisks {
h: HeaderMap,
opts: &ObjectOptions,
) -> Result<GetObjectReader> {
self.get_object_reader(bucket, object, range, h, opts).await
self.source.get_object_reader(bucket, object, range, h, opts).await
}
async fn move_remote_version_for_migration(
@@ -90,7 +130,7 @@ impl MigrationBackend for SetDisks {
fi: &FileInfo,
opts: &ObjectOptions,
) -> Result<()> {
self.decommission_tiered_object(bucket, object, fi, opts).await
self.store.decommission_tiered_object(bucket, object, fi, opts).await
}
}
@@ -101,6 +141,7 @@ pub(crate) async fn migrate_entry_version<Backend, F, Fut, D, DFut>(
pool_index: usize,
version: &FileInfo,
version_id: Option<String>,
expected_bucket_incarnation_id: Option<uuid::Uuid>,
max_attempts: usize,
ignore_data_usage_cache: bool,
transfer: F,
@@ -113,12 +154,13 @@ where
D: FnMut(String, String, ObjectOptions) -> DFut + Send,
DFut: Future<Output = Result<ObjectInfo>> + Send,
{
migrate_entry_version_with_retry_wait(
migrate_entry_version_with_retry_wait_and_incarnation(
set,
bucket,
pool_index,
version,
version_id,
expected_bucket_incarnation_id,
max_attempts,
ignore_data_usage_cache,
transfer,
@@ -137,6 +179,45 @@ pub(super) async fn migrate_entry_version_with_retry_wait<Backend, F, Fut, D, DF
version_id: Option<String>,
max_attempts: usize,
ignore_data_usage_cache: bool,
transfer: F,
delete_marker: D,
wait_retry: W,
) -> MigrationVersionResult
where
Backend: MigrationBackend + ?Sized,
F: FnMut(usize, String, GetObjectReader) -> Fut + Send,
Fut: Future<Output = Result<()>> + Send,
D: FnMut(String, String, ObjectOptions) -> DFut + Send,
DFut: Future<Output = Result<ObjectInfo>> + Send,
W: FnMut(Duration) -> WFut + Send,
WFut: Future<Output = ()> + Send,
{
migrate_entry_version_with_retry_wait_and_incarnation(
set,
bucket,
pool_index,
version,
version_id,
None,
max_attempts,
ignore_data_usage_cache,
transfer,
delete_marker,
wait_retry,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn migrate_entry_version_with_retry_wait_and_incarnation<Backend, F, Fut, D, DFut, W, WFut>(
set: &Backend,
bucket: String,
pool_index: usize,
version: &FileInfo,
version_id: Option<String>,
expected_bucket_incarnation_id: Option<uuid::Uuid>,
max_attempts: usize,
ignore_data_usage_cache: bool,
mut transfer: F,
mut delete_marker: D,
mut wait_retry: W,
@@ -169,7 +250,7 @@ where
&bucket,
&version.name,
version,
&rebalance_remote_tiered_opts(version, version_id, pool_index),
&rebalance_remote_tiered_opts(version, version_id, pool_index, expected_bucket_incarnation_id),
)
.await
{
@@ -212,7 +293,7 @@ where
if let Err(err) = delete_marker(
bucket.clone(),
version.name.clone(),
rebalance_delete_marker_opts(version, version_id, pool_index),
rebalance_delete_marker_opts(version, version_id, pool_index, expected_bucket_incarnation_id),
)
.await
{
@@ -255,11 +336,7 @@ where
&encode_dir_object(&version.name),
None,
HeaderMap::new(),
&ObjectOptions {
version_id: version_id.clone(),
no_lock: true,
..Default::default()
},
&rebalance_object_migration_read_opts(version_id.clone()),
)
.await
{
crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs
+68 -3
View File
@@ -113,6 +113,8 @@ struct LegacyRebalanceMeta {
struct MigrationBackendSpy {
get_object_reader: Mutex<Option<core::result::Result<GetObjectReader, Error>>>,
move_remote: Mutex<Option<core::result::Result<(), Error>>>,
get_opts: Mutex<Vec<ObjectOptions>>,
move_remote_opts: Mutex<Vec<ObjectOptions>>,
get_calls: AtomicUsize,
move_remote_calls: AtomicUsize,
}
@@ -125,6 +127,8 @@ impl MigrationBackendSpy {
Self {
get_object_reader: Mutex::new(get_object_reader),
move_remote: Mutex::new(move_remote),
get_opts: Mutex::new(Vec::new()),
move_remote_opts: Mutex::new(Vec::new()),
get_calls: AtomicUsize::new(0),
move_remote_calls: AtomicUsize::new(0),
}
@@ -138,6 +142,24 @@ impl MigrationBackendSpy {
self.move_remote_calls.load(Ordering::SeqCst)
}
fn last_get_opts(&self) -> ObjectOptions {
self.get_opts
.lock()
.unwrap()
.last()
.cloned()
.expect("reader opts should be captured")
}
fn last_move_remote_opts(&self) -> ObjectOptions {
self.move_remote_opts
.lock()
.unwrap()
.last()
.cloned()
.expect("remote opts should be captured")
}
fn make_reader() -> GetObjectReader {
GetObjectReader {
stream: Box::new(Cursor::new(vec![0_u8; 3])),
@@ -156,9 +178,10 @@ impl MigrationBackend for MigrationBackendSpy {
_object: &str,
_range: Option<HTTPRangeSpec>,
_h: http::HeaderMap,
_opts: &ObjectOptions,
opts: &ObjectOptions,
) -> Result<GetObjectReader> {
self.get_calls.fetch_add(1, Ordering::SeqCst);
self.get_opts.lock().unwrap().push(opts.clone());
if let Some(result) = self.get_object_reader.lock().unwrap().take() {
return result;
}
@@ -171,9 +194,10 @@ impl MigrationBackend for MigrationBackendSpy {
_bucket: &str,
_object: &str,
_fi: &FileInfo,
_opts: &ObjectOptions,
opts: &ObjectOptions,
) -> Result<()> {
self.move_remote_calls.fetch_add(1, Ordering::SeqCst);
self.move_remote_opts.lock().unwrap().push(opts.clone());
if let Some(result) = self.move_remote.lock().unwrap().take() {
return result;
}
@@ -217,7 +241,8 @@ fn test_rebalance_delete_marker_opts_preserves_replication_state() {
..version_deleted()
};
let opts = rebalance_delete_marker_opts(&version, Some("version-id".to_string()), 7);
let incarnation = uuid::Uuid::new_v4();
let opts = rebalance_delete_marker_opts(&version, Some("version-id".to_string()), 7, Some(incarnation));
let replication = opts.delete_replication.expect("replication state should be preserved");
assert!(opts.versioned);
@@ -227,11 +252,22 @@ fn test_rebalance_delete_marker_opts_preserves_replication_state() {
assert_eq!(opts.src_pool_idx, 7);
assert_eq!(opts.version_id.as_deref(), Some("version-id"));
assert_eq!(opts.mod_time, Some(mod_time));
assert_eq!(opts.expected_bucket_incarnation_id, Some(incarnation));
assert_eq!(replication.replica_status, ReplicationStatusType::Replica);
assert!(replication.delete_marker);
assert_eq!(replication.replicate_decision_str, "existing");
}
#[test]
fn test_rebalance_delete_marker_opts_preserves_suspended_null_version() {
let version = version_deleted();
let opts = rebalance_delete_marker_opts(&version, None, 7, None);
assert!(!opts.versioned);
assert!(opts.version_suspended);
assert_eq!(opts.version_id.as_deref(), Some(uuid::Uuid::nil().to_string().as_str()));
}
#[tokio::test]
async fn test_migrate_entry_version_remote_version_is_moved_without_transfer() {
let backend = MigrationBackendSpy::new(None, Some(Ok(())));
@@ -248,12 +284,14 @@ async fn test_migrate_entry_version_remote_version_is_moved_without_transfer() {
}
};
let incarnation = uuid::Uuid::new_v4();
let result = migrate_entry_version(
&backend,
"bucket".to_string(),
0,
&version,
version.version_id.map(|v| v.to_string()),
Some(incarnation),
3,
false,
&mut transfer,
@@ -269,6 +307,10 @@ async fn test_migrate_entry_version_remote_version_is_moved_without_transfer() {
assert_eq!(transfer_count.load(Ordering::SeqCst), 0);
assert_eq!(backend.move_remote_calls(), 1);
assert_eq!(backend.get_calls(), 0);
let remote_opts = backend.last_move_remote_opts();
assert!(remote_opts.include_part_checksums);
assert!(remote_opts.http_preconditions.is_some());
assert_eq!(remote_opts.expected_bucket_incarnation_id, Some(incarnation));
}
#[tokio::test]
@@ -294,6 +336,7 @@ async fn test_migrate_entry_version_remote_not_found_is_cleanup_ignored() {
0,
&version,
version.version_id.map(|v| v.to_string()),
None,
3,
false,
&mut transfer,
@@ -330,6 +373,7 @@ async fn test_migrate_entry_version_remote_overwrite_is_not_ignored() {
0,
&version,
Some("vid-1".to_string()),
None,
3,
false,
&mut transfer,
@@ -368,6 +412,7 @@ async fn test_migrate_entry_version_remote_failure_is_reported() {
0,
&version,
version.version_id.map(|v| v.to_string()),
None,
3,
false,
&mut transfer,
@@ -410,6 +455,7 @@ async fn test_migrate_entry_version_deleted_version_routes_delete_through_store_
1,
&version,
version.version_id.map(|v| v.to_string()),
None,
3,
false,
&mut transfer,
@@ -449,6 +495,7 @@ async fn test_migrate_entry_version_deleted_version_not_found_is_ignored() {
1,
&version,
version.version_id.map(|v| v.to_string()),
None,
3,
false,
&mut transfer,
@@ -491,6 +538,7 @@ async fn test_migrate_entry_version_deleted_version_overwrite_is_not_ignored() {
1,
&version,
Some("vid-1".to_string()),
None,
3,
false,
&mut transfer,
@@ -520,6 +568,7 @@ async fn test_migrate_entry_version_reader_not_found_is_ignored() {
1,
&version,
version.version_id.map(|v| v.to_string()),
None,
3,
false,
&mut transfer,
@@ -647,6 +696,7 @@ async fn test_migrate_entry_version_reader_fails_after_retries() {
1,
&version,
version.version_id.map(|v| v.to_string()),
None,
3,
false,
&mut transfer,
@@ -685,6 +735,7 @@ async fn test_migrate_entry_version_zero_max_attempts_still_attempts_once() {
1,
&version,
version.version_id.map(|v| v.to_string()),
None,
0,
false,
&mut transfer,
@@ -750,6 +801,13 @@ async fn test_migrate_entry_version_transfer_retries_before_success() {
assert_eq!(backend.get_calls(), 2);
assert_eq!(transfer_count.load(Ordering::SeqCst), 2);
assert_eq!(wait_count.load(Ordering::SeqCst), 1);
let read_opts = backend.last_get_opts();
assert_eq!(read_opts.version_id.as_deref(), version.version_id.map(|id| id.to_string()).as_deref());
assert!(read_opts.no_lock);
assert!(read_opts.data_movement);
assert!(read_opts.raw_data_movement_read);
assert!(read_opts.skip_decommissioned);
assert!(read_opts.skip_rebalancing);
}
#[tokio::test]
@@ -822,6 +880,7 @@ async fn test_migrate_entry_version_transfer_fails_after_retries() {
1,
&version,
version.version_id.map(|v| v.to_string()),
None,
2,
false,
&mut transfer,
@@ -860,6 +919,7 @@ async fn test_migrate_entry_version_transfer_not_found_is_ignored() {
1,
&version,
version.version_id.map(|v| v.to_string()),
None,
3,
false,
&mut transfer,
@@ -901,6 +961,7 @@ async fn test_migrate_entry_version_transfer_overwrite_is_not_ignored() {
1,
&version,
Some("vid-1".to_string()),
None,
3,
false,
&mut transfer,
@@ -943,6 +1004,7 @@ async fn test_migrate_entry_version_ignores_data_usage_cache_when_enabled() {
1,
&version,
version.version_id.map(|v| v.to_string()),
None,
2,
true,
&mut transfer,
@@ -985,6 +1047,7 @@ async fn test_migrate_entry_version_data_usage_cache_moves_when_ignore_disabled(
1,
&version,
version.version_id.map(|v| v.to_string()),
None,
2,
false,
&mut transfer,
@@ -2026,6 +2089,7 @@ async fn test_migrate_entry_version_transfer_failure_reports_write_target_stage(
1,
&version,
version.version_id.map(|v| v.to_string()),
None,
1,
false,
&mut transfer,
@@ -2050,6 +2114,7 @@ async fn test_migrate_entry_version_reader_failure_reports_read_source_stage() {
1,
&version,
version.version_id.map(|v| v.to_string()),
None,
1,
false,
&mut transfer,
crates/ecstore/src/services/rebalance/types.rs
+1
View File
@@ -36,6 +36,7 @@ pub type RStats = Vec<Arc<RebalanceStats>>;
#[derive(Debug, Default)]
pub(super) struct RebalanceBucketConfigs {
pub(super) bucket_incarnation_id: Option<uuid::Uuid>,
pub(super) lifecycle_config: Option<s3s::dto::BucketLifecycleConfiguration>,
pub(super) object_lock_config: Option<s3s::dto::ObjectLockConfiguration>,
pub(super) replication_config: Option<(s3s::dto::ReplicationConfiguration, OffsetDateTime)>,
crates/ecstore/src/services/rebalance/worker.rs
+1
View File
@@ -406,6 +406,7 @@ pub(super) async fn load_rebalance_bucket_configs(api: &ECStore, bucket: &str) -
let expiry_configs = crate::bucket::lifecycle::get_expiry_configs(api, bucket).await?;
Ok(RebalanceBucketConfigs {
bucket_incarnation_id: Some(api.bucket_incarnation_id_from_disk(bucket).await?),
lifecycle_config: expiry_configs.lifecycle.map(|config| (*config).clone()),
object_lock_config: expiry_configs.object_lock.map(|config| (*config).clone()),
replication_config: resolve_rebalance_optional_bucket_config_result(
crates/ecstore/src/set_disk/core/io_primitives.rs
+47 -11
View File
@@ -2443,13 +2443,14 @@ impl SetDisks {
bucket: &str,
object: &str,
) -> Result<Option<rustfs_filemeta::FileInfoVersions>> {
let disk_object = rustfs_utils::path::encode_dir_object(object);
let disks = self.get_disks_internal().await;
if disks.is_empty() {
return Err(to_object_err(StorageError::ErasureReadQuorum, vec![bucket, object]));
}
let read_quorum = disks.len().div_ceil(2).max(1);
let (raw_fileinfos, errs) = Self::read_all_raw_file_info(&disks, bucket, object, false).await;
let (raw_fileinfos, errs) = Self::read_all_raw_file_info(&disks, bucket, disk_object.as_str(), false).await;
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
let object_err = to_object_err(err.into(), vec![bucket, object]);
@@ -2605,7 +2606,7 @@ impl SetDisks {
//
// `into_fileinfo` with an empty version_id selects the first non-free version
// (see FileMeta::into_fileinfo); replicate that selection from the header here.
let vid = match meta.into_fileinfo(bucket, object, "", true, incl_free_vers, true) {
let vid = match meta.into_fileinfo_without_part_checksums(bucket, object, "", true, incl_free_vers) {
Ok(finfo) if file_info_is_valid_for_metadata(&finfo) => finfo.version_id.unwrap_or(Uuid::nil()),
_ => match meta
.versions
@@ -2628,7 +2629,13 @@ impl SetDisks {
for (idx, meta_op) in metadata_array.iter().enumerate() {
if let Some(meta) = meta_op {
match meta.into_fileinfo(bucket, object, vid.to_string().as_str(), read_data, incl_free_vers, true) {
match meta.into_fileinfo_without_part_checksums(
bucket,
object,
vid.to_string().as_str(),
read_data,
incl_free_vers,
) {
Ok(res) => match res.validate_for_metadata_read() {
Ok(_) => meta_file_infos[idx] = res,
Err(err) => errs[idx] = Some(err.into()),
@@ -4533,26 +4540,29 @@ impl SetDisks {
object: &str,
opts: &ObjectOptions,
) -> Option<StorageError> {
let mut opts = opts.clone();
let mut lookup_opts = opts.clone();
let http_preconditions = opts.http_preconditions?;
opts.http_preconditions = None;
let http_preconditions = lookup_opts.http_preconditions?;
lookup_opts.http_preconditions = None;
// Never claim a lock here, to avoid deadlock
// - If no_lock is false, we must have obtained the lock out side of this function
// - If no_lock is true, we should not obtain locks
opts.no_lock = true;
let oi = self.get_object_info(bucket, object, &opts).await;
lookup_opts.no_lock = true;
let oi = self.get_object_info(bucket, object, &lookup_opts).await;
match oi {
Ok(oi) => {
// If top level is a delete marker proceed to upload.
// Ordinary writes may proceed past a top-level delete marker;
// data movement must not replace an acknowledged deletion.
if oi.delete_marker {
return None;
return opts.data_movement.then_some(StorageError::PreconditionFailed);
}
let if_none_match = http_preconditions.if_none_match_value().map(str::to_owned);
let if_match = http_preconditions.if_match_value().map(str::to_owned);
if should_prevent_write(&oi, if_none_match, if_match) {
if should_prevent_write(&oi, if_none_match, if_match)
&& !crate::data_movement::can_replace_stale_data_movement_target(&oi, opts)
{
return Some(StorageError::PreconditionFailed);
}
}
@@ -6449,6 +6459,32 @@ mod tests {
assert_eq!(versions.versions[0].name, object);
}
#[tokio::test]
async fn load_file_info_versions_exact_encodes_directory_key_but_returns_logical_name() {
let bucket = "exact-directory-versions-bucket";
let object = "prefix/directory/";
let disk_object = rustfs_utils::path::encode_dir_object(object);
let (_dir, disk) = read_multiple_test_disk(bucket, &[]).await;
let mut fi = metadata_test_fileinfo(object);
fi.version_id = Some(Uuid::new_v4());
fi.mod_time = Some(OffsetDateTime::now_utc());
disk.write_metadata(bucket, bucket, disk_object.as_str(), fi.clone())
.await
.expect("directory metadata should be written under the encoded key");
let set = io_primitives_test_set(vec![Some(disk)], 0).await;
let versions = set
.load_file_info_versions_exact(bucket, object)
.await
.expect("exact directory version load should succeed")
.expect("exact directory version load should find metadata");
assert_eq!(versions.name, object);
assert_eq!(versions.versions.len(), 1);
assert_eq!(versions.versions[0].name, object);
assert_eq!(versions.versions[0].version_id, fi.version_id);
}
#[tokio::test]
async fn load_file_info_versions_exact_rejects_transitioned_duplicate_parts() {
let bucket = "exact-versions-bucket";
crates/ecstore/src/set_disk/metadata.rs
+31
View File
@@ -85,6 +85,15 @@ impl SetDisks {
format!("{}/{}", Self::get_multipart_sha_dir(bucket, object), upload_uuid)
}
pub(super) fn get_multipart_upload_dir(bucket: &str, object: &str, upload_id: &str, data_movement: bool) -> String {
let upload_dir = Self::get_upload_id_dir(bucket, object, upload_id);
if data_movement {
format!("{DATA_MOVEMENT_MULTIPART_PREFIX}/{upload_dir}")
} else {
upload_dir
}
}
pub(super) fn get_multipart_sha_dir(bucket: &str, object: &str) -> String {
let path = format!("{bucket}/{object}");
let mut hasher = Sha256::new();
@@ -466,6 +475,28 @@ impl SetDisks {
Self::find_file_info_in_quorum(metas, &mod_time, &etag, quorum)
}
pub(crate) fn hydrate_selected_fileinfo_part_checksums(fi: &mut FileInfo) -> disk::error::Result<()> {
fi.hydrate_data_movement_part_checksums().map_err(DiskError::from)?;
for part in &fi.parts {
let Some(checksums) = part.checksums.as_ref() else {
continue;
};
let mut algorithms = HashSet::with_capacity(checksums.len());
for (name, value) in checksums {
let Some(checksum) = rustfs_rio::Checksum::new_from_string(name, value) else {
return Err(DiskError::FileCorrupt);
};
if checksum.checksum_type.is(rustfs_rio::ChecksumType::MULTIPART) {
return Err(DiskError::FileCorrupt);
}
if !algorithms.insert(checksum.checksum_type.base().0) {
return Err(DiskError::FileCorrupt);
}
}
}
Ok(())
}
fn update_hash_bytes(hasher: &mut Sha256, value: &[u8]) {
hasher.update(value.len().to_le_bytes());
hasher.update(value);
crates/ecstore/src/set_disk/mod.rs
+152 -10
View File
@@ -288,6 +288,7 @@ pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024;
pub const MAX_PARTS_COUNT: usize = 10000;
pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket";
pub(crate) const RUSTFS_MULTIPART_OBJECT_KEY: &str = "x-rustfs-internal-multipart-object";
pub(crate) const DATA_MOVEMENT_MULTIPART_PREFIX: &str = "data-movement";
const ENV_ISSUE3031_DIAG_ENABLE: &str = "RUSTFS_ISSUE3031_DIAG_ENABLE";
/// Validate disk metadata at a boundary that may legitimately return a delete
@@ -3780,8 +3781,10 @@ fn resolve_delete_version_state(opts: &ObjectOptions, goi: &ObjectInfo, version_
if opts.version_id.is_some() {
// Decommission/rebalance may recreate a delete marker on a new pool before that
// exact version exists there, so we must still treat it as a mark-delete write.
if opts.data_movement && opts.delete_marker && !version_found {
let data_movement_missing_delete_marker = opts.data_movement && opts.delete_marker && !version_found;
if data_movement_missing_delete_marker {
mark_delete = true;
delete_marker = true;
}
let delete_marker_version_purge = version_found && goi.delete_marker && !opts.version_purge_status().is_empty();
@@ -3790,7 +3793,10 @@ fn resolve_delete_version_state(opts: &ObjectOptions, goi: &ObjectInfo, version_
mark_delete = false;
}
if opts.version_purge_status().is_empty() && opts.delete_marker_replication_status().is_empty() {
if !data_movement_missing_delete_marker
&& opts.version_purge_status().is_empty()
&& opts.delete_marker_replication_status().is_empty()
{
mark_delete = false;
}
@@ -3832,6 +3838,19 @@ impl SetDisks {
opts: &ObjectOptions,
) -> Result<()> {
let storage_class_config = self.storage_class_config_snapshot();
let bucket_lifecycle_guard = if let Some(expected_incarnation_id) = opts.expected_bucket_incarnation_id
&& opts.bucket_lifecycle_lock_fence.is_none()
&& !crate::bucket::utils::is_meta_bucketname(bucket)
{
Some(
metadata_sys::object_store_in(&self.ctx)
.await?
.acquire_bucket_incarnation_fence(bucket, expected_incarnation_id)
.await?,
)
} else {
None
};
let _lock_guard = if !opts.no_lock {
Some(
self.new_ns_lock(bucket, object)
@@ -3844,6 +3863,12 @@ impl SetDisks {
None
};
if opts.http_preconditions.is_some()
&& let Some(err) = self.check_write_precondition(bucket, object, opts).await
{
return Err(err);
}
let disks = self.disks.read().await.clone();
let storage_class = opts.user_defined.get(AMZ_STORAGE_CLASS).map(String::as_str);
let layout = resolve_write_layout(
@@ -3856,6 +3881,20 @@ impl SetDisks {
)?;
let fi = build_tiered_decommission_file_info(bucket, object, fi, layout);
let write_quorum = layout.write_quorum;
if opts
.bucket_lifecycle_lock_fence
.as_ref()
.is_some_and(NamespaceLockFence::is_lock_lost)
|| bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
{
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode: "decommission_tiered_object_commit",
bucket: bucket.to_string(),
object: object.to_string(),
required: 1,
achieved: 0,
});
}
let parts_metadata = vec![fi.clone(); disks.len()];
let (shuffle_disks, parts_metadata) = Self::shuffle_disks_and_parts_metadata(&disks, &parts_metadata, &fi);
@@ -5283,6 +5322,22 @@ mod tests {
assert!(delete_marker);
}
#[test]
fn resolve_delete_version_state_creates_missing_suspended_data_movement_marker() {
let opts = ObjectOptions {
version_suspended: true,
version_id: Some(Uuid::nil().to_string()),
data_movement: true,
delete_marker: true,
..Default::default()
};
let (mark_delete, delete_marker) = resolve_delete_version_state(&opts, &ObjectInfo::default(), false);
assert!(mark_delete);
assert!(delete_marker);
}
#[test]
fn should_force_delete_marker_for_missing_version_rejects_data_movement_latest_delete() {
let opts = ObjectOptions {
@@ -6966,6 +7021,7 @@ mod tests {
rustfs_filemeta::FileInfoOpts {
data: false,
include_free_versions: false,
include_part_checksums: true,
},
)
.expect("test file metadata should decode as file info")
@@ -7094,20 +7150,92 @@ mod tests {
fn test_latest_fileinfo_selection_preserves_degraded_read_quorum_without_competing_latest() {
let mod_time = OffsetDateTime::now_utc();
let data_dir = Uuid::new_v4();
let metas = vec![
quorum_test_fileinfo(mod_time, data_dir, "part-etag-old", 1),
quorum_test_fileinfo(mod_time, data_dir, "part-etag-old", 2),
FileInfo::default(),
FileInfo::default(),
];
let mut first = quorum_test_fileinfo(mod_time, data_dir, "part-etag-old", 1);
rustfs_utils::http::insert_str(
&mut first.metadata,
rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string(),
);
let mut second = first.clone();
second.erasure.index = 2;
let metas = vec![first, second, FileInfo::default(), FileInfo::default()];
let errs = vec![None, None, Some(DiskError::DiskNotFound), Some(DiskError::DiskNotFound)];
let (_, selected, selected_quorum) = SetDisks::select_valid_fileinfo(&vec![None; metas.len()], &metas, &errs, "", 2, 3)
.expect("read quorum should remain enough when no competing latest is visible");
let (_, mut selected, selected_quorum) =
SetDisks::select_valid_fileinfo(&vec![None; metas.len()], &metas, &errs, "", 2, 3)
.expect("read quorum should remain enough when no competing latest is visible");
assert_eq!(selected_quorum, 2);
assert_eq!(selected.data_dir, Some(data_dir));
assert_eq!(selected.parts[0].etag, "part-etag-old");
assert!(selected.parts[0].checksums.is_none());
SetDisks::hydrate_selected_fileinfo_part_checksums(&mut selected)
.expect("requested part checksums should hydrate after winner selection");
assert_eq!(
selected.parts[0]
.checksums
.as_ref()
.and_then(|checksums| checksums.get("CRC32C"))
.map(String::as_str),
Some("AAAAAA==")
);
}
#[test]
fn test_degraded_fileinfo_selection_rejects_malformed_part_checksum_metadata() {
let mod_time = OffsetDateTime::now_utc();
let data_dir = Uuid::new_v4();
let mut first = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 1);
rustfs_utils::http::insert_str(&mut first.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS, "not-json".to_string());
let mut second = first.clone();
second.erasure.index = 2;
let metas = vec![first, second, FileInfo::default(), FileInfo::default()];
let errs = vec![None, None, Some(DiskError::DiskNotFound), Some(DiskError::DiskNotFound)];
let (_, mut selected, _) = SetDisks::select_valid_fileinfo(&vec![None; metas.len()], &metas, &errs, "", 2, 3)
.expect("winner selection should defer sidecar decoding");
let err = SetDisks::hydrate_selected_fileinfo_part_checksums(&mut selected)
.expect_err("a malformed degraded winner must fail closed when checksums are requested");
assert_eq!(err, DiskError::FileCorrupt);
}
#[test]
fn test_pick_valid_fileinfo_rejects_malformed_part_checksum_metadata() {
let mod_time = OffsetDateTime::now_utc();
let data_dir = Uuid::new_v4();
let mut meta = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 1);
rustfs_utils::http::insert_str(&mut meta.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS, "not-json".to_string());
let mut second = meta.clone();
second.erasure.index = 2;
let mut selected = SetDisks::pick_valid_fileinfo(&[meta, second], Some(mod_time), None, 2)
.expect("winner selection should defer sidecar decoding");
let err = SetDisks::hydrate_selected_fileinfo_part_checksums(&mut selected)
.expect_err("a malformed winning part-checksum sidecar must fail closed when checksums are requested");
assert_eq!(err, DiskError::FileCorrupt);
}
#[test]
fn test_part_checksum_hydration_rejects_invalid_algorithm_and_value() {
let mod_time = OffsetDateTime::now_utc();
let data_dir = Uuid::new_v4();
for encoded in [
r#"[[1,[["UNKNOWN","AAAAAA=="]]]]"#,
r#"[[1,[["CRC32C","not-base64"]]]]"#,
r#"[[1,[["CRC32C","AA=="]]]]"#,
r#"[[1,[["CRC32C","AAAAAA==-0"]]]]"#,
r#"[[1,[["CRC32C","AAAAAA==-1"]]]]"#,
r#"[[1,[["CRC32C","AAAAAA=="],["crc32c","BBBBBB=="]]]]"#,
] {
let mut meta = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 1);
rustfs_utils::http::insert_str(&mut meta.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS, encoded.to_string());
let err = SetDisks::hydrate_selected_fileinfo_part_checksums(&mut meta)
.expect_err("invalid persisted part checksum metadata must fail closed");
assert_eq!(err, DiskError::FileCorrupt);
}
}
#[test]
@@ -10744,6 +10872,20 @@ mod tests {
assert!(marker.delete_marker);
let marker_version = marker.version_id.expect("versioned delete marker should carry a version id");
let create_only = ObjectOptions {
versioned: true,
data_movement: true,
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
}),
..Default::default()
};
assert_eq!(
set_disks.check_write_precondition(bucket, object, &create_only).await,
Some(StorageError::PreconditionFailed),
"data movement must not replace a target delete marker"
);
let err = match set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
.await
crates/ecstore/src/set_disk/ops/heal.rs
+2 -1
View File
@@ -542,7 +542,8 @@ impl SetDisks {
let filter_by_etag = quorum_etag.is_some();
match Self::pick_valid_fileinfo(&parts_metadata, quorum_mod_time, quorum_etag.clone(), read_quorum as usize) {
Ok(latest_meta) => {
Ok(mut latest_meta) => {
Self::hydrate_selected_fileinfo_part_checksums(&mut latest_meta)?;
trace!(
event = EVENT_SET_DISK_HEAL,
component = LOG_COMPONENT_ECSTORE,
crates/ecstore/src/set_disk/ops/multipart.rs
+618 -23
View File
@@ -33,6 +33,29 @@ use tokio::task::JoinSet;
const MULTIPART_LIST_IO_CONCURRENCY: usize = 16;
pub(crate) struct StaleMultipartCleanupGuard {
file_info: FileInfo,
upload_path: String,
write_quorum: usize,
lock_guard: ObjectLockDiagGuard,
}
impl StaleMultipartCleanupGuard {
pub(crate) fn file_info(&self) -> &FileInfo {
&self.file_info
}
pub(crate) fn is_lock_lost(&self) -> bool {
self.lock_guard.is_lock_lost()
}
pub(crate) async fn delete(self, set: &SetDisks) -> Result<()> {
fence_commit_on_lock_loss(Some(&self.lock_guard), "stale_multipart_cleanup", &self.upload_path)?;
set.delete_all_with_quorum(RUSTFS_META_MULTIPART_BUCKET, &self.upload_path, self.write_quorum)
.await
}
}
#[cfg(test)]
#[derive(Clone, Copy, PartialEq, Eq)]
pub(crate) enum MultipartCommitPause {
@@ -223,6 +246,20 @@ fn validate_multipart_bucket_incarnation(
Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()))
}
fn ensure_data_movement_upload_access(
fi: &FileInfo,
bucket: &str,
object: &str,
upload_id: &str,
opts: &ObjectOptions,
) -> Result<()> {
if rustfs_utils::http::contains_key_str(&fi.metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD) && !opts.data_movement
{
return Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()));
}
Ok(())
}
async fn ensure_multipart_bucket_incarnation(
ctx: &crate::runtime::instance::InstanceContext,
fi: &FileInfo,
@@ -445,7 +482,7 @@ impl SetDisks {
return Ok(None);
}
let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id);
let upload_id_path = Self::get_multipart_upload_dir(bucket, object, upload_id, opts.data_movement);
self.acquire_read_lock_diag(op, RUSTFS_META_MULTIPART_BUCKET, &upload_id_path)
.await
.map(Some)
@@ -463,7 +500,7 @@ impl SetDisks {
return Ok(None);
}
let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id);
let upload_id_path = Self::get_multipart_upload_dir(bucket, object, upload_id, opts.data_movement);
self.acquire_write_lock_diag(op, RUSTFS_META_MULTIPART_BUCKET, &upload_id_path)
.await
.map(Some)
@@ -511,13 +548,49 @@ impl SetDisks {
upload_id: &str,
write: bool,
) -> Result<(FileInfo, Vec<FileInfo>)> {
let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id);
self.check_upload_id_exists_for_data_movement(bucket, object, upload_id, write, false)
.await
}
async fn check_upload_id_exists_with_opts(
&self,
bucket: &str,
object: &str,
upload_id: &str,
write: bool,
opts: &ObjectOptions,
) -> Result<(FileInfo, Vec<FileInfo>)> {
self.check_upload_id_exists_for_data_movement(bucket, object, upload_id, write, opts.data_movement)
.await
}
async fn check_upload_id_exists_for_data_movement(
&self,
bucket: &str,
object: &str,
upload_id: &str,
write: bool,
data_movement: bool,
) -> Result<(FileInfo, Vec<FileInfo>)> {
let upload_id_path = Self::get_multipart_upload_dir(bucket, object, upload_id, data_movement);
self.check_multipart_upload_path_exists(bucket, object, upload_id, &upload_id_path, write)
.await
}
async fn check_multipart_upload_path_exists(
&self,
bucket: &str,
object: &str,
upload_id: &str,
upload_id_path: &str,
write: bool,
) -> Result<(FileInfo, Vec<FileInfo>)> {
let disks = self.disks.read().await;
let disks = disks.clone();
let (parts_metadata, errs) =
Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_id_path, "", false, false, false)
Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, upload_id_path, "", false, false, false)
.await?;
let (read_quorum, write_quorum) = Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count)
@@ -562,6 +635,20 @@ impl SetDisks {
Ok((fi, parts_metadata))
}
pub(crate) async fn lock_stale_multipart_cleanup(&self, upload_path: &str) -> Result<StaleMultipartCleanupGuard> {
let lock_guard = self
.acquire_write_lock_diag("stale_multipart_cleanup", RUSTFS_META_MULTIPART_BUCKET, upload_path)
.await?;
let (file_info, _) = self.check_multipart_upload_path_exists("", "", "", upload_path, true).await?;
let write_quorum = file_info.write_quorum(self.default_write_quorum());
Ok(StaleMultipartCleanupGuard {
file_info,
upload_path: upload_path.to_string(),
write_quorum,
lock_guard,
})
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn list_multipart_uploads_for_incarnation(
&self,
@@ -704,6 +791,12 @@ impl SetDisks {
{
return Ok(None);
}
if rustfs_utils::http::contains_key_str(
&file_info.metadata,
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
) {
return Ok(None);
}
let object = match (
file_info.metadata.get(RUSTFS_MULTIPART_BUCKET_KEY),
@@ -852,9 +945,12 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
opts: &ObjectOptions,
) -> Result<PartInfo> {
crate::hp_guard!("SetDisks::put_object_part");
let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id);
let upload_id_path = Self::get_multipart_upload_dir(bucket, object, upload_id, opts.data_movement);
let (fi, _) = self.check_upload_id_exists(bucket, object, upload_id, true).await?;
let (fi, _) = self
.check_upload_id_exists_with_opts(bucket, object, upload_id, true, opts)
.await?;
ensure_data_movement_upload_access(&fi, bucket, object, upload_id, opts)?;
ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
.await?;
@@ -1120,7 +1216,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
.await?;
(Some(upload_guard), Some(part_guard))
};
let (commit_fi, _) = self.check_upload_id_exists(bucket, object, upload_id, false).await?;
let (commit_fi, _) = self
.check_upload_id_exists_with_opts(bucket, object, upload_id, false, opts)
.await?;
ensure_data_movement_upload_access(&commit_fi, bucket, object, upload_id, opts)?;
ensure_multipart_bucket_incarnation(
&self.ctx,
&commit_fi,
@@ -1196,11 +1295,14 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let _upload_guard = self
.acquire_multipart_upload_read_lock("list_object_parts", bucket, object, upload_id, opts)
.await?;
let (fi, _) = self.check_upload_id_exists(bucket, object, upload_id, false).await?;
let (fi, _) = self
.check_upload_id_exists_with_opts(bucket, object, upload_id, false, opts)
.await?;
ensure_data_movement_upload_access(&fi, bucket, object, upload_id, opts)?;
ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
.await?;
let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id);
let upload_id_path = Self::get_multipart_upload_dir(bucket, object, upload_id, opts.data_movement);
if max_parts > MAX_PARTS_COUNT {
max_parts = MAX_PARTS_COUNT;
@@ -1384,6 +1486,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let disks = disks.clone();
let mut user_defined = opts.user_defined.clone();
rustfs_utils::http::remove_str(&mut user_defined, rustfs_utils::http::SUFFIX_PART_CHECKSUMS);
if !opts.data_movement {
rustfs_utils::http::remove_str(&mut user_defined, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD);
}
rustfs_utils::http::metadata_compat::remove_str(
&mut user_defined,
crate::object_api::ENCRYPTED_PART_LAYOUT_QUORUM_SUFFIX,
@@ -1489,7 +1595,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let upload_id = runtime_sources::deployment_upload_id(&upload_uuid);
let upload_path = Self::get_upload_id_dir(bucket, object, upload_uuid.as_str());
let upload_path = Self::get_multipart_upload_dir(bucket, object, upload_uuid.as_str(), opts.data_movement);
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
Self::write_unique_file_info(
@@ -1524,9 +1630,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
.acquire_multipart_upload_read_lock("get_multipart_info", bucket, object, upload_id, opts)
.await?;
let (mut fi, _) = self
.check_upload_id_exists(bucket, object, upload_id, false)
.check_upload_id_exists_with_opts(bucket, object, upload_id, false, opts)
.await
.map_err(|e| to_object_err(e, vec![bucket, object, upload_id]))?;
ensure_data_movement_upload_access(&fi, bucket, object, upload_id, opts)?;
ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
.await?;
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
@@ -1548,11 +1655,14 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let _upload_guard = self
.acquire_multipart_upload_write_lock("abort_multipart_upload", bucket, object, upload_id, opts)
.await?;
let (fi, _) = self.check_upload_id_exists(bucket, object, upload_id, true).await?;
let (fi, _) = self
.check_upload_id_exists_with_opts(bucket, object, upload_id, true, opts)
.await?;
ensure_data_movement_upload_access(&fi, bucket, object, upload_id, opts)?;
ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
.await?;
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id);
let upload_id_path = Self::get_multipart_upload_dir(bucket, object, upload_id, opts.data_movement);
self.delete_all_with_quorum(
RUSTFS_META_MULTIPART_BUCKET,
@@ -1574,7 +1684,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
crate::hp_guard!("SetDisks::complete_multipart_upload");
self.invalidate_get_object_metadata_cache(bucket, object).await;
let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id);
let upload_id_path = Self::get_multipart_upload_dir(bucket, object, upload_id, opts.data_movement);
let range_seek_rollout_enabled = crate::object_api::legacy_encrypted_range_seek_enabled() && !opts.no_lock;
let mut object_lock_guard = None;
@@ -1602,7 +1712,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
.await?;
let expected_restore_operation_id = restore_commit_operation_id_from_metadata(&opts.user_defined)?;
let (mut fi, files_metas) = self.check_upload_id_exists(bucket, object, upload_id, true).await?;
let (mut fi, files_metas) = self
.check_upload_id_exists_with_opts(bucket, object, upload_id, true, opts)
.await?;
ensure_data_movement_upload_access(&fi, bucket, object, upload_id, opts)?;
ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
.await?;
let has_layout_candidate = range_seek_rollout_enabled
@@ -1771,7 +1884,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
// Build a lookup map for O(1) part resolution instead of O(n) find() in the loop
// This optimizes from O(n^2) to O(n) when processing many parts
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
let part_lookup: HashMap<usize, &ObjectPartInfo> = curr_fi.parts.iter().map(|part| (part.number, part)).collect();
for (i, p) in uploaded_parts.iter().enumerate() {
@@ -1807,7 +1920,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
// TODO: crypto
if (i < uploaded_parts.len() - 1) && !is_min_allowed_part_size(ext_part.actual_size) {
if (i < uploaded_parts.len() - 1)
&& !(opts.data_movement && ext_part.actual_size < 0)
&& !is_min_allowed_part_size(ext_part.actual_size)
{
error!(
"complete_multipart_upload part size too small: part {} size {} is less than minimum {}",
p.part_num,
@@ -1978,7 +2094,27 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
);
}
if opts.replication_request {
let data_movement_actual_size = if opts.data_movement {
rustfs_utils::http::get_consistent_str(&opts.user_defined, SUFFIX_ACTUAL_SIZE)
.map(|value| {
value
.parse::<i64>()
.ok()
.filter(|value| *value >= 0)
.ok_or_else(|| Error::other("data movement actual size metadata is invalid"))
})
.transpose()?
} else {
None
};
if let Some(actual_size) = data_movement_actual_size {
insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string());
if persist_encryption_original_size {
fi.metadata
.insert("x-rustfs-encryption-original-size".to_string(), actual_size.to_string());
}
} else if opts.replication_request {
if let Some(actual_size) = get_str(&opts.user_defined, SUFFIX_ACTUAL_OBJECT_SIZE_CAP) {
insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.clone());
if persist_encryption_original_size {
@@ -1998,7 +2134,15 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
insert_str(&mut fi.metadata, SUFFIX_COMPRESSION_SIZE, object_size.to_string());
}
rustfs_utils::http::remove_str(&mut fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS);
if let Some(part_checksums) =
rustfs_utils::http::get_consistent_str(&opts.user_defined, rustfs_utils::http::SUFFIX_PART_CHECKSUMS)
{
insert_str(&mut fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS, part_checksums.to_string());
}
if opts.data_movement {
rustfs_utils::http::remove_str(&mut fi.metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD);
fi.set_data_moved();
}
@@ -2014,19 +2158,21 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
}
}
let completed_part_numbers = fi.parts.iter().map(|part| part.number).collect::<HashSet<_>>();
let upload_data_dir = curr_fi.data_dir.unwrap_or(Uuid::nil()).to_string();
let mut parts = Vec::with_capacity(curr_fi.parts.len());
for p in curr_fi.parts.iter() {
parts.push(path_join_buf(&[
&upload_id_path,
curr_fi.data_dir.unwrap_or(Uuid::nil()).to_string().as_str(),
upload_data_dir.as_str(),
format!("part.{}.meta", p.number).as_str(),
]));
if !fi.parts.iter().any(|v| v.number == p.number) {
if !completed_part_numbers.contains(&p.number) {
parts.push(path_join_buf(&[
&upload_id_path,
curr_fi.data_dir.unwrap_or(Uuid::nil()).to_string().as_str(),
upload_data_dir.as_str(),
format!("part.{}", p.number).as_str(),
]));
}
@@ -2049,6 +2195,19 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
achieved: 0,
});
}
if opts
.namespace_lock_fence
.as_ref()
.is_some_and(NamespaceLockFence::is_lock_lost)
{
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode: "complete_multipart_upload_outer_lock",
bucket: bucket.to_string(),
object: object.to_string(),
required: 1,
achieved: 0,
});
}
if upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode: "complete_multipart_upload_commit",
@@ -2069,6 +2228,74 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
)
.await?;
if opts.data_movement
&& opts.http_preconditions.is_some()
&& let Some(err) = self.check_write_precondition(bucket, object, opts).await
{
return Err(err);
}
if opts.data_movement && opts.http_preconditions.is_some() && !crate::bucket::utils::is_meta_bucketname(bucket) {
let current = self
.get_object_info(
bucket,
object,
&ObjectOptions {
version_id: opts.version_id.clone(),
no_lock: true,
metadata_cache_safe: false,
versioned: opts.versioned,
version_suspended: opts.version_suspended,
..Default::default()
},
)
.await;
match current {
Ok(existing) if crate::data_movement::can_replace_stale_data_movement_target(&existing, opts) => {
let object_lock_config = opts.object_lock_config_snapshot.as_deref().ok_or_else(|| {
Error::other("data movement completion is missing its Object Lock configuration snapshot")
})?;
if check_object_lock_for_deletion_with_state(object_lock_config.state(), &existing, false)?.is_some() {
return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object.to_string()));
}
}
Ok(_) => return Err(StorageError::PreconditionFailed),
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
Err(err) => return Err(err),
}
}
if object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode: "complete_multipart_upload_commit",
bucket: bucket.to_string(),
object: object.to_string(),
required: 1,
achieved: 0,
});
}
if opts
.namespace_lock_fence
.as_ref()
.is_some_and(NamespaceLockFence::is_lock_lost)
{
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode: "complete_multipart_upload_outer_lock",
bucket: bucket.to_string(),
object: object.to_string(),
required: 1,
achieved: 0,
});
}
if upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode: "complete_multipart_upload_commit",
bucket: RUSTFS_META_MULTIPART_BUCKET.to_string(),
object: upload_id_path.clone(),
required: 1,
achieved: 0,
});
}
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
let complete_tail_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
// Crash-consistency injection: hard power loss after the upload is fully
@@ -2589,8 +2816,29 @@ mod tests {
.new_multipart_upload(bucket, object, create_opts)
.await
.expect("multipart upload should be created");
let part = put_test_part(set_disks, bucket, object, &upload.upload_id, 1, content, content.len() as i64).await;
(upload.upload_id, vec![part])
let mut reader = PutObjReader::new(
HashReader::from_stream(
Cursor::new(content.to_vec()),
content.len() as i64,
content.len() as i64,
None,
None,
false,
)
.expect("hash reader should be constructed"),
);
let part = set_disks
.put_object_part(bucket, object, &upload.upload_id, 1, &mut reader, create_opts)
.await
.expect("uploading the part should succeed");
(
upload.upload_id,
vec![CompletePart {
part_num: part.part_num,
etag: part.etag,
..Default::default()
}],
)
}
async fn put_test_part(
@@ -2742,6 +2990,353 @@ mod tests {
assert!(matches!(overflow_err, StorageError::PartMissingOrCorrupt));
}
#[tokio::test]
#[serial]
async fn data_movement_upload_is_hidden_from_external_multipart_operations() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "data-movement-upload-complete-bucket";
make_bucket_on_all(&disk_stores, bucket).await;
let mut durable_metadata = HashMap::new();
rustfs_utils::http::insert_str(&mut durable_metadata, rustfs_utils::http::SUFFIX_DATA_MOVED, "true".to_string());
rustfs_utils::http::insert_str(
&mut durable_metadata,
rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string(),
);
rustfs_utils::http::insert_str(
&mut durable_metadata,
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
"residual-final-marker".to_string(),
);
let ordinary_opts = ObjectOptions {
user_defined: durable_metadata,
..Default::default()
};
let (upload_id, parts) =
stage_upload_with_create_opts(&set_disks, bucket, "ordinary-object", b"ordinary multipart body", &ordinary_opts)
.await;
set_disks
.clone()
.complete_multipart_upload(bucket, "ordinary-object", &upload_id, parts, &ordinary_opts)
.await
.expect("durable object metadata must not claim upload ownership");
let completed = set_disks
.get_object_info(
bucket,
"ordinary-object",
&ObjectOptions {
include_part_checksums: true,
..Default::default()
},
)
.await
.expect("completed staging checksum metadata should remain readable");
assert_eq!(
completed.parts[0]
.checksums
.as_ref()
.and_then(|checksums| checksums.get("CRC32C"))
.map(String::as_str),
Some("AAAAAA==")
);
let object = "owned-object";
let mut metadata = HashMap::new();
rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD, "true".to_string());
rustfs_utils::http::insert_str(
&mut metadata,
rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string(),
);
let create_opts = ObjectOptions {
data_movement: true,
user_defined: metadata,
..Default::default()
};
let upload = set_disks
.new_multipart_upload(bucket, object, &create_opts)
.await
.expect("data movement upload should be created");
let upload_id = upload.upload_id;
let ordinary_upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
let data_movement_upload_path = SetDisks::get_multipart_upload_dir(bucket, object, &upload_id, true);
assert_eq!(data_movement_upload_path, format!("data-movement/{ordinary_upload_path}"));
assert!(matches!(
set_disks.check_upload_id_exists(bucket, object, &upload_id, false).await,
Err(StorageError::InvalidUploadID(..))
));
set_disks
.check_upload_id_exists_with_opts(bucket, object, &upload_id, false, &create_opts)
.await
.expect("data movement lookup should find the isolated upload");
let mut part_reader = PutObjReader::from_vec(b"data movement multipart body".to_vec());
let uploaded_part = set_disks
.put_object_part(bucket, object, &upload_id, 1, &mut part_reader, &create_opts)
.await
.expect("data movement part upload should retain ownership of its upload");
let parts = vec![CompletePart {
part_num: uploaded_part.part_num,
etag: uploaded_part.etag,
..Default::default()
}];
let listed = set_disks
.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
.await
.expect("external multipart listing should succeed");
assert!(!listed.uploads.iter().any(|upload| upload.upload_id == upload_id));
let get_err = set_disks
.get_multipart_info(bucket, object, &upload_id, &ObjectOptions::default())
.await
.expect_err("external multipart metadata reads must not expose a data movement upload");
assert!(matches!(get_err, StorageError::InvalidUploadID(..)));
let list_parts_err = set_disks
.list_object_parts(bucket, object, &upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
.await
.expect_err("external part listings must not expose a data movement upload");
assert!(matches!(list_parts_err, StorageError::InvalidUploadID(..)));
let mut external_part = PutObjReader::from_vec(b"external overwrite".to_vec());
let put_err = set_disks
.put_object_part(bucket, object, &upload_id, 1, &mut external_part, &ObjectOptions::default())
.await
.expect_err("external part uploads must not modify a data movement upload");
assert!(matches!(put_err, StorageError::InvalidUploadID(..)));
let abort_err = set_disks
.abort_multipart_upload(bucket, object, &upload_id, &ObjectOptions::default())
.await
.expect_err("external aborts must not remove a data movement upload");
assert!(matches!(abort_err, StorageError::InvalidUploadID(..)));
let external_err = set_disks
.clone()
.complete_multipart_upload(bucket, object, &upload_id, parts.clone(), &ObjectOptions::default())
.await
.expect_err("external completion must not finalize a data movement upload");
assert!(matches!(external_err, StorageError::InvalidUploadID(..)));
let internal_parts = set_disks
.list_object_parts(bucket, object, &upload_id, None, MAX_PARTS_COUNT, &create_opts)
.await
.expect("data movement part listing should retain ownership of its upload");
assert_eq!(internal_parts.parts.len(), 1);
set_disks
.clone()
.complete_multipart_upload(bucket, object, &upload_id, parts, &create_opts)
.await
.expect("data movement completion should retain ownership of its upload");
let completed = set_disks
.get_object_info(
bucket,
object,
&ObjectOptions {
include_part_checksums: true,
..Default::default()
},
)
.await
.expect("completed data movement object should be readable");
assert!(!rustfs_utils::http::contains_key_str(
&completed.user_defined,
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD
));
for suffix in [
rustfs_utils::http::SUFFIX_DATA_MOVED,
rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
] {
assert!(
completed
.user_defined
.contains_key(&rustfs_utils::http::internal_key_rustfs(suffix))
);
assert!(
completed
.user_defined
.contains_key(&format!("{}{suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX))
);
}
assert_eq!(
completed.parts[0]
.checksums
.as_ref()
.and_then(|checksums| checksums.get("CRC32C"))
.map(String::as_str),
Some("AAAAAA==")
);
let abort_object = "owned-abort-object";
let abort_upload = set_disks
.new_multipart_upload(bucket, abort_object, &create_opts)
.await
.expect("data movement abort upload should be created");
let abort_err = set_disks
.abort_multipart_upload(bucket, abort_object, &abort_upload.upload_id, &ObjectOptions::default())
.await
.expect_err("external abort must not remove the second data movement upload");
assert!(matches!(abort_err, StorageError::InvalidUploadID(..)));
set_disks
.abort_multipart_upload(bucket, abort_object, &abort_upload.upload_id, &create_opts)
.await
.expect("data movement abort should retain ownership of its upload");
}
#[tokio::test]
async fn stale_data_movement_replacement_fails_before_commit_on_outer_fence_loss() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "data-movement-stale-fence-bucket";
make_bucket_on_all(&disk_stores, bucket).await;
let old_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
let new_time = old_time + time::Duration::SECOND;
for (object, namespace_lock_fence, bucket_lifecycle_lock_fence) in [
("metadata-fence", Some(NamespaceLockFence::lost_for_test()), None),
("bucket-fence", None, Some(NamespaceLockFence::lost_for_test())),
] {
let version_id = Uuid::new_v4();
let old_body = format!("old-{object}").into_bytes();
let mut old_reader = PutObjReader::from_vec(old_body.clone());
set_disks
.put_object(
bucket,
object,
&mut old_reader,
&ObjectOptions {
data_movement: true,
versioned: true,
version_id: Some(version_id.to_string()),
mod_time: Some(old_time),
..Default::default()
},
)
.await
.expect("seed old data movement target");
let replacement_body = format!("new-{object}").into_bytes();
let create_opts = ObjectOptions {
data_movement: true,
..Default::default()
};
let (upload_id, parts) =
stage_upload_with_create_opts(&set_disks, bucket, object, &replacement_body, &create_opts).await;
let complete_opts = ObjectOptions {
data_movement: true,
versioned: true,
version_id: Some(version_id.to_string()),
mod_time: Some(new_time),
http_preconditions: Some(crate::data_movement::data_movement_target_precondition()),
namespace_lock_fence,
bucket_lifecycle_lock_fence,
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
ObjectLockConfigState::ConfirmedAbsent,
))),
..Default::default()
};
let err = set_disks
.clone()
.complete_multipart_upload(bucket, object, &upload_id, parts, &complete_opts)
.await
.expect_err("a lost outer fence must abort stale target replacement");
assert!(matches!(err, StorageError::NamespaceLockQuorumUnavailable { .. }));
let mut preserved = set_disks
.get_object_reader(
bucket,
object,
None,
HeaderMap::new(),
&ObjectOptions {
versioned: true,
version_id: Some(version_id.to_string()),
..Default::default()
},
)
.await
.expect("read target after rejected replacement");
let mut preserved_body = Vec::new();
preserved
.stream
.read_to_end(&mut preserved_body)
.await
.expect("drain target after rejected replacement");
assert_eq!(preserved_body, old_body);
set_disks
.check_upload_id_exists_with_opts(bucket, object, &upload_id, true, &complete_opts)
.await
.expect("fence loss must leave replacement staging retryable");
}
}
#[tokio::test]
#[serial]
async fn data_movement_complete_accepts_unknown_compressed_part_actual_size() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "data-movement-unknown-actual-size-bucket";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
let mut metadata = HashMap::new();
rustfs_utils::http::insert_str(
&mut metadata,
rustfs_utils::http::SUFFIX_COMPRESSION,
crate::io_support::rio::compression_metadata_value(rustfs_utils::CompressionAlgorithm::default()),
);
rustfs_utils::http::insert_str(
&mut metadata,
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
"source-generation".to_string(),
);
let create_opts = ObjectOptions {
data_movement: true,
user_defined: metadata.clone(),
..Default::default()
};
let upload = set_disks
.new_multipart_upload(bucket, object, &create_opts)
.await
.expect("data movement upload should be created");
let mut completed_parts = Vec::new();
for (number, actual_size) in [(1, -1), (2, 1)] {
let mut reader = PutObjReader::new(
HashReader::from_stream(Cursor::new(vec![number as u8]), 1, actual_size, None, None, false)
.expect("part reader should be constructed"),
);
let part = set_disks
.put_object_part(bucket, object, &upload.upload_id, number, &mut reader, &create_opts)
.await
.expect("data movement part should be written");
completed_parts.push(CompletePart {
part_num: number,
etag: part.etag,
..Default::default()
});
}
rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "2".to_string());
let completed = set_disks
.clone()
.complete_multipart_upload(
bucket,
object,
&upload.upload_id,
completed_parts,
&ObjectOptions {
data_movement: true,
user_defined: metadata,
..Default::default()
},
)
.await
.expect("data movement completion should accept the persisted unknown-size sentinel");
assert_eq!(completed.parts[0].actual_size, -1);
assert_eq!(completed.get_actual_size().expect("completed object actual size"), 2);
}
async fn assert_complete_first_linearizes(bucket: &'static str, object: &'static str, create_opts: ObjectOptions) {
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
crates/ecstore/src/set_disk/ops/object.rs
+466 -16
View File
@@ -3292,7 +3292,11 @@ impl SetDisks {
// Force the full quorum fanout (allow_early_stop=false): `disks` is the
// write target below, and an early-stop subset would only carry read
// quorum, failing write quorum on update_object_meta (backlog#872).
let (fi, _, disks) = self.get_object_fileinfo_gated(bucket, object, opts, false, false).await?;
let mut read_opts = opts.clone();
read_opts.include_part_checksums = true;
let (fi, _, disks) = self
.get_object_fileinfo_gated(bucket, object, &read_opts, false, false)
.await?;
let mut fi = fi.into_owned();
fi.metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags.to_owned());
@@ -3301,6 +3305,7 @@ impl SetDisks {
fi.metadata.insert(key.clone(), value.clone());
}
}
fi.acknowledge_data_movement();
#[cfg(test)]
pause_object_tagging_commit(bucket, object).await;
@@ -3503,11 +3508,26 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
};
fi.metadata = (*src_info.user_defined).clone();
if let Some(etag) = &src_info.etag {
fi.metadata.insert("etag".to_owned(), etag.clone());
let preserved_part_checksums = if (src_info.metadata_only || src_info.version_only)
&& rustfs_utils::http::contains_key_str(&fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS)
{
Self::hydrate_selected_fileinfo_part_checksums(&mut fi).map_err(|_| Error::FileCorrupt)?;
Some(
rustfs_utils::http::get_consistent_str(&fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS)
.ok_or(Error::FileCorrupt)?
.to_string(),
)
} else {
None
};
let mut replacement_metadata = (*src_info.user_defined).clone();
if let Some(part_checksums) = preserved_part_checksums {
rustfs_utils::http::insert_str(&mut replacement_metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS, part_checksums);
}
if let Some(etag) = &src_info.etag {
replacement_metadata.insert("etag".to_owned(), etag.clone());
}
fi.metadata = replacement_metadata.clone();
let mod_time = OffsetDateTime::now_utc();
fi.mod_time = Some(mod_time);
@@ -3538,10 +3558,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
for fi in metas.iter_mut() {
if fi.has_valid_erasure_geometry() {
fi.metadata = (*src_info.user_defined).clone();
if let Some(etag) = &src_info.etag {
fi.metadata.insert("etag".to_owned(), etag.clone());
}
fi.metadata.clone_from(&replacement_metadata);
fi.mod_time = Some(mod_time);
fi.version_id = version_id;
fi.versioned = src_opts.versioned || src_opts.version_suspended;
@@ -4482,7 +4499,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
fi.version_id = if let Some(vid) = opts.version_id.as_ref() {
Some(Uuid::parse_str(vid.as_str())?)
let vid = Uuid::parse_str(vid.as_str())?;
(!opts.version_suspended || !vid.is_nil()).then_some(vid)
} else if opts.version_suspended {
None
} else if opts.versioned {
Some(Uuid::new_v4())
} else {
@@ -4507,7 +4527,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
// Create a single object deletion request
let mut dfi = FileInfo {
name: object.to_string(),
version_id: opts.version_id.as_ref().and_then(|v| Uuid::parse_str(v).ok()),
version_id: opts
.version_id
.as_ref()
.and_then(|v| Uuid::parse_str(v).ok())
.filter(|vid| !opts.version_suspended || !vid.is_nil()),
mark_deleted: mark_delete,
deleted: delete_marker,
mod_time: Some(mod_time),
@@ -4654,6 +4678,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
}
fi.acknowledge_data_movement();
if opts.mod_time.is_some() {
fi.mod_time = opts.mod_time;
}
@@ -4720,7 +4746,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
// _lock_guard = guard_opt;
// }
let (fi, meta_arr, online_disks) = self.get_object_fileinfo(bucket, object, opts, true, false).await?;
let mut transition_read_opts = opts.clone();
transition_read_opts.include_part_checksums = true;
let (fi, meta_arr, online_disks) = self
.get_object_fileinfo(bucket, object, &transition_read_opts, true, false)
.await?;
let mut fi = fi.into_owned();
/*if err != nil {
return Err(to_object_err(err, vec![bucket, object]));
@@ -4778,6 +4808,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
let transaction_id = transaction.transaction_id;
let dest_obj = transaction.remote_object.clone();
let mut transition_meta = (*oi.user_defined).clone();
rustfs_utils::http::remove_str(&mut transition_meta, rustfs_utils::http::SUFFIX_PART_CHECKSUMS);
transition_meta.insert("name".to_string(), object.to_string());
rustfs_utils::http::metadata_compat::insert_str(
&mut transition_meta,
@@ -4938,6 +4969,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
let mut commit_opts = opts.clone();
commit_opts.no_lock = true;
commit_opts.metadata_cache_safe = false;
commit_opts.include_part_checksums = true;
let transition_lock_guard = if opts.no_lock {
None
} else {
@@ -5191,7 +5223,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
achieved: 0,
});
}
let fi = self.clone().get_object_fileinfo(bucket, object, opts, true, false).await;
let mut restore_read_opts = opts.clone();
restore_read_opts.include_part_checksums = true;
let fi = self
.clone()
.get_object_fileinfo(bucket, object, &restore_read_opts, true, false)
.await;
drop(bucket_lifecycle_guard);
if let Err(err) = fi {
return set_restore_header_fn(&mut oi, Some(to_object_err(err, vec![bucket, object]))).await;
@@ -5211,7 +5248,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
expected_operation_id.to_string(),
);
}
let restore_commit_metadata = if let Some(expected_operation_id) = expected_operation_id {
let mut restore_commit_metadata = if let Some(expected_operation_id) = expected_operation_id {
let mut metadata = HashMap::new();
metadata.insert(X_AMZ_RESTORE.as_str().to_string(), "ongoing-request=\"false\"".to_string());
rustfs_utils::http::metadata_compat::insert_str(
@@ -5223,6 +5260,15 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
} else {
HashMap::new()
};
if let Some(part_checksums) =
rustfs_utils::http::get_consistent_str(&actual_fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS)
{
rustfs_utils::http::insert_str(
&mut restore_commit_metadata,
rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
part_checksums.to_string(),
);
}
// The restore copy-back re-writes this same object via put_object /
// new_multipart_upload / complete_multipart_upload, each of which takes
// the object write lock in its commit phase. The caller
@@ -6103,7 +6149,7 @@ mod get_object_downstream_close_accounting_tests {
mod metadata_mutation_generation_tests {
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
use super::*;
use crate::disk::DiskAPI as _;
use crate::disk::{DiskAPI as _, ReadOptions};
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
async fn put_and_prime(
@@ -6140,6 +6186,31 @@ mod metadata_mutation_generation_tests {
);
}
async fn persist_part_checksum_sidecar(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, value: &str) {
let (fi, _, disks) = set_disks
.get_object_fileinfo(
bucket,
object,
&ObjectOptions {
no_lock: true,
metadata_cache_safe: false,
..Default::default()
},
true,
false,
)
.await
.expect("object metadata should be readable before adding the checksum sidecar");
let mut fi = fi.into_owned();
let disks = disks.into_owned();
rustfs_utils::http::insert_str(&mut fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS, value.to_string());
set_disks
.update_object_meta(bucket, object, fi, &disks)
.await
.expect("checksum sidecar should be persisted");
set_disks.invalidate_get_object_metadata_cache(bucket, object).await;
}
#[tokio::test]
#[serial_test::serial(metadata_cache_invalidation_probe)]
async fn metadata_semantic_mutation_generation_matrix_retires_cached_snapshot() {
@@ -6226,6 +6297,112 @@ mod metadata_mutation_generation_tests {
);
assert_retired(&set_disks, &metadata_key).await;
}
#[tokio::test]
async fn metadata_only_copy_preserves_valid_part_checksums_and_rejects_conflicting_aliases() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "metadata-copy-part-checksums-bucket";
for disk in &disk_stores {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
let valid_object = "valid-sidecar";
let (mut valid_source, _) = put_and_prime(&set_disks, bucket, valid_object, b"valid sidecar body").await;
persist_part_checksum_sidecar(&set_disks, bucket, valid_object, r#"[[1,[["CRC32C","AAAAAA=="]]]]"#).await;
valid_source.metadata_only = true;
Arc::make_mut(&mut valid_source.user_defined).insert("x-amz-meta-copy".to_string(), "updated".to_string());
set_disks
.copy_object(
bucket,
valid_object,
bucket,
valid_object,
&mut valid_source,
&ObjectOptions::default(),
&ObjectOptions::default(),
)
.await
.expect("metadata-only copy should preserve a valid checksum sidecar");
let copied = set_disks
.get_object_info(
bucket,
valid_object,
&ObjectOptions {
include_part_checksums: true,
..Default::default()
},
)
.await
.expect("copied object should retain readable part checksums");
assert_eq!(
copied.parts[0]
.checksums
.as_ref()
.and_then(|checksums| checksums.get("CRC32C"))
.map(String::as_str),
Some("AAAAAA==")
);
let conflicting_object = "conflicting-sidecar";
let (mut conflicting_source, _) =
put_and_prime(&set_disks, bucket, conflicting_object, b"conflicting sidecar body").await;
let (fi, _, disks) = set_disks
.get_object_fileinfo(
bucket,
conflicting_object,
&ObjectOptions {
no_lock: true,
metadata_cache_safe: false,
..Default::default()
},
true,
false,
)
.await
.expect("conflicting object metadata should be readable before corruption is injected");
let mut fi = fi.into_owned();
let disks = disks.into_owned();
let rustfs_key = format!(
"{}{}",
rustfs_utils::http::RUSTFS_INTERNAL_PREFIX,
rustfs_utils::http::SUFFIX_PART_CHECKSUMS
);
let minio_key = format!(
"{}{}",
rustfs_utils::http::MINIO_INTERNAL_PREFIX,
rustfs_utils::http::SUFFIX_PART_CHECKSUMS
);
fi.metadata
.insert(rustfs_key.clone(), r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string());
fi.metadata
.insert(minio_key.clone(), r#"[[1,[["CRC32C","AQAAAA=="]]]]"#.to_string());
set_disks
.update_object_meta(bucket, conflicting_object, fi, &disks)
.await
.expect("conflicting aliases should be persisted for the fail-closed regression");
set_disks
.invalidate_get_object_metadata_cache(bucket, conflicting_object)
.await;
conflicting_source.metadata_only = true;
let err = set_disks
.copy_object(
bucket,
conflicting_object,
bucket,
conflicting_object,
&mut conflicting_source,
&ObjectOptions::default(),
&ObjectOptions::default(),
)
.await
.expect_err("metadata-only copy must reject conflicting checksum aliases");
assert!(matches!(err, Error::FileCorrupt));
let raw_err = disk_stores[0]
.read_version("", bucket, conflicting_object, "", &ReadOptions::default())
.await
.expect_err("the rejected copy must leave the conflicting persisted aliases fail-closed");
assert!(matches!(raw_err, crate::disk::error::DiskError::FileCorrupt));
}
}
#[cfg(all(test, feature = "test-util"))]
@@ -6287,6 +6464,7 @@ mod transition_commit_failure_tests {
}
}
#[tokio::test]
#[serial_test::serial(restore_multipart_failure_point)]
async fn multipart_restore_aborts_every_post_create_failure() {
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
@@ -6333,6 +6511,32 @@ mod transition_commit_failure_tests {
)
.await
.expect("source multipart upload should complete");
let (source_fi, _, online_disks) = set_disks
.get_object_fileinfo(
bucket,
object,
&ObjectOptions {
no_lock: true,
metadata_cache_safe: false,
..Default::default()
},
true,
false,
)
.await
.expect("source metadata should be readable before adding the checksum sidecar");
let mut source_fi = source_fi.into_owned();
let online_disks = online_disks.into_owned();
rustfs_utils::http::insert_str(
&mut source_fi.metadata,
rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
r#"[[1,[["CRC32C","AAAAAA=="]]],[2,[["CRC32C","AQAAAA=="]]]]"#.to_string(),
);
set_disks
.update_object_meta(bucket, object, source_fi, &online_disks)
.await
.expect("source checksum sidecar should be persisted before transition");
set_disks.invalidate_get_object_metadata_cache(bucket, object).await;
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
set_disks
@@ -6416,9 +6620,32 @@ mod transition_commit_failure_tests {
"successful multipart completion must disarm cleanup without aborting"
);
let restored = set_disks
.get_object_info(bucket, object, &ObjectOptions::default())
.get_object_info(
bucket,
object,
&ObjectOptions {
include_part_checksums: true,
..Default::default()
},
)
.await
.expect("successful multipart restore must leave the committed object intact");
assert_eq!(
restored.parts[0]
.checksums
.as_ref()
.and_then(|checksums| checksums.get("CRC32C"))
.map(String::as_str),
Some("AAAAAA==")
);
assert_eq!(
restored.parts[1]
.checksums
.as_ref()
.and_then(|checksums| checksums.get("CRC32C"))
.map(String::as_str),
Some("AQAAAA==")
);
let restore_header = restored
.user_defined
.get(s3s::header::X_AMZ_RESTORE.as_str())
@@ -7900,6 +8127,88 @@ mod transition_upload_integrity_tests {
.expect("source object should be written")
}
#[tokio::test]
#[serial_test::serial]
async fn data_movement_tiered_metadata_is_create_only_under_object_lock() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "tiered-data-movement-create-only";
let object = "object.bin";
let version_id = Uuid::new_v4();
for disk in &disk_stores {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
let mut reader = PutObjReader::from_vec(b"existing target".to_vec());
set_disks
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
versioned: true,
version_id: Some(version_id.to_string()),
..Default::default()
},
)
.await
.expect("existing target should be written");
let conflicting = FileInfo {
volume: bucket.to_string(),
name: object.to_string(),
version_id: Some(version_id),
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND),
size: 12,
parts: vec![ObjectPartInfo {
number: 1,
size: 12,
actual_size: 12,
etag: "part-etag".to_string(),
..Default::default()
}],
transition_status: TRANSITION_COMPLETE.to_string(),
transition_tier: "WARM".to_string(),
transitioned_objname: "remote/object-a".to_string(),
transition_version: Some("remote-version-a".to_string()),
transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
fresh: true,
..Default::default()
};
let err = set_disks
.decommission_tiered_object(
bucket,
object,
&conflicting,
&ObjectOptions {
versioned: true,
version_id: Some(version_id.to_string()),
mod_time: conflicting.mod_time,
data_movement: true,
http_preconditions: Some(crate::data_movement::data_movement_target_precondition()),
..Default::default()
},
)
.await
.expect_err("data movement must not overwrite an existing tiered version");
assert!(matches!(err, StorageError::PreconditionFailed));
let (stored, _, _) = set_disks
.get_object_fileinfo(
bucket,
object,
&ObjectOptions {
version_id: Some(version_id.to_string()),
no_lock: true,
metadata_cache_safe: false,
..Default::default()
},
true,
false,
)
.await
.expect("the existing target should remain readable");
assert_ne!(stored.transition_status, TRANSITION_COMPLETE);
assert!(stored.transition_version.is_none());
}
fn transition_options(original: &ObjectInfo, tier_name: String) -> ObjectOptions {
ObjectOptions {
no_lock: true,
@@ -7954,6 +8263,31 @@ mod transition_upload_integrity_tests {
let object = "object.bin";
let payload = b"transition remote object must be bound to its transaction id".repeat(1024);
let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await;
let (source_fi, _, online_disks) = set_disks
.get_object_fileinfo(
bucket,
object,
&ObjectOptions {
no_lock: true,
metadata_cache_safe: false,
..Default::default()
},
true,
false,
)
.await
.expect("source metadata should be readable");
let mut source_fi = source_fi.into_owned();
let online_disks = online_disks.into_owned();
rustfs_utils::http::insert_str(
&mut source_fi.metadata,
rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string(),
);
set_disks
.update_object_meta(bucket, object, source_fi, &online_disks)
.await
.expect("source checksum sidecar should be persisted");
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
let remote_version = Uuid::new_v4().to_string();
let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
@@ -7978,6 +8312,7 @@ mod transition_upload_integrity_tests {
&ObjectOptions {
no_lock: true,
metadata_cache_safe: false,
include_part_checksums: true,
..Default::default()
},
true,
@@ -7991,6 +8326,22 @@ mod transition_upload_integrity_tests {
fi.transition_version_id,
Some(Uuid::parse_str(&remote_version).expect("test version id should parse"))
);
assert_eq!(
fi.parts[0]
.checksums
.as_ref()
.and_then(|checksums| checksums.get("CRC32C"))
.map(String::as_str),
Some("AAAAAA==")
);
let remote_metadata = backend
.metadata(remote_object)
.await
.expect("remote metadata should be stored");
assert!(
!rustfs_utils::http::contains_key_str(&remote_metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS),
"the internal checksum sidecar must not be uploaded as remote user metadata"
);
assert!(backend.contains(remote_object).await, "committed remote object should remain available");
}
@@ -8067,6 +8418,7 @@ mod transition_upload_integrity_tests {
object,
&expected,
&[],
crate::data_movement::SourceCleanupBucketFence::default(),
"test_data_movement",
)
.await
@@ -8091,6 +8443,72 @@ mod transition_upload_integrity_tests {
assert_local_source_intact(&set_disks, bucket, object, &payload).await;
}
#[tokio::test(flavor = "current_thread", start_paused = true)]
#[serial_test::serial]
async fn data_movement_cleanup_aborts_after_bucket_fence_loss() {
let refresh_calls = Arc::new(AtomicUsize::new(0));
let lockers: Vec<Arc<dyn LockClient>> = (0..4)
.map(|_| Arc::new(LockLostRefreshClient::new(Arc::clone(&refresh_calls))) as Arc<dyn LockClient>)
.collect();
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
let bucket = "data-movement-cleanup-bucket-fence-lost";
let object = "object.bin";
let payload = b"lost bucket fence must preserve the source".repeat(1024);
write_source(&set_disks, &disk_stores, bucket, object, &payload).await;
let expected = set_disks
.load_file_info_versions_exact(bucket, object)
.await
.expect("source versions should be readable")
.expect("source versions should exist");
let distributed_setup = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
let bucket_guard = set_disks
.new_ns_lock(bucket, crate::storage_api_contracts::bucket::BUCKET_LIFECYCLE_LOCK_OBJECT)
.await
.expect("create bucket lifecycle lock")
.get_read_lock(get_lock_acquire_timeout())
.await
.expect("acquire bucket lifecycle read lock");
let local_cleanup_setup = SetupTypeGuard::switch_to(SetupType::Erasure).await;
let barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(bucket, object);
let cleanup_set = Arc::clone(&set_disks);
let cleanup = tokio::spawn(async move {
let result = crate::data_movement::cleanup_source_entry_if_unchanged(
cleanup_set,
bucket,
object,
&expected,
&[],
crate::data_movement::SourceCleanupBucketFence {
expected_incarnation_id: None,
lifecycle_guard: Some(&bucket_guard),
},
"test_data_movement",
)
.await;
drop(bucket_guard);
result
});
barrier.wait_until_paused().await;
tokio::time::advance(Duration::from_secs(11)).await;
tokio::task::yield_now().await;
assert!(refresh_calls.load(Ordering::SeqCst) > 0, "bucket guard refresh must run before commit");
barrier.release();
let error = cleanup
.await
.expect("cleanup task should not panic")
.expect_err("cleanup must fail after its bucket lifecycle fence loses refresh quorum");
assert!(matches!(
error,
crate::data_movement::SourceCleanupError::Storage(StorageError::NamespaceLockQuorumUnavailable { .. })
));
assert_local_source_intact(&set_disks, bucket, object, &payload).await;
drop(local_cleanup_setup);
drop(distributed_setup);
}
#[tokio::test]
#[serial_test::serial]
async fn partial_remote_acceptance_cleans_exact_candidate_and_preserves_source() {
@@ -9925,6 +10343,33 @@ mod put_object_tags_early_stop_regression_tests {
.await
.expect("put_object should succeed");
let (fi, _, disks) = set_disks
.get_object_fileinfo(
bucket,
object,
&ObjectOptions {
no_lock: true,
metadata_cache_safe: false,
..Default::default()
},
true,
false,
)
.await
.expect("object metadata should be readable before adding the checksum sidecar");
let mut fi = fi.into_owned();
let disks = disks.into_owned();
rustfs_utils::http::insert_str(
&mut fi.metadata,
rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string(),
);
set_disks
.update_object_meta(bucket, object, fi, &disks)
.await
.expect("checksum sidecar should be persisted before tagging");
set_disks.invalidate_get_object_metadata_cache(bucket, object).await;
let tags = "unit=backlog881&stage=regression";
set_disks
.put_object_tags(bucket, object, tags, &ObjectOptions::default())
@@ -9944,6 +10389,11 @@ mod put_object_tags_early_stop_regression_tests {
Some(tags),
"disk {idx} must carry the tag written under early-stop (write set not shrunk to a read-quorum subset)"
);
assert_eq!(
rustfs_utils::http::get_consistent_str(&fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS),
Some(r#"[[1,[["CRC32C","AAAAAA=="]]]]"#),
"disk {idx} must retain the checksum sidecar across the tag metadata update"
);
}
},
)
crates/ecstore/src/set_disk/read.rs
+25 -3
View File
@@ -19,7 +19,7 @@ use crate::diagnostics::get::{
GET_METADATA_CACHE_REASON_DATA_MOVEMENT, GET_METADATA_CACHE_REASON_DELETE_MARKER, GET_METADATA_CACHE_REASON_DIST_ERASURE,
GET_METADATA_CACHE_REASON_INCL_FREE_VERSIONS, GET_METADATA_CACHE_REASON_INSUFFICIENT_CACHED_QUORUM,
GET_METADATA_CACHE_REASON_META_BUCKET, GET_METADATA_CACHE_REASON_NO_LOCK, GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED,
GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_NUMBER,
GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_CHECKSUMS, GET_METADATA_CACHE_REASON_PART_NUMBER,
GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_STALE_PUBLICATION,
GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID, GET_METADATA_CACHE_REASON_VERSION_SUSPENDED,
GET_METADATA_CACHE_REASON_VERSIONED, GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA,
@@ -340,7 +340,7 @@ impl SetDisks {
// read_all_fileinfo_observed (see read_all_fileinfo_early_stop in
// core/io_primitives.rs); unsafe requests and callers that opt out
// (allow_early_stop=false) fall back to full-wait.
let (parts_metadata, errs, metadata_fanout_diagnostics) = Self::read_all_fileinfo_observed(
let (mut parts_metadata, errs, metadata_fanout_diagnostics) = Self::read_all_fileinfo_observed(
&disks,
"",
bucket,
@@ -394,8 +394,17 @@ impl SetDisks {
return Err(to_object_err(err.into(), vec![bucket, object]));
}
let (op_online_disks, fi, fileinfo_selection_quorum) =
let (op_online_disks, mut fi, fileinfo_selection_quorum) =
Self::select_valid_fileinfo(&disks, &parts_metadata, &errs, vid.as_str(), read_quorum, write_quorum)?;
let include_part_checksums =
opts.include_part_checksums || opts.part_number.is_some() || opts.data_movement || opts.raw_data_movement_read;
if include_part_checksums {
Self::hydrate_selected_fileinfo_part_checksums(&mut fi)?;
} else {
for metadata in std::iter::once(&mut fi).chain(parts_metadata.iter_mut()) {
rustfs_utils::http::remove_str(&mut metadata.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS);
}
}
metadata_fanout_diagnostics.record_quorum_candidate_latency(metadata_metrics_path, fileinfo_selection_quorum);
if errs.iter().any(|err| err.is_some()) {
let version_id = resolved_read_repair_version_id(&fi, opts.version_id.as_deref());
@@ -1826,6 +1835,9 @@ fn get_object_metadata_cache_request_bypass_reason(bucket: &str, opts: &ObjectOp
if opts.part_number.is_some() {
return Some(GET_METADATA_CACHE_REASON_PART_NUMBER);
}
if opts.include_part_checksums {
return Some(GET_METADATA_CACHE_REASON_PART_CHECKSUMS);
}
if opts.data_movement {
return Some(GET_METADATA_CACHE_REASON_DATA_MOVEMENT);
}
@@ -2497,6 +2509,16 @@ mod metadata_cache_tests {
Some(GET_METADATA_CACHE_REASON_PART_NUMBER)
);
opts = ObjectOptions {
include_part_checksums: true,
..Default::default()
};
assert!(!is_get_object_metadata_cache_request_eligible("bucket", &opts, true));
assert_eq!(
get_object_metadata_cache_request_bypass_reason("bucket", &opts, true),
Some(GET_METADATA_CACHE_REASON_PART_CHECKSUMS)
);
opts = ObjectOptions {
data_movement: true,
..Default::default()
crates/ecstore/src/set_disk/replication.rs
+2
View File
@@ -75,6 +75,7 @@ impl SetDisks {
version_id,
versioned: opts.versioned,
version_suspended: opts.version_suspended,
include_part_checksums: true,
..Default::default()
};
let (fi, _, disks) = self
@@ -142,6 +143,7 @@ impl SetDisks {
version_id,
versioned: opts.versioned,
version_suspended: opts.version_suspended,
include_part_checksums: true,
..Default::default()
};
let (fi, _, disks) = self
crates/ecstore/src/store/bucket_fence.rs
+4
View File
@@ -161,6 +161,10 @@ impl BucketIncarnationFenceGuard {
pub(crate) fn is_lock_lost(&self) -> bool {
self.inner.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
}
pub(crate) fn namespace_lock_guard(&self) -> Option<&NamespaceLockGuard> {
self.inner.as_ref()
}
}
impl Drop for BucketIncarnationFenceGuard {
crates/ecstore/src/store/init.rs
+1493 -108
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/store/multipart.rs
+188 -5
View File
@@ -66,6 +66,76 @@ fn ensure_multipart_bucket_lifecycle_guard_held(
Ok(())
}
#[cfg(test)]
struct DataMovementMultipartCompletionBarrierState {
bucket: String,
arrived: tokio::sync::Notify,
release: tokio::sync::Notify,
}
#[cfg(test)]
pub(crate) struct DataMovementMultipartCompletionBarrier {
state: Arc<DataMovementMultipartCompletionBarrierState>,
}
#[cfg(test)]
static DATA_MOVEMENT_MULTIPART_COMPLETION_BARRIER: std::sync::OnceLock<
std::sync::Mutex<Option<Arc<DataMovementMultipartCompletionBarrierState>>>,
> = std::sync::OnceLock::new();
#[cfg(test)]
impl DataMovementMultipartCompletionBarrier {
pub(crate) fn install(bucket: &str) -> Self {
let state = Arc::new(DataMovementMultipartCompletionBarrierState {
bucket: bucket.to_string(),
arrived: tokio::sync::Notify::new(),
release: tokio::sync::Notify::new(),
});
let mut slot = DATA_MOVEMENT_MULTIPART_COMPLETION_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
.lock()
.expect("data movement multipart completion barrier mutex should not poison");
assert!(slot.is_none(), "data movement multipart completion barrier must be unique");
*slot = Some(Arc::clone(&state));
Self { state }
}
pub(crate) async fn wait_until_paused(&self) {
tokio::time::timeout(std::time::Duration::from_secs(30), self.state.arrived.notified())
.await
.expect("data movement multipart operation should reach selected completion");
}
}
#[cfg(test)]
impl Drop for DataMovementMultipartCompletionBarrier {
fn drop(&mut self) {
self.state.release.notify_one();
let mut slot = DATA_MOVEMENT_MULTIPART_COMPLETION_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
.lock()
.expect("data movement multipart completion barrier mutex should not poison");
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
*slot = None;
}
}
}
#[cfg(test)]
async fn pause_data_movement_multipart_before_selected_completion(bucket: &str) {
let barrier = DATA_MOVEMENT_MULTIPART_COMPLETION_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
.lock()
.expect("data movement multipart completion barrier mutex should not poison")
.as_ref()
.filter(|barrier| barrier.bucket == bucket)
.cloned();
if let Some(barrier) = barrier {
barrier.arrived.notify_one();
barrier.release.notified().await;
}
}
async fn list_pool_multipart_uploads_for_incarnation(
pool: &crate::core::sets::Sets,
bucket: &str,
@@ -332,7 +402,7 @@ impl ECStore {
) -> Result<MultipartUploadResult> {
self.handle_new_multipart_upload_with_pool_idx(bucket, object, opts)
.await
.map(|(res, _)| res)
.map(|(res, _, _)| res)
}
pub(crate) async fn handle_new_multipart_upload_with_pool_idx(
@@ -340,7 +410,7 @@ impl ECStore {
bucket: &str,
object: &str,
opts: &ObjectOptions,
) -> Result<(MultipartUploadResult, usize)> {
) -> Result<(MultipartUploadResult, usize, Option<Uuid>)> {
check_new_multipart_args(bucket, object)?;
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
let opts = &opts;
@@ -349,7 +419,20 @@ impl ECStore {
return self.pools[0]
.new_multipart_upload(bucket, object, opts)
.await
.map(|res| (res, 0));
.map(|res| (res, 0, opts.expected_bucket_incarnation_id));
}
if opts.data_movement && opts.version_id.is_some() {
let idx = self.select_data_movement_pool_idx(bucket, object, -1, opts, false).await?;
if idx == opts.src_pool_idx {
return Err(StorageError::DataMovementOverwriteErr(
bucket.to_owned(),
object.to_owned(),
opts.version_id.clone().unwrap_or_default(),
));
}
let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
return Ok((res, idx, opts.expected_bucket_incarnation_id));
}
for (idx, pool) in self.pools.iter().enumerate() {
@@ -372,7 +455,7 @@ impl ECStore {
if !res.uploads.is_empty() {
let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
return Ok((res, idx));
return Ok((res, idx, opts.expected_bucket_incarnation_id));
}
}
let idx = self.get_pool_idx(bucket, object, -1).await?;
@@ -385,7 +468,7 @@ impl ECStore {
}
let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
Ok((res, idx))
Ok((res, idx, opts.expected_bucket_incarnation_id))
}
#[instrument(skip(self))]
@@ -456,6 +539,30 @@ impl ECStore {
Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()))
}
pub(crate) async fn put_object_part_for_data_movement(
&self,
target_pool_idx: usize,
bucket: &str,
object: &str,
upload_id: &str,
data: &mut PutObjReader,
opts: &ObjectOptions,
) -> Result<PartInfo> {
let part_id = opts
.part_number
.ok_or_else(|| Error::other("targeted multipart upload requires a part number"))?;
check_put_object_part_args(bucket, object, upload_id)?;
if !opts.data_movement {
return Err(Error::other("targeted multipart upload requires data_movement options"));
}
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
let pool = self
.pools
.get(target_pool_idx)
.ok_or_else(|| Error::other(format!("data movement target pool {target_pool_idx} is out of range")))?;
pool.put_object_part(bucket, object, upload_id, part_id, data, &opts).await
}
#[instrument(skip(self))]
pub(super) async fn handle_get_multipart_info(
&self,
@@ -530,6 +637,26 @@ impl ECStore {
Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()))
}
pub(crate) async fn abort_multipart_upload_for_data_movement(
&self,
target_pool_idx: usize,
bucket: &str,
object: &str,
upload_id: &str,
opts: &ObjectOptions,
) -> Result<()> {
check_abort_multipart_args(bucket, object, upload_id)?;
if !opts.data_movement {
return Err(Error::other("targeted multipart abort requires data_movement options"));
}
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
let pool = self
.pools
.get(target_pool_idx)
.ok_or_else(|| Error::other(format!("data movement target pool {target_pool_idx} is out of range")))?;
pool.abort_multipart_upload(bucket, object, upload_id, &opts).await
}
#[instrument(skip(self))]
pub(super) async fn handle_complete_multipart_upload(
self: Arc<Self>,
@@ -574,6 +701,62 @@ impl ECStore {
Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()))
}
pub(crate) async fn complete_multipart_upload_for_data_movement(
self: Arc<Self>,
target_pool_idx: usize,
bucket: &str,
object: &str,
upload_id: &str,
uploaded_parts: Vec<CompletePart>,
opts: &ObjectOptions,
) -> Result<ObjectInfo> {
check_complete_multipart_args(bucket, object, upload_id)?;
if !opts.data_movement {
return Err(Error::other("targeted multipart completion requires data_movement options"));
}
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
let expected_incarnation_id = opts
.expected_bucket_incarnation_id
.ok_or_else(|| Error::other("data movement completion is missing its bucket incarnation"))?;
let lifecycle_fence = opts
.bucket_lifecycle_lock_fence
.as_ref()
.ok_or_else(|| Error::other("data movement completion is missing its bucket lifecycle fence"))?;
let snapshot = match opts.object_lock_config_snapshot.as_ref() {
Some(snapshot) => Arc::clone(snapshot),
None => {
self.object_lock_config_snapshot_under_lifecycle_fence(bucket, lifecycle_fence)
.await?
}
};
if !snapshot.is_valid_for_destructive_put(self.id, bucket, expected_incarnation_id) {
return Err(Error::other(
"data movement Object Lock snapshot does not match the target bucket generation",
));
}
snapshot.add_lock_fences(&mut opts);
opts.object_lock_config_snapshot = Some(snapshot);
}
#[cfg(test)]
pause_data_movement_multipart_before_selected_completion(bucket).await;
let pool = self
.pools
.get(target_pool_idx)
.ok_or_else(|| Error::other(format!("data movement target pool {target_pool_idx} is out of range")))?
.clone();
let result = enqueue_transition_after_write(
pool.complete_multipart_upload(bucket, object, upload_id, uploaded_parts, &opts)
.await,
LcEventSrc::S3CompleteMultipartUpload,
)
.await;
if result.is_ok() {
list_objects::observe_list_objects_mutation(self.as_ref(), bucket).await;
}
result
}
}
/// Merges per-pool `ListMultipartUploads` pages into a single globally paginated
crates/ecstore/src/store/object.rs
+620 -34
View File
@@ -918,7 +918,7 @@ fn is_equivalent_data_movement_delete_marker(source: &ObjectInfo, target: &Objec
&& is_data_movement_delete_marker(target)
&& source.version_id == target.version_id
&& source.mod_time == target.mod_time
&& source.user_defined == target.user_defined
&& is_equivalent_data_movement_delete_marker_metadata(&source.user_defined, &target.user_defined)
&& source.user_tags == target.user_tags
&& source.replication_status_internal == target.replication_status_internal
&& source.replication_status == target.replication_status
@@ -926,24 +926,185 @@ fn is_equivalent_data_movement_delete_marker(source: &ObjectInfo, target: &Objec
&& source.version_purge_status == target.version_purge_status
}
fn is_equivalent_data_movement_delete_marker_metadata(
source: &HashMap<String, String>,
target: &HashMap<String, String>,
) -> bool {
matches!(
(
data_movement_delete_marker_metadata_identity(source),
data_movement_delete_marker_metadata_identity(target)
),
(Some(source), Some(target)) if source == target
)
}
fn data_movement_delete_marker_metadata_identity(metadata: &HashMap<String, String>) -> Option<HashMap<String, String>> {
let mut identity = HashMap::with_capacity(metadata.len());
let mut local_tier_free_version_id = None;
for (key, value) in metadata {
let Some(suffix) = rustfs_utils::http::strip_internal_prefix_preserving_case(key) else {
identity.insert(key.clone(), value.clone());
continue;
};
if suffix.eq_ignore_ascii_case(rustfs_utils::http::SUFFIX_TIER_FV_ID) {
let version_id = Uuid::parse_str(value).ok().filter(|version_id| !version_id.is_nil())?;
if local_tier_free_version_id.is_some_and(|expected| expected != version_id) {
return None;
}
local_tier_free_version_id = Some(version_id);
continue;
}
let canonical_suffix = [
rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP,
rustfs_utils::http::SUFFIX_REPLICA_STATUS,
rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,
rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
rustfs_utils::http::SUFFIX_PURGESTATUS,
]
.into_iter()
.find(|candidate| suffix.eq_ignore_ascii_case(candidate))
.map(str::to_string)
.or_else(|| {
[
rustfs_utils::http::SUFFIX_REPLICATION_RESET_ARN_PREFIX,
rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX,
]
.into_iter()
.find_map(|prefix| {
suffix
.get(..prefix.len())
.is_some_and(|candidate| candidate.eq_ignore_ascii_case(prefix))
.then(|| format!("{prefix}{}", &suffix[prefix.len()..]))
})
})
.unwrap_or_else(|| suffix.to_string());
let canonical_value = if canonical_suffix.eq_ignore_ascii_case(rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP)
|| canonical_suffix.eq_ignore_ascii_case(rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP)
{
rustfs_filemeta::parse_replication_timestamp(value)?
.unix_timestamp_nanos()
.to_string()
} else {
value.clone()
};
let canonical_key = format!("{}{canonical_suffix}", rustfs_utils::http::RUSTFS_INTERNAL_PREFIX);
if identity
.insert(canonical_key, canonical_value.clone())
.is_some_and(|existing| existing != canonical_value)
{
return None;
}
}
for (status_suffix, timestamp_suffix) in [
(rustfs_utils::http::SUFFIX_REPLICA_STATUS, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP),
(
rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,
),
] {
let status_key = format!("{}{status_suffix}", rustfs_utils::http::RUSTFS_INTERNAL_PREFIX);
let timestamp_key = format!("{}{timestamp_suffix}", rustfs_utils::http::RUSTFS_INTERNAL_PREFIX);
match (identity.contains_key(&status_key), identity.contains_key(&timestamp_key)) {
(true, false) => {
identity.insert(timestamp_key, OffsetDateTime::UNIX_EPOCH.unix_timestamp_nanos().to_string());
}
(false, true) => return None,
_ => {}
}
}
Some(identity)
}
fn is_data_movement_delete_marker(info: &ObjectInfo) -> bool {
info.delete_marker
}
fn is_expected_data_movement_delete_marker_source(source: &ObjectInfo, expected_mod_time: Option<OffsetDateTime>) -> bool {
is_data_movement_delete_marker(source)
&& source.mod_time.is_some()
&& source.mod_time == expected_mod_time
&& data_movement_delete_marker_metadata_identity(&source.user_defined).is_some()
}
fn current_data_movement_delete_marker_opts(source: &ObjectInfo, opts: &ObjectOptions) -> Option<ObjectOptions> {
let replica_status = rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICA_STATUS);
let replica_timestamp = rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP);
let (replica_status, replica_timestamp) = match (replica_status, replica_timestamp) {
(None, None) => Default::default(),
(Some(status), timestamp) => {
let status = crate::bucket::replication::ReplicationStatusType::from(status.as_str());
if status.is_empty() {
return None;
}
let timestamp = match timestamp {
Some(timestamp) => rustfs_filemeta::parse_replication_timestamp(&timestamp)?,
None => OffsetDateTime::UNIX_EPOCH,
};
(status, Some(timestamp))
}
(None, Some(_)) => return None,
};
let replication_status = rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_STATUS);
let replication_timestamp =
rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP);
let (replication_status, replication_timestamp, replication_targets) = match (replication_status, replication_timestamp) {
(None, None) => Default::default(),
(Some(status), timestamp) => {
let direct_status = crate::bucket::replication::ReplicationStatusType::from(status.as_str());
let targets = crate::bucket::replication::replication_statuses_map(status.as_str());
if direct_status.is_empty() && targets.is_empty() {
return None;
}
let timestamp = match timestamp {
Some(timestamp) => rustfs_filemeta::parse_replication_timestamp(&timestamp)?,
None => OffsetDateTime::UNIX_EPOCH,
};
(Some(status), Some(timestamp), targets)
}
(None, Some(_)) => return None,
};
let mut state = source.replication_state();
if state.target_delete_marker_version_ids_corrupt {
return None;
}
state.replica_status = replica_status;
state.replica_timestamp = replica_timestamp;
state.replication_status_internal = replication_status;
state.replication_timestamp = replication_timestamp;
state.targets = replication_targets;
state.replicate_decision_str = source.replication_decision.clone();
state.delete_marker = true;
let mut target_opts = opts.clone();
target_opts.mod_time = source.mod_time;
target_opts.delete_replication = Some(state);
Some(target_opts)
}
fn expected_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo) -> ObjectInfo {
ObjectInfo::from_file_info(source, "", &source.name, source.version_id.is_some())
}
fn is_equivalent_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo, target: &ObjectInfo) -> bool {
let expected = expected_data_movement_tiered_object(source);
let Some(source_actual_size) = effective_object_actual_size(&expected) else {
return false;
};
let Some(target_actual_size) = effective_object_actual_size(target) else {
return false;
};
source.version_id == target.version_id
&& !target.delete_marker
&& source.size == target.size
&& source.get_etag() == target.etag
&& source.checksum == target.checksum
&& crate::data_movement::are_equivalent_data_movement_parts(&source.parts, &target.parts)
&& source.mod_time == target.mod_time
&& expected.user_defined == target.user_defined
&& crate::data_movement::is_equivalent_data_movement_metadata(&expected, target, source_actual_size, target_actual_size)
&& expected.user_tags == target.user_tags
&& expected.expires == target.expires
&& expected.storage_class == target.storage_class
@@ -952,11 +1113,12 @@ fn is_equivalent_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo,
&& expected.version_purge_status_internal == target.version_purge_status_internal
&& expected.version_purge_status == target.version_purge_status
&& expected.transitioned_object.status == target.transitioned_object.status
&& expected.transition_version_state == target.transition_version_state
&& expected.transitioned_object.name == target.transitioned_object.name
&& expected.transitioned_object.tier == target.transitioned_object.tier
&& expected.transitioned_object.version_id == target.transitioned_object.version_id
&& expected.transitioned_object.free_version == target.transitioned_object.free_version
&& effective_object_actual_size(target) == Some(source.size)
&& source_actual_size == target_actual_size
}
fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
@@ -1054,7 +1216,7 @@ impl ECStore {
)))
}
async fn object_lock_config_snapshot_under_lifecycle_fence(
pub(super) async fn object_lock_config_snapshot_under_lifecycle_fence(
&self,
bucket: &str,
lifecycle_fence: &NamespaceLockFence,
@@ -1454,7 +1616,8 @@ impl ECStore {
target_pool_idx: usize,
opts: &ObjectOptions,
) -> Result<Option<ObjectInfo>> {
let lookup_opts = version_aware_lookup_opts(opts, true);
let mut lookup_opts = version_aware_lookup_opts(opts, true);
lookup_opts.include_part_checksums = true;
let Some(pool) = self.pools.get(target_pool_idx) else {
return Err(Error::other(format!(
@@ -1521,6 +1684,25 @@ impl ECStore {
) -> Result<()> {
check_put_object_args(bucket, object)?;
let mut opts = opts.clone();
let bucket_incarnation_fence = if is_meta_bucketname(bucket) {
None
} else {
let expected = opts
.expected_bucket_incarnation_id
.ok_or_else(|| Error::other("tiered data movement is missing its bucket incarnation snapshot"))?;
let guard = self.acquire_bucket_incarnation_fence(bucket, expected).await?;
if let Some(namespace_guard) = guard.namespace_lock_guard() {
opts.add_bucket_lifecycle_lock_guard(namespace_guard);
}
Some(guard)
};
let mut fi = fi.clone();
if opts.data_movement {
crate::data_movement::prepare_tiered_data_movement_file_info(&mut fi)?;
}
let object = encode_dir_object(object);
if self.single_pool() {
@@ -1533,7 +1715,8 @@ impl ECStore {
let idx = if opts.data_movement && opts.version_id.is_some() {
Self::resolve_decommission_target_pool_idx_result(
self.select_data_movement_pool_idx(bucket, &object, fi.size, opts, true).await,
self.select_data_movement_pool_idx(bucket, &object, fi.size, &opts, true)
.await,
bucket,
&object,
)?
@@ -1550,7 +1733,7 @@ impl ECStore {
.await;
let target_pool_idx = resolve_data_movement_resume_target_pool(idx, resume_target_pool_idx, opts.src_pool_idx);
if self
.has_equivalent_data_movement_tiered_object(bucket, &object, fi, opts, target_pool_idx)
.has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, target_pool_idx)
.await?
{
return Ok(());
@@ -1563,14 +1746,27 @@ impl ECStore {
));
}
Self::resolve_decommission_tiered_object_result(
self.pools[idx]
.get_disks_by_key(&object)
.decommission_tiered_object(bucket, &object, fi, opts)
.await,
bucket,
&object,
)
let result = self.pools[idx]
.get_disks_by_key(&object)
.decommission_tiered_object(bucket, &object, &fi, &opts)
.await;
if matches!(result, Err(Error::PreconditionFailed)) {
if self
.has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, idx)
.await?
{
return Ok(());
}
return Err(StorageError::DataMovementOverwriteErr(
bucket.to_owned(),
object,
opts.version_id.clone().unwrap_or_default(),
));
}
if bucket_incarnation_fence.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
return Err(Error::other("tiered data movement bucket incarnation fence was lost during target write"));
}
Self::resolve_decommission_tiered_object_result(result, bucket, &object)
}
#[instrument(level = "debug", skip(self))]
@@ -1607,15 +1803,7 @@ impl ECStore {
Ok(Self::attach_read_lock_guard(reader, read_lock_guard))
}
#[instrument(level = "debug", skip(self, data))]
#[hotpath::measure(impl_type = "ECStore")]
pub(super) async fn handle_put_object(
&self,
bucket: &str,
object: &str,
data: &mut PutObjReader,
opts: &ObjectOptions,
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
async fn prepare_put_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<(String, ObjectOptions)> {
check_put_object_args(bucket, object)?;
let object = encode_dir_object(object);
@@ -1642,22 +1830,20 @@ impl ECStore {
};
snapshot.add_lock_fences(&mut opts);
}
Ok((object, opts))
}
// Keep PUT atomic-read friendly: SetDisks takes the object write lock only
// around precondition checks and the final rename/commit.
async fn select_put_object_pool_idx(&self, bucket: &str, object: &str, size: i64, opts: &ObjectOptions) -> Result<usize> {
if self.single_pool() {
return self.pools[0]
.put_object_with_old_current_size(bucket, object.as_str(), data, &opts)
.await;
return Ok(0);
}
let idx = if opts.data_movement && opts.version_id.is_some() {
self.select_data_movement_pool_idx(bucket, &object, data.size(), &opts, false)
.await?
self.select_data_movement_pool_idx(bucket, object, size, opts, false).await?
} else if opts.no_lock {
self.get_pool_idx_no_lock(bucket, &object, data.size()).await?
self.get_pool_idx_no_lock(bucket, object, size).await?
} else {
self.get_pool_idx(bucket, &object, data.size()).await?
self.get_pool_idx(bucket, object, size).await?
};
if opts.data_movement && idx == opts.src_pool_idx {
@@ -1667,7 +1853,50 @@ impl ECStore {
opts.version_id.clone().unwrap_or_default(),
));
}
Ok(idx)
}
pub(crate) async fn put_object_for_data_movement(
&self,
bucket: &str,
object: &str,
data: &mut PutObjReader,
opts: &ObjectOptions,
) -> Result<(usize, Result<ObjectInfo>)> {
if !opts.data_movement {
return Err(Error::other("data movement PUT requires data_movement options"));
}
let (object, opts) = self.prepare_put_object(bucket, object, opts).await?;
let idx = self
.select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts)
.await?;
let result = self.pools[idx]
.put_object_with_old_current_size(bucket, &object, data, &opts)
.await
.map(|(object_info, _)| object_info);
let result = enqueue_transition_after_write(result, LcEventSrc::S3PutObject).await;
if result.is_ok() {
list_objects::observe_list_objects_mutation(self, bucket).await;
}
Ok((idx, result))
}
#[instrument(level = "debug", skip(self, data))]
#[hotpath::measure(impl_type = "ECStore")]
pub(super) async fn handle_put_object(
&self,
bucket: &str,
object: &str,
data: &mut PutObjReader,
opts: &ObjectOptions,
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
let (object, opts) = self.prepare_put_object(bucket, object, opts).await?;
let idx = self
.select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts)
.await?;
// Keep PUT atomic-read friendly: SetDisks takes the object write lock only
// around precondition checks and the final rename/commit.
self.pools[idx]
.put_object_with_old_current_size(bucket, &object, data, &opts)
.await
@@ -2110,6 +2339,50 @@ impl ECStore {
};
let target_pool_idx =
resolve_data_movement_resume_target_pool(selected_target_pool_idx, resume_target_pool_idx, opts.src_pool_idx);
let mut delete_marker_target_opts = None;
if opts.delete_marker && should_check_data_movement_resume_target(opts.src_pool_idx, target_pool_idx) {
let source = self
.find_data_movement_target_info(bucket, object, opts.src_pool_idx, &opts)
.await?;
let Some(source) = source else {
return Err(StorageError::DataMovementOverwriteErr(
bucket.to_owned(),
object.to_owned(),
opts.version_id.unwrap_or_default(),
));
};
if !is_expected_data_movement_delete_marker_source(&source, opts.mod_time) {
return Err(StorageError::DataMovementOverwriteErr(
bucket.to_owned(),
object.to_owned(),
opts.version_id.unwrap_or_default(),
));
}
let Some(target_opts) = current_data_movement_delete_marker_opts(&source, &opts) else {
return Err(StorageError::DataMovementOverwriteErr(
bucket.to_owned(),
object.to_owned(),
opts.version_id.unwrap_or_default(),
));
};
let target = self
.find_data_movement_target_info(bucket, object, target_pool_idx, &target_opts)
.await?;
if let Some(target) = target {
if is_equivalent_data_movement_delete_marker(&source, &target) {
let mut target = target;
target.name = decode_dir_object(object);
return Ok(target);
}
return Err(StorageError::DataMovementOverwriteErr(
bucket.to_owned(),
object.to_owned(),
opts.version_id.unwrap_or_default(),
));
}
delete_marker_target_opts = Some(target_opts);
}
if !should_check_data_movement_resume_target(opts.src_pool_idx, target_pool_idx) {
if let Ok((source_pool_info, _)) = existing_pool_info
@@ -2137,7 +2410,8 @@ impl ECStore {
));
}
let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, opts).await?;
let target_opts = delete_marker_target_opts.unwrap_or(opts);
let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, target_opts).await?;
obj.name = decode_dir_object(obj.name.as_str());
return Ok(obj);
}
@@ -3200,6 +3474,247 @@ mod tests {
assert!(!is_equivalent_data_movement_delete_marker(&source, &mismatched));
}
#[test]
fn equivalent_data_movement_delete_marker_accepts_distinct_local_free_version_ids() {
let mut source = ObjectInfo {
version_id: Some(Uuid::from_u128(1)),
delete_marker: true,
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
};
rustfs_utils::http::insert_str(
Arc::make_mut(&mut source.user_defined),
rustfs_utils::http::SUFFIX_TIER_FV_ID,
Uuid::from_u128(2).to_string(),
);
let mut target = source.clone();
rustfs_utils::http::insert_str(
Arc::make_mut(&mut target.user_defined),
rustfs_utils::http::SUFFIX_TIER_FV_ID,
Uuid::from_u128(3).to_string(),
);
assert!(is_equivalent_data_movement_delete_marker(&source, &target));
Arc::make_mut(&mut target.user_defined).insert(
format!("{}{}", rustfs_utils::http::MINIO_INTERNAL_PREFIX, rustfs_utils::http::SUFFIX_TIER_FV_ID),
Uuid::from_u128(4).to_string(),
);
assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
}
#[test]
fn equivalent_data_movement_delete_marker_accepts_replication_alias_expansion() {
let key = format!(
"{}{}",
rustfs_utils::http::MINIO_INTERNAL_PREFIX,
rustfs_utils::http::SUFFIX_REPLICATION_STATUS
);
let timestamp_key = format!(
"{}{}",
rustfs_utils::http::MINIO_INTERNAL_PREFIX,
rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP
);
let source = ObjectInfo {
version_id: Some(Uuid::from_u128(1)),
delete_marker: true,
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
user_defined: Arc::new(HashMap::from([
(key.clone(), "arn=COMPLETED;".to_string()),
(timestamp_key, "1970-01-01T00:00:01Z".to_string()),
])),
..Default::default()
};
let mut target = source.clone();
rustfs_utils::http::insert_str(
Arc::make_mut(&mut target.user_defined),
rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
"arn=COMPLETED;".to_string(),
);
rustfs_utils::http::insert_str(
Arc::make_mut(&mut target.user_defined),
rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,
(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND).to_string(),
);
assert!(is_equivalent_data_movement_delete_marker(&source, &target));
Arc::make_mut(&mut target.user_defined).insert(key, "arn=FAILED;".to_string());
assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
}
#[test]
fn data_movement_delete_marker_source_requires_persisted_mod_time() {
let source = ObjectInfo {
delete_marker: true,
..Default::default()
};
assert!(!is_expected_data_movement_delete_marker_source(&source, None));
let source = ObjectInfo {
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
..source
};
assert!(is_expected_data_movement_delete_marker_source(&source, Some(OffsetDateTime::UNIX_EPOCH)));
assert!(!is_expected_data_movement_delete_marker_source(&source, None));
}
#[test]
fn data_movement_delete_marker_uses_current_source_replication_state() {
let expected_timestamp = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
let timestamp = expected_timestamp.to_string();
let mut metadata = HashMap::new();
rustfs_utils::http::insert_str(
&mut metadata,
rustfs_utils::http::SUFFIX_REPLICA_STATUS,
ReplicationStatusType::Replica.to_string(),
);
rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP, timestamp.clone());
rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP, timestamp);
rustfs_utils::http::insert_str(
&mut metadata,
rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
"arn=COMPLETED;".to_string(),
);
rustfs_utils::http::insert_str(
&mut metadata,
&format!(
"{}{}",
rustfs_utils::http::SUFFIX_REPLICATION_RESET_ARN_PREFIX,
"arn:minio:replication::TenantA:bucket"
),
"reset-id".to_string(),
);
rustfs_utils::http::insert_str(
&mut metadata,
&format!(
"{}{}",
rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX,
"arn:minio:replication::TenantA:bucket"
),
"target-version".to_string(),
);
let source = ObjectInfo {
delete_marker: true,
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
replication_status_internal: Some("arn=COMPLETED;".to_string()),
replication_decision: "arn=replicate;".to_string(),
user_defined: Arc::new(metadata),
..Default::default()
};
let opts = ObjectOptions {
mod_time: source.mod_time,
delete_replication: Some(ReplicationState {
replication_status_internal: Some("arn=PENDING;".to_string()),
..Default::default()
}),
..Default::default()
};
let target_opts = current_data_movement_delete_marker_opts(&source, &opts).expect("valid current source state");
let state = target_opts.delete_replication.as_ref().expect("current replication state");
assert_eq!(state.replication_status_internal.as_deref(), Some("arn=COMPLETED;"));
assert_eq!(state.replica_status, crate::bucket::replication::ReplicationStatusType::Replica);
assert_eq!(state.replica_timestamp, Some(expected_timestamp));
assert_eq!(state.replication_timestamp, state.replica_timestamp);
assert_eq!(state.replicate_decision_str, "arn=replicate;");
assert_eq!(
state
.reset_statuses_map
.get("arn:minio:replication::TenantA:bucket")
.map(String::as_str),
Some("reset-id")
);
assert_eq!(
state
.target_delete_marker_version_ids
.get("arn:minio:replication::TenantA:bucket")
.map(String::as_str),
Some("target-version")
);
}
#[test]
fn data_movement_delete_marker_rejects_corrupt_target_version_maps() {
let suffix = format!("{}not-an-arn", rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX);
let mut malformed = HashMap::new();
rustfs_utils::http::insert_str(&mut malformed, &suffix, "target-version".to_string());
let malformed_source = ObjectInfo {
delete_marker: true,
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
user_defined: Arc::new(malformed),
..Default::default()
};
assert!(current_data_movement_delete_marker_opts(&malformed_source, &ObjectOptions::default()).is_none());
let mut conflicted = HashMap::new();
let suffix = format!(
"{}arn:minio:replication::target:bucket",
rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX
);
rustfs_utils::http::insert_str(&mut conflicted, &suffix, "target-version-a".to_string());
conflicted.insert(
format!("{}{suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX),
"target-version-b".to_string(),
);
let conflicted_source = ObjectInfo {
user_defined: Arc::new(conflicted),
..malformed_source.clone()
};
assert!(current_data_movement_delete_marker_opts(&conflicted_source, &ObjectOptions::default()).is_none());
let mut over_cap = HashMap::new();
for index in 0..=1_000 {
let suffix = format!(
"{}arn:minio:replication::target:bucket-{index}",
rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX
);
rustfs_utils::http::insert_str(&mut over_cap, &suffix, format!("target-version-{index}"));
}
let over_cap_source = ObjectInfo {
user_defined: Arc::new(over_cap),
..malformed_source
};
assert!(current_data_movement_delete_marker_opts(&over_cap_source, &ObjectOptions::default()).is_none());
}
#[test]
fn data_movement_delete_marker_normalizes_legacy_missing_replication_timestamps() {
let mut source_metadata = HashMap::new();
rustfs_utils::http::insert_str(
&mut source_metadata,
rustfs_utils::http::SUFFIX_REPLICA_STATUS,
ReplicationStatusType::Replica.to_string(),
);
rustfs_utils::http::insert_str(
&mut source_metadata,
rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
"arn=COMPLETED;".to_string(),
);
let source = ObjectInfo {
delete_marker: true,
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
replication_status_internal: Some("arn=COMPLETED;".to_string()),
user_defined: Arc::new(source_metadata),
..Default::default()
};
let target_opts = current_data_movement_delete_marker_opts(&source, &ObjectOptions::default())
.expect("legacy status-only metadata should remain migratable");
let state = target_opts
.delete_replication
.expect("replication state should be reconstructed");
assert_eq!(state.replica_timestamp, Some(OffsetDateTime::UNIX_EPOCH));
assert_eq!(state.replication_timestamp, Some(OffsetDateTime::UNIX_EPOCH));
let mut target_metadata = (*source.user_defined).clone();
let epoch = OffsetDateTime::UNIX_EPOCH
.format(&time::format_description::well_known::Rfc3339)
.unwrap();
rustfs_utils::http::insert_str(&mut target_metadata, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP, epoch.clone());
rustfs_utils::http::insert_str(&mut target_metadata, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP, epoch);
assert!(is_equivalent_data_movement_delete_marker_metadata(&source.user_defined, &target_metadata));
}
#[test]
fn equivalent_data_movement_delete_marker_rejects_metadata_and_replication_mismatch() {
let version_id = Uuid::nil();
@@ -3344,6 +3859,67 @@ mod tests {
assert!(is_equivalent_data_movement_tiered_object(&source, &target));
}
#[test]
fn equivalent_data_movement_tiered_object_uses_logical_compressed_and_encrypted_sizes() {
let mut compressed = tiered_equivalence_source();
compressed.size = 600;
rustfs_utils::http::insert_str(&mut compressed.metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "S2".to_string());
rustfs_utils::http::insert_str(&mut compressed.metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1024".to_string());
let compressed_target = tiered_equivalence_target(&compressed);
assert!(is_equivalent_data_movement_tiered_object(&compressed, &compressed_target));
let mut encrypted = tiered_equivalence_source();
encrypted.size = 640;
encrypted.metadata.insert(
rustfs_utils::http::object_encryption_keys::INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(),
"key-id".to_string(),
);
encrypted.metadata.insert(
rustfs_utils::http::object_encryption_keys::INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER.to_string(),
"1024".to_string(),
);
let encrypted_target = tiered_equivalence_target(&encrypted);
assert!(is_equivalent_data_movement_tiered_object(&encrypted, &encrypted_target));
}
#[test]
fn equivalent_data_movement_tiered_object_accepts_transition_alias_expansion() {
let mut source = tiered_equivalence_source();
let suffix = rustfs_utils::http::SUFFIX_TRANSITION_TIER;
source.metadata.insert(
format!("{}{suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX),
source.transition_tier.clone(),
);
let mut target = tiered_equivalence_target(&source);
Arc::make_mut(&mut target.user_defined)
.insert(rustfs_utils::http::internal_key_rustfs(suffix), source.transition_tier.clone());
assert!(is_equivalent_data_movement_tiered_object(&source, &target));
}
#[test]
fn equivalent_data_movement_tiered_object_requires_hydrated_part_checksums() {
let mut source = tiered_equivalence_source();
source.parts = vec![rustfs_filemeta::ObjectPartInfo {
number: 1,
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND),
checksums: Some(HashMap::from([("CRC32C".to_string(), "AAAAAA==".to_string())])),
..Default::default()
}];
rustfs_utils::http::insert_str(
&mut source.metadata,
rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string(),
);
let mut target = tiered_equivalence_target(&source);
Arc::make_mut(&mut target.parts)[0].mod_time = None;
assert!(is_equivalent_data_movement_tiered_object(&source, &target));
let mut missing = target;
Arc::make_mut(&mut missing.parts)[0].checksums = None;
assert!(!is_equivalent_data_movement_tiered_object(&source, &missing));
}
#[test]
fn equivalent_data_movement_tiered_object_rejects_transition_mismatch() {
let source = tiered_equivalence_source();
@@ -3353,6 +3929,16 @@ mod tests {
assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
}
#[test]
fn equivalent_data_movement_tiered_object_rejects_transition_version_state_mismatch() {
let mut source = tiered_equivalence_source();
source.transition_version_state = rustfs_filemeta::TransitionVersionState::Exact;
let mut target = tiered_equivalence_target(&source);
target.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown;
assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
}
#[test]
fn equivalent_data_movement_tiered_object_rejects_user_metadata_mismatch() {
let source = tiered_equivalence_source();
crates/ecstore/tests/legacy_bitrot_read_test.rs
+1
View File
@@ -109,6 +109,7 @@ async fn run_legacy_bitrot_test_for_object(root: &std::path::Path, disk_name: &s
FileInfoOpts {
data: true, // need inline data for inline objects
include_free_versions: false,
include_part_checksums: true,
},
) {
Ok(f) => f,