From e11fcfbd087f8a8dae2c0f2c62bc0f6e40e3f10a Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 13 Aug 2026 14:12:26 +0800 Subject: [PATCH] fix(rebalance): converge multipart data movement retries (#6057) * fix(rebalance): converge multipart data movement retries * fix(rebalance): harden multipart retry replacement * fix(rebalance): isolate internal multipart uploads * test(ecstore): adapt metadata mutation fixtures * fix(rebalance): preserve transition metadata semantics * refactor(ecstore): reuse internal metadata matcher * Revert "refactor(ecstore): reuse internal metadata matcher" This reverts commit c87ca0328fff82e18fb88d8717d8894571950d50. * refactor(rebalance): reuse data movement log constants * fix(rebalance): isolate migration-owned state * fix(rebalance): preserve pre-gate retry compatibility --- crates/config/src/constants/object.rs | 24 + .../bucket/lifecycle/bucket_lifecycle_ops.rs | 483 +++-- crates/ecstore/src/core/pools.rs | 85 +- crates/ecstore/src/data_movement/mod.rs | 1807 ++++++++++++++--- crates/ecstore/src/diagnostics/get.rs | 2 + crates/ecstore/src/disk/local.rs | 1 + crates/ecstore/src/object_api/types.rs | 3 + .../ecstore/src/services/rebalance/entry.rs | 36 +- .../src/services/rebalance/migration.rs | 107 +- .../rebalance/rebalance_unit_tests.rs | 71 +- .../ecstore/src/services/rebalance/types.rs | 1 + .../ecstore/src/services/rebalance/worker.rs | 1 + .../src/set_disk/core/io_primitives.rs | 58 +- crates/ecstore/src/set_disk/metadata.rs | 31 + crates/ecstore/src/set_disk/mod.rs | 162 +- crates/ecstore/src/set_disk/ops/heal.rs | 3 +- crates/ecstore/src/set_disk/ops/multipart.rs | 641 +++++- crates/ecstore/src/set_disk/ops/object.rs | 482 ++++- crates/ecstore/src/set_disk/read.rs | 28 +- crates/ecstore/src/set_disk/replication.rs | 2 + crates/ecstore/src/store/bucket_fence.rs | 4 + crates/ecstore/src/store/init.rs | 1601 ++++++++++++++- crates/ecstore/src/store/multipart.rs | 193 +- crates/ecstore/src/store/object.rs | 654 +++++- .../ecstore/tests/legacy_bitrot_read_test.rs | 1 + crates/filemeta/Cargo.toml | 2 +- crates/filemeta/examples/dump_fileinfo.rs | 1 + crates/filemeta/src/fileinfo.rs | 24 +- crates/filemeta/src/filemeta.rs | 253 ++- crates/filemeta/src/filemeta/version.rs | 723 +++++-- crates/filemeta/src/metacache.rs | 37 +- .../rio-v2/tests/minio_generated_fixtures.rs | 1 + crates/utils/src/http/metadata_compat.rs | 24 +- docs/operations/rolling-restart.md | 18 +- rustfs/src/admin/site_replication_state.rs | 3 +- rustfs/src/app/object_usecase.rs | 3 +- 36 files changed, 6725 insertions(+), 845 deletions(-) diff --git a/crates/config/src/constants/object.rs b/crates/config/src/constants/object.rs index bab33270b..279789719 100644 --- a/crates/config/src/constants/object.rs +++ b/crates/config/src/constants/object.rs @@ -137,6 +137,21 @@ pub const DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: bool = false; const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE); const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED); +/// Request preserving legacy per-part checksum metadata during data movement. +/// +/// This remains ineffective until +/// [`ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED`] is also enabled. +pub const ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE: &str = "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_WRITE"; +pub const DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_WRITE: bool = false; + +/// Operator-attested confirmation that every serving node understands the +/// data-movement per-part checksum sidecar. +pub const ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED: &str = "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED"; +pub const DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED: bool = false; + +const _: () = assert!(!DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_WRITE); +const _: () = assert!(!DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED); + // ============================================================================= // Concurrent Request Fix - Timeout and Backpressure Configuration // ============================================================================= @@ -649,4 +664,13 @@ mod remote_version_state_tests { "RUSTFS_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED" ); } + + #[test] + fn data_movement_part_checksum_gate_uses_stable_environment_names() { + assert_eq!(super::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_WRITE"); + assert_eq!( + super::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, + "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED" + ); + } } diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index e6174be17..72d9cca86 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -2871,11 +2871,32 @@ fn stale_upload_default_due(initiated: OffsetDateTime, default_expiry: StdDurati } async fn stale_upload_current_size(set: &Arc, metadata: &HashMap, upload_dir: &str) -> Option { + stale_upload_current_size_with_opts(set, metadata, upload_dir, false).await +} + +async fn stale_upload_current_size_with_opts( + set: &Arc, + metadata: &HashMap, + upload_dir: &str, + no_lock: bool, +) -> Option { let bucket = metadata.get(RUSTFS_MULTIPART_BUCKET_KEY)?; let object = metadata.get(RUSTFS_MULTIPART_OBJECT_KEY)?; let upload_id = encode_stale_upload_id(upload_dir); + let data_movement = rustfs_utils::http::contains_key_str(metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD); let parts = set - .list_object_parts(bucket, object, &upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default()) + .list_object_parts( + bucket, + object, + &upload_id, + None, + MAX_PARTS_COUNT, + &ObjectOptions { + data_movement, + no_lock, + ..Default::default() + }, + ) .await .ok()?; @@ -2893,7 +2914,12 @@ async fn stale_upload_lifecycle_due( metadata: &HashMap, initiated: OffsetDateTime, upload_dir: &str, + no_lock: bool, ) -> Option { + if rustfs_utils::http::contains_key_str(metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD) { + return None; + } + let bucket = metadata.get(RUSTFS_MULTIPART_BUCKET_KEY)?; let object = metadata.get(RUSTFS_MULTIPART_OBJECT_KEY)?; @@ -2906,7 +2932,9 @@ async fn stale_upload_lifecycle_due( name: object.clone(), user_tags: metadata.get(AMZ_OBJECT_TAGGING).cloned().unwrap_or_default(), mod_time: Some(initiated), - size: stale_upload_current_size(set, metadata, upload_dir).await.unwrap_or_default(), + size: stale_upload_current_size_with_opts(set, metadata, upload_dir, no_lock) + .await + .unwrap_or_default(), is_latest: true, delete_marker: false, user_defined: metadata.clone(), @@ -2934,6 +2962,7 @@ async fn read_stale_multipart_candidate( FileInfoOpts { data: false, include_free_versions: false, + include_part_checksums: false, }, ) { Ok(file_info) => (Some(file_info.metadata), file_info.mod_time), @@ -2973,36 +3002,30 @@ fn merge_stale_multipart_candidate( } } +fn is_multipart_sha_dir(path: &str) -> bool { + path.len() == 64 && path.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +fn multipart_sha_path(root: &str, entry: &str) -> Option { + let sha_dir = entry.trim_end_matches('/'); + is_multipart_sha_dir(sha_dir).then(|| { + if root.is_empty() { + sha_dir.to_string() + } else { + format!("{root}/{sha_dir}") + } + }) +} + async fn cleanup_empty_multipart_sha_dirs_on_local_disks(set: &Arc) { for disk in set.get_local_disks().await.into_iter().flatten() { if !disk.is_online().await { continue; } - let sha_dirs = match disk - .list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, "", -1) - .await - { - Ok(entries) => entries, - Err(err) => { - if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound { - debug!( - event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_LIFECYCLE, - error = ?err, - reason = "multipart_root_list_failed", - "Skipped empty multipart sha cleanup" - ); - } - continue; - } - }; - - for sha_dir in sha_dirs { - let sha_dir = sha_dir.trim_end_matches('/').to_string(); - let upload_dirs = match disk - .list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, &sha_dir, -1) + for root in ["", crate::set_disk::DATA_MOVEMENT_MULTIPART_PREFIX] { + let sha_dirs = match disk + .list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, root, -1) .await { Ok(entries) => entries, @@ -3012,9 +3035,8 @@ async fn cleanup_empty_multipart_sha_dirs_on_local_disks(set: &Arc) { event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP, component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_LIFECYCLE, - sha_dir = %sha_dir, error = ?err, - reason = "multipart_sha_dir_list_failed", + reason = "multipart_root_list_failed", "Skipped empty multipart sha cleanup" ); } @@ -3022,25 +3044,48 @@ async fn cleanup_empty_multipart_sha_dirs_on_local_disks(set: &Arc) { } }; - if !upload_dirs.is_empty() { - continue; - } + for sha_dir in sha_dirs.into_iter().filter_map(|entry| multipart_sha_path(root, &entry)) { + let upload_dirs = match disk + .list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, &sha_dir, -1) + .await + { + Ok(entries) => entries, + Err(err) => { + if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound { + debug!( + event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + sha_dir = %sha_dir, + error = ?err, + reason = "multipart_sha_dir_list_failed", + "Skipped empty multipart sha cleanup" + ); + } + continue; + } + }; - if let Err(err) = disk - .delete(RUSTFS_META_MULTIPART_BUCKET, &sha_dir, DeleteOptions::default()) - .await - && err != DiskError::FileNotFound - && err != DiskError::VolumeNotFound - { - debug!( - event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_LIFECYCLE, - sha_dir = %sha_dir, - error = ?err, - reason = "multipart_sha_dir_remove_failed", - "Failed to remove empty multipart sha dir" - ); + if !upload_dirs.is_empty() { + continue; + } + + if let Err(err) = disk + .delete(RUSTFS_META_MULTIPART_BUCKET, &sha_dir, DeleteOptions::default()) + .await + && err != DiskError::FileNotFound + && err != DiskError::VolumeNotFound + { + debug!( + event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + sha_dir = %sha_dir, + error = ?err, + reason = "multipart_sha_dir_remove_failed", + "Failed to remove empty multipart sha dir" + ); + } } } } @@ -3058,30 +3103,9 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc, now: Offset continue; } - let sha_dirs = match disk - .list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, "", -1) - .await - { - Ok(entries) => entries, - Err(err) => { - if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound { - debug!( - event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_LIFECYCLE, - error = ?err, - reason = "multipart_root_list_failed", - "Skipped stale multipart cleanup" - ); - } - continue; - } - }; - - for sha_dir in sha_dirs { - let sha_dir = sha_dir.trim_end_matches('/').to_string(); - let upload_dirs = match disk - .list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, &sha_dir, -1) + for root in ["", crate::set_disk::DATA_MOVEMENT_MULTIPART_PREFIX] { + let sha_dirs = match disk + .list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, root, -1) .await { Ok(entries) => entries, @@ -3091,9 +3115,8 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc, now: Offset event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP, component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_LIFECYCLE, - sha_dir = %sha_dir, error = ?err, - reason = "multipart_sha_dir_list_failed", + reason = "multipart_root_list_failed", "Skipped stale multipart cleanup" ); } @@ -3101,39 +3124,62 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc, now: Offset } }; - for upload_dir in upload_dirs { - let upload_dir = upload_dir.trim_end_matches('/').to_string(); - let candidate_path = format!("{sha_dir}/{upload_dir}"); - if candidates - .get(&candidate_path) - .is_some_and(|existing: &StaleMultipartUploadCandidate| existing.metadata.is_some()) + for sha_dir in sha_dirs.into_iter().filter_map(|entry| multipart_sha_path(root, &entry)) { + let upload_dirs = match disk + .list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, &sha_dir, -1) + .await { - continue; - } - - let candidate = match read_stale_multipart_candidate(disk.as_ref(), &sha_dir, &upload_dir).await { - Ok(candidate) => candidate, + Ok(entries) => entries, Err(err) => { - if err != DiskError::FileNotFound { + if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound { debug!( event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP, component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_LIFECYCLE, - path = %candidate_path, + sha_dir = %sha_dir, error = ?err, - reason = "multipart_metadata_read_failed", - "Multipart metadata unavailable during stale cleanup" + reason = "multipart_sha_dir_list_failed", + "Skipped stale multipart cleanup" ); } - let initiated = initiated_from_upload_dir(&upload_dir, None); - StaleMultipartUploadCandidate { - path: candidate_path, - initiated, - metadata: None, - } + continue; } }; - merge_stale_multipart_candidate(&mut candidates, candidate); + + for upload_dir in upload_dirs { + let upload_dir = upload_dir.trim_end_matches('/').to_string(); + let candidate_path = format!("{sha_dir}/{upload_dir}"); + if candidates + .get(&candidate_path) + .is_some_and(|existing: &StaleMultipartUploadCandidate| existing.metadata.is_some()) + { + continue; + } + + let candidate = match read_stale_multipart_candidate(disk.as_ref(), &sha_dir, &upload_dir).await { + Ok(candidate) => candidate, + Err(err) => { + if err != DiskError::FileNotFound { + debug!( + event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + path = %candidate_path, + error = ?err, + reason = "multipart_metadata_read_failed", + "Multipart metadata unavailable during stale cleanup" + ); + } + let initiated = initiated_from_upload_dir(&upload_dir, None); + StaleMultipartUploadCandidate { + path: candidate_path, + initiated, + metadata: None, + } + } + }; + merge_stale_multipart_candidate(&mut candidates, candidate); + } } } } @@ -3142,7 +3188,7 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc, now: Offset let upload_dir = candidate.path.rsplit('/').next().unwrap_or_default().to_string(); let mut due = stale_upload_default_due(candidate.initiated, default_expiry); if let Some(metadata) = candidate.metadata.as_ref() - && let Some(lifecycle_due) = stale_upload_lifecycle_due(set, metadata, candidate.initiated, &upload_dir).await + && let Some(lifecycle_due) = stale_upload_lifecycle_due(set, metadata, candidate.initiated, &upload_dir, false).await && lifecycle_due < due { due = lifecycle_due; @@ -3152,34 +3198,49 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc, now: Offset continue; } - match set.delete_all(RUSTFS_META_MULTIPART_BUCKET, &candidate.path).await { + let cleanup_guard = match set.lock_stale_multipart_cleanup(&candidate.path).await { + Ok(guard) => guard, + Err(err) => { + debug!( + event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + path = %candidate.path, + error = ?err, + reason = "multipart_cleanup_lock_or_recheck_failed", + "Skipped stale multipart cleanup" + ); + continue; + } + }; + let current_metadata = cleanup_guard.file_info().metadata.clone(); + let current_initiated = initiated_from_upload_dir(&upload_dir, cleanup_guard.file_info().mod_time); + let mut current_due = stale_upload_default_due(current_initiated, default_expiry); + if let Some(lifecycle_due) = + stale_upload_lifecycle_due(set, ¤t_metadata, current_initiated, &upload_dir, true).await + && lifecycle_due < current_due + { + current_due = lifecycle_due; + } + if now < current_due || cleanup_guard.is_lock_lost() { + continue; + } + + match cleanup_guard.delete(set).await { Ok(()) => { deleted += 1; let upload_id = encode_stale_upload_id(&upload_dir); - if let Some(metadata) = candidate.metadata.as_ref() { - debug!( - bucket = metadata.get(RUSTFS_MULTIPART_BUCKET_KEY).cloned().unwrap_or_default(), - object = metadata.get(RUSTFS_MULTIPART_OBJECT_KEY).cloned().unwrap_or_default(), - upload_id = %upload_id, - due = ?due, - event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_LIFECYCLE, - state = "removed", - "Removed stale multipart upload" - ); - } else { - debug!( - path = %candidate.path, - upload_id = %upload_id, - due = ?due, - event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_LIFECYCLE, - state = "removed", - "Removed stale multipart upload" - ); - } + debug!( + bucket = current_metadata.get(RUSTFS_MULTIPART_BUCKET_KEY).cloned().unwrap_or_default(), + object = current_metadata.get(RUSTFS_MULTIPART_OBJECT_KEY).cloned().unwrap_or_default(), + upload_id = %upload_id, + due = ?current_due, + event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + state = "removed", + "Removed stale multipart upload" + ); } Err(err) => debug!( event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP, @@ -5162,6 +5223,7 @@ mod tests { use crate::services::tier::tier::TierConfigMgr; #[cfg(feature = "test-util")] use crate::services::tier::warm_backend::WarmBackend as _; + use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause}; use crate::set_disk::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY}; use crate::storage_api_contracts::namespace::NamespaceLocking as _; use crate::storage_api_contracts::{ @@ -11926,6 +11988,135 @@ mod tests { assert!(is_err_invalid_upload_id(&err)); } + #[tokio::test] + #[serial] + async fn stale_multipart_cleanup_handles_data_movement_namespace() { + let (_paths, ecstore) = setup_test_env().await; + let bucket = format!("stale-data-movement-{}", Uuid::new_v4().simple()); + create_test_bucket(&ecstore, &bucket).await; + + let create_upload = |object: &'static str, mod_time| { + let ecstore = ecstore.clone(); + let bucket = bucket.clone(); + async move { + let mut metadata = HashMap::new(); + rustfs_utils::http::insert_str( + &mut metadata, + rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD, + "cleanup-test".to_string(), + ); + ecstore + .new_multipart_upload( + &bucket, + object, + &ObjectOptions { + data_movement: true, + mod_time: Some(mod_time), + user_defined: metadata, + ..Default::default() + }, + ) + .await + .expect("data movement multipart upload should be created") + .upload_id + } + }; + + let stale_object = "stale-internal.bin"; + let active_object = "active-internal.bin"; + let now = OffsetDateTime::now_utc(); + let stale_upload_id = create_upload(stale_object, now - time::Duration::hours(30)).await; + let active_upload_id = create_upload(active_object, now).await; + + let deleted = cleanup_stale_multipart_uploads_once_at(ecstore.clone(), now, StdDuration::from_secs(24 * 60 * 60)).await; + assert!(deleted >= 1, "expected stale data movement upload to be removed"); + + let internal_opts = ObjectOptions { + data_movement: true, + ..Default::default() + }; + let stale_err = ecstore + .get_multipart_info(&bucket, stale_object, &stale_upload_id, &internal_opts) + .await + .expect_err("stale data movement upload should be removed"); + assert!(is_err_invalid_upload_id(&stale_err)); + ecstore + .get_multipart_info(&bucket, active_object, &active_upload_id, &internal_opts) + .await + .expect("active data movement upload should remain available"); + } + + #[tokio::test] + #[serial] + async fn stale_multipart_cleanup_waits_for_data_movement_part_commit() { + let (_paths, ecstore) = setup_test_env().await; + let bucket = format!("stale-data-movement-lock-{}", Uuid::new_v4().simple()); + let object = "stale-internal.bin"; + create_test_bucket(&ecstore, &bucket).await; + + let mut metadata = HashMap::new(); + rustfs_utils::http::insert_str( + &mut metadata, + rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD, + "cleanup-lock-test".to_string(), + ); + let opts = ObjectOptions { + data_movement: true, + mod_time: Some(OffsetDateTime::now_utc() - time::Duration::hours(30)), + user_defined: metadata, + ..Default::default() + }; + let upload = ecstore + .new_multipart_upload(&bucket, object, &opts) + .await + .expect("data movement multipart upload should be created"); + let barrier = MultipartCommitBarrier::install(bucket.as_str(), object, MultipartCommitPause::PutPartAfterRename); + let put_store = ecstore.clone(); + let put_bucket = bucket.clone(); + let upload_id = upload.upload_id.clone(); + let put_task = tokio::spawn(async move { + let mut data = PutObjReader::from_vec(vec![1, 2, 3, 4]); + put_store + .put_object_part( + &put_bucket, + object, + &upload_id, + 1, + &mut data, + &ObjectOptions { + data_movement: true, + ..Default::default() + }, + ) + .await + }); + barrier.wait_until_paused().await; + + let cleanup_store = ecstore.clone(); + let mut cleanup_task = tokio::spawn(async move { + cleanup_stale_multipart_uploads_once_at( + cleanup_store, + OffsetDateTime::now_utc(), + StdDuration::from_secs(24 * 60 * 60), + ) + .await + }); + assert!( + tokio::time::timeout(StdDuration::from_millis(200), &mut cleanup_task) + .await + .is_err(), + "stale cleanup must wait for the in-flight part commit upload lock" + ); + + barrier.release(); + put_task + .await + .expect("part upload task should join") + .expect("part upload should commit before stale cleanup"); + let deleted = cleanup_task.await.expect("stale cleanup task should join"); + assert!(deleted >= 1, "stale cleanup should proceed after the part commit releases its lock"); + } + #[tokio::test] #[serial] async fn stale_multipart_cleanup_applies_abort_incomplete_lifecycle_before_default_expiry() { @@ -12000,6 +12191,58 @@ mod tests { assert!(is_err_invalid_upload_id(&err)); } + #[tokio::test] + #[serial] + async fn stale_multipart_cleanup_excludes_data_movement_from_abort_lifecycle() { + let (_paths, ecstore) = setup_test_env().await; + let bucket = format!("stale-internal-lifecycle-{}", Uuid::new_v4().simple()); + let object = "logs/internal/object.bin"; + create_test_bucket(&ecstore, &bucket).await; + set_abort_incomplete_lifecycle(&bucket, "logs/", 0).await; + + let initiated = OffsetDateTime::now_utc() - time::Duration::minutes(5); + let mut metadata = HashMap::new(); + rustfs_utils::http::insert_str( + &mut metadata, + rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD, + "lifecycle-exclusion-test".to_string(), + ); + let upload = ecstore + .new_multipart_upload( + &bucket, + object, + &ObjectOptions { + data_movement: true, + mod_time: Some(initiated), + user_defined: metadata, + ..Default::default() + }, + ) + .await + .expect("data movement multipart upload should be created"); + + let deleted = cleanup_stale_multipart_uploads_once_at( + ecstore.clone(), + OffsetDateTime::now_utc(), + StdDuration::from_secs(7 * 24 * 60 * 60), + ) + .await; + assert_eq!(deleted, 0, "bucket lifecycle must not remove active data movement uploads"); + + ecstore + .get_multipart_info( + &bucket, + object, + &upload.upload_id, + &ObjectOptions { + data_movement: true, + ..Default::default() + }, + ) + .await + .expect("active data movement upload should remain available"); + } + #[tokio::test] #[serial] async fn stale_multipart_cleanup_applies_abort_lifecycle_with_size_filter() { diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index 14d095284..70aa73bde 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -2266,15 +2266,19 @@ fn decommission_delete_marker_opts( version: &rustfs_filemeta::FileInfo, version_id: Option, src_pool_idx: usize, + expected_bucket_incarnation_id: Option, ) -> ObjectOptions { + let version_suspended = version.version_id.is_none() && version_id.is_none(); ObjectOptions { - versioned: true, - version_id, + versioned: !version_suspended, + version_suspended, + version_id: version_id.or_else(|| version_suspended.then(|| uuid::Uuid::nil().to_string())), mod_time: version.mod_time, src_pool_idx, data_movement: true, delete_marker: true, skip_decommissioned: true, + expected_bucket_incarnation_id, delete_replication: version .replication_state_internal .as_ref() @@ -2299,6 +2303,7 @@ fn decommission_remote_tiered_opts( version: &rustfs_filemeta::FileInfo, version_id: Option, src_pool_idx: usize, + expected_bucket_incarnation_id: Option, ) -> ObjectOptions { ObjectOptions { versioned: version_id.is_some(), @@ -2307,6 +2312,9 @@ fn decommission_remote_tiered_opts( user_defined: version.metadata.clone(), src_pool_idx, data_movement: true, + include_part_checksums: true, + http_preconditions: Some(crate::data_movement::data_movement_target_precondition()), + expected_bucket_incarnation_id, ..Default::default() } } @@ -2805,6 +2813,7 @@ impl ECStore { lifecycle_config: Option, object_lock_config: Option, replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>, + expected_bucket_incarnation_id: Option, ) -> Result<()> { debug!( event = EVENT_DECOMMISSION_ENTRY, @@ -2834,6 +2843,11 @@ impl ECStore { } decommission_cancel_signal_result(rx.is_cancelled())?; + let bucket_incarnation_fence = match expected_bucket_incarnation_id { + Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?), + None => None, + }; + let mut fivs = load_decommission_entry_exact_versions(&set, &entry, &bucket, "file_info_versions").await?; fivs.versions @@ -2894,7 +2908,7 @@ impl ECStore { .delete_object( bucket.as_str(), &version.name, - decommission_delete_marker_opts(version, version_id.clone(), idx), + decommission_delete_marker_opts(version, version_id.clone(), idx, expected_bucket_incarnation_id), ) .await { @@ -2984,7 +2998,7 @@ impl ECStore { bucket.as_str(), &version.name, version, - &decommission_remote_tiered_opts(version, version_id.clone(), idx), + &decommission_remote_tiered_opts(version, version_id.clone(), idx, expected_bucket_incarnation_id), ) .await { @@ -3056,7 +3070,11 @@ impl ECStore { ) .await?; - if let Err(err) = self.clone().decommission_object(idx, bucket, rd).await { + if let Err(err) = self + .clone() + .decommission_object(idx, bucket, rd, expected_bucket_incarnation_id) + .await + { if is_decommission_copy_cleanup_safe_error(&err) { ignore = true; cleanup_ignored = true; @@ -3133,6 +3151,9 @@ impl ECStore { } if should_cleanup_decommission_source_entry(decommissioned, fivs.versions.len(), expired) { + if bucket_incarnation_fence.as_ref().is_some_and(|guard| guard.is_lock_lost()) { + return Err(Error::other("decommission bucket incarnation fence was lost before source cleanup")); + } decommission_cancel_signal_result(rx.is_cancelled())?; self.save_decommission_entry_progress_stage( @@ -3157,6 +3178,12 @@ impl ECStore { entry.name.as_str(), &fivs, &cleanup_preflight_allowed_missing, + data_movement::SourceCleanupBucketFence { + expected_incarnation_id: expected_bucket_incarnation_id, + lifecycle_guard: bucket_incarnation_fence + .as_ref() + .and_then(|guard| guard.namespace_lock_guard()), + }, "decommission", ) .await @@ -3268,6 +3295,11 @@ impl ECStore { let mut lifecycle_config = None; let mut object_lock_config = None; let mut replication_config = None; + let expected_bucket_incarnation_id = if bi.name == RUSTFS_META_BUCKET { + None + } else { + Some(self.bucket_incarnation_id_from_disk(&bi.name).await?) + }; if bi.name != RUSTFS_META_BUCKET { let _ = resolve_decommission_optional_bucket_config_result( @@ -3321,6 +3353,7 @@ impl ECStore { let lifecycle_config = lifecycle_config.clone(); let object_lock_config = object_lock_config.clone(); let replication_config = replication_config.clone(); + let expected_bucket_incarnation_id = expected_bucket_incarnation_id; let entry_error = entry_error.clone(); let callback_rx = callback_rx.clone(); @@ -3383,6 +3416,7 @@ impl ECStore { lifecycle_config, object_lock_config, replication_config, + expected_bucket_incarnation_id, ) .await { @@ -4168,10 +4202,24 @@ impl ECStore { } #[tracing::instrument(skip(self, rd))] - async fn decommission_object(self: Arc, pool_idx: usize, bucket: String, rd: GetObjectReader) -> Result<()> { + async fn decommission_object( + self: Arc, + pool_idx: usize, + bucket: String, + rd: GetObjectReader, + expected_bucket_incarnation_id: Option, + ) -> Result<()> { warn!("decommission_object: start {} {}", &bucket, &rd.object_info.name); let object_name = rd.object_info.name.clone(); - let result = data_movement::migrate_object(self, pool_idx, bucket.clone(), rd, "decommission_object").await; + let result = data_movement::migrate_object( + self, + pool_idx, + bucket.clone(), + rd, + expected_bucket_incarnation_id, + "decommission_object", + ) + .await; if result.is_ok() { warn!("decommission_object: migrated {} {}", &bucket, &object_name); } @@ -4347,7 +4395,8 @@ mod tests { ..Default::default() }; - let opts = decommission_delete_marker_opts(&version, Some("version-id".to_string()), 7); + let incarnation = uuid::Uuid::new_v4(); + let opts = decommission_delete_marker_opts(&version, Some("version-id".to_string()), 7, Some(incarnation)); let replication = opts.delete_replication.expect("replication state should be preserved"); assert!(opts.versioned); @@ -4357,11 +4406,25 @@ mod tests { assert_eq!(opts.src_pool_idx, 7); assert_eq!(opts.version_id.as_deref(), Some("version-id")); assert_eq!(opts.mod_time, Some(mod_time)); + assert_eq!(opts.expected_bucket_incarnation_id, Some(incarnation)); assert_eq!(replication.replica_status, ReplicationStatusType::Replica); assert!(replication.delete_marker); assert_eq!(replication.replicate_decision_str, "existing"); } + #[test] + fn decommission_delete_marker_opts_preserves_suspended_null_version() { + let version = rustfs_filemeta::FileInfo { + deleted: true, + ..Default::default() + }; + let opts = decommission_delete_marker_opts(&version, None, 7, None); + + assert!(!opts.versioned); + assert!(opts.version_suspended); + assert_eq!(opts.version_id.as_deref(), Some(uuid::Uuid::nil().to_string().as_str())); + } + #[test] fn test_decommission_object_migration_read_opts_are_raw_data_movement() { let opts = decommission_object_migration_read_opts(Some("vid-1".to_string())); @@ -4383,7 +4446,8 @@ mod tests { ..Default::default() }; - let opts = decommission_remote_tiered_opts(&version, Some("version-id".to_string()), 9); + let incarnation = uuid::Uuid::new_v4(); + let opts = decommission_remote_tiered_opts(&version, Some("version-id".to_string()), 9, Some(incarnation)); assert!(opts.versioned); assert!(opts.data_movement); @@ -4391,6 +4455,9 @@ mod tests { assert_eq!(opts.version_id.as_deref(), Some("version-id")); assert_eq!(opts.mod_time, Some(mod_time)); assert_eq!(opts.user_defined.get("x-amz-meta-key").map(String::as_str), Some("value")); + assert!(opts.include_part_checksums); + assert!(opts.http_preconditions.is_some()); + assert_eq!(opts.expected_bucket_incarnation_id, Some(incarnation)); } #[test] diff --git a/crates/ecstore/src/data_movement/mod.rs b/crates/ecstore/src/data_movement/mod.rs index 67bbfc50f..a5f631579 100644 --- a/crates/ecstore/src/data_movement/mod.rs +++ b/crates/ecstore/src/data_movement/mod.rs @@ -25,13 +25,18 @@ use crate::set_disk::{SetDisks, get_lock_acquire_timeout}; use crate::storage_api_contracts::{ multipart::{CompletePart, MultipartOperations as _}, namespace::NamespaceLocking as _, - object::{HTTPPreconditions, ObjectIO as _, ObjectOperations as _}, + object::{HTTPPreconditions, ObjectOperations as _}, }; use crate::store::ECStore; use bytes::Bytes; use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo}; -use rustfs_rio::{ChecksumType, EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex}; -use rustfs_utils::http::AMZ_OBJECT_TAGGING; +use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex}; +use rustfs_utils::http::{ + AMZ_OBJECT_TAGGING, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION_SIZE, SUFFIX_CRC, SUFFIX_DATA_MOVED, SUFFIX_DATA_MOVED_TAGS, + SUFFIX_DATA_MOVEMENT_UPLOAD, SUFFIX_PART_CHECKSUMS, SUFFIX_TRANSITION_STATUS, SUFFIX_TRANSITION_TIER, + SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, + strip_internal_prefix_preserving_case, +}; use rustfs_utils::path::encode_dir_object; use std::collections::{BTreeMap, HashMap}; use std::pin::Pin; @@ -46,6 +51,9 @@ use tokio::io::{AsyncRead, BufReader, ReadBuf}; use tracing::{error, info}; type SharedDataMovementStream = Arc>>; +const LOG_COMPONENT_ECSTORE: &str = "ecstore"; +const LOG_SUBSYSTEM_DATA_MOVEMENT: &str = "data_movement"; +const EVENT_DATA_MOVEMENT_MULTIPART_ABORT_FAILED: &str = "data_movement_multipart_abort_failed"; const DATA_MOVEMENT_MULTIPART_ABORT_RETRY_ATTEMPTS: usize = 3; const DATA_MOVEMENT_MULTIPART_ABORT_RETRY_DELAY_SECS: u64 = 60; @@ -141,52 +149,6 @@ fn put_obj_reader_from_part_stream( Ok(PutObjReader::new(hash_reader)) } -fn data_movement_object_checksum_type(object_info: &ObjectInfo) -> Option { - let checksum = object_info.checksum.as_ref()?; - let (checksums, _) = rustfs_rio::read_checksums(checksum.as_ref(), 0); - rustfs_rio::BASE_CHECKSUM_TYPES - .iter() - .copied() - .find(|checksum_type| checksums.contains_key(checksum_type.to_string().as_str())) -} - -fn data_movement_multipart_checksum_type(object_info: &ObjectInfo) -> Option { - let checksum = object_info.user_defined.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM)?; - let checksum_type = object_info - .user_defined - .get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE) - .map(String::as_str) - .unwrap_or_default(); - let checksum_type = ChecksumType::from_string_with_obj_type(checksum, checksum_type); - checksum_type.is_set().then_some(checksum_type) -} - -fn add_data_movement_calculated_checksum(data: &mut PutObjReader, checksum_type: Option) -> Result<()> { - if let Some(checksum_type) = checksum_type { - data.stream.add_calculated_checksum(checksum_type).map_err(Error::from)?; - } - Ok(()) -} - -fn data_movement_part_checksum(part: &ObjectPartInfo, checksum_type: ChecksumType) -> Option { - part.checksums - .as_ref() - .and_then(|checksums| checksums.get(checksum_type.to_string().as_str())) - .cloned() -} - -fn data_movement_complete_part(part_num: usize, etag: Option, source_part: &ObjectPartInfo) -> CompletePart { - CompletePart { - part_num, - etag, - checksum_crc32: data_movement_part_checksum(source_part, ChecksumType::CRC32), - checksum_crc32c: data_movement_part_checksum(source_part, ChecksumType::CRC32C), - checksum_sha1: data_movement_part_checksum(source_part, ChecksumType::SHA1), - checksum_sha256: data_movement_part_checksum(source_part, ChecksumType::SHA256), - checksum_crc64nvme: data_movement_part_checksum(source_part, ChecksumType::CRC64_NVME), - } -} - pub fn new_multipart_abort_flag() -> Arc { Arc::new(AtomicBool::new(true)) } @@ -199,11 +161,35 @@ pub fn mark_multipart_upload_completed(flag: &Arc) { flag.store(false, Ordering::Relaxed); } +fn insert_data_movement_checksum(user_defined: &mut HashMap, object_info: &ObjectInfo) { + rustfs_utils::http::remove_header_map(user_defined, rustfs_utils::http::SUFFIX_REPLICATION_SSEC_CRC); + if let Some(checksum) = object_info.checksum.as_ref().filter(|checksum| !checksum.is_empty()) { + rustfs_utils::http::insert_header_map( + user_defined, + rustfs_utils::http::SUFFIX_REPLICATION_SSEC_CRC, + base64_simd::STANDARD.encode_to_string(checksum), + ); + } +} + +fn data_movement_upload_identity(object_info: &ObjectInfo) -> String { + let version_id = object_info + .version_id + .map_or_else(|| "none".to_string(), |version_id| version_id.to_string()); + let mod_time = object_info + .mod_time + .map_or_else(|| "none".to_string(), |mod_time| mod_time.unix_timestamp_nanos().to_string()); + format!("v1:{version_id}:{mod_time}") +} + fn data_movement_new_multipart_opts(object_info: &ObjectInfo, src_pool_idx: usize) -> ObjectOptions { + let mut user_defined = data_movement_user_defined(object_info); + let upload_identity = data_movement_upload_identity(object_info); + rustfs_utils::http::insert_str(&mut user_defined, SUFFIX_DATA_MOVEMENT_UPLOAD, upload_identity); ObjectOptions { versioned: object_info.version_id.is_some(), version_id: object_info.version_id.as_ref().map(|v| v.to_string()), - user_defined: data_movement_user_defined(object_info), + user_defined, preserve_etag: object_info.etag.clone(), src_pool_idx, data_movement: true, @@ -212,7 +198,67 @@ fn data_movement_new_multipart_opts(object_info: &ObjectInfo, src_pool_idx: usiz } fn data_movement_user_defined(object_info: &ObjectInfo) -> HashMap { - let mut user_defined = (*object_info.user_defined).clone(); + let mut user_defined = object_info + .user_defined + .iter() + .filter(|(key, _)| { + !is_data_movement_internal_metadata(key, SUFFIX_DATA_MOVEMENT_UPLOAD) + && !is_data_movement_internal_metadata(key, SUFFIX_PART_CHECKSUMS) + }) + .map(|(key, value)| (key.clone(), value.clone())) + .collect::>(); + let remove_canonical = |metadata: &mut HashMap, suffix: &str| { + metadata.remove(&rustfs_utils::http::internal_key_rustfs(suffix)); + metadata.remove(&format!("{}{suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX)); + }; + if object_info.checksum.as_ref().is_some_and(|checksum| !checksum.is_empty()) { + rustfs_utils::http::remove_str(&mut user_defined, SUFFIX_CRC); + } else { + remove_canonical(&mut user_defined, SUFFIX_CRC); + } + for (suffix, value) in [ + (SUFFIX_TRANSITION_STATUS, object_info.transitioned_object.status.as_str()), + (SUFFIX_TRANSITIONED_OBJECTNAME, object_info.transitioned_object.name.as_str()), + (SUFFIX_TRANSITION_TIER, object_info.transitioned_object.tier.as_str()), + ] { + if value.is_empty() { + remove_canonical(&mut user_defined, suffix); + continue; + } + rustfs_utils::http::remove_str(&mut user_defined, suffix); + rustfs_utils::http::insert_str(&mut user_defined, suffix, value.to_string()); + } + if object_info.transitioned_object.version_id.is_empty() { + let version_is_semantically_empty = user_defined + .iter() + .filter(|(key, _)| is_data_movement_internal_metadata(key, SUFFIX_TRANSITIONED_VERSION_ID)) + .all(|(_, value)| is_empty_data_movement_transition_version(value)); + if version_is_semantically_empty { + remove_canonical(&mut user_defined, SUFFIX_TRANSITIONED_VERSION_ID); + } + } else { + rustfs_utils::http::remove_str(&mut user_defined, SUFFIX_TRANSITIONED_VERSION_ID); + rustfs_utils::http::insert_str( + &mut user_defined, + SUFFIX_TRANSITIONED_VERSION_ID, + object_info.transitioned_object.version_id.clone(), + ); + } + let transition_version_state = (object_info.transition_version_state != rustfs_filemeta::TransitionVersionState::Unknown) + .then(|| object_info.transition_version_state.as_str()); + if let Some(transition_version_state) = transition_version_state { + rustfs_utils::http::remove_str(&mut user_defined, SUFFIX_TRANSITIONED_VERSION_STATE); + rustfs_utils::http::insert_str( + &mut user_defined, + SUFFIX_TRANSITIONED_VERSION_STATE, + transition_version_state.to_string(), + ); + } else { + remove_canonical(&mut user_defined, SUFFIX_TRANSITIONED_VERSION_STATE); + } + user_defined.remove(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM); + user_defined.remove(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE); + insert_data_movement_checksum(&mut user_defined, object_info); if !object_info.user_tags.is_empty() { user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), (*object_info.user_tags).clone()); } @@ -224,17 +270,102 @@ fn data_movement_user_defined(object_info: &ObjectInfo) -> HashMap ObjectOptions { - ObjectOptions { +fn data_movement_part_checksums(parts: &[ObjectPartInfo]) -> Result> { + let mut part_checksums = BTreeMap::>::new(); + for part in parts { + let Some(checksums) = part.checksums.as_ref().filter(|checksums| !checksums.is_empty()) else { + continue; + }; + let checksums = checksums.iter().map(|(key, value)| (key.clone(), value.clone())).collect(); + if part_checksums.insert(part.number, checksums).is_some() { + return Err(Error::other("data movement source has duplicate part numbers")); + } + } + if part_checksums.is_empty() { + return Ok(None); + } + let part_checksums = part_checksums + .into_iter() + .map(|(part_number, checksums)| (part_number, checksums.into_iter().collect::>())) + .collect::>(); + serde_json::to_string(&part_checksums) + .map(Some) + .map_err(|err| Error::other(format!("data movement part checksum metadata encode failed: {err}"))) +} + +pub(crate) fn prepare_tiered_data_movement_file_info(file_info: &mut rustfs_filemeta::FileInfo) -> Result<()> { + prepare_tiered_data_movement_file_info_for(file_info, data_movement_part_checksum_writer_enabled()) +} + +fn prepare_tiered_data_movement_file_info_for(file_info: &mut rustfs_filemeta::FileInfo, writer_enabled: bool) -> Result<()> { + if !writer_enabled { + rustfs_utils::http::remove_str(&mut file_info.metadata, SUFFIX_PART_CHECKSUMS); + for part in &mut file_info.parts { + part.checksums = None; + } + return Ok(()); + } + + SetDisks::hydrate_selected_fileinfo_part_checksums(file_info).map_err(|_| Error::FileCorrupt)?; + rustfs_utils::http::remove_str(&mut file_info.metadata, SUFFIX_PART_CHECKSUMS); + if let Some(encoded) = data_movement_part_checksums(&file_info.parts)? { + rustfs_utils::http::insert_str(&mut file_info.metadata, SUFFIX_PART_CHECKSUMS, encoded); + } + Ok(()) +} + +fn data_movement_part_checksum_writer_enabled_for(requested: bool, fleet_confirmed: bool) -> bool { + requested && fleet_confirmed +} + +fn data_movement_part_checksum_writer_enabled() -> bool { + data_movement_part_checksum_writer_enabled_for( + rustfs_utils::get_env_bool( + rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, + rustfs_config::DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, + ), + rustfs_utils::get_env_bool( + rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, + rustfs_config::DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, + ), + ) +} + +fn should_use_multipart_data_movement(object_info: &ObjectInfo, has_part_checksums: bool) -> bool { + object_info.is_multipart() + || has_part_checksums + || object_info.parts.len() > 1 + || object_info.parts.first().is_some_and(|part| part.number != 1) +} + +fn data_movement_complete_multipart_opts( + object_info: &ObjectInfo, + src_pool_idx: usize, + preserve_part_checksums: bool, +) -> Result { + let mut user_defined = HashMap::new(); + insert_data_movement_checksum(&mut user_defined, object_info); + let actual_size = object_info + .get_actual_size() + .map_err(|err| Error::other(format!("data movement source actual size is invalid: {err}")))?; + if actual_size < 0 { + return Err(Error::other("data movement source actual size is unknown")); + } + rustfs_utils::http::insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, actual_size.to_string()); + if preserve_part_checksums && let Some(encoded) = data_movement_part_checksums(&object_info.parts)? { + rustfs_utils::http::insert_str(&mut user_defined, SUFFIX_PART_CHECKSUMS, encoded); + } + Ok(ObjectOptions { versioned: object_info.version_id.is_some(), version_id: object_info.version_id.as_ref().map(|v| v.to_string()), - http_preconditions: data_movement_unversioned_target_precondition(object_info), + http_preconditions: Some(data_movement_target_precondition()), data_movement: true, mod_time: object_info.mod_time, preserve_etag: object_info.etag.clone(), + user_defined, src_pool_idx, ..Default::default() - } + }) } fn data_movement_put_object_opts(object_info: &ObjectInfo, src_pool_idx: usize) -> ObjectOptions { @@ -243,7 +374,7 @@ fn data_movement_put_object_opts(object_info: &ObjectInfo, src_pool_idx: usize) src_pool_idx, data_movement: true, version_id: object_info.version_id.as_ref().map(|v| v.to_string()), - http_preconditions: data_movement_unversioned_target_precondition(object_info), + http_preconditions: Some(data_movement_target_precondition()), mod_time: object_info.mod_time, user_defined: data_movement_user_defined(object_info), preserve_etag: object_info.etag.clone(), @@ -255,11 +386,58 @@ fn is_unversioned_data_movement_object(object_info: &ObjectInfo) -> bool { object_info.version_id.is_none_or(|version_id| version_id.is_nil()) } -fn data_movement_unversioned_target_precondition(object_info: &ObjectInfo) -> Option { - is_unversioned_data_movement_object(object_info).then(|| HTTPPreconditions { +pub(crate) fn data_movement_target_precondition() -> HTTPPreconditions { + HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() - }) + } +} + +fn is_owned_data_movement_target(target: &ObjectInfo) -> bool { + let rustfs_marker = rustfs_utils::http::internal_key_rustfs(SUFFIX_DATA_MOVED); + let minio_marker = format!("{}{SUFFIX_DATA_MOVED}", rustfs_utils::http::MINIO_INTERNAL_PREFIX); + if rustfs_utils::http::get_consistent_str(&target.user_defined, SUFFIX_DATA_MOVED) != Some("true") + || target.user_defined.get(&rustfs_marker).map(String::as_str) != Some("true") + || target.user_defined.get(&minio_marker).map(String::as_str) != Some("true") + { + return false; + } + + let tags_proof = format!("v1:{}", target.user_tags); + let rustfs_tags_proof = rustfs_utils::http::internal_key_rustfs(SUFFIX_DATA_MOVED_TAGS); + let minio_tags_proof = format!("{}{SUFFIX_DATA_MOVED_TAGS}", rustfs_utils::http::MINIO_INTERNAL_PREFIX); + rustfs_utils::http::get_consistent_str(&target.user_defined, SUFFIX_DATA_MOVED_TAGS) == Some(tags_proof.as_str()) + && target.user_defined.get(&rustfs_tags_proof).map(String::as_str) == Some(tags_proof.as_str()) + && target.user_defined.get(&minio_tags_proof).map(String::as_str) == Some(tags_proof.as_str()) +} + +pub(crate) fn can_replace_stale_data_movement_target(target: &ObjectInfo, opts: &ObjectOptions) -> bool { + let Some(preconditions) = opts.http_preconditions.as_ref() else { + return false; + }; + if !opts.data_movement + || preconditions.if_none_match_value() != Some("*") + || preconditions.if_match_value().is_some() + || target.delete_marker + { + return false; + } + + if !is_owned_data_movement_target(target) { + return false; + } + + let version_matches = match (opts.version_id.as_deref(), target.version_id) { + (None, None) => true, + (Some(expected), Some(actual)) => uuid::Uuid::parse_str(expected).ok() == Some(actual), + _ => false, + }; + + version_matches + && target + .mod_time + .zip(opts.mod_time) + .is_some_and(|(target_time, source_time)| target_time < source_time) } fn data_movement_put_object_reader( @@ -278,10 +456,7 @@ fn data_movement_put_object_reader( let reader = IndexedDataMovementReader::new(BufReader::new(rd.stream), index); let hrd = HashReader::from_stream(reader, object_info.size, actual_size, None, None, false) .map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", bucket, object_info.name.as_str(), err))?; - let mut data = PutObjReader::new(hrd); - add_data_movement_calculated_checksum(&mut data, data_movement_object_checksum_type(object_info)) - .map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", bucket, object_info.name.as_str(), err))?; - Ok(data) + Ok(PutObjReader::new(hrd)) } fn resolve_data_movement_abort_result( @@ -322,7 +497,15 @@ fn schedule_data_movement_multipart_abort_cleanup( }; match pool - .abort_multipart_upload(&bucket, &object, &upload_id, &ObjectOptions::default()) + .abort_multipart_upload( + &bucket, + &object, + &upload_id, + &ObjectOptions { + data_movement: true, + ..Default::default() + }, + ) .await { Ok(()) => { @@ -350,21 +533,33 @@ fn schedule_data_movement_multipart_abort_cleanup( } fn should_check_data_movement_overwrite_resume(err: &Error) -> bool { - is_err_data_movement_overwrite(err) || matches!(err, Error::PreconditionFailed) + is_err_data_movement_overwrite(err) || is_err_invalid_upload_id(err) || matches!(err, Error::PreconditionFailed) } fn effective_actual_size(info: &ObjectInfo) -> Option { info.get_actual_size().ok() } -fn is_equivalent_data_movement_part(source: &ObjectPartInfo, target: &ObjectPartInfo) -> bool { +fn effective_part_actual_size(part: &ObjectPartInfo) -> Option { + (part.actual_size != 0) + .then_some(part.actual_size) + .or_else(|| i64::try_from(part.size).ok()) +} + +fn is_equivalent_data_movement_part(source: &ObjectPartInfo, target: &ObjectPartInfo, compare_checksums: bool) -> bool { + // Multipart migration rewrites part timestamps. source.number == target.number && source.etag == target.etag && source.size == target.size - && source.actual_size == target.actual_size - && source.mod_time == target.mod_time - && source.index == target.index - && source.checksums == target.checksums + && matches!( + (effective_part_actual_size(source), effective_part_actual_size(target)), + (Some(source_size), Some(target_size)) if source_size == target_size + ) + // A missing target compression index selects the safe full-read fallback. + && (target.index.is_none() || source.index == target.index) + && (!compare_checksums + || source.checksums.as_ref().filter(|checksums| !checksums.is_empty()) + == target.checksums.as_ref().filter(|checksums| !checksums.is_empty())) } fn data_movement_parts_by_number(parts: &[ObjectPartInfo]) -> Option> { @@ -378,7 +573,11 @@ fn data_movement_parts_by_number(parts: &[ObjectPartInfo]) -> Option bool { +pub(crate) fn are_equivalent_data_movement_parts(source: &[ObjectPartInfo], target: &[ObjectPartInfo]) -> bool { + are_equivalent_data_movement_parts_for(source, target, true) +} + +fn are_equivalent_data_movement_parts_for(source: &[ObjectPartInfo], target: &[ObjectPartInfo], compare_checksums: bool) -> bool { if source.len() != target.len() { return false; } @@ -393,27 +592,214 @@ fn are_equivalent_data_movement_parts(source: &[ObjectPartInfo], target: &[Objec source_parts.iter().all(|(number, source_part)| { target_parts .get(number) - .is_some_and(|target_part| is_equivalent_data_movement_part(source_part, target_part)) + .is_some_and(|target_part| is_equivalent_data_movement_part(source_part, target_part, compare_checksums)) }) } -fn is_equivalent_data_movement_object(source: &ObjectInfo, target: &ObjectInfo) -> bool { +fn is_data_movement_internal_metadata(key: &str, suffix: &str) -> bool { + strip_internal_prefix_preserving_case(key).is_some_and(|candidate| candidate.eq_ignore_ascii_case(suffix)) +} + +fn is_canonical_data_movement_internal_metadata(key: &str, suffix: &str) -> bool { + key.strip_prefix(rustfs_utils::http::RUSTFS_INTERNAL_PREFIX) == Some(suffix) + || key.strip_prefix(rustfs_utils::http::MINIO_INTERNAL_PREFIX) == Some(suffix) +} + +fn data_movement_layout_marker_presence(object_info: &ObjectInfo) -> Option { + if !rustfs_utils::http::contains_key_str(&object_info.user_defined, crate::object_api::ENCRYPTED_PART_LAYOUT_QUORUM_SUFFIX) { + return Some(false); + } + let data_dir = object_info.data_dir.filter(|data_dir| !data_dir.is_nil())?; + let mut expected_buf = [0_u8; 36]; + let expected = data_dir.hyphenated().encode_lower(&mut expected_buf); + crate::object_api::has_encrypted_part_layout_marker( + &object_info.user_defined, + crate::object_api::ENCRYPTED_PART_LAYOUT_QUORUM_SUFFIX, + expected, + ) + .then_some(true) +} + +fn data_movement_size_marker_presence(object_info: &ObjectInfo, suffix: &str, expected: i64) -> Option { + let mut present = false; + for (key, value) in object_info.user_defined.iter() { + if !is_data_movement_internal_metadata(key, suffix) { + continue; + } + present = true; + if value.parse::().ok() != Some(expected) { + return None; + } + } + Some(present) +} + +fn data_movement_checksum_marker_presence(object_info: &ObjectInfo) -> Option { + let mut present = false; + for (key, value) in object_info.user_defined.iter() { + if !is_data_movement_internal_metadata(key, SUFFIX_CRC) { + continue; + } + let Some(checksum) = object_info.checksum.as_deref().filter(|checksum| !checksum.is_empty()) else { + if value.is_empty() { + continue; + } + return None; + }; + present = true; + match std::str::from_utf8(checksum) { + Ok(checksum) if value != checksum => return None, + Err(_) if !value.is_empty() => return None, + _ => {} + } + } + Some(present) +} + +fn is_compatible_data_movement_marker_presence(source: Option, target: Option) -> bool { + matches!((source, target), (Some(false), Some(_)) | (Some(true), Some(true))) +} + +fn is_empty_data_movement_transition_version(value: &str) -> bool { + value.is_empty() + || uuid::Uuid::from_slice(value.as_bytes()).is_ok_and(|version_id| version_id.is_nil()) + || uuid::Uuid::parse_str(value).is_ok_and(|version_id| version_id.is_nil()) +} + +fn is_data_movement_rewritten_transition_metadata(object_info: &ObjectInfo, key: &str) -> bool { + let state = (object_info.transition_version_state != rustfs_filemeta::TransitionVersionState::Unknown) + .then(|| object_info.transition_version_state.as_str()); + [ + (SUFFIX_TRANSITION_STATUS, Some(object_info.transitioned_object.status.as_str())), + (SUFFIX_TRANSITIONED_OBJECTNAME, Some(object_info.transitioned_object.name.as_str())), + (SUFFIX_TRANSITIONED_VERSION_ID, Some(object_info.transitioned_object.version_id.as_str())), + (SUFFIX_TRANSITIONED_VERSION_STATE, state), + (SUFFIX_TRANSITION_TIER, Some(object_info.transitioned_object.tier.as_str())), + ] + .iter() + .any(|(suffix, expected)| { + let canonical = is_canonical_data_movement_internal_metadata(key, suffix); + let preserves_unusable_version = *suffix == SUFFIX_TRANSITIONED_VERSION_ID + && expected == &Some("") + && object_info + .user_defined + .get(key) + .is_some_and(|value| !is_empty_data_movement_transition_version(value)); + canonical && !preserves_unusable_version + || expected.is_some_and(|expected| { + !expected.is_empty() + && is_data_movement_internal_metadata(key, suffix) + && rustfs_utils::http::get_consistent_str(&object_info.user_defined, suffix) == Some(expected) + }) + }) +} + +fn is_data_movement_rewritten_metadata(object_info: &ObjectInfo, key: &str, normalize_compression_size: bool) -> bool { + [ + SUFFIX_DATA_MOVED, + SUFFIX_DATA_MOVED_TAGS, + SUFFIX_DATA_MOVEMENT_UPLOAD, + SUFFIX_ACTUAL_SIZE, + SUFFIX_CRC, + SUFFIX_PART_CHECKSUMS, + crate::object_api::ENCRYPTED_PART_LAYOUT_QUORUM_SUFFIX, + ] + .iter() + .any(|suffix| is_data_movement_internal_metadata(key, suffix)) + || is_data_movement_rewritten_transition_metadata(object_info, key) + || key == rustfs_rio::RUSTFS_MULTIPART_CHECKSUM + || key == rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE + || normalize_compression_size && is_data_movement_internal_metadata(key, SUFFIX_COMPRESSION_SIZE) +} + +pub(crate) fn is_equivalent_data_movement_metadata( + source: &ObjectInfo, + target: &ObjectInfo, + source_actual_size: i64, + target_actual_size: i64, +) -> bool { + if !is_compatible_data_movement_marker_presence( + data_movement_size_marker_presence(source, SUFFIX_ACTUAL_SIZE, source_actual_size), + data_movement_size_marker_presence(target, SUFFIX_ACTUAL_SIZE, target_actual_size), + ) || !is_compatible_data_movement_marker_presence( + data_movement_checksum_marker_presence(source), + data_movement_checksum_marker_presence(target), + ) { + return false; + } + + let normalize_compression_size = source.is_compressed() && target.is_compressed(); + if normalize_compression_size + && !is_compatible_data_movement_marker_presence( + data_movement_size_marker_presence(source, SUFFIX_COMPRESSION_SIZE, source.size), + data_movement_size_marker_presence(target, SUFFIX_COMPRESSION_SIZE, target.size), + ) + { + return false; + } + let Some(source_layout) = data_movement_layout_marker_presence(source) else { + return false; + }; + let Some(target_layout) = data_movement_layout_marker_presence(target) else { + return false; + }; + if (source_layout || target_layout) + && !(source.data_dir.is_some_and(|data_dir| !data_dir.is_nil()) + && target.data_dir.is_some_and(|data_dir| !data_dir.is_nil())) + { + return false; + } + + source + .user_defined + .iter() + .filter(|(key, _)| !is_data_movement_rewritten_metadata(source, key, normalize_compression_size)) + .all(|(key, value)| target.user_defined.get(key) == Some(value)) + && target + .user_defined + .iter() + .filter(|(key, _)| !is_data_movement_rewritten_metadata(target, key, normalize_compression_size)) + .all(|(key, value)| source.user_defined.get(key) == Some(value)) +} + +fn is_equivalent_data_movement_object_identity( + source: &ObjectInfo, + target: &ObjectInfo, + compare_mod_time: bool, + compare_part_checksums: bool, +) -> bool { + let (Some(source_actual_size), Some(target_actual_size)) = (effective_actual_size(source), effective_actual_size(target)) + else { + return false; + }; + source.version_id == target.version_id && source.delete_marker == target.delete_marker && source.size == target.size - && effective_actual_size(source) == effective_actual_size(target) + && source_actual_size == target_actual_size && source.etag == target.etag && source.checksum == target.checksum - && source.mod_time == target.mod_time + && (!compare_mod_time || source.mod_time == target.mod_time) && source.storage_class == target.storage_class - && source.user_defined == target.user_defined + && is_equivalent_data_movement_metadata(source, target, source_actual_size, target_actual_size) && source.user_tags == target.user_tags && source.expires == target.expires && source.replication_status_internal == target.replication_status_internal && source.replication_status == target.replication_status && source.version_purge_status_internal == target.version_purge_status_internal && source.version_purge_status == target.version_purge_status - && are_equivalent_data_movement_parts(&source.parts, &target.parts) + && source.transitioned_object.name == target.transitioned_object.name + && source.transitioned_object.version_id == target.transitioned_object.version_id + && source.transitioned_object.tier == target.transitioned_object.tier + && source.transitioned_object.free_version == target.transitioned_object.free_version + && source.transitioned_object.status == target.transitioned_object.status + && source.transition_version_state == target.transition_version_state + && are_equivalent_data_movement_parts_for(&source.parts, &target.parts, compare_part_checksums) +} + +#[cfg(test)] +fn is_equivalent_data_movement_object(source: &ObjectInfo, target: &ObjectInfo) -> bool { + is_equivalent_data_movement_object_identity(source, target, true, true) } fn is_superseding_unversioned_data_movement_object(source: &ObjectInfo, target: &ObjectInfo) -> bool { @@ -426,6 +812,22 @@ fn is_superseding_unversioned_data_movement_object(source: &ObjectInfo, target: .is_some_and(|(source_time, target_time)| target_time > source_time) } +fn is_data_movement_upload_takeover_target(source: &ObjectInfo, target: &ObjectInfo, compare_part_checksums: bool) -> bool { + let identity = data_movement_upload_identity(source); + source.mod_time.is_some() + && rustfs_utils::http::get_consistent_str(&target.user_defined, SUFFIX_DATA_MOVEMENT_UPLOAD) == Some(identity.as_str()) + && is_equivalent_data_movement_object_identity(source, target, false, compare_part_checksums) +} + +fn is_legacy_data_movement_checksum_target(source: &ObjectInfo, target: &ObjectInfo) -> bool { + let has_checksums = |part: &ObjectPartInfo| part.checksums.as_ref().is_some_and(|checksums| !checksums.is_empty()); + source.parts.iter().any(has_checksums) + && target.parts.iter().all(|part| part.checksums.is_none()) + && !rustfs_utils::http::contains_key_str(&target.user_defined, SUFFIX_PART_CHECKSUMS) + && is_owned_data_movement_target(target) + && is_equivalent_data_movement_object_identity(source, target, true, false) +} + #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] struct SourceCleanupPartIdentity { number: usize, @@ -461,7 +863,7 @@ pub(crate) struct SourceCleanupVersionIdentity { transition_tier: String, transition_version_id: Option, transition_version: Option, - transition_version_state: u8, + transition_version_state: &'static str, expire_restored: bool, erasure: SourceCleanupErasureIdentity, metadata: BTreeMap, @@ -512,12 +914,7 @@ pub(crate) fn source_cleanup_version_identity(version: &FileInfo) -> SourceClean transition_tier: version.transition_tier.clone(), transition_version_id: version.transition_version_id, transition_version: version.transition_version.clone(), - transition_version_state: match version.transition_version_state { - rustfs_filemeta::TransitionVersionState::Unknown => 0, - rustfs_filemeta::TransitionVersionState::KnownDisabled => 1, - rustfs_filemeta::TransitionVersionState::SuspendedNull => 2, - rustfs_filemeta::TransitionVersionState::Exact => 3, - }, + transition_version_state: version.transition_version_state.as_str(), expire_restored: version.expire_restored, erasure: source_cleanup_erasure_identity(&version.erasure), metadata: version @@ -529,23 +926,25 @@ pub(crate) fn source_cleanup_version_identity(version: &FileInfo) -> SourceClean } } -fn source_cleanup_version_identities(fivs: &FileInfoVersions) -> Vec { - let mut identities: Vec<_> = fivs.versions.iter().map(source_cleanup_version_identity).collect(); - identities.sort(); - identities -} - fn source_cleanup_versions_match_with_allowed_missing( expected: &FileInfoVersions, current: &FileInfoVersions, allowed_missing: &[SourceCleanupVersionIdentity], ) -> bool { + let mut expected_free_versions: Vec<_> = expected.free_versions.iter().map(source_cleanup_version_identity).collect(); + let mut current_free_versions: Vec<_> = current.free_versions.iter().map(source_cleanup_version_identity).collect(); + expected_free_versions.sort(); + current_free_versions.sort(); + if expected_free_versions != current_free_versions { + return false; + } + let mut expected_counts = BTreeMap::new(); - for identity in source_cleanup_version_identities(expected) { + for identity in expected.versions.iter().map(source_cleanup_version_identity) { *expected_counts.entry(identity).or_insert(0usize) += 1; } - for identity in source_cleanup_version_identities(current) { + for identity in current.versions.iter().map(source_cleanup_version_identity) { let Some(count) = expected_counts.get_mut(&identity) else { return false; }; @@ -574,6 +973,12 @@ pub(crate) enum SourceCleanupError { Storage(#[from] Error), } +#[derive(Clone, Copy, Default)] +pub(crate) struct SourceCleanupBucketFence<'a> { + pub(crate) expected_incarnation_id: Option, + pub(crate) lifecycle_guard: Option<&'a rustfs_lock::NamespaceLockGuard>, +} + fn ensure_source_cleanup_versions_match( expected: &FileInfoVersions, current: &FileInfoVersions, @@ -586,21 +991,6 @@ fn ensure_source_cleanup_versions_match( } } -fn source_cleanup_preflight_error(op_label: &str, bucket: &str, object: &str, err: impl std::fmt::Display) -> Error { - Error::other(format!("{op_label}: source cleanup preflight failed for {bucket}/{object}: {err}")) -} - -async fn load_source_cleanup_versions( - set: Arc, - bucket: &str, - object: &str, - op_label: &str, -) -> Result> { - set.load_file_info_versions_exact(bucket, object) - .await - .map_err(|err| source_cleanup_preflight_error(op_label, bucket, object, err)) -} - pub(crate) async fn ensure_source_cleanup_versions_unchanged( set: Arc, bucket: &str, @@ -609,7 +999,11 @@ pub(crate) async fn ensure_source_cleanup_versions_unchanged( allowed_missing: &[SourceCleanupVersionIdentity], op_label: &str, ) -> std::result::Result<(), SourceCleanupError> { - let Some(current) = load_source_cleanup_versions(set, bucket, object, op_label).await? else { + let Some(current) = set + .load_file_info_versions_exact(bucket, object) + .await + .map_err(|err| Error::other(format!("{op_label}: source cleanup preflight failed for {bucket}/{object}: {err}")))? + else { return Ok(()); }; @@ -697,6 +1091,7 @@ pub(crate) async fn cleanup_source_entry_if_unchanged( object: &str, expected: &FileInfoVersions, allowed_missing: &[SourceCleanupVersionIdentity], + bucket_fence: SourceCleanupBucketFence<'_>, op_label: &str, ) -> std::result::Result { let cleanup_key = encode_dir_object(object); @@ -706,6 +1101,15 @@ pub(crate) async fn cleanup_source_entry_if_unchanged( .await .map_err(Error::from)?; + if bucket_fence + .lifecycle_guard + .is_some_and(rustfs_lock::NamespaceLockGuard::is_lock_lost) + { + return Err(SourceCleanupError::Storage(Error::other(format!( + "{op_label}: bucket incarnation fence was lost before source cleanup" + )))); + } + ensure_source_cleanup_versions_unchanged(set.clone(), bucket, object, expected, allowed_missing, op_label).await?; #[cfg(test)] @@ -716,9 +1120,13 @@ pub(crate) async fn cleanup_source_entry_if_unchanged( delete_prefix_object: true, data_movement: true, no_lock: true, + expected_bucket_incarnation_id: bucket_fence.expected_incarnation_id, ..Default::default() }; opts.add_namespace_lock_guard(&_guard); + if let Some(bucket_lifecycle_guard) = bucket_fence.lifecycle_guard { + opts.add_bucket_lifecycle_lock_guard(bucket_lifecycle_guard); + } let result = set.delete_object(bucket, cleanup_key.as_str(), opts).await; if result.is_ok() { crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1); @@ -740,6 +1148,7 @@ async fn find_data_movement_target_info( versioned: object_info.version_id.is_some(), version_id: object_info.version_id.as_ref().map(|v| v.to_string()), no_lock: true, + include_part_checksums: true, ..Default::default() }; let object = encode_dir_object(object_info.name.as_str()); @@ -763,6 +1172,24 @@ fn resolve_data_movement_overwrite_resume_result( source: &ObjectInfo, src_pool_idx: usize, target_pool_idx: usize, +) -> Result { + resolve_data_movement_overwrite_resume_result_for( + err, + target_result, + source, + src_pool_idx, + target_pool_idx, + data_movement_part_checksum_writer_enabled(), + ) +} + +fn resolve_data_movement_overwrite_resume_result_for( + err: &Error, + target_result: Result>, + source: &ObjectInfo, + src_pool_idx: usize, + target_pool_idx: usize, + compare_part_checksums: bool, ) -> Result { if !should_check_data_movement_overwrite_resume(err) || !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx) @@ -774,7 +1201,15 @@ fn resolve_data_movement_overwrite_resume_result( return Ok(false); }; - if is_equivalent_data_movement_object(source, &target) { + if is_equivalent_data_movement_object_identity(source, &target, true, compare_part_checksums) { + return Ok(true); + } + + if compare_part_checksums && is_legacy_data_movement_checksum_target(source, &target) { + return Ok(true); + } + + if is_data_movement_upload_takeover_target(source, &target, compare_part_checksums) { return Ok(true); } @@ -788,46 +1223,22 @@ async fn should_treat_data_movement_overwrite_as_complete( bucket: &str, object_info: &ObjectInfo, err: &Error, + compare_part_checksums: bool, ) -> Result { if !should_check_data_movement_overwrite_resume(err) { return Ok(false); } - resolve_data_movement_overwrite_resume_result( + resolve_data_movement_overwrite_resume_result_for( err, find_data_movement_target_info(store, target_pool_idx, bucket, object_info).await, object_info, src_pool_idx, target_pool_idx, + compare_part_checksums, ) } -async fn should_treat_data_movement_overwrite_as_complete_in_any_target_pool( - store: &ECStore, - src_pool_idx: usize, - bucket: &str, - object_info: &ObjectInfo, - err: &Error, -) -> Result { - if !should_check_data_movement_overwrite_resume(err) { - return Ok(false); - } - - for target_pool_idx in 0..store.pools.len() { - if target_pool_idx == src_pool_idx { - continue; - } - - if should_treat_data_movement_overwrite_as_complete(store, src_pool_idx, target_pool_idx, bucket, object_info, err) - .await? - { - return Ok(true); - } - } - - Ok(false) -} - fn data_movement_part_stage_error( op_label: &str, stage: &str, @@ -864,17 +1275,22 @@ pub(crate) async fn migrate_object( pool_idx: usize, bucket: String, rd: GetObjectReader, + source_bucket_incarnation_id: Option, op_label: &str, ) -> Result<()> { let object_info = rd.object_info.clone(); + let has_part_checksums = object_info + .parts + .iter() + .any(|part| part.checksums.as_ref().is_some_and(|checksums| !checksums.is_empty())); - if object_info.is_multipart() { - let (res, target_pool_idx) = match store - .handle_new_multipart_upload_with_pool_idx( - &bucket, - &object_info.name, - &data_movement_new_multipart_opts(&object_info, pool_idx), - ) + let preserve_part_checksums = data_movement_part_checksum_writer_enabled(); + + if should_use_multipart_data_movement(&object_info, has_part_checksums) { + let mut new_multipart_opts = data_movement_new_multipart_opts(&object_info, pool_idx); + new_multipart_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id; + let (res, target_pool_idx, expected_bucket_incarnation_id) = match store + .handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts) .await { Ok(res) => res, @@ -889,13 +1305,10 @@ pub(crate) async fn migrate_object( )); } }; - let abort_multipart_flag = new_multipart_abort_flag(); let multipart_result: Result<()> = async { let mut parts = vec![CompletePart::default(); object_info.parts.len()]; let reader = Arc::new(Mutex::new(rd.stream)); - let multipart_checksum_type = data_movement_multipart_checksum_type(&object_info); - for (i, part) in object_info.parts.iter().enumerate() { let part_size = i64::try_from(part.size).map_err(|_| { data_movement_part_stage_error( @@ -907,7 +1320,7 @@ pub(crate) async fn migrate_object( Error::other("part size overflow"), ) })?; - let part_actual_size = if part.actual_size > 0 { part.actual_size } else { part_size }; + let part_actual_size = if part.actual_size == 0 { part_size } else { part.actual_size }; let index = decode_part_index(part.index.as_ref()); let mut data = put_obj_reader_from_part_stream(reader.clone(), part_size, part_actual_size, index).map_err(|err| { @@ -920,28 +1333,22 @@ pub(crate) async fn migrate_object( err, ) })?; - add_data_movement_calculated_checksum(&mut data, multipart_checksum_type).map_err(|err| { - data_movement_part_stage_error( - op_label, - "prepare_part", - bucket.as_str(), - object_info.name.as_str(), - part.number, - err, - ) - })?; - + let part_opts = ObjectOptions { + part_number: Some(part.number), + preserve_etag: Some(part.etag.clone()), + data_movement: true, + src_pool_idx: pool_idx, + expected_bucket_incarnation_id, + ..Default::default() + }; let pi = match store - .put_object_part( + .put_object_part_for_data_movement( + target_pool_idx, &bucket, &object_info.name, &res.upload_id, - part.number, &mut data, - &ObjectOptions { - preserve_etag: Some(part.etag.clone()), - ..Default::default() - }, + &part_opts, ) .await { @@ -960,17 +1367,33 @@ pub(crate) async fn migrate_object( } }; - parts[i] = data_movement_complete_part(pi.part_num, pi.etag, part); + parts[i] = CompletePart { + part_num: pi.part_num, + etag: pi.etag, + ..Default::default() + }; } + let mut complete_multipart_opts = + data_movement_complete_multipart_opts(&object_info, pool_idx, preserve_part_checksums).map_err(|err| { + data_movement_stage_error( + op_label, + "prepare_complete_multipart", + bucket.as_str(), + object_info.name.as_str(), + err, + ) + })?; + complete_multipart_opts.expected_bucket_incarnation_id = expected_bucket_incarnation_id; if let Err(err) = store .clone() - .complete_multipart_upload( + .complete_multipart_upload_for_data_movement( + target_pool_idx, &bucket, &object_info.name, &res.upload_id, parts, - &data_movement_complete_multipart_opts(&object_info, pool_idx), + &complete_multipart_opts, ) .await { @@ -981,6 +1404,7 @@ pub(crate) async fn migrate_object( bucket.as_str(), &object_info, &err, + preserve_part_checksums, ) .await? { @@ -1008,35 +1432,88 @@ pub(crate) async fn migrate_object( .await; if multipart_result.is_ok() && should_abort_multipart_upload(&abort_multipart_flag) { - let abort_result = match store.pools.get(target_pool_idx) { - Some(pool) => { - pool.abort_multipart_upload(&bucket, &object_info.name, &res.upload_id, &ObjectOptions::default()) - .await - } - None => Err(Error::other(format!( - "{op_label}: target pool {target_pool_idx} is out of range while aborting superseded multipart upload" - ))), - }; - if let Err(abort_err) = abort_result - && !is_err_invalid_upload_id(&abort_err) - { - error!("{op_label}: abort superseded multipart upload err {:?}", &abort_err); - schedule_data_movement_multipart_abort_cleanup( - store.clone(), + let abort_result = store + .abort_multipart_upload_for_data_movement( target_pool_idx, - bucket.clone(), - object_info.name.clone(), - res.upload_id.clone(), - op_label, - ); + &bucket, + &object_info.name, + &res.upload_id, + &ObjectOptions { + data_movement: true, + src_pool_idx: pool_idx, + expected_bucket_incarnation_id, + ..Default::default() + }, + ) + .await; + match abort_result { + Ok(()) => return Ok(()), + Err(abort_err) if is_err_invalid_upload_id(&abort_err) => { + if should_treat_data_movement_overwrite_as_complete( + store.as_ref(), + pool_idx, + target_pool_idx, + bucket.as_str(), + &object_info, + &abort_err, + preserve_part_checksums, + ) + .await? + { + return Ok(()); + } + return Err(data_movement_stage_error( + op_label, + "verify_superseded_multipart", + bucket.as_str(), + object_info.name.as_str(), + abort_err, + )); + } + Err(abort_err) => { + error!( + event = EVENT_DATA_MOVEMENT_MULTIPART_ABORT_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DATA_MOVEMENT, + result = "error", + operation = op_label, + error = ?abort_err, + "data movement multipart abort failed" + ); + schedule_data_movement_multipart_abort_cleanup( + store.clone(), + target_pool_idx, + bucket.clone(), + object_info.name.clone(), + res.upload_id.clone(), + op_label, + ); + return Err(data_movement_stage_error( + op_label, + "abort_superseded_multipart", + bucket.as_str(), + object_info.name.as_str(), + abort_err, + )); + } } - return Ok(()); } if let Err(primary_err) = multipart_result { if should_abort_multipart_upload(&abort_multipart_flag) { return match store - .abort_multipart_upload(&bucket, &object_info.name, &res.upload_id, &ObjectOptions::default()) + .abort_multipart_upload_for_data_movement( + target_pool_idx, + &bucket, + &object_info.name, + &res.upload_id, + &ObjectOptions { + data_movement: true, + src_pool_idx: pool_idx, + expected_bucket_incarnation_id, + ..Default::default() + }, + ) .await { Ok(()) => Err(primary_err), @@ -1069,21 +1546,21 @@ pub(crate) async fn migrate_object( let mut data = data_movement_put_object_reader(bucket.as_str(), &object_info, rd, op_label)?; - if let Err(err) = store - .put_object( - &bucket, - &object_info.name, - &mut data, - &data_movement_put_object_opts(&object_info, pool_idx), - ) + let mut put_opts = data_movement_put_object_opts(&object_info, pool_idx); + put_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id; + let (target_pool_idx, put_result) = store + .put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts) .await - { - if should_treat_data_movement_overwrite_as_complete_in_any_target_pool( + .map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", &bucket, &object_info.name, err))?; + if let Err(err) = put_result { + if should_treat_data_movement_overwrite_as_complete( store.as_ref(), pool_idx, + target_pool_idx, bucket.as_str(), &object_info, &err, + preserve_part_checksums, ) .await? { @@ -1112,7 +1589,7 @@ pub(crate) async fn migrate_object( mod tests { use super::*; use crate::bucket::replication::{ReplicationStatusType, VersionPurgeStatusType}; - use rustfs_rio::HashReaderMut; + use rustfs_rio::{Checksum, ChecksumType}; use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE}; use std::collections::HashMap; use std::io::Cursor; @@ -1245,6 +1722,22 @@ mod tests { assert!(matches!(err, SourceCleanupError::SourceChanged)); } + #[test] + fn test_source_cleanup_preflight_rejects_changed_or_missing_free_version() { + let mut expected = cleanup_test_versions(vec![cleanup_test_file_info("object.txt", Uuid::from_u128(1), "source")]); + expected.free_versions = vec![cleanup_test_file_info("object.txt", Uuid::from_u128(2), "tier-cleanup")]; + + let mut changed = expected.clone(); + changed.free_versions[0] + .metadata + .insert("x-amz-meta-key".to_string(), "changed".to_string()); + assert!(!source_cleanup_versions_match_with_allowed_missing(&expected, &changed, &[])); + + let mut missing = expected.clone(); + missing.free_versions.clear(); + assert!(!source_cleanup_versions_match_with_allowed_missing(&expected, &missing, &[])); + } + #[test] fn test_source_cleanup_preflight_rejects_changed_transition_or_erasure() { let expected = cleanup_test_versions(vec![cleanup_test_file_info("object.txt", Uuid::from_u128(1), "source")]); @@ -1422,30 +1915,24 @@ mod tests { assert_eq!(decoded.total_compressed, 2_097_152); } - #[tokio::test] - async fn test_data_movement_single_part_checksum_is_recalculated_from_source_type() { - let payload = b"checksum-payload"; - let checksum = - rustfs_rio::Checksum::new_from_data(ChecksumType::CRC32C, payload).expect("source checksum should be created"); + #[test] + fn test_data_movement_checksum_is_preserved_opaque() { + let checksum = Bytes::from_static(b"sealed-or-plaintext-checksum"); let object_info = ObjectInfo { - checksum: Some(checksum.to_bytes(&[])), + checksum: Some(checksum.clone()), ..Default::default() }; - let mut data = PutObjReader::from_vec(payload.to_vec()); + let opts = data_movement_put_object_opts(&object_info, 0); + let encoded = rustfs_utils::http::get_header_map(&opts.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_SSEC_CRC) + .expect("data movement must carry the persisted checksum out of band"); - add_data_movement_calculated_checksum(&mut data, data_movement_object_checksum_type(&object_info)) - .expect("source checksum type should be enabled on the migrated reader"); - data.stream - .read_to_end(&mut Vec::new()) - .await - .expect("reader should consume payload and calculate checksum"); - - let migrated = data - .stream - .content_hash() - .as_ref() - .expect("migrated reader should contain calculated checksum"); - assert_eq!(migrated.to_bytes(&[]), checksum.to_bytes(&[])); + assert_eq!( + base64_simd::STANDARD + .decode_to_vec(&encoded) + .expect("checksum marker should decode"), + checksum + ); + assert!(!rustfs_utils::http::contains_key_str(&opts.user_defined, SUFFIX_CRC)); } #[tokio::test] @@ -1477,22 +1964,29 @@ mod tests { } #[test] - fn test_data_movement_single_part_checksum_uses_raw_source_size() { - let object_info = ObjectInfo { + fn test_data_movement_empty_checksum_adds_no_passthrough_marker() { + let mut object_info = ObjectInfo { size: 32, actual_size: 128, etag: Some("etag-value".to_string()), checksum: Some(Bytes::new()), ..Default::default() }; + rustfs_utils::http::insert_header_map( + Arc::make_mut(&mut object_info.user_defined), + rustfs_utils::http::SUFFIX_REPLICATION_SSEC_CRC, + "stale-checksum", + ); - assert_eq!(object_info.size, 32); - assert_eq!(object_info.get_actual_size().expect("actual size should resolve"), 128); - assert_eq!(data_movement_object_checksum_type(&object_info), None); + let opts = data_movement_put_object_opts(&object_info, 0); + + assert!( + rustfs_utils::http::get_header_map(&opts.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_SSEC_CRC,).is_none() + ); } #[test] - fn test_data_movement_multipart_checksum_type_uses_source_metadata() { + fn test_data_movement_multipart_opts_strip_upload_checksum_contract() { let object_info = ObjectInfo { user_defined: Arc::new(HashMap::from([ (rustfs_rio::RUSTFS_MULTIPART_CHECKSUM.to_string(), ChecksumType::CRC64_NVME.to_string()), @@ -1504,33 +1998,146 @@ mod tests { ..Default::default() }; - assert_eq!(data_movement_multipart_checksum_type(&object_info), Some(ChecksumType::CRC64_NVME)); + let opts = data_movement_new_multipart_opts(&object_info, 0); + + assert!(!opts.user_defined.contains_key(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM)); + assert!(!opts.user_defined.contains_key(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE)); } #[test] - fn test_data_movement_complete_part_preserves_source_part_checksums() { - let source_part = ObjectPartInfo { - number: 2, - etag: "etag-2".to_string(), - checksums: Some(HashMap::from([ - (ChecksumType::CRC32.to_string(), "crc32-value".to_string()), - (ChecksumType::CRC32C.to_string(), "crc32c-value".to_string()), - (ChecksumType::SHA1.to_string(), "sha1-value".to_string()), - (ChecksumType::SHA256.to_string(), "sha256-value".to_string()), - (ChecksumType::CRC64_NVME.to_string(), "crc64-value".to_string()), - ])), + fn test_data_movement_multipart_opts_defer_part_checksums_until_completion() { + let object_info = ObjectInfo { + parts: Arc::new(vec![ObjectPartInfo { + number: 2, + checksums: Some(HashMap::from([(ChecksumType::CRC32C.to_string(), "crc32c-value".to_string())])), + ..Default::default() + }]), ..Default::default() }; - let complete = data_movement_complete_part(2, Some("etag-2".to_string()), &source_part); + let new_opts = data_movement_new_multipart_opts(&object_info, 0); + let compatible_opts = + data_movement_complete_multipart_opts(&object_info, 0, false).expect("compatible opts should be created"); + let complete_opts = + data_movement_complete_multipart_opts(&object_info, 0, true).expect("complete opts should be created"); - assert_eq!(complete.part_num, 2); - assert_eq!(complete.etag.as_deref(), Some("etag-2")); - assert_eq!(complete.checksum_crc32.as_deref(), Some("crc32-value")); - assert_eq!(complete.checksum_crc32c.as_deref(), Some("crc32c-value")); - assert_eq!(complete.checksum_sha1.as_deref(), Some("sha1-value")); - assert_eq!(complete.checksum_sha256.as_deref(), Some("sha256-value")); - assert_eq!(complete.checksum_crc64nvme.as_deref(), Some("crc64-value")); + assert!(!rustfs_utils::http::contains_key_str(&new_opts.user_defined, SUFFIX_PART_CHECKSUMS)); + assert!(rustfs_utils::http::contains_key_str(&new_opts.user_defined, SUFFIX_DATA_MOVEMENT_UPLOAD)); + assert!(!rustfs_utils::http::contains_key_str( + &compatible_opts.user_defined, + SUFFIX_PART_CHECKSUMS + )); + assert_eq!( + rustfs_utils::http::get_consistent_str(&complete_opts.user_defined, SUFFIX_PART_CHECKSUMS), + Some(r#"[[2,[["CRC32C","crc32c-value"]]]]"#) + ); + } + + #[test] + fn test_data_movement_part_checksum_writer_requires_fleet_confirmation() { + let object_info = ObjectInfo { + parts: Arc::new(vec![ObjectPartInfo { + checksums: Some(HashMap::from([("CRC32C".to_string(), "AAAAAA==".to_string())])), + ..Default::default() + }]), + ..Default::default() + }; + + assert!(!data_movement_part_checksum_writer_enabled_for(false, false)); + assert!(!data_movement_part_checksum_writer_enabled_for(true, false)); + assert!(!data_movement_part_checksum_writer_enabled_for(false, true)); + assert!(data_movement_part_checksum_writer_enabled_for(true, true)); + let mut compatible = FileInfo { + parts: object_info.parts.as_ref().clone(), + ..Default::default() + }; + prepare_tiered_data_movement_file_info_for(&mut compatible, false) + .expect("disabled sidecar writer should preserve data movement compatibility"); + assert!(compatible.parts.iter().all(|part| part.checksums.is_none())); + assert!(!rustfs_utils::http::contains_key_str(&compatible.metadata, SUFFIX_PART_CHECKSUMS)); + + let empty = ObjectInfo { + parts: Arc::new(vec![ObjectPartInfo { + checksums: Some(HashMap::new()), + ..Default::default() + }]), + ..Default::default() + }; + assert_eq!( + data_movement_part_checksums(&empty.parts).expect("empty checksum maps should normalize"), + None + ); + assert!( + !empty + .parts + .iter() + .any(|part| part.checksums.as_ref().is_some_and(|checksums| !checksums.is_empty())) + ); + } + + #[test] + fn test_tiered_data_movement_prepares_and_validates_part_checksum_sidecar() { + let valid_checksums = HashMap::from([("CRC32C".to_string(), "AAAAAA==".to_string())]); + let mut valid = FileInfo { + parts: vec![ObjectPartInfo { + number: 1, + checksums: Some(valid_checksums), + ..Default::default() + }], + ..Default::default() + }; + let mut compatible = valid.clone(); + prepare_tiered_data_movement_file_info_for(&mut compatible, false) + .expect("disabled sidecar writer should omit optional part checksums"); + assert!(compatible.parts.iter().all(|part| part.checksums.is_none())); + assert!(!rustfs_utils::http::contains_key_str(&compatible.metadata, SUFFIX_PART_CHECKSUMS)); + prepare_tiered_data_movement_file_info_for(&mut valid, true).expect("valid legacy part checksums should be encoded"); + assert_eq!( + rustfs_utils::http::get_consistent_str(&valid.metadata, SUFFIX_PART_CHECKSUMS), + Some(r#"[[1,[["CRC32C","AAAAAA=="]]]]"#) + ); + + let mut invalid = FileInfo { + parts: vec![ObjectPartInfo { + number: 1, + checksums: Some(HashMap::from([("CRC32C".to_string(), "not-base64".to_string())])), + ..Default::default() + }], + ..Default::default() + }; + assert!(matches!( + prepare_tiered_data_movement_file_info_for(&mut invalid, true), + Err(Error::FileCorrupt) + )); + } + + #[test] + fn test_data_movement_preserves_multipart_topology_with_opaque_etag() { + let object_info = ObjectInfo { + etag: Some("0123456789abcdef0123456789abcdef".to_string()), + parts: Arc::new(vec![ + ObjectPartInfo { + number: 2, + ..Default::default() + }, + ObjectPartInfo { + number: 7, + ..Default::default() + }, + ]), + ..Default::default() + }; + assert!(!object_info.is_multipart()); + assert!(should_use_multipart_data_movement(&object_info, false)); + + let single_nonstandard_part = ObjectInfo { + parts: Arc::new(vec![ObjectPartInfo { + number: 7, + ..Default::default() + }]), + ..object_info + }; + assert!(should_use_multipart_data_movement(&single_nonstandard_part, false)); } #[test] @@ -1744,6 +2351,65 @@ mod tests { assert_eq!(put_opts.user_defined.get("x-amz-meta-key").map(String::as_str), Some("value")); } + #[test] + fn test_data_movement_opts_canonicalize_derived_binary_metadata() { + let transition_version = Uuid::from_u128(400).to_string(); + let checksum = Checksum::new_from_data(ChecksumType::CRC32C, b"checksum-payload") + .expect("checksum should be created") + .to_bytes(&[]); + let mut object_info = ObjectInfo { + checksum: Some(checksum), + transitioned_object: crate::storage_api_contracts::lifecycle::TransitionedObject { + name: "remote/object".to_string(), + version_id: transition_version.clone(), + tier: "WARM".to_string(), + status: rustfs_filemeta::TRANSITION_COMPLETE.to_string(), + ..Default::default() + }, + transition_version_state: rustfs_filemeta::TransitionVersionState::Exact, + ..Default::default() + }; + Arc::make_mut(&mut object_info.user_defined) + .insert(format!("{}{SUFFIX_CRC}", rustfs_utils::http::MINIO_INTERNAL_PREFIX), String::new()); + Arc::make_mut(&mut object_info.user_defined).insert( + format!("{}{SUFFIX_TRANSITIONED_VERSION_ID}", rustfs_utils::http::MINIO_INTERNAL_PREFIX), + String::new(), + ); + + let metadata = data_movement_new_multipart_opts(&object_info, 1).user_defined; + + assert!(!rustfs_utils::http::contains_key_str(&metadata, SUFFIX_CRC)); + assert_eq!( + rustfs_utils::http::get_consistent_str(&metadata, SUFFIX_TRANSITIONED_VERSION_ID), + Some(transition_version.as_str()) + ); + assert_eq!( + rustfs_utils::http::get_consistent_str(&metadata, SUFFIX_TRANSITIONED_VERSION_STATE), + Some("exact") + ); + } + + #[test] + fn test_data_movement_opts_preserve_unusable_transition_versions() { + for value in ["opaque\0version".to_string(), "x".repeat(1_025)] { + let key = rustfs_utils::http::internal_key_rustfs(SUFFIX_TRANSITIONED_VERSION_ID); + let mut source = overwrite_equivalence_source(); + source.user_defined = Arc::new(HashMap::from([(key.clone(), value.clone())])); + + let opts = data_movement_new_multipart_opts(&source, 0); + assert_eq!(opts.user_defined.get(&key), Some(&value)); + + let mut target = source.clone(); + let different_value = if value.contains('\0') { + "different\0version".to_string() + } else { + "y".repeat(1_025) + }; + Arc::make_mut(&mut target.user_defined).insert(key, different_value); + assert!(!overwrite_resume_for_target(&source, target)); + } + } + #[test] fn test_data_movement_new_multipart_opts_preserves_etag_and_version() { let version_id = Uuid::nil(); @@ -1775,7 +2441,7 @@ mod tests { ..Default::default() }; - let opts = data_movement_complete_multipart_opts(&object_info, 7); + let opts = data_movement_complete_multipart_opts(&object_info, 7, false).expect("complete opts should encode metadata"); assert!(opts.versioned); assert!(opts.data_movement); @@ -1783,7 +2449,12 @@ mod tests { assert_eq!(opts.version_id.as_deref(), Some(version_id.to_string().as_str())); assert_eq!(opts.preserve_etag.as_deref(), Some("etag-value")); assert_eq!(opts.src_pool_idx, 7); - assert!(opts.http_preconditions.is_none()); + assert_eq!( + opts.http_preconditions + .as_ref() + .and_then(HTTPPreconditions::if_none_match_value), + Some("*") + ); } #[test] @@ -1806,19 +2477,50 @@ mod tests { assert_eq!(opts.src_pool_idx, 9); assert!(opts.data_movement); assert_eq!(opts.mod_time, object_info.mod_time); - assert!(opts.http_preconditions.is_none()); + assert_eq!( + opts.http_preconditions + .as_ref() + .and_then(HTTPPreconditions::if_none_match_value), + Some("*") + ); } #[test] - fn test_data_movement_unversioned_put_and_complete_require_absent_target() { - for version_id in [None, Some(Uuid::nil())] { + fn test_data_movement_put_opts_do_not_persist_multipart_part_checksums() { + let object_info = ObjectInfo { + etag: Some("0123456789abcdef0123456789abcdef".to_string()), + parts: Arc::new(vec![ + ObjectPartInfo { + number: 1, + checksums: Some(HashMap::from([("CRC32C".to_string(), "AAAAAA==".to_string())])), + ..Default::default() + }, + ObjectPartInfo { + number: 2, + checksums: Some(HashMap::from([("CRC32C".to_string(), "BBBBBB==".to_string())])), + ..Default::default() + }, + ]), + ..Default::default() + }; + + assert!(!object_info.is_multipart()); + assert!(object_info.parts.iter().any(|part| part.checksums.is_some())); + let opts = data_movement_put_object_opts(&object_info, 0); + assert!(!rustfs_utils::http::contains_key_str(&opts.user_defined, SUFFIX_PART_CHECKSUMS)); + } + + #[test] + fn test_data_movement_put_and_complete_require_absent_target() { + for version_id in [None, Some(Uuid::nil()), Some(Uuid::from_u128(1))] { let object_info = ObjectInfo { version_id, ..Default::default() }; let put_opts = data_movement_put_object_opts(&object_info, 9); - let complete_opts = data_movement_complete_multipart_opts(&object_info, 9); + let complete_opts = + data_movement_complete_multipart_opts(&object_info, 9, false).expect("complete opts should encode metadata"); assert_eq!( put_opts @@ -1837,6 +2539,71 @@ mod tests { } } + #[test] + fn test_stale_data_movement_target_replacement_requires_exact_owned_generation() { + let version_id = Uuid::from_u128(41); + let source_time = OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2); + let opts = ObjectOptions { + data_movement: true, + versioned: true, + version_id: Some(version_id.to_string()), + mod_time: Some(source_time), + http_preconditions: Some(data_movement_target_precondition()), + ..Default::default() + }; + let mut metadata = HashMap::new(); + rustfs_utils::http::insert_str(&mut metadata, SUFFIX_DATA_MOVED, "true".to_string()); + rustfs_utils::http::insert_str(&mut metadata, SUFFIX_DATA_MOVED_TAGS, "v1:".to_string()); + let target = ObjectInfo { + version_id: Some(version_id), + mod_time: Some(source_time - time::Duration::SECOND), + user_defined: Arc::new(metadata), + ..Default::default() + }; + + assert!(can_replace_stale_data_movement_target(&target, &opts)); + + let mut client_target = target.clone(); + client_target.user_defined = Arc::new(HashMap::new()); + assert!(!can_replace_stale_data_movement_target(&client_target, &opts)); + + let mut single_marker = target.clone(); + Arc::make_mut(&mut single_marker.user_defined) + .remove(&format!("{}{SUFFIX_DATA_MOVED}", rustfs_utils::http::MINIO_INTERNAL_PREFIX)); + assert!(!can_replace_stale_data_movement_target(&single_marker, &opts)); + + let mut conflicting_marker = target.clone(); + Arc::make_mut(&mut conflicting_marker.user_defined).insert( + format!("{}{SUFFIX_DATA_MOVED}", rustfs_utils::http::MINIO_INTERNAL_PREFIX), + "false".to_string(), + ); + assert!(!can_replace_stale_data_movement_target(&conflicting_marker, &opts)); + + let mut retagged_target = target.clone(); + retagged_target.user_tags = Arc::new("acknowledged=true".to_string()); + assert!(!can_replace_stale_data_movement_target(&retagged_target, &opts)); + + let mut retagged_owned_target = retagged_target; + rustfs_utils::http::insert_str( + Arc::make_mut(&mut retagged_owned_target.user_defined), + SUFFIX_DATA_MOVED_TAGS, + "v1:acknowledged=true".to_string(), + ); + assert!(can_replace_stale_data_movement_target(&retagged_owned_target, &opts)); + + let mut different_version = target.clone(); + different_version.version_id = Some(Uuid::from_u128(42)); + assert!(!can_replace_stale_data_movement_target(&different_version, &opts)); + + let mut same_generation = target.clone(); + same_generation.mod_time = Some(source_time); + assert!(!can_replace_stale_data_movement_target(&same_generation, &opts)); + + let mut delete_marker = target; + delete_marker.delete_marker = true; + assert!(!can_replace_stale_data_movement_target(&delete_marker, &opts)); + } + #[test] fn test_is_equivalent_data_movement_object_accepts_matching_metadata() { let version_id = Uuid::nil(); @@ -1959,8 +2726,12 @@ mod tests { } fn overwrite_resume_for_target(source: &ObjectInfo, target: ObjectInfo) -> bool { + overwrite_resume_for_target_with_checksums(source, target, data_movement_part_checksum_writer_enabled()) + } + + fn overwrite_resume_for_target_with_checksums(source: &ObjectInfo, target: ObjectInfo, compare_part_checksums: bool) -> bool { let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string()); - resolve_data_movement_overwrite_resume_result(&err, Ok(Some(target)), source, 0, 1) + resolve_data_movement_overwrite_resume_result_for(&err, Ok(Some(target)), source, 0, 1, compare_part_checksums) .expect("overwrite target should be evaluated") } @@ -1971,6 +2742,348 @@ mod tests { assert!(overwrite_resume_for_target(&source, source.clone())); } + #[test] + fn test_data_movement_overwrite_resume_accepts_part_mod_time_drift() { + let source = overwrite_equivalence_source(); + let mut target = source.clone(); + let mut parts = target.parts.as_ref().clone(); + parts[0].mod_time = Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND); + target.parts = Arc::new(parts); + + assert!(overwrite_resume_for_target(&source, target)); + } + + #[test] + fn test_data_movement_overwrite_resume_accepts_legacy_part_actual_size_fallback() { + let mut source = overwrite_equivalence_source(); + let mut source_parts = source.parts.as_ref().clone(); + source_parts[0].actual_size = 0; + source.parts = Arc::new(source_parts); + let mut target = source.clone(); + let mut target_parts = target.parts.as_ref().clone(); + target_parts[0].actual_size = i64::try_from(target_parts[0].size).expect("part size should fit i64"); + target.parts = Arc::new(target_parts); + + assert!(overwrite_resume_for_target(&source, target)); + } + + #[test] + fn test_data_movement_overwrite_resume_preserves_negative_part_actual_size() { + let mut source = overwrite_equivalence_source(); + let mut source_parts = source.parts.as_ref().clone(); + source_parts[0].actual_size = -1; + source.parts = Arc::new(source_parts); + + let mut target = source.clone(); + let mut target_parts = target.parts.as_ref().clone(); + target_parts[0].actual_size = i64::try_from(target_parts[0].size).expect("part size should fit i64"); + target.parts = Arc::new(target_parts); + assert!(!overwrite_resume_for_target(&source, target)); + + assert!(overwrite_resume_for_target(&source, source.clone())); + } + + #[test] + fn test_data_movement_overwrite_resume_accepts_destination_marker() { + let source = overwrite_equivalence_source(); + let mut target = source.clone(); + let mut metadata = target.user_defined.as_ref().clone(); + rustfs_utils::http::insert_str(&mut metadata, SUFFIX_DATA_MOVED, "true".to_string()); + target.user_defined = Arc::new(metadata); + + assert!(overwrite_resume_for_target(&source, target)); + } + + #[test] + fn test_data_movement_overwrite_resume_accepts_compatible_internal_aliases() { + for suffix in [SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION_SIZE] { + let mut source = overwrite_equivalence_source(); + let mut source_metadata = source.user_defined.as_ref().clone(); + if suffix == SUFFIX_COMPRESSION_SIZE { + rustfs_utils::http::insert_str( + &mut source_metadata, + rustfs_utils::http::SUFFIX_COMPRESSION, + "klauspost/compress/s2".to_string(), + ); + } + source_metadata.insert(format!("X-Minio-Internal-{suffix}"), "000128".to_string()); + source.user_defined = Arc::new(source_metadata); + + let mut target = source.clone(); + let mut target_metadata = target.user_defined.as_ref().clone(); + rustfs_utils::http::insert_str(&mut target_metadata, suffix, "128".to_string()); + target.user_defined = Arc::new(target_metadata); + + assert!( + overwrite_resume_for_target(&source, target), + "compatible aliases for {suffix} should match" + ); + + let mut source_without_marker = source.clone(); + rustfs_utils::http::remove_str(Arc::make_mut(&mut source_without_marker.user_defined), suffix); + let mut target_with_generated_marker = source_without_marker.clone(); + rustfs_utils::http::insert_str( + Arc::make_mut(&mut target_with_generated_marker.user_defined), + suffix, + "128".to_string(), + ); + assert!( + overwrite_resume_for_target(&source_without_marker, target_with_generated_marker), + "a generated target marker for {suffix} should match" + ); + } + } + + #[test] + fn test_data_movement_overwrite_resume_rejects_conflicting_internal_aliases() { + for suffix in [SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION_SIZE] { + let mut source = overwrite_equivalence_source(); + if suffix == SUFFIX_COMPRESSION_SIZE { + rustfs_utils::http::insert_str( + Arc::make_mut(&mut source.user_defined), + rustfs_utils::http::SUFFIX_COMPRESSION, + "klauspost/compress/s2".to_string(), + ); + } + let mut target = source.clone(); + let mut metadata = target.user_defined.as_ref().clone(); + rustfs_utils::http::insert_str(&mut metadata, suffix, "128".to_string()); + metadata.insert(rustfs_utils::http::internal_key_rustfs(suffix), "64".to_string()); + target.user_defined = Arc::new(metadata); + + assert!( + !overwrite_resume_for_target(&source, target), + "conflicting aliases for {suffix} must fail closed" + ); + + rustfs_utils::http::insert_str(Arc::make_mut(&mut source.user_defined), suffix, "128".to_string()); + let mut target_without_marker = source.clone(); + rustfs_utils::http::remove_str(Arc::make_mut(&mut target_without_marker.user_defined), suffix); + assert!( + !overwrite_resume_for_target(&source, target_without_marker), + "a missing target marker for {suffix} must fail closed" + ); + } + } + + #[test] + fn test_data_movement_overwrite_resume_rejects_unreadable_actual_size() { + let mut source = overwrite_equivalence_source(); + source.actual_size = 0; + rustfs_utils::http::insert_str( + Arc::make_mut(&mut source.user_defined), + rustfs_utils::http::SUFFIX_COMPRESSION, + "klauspost/compress/s2".to_string(), + ); + rustfs_utils::http::insert_str( + Arc::make_mut(&mut source.user_defined), + SUFFIX_ACTUAL_SIZE, + "invalid-source-size".to_string(), + ); + let mut target = source.clone(); + rustfs_utils::http::insert_str( + Arc::make_mut(&mut target.user_defined), + SUFFIX_ACTUAL_SIZE, + "invalid-target-size".to_string(), + ); + + assert!(!overwrite_resume_for_target(&source, target)); + } + + #[test] + fn test_data_movement_overwrite_resume_rejects_missing_unreadable_actual_size() { + let mut source = overwrite_equivalence_source(); + source.actual_size = 0; + rustfs_utils::http::insert_str( + Arc::make_mut(&mut source.user_defined), + rustfs_utils::http::SUFFIX_COMPRESSION, + "klauspost/compress/s2".to_string(), + ); + let mut parts = source.parts.as_ref().clone(); + parts[0].actual_size = 0; + source.parts = Arc::new(parts); + + assert!(!overwrite_resume_for_target(&source, source.clone())); + } + + #[test] + fn test_data_movement_overwrite_resume_accepts_checksum_alias_expansion() { + let mut source = overwrite_equivalence_source(); + let checksum = rustfs_rio::Checksum::new_from_data(ChecksumType::CRC32C, b"checksum-payload") + .expect("checksum should be created") + .to_bytes(&[]); + assert!( + std::str::from_utf8(&checksum).is_err(), + "wire checksum should exercise non-UTF-8 metadata" + ); + source.checksum = Some(checksum); + Arc::make_mut(&mut source.user_defined) + .insert(format!("{}{SUFFIX_CRC}", rustfs_utils::http::MINIO_INTERNAL_PREFIX), String::new()); + + let mut target = source.clone(); + rustfs_utils::http::insert_str(Arc::make_mut(&mut target.user_defined), SUFFIX_CRC, String::new()); + + assert!(overwrite_resume_for_target(&source, target)); + } + + #[test] + fn test_data_movement_overwrite_resume_accepts_empty_checksum_normalization() { + let mut source = overwrite_equivalence_source(); + source.checksum = None; + source.user_defined = Arc::new(HashMap::from([( + format!("{}{SUFFIX_CRC}", rustfs_utils::http::MINIO_INTERNAL_PREFIX), + String::new(), + )])); + let mut target = source.clone(); + target.user_defined = Arc::new(HashMap::new()); + + assert!(is_equivalent_data_movement_object(&source, &target)); + } + + #[test] + fn test_data_movement_overwrite_resume_rejects_checksum_alias_conflict() { + let mut source = overwrite_equivalence_source(); + rustfs_utils::http::insert_str(Arc::make_mut(&mut source.user_defined), SUFFIX_CRC, "object-checksum".to_string()); + let mut target_without_marker = source.clone(); + rustfs_utils::http::remove_str(Arc::make_mut(&mut target_without_marker.user_defined), SUFFIX_CRC); + assert!(!overwrite_resume_for_target(&source, target_without_marker)); + + let mut target = source.clone(); + rustfs_utils::http::insert_str(Arc::make_mut(&mut target.user_defined), SUFFIX_CRC, "different".to_string()); + + assert!(!overwrite_resume_for_target(&source, target)); + } + + #[test] + fn test_data_movement_overwrite_resume_accepts_transition_alias_expansion() { + let mut source = overwrite_equivalence_source(); + source.transitioned_object.name = "remote/object".to_string(); + source.transitioned_object.version_id = Uuid::from_u128(300).to_string(); + source.transitioned_object.tier = "WARM".to_string(); + source.transitioned_object.status = rustfs_filemeta::TRANSITION_COMPLETE.to_string(); + source.transition_version_state = rustfs_filemeta::TransitionVersionState::Exact; + let markers = [ + (SUFFIX_TRANSITION_STATUS, source.transitioned_object.status.as_str()), + (SUFFIX_TRANSITIONED_OBJECTNAME, source.transitioned_object.name.as_str()), + (SUFFIX_TRANSITIONED_VERSION_ID, source.transitioned_object.version_id.as_str()), + (SUFFIX_TRANSITIONED_VERSION_STATE, "exact"), + (SUFFIX_TRANSITION_TIER, source.transitioned_object.tier.as_str()), + ]; + for (suffix, value) in markers { + Arc::make_mut(&mut source.user_defined) + .insert(format!("{}{suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX), value.to_string()); + } + + let mut target = source.clone(); + for (suffix, value) in markers { + rustfs_utils::http::insert_str(Arc::make_mut(&mut target.user_defined), suffix, value.to_string()); + } + assert!(overwrite_resume_for_target(&source, target.clone())); + + target.transitioned_object.tier = "OTHER".to_string(); + assert!(!overwrite_resume_for_target(&source, target)); + } + + #[test] + fn test_data_movement_overwrite_resume_accepts_empty_transition_marker_normalization() { + for value in [ + String::new(), + "\0".repeat(16), + Uuid::nil().to_string(), + Uuid::nil().simple().to_string(), + ] { + let mut source = overwrite_equivalence_source(); + Arc::make_mut(&mut source.user_defined).insert( + format!("{}{SUFFIX_TRANSITIONED_VERSION_ID}", rustfs_utils::http::MINIO_INTERNAL_PREFIX), + value, + ); + Arc::make_mut(&mut source.user_defined).insert( + format!("{}{SUFFIX_TRANSITIONED_VERSION_STATE}", rustfs_utils::http::MINIO_INTERNAL_PREFIX), + "unknown".to_string(), + ); + let mut target = source.clone(); + rustfs_utils::http::remove_str(Arc::make_mut(&mut target.user_defined), SUFFIX_TRANSITIONED_VERSION_ID); + rustfs_utils::http::remove_str(Arc::make_mut(&mut target.user_defined), SUFFIX_TRANSITIONED_VERSION_STATE); + + assert!(overwrite_resume_for_target(&source, target)); + } + } + + #[test] + fn test_data_movement_overwrite_resume_rejects_unparsed_transition_case_alias_mismatch() { + let mut source = overwrite_equivalence_source(); + Arc::make_mut(&mut source.user_defined).insert("X-Minio-Internal-transition-tier".to_string(), "source-tier".to_string()); + let mut target = source.clone(); + Arc::make_mut(&mut target.user_defined).insert("X-Minio-Internal-transition-tier".to_string(), "target-tier".to_string()); + + assert!(!overwrite_resume_for_target(&source, target)); + } + + #[test] + fn test_data_movement_overwrite_resume_accepts_rebuilt_transition_case_alias() { + let mut source = overwrite_equivalence_source(); + source.transitioned_object.tier = "WARM".to_string(); + rustfs_utils::http::insert_str(Arc::make_mut(&mut source.user_defined), SUFFIX_TRANSITION_TIER, "WARM".to_string()); + Arc::make_mut(&mut source.user_defined).insert("X-Minio-Internal-transition-tier".to_string(), "WARM".to_string()); + let mut target = source.clone(); + Arc::make_mut(&mut target.user_defined).remove("X-Minio-Internal-transition-tier"); + + assert!(overwrite_resume_for_target(&source, target)); + } + + #[test] + fn test_data_movement_overwrite_resume_accepts_target_local_layout_marker() { + let source_data_dir = Uuid::from_u128(100); + let mut source = overwrite_equivalence_source(); + source.data_dir = Some(source_data_dir); + let mut source_metadata = source.user_defined.as_ref().clone(); + rustfs_utils::http::insert_str( + &mut source_metadata, + crate::object_api::ENCRYPTED_PART_LAYOUT_QUORUM_SUFFIX, + source_data_dir.to_string(), + ); + source.user_defined = Arc::new(source_metadata); + + let target_data_dir = Uuid::from_u128(200); + let mut target = source.clone(); + target.data_dir = Some(target_data_dir); + let mut target_metadata = target.user_defined.as_ref().clone(); + rustfs_utils::http::insert_str( + &mut target_metadata, + crate::object_api::ENCRYPTED_PART_LAYOUT_QUORUM_SUFFIX, + target_data_dir.to_string(), + ); + target.user_defined = Arc::new(target_metadata); + assert!(overwrite_resume_for_target(&source, target.clone())); + + let mut target_without_marker = target.clone(); + rustfs_utils::http::remove_str( + Arc::make_mut(&mut target_without_marker.user_defined), + crate::object_api::ENCRYPTED_PART_LAYOUT_QUORUM_SUFFIX, + ); + assert!(overwrite_resume_for_target(&source, target_without_marker.clone())); + target_without_marker.data_dir = None; + assert!(!overwrite_resume_for_target(&source, target_without_marker)); + + let mut source_without_marker = source.clone(); + rustfs_utils::http::remove_str( + Arc::make_mut(&mut source_without_marker.user_defined), + crate::object_api::ENCRYPTED_PART_LAYOUT_QUORUM_SUFFIX, + ); + assert!(overwrite_resume_for_target(&source_without_marker, target.clone())); + source_without_marker.data_dir = Some(Uuid::nil()); + assert!(!overwrite_resume_for_target(&source_without_marker, target.clone())); + + let mut invalid_target_metadata = target.user_defined.as_ref().clone(); + rustfs_utils::http::insert_str( + &mut invalid_target_metadata, + crate::object_api::ENCRYPTED_PART_LAYOUT_QUORUM_SUFFIX, + source_data_dir.to_string(), + ); + target.user_defined = Arc::new(invalid_target_metadata); + assert!(!overwrite_resume_for_target(&source, target)); + } + #[test] fn test_data_movement_overwrite_resume_rejects_missing_part_checksum() { let source = overwrite_equivalence_source(); @@ -1979,9 +3092,47 @@ mod tests { parts[0].checksums = None; target.parts = Arc::new(parts); + assert!(!overwrite_resume_for_target_with_checksums(&source, target, true)); + } + + #[test] + fn test_data_movement_overwrite_resume_accepts_dropped_part_index() { + let source = overwrite_equivalence_source(); + let mut target = source.clone(); + let mut parts = target.parts.as_ref().clone(); + parts[0].index = None; + target.parts = Arc::new(parts); + + assert!(overwrite_resume_for_target(&source, target)); + } + + #[test] + fn test_data_movement_overwrite_resume_rejects_target_only_part_index() { + let mut source = overwrite_equivalence_source(); + let mut source_parts = source.parts.as_ref().clone(); + source_parts[0].index = None; + source.parts = Arc::new(source_parts); + let mut target = source.clone(); + let mut target_parts = target.parts.as_ref().clone(); + target_parts[0].index = Some(Bytes::from_static(&[9])); + target.parts = Arc::new(target_parts); + assert!(!overwrite_resume_for_target(&source, target)); } + #[test] + fn test_data_movement_overwrite_resume_accepts_generated_part_checksum_marker() { + let source = overwrite_equivalence_source(); + let mut target = source.clone(); + rustfs_utils::http::insert_str( + Arc::make_mut(&mut target.user_defined), + SUFFIX_PART_CHECKSUMS, + r#"[[1,[["CRC32C","part-checksum"]]]]"#.to_string(), + ); + + assert!(overwrite_resume_for_target(&source, target)); + } + fn overwrite_equivalence_source_with_two_parts() -> ObjectInfo { let source = overwrite_equivalence_source(); let mut parts = source.parts.as_ref().clone(); @@ -2083,6 +3234,158 @@ mod tests { assert!(should_resume); } + #[test] + fn test_overwrite_resume_accepts_owned_target_without_legacy_checksum_sidecar() { + let mut source = overwrite_equivalence_source(); + let mut source_parts = source.parts.as_ref().clone(); + source_parts.push(ObjectPartInfo { + number: 2, + etag: "second-part-etag".to_string(), + size: 64, + actual_size: 64, + checksums: Some(HashMap::from([(ChecksumType::CRC32C.to_string(), "second-part-checksum".to_string())])), + ..Default::default() + }); + source.parts = Arc::new(source_parts); + let mut target = source.clone(); + let mut target_parts = target.parts.as_ref().clone(); + for part in &mut target_parts { + part.checksums = None; + } + target.parts = Arc::new(target_parts); + let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string()); + + assert!( + resolve_data_movement_overwrite_resume_result_for(&err, Ok(Some(target.clone())), &source, 0, 1, false) + .expect("compatible migration should accept an omitted optional checksum sidecar") + ); + assert!( + !resolve_data_movement_overwrite_resume_result_for(&err, Ok(Some(target)), &source, 0, 1, true) + .expect("an unowned target must not bypass fleet-confirmed checksum comparison") + ); + + let mut owned_target = source.clone(); + let mut owned_target_parts = owned_target.parts.as_ref().clone(); + for part in &mut owned_target_parts { + part.checksums = None; + } + owned_target.parts = Arc::new(owned_target_parts); + rustfs_utils::http::insert_str(Arc::make_mut(&mut owned_target.user_defined), SUFFIX_DATA_MOVED, "true".to_string()); + rustfs_utils::http::insert_str( + Arc::make_mut(&mut owned_target.user_defined), + SUFFIX_DATA_MOVED_TAGS, + "v1:tag=value".to_string(), + ); + assert!( + resolve_data_movement_overwrite_resume_result_for(&err, Ok(Some(owned_target.clone())), &source, 0, 1, true,) + .expect("an owned pre-gate target should remain compatible after enabling checksum persistence") + ); + + let mut partial_target = owned_target.clone(); + let mut partial_target_parts = partial_target.parts.as_ref().clone(); + partial_target_parts[0].checksums.clone_from(&source.parts[0].checksums); + partial_target.parts = Arc::new(partial_target_parts); + assert!( + !resolve_data_movement_overwrite_resume_result_for(&err, Ok(Some(partial_target)), &source, 0, 1, true) + .expect("a partially missing checksum sidecar must fail closed") + ); + + let mut corrupt_target = owned_target.clone(); + rustfs_utils::http::insert_str(Arc::make_mut(&mut corrupt_target.user_defined), SUFFIX_PART_CHECKSUMS, String::new()); + assert!( + !resolve_data_movement_overwrite_resume_result_for(&err, Ok(Some(corrupt_target)), &source, 0, 1, true) + .expect("a present but empty checksum sidecar must fail closed") + ); + + let mut conflicting_target = owned_target; + let mut conflicting_target_parts = conflicting_target.parts.as_ref().clone(); + conflicting_target_parts[0].checksums = Some(HashMap::from([( + ChecksumType::CRC32C.to_string(), + "conflicting-part-checksum".to_string(), + )])); + conflicting_target.parts = Arc::new(conflicting_target_parts); + assert!( + !resolve_data_movement_overwrite_resume_result_for(&err, Ok(Some(conflicting_target)), &source, 0, 1, true) + .expect("a conflicting checksum sidecar must fail closed") + ); + } + + #[test] + fn test_invalid_upload_accepts_versioned_target_taken_over_by_old_node() { + let mut source = overwrite_equivalence_source(); + let mut source_parts = source.parts.as_ref().clone(); + source_parts[0].checksums = None; + source.parts = Arc::new(source_parts); + let mut target = ObjectInfo { + mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND), + ..source.clone() + }; + let err = Error::InvalidUploadID("bucket".to_string(), "object".to_string(), "upload-id".to_string()); + assert!( + !resolve_data_movement_overwrite_resume_result(&err, Ok(Some(target.clone())), &source, 0, 1) + .expect("an unrelated invalid upload must not accept a different target") + ); + let upload_identity = data_movement_upload_identity(&source); + rustfs_utils::http::insert_str(Arc::make_mut(&mut target.user_defined), SUFFIX_DATA_MOVEMENT_UPLOAD, upload_identity); + let should_resume = resolve_data_movement_overwrite_resume_result(&err, Ok(Some(target.clone())), &source, 0, 1) + .expect("an old-node completion must make its target version authoritative"); + + assert!(should_resume); + + target.etag = Some("etag-client-write".to_string()); + assert!( + !resolve_data_movement_overwrite_resume_result(&err, Ok(Some(target.clone())), &source, 0, 1) + .expect("a takeover target with changed content must be rejected") + ); + + target.etag.clone_from(&source.etag); + target.parts = Arc::new(Vec::new()); + assert!( + !resolve_data_movement_overwrite_resume_result(&err, Ok(Some(target.clone())), &source, 0, 1) + .expect("a partial old-node completion must not replace the source") + ); + + target.parts.clone_from(&source.parts); + let mut newer_source = source.clone(); + newer_source.mod_time = OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND * 2); + assert!( + !resolve_data_movement_overwrite_resume_result(&err, Ok(Some(target)), &newer_source, 0, 1) + .expect("a stale takeover marker must not accept a newer source generation") + ); + + let mut missing_time_source = source.clone(); + missing_time_source.mod_time = None; + let mut missing_time_target = missing_time_source.clone(); + missing_time_target.mod_time = OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND); + let upload_identity = data_movement_upload_identity(&missing_time_source); + rustfs_utils::http::insert_str( + Arc::make_mut(&mut missing_time_target.user_defined), + SUFFIX_DATA_MOVEMENT_UPLOAD, + upload_identity, + ); + assert!( + !resolve_data_movement_overwrite_resume_result(&err, Ok(Some(missing_time_target)), &missing_time_source, 0, 1) + .expect("a takeover target must not replace a source with no generation timestamp") + ); + + let legacy_source = overwrite_equivalence_source(); + let mut legacy_target = legacy_source.clone(); + legacy_target.mod_time = OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND); + let mut legacy_target_parts = legacy_target.parts.as_ref().clone(); + legacy_target_parts[0].checksums = None; + legacy_target.parts = Arc::new(legacy_target_parts); + let upload_identity = data_movement_upload_identity(&legacy_source); + rustfs_utils::http::insert_str( + Arc::make_mut(&mut legacy_target.user_defined), + SUFFIX_DATA_MOVEMENT_UPLOAD, + upload_identity, + ); + assert!( + !resolve_data_movement_overwrite_resume_result_for(&err, Ok(Some(legacy_target)), &legacy_source, 0, 1, true) + .expect("an old-node takeover must not discard legacy part checksums") + ); + } + #[test] fn test_precondition_conflict_accepts_newer_unversioned_target() { for version_id in [None, Some(Uuid::nil())] { diff --git a/crates/ecstore/src/diagnostics/get.rs b/crates/ecstore/src/diagnostics/get.rs index ad3b514f4..4523ba42b 100644 --- a/crates/ecstore/src/diagnostics/get.rs +++ b/crates/ecstore/src/diagnostics/get.rs @@ -137,6 +137,7 @@ pub(crate) const GET_METADATA_CACHE_REASON_NO_LOCK: &str = "no_lock"; pub(crate) const GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED: &str = "not_found_or_expired"; pub(crate) const GET_METADATA_CACHE_REASON_NOT_READ_DATA: &str = "not_read_data"; pub(crate) const GET_METADATA_CACHE_REASON_PART_NUMBER: &str = "part_number"; +pub(crate) const GET_METADATA_CACHE_REASON_PART_CHECKSUMS: &str = "part_checksums"; pub(crate) const GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ: &str = "raw_data_movement_read"; pub(crate) const GET_METADATA_CACHE_REASON_STALE_PUBLICATION: &str = "stale_publication"; pub(crate) const GET_METADATA_CACHE_REASON_USABLE: &str = "usable"; @@ -480,6 +481,7 @@ mod tests { assert_eq!(GET_METADATA_CACHE_REASON_NO_LOCK, "no_lock"); assert_eq!(GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED, "not_found_or_expired"); assert_eq!(GET_METADATA_CACHE_REASON_NOT_READ_DATA, "not_read_data"); + assert_eq!(GET_METADATA_CACHE_REASON_PART_CHECKSUMS, "part_checksums"); assert_eq!(GET_METADATA_CACHE_REASON_PART_NUMBER, "part_number"); assert_eq!(GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, "raw_data_movement_read"); assert_eq!(GET_METADATA_CACHE_REASON_STALE_PUBLICATION, "stale_publication"); diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 3778d837e..258904f9d 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -9875,6 +9875,7 @@ impl DiskAPI for LocalDisk { FileInfoOpts { data: read_data, include_free_versions: opts.incl_free_versions, + include_part_checksums: false, }, )?; diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index 142ae9307..ec79567c8 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -260,6 +260,9 @@ pub struct ObjectOptions { pub data_movement: bool, pub raw_data_movement_read: bool, + /// Materialize the data-movement per-part checksum sidecar for APIs that + /// return part checksums. Ordinary object reads leave it encoded. + pub include_part_checksums: bool, pub src_pool_idx: usize, pub user_defined: HashMap, pub preserve_etag: Option, diff --git a/crates/ecstore/src/services/rebalance/entry.rs b/crates/ecstore/src/services/rebalance/entry.rs index b066a0235..764a68500 100644 --- a/crates/ecstore/src/services/rebalance/entry.rs +++ b/crates/ecstore/src/services/rebalance/entry.rs @@ -16,7 +16,7 @@ use super::meta::{ clone_arc_by_index, ensure_valid_rebalance_pool_index, invalid_rebalance_pool_index_error, rebalance_metadata_not_initialized_error, should_ignore_rebalance_data_usage_cache, }; -use super::migration::migrate_entry_version; +use super::migration::{RebalanceMigrationBackend, migrate_entry_version}; use super::worker::{ RebalanceEntryCleanupResult, RebalanceEntryTask, load_rebalance_bucket_configs, rebalance_max_attempts, resolve_rebalance_bucket_error, resolve_rebalance_entry_cleanup_delete_result, resolve_rebalance_file_info_versions_result, @@ -144,6 +144,11 @@ impl ECStore { return Ok(RebalanceEntryOutcome::Completed); } + let bucket_incarnation_fence = match bucket_configs.bucket_incarnation_id { + Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?), + None => None, + }; + let mut fivs = resolve_rebalance_file_info_versions_result(entry.file_info_versions(&bucket), bucket.as_str(), entry.name.as_str())?; @@ -203,9 +208,14 @@ impl ECStore { } let version_id = version.version_id.map(|v| v.to_string()); + let expected_bucket_incarnation_id = bucket_configs.bucket_incarnation_id; let mut transfer = |src_pool_idx: usize, bucket: String, rd: GetObjectReader| { let store = self.clone(); - async move { store.rebalance_object(src_pool_idx, bucket, rd).await } + async move { + store + .rebalance_object(src_pool_idx, bucket, rd, expected_bucket_incarnation_id) + .await + } }; // Route delete-marker migration through the store layer so it lands on the // cross-pool target (excluding the source pool), not back onto the source set. @@ -214,11 +224,12 @@ impl ECStore { async move { store.delete_object(&bucket, &object, opts).await } }; let result = migrate_entry_version( - set.as_ref(), + &RebalanceMigrationBackend::new(set.as_ref(), self.as_ref()), bucket.clone(), pool_index, version, version_id.clone(), + expected_bucket_incarnation_id, rebalance_max_attempts(), should_ignore_rebalance_data_usage_cache(bucket.as_str()), &mut transfer, @@ -303,6 +314,9 @@ impl ECStore { } if should_cleanup_rebalance_source_entry(rebalanced, fivs.versions.len(), expired) { + if bucket_incarnation_fence.as_ref().is_some_and(|guard| guard.is_lock_lost()) { + return Err(Error::other("rebalance bucket incarnation fence was lost before source cleanup")); + } let cleanup_result = self .finish_rebalance_entry_after_cleanup( pool_index, @@ -315,6 +329,12 @@ impl ECStore { entry.name.as_str(), &fivs, &cleanup_preflight_allowed_missing, + data_movement::SourceCleanupBucketFence { + expected_incarnation_id: bucket_configs.bucket_incarnation_id, + lifecycle_guard: bucket_incarnation_fence + .as_ref() + .and_then(|guard| guard.namespace_lock_guard()), + }, "rebalance", ), ) @@ -389,8 +409,14 @@ impl ECStore { } #[tracing::instrument(skip(self, rd))] - async fn rebalance_object(self: Arc, pool_idx: usize, bucket: String, rd: GetObjectReader) -> Result<()> { - data_movement::migrate_object(self, pool_idx, bucket, rd, "rebalance_object").await + async fn rebalance_object( + self: Arc, + pool_idx: usize, + bucket: String, + rd: GetObjectReader, + expected_bucket_incarnation_id: Option, + ) -> Result<()> { + data_movement::migrate_object(self, pool_idx, bucket, rd, expected_bucket_incarnation_id, "rebalance_object").await } async fn update_rebalance_last_error(&self, pool_idx: usize, message: String) -> Result<()> { diff --git a/crates/ecstore/src/services/rebalance/migration.rs b/crates/ecstore/src/services/rebalance/migration.rs index fc49a7622..7e23c9ed1 100644 --- a/crates/ecstore/src/services/rebalance/migration.rs +++ b/crates/ecstore/src/services/rebalance/migration.rs @@ -5,6 +5,7 @@ use crate::error::{Error, Result, is_err_object_not_found, is_err_version_not_fo use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions}; use crate::set_disk::SetDisks; use crate::storage_api_contracts::{object::ObjectIO, range::HTTPRangeSpec}; +use crate::store::ECStore; use http::HeaderMap; use rustfs_filemeta::FileInfo; use rustfs_utils::path::encode_dir_object; @@ -21,15 +22,23 @@ pub(crate) struct MigrationVersionResult { pub error: Option, } -pub(super) fn rebalance_delete_marker_opts(version: &FileInfo, version_id: Option, src_pool_idx: usize) -> ObjectOptions { +pub(super) fn rebalance_delete_marker_opts( + version: &FileInfo, + version_id: Option, + src_pool_idx: usize, + expected_bucket_incarnation_id: Option, +) -> ObjectOptions { + let version_suspended = version.version_id.is_none() && version_id.is_none(); ObjectOptions { - versioned: true, - version_id, + versioned: !version_suspended, + version_suspended, + version_id: version_id.or_else(|| version_suspended.then(|| uuid::Uuid::nil().to_string())), mod_time: version.mod_time, src_pool_idx, data_movement: true, delete_marker: true, skip_decommissioned: true, + expected_bucket_incarnation_id, delete_replication: version .replication_state_internal .as_ref() @@ -38,7 +47,12 @@ pub(super) fn rebalance_delete_marker_opts(version: &FileInfo, version_id: Optio } } -fn rebalance_remote_tiered_opts(version: &FileInfo, version_id: Option, src_pool_idx: usize) -> ObjectOptions { +fn rebalance_remote_tiered_opts( + version: &FileInfo, + version_id: Option, + src_pool_idx: usize, + expected_bucket_incarnation_id: Option, +) -> ObjectOptions { ObjectOptions { versioned: version_id.is_some(), version_id, @@ -46,6 +60,21 @@ fn rebalance_remote_tiered_opts(version: &FileInfo, version_id: Option, user_defined: version.metadata.clone(), src_pool_idx, data_movement: true, + include_part_checksums: true, + http_preconditions: Some(crate::data_movement::data_movement_target_precondition()), + expected_bucket_incarnation_id, + ..Default::default() + } +} + +pub(super) fn rebalance_object_migration_read_opts(version_id: Option) -> ObjectOptions { + ObjectOptions { + version_id, + no_lock: true, + data_movement: true, + raw_data_movement_read: true, + skip_decommissioned: true, + skip_rebalancing: true, ..Default::default() } } @@ -70,8 +99,19 @@ pub(crate) trait MigrationBackend: Send + Sync { ) -> Result<()>; } +pub(crate) struct RebalanceMigrationBackend<'a> { + source: &'a SetDisks, + store: &'a ECStore, +} + +impl<'a> RebalanceMigrationBackend<'a> { + pub(crate) fn new(source: &'a SetDisks, store: &'a ECStore) -> Self { + Self { source, store } + } +} + #[async_trait::async_trait] -impl MigrationBackend for SetDisks { +impl MigrationBackend for RebalanceMigrationBackend<'_> { async fn get_object_reader_for_migration( &self, bucket: &str, @@ -80,7 +120,7 @@ impl MigrationBackend for SetDisks { h: HeaderMap, opts: &ObjectOptions, ) -> Result { - self.get_object_reader(bucket, object, range, h, opts).await + self.source.get_object_reader(bucket, object, range, h, opts).await } async fn move_remote_version_for_migration( @@ -90,7 +130,7 @@ impl MigrationBackend for SetDisks { fi: &FileInfo, opts: &ObjectOptions, ) -> Result<()> { - self.decommission_tiered_object(bucket, object, fi, opts).await + self.store.decommission_tiered_object(bucket, object, fi, opts).await } } @@ -101,6 +141,7 @@ pub(crate) async fn migrate_entry_version( pool_index: usize, version: &FileInfo, version_id: Option, + expected_bucket_incarnation_id: Option, max_attempts: usize, ignore_data_usage_cache: bool, transfer: F, @@ -113,12 +154,13 @@ where D: FnMut(String, String, ObjectOptions) -> DFut + Send, DFut: Future> + Send, { - migrate_entry_version_with_retry_wait( + migrate_entry_version_with_retry_wait_and_incarnation( set, bucket, pool_index, version, version_id, + expected_bucket_incarnation_id, max_attempts, ignore_data_usage_cache, transfer, @@ -137,6 +179,45 @@ pub(super) async fn migrate_entry_version_with_retry_wait, max_attempts: usize, ignore_data_usage_cache: bool, + transfer: F, + delete_marker: D, + wait_retry: W, +) -> MigrationVersionResult +where + Backend: MigrationBackend + ?Sized, + F: FnMut(usize, String, GetObjectReader) -> Fut + Send, + Fut: Future> + Send, + D: FnMut(String, String, ObjectOptions) -> DFut + Send, + DFut: Future> + Send, + W: FnMut(Duration) -> WFut + Send, + WFut: Future + Send, +{ + migrate_entry_version_with_retry_wait_and_incarnation( + set, + bucket, + pool_index, + version, + version_id, + None, + max_attempts, + ignore_data_usage_cache, + transfer, + delete_marker, + wait_retry, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn migrate_entry_version_with_retry_wait_and_incarnation( + set: &Backend, + bucket: String, + pool_index: usize, + version: &FileInfo, + version_id: Option, + expected_bucket_incarnation_id: Option, + max_attempts: usize, + ignore_data_usage_cache: bool, mut transfer: F, mut delete_marker: D, mut wait_retry: W, @@ -169,7 +250,7 @@ where &bucket, &version.name, version, - &rebalance_remote_tiered_opts(version, version_id, pool_index), + &rebalance_remote_tiered_opts(version, version_id, pool_index, expected_bucket_incarnation_id), ) .await { @@ -212,7 +293,7 @@ where if let Err(err) = delete_marker( bucket.clone(), version.name.clone(), - rebalance_delete_marker_opts(version, version_id, pool_index), + rebalance_delete_marker_opts(version, version_id, pool_index, expected_bucket_incarnation_id), ) .await { @@ -255,11 +336,7 @@ where &encode_dir_object(&version.name), None, HeaderMap::new(), - &ObjectOptions { - version_id: version_id.clone(), - no_lock: true, - ..Default::default() - }, + &rebalance_object_migration_read_opts(version_id.clone()), ) .await { diff --git a/crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs b/crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs index aade4cb28..38d4d7460 100644 --- a/crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs +++ b/crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs @@ -113,6 +113,8 @@ struct LegacyRebalanceMeta { struct MigrationBackendSpy { get_object_reader: Mutex>>, move_remote: Mutex>>, + get_opts: Mutex>, + move_remote_opts: Mutex>, get_calls: AtomicUsize, move_remote_calls: AtomicUsize, } @@ -125,6 +127,8 @@ impl MigrationBackendSpy { Self { get_object_reader: Mutex::new(get_object_reader), move_remote: Mutex::new(move_remote), + get_opts: Mutex::new(Vec::new()), + move_remote_opts: Mutex::new(Vec::new()), get_calls: AtomicUsize::new(0), move_remote_calls: AtomicUsize::new(0), } @@ -138,6 +142,24 @@ impl MigrationBackendSpy { self.move_remote_calls.load(Ordering::SeqCst) } + fn last_get_opts(&self) -> ObjectOptions { + self.get_opts + .lock() + .unwrap() + .last() + .cloned() + .expect("reader opts should be captured") + } + + fn last_move_remote_opts(&self) -> ObjectOptions { + self.move_remote_opts + .lock() + .unwrap() + .last() + .cloned() + .expect("remote opts should be captured") + } + fn make_reader() -> GetObjectReader { GetObjectReader { stream: Box::new(Cursor::new(vec![0_u8; 3])), @@ -156,9 +178,10 @@ impl MigrationBackend for MigrationBackendSpy { _object: &str, _range: Option, _h: http::HeaderMap, - _opts: &ObjectOptions, + opts: &ObjectOptions, ) -> Result { self.get_calls.fetch_add(1, Ordering::SeqCst); + self.get_opts.lock().unwrap().push(opts.clone()); if let Some(result) = self.get_object_reader.lock().unwrap().take() { return result; } @@ -171,9 +194,10 @@ impl MigrationBackend for MigrationBackendSpy { _bucket: &str, _object: &str, _fi: &FileInfo, - _opts: &ObjectOptions, + opts: &ObjectOptions, ) -> Result<()> { self.move_remote_calls.fetch_add(1, Ordering::SeqCst); + self.move_remote_opts.lock().unwrap().push(opts.clone()); if let Some(result) = self.move_remote.lock().unwrap().take() { return result; } @@ -217,7 +241,8 @@ fn test_rebalance_delete_marker_opts_preserves_replication_state() { ..version_deleted() }; - let opts = rebalance_delete_marker_opts(&version, Some("version-id".to_string()), 7); + let incarnation = uuid::Uuid::new_v4(); + let opts = rebalance_delete_marker_opts(&version, Some("version-id".to_string()), 7, Some(incarnation)); let replication = opts.delete_replication.expect("replication state should be preserved"); assert!(opts.versioned); @@ -227,11 +252,22 @@ fn test_rebalance_delete_marker_opts_preserves_replication_state() { assert_eq!(opts.src_pool_idx, 7); assert_eq!(opts.version_id.as_deref(), Some("version-id")); assert_eq!(opts.mod_time, Some(mod_time)); + assert_eq!(opts.expected_bucket_incarnation_id, Some(incarnation)); assert_eq!(replication.replica_status, ReplicationStatusType::Replica); assert!(replication.delete_marker); assert_eq!(replication.replicate_decision_str, "existing"); } +#[test] +fn test_rebalance_delete_marker_opts_preserves_suspended_null_version() { + let version = version_deleted(); + let opts = rebalance_delete_marker_opts(&version, None, 7, None); + + assert!(!opts.versioned); + assert!(opts.version_suspended); + assert_eq!(opts.version_id.as_deref(), Some(uuid::Uuid::nil().to_string().as_str())); +} + #[tokio::test] async fn test_migrate_entry_version_remote_version_is_moved_without_transfer() { let backend = MigrationBackendSpy::new(None, Some(Ok(()))); @@ -248,12 +284,14 @@ async fn test_migrate_entry_version_remote_version_is_moved_without_transfer() { } }; + let incarnation = uuid::Uuid::new_v4(); let result = migrate_entry_version( &backend, "bucket".to_string(), 0, &version, version.version_id.map(|v| v.to_string()), + Some(incarnation), 3, false, &mut transfer, @@ -269,6 +307,10 @@ async fn test_migrate_entry_version_remote_version_is_moved_without_transfer() { assert_eq!(transfer_count.load(Ordering::SeqCst), 0); assert_eq!(backend.move_remote_calls(), 1); assert_eq!(backend.get_calls(), 0); + let remote_opts = backend.last_move_remote_opts(); + assert!(remote_opts.include_part_checksums); + assert!(remote_opts.http_preconditions.is_some()); + assert_eq!(remote_opts.expected_bucket_incarnation_id, Some(incarnation)); } #[tokio::test] @@ -294,6 +336,7 @@ async fn test_migrate_entry_version_remote_not_found_is_cleanup_ignored() { 0, &version, version.version_id.map(|v| v.to_string()), + None, 3, false, &mut transfer, @@ -330,6 +373,7 @@ async fn test_migrate_entry_version_remote_overwrite_is_not_ignored() { 0, &version, Some("vid-1".to_string()), + None, 3, false, &mut transfer, @@ -368,6 +412,7 @@ async fn test_migrate_entry_version_remote_failure_is_reported() { 0, &version, version.version_id.map(|v| v.to_string()), + None, 3, false, &mut transfer, @@ -410,6 +455,7 @@ async fn test_migrate_entry_version_deleted_version_routes_delete_through_store_ 1, &version, version.version_id.map(|v| v.to_string()), + None, 3, false, &mut transfer, @@ -449,6 +495,7 @@ async fn test_migrate_entry_version_deleted_version_not_found_is_ignored() { 1, &version, version.version_id.map(|v| v.to_string()), + None, 3, false, &mut transfer, @@ -491,6 +538,7 @@ async fn test_migrate_entry_version_deleted_version_overwrite_is_not_ignored() { 1, &version, Some("vid-1".to_string()), + None, 3, false, &mut transfer, @@ -520,6 +568,7 @@ async fn test_migrate_entry_version_reader_not_found_is_ignored() { 1, &version, version.version_id.map(|v| v.to_string()), + None, 3, false, &mut transfer, @@ -647,6 +696,7 @@ async fn test_migrate_entry_version_reader_fails_after_retries() { 1, &version, version.version_id.map(|v| v.to_string()), + None, 3, false, &mut transfer, @@ -685,6 +735,7 @@ async fn test_migrate_entry_version_zero_max_attempts_still_attempts_once() { 1, &version, version.version_id.map(|v| v.to_string()), + None, 0, false, &mut transfer, @@ -750,6 +801,13 @@ async fn test_migrate_entry_version_transfer_retries_before_success() { assert_eq!(backend.get_calls(), 2); assert_eq!(transfer_count.load(Ordering::SeqCst), 2); assert_eq!(wait_count.load(Ordering::SeqCst), 1); + let read_opts = backend.last_get_opts(); + assert_eq!(read_opts.version_id.as_deref(), version.version_id.map(|id| id.to_string()).as_deref()); + assert!(read_opts.no_lock); + assert!(read_opts.data_movement); + assert!(read_opts.raw_data_movement_read); + assert!(read_opts.skip_decommissioned); + assert!(read_opts.skip_rebalancing); } #[tokio::test] @@ -822,6 +880,7 @@ async fn test_migrate_entry_version_transfer_fails_after_retries() { 1, &version, version.version_id.map(|v| v.to_string()), + None, 2, false, &mut transfer, @@ -860,6 +919,7 @@ async fn test_migrate_entry_version_transfer_not_found_is_ignored() { 1, &version, version.version_id.map(|v| v.to_string()), + None, 3, false, &mut transfer, @@ -901,6 +961,7 @@ async fn test_migrate_entry_version_transfer_overwrite_is_not_ignored() { 1, &version, Some("vid-1".to_string()), + None, 3, false, &mut transfer, @@ -943,6 +1004,7 @@ async fn test_migrate_entry_version_ignores_data_usage_cache_when_enabled() { 1, &version, version.version_id.map(|v| v.to_string()), + None, 2, true, &mut transfer, @@ -985,6 +1047,7 @@ async fn test_migrate_entry_version_data_usage_cache_moves_when_ignore_disabled( 1, &version, version.version_id.map(|v| v.to_string()), + None, 2, false, &mut transfer, @@ -2026,6 +2089,7 @@ async fn test_migrate_entry_version_transfer_failure_reports_write_target_stage( 1, &version, version.version_id.map(|v| v.to_string()), + None, 1, false, &mut transfer, @@ -2050,6 +2114,7 @@ async fn test_migrate_entry_version_reader_failure_reports_read_source_stage() { 1, &version, version.version_id.map(|v| v.to_string()), + None, 1, false, &mut transfer, diff --git a/crates/ecstore/src/services/rebalance/types.rs b/crates/ecstore/src/services/rebalance/types.rs index bf6fb3bb8..c8c8378ea 100644 --- a/crates/ecstore/src/services/rebalance/types.rs +++ b/crates/ecstore/src/services/rebalance/types.rs @@ -36,6 +36,7 @@ pub type RStats = Vec>; #[derive(Debug, Default)] pub(super) struct RebalanceBucketConfigs { + pub(super) bucket_incarnation_id: Option, pub(super) lifecycle_config: Option, pub(super) object_lock_config: Option, pub(super) replication_config: Option<(s3s::dto::ReplicationConfiguration, OffsetDateTime)>, diff --git a/crates/ecstore/src/services/rebalance/worker.rs b/crates/ecstore/src/services/rebalance/worker.rs index 2ad077f30..5a6b7abc6 100644 --- a/crates/ecstore/src/services/rebalance/worker.rs +++ b/crates/ecstore/src/services/rebalance/worker.rs @@ -406,6 +406,7 @@ pub(super) async fn load_rebalance_bucket_configs(api: &ECStore, bucket: &str) - let expiry_configs = crate::bucket::lifecycle::get_expiry_configs(api, bucket).await?; Ok(RebalanceBucketConfigs { + bucket_incarnation_id: Some(api.bucket_incarnation_id_from_disk(bucket).await?), lifecycle_config: expiry_configs.lifecycle.map(|config| (*config).clone()), object_lock_config: expiry_configs.object_lock.map(|config| (*config).clone()), replication_config: resolve_rebalance_optional_bucket_config_result( diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index a6c3095e0..29ada8dfd 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -2443,13 +2443,14 @@ impl SetDisks { bucket: &str, object: &str, ) -> Result> { + let disk_object = rustfs_utils::path::encode_dir_object(object); let disks = self.get_disks_internal().await; if disks.is_empty() { return Err(to_object_err(StorageError::ErasureReadQuorum, vec![bucket, object])); } let read_quorum = disks.len().div_ceil(2).max(1); - let (raw_fileinfos, errs) = Self::read_all_raw_file_info(&disks, bucket, object, false).await; + let (raw_fileinfos, errs) = Self::read_all_raw_file_info(&disks, bucket, disk_object.as_str(), false).await; if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) { let object_err = to_object_err(err.into(), vec![bucket, object]); @@ -2605,7 +2606,7 @@ impl SetDisks { // // `into_fileinfo` with an empty version_id selects the first non-free version // (see FileMeta::into_fileinfo); replicate that selection from the header here. - let vid = match meta.into_fileinfo(bucket, object, "", true, incl_free_vers, true) { + let vid = match meta.into_fileinfo_without_part_checksums(bucket, object, "", true, incl_free_vers) { Ok(finfo) if file_info_is_valid_for_metadata(&finfo) => finfo.version_id.unwrap_or(Uuid::nil()), _ => match meta .versions @@ -2628,7 +2629,13 @@ impl SetDisks { for (idx, meta_op) in metadata_array.iter().enumerate() { if let Some(meta) = meta_op { - match meta.into_fileinfo(bucket, object, vid.to_string().as_str(), read_data, incl_free_vers, true) { + match meta.into_fileinfo_without_part_checksums( + bucket, + object, + vid.to_string().as_str(), + read_data, + incl_free_vers, + ) { Ok(res) => match res.validate_for_metadata_read() { Ok(_) => meta_file_infos[idx] = res, Err(err) => errs[idx] = Some(err.into()), @@ -4533,26 +4540,29 @@ impl SetDisks { object: &str, opts: &ObjectOptions, ) -> Option { - let mut opts = opts.clone(); + let mut lookup_opts = opts.clone(); - let http_preconditions = opts.http_preconditions?; - opts.http_preconditions = None; + let http_preconditions = lookup_opts.http_preconditions?; + lookup_opts.http_preconditions = None; // Never claim a lock here, to avoid deadlock // - If no_lock is false, we must have obtained the lock out side of this function // - If no_lock is true, we should not obtain locks - opts.no_lock = true; - let oi = self.get_object_info(bucket, object, &opts).await; + lookup_opts.no_lock = true; + let oi = self.get_object_info(bucket, object, &lookup_opts).await; match oi { Ok(oi) => { - // If top level is a delete marker proceed to upload. + // Ordinary writes may proceed past a top-level delete marker; + // data movement must not replace an acknowledged deletion. if oi.delete_marker { - return None; + return opts.data_movement.then_some(StorageError::PreconditionFailed); } let if_none_match = http_preconditions.if_none_match_value().map(str::to_owned); let if_match = http_preconditions.if_match_value().map(str::to_owned); - if should_prevent_write(&oi, if_none_match, if_match) { + if should_prevent_write(&oi, if_none_match, if_match) + && !crate::data_movement::can_replace_stale_data_movement_target(&oi, opts) + { return Some(StorageError::PreconditionFailed); } } @@ -6449,6 +6459,32 @@ mod tests { assert_eq!(versions.versions[0].name, object); } + #[tokio::test] + async fn load_file_info_versions_exact_encodes_directory_key_but_returns_logical_name() { + let bucket = "exact-directory-versions-bucket"; + let object = "prefix/directory/"; + let disk_object = rustfs_utils::path::encode_dir_object(object); + let (_dir, disk) = read_multiple_test_disk(bucket, &[]).await; + let mut fi = metadata_test_fileinfo(object); + fi.version_id = Some(Uuid::new_v4()); + fi.mod_time = Some(OffsetDateTime::now_utc()); + disk.write_metadata(bucket, bucket, disk_object.as_str(), fi.clone()) + .await + .expect("directory metadata should be written under the encoded key"); + let set = io_primitives_test_set(vec![Some(disk)], 0).await; + + let versions = set + .load_file_info_versions_exact(bucket, object) + .await + .expect("exact directory version load should succeed") + .expect("exact directory version load should find metadata"); + + assert_eq!(versions.name, object); + assert_eq!(versions.versions.len(), 1); + assert_eq!(versions.versions[0].name, object); + assert_eq!(versions.versions[0].version_id, fi.version_id); + } + #[tokio::test] async fn load_file_info_versions_exact_rejects_transitioned_duplicate_parts() { let bucket = "exact-versions-bucket"; diff --git a/crates/ecstore/src/set_disk/metadata.rs b/crates/ecstore/src/set_disk/metadata.rs index b07ed428c..57a51f861 100644 --- a/crates/ecstore/src/set_disk/metadata.rs +++ b/crates/ecstore/src/set_disk/metadata.rs @@ -85,6 +85,15 @@ impl SetDisks { format!("{}/{}", Self::get_multipart_sha_dir(bucket, object), upload_uuid) } + pub(super) fn get_multipart_upload_dir(bucket: &str, object: &str, upload_id: &str, data_movement: bool) -> String { + let upload_dir = Self::get_upload_id_dir(bucket, object, upload_id); + if data_movement { + format!("{DATA_MOVEMENT_MULTIPART_PREFIX}/{upload_dir}") + } else { + upload_dir + } + } + pub(super) fn get_multipart_sha_dir(bucket: &str, object: &str) -> String { let path = format!("{bucket}/{object}"); let mut hasher = Sha256::new(); @@ -466,6 +475,28 @@ impl SetDisks { Self::find_file_info_in_quorum(metas, &mod_time, &etag, quorum) } + pub(crate) fn hydrate_selected_fileinfo_part_checksums(fi: &mut FileInfo) -> disk::error::Result<()> { + fi.hydrate_data_movement_part_checksums().map_err(DiskError::from)?; + for part in &fi.parts { + let Some(checksums) = part.checksums.as_ref() else { + continue; + }; + let mut algorithms = HashSet::with_capacity(checksums.len()); + for (name, value) in checksums { + let Some(checksum) = rustfs_rio::Checksum::new_from_string(name, value) else { + return Err(DiskError::FileCorrupt); + }; + if checksum.checksum_type.is(rustfs_rio::ChecksumType::MULTIPART) { + return Err(DiskError::FileCorrupt); + } + if !algorithms.insert(checksum.checksum_type.base().0) { + return Err(DiskError::FileCorrupt); + } + } + } + Ok(()) + } + fn update_hash_bytes(hasher: &mut Sha256, value: &[u8]) { hasher.update(value.len().to_le_bytes()); hasher.update(value); diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 9a1a29ee6..3b37b64a6 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -288,6 +288,7 @@ pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024; pub const MAX_PARTS_COUNT: usize = 10000; pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket"; pub(crate) const RUSTFS_MULTIPART_OBJECT_KEY: &str = "x-rustfs-internal-multipart-object"; +pub(crate) const DATA_MOVEMENT_MULTIPART_PREFIX: &str = "data-movement"; const ENV_ISSUE3031_DIAG_ENABLE: &str = "RUSTFS_ISSUE3031_DIAG_ENABLE"; /// Validate disk metadata at a boundary that may legitimately return a delete @@ -3780,8 +3781,10 @@ fn resolve_delete_version_state(opts: &ObjectOptions, goi: &ObjectInfo, version_ if opts.version_id.is_some() { // Decommission/rebalance may recreate a delete marker on a new pool before that // exact version exists there, so we must still treat it as a mark-delete write. - if opts.data_movement && opts.delete_marker && !version_found { + let data_movement_missing_delete_marker = opts.data_movement && opts.delete_marker && !version_found; + if data_movement_missing_delete_marker { mark_delete = true; + delete_marker = true; } let delete_marker_version_purge = version_found && goi.delete_marker && !opts.version_purge_status().is_empty(); @@ -3790,7 +3793,10 @@ fn resolve_delete_version_state(opts: &ObjectOptions, goi: &ObjectInfo, version_ mark_delete = false; } - if opts.version_purge_status().is_empty() && opts.delete_marker_replication_status().is_empty() { + if !data_movement_missing_delete_marker + && opts.version_purge_status().is_empty() + && opts.delete_marker_replication_status().is_empty() + { mark_delete = false; } @@ -3832,6 +3838,19 @@ impl SetDisks { opts: &ObjectOptions, ) -> Result<()> { let storage_class_config = self.storage_class_config_snapshot(); + let bucket_lifecycle_guard = if let Some(expected_incarnation_id) = opts.expected_bucket_incarnation_id + && opts.bucket_lifecycle_lock_fence.is_none() + && !crate::bucket::utils::is_meta_bucketname(bucket) + { + Some( + metadata_sys::object_store_in(&self.ctx) + .await? + .acquire_bucket_incarnation_fence(bucket, expected_incarnation_id) + .await?, + ) + } else { + None + }; let _lock_guard = if !opts.no_lock { Some( self.new_ns_lock(bucket, object) @@ -3844,6 +3863,12 @@ impl SetDisks { None }; + if opts.http_preconditions.is_some() + && let Some(err) = self.check_write_precondition(bucket, object, opts).await + { + return Err(err); + } + let disks = self.disks.read().await.clone(); let storage_class = opts.user_defined.get(AMZ_STORAGE_CLASS).map(String::as_str); let layout = resolve_write_layout( @@ -3856,6 +3881,20 @@ impl SetDisks { )?; let fi = build_tiered_decommission_file_info(bucket, object, fi, layout); let write_quorum = layout.write_quorum; + if opts + .bucket_lifecycle_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + || bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "decommission_tiered_object_commit", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + }); + } let parts_metadata = vec![fi.clone(); disks.len()]; let (shuffle_disks, parts_metadata) = Self::shuffle_disks_and_parts_metadata(&disks, &parts_metadata, &fi); @@ -5283,6 +5322,22 @@ mod tests { assert!(delete_marker); } + #[test] + fn resolve_delete_version_state_creates_missing_suspended_data_movement_marker() { + let opts = ObjectOptions { + version_suspended: true, + version_id: Some(Uuid::nil().to_string()), + data_movement: true, + delete_marker: true, + ..Default::default() + }; + + let (mark_delete, delete_marker) = resolve_delete_version_state(&opts, &ObjectInfo::default(), false); + + assert!(mark_delete); + assert!(delete_marker); + } + #[test] fn should_force_delete_marker_for_missing_version_rejects_data_movement_latest_delete() { let opts = ObjectOptions { @@ -6966,6 +7021,7 @@ mod tests { rustfs_filemeta::FileInfoOpts { data: false, include_free_versions: false, + include_part_checksums: true, }, ) .expect("test file metadata should decode as file info") @@ -7094,20 +7150,92 @@ mod tests { fn test_latest_fileinfo_selection_preserves_degraded_read_quorum_without_competing_latest() { let mod_time = OffsetDateTime::now_utc(); let data_dir = Uuid::new_v4(); - let metas = vec![ - quorum_test_fileinfo(mod_time, data_dir, "part-etag-old", 1), - quorum_test_fileinfo(mod_time, data_dir, "part-etag-old", 2), - FileInfo::default(), - FileInfo::default(), - ]; + let mut first = quorum_test_fileinfo(mod_time, data_dir, "part-etag-old", 1); + rustfs_utils::http::insert_str( + &mut first.metadata, + rustfs_utils::http::SUFFIX_PART_CHECKSUMS, + r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string(), + ); + let mut second = first.clone(); + second.erasure.index = 2; + let metas = vec![first, second, FileInfo::default(), FileInfo::default()]; let errs = vec![None, None, Some(DiskError::DiskNotFound), Some(DiskError::DiskNotFound)]; - let (_, selected, selected_quorum) = SetDisks::select_valid_fileinfo(&vec![None; metas.len()], &metas, &errs, "", 2, 3) - .expect("read quorum should remain enough when no competing latest is visible"); + let (_, mut selected, selected_quorum) = + SetDisks::select_valid_fileinfo(&vec![None; metas.len()], &metas, &errs, "", 2, 3) + .expect("read quorum should remain enough when no competing latest is visible"); assert_eq!(selected_quorum, 2); assert_eq!(selected.data_dir, Some(data_dir)); assert_eq!(selected.parts[0].etag, "part-etag-old"); + assert!(selected.parts[0].checksums.is_none()); + SetDisks::hydrate_selected_fileinfo_part_checksums(&mut selected) + .expect("requested part checksums should hydrate after winner selection"); + assert_eq!( + selected.parts[0] + .checksums + .as_ref() + .and_then(|checksums| checksums.get("CRC32C")) + .map(String::as_str), + Some("AAAAAA==") + ); + } + + #[test] + fn test_degraded_fileinfo_selection_rejects_malformed_part_checksum_metadata() { + let mod_time = OffsetDateTime::now_utc(); + let data_dir = Uuid::new_v4(); + let mut first = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 1); + rustfs_utils::http::insert_str(&mut first.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS, "not-json".to_string()); + let mut second = first.clone(); + second.erasure.index = 2; + let metas = vec![first, second, FileInfo::default(), FileInfo::default()]; + let errs = vec![None, None, Some(DiskError::DiskNotFound), Some(DiskError::DiskNotFound)]; + + let (_, mut selected, _) = SetDisks::select_valid_fileinfo(&vec![None; metas.len()], &metas, &errs, "", 2, 3) + .expect("winner selection should defer sidecar decoding"); + let err = SetDisks::hydrate_selected_fileinfo_part_checksums(&mut selected) + .expect_err("a malformed degraded winner must fail closed when checksums are requested"); + + assert_eq!(err, DiskError::FileCorrupt); + } + + #[test] + fn test_pick_valid_fileinfo_rejects_malformed_part_checksum_metadata() { + let mod_time = OffsetDateTime::now_utc(); + let data_dir = Uuid::new_v4(); + let mut meta = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 1); + rustfs_utils::http::insert_str(&mut meta.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS, "not-json".to_string()); + let mut second = meta.clone(); + second.erasure.index = 2; + + let mut selected = SetDisks::pick_valid_fileinfo(&[meta, second], Some(mod_time), None, 2) + .expect("winner selection should defer sidecar decoding"); + let err = SetDisks::hydrate_selected_fileinfo_part_checksums(&mut selected) + .expect_err("a malformed winning part-checksum sidecar must fail closed when checksums are requested"); + + assert_eq!(err, DiskError::FileCorrupt); + } + + #[test] + fn test_part_checksum_hydration_rejects_invalid_algorithm_and_value() { + let mod_time = OffsetDateTime::now_utc(); + let data_dir = Uuid::new_v4(); + for encoded in [ + r#"[[1,[["UNKNOWN","AAAAAA=="]]]]"#, + r#"[[1,[["CRC32C","not-base64"]]]]"#, + r#"[[1,[["CRC32C","AA=="]]]]"#, + r#"[[1,[["CRC32C","AAAAAA==-0"]]]]"#, + r#"[[1,[["CRC32C","AAAAAA==-1"]]]]"#, + r#"[[1,[["CRC32C","AAAAAA=="],["crc32c","BBBBBB=="]]]]"#, + ] { + let mut meta = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 1); + rustfs_utils::http::insert_str(&mut meta.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS, encoded.to_string()); + + let err = SetDisks::hydrate_selected_fileinfo_part_checksums(&mut meta) + .expect_err("invalid persisted part checksum metadata must fail closed"); + assert_eq!(err, DiskError::FileCorrupt); + } } #[test] @@ -10744,6 +10872,20 @@ mod tests { assert!(marker.delete_marker); let marker_version = marker.version_id.expect("versioned delete marker should carry a version id"); + let create_only = ObjectOptions { + versioned: true, + data_movement: true, + http_preconditions: Some(HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }), + ..Default::default() + }; + assert_eq!( + set_disks.check_write_precondition(bucket, object, &create_only).await, + Some(StorageError::PreconditionFailed), + "data movement must not replace a target delete marker" + ); let err = match set_disks .get_object_reader(bucket, object, None, HeaderMap::new(), &opts) .await diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index 4e6c81d96..718050533 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -542,7 +542,8 @@ impl SetDisks { let filter_by_etag = quorum_etag.is_some(); match Self::pick_valid_fileinfo(&parts_metadata, quorum_mod_time, quorum_etag.clone(), read_quorum as usize) { - Ok(latest_meta) => { + Ok(mut latest_meta) => { + Self::hydrate_selected_fileinfo_part_checksums(&mut latest_meta)?; trace!( event = EVENT_SET_DISK_HEAL, component = LOG_COMPONENT_ECSTORE, diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index 2d7a5ff27..1cbad8cc1 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -33,6 +33,29 @@ use tokio::task::JoinSet; const MULTIPART_LIST_IO_CONCURRENCY: usize = 16; +pub(crate) struct StaleMultipartCleanupGuard { + file_info: FileInfo, + upload_path: String, + write_quorum: usize, + lock_guard: ObjectLockDiagGuard, +} + +impl StaleMultipartCleanupGuard { + pub(crate) fn file_info(&self) -> &FileInfo { + &self.file_info + } + + pub(crate) fn is_lock_lost(&self) -> bool { + self.lock_guard.is_lock_lost() + } + + pub(crate) async fn delete(self, set: &SetDisks) -> Result<()> { + fence_commit_on_lock_loss(Some(&self.lock_guard), "stale_multipart_cleanup", &self.upload_path)?; + set.delete_all_with_quorum(RUSTFS_META_MULTIPART_BUCKET, &self.upload_path, self.write_quorum) + .await + } +} + #[cfg(test)] #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum MultipartCommitPause { @@ -223,6 +246,20 @@ fn validate_multipart_bucket_incarnation( Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned())) } +fn ensure_data_movement_upload_access( + fi: &FileInfo, + bucket: &str, + object: &str, + upload_id: &str, + opts: &ObjectOptions, +) -> Result<()> { + if rustfs_utils::http::contains_key_str(&fi.metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD) && !opts.data_movement + { + return Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned())); + } + Ok(()) +} + async fn ensure_multipart_bucket_incarnation( ctx: &crate::runtime::instance::InstanceContext, fi: &FileInfo, @@ -445,7 +482,7 @@ impl SetDisks { return Ok(None); } - let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id); + let upload_id_path = Self::get_multipart_upload_dir(bucket, object, upload_id, opts.data_movement); self.acquire_read_lock_diag(op, RUSTFS_META_MULTIPART_BUCKET, &upload_id_path) .await .map(Some) @@ -463,7 +500,7 @@ impl SetDisks { return Ok(None); } - let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id); + let upload_id_path = Self::get_multipart_upload_dir(bucket, object, upload_id, opts.data_movement); self.acquire_write_lock_diag(op, RUSTFS_META_MULTIPART_BUCKET, &upload_id_path) .await .map(Some) @@ -511,13 +548,49 @@ impl SetDisks { upload_id: &str, write: bool, ) -> Result<(FileInfo, Vec)> { - let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id); + self.check_upload_id_exists_for_data_movement(bucket, object, upload_id, write, false) + .await + } + + async fn check_upload_id_exists_with_opts( + &self, + bucket: &str, + object: &str, + upload_id: &str, + write: bool, + opts: &ObjectOptions, + ) -> Result<(FileInfo, Vec)> { + self.check_upload_id_exists_for_data_movement(bucket, object, upload_id, write, opts.data_movement) + .await + } + + async fn check_upload_id_exists_for_data_movement( + &self, + bucket: &str, + object: &str, + upload_id: &str, + write: bool, + data_movement: bool, + ) -> Result<(FileInfo, Vec)> { + let upload_id_path = Self::get_multipart_upload_dir(bucket, object, upload_id, data_movement); + self.check_multipart_upload_path_exists(bucket, object, upload_id, &upload_id_path, write) + .await + } + + async fn check_multipart_upload_path_exists( + &self, + bucket: &str, + object: &str, + upload_id: &str, + upload_id_path: &str, + write: bool, + ) -> Result<(FileInfo, Vec)> { let disks = self.disks.read().await; let disks = disks.clone(); let (parts_metadata, errs) = - Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_id_path, "", false, false, false) + Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, upload_id_path, "", false, false, false) .await?; let (read_quorum, write_quorum) = Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count) @@ -562,6 +635,20 @@ impl SetDisks { Ok((fi, parts_metadata)) } + pub(crate) async fn lock_stale_multipart_cleanup(&self, upload_path: &str) -> Result { + let lock_guard = self + .acquire_write_lock_diag("stale_multipart_cleanup", RUSTFS_META_MULTIPART_BUCKET, upload_path) + .await?; + let (file_info, _) = self.check_multipart_upload_path_exists("", "", "", upload_path, true).await?; + let write_quorum = file_info.write_quorum(self.default_write_quorum()); + Ok(StaleMultipartCleanupGuard { + file_info, + upload_path: upload_path.to_string(), + write_quorum, + lock_guard, + }) + } + #[allow(clippy::too_many_arguments)] pub(crate) async fn list_multipart_uploads_for_incarnation( &self, @@ -704,6 +791,12 @@ impl SetDisks { { return Ok(None); } + if rustfs_utils::http::contains_key_str( + &file_info.metadata, + rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD, + ) { + return Ok(None); + } let object = match ( file_info.metadata.get(RUSTFS_MULTIPART_BUCKET_KEY), @@ -852,9 +945,12 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { opts: &ObjectOptions, ) -> Result { crate::hp_guard!("SetDisks::put_object_part"); - let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id); + let upload_id_path = Self::get_multipart_upload_dir(bucket, object, upload_id, opts.data_movement); - let (fi, _) = self.check_upload_id_exists(bucket, object, upload_id, true).await?; + let (fi, _) = self + .check_upload_id_exists_with_opts(bucket, object, upload_id, true, opts) + .await?; + ensure_data_movement_upload_access(&fi, bucket, object, upload_id, opts)?; ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id) .await?; @@ -1120,7 +1216,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { .await?; (Some(upload_guard), Some(part_guard)) }; - let (commit_fi, _) = self.check_upload_id_exists(bucket, object, upload_id, false).await?; + let (commit_fi, _) = self + .check_upload_id_exists_with_opts(bucket, object, upload_id, false, opts) + .await?; + ensure_data_movement_upload_access(&commit_fi, bucket, object, upload_id, opts)?; ensure_multipart_bucket_incarnation( &self.ctx, &commit_fi, @@ -1196,11 +1295,14 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { let _upload_guard = self .acquire_multipart_upload_read_lock("list_object_parts", bucket, object, upload_id, opts) .await?; - let (fi, _) = self.check_upload_id_exists(bucket, object, upload_id, false).await?; + let (fi, _) = self + .check_upload_id_exists_with_opts(bucket, object, upload_id, false, opts) + .await?; + ensure_data_movement_upload_access(&fi, bucket, object, upload_id, opts)?; ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id) .await?; - let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id); + let upload_id_path = Self::get_multipart_upload_dir(bucket, object, upload_id, opts.data_movement); if max_parts > MAX_PARTS_COUNT { max_parts = MAX_PARTS_COUNT; @@ -1384,6 +1486,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { let disks = disks.clone(); let mut user_defined = opts.user_defined.clone(); + rustfs_utils::http::remove_str(&mut user_defined, rustfs_utils::http::SUFFIX_PART_CHECKSUMS); + if !opts.data_movement { + rustfs_utils::http::remove_str(&mut user_defined, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD); + } rustfs_utils::http::metadata_compat::remove_str( &mut user_defined, crate::object_api::ENCRYPTED_PART_LAYOUT_QUORUM_SUFFIX, @@ -1489,7 +1595,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { let upload_id = runtime_sources::deployment_upload_id(&upload_uuid); - let upload_path = Self::get_upload_id_dir(bucket, object, upload_uuid.as_str()); + let upload_path = Self::get_multipart_upload_dir(bucket, object, upload_uuid.as_str(), opts.data_movement); ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?; Self::write_unique_file_info( @@ -1524,9 +1630,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { .acquire_multipart_upload_read_lock("get_multipart_info", bucket, object, upload_id, opts) .await?; let (mut fi, _) = self - .check_upload_id_exists(bucket, object, upload_id, false) + .check_upload_id_exists_with_opts(bucket, object, upload_id, false, opts) .await .map_err(|e| to_object_err(e, vec![bucket, object, upload_id]))?; + ensure_data_movement_upload_access(&fi, bucket, object, upload_id, opts)?; ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id) .await?; ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?; @@ -1548,11 +1655,14 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { let _upload_guard = self .acquire_multipart_upload_write_lock("abort_multipart_upload", bucket, object, upload_id, opts) .await?; - let (fi, _) = self.check_upload_id_exists(bucket, object, upload_id, true).await?; + let (fi, _) = self + .check_upload_id_exists_with_opts(bucket, object, upload_id, true, opts) + .await?; + ensure_data_movement_upload_access(&fi, bucket, object, upload_id, opts)?; ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id) .await?; ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?; - let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id); + let upload_id_path = Self::get_multipart_upload_dir(bucket, object, upload_id, opts.data_movement); self.delete_all_with_quorum( RUSTFS_META_MULTIPART_BUCKET, @@ -1574,7 +1684,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { crate::hp_guard!("SetDisks::complete_multipart_upload"); self.invalidate_get_object_metadata_cache(bucket, object).await; - let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id); + let upload_id_path = Self::get_multipart_upload_dir(bucket, object, upload_id, opts.data_movement); let range_seek_rollout_enabled = crate::object_api::legacy_encrypted_range_seek_enabled() && !opts.no_lock; let mut object_lock_guard = None; @@ -1602,7 +1712,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { .await?; let expected_restore_operation_id = restore_commit_operation_id_from_metadata(&opts.user_defined)?; - let (mut fi, files_metas) = self.check_upload_id_exists(bucket, object, upload_id, true).await?; + let (mut fi, files_metas) = self + .check_upload_id_exists_with_opts(bucket, object, upload_id, true, opts) + .await?; + ensure_data_movement_upload_access(&fi, bucket, object, upload_id, opts)?; ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id) .await?; let has_layout_candidate = range_seek_rollout_enabled @@ -1771,7 +1884,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { // Build a lookup map for O(1) part resolution instead of O(n) find() in the loop // This optimizes from O(n^2) to O(n) when processing many parts - use std::collections::HashMap; + use std::collections::{HashMap, HashSet}; let part_lookup: HashMap = curr_fi.parts.iter().map(|part| (part.number, part)).collect(); for (i, p) in uploaded_parts.iter().enumerate() { @@ -1807,7 +1920,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { // TODO: crypto - if (i < uploaded_parts.len() - 1) && !is_min_allowed_part_size(ext_part.actual_size) { + if (i < uploaded_parts.len() - 1) + && !(opts.data_movement && ext_part.actual_size < 0) + && !is_min_allowed_part_size(ext_part.actual_size) + { error!( "complete_multipart_upload part size too small: part {} size {} is less than minimum {}", p.part_num, @@ -1978,7 +2094,27 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { ); } - if opts.replication_request { + let data_movement_actual_size = if opts.data_movement { + rustfs_utils::http::get_consistent_str(&opts.user_defined, SUFFIX_ACTUAL_SIZE) + .map(|value| { + value + .parse::() + .ok() + .filter(|value| *value >= 0) + .ok_or_else(|| Error::other("data movement actual size metadata is invalid")) + }) + .transpose()? + } else { + None + }; + + if let Some(actual_size) = data_movement_actual_size { + insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string()); + if persist_encryption_original_size { + fi.metadata + .insert("x-rustfs-encryption-original-size".to_string(), actual_size.to_string()); + } + } else if opts.replication_request { if let Some(actual_size) = get_str(&opts.user_defined, SUFFIX_ACTUAL_OBJECT_SIZE_CAP) { insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.clone()); if persist_encryption_original_size { @@ -1998,7 +2134,15 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { insert_str(&mut fi.metadata, SUFFIX_COMPRESSION_SIZE, object_size.to_string()); } + rustfs_utils::http::remove_str(&mut fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS); + if let Some(part_checksums) = + rustfs_utils::http::get_consistent_str(&opts.user_defined, rustfs_utils::http::SUFFIX_PART_CHECKSUMS) + { + insert_str(&mut fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS, part_checksums.to_string()); + } + if opts.data_movement { + rustfs_utils::http::remove_str(&mut fi.metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD); fi.set_data_moved(); } @@ -2014,19 +2158,21 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { } } + let completed_part_numbers = fi.parts.iter().map(|part| part.number).collect::>(); + let upload_data_dir = curr_fi.data_dir.unwrap_or(Uuid::nil()).to_string(); let mut parts = Vec::with_capacity(curr_fi.parts.len()); for p in curr_fi.parts.iter() { parts.push(path_join_buf(&[ &upload_id_path, - curr_fi.data_dir.unwrap_or(Uuid::nil()).to_string().as_str(), + upload_data_dir.as_str(), format!("part.{}.meta", p.number).as_str(), ])); - if !fi.parts.iter().any(|v| v.number == p.number) { + if !completed_part_numbers.contains(&p.number) { parts.push(path_join_buf(&[ &upload_id_path, - curr_fi.data_dir.unwrap_or(Uuid::nil()).to_string().as_str(), + upload_data_dir.as_str(), format!("part.{}", p.number).as_str(), ])); } @@ -2049,6 +2195,19 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { achieved: 0, }); } + if opts + .namespace_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "complete_multipart_upload_outer_lock", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + }); + } if upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) { return Err(StorageError::NamespaceLockQuorumUnavailable { mode: "complete_multipart_upload_commit", @@ -2069,6 +2228,74 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { ) .await?; + if opts.data_movement + && opts.http_preconditions.is_some() + && let Some(err) = self.check_write_precondition(bucket, object, opts).await + { + return Err(err); + } + if opts.data_movement && opts.http_preconditions.is_some() && !crate::bucket::utils::is_meta_bucketname(bucket) { + let current = self + .get_object_info( + bucket, + object, + &ObjectOptions { + version_id: opts.version_id.clone(), + no_lock: true, + metadata_cache_safe: false, + versioned: opts.versioned, + version_suspended: opts.version_suspended, + ..Default::default() + }, + ) + .await; + match current { + Ok(existing) if crate::data_movement::can_replace_stale_data_movement_target(&existing, opts) => { + let object_lock_config = opts.object_lock_config_snapshot.as_deref().ok_or_else(|| { + Error::other("data movement completion is missing its Object Lock configuration snapshot") + })?; + if check_object_lock_for_deletion_with_state(object_lock_config.state(), &existing, false)?.is_some() { + return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object.to_string())); + } + } + Ok(_) => return Err(StorageError::PreconditionFailed), + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {} + Err(err) => return Err(err), + } + } + if object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "complete_multipart_upload_commit", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + }); + } + if opts + .namespace_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "complete_multipart_upload_outer_lock", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + }); + } + if upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "complete_multipart_upload_commit", + bucket: RUSTFS_META_MULTIPART_BUCKET.to_string(), + object: upload_id_path.clone(), + required: 1, + achieved: 0, + }); + } + ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?; + let complete_tail_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now); // Crash-consistency injection: hard power loss after the upload is fully @@ -2589,8 +2816,29 @@ mod tests { .new_multipart_upload(bucket, object, create_opts) .await .expect("multipart upload should be created"); - let part = put_test_part(set_disks, bucket, object, &upload.upload_id, 1, content, content.len() as i64).await; - (upload.upload_id, vec![part]) + let mut reader = PutObjReader::new( + HashReader::from_stream( + Cursor::new(content.to_vec()), + content.len() as i64, + content.len() as i64, + None, + None, + false, + ) + .expect("hash reader should be constructed"), + ); + let part = set_disks + .put_object_part(bucket, object, &upload.upload_id, 1, &mut reader, create_opts) + .await + .expect("uploading the part should succeed"); + ( + upload.upload_id, + vec![CompletePart { + part_num: part.part_num, + etag: part.etag, + ..Default::default() + }], + ) } async fn put_test_part( @@ -2742,6 +2990,353 @@ mod tests { assert!(matches!(overflow_err, StorageError::PartMissingOrCorrupt)); } + #[tokio::test] + #[serial] + async fn data_movement_upload_is_hidden_from_external_multipart_operations() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "data-movement-upload-complete-bucket"; + make_bucket_on_all(&disk_stores, bucket).await; + let mut durable_metadata = HashMap::new(); + rustfs_utils::http::insert_str(&mut durable_metadata, rustfs_utils::http::SUFFIX_DATA_MOVED, "true".to_string()); + rustfs_utils::http::insert_str( + &mut durable_metadata, + rustfs_utils::http::SUFFIX_PART_CHECKSUMS, + r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string(), + ); + rustfs_utils::http::insert_str( + &mut durable_metadata, + rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD, + "residual-final-marker".to_string(), + ); + let ordinary_opts = ObjectOptions { + user_defined: durable_metadata, + ..Default::default() + }; + let (upload_id, parts) = + stage_upload_with_create_opts(&set_disks, bucket, "ordinary-object", b"ordinary multipart body", &ordinary_opts) + .await; + set_disks + .clone() + .complete_multipart_upload(bucket, "ordinary-object", &upload_id, parts, &ordinary_opts) + .await + .expect("durable object metadata must not claim upload ownership"); + let completed = set_disks + .get_object_info( + bucket, + "ordinary-object", + &ObjectOptions { + include_part_checksums: true, + ..Default::default() + }, + ) + .await + .expect("completed staging checksum metadata should remain readable"); + assert_eq!( + completed.parts[0] + .checksums + .as_ref() + .and_then(|checksums| checksums.get("CRC32C")) + .map(String::as_str), + Some("AAAAAA==") + ); + + let object = "owned-object"; + let mut metadata = HashMap::new(); + rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD, "true".to_string()); + rustfs_utils::http::insert_str( + &mut metadata, + rustfs_utils::http::SUFFIX_PART_CHECKSUMS, + r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string(), + ); + let create_opts = ObjectOptions { + data_movement: true, + user_defined: metadata, + ..Default::default() + }; + let upload = set_disks + .new_multipart_upload(bucket, object, &create_opts) + .await + .expect("data movement upload should be created"); + let upload_id = upload.upload_id; + let ordinary_upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id); + let data_movement_upload_path = SetDisks::get_multipart_upload_dir(bucket, object, &upload_id, true); + assert_eq!(data_movement_upload_path, format!("data-movement/{ordinary_upload_path}")); + assert!(matches!( + set_disks.check_upload_id_exists(bucket, object, &upload_id, false).await, + Err(StorageError::InvalidUploadID(..)) + )); + set_disks + .check_upload_id_exists_with_opts(bucket, object, &upload_id, false, &create_opts) + .await + .expect("data movement lookup should find the isolated upload"); + let mut part_reader = PutObjReader::from_vec(b"data movement multipart body".to_vec()); + let uploaded_part = set_disks + .put_object_part(bucket, object, &upload_id, 1, &mut part_reader, &create_opts) + .await + .expect("data movement part upload should retain ownership of its upload"); + let parts = vec![CompletePart { + part_num: uploaded_part.part_num, + etag: uploaded_part.etag, + ..Default::default() + }]; + + let listed = set_disks + .list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None) + .await + .expect("external multipart listing should succeed"); + assert!(!listed.uploads.iter().any(|upload| upload.upload_id == upload_id)); + + let get_err = set_disks + .get_multipart_info(bucket, object, &upload_id, &ObjectOptions::default()) + .await + .expect_err("external multipart metadata reads must not expose a data movement upload"); + assert!(matches!(get_err, StorageError::InvalidUploadID(..))); + + let list_parts_err = set_disks + .list_object_parts(bucket, object, &upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default()) + .await + .expect_err("external part listings must not expose a data movement upload"); + assert!(matches!(list_parts_err, StorageError::InvalidUploadID(..))); + + let mut external_part = PutObjReader::from_vec(b"external overwrite".to_vec()); + let put_err = set_disks + .put_object_part(bucket, object, &upload_id, 1, &mut external_part, &ObjectOptions::default()) + .await + .expect_err("external part uploads must not modify a data movement upload"); + assert!(matches!(put_err, StorageError::InvalidUploadID(..))); + + let abort_err = set_disks + .abort_multipart_upload(bucket, object, &upload_id, &ObjectOptions::default()) + .await + .expect_err("external aborts must not remove a data movement upload"); + assert!(matches!(abort_err, StorageError::InvalidUploadID(..))); + + let external_err = set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload_id, parts.clone(), &ObjectOptions::default()) + .await + .expect_err("external completion must not finalize a data movement upload"); + assert!(matches!(external_err, StorageError::InvalidUploadID(..))); + + let internal_parts = set_disks + .list_object_parts(bucket, object, &upload_id, None, MAX_PARTS_COUNT, &create_opts) + .await + .expect("data movement part listing should retain ownership of its upload"); + assert_eq!(internal_parts.parts.len(), 1); + + set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload_id, parts, &create_opts) + .await + .expect("data movement completion should retain ownership of its upload"); + let completed = set_disks + .get_object_info( + bucket, + object, + &ObjectOptions { + include_part_checksums: true, + ..Default::default() + }, + ) + .await + .expect("completed data movement object should be readable"); + assert!(!rustfs_utils::http::contains_key_str( + &completed.user_defined, + rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD + )); + for suffix in [ + rustfs_utils::http::SUFFIX_DATA_MOVED, + rustfs_utils::http::SUFFIX_PART_CHECKSUMS, + ] { + assert!( + completed + .user_defined + .contains_key(&rustfs_utils::http::internal_key_rustfs(suffix)) + ); + assert!( + completed + .user_defined + .contains_key(&format!("{}{suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX)) + ); + } + assert_eq!( + completed.parts[0] + .checksums + .as_ref() + .and_then(|checksums| checksums.get("CRC32C")) + .map(String::as_str), + Some("AAAAAA==") + ); + + let abort_object = "owned-abort-object"; + let abort_upload = set_disks + .new_multipart_upload(bucket, abort_object, &create_opts) + .await + .expect("data movement abort upload should be created"); + let abort_err = set_disks + .abort_multipart_upload(bucket, abort_object, &abort_upload.upload_id, &ObjectOptions::default()) + .await + .expect_err("external abort must not remove the second data movement upload"); + assert!(matches!(abort_err, StorageError::InvalidUploadID(..))); + set_disks + .abort_multipart_upload(bucket, abort_object, &abort_upload.upload_id, &create_opts) + .await + .expect("data movement abort should retain ownership of its upload"); + } + + #[tokio::test] + async fn stale_data_movement_replacement_fails_before_commit_on_outer_fence_loss() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "data-movement-stale-fence-bucket"; + make_bucket_on_all(&disk_stores, bucket).await; + let old_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND; + let new_time = old_time + time::Duration::SECOND; + + for (object, namespace_lock_fence, bucket_lifecycle_lock_fence) in [ + ("metadata-fence", Some(NamespaceLockFence::lost_for_test()), None), + ("bucket-fence", None, Some(NamespaceLockFence::lost_for_test())), + ] { + let version_id = Uuid::new_v4(); + let old_body = format!("old-{object}").into_bytes(); + let mut old_reader = PutObjReader::from_vec(old_body.clone()); + set_disks + .put_object( + bucket, + object, + &mut old_reader, + &ObjectOptions { + data_movement: true, + versioned: true, + version_id: Some(version_id.to_string()), + mod_time: Some(old_time), + ..Default::default() + }, + ) + .await + .expect("seed old data movement target"); + + let replacement_body = format!("new-{object}").into_bytes(); + let create_opts = ObjectOptions { + data_movement: true, + ..Default::default() + }; + let (upload_id, parts) = + stage_upload_with_create_opts(&set_disks, bucket, object, &replacement_body, &create_opts).await; + let complete_opts = ObjectOptions { + data_movement: true, + versioned: true, + version_id: Some(version_id.to_string()), + mod_time: Some(new_time), + http_preconditions: Some(crate::data_movement::data_movement_target_precondition()), + namespace_lock_fence, + bucket_lifecycle_lock_fence, + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), + ..Default::default() + }; + let err = set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload_id, parts, &complete_opts) + .await + .expect_err("a lost outer fence must abort stale target replacement"); + assert!(matches!(err, StorageError::NamespaceLockQuorumUnavailable { .. })); + + let mut preserved = set_disks + .get_object_reader( + bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + versioned: true, + version_id: Some(version_id.to_string()), + ..Default::default() + }, + ) + .await + .expect("read target after rejected replacement"); + let mut preserved_body = Vec::new(); + preserved + .stream + .read_to_end(&mut preserved_body) + .await + .expect("drain target after rejected replacement"); + assert_eq!(preserved_body, old_body); + set_disks + .check_upload_id_exists_with_opts(bucket, object, &upload_id, true, &complete_opts) + .await + .expect("fence loss must leave replacement staging retryable"); + } + } + + #[tokio::test] + #[serial] + async fn data_movement_complete_accepts_unknown_compressed_part_actual_size() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "data-movement-unknown-actual-size-bucket"; + let object = "object"; + make_bucket_on_all(&disk_stores, bucket).await; + + let mut metadata = HashMap::new(); + rustfs_utils::http::insert_str( + &mut metadata, + rustfs_utils::http::SUFFIX_COMPRESSION, + crate::io_support::rio::compression_metadata_value(rustfs_utils::CompressionAlgorithm::default()), + ); + rustfs_utils::http::insert_str( + &mut metadata, + rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD, + "source-generation".to_string(), + ); + let create_opts = ObjectOptions { + data_movement: true, + user_defined: metadata.clone(), + ..Default::default() + }; + let upload = set_disks + .new_multipart_upload(bucket, object, &create_opts) + .await + .expect("data movement upload should be created"); + + let mut completed_parts = Vec::new(); + for (number, actual_size) in [(1, -1), (2, 1)] { + let mut reader = PutObjReader::new( + HashReader::from_stream(Cursor::new(vec![number as u8]), 1, actual_size, None, None, false) + .expect("part reader should be constructed"), + ); + let part = set_disks + .put_object_part(bucket, object, &upload.upload_id, number, &mut reader, &create_opts) + .await + .expect("data movement part should be written"); + completed_parts.push(CompletePart { + part_num: number, + etag: part.etag, + ..Default::default() + }); + } + + rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "2".to_string()); + let completed = set_disks + .clone() + .complete_multipart_upload( + bucket, + object, + &upload.upload_id, + completed_parts, + &ObjectOptions { + data_movement: true, + user_defined: metadata, + ..Default::default() + }, + ) + .await + .expect("data movement completion should accept the persisted unknown-size sentinel"); + + assert_eq!(completed.parts[0].actual_size, -1); + assert_eq!(completed.get_actual_size().expect("completed object actual size"), 2); + } + async fn assert_complete_first_linearizes(bucket: &'static str, object: &'static str, create_opts: ObjectOptions) { let manager = Arc::new(rustfs_lock::GlobalLockManager::new()); let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager)))); diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index df11384e3..a38ec657b 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -3292,7 +3292,11 @@ impl SetDisks { // Force the full quorum fanout (allow_early_stop=false): `disks` is the // write target below, and an early-stop subset would only carry read // quorum, failing write quorum on update_object_meta (backlog#872). - let (fi, _, disks) = self.get_object_fileinfo_gated(bucket, object, opts, false, false).await?; + let mut read_opts = opts.clone(); + read_opts.include_part_checksums = true; + let (fi, _, disks) = self + .get_object_fileinfo_gated(bucket, object, &read_opts, false, false) + .await?; let mut fi = fi.into_owned(); fi.metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags.to_owned()); @@ -3301,6 +3305,7 @@ impl SetDisks { fi.metadata.insert(key.clone(), value.clone()); } } + fi.acknowledge_data_movement(); #[cfg(test)] pause_object_tagging_commit(bucket, object).await; @@ -3503,11 +3508,26 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } }; - fi.metadata = (*src_info.user_defined).clone(); - - if let Some(etag) = &src_info.etag { - fi.metadata.insert("etag".to_owned(), etag.clone()); + let preserved_part_checksums = if (src_info.metadata_only || src_info.version_only) + && rustfs_utils::http::contains_key_str(&fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS) + { + Self::hydrate_selected_fileinfo_part_checksums(&mut fi).map_err(|_| Error::FileCorrupt)?; + Some( + rustfs_utils::http::get_consistent_str(&fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS) + .ok_or(Error::FileCorrupt)? + .to_string(), + ) + } else { + None + }; + let mut replacement_metadata = (*src_info.user_defined).clone(); + if let Some(part_checksums) = preserved_part_checksums { + rustfs_utils::http::insert_str(&mut replacement_metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS, part_checksums); } + if let Some(etag) = &src_info.etag { + replacement_metadata.insert("etag".to_owned(), etag.clone()); + } + fi.metadata = replacement_metadata.clone(); let mod_time = OffsetDateTime::now_utc(); fi.mod_time = Some(mod_time); @@ -3538,10 +3558,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { for fi in metas.iter_mut() { if fi.has_valid_erasure_geometry() { - fi.metadata = (*src_info.user_defined).clone(); - if let Some(etag) = &src_info.etag { - fi.metadata.insert("etag".to_owned(), etag.clone()); - } + fi.metadata.clone_from(&replacement_metadata); fi.mod_time = Some(mod_time); fi.version_id = version_id; fi.versioned = src_opts.versioned || src_opts.version_suspended; @@ -4482,7 +4499,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } fi.version_id = if let Some(vid) = opts.version_id.as_ref() { - Some(Uuid::parse_str(vid.as_str())?) + let vid = Uuid::parse_str(vid.as_str())?; + (!opts.version_suspended || !vid.is_nil()).then_some(vid) + } else if opts.version_suspended { + None } else if opts.versioned { Some(Uuid::new_v4()) } else { @@ -4507,7 +4527,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { // Create a single object deletion request let mut dfi = FileInfo { name: object.to_string(), - version_id: opts.version_id.as_ref().and_then(|v| Uuid::parse_str(v).ok()), + version_id: opts + .version_id + .as_ref() + .and_then(|v| Uuid::parse_str(v).ok()) + .filter(|vid| !opts.version_suspended || !vid.is_nil()), mark_deleted: mark_delete, deleted: delete_marker, mod_time: Some(mod_time), @@ -4654,6 +4678,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } } + fi.acknowledge_data_movement(); + if opts.mod_time.is_some() { fi.mod_time = opts.mod_time; } @@ -4720,7 +4746,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { // _lock_guard = guard_opt; // } - let (fi, meta_arr, online_disks) = self.get_object_fileinfo(bucket, object, opts, true, false).await?; + let mut transition_read_opts = opts.clone(); + transition_read_opts.include_part_checksums = true; + let (fi, meta_arr, online_disks) = self + .get_object_fileinfo(bucket, object, &transition_read_opts, true, false) + .await?; let mut fi = fi.into_owned(); /*if err != nil { return Err(to_object_err(err, vec![bucket, object])); @@ -4778,6 +4808,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let transaction_id = transaction.transaction_id; let dest_obj = transaction.remote_object.clone(); let mut transition_meta = (*oi.user_defined).clone(); + rustfs_utils::http::remove_str(&mut transition_meta, rustfs_utils::http::SUFFIX_PART_CHECKSUMS); transition_meta.insert("name".to_string(), object.to_string()); rustfs_utils::http::metadata_compat::insert_str( &mut transition_meta, @@ -4938,6 +4969,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let mut commit_opts = opts.clone(); commit_opts.no_lock = true; commit_opts.metadata_cache_safe = false; + commit_opts.include_part_checksums = true; let transition_lock_guard = if opts.no_lock { None } else { @@ -5191,7 +5223,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { achieved: 0, }); } - let fi = self.clone().get_object_fileinfo(bucket, object, opts, true, false).await; + let mut restore_read_opts = opts.clone(); + restore_read_opts.include_part_checksums = true; + let fi = self + .clone() + .get_object_fileinfo(bucket, object, &restore_read_opts, true, false) + .await; drop(bucket_lifecycle_guard); if let Err(err) = fi { return set_restore_header_fn(&mut oi, Some(to_object_err(err, vec![bucket, object]))).await; @@ -5211,7 +5248,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { expected_operation_id.to_string(), ); } - let restore_commit_metadata = if let Some(expected_operation_id) = expected_operation_id { + let mut restore_commit_metadata = if let Some(expected_operation_id) = expected_operation_id { let mut metadata = HashMap::new(); metadata.insert(X_AMZ_RESTORE.as_str().to_string(), "ongoing-request=\"false\"".to_string()); rustfs_utils::http::metadata_compat::insert_str( @@ -5223,6 +5260,15 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } else { HashMap::new() }; + if let Some(part_checksums) = + rustfs_utils::http::get_consistent_str(&actual_fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS) + { + rustfs_utils::http::insert_str( + &mut restore_commit_metadata, + rustfs_utils::http::SUFFIX_PART_CHECKSUMS, + part_checksums.to_string(), + ); + } // The restore copy-back re-writes this same object via put_object / // new_multipart_upload / complete_multipart_upload, each of which takes // the object write lock in its commit phase. The caller @@ -6103,7 +6149,7 @@ mod get_object_downstream_close_accounting_tests { mod metadata_mutation_generation_tests { use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks; use super::*; - use crate::disk::DiskAPI as _; + use crate::disk::{DiskAPI as _, ReadOptions}; use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; async fn put_and_prime( @@ -6140,6 +6186,31 @@ mod metadata_mutation_generation_tests { ); } + async fn persist_part_checksum_sidecar(set_disks: &Arc, bucket: &str, object: &str, value: &str) { + let (fi, _, disks) = set_disks + .get_object_fileinfo( + bucket, + object, + &ObjectOptions { + no_lock: true, + metadata_cache_safe: false, + ..Default::default() + }, + true, + false, + ) + .await + .expect("object metadata should be readable before adding the checksum sidecar"); + let mut fi = fi.into_owned(); + let disks = disks.into_owned(); + rustfs_utils::http::insert_str(&mut fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS, value.to_string()); + set_disks + .update_object_meta(bucket, object, fi, &disks) + .await + .expect("checksum sidecar should be persisted"); + set_disks.invalidate_get_object_metadata_cache(bucket, object).await; + } + #[tokio::test] #[serial_test::serial(metadata_cache_invalidation_probe)] async fn metadata_semantic_mutation_generation_matrix_retires_cached_snapshot() { @@ -6226,6 +6297,112 @@ mod metadata_mutation_generation_tests { ); assert_retired(&set_disks, &metadata_key).await; } + + #[tokio::test] + async fn metadata_only_copy_preserves_valid_part_checksums_and_rejects_conflicting_aliases() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "metadata-copy-part-checksums-bucket"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let valid_object = "valid-sidecar"; + let (mut valid_source, _) = put_and_prime(&set_disks, bucket, valid_object, b"valid sidecar body").await; + persist_part_checksum_sidecar(&set_disks, bucket, valid_object, r#"[[1,[["CRC32C","AAAAAA=="]]]]"#).await; + valid_source.metadata_only = true; + Arc::make_mut(&mut valid_source.user_defined).insert("x-amz-meta-copy".to_string(), "updated".to_string()); + set_disks + .copy_object( + bucket, + valid_object, + bucket, + valid_object, + &mut valid_source, + &ObjectOptions::default(), + &ObjectOptions::default(), + ) + .await + .expect("metadata-only copy should preserve a valid checksum sidecar"); + let copied = set_disks + .get_object_info( + bucket, + valid_object, + &ObjectOptions { + include_part_checksums: true, + ..Default::default() + }, + ) + .await + .expect("copied object should retain readable part checksums"); + assert_eq!( + copied.parts[0] + .checksums + .as_ref() + .and_then(|checksums| checksums.get("CRC32C")) + .map(String::as_str), + Some("AAAAAA==") + ); + + let conflicting_object = "conflicting-sidecar"; + let (mut conflicting_source, _) = + put_and_prime(&set_disks, bucket, conflicting_object, b"conflicting sidecar body").await; + let (fi, _, disks) = set_disks + .get_object_fileinfo( + bucket, + conflicting_object, + &ObjectOptions { + no_lock: true, + metadata_cache_safe: false, + ..Default::default() + }, + true, + false, + ) + .await + .expect("conflicting object metadata should be readable before corruption is injected"); + let mut fi = fi.into_owned(); + let disks = disks.into_owned(); + let rustfs_key = format!( + "{}{}", + rustfs_utils::http::RUSTFS_INTERNAL_PREFIX, + rustfs_utils::http::SUFFIX_PART_CHECKSUMS + ); + let minio_key = format!( + "{}{}", + rustfs_utils::http::MINIO_INTERNAL_PREFIX, + rustfs_utils::http::SUFFIX_PART_CHECKSUMS + ); + fi.metadata + .insert(rustfs_key.clone(), r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string()); + fi.metadata + .insert(minio_key.clone(), r#"[[1,[["CRC32C","AQAAAA=="]]]]"#.to_string()); + set_disks + .update_object_meta(bucket, conflicting_object, fi, &disks) + .await + .expect("conflicting aliases should be persisted for the fail-closed regression"); + set_disks + .invalidate_get_object_metadata_cache(bucket, conflicting_object) + .await; + conflicting_source.metadata_only = true; + let err = set_disks + .copy_object( + bucket, + conflicting_object, + bucket, + conflicting_object, + &mut conflicting_source, + &ObjectOptions::default(), + &ObjectOptions::default(), + ) + .await + .expect_err("metadata-only copy must reject conflicting checksum aliases"); + assert!(matches!(err, Error::FileCorrupt)); + let raw_err = disk_stores[0] + .read_version("", bucket, conflicting_object, "", &ReadOptions::default()) + .await + .expect_err("the rejected copy must leave the conflicting persisted aliases fail-closed"); + assert!(matches!(raw_err, crate::disk::error::DiskError::FileCorrupt)); + } } #[cfg(all(test, feature = "test-util"))] @@ -6287,6 +6464,7 @@ mod transition_commit_failure_tests { } } + #[tokio::test] #[serial_test::serial(restore_multipart_failure_point)] async fn multipart_restore_aborts_every_post_create_failure() { let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; @@ -6333,6 +6511,32 @@ mod transition_commit_failure_tests { ) .await .expect("source multipart upload should complete"); + let (source_fi, _, online_disks) = set_disks + .get_object_fileinfo( + bucket, + object, + &ObjectOptions { + no_lock: true, + metadata_cache_safe: false, + ..Default::default() + }, + true, + false, + ) + .await + .expect("source metadata should be readable before adding the checksum sidecar"); + let mut source_fi = source_fi.into_owned(); + let online_disks = online_disks.into_owned(); + rustfs_utils::http::insert_str( + &mut source_fi.metadata, + rustfs_utils::http::SUFFIX_PART_CHECKSUMS, + r#"[[1,[["CRC32C","AAAAAA=="]]],[2,[["CRC32C","AQAAAA=="]]]]"#.to_string(), + ); + set_disks + .update_object_meta(bucket, object, source_fi, &online_disks) + .await + .expect("source checksum sidecar should be persisted before transition"); + set_disks.invalidate_get_object_metadata_cache(bucket, object).await; let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await; set_disks @@ -6416,9 +6620,32 @@ mod transition_commit_failure_tests { "successful multipart completion must disarm cleanup without aborting" ); let restored = set_disks - .get_object_info(bucket, object, &ObjectOptions::default()) + .get_object_info( + bucket, + object, + &ObjectOptions { + include_part_checksums: true, + ..Default::default() + }, + ) .await .expect("successful multipart restore must leave the committed object intact"); + assert_eq!( + restored.parts[0] + .checksums + .as_ref() + .and_then(|checksums| checksums.get("CRC32C")) + .map(String::as_str), + Some("AAAAAA==") + ); + assert_eq!( + restored.parts[1] + .checksums + .as_ref() + .and_then(|checksums| checksums.get("CRC32C")) + .map(String::as_str), + Some("AQAAAA==") + ); let restore_header = restored .user_defined .get(s3s::header::X_AMZ_RESTORE.as_str()) @@ -7900,6 +8127,88 @@ mod transition_upload_integrity_tests { .expect("source object should be written") } + #[tokio::test] + #[serial_test::serial] + async fn data_movement_tiered_metadata_is_create_only_under_object_lock() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "tiered-data-movement-create-only"; + let object = "object.bin"; + let version_id = Uuid::new_v4(); + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + let mut reader = PutObjReader::from_vec(b"existing target".to_vec()); + set_disks + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + versioned: true, + version_id: Some(version_id.to_string()), + ..Default::default() + }, + ) + .await + .expect("existing target should be written"); + let conflicting = FileInfo { + volume: bucket.to_string(), + name: object.to_string(), + version_id: Some(version_id), + mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND), + size: 12, + parts: vec![ObjectPartInfo { + number: 1, + size: 12, + actual_size: 12, + etag: "part-etag".to_string(), + ..Default::default() + }], + transition_status: TRANSITION_COMPLETE.to_string(), + transition_tier: "WARM".to_string(), + transitioned_objname: "remote/object-a".to_string(), + transition_version: Some("remote-version-a".to_string()), + transition_version_state: rustfs_filemeta::TransitionVersionState::Exact, + fresh: true, + ..Default::default() + }; + let err = set_disks + .decommission_tiered_object( + bucket, + object, + &conflicting, + &ObjectOptions { + versioned: true, + version_id: Some(version_id.to_string()), + mod_time: conflicting.mod_time, + data_movement: true, + http_preconditions: Some(crate::data_movement::data_movement_target_precondition()), + ..Default::default() + }, + ) + .await + .expect_err("data movement must not overwrite an existing tiered version"); + assert!(matches!(err, StorageError::PreconditionFailed)); + + let (stored, _, _) = set_disks + .get_object_fileinfo( + bucket, + object, + &ObjectOptions { + version_id: Some(version_id.to_string()), + no_lock: true, + metadata_cache_safe: false, + ..Default::default() + }, + true, + false, + ) + .await + .expect("the existing target should remain readable"); + assert_ne!(stored.transition_status, TRANSITION_COMPLETE); + assert!(stored.transition_version.is_none()); + } + fn transition_options(original: &ObjectInfo, tier_name: String) -> ObjectOptions { ObjectOptions { no_lock: true, @@ -7954,6 +8263,31 @@ mod transition_upload_integrity_tests { let object = "object.bin"; let payload = b"transition remote object must be bound to its transaction id".repeat(1024); let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await; + let (source_fi, _, online_disks) = set_disks + .get_object_fileinfo( + bucket, + object, + &ObjectOptions { + no_lock: true, + metadata_cache_safe: false, + ..Default::default() + }, + true, + false, + ) + .await + .expect("source metadata should be readable"); + let mut source_fi = source_fi.into_owned(); + let online_disks = online_disks.into_owned(); + rustfs_utils::http::insert_str( + &mut source_fi.metadata, + rustfs_utils::http::SUFFIX_PART_CHECKSUMS, + r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string(), + ); + set_disks + .update_object_meta(bucket, object, source_fi, &online_disks) + .await + .expect("source checksum sidecar should be persisted"); let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); let remote_version = Uuid::new_v4().to_string(); let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await; @@ -7978,6 +8312,7 @@ mod transition_upload_integrity_tests { &ObjectOptions { no_lock: true, metadata_cache_safe: false, + include_part_checksums: true, ..Default::default() }, true, @@ -7991,6 +8326,22 @@ mod transition_upload_integrity_tests { fi.transition_version_id, Some(Uuid::parse_str(&remote_version).expect("test version id should parse")) ); + assert_eq!( + fi.parts[0] + .checksums + .as_ref() + .and_then(|checksums| checksums.get("CRC32C")) + .map(String::as_str), + Some("AAAAAA==") + ); + let remote_metadata = backend + .metadata(remote_object) + .await + .expect("remote metadata should be stored"); + assert!( + !rustfs_utils::http::contains_key_str(&remote_metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS), + "the internal checksum sidecar must not be uploaded as remote user metadata" + ); assert!(backend.contains(remote_object).await, "committed remote object should remain available"); } @@ -8067,6 +8418,7 @@ mod transition_upload_integrity_tests { object, &expected, &[], + crate::data_movement::SourceCleanupBucketFence::default(), "test_data_movement", ) .await @@ -8091,6 +8443,72 @@ mod transition_upload_integrity_tests { assert_local_source_intact(&set_disks, bucket, object, &payload).await; } + #[tokio::test(flavor = "current_thread", start_paused = true)] + #[serial_test::serial] + async fn data_movement_cleanup_aborts_after_bucket_fence_loss() { + let refresh_calls = Arc::new(AtomicUsize::new(0)); + let lockers: Vec> = (0..4) + .map(|_| Arc::new(LockLostRefreshClient::new(Arc::clone(&refresh_calls))) as Arc) + .collect(); + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await; + let bucket = "data-movement-cleanup-bucket-fence-lost"; + let object = "object.bin"; + let payload = b"lost bucket fence must preserve the source".repeat(1024); + write_source(&set_disks, &disk_stores, bucket, object, &payload).await; + let expected = set_disks + .load_file_info_versions_exact(bucket, object) + .await + .expect("source versions should be readable") + .expect("source versions should exist"); + + let distributed_setup = SetupTypeGuard::switch_to(SetupType::DistErasure).await; + let bucket_guard = set_disks + .new_ns_lock(bucket, crate::storage_api_contracts::bucket::BUCKET_LIFECYCLE_LOCK_OBJECT) + .await + .expect("create bucket lifecycle lock") + .get_read_lock(get_lock_acquire_timeout()) + .await + .expect("acquire bucket lifecycle read lock"); + let local_cleanup_setup = SetupTypeGuard::switch_to(SetupType::Erasure).await; + let barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(bucket, object); + + let cleanup_set = Arc::clone(&set_disks); + let cleanup = tokio::spawn(async move { + let result = crate::data_movement::cleanup_source_entry_if_unchanged( + cleanup_set, + bucket, + object, + &expected, + &[], + crate::data_movement::SourceCleanupBucketFence { + expected_incarnation_id: None, + lifecycle_guard: Some(&bucket_guard), + }, + "test_data_movement", + ) + .await; + drop(bucket_guard); + result + }); + barrier.wait_until_paused().await; + tokio::time::advance(Duration::from_secs(11)).await; + tokio::task::yield_now().await; + assert!(refresh_calls.load(Ordering::SeqCst) > 0, "bucket guard refresh must run before commit"); + barrier.release(); + + let error = cleanup + .await + .expect("cleanup task should not panic") + .expect_err("cleanup must fail after its bucket lifecycle fence loses refresh quorum"); + assert!(matches!( + error, + crate::data_movement::SourceCleanupError::Storage(StorageError::NamespaceLockQuorumUnavailable { .. }) + )); + assert_local_source_intact(&set_disks, bucket, object, &payload).await; + drop(local_cleanup_setup); + drop(distributed_setup); + } + #[tokio::test] #[serial_test::serial] async fn partial_remote_acceptance_cleans_exact_candidate_and_preserves_source() { @@ -9925,6 +10343,33 @@ mod put_object_tags_early_stop_regression_tests { .await .expect("put_object should succeed"); + let (fi, _, disks) = set_disks + .get_object_fileinfo( + bucket, + object, + &ObjectOptions { + no_lock: true, + metadata_cache_safe: false, + ..Default::default() + }, + true, + false, + ) + .await + .expect("object metadata should be readable before adding the checksum sidecar"); + let mut fi = fi.into_owned(); + let disks = disks.into_owned(); + rustfs_utils::http::insert_str( + &mut fi.metadata, + rustfs_utils::http::SUFFIX_PART_CHECKSUMS, + r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string(), + ); + set_disks + .update_object_meta(bucket, object, fi, &disks) + .await + .expect("checksum sidecar should be persisted before tagging"); + set_disks.invalidate_get_object_metadata_cache(bucket, object).await; + let tags = "unit=backlog881&stage=regression"; set_disks .put_object_tags(bucket, object, tags, &ObjectOptions::default()) @@ -9944,6 +10389,11 @@ mod put_object_tags_early_stop_regression_tests { Some(tags), "disk {idx} must carry the tag written under early-stop (write set not shrunk to a read-quorum subset)" ); + assert_eq!( + rustfs_utils::http::get_consistent_str(&fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS), + Some(r#"[[1,[["CRC32C","AAAAAA=="]]]]"#), + "disk {idx} must retain the checksum sidecar across the tag metadata update" + ); } }, ) diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index d4a98ccd4..119d6599d 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -19,7 +19,7 @@ use crate::diagnostics::get::{ GET_METADATA_CACHE_REASON_DATA_MOVEMENT, GET_METADATA_CACHE_REASON_DELETE_MARKER, GET_METADATA_CACHE_REASON_DIST_ERASURE, GET_METADATA_CACHE_REASON_INCL_FREE_VERSIONS, GET_METADATA_CACHE_REASON_INSUFFICIENT_CACHED_QUORUM, GET_METADATA_CACHE_REASON_META_BUCKET, GET_METADATA_CACHE_REASON_NO_LOCK, GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED, - GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_NUMBER, + GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_CHECKSUMS, GET_METADATA_CACHE_REASON_PART_NUMBER, GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_STALE_PUBLICATION, GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID, GET_METADATA_CACHE_REASON_VERSION_SUSPENDED, GET_METADATA_CACHE_REASON_VERSIONED, GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, @@ -340,7 +340,7 @@ impl SetDisks { // read_all_fileinfo_observed (see read_all_fileinfo_early_stop in // core/io_primitives.rs); unsafe requests and callers that opt out // (allow_early_stop=false) fall back to full-wait. - let (parts_metadata, errs, metadata_fanout_diagnostics) = Self::read_all_fileinfo_observed( + let (mut parts_metadata, errs, metadata_fanout_diagnostics) = Self::read_all_fileinfo_observed( &disks, "", bucket, @@ -394,8 +394,17 @@ impl SetDisks { return Err(to_object_err(err.into(), vec![bucket, object])); } - let (op_online_disks, fi, fileinfo_selection_quorum) = + let (op_online_disks, mut fi, fileinfo_selection_quorum) = Self::select_valid_fileinfo(&disks, &parts_metadata, &errs, vid.as_str(), read_quorum, write_quorum)?; + let include_part_checksums = + opts.include_part_checksums || opts.part_number.is_some() || opts.data_movement || opts.raw_data_movement_read; + if include_part_checksums { + Self::hydrate_selected_fileinfo_part_checksums(&mut fi)?; + } else { + for metadata in std::iter::once(&mut fi).chain(parts_metadata.iter_mut()) { + rustfs_utils::http::remove_str(&mut metadata.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS); + } + } metadata_fanout_diagnostics.record_quorum_candidate_latency(metadata_metrics_path, fileinfo_selection_quorum); if errs.iter().any(|err| err.is_some()) { let version_id = resolved_read_repair_version_id(&fi, opts.version_id.as_deref()); @@ -1826,6 +1835,9 @@ fn get_object_metadata_cache_request_bypass_reason(bucket: &str, opts: &ObjectOp if opts.part_number.is_some() { return Some(GET_METADATA_CACHE_REASON_PART_NUMBER); } + if opts.include_part_checksums { + return Some(GET_METADATA_CACHE_REASON_PART_CHECKSUMS); + } if opts.data_movement { return Some(GET_METADATA_CACHE_REASON_DATA_MOVEMENT); } @@ -2497,6 +2509,16 @@ mod metadata_cache_tests { Some(GET_METADATA_CACHE_REASON_PART_NUMBER) ); + opts = ObjectOptions { + include_part_checksums: true, + ..Default::default() + }; + assert!(!is_get_object_metadata_cache_request_eligible("bucket", &opts, true)); + assert_eq!( + get_object_metadata_cache_request_bypass_reason("bucket", &opts, true), + Some(GET_METADATA_CACHE_REASON_PART_CHECKSUMS) + ); + opts = ObjectOptions { data_movement: true, ..Default::default() diff --git a/crates/ecstore/src/set_disk/replication.rs b/crates/ecstore/src/set_disk/replication.rs index e1f1d512a..523053f9d 100644 --- a/crates/ecstore/src/set_disk/replication.rs +++ b/crates/ecstore/src/set_disk/replication.rs @@ -75,6 +75,7 @@ impl SetDisks { version_id, versioned: opts.versioned, version_suspended: opts.version_suspended, + include_part_checksums: true, ..Default::default() }; let (fi, _, disks) = self @@ -142,6 +143,7 @@ impl SetDisks { version_id, versioned: opts.versioned, version_suspended: opts.version_suspended, + include_part_checksums: true, ..Default::default() }; let (fi, _, disks) = self diff --git a/crates/ecstore/src/store/bucket_fence.rs b/crates/ecstore/src/store/bucket_fence.rs index a705c4d5c..1800d53a3 100644 --- a/crates/ecstore/src/store/bucket_fence.rs +++ b/crates/ecstore/src/store/bucket_fence.rs @@ -161,6 +161,10 @@ impl BucketIncarnationFenceGuard { pub(crate) fn is_lock_lost(&self) -> bool { self.inner.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost) } + + pub(crate) fn namespace_lock_guard(&self) -> Option<&NamespaceLockGuard> { + self.inner.as_ref() + } } impl Drop for BucketIncarnationFenceGuard { diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index a979328e7..2d35b551b 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -597,9 +597,11 @@ mod tests { storage_api_contracts::list::ListOperations as _, }; use crate::{ + bucket::replication::{ReplicationState, ReplicationStatusType, replication_statuses_map}, core::pools::{POOL_META_VERSION, PoolDecommissionInfo, PoolMeta, PoolStatus}, disk::endpoint::Endpoint, error::{Error, Result, StorageError}, + io_support::rio::{WritePlan, compression_metadata_value}, layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}, object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}, services::rebalance::{RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats}, @@ -615,6 +617,11 @@ mod tests { use rustfs_filemeta::ObjectPartInfo; #[cfg(feature = "test-util")] use rustfs_protos::{TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase}; + use rustfs_rio::{Checksum, ChecksumType}; + use rustfs_utils::{ + CompressionAlgorithm, + http::{SUFFIX_COMPRESSION, insert_str}, + }; use std::{ collections::HashMap, future::Future, @@ -1407,143 +1414,1521 @@ mod tests { shutdown.cancel(); } + #[test] + #[serial_test::serial(storage_class_env)] + fn data_movement_conflicts_and_multipart_retries_converge_safely() { + // Run this large async scenario on a larger dedicated stack so debug + // test threads do not overflow before the regression assertions. + std::thread::Builder::new() + .stack_size(16 * 1024 * 1024) + .spawn(|| { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .worker_threads(2) + .build() + .expect("build data movement test runtime"); + runtime.block_on(async move { + let temp_dir = tempfile::tempdir().expect("create data movement store dir"); + let (_ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store( + temp_dir.path(), + "data-movement-conflict-convergence", + &[4, 4], + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("data-movement-conflict-{}", uuid::Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create data movement bucket"); + let source_mod_time = OffsetDateTime::UNIX_EPOCH; + let target_mod_time = source_mod_time + time::Duration::SECOND; + + let object = "single-object"; + let target_body = b"newer client body".to_vec(); + let mut target_reader = PutObjReader::from_vec(target_body.clone()); + store.pools[1] + .put_object( + &bucket, + object, + &mut target_reader, + &ObjectOptions { + mod_time: Some(target_mod_time), + ..Default::default() + }, + ) + .await + .expect("write newer single-part target"); + + let source_body = b"stale migration body".to_vec(); + crate::data_movement::migrate_object( + store.clone(), + 0, + bucket.clone(), + GetObjectReader { + stream: Box::new(Cursor::new(source_body.clone())), + object_info: ObjectInfo { + bucket: bucket.clone(), + name: object.to_string(), + size: i64::try_from(source_body.len()).expect("single source size should fit i64"), + actual_size: i64::try_from(source_body.len()).expect("single source size should fit i64"), + etag: Some("0123456789abcdef0123456789abcdef".to_string()), + mod_time: Some(source_mod_time), + ..Default::default() + }, + buffered_body: None, + body_source: Default::default(), + }, + None, + "test_data_movement", + ) + .await + .expect("newer single-part target should converge migration"); + + let mut reader = store.pools[1] + .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("read converged single-part target"); + let mut body = Vec::new(); + reader.stream.read_to_end(&mut body).await.expect("drain single-part target"); + assert_eq!(body, target_body); + + let multipart_object = "multipart-object"; + let multipart_target_body = b"newer multipart client body".to_vec(); + let mut multipart_target_reader = PutObjReader::from_vec(multipart_target_body.clone()); + store.pools[1] + .put_object( + &bucket, + multipart_object, + &mut multipart_target_reader, + &ObjectOptions { + mod_time: Some(target_mod_time), + ..Default::default() + }, + ) + .await + .expect("write newer multipart target"); + + let first_part_size = 5 * 1024 * 1024; + let mut multipart_source_body = vec![b'a'; first_part_size]; + multipart_source_body.push(b'b'); + let multipart_source_size = + i64::try_from(multipart_source_body.len()).expect("multipart source size should fit i64"); + crate::data_movement::migrate_object( + store.clone(), + 0, + bucket.clone(), + GetObjectReader { + stream: Box::new(Cursor::new(multipart_source_body)), + object_info: ObjectInfo { + bucket: bucket.clone(), + name: multipart_object.to_string(), + size: multipart_source_size, + actual_size: multipart_source_size, + etag: Some("source-multipart-etag-2".to_string()), + mod_time: Some(source_mod_time), + parts: Arc::new(vec![ + ObjectPartInfo { + number: 1, + size: first_part_size, + actual_size: i64::try_from(first_part_size).expect("first part size should fit i64"), + etag: "source-part-1".to_string(), + ..Default::default() + }, + ObjectPartInfo { + number: 2, + size: 1, + actual_size: 1, + etag: "source-part-2".to_string(), + ..Default::default() + }, + ]), + ..Default::default() + }, + buffered_body: None, + body_source: Default::default(), + }, + None, + "test_data_movement", + ) + .await + .expect("newer multipart target should converge migration"); + + let uploads = store.pools[1] + .list_multipart_uploads(&bucket, multipart_object, None, None, None, 100) + .await + .expect("list target pool multipart uploads"); + assert!(uploads.uploads.is_empty(), "superseded migration staging must be aborted"); + + let mut reader = store.pools[1] + .get_object_reader(&bucket, multipart_object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("read converged multipart target"); + let mut body = Vec::new(); + reader.stream.read_to_end(&mut body).await.expect("drain multipart target"); + assert_eq!(body, multipart_target_body); + + let retry_object = "multipart-retry-object"; + let retry_first_part_size = 5 * 1024 * 1024; + let retry_object_mod_time = + OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed retry timestamp should be valid"); + let mut retry_source_body = vec![b'c'; retry_first_part_size]; + retry_source_body.push(b'd'); + let retry_first_checksum = + Checksum::new_from_data(ChecksumType::CRC32C, &retry_source_body[..retry_first_part_size]) + .expect("first source part checksum should be created"); + let retry_second_checksum = + Checksum::new_from_data(ChecksumType::CRC32C, &retry_source_body[retry_first_part_size..]) + .expect("second source part checksum should be created"); + let mut retry_checksum_parts = retry_first_checksum.raw.clone(); + retry_checksum_parts.extend_from_slice(&retry_second_checksum.raw); + let mut retry_checksum_type = ChecksumType::CRC32C; + retry_checksum_type + .merge(ChecksumType::MULTIPART) + .merge(ChecksumType::INCLUDES_MULTIPART); + let retry_object_checksum = Checksum::new_from_data(retry_checksum_type, &retry_checksum_parts) + .expect("source multipart checksum should be created"); + let retry_object_checksum_bytes = retry_object_checksum.to_bytes(&retry_checksum_parts); + let mut retry_metadata = HashMap::from([("x-amz-meta-retry".to_string(), "stable".to_string())]); + insert_str( + &mut retry_metadata, + SUFFIX_COMPRESSION, + compression_metadata_value(CompressionAlgorithm::default()), + ); + let retry_upload = store.pools[0] + .new_multipart_upload( + &bucket, + retry_object, + &ObjectOptions { + user_defined: retry_metadata, + want_checksum: Some(retry_object_checksum.clone()), + ..Default::default() + }, + ) + .await + .expect("create source multipart upload"); + let mut retry_parts = Vec::with_capacity(2); + for (part_number, plaintext, checksum) in [ + (1, &retry_source_body[..retry_first_part_size], &retry_first_checksum), + (2, &retry_source_body[retry_first_part_size..], &retry_second_checksum), + ] { + let part_size = i64::try_from(plaintext.len()).expect("source part size should fit i64"); + let mut plaintext_reader = rustfs_rio::HashReader::from_stream( + Cursor::new(plaintext.to_vec()), + part_size, + part_size, + None, + None, + false, + ) + .expect("create source part plaintext reader"); + plaintext_reader + .add_non_trailing_checksum(Some(checksum.clone()), false) + .expect("set source part checksum"); + let compressed = WritePlan::new() + .with_compression(CompressionAlgorithm::default()) + .apply(plaintext_reader, part_size) + .expect("compress source part"); + let mut reader = PutObjReader::new(compressed); + let part = store.pools[0] + .put_object_part( + &bucket, + retry_object, + &retry_upload.upload_id, + part_number, + &mut reader, + &ObjectOptions::default(), + ) + .await + .expect("write source multipart part"); + retry_parts.push(crate::storage_api_contracts::multipart::CompletePart { + part_num: part.part_num, + etag: part.etag, + checksum_crc32c: Some(checksum.encoded.clone()), + ..Default::default() + }); + } + store.pools[0] + .clone() + .complete_multipart_upload( + &bucket, + retry_object, + &retry_upload.upload_id, + retry_parts, + &ObjectOptions { + mod_time: Some(retry_object_mod_time), + want_checksum: Some(retry_object_checksum), + ..Default::default() + }, + ) + .await + .expect("complete source multipart object"); + + *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0)); + assert!(store.is_pool_rebalancing(0).await, "pool 0 must be marked as an active rebalance source"); + + let mut retry_reader = store.pools[0] + .get_object_reader( + &bucket, + retry_object, + None, + HeaderMap::new(), + &ObjectOptions { + raw_data_movement_read: true, + ..Default::default() + }, + ) + .await + .expect("read source multipart object for first migration"); + let mut retry_source_info = retry_reader.object_info.clone(); + let mut retry_source_parts = retry_source_info.parts.as_ref().clone(); + for (part, checksum) in retry_source_parts + .iter_mut() + .zip([&retry_first_checksum, &retry_second_checksum]) + { + part.mod_time = Some(OffsetDateTime::UNIX_EPOCH); + part.checksums = Some(HashMap::from([(ChecksumType::CRC32C.to_string(), checksum.encoded.clone())])); + } + retry_source_info.parts = Arc::new(retry_source_parts); + retry_source_info.etag = Some("0123456789abcdef0123456789abcdef".to_string()); + assert!(!retry_source_info.is_multipart()); + assert!(retry_source_info.parts.iter().all(|part| part.checksums.is_some())); + assert_eq!(retry_source_info.checksum.as_deref(), Some(retry_object_checksum_bytes.as_ref())); + assert!( + !retry_source_info + .user_defined + .contains_key(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM) + ); + assert!( + !retry_source_info + .user_defined + .contains_key(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE) + ); + retry_reader.object_info = retry_source_info.clone(); + temp_env::async_with_vars( + [ + (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, None::<&str>), + (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, None::<&str>), + ], + crate::data_movement::migrate_object( + store.clone(), + 0, + bucket.clone(), + retry_reader, + None, + "test_data_movement_compatible_retry", + ), + ) + .await + .expect("default multipart migration should not require the checksum sidecar capability"); + + let compatible_target = store.pools[1] + .get_object_info( + &bucket, + retry_object, + &ObjectOptions { + include_part_checksums: true, + ..Default::default() + }, + ) + .await + .expect("read compatible multipart target metadata"); + assert_eq!(compatible_target.checksum, retry_source_info.checksum); + assert!(compatible_target.parts.iter().all(|part| part.checksums.is_none())); + assert_eq!( + rustfs_utils::http::get_consistent_str( + &compatible_target.user_defined, + rustfs_utils::http::SUFFIX_DATA_MOVED + ), + Some("true") + ); + assert!(!rustfs_utils::http::contains_key_str( + &compatible_target.user_defined, + rustfs_utils::http::SUFFIX_PART_CHECKSUMS + )); + let mut compatible_reader = store.pools[1] + .get_object_reader(&bucket, retry_object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("read compatible multipart target body"); + let mut compatible_body = Vec::new(); + compatible_reader + .stream + .read_to_end(&mut compatible_body) + .await + .expect("drain compatible multipart target body"); + assert_eq!(compatible_body, retry_source_body); + + let mut compatible_retry_reader = store.pools[0] + .get_object_reader( + &bucket, + retry_object, + None, + HeaderMap::new(), + &ObjectOptions { + raw_data_movement_read: true, + ..Default::default() + }, + ) + .await + .expect("read source multipart object for compatible retry"); + compatible_retry_reader.object_info = retry_source_info.clone(); + temp_env::async_with_vars( + [ + (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, Some("true")), + (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, Some("true")), + ], + crate::data_movement::migrate_object( + store.clone(), + 0, + bucket.clone(), + compatible_retry_reader, + None, + "test_data_movement_compatible_retry", + ), + ) + .await + .expect("pre-gate multipart target should converge after enabling checksum persistence"); + let compatible_target_after_retry = store.pools[1] + .get_object_info( + &bucket, + retry_object, + &ObjectOptions { + include_part_checksums: true, + ..Default::default() + }, + ) + .await + .expect("read pre-gate target after fleet-confirmed retry"); + assert_eq!(compatible_target_after_retry.data_dir, compatible_target.data_dir); + assert_eq!(compatible_target_after_retry.parts, compatible_target.parts); + let compatible_uploads = store.pools[1] + .list_multipart_uploads(&bucket, retry_object, None, None, None, 100) + .await + .expect("list compatible target multipart uploads"); + assert!(compatible_uploads.uploads.is_empty(), "compatible retry staging must be aborted"); + + store.pools[1] + .delete_object(&bucket, retry_object, ObjectOptions::default()) + .await + .expect("remove compatible target before fleet-confirmed migration"); + let mut retry_reader = store.pools[0] + .get_object_reader( + &bucket, + retry_object, + None, + HeaderMap::new(), + &ObjectOptions { + raw_data_movement_read: true, + ..Default::default() + }, + ) + .await + .expect("read source multipart object for fleet-confirmed migration"); + retry_reader.object_info = retry_source_info.clone(); + temp_env::async_with_vars( + [ + (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, Some("true")), + (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, Some("true")), + ], + crate::data_movement::migrate_object( + store.clone(), + 0, + bucket.clone(), + retry_reader, + None, + "test_data_movement_retry", + ), + ) + .await + .expect("first multipart migration should succeed"); + + let target_before_retry = store.pools[1] + .get_object_info( + &bucket, + retry_object, + &ObjectOptions { + include_part_checksums: true, + ..Default::default() + }, + ) + .await + .expect("read target metadata before retry"); + assert_eq!(target_before_retry.checksum, retry_source_info.checksum); + assert_eq!( + target_before_retry + .parts + .iter() + .map(|part| (&part.number, &part.checksums)) + .collect::>(), + retry_source_info + .parts + .iter() + .map(|part| (&part.number, &part.checksums)) + .collect::>() + ); + assert_eq!(retry_source_info.mod_time, target_before_retry.mod_time); + assert!( + retry_source_info.parts.iter().any(|source_part| { + target_before_retry + .parts + .iter() + .find(|target_part| target_part.number == source_part.number) + .is_some_and(|target_part| target_part.mod_time != source_part.mod_time) + }), + "multipart migration must rewrite at least one target part timestamp" + ); + let mut retry_reader = store.pools[0] + .get_object_reader( + &bucket, + retry_object, + None, + HeaderMap::new(), + &ObjectOptions { + raw_data_movement_read: true, + ..Default::default() + }, + ) + .await + .expect("read source multipart object for retry"); + retry_reader.object_info = retry_source_info; + temp_env::async_with_vars( + [ + (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, Some("true")), + (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, Some("true")), + ], + crate::data_movement::migrate_object( + store.clone(), + 0, + bucket.clone(), + retry_reader, + None, + "test_data_movement_retry", + ), + ) + .await + .expect("equivalent multipart migration retry should converge"); + + let target_after_retry = store.pools[1] + .get_object_info(&bucket, retry_object, &ObjectOptions::default()) + .await + .expect("read target metadata after retry"); + assert_eq!(target_after_retry.data_dir, target_before_retry.data_dir); + assert_eq!(target_after_retry.etag, target_before_retry.etag); + assert_eq!(target_after_retry.checksum, target_before_retry.checksum); + assert_eq!(target_after_retry.mod_time, target_before_retry.mod_time); + assert!(!rustfs_utils::http::contains_key_str( + &target_after_retry.user_defined, + rustfs_utils::http::SUFFIX_PART_CHECKSUMS + )); + assert!(target_after_retry.parts.iter().all(|part| part.checksums.is_none())); + let cached_target_after_retry = store.pools[1] + .get_object_info(&bucket, retry_object, &ObjectOptions::default()) + .await + .expect("read target metadata from the ordinary cache path"); + assert!(!rustfs_utils::http::contains_key_str( + &cached_target_after_retry.user_defined, + rustfs_utils::http::SUFFIX_PART_CHECKSUMS + )); + assert!(cached_target_after_retry.parts.iter().all(|part| part.checksums.is_none())); + let hydrated_target_after_retry = store.pools[1] + .get_object_info( + &bucket, + retry_object, + &ObjectOptions { + include_part_checksums: true, + ..Default::default() + }, + ) + .await + .expect("hydrate target metadata after ordinary cache reads"); + assert_eq!(hydrated_target_after_retry.user_defined, target_before_retry.user_defined); + assert_eq!(hydrated_target_after_retry.parts, target_before_retry.parts); + + let mut target_reader = store.pools[1] + .get_object_reader(&bucket, retry_object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("read target multipart body after retry"); + let mut target_body = Vec::new(); + target_reader + .stream + .read_to_end(&mut target_body) + .await + .expect("drain target multipart body after retry"); + assert_eq!(target_body, retry_source_body); + + let uploads = store.pools[1] + .list_multipart_uploads(&bucket, retry_object, None, None, None, 100) + .await + .expect("list retry target multipart uploads"); + assert!(uploads.uploads.is_empty(), "retry migration staging must be aborted"); + + Box::pin(async { + store.pools[0] + .delete_object(&bucket, retry_object, ObjectOptions::default()) + .await + .expect("remove first-hop source before second migration"); + *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 1)); + let second_hop_reader = store.pools[1] + .get_object_reader( + &bucket, + retry_object, + None, + HeaderMap::new(), + &ObjectOptions { + data_movement: true, + raw_data_movement_read: true, + ..Default::default() + }, + ) + .await + .expect("read first target as the second migration source"); + assert!( + second_hop_reader + .object_info + .parts + .iter() + .all(|part| part.checksums.is_some()), + "data-movement source reads must hydrate persisted part checksums" + ); + temp_env::async_with_vars( + [ + (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, Some("true")), + (rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, Some("true")), + ], + crate::data_movement::migrate_object( + store.clone(), + 1, + bucket.clone(), + second_hop_reader, + None, + "test_data_movement_second_hop", + ), + ) + .await + .expect("second multipart migration should preserve part checksums"); + + let second_hop_target = store.pools[0] + .get_object_info( + &bucket, + retry_object, + &ObjectOptions { + include_part_checksums: true, + ..Default::default() + }, + ) + .await + .expect("read second-hop target metadata"); + assert_eq!(second_hop_target.checksum, target_before_retry.checksum); + assert_eq!( + second_hop_target + .parts + .iter() + .map(|part| (&part.number, &part.checksums)) + .collect::>(), + target_before_retry + .parts + .iter() + .map(|part| (&part.number, &part.checksums)) + .collect::>() + ); + }) + .await; + }); + }) + .expect("spawn data movement test thread") + .join() + .expect("join data movement test thread"); + } + + #[test] + #[serial_test::serial(storage_class_env)] + fn data_movement_multipart_replaces_only_unlocked_owned_generation() { + std::thread::Builder::new() + .stack_size(16 * 1024 * 1024) + .spawn(|| { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .worker_threads(2) + .build() + .expect("build data movement replacement test runtime"); + runtime.block_on(async move { + let temp_dir = tempfile::tempdir().expect("create data movement replacement store dir"); + let (_ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store( + temp_dir.path(), + "data-movement-stale-replacement", + &[4, 4], + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("dm-stale-replacement-{}", uuid::Uuid::new_v4()); + store + .make_bucket( + &bucket, + &MakeBucketOptions { + lock_enabled: true, + ..Default::default() + }, + ) + .await + .expect("create data movement replacement bucket"); + *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0)); + + let version_id = uuid::Uuid::new_v4(); + let old_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND; + let new_time = old_time + time::Duration::SECOND; + let first_part_size = 5 * 1024 * 1024; + let mut old_body = vec![b'a'; first_part_size]; + old_body.push(b'b'); + let mut new_body = vec![b'c'; first_part_size]; + new_body.push(b'd'); + + let source_reader = |name: &str, body: Vec, mod_time, metadata: HashMap| { + let size = i64::try_from(body.len()).expect("source body size should fit i64"); + GetObjectReader { + stream: Box::new(Cursor::new(body)), + object_info: ObjectInfo { + bucket: bucket.clone(), + name: name.to_string(), + version_id: Some(version_id), + size, + actual_size: size, + etag: Some(format!("{name}-multipart-etag-2")), + mod_time: Some(mod_time), + user_defined: Arc::new(metadata), + parts: Arc::new(vec![ + ObjectPartInfo { + number: 1, + size: first_part_size, + actual_size: i64::try_from(first_part_size).expect("first part size should fit i64"), + etag: format!("{name}-part-1"), + ..Default::default() + }, + ObjectPartInfo { + number: 2, + size: 1, + actual_size: 1, + etag: format!("{name}-part-2"), + ..Default::default() + }, + ]), + ..Default::default() + }, + buffered_body: None, + body_source: Default::default(), + } + }; + + let replaceable = "replaceable.bin"; + crate::data_movement::migrate_object( + store.clone(), + 0, + bucket.clone(), + source_reader( + replaceable, + old_body.clone(), + old_time, + HashMap::from([("x-amz-meta-generation".to_string(), "old".to_string())]), + ), + None, + "test_stale_target_seed", + ) + .await + .expect("seed old migrated target"); + let seeded_target = store.pools[1] + .get_object_info( + &bucket, + replaceable, + &ObjectOptions { + versioned: true, + version_id: Some(version_id.to_string()), + ..Default::default() + }, + ) + .await + .expect("read old migrated target"); + let replacement_opts = ObjectOptions { + data_movement: true, + versioned: true, + version_id: Some(version_id.to_string()), + mod_time: Some(new_time), + http_preconditions: Some(crate::data_movement::data_movement_target_precondition()), + ..Default::default() + }; + assert!( + crate::data_movement::can_replace_stale_data_movement_target(&seeded_target, &replacement_opts), + "seeded target should be replaceable: {seeded_target:?}" + ); + crate::data_movement::migrate_object( + store.clone(), + 0, + bucket.clone(), + source_reader( + replaceable, + new_body.clone(), + new_time, + HashMap::from([("x-amz-meta-generation".to_string(), "new".to_string())]), + ), + None, + "test_stale_target_replace", + ) + .await + .expect("newer source generation should replace the old migrated target"); + + let replacement = store.pools[1] + .get_object_info( + &bucket, + replaceable, + &ObjectOptions { + versioned: true, + version_id: Some(version_id.to_string()), + ..Default::default() + }, + ) + .await + .expect("read replaced target metadata"); + assert_eq!(replacement.mod_time, Some(new_time)); + assert_eq!(replacement.user_defined.get("x-amz-meta-generation").map(String::as_str), Some("new")); + let mut replacement_reader = store.pools[1] + .get_object_reader( + &bucket, + replaceable, + None, + HeaderMap::new(), + &ObjectOptions { + versioned: true, + version_id: Some(version_id.to_string()), + ..Default::default() + }, + ) + .await + .expect("read replaced target body"); + let mut replacement_body = Vec::new(); + replacement_reader + .stream + .read_to_end(&mut replacement_body) + .await + .expect("drain replaced target body"); + assert_eq!(replacement_body, new_body); + + let client_target = "client-target.bin"; + let client_body = b"client-owned exact version".to_vec(); + let mut client_reader = PutObjReader::from_vec(client_body.clone()); + store.pools[1] + .put_object( + &bucket, + client_target, + &mut client_reader, + &ObjectOptions { + versioned: true, + version_id: Some(version_id.to_string()), + mod_time: Some(old_time), + user_defined: HashMap::from([("x-amz-meta-owner".to_string(), "client".to_string())]), + ..Default::default() + }, + ) + .await + .expect("seed client-owned exact version"); + let err = crate::data_movement::migrate_object( + store.clone(), + 0, + bucket.clone(), + source_reader(client_target, new_body.clone(), new_time, HashMap::new()), + None, + "test_client_target_reject", + ) + .await + .expect_err("data movement must not replace a client-owned exact version"); + assert!(err.to_string().contains("complete_multipart_upload"), "unexpected migration error: {err}"); + let mut preserved_reader = store.pools[1] + .get_object_reader( + &bucket, + client_target, + None, + HeaderMap::new(), + &ObjectOptions { + versioned: true, + version_id: Some(version_id.to_string()), + ..Default::default() + }, + ) + .await + .expect("read preserved client target"); + let mut preserved_body = Vec::new(); + preserved_reader + .stream + .read_to_end(&mut preserved_body) + .await + .expect("drain preserved client target"); + assert_eq!(preserved_body, client_body); + + let acknowledged_target = "acknowledged-target.bin"; + crate::data_movement::migrate_object( + store.clone(), + 0, + bucket.clone(), + source_reader( + acknowledged_target, + old_body.clone(), + old_time, + HashMap::from([("x-amz-meta-generation".to_string(), "old".to_string())]), + ), + None, + "test_acknowledged_target_seed", + ) + .await + .expect("seed migrated target before tag acknowledgement"); + let target_version_opts = ObjectOptions { + versioned: true, + version_id: Some(version_id.to_string()), + ..Default::default() + }; + store + .put_object_tags(&bucket, acknowledged_target, "acknowledged=true", &target_version_opts) + .await + .expect("tag update should acknowledge the migrated target"); + let acknowledged = store.pools[1] + .get_object_info(&bucket, acknowledged_target, &target_version_opts) + .await + .expect("read acknowledged migrated target"); + let rustfs_data_moved_key = rustfs_utils::http::internal_key_rustfs(rustfs_utils::http::SUFFIX_DATA_MOVED); + let minio_data_moved_key = + format!("{}{}", rustfs_utils::http::MINIO_INTERNAL_PREFIX, rustfs_utils::http::SUFFIX_DATA_MOVED); + assert_eq!(acknowledged.user_defined.get(&rustfs_data_moved_key).map(String::as_str), Some("")); + assert_eq!(acknowledged.user_defined.get(&minio_data_moved_key).map(String::as_str), Some("")); + assert_eq!( + rustfs_utils::http::get_consistent_str(&acknowledged.user_defined, rustfs_utils::http::SUFFIX_DATA_MOVED), + None + ); + + let err = crate::data_movement::migrate_object( + store.clone(), + 0, + bucket.clone(), + source_reader( + acknowledged_target, + new_body.clone(), + new_time, + HashMap::from([("x-amz-meta-generation".to_string(), "new".to_string())]), + ), + None, + "test_acknowledged_target_reject", + ) + .await + .expect_err("data movement must not replace a target acknowledged by a tag update"); + assert!(err.to_string().contains("complete_multipart_upload"), "unexpected migration error: {err}"); + + let acknowledged_tags = store.pools[1] + .get_object_tags(&bucket, acknowledged_target, &target_version_opts) + .await + .expect("read preserved acknowledged target tags"); + assert_eq!(acknowledged_tags, "acknowledged=true"); + let mut acknowledged_reader = store.pools[1] + .get_object_reader(&bucket, acknowledged_target, None, HeaderMap::new(), &target_version_opts) + .await + .expect("read preserved acknowledged target body"); + let mut acknowledged_body = Vec::new(); + acknowledged_reader + .stream + .read_to_end(&mut acknowledged_body) + .await + .expect("drain preserved acknowledged target body"); + assert_eq!(acknowledged_body, old_body); + + let metadata_acknowledged_target = "metadata-acknowledged-target.bin"; + crate::data_movement::migrate_object( + store.clone(), + 0, + bucket.clone(), + source_reader( + metadata_acknowledged_target, + old_body.clone(), + old_time, + HashMap::from([("x-amz-meta-generation".to_string(), "old".to_string())]), + ), + None, + "test_metadata_acknowledged_target_seed", + ) + .await + .expect("seed migrated target before metadata acknowledgement"); + store + .put_object_metadata( + &bucket, + metadata_acknowledged_target, + &ObjectOptions { + eval_metadata: Some(HashMap::from([( + s3s::header::X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), + s3s::dto::ObjectLockLegalHoldStatus::OFF.to_string(), + )])), + ..target_version_opts.clone() + }, + ) + .await + .expect("legal hold metadata update should acknowledge the migrated target"); + let metadata_acknowledged = store.pools[1] + .get_object_info(&bucket, metadata_acknowledged_target, &target_version_opts) + .await + .expect("read metadata-acknowledged migrated target"); + assert_eq!( + metadata_acknowledged + .user_defined + .get(&rustfs_data_moved_key) + .map(String::as_str), + Some("") + ); + assert_eq!( + metadata_acknowledged + .user_defined + .get(&minio_data_moved_key) + .map(String::as_str), + Some("") + ); + assert_eq!( + metadata_acknowledged + .user_defined + .get(s3s::header::X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()) + .map(String::as_str), + Some("OFF") + ); + let err = crate::data_movement::migrate_object( + store.clone(), + 0, + bucket.clone(), + source_reader( + metadata_acknowledged_target, + new_body.clone(), + new_time, + HashMap::from([("x-amz-meta-generation".to_string(), "new".to_string())]), + ), + None, + "test_metadata_acknowledged_target_reject", + ) + .await + .expect_err("data movement must not replace a target acknowledged by a metadata update"); + assert!(err.to_string().contains("complete_multipart_upload"), "unexpected migration error: {err}"); + + let retain_until = (OffsetDateTime::now_utc() + time::Duration::days(1)) + .format(&time::format_description::well_known::Rfc3339) + .expect("retain-until date should format"); + for (object, mode) in [ + ("compliance-target.bin", s3s::dto::ObjectLockRetentionMode::COMPLIANCE), + ("governance-target.bin", s3s::dto::ObjectLockRetentionMode::GOVERNANCE), + ] { + let retained_metadata = HashMap::from([ + (s3s::header::X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(), mode.to_string()), + ( + s3s::header::X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(), + retain_until.clone(), + ), + ]); + crate::data_movement::migrate_object( + store.clone(), + 0, + bucket.clone(), + source_reader(object, old_body.clone(), old_time, retained_metadata.clone()), + None, + "test_retained_target_seed", + ) + .await + .expect("seed retained migrated target"); + let err = crate::data_movement::migrate_object( + store.clone(), + 0, + bucket.clone(), + source_reader(object, new_body.clone(), new_time, retained_metadata), + None, + "test_retained_target_replace", + ) + .await + .expect_err("active retention must block stale target replacement"); + assert!(err.to_string().contains("complete_multipart_upload"), "unexpected retention error: {err}"); + + let mut retained_reader = store.pools[1] + .get_object_reader( + &bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + versioned: true, + version_id: Some(version_id.to_string()), + ..Default::default() + }, + ) + .await + .expect("read retained target"); + let mut retained_body = Vec::new(); + retained_reader + .stream + .read_to_end(&mut retained_body) + .await + .expect("drain retained target"); + assert_eq!(retained_body, old_body); + } + + for object in [ + replaceable, + client_target, + acknowledged_target, + metadata_acknowledged_target, + "compliance-target.bin", + "governance-target.bin", + ] { + let uploads = store.pools[1] + .list_multipart_uploads(&bucket, object, None, None, None, 100) + .await + .expect("list target multipart staging"); + assert!(uploads.uploads.is_empty(), "data movement staging must be cleaned for {object}"); + } + }); + }) + .expect("spawn data movement replacement test thread") + .join() + .expect("join data movement replacement test thread"); + } + #[tokio::test] #[serial_test::serial(storage_class_env)] - async fn data_movement_conflicts_preserve_newer_target_and_abort_staging() { - let temp_dir = tempfile::tempdir().expect("create data movement store dir"); + async fn data_movement_delete_marker_retries_converge_safely() { + let temp_dir = tempfile::tempdir().expect("create delete-marker data movement store dir"); let (_ctx, store, _shutdown) = - without_storage_class_env(build_isolated_test_store(temp_dir.path(), "data-movement-conflict-convergence", &[4, 4])) - .await; + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "data-movement-delete-marker", &[4, 4])).await; crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; - let bucket = format!("data-movement-conflict-{}", uuid::Uuid::new_v4()); + let bucket = format!("dm-delete-{}", uuid::Uuid::new_v4()); store .make_bucket(&bucket, &MakeBucketOptions::default()) .await - .expect("create data movement bucket"); - let source_mod_time = OffsetDateTime::UNIX_EPOCH; - let target_mod_time = source_mod_time + time::Duration::SECOND; - - let object = "single-object"; - let target_body = b"newer client body".to_vec(); - let mut target_reader = PutObjReader::from_vec(target_body.clone()); - store.pools[1] - .put_object( + .expect("create delete-marker data movement bucket"); + let object = "delete-marker-retry"; + let version = uuid::Uuid::new_v4(); + let mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND; + let replica_timestamp = mod_time + time::Duration::SECOND; + let replication_timestamp = mod_time; + let replica_timestamp_string = replica_timestamp + .format(&time::format_description::well_known::Rfc3339) + .expect("replica timestamp should format as RFC3339"); + let replication_timestamp_string = replication_timestamp + .format(&time::format_description::well_known::Rfc3339) + .expect("replication timestamp should format as RFC3339"); + let replication_status = "arn:minio:replication::TenantA:bucket=COMPLETED;"; + store.pools[0] + .delete_object( &bucket, object, - &mut target_reader, - &ObjectOptions { - mod_time: Some(target_mod_time), + ObjectOptions { + versioned: true, + version_id: Some(version.to_string()), + delete_marker: true, + mod_time: Some(mod_time), + delete_replication: Some(ReplicationState { + replica_status: ReplicationStatusType::Replica, + replica_timestamp: Some(replica_timestamp), + replication_status_internal: Some(replication_status.to_string()), + replication_timestamp: Some(replication_timestamp), + targets: replication_statuses_map(replication_status), + delete_marker: true, + ..Default::default() + }), ..Default::default() }, ) .await - .expect("write newer single-part target"); - - let source_body = b"stale migration body".to_vec(); - crate::data_movement::migrate_object( - store.clone(), - 0, - bucket.clone(), - GetObjectReader { - stream: Box::new(Cursor::new(source_body.clone())), - object_info: ObjectInfo { - bucket: bucket.clone(), - name: object.to_string(), - size: i64::try_from(source_body.len()).expect("single source size should fit i64"), - actual_size: i64::try_from(source_body.len()).expect("single source size should fit i64"), - etag: Some("0123456789abcdef0123456789abcdef".to_string()), - mod_time: Some(source_mod_time), + .expect("write source delete marker"); + let source = store.pools[0] + .get_object_info( + &bucket, + object, + &ObjectOptions { + versioned: true, + version_id: Some(version.to_string()), ..Default::default() }, - buffered_body: None, - body_source: Default::default(), - }, - "test_data_movement", - ) - .await - .expect("newer single-part target should converge migration"); - - let mut reader = store - .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + ) .await - .expect("read converged single-part target"); - let mut body = Vec::new(); - reader.stream.read_to_end(&mut body).await.expect("drain single-part target"); - assert_eq!(body, target_body); + .expect("read source delete marker replication metadata"); + assert_eq!( + rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICA_STATUS).as_deref(), + Some("REPLICA"), + "source metadata: {:?}", + source.user_defined + ); - let multipart_object = "multipart-object"; - let multipart_target_body = b"newer multipart client body".to_vec(); - let mut multipart_target_reader = PutObjReader::from_vec(multipart_target_body.clone()); - store.pools[1] + *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0)); + let missing_object = "missing-delete-marker"; + let missing_version = uuid::Uuid::new_v4(); + let missing_opts = ObjectOptions { + versioned: true, + version_id: Some(missing_version.to_string()), + delete_marker: true, + mod_time: Some(mod_time), + src_pool_idx: 0, + data_movement: true, + skip_decommissioned: true, + ..Default::default() + }; + let missing_source_err = store + .delete_object(&bucket, missing_object, missing_opts.clone()) + .await + .expect_err("data movement must not recreate a delete marker missing from the source"); + assert!(matches!(missing_source_err, StorageError::DataMovementOverwriteErr(_, _, _))); + let missing_target_err = store.pools[1] + .get_object_info(&bucket, missing_object, &missing_opts) + .await + .expect_err("source-missing preflight must not write a target delete marker"); + assert!(matches!( + missing_target_err, + StorageError::ObjectNotFound(_, _) | StorageError::VersionNotFound(_, _, _) + )); + + let movement_opts = ObjectOptions { + versioned: true, + version_id: Some(version.to_string()), + delete_marker: true, + mod_time: Some(mod_time), + src_pool_idx: 0, + data_movement: true, + skip_decommissioned: true, + ..Default::default() + }; + for attempt in 1..=2 { + let moved = store + .delete_object(&bucket, object, movement_opts.clone()) + .await + .unwrap_or_else(|err| panic!("delete marker migration attempt {attempt} should converge: {err}")); + assert!(moved.delete_marker); + assert_eq!(moved.version_id, Some(version)); + } + let target = store.pools[1] + .get_object_info(&bucket, object, &movement_opts) + .await + .expect("retry must retain the target delete marker"); + assert!(target.delete_marker); + assert_eq!(target.mod_time, Some(mod_time)); + assert_eq!( + rustfs_utils::http::get_str(&target.user_defined, rustfs_utils::http::SUFFIX_REPLICA_STATUS).as_deref(), + Some("REPLICA") + ); + assert_eq!( + rustfs_utils::http::get_str(&target.user_defined, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP).as_deref(), + Some(replica_timestamp_string.as_str()) + ); + assert_eq!( + rustfs_utils::http::get_str(&target.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_STATUS).as_deref(), + Some(replication_status) + ); + assert_eq!( + rustfs_utils::http::get_str(&target.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP).as_deref(), + Some(replication_timestamp_string.as_str()) + ); + + let null_object = "null-delete-marker-retry"; + let null_mod_time = mod_time + time::Duration::seconds(3); + let null_marker = store.pools[0] + .delete_object( + &bucket, + null_object, + ObjectOptions { + version_suspended: true, + version_id: Some(uuid::Uuid::nil().to_string()), + data_movement: true, + delete_marker: true, + mod_time: Some(null_mod_time), + ..Default::default() + }, + ) + .await + .expect("write source null delete marker"); + assert_eq!(null_marker.version_id, Some(uuid::Uuid::nil())); + let null_opts = ObjectOptions { + version_suspended: true, + version_id: Some(uuid::Uuid::nil().to_string()), + delete_marker: true, + mod_time: Some(null_mod_time), + src_pool_idx: 0, + data_movement: true, + skip_decommissioned: true, + ..Default::default() + }; + for attempt in 1..=2 { + let moved = store + .delete_object(&bucket, null_object, null_opts.clone()) + .await + .unwrap_or_else(|err| panic!("null delete marker migration attempt {attempt} should converge: {err}")); + assert!(moved.delete_marker); + assert_eq!(moved.version_id, Some(uuid::Uuid::nil())); + } + let target_null_marker = store.pools[1] + .get_object_info(&bucket, null_object, &null_opts) + .await + .expect("target must retain the null delete marker identity"); + assert!(target_null_marker.delete_marker); + assert_eq!(target_null_marker.version_id, Some(uuid::Uuid::nil())); + + let explicit_object = "suspended-explicit-version-delete"; + let explicit_version = uuid::Uuid::new_v4(); + let mut explicit_reader = PutObjReader::from_vec(b"explicit version".to_vec()); + store.pools[0] .put_object( &bucket, - multipart_object, - &mut multipart_target_reader, + explicit_object, + &mut explicit_reader, &ObjectOptions { - mod_time: Some(target_mod_time), + versioned: true, + version_id: Some(explicit_version.to_string()), ..Default::default() }, ) .await - .expect("write newer multipart target"); - - let first_part_size = 5 * 1024 * 1024; - let mut multipart_source_body = vec![b'a'; first_part_size]; - multipart_source_body.push(b'b'); - let multipart_source_size = i64::try_from(multipart_source_body.len()).expect("multipart source size should fit i64"); - crate::data_movement::migrate_object( - store.clone(), - 0, - bucket.clone(), - GetObjectReader { - stream: Box::new(Cursor::new(multipart_source_body)), - object_info: ObjectInfo { - bucket: bucket.clone(), - name: multipart_object.to_string(), - size: multipart_source_size, - actual_size: multipart_source_size, - etag: Some("source-multipart-etag-2".to_string()), - mod_time: Some(source_mod_time), - parts: Arc::new(vec![ - ObjectPartInfo { - number: 1, - size: first_part_size, - actual_size: i64::try_from(first_part_size).expect("first part size should fit i64"), - etag: "source-part-1".to_string(), - ..Default::default() - }, - ObjectPartInfo { - number: 2, - size: 1, - actual_size: 1, - etag: "source-part-2".to_string(), - ..Default::default() - }, - ]), + .expect("write explicit version before suspended delete"); + store.pools[0] + .delete_object( + &bucket, + explicit_object, + ObjectOptions { + version_suspended: true, + version_id: Some(explicit_version.to_string()), ..Default::default() }, - buffered_body: None, - body_source: Default::default(), - }, - "test_data_movement", - ) - .await - .expect("newer multipart target should converge migration"); - - let uploads = store.pools[1] - .list_multipart_uploads(&bucket, multipart_object, None, None, None, 100) + ) .await - .expect("list target pool multipart uploads"); - assert!(uploads.uploads.is_empty(), "superseded migration staging must be aborted"); - - let mut reader = store - .get_object_reader(&bucket, multipart_object, None, HeaderMap::new(), &ObjectOptions::default()) + .expect("delete explicit version from suspended bucket"); + store.pools[0] + .get_object_info( + &bucket, + explicit_object, + &ObjectOptions { + versioned: true, + version_id: Some(explicit_version.to_string()), + ..Default::default() + }, + ) .await - .expect("read converged multipart target"); - let mut body = Vec::new(); - reader.stream.read_to_end(&mut body).await.expect("drain multipart target"); - assert_eq!(body, multipart_target_body); + .expect_err("suspended delete must remove the requested UUID version"); + } + + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn data_movement_put_conflict_validates_only_selected_target_pool() { + let temp_dir = tempfile::tempdir().expect("create three-pool data movement store dir"); + let (_ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "data-movement-selected-target", &[4, 4, 4])) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("dm-selected-target-{}", uuid::Uuid::new_v4()); + let object = "object.bin"; + let source_body = b"source-and-equivalent".to_vec(); + let conflicting_body = b"newer-conflicting-target".to_vec(); + let source_mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND; + let version = uuid::Uuid::new_v4(); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create selected-target bucket"); + for (pool_idx, body, mod_time) in [ + (0, source_body.clone(), source_mod_time), + (1, conflicting_body.clone(), source_mod_time + time::Duration::SECOND), + (2, source_body.clone(), source_mod_time), + ] { + let mut reader = PutObjReader::from_vec(body); + store.pools[pool_idx] + .put_object( + &bucket, + object, + &mut reader, + &ObjectOptions { + versioned: true, + version_id: Some(version.to_string()), + mod_time: Some(mod_time), + ..Default::default() + }, + ) + .await + .expect("seed data movement pool"); + } + *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0)); + + let source_reader = store.pools[0] + .get_object_reader( + &bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + versioned: true, + version_id: Some(version.to_string()), + raw_data_movement_read: true, + ..Default::default() + }, + ) + .await + .expect("read source for selected-target migration"); + crate::data_movement::migrate_object(store.clone(), 0, bucket.clone(), source_reader, None, "test_selected_target") + .await + .expect_err("an equivalent object in another pool must not mask the selected target conflict"); + + let mut selected_target = store.pools[1] + .get_object_reader( + &bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + versioned: true, + version_id: Some(version.to_string()), + ..Default::default() + }, + ) + .await + .expect("read selected conflicting target"); + let mut selected_body = Vec::new(); + selected_target + .stream + .read_to_end(&mut selected_body) + .await + .expect("drain selected conflicting target"); + assert_eq!(selected_body, conflicting_body); + } + + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn data_movement_multipart_conflict_validates_exact_version_target_pool() { + let temp_dir = tempfile::tempdir().expect("create three-pool multipart data movement store dir"); + let (_ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "data-movement-multipart-target", &[4, 4, 4])) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("dm-multipart-target-{}", uuid::Uuid::new_v4()); + let object = "object.bin"; + let source_version = uuid::Uuid::new_v4(); + let other_version = uuid::Uuid::new_v4(); + let source_mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create multipart selected-target bucket"); + for (pool_idx, version_id, body, mod_time) in [ + (0, source_version, b"source version".to_vec(), source_mod_time), + ( + 1, + other_version, + b"newer different version".to_vec(), + source_mod_time + time::Duration::seconds(2), + ), + (2, source_version, b"conflicting exact version".to_vec(), source_mod_time), + ] { + let mut reader = PutObjReader::from_vec(body); + store.pools[pool_idx] + .put_object( + &bucket, + object, + &mut reader, + &ObjectOptions { + versioned: true, + version_id: Some(version_id.to_string()), + mod_time: Some(mod_time), + ..Default::default() + }, + ) + .await + .expect("seed multipart data movement pool"); + } + *store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0)); + + let first_part_size = 5 * 1024 * 1024; + let mut source_body = vec![b'a'; first_part_size]; + source_body.push(b'b'); + let source_size = i64::try_from(source_body.len()).expect("multipart source size should fit i64"); + let completion_barrier = crate::store::multipart::DataMovementMultipartCompletionBarrier::install(&bucket); + let migration_store = store.clone(); + let migration_bucket = bucket.clone(); + let migration = tokio::spawn(async move { + let object_bucket = migration_bucket.clone(); + crate::data_movement::migrate_object( + migration_store, + 0, + migration_bucket, + GetObjectReader { + stream: Box::new(Cursor::new(source_body)), + object_info: ObjectInfo { + bucket: object_bucket, + name: object.to_string(), + version_id: Some(source_version), + size: source_size, + actual_size: source_size, + etag: Some("source-multipart-etag-2".to_string()), + mod_time: Some(source_mod_time), + parts: Arc::new(vec![ + ObjectPartInfo { + number: 1, + size: first_part_size, + actual_size: i64::try_from(first_part_size).expect("first part size should fit i64"), + etag: "source-part-1".to_string(), + ..Default::default() + }, + ObjectPartInfo { + number: 2, + size: 1, + actual_size: 1, + etag: "source-part-2".to_string(), + ..Default::default() + }, + ]), + ..Default::default() + }, + buffered_body: None, + body_source: Default::default(), + }, + None, + "test_multipart_selected_target", + ) + .await + }); + completion_barrier.wait_until_paused().await; + let unrelated_set = store.pools[1].get_disks_by_key(object); + let original_disks = { + let mut disks = unrelated_set.disks.write().await; + let original = disks.clone(); + for disk in disks.iter_mut().take(3) { + *disk = None; + } + original + }; + drop(completion_barrier); + let err = migration + .await + .expect("multipart migration task should join") + .expect_err("the exact-version target conflict must not be bypassed by a newer different version"); + *unrelated_set.disks.write().await = original_disks; + let rendered = err.to_string(); + assert!( + rendered.contains("complete_multipart_upload"), + "the selected target should be reached despite an unrelated degraded pool: {rendered}" + ); + assert!( + !rendered.contains("put_object_part failed"), + "part upload must not scan the unrelated degraded pool: {rendered}" + ); + + let unexpected_version = store.pools[1] + .get_object_info( + &bucket, + object, + &ObjectOptions { + versioned: true, + version_id: Some(source_version.to_string()), + ..Default::default() + }, + ) + .await + .expect_err("multipart migration must not duplicate the source version into the latest-version pool"); + assert!(matches!( + unexpected_version, + StorageError::ObjectNotFound(_, _) | StorageError::VersionNotFound(_, _, _) + )); } #[cfg(feature = "test-util")] diff --git a/crates/ecstore/src/store/multipart.rs b/crates/ecstore/src/store/multipart.rs index d04883795..3d460d4d5 100644 --- a/crates/ecstore/src/store/multipart.rs +++ b/crates/ecstore/src/store/multipart.rs @@ -66,6 +66,76 @@ fn ensure_multipart_bucket_lifecycle_guard_held( Ok(()) } +#[cfg(test)] +struct DataMovementMultipartCompletionBarrierState { + bucket: String, + arrived: tokio::sync::Notify, + release: tokio::sync::Notify, +} + +#[cfg(test)] +pub(crate) struct DataMovementMultipartCompletionBarrier { + state: Arc, +} + +#[cfg(test)] +static DATA_MOVEMENT_MULTIPART_COMPLETION_BARRIER: std::sync::OnceLock< + std::sync::Mutex>>, +> = std::sync::OnceLock::new(); + +#[cfg(test)] +impl DataMovementMultipartCompletionBarrier { + pub(crate) fn install(bucket: &str) -> Self { + let state = Arc::new(DataMovementMultipartCompletionBarrierState { + bucket: bucket.to_string(), + arrived: tokio::sync::Notify::new(), + release: tokio::sync::Notify::new(), + }); + let mut slot = DATA_MOVEMENT_MULTIPART_COMPLETION_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("data movement multipart completion barrier mutex should not poison"); + assert!(slot.is_none(), "data movement multipart completion barrier must be unique"); + *slot = Some(Arc::clone(&state)); + Self { state } + } + + pub(crate) async fn wait_until_paused(&self) { + tokio::time::timeout(std::time::Duration::from_secs(30), self.state.arrived.notified()) + .await + .expect("data movement multipart operation should reach selected completion"); + } +} + +#[cfg(test)] +impl Drop for DataMovementMultipartCompletionBarrier { + fn drop(&mut self) { + self.state.release.notify_one(); + let mut slot = DATA_MOVEMENT_MULTIPART_COMPLETION_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("data movement multipart completion barrier mutex should not poison"); + if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { + *slot = None; + } + } +} + +#[cfg(test)] +async fn pause_data_movement_multipart_before_selected_completion(bucket: &str) { + let barrier = DATA_MOVEMENT_MULTIPART_COMPLETION_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("data movement multipart completion barrier mutex should not poison") + .as_ref() + .filter(|barrier| barrier.bucket == bucket) + .cloned(); + if let Some(barrier) = barrier { + barrier.arrived.notify_one(); + barrier.release.notified().await; + } +} + async fn list_pool_multipart_uploads_for_incarnation( pool: &crate::core::sets::Sets, bucket: &str, @@ -332,7 +402,7 @@ impl ECStore { ) -> Result { self.handle_new_multipart_upload_with_pool_idx(bucket, object, opts) .await - .map(|(res, _)| res) + .map(|(res, _, _)| res) } pub(crate) async fn handle_new_multipart_upload_with_pool_idx( @@ -340,7 +410,7 @@ impl ECStore { bucket: &str, object: &str, opts: &ObjectOptions, - ) -> Result<(MultipartUploadResult, usize)> { + ) -> Result<(MultipartUploadResult, usize, Option)> { check_new_multipart_args(bucket, object)?; let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?; let opts = &opts; @@ -349,7 +419,20 @@ impl ECStore { return self.pools[0] .new_multipart_upload(bucket, object, opts) .await - .map(|res| (res, 0)); + .map(|res| (res, 0, opts.expected_bucket_incarnation_id)); + } + + if opts.data_movement && opts.version_id.is_some() { + let idx = self.select_data_movement_pool_idx(bucket, object, -1, opts, false).await?; + if idx == opts.src_pool_idx { + return Err(StorageError::DataMovementOverwriteErr( + bucket.to_owned(), + object.to_owned(), + opts.version_id.clone().unwrap_or_default(), + )); + } + let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?; + return Ok((res, idx, opts.expected_bucket_incarnation_id)); } for (idx, pool) in self.pools.iter().enumerate() { @@ -372,7 +455,7 @@ impl ECStore { if !res.uploads.is_empty() { let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?; - return Ok((res, idx)); + return Ok((res, idx, opts.expected_bucket_incarnation_id)); } } let idx = self.get_pool_idx(bucket, object, -1).await?; @@ -385,7 +468,7 @@ impl ECStore { } let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?; - Ok((res, idx)) + Ok((res, idx, opts.expected_bucket_incarnation_id)) } #[instrument(skip(self))] @@ -456,6 +539,30 @@ impl ECStore { Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned())) } + pub(crate) async fn put_object_part_for_data_movement( + &self, + target_pool_idx: usize, + bucket: &str, + object: &str, + upload_id: &str, + data: &mut PutObjReader, + opts: &ObjectOptions, + ) -> Result { + let part_id = opts + .part_number + .ok_or_else(|| Error::other("targeted multipart upload requires a part number"))?; + check_put_object_part_args(bucket, object, upload_id)?; + if !opts.data_movement { + return Err(Error::other("targeted multipart upload requires data_movement options")); + } + let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?; + let pool = self + .pools + .get(target_pool_idx) + .ok_or_else(|| Error::other(format!("data movement target pool {target_pool_idx} is out of range")))?; + pool.put_object_part(bucket, object, upload_id, part_id, data, &opts).await + } + #[instrument(skip(self))] pub(super) async fn handle_get_multipart_info( &self, @@ -530,6 +637,26 @@ impl ECStore { Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned())) } + pub(crate) async fn abort_multipart_upload_for_data_movement( + &self, + target_pool_idx: usize, + bucket: &str, + object: &str, + upload_id: &str, + opts: &ObjectOptions, + ) -> Result<()> { + check_abort_multipart_args(bucket, object, upload_id)?; + if !opts.data_movement { + return Err(Error::other("targeted multipart abort requires data_movement options")); + } + let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?; + let pool = self + .pools + .get(target_pool_idx) + .ok_or_else(|| Error::other(format!("data movement target pool {target_pool_idx} is out of range")))?; + pool.abort_multipart_upload(bucket, object, upload_id, &opts).await + } + #[instrument(skip(self))] pub(super) async fn handle_complete_multipart_upload( self: Arc, @@ -574,6 +701,62 @@ impl ECStore { Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned())) } + + pub(crate) async fn complete_multipart_upload_for_data_movement( + self: Arc, + target_pool_idx: usize, + bucket: &str, + object: &str, + upload_id: &str, + uploaded_parts: Vec, + opts: &ObjectOptions, + ) -> Result { + check_complete_multipart_args(bucket, object, upload_id)?; + if !opts.data_movement { + return Err(Error::other("targeted multipart completion requires data_movement options")); + } + let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?; + if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) { + let expected_incarnation_id = opts + .expected_bucket_incarnation_id + .ok_or_else(|| Error::other("data movement completion is missing its bucket incarnation"))?; + let lifecycle_fence = opts + .bucket_lifecycle_lock_fence + .as_ref() + .ok_or_else(|| Error::other("data movement completion is missing its bucket lifecycle fence"))?; + let snapshot = match opts.object_lock_config_snapshot.as_ref() { + Some(snapshot) => Arc::clone(snapshot), + None => { + self.object_lock_config_snapshot_under_lifecycle_fence(bucket, lifecycle_fence) + .await? + } + }; + if !snapshot.is_valid_for_destructive_put(self.id, bucket, expected_incarnation_id) { + return Err(Error::other( + "data movement Object Lock snapshot does not match the target bucket generation", + )); + } + snapshot.add_lock_fences(&mut opts); + opts.object_lock_config_snapshot = Some(snapshot); + } + #[cfg(test)] + pause_data_movement_multipart_before_selected_completion(bucket).await; + let pool = self + .pools + .get(target_pool_idx) + .ok_or_else(|| Error::other(format!("data movement target pool {target_pool_idx} is out of range")))? + .clone(); + let result = enqueue_transition_after_write( + pool.complete_multipart_upload(bucket, object, upload_id, uploaded_parts, &opts) + .await, + LcEventSrc::S3CompleteMultipartUpload, + ) + .await; + if result.is_ok() { + list_objects::observe_list_objects_mutation(self.as_ref(), bucket).await; + } + result + } } /// Merges per-pool `ListMultipartUploads` pages into a single globally paginated diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index a9358ed6b..df915af7f 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -918,7 +918,7 @@ fn is_equivalent_data_movement_delete_marker(source: &ObjectInfo, target: &Objec && is_data_movement_delete_marker(target) && source.version_id == target.version_id && source.mod_time == target.mod_time - && source.user_defined == target.user_defined + && is_equivalent_data_movement_delete_marker_metadata(&source.user_defined, &target.user_defined) && source.user_tags == target.user_tags && source.replication_status_internal == target.replication_status_internal && source.replication_status == target.replication_status @@ -926,24 +926,185 @@ fn is_equivalent_data_movement_delete_marker(source: &ObjectInfo, target: &Objec && source.version_purge_status == target.version_purge_status } +fn is_equivalent_data_movement_delete_marker_metadata( + source: &HashMap, + target: &HashMap, +) -> bool { + matches!( + ( + data_movement_delete_marker_metadata_identity(source), + data_movement_delete_marker_metadata_identity(target) + ), + (Some(source), Some(target)) if source == target + ) +} + +fn data_movement_delete_marker_metadata_identity(metadata: &HashMap) -> Option> { + let mut identity = HashMap::with_capacity(metadata.len()); + let mut local_tier_free_version_id = None; + for (key, value) in metadata { + let Some(suffix) = rustfs_utils::http::strip_internal_prefix_preserving_case(key) else { + identity.insert(key.clone(), value.clone()); + continue; + }; + + if suffix.eq_ignore_ascii_case(rustfs_utils::http::SUFFIX_TIER_FV_ID) { + let version_id = Uuid::parse_str(value).ok().filter(|version_id| !version_id.is_nil())?; + if local_tier_free_version_id.is_some_and(|expected| expected != version_id) { + return None; + } + local_tier_free_version_id = Some(version_id); + continue; + } + + let canonical_suffix = [ + rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP, + rustfs_utils::http::SUFFIX_REPLICA_STATUS, + rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP, + rustfs_utils::http::SUFFIX_REPLICATION_STATUS, + rustfs_utils::http::SUFFIX_PURGESTATUS, + ] + .into_iter() + .find(|candidate| suffix.eq_ignore_ascii_case(candidate)) + .map(str::to_string) + .or_else(|| { + [ + rustfs_utils::http::SUFFIX_REPLICATION_RESET_ARN_PREFIX, + rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX, + ] + .into_iter() + .find_map(|prefix| { + suffix + .get(..prefix.len()) + .is_some_and(|candidate| candidate.eq_ignore_ascii_case(prefix)) + .then(|| format!("{prefix}{}", &suffix[prefix.len()..])) + }) + }) + .unwrap_or_else(|| suffix.to_string()); + let canonical_value = if canonical_suffix.eq_ignore_ascii_case(rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP) + || canonical_suffix.eq_ignore_ascii_case(rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP) + { + rustfs_filemeta::parse_replication_timestamp(value)? + .unix_timestamp_nanos() + .to_string() + } else { + value.clone() + }; + let canonical_key = format!("{}{canonical_suffix}", rustfs_utils::http::RUSTFS_INTERNAL_PREFIX); + if identity + .insert(canonical_key, canonical_value.clone()) + .is_some_and(|existing| existing != canonical_value) + { + return None; + } + } + for (status_suffix, timestamp_suffix) in [ + (rustfs_utils::http::SUFFIX_REPLICA_STATUS, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP), + ( + rustfs_utils::http::SUFFIX_REPLICATION_STATUS, + rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP, + ), + ] { + let status_key = format!("{}{status_suffix}", rustfs_utils::http::RUSTFS_INTERNAL_PREFIX); + let timestamp_key = format!("{}{timestamp_suffix}", rustfs_utils::http::RUSTFS_INTERNAL_PREFIX); + match (identity.contains_key(&status_key), identity.contains_key(×tamp_key)) { + (true, false) => { + identity.insert(timestamp_key, OffsetDateTime::UNIX_EPOCH.unix_timestamp_nanos().to_string()); + } + (false, true) => return None, + _ => {} + } + } + Some(identity) +} + fn is_data_movement_delete_marker(info: &ObjectInfo) -> bool { info.delete_marker } +fn is_expected_data_movement_delete_marker_source(source: &ObjectInfo, expected_mod_time: Option) -> bool { + is_data_movement_delete_marker(source) + && source.mod_time.is_some() + && source.mod_time == expected_mod_time + && data_movement_delete_marker_metadata_identity(&source.user_defined).is_some() +} + +fn current_data_movement_delete_marker_opts(source: &ObjectInfo, opts: &ObjectOptions) -> Option { + let replica_status = rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICA_STATUS); + let replica_timestamp = rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP); + let (replica_status, replica_timestamp) = match (replica_status, replica_timestamp) { + (None, None) => Default::default(), + (Some(status), timestamp) => { + let status = crate::bucket::replication::ReplicationStatusType::from(status.as_str()); + if status.is_empty() { + return None; + } + let timestamp = match timestamp { + Some(timestamp) => rustfs_filemeta::parse_replication_timestamp(×tamp)?, + None => OffsetDateTime::UNIX_EPOCH, + }; + (status, Some(timestamp)) + } + (None, Some(_)) => return None, + }; + let replication_status = rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_STATUS); + let replication_timestamp = + rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP); + let (replication_status, replication_timestamp, replication_targets) = match (replication_status, replication_timestamp) { + (None, None) => Default::default(), + (Some(status), timestamp) => { + let direct_status = crate::bucket::replication::ReplicationStatusType::from(status.as_str()); + let targets = crate::bucket::replication::replication_statuses_map(status.as_str()); + if direct_status.is_empty() && targets.is_empty() { + return None; + } + let timestamp = match timestamp { + Some(timestamp) => rustfs_filemeta::parse_replication_timestamp(×tamp)?, + None => OffsetDateTime::UNIX_EPOCH, + }; + (Some(status), Some(timestamp), targets) + } + (None, Some(_)) => return None, + }; + let mut state = source.replication_state(); + if state.target_delete_marker_version_ids_corrupt { + return None; + } + state.replica_status = replica_status; + state.replica_timestamp = replica_timestamp; + state.replication_status_internal = replication_status; + state.replication_timestamp = replication_timestamp; + state.targets = replication_targets; + state.replicate_decision_str = source.replication_decision.clone(); + state.delete_marker = true; + + let mut target_opts = opts.clone(); + target_opts.mod_time = source.mod_time; + target_opts.delete_replication = Some(state); + Some(target_opts) +} + fn expected_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo) -> ObjectInfo { ObjectInfo::from_file_info(source, "", &source.name, source.version_id.is_some()) } fn is_equivalent_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo, target: &ObjectInfo) -> bool { let expected = expected_data_movement_tiered_object(source); + let Some(source_actual_size) = effective_object_actual_size(&expected) else { + return false; + }; + let Some(target_actual_size) = effective_object_actual_size(target) else { + return false; + }; source.version_id == target.version_id && !target.delete_marker && source.size == target.size && source.get_etag() == target.etag && source.checksum == target.checksum + && crate::data_movement::are_equivalent_data_movement_parts(&source.parts, &target.parts) && source.mod_time == target.mod_time - && expected.user_defined == target.user_defined + && crate::data_movement::is_equivalent_data_movement_metadata(&expected, target, source_actual_size, target_actual_size) && expected.user_tags == target.user_tags && expected.expires == target.expires && expected.storage_class == target.storage_class @@ -952,11 +1113,12 @@ fn is_equivalent_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo, && expected.version_purge_status_internal == target.version_purge_status_internal && expected.version_purge_status == target.version_purge_status && expected.transitioned_object.status == target.transitioned_object.status + && expected.transition_version_state == target.transition_version_state && expected.transitioned_object.name == target.transitioned_object.name && expected.transitioned_object.tier == target.transitioned_object.tier && expected.transitioned_object.version_id == target.transitioned_object.version_id && expected.transitioned_object.free_version == target.transitioned_object.free_version - && effective_object_actual_size(target) == Some(source.size) + && source_actual_size == target_actual_size } fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool { @@ -1054,7 +1216,7 @@ impl ECStore { ))) } - async fn object_lock_config_snapshot_under_lifecycle_fence( + pub(super) async fn object_lock_config_snapshot_under_lifecycle_fence( &self, bucket: &str, lifecycle_fence: &NamespaceLockFence, @@ -1454,7 +1616,8 @@ impl ECStore { target_pool_idx: usize, opts: &ObjectOptions, ) -> Result> { - let lookup_opts = version_aware_lookup_opts(opts, true); + let mut lookup_opts = version_aware_lookup_opts(opts, true); + lookup_opts.include_part_checksums = true; let Some(pool) = self.pools.get(target_pool_idx) else { return Err(Error::other(format!( @@ -1521,6 +1684,25 @@ impl ECStore { ) -> Result<()> { check_put_object_args(bucket, object)?; + let mut opts = opts.clone(); + let bucket_incarnation_fence = if is_meta_bucketname(bucket) { + None + } else { + let expected = opts + .expected_bucket_incarnation_id + .ok_or_else(|| Error::other("tiered data movement is missing its bucket incarnation snapshot"))?; + let guard = self.acquire_bucket_incarnation_fence(bucket, expected).await?; + if let Some(namespace_guard) = guard.namespace_lock_guard() { + opts.add_bucket_lifecycle_lock_guard(namespace_guard); + } + Some(guard) + }; + + let mut fi = fi.clone(); + if opts.data_movement { + crate::data_movement::prepare_tiered_data_movement_file_info(&mut fi)?; + } + let object = encode_dir_object(object); if self.single_pool() { @@ -1533,7 +1715,8 @@ impl ECStore { let idx = if opts.data_movement && opts.version_id.is_some() { Self::resolve_decommission_target_pool_idx_result( - self.select_data_movement_pool_idx(bucket, &object, fi.size, opts, true).await, + self.select_data_movement_pool_idx(bucket, &object, fi.size, &opts, true) + .await, bucket, &object, )? @@ -1550,7 +1733,7 @@ impl ECStore { .await; let target_pool_idx = resolve_data_movement_resume_target_pool(idx, resume_target_pool_idx, opts.src_pool_idx); if self - .has_equivalent_data_movement_tiered_object(bucket, &object, fi, opts, target_pool_idx) + .has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, target_pool_idx) .await? { return Ok(()); @@ -1563,14 +1746,27 @@ impl ECStore { )); } - Self::resolve_decommission_tiered_object_result( - self.pools[idx] - .get_disks_by_key(&object) - .decommission_tiered_object(bucket, &object, fi, opts) - .await, - bucket, - &object, - ) + let result = self.pools[idx] + .get_disks_by_key(&object) + .decommission_tiered_object(bucket, &object, &fi, &opts) + .await; + if matches!(result, Err(Error::PreconditionFailed)) { + if self + .has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, idx) + .await? + { + return Ok(()); + } + return Err(StorageError::DataMovementOverwriteErr( + bucket.to_owned(), + object, + opts.version_id.clone().unwrap_or_default(), + )); + } + if bucket_incarnation_fence.as_ref().is_some_and(|guard| guard.is_lock_lost()) { + return Err(Error::other("tiered data movement bucket incarnation fence was lost during target write")); + } + Self::resolve_decommission_tiered_object_result(result, bucket, &object) } #[instrument(level = "debug", skip(self))] @@ -1607,15 +1803,7 @@ impl ECStore { Ok(Self::attach_read_lock_guard(reader, read_lock_guard)) } - #[instrument(level = "debug", skip(self, data))] - #[hotpath::measure(impl_type = "ECStore")] - pub(super) async fn handle_put_object( - &self, - bucket: &str, - object: &str, - data: &mut PutObjReader, - opts: &ObjectOptions, - ) -> Result<(ObjectInfo, Option)> { + async fn prepare_put_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<(String, ObjectOptions)> { check_put_object_args(bucket, object)?; let object = encode_dir_object(object); @@ -1642,22 +1830,20 @@ impl ECStore { }; snapshot.add_lock_fences(&mut opts); } + Ok((object, opts)) + } - // Keep PUT atomic-read friendly: SetDisks takes the object write lock only - // around precondition checks and the final rename/commit. + async fn select_put_object_pool_idx(&self, bucket: &str, object: &str, size: i64, opts: &ObjectOptions) -> Result { if self.single_pool() { - return self.pools[0] - .put_object_with_old_current_size(bucket, object.as_str(), data, &opts) - .await; + return Ok(0); } let idx = if opts.data_movement && opts.version_id.is_some() { - self.select_data_movement_pool_idx(bucket, &object, data.size(), &opts, false) - .await? + self.select_data_movement_pool_idx(bucket, object, size, opts, false).await? } else if opts.no_lock { - self.get_pool_idx_no_lock(bucket, &object, data.size()).await? + self.get_pool_idx_no_lock(bucket, object, size).await? } else { - self.get_pool_idx(bucket, &object, data.size()).await? + self.get_pool_idx(bucket, object, size).await? }; if opts.data_movement && idx == opts.src_pool_idx { @@ -1667,7 +1853,50 @@ impl ECStore { opts.version_id.clone().unwrap_or_default(), )); } + Ok(idx) + } + pub(crate) async fn put_object_for_data_movement( + &self, + bucket: &str, + object: &str, + data: &mut PutObjReader, + opts: &ObjectOptions, + ) -> Result<(usize, Result)> { + if !opts.data_movement { + return Err(Error::other("data movement PUT requires data_movement options")); + } + let (object, opts) = self.prepare_put_object(bucket, object, opts).await?; + let idx = self + .select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts) + .await?; + let result = self.pools[idx] + .put_object_with_old_current_size(bucket, &object, data, &opts) + .await + .map(|(object_info, _)| object_info); + let result = enqueue_transition_after_write(result, LcEventSrc::S3PutObject).await; + if result.is_ok() { + list_objects::observe_list_objects_mutation(self, bucket).await; + } + Ok((idx, result)) + } + + #[instrument(level = "debug", skip(self, data))] + #[hotpath::measure(impl_type = "ECStore")] + pub(super) async fn handle_put_object( + &self, + bucket: &str, + object: &str, + data: &mut PutObjReader, + opts: &ObjectOptions, + ) -> Result<(ObjectInfo, Option)> { + let (object, opts) = self.prepare_put_object(bucket, object, opts).await?; + let idx = self + .select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts) + .await?; + + // Keep PUT atomic-read friendly: SetDisks takes the object write lock only + // around precondition checks and the final rename/commit. self.pools[idx] .put_object_with_old_current_size(bucket, &object, data, &opts) .await @@ -2110,6 +2339,50 @@ impl ECStore { }; let target_pool_idx = resolve_data_movement_resume_target_pool(selected_target_pool_idx, resume_target_pool_idx, opts.src_pool_idx); + let mut delete_marker_target_opts = None; + + if opts.delete_marker && should_check_data_movement_resume_target(opts.src_pool_idx, target_pool_idx) { + let source = self + .find_data_movement_target_info(bucket, object, opts.src_pool_idx, &opts) + .await?; + let Some(source) = source else { + return Err(StorageError::DataMovementOverwriteErr( + bucket.to_owned(), + object.to_owned(), + opts.version_id.unwrap_or_default(), + )); + }; + if !is_expected_data_movement_delete_marker_source(&source, opts.mod_time) { + return Err(StorageError::DataMovementOverwriteErr( + bucket.to_owned(), + object.to_owned(), + opts.version_id.unwrap_or_default(), + )); + } + let Some(target_opts) = current_data_movement_delete_marker_opts(&source, &opts) else { + return Err(StorageError::DataMovementOverwriteErr( + bucket.to_owned(), + object.to_owned(), + opts.version_id.unwrap_or_default(), + )); + }; + let target = self + .find_data_movement_target_info(bucket, object, target_pool_idx, &target_opts) + .await?; + if let Some(target) = target { + if is_equivalent_data_movement_delete_marker(&source, &target) { + let mut target = target; + target.name = decode_dir_object(object); + return Ok(target); + } + return Err(StorageError::DataMovementOverwriteErr( + bucket.to_owned(), + object.to_owned(), + opts.version_id.unwrap_or_default(), + )); + } + delete_marker_target_opts = Some(target_opts); + } if !should_check_data_movement_resume_target(opts.src_pool_idx, target_pool_idx) { if let Ok((source_pool_info, _)) = existing_pool_info @@ -2137,7 +2410,8 @@ impl ECStore { )); } - let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, opts).await?; + let target_opts = delete_marker_target_opts.unwrap_or(opts); + let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, target_opts).await?; obj.name = decode_dir_object(obj.name.as_str()); return Ok(obj); } @@ -3200,6 +3474,247 @@ mod tests { assert!(!is_equivalent_data_movement_delete_marker(&source, &mismatched)); } + #[test] + fn equivalent_data_movement_delete_marker_accepts_distinct_local_free_version_ids() { + let mut source = ObjectInfo { + version_id: Some(Uuid::from_u128(1)), + delete_marker: true, + mod_time: Some(OffsetDateTime::UNIX_EPOCH), + ..Default::default() + }; + rustfs_utils::http::insert_str( + Arc::make_mut(&mut source.user_defined), + rustfs_utils::http::SUFFIX_TIER_FV_ID, + Uuid::from_u128(2).to_string(), + ); + let mut target = source.clone(); + rustfs_utils::http::insert_str( + Arc::make_mut(&mut target.user_defined), + rustfs_utils::http::SUFFIX_TIER_FV_ID, + Uuid::from_u128(3).to_string(), + ); + + assert!(is_equivalent_data_movement_delete_marker(&source, &target)); + + Arc::make_mut(&mut target.user_defined).insert( + format!("{}{}", rustfs_utils::http::MINIO_INTERNAL_PREFIX, rustfs_utils::http::SUFFIX_TIER_FV_ID), + Uuid::from_u128(4).to_string(), + ); + assert!(!is_equivalent_data_movement_delete_marker(&source, &target)); + } + + #[test] + fn equivalent_data_movement_delete_marker_accepts_replication_alias_expansion() { + let key = format!( + "{}{}", + rustfs_utils::http::MINIO_INTERNAL_PREFIX, + rustfs_utils::http::SUFFIX_REPLICATION_STATUS + ); + let timestamp_key = format!( + "{}{}", + rustfs_utils::http::MINIO_INTERNAL_PREFIX, + rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP + ); + let source = ObjectInfo { + version_id: Some(Uuid::from_u128(1)), + delete_marker: true, + mod_time: Some(OffsetDateTime::UNIX_EPOCH), + user_defined: Arc::new(HashMap::from([ + (key.clone(), "arn=COMPLETED;".to_string()), + (timestamp_key, "1970-01-01T00:00:01Z".to_string()), + ])), + ..Default::default() + }; + let mut target = source.clone(); + rustfs_utils::http::insert_str( + Arc::make_mut(&mut target.user_defined), + rustfs_utils::http::SUFFIX_REPLICATION_STATUS, + "arn=COMPLETED;".to_string(), + ); + rustfs_utils::http::insert_str( + Arc::make_mut(&mut target.user_defined), + rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP, + (OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND).to_string(), + ); + assert!(is_equivalent_data_movement_delete_marker(&source, &target)); + + Arc::make_mut(&mut target.user_defined).insert(key, "arn=FAILED;".to_string()); + assert!(!is_equivalent_data_movement_delete_marker(&source, &target)); + } + + #[test] + fn data_movement_delete_marker_source_requires_persisted_mod_time() { + let source = ObjectInfo { + delete_marker: true, + ..Default::default() + }; + assert!(!is_expected_data_movement_delete_marker_source(&source, None)); + + let source = ObjectInfo { + mod_time: Some(OffsetDateTime::UNIX_EPOCH), + ..source + }; + assert!(is_expected_data_movement_delete_marker_source(&source, Some(OffsetDateTime::UNIX_EPOCH))); + assert!(!is_expected_data_movement_delete_marker_source(&source, None)); + } + + #[test] + fn data_movement_delete_marker_uses_current_source_replication_state() { + let expected_timestamp = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND; + let timestamp = expected_timestamp.to_string(); + let mut metadata = HashMap::new(); + rustfs_utils::http::insert_str( + &mut metadata, + rustfs_utils::http::SUFFIX_REPLICA_STATUS, + ReplicationStatusType::Replica.to_string(), + ); + rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP, timestamp.clone()); + rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP, timestamp); + rustfs_utils::http::insert_str( + &mut metadata, + rustfs_utils::http::SUFFIX_REPLICATION_STATUS, + "arn=COMPLETED;".to_string(), + ); + rustfs_utils::http::insert_str( + &mut metadata, + &format!( + "{}{}", + rustfs_utils::http::SUFFIX_REPLICATION_RESET_ARN_PREFIX, + "arn:minio:replication::TenantA:bucket" + ), + "reset-id".to_string(), + ); + rustfs_utils::http::insert_str( + &mut metadata, + &format!( + "{}{}", + rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX, + "arn:minio:replication::TenantA:bucket" + ), + "target-version".to_string(), + ); + let source = ObjectInfo { + delete_marker: true, + mod_time: Some(OffsetDateTime::UNIX_EPOCH), + replication_status_internal: Some("arn=COMPLETED;".to_string()), + replication_decision: "arn=replicate;".to_string(), + user_defined: Arc::new(metadata), + ..Default::default() + }; + let opts = ObjectOptions { + mod_time: source.mod_time, + delete_replication: Some(ReplicationState { + replication_status_internal: Some("arn=PENDING;".to_string()), + ..Default::default() + }), + ..Default::default() + }; + + let target_opts = current_data_movement_delete_marker_opts(&source, &opts).expect("valid current source state"); + let state = target_opts.delete_replication.as_ref().expect("current replication state"); + assert_eq!(state.replication_status_internal.as_deref(), Some("arn=COMPLETED;")); + assert_eq!(state.replica_status, crate::bucket::replication::ReplicationStatusType::Replica); + assert_eq!(state.replica_timestamp, Some(expected_timestamp)); + assert_eq!(state.replication_timestamp, state.replica_timestamp); + assert_eq!(state.replicate_decision_str, "arn=replicate;"); + assert_eq!( + state + .reset_statuses_map + .get("arn:minio:replication::TenantA:bucket") + .map(String::as_str), + Some("reset-id") + ); + assert_eq!( + state + .target_delete_marker_version_ids + .get("arn:minio:replication::TenantA:bucket") + .map(String::as_str), + Some("target-version") + ); + } + + #[test] + fn data_movement_delete_marker_rejects_corrupt_target_version_maps() { + let suffix = format!("{}not-an-arn", rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX); + let mut malformed = HashMap::new(); + rustfs_utils::http::insert_str(&mut malformed, &suffix, "target-version".to_string()); + let malformed_source = ObjectInfo { + delete_marker: true, + mod_time: Some(OffsetDateTime::UNIX_EPOCH), + user_defined: Arc::new(malformed), + ..Default::default() + }; + assert!(current_data_movement_delete_marker_opts(&malformed_source, &ObjectOptions::default()).is_none()); + + let mut conflicted = HashMap::new(); + let suffix = format!( + "{}arn:minio:replication::target:bucket", + rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX + ); + rustfs_utils::http::insert_str(&mut conflicted, &suffix, "target-version-a".to_string()); + conflicted.insert( + format!("{}{suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX), + "target-version-b".to_string(), + ); + let conflicted_source = ObjectInfo { + user_defined: Arc::new(conflicted), + ..malformed_source.clone() + }; + assert!(current_data_movement_delete_marker_opts(&conflicted_source, &ObjectOptions::default()).is_none()); + + let mut over_cap = HashMap::new(); + for index in 0..=1_000 { + let suffix = format!( + "{}arn:minio:replication::target:bucket-{index}", + rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX + ); + rustfs_utils::http::insert_str(&mut over_cap, &suffix, format!("target-version-{index}")); + } + let over_cap_source = ObjectInfo { + user_defined: Arc::new(over_cap), + ..malformed_source + }; + assert!(current_data_movement_delete_marker_opts(&over_cap_source, &ObjectOptions::default()).is_none()); + } + + #[test] + fn data_movement_delete_marker_normalizes_legacy_missing_replication_timestamps() { + let mut source_metadata = HashMap::new(); + rustfs_utils::http::insert_str( + &mut source_metadata, + rustfs_utils::http::SUFFIX_REPLICA_STATUS, + ReplicationStatusType::Replica.to_string(), + ); + rustfs_utils::http::insert_str( + &mut source_metadata, + rustfs_utils::http::SUFFIX_REPLICATION_STATUS, + "arn=COMPLETED;".to_string(), + ); + let source = ObjectInfo { + delete_marker: true, + mod_time: Some(OffsetDateTime::UNIX_EPOCH), + replication_status_internal: Some("arn=COMPLETED;".to_string()), + user_defined: Arc::new(source_metadata), + ..Default::default() + }; + + let target_opts = current_data_movement_delete_marker_opts(&source, &ObjectOptions::default()) + .expect("legacy status-only metadata should remain migratable"); + let state = target_opts + .delete_replication + .expect("replication state should be reconstructed"); + assert_eq!(state.replica_timestamp, Some(OffsetDateTime::UNIX_EPOCH)); + assert_eq!(state.replication_timestamp, Some(OffsetDateTime::UNIX_EPOCH)); + + let mut target_metadata = (*source.user_defined).clone(); + let epoch = OffsetDateTime::UNIX_EPOCH + .format(&time::format_description::well_known::Rfc3339) + .unwrap(); + rustfs_utils::http::insert_str(&mut target_metadata, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP, epoch.clone()); + rustfs_utils::http::insert_str(&mut target_metadata, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP, epoch); + assert!(is_equivalent_data_movement_delete_marker_metadata(&source.user_defined, &target_metadata)); + } + #[test] fn equivalent_data_movement_delete_marker_rejects_metadata_and_replication_mismatch() { let version_id = Uuid::nil(); @@ -3344,6 +3859,67 @@ mod tests { assert!(is_equivalent_data_movement_tiered_object(&source, &target)); } + #[test] + fn equivalent_data_movement_tiered_object_uses_logical_compressed_and_encrypted_sizes() { + let mut compressed = tiered_equivalence_source(); + compressed.size = 600; + rustfs_utils::http::insert_str(&mut compressed.metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "S2".to_string()); + rustfs_utils::http::insert_str(&mut compressed.metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1024".to_string()); + let compressed_target = tiered_equivalence_target(&compressed); + assert!(is_equivalent_data_movement_tiered_object(&compressed, &compressed_target)); + + let mut encrypted = tiered_equivalence_source(); + encrypted.size = 640; + encrypted.metadata.insert( + rustfs_utils::http::object_encryption_keys::INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), + "key-id".to_string(), + ); + encrypted.metadata.insert( + rustfs_utils::http::object_encryption_keys::INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER.to_string(), + "1024".to_string(), + ); + let encrypted_target = tiered_equivalence_target(&encrypted); + assert!(is_equivalent_data_movement_tiered_object(&encrypted, &encrypted_target)); + } + + #[test] + fn equivalent_data_movement_tiered_object_accepts_transition_alias_expansion() { + let mut source = tiered_equivalence_source(); + let suffix = rustfs_utils::http::SUFFIX_TRANSITION_TIER; + source.metadata.insert( + format!("{}{suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX), + source.transition_tier.clone(), + ); + let mut target = tiered_equivalence_target(&source); + Arc::make_mut(&mut target.user_defined) + .insert(rustfs_utils::http::internal_key_rustfs(suffix), source.transition_tier.clone()); + + assert!(is_equivalent_data_movement_tiered_object(&source, &target)); + } + + #[test] + fn equivalent_data_movement_tiered_object_requires_hydrated_part_checksums() { + let mut source = tiered_equivalence_source(); + source.parts = vec![rustfs_filemeta::ObjectPartInfo { + number: 1, + mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND), + checksums: Some(HashMap::from([("CRC32C".to_string(), "AAAAAA==".to_string())])), + ..Default::default() + }]; + rustfs_utils::http::insert_str( + &mut source.metadata, + rustfs_utils::http::SUFFIX_PART_CHECKSUMS, + r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string(), + ); + let mut target = tiered_equivalence_target(&source); + Arc::make_mut(&mut target.parts)[0].mod_time = None; + assert!(is_equivalent_data_movement_tiered_object(&source, &target)); + + let mut missing = target; + Arc::make_mut(&mut missing.parts)[0].checksums = None; + assert!(!is_equivalent_data_movement_tiered_object(&source, &missing)); + } + #[test] fn equivalent_data_movement_tiered_object_rejects_transition_mismatch() { let source = tiered_equivalence_source(); @@ -3353,6 +3929,16 @@ mod tests { assert!(!is_equivalent_data_movement_tiered_object(&source, &target)); } + #[test] + fn equivalent_data_movement_tiered_object_rejects_transition_version_state_mismatch() { + let mut source = tiered_equivalence_source(); + source.transition_version_state = rustfs_filemeta::TransitionVersionState::Exact; + let mut target = tiered_equivalence_target(&source); + target.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown; + + assert!(!is_equivalent_data_movement_tiered_object(&source, &target)); + } + #[test] fn equivalent_data_movement_tiered_object_rejects_user_metadata_mismatch() { let source = tiered_equivalence_source(); diff --git a/crates/ecstore/tests/legacy_bitrot_read_test.rs b/crates/ecstore/tests/legacy_bitrot_read_test.rs index 7b8df7815..4e04518b3 100644 --- a/crates/ecstore/tests/legacy_bitrot_read_test.rs +++ b/crates/ecstore/tests/legacy_bitrot_read_test.rs @@ -109,6 +109,7 @@ async fn run_legacy_bitrot_test_for_object(root: &std::path::Path, disk_name: &s FileInfoOpts { data: true, // need inline data for inline objects include_free_versions: false, + include_part_checksums: true, }, ) { Ok(f) => f, diff --git a/crates/filemeta/Cargo.toml b/crates/filemeta/Cargo.toml index c2e7ee95f..ef6095d65 100644 --- a/crates/filemeta/Cargo.toml +++ b/crates/filemeta/Cargo.toml @@ -37,6 +37,7 @@ crc-fast = { workspace = true } rmp.workspace = true rmp-serde.workspace = true serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] } uuid = { workspace = true, features = ["v4", "fast-rng", "serde", "macro-diagnostics"] } tokio = { workspace = true, features = ["io-util", "macros", "sync", "fs", "rt-multi-thread"] } @@ -54,7 +55,6 @@ arc-swap.workspace = true criterion = { workspace = true, features = ["html_reports"] } tempfile = { workspace = true } proptest = "1" -serde_json.workspace = true [[bench]] name = "xl_meta_bench" diff --git a/crates/filemeta/examples/dump_fileinfo.rs b/crates/filemeta/examples/dump_fileinfo.rs index 41d819710..4d148703c 100644 --- a/crates/filemeta/examples/dump_fileinfo.rs +++ b/crates/filemeta/examples/dump_fileinfo.rs @@ -52,6 +52,7 @@ fn main() { FileInfoOpts { data: false, include_free_versions: true, + include_part_checksums: true, }, ) .expect("decode file info"); diff --git a/crates/filemeta/src/fileinfo.rs b/crates/filemeta/src/fileinfo.rs index 454b9d6c0..0f799a40e 100644 --- a/crates/filemeta/src/fileinfo.rs +++ b/crates/filemeta/src/fileinfo.rs @@ -17,9 +17,9 @@ use bytes::Bytes; use rmp_serde::Serializer; use rustfs_utils::HashAlgorithm; use rustfs_utils::http::{ - SUFFIX_COMPRESSION, SUFFIX_DATA_MOVED, SUFFIX_FREE_VERSION, SUFFIX_HEALING, SUFFIX_INLINE_DATA, SUFFIX_TIER_FV_ID, - SUFFIX_TIER_FV_MARKER, SUFFIX_TIER_SKIP_FV_ID, contains_key_str, get_str, has_internal_suffix, insert_str, - is_encryption_metadata_key, starts_with_ignore_ascii_case, + AMZ_OBJECT_TAGGING, SUFFIX_COMPRESSION, SUFFIX_DATA_MOVED, SUFFIX_DATA_MOVED_TAGS, SUFFIX_FREE_VERSION, SUFFIX_HEALING, + SUFFIX_INLINE_DATA, SUFFIX_TIER_FV_ID, SUFFIX_TIER_FV_MARKER, SUFFIX_TIER_SKIP_FV_ID, contains_key_str, get_str, + has_internal_suffix, insert_str, is_encryption_metadata_key, starts_with_ignore_ascii_case, }; use s3s::dto::{RestoreStatus, Timestamp}; use s3s::header::X_AMZ_RESTORE; @@ -232,6 +232,17 @@ pub enum TransitionVersionState { Exact, } +impl TransitionVersionState { + pub const fn as_str(self) -> &'static str { + match self { + Self::Unknown => "unknown", + Self::KnownDisabled => "known-disabled", + Self::SuspendedNull => "suspended-null", + Self::Exact => "exact", + } + } +} + #[derive(PartialEq, Clone, Default)] pub struct FileInfo { pub volume: String, @@ -1151,9 +1162,16 @@ impl FileInfo { } pub fn set_data_moved(&mut self) { + let tags_proof = format!("v1:{}", self.metadata.get(AMZ_OBJECT_TAGGING).map(String::as_str).unwrap_or_default()); + insert_str(&mut self.metadata, SUFFIX_DATA_MOVED_TAGS, tags_proof); insert_str(&mut self.metadata, SUFFIX_DATA_MOVED, "true".to_string()); } + pub fn acknowledge_data_movement(&mut self) { + // Keep both empty aliases so mixed-version disks retain one metadata identity. + insert_str(&mut self.metadata, SUFFIX_DATA_MOVED, String::new()); + } + pub fn inline_data(&self) -> bool { contains_key_str(&self.metadata, SUFFIX_INLINE_DATA) && !self.is_remote() } diff --git a/crates/filemeta/src/filemeta.rs b/crates/filemeta/src/filemeta.rs index e31fd4ba2..d2569447c 100644 --- a/crates/filemeta/src/filemeta.rs +++ b/crates/filemeta/src/filemeta.rs @@ -206,6 +206,42 @@ fn persist_reset_statuses(meta_sys: &mut HashMap>, reset_statuse } } +pub fn parse_replication_timestamp(value: &str) -> Option { + const DISPLAY_FORMAT: &[time::format_description::BorrowedFormatItem<'_>] = time::macros::format_description!( + "[year sign:automatic]-[month]-[day] [hour padding:none]:[minute]:[second].[subsecond] [offset_hour sign:mandatory]:[offset_minute]:[offset_second]" + ); + OffsetDateTime::parse(value, &Rfc3339) + .or_else(|_| OffsetDateTime::parse(value, DISPLAY_FORMAT)) + .ok() +} + +fn format_replication_timestamp(value: Option) -> String { + let value = value.unwrap_or(OffsetDateTime::UNIX_EPOCH); + value + .to_offset(time::UtcOffset::UTC) + .format(&Rfc3339) + .unwrap_or_else(|_| value.to_string()) +} + +fn persist_delete_marker_replication_state(meta_sys: &mut HashMap>, state: &ReplicationState) { + if !state.replica_status.is_empty() { + insert_bytes(meta_sys, SUFFIX_REPLICA_STATUS, state.replica_status.as_str().as_bytes().to_vec()); + insert_bytes( + meta_sys, + SUFFIX_REPLICA_TIMESTAMP, + format_replication_timestamp(state.replica_timestamp).into_bytes(), + ); + } + if let Some(status) = state.replication_status_internal.as_ref().filter(|status| !status.is_empty()) { + insert_bytes(meta_sys, SUFFIX_REPLICATION_STATUS, status.as_bytes().to_vec()); + insert_bytes( + meta_sys, + SUFFIX_REPLICATION_TIMESTAMP, + format_replication_timestamp(state.replication_timestamp).into_bytes(), + ); + } +} + #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct FileMeta { pub versions: Vec, @@ -213,6 +249,13 @@ pub struct FileMeta { pub meta_ver: u8, } +struct FileInfoDecodeOptions { + read_data: bool, + include_free_versions: bool, + all_parts: bool, + include_part_checksums: bool, +} + impl FileMeta { pub fn new() -> Self { Self { @@ -511,53 +554,8 @@ impl FileMeta { } if fi.deleted { - if !fi.delete_marker_replication_status().is_empty() - && let Some(delete_marker) = ventry.delete_marker.as_mut() - { - if fi.delete_marker_replication_status() == ReplicationStatusType::Replica { - insert_bytes( - &mut delete_marker.meta_sys, - SUFFIX_REPLICA_STATUS, - fi.replication_state_internal - .as_ref() - .map(|v| v.replica_status.clone()) - .unwrap_or_default() - .as_str() - .as_bytes() - .to_vec(), - ); - insert_bytes( - &mut delete_marker.meta_sys, - SUFFIX_REPLICA_TIMESTAMP, - fi.replication_state_internal - .as_ref() - .map(|v| v.replica_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string()) - .unwrap_or_default() - .as_bytes() - .to_vec(), - ); - } else { - insert_bytes( - &mut delete_marker.meta_sys, - SUFFIX_REPLICATION_STATUS, - fi.replication_state_internal - .as_ref() - .map(|v| v.replication_status_internal.clone().unwrap_or_default()) - .unwrap_or_default() - .as_bytes() - .to_vec(), - ); - insert_bytes( - &mut delete_marker.meta_sys, - SUFFIX_REPLICATION_TIMESTAMP, - fi.replication_state_internal - .as_ref() - .map(|v| v.replication_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string()) - .unwrap_or_default() - .as_bytes() - .to_vec(), - ); - } + if let (Some(delete_marker), Some(state)) = (ventry.delete_marker.as_mut(), fi.replication_state_internal.as_ref()) { + persist_delete_marker_replication_state(&mut delete_marker.meta_sys, state); } if !fi.version_purge_status().is_empty() @@ -609,51 +607,8 @@ impl FileMeta { } if let Some(delete_marker) = v.delete_marker.as_mut() { - if !fi.delete_marker_replication_status().is_empty() { - if fi.delete_marker_replication_status() == ReplicationStatusType::Replica { - insert_bytes( - &mut delete_marker.meta_sys, - SUFFIX_REPLICA_STATUS, - fi.replication_state_internal - .as_ref() - .map(|v| v.replica_status.clone()) - .unwrap_or_default() - .as_str() - .as_bytes() - .to_vec(), - ); - insert_bytes( - &mut delete_marker.meta_sys, - SUFFIX_REPLICA_TIMESTAMP, - fi.replication_state_internal - .as_ref() - .map(|v| v.replica_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string()) - .unwrap_or_default() - .as_bytes() - .to_vec(), - ); - } else { - insert_bytes( - &mut delete_marker.meta_sys, - SUFFIX_REPLICATION_STATUS, - fi.replication_state_internal - .as_ref() - .map(|v| v.replication_status_internal.clone().unwrap_or_default()) - .unwrap_or_default() - .as_bytes() - .to_vec(), - ); - insert_bytes( - &mut delete_marker.meta_sys, - SUFFIX_REPLICATION_TIMESTAMP, - fi.replication_state_internal - .as_ref() - .map(|v| v.replication_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string()) - .unwrap_or_default() - .as_bytes() - .to_vec(), - ); - } + if let Some(state) = fi.replication_state_internal.as_ref() { + persist_delete_marker_replication_state(&mut delete_marker.meta_sys, state); } if let Some(state) = fi.replication_state_internal.as_ref() { @@ -773,6 +728,47 @@ impl FileMeta { read_data: bool, include_free_versions: bool, all_parts: bool, + ) -> Result { + self.to_fileinfo_with_part_checksums( + volume, + path, + version_id, + FileInfoDecodeOptions { + read_data, + include_free_versions, + all_parts, + include_part_checksums: true, + }, + ) + } + + pub fn into_fileinfo_without_part_checksums( + &self, + volume: &str, + path: &str, + version_id: &str, + read_data: bool, + include_free_versions: bool, + ) -> Result { + self.to_fileinfo_with_part_checksums( + volume, + path, + version_id, + FileInfoDecodeOptions { + read_data, + include_free_versions, + all_parts: true, + include_part_checksums: false, + }, + ) + } + + fn to_fileinfo_with_part_checksums( + &self, + volume: &str, + path: &str, + version_id: &str, + opts: FileInfoDecodeOptions, ) -> Result { let vid = { if !version_id.is_empty() { @@ -795,7 +791,7 @@ impl FileMeta { if header.free_version() { non_free_versions -= 1; - if include_free_versions + if opts.include_free_versions && found_free_version.is_none() && let Ok(found_free_fi) = ver.parse_version_meta() && found_free_fi.version_type != VersionType::Invalid @@ -806,7 +802,8 @@ impl FileMeta { // Known side effect: if a disk holds only free versions and they are // corrupt, `into_fileinfo` falls through to `FileNotFound` (not // `FileCorrupt`), so that disk is not enqueued for heal. - match found_free_fi.into_fileinfo(volume, path, all_parts) { + match found_free_fi.to_fileinfo_with_part_checksums(volume, path, opts.all_parts, opts.include_part_checksums) + { Ok(mut free_fi) => { free_fi.is_latest = true; found_free_version = Some(free_fi); @@ -834,14 +831,14 @@ impl FileMeta { found = true; - let mut fi = ver.into_fileinfo(volume, path, all_parts)?; + let mut fi = ver.to_fileinfo_with_part_checksums(volume, path, opts.all_parts, opts.include_part_checksums)?; fi.is_latest = is_latest; if let Some(_d) = succ_mod_time { fi.successor_mod_time = succ_mod_time; } - if read_data && fi.inline_data() { + if opts.read_data && fi.inline_data() { fi.data = self.find_inline_data_for_version(fi.version_id)?.map(bytes::Bytes::from); } @@ -850,7 +847,7 @@ impl FileMeta { if !found { if version_id.is_empty() { - if include_free_versions + if opts.include_free_versions && non_free_versions == 0 && let Some(free_version) = found_free_version { @@ -1537,6 +1534,68 @@ mod test { assert_eq!(meta_sys2.len(), 2, "must not create a double-prefixed key"); } + #[test] + fn persist_delete_marker_replication_state_keeps_replica_and_target_statuses() { + let replica_timestamp = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND; + let replication_timestamp = replica_timestamp + time::Duration::SECOND; + let state = ReplicationState { + replica_status: ReplicationStatusType::Replica, + replica_timestamp: Some(replica_timestamp), + replication_status_internal: Some("arn:target=COMPLETED;".to_string()), + replication_timestamp: Some(replication_timestamp), + ..Default::default() + }; + let mut meta_sys = HashMap::new(); + let replica_timestamp_string = replica_timestamp + .format(&Rfc3339) + .expect("timestamp should format as RFC3339"); + let replication_timestamp_string = replication_timestamp + .format(&Rfc3339) + .expect("timestamp should format as RFC3339"); + + persist_delete_marker_replication_state(&mut meta_sys, &state); + + assert_eq!( + rustfs_utils::http::get_bytes(&meta_sys, SUFFIX_REPLICA_STATUS).as_deref(), + Some(b"REPLICA".as_slice()) + ); + assert_eq!( + rustfs_utils::http::get_bytes(&meta_sys, SUFFIX_REPLICA_TIMESTAMP).as_deref(), + Some(replica_timestamp_string.as_bytes()) + ); + assert_eq!( + rustfs_utils::http::get_bytes(&meta_sys, SUFFIX_REPLICATION_STATUS).as_deref(), + Some(b"arn:target=COMPLETED;".as_slice()) + ); + assert_eq!( + rustfs_utils::http::get_bytes(&meta_sys, SUFFIX_REPLICATION_TIMESTAMP).as_deref(), + Some(replication_timestamp_string.as_bytes()) + ); + } + + #[test] + fn persist_delete_marker_replication_timestamp_normalizes_second_offset_to_utc() { + let timestamp = OffsetDateTime::UNIX_EPOCH.to_offset(time::UtcOffset::from_hms(5, 30, 15).expect("valid offset")); + assert_eq!(parse_replication_timestamp(×tamp.to_string()), Some(timestamp)); + let state = ReplicationState { + replica_status: ReplicationStatusType::Replica, + replica_timestamp: Some(timestamp), + replication_status_internal: Some("arn:target=COMPLETED;".to_string()), + replication_timestamp: Some(timestamp), + ..Default::default() + }; + let mut meta_sys = HashMap::new(); + + persist_delete_marker_replication_state(&mut meta_sys, &state); + + for suffix in [SUFFIX_REPLICA_TIMESTAMP, SUFFIX_REPLICATION_TIMESTAMP] { + let persisted = rustfs_utils::http::get_bytes(&meta_sys, suffix).expect("timestamp must be persisted"); + let persisted = std::str::from_utf8(&persisted).expect("timestamp must be UTF-8"); + assert_eq!(parse_replication_timestamp(persisted), Some(timestamp)); + assert_ne!(persisted, OffsetDateTime::UNIX_EPOCH.to_string()); + } + } + /// Regression test for rustfs/rustfs#2715: a corrupted version count in /// xl.meta must yield a decode error instead of sizing a huge allocation /// from the bogus count (which aborts the whole process). diff --git a/crates/filemeta/src/filemeta/version.rs b/crates/filemeta/src/filemeta/version.rs index e01cbf7bb..37c22b125 100644 --- a/crates/filemeta/src/filemeta/version.rs +++ b/crates/filemeta/src/filemeta/version.rs @@ -29,11 +29,12 @@ use super::*; use crate::{ChecksumInfo, TransitionVersionState}; use rustfs_utils::HashAlgorithm; use rustfs_utils::http::{ - RUSTFS_INTERNAL_PREFIX, SUFFIX_CRC, SUFFIX_FREE_VERSION, SUFFIX_INLINE_DATA, SUFFIX_PURGESTATUS, SUFFIX_TIER_FV_ID, + RUSTFS_INTERNAL_PREFIX, SUFFIX_CRC, SUFFIX_FREE_VERSION, SUFFIX_INLINE_DATA, SUFFIX_PART_CHECKSUMS, SUFFIX_PURGESTATUS, + SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX, SUFFIX_REPLICATION_RESET_ARN_PREFIX, SUFFIX_TIER_FV_ID, SUFFIX_TIER_FV_MARKER, SUFFIX_TRANSITION_STATUS, SUFFIX_TRANSITION_TIER, SUFFIX_TRANSITION_TIER_DESTINATION_ID, SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, contains_key_bytes, get_bytes, get_consistent_bytes, get_str, has_internal_suffix, insert_bytes, is_internal_key, remove_bytes, - strip_internal_prefix, target_delete_marker_versions, + strip_internal_prefix, strip_internal_prefix_preserving_case, target_delete_marker_versions, }; const MSGPACK_EXT8: u8 = 0xc7; @@ -258,63 +259,181 @@ fn parse_legacy_uuid_bytes(bytes: &[u8], field: &str) -> Result> { /// Legacy RustFS writes used 16 raw UUID bytes. New writes and MinIO-migrated /// records use the provider's exact UTF-8 version text. Empty, nil UUID, and /// malformed bytes are not usable remote versions. -fn transitioned_version_from_meta_sys(meta_sys: &HashMap>) -> Result> { - if !contains_key_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_ID) { - return Ok(None); - } - let Some(value) = get_consistent_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_ID) else { - return Ok(None); +fn transition_version_state_from_bytes(value: Option<&[u8]>) -> Result { + let Some(value) = value else { + return Ok(TransitionVersionState::Unknown); }; - let value = value.to_vec(); + match value { + b"known-disabled" => Ok(TransitionVersionState::KnownDisabled), + b"suspended-null" => Ok(TransitionVersionState::SuspendedNull), + b"exact" => Ok(TransitionVersionState::Exact), + b"unknown" => Ok(TransitionVersionState::Unknown), + _ => Err(Error::FileCorrupt), + } +} + +fn transitioned_version_from_bytes(value: Option<&[u8]>, state: TransitionVersionState) -> Option { + let value = value?; if value.is_empty() { - return Ok(None); + return None; } - if let Ok(id) = Uuid::from_slice(&value) { - return Ok((!id.is_nil()).then(|| id.to_string())); + if state == TransitionVersionState::Unknown + && let Ok(id) = Uuid::from_slice(value) + { + return (!id.is_nil()).then(|| id.to_string()); } - let Ok(value) = String::from_utf8(value) else { - return Ok(None); + let Ok(value) = std::str::from_utf8(value) else { + return None; }; if value.is_empty() || value.len() > MAX_TRANSITION_VERSION_LEN || value.chars().any(char::is_control) - || Uuid::parse_str(&value).is_ok_and(|id| id.is_nil()) + || Uuid::parse_str(value).is_ok_and(|id| id.is_nil()) { - Ok(None) + None } else { - Ok(Some(value)) + Some(value.to_string()) } } -fn transition_version_state_from_meta_sys( - meta_sys: &HashMap>, - version: Option<&str>, -) -> Result { - if !contains_key_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE) { - return Ok(TransitionVersionState::Unknown); - } - let value = get_consistent_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE).ok_or(Error::FileCorrupt)?; - let state = match value { - b"known-disabled" => TransitionVersionState::KnownDisabled, - b"suspended-null" => TransitionVersionState::SuspendedNull, - b"exact" => TransitionVersionState::Exact, - b"unknown" => TransitionVersionState::Unknown, - _ => return Err(Error::FileCorrupt), - }; +fn validate_transition_version_state(state: TransitionVersionState, version: Option<&str>) -> Result<()> { let valid = match state { TransitionVersionState::Unknown | TransitionVersionState::KnownDisabled => version.is_none(), TransitionVersionState::SuspendedNull => version == Some("null"), TransitionVersionState::Exact => version.is_some_and(|value| value != "null"), }; - valid.then_some(state).ok_or(Error::FileCorrupt) + valid.then_some(()).ok_or(Error::FileCorrupt) } -fn transition_version_state_bytes(state: TransitionVersionState) -> &'static [u8] { - match state { - TransitionVersionState::Unknown => b"unknown", - TransitionVersionState::KnownDisabled => b"known-disabled", - TransitionVersionState::SuspendedNull => b"suspended-null", - TransitionVersionState::Exact => b"exact", +#[derive(Default)] +struct DerivedInternalMetadata<'a> { + checksum: Option<&'a [u8]>, + part_checksums: Option<&'a [u8]>, + transition_status: Option<&'a [u8]>, + transitioned_object: Option<&'a [u8]>, + transitioned_version: Option<&'a [u8]>, + transitioned_version_state: Option<&'a [u8]>, + transition_tier: Option<&'a [u8]>, +} + +impl<'a> DerivedInternalMetadata<'a> { + fn from_meta_sys(meta_sys: &'a HashMap>) -> Result { + let mut canonical = Self::default(); + let mut legacy = Self::default(); + for (key, value) in meta_sys { + let Some(suffix) = rustfs_utils::http::strip_internal_prefix_preserving_case(key) else { + continue; + }; + let (canonical_slot, legacy_slot, expected_suffix) = if suffix.eq_ignore_ascii_case(SUFFIX_CRC) { + (&mut canonical.checksum, &mut legacy.checksum, SUFFIX_CRC) + } else if suffix.eq_ignore_ascii_case(SUFFIX_PART_CHECKSUMS) { + (&mut canonical.part_checksums, &mut legacy.part_checksums, SUFFIX_PART_CHECKSUMS) + } else if suffix.eq_ignore_ascii_case(SUFFIX_TRANSITION_STATUS) { + (&mut canonical.transition_status, &mut legacy.transition_status, SUFFIX_TRANSITION_STATUS) + } else if suffix.eq_ignore_ascii_case(SUFFIX_TRANSITIONED_OBJECTNAME) { + ( + &mut canonical.transitioned_object, + &mut legacy.transitioned_object, + SUFFIX_TRANSITIONED_OBJECTNAME, + ) + } else if suffix.eq_ignore_ascii_case(SUFFIX_TRANSITIONED_VERSION_ID) { + ( + &mut canonical.transitioned_version, + &mut legacy.transitioned_version, + SUFFIX_TRANSITIONED_VERSION_ID, + ) + } else if suffix.eq_ignore_ascii_case(SUFFIX_TRANSITIONED_VERSION_STATE) { + ( + &mut canonical.transitioned_version_state, + &mut legacy.transitioned_version_state, + SUFFIX_TRANSITIONED_VERSION_STATE, + ) + } else if suffix.eq_ignore_ascii_case(SUFFIX_TRANSITION_TIER) { + (&mut canonical.transition_tier, &mut legacy.transition_tier, SUFFIX_TRANSITION_TIER) + } else { + continue; + }; + let slot = if suffix == expected_suffix + && (key.starts_with(RUSTFS_INTERNAL_PREFIX) || key.starts_with(rustfs_utils::http::MINIO_INTERNAL_PREFIX)) + { + canonical_slot + } else { + legacy_slot + }; + if slot.is_some_and(|current| current != value.as_slice()) { + return Err(Error::FileCorrupt); + } + *slot = Some(value.as_slice()); + } + Ok(Self { + checksum: canonical.checksum.or(legacy.checksum), + part_checksums: canonical.part_checksums.or(legacy.part_checksums), + transition_status: canonical.transition_status.or(legacy.transition_status), + transitioned_object: canonical.transitioned_object.or(legacy.transitioned_object), + transitioned_version: canonical.transitioned_version.or(legacy.transitioned_version), + transitioned_version_state: canonical.transitioned_version_state.or(legacy.transitioned_version_state), + transition_tier: canonical.transition_tier.or(legacy.transition_tier), + }) + } +} + +struct UniquePartChecksums(HashMap); + +impl<'de> serde::Deserialize<'de> for UniquePartChecksums { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct UniquePartChecksumsVisitor; + + impl<'de> serde::de::Visitor<'de> for UniquePartChecksumsVisitor { + type Value = UniquePartChecksums; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("an array of unique checksum name and value pairs") + } + + fn visit_seq(self, mut seq: A) -> std::result::Result + where + A: serde::de::SeqAccess<'de>, + { + let mut checksums = HashMap::with_capacity(seq.size_hint().unwrap_or_default()); + while let Some((key, value)) = seq.next_element::<(String, String)>()? { + if checksums.insert(key, value).is_some() { + return Err(serde::de::Error::custom("duplicate part checksum name")); + } + } + Ok(UniquePartChecksums(checksums)) + } + } + + deserializer.deserialize_seq(UniquePartChecksumsVisitor) + } +} + +impl FileInfo { + pub fn hydrate_data_movement_part_checksums(&mut self) -> Result<()> { + let present = self + .metadata + .keys() + .any(|key| has_internal_suffix(key, SUFFIX_PART_CHECKSUMS)); + if !present { + return Ok(()); + } + let encoded = rustfs_utils::http::get_consistent_str(&self.metadata, SUFFIX_PART_CHECKSUMS).ok_or(Error::FileCorrupt)?; + let persisted = serde_json::from_str::>(encoded).map_err(|_| Error::FileCorrupt)?; + let mut part_indices = HashMap::with_capacity(self.parts.len()); + for (index, part) in self.parts.iter().enumerate() { + if part_indices.insert(part.number, index).is_some() { + return Err(Error::FileCorrupt); + } + } + for (part_number, UniquePartChecksums(checksums)) in persisted { + let index = part_indices.remove(&part_number).ok_or(Error::FileCorrupt)?; + let part = self.parts.get_mut(index).ok_or(Error::FileCorrupt)?; + part.checksums = Some(checksums); + } + Ok(()) } } @@ -322,21 +441,10 @@ fn set_transition_version_state(meta_sys: &mut HashMap>, state: if state == TransitionVersionState::Unknown { remove_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE); } else { - insert_bytes( - meta_sys, - SUFFIX_TRANSITIONED_VERSION_STATE, - transition_version_state_bytes(state).to_vec(), - ); + insert_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE, state.as_str().as_bytes().to_vec()); } } -fn legacy_transitioned_version_id_from_meta_sys(meta_sys: &HashMap>) -> Option { - transitioned_version_from_meta_sys(meta_sys) - .ok() - .flatten() - .and_then(|value| Uuid::parse_str(&value).ok()) -} - fn transitioned_version_bytes(fi: &FileInfo) -> Option> { fi.transition_version .as_ref() @@ -448,6 +556,17 @@ impl FileMetaShallowVersion { pub fn into_fileinfo(&self, volume: &str, path: &str, all_parts: bool) -> Result { self.parse_version_meta()?.into_fileinfo(volume, path, all_parts) } + + pub(super) fn to_fileinfo_with_part_checksums( + &self, + volume: &str, + path: &str, + all_parts: bool, + include_part_checksums: bool, + ) -> Result { + self.parse_version_meta()? + .to_fileinfo_with_part_checksums(volume, path, all_parts, include_part_checksums) + } } impl TryFrom for FileMetaShallowVersion { @@ -769,8 +888,16 @@ impl FileMetaVersion { } pub fn into_fileinfo(&self, volume: &str, path: &str, all_parts: bool) -> Result { - // Only the Object arm carries part arrays and can fail the length guard; the - // Legacy and Delete arms have no part arrays and stay infallible. + self.to_fileinfo_with_part_checksums(volume, path, all_parts, true) + } + + pub(super) fn to_fileinfo_with_part_checksums( + &self, + volume: &str, + path: &str, + all_parts: bool, + include_part_checksums: bool, + ) -> Result { let mut fi = match self.version_type { VersionType::Invalid | VersionType::Legacy => { if let Some(ref legacy) = self.legacy_object { @@ -788,14 +915,14 @@ impl FileMetaVersion { self.object .as_ref() .unwrap_or(&default_object) - .into_fileinfo(volume, path, all_parts)? + .to_fileinfo_with_part_checksums(volume, path, all_parts, include_part_checksums)? } VersionType::Delete => { let default_marker = MetaDeleteMarker::default(); self.delete_marker .as_ref() .unwrap_or(&default_marker) - .into_fileinfo(volume, path, all_parts) + .into_fileinfo(volume, path, all_parts)? } }; fi.uses_legacy_checksum = self.uses_legacy_checksum; @@ -2390,7 +2517,18 @@ impl MetaObject { } pub fn into_fileinfo(&self, volume: &str, path: &str, all_parts: bool) -> Result { + self.to_fileinfo_with_part_checksums(volume, path, all_parts, true) + } + + fn to_fileinfo_with_part_checksums( + &self, + volume: &str, + path: &str, + all_parts: bool, + include_part_checksums: bool, + ) -> Result { let version_id = self.version_id.filter(|&vid| !vid.is_nil()); + let derived_metadata = DerivedInternalMetadata::from_meta_sys(&self.meta_sys)?; let parts = if all_parts { let n = self.part_numbers.len(); @@ -2474,7 +2612,10 @@ impl MetaObject { } } - let checksum = get_bytes(&self.meta_sys, SUFFIX_CRC).map(Bytes::from); + let checksum = derived_metadata + .checksum + .filter(|checksum| !checksum.is_empty()) + .map(Bytes::copy_from_slice); let erasure = ErasureInfo { algorithm: self.erasure_algorithm.to_string(), @@ -2486,20 +2627,29 @@ impl MetaObject { ..Default::default() }; - let transition_status = get_bytes(&self.meta_sys, SUFFIX_TRANSITION_STATUS) - .map(|v| String::from_utf8_lossy(&v).to_string()) + let transition_status = derived_metadata + .transition_status + .filter(|value| !value.is_empty()) + .map(|v| String::from_utf8_lossy(v).to_string()) .unwrap_or_default(); - let transitioned_objname = get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_OBJECTNAME) - .map(|v| String::from_utf8_lossy(&v).to_string()) + let transitioned_objname = derived_metadata + .transitioned_object + .filter(|value| !value.is_empty()) + .map(|v| String::from_utf8_lossy(v).to_string()) .unwrap_or_default(); - let transition_version = transitioned_version_from_meta_sys(&self.meta_sys)?; - let transition_version_state = transition_version_state_from_meta_sys(&self.meta_sys, transition_version.as_deref())?; + let transition_version_state = transition_version_state_from_bytes(derived_metadata.transitioned_version_state)?; + let transition_version = transitioned_version_from_bytes(derived_metadata.transitioned_version, transition_version_state); + if derived_metadata.transitioned_version_state.is_some() { + validate_transition_version_state(transition_version_state, transition_version.as_deref())?; + } let transition_version_id = transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok()); - let transition_tier = get_bytes(&self.meta_sys, SUFFIX_TRANSITION_TIER) - .map(|v| String::from_utf8_lossy(&v).to_string()) + let transition_tier = derived_metadata + .transition_tier + .filter(|value| !value.is_empty()) + .map(|v| String::from_utf8_lossy(v).to_string()) .unwrap_or_default(); - Ok(FileInfo { + let mut file_info = FileInfo { version_id, erasure, data_dir: self.data_dir, @@ -2519,7 +2669,11 @@ impl MetaObject { transition_version_state, transition_tier, ..Default::default() - }) + }; + if all_parts && include_part_checksums { + file_info.hydrate_data_movement_part_checksums()?; + } + Ok(file_info) } pub fn set_transition(&mut self, fi: &FileInfo) { @@ -2710,39 +2864,31 @@ fn get_internal_replication_state(metadata: &HashMap) -> Option< continue; } - let sub_key_opt = strip_internal_prefix(k); - if let Some(ref sub_key) = sub_key_opt { - match sub_key.as_str() { - "replica-timestamp" => { - has = true; - rs.replica_timestamp = Some(OffsetDateTime::parse(v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)); - } - "replica-status" => { - has = true; - rs.replica_status = ReplicationStatusType::from(v.as_str()); - } - "replication-timestamp" => { - has = true; - rs.replication_timestamp = Some(OffsetDateTime::parse(v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)) - } - "replication-status" => { - has = true; - rs.replication_status_internal = Some(v.clone()); - rs.targets = replication_statuses_map(v.as_str()); - } - _ => { - if let Some(arn) = sub_key.strip_prefix("replication-reset-") { - has = true; - // Store the canonical full-header key so the map matches - // the key `target_reset_header()` produces on the - // write/lookup side. Storing the bare ARN keyed the map - // inconsistently (bare on read, full on write), which - // could drop reset state across merge/reflatten cycles - // (backlog#799 B16). - rs.reset_statuses_map - .insert(crate::replication::target_reset_header(arn), v.clone()); - } - } + if let Some(sub_key) = strip_internal_prefix_preserving_case(k) { + if sub_key.eq_ignore_ascii_case(SUFFIX_REPLICA_TIMESTAMP) { + has = true; + rs.replica_timestamp = Some(parse_replication_timestamp(v).unwrap_or(OffsetDateTime::UNIX_EPOCH)); + } else if sub_key.eq_ignore_ascii_case(SUFFIX_REPLICA_STATUS) { + has = true; + rs.replica_status = ReplicationStatusType::from(v.as_str()); + } else if sub_key.eq_ignore_ascii_case(SUFFIX_REPLICATION_TIMESTAMP) { + has = true; + rs.replication_timestamp = Some(parse_replication_timestamp(v).unwrap_or(OffsetDateTime::UNIX_EPOCH)) + } else if sub_key.eq_ignore_ascii_case(SUFFIX_REPLICATION_STATUS) { + has = true; + rs.replication_status_internal = Some(v.clone()); + rs.targets = replication_statuses_map(v.as_str()); + } else if let Some(arn) = rustfs_utils::http::internal_key_strip_suffix_prefix(k, SUFFIX_REPLICATION_RESET_ARN_PREFIX) + { + has = true; + // Store the canonical full-header key so the map matches + // the key `target_reset_header()` produces on the + // write/lookup side. Storing the bare ARN keyed the map + // inconsistently (bare on read, full on write), which + // could drop reset state across merge/reflatten cycles + // (backlog#799 B16). + rs.reset_statuses_map + .insert(crate::replication::target_reset_header(&arn), v.clone()); } } } @@ -2786,7 +2932,7 @@ impl MetaDeleteMarker { contains_key_bytes(&self.meta_sys, SUFFIX_FREE_VERSION) } - pub fn into_fileinfo(&self, volume: &str, path: &str, _all_parts: bool) -> FileInfo { + pub fn into_fileinfo(&self, volume: &str, path: &str, _all_parts: bool) -> Result { let metadata = self .meta_sys .clone() @@ -2808,22 +2954,27 @@ impl MetaDeleteMarker { if self.free_version() { fi.set_tier_free_version(); - fi.transition_tier = get_bytes(&self.meta_sys, SUFFIX_TRANSITION_TIER) - .map(|v| String::from_utf8_lossy(&v).to_string()) + let derived_metadata = DerivedInternalMetadata::from_meta_sys(&self.meta_sys)?; + fi.transition_tier = derived_metadata + .transition_tier + .filter(|value| !value.is_empty()) + .map(|value| String::from_utf8_lossy(value).to_string()) .unwrap_or_default(); - - fi.transitioned_objname = get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_OBJECTNAME) - .map(|v| String::from_utf8_lossy(&v).to_string()) + fi.transitioned_objname = derived_metadata + .transitioned_object + .filter(|value| !value.is_empty()) + .map(|value| String::from_utf8_lossy(value).to_string()) .unwrap_or_default(); - - fi.transition_version = transitioned_version_from_meta_sys(&self.meta_sys).ok().flatten(); - fi.transition_version_id = legacy_transitioned_version_id_from_meta_sys(&self.meta_sys); - fi.transition_version_state = - transition_version_state_from_meta_sys(&self.meta_sys, fi.transition_version.as_deref()) - .unwrap_or(TransitionVersionState::Unknown); + fi.transition_version_state = transition_version_state_from_bytes(derived_metadata.transitioned_version_state)?; + fi.transition_version = + transitioned_version_from_bytes(derived_metadata.transitioned_version, fi.transition_version_state); + fi.transition_version_id = fi.transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok()); + if derived_metadata.transitioned_version_state.is_some() { + validate_transition_version_state(fi.transition_version_state, fi.transition_version.as_deref())?; + } } - fi + Ok(fi) } pub fn encode_to(&self, wr: &mut W) -> Result<()> { @@ -2941,6 +3092,13 @@ impl From for MetaDeleteMarker { if !is_internal_key(key) || is_skip_meta_key(key) { continue; } + if rustfs_utils::http::internal_key_strip_suffix_prefix(key, SUFFIX_REPLICATION_RESET_ARN_PREFIX).is_some() + || rustfs_utils::http::internal_key_strip_suffix_prefix(key, SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX) + .is_some() + { + meta_sys.insert(key.clone(), metadata_value.as_bytes().to_vec()); + continue; + } let Some(suffix) = strip_internal_prefix(key) else { continue; }; @@ -2982,6 +3140,11 @@ impl From for MetaDeleteMarker { if !value.transition_tier.is_empty() { insert_bytes(&mut meta_sys, SUFFIX_TRANSITION_TIER, value.transition_tier.as_bytes().to_vec()); } + if let Some(state) = value.replication_state_internal.as_ref() { + persist_delete_marker_replication_state(&mut meta_sys, state); + persist_reset_statuses(&mut meta_sys, &state.reset_statuses_map); + persist_target_delete_marker_versions(&mut meta_sys, &state.target_delete_marker_version_ids, &value.metadata); + } Self { version_id: value.version_id, mod_time: value.mod_time, @@ -3278,6 +3441,7 @@ pub fn file_info_from_raw( FileInfoOpts { data: read_data, include_free_versions, + include_part_checksums: true, }, ) } @@ -3285,6 +3449,7 @@ pub fn file_info_from_raw( pub struct FileInfoOpts { pub data: bool, pub include_free_versions: bool, + pub include_part_checksums: bool, } pub fn get_file_info(buf: &[u8], volume: &str, path: &str, version_id: &str, opts: FileInfoOpts) -> Result { @@ -3309,7 +3474,11 @@ pub fn get_file_info(buf: &[u8], volume: &str, path: &str, version_id: &str, opt }); } - let fi = meta.into_fileinfo(volume, path, version_id, opts.data, opts.include_free_versions, true)?; + let fi = if opts.include_part_checksums { + meta.into_fileinfo(volume, path, version_id, opts.data, opts.include_free_versions, true)? + } else { + meta.into_fileinfo_without_part_checksums(volume, path, version_id, opts.data, opts.include_free_versions)? + }; Ok(fi) } @@ -3555,6 +3724,56 @@ mod tests { assert!(!converted.meta_sys.contains_key("content-type")); } + #[test] + fn delete_marker_conversion_does_not_lowercase_dynamic_replication_targets() { + let arn = "arn:rustfs:replication:us-east-1:TenantA:bucket"; + let reset_suffix = format!("{SUFFIX_REPLICATION_RESET_ARN_PREFIX}{arn}"); + let version_suffix = format!("{SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX}{arn}"); + let mut marker = FileInfo::default(); + marker.metadata.insert( + format!("{}{reset_suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX), + "2026-08-12T00:00:00Z;COMPLETED".to_string(), + ); + marker.metadata.insert( + format!("{}{version_suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX), + "remote-version".to_string(), + ); + marker.replication_state_internal = get_internal_replication_state(&marker.metadata); + + let converted = MetaDeleteMarker::from(marker); + + assert!(converted.meta_sys.keys().any(|key| key.ends_with(&reset_suffix))); + assert!(converted.meta_sys.keys().any(|key| key.ends_with(&version_suffix))); + assert!(!converted.meta_sys.keys().any(|key| key.contains("tenanta"))); + + let mut conflicting = FileInfo::default(); + conflicting + .metadata + .insert(format!("{RUSTFS_INTERNAL_PREFIX}{version_suffix}"), "remote-version-a".to_string()); + conflicting.metadata.insert( + format!("{}{version_suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX), + "remote-version-b".to_string(), + ); + conflicting.replication_state_internal = get_internal_replication_state(&conflicting.metadata); + assert!( + conflicting + .replication_state_internal + .as_ref() + .is_some_and(|state| state.target_delete_marker_version_ids_corrupt) + ); + + let roundtrip = MetaDeleteMarker::from(conflicting) + .into_fileinfo("bucket", "object", false) + .expect("dynamic replication aliases should remain decodable"); + assert!( + roundtrip + .replication_state_internal + .as_ref() + .is_some_and(|state| state.target_delete_marker_version_ids_corrupt), + "conflicting dynamic aliases must remain corrupt across persistence" + ); + } + #[derive(Serialize)] enum LegacyDeleteVersionTypeFixture { #[serde(rename = "DeleteMarker")] @@ -3663,6 +3882,138 @@ mod tests { assert!(matches!(res, Err(Error::FileCorrupt)), "short part_sizes must map to FileCorrupt"); } + #[test] + fn into_fileinfo_rejects_conflicting_derived_internal_aliases() { + for suffix in [ + SUFFIX_CRC, + SUFFIX_TRANSITION_STATUS, + SUFFIX_TRANSITIONED_OBJECTNAME, + SUFFIX_TRANSITIONED_VERSION_ID, + SUFFIX_TRANSITIONED_VERSION_STATE, + SUFFIX_TRANSITION_TIER, + ] { + let mut meta_sys = HashMap::from([(format!("{RUSTFS_INTERNAL_PREFIX}{suffix}"), vec![0xff, 1])]); + meta_sys.insert(format!("{}{suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX), vec![0xfe, 2]); + let object = MetaObject { + meta_sys, + ..Default::default() + }; + + assert_eq!( + object + .into_fileinfo("bucket", "key", false) + .expect_err("conflicting aliases must fail closed"), + Error::FileCorrupt, + "suffix {suffix}" + ); + } + } + + #[test] + fn into_fileinfo_recovers_noncanonical_binary_checksum_alias() { + let checksum = vec![0xff, 0x00, 0x80, 0x01]; + let object = MetaObject { + meta_sys: HashMap::from([("X-Minio-Internal-crc".to_string(), checksum.clone())]), + ..Default::default() + }; + + let file_info = object + .into_fileinfo("bucket", "key", false) + .expect("a single legacy checksum alias should remain readable"); + + assert_eq!(file_info.checksum.as_deref(), Some(checksum.as_slice())); + } + + #[test] + fn into_fileinfo_prefers_canonical_rewrite_over_stale_mixed_case_alias() { + let checksum = vec![0xff, 0x00, 0x80, 0x01]; + let mut object = MetaObject { + meta_sys: HashMap::from([("X-Minio-Internal-crc".to_string(), b"stale".to_vec())]), + ..Default::default() + }; + insert_bytes(&mut object.meta_sys, SUFFIX_CRC, checksum.clone()); + + let file_info = object + .into_fileinfo("bucket", "key", false) + .expect("canonical rewrites should supersede legacy mixed-case aliases"); + + assert_eq!(file_info.checksum.as_deref(), Some(checksum.as_slice())); + } + + #[test] + fn into_fileinfo_recovers_data_movement_part_checksums() { + let mut object = object_with_parts(vec![1], vec![16], vec![16]); + insert_bytes(&mut object.meta_sys, SUFFIX_PART_CHECKSUMS, br#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_vec()); + + let file_info = object + .into_fileinfo("bucket", "key", true) + .expect("data movement part checksums should decode"); + + assert_eq!( + file_info.parts[0] + .checksums + .as_ref() + .and_then(|checksums| checksums.get("CRC32C")) + .map(String::as_str), + Some("AAAAAA==") + ); + + let mut deferred = object + .to_fileinfo_with_part_checksums("bucket", "key", true, false) + .expect("quorum candidates should retain raw checksum metadata"); + assert!(deferred.parts[0].checksums.is_none()); + deferred + .hydrate_data_movement_part_checksums() + .expect("the selected candidate should hydrate checksums once"); + assert_eq!(deferred.parts[0].checksums, file_info.parts[0].checksums); + + insert_bytes(&mut object.meta_sys, SUFFIX_PART_CHECKSUMS, b"not-json".to_vec()); + assert_eq!( + object + .into_fileinfo("bucket", "key", true) + .expect_err("malformed data movement part checksums must fail closed"), + Error::FileCorrupt + ); + + for encoded in [ + br#"[[1,[["CRC32C","AAAAAA=="]]],[1,[["CRC32C","BBBBBB=="]]]]"#.as_slice(), + br#"[[1,[["CRC32C","AAAAAA=="],["CRC32C","BBBBBB=="]]]]"#.as_slice(), + ] { + insert_bytes(&mut object.meta_sys, SUFFIX_PART_CHECKSUMS, encoded.to_vec()); + assert_eq!( + object + .into_fileinfo("bucket", "key", true) + .expect_err("duplicate part checksum keys must fail closed"), + Error::FileCorrupt + ); + } + + insert_bytes(&mut object.meta_sys, SUFFIX_PART_CHECKSUMS, br#"[[2,[["CRC32C","AAAAAA=="]]]]"#.to_vec()); + assert_eq!( + object + .into_fileinfo("bucket", "key", true) + .expect_err("a sidecar for an unknown part must fail closed"), + Error::FileCorrupt + ); + + object.meta_sys = HashMap::from([ + ( + format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_PART_CHECKSUMS}"), + br#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_vec(), + ), + ( + format!("{}{}", rustfs_utils::http::MINIO_INTERNAL_PREFIX, SUFFIX_PART_CHECKSUMS), + br#"[[1,[["CRC32C","BBBBBB=="]]]]"#.to_vec(), + ), + ]); + assert_eq!( + object + .into_fileinfo("bucket", "key", true) + .expect_err("conflicting sidecar aliases must fail closed"), + Error::FileCorrupt + ); + } + #[test] fn into_fileinfo_rejects_short_part_actual_sizes_including_empty() { let obj = object_with_parts(vec![1, 2], vec![10, 20], vec![]); @@ -4230,6 +4581,31 @@ mod tests { assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown); } + #[test] + fn meta_object_transition_exact_rejects_legacy_raw_uuid_encoding() { + let mut sys = HashMap::new(); + insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, sample_version_id().as_bytes().to_vec()); + insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_STATE, b"exact".to_vec()); + + let err = make_meta_object_with_sys(sys) + .into_fileinfo("b", "k", false) + .expect_err("exact remote versions must use their UTF-8 provider representation"); + + assert_eq!(err, Error::FileCorrupt); + } + + #[test] + fn meta_object_transition_version_id_mixed_case_alias_is_recovered() { + let id = sample_version_id(); + let sys = HashMap::from([("X-Minio-Internal-transitioned-versionID".to_string(), id.as_bytes().to_vec())]); + let fi = make_meta_object_with_sys(sys) + .into_fileinfo("b", "k", false) + .expect("a legacy mixed-case transition version alias should decode"); + + assert_eq!(fi.transition_version_id, Some(id)); + assert_eq!(fi.transition_version, Some(id.to_string())); + } + #[test] fn meta_object_transition_version_id_opaque_text_is_preserved() { let mut sys = HashMap::new(); @@ -4271,8 +4647,10 @@ mod tests { .map(Vec::as_slice), Some(b"exact".as_slice()) ); + let persisted_version = get_consistent_bytes(&object.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID); assert_eq!( - legacy_transitioned_version_id_from_meta_sys(&object.meta_sys), + transitioned_version_from_bytes(persisted_version, TransitionVersionState::Unknown) + .and_then(|value| Uuid::parse_str(&value).ok()), Some(id), "UUID exact writes must remain readable by the legacy UUID consumer" ); @@ -4281,6 +4659,26 @@ mod tests { assert_eq!(decoded.transition_version.as_deref(), Some(expected_version.as_str())); } + #[test] + fn meta_object_transition_version_state_exact_preserves_sixteen_byte_opaque_text() { + let expected_version = "opaque.wasabi_01"; + assert_eq!(expected_version.len(), 16); + let fi = FileInfo { + transition_status: "complete".to_string(), + transition_version: Some(expected_version.to_string()), + transition_version_state: TransitionVersionState::Exact, + ..Default::default() + }; + + let decoded = MetaObject::from(fi) + .into_fileinfo("b", "k", false) + .expect("exact opaque transition version should round trip"); + + assert_eq!(decoded.transition_version.as_deref(), Some(expected_version)); + assert_eq!(decoded.transition_version_id, None); + assert_eq!(decoded.transition_version_state, TransitionVersionState::Exact); + } + #[test] fn set_transition_known_disabled_removes_stale_version_dual_keys() { let mut meta_sys = HashMap::new(); @@ -4375,7 +4773,8 @@ mod tests { mod_time: None, meta_sys: sys, } - .into_fileinfo("b", "k", false); + .into_fileinfo("b", "k", false) + .expect("nil tier version should remain an absent remote version"); assert_eq!(fi.transition_version_id, None); } @@ -4390,7 +4789,8 @@ mod tests { mod_time: None, meta_sys: sys, } - .into_fileinfo("b", "k", false); + .into_fileinfo("b", "k", false) + .expect("legacy binary UUID tier version should decode"); assert_eq!(fi.transition_version_id, Some(id)); assert_eq!(fi.transition_version, Some(id.to_string())); } @@ -4407,7 +4807,8 @@ mod tests { mod_time: Some(sample_mod_time()), meta_sys: sys, } - .into_fileinfo("b", "k", false); + .into_fileinfo("b", "k", false) + .expect("opaque tier version should remain readable"); assert_eq!(fi.transition_version_id, None); assert_eq!(fi.transition_version.as_deref(), Some("opaque-generation-42")); @@ -4429,12 +4830,67 @@ mod tests { mod_time: Some(sample_mod_time()), meta_sys: sys, } - .into_fileinfo("b", "k", false); + .into_fileinfo("b", "k", false) + .expect("mixed-case tier aliases should decode"); assert_eq!(fi.transition_version_id, Some(id)); assert_eq!(fi.transition_version, Some(id.to_string())); } + #[test] + fn delete_marker_free_version_recovers_mixed_case_transition_aliases() { + let id = sample_version_id(); + let id_text = id.to_string(); + let mut sys = HashMap::new(); + insert_bytes(&mut sys, SUFFIX_FREE_VERSION, vec![]); + for (suffix, value) in [ + (SUFFIX_TRANSITIONED_VERSION_ID, id_text.as_bytes()), + (SUFFIX_TRANSITIONED_VERSION_STATE, b"exact".as_slice()), + (SUFFIX_TRANSITION_TIER, b"WARM".as_slice()), + (SUFFIX_TRANSITIONED_OBJECTNAME, b"remote-object".as_slice()), + ] { + sys.insert(format!("X-Minio-Internal-{suffix}"), value.to_vec()); + } + + let fi = MetaDeleteMarker { + version_id: Some(sample_version_id()), + mod_time: Some(sample_mod_time()), + meta_sys: sys, + } + .into_fileinfo("b", "k", false) + .expect("mixed-case tier aliases should decode"); + + assert_eq!(fi.transition_version_id, Some(id)); + assert_eq!(fi.transition_version, Some(id.to_string())); + assert_eq!(fi.transition_version_state, TransitionVersionState::Exact); + assert_eq!(fi.transition_tier, "WARM"); + assert_eq!(fi.transitioned_objname, "remote-object"); + } + + #[test] + fn delete_marker_free_version_rejects_conflicting_transition_aliases() { + let mut sys = HashMap::new(); + insert_bytes(&mut sys, SUFFIX_FREE_VERSION, vec![]); + sys.insert( + format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_ID}"), + b"source-version".to_vec(), + ); + sys.insert( + format!("{}{}", rustfs_utils::http::MINIO_INTERNAL_PREFIX, SUFFIX_TRANSITIONED_VERSION_ID), + b"target-version".to_vec(), + ); + + let err = MetaDeleteMarker { + version_id: Some(sample_version_id()), + mod_time: Some(sample_mod_time()), + meta_sys: sys, + } + .into_fileinfo("b", "k", false) + .expect_err("conflicting transition aliases must fail closed"); + + assert_eq!(err, Error::FileCorrupt); + } + #[test] fn version_header_sorts_before_prefers_object_over_delete_marker_on_equal_mod_time() { let object = FileMetaVersionHeader { @@ -4758,7 +5214,7 @@ mod tests { #[test] fn target_delete_marker_version_metadata_is_forward_and_backward_compatible() { - let arn = "arn:rustfs:replication:us-east-1:target:bucket"; + let arn = "arn:rustfs:replication:us-east-1:TenantA:bucket"; let suffix = format!("{}{arn}", rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX); let mut metadata = HashMap::from([(format!("{RUSTFS_INTERNAL_PREFIX}replication-status"), format!("{arn}=COMPLETED;"))]); @@ -4803,7 +5259,7 @@ mod tests { // must keep it keyed by `target_reset_header(arn)` (not the bare ARN) so // `ReplicationState::target_state` finds it after a round trip // (backlog#799 B16). - let arn = "arn:rustfs:replication:us-east-1:target:bucket"; + let arn = "arn:rustfs:replication:us-east-1:TenantA:bucket"; let ts = "2026-06-30T00:00:00Z;reset-1".to_string(); let key = crate::replication::target_reset_header(arn); let mut metadata = HashMap::new(); @@ -4822,6 +5278,33 @@ mod tests { ); } + #[test] + fn get_internal_replication_state_accepts_rfc3339_and_rustfs_display_timestamps() { + let replica_timestamp = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND; + let replication_timestamp = replica_timestamp + time::Duration::SECOND; + let metadata = HashMap::from([ + (format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_REPLICA_STATUS}"), "REPLICA".to_string()), + ( + format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_REPLICA_TIMESTAMP}"), + replica_timestamp.to_string(), + ), + ( + format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_REPLICATION_STATUS}"), + "arn:rustfs:replication:us-east-1:TenantA:bucket=COMPLETED;".to_string(), + ), + ( + format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_REPLICATION_TIMESTAMP}"), + replication_timestamp + .format(&Rfc3339) + .expect("RFC3339 timestamp should format"), + ), + ]); + + let state = get_internal_replication_state(&metadata).expect("replication metadata should parse"); + assert_eq!(state.replica_timestamp, Some(replica_timestamp)); + assert_eq!(state.replication_timestamp, Some(replication_timestamp)); + } + // ---- Header signature (backlog#861 / B12) ---- fn signed_object() -> MetaObject { diff --git a/crates/filemeta/src/metacache.rs b/crates/filemeta/src/metacache.rs index ab602e396..462ce6fae 100644 --- a/crates/filemeta/src/metacache.rs +++ b/crates/filemeta/src/metacache.rs @@ -157,7 +157,7 @@ impl MetaCacheEntry { }); } - let fi = fm.into_fileinfo(bucket, self.name.as_str(), "", false, false, true)?; + let fi = fm.into_fileinfo_without_part_checksums(bucket, self.name.as_str(), "", false, false)?; return Ok(fi); } @@ -169,6 +169,7 @@ impl MetaCacheEntry { FileInfoOpts { data: false, include_free_versions: false, + include_part_checksums: false, }, ) } @@ -2157,4 +2158,38 @@ mod tests { let inner = caught.expect("file_info_versions must not panic"); assert!(matches!(inner, Err(Error::FileCorrupt)), "expected FileCorrupt"); } + + #[test] + fn metacache_to_fileinfo_ignores_part_checksum_sidecar_regardless_of_cache_state() { + let mut meta = FileMeta::load(&create_real_xlmeta().expect("create real xl.meta")).expect("load real xl.meta"); + let version_id = Uuid::parse_str("01234567-89ab-cdef-0123-456789abcdef").expect("valid fixture version id"); + let (index, mut version) = meta.find_version(Some(version_id)).expect("find fixture object version"); + rustfs_utils::http::insert_bytes( + &mut version.object.as_mut().expect("fixture object").meta_sys, + rustfs_utils::http::SUFFIX_PART_CHECKSUMS, + b"not-json".to_vec(), + ); + meta.versions[index] = FileMetaShallowVersion::try_from(version).expect("replace fixture object version"); + let encoded = meta.marshal_msg().expect("marshal object metadata"); + + let uncached = MetaCacheEntry { + name: "object".to_string(), + metadata: encoded.clone(), + cached: None, + reusable: false, + } + .to_fileinfo("bucket") + .expect("uncached metacache conversion must stay lazy"); + let cached = MetaCacheEntry { + name: "object".to_string(), + metadata: encoded, + cached: Some(meta), + reusable: false, + } + .to_fileinfo("bucket") + .expect("cached metacache conversion must stay lazy"); + + assert_eq!(cached, uncached); + assert!(cached.parts.iter().all(|part| part.checksums.is_none())); + } } diff --git a/crates/rio-v2/tests/minio_generated_fixtures.rs b/crates/rio-v2/tests/minio_generated_fixtures.rs index d1181e923..78888ced4 100644 --- a/crates/rio-v2/tests/minio_generated_fixtures.rs +++ b/crates/rio-v2/tests/minio_generated_fixtures.rs @@ -85,6 +85,7 @@ fn load_file_info(case_id: &str) -> FileInfo { FileInfoOpts { data: false, include_free_versions: true, + include_part_checksums: true, }, ) .unwrap_or_else(|err| panic!("decode {}: {err}", xl_meta.display())) diff --git a/crates/utils/src/http/metadata_compat.rs b/crates/utils/src/http/metadata_compat.rs index b72eaadd5..bc71f81ec 100644 --- a/crates/utils/src/http/metadata_compat.rs +++ b/crates/utils/src/http/metadata_compat.rs @@ -30,6 +30,10 @@ pub const MINIO_INTERNAL_PREFIX: &str = "x-minio-internal-"; // Key suffixes (lowercase, no prefix) pub const SUFFIX_INLINE_DATA: &str = "inline-data"; pub const SUFFIX_DATA_MOVED: &str = "data-moved"; +/// Tags snapshot bound to a data-movement-owned target. +pub const SUFFIX_DATA_MOVED_TAGS: &str = "data-moved-tags"; +/// Internal ownership marker for a data-movement multipart upload. +pub const SUFFIX_DATA_MOVEMENT_UPLOAD: &str = "data-movement-upload"; /// Transient flag for data movement pub const SUFFIX_DATA_MOV: &str = "data-mov"; /// Transient flag for healing @@ -44,6 +48,8 @@ pub const SUFFIX_ACTUAL_OBJECT_SIZE: &str = "actual-object-size"; /// Used by replication; key stored with capital A pub const SUFFIX_ACTUAL_OBJECT_SIZE_CAP: &str = "Actual-Object-Size"; pub const SUFFIX_CRC: &str = "crc"; +/// JSON-encoded per-part S3 checksum maps retained across raw data movement. +pub const SUFFIX_PART_CHECKSUMS: &str = "part-checksums"; pub const SUFFIX_TRANSITION_STATUS: &str = "transition-status"; pub const SUFFIX_TRANSITIONED_OBJECTNAME: &str = "transitioned-object"; pub const SUFFIX_TRANSITIONED_VERSION_ID: &str = "transitioned-versionID"; @@ -132,8 +138,10 @@ pub fn internal_key_starts_with(key: &str, suffix_prefix: &str) -> bool { /// For keys like x-rustfs-internal-replication-reset-{arn}, strips the internal prefix and suffix_prefix, /// returning the remainder (e.g. "arn1"). Returns None if key does not match. pub fn internal_key_strip_suffix_prefix(key: &str, suffix_prefix: &str) -> Option { - let rest = strip_internal_prefix(key)?; - rest.strip_prefix(suffix_prefix).map(|s| s.to_string()) + let rest = strip_internal_prefix_preserving_case(key)?; + rest.get(..suffix_prefix.len()) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case(suffix_prefix)) + .then(|| rest[suffix_prefix.len()..].to_string()) } fn both_keys(suffix: &str) -> (String, String) { @@ -529,6 +537,18 @@ mod tests { assert!(!has_internal_suffix(key, SUFFIX_COMPRESSION)); } + #[test] + fn internal_suffix_prefix_preserves_dynamic_identifier_case() { + assert_eq!( + internal_key_strip_suffix_prefix( + "X-Minio-Internal-Replication-Reset-arn:minio:replication::TenantA:bucket", + SUFFIX_REPLICATION_RESET_ARN_PREFIX, + ) + .as_deref(), + Some("arn:minio:replication::TenantA:bucket") + ); + } + #[test] fn test_get_str_case_insensitive_fallback() { // Non-canonical mixed-case key must still be found via the case-insensitive scan. diff --git a/docs/operations/rolling-restart.md b/docs/operations/rolling-restart.md index 83a52bfef..63b8bb9ee 100644 --- a/docs/operations/rolling-restart.md +++ b/docs/operations/rolling-restart.md @@ -5,9 +5,10 @@ availability, what to expect when several nodes are down at once (sequential cold start), and how to read the degraded-mode signals. Written for the failure pattern reported in rustfs/rustfs#4304. -> Upgrading the binary or container image never changes the on-disk data -> format. Replacing the executable and restarting is safe; no migration step -> runs on startup. +> Upgrading the binary or container image does not change the on-disk data +> format unless an explicitly enabled feature documents a version floor. +> Replacing the executable and restarting does not run a migration step on +> startup. > [!WARNING] > The release that switches local SSE wrapped DEKs from the legacy @@ -19,6 +20,17 @@ failure pattern reported in rustfs/rustfs#4304. > and then resume traffic. Downgrading or rolling back after new encrypted > objects are written is not supported. +> [!WARNING] +> `RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_WRITE` remains inactive unless +> `RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED` is also `true`. Enable +> both only after every node that can read or write object metadata supports +> the `part-checksums` sidecar and the fleet has adopted that version as its +> rollback floor. Leave either setting disabled throughout a mixed-version +> rolling upgrade. Once rebalance or decommission has migrated a legacy +> checksummed multipart object with both settings enabled, rolling back to an +> older build is not supported: older readers ignore the sidecar and can +> report an object checksum in place of the requested part checksum. + ## TL;DR - **Rolling restart (no downtime):** restart **one node at a time**, and wait diff --git a/rustfs/src/admin/site_replication_state.rs b/rustfs/src/admin/site_replication_state.rs index 6d924a218..eb0414665 100644 --- a/rustfs/src/admin/site_replication_state.rs +++ b/rustfs/src/admin/site_replication_state.rs @@ -77,8 +77,7 @@ where F: FnOnce() -> Fut + Send + 'static, Fut: std::future::Future> + Send + 'static, { - let store = - current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?; + let store = current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init"))?; with_site_replication_state_lock_on(store, operation).await } diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index d84faf1fc..d531b0fcd 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -6737,9 +6737,10 @@ impl DefaultObjectUsecase { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; - let opts: ObjectOptions = get_opts(&bucket, &key, version_id.clone(), None, &req.headers) + let mut opts: ObjectOptions = get_opts(&bucket, &key, version_id.clone(), None, &req.headers) .await .map_err(ApiError::from)?; + opts.include_part_checksums = object_attributes_requested(&object_attributes, ObjectAttributes::OBJECT_PARTS); let info = match store.get_object_info(&bucket, &key, &opts).await { Ok(info) => info,