// Copyright 2024 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // #730: data-movement migration keeps staged cleanup helpers until copy paths converge. #![allow(dead_code)] pub(crate) mod backpressure; use crate::error::{ Error, Result, is_err_data_movement_overwrite, is_err_invalid_upload_id, is_err_object_not_found, is_err_version_not_found, }; use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}; use crate::set_disk::{SetDisks, get_lock_acquire_timeout}; use crate::storage_api_contracts::{ multipart::{CompletePart, MultipartOperations as _}, namespace::NamespaceLocking as _, object::{HTTPPreconditions, ObjectOperations as _}, }; use crate::store::ECStore; use bytes::Bytes; use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo}; use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex}; use rustfs_utils::http::{ AMZ_OBJECT_TAGGING, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION_SIZE, SUFFIX_CRC, SUFFIX_DATA_MOVED, SUFFIX_DATA_MOVED_TAGS, SUFFIX_DATA_MOVEMENT_UPLOAD, SUFFIX_PART_CHECKSUMS, SUFFIX_TRANSITION_STATUS, SUFFIX_TRANSITION_TIER, SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, strip_internal_prefix_preserving_case, }; use rustfs_utils::path::encode_dir_object; use std::collections::{BTreeMap, HashMap}; use std::pin::Pin; use std::sync::{ Arc, Mutex, atomic::{AtomicBool, Ordering}, }; use std::task::{Context, Poll}; use std::time::Duration as StdDuration; use time::format_description::well_known::Rfc3339; use tokio::io::{AsyncRead, BufReader, ReadBuf}; use tracing::{error, info}; type SharedDataMovementStream = Arc>>; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; const LOG_SUBSYSTEM_DATA_MOVEMENT: &str = "data_movement"; const EVENT_DATA_MOVEMENT_MULTIPART_ABORT_FAILED: &str = "data_movement_multipart_abort_failed"; const DATA_MOVEMENT_MULTIPART_ABORT_RETRY_ATTEMPTS: usize = 3; const DATA_MOVEMENT_MULTIPART_ABORT_RETRY_DELAY_SECS: u64 = 60; pub struct IndexedDataMovementReader { inner: R, index: Option, } impl IndexedDataMovementReader { pub fn new(inner: R, index: Option) -> Self { Self { inner, index } } } impl AsyncRead for IndexedDataMovementReader { fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { Pin::new(&mut self.inner).poll_read(cx, buf) } } impl EtagResolvable for IndexedDataMovementReader {} impl HashReaderDetector for IndexedDataMovementReader {} impl TryGetIndex for IndexedDataMovementReader { fn try_get_index(&self) -> Option<&Index> { self.index.as_ref() } } pub fn decode_part_index(index: Option<&Bytes>) -> Option { let bytes = index?; let mut decoded = Index::new(); if decoded.load(bytes.as_ref()).is_ok() { Some(decoded) } else { None } } struct DataMovementPartReader { inner: SharedDataMovementStream, remaining: u64, } impl DataMovementPartReader { fn new(inner: SharedDataMovementStream, size: u64) -> Self { Self { inner, remaining: size } } } impl AsyncRead for DataMovementPartReader { fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { if self.remaining == 0 || buf.remaining() == 0 { return Poll::Ready(Ok(())); } let allowed = buf.remaining().min(usize::try_from(self.remaining).unwrap_or(usize::MAX)); let target = buf.initialize_unfilled_to(allowed); let mut limited_buf = ReadBuf::new(target); let poll = { let mut inner = self .inner .lock() .map_err(|_| std::io::Error::other("data movement stream lock poisoned"))?; Pin::new(&mut **inner).poll_read(cx, &mut limited_buf) }; if let Poll::Ready(Ok(())) = &poll { let read = limited_buf.filled().len(); buf.advance(read); self.remaining = self.remaining.saturating_sub(u64::try_from(read).unwrap_or(u64::MAX)); } poll } } impl EtagResolvable for DataMovementPartReader {} impl HashReaderDetector for DataMovementPartReader {} fn put_obj_reader_from_part_stream( stream: SharedDataMovementStream, size: i64, actual_size: i64, index: Option, ) -> Result { let limit = u64::try_from(size).map_err(|_| Error::other("part size overflow"))?; let reader = IndexedDataMovementReader::new(DataMovementPartReader::new(stream, limit), index); let hash_reader = HashReader::from_reader(reader, size, actual_size, None, None, false)?; Ok(PutObjReader::new(hash_reader)) } pub fn new_multipart_abort_flag() -> Arc { Arc::new(AtomicBool::new(true)) } pub fn should_abort_multipart_upload(flag: &Arc) -> bool { flag.load(Ordering::Relaxed) } pub fn mark_multipart_upload_completed(flag: &Arc) { flag.store(false, Ordering::Relaxed); } fn insert_data_movement_checksum(user_defined: &mut HashMap, object_info: &ObjectInfo) { rustfs_utils::http::remove_header_map(user_defined, rustfs_utils::http::SUFFIX_REPLICATION_SSEC_CRC); if let Some(checksum) = object_info.checksum.as_ref().filter(|checksum| !checksum.is_empty()) { rustfs_utils::http::insert_header_map( user_defined, rustfs_utils::http::SUFFIX_REPLICATION_SSEC_CRC, base64_simd::STANDARD.encode_to_string(checksum), ); } } fn data_movement_upload_identity(object_info: &ObjectInfo) -> String { let version_id = object_info .version_id .map_or_else(|| "none".to_string(), |version_id| version_id.to_string()); let mod_time = object_info .mod_time .map_or_else(|| "none".to_string(), |mod_time| mod_time.unix_timestamp_nanos().to_string()); format!("v1:{version_id}:{mod_time}") } fn data_movement_new_multipart_opts(object_info: &ObjectInfo, src_pool_idx: usize) -> ObjectOptions { let mut user_defined = data_movement_user_defined(object_info); let upload_identity = data_movement_upload_identity(object_info); rustfs_utils::http::insert_str(&mut user_defined, SUFFIX_DATA_MOVEMENT_UPLOAD, upload_identity); ObjectOptions { versioned: object_info.version_id.is_some(), version_id: object_info.version_id.as_ref().map(|v| v.to_string()), user_defined, preserve_etag: object_info.etag.clone(), src_pool_idx, data_movement: true, ..Default::default() } } fn data_movement_user_defined(object_info: &ObjectInfo) -> HashMap { let mut user_defined = object_info .user_defined .iter() .filter(|(key, _)| { !is_data_movement_internal_metadata(key, SUFFIX_DATA_MOVEMENT_UPLOAD) && !is_data_movement_internal_metadata(key, SUFFIX_PART_CHECKSUMS) }) .map(|(key, value)| (key.clone(), value.clone())) .collect::>(); let remove_canonical = |metadata: &mut HashMap, suffix: &str| { metadata.remove(&rustfs_utils::http::internal_key_rustfs(suffix)); metadata.remove(&format!("{}{suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX)); }; if object_info.checksum.as_ref().is_some_and(|checksum| !checksum.is_empty()) { rustfs_utils::http::remove_str(&mut user_defined, SUFFIX_CRC); } else { remove_canonical(&mut user_defined, SUFFIX_CRC); } for (suffix, value) in [ (SUFFIX_TRANSITION_STATUS, object_info.transitioned_object.status.as_str()), (SUFFIX_TRANSITIONED_OBJECTNAME, object_info.transitioned_object.name.as_str()), (SUFFIX_TRANSITION_TIER, object_info.transitioned_object.tier.as_str()), ] { if value.is_empty() { remove_canonical(&mut user_defined, suffix); continue; } rustfs_utils::http::remove_str(&mut user_defined, suffix); rustfs_utils::http::insert_str(&mut user_defined, suffix, value.to_string()); } if object_info.transitioned_object.version_id.is_empty() { let version_is_semantically_empty = user_defined .iter() .filter(|(key, _)| is_data_movement_internal_metadata(key, SUFFIX_TRANSITIONED_VERSION_ID)) .all(|(_, value)| is_empty_data_movement_transition_version(value)); if version_is_semantically_empty { remove_canonical(&mut user_defined, SUFFIX_TRANSITIONED_VERSION_ID); } } else { rustfs_utils::http::remove_str(&mut user_defined, SUFFIX_TRANSITIONED_VERSION_ID); rustfs_utils::http::insert_str( &mut user_defined, SUFFIX_TRANSITIONED_VERSION_ID, object_info.transitioned_object.version_id.clone(), ); } let transition_version_state = (object_info.transition_version_state != rustfs_filemeta::TransitionVersionState::Unknown) .then(|| object_info.transition_version_state.as_str()); if let Some(transition_version_state) = transition_version_state { rustfs_utils::http::remove_str(&mut user_defined, SUFFIX_TRANSITIONED_VERSION_STATE); rustfs_utils::http::insert_str( &mut user_defined, SUFFIX_TRANSITIONED_VERSION_STATE, transition_version_state.to_string(), ); } else { remove_canonical(&mut user_defined, SUFFIX_TRANSITIONED_VERSION_STATE); } user_defined.remove(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM); user_defined.remove(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE); insert_data_movement_checksum(&mut user_defined, object_info); if !object_info.user_tags.is_empty() { user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), (*object_info.user_tags).clone()); } if let Some(expires) = object_info.expires && let Ok(expires) = expires.format(&Rfc3339) { user_defined.insert("expires".to_string(), expires); } user_defined } fn data_movement_part_checksums(parts: &[ObjectPartInfo]) -> Result> { let mut part_checksums = BTreeMap::>::new(); for part in parts { let Some(checksums) = part.checksums.as_ref().filter(|checksums| !checksums.is_empty()) else { continue; }; let checksums = checksums.iter().map(|(key, value)| (key.clone(), value.clone())).collect(); if part_checksums.insert(part.number, checksums).is_some() { return Err(Error::other("data movement source has duplicate part numbers")); } } if part_checksums.is_empty() { return Ok(None); } let part_checksums = part_checksums .into_iter() .map(|(part_number, checksums)| (part_number, checksums.into_iter().collect::>())) .collect::>(); serde_json::to_string(&part_checksums) .map(Some) .map_err(|err| Error::other(format!("data movement part checksum metadata encode failed: {err}"))) } pub(crate) fn prepare_tiered_data_movement_file_info(file_info: &mut rustfs_filemeta::FileInfo) -> Result<()> { prepare_tiered_data_movement_file_info_for(file_info, data_movement_part_checksum_writer_enabled()) } fn prepare_tiered_data_movement_file_info_for(file_info: &mut rustfs_filemeta::FileInfo, writer_enabled: bool) -> Result<()> { if !writer_enabled { rustfs_utils::http::remove_str(&mut file_info.metadata, SUFFIX_PART_CHECKSUMS); for part in &mut file_info.parts { part.checksums = None; } return Ok(()); } SetDisks::hydrate_selected_fileinfo_part_checksums(file_info).map_err(|_| Error::FileCorrupt)?; rustfs_utils::http::remove_str(&mut file_info.metadata, SUFFIX_PART_CHECKSUMS); if let Some(encoded) = data_movement_part_checksums(&file_info.parts)? { rustfs_utils::http::insert_str(&mut file_info.metadata, SUFFIX_PART_CHECKSUMS, encoded); } Ok(()) } fn data_movement_part_checksum_writer_enabled_for(requested: bool, fleet_confirmed: bool) -> bool { requested && fleet_confirmed } fn data_movement_part_checksum_writer_enabled() -> bool { data_movement_part_checksum_writer_enabled_for( rustfs_utils::get_env_bool( rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, rustfs_config::DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, ), rustfs_utils::get_env_bool( rustfs_config::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, rustfs_config::DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED, ), ) } fn should_use_multipart_data_movement(object_info: &ObjectInfo, has_part_checksums: bool) -> bool { object_info.is_multipart() || has_part_checksums || object_info.parts.len() > 1 || object_info.parts.first().is_some_and(|part| part.number != 1) } fn data_movement_complete_multipart_opts( object_info: &ObjectInfo, src_pool_idx: usize, preserve_part_checksums: bool, ) -> Result { let mut user_defined = HashMap::new(); insert_data_movement_checksum(&mut user_defined, object_info); let actual_size = object_info .get_actual_size() .map_err(|err| Error::other(format!("data movement source actual size is invalid: {err}")))?; if actual_size < 0 { return Err(Error::other("data movement source actual size is unknown")); } rustfs_utils::http::insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, actual_size.to_string()); if preserve_part_checksums && let Some(encoded) = data_movement_part_checksums(&object_info.parts)? { rustfs_utils::http::insert_str(&mut user_defined, SUFFIX_PART_CHECKSUMS, encoded); } Ok(ObjectOptions { versioned: object_info.version_id.is_some(), version_id: object_info.version_id.as_ref().map(|v| v.to_string()), http_preconditions: Some(data_movement_target_precondition()), data_movement: true, mod_time: object_info.mod_time, preserve_etag: object_info.etag.clone(), user_defined, src_pool_idx, ..Default::default() }) } fn data_movement_put_object_opts(object_info: &ObjectInfo, src_pool_idx: usize) -> ObjectOptions { ObjectOptions { versioned: object_info.version_id.is_some(), src_pool_idx, data_movement: true, version_id: object_info.version_id.as_ref().map(|v| v.to_string()), http_preconditions: Some(data_movement_target_precondition()), mod_time: object_info.mod_time, user_defined: data_movement_user_defined(object_info), preserve_etag: object_info.etag.clone(), ..Default::default() } } fn is_unversioned_data_movement_object(object_info: &ObjectInfo) -> bool { object_info.version_id.is_none_or(|version_id| version_id.is_nil()) } pub(crate) fn data_movement_target_precondition() -> HTTPPreconditions { HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() } } fn is_owned_data_movement_target(target: &ObjectInfo) -> bool { let rustfs_marker = rustfs_utils::http::internal_key_rustfs(SUFFIX_DATA_MOVED); let minio_marker = format!("{}{SUFFIX_DATA_MOVED}", rustfs_utils::http::MINIO_INTERNAL_PREFIX); if rustfs_utils::http::get_consistent_str(&target.user_defined, SUFFIX_DATA_MOVED) != Some("true") || target.user_defined.get(&rustfs_marker).map(String::as_str) != Some("true") || target.user_defined.get(&minio_marker).map(String::as_str) != Some("true") { return false; } let tags_proof = format!("v1:{}", target.user_tags); let rustfs_tags_proof = rustfs_utils::http::internal_key_rustfs(SUFFIX_DATA_MOVED_TAGS); let minio_tags_proof = format!("{}{SUFFIX_DATA_MOVED_TAGS}", rustfs_utils::http::MINIO_INTERNAL_PREFIX); rustfs_utils::http::get_consistent_str(&target.user_defined, SUFFIX_DATA_MOVED_TAGS) == Some(tags_proof.as_str()) && target.user_defined.get(&rustfs_tags_proof).map(String::as_str) == Some(tags_proof.as_str()) && target.user_defined.get(&minio_tags_proof).map(String::as_str) == Some(tags_proof.as_str()) } pub(crate) fn can_replace_stale_data_movement_target(target: &ObjectInfo, opts: &ObjectOptions) -> bool { let Some(preconditions) = opts.http_preconditions.as_ref() else { return false; }; if !opts.data_movement || preconditions.if_none_match_value() != Some("*") || preconditions.if_match_value().is_some() || target.delete_marker { return false; } if !is_owned_data_movement_target(target) { return false; } let version_matches = match (opts.version_id.as_deref(), target.version_id) { (None, None) => true, (Some(expected), Some(actual)) => uuid::Uuid::parse_str(expected).ok() == Some(actual), _ => false, }; version_matches && target .mod_time .zip(opts.mod_time) .is_some_and(|(target_time, source_time)| target_time < source_time) } fn data_movement_put_object_reader( bucket: &str, object_info: &ObjectInfo, rd: GetObjectReader, op_label: &str, ) -> Result { let actual_size = object_info .get_actual_size() .map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", bucket, object_info.name.as_str(), err))?; let index = object_info .parts .first() .and_then(|part| decode_part_index(part.index.as_ref())); let reader = IndexedDataMovementReader::new(BufReader::new(rd.stream), index); let hrd = HashReader::from_stream(reader, object_info.size, actual_size, None, None, false) .map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", bucket, object_info.name.as_str(), err))?; Ok(PutObjReader::new(hrd)) } fn resolve_data_movement_abort_result( op_label: &str, bucket: &str, object: &str, upload_id: &str, primary_err: Error, abort_err: Error, ) -> Error { Error::other(format!( "{op_label}: abort_multipart_upload failed for {bucket}/{object} upload {upload_id} after error {primary_err}: {abort_err}" )) } fn data_movement_stage_error(op_label: &str, stage: &str, bucket: &str, object: &str, err: impl std::fmt::Display) -> Error { Error::other(format!("{op_label}: {stage} failed for {bucket}/{object}: {err}")) } fn schedule_data_movement_multipart_abort_cleanup( store: Arc, target_pool_idx: usize, bucket: String, object: String, upload_id: String, op_label: &str, ) { let op_label = op_label.to_string(); tokio::spawn(async move { for attempt in 1..=DATA_MOVEMENT_MULTIPART_ABORT_RETRY_ATTEMPTS { tokio::time::sleep(StdDuration::from_secs(DATA_MOVEMENT_MULTIPART_ABORT_RETRY_DELAY_SECS)).await; let Some(pool) = store.pools.get(target_pool_idx).cloned() else { error!( "{op_label}: background abort_multipart_upload cleanup skipped for {bucket}/{object} upload {upload_id}: target pool {target_pool_idx} is out of range" ); return; }; match pool .abort_multipart_upload( &bucket, &object, &upload_id, &ObjectOptions { data_movement: true, ..Default::default() }, ) .await { Ok(()) => { info!( "{op_label}: background abort_multipart_upload cleanup succeeded for {bucket}/{object} upload {upload_id} on attempt {attempt}" ); return; } Err(err) if is_err_invalid_upload_id(&err) => { info!( "{op_label}: background abort_multipart_upload cleanup found {bucket}/{object} upload {upload_id} already removed" ); return; } Err(err) => { error!( "{op_label}: background abort_multipart_upload cleanup attempt {attempt} failed for {bucket}/{object} upload {upload_id}: {err:?}" ); } } } crate::bucket::lifecycle::bucket_lifecycle_ops::schedule_stale_multipart_upload_cleanup_once(store); }); } fn should_check_data_movement_overwrite_resume(err: &Error) -> bool { is_err_data_movement_overwrite(err) || is_err_invalid_upload_id(err) || matches!(err, Error::PreconditionFailed) } fn effective_actual_size(info: &ObjectInfo) -> Option { info.get_actual_size().ok() } fn effective_part_actual_size(part: &ObjectPartInfo) -> Option { (part.actual_size != 0) .then_some(part.actual_size) .or_else(|| i64::try_from(part.size).ok()) } fn is_equivalent_data_movement_part(source: &ObjectPartInfo, target: &ObjectPartInfo, compare_checksums: bool) -> bool { // Multipart migration rewrites part timestamps. source.number == target.number && source.etag == target.etag && source.size == target.size && matches!( (effective_part_actual_size(source), effective_part_actual_size(target)), (Some(source_size), Some(target_size)) if source_size == target_size ) // A missing target compression index selects the safe full-read fallback. && (target.index.is_none() || source.index == target.index) && (!compare_checksums || source.checksums.as_ref().filter(|checksums| !checksums.is_empty()) == target.checksums.as_ref().filter(|checksums| !checksums.is_empty())) } fn data_movement_parts_by_number(parts: &[ObjectPartInfo]) -> Option> { let mut parts_by_number = BTreeMap::new(); for part in parts { if parts_by_number.insert(part.number, part).is_some() { return None; } } Some(parts_by_number) } pub(crate) fn are_equivalent_data_movement_parts(source: &[ObjectPartInfo], target: &[ObjectPartInfo]) -> bool { are_equivalent_data_movement_parts_for(source, target, true) } fn are_equivalent_data_movement_parts_for(source: &[ObjectPartInfo], target: &[ObjectPartInfo], compare_checksums: bool) -> bool { if source.len() != target.len() { return false; } let Some(source_parts) = data_movement_parts_by_number(source) else { return false; }; let Some(target_parts) = data_movement_parts_by_number(target) else { return false; }; source_parts.iter().all(|(number, source_part)| { target_parts .get(number) .is_some_and(|target_part| is_equivalent_data_movement_part(source_part, target_part, compare_checksums)) }) } fn is_data_movement_internal_metadata(key: &str, suffix: &str) -> bool { strip_internal_prefix_preserving_case(key).is_some_and(|candidate| candidate.eq_ignore_ascii_case(suffix)) } fn is_canonical_data_movement_internal_metadata(key: &str, suffix: &str) -> bool { key.strip_prefix(rustfs_utils::http::RUSTFS_INTERNAL_PREFIX) == Some(suffix) || key.strip_prefix(rustfs_utils::http::MINIO_INTERNAL_PREFIX) == Some(suffix) } fn data_movement_layout_marker_presence(object_info: &ObjectInfo) -> Option { if !rustfs_utils::http::contains_key_str(&object_info.user_defined, crate::object_api::ENCRYPTED_PART_LAYOUT_QUORUM_SUFFIX) { return Some(false); } let data_dir = object_info.data_dir.filter(|data_dir| !data_dir.is_nil())?; let mut expected_buf = [0_u8; 36]; let expected = data_dir.hyphenated().encode_lower(&mut expected_buf); crate::object_api::has_encrypted_part_layout_marker( &object_info.user_defined, crate::object_api::ENCRYPTED_PART_LAYOUT_QUORUM_SUFFIX, expected, ) .then_some(true) } fn data_movement_size_marker_presence(object_info: &ObjectInfo, suffix: &str, expected: i64) -> Option { let mut present = false; for (key, value) in object_info.user_defined.iter() { if !is_data_movement_internal_metadata(key, suffix) { continue; } present = true; if value.parse::().ok() != Some(expected) { return None; } } Some(present) } fn data_movement_checksum_marker_presence(object_info: &ObjectInfo) -> Option { let mut present = false; for (key, value) in object_info.user_defined.iter() { if !is_data_movement_internal_metadata(key, SUFFIX_CRC) { continue; } let Some(checksum) = object_info.checksum.as_deref().filter(|checksum| !checksum.is_empty()) else { if value.is_empty() { continue; } return None; }; present = true; match std::str::from_utf8(checksum) { Ok(checksum) if value != checksum => return None, Err(_) if !value.is_empty() => return None, _ => {} } } Some(present) } fn is_compatible_data_movement_marker_presence(source: Option, target: Option) -> bool { matches!((source, target), (Some(false), Some(_)) | (Some(true), Some(true))) } fn is_empty_data_movement_transition_version(value: &str) -> bool { value.is_empty() || uuid::Uuid::from_slice(value.as_bytes()).is_ok_and(|version_id| version_id.is_nil()) || uuid::Uuid::parse_str(value).is_ok_and(|version_id| version_id.is_nil()) } fn is_data_movement_rewritten_transition_metadata(object_info: &ObjectInfo, key: &str) -> bool { let state = (object_info.transition_version_state != rustfs_filemeta::TransitionVersionState::Unknown) .then(|| object_info.transition_version_state.as_str()); [ (SUFFIX_TRANSITION_STATUS, Some(object_info.transitioned_object.status.as_str())), (SUFFIX_TRANSITIONED_OBJECTNAME, Some(object_info.transitioned_object.name.as_str())), (SUFFIX_TRANSITIONED_VERSION_ID, Some(object_info.transitioned_object.version_id.as_str())), (SUFFIX_TRANSITIONED_VERSION_STATE, state), (SUFFIX_TRANSITION_TIER, Some(object_info.transitioned_object.tier.as_str())), ] .iter() .any(|(suffix, expected)| { let canonical = is_canonical_data_movement_internal_metadata(key, suffix); let preserves_unusable_version = *suffix == SUFFIX_TRANSITIONED_VERSION_ID && expected == &Some("") && object_info .user_defined .get(key) .is_some_and(|value| !is_empty_data_movement_transition_version(value)); canonical && !preserves_unusable_version || expected.is_some_and(|expected| { !expected.is_empty() && is_data_movement_internal_metadata(key, suffix) && rustfs_utils::http::get_consistent_str(&object_info.user_defined, suffix) == Some(expected) }) }) } fn is_data_movement_rewritten_metadata(object_info: &ObjectInfo, key: &str, normalize_compression_size: bool) -> bool { [ SUFFIX_DATA_MOVED, SUFFIX_DATA_MOVED_TAGS, SUFFIX_DATA_MOVEMENT_UPLOAD, SUFFIX_ACTUAL_SIZE, SUFFIX_CRC, SUFFIX_PART_CHECKSUMS, crate::object_api::ENCRYPTED_PART_LAYOUT_QUORUM_SUFFIX, ] .iter() .any(|suffix| is_data_movement_internal_metadata(key, suffix)) || is_data_movement_rewritten_transition_metadata(object_info, key) || key == rustfs_rio::RUSTFS_MULTIPART_CHECKSUM || key == rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE || normalize_compression_size && is_data_movement_internal_metadata(key, SUFFIX_COMPRESSION_SIZE) } pub(crate) fn is_equivalent_data_movement_metadata( source: &ObjectInfo, target: &ObjectInfo, source_actual_size: i64, target_actual_size: i64, ) -> bool { if !is_compatible_data_movement_marker_presence( data_movement_size_marker_presence(source, SUFFIX_ACTUAL_SIZE, source_actual_size), data_movement_size_marker_presence(target, SUFFIX_ACTUAL_SIZE, target_actual_size), ) || !is_compatible_data_movement_marker_presence( data_movement_checksum_marker_presence(source), data_movement_checksum_marker_presence(target), ) { return false; } let normalize_compression_size = source.is_compressed() && target.is_compressed(); if normalize_compression_size && !is_compatible_data_movement_marker_presence( data_movement_size_marker_presence(source, SUFFIX_COMPRESSION_SIZE, source.size), data_movement_size_marker_presence(target, SUFFIX_COMPRESSION_SIZE, target.size), ) { return false; } let Some(source_layout) = data_movement_layout_marker_presence(source) else { return false; }; let Some(target_layout) = data_movement_layout_marker_presence(target) else { return false; }; if (source_layout || target_layout) && !(source.data_dir.is_some_and(|data_dir| !data_dir.is_nil()) && target.data_dir.is_some_and(|data_dir| !data_dir.is_nil())) { return false; } source .user_defined .iter() .filter(|(key, _)| !is_data_movement_rewritten_metadata(source, key, normalize_compression_size)) .all(|(key, value)| target.user_defined.get(key) == Some(value)) && target .user_defined .iter() .filter(|(key, _)| !is_data_movement_rewritten_metadata(target, key, normalize_compression_size)) .all(|(key, value)| source.user_defined.get(key) == Some(value)) } fn is_equivalent_data_movement_object_identity( source: &ObjectInfo, target: &ObjectInfo, compare_mod_time: bool, compare_part_checksums: bool, ) -> bool { let (Some(source_actual_size), Some(target_actual_size)) = (effective_actual_size(source), effective_actual_size(target)) else { return false; }; source.version_id == target.version_id && source.delete_marker == target.delete_marker && source.size == target.size && source_actual_size == target_actual_size && source.etag == target.etag && source.checksum == target.checksum && (!compare_mod_time || source.mod_time == target.mod_time) && source.storage_class == target.storage_class && is_equivalent_data_movement_metadata(source, target, source_actual_size, target_actual_size) && source.user_tags == target.user_tags && source.expires == target.expires && source.replication_status_internal == target.replication_status_internal && source.replication_status == target.replication_status && source.version_purge_status_internal == target.version_purge_status_internal && source.version_purge_status == target.version_purge_status && source.transitioned_object.name == target.transitioned_object.name && source.transitioned_object.version_id == target.transitioned_object.version_id && source.transitioned_object.tier == target.transitioned_object.tier && source.transitioned_object.free_version == target.transitioned_object.free_version && source.transitioned_object.status == target.transitioned_object.status && source.transition_version_state == target.transition_version_state && are_equivalent_data_movement_parts_for(&source.parts, &target.parts, compare_part_checksums) } #[cfg(test)] fn is_equivalent_data_movement_object(source: &ObjectInfo, target: &ObjectInfo) -> bool { is_equivalent_data_movement_object_identity(source, target, true, true) } fn is_superseding_unversioned_data_movement_object(source: &ObjectInfo, target: &ObjectInfo) -> bool { is_unversioned_data_movement_object(source) && is_unversioned_data_movement_object(target) && !target.delete_marker && source .mod_time .zip(target.mod_time) .is_some_and(|(source_time, target_time)| target_time > source_time) } fn is_data_movement_upload_takeover_target(source: &ObjectInfo, target: &ObjectInfo, compare_part_checksums: bool) -> bool { let identity = data_movement_upload_identity(source); source.mod_time.is_some() && rustfs_utils::http::get_consistent_str(&target.user_defined, SUFFIX_DATA_MOVEMENT_UPLOAD) == Some(identity.as_str()) && is_equivalent_data_movement_object_identity(source, target, false, compare_part_checksums) } fn is_legacy_data_movement_checksum_target(source: &ObjectInfo, target: &ObjectInfo) -> bool { let has_checksums = |part: &ObjectPartInfo| part.checksums.as_ref().is_some_and(|checksums| !checksums.is_empty()); source.parts.iter().any(has_checksums) && target.parts.iter().all(|part| part.checksums.is_none()) && !rustfs_utils::http::contains_key_str(&target.user_defined, SUFFIX_PART_CHECKSUMS) && is_owned_data_movement_target(target) && is_equivalent_data_movement_object_identity(source, target, true, false) } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] struct SourceCleanupPartIdentity { number: usize, etag: String, size: usize, actual_size: i64, mod_time: Option, index: Option>, checksums: BTreeMap, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] struct SourceCleanupErasureIdentity { algorithm: String, data_blocks: usize, parity_blocks: usize, block_size: usize, distribution: Vec, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] pub(crate) struct SourceCleanupVersionIdentity { name: String, version_id: Option, deleted: bool, mod_time: Option, size: i64, etag: Option, checksum: Option>, data_dir: Option, transition_status: String, transitioned_objname: String, transition_tier: String, transition_version_id: Option, transition_version: Option, transition_version_state: &'static str, expire_restored: bool, erasure: SourceCleanupErasureIdentity, metadata: BTreeMap, parts: Vec, } fn source_cleanup_erasure_identity(erasure: &rustfs_filemeta::ErasureInfo) -> SourceCleanupErasureIdentity { SourceCleanupErasureIdentity { algorithm: erasure.algorithm.clone(), data_blocks: erasure.data_blocks, parity_blocks: erasure.parity_blocks, block_size: erasure.block_size, distribution: erasure.distribution.clone(), } } fn source_cleanup_part_identity(part: &ObjectPartInfo) -> SourceCleanupPartIdentity { SourceCleanupPartIdentity { number: part.number, etag: part.etag.clone(), size: part.size, actual_size: part.actual_size, mod_time: part.mod_time, index: part.index.as_ref().map(|index| index.to_vec()), checksums: part .checksums .as_ref() .map(|checksums| checksums.iter().map(|(key, value)| (key.clone(), value.clone())).collect()) .unwrap_or_default(), } } pub(crate) fn source_cleanup_version_identity(version: &FileInfo) -> SourceCleanupVersionIdentity { let mut parts: Vec<_> = version.parts.iter().map(source_cleanup_part_identity).collect(); parts.sort(); SourceCleanupVersionIdentity { name: version.name.clone(), version_id: version.version_id, deleted: version.deleted, mod_time: version.mod_time, size: version.size, etag: version.get_etag(), checksum: version.checksum.as_ref().map(|checksum| checksum.to_vec()), data_dir: version.data_dir, transition_status: version.transition_status.clone(), transitioned_objname: version.transitioned_objname.clone(), transition_tier: version.transition_tier.clone(), transition_version_id: version.transition_version_id, transition_version: version.transition_version.clone(), transition_version_state: version.transition_version_state.as_str(), expire_restored: version.expire_restored, erasure: source_cleanup_erasure_identity(&version.erasure), metadata: version .metadata .iter() .map(|(key, value)| (key.clone(), value.clone())) .collect(), parts, } } fn source_cleanup_versions_match_with_allowed_missing( expected: &FileInfoVersions, current: &FileInfoVersions, allowed_missing: &[SourceCleanupVersionIdentity], ) -> bool { let mut expected_free_versions: Vec<_> = expected.free_versions.iter().map(source_cleanup_version_identity).collect(); let mut current_free_versions: Vec<_> = current.free_versions.iter().map(source_cleanup_version_identity).collect(); expected_free_versions.sort(); current_free_versions.sort(); if expected_free_versions != current_free_versions { return false; } let mut expected_counts = BTreeMap::new(); for identity in expected.versions.iter().map(source_cleanup_version_identity) { *expected_counts.entry(identity).or_insert(0usize) += 1; } for identity in current.versions.iter().map(source_cleanup_version_identity) { let Some(count) = expected_counts.get_mut(&identity) else { return false; }; *count = count.saturating_sub(1); if *count == 0 { expected_counts.remove(&identity); } } let mut allowed_counts = BTreeMap::new(); for identity in allowed_missing.iter().cloned() { *allowed_counts.entry(identity).or_insert(0usize) += 1; } expected_counts .into_iter() .all(|(identity, count)| allowed_counts.get(&identity).copied().unwrap_or_default() >= count) } #[derive(Debug, thiserror::Error)] pub(crate) enum SourceCleanupError { #[error("source versions changed after migration started")] SourceChanged, #[error(transparent)] Storage(#[from] Error), } #[derive(Clone, Copy, Default)] pub(crate) struct SourceCleanupBucketFence<'a> { pub(crate) expected_incarnation_id: Option, pub(crate) lifecycle_guard: Option<&'a rustfs_lock::NamespaceLockGuard>, } fn ensure_source_cleanup_versions_match( expected: &FileInfoVersions, current: &FileInfoVersions, allowed_missing: &[SourceCleanupVersionIdentity], ) -> std::result::Result<(), SourceCleanupError> { if source_cleanup_versions_match_with_allowed_missing(expected, current, allowed_missing) { Ok(()) } else { Err(SourceCleanupError::SourceChanged) } } pub(crate) async fn ensure_source_cleanup_versions_unchanged( set: Arc, bucket: &str, object: &str, expected: &FileInfoVersions, allowed_missing: &[SourceCleanupVersionIdentity], op_label: &str, ) -> std::result::Result<(), SourceCleanupError> { let Some(current) = set .load_file_info_versions_exact(bucket, object) .await .map_err(|err| Error::other(format!("{op_label}: source cleanup preflight failed for {bucket}/{object}: {err}")))? else { return Ok(()); }; ensure_source_cleanup_versions_match(expected, ¤t, allowed_missing) } #[cfg(test)] struct SourceCleanupDeleteBarrierState { bucket: String, object: String, arrived: tokio::sync::Notify, release: tokio::sync::Notify, } #[cfg(test)] pub(crate) struct SourceCleanupDeleteBarrier { state: Arc, } #[cfg(test)] static SOURCE_CLEANUP_DELETE_BARRIER: std::sync::OnceLock>>> = std::sync::OnceLock::new(); #[cfg(test)] impl SourceCleanupDeleteBarrier { pub(crate) fn install(bucket: &str, object: &str) -> Self { let state = Arc::new(SourceCleanupDeleteBarrierState { bucket: bucket.to_string(), object: object.to_string(), arrived: tokio::sync::Notify::new(), release: tokio::sync::Notify::new(), }); let mut slot = SOURCE_CLEANUP_DELETE_BARRIER .get_or_init(|| std::sync::Mutex::new(None)) .lock() .expect("source cleanup delete barrier mutex should not poison"); assert!(slot.is_none(), "source cleanup delete barrier must be unique"); *slot = Some(Arc::clone(&state)); Self { state } } pub(crate) async fn wait_until_paused(&self) { tokio::time::timeout(StdDuration::from_secs(30), self.state.arrived.notified()) .await .expect("source cleanup should reach the pre-delete barrier"); } pub(crate) fn release(&self) { self.state.release.notify_one(); } } #[cfg(test)] impl Drop for SourceCleanupDeleteBarrier { fn drop(&mut self) { self.state.release.notify_one(); let mut slot = SOURCE_CLEANUP_DELETE_BARRIER .get_or_init(|| std::sync::Mutex::new(None)) .lock() .expect("source cleanup delete barrier mutex should not poison"); if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { *slot = None; } } } #[cfg(test)] async fn pause_source_cleanup_before_delete(bucket: &str, object: &str) { let barrier = SOURCE_CLEANUP_DELETE_BARRIER .get_or_init(|| std::sync::Mutex::new(None)) .lock() .expect("source cleanup delete barrier mutex should not poison") .as_ref() .filter(|barrier| barrier.bucket == bucket && barrier.object == object) .cloned(); if let Some(barrier) = barrier { barrier.arrived.notify_one(); barrier.release.notified().await; } } pub(crate) async fn cleanup_source_entry_if_unchanged( set: Arc, bucket: &str, object: &str, expected: &FileInfoVersions, allowed_missing: &[SourceCleanupVersionIdentity], bucket_fence: SourceCleanupBucketFence<'_>, op_label: &str, ) -> std::result::Result { let cleanup_key = encode_dir_object(object); let ns_lock = set.new_ns_lock(bucket, cleanup_key.as_str()).await?; let _guard = ns_lock .get_write_lock(get_lock_acquire_timeout()) .await .map_err(Error::from)?; if bucket_fence .lifecycle_guard .is_some_and(rustfs_lock::NamespaceLockGuard::is_lock_lost) { return Err(SourceCleanupError::Storage(Error::other(format!( "{op_label}: bucket incarnation fence was lost before source cleanup" )))); } ensure_source_cleanup_versions_unchanged(set.clone(), bucket, object, expected, allowed_missing, op_label).await?; #[cfg(test)] pause_source_cleanup_before_delete(bucket, object).await; let mut opts = ObjectOptions { delete_prefix: true, delete_prefix_object: true, data_movement: true, no_lock: true, expected_bucket_incarnation_id: bucket_fence.expected_incarnation_id, ..Default::default() }; opts.add_namespace_lock_guard(&_guard); if let Some(bucket_lifecycle_guard) = bucket_fence.lifecycle_guard { opts.add_bucket_lifecycle_lock_guard(bucket_lifecycle_guard); } let result = set.delete_object(bucket, cleanup_key.as_str(), opts).await; if result.is_ok() { crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1); } result.map_err(SourceCleanupError::from) } fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool { target_pool_idx != src_pool_idx } async fn find_data_movement_target_info( store: &ECStore, target_pool_idx: usize, bucket: &str, object_info: &ObjectInfo, ) -> Result> { let opts = ObjectOptions { versioned: object_info.version_id.is_some(), version_id: object_info.version_id.as_ref().map(|v| v.to_string()), no_lock: true, include_part_checksums: true, ..Default::default() }; let object = encode_dir_object(object_info.name.as_str()); let Some(pool) = store.pools.get(target_pool_idx) else { return Err(Error::other(format!( "data movement resume target pool {target_pool_idx} is out of range for {bucket}/{object}" ))); }; match pool.get_object_info(bucket, object.as_str(), &opts).await { Ok(target_info) => Ok(Some(target_info)), Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(None), Err(err) => Err(err), } } fn resolve_data_movement_overwrite_resume_result( err: &Error, target_result: Result>, source: &ObjectInfo, src_pool_idx: usize, target_pool_idx: usize, ) -> Result { resolve_data_movement_overwrite_resume_result_for( err, target_result, source, src_pool_idx, target_pool_idx, data_movement_part_checksum_writer_enabled(), ) } fn resolve_data_movement_overwrite_resume_result_for( err: &Error, target_result: Result>, source: &ObjectInfo, src_pool_idx: usize, target_pool_idx: usize, compare_part_checksums: bool, ) -> Result { if !should_check_data_movement_overwrite_resume(err) || !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx) { return Ok(false); } let Some(target) = target_result? else { return Ok(false); }; if is_equivalent_data_movement_object_identity(source, &target, true, compare_part_checksums) { return Ok(true); } if compare_part_checksums && is_legacy_data_movement_checksum_target(source, &target) { return Ok(true); } if is_data_movement_upload_takeover_target(source, &target, compare_part_checksums) { return Ok(true); } Ok(matches!(err, Error::PreconditionFailed) && is_superseding_unversioned_data_movement_object(source, &target)) } async fn should_treat_data_movement_overwrite_as_complete( store: &ECStore, src_pool_idx: usize, target_pool_idx: usize, bucket: &str, object_info: &ObjectInfo, err: &Error, compare_part_checksums: bool, ) -> Result { if !should_check_data_movement_overwrite_resume(err) { return Ok(false); } resolve_data_movement_overwrite_resume_result_for( err, find_data_movement_target_info(store, target_pool_idx, bucket, object_info).await, object_info, src_pool_idx, target_pool_idx, compare_part_checksums, ) } fn data_movement_part_stage_error( op_label: &str, stage: &str, bucket: &str, object: &str, part_number: usize, err: impl std::fmt::Display, ) -> Error { Error::other(format!("{op_label}: {stage} failed for {bucket}/{object} part {part_number}: {err}")) } fn is_data_movement_part_read_error(err: &Error) -> bool { fn is_unexpected_eof(err: &std::io::Error) -> bool { err.kind() == std::io::ErrorKind::UnexpectedEof || err .get_ref() .and_then(|inner| inner.downcast_ref::()) .is_some_and(is_unexpected_eof) } matches!(err, Error::Io(io_err) if is_unexpected_eof(io_err)) } fn data_movement_part_upload_failure_stage(err: &Error) -> &'static str { if is_data_movement_part_read_error(err) { "read_part" } else { "put_object_part" } } pub(crate) async fn migrate_object( store: Arc, pool_idx: usize, bucket: String, rd: GetObjectReader, source_bucket_incarnation_id: Option, op_label: &str, ) -> Result<()> { let object_info = rd.object_info.clone(); let has_part_checksums = object_info .parts .iter() .any(|part| part.checksums.as_ref().is_some_and(|checksums| !checksums.is_empty())); let preserve_part_checksums = data_movement_part_checksum_writer_enabled(); if should_use_multipart_data_movement(&object_info, has_part_checksums) { let mut new_multipart_opts = data_movement_new_multipart_opts(&object_info, pool_idx); new_multipart_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id; let (res, target_pool_idx, expected_bucket_incarnation_id) = match store .handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts) .await { Ok(res) => res, Err(err) => { error!("{op_label}: new_multipart_upload err {:?}", &err); return Err(data_movement_stage_error( op_label, "new_multipart_upload", bucket.as_str(), object_info.name.as_str(), err, )); } }; let abort_multipart_flag = new_multipart_abort_flag(); let multipart_result: Result<()> = async { let mut parts = vec![CompletePart::default(); object_info.parts.len()]; let reader = Arc::new(Mutex::new(rd.stream)); for (i, part) in object_info.parts.iter().enumerate() { let part_size = i64::try_from(part.size).map_err(|_| { data_movement_part_stage_error( op_label, "prepare_part", bucket.as_str(), object_info.name.as_str(), part.number, Error::other("part size overflow"), ) })?; let part_actual_size = if part.actual_size == 0 { part_size } else { part.actual_size }; let index = decode_part_index(part.index.as_ref()); let mut data = put_obj_reader_from_part_stream(reader.clone(), part_size, part_actual_size, index).map_err(|err| { data_movement_part_stage_error( op_label, "prepare_part", bucket.as_str(), object_info.name.as_str(), part.number, err, ) })?; let part_opts = ObjectOptions { part_number: Some(part.number), preserve_etag: Some(part.etag.clone()), data_movement: true, src_pool_idx: pool_idx, expected_bucket_incarnation_id, ..Default::default() }; let pi = match store .put_object_part_for_data_movement( target_pool_idx, &bucket, &object_info.name, &res.upload_id, &mut data, &part_opts, ) .await { Ok(pi) => pi, Err(err) => { error!("{op_label}: put_object_part {i} err {:?}", &err); let stage = data_movement_part_upload_failure_stage(&err); return Err(data_movement_part_stage_error( op_label, stage, bucket.as_str(), object_info.name.as_str(), part.number, err, )); } }; parts[i] = CompletePart { part_num: pi.part_num, etag: pi.etag, ..Default::default() }; } let mut complete_multipart_opts = data_movement_complete_multipart_opts(&object_info, pool_idx, preserve_part_checksums).map_err(|err| { data_movement_stage_error( op_label, "prepare_complete_multipart", bucket.as_str(), object_info.name.as_str(), err, ) })?; complete_multipart_opts.expected_bucket_incarnation_id = expected_bucket_incarnation_id; if let Err(err) = store .clone() .complete_multipart_upload_for_data_movement( target_pool_idx, &bucket, &object_info.name, &res.upload_id, parts, &complete_multipart_opts, ) .await { if should_treat_data_movement_overwrite_as_complete( store.as_ref(), pool_idx, target_pool_idx, bucket.as_str(), &object_info, &err, preserve_part_checksums, ) .await? { info!( "{op_label}: complete_multipart_upload overwrite resolved by equivalent target for {}/{}", bucket.as_str(), object_info.name.as_str() ); return Ok(()); } error!("{op_label}: complete_multipart_upload err {:?}", &err); return Err(data_movement_stage_error( op_label, "complete_multipart_upload", bucket.as_str(), object_info.name.as_str(), err, )); } mark_multipart_upload_completed(&abort_multipart_flag); Ok(()) } .await; if multipart_result.is_ok() && should_abort_multipart_upload(&abort_multipart_flag) { let abort_result = store .abort_multipart_upload_for_data_movement( target_pool_idx, &bucket, &object_info.name, &res.upload_id, &ObjectOptions { data_movement: true, src_pool_idx: pool_idx, expected_bucket_incarnation_id, ..Default::default() }, ) .await; match abort_result { Ok(()) => return Ok(()), Err(abort_err) if is_err_invalid_upload_id(&abort_err) => { if should_treat_data_movement_overwrite_as_complete( store.as_ref(), pool_idx, target_pool_idx, bucket.as_str(), &object_info, &abort_err, preserve_part_checksums, ) .await? { return Ok(()); } return Err(data_movement_stage_error( op_label, "verify_superseded_multipart", bucket.as_str(), object_info.name.as_str(), abort_err, )); } Err(abort_err) => { error!( event = EVENT_DATA_MOVEMENT_MULTIPART_ABORT_FAILED, component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_DATA_MOVEMENT, result = "error", operation = op_label, error = ?abort_err, "data movement multipart abort failed" ); schedule_data_movement_multipart_abort_cleanup( store.clone(), target_pool_idx, bucket.clone(), object_info.name.clone(), res.upload_id.clone(), op_label, ); return Err(data_movement_stage_error( op_label, "abort_superseded_multipart", bucket.as_str(), object_info.name.as_str(), abort_err, )); } } } if let Err(primary_err) = multipart_result { if should_abort_multipart_upload(&abort_multipart_flag) { return match store .abort_multipart_upload_for_data_movement( target_pool_idx, &bucket, &object_info.name, &res.upload_id, &ObjectOptions { data_movement: true, src_pool_idx: pool_idx, expected_bucket_incarnation_id, ..Default::default() }, ) .await { Ok(()) => Err(primary_err), Err(abort_err) => { error!("{op_label}: abort_multipart_upload err {:?}", &abort_err); schedule_data_movement_multipart_abort_cleanup( store.clone(), target_pool_idx, bucket.clone(), object_info.name.clone(), res.upload_id.clone(), op_label, ); Err(resolve_data_movement_abort_result( op_label, bucket.as_str(), object_info.name.as_str(), res.upload_id.as_str(), primary_err, abort_err, )) } }; } return Err(primary_err); } return Ok(()); } let mut data = data_movement_put_object_reader(bucket.as_str(), &object_info, rd, op_label)?; let mut put_opts = data_movement_put_object_opts(&object_info, pool_idx); put_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id; let (target_pool_idx, put_result) = store .put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts) .await .map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", &bucket, &object_info.name, err))?; if let Err(err) = put_result { if should_treat_data_movement_overwrite_as_complete( store.as_ref(), pool_idx, target_pool_idx, bucket.as_str(), &object_info, &err, preserve_part_checksums, ) .await? { info!( "{op_label}: put_object overwrite resolved by equivalent target for {}/{}", bucket.as_str(), object_info.name.as_str() ); return Ok(()); } error!("{op_label}: put_object err {:?}", &err); return Err(data_movement_stage_error( op_label, "put_object", bucket.as_str(), object_info.name.as_str(), err, )); } Ok(()) } #[cfg(test)] mod tests { use super::*; use crate::bucket::replication::{ReplicationStatusType, VersionPurgeStatusType}; use rustfs_rio::{Checksum, ChecksumType}; use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE}; use std::collections::HashMap; use std::io::Cursor; use std::sync::atomic::AtomicUsize; use time::OffsetDateTime; use tokio::io::AsyncReadExt; use uuid::Uuid; struct MaxReadRequestReader { remaining: u64, max_request: usize, largest_request: Arc, } impl MaxReadRequestReader { fn new(remaining: u64, max_request: usize, largest_request: Arc) -> Self { Self { remaining, max_request, largest_request, } } } impl AsyncRead for MaxReadRequestReader { fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { let requested = buf.remaining(); self.largest_request.fetch_max(requested, Ordering::Relaxed); if requested > self.max_request { return Poll::Ready(Err(std::io::Error::other(format!("oversized read request: {requested}")))); } if self.remaining == 0 || requested == 0 { return Poll::Ready(Ok(())); } let read = requested.min(usize::try_from(self.remaining).unwrap_or(usize::MAX)); let target = buf.initialize_unfilled_to(read); target.fill(b'x'); buf.advance(read); self.remaining = self.remaining.saturating_sub(u64::try_from(read).unwrap_or(u64::MAX)); Poll::Ready(Ok(())) } } fn assert_data_movement_metadata_equivalent(source: &ObjectInfo, target: &ObjectInfo) { assert_eq!(source.version_id, target.version_id); assert_eq!(source.etag, target.etag); assert_eq!(source.size, target.size); assert_eq!(effective_actual_size(source), effective_actual_size(target)); assert_eq!(source.mod_time, target.mod_time); assert_eq!(source.user_defined, target.user_defined); assert_eq!(source.storage_class, target.storage_class); assert_eq!(source.checksum, target.checksum); assert_eq!(source.replication_status_internal, target.replication_status_internal); assert_eq!(source.replication_status, target.replication_status); assert_eq!(source.version_purge_status_internal, target.version_purge_status_internal); assert_eq!(source.version_purge_status, target.version_purge_status); assert_eq!( source.user_defined.get(X_AMZ_OBJECT_LOCK_MODE.as_str()), target.user_defined.get(X_AMZ_OBJECT_LOCK_MODE.as_str()) ); assert_eq!( source.user_defined.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str()), target.user_defined.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str()) ); assert_eq!( source.user_defined.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()), target.user_defined.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()) ); assert_eq!(source.parts.len(), target.parts.len()); for (source_part, target_part) in source.parts.iter().zip(target.parts.iter()) { assert_eq!(source_part.number, target_part.number); assert_eq!(source_part.etag, target_part.etag); assert_eq!(source_part.size, target_part.size); assert_eq!(source_part.actual_size, target_part.actual_size); assert_eq!(source_part.checksums, target_part.checksums); } } fn cleanup_test_file_info(name: &str, version_id: Uuid, metadata_value: &str) -> FileInfo { FileInfo { name: name.to_string(), version_id: Some(version_id), size: 128, mod_time: Some(OffsetDateTime::UNIX_EPOCH), data_dir: Some(Uuid::from_u128(100)), checksum: Some(Bytes::from_static(b"object-checksum")), metadata: HashMap::from([ ("etag".to_string(), "etag-value".to_string()), ("x-amz-meta-key".to_string(), metadata_value.to_string()), ]), parts: vec![ObjectPartInfo { number: 1, etag: "part-etag".to_string(), size: 128, actual_size: 128, mod_time: Some(OffsetDateTime::UNIX_EPOCH), checksums: Some(HashMap::from([(ChecksumType::CRC32C.to_string(), "part-checksum".to_string())])), ..Default::default() }], ..Default::default() } } fn cleanup_test_versions(versions: Vec) -> FileInfoVersions { FileInfoVersions { name: "object.txt".to_string(), versions, ..Default::default() } } #[test] fn test_source_cleanup_version_identities_accept_same_versions_out_of_order() { let first = cleanup_test_file_info("object.txt", Uuid::from_u128(1), "first"); let second = cleanup_test_file_info("object.txt", Uuid::from_u128(2), "second"); let expected = cleanup_test_versions(vec![first.clone(), second.clone()]); let current = cleanup_test_versions(vec![second, first]); assert!(source_cleanup_versions_match_with_allowed_missing(&expected, ¤t, &[])); } #[test] fn test_rebalance_entry_cleanup_preflight_rejects_changed_source_metadata() { let expected = cleanup_test_versions(vec![cleanup_test_file_info("object.txt", Uuid::from_u128(1), "source")]); let current = cleanup_test_versions(vec![cleanup_test_file_info("object.txt", Uuid::from_u128(1), "changed")]); let err = ensure_source_cleanup_versions_match(&expected, ¤t, &[]) .expect_err("changed source metadata must defer cleanup"); assert!(matches!(err, SourceCleanupError::SourceChanged)); } #[test] fn test_source_cleanup_preflight_rejects_changed_or_missing_free_version() { let mut expected = cleanup_test_versions(vec![cleanup_test_file_info("object.txt", Uuid::from_u128(1), "source")]); expected.free_versions = vec![cleanup_test_file_info("object.txt", Uuid::from_u128(2), "tier-cleanup")]; let mut changed = expected.clone(); changed.free_versions[0] .metadata .insert("x-amz-meta-key".to_string(), "changed".to_string()); assert!(!source_cleanup_versions_match_with_allowed_missing(&expected, &changed, &[])); let mut missing = expected.clone(); missing.free_versions.clear(); assert!(!source_cleanup_versions_match_with_allowed_missing(&expected, &missing, &[])); } #[test] fn test_source_cleanup_preflight_rejects_changed_transition_or_erasure() { let expected = cleanup_test_versions(vec![cleanup_test_file_info("object.txt", Uuid::from_u128(1), "source")]); let mut current = expected.clone(); current.versions[0].transition_tier = "COLD".to_string(); let err = ensure_source_cleanup_versions_match(&expected, ¤t, &[]) .expect_err("transition metadata changes must defer cleanup"); assert!(matches!(err, SourceCleanupError::SourceChanged)); let mut current = expected.clone(); current.versions[0].erasure.algorithm = "changed".to_string(); let err = ensure_source_cleanup_versions_match(&expected, ¤t, &[]) .expect_err("erasure metadata changes must defer cleanup"); assert!(matches!(err, SourceCleanupError::SourceChanged)); } #[test] fn test_source_cleanup_preflight_ignores_per_disk_erasure_fields() { let mut expected = cleanup_test_versions(vec![cleanup_test_file_info("object.txt", Uuid::from_u128(1), "source")]); expected.versions[0].erasure.checksums = vec![rustfs_filemeta::ChecksumInfo { part_number: 1, hash: Bytes::from_static(b"disk-a-checksum"), ..Default::default() }]; let mut current = expected.clone(); current.versions[0].erasure.index = 7; current.versions[0].erasure.checksums[0].hash = Bytes::from_static(b"disk-b-checksum"); assert!(source_cleanup_versions_match_with_allowed_missing(&expected, ¤t, &[])); } #[test] fn test_decommission_entry_cleanup_preflight_rejects_added_source_version() { let expected = cleanup_test_versions(vec![cleanup_test_file_info("object.txt", Uuid::from_u128(1), "source")]); let current = cleanup_test_versions(vec![ cleanup_test_file_info("object.txt", Uuid::from_u128(1), "source"), cleanup_test_file_info("object.txt", Uuid::from_u128(2), "new-version"), ]); let err = ensure_source_cleanup_versions_match(&expected, ¤t, &[]) .expect_err("an added source version must defer cleanup"); assert!(matches!(err, SourceCleanupError::SourceChanged)); } #[test] fn test_decommission_cleanup_preflight_accepts_allowed_expired_missing_version() { let migrated = cleanup_test_file_info("object.txt", Uuid::from_u128(1), "migrated"); let expired = cleanup_test_file_info("object.txt", Uuid::from_u128(2), "expired"); let expected = cleanup_test_versions(vec![migrated.clone(), expired.clone()]); let current = cleanup_test_versions(vec![migrated]); let allowed_missing = vec![source_cleanup_version_identity(&expired)]; assert!(source_cleanup_versions_match_with_allowed_missing(&expected, ¤t, &allowed_missing)); } #[test] fn test_decommission_cleanup_preflight_rejects_unexpected_missing_version() { let migrated = cleanup_test_file_info("object.txt", Uuid::from_u128(1), "migrated"); let protected = cleanup_test_file_info("object.txt", Uuid::from_u128(2), "protected"); let expected = cleanup_test_versions(vec![migrated.clone(), protected]); let current = cleanup_test_versions(vec![migrated]); let err = ensure_source_cleanup_versions_match(&expected, ¤t, &[]) .expect_err("an unexpected missing version must defer cleanup"); assert!(matches!(err, SourceCleanupError::SourceChanged)); } #[test] fn test_decommission_cleanup_preflight_rejects_new_version_with_allowed_missing() { let migrated = cleanup_test_file_info("object.txt", Uuid::from_u128(1), "migrated"); let expired = cleanup_test_file_info("object.txt", Uuid::from_u128(2), "expired"); let new_version = cleanup_test_file_info("object.txt", Uuid::from_u128(3), "new-version"); let expected = cleanup_test_versions(vec![migrated.clone(), expired.clone()]); let current = cleanup_test_versions(vec![migrated, new_version]); let allowed_missing = vec![source_cleanup_version_identity(&expired)]; let err = ensure_source_cleanup_versions_match(&expected, ¤t, &allowed_missing) .expect_err("a new source version must defer cleanup even when an expired version may be missing"); assert!(matches!(err, SourceCleanupError::SourceChanged)); } #[test] fn test_new_multipart_abort_flag_defaults_to_abort_enabled() { let flag = new_multipart_abort_flag(); assert!(should_abort_multipart_upload(&flag)); } #[test] fn test_mark_multipart_upload_completed_disables_abort_cleanup() { let flag = new_multipart_abort_flag(); mark_multipart_upload_completed(&flag); assert!(!should_abort_multipart_upload(&flag)); } #[test] fn test_resolve_data_movement_abort_result_wraps_abort_context() { let err = resolve_data_movement_abort_result( "rebalance_object", "bucket-a", "object-a", "upload-1", Error::SlowDown, Error::OperationCanceled, ); let message = err.to_string(); assert!(message.contains("rebalance_object: abort_multipart_upload failed")); assert!(message.contains("bucket-a/object-a")); assert!(message.contains("upload upload-1")); assert!(message.contains(Error::SlowDown.to_string().as_str())); } #[test] fn test_data_movement_stage_error_includes_stage_and_object() { let err = data_movement_stage_error("rebalance_object", "put_object", "bucket-a", "object-a", Error::SlowDown); let message = err.to_string(); assert!(message.contains("rebalance_object: put_object failed for bucket-a/object-a")); assert!(message.contains(Error::SlowDown.to_string().as_str())); } #[test] fn test_data_movement_part_stage_error_includes_stage_object_and_part() { let err = data_movement_part_stage_error("rebalance_object", "put_object_part", "bucket-a", "object-a", 7, Error::SlowDown); let message = err.to_string(); assert!(message.contains("rebalance_object: put_object_part failed for bucket-a/object-a part 7")); assert!(message.contains(Error::SlowDown.to_string().as_str())); } #[test] fn test_data_movement_part_upload_failure_stage_reports_short_read() { let err = Error::Io(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "short part")); assert_eq!(data_movement_part_upload_failure_stage(&err), "read_part"); } #[test] fn test_data_movement_part_upload_failure_stage_keeps_write_errors() { assert_eq!(data_movement_part_upload_failure_stage(&Error::SlowDown), "put_object_part"); } #[test] fn test_should_check_data_movement_overwrite_resume_accepts_conflict_errors() { assert!(should_check_data_movement_overwrite_resume(&Error::DataMovementOverwriteErr( "bucket-a".to_string(), "object-a".to_string(), "version-a".to_string(), ))); assert!(should_check_data_movement_overwrite_resume(&Error::PreconditionFailed)); assert!(!should_check_data_movement_overwrite_resume(&Error::SlowDown)); } #[test] fn test_decode_part_index_returns_none_when_absent() { assert!(decode_part_index(None).is_none()); } #[test] fn test_decode_part_index_returns_none_for_invalid_payload() { let invalid = Bytes::from_static(b"not-a-valid-index"); assert!(decode_part_index(Some(&invalid)).is_none()); } #[test] fn test_decode_part_index_returns_some_for_valid_payload() { let mut index = Index::new(); index.add(0, 0).expect("first index entry should be accepted"); index .add(2_097_152, 2_097_152) .expect("second index entry should advance totals"); let encoded = index.into_vec(); let decoded = decode_part_index(Some(&encoded)).expect("valid index payload should decode"); assert_eq!(decoded.total_uncompressed, 2_097_152); assert_eq!(decoded.total_compressed, 2_097_152); } #[test] fn test_data_movement_checksum_is_preserved_opaque() { let checksum = Bytes::from_static(b"sealed-or-plaintext-checksum"); let object_info = ObjectInfo { checksum: Some(checksum.clone()), ..Default::default() }; let opts = data_movement_put_object_opts(&object_info, 0); let encoded = rustfs_utils::http::get_header_map(&opts.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_SSEC_CRC) .expect("data movement must carry the persisted checksum out of band"); assert_eq!( base64_simd::STANDARD .decode_to_vec(&encoded) .expect("checksum marker should decode"), checksum ); assert!(!rustfs_utils::http::contains_key_str(&opts.user_defined, SUFFIX_CRC)); } #[tokio::test] async fn test_data_movement_single_part_raw_reader_does_not_validate_source_etag() { let raw_payload = b"raw-encrypted-or-compressed-bytes".to_vec(); let object_info = ObjectInfo { name: "object.txt".to_string(), size: i64::try_from(raw_payload.len()).expect("test payload size should fit i64"), actual_size: 128, etag: Some("logical-source-etag".to_string()), ..Default::default() }; let rd = GetObjectReader { stream: Box::new(Cursor::new(raw_payload.clone())), object_info: object_info.clone(), buffered_body: None, body_source: Default::default(), }; let mut data = data_movement_put_object_reader("bucket-a", &object_info, rd, "test_migration") .expect("raw data movement reader should ignore source ETag during stream validation"); let mut migrated = Vec::new(); data.stream .read_to_end(&mut migrated) .await .expect("raw data movement reader should consume payload without ETag mismatch"); assert_eq!(migrated, raw_payload); } #[test] fn test_data_movement_empty_checksum_adds_no_passthrough_marker() { let mut object_info = ObjectInfo { size: 32, actual_size: 128, etag: Some("etag-value".to_string()), checksum: Some(Bytes::new()), ..Default::default() }; rustfs_utils::http::insert_header_map( Arc::make_mut(&mut object_info.user_defined), rustfs_utils::http::SUFFIX_REPLICATION_SSEC_CRC, "stale-checksum", ); let opts = data_movement_put_object_opts(&object_info, 0); assert!( rustfs_utils::http::get_header_map(&opts.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_SSEC_CRC,).is_none() ); } #[test] fn test_data_movement_multipart_opts_strip_upload_checksum_contract() { let object_info = ObjectInfo { user_defined: Arc::new(HashMap::from([ (rustfs_rio::RUSTFS_MULTIPART_CHECKSUM.to_string(), ChecksumType::CRC64_NVME.to_string()), ( rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE.to_string(), ChecksumType::CRC64_NVME.obj_type().to_string(), ), ])), ..Default::default() }; let opts = data_movement_new_multipart_opts(&object_info, 0); assert!(!opts.user_defined.contains_key(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM)); assert!(!opts.user_defined.contains_key(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE)); } #[test] fn test_data_movement_multipart_opts_defer_part_checksums_until_completion() { let object_info = ObjectInfo { parts: Arc::new(vec![ObjectPartInfo { number: 2, checksums: Some(HashMap::from([(ChecksumType::CRC32C.to_string(), "crc32c-value".to_string())])), ..Default::default() }]), ..Default::default() }; let new_opts = data_movement_new_multipart_opts(&object_info, 0); let compatible_opts = data_movement_complete_multipart_opts(&object_info, 0, false).expect("compatible opts should be created"); let complete_opts = data_movement_complete_multipart_opts(&object_info, 0, true).expect("complete opts should be created"); assert!(!rustfs_utils::http::contains_key_str(&new_opts.user_defined, SUFFIX_PART_CHECKSUMS)); assert!(rustfs_utils::http::contains_key_str(&new_opts.user_defined, SUFFIX_DATA_MOVEMENT_UPLOAD)); assert!(!rustfs_utils::http::contains_key_str( &compatible_opts.user_defined, SUFFIX_PART_CHECKSUMS )); assert_eq!( rustfs_utils::http::get_consistent_str(&complete_opts.user_defined, SUFFIX_PART_CHECKSUMS), Some(r#"[[2,[["CRC32C","crc32c-value"]]]]"#) ); } #[test] fn test_data_movement_part_checksum_writer_requires_fleet_confirmation() { let object_info = ObjectInfo { parts: Arc::new(vec![ObjectPartInfo { checksums: Some(HashMap::from([("CRC32C".to_string(), "AAAAAA==".to_string())])), ..Default::default() }]), ..Default::default() }; assert!(!data_movement_part_checksum_writer_enabled_for(false, false)); assert!(!data_movement_part_checksum_writer_enabled_for(true, false)); assert!(!data_movement_part_checksum_writer_enabled_for(false, true)); assert!(data_movement_part_checksum_writer_enabled_for(true, true)); let mut compatible = FileInfo { parts: object_info.parts.as_ref().clone(), ..Default::default() }; prepare_tiered_data_movement_file_info_for(&mut compatible, false) .expect("disabled sidecar writer should preserve data movement compatibility"); assert!(compatible.parts.iter().all(|part| part.checksums.is_none())); assert!(!rustfs_utils::http::contains_key_str(&compatible.metadata, SUFFIX_PART_CHECKSUMS)); let empty = ObjectInfo { parts: Arc::new(vec![ObjectPartInfo { checksums: Some(HashMap::new()), ..Default::default() }]), ..Default::default() }; assert_eq!( data_movement_part_checksums(&empty.parts).expect("empty checksum maps should normalize"), None ); assert!( !empty .parts .iter() .any(|part| part.checksums.as_ref().is_some_and(|checksums| !checksums.is_empty())) ); } #[test] fn test_tiered_data_movement_prepares_and_validates_part_checksum_sidecar() { let valid_checksums = HashMap::from([("CRC32C".to_string(), "AAAAAA==".to_string())]); let mut valid = FileInfo { parts: vec![ObjectPartInfo { number: 1, checksums: Some(valid_checksums), ..Default::default() }], ..Default::default() }; let mut compatible = valid.clone(); prepare_tiered_data_movement_file_info_for(&mut compatible, false) .expect("disabled sidecar writer should omit optional part checksums"); assert!(compatible.parts.iter().all(|part| part.checksums.is_none())); assert!(!rustfs_utils::http::contains_key_str(&compatible.metadata, SUFFIX_PART_CHECKSUMS)); prepare_tiered_data_movement_file_info_for(&mut valid, true).expect("valid legacy part checksums should be encoded"); assert_eq!( rustfs_utils::http::get_consistent_str(&valid.metadata, SUFFIX_PART_CHECKSUMS), Some(r#"[[1,[["CRC32C","AAAAAA=="]]]]"#) ); let mut invalid = FileInfo { parts: vec![ObjectPartInfo { number: 1, checksums: Some(HashMap::from([("CRC32C".to_string(), "not-base64".to_string())])), ..Default::default() }], ..Default::default() }; assert!(matches!( prepare_tiered_data_movement_file_info_for(&mut invalid, true), Err(Error::FileCorrupt) )); } #[test] fn test_data_movement_preserves_multipart_topology_with_opaque_etag() { let object_info = ObjectInfo { etag: Some("0123456789abcdef0123456789abcdef".to_string()), parts: Arc::new(vec![ ObjectPartInfo { number: 2, ..Default::default() }, ObjectPartInfo { number: 7, ..Default::default() }, ]), ..Default::default() }; assert!(!object_info.is_multipart()); assert!(should_use_multipart_data_movement(&object_info, false)); let single_nonstandard_part = ObjectInfo { parts: Arc::new(vec![ObjectPartInfo { number: 7, ..Default::default() }]), ..object_info }; assert!(should_use_multipart_data_movement(&single_nonstandard_part, false)); } #[test] fn test_data_movement_part_reader_uses_stored_part_size_for_raw_stream() { let source_part = ObjectPartInfo { number: 1, size: 32, actual_size: 128, etag: "etag-1".to_string(), ..Default::default() }; let part_size = i64::try_from(source_part.size).expect("part size fits"); let part_actual_size = if source_part.actual_size > 0 { source_part.actual_size } else { part_size }; assert_eq!(part_size, 32); assert_eq!(part_actual_size, 128); } #[tokio::test] async fn test_multipart_part_stream_preserves_boundaries() { let stream: Box = Box::new(Cursor::new(b"abcdef".to_vec())); let shared = Arc::new(Mutex::new(stream)); let mut first = put_obj_reader_from_part_stream(shared.clone(), 3, 3, None).expect("first bounded part reader should be created"); let mut first_data = Vec::new(); first .stream .read_to_end(&mut first_data) .await .expect("first part should read only its boundary"); let mut second = put_obj_reader_from_part_stream(shared, 3, 3, None).expect("second bounded part reader should be created"); let mut second_data = Vec::new(); second .stream .read_to_end(&mut second_data) .await .expect("second part should continue at the next boundary"); assert_eq!(first_data, b"abc"); assert_eq!(second_data, b"def"); } #[tokio::test] async fn test_multipart_part_stream_does_not_request_full_large_part() { let largest_request = Arc::new(AtomicUsize::new(0)); let stream: Box = Box::new(MaxReadRequestReader::new(16 * 1024 * 1024, 8 * 1024, largest_request.clone())); let shared = Arc::new(Mutex::new(stream)); let mut data = put_obj_reader_from_part_stream(shared, 16 * 1024 * 1024, 16 * 1024 * 1024, None) .expect("large bounded part reader should be created without allocating part size"); let mut buf = [0u8; 8 * 1024]; let read = data .stream .read(&mut buf) .await .expect("bounded reader should satisfy a small read against a large advertised part"); assert_eq!(read, buf.len()); assert!(largest_request.load(Ordering::Relaxed) <= buf.len()); } #[tokio::test] async fn test_multipart_part_stream_reports_short_part() { let stream: Box = Box::new(Cursor::new(b"abc".to_vec())); let shared = Arc::new(Mutex::new(stream)); let mut data = put_obj_reader_from_part_stream(shared, 5, 5, None).expect("short bounded part reader should be created"); let err = data .stream .read_to_end(&mut Vec::new()) .await .expect_err("short source stream should fail the part reader"); assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof); } #[test] fn test_multipart_part_stream_preserves_index() { let mut index = Index::new(); index.add(0, 0).expect("index entry should be accepted"); let stream: Box = Box::new(Cursor::new(b"abc".to_vec())); let shared = Arc::new(Mutex::new(stream)); let data = put_obj_reader_from_part_stream(shared, 3, 3, Some(index)) .expect("bounded part reader should retain compression index"); assert!(data.stream.try_get_index().is_some()); } #[test] fn test_data_movement_metadata_equivalence_accepts_required_fields() { let version_id = Uuid::nil(); let mod_time = OffsetDateTime::UNIX_EPOCH; let metadata = Arc::new(HashMap::from([ ("x-amz-meta-key".to_string(), "value".to_string()), (rustfs_utils::http::AMZ_STORAGE_CLASS.to_string(), "STANDARD_IA".to_string()), (X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(), "GOVERNANCE".to_string()), ( X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(), "2030-01-01T00:00:00Z".to_string(), ), (X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), "ON".to_string()), ])); let part = ObjectPartInfo { number: 1, etag: "part-etag".to_string(), size: 128, actual_size: 128, checksums: Some(HashMap::from([(ChecksumType::CRC32C.to_string(), "part-checksum".to_string())])), ..Default::default() }; let info = ObjectInfo { version_id: Some(version_id), etag: Some("etag-value".to_string()), size: 128, actual_size: 128, mod_time: Some(mod_time), user_defined: metadata, storage_class: Some("STANDARD_IA".to_string()), checksum: Some(Bytes::from_static(b"object-checksum")), replication_status_internal: Some("arn:minio:replication:target=COMPLETED;".to_string()), replication_status: ReplicationStatusType::Completed, version_purge_status_internal: Some("arn:minio:replication:target=PENDING;".to_string()), version_purge_status: VersionPurgeStatusType::Pending, parts: Arc::new(vec![part]), ..Default::default() }; assert_data_movement_metadata_equivalent(&info, &info.clone()); } #[test] fn test_data_movement_opts_preserve_replication_and_object_lock_metadata() { let version_id = Uuid::nil(); let object_info = ObjectInfo { version_id: Some(version_id), user_defined: Arc::new(HashMap::from([ ( rustfs_utils::http::SUFFIX_REPLICATION_STATUS.to_string(), "arn:minio:target=PENDING;".to_string(), ), (X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(), "COMPLIANCE".to_string()), ( X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(), "2031-01-01T00:00:00Z".to_string(), ), (X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), "ON".to_string()), ])), ..Default::default() }; let put_opts = data_movement_put_object_opts(&object_info, 3); let new_multipart_opts = data_movement_new_multipart_opts(&object_info, 3); assert_eq!( put_opts.user_defined.get(rustfs_utils::http::SUFFIX_REPLICATION_STATUS), Some(&"arn:minio:target=PENDING;".to_string()) ); assert_eq!( new_multipart_opts.user_defined.get(X_AMZ_OBJECT_LOCK_MODE.as_str()), Some(&"COMPLIANCE".to_string()) ); assert_eq!( put_opts.user_defined.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str()), Some(&"2031-01-01T00:00:00Z".to_string()) ); assert_eq!( new_multipart_opts.user_defined.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()), Some(&"ON".to_string()) ); } #[test] fn test_data_movement_opts_preserve_tags_and_expires() { let expires = OffsetDateTime::from_unix_timestamp(2_000).expect("valid timestamp"); let object_info = ObjectInfo { user_defined: Arc::new(HashMap::from([("x-amz-meta-key".to_string(), "value".to_string())])), user_tags: Arc::new("tag=value".to_string()), expires: Some(expires), ..Default::default() }; let put_opts = data_movement_put_object_opts(&object_info, 1); let multipart_opts = data_movement_new_multipart_opts(&object_info, 1); assert_eq!( put_opts .user_defined .get(rustfs_utils::http::AMZ_OBJECT_TAGGING) .map(String::as_str), Some("tag=value") ); assert_eq!( multipart_opts .user_defined .get(rustfs_utils::http::AMZ_OBJECT_TAGGING) .map(String::as_str), Some("tag=value") ); assert!(put_opts.user_defined.contains_key("expires")); assert!(multipart_opts.user_defined.contains_key("expires")); assert_eq!(put_opts.user_defined.get("x-amz-meta-key").map(String::as_str), Some("value")); } #[test] fn test_data_movement_opts_canonicalize_derived_binary_metadata() { let transition_version = Uuid::from_u128(400).to_string(); let checksum = Checksum::new_from_data(ChecksumType::CRC32C, b"checksum-payload") .expect("checksum should be created") .to_bytes(&[]); let mut object_info = ObjectInfo { checksum: Some(checksum), transitioned_object: crate::storage_api_contracts::lifecycle::TransitionedObject { name: "remote/object".to_string(), version_id: transition_version.clone(), tier: "WARM".to_string(), status: rustfs_filemeta::TRANSITION_COMPLETE.to_string(), ..Default::default() }, transition_version_state: rustfs_filemeta::TransitionVersionState::Exact, ..Default::default() }; Arc::make_mut(&mut object_info.user_defined) .insert(format!("{}{SUFFIX_CRC}", rustfs_utils::http::MINIO_INTERNAL_PREFIX), String::new()); Arc::make_mut(&mut object_info.user_defined).insert( format!("{}{SUFFIX_TRANSITIONED_VERSION_ID}", rustfs_utils::http::MINIO_INTERNAL_PREFIX), String::new(), ); let metadata = data_movement_new_multipart_opts(&object_info, 1).user_defined; assert!(!rustfs_utils::http::contains_key_str(&metadata, SUFFIX_CRC)); assert_eq!( rustfs_utils::http::get_consistent_str(&metadata, SUFFIX_TRANSITIONED_VERSION_ID), Some(transition_version.as_str()) ); assert_eq!( rustfs_utils::http::get_consistent_str(&metadata, SUFFIX_TRANSITIONED_VERSION_STATE), Some("exact") ); } #[test] fn test_data_movement_opts_preserve_unusable_transition_versions() { for value in ["opaque\0version".to_string(), "x".repeat(1_025)] { let key = rustfs_utils::http::internal_key_rustfs(SUFFIX_TRANSITIONED_VERSION_ID); let mut source = overwrite_equivalence_source(); source.user_defined = Arc::new(HashMap::from([(key.clone(), value.clone())])); let opts = data_movement_new_multipart_opts(&source, 0); assert_eq!(opts.user_defined.get(&key), Some(&value)); let mut target = source.clone(); let different_value = if value.contains('\0') { "different\0version".to_string() } else { "y".repeat(1_025) }; Arc::make_mut(&mut target.user_defined).insert(key, different_value); assert!(!overwrite_resume_for_target(&source, target)); } } #[test] fn test_data_movement_new_multipart_opts_preserves_etag_and_version() { let version_id = Uuid::nil(); let object_info = ObjectInfo { version_id: Some(version_id), etag: Some("etag-value".to_string()), user_defined: Arc::new(std::collections::HashMap::from([("x-amz-meta-key".to_string(), "value".to_string())])), ..Default::default() }; let opts = data_movement_new_multipart_opts(&object_info, 7); assert!(opts.versioned); assert_eq!(opts.version_id.as_deref(), Some(version_id.to_string().as_str())); assert_eq!(opts.preserve_etag.as_deref(), Some("etag-value")); assert_eq!(opts.user_defined.get("x-amz-meta-key").map(String::as_str), Some("value")); assert_eq!(opts.src_pool_idx, 7); assert!(opts.data_movement); } #[test] fn test_data_movement_complete_multipart_opts_preserves_mod_time_version_and_etag() { let mod_time = OffsetDateTime::now_utc(); let version_id = Uuid::from_u128(7); let object_info = ObjectInfo { version_id: Some(version_id), mod_time: Some(mod_time), etag: Some("etag-value".to_string()), ..Default::default() }; let opts = data_movement_complete_multipart_opts(&object_info, 7, false).expect("complete opts should encode metadata"); assert!(opts.versioned); assert!(opts.data_movement); assert_eq!(opts.mod_time, Some(mod_time)); assert_eq!(opts.version_id.as_deref(), Some(version_id.to_string().as_str())); assert_eq!(opts.preserve_etag.as_deref(), Some("etag-value")); assert_eq!(opts.src_pool_idx, 7); assert_eq!( opts.http_preconditions .as_ref() .and_then(HTTPPreconditions::if_none_match_value), Some("*") ); } #[test] fn test_data_movement_put_object_opts_preserves_version_and_etag() { let version_id = Uuid::from_u128(9); let object_info = ObjectInfo { version_id: Some(version_id), mod_time: Some(OffsetDateTime::UNIX_EPOCH), etag: Some("etag-value".to_string()), user_defined: Arc::new(std::collections::HashMap::from([("x-amz-meta-key".to_string(), "value".to_string())])), ..Default::default() }; let opts = data_movement_put_object_opts(&object_info, 9); assert!(opts.versioned); assert_eq!(opts.version_id.as_deref(), Some(version_id.to_string().as_str())); assert_eq!(opts.preserve_etag.as_deref(), Some("etag-value")); assert_eq!(opts.user_defined.get("x-amz-meta-key").map(String::as_str), Some("value")); assert_eq!(opts.src_pool_idx, 9); assert!(opts.data_movement); assert_eq!(opts.mod_time, object_info.mod_time); assert_eq!( opts.http_preconditions .as_ref() .and_then(HTTPPreconditions::if_none_match_value), Some("*") ); } #[test] fn test_data_movement_put_opts_do_not_persist_multipart_part_checksums() { let object_info = ObjectInfo { etag: Some("0123456789abcdef0123456789abcdef".to_string()), parts: Arc::new(vec![ ObjectPartInfo { number: 1, checksums: Some(HashMap::from([("CRC32C".to_string(), "AAAAAA==".to_string())])), ..Default::default() }, ObjectPartInfo { number: 2, checksums: Some(HashMap::from([("CRC32C".to_string(), "BBBBBB==".to_string())])), ..Default::default() }, ]), ..Default::default() }; assert!(!object_info.is_multipart()); assert!(object_info.parts.iter().any(|part| part.checksums.is_some())); let opts = data_movement_put_object_opts(&object_info, 0); assert!(!rustfs_utils::http::contains_key_str(&opts.user_defined, SUFFIX_PART_CHECKSUMS)); } #[test] fn test_data_movement_put_and_complete_require_absent_target() { for version_id in [None, Some(Uuid::nil()), Some(Uuid::from_u128(1))] { let object_info = ObjectInfo { version_id, ..Default::default() }; let put_opts = data_movement_put_object_opts(&object_info, 9); let complete_opts = data_movement_complete_multipart_opts(&object_info, 9, false).expect("complete opts should encode metadata"); assert_eq!( put_opts .http_preconditions .as_ref() .and_then(HTTPPreconditions::if_none_match_value), Some("*") ); assert_eq!( complete_opts .http_preconditions .as_ref() .and_then(HTTPPreconditions::if_none_match_value), Some("*") ); } } #[test] fn test_stale_data_movement_target_replacement_requires_exact_owned_generation() { let version_id = Uuid::from_u128(41); let source_time = OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2); let opts = ObjectOptions { data_movement: true, versioned: true, version_id: Some(version_id.to_string()), mod_time: Some(source_time), http_preconditions: Some(data_movement_target_precondition()), ..Default::default() }; let mut metadata = HashMap::new(); rustfs_utils::http::insert_str(&mut metadata, SUFFIX_DATA_MOVED, "true".to_string()); rustfs_utils::http::insert_str(&mut metadata, SUFFIX_DATA_MOVED_TAGS, "v1:".to_string()); let target = ObjectInfo { version_id: Some(version_id), mod_time: Some(source_time - time::Duration::SECOND), user_defined: Arc::new(metadata), ..Default::default() }; assert!(can_replace_stale_data_movement_target(&target, &opts)); let mut client_target = target.clone(); client_target.user_defined = Arc::new(HashMap::new()); assert!(!can_replace_stale_data_movement_target(&client_target, &opts)); let mut single_marker = target.clone(); Arc::make_mut(&mut single_marker.user_defined) .remove(&format!("{}{SUFFIX_DATA_MOVED}", rustfs_utils::http::MINIO_INTERNAL_PREFIX)); assert!(!can_replace_stale_data_movement_target(&single_marker, &opts)); let mut conflicting_marker = target.clone(); Arc::make_mut(&mut conflicting_marker.user_defined).insert( format!("{}{SUFFIX_DATA_MOVED}", rustfs_utils::http::MINIO_INTERNAL_PREFIX), "false".to_string(), ); assert!(!can_replace_stale_data_movement_target(&conflicting_marker, &opts)); let mut retagged_target = target.clone(); retagged_target.user_tags = Arc::new("acknowledged=true".to_string()); assert!(!can_replace_stale_data_movement_target(&retagged_target, &opts)); let mut retagged_owned_target = retagged_target; rustfs_utils::http::insert_str( Arc::make_mut(&mut retagged_owned_target.user_defined), SUFFIX_DATA_MOVED_TAGS, "v1:acknowledged=true".to_string(), ); assert!(can_replace_stale_data_movement_target(&retagged_owned_target, &opts)); let mut different_version = target.clone(); different_version.version_id = Some(Uuid::from_u128(42)); assert!(!can_replace_stale_data_movement_target(&different_version, &opts)); let mut same_generation = target.clone(); same_generation.mod_time = Some(source_time); assert!(!can_replace_stale_data_movement_target(&same_generation, &opts)); let mut delete_marker = target; delete_marker.delete_marker = true; assert!(!can_replace_stale_data_movement_target(&delete_marker, &opts)); } #[test] fn test_is_equivalent_data_movement_object_accepts_matching_metadata() { let version_id = Uuid::nil(); let info = ObjectInfo { version_id: Some(version_id), size: 128, actual_size: 96, etag: Some("etag-value".to_string()), checksum: Some(Bytes::from_static(b"checksum")), mod_time: Some(OffsetDateTime::UNIX_EPOCH), ..Default::default() }; assert!(is_equivalent_data_movement_object(&info, &info.clone())); } #[test] fn test_is_equivalent_data_movement_object_rejects_content_mismatch() { let source = ObjectInfo { version_id: Some(Uuid::nil()), size: 128, actual_size: 96, etag: Some("etag-source".to_string()), checksum: Some(Bytes::from_static(b"checksum-source")), mod_time: Some(OffsetDateTime::UNIX_EPOCH), ..Default::default() }; let target = ObjectInfo { etag: Some("etag-target".to_string()), checksum: Some(Bytes::from_static(b"checksum-target")), ..source.clone() }; assert!(!is_equivalent_data_movement_object(&source, &target)); } #[test] fn test_is_equivalent_data_movement_object_rejects_user_metadata_mismatch() { let source = ObjectInfo { version_id: Some(Uuid::nil()), size: 128, etag: Some("etag-value".to_string()), user_defined: Arc::new(HashMap::from([("x-amz-meta-key".to_string(), "source".to_string())])), storage_class: Some("STANDARD_IA".to_string()), ..Default::default() }; let target = ObjectInfo { user_defined: Arc::new(HashMap::from([("x-amz-meta-key".to_string(), "target".to_string())])), ..source.clone() }; assert!(!is_equivalent_data_movement_object(&source, &target)); } #[test] fn test_is_equivalent_data_movement_object_rejects_storage_class_mismatch() { let source = ObjectInfo { version_id: Some(Uuid::nil()), size: 128, etag: Some("etag-value".to_string()), storage_class: Some("STANDARD_IA".to_string()), ..Default::default() }; let target = ObjectInfo { storage_class: Some("STANDARD".to_string()), ..source.clone() }; assert!(!is_equivalent_data_movement_object(&source, &target)); } #[test] fn test_is_equivalent_data_movement_object_uses_effective_actual_size() { let source = ObjectInfo { size: 128, actual_size: 0, etag: Some("etag-value".to_string()), ..Default::default() }; let target = ObjectInfo { size: 128, actual_size: 128, etag: Some("etag-value".to_string()), ..Default::default() }; assert!(is_equivalent_data_movement_object(&source, &target)); } fn overwrite_equivalence_source() -> ObjectInfo { let part = ObjectPartInfo { number: 1, etag: "part-etag".to_string(), size: 128, actual_size: 128, mod_time: Some(OffsetDateTime::UNIX_EPOCH), index: Some(Bytes::from_static(&[1, 2, 3])), checksums: Some(HashMap::from([(ChecksumType::CRC32C.to_string(), "part-checksum".to_string())])), ..Default::default() }; ObjectInfo { version_id: Some(Uuid::from_u128(1)), size: 128, actual_size: 128, etag: Some("etag-value".to_string()), checksum: Some(Bytes::from_static(b"object-checksum")), mod_time: Some(OffsetDateTime::UNIX_EPOCH), user_defined: Arc::new(HashMap::from([("x-amz-meta-key".to_string(), "value".to_string())])), user_tags: Arc::new("tag=value".to_string()), expires: Some(OffsetDateTime::from_unix_timestamp(2_000).expect("valid expires timestamp")), storage_class: Some("STANDARD_IA".to_string()), replication_status_internal: Some("arn:minio:replication:target=COMPLETED;".to_string()), replication_status: ReplicationStatusType::Completed, version_purge_status_internal: Some("arn:minio:replication:target=PENDING;".to_string()), version_purge_status: VersionPurgeStatusType::Pending, parts: Arc::new(vec![part]), ..Default::default() } } fn overwrite_resume_for_target(source: &ObjectInfo, target: ObjectInfo) -> bool { overwrite_resume_for_target_with_checksums(source, target, data_movement_part_checksum_writer_enabled()) } fn overwrite_resume_for_target_with_checksums(source: &ObjectInfo, target: ObjectInfo, compare_part_checksums: bool) -> bool { let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string()); resolve_data_movement_overwrite_resume_result_for(&err, Ok(Some(target)), source, 0, 1, compare_part_checksums) .expect("overwrite target should be evaluated") } #[test] fn test_data_movement_overwrite_resume_accepts_full_equivalence() { let source = overwrite_equivalence_source(); assert!(overwrite_resume_for_target(&source, source.clone())); } #[test] fn test_data_movement_overwrite_resume_accepts_part_mod_time_drift() { let source = overwrite_equivalence_source(); let mut target = source.clone(); let mut parts = target.parts.as_ref().clone(); parts[0].mod_time = Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND); target.parts = Arc::new(parts); assert!(overwrite_resume_for_target(&source, target)); } #[test] fn test_data_movement_overwrite_resume_accepts_legacy_part_actual_size_fallback() { let mut source = overwrite_equivalence_source(); let mut source_parts = source.parts.as_ref().clone(); source_parts[0].actual_size = 0; source.parts = Arc::new(source_parts); let mut target = source.clone(); let mut target_parts = target.parts.as_ref().clone(); target_parts[0].actual_size = i64::try_from(target_parts[0].size).expect("part size should fit i64"); target.parts = Arc::new(target_parts); assert!(overwrite_resume_for_target(&source, target)); } #[test] fn test_data_movement_overwrite_resume_preserves_negative_part_actual_size() { let mut source = overwrite_equivalence_source(); let mut source_parts = source.parts.as_ref().clone(); source_parts[0].actual_size = -1; source.parts = Arc::new(source_parts); let mut target = source.clone(); let mut target_parts = target.parts.as_ref().clone(); target_parts[0].actual_size = i64::try_from(target_parts[0].size).expect("part size should fit i64"); target.parts = Arc::new(target_parts); assert!(!overwrite_resume_for_target(&source, target)); assert!(overwrite_resume_for_target(&source, source.clone())); } #[test] fn test_data_movement_overwrite_resume_accepts_destination_marker() { let source = overwrite_equivalence_source(); let mut target = source.clone(); let mut metadata = target.user_defined.as_ref().clone(); rustfs_utils::http::insert_str(&mut metadata, SUFFIX_DATA_MOVED, "true".to_string()); target.user_defined = Arc::new(metadata); assert!(overwrite_resume_for_target(&source, target)); } #[test] fn test_data_movement_overwrite_resume_accepts_compatible_internal_aliases() { for suffix in [SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION_SIZE] { let mut source = overwrite_equivalence_source(); let mut source_metadata = source.user_defined.as_ref().clone(); if suffix == SUFFIX_COMPRESSION_SIZE { rustfs_utils::http::insert_str( &mut source_metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "klauspost/compress/s2".to_string(), ); } source_metadata.insert(format!("X-Minio-Internal-{suffix}"), "000128".to_string()); source.user_defined = Arc::new(source_metadata); let mut target = source.clone(); let mut target_metadata = target.user_defined.as_ref().clone(); rustfs_utils::http::insert_str(&mut target_metadata, suffix, "128".to_string()); target.user_defined = Arc::new(target_metadata); assert!( overwrite_resume_for_target(&source, target), "compatible aliases for {suffix} should match" ); let mut source_without_marker = source.clone(); rustfs_utils::http::remove_str(Arc::make_mut(&mut source_without_marker.user_defined), suffix); let mut target_with_generated_marker = source_without_marker.clone(); rustfs_utils::http::insert_str( Arc::make_mut(&mut target_with_generated_marker.user_defined), suffix, "128".to_string(), ); assert!( overwrite_resume_for_target(&source_without_marker, target_with_generated_marker), "a generated target marker for {suffix} should match" ); } } #[test] fn test_data_movement_overwrite_resume_rejects_conflicting_internal_aliases() { for suffix in [SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION_SIZE] { let mut source = overwrite_equivalence_source(); if suffix == SUFFIX_COMPRESSION_SIZE { rustfs_utils::http::insert_str( Arc::make_mut(&mut source.user_defined), rustfs_utils::http::SUFFIX_COMPRESSION, "klauspost/compress/s2".to_string(), ); } let mut target = source.clone(); let mut metadata = target.user_defined.as_ref().clone(); rustfs_utils::http::insert_str(&mut metadata, suffix, "128".to_string()); metadata.insert(rustfs_utils::http::internal_key_rustfs(suffix), "64".to_string()); target.user_defined = Arc::new(metadata); assert!( !overwrite_resume_for_target(&source, target), "conflicting aliases for {suffix} must fail closed" ); rustfs_utils::http::insert_str(Arc::make_mut(&mut source.user_defined), suffix, "128".to_string()); let mut target_without_marker = source.clone(); rustfs_utils::http::remove_str(Arc::make_mut(&mut target_without_marker.user_defined), suffix); assert!( !overwrite_resume_for_target(&source, target_without_marker), "a missing target marker for {suffix} must fail closed" ); } } #[test] fn test_data_movement_overwrite_resume_rejects_unreadable_actual_size() { let mut source = overwrite_equivalence_source(); source.actual_size = 0; rustfs_utils::http::insert_str( Arc::make_mut(&mut source.user_defined), rustfs_utils::http::SUFFIX_COMPRESSION, "klauspost/compress/s2".to_string(), ); rustfs_utils::http::insert_str( Arc::make_mut(&mut source.user_defined), SUFFIX_ACTUAL_SIZE, "invalid-source-size".to_string(), ); let mut target = source.clone(); rustfs_utils::http::insert_str( Arc::make_mut(&mut target.user_defined), SUFFIX_ACTUAL_SIZE, "invalid-target-size".to_string(), ); assert!(!overwrite_resume_for_target(&source, target)); } #[test] fn test_data_movement_overwrite_resume_rejects_missing_unreadable_actual_size() { let mut source = overwrite_equivalence_source(); source.actual_size = 0; rustfs_utils::http::insert_str( Arc::make_mut(&mut source.user_defined), rustfs_utils::http::SUFFIX_COMPRESSION, "klauspost/compress/s2".to_string(), ); let mut parts = source.parts.as_ref().clone(); parts[0].actual_size = 0; source.parts = Arc::new(parts); assert!(!overwrite_resume_for_target(&source, source.clone())); } #[test] fn test_data_movement_overwrite_resume_accepts_checksum_alias_expansion() { let mut source = overwrite_equivalence_source(); let checksum = rustfs_rio::Checksum::new_from_data(ChecksumType::CRC32C, b"checksum-payload") .expect("checksum should be created") .to_bytes(&[]); assert!( std::str::from_utf8(&checksum).is_err(), "wire checksum should exercise non-UTF-8 metadata" ); source.checksum = Some(checksum); Arc::make_mut(&mut source.user_defined) .insert(format!("{}{SUFFIX_CRC}", rustfs_utils::http::MINIO_INTERNAL_PREFIX), String::new()); let mut target = source.clone(); rustfs_utils::http::insert_str(Arc::make_mut(&mut target.user_defined), SUFFIX_CRC, String::new()); assert!(overwrite_resume_for_target(&source, target)); } #[test] fn test_data_movement_overwrite_resume_accepts_empty_checksum_normalization() { let mut source = overwrite_equivalence_source(); source.checksum = None; source.user_defined = Arc::new(HashMap::from([( format!("{}{SUFFIX_CRC}", rustfs_utils::http::MINIO_INTERNAL_PREFIX), String::new(), )])); let mut target = source.clone(); target.user_defined = Arc::new(HashMap::new()); assert!(is_equivalent_data_movement_object(&source, &target)); } #[test] fn test_data_movement_overwrite_resume_rejects_checksum_alias_conflict() { let mut source = overwrite_equivalence_source(); rustfs_utils::http::insert_str(Arc::make_mut(&mut source.user_defined), SUFFIX_CRC, "object-checksum".to_string()); let mut target_without_marker = source.clone(); rustfs_utils::http::remove_str(Arc::make_mut(&mut target_without_marker.user_defined), SUFFIX_CRC); assert!(!overwrite_resume_for_target(&source, target_without_marker)); let mut target = source.clone(); rustfs_utils::http::insert_str(Arc::make_mut(&mut target.user_defined), SUFFIX_CRC, "different".to_string()); assert!(!overwrite_resume_for_target(&source, target)); } #[test] fn test_data_movement_overwrite_resume_accepts_transition_alias_expansion() { let mut source = overwrite_equivalence_source(); source.transitioned_object.name = "remote/object".to_string(); source.transitioned_object.version_id = Uuid::from_u128(300).to_string(); source.transitioned_object.tier = "WARM".to_string(); source.transitioned_object.status = rustfs_filemeta::TRANSITION_COMPLETE.to_string(); source.transition_version_state = rustfs_filemeta::TransitionVersionState::Exact; let markers = [ (SUFFIX_TRANSITION_STATUS, source.transitioned_object.status.as_str()), (SUFFIX_TRANSITIONED_OBJECTNAME, source.transitioned_object.name.as_str()), (SUFFIX_TRANSITIONED_VERSION_ID, source.transitioned_object.version_id.as_str()), (SUFFIX_TRANSITIONED_VERSION_STATE, "exact"), (SUFFIX_TRANSITION_TIER, source.transitioned_object.tier.as_str()), ]; for (suffix, value) in markers { Arc::make_mut(&mut source.user_defined) .insert(format!("{}{suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX), value.to_string()); } let mut target = source.clone(); for (suffix, value) in markers { rustfs_utils::http::insert_str(Arc::make_mut(&mut target.user_defined), suffix, value.to_string()); } assert!(overwrite_resume_for_target(&source, target.clone())); target.transitioned_object.tier = "OTHER".to_string(); assert!(!overwrite_resume_for_target(&source, target)); } #[test] fn test_data_movement_overwrite_resume_accepts_empty_transition_marker_normalization() { for value in [ String::new(), "\0".repeat(16), Uuid::nil().to_string(), Uuid::nil().simple().to_string(), ] { let mut source = overwrite_equivalence_source(); Arc::make_mut(&mut source.user_defined).insert( format!("{}{SUFFIX_TRANSITIONED_VERSION_ID}", rustfs_utils::http::MINIO_INTERNAL_PREFIX), value, ); Arc::make_mut(&mut source.user_defined).insert( format!("{}{SUFFIX_TRANSITIONED_VERSION_STATE}", rustfs_utils::http::MINIO_INTERNAL_PREFIX), "unknown".to_string(), ); let mut target = source.clone(); rustfs_utils::http::remove_str(Arc::make_mut(&mut target.user_defined), SUFFIX_TRANSITIONED_VERSION_ID); rustfs_utils::http::remove_str(Arc::make_mut(&mut target.user_defined), SUFFIX_TRANSITIONED_VERSION_STATE); assert!(overwrite_resume_for_target(&source, target)); } } #[test] fn test_data_movement_overwrite_resume_rejects_unparsed_transition_case_alias_mismatch() { let mut source = overwrite_equivalence_source(); Arc::make_mut(&mut source.user_defined).insert("X-Minio-Internal-transition-tier".to_string(), "source-tier".to_string()); let mut target = source.clone(); Arc::make_mut(&mut target.user_defined).insert("X-Minio-Internal-transition-tier".to_string(), "target-tier".to_string()); assert!(!overwrite_resume_for_target(&source, target)); } #[test] fn test_data_movement_overwrite_resume_accepts_rebuilt_transition_case_alias() { let mut source = overwrite_equivalence_source(); source.transitioned_object.tier = "WARM".to_string(); rustfs_utils::http::insert_str(Arc::make_mut(&mut source.user_defined), SUFFIX_TRANSITION_TIER, "WARM".to_string()); Arc::make_mut(&mut source.user_defined).insert("X-Minio-Internal-transition-tier".to_string(), "WARM".to_string()); let mut target = source.clone(); Arc::make_mut(&mut target.user_defined).remove("X-Minio-Internal-transition-tier"); assert!(overwrite_resume_for_target(&source, target)); } #[test] fn test_data_movement_overwrite_resume_accepts_target_local_layout_marker() { let source_data_dir = Uuid::from_u128(100); let mut source = overwrite_equivalence_source(); source.data_dir = Some(source_data_dir); let mut source_metadata = source.user_defined.as_ref().clone(); rustfs_utils::http::insert_str( &mut source_metadata, crate::object_api::ENCRYPTED_PART_LAYOUT_QUORUM_SUFFIX, source_data_dir.to_string(), ); source.user_defined = Arc::new(source_metadata); let target_data_dir = Uuid::from_u128(200); let mut target = source.clone(); target.data_dir = Some(target_data_dir); let mut target_metadata = target.user_defined.as_ref().clone(); rustfs_utils::http::insert_str( &mut target_metadata, crate::object_api::ENCRYPTED_PART_LAYOUT_QUORUM_SUFFIX, target_data_dir.to_string(), ); target.user_defined = Arc::new(target_metadata); assert!(overwrite_resume_for_target(&source, target.clone())); let mut target_without_marker = target.clone(); rustfs_utils::http::remove_str( Arc::make_mut(&mut target_without_marker.user_defined), crate::object_api::ENCRYPTED_PART_LAYOUT_QUORUM_SUFFIX, ); assert!(overwrite_resume_for_target(&source, target_without_marker.clone())); target_without_marker.data_dir = None; assert!(!overwrite_resume_for_target(&source, target_without_marker)); let mut source_without_marker = source.clone(); rustfs_utils::http::remove_str( Arc::make_mut(&mut source_without_marker.user_defined), crate::object_api::ENCRYPTED_PART_LAYOUT_QUORUM_SUFFIX, ); assert!(overwrite_resume_for_target(&source_without_marker, target.clone())); source_without_marker.data_dir = Some(Uuid::nil()); assert!(!overwrite_resume_for_target(&source_without_marker, target.clone())); let mut invalid_target_metadata = target.user_defined.as_ref().clone(); rustfs_utils::http::insert_str( &mut invalid_target_metadata, crate::object_api::ENCRYPTED_PART_LAYOUT_QUORUM_SUFFIX, source_data_dir.to_string(), ); target.user_defined = Arc::new(invalid_target_metadata); assert!(!overwrite_resume_for_target(&source, target)); } #[test] fn test_data_movement_overwrite_resume_rejects_missing_part_checksum() { let source = overwrite_equivalence_source(); let mut target = source.clone(); let mut parts = target.parts.as_ref().clone(); parts[0].checksums = None; target.parts = Arc::new(parts); assert!(!overwrite_resume_for_target_with_checksums(&source, target, true)); } #[test] fn test_data_movement_overwrite_resume_accepts_dropped_part_index() { let source = overwrite_equivalence_source(); let mut target = source.clone(); let mut parts = target.parts.as_ref().clone(); parts[0].index = None; target.parts = Arc::new(parts); assert!(overwrite_resume_for_target(&source, target)); } #[test] fn test_data_movement_overwrite_resume_rejects_target_only_part_index() { let mut source = overwrite_equivalence_source(); let mut source_parts = source.parts.as_ref().clone(); source_parts[0].index = None; source.parts = Arc::new(source_parts); let mut target = source.clone(); let mut target_parts = target.parts.as_ref().clone(); target_parts[0].index = Some(Bytes::from_static(&[9])); target.parts = Arc::new(target_parts); assert!(!overwrite_resume_for_target(&source, target)); } #[test] fn test_data_movement_overwrite_resume_accepts_generated_part_checksum_marker() { let source = overwrite_equivalence_source(); let mut target = source.clone(); rustfs_utils::http::insert_str( Arc::make_mut(&mut target.user_defined), SUFFIX_PART_CHECKSUMS, r#"[[1,[["CRC32C","part-checksum"]]]]"#.to_string(), ); assert!(overwrite_resume_for_target(&source, target)); } fn overwrite_equivalence_source_with_two_parts() -> ObjectInfo { let source = overwrite_equivalence_source(); let mut parts = source.parts.as_ref().clone(); let mut second_part = parts[0].clone(); second_part.number = 2; second_part.etag = "part-etag-2".to_string(); second_part.index = Some(Bytes::from_static(&[4, 5, 6])); second_part.checksums = Some(HashMap::from([(ChecksumType::CRC32C.to_string(), "part-checksum-2".to_string())])); parts.push(second_part); ObjectInfo { parts: Arc::new(parts), ..source } } #[test] fn test_data_movement_overwrite_resume_accepts_parts_reordered_by_number() { let source = overwrite_equivalence_source_with_two_parts(); let mut target = source.clone(); let mut parts = target.parts.as_ref().clone(); parts.reverse(); target.parts = Arc::new(parts); assert!(overwrite_resume_for_target(&source, target)); } #[test] fn test_data_movement_overwrite_resume_rejects_duplicate_part_number() { let source = overwrite_equivalence_source_with_two_parts(); let mut target = source.clone(); let mut parts = target.parts.as_ref().clone(); parts[1].number = parts[0].number; target.parts = Arc::new(parts); assert!(!overwrite_resume_for_target(&source, target)); } #[test] fn test_data_movement_overwrite_resume_rejects_version_purge_mismatch() { let source = overwrite_equivalence_source(); let target = ObjectInfo { version_purge_status_internal: Some("arn:minio:replication:target=COMPLETE;".to_string()), version_purge_status: VersionPurgeStatusType::Complete, ..source.clone() }; assert!(!overwrite_resume_for_target(&source, target)); } #[test] fn test_data_movement_overwrite_resume_rejects_replication_mismatch() { let source = overwrite_equivalence_source(); let target = ObjectInfo { replication_status_internal: Some("arn:minio:replication:target=FAILED;".to_string()), replication_status: ReplicationStatusType::Failed, ..source.clone() }; assert!(!overwrite_resume_for_target(&source, target)); } #[test] fn test_data_movement_overwrite_resume_rejects_tag_mismatch() { let source = overwrite_equivalence_source(); let target = ObjectInfo { user_tags: Arc::new(String::new()), ..source.clone() }; assert!(!overwrite_resume_for_target(&source, target)); } #[test] fn test_data_movement_overwrite_resume_rejects_expires_mismatch() { let source = overwrite_equivalence_source(); let target = ObjectInfo { expires: None, ..source.clone() }; assert!(!overwrite_resume_for_target(&source, target)); } #[test] fn test_resolve_data_movement_overwrite_resume_result_accepts_equivalent_target() { let source = ObjectInfo { version_id: Some(Uuid::nil()), size: 128, etag: Some("etag-value".to_string()), mod_time: Some(OffsetDateTime::UNIX_EPOCH), ..Default::default() }; let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string()); let should_resume = resolve_data_movement_overwrite_resume_result(&err, Ok(Some(source.clone())), &source, 0, 1) .expect("equivalent overwrite target should be evaluated"); assert!(should_resume); } #[test] fn test_overwrite_resume_accepts_owned_target_without_legacy_checksum_sidecar() { let mut source = overwrite_equivalence_source(); let mut source_parts = source.parts.as_ref().clone(); source_parts.push(ObjectPartInfo { number: 2, etag: "second-part-etag".to_string(), size: 64, actual_size: 64, checksums: Some(HashMap::from([(ChecksumType::CRC32C.to_string(), "second-part-checksum".to_string())])), ..Default::default() }); source.parts = Arc::new(source_parts); let mut target = source.clone(); let mut target_parts = target.parts.as_ref().clone(); for part in &mut target_parts { part.checksums = None; } target.parts = Arc::new(target_parts); let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string()); assert!( resolve_data_movement_overwrite_resume_result_for(&err, Ok(Some(target.clone())), &source, 0, 1, false) .expect("compatible migration should accept an omitted optional checksum sidecar") ); assert!( !resolve_data_movement_overwrite_resume_result_for(&err, Ok(Some(target)), &source, 0, 1, true) .expect("an unowned target must not bypass fleet-confirmed checksum comparison") ); let mut owned_target = source.clone(); let mut owned_target_parts = owned_target.parts.as_ref().clone(); for part in &mut owned_target_parts { part.checksums = None; } owned_target.parts = Arc::new(owned_target_parts); rustfs_utils::http::insert_str(Arc::make_mut(&mut owned_target.user_defined), SUFFIX_DATA_MOVED, "true".to_string()); rustfs_utils::http::insert_str( Arc::make_mut(&mut owned_target.user_defined), SUFFIX_DATA_MOVED_TAGS, "v1:tag=value".to_string(), ); assert!( resolve_data_movement_overwrite_resume_result_for(&err, Ok(Some(owned_target.clone())), &source, 0, 1, true,) .expect("an owned pre-gate target should remain compatible after enabling checksum persistence") ); let mut partial_target = owned_target.clone(); let mut partial_target_parts = partial_target.parts.as_ref().clone(); partial_target_parts[0].checksums.clone_from(&source.parts[0].checksums); partial_target.parts = Arc::new(partial_target_parts); assert!( !resolve_data_movement_overwrite_resume_result_for(&err, Ok(Some(partial_target)), &source, 0, 1, true) .expect("a partially missing checksum sidecar must fail closed") ); let mut corrupt_target = owned_target.clone(); rustfs_utils::http::insert_str(Arc::make_mut(&mut corrupt_target.user_defined), SUFFIX_PART_CHECKSUMS, String::new()); assert!( !resolve_data_movement_overwrite_resume_result_for(&err, Ok(Some(corrupt_target)), &source, 0, 1, true) .expect("a present but empty checksum sidecar must fail closed") ); let mut conflicting_target = owned_target; let mut conflicting_target_parts = conflicting_target.parts.as_ref().clone(); conflicting_target_parts[0].checksums = Some(HashMap::from([( ChecksumType::CRC32C.to_string(), "conflicting-part-checksum".to_string(), )])); conflicting_target.parts = Arc::new(conflicting_target_parts); assert!( !resolve_data_movement_overwrite_resume_result_for(&err, Ok(Some(conflicting_target)), &source, 0, 1, true) .expect("a conflicting checksum sidecar must fail closed") ); } #[test] fn test_invalid_upload_accepts_versioned_target_taken_over_by_old_node() { let mut source = overwrite_equivalence_source(); let mut source_parts = source.parts.as_ref().clone(); source_parts[0].checksums = None; source.parts = Arc::new(source_parts); let mut target = ObjectInfo { mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND), ..source.clone() }; let err = Error::InvalidUploadID("bucket".to_string(), "object".to_string(), "upload-id".to_string()); assert!( !resolve_data_movement_overwrite_resume_result(&err, Ok(Some(target.clone())), &source, 0, 1) .expect("an unrelated invalid upload must not accept a different target") ); let upload_identity = data_movement_upload_identity(&source); rustfs_utils::http::insert_str(Arc::make_mut(&mut target.user_defined), SUFFIX_DATA_MOVEMENT_UPLOAD, upload_identity); let should_resume = resolve_data_movement_overwrite_resume_result(&err, Ok(Some(target.clone())), &source, 0, 1) .expect("an old-node completion must make its target version authoritative"); assert!(should_resume); target.etag = Some("etag-client-write".to_string()); assert!( !resolve_data_movement_overwrite_resume_result(&err, Ok(Some(target.clone())), &source, 0, 1) .expect("a takeover target with changed content must be rejected") ); target.etag.clone_from(&source.etag); target.parts = Arc::new(Vec::new()); assert!( !resolve_data_movement_overwrite_resume_result(&err, Ok(Some(target.clone())), &source, 0, 1) .expect("a partial old-node completion must not replace the source") ); target.parts.clone_from(&source.parts); let mut newer_source = source.clone(); newer_source.mod_time = OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND * 2); assert!( !resolve_data_movement_overwrite_resume_result(&err, Ok(Some(target)), &newer_source, 0, 1) .expect("a stale takeover marker must not accept a newer source generation") ); let mut missing_time_source = source.clone(); missing_time_source.mod_time = None; let mut missing_time_target = missing_time_source.clone(); missing_time_target.mod_time = OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND); let upload_identity = data_movement_upload_identity(&missing_time_source); rustfs_utils::http::insert_str( Arc::make_mut(&mut missing_time_target.user_defined), SUFFIX_DATA_MOVEMENT_UPLOAD, upload_identity, ); assert!( !resolve_data_movement_overwrite_resume_result(&err, Ok(Some(missing_time_target)), &missing_time_source, 0, 1) .expect("a takeover target must not replace a source with no generation timestamp") ); let legacy_source = overwrite_equivalence_source(); let mut legacy_target = legacy_source.clone(); legacy_target.mod_time = OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND); let mut legacy_target_parts = legacy_target.parts.as_ref().clone(); legacy_target_parts[0].checksums = None; legacy_target.parts = Arc::new(legacy_target_parts); let upload_identity = data_movement_upload_identity(&legacy_source); rustfs_utils::http::insert_str( Arc::make_mut(&mut legacy_target.user_defined), SUFFIX_DATA_MOVEMENT_UPLOAD, upload_identity, ); assert!( !resolve_data_movement_overwrite_resume_result_for(&err, Ok(Some(legacy_target)), &legacy_source, 0, 1, true) .expect("an old-node takeover must not discard legacy part checksums") ); } #[test] fn test_precondition_conflict_accepts_newer_unversioned_target() { for version_id in [None, Some(Uuid::nil())] { let source = ObjectInfo { version_id, size: 128, etag: Some("etag-source".to_string()), mod_time: Some(OffsetDateTime::UNIX_EPOCH), ..Default::default() }; let target = ObjectInfo { etag: Some("etag-client-write".to_string()), mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND), ..source.clone() }; let should_resume = resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1) .expect("precondition conflict target should be evaluated"); assert!(should_resume); } } #[test] fn test_precondition_conflict_accepts_equivalent_target() { let source = ObjectInfo { size: 128, etag: Some("etag-source".to_string()), mod_time: Some(OffsetDateTime::UNIX_EPOCH), ..Default::default() }; let should_resume = resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(source.clone())), &source, 0, 1) .expect("equivalent precondition target should be evaluated"); assert!(should_resume); } #[test] fn test_precondition_conflict_rejects_non_newer_unversioned_target() { let source = ObjectInfo { size: 128, etag: Some("etag-source".to_string()), mod_time: Some(OffsetDateTime::UNIX_EPOCH), ..Default::default() }; let target = ObjectInfo { etag: Some("etag-conflict".to_string()), ..source.clone() }; let should_resume = resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1) .expect("precondition conflict target should be evaluated"); assert!(!should_resume); } #[test] fn test_precondition_conflict_rejects_newer_delete_marker() { let source = ObjectInfo { size: 128, etag: Some("etag-source".to_string()), mod_time: Some(OffsetDateTime::UNIX_EPOCH), ..Default::default() }; let target = ObjectInfo { delete_marker: true, etag: None, mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND), ..source.clone() }; let should_resume = resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1) .expect("delete marker conflict should be evaluated"); assert!(!should_resume); } #[test] fn test_overwrite_error_rejects_newer_unversioned_target() { let source = ObjectInfo { size: 128, etag: Some("etag-source".to_string()), mod_time: Some(OffsetDateTime::UNIX_EPOCH), ..Default::default() }; let target = ObjectInfo { etag: Some("etag-client-write".to_string()), mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND), ..source.clone() }; let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string()); let should_resume = resolve_data_movement_overwrite_resume_result(&err, Ok(Some(target)), &source, 0, 1) .expect("pool-selection overwrite must require target equivalence"); assert!(!should_resume); } #[test] fn test_precondition_conflict_rejects_newer_versioned_target() { let source = ObjectInfo { size: 128, etag: Some("etag-source".to_string()), mod_time: Some(OffsetDateTime::UNIX_EPOCH), ..Default::default() }; let target = ObjectInfo { version_id: Some(Uuid::from_u128(2)), etag: Some("etag-conflict".to_string()), mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND), ..source.clone() }; let should_resume = resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1) .expect("versioned conflict target should be evaluated"); assert!(!should_resume); } #[test] fn test_precondition_conflict_rejects_versioned_source_with_unversioned_target() { let source = ObjectInfo { version_id: Some(Uuid::from_u128(1)), size: 128, etag: Some("etag-source".to_string()), mod_time: Some(OffsetDateTime::UNIX_EPOCH), ..Default::default() }; let target = ObjectInfo { version_id: None, etag: Some("etag-conflict".to_string()), mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND), ..source.clone() }; let should_resume = resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1) .expect("versioned source conflict should be evaluated"); assert!(!should_resume); } #[test] fn test_rebalance_overwrite_resume_accepts_equivalent_target_version() { let source = ObjectInfo { version_id: Some(Uuid::from_u128(1)), size: 128, etag: Some("etag-value".to_string()), mod_time: Some(OffsetDateTime::UNIX_EPOCH), user_defined: Arc::new(HashMap::from([("x-amz-meta-key".to_string(), "value".to_string())])), ..Default::default() }; let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string()); let should_resume = resolve_data_movement_overwrite_resume_result(&err, Ok(Some(source.clone())), &source, 2, 3) .expect("rebalance overwrite should converge when the target version is equivalent"); assert!(should_resume); } #[test] fn test_resolve_data_movement_overwrite_resume_result_rejects_source_pool_target() { let source = ObjectInfo { version_id: Some(Uuid::nil()), size: 128, etag: Some("etag-value".to_string()), mod_time: Some(OffsetDateTime::UNIX_EPOCH), ..Default::default() }; let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string()); let should_resume = resolve_data_movement_overwrite_resume_result(&err, Ok(Some(source.clone())), &source, 0, 0) .expect("source-pool target should be rejected before target lookup"); assert!(!should_resume); } #[test] fn test_rebalance_overwrite_resume_rejects_different_target_version() { let source = ObjectInfo { version_id: Some(Uuid::from_u128(1)), size: 128, etag: Some("etag-value".to_string()), mod_time: Some(OffsetDateTime::UNIX_EPOCH), ..Default::default() }; let target = ObjectInfo { version_id: Some(Uuid::from_u128(2)), ..source.clone() }; let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string()); let should_resume = resolve_data_movement_overwrite_resume_result(&err, Ok(Some(target)), &source, 2, 3) .expect("rebalance overwrite should evaluate a different target version"); assert!(!should_resume); } #[test] fn test_resolve_data_movement_overwrite_resume_result_rejects_non_equivalent_target() { let source = ObjectInfo { version_id: Some(Uuid::nil()), size: 128, etag: Some("etag-source".to_string()), ..Default::default() }; let target = ObjectInfo { etag: Some("etag-target".to_string()), ..source.clone() }; let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string()); let should_resume = resolve_data_movement_overwrite_resume_result(&err, Ok(Some(target)), &source, 0, 1) .expect("non-equivalent overwrite target should be evaluated"); assert!(!should_resume); } #[test] fn test_resolve_data_movement_overwrite_resume_result_propagates_target_lookup_error() { let source = ObjectInfo::default(); let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string()); let result = resolve_data_movement_overwrite_resume_result(&err, Err(Error::SlowDown), &source, 0, 1); assert!(matches!(result, Err(Error::SlowDown))); } #[test] fn test_resolve_data_movement_overwrite_resume_result_ignores_non_overwrite_error() { let source = ObjectInfo::default(); let result = resolve_data_movement_overwrite_resume_result(&Error::SlowDown, Err(Error::FileAccessDenied), &source, 0, 1) .expect("non-overwrite errors should not query target equivalence"); assert!(!result); } }