mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-13 16:46:55 +00:00
fix(rebalance): converge multipart data movement retries (#6057)
* fix(rebalance): converge multipart data movement retries
* fix(rebalance): harden multipart retry replacement
* fix(rebalance): isolate internal multipart uploads
* test(ecstore): adapt metadata mutation fixtures
* fix(rebalance): preserve transition metadata semantics
* refactor(ecstore): reuse internal metadata matcher
* Revert "refactor(ecstore): reuse internal metadata matcher"
This reverts commit c87ca0328f.
* refactor(rebalance): reuse data movement log constants
* fix(rebalance): isolate migration-owned state
* fix(rebalance): preserve pre-gate retry compatibility
This commit is contained in:
@@ -137,6 +137,21 @@ pub const DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: bool = false;
|
|||||||
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE);
|
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE);
|
||||||
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED);
|
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED);
|
||||||
|
|
||||||
|
/// Request preserving legacy per-part checksum metadata during data movement.
|
||||||
|
///
|
||||||
|
/// This remains ineffective until
|
||||||
|
/// [`ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED`] is also enabled.
|
||||||
|
pub const ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE: &str = "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_WRITE";
|
||||||
|
pub const DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_WRITE: bool = false;
|
||||||
|
|
||||||
|
/// Operator-attested confirmation that every serving node understands the
|
||||||
|
/// data-movement per-part checksum sidecar.
|
||||||
|
pub const ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED: &str = "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED";
|
||||||
|
pub const DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED: bool = false;
|
||||||
|
|
||||||
|
const _: () = assert!(!DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_WRITE);
|
||||||
|
const _: () = assert!(!DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED);
|
||||||
|
|
||||||
// =============================================================================
|
// =============================================================================
|
||||||
// Concurrent Request Fix - Timeout and Backpressure Configuration
|
// Concurrent Request Fix - Timeout and Backpressure Configuration
|
||||||
// =============================================================================
|
// =============================================================================
|
||||||
@@ -649,4 +664,13 @@ mod remote_version_state_tests {
|
|||||||
"RUSTFS_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED"
|
"RUSTFS_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn data_movement_part_checksum_gate_uses_stable_environment_names() {
|
||||||
|
assert_eq!(super::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_WRITE");
|
||||||
|
assert_eq!(
|
||||||
|
super::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED,
|
||||||
|
"RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2871,11 +2871,32 @@ fn stale_upload_default_due(initiated: OffsetDateTime, default_expiry: StdDurati
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn stale_upload_current_size(set: &Arc<SetDisks>, metadata: &HashMap<String, String>, upload_dir: &str) -> Option<usize> {
|
async fn stale_upload_current_size(set: &Arc<SetDisks>, metadata: &HashMap<String, String>, upload_dir: &str) -> Option<usize> {
|
||||||
|
stale_upload_current_size_with_opts(set, metadata, upload_dir, false).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn stale_upload_current_size_with_opts(
|
||||||
|
set: &Arc<SetDisks>,
|
||||||
|
metadata: &HashMap<String, String>,
|
||||||
|
upload_dir: &str,
|
||||||
|
no_lock: bool,
|
||||||
|
) -> Option<usize> {
|
||||||
let bucket = metadata.get(RUSTFS_MULTIPART_BUCKET_KEY)?;
|
let bucket = metadata.get(RUSTFS_MULTIPART_BUCKET_KEY)?;
|
||||||
let object = metadata.get(RUSTFS_MULTIPART_OBJECT_KEY)?;
|
let object = metadata.get(RUSTFS_MULTIPART_OBJECT_KEY)?;
|
||||||
let upload_id = encode_stale_upload_id(upload_dir);
|
let upload_id = encode_stale_upload_id(upload_dir);
|
||||||
|
let data_movement = rustfs_utils::http::contains_key_str(metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD);
|
||||||
let parts = set
|
let parts = set
|
||||||
.list_object_parts(bucket, object, &upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
|
.list_object_parts(
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
&upload_id,
|
||||||
|
None,
|
||||||
|
MAX_PARTS_COUNT,
|
||||||
|
&ObjectOptions {
|
||||||
|
data_movement,
|
||||||
|
no_lock,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
.ok()?;
|
.ok()?;
|
||||||
|
|
||||||
@@ -2893,7 +2914,12 @@ async fn stale_upload_lifecycle_due(
|
|||||||
metadata: &HashMap<String, String>,
|
metadata: &HashMap<String, String>,
|
||||||
initiated: OffsetDateTime,
|
initiated: OffsetDateTime,
|
||||||
upload_dir: &str,
|
upload_dir: &str,
|
||||||
|
no_lock: bool,
|
||||||
) -> Option<OffsetDateTime> {
|
) -> Option<OffsetDateTime> {
|
||||||
|
if rustfs_utils::http::contains_key_str(metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
let bucket = metadata.get(RUSTFS_MULTIPART_BUCKET_KEY)?;
|
let bucket = metadata.get(RUSTFS_MULTIPART_BUCKET_KEY)?;
|
||||||
let object = metadata.get(RUSTFS_MULTIPART_OBJECT_KEY)?;
|
let object = metadata.get(RUSTFS_MULTIPART_OBJECT_KEY)?;
|
||||||
|
|
||||||
@@ -2906,7 +2932,9 @@ async fn stale_upload_lifecycle_due(
|
|||||||
name: object.clone(),
|
name: object.clone(),
|
||||||
user_tags: metadata.get(AMZ_OBJECT_TAGGING).cloned().unwrap_or_default(),
|
user_tags: metadata.get(AMZ_OBJECT_TAGGING).cloned().unwrap_or_default(),
|
||||||
mod_time: Some(initiated),
|
mod_time: Some(initiated),
|
||||||
size: stale_upload_current_size(set, metadata, upload_dir).await.unwrap_or_default(),
|
size: stale_upload_current_size_with_opts(set, metadata, upload_dir, no_lock)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default(),
|
||||||
is_latest: true,
|
is_latest: true,
|
||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
user_defined: metadata.clone(),
|
user_defined: metadata.clone(),
|
||||||
@@ -2934,6 +2962,7 @@ async fn read_stale_multipart_candidate(
|
|||||||
FileInfoOpts {
|
FileInfoOpts {
|
||||||
data: false,
|
data: false,
|
||||||
include_free_versions: false,
|
include_free_versions: false,
|
||||||
|
include_part_checksums: false,
|
||||||
},
|
},
|
||||||
) {
|
) {
|
||||||
Ok(file_info) => (Some(file_info.metadata), file_info.mod_time),
|
Ok(file_info) => (Some(file_info.metadata), file_info.mod_time),
|
||||||
@@ -2973,36 +3002,30 @@ fn merge_stale_multipart_candidate(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn is_multipart_sha_dir(path: &str) -> bool {
|
||||||
|
path.len() == 64 && path.bytes().all(|byte| byte.is_ascii_hexdigit())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn multipart_sha_path(root: &str, entry: &str) -> Option<String> {
|
||||||
|
let sha_dir = entry.trim_end_matches('/');
|
||||||
|
is_multipart_sha_dir(sha_dir).then(|| {
|
||||||
|
if root.is_empty() {
|
||||||
|
sha_dir.to_string()
|
||||||
|
} else {
|
||||||
|
format!("{root}/{sha_dir}")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
async fn cleanup_empty_multipart_sha_dirs_on_local_disks(set: &Arc<SetDisks>) {
|
async fn cleanup_empty_multipart_sha_dirs_on_local_disks(set: &Arc<SetDisks>) {
|
||||||
for disk in set.get_local_disks().await.into_iter().flatten() {
|
for disk in set.get_local_disks().await.into_iter().flatten() {
|
||||||
if !disk.is_online().await {
|
if !disk.is_online().await {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
let sha_dirs = match disk
|
for root in ["", crate::set_disk::DATA_MOVEMENT_MULTIPART_PREFIX] {
|
||||||
.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, "", -1)
|
let sha_dirs = match disk
|
||||||
.await
|
.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, root, -1)
|
||||||
{
|
|
||||||
Ok(entries) => entries,
|
|
||||||
Err(err) => {
|
|
||||||
if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound {
|
|
||||||
debug!(
|
|
||||||
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
|
||||||
error = ?err,
|
|
||||||
reason = "multipart_root_list_failed",
|
|
||||||
"Skipped empty multipart sha cleanup"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
for sha_dir in sha_dirs {
|
|
||||||
let sha_dir = sha_dir.trim_end_matches('/').to_string();
|
|
||||||
let upload_dirs = match disk
|
|
||||||
.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, &sha_dir, -1)
|
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(entries) => entries,
|
Ok(entries) => entries,
|
||||||
@@ -3012,9 +3035,8 @@ async fn cleanup_empty_multipart_sha_dirs_on_local_disks(set: &Arc<SetDisks>) {
|
|||||||
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||||
sha_dir = %sha_dir,
|
|
||||||
error = ?err,
|
error = ?err,
|
||||||
reason = "multipart_sha_dir_list_failed",
|
reason = "multipart_root_list_failed",
|
||||||
"Skipped empty multipart sha cleanup"
|
"Skipped empty multipart sha cleanup"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -3022,25 +3044,48 @@ async fn cleanup_empty_multipart_sha_dirs_on_local_disks(set: &Arc<SetDisks>) {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
if !upload_dirs.is_empty() {
|
for sha_dir in sha_dirs.into_iter().filter_map(|entry| multipart_sha_path(root, &entry)) {
|
||||||
continue;
|
let upload_dirs = match disk
|
||||||
}
|
.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, &sha_dir, -1)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(entries) => entries,
|
||||||
|
Err(err) => {
|
||||||
|
if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound {
|
||||||
|
debug!(
|
||||||
|
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||||
|
sha_dir = %sha_dir,
|
||||||
|
error = ?err,
|
||||||
|
reason = "multipart_sha_dir_list_failed",
|
||||||
|
"Skipped empty multipart sha cleanup"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
if let Err(err) = disk
|
if !upload_dirs.is_empty() {
|
||||||
.delete(RUSTFS_META_MULTIPART_BUCKET, &sha_dir, DeleteOptions::default())
|
continue;
|
||||||
.await
|
}
|
||||||
&& err != DiskError::FileNotFound
|
|
||||||
&& err != DiskError::VolumeNotFound
|
if let Err(err) = disk
|
||||||
{
|
.delete(RUSTFS_META_MULTIPART_BUCKET, &sha_dir, DeleteOptions::default())
|
||||||
debug!(
|
.await
|
||||||
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
&& err != DiskError::FileNotFound
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
&& err != DiskError::VolumeNotFound
|
||||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
{
|
||||||
sha_dir = %sha_dir,
|
debug!(
|
||||||
error = ?err,
|
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
||||||
reason = "multipart_sha_dir_remove_failed",
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
"Failed to remove empty multipart sha dir"
|
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||||
);
|
sha_dir = %sha_dir,
|
||||||
|
error = ?err,
|
||||||
|
reason = "multipart_sha_dir_remove_failed",
|
||||||
|
"Failed to remove empty multipart sha dir"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3058,30 +3103,9 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
let sha_dirs = match disk
|
for root in ["", crate::set_disk::DATA_MOVEMENT_MULTIPART_PREFIX] {
|
||||||
.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, "", -1)
|
let sha_dirs = match disk
|
||||||
.await
|
.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, root, -1)
|
||||||
{
|
|
||||||
Ok(entries) => entries,
|
|
||||||
Err(err) => {
|
|
||||||
if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound {
|
|
||||||
debug!(
|
|
||||||
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
|
||||||
error = ?err,
|
|
||||||
reason = "multipart_root_list_failed",
|
|
||||||
"Skipped stale multipart cleanup"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
for sha_dir in sha_dirs {
|
|
||||||
let sha_dir = sha_dir.trim_end_matches('/').to_string();
|
|
||||||
let upload_dirs = match disk
|
|
||||||
.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, &sha_dir, -1)
|
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(entries) => entries,
|
Ok(entries) => entries,
|
||||||
@@ -3091,9 +3115,8 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
|
|||||||
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||||
sha_dir = %sha_dir,
|
|
||||||
error = ?err,
|
error = ?err,
|
||||||
reason = "multipart_sha_dir_list_failed",
|
reason = "multipart_root_list_failed",
|
||||||
"Skipped stale multipart cleanup"
|
"Skipped stale multipart cleanup"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -3101,39 +3124,62 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
for upload_dir in upload_dirs {
|
for sha_dir in sha_dirs.into_iter().filter_map(|entry| multipart_sha_path(root, &entry)) {
|
||||||
let upload_dir = upload_dir.trim_end_matches('/').to_string();
|
let upload_dirs = match disk
|
||||||
let candidate_path = format!("{sha_dir}/{upload_dir}");
|
.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, &sha_dir, -1)
|
||||||
if candidates
|
.await
|
||||||
.get(&candidate_path)
|
|
||||||
.is_some_and(|existing: &StaleMultipartUploadCandidate| existing.metadata.is_some())
|
|
||||||
{
|
{
|
||||||
continue;
|
Ok(entries) => entries,
|
||||||
}
|
|
||||||
|
|
||||||
let candidate = match read_stale_multipart_candidate(disk.as_ref(), &sha_dir, &upload_dir).await {
|
|
||||||
Ok(candidate) => candidate,
|
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
if err != DiskError::FileNotFound {
|
if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound {
|
||||||
debug!(
|
debug!(
|
||||||
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||||
path = %candidate_path,
|
sha_dir = %sha_dir,
|
||||||
error = ?err,
|
error = ?err,
|
||||||
reason = "multipart_metadata_read_failed",
|
reason = "multipart_sha_dir_list_failed",
|
||||||
"Multipart metadata unavailable during stale cleanup"
|
"Skipped stale multipart cleanup"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
let initiated = initiated_from_upload_dir(&upload_dir, None);
|
continue;
|
||||||
StaleMultipartUploadCandidate {
|
|
||||||
path: candidate_path,
|
|
||||||
initiated,
|
|
||||||
metadata: None,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
merge_stale_multipart_candidate(&mut candidates, candidate);
|
|
||||||
|
for upload_dir in upload_dirs {
|
||||||
|
let upload_dir = upload_dir.trim_end_matches('/').to_string();
|
||||||
|
let candidate_path = format!("{sha_dir}/{upload_dir}");
|
||||||
|
if candidates
|
||||||
|
.get(&candidate_path)
|
||||||
|
.is_some_and(|existing: &StaleMultipartUploadCandidate| existing.metadata.is_some())
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let candidate = match read_stale_multipart_candidate(disk.as_ref(), &sha_dir, &upload_dir).await {
|
||||||
|
Ok(candidate) => candidate,
|
||||||
|
Err(err) => {
|
||||||
|
if err != DiskError::FileNotFound {
|
||||||
|
debug!(
|
||||||
|
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||||
|
path = %candidate_path,
|
||||||
|
error = ?err,
|
||||||
|
reason = "multipart_metadata_read_failed",
|
||||||
|
"Multipart metadata unavailable during stale cleanup"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let initiated = initiated_from_upload_dir(&upload_dir, None);
|
||||||
|
StaleMultipartUploadCandidate {
|
||||||
|
path: candidate_path,
|
||||||
|
initiated,
|
||||||
|
metadata: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
merge_stale_multipart_candidate(&mut candidates, candidate);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3142,7 +3188,7 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
|
|||||||
let upload_dir = candidate.path.rsplit('/').next().unwrap_or_default().to_string();
|
let upload_dir = candidate.path.rsplit('/').next().unwrap_or_default().to_string();
|
||||||
let mut due = stale_upload_default_due(candidate.initiated, default_expiry);
|
let mut due = stale_upload_default_due(candidate.initiated, default_expiry);
|
||||||
if let Some(metadata) = candidate.metadata.as_ref()
|
if let Some(metadata) = candidate.metadata.as_ref()
|
||||||
&& let Some(lifecycle_due) = stale_upload_lifecycle_due(set, metadata, candidate.initiated, &upload_dir).await
|
&& let Some(lifecycle_due) = stale_upload_lifecycle_due(set, metadata, candidate.initiated, &upload_dir, false).await
|
||||||
&& lifecycle_due < due
|
&& lifecycle_due < due
|
||||||
{
|
{
|
||||||
due = lifecycle_due;
|
due = lifecycle_due;
|
||||||
@@ -3152,34 +3198,49 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
match set.delete_all(RUSTFS_META_MULTIPART_BUCKET, &candidate.path).await {
|
let cleanup_guard = match set.lock_stale_multipart_cleanup(&candidate.path).await {
|
||||||
|
Ok(guard) => guard,
|
||||||
|
Err(err) => {
|
||||||
|
debug!(
|
||||||
|
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||||
|
path = %candidate.path,
|
||||||
|
error = ?err,
|
||||||
|
reason = "multipart_cleanup_lock_or_recheck_failed",
|
||||||
|
"Skipped stale multipart cleanup"
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let current_metadata = cleanup_guard.file_info().metadata.clone();
|
||||||
|
let current_initiated = initiated_from_upload_dir(&upload_dir, cleanup_guard.file_info().mod_time);
|
||||||
|
let mut current_due = stale_upload_default_due(current_initiated, default_expiry);
|
||||||
|
if let Some(lifecycle_due) =
|
||||||
|
stale_upload_lifecycle_due(set, ¤t_metadata, current_initiated, &upload_dir, true).await
|
||||||
|
&& lifecycle_due < current_due
|
||||||
|
{
|
||||||
|
current_due = lifecycle_due;
|
||||||
|
}
|
||||||
|
if now < current_due || cleanup_guard.is_lock_lost() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
match cleanup_guard.delete(set).await {
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
deleted += 1;
|
deleted += 1;
|
||||||
let upload_id = encode_stale_upload_id(&upload_dir);
|
let upload_id = encode_stale_upload_id(&upload_dir);
|
||||||
if let Some(metadata) = candidate.metadata.as_ref() {
|
debug!(
|
||||||
debug!(
|
bucket = current_metadata.get(RUSTFS_MULTIPART_BUCKET_KEY).cloned().unwrap_or_default(),
|
||||||
bucket = metadata.get(RUSTFS_MULTIPART_BUCKET_KEY).cloned().unwrap_or_default(),
|
object = current_metadata.get(RUSTFS_MULTIPART_OBJECT_KEY).cloned().unwrap_or_default(),
|
||||||
object = metadata.get(RUSTFS_MULTIPART_OBJECT_KEY).cloned().unwrap_or_default(),
|
upload_id = %upload_id,
|
||||||
upload_id = %upload_id,
|
due = ?current_due,
|
||||||
due = ?due,
|
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
||||||
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
state = "removed",
|
||||||
state = "removed",
|
"Removed stale multipart upload"
|
||||||
"Removed stale multipart upload"
|
);
|
||||||
);
|
|
||||||
} else {
|
|
||||||
debug!(
|
|
||||||
path = %candidate.path,
|
|
||||||
upload_id = %upload_id,
|
|
||||||
due = ?due,
|
|
||||||
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
|
||||||
state = "removed",
|
|
||||||
"Removed stale multipart upload"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
Err(err) => debug!(
|
Err(err) => debug!(
|
||||||
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
|
||||||
@@ -5162,6 +5223,7 @@ mod tests {
|
|||||||
use crate::services::tier::tier::TierConfigMgr;
|
use crate::services::tier::tier::TierConfigMgr;
|
||||||
#[cfg(feature = "test-util")]
|
#[cfg(feature = "test-util")]
|
||||||
use crate::services::tier::warm_backend::WarmBackend as _;
|
use crate::services::tier::warm_backend::WarmBackend as _;
|
||||||
|
use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause};
|
||||||
use crate::set_disk::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY};
|
use crate::set_disk::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY};
|
||||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||||
use crate::storage_api_contracts::{
|
use crate::storage_api_contracts::{
|
||||||
@@ -11926,6 +11988,135 @@ mod tests {
|
|||||||
assert!(is_err_invalid_upload_id(&err));
|
assert!(is_err_invalid_upload_id(&err));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn stale_multipart_cleanup_handles_data_movement_namespace() {
|
||||||
|
let (_paths, ecstore) = setup_test_env().await;
|
||||||
|
let bucket = format!("stale-data-movement-{}", Uuid::new_v4().simple());
|
||||||
|
create_test_bucket(&ecstore, &bucket).await;
|
||||||
|
|
||||||
|
let create_upload = |object: &'static str, mod_time| {
|
||||||
|
let ecstore = ecstore.clone();
|
||||||
|
let bucket = bucket.clone();
|
||||||
|
async move {
|
||||||
|
let mut metadata = HashMap::new();
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
&mut metadata,
|
||||||
|
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
|
||||||
|
"cleanup-test".to_string(),
|
||||||
|
);
|
||||||
|
ecstore
|
||||||
|
.new_multipart_upload(
|
||||||
|
&bucket,
|
||||||
|
object,
|
||||||
|
&ObjectOptions {
|
||||||
|
data_movement: true,
|
||||||
|
mod_time: Some(mod_time),
|
||||||
|
user_defined: metadata,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("data movement multipart upload should be created")
|
||||||
|
.upload_id
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let stale_object = "stale-internal.bin";
|
||||||
|
let active_object = "active-internal.bin";
|
||||||
|
let now = OffsetDateTime::now_utc();
|
||||||
|
let stale_upload_id = create_upload(stale_object, now - time::Duration::hours(30)).await;
|
||||||
|
let active_upload_id = create_upload(active_object, now).await;
|
||||||
|
|
||||||
|
let deleted = cleanup_stale_multipart_uploads_once_at(ecstore.clone(), now, StdDuration::from_secs(24 * 60 * 60)).await;
|
||||||
|
assert!(deleted >= 1, "expected stale data movement upload to be removed");
|
||||||
|
|
||||||
|
let internal_opts = ObjectOptions {
|
||||||
|
data_movement: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let stale_err = ecstore
|
||||||
|
.get_multipart_info(&bucket, stale_object, &stale_upload_id, &internal_opts)
|
||||||
|
.await
|
||||||
|
.expect_err("stale data movement upload should be removed");
|
||||||
|
assert!(is_err_invalid_upload_id(&stale_err));
|
||||||
|
ecstore
|
||||||
|
.get_multipart_info(&bucket, active_object, &active_upload_id, &internal_opts)
|
||||||
|
.await
|
||||||
|
.expect("active data movement upload should remain available");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn stale_multipart_cleanup_waits_for_data_movement_part_commit() {
|
||||||
|
let (_paths, ecstore) = setup_test_env().await;
|
||||||
|
let bucket = format!("stale-data-movement-lock-{}", Uuid::new_v4().simple());
|
||||||
|
let object = "stale-internal.bin";
|
||||||
|
create_test_bucket(&ecstore, &bucket).await;
|
||||||
|
|
||||||
|
let mut metadata = HashMap::new();
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
&mut metadata,
|
||||||
|
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
|
||||||
|
"cleanup-lock-test".to_string(),
|
||||||
|
);
|
||||||
|
let opts = ObjectOptions {
|
||||||
|
data_movement: true,
|
||||||
|
mod_time: Some(OffsetDateTime::now_utc() - time::Duration::hours(30)),
|
||||||
|
user_defined: metadata,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let upload = ecstore
|
||||||
|
.new_multipart_upload(&bucket, object, &opts)
|
||||||
|
.await
|
||||||
|
.expect("data movement multipart upload should be created");
|
||||||
|
let barrier = MultipartCommitBarrier::install(bucket.as_str(), object, MultipartCommitPause::PutPartAfterRename);
|
||||||
|
let put_store = ecstore.clone();
|
||||||
|
let put_bucket = bucket.clone();
|
||||||
|
let upload_id = upload.upload_id.clone();
|
||||||
|
let put_task = tokio::spawn(async move {
|
||||||
|
let mut data = PutObjReader::from_vec(vec![1, 2, 3, 4]);
|
||||||
|
put_store
|
||||||
|
.put_object_part(
|
||||||
|
&put_bucket,
|
||||||
|
object,
|
||||||
|
&upload_id,
|
||||||
|
1,
|
||||||
|
&mut data,
|
||||||
|
&ObjectOptions {
|
||||||
|
data_movement: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
barrier.wait_until_paused().await;
|
||||||
|
|
||||||
|
let cleanup_store = ecstore.clone();
|
||||||
|
let mut cleanup_task = tokio::spawn(async move {
|
||||||
|
cleanup_stale_multipart_uploads_once_at(
|
||||||
|
cleanup_store,
|
||||||
|
OffsetDateTime::now_utc(),
|
||||||
|
StdDuration::from_secs(24 * 60 * 60),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
assert!(
|
||||||
|
tokio::time::timeout(StdDuration::from_millis(200), &mut cleanup_task)
|
||||||
|
.await
|
||||||
|
.is_err(),
|
||||||
|
"stale cleanup must wait for the in-flight part commit upload lock"
|
||||||
|
);
|
||||||
|
|
||||||
|
barrier.release();
|
||||||
|
put_task
|
||||||
|
.await
|
||||||
|
.expect("part upload task should join")
|
||||||
|
.expect("part upload should commit before stale cleanup");
|
||||||
|
let deleted = cleanup_task.await.expect("stale cleanup task should join");
|
||||||
|
assert!(deleted >= 1, "stale cleanup should proceed after the part commit releases its lock");
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
#[serial]
|
||||||
async fn stale_multipart_cleanup_applies_abort_incomplete_lifecycle_before_default_expiry() {
|
async fn stale_multipart_cleanup_applies_abort_incomplete_lifecycle_before_default_expiry() {
|
||||||
@@ -12000,6 +12191,58 @@ mod tests {
|
|||||||
assert!(is_err_invalid_upload_id(&err));
|
assert!(is_err_invalid_upload_id(&err));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn stale_multipart_cleanup_excludes_data_movement_from_abort_lifecycle() {
|
||||||
|
let (_paths, ecstore) = setup_test_env().await;
|
||||||
|
let bucket = format!("stale-internal-lifecycle-{}", Uuid::new_v4().simple());
|
||||||
|
let object = "logs/internal/object.bin";
|
||||||
|
create_test_bucket(&ecstore, &bucket).await;
|
||||||
|
set_abort_incomplete_lifecycle(&bucket, "logs/", 0).await;
|
||||||
|
|
||||||
|
let initiated = OffsetDateTime::now_utc() - time::Duration::minutes(5);
|
||||||
|
let mut metadata = HashMap::new();
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
&mut metadata,
|
||||||
|
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
|
||||||
|
"lifecycle-exclusion-test".to_string(),
|
||||||
|
);
|
||||||
|
let upload = ecstore
|
||||||
|
.new_multipart_upload(
|
||||||
|
&bucket,
|
||||||
|
object,
|
||||||
|
&ObjectOptions {
|
||||||
|
data_movement: true,
|
||||||
|
mod_time: Some(initiated),
|
||||||
|
user_defined: metadata,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("data movement multipart upload should be created");
|
||||||
|
|
||||||
|
let deleted = cleanup_stale_multipart_uploads_once_at(
|
||||||
|
ecstore.clone(),
|
||||||
|
OffsetDateTime::now_utc(),
|
||||||
|
StdDuration::from_secs(7 * 24 * 60 * 60),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(deleted, 0, "bucket lifecycle must not remove active data movement uploads");
|
||||||
|
|
||||||
|
ecstore
|
||||||
|
.get_multipart_info(
|
||||||
|
&bucket,
|
||||||
|
object,
|
||||||
|
&upload.upload_id,
|
||||||
|
&ObjectOptions {
|
||||||
|
data_movement: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("active data movement upload should remain available");
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
#[serial]
|
||||||
async fn stale_multipart_cleanup_applies_abort_lifecycle_with_size_filter() {
|
async fn stale_multipart_cleanup_applies_abort_lifecycle_with_size_filter() {
|
||||||
|
|||||||
@@ -2266,15 +2266,19 @@ fn decommission_delete_marker_opts(
|
|||||||
version: &rustfs_filemeta::FileInfo,
|
version: &rustfs_filemeta::FileInfo,
|
||||||
version_id: Option<String>,
|
version_id: Option<String>,
|
||||||
src_pool_idx: usize,
|
src_pool_idx: usize,
|
||||||
|
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||||
) -> ObjectOptions {
|
) -> ObjectOptions {
|
||||||
|
let version_suspended = version.version_id.is_none() && version_id.is_none();
|
||||||
ObjectOptions {
|
ObjectOptions {
|
||||||
versioned: true,
|
versioned: !version_suspended,
|
||||||
version_id,
|
version_suspended,
|
||||||
|
version_id: version_id.or_else(|| version_suspended.then(|| uuid::Uuid::nil().to_string())),
|
||||||
mod_time: version.mod_time,
|
mod_time: version.mod_time,
|
||||||
src_pool_idx,
|
src_pool_idx,
|
||||||
data_movement: true,
|
data_movement: true,
|
||||||
delete_marker: true,
|
delete_marker: true,
|
||||||
skip_decommissioned: true,
|
skip_decommissioned: true,
|
||||||
|
expected_bucket_incarnation_id,
|
||||||
delete_replication: version
|
delete_replication: version
|
||||||
.replication_state_internal
|
.replication_state_internal
|
||||||
.as_ref()
|
.as_ref()
|
||||||
@@ -2299,6 +2303,7 @@ fn decommission_remote_tiered_opts(
|
|||||||
version: &rustfs_filemeta::FileInfo,
|
version: &rustfs_filemeta::FileInfo,
|
||||||
version_id: Option<String>,
|
version_id: Option<String>,
|
||||||
src_pool_idx: usize,
|
src_pool_idx: usize,
|
||||||
|
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||||
) -> ObjectOptions {
|
) -> ObjectOptions {
|
||||||
ObjectOptions {
|
ObjectOptions {
|
||||||
versioned: version_id.is_some(),
|
versioned: version_id.is_some(),
|
||||||
@@ -2307,6 +2312,9 @@ fn decommission_remote_tiered_opts(
|
|||||||
user_defined: version.metadata.clone(),
|
user_defined: version.metadata.clone(),
|
||||||
src_pool_idx,
|
src_pool_idx,
|
||||||
data_movement: true,
|
data_movement: true,
|
||||||
|
include_part_checksums: true,
|
||||||
|
http_preconditions: Some(crate::data_movement::data_movement_target_precondition()),
|
||||||
|
expected_bucket_incarnation_id,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2805,6 +2813,7 @@ impl ECStore {
|
|||||||
lifecycle_config: Option<BucketLifecycleConfiguration>,
|
lifecycle_config: Option<BucketLifecycleConfiguration>,
|
||||||
object_lock_config: Option<ObjectLockConfiguration>,
|
object_lock_config: Option<ObjectLockConfiguration>,
|
||||||
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
|
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
|
||||||
|
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
debug!(
|
debug!(
|
||||||
event = EVENT_DECOMMISSION_ENTRY,
|
event = EVENT_DECOMMISSION_ENTRY,
|
||||||
@@ -2834,6 +2843,11 @@ impl ECStore {
|
|||||||
}
|
}
|
||||||
decommission_cancel_signal_result(rx.is_cancelled())?;
|
decommission_cancel_signal_result(rx.is_cancelled())?;
|
||||||
|
|
||||||
|
let bucket_incarnation_fence = match expected_bucket_incarnation_id {
|
||||||
|
Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?),
|
||||||
|
None => None,
|
||||||
|
};
|
||||||
|
|
||||||
let mut fivs = load_decommission_entry_exact_versions(&set, &entry, &bucket, "file_info_versions").await?;
|
let mut fivs = load_decommission_entry_exact_versions(&set, &entry, &bucket, "file_info_versions").await?;
|
||||||
|
|
||||||
fivs.versions
|
fivs.versions
|
||||||
@@ -2894,7 +2908,7 @@ impl ECStore {
|
|||||||
.delete_object(
|
.delete_object(
|
||||||
bucket.as_str(),
|
bucket.as_str(),
|
||||||
&version.name,
|
&version.name,
|
||||||
decommission_delete_marker_opts(version, version_id.clone(), idx),
|
decommission_delete_marker_opts(version, version_id.clone(), idx, expected_bucket_incarnation_id),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
@@ -2984,7 +2998,7 @@ impl ECStore {
|
|||||||
bucket.as_str(),
|
bucket.as_str(),
|
||||||
&version.name,
|
&version.name,
|
||||||
version,
|
version,
|
||||||
&decommission_remote_tiered_opts(version, version_id.clone(), idx),
|
&decommission_remote_tiered_opts(version, version_id.clone(), idx, expected_bucket_incarnation_id),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
@@ -3056,7 +3070,11 @@ impl ECStore {
|
|||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
if let Err(err) = self.clone().decommission_object(idx, bucket, rd).await {
|
if let Err(err) = self
|
||||||
|
.clone()
|
||||||
|
.decommission_object(idx, bucket, rd, expected_bucket_incarnation_id)
|
||||||
|
.await
|
||||||
|
{
|
||||||
if is_decommission_copy_cleanup_safe_error(&err) {
|
if is_decommission_copy_cleanup_safe_error(&err) {
|
||||||
ignore = true;
|
ignore = true;
|
||||||
cleanup_ignored = true;
|
cleanup_ignored = true;
|
||||||
@@ -3133,6 +3151,9 @@ impl ECStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if should_cleanup_decommission_source_entry(decommissioned, fivs.versions.len(), expired) {
|
if should_cleanup_decommission_source_entry(decommissioned, fivs.versions.len(), expired) {
|
||||||
|
if bucket_incarnation_fence.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||||
|
return Err(Error::other("decommission bucket incarnation fence was lost before source cleanup"));
|
||||||
|
}
|
||||||
decommission_cancel_signal_result(rx.is_cancelled())?;
|
decommission_cancel_signal_result(rx.is_cancelled())?;
|
||||||
|
|
||||||
self.save_decommission_entry_progress_stage(
|
self.save_decommission_entry_progress_stage(
|
||||||
@@ -3157,6 +3178,12 @@ impl ECStore {
|
|||||||
entry.name.as_str(),
|
entry.name.as_str(),
|
||||||
&fivs,
|
&fivs,
|
||||||
&cleanup_preflight_allowed_missing,
|
&cleanup_preflight_allowed_missing,
|
||||||
|
data_movement::SourceCleanupBucketFence {
|
||||||
|
expected_incarnation_id: expected_bucket_incarnation_id,
|
||||||
|
lifecycle_guard: bucket_incarnation_fence
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|guard| guard.namespace_lock_guard()),
|
||||||
|
},
|
||||||
"decommission",
|
"decommission",
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -3268,6 +3295,11 @@ impl ECStore {
|
|||||||
let mut lifecycle_config = None;
|
let mut lifecycle_config = None;
|
||||||
let mut object_lock_config = None;
|
let mut object_lock_config = None;
|
||||||
let mut replication_config = None;
|
let mut replication_config = None;
|
||||||
|
let expected_bucket_incarnation_id = if bi.name == RUSTFS_META_BUCKET {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(self.bucket_incarnation_id_from_disk(&bi.name).await?)
|
||||||
|
};
|
||||||
|
|
||||||
if bi.name != RUSTFS_META_BUCKET {
|
if bi.name != RUSTFS_META_BUCKET {
|
||||||
let _ = resolve_decommission_optional_bucket_config_result(
|
let _ = resolve_decommission_optional_bucket_config_result(
|
||||||
@@ -3321,6 +3353,7 @@ impl ECStore {
|
|||||||
let lifecycle_config = lifecycle_config.clone();
|
let lifecycle_config = lifecycle_config.clone();
|
||||||
let object_lock_config = object_lock_config.clone();
|
let object_lock_config = object_lock_config.clone();
|
||||||
let replication_config = replication_config.clone();
|
let replication_config = replication_config.clone();
|
||||||
|
let expected_bucket_incarnation_id = expected_bucket_incarnation_id;
|
||||||
let entry_error = entry_error.clone();
|
let entry_error = entry_error.clone();
|
||||||
let callback_rx = callback_rx.clone();
|
let callback_rx = callback_rx.clone();
|
||||||
|
|
||||||
@@ -3383,6 +3416,7 @@ impl ECStore {
|
|||||||
lifecycle_config,
|
lifecycle_config,
|
||||||
object_lock_config,
|
object_lock_config,
|
||||||
replication_config,
|
replication_config,
|
||||||
|
expected_bucket_incarnation_id,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
@@ -4168,10 +4202,24 @@ impl ECStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tracing::instrument(skip(self, rd))]
|
#[tracing::instrument(skip(self, rd))]
|
||||||
async fn decommission_object(self: Arc<Self>, pool_idx: usize, bucket: String, rd: GetObjectReader) -> Result<()> {
|
async fn decommission_object(
|
||||||
|
self: Arc<Self>,
|
||||||
|
pool_idx: usize,
|
||||||
|
bucket: String,
|
||||||
|
rd: GetObjectReader,
|
||||||
|
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||||
|
) -> Result<()> {
|
||||||
warn!("decommission_object: start {} {}", &bucket, &rd.object_info.name);
|
warn!("decommission_object: start {} {}", &bucket, &rd.object_info.name);
|
||||||
let object_name = rd.object_info.name.clone();
|
let object_name = rd.object_info.name.clone();
|
||||||
let result = data_movement::migrate_object(self, pool_idx, bucket.clone(), rd, "decommission_object").await;
|
let result = data_movement::migrate_object(
|
||||||
|
self,
|
||||||
|
pool_idx,
|
||||||
|
bucket.clone(),
|
||||||
|
rd,
|
||||||
|
expected_bucket_incarnation_id,
|
||||||
|
"decommission_object",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
if result.is_ok() {
|
if result.is_ok() {
|
||||||
warn!("decommission_object: migrated {} {}", &bucket, &object_name);
|
warn!("decommission_object: migrated {} {}", &bucket, &object_name);
|
||||||
}
|
}
|
||||||
@@ -4347,7 +4395,8 @@ mod tests {
|
|||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
let opts = decommission_delete_marker_opts(&version, Some("version-id".to_string()), 7);
|
let incarnation = uuid::Uuid::new_v4();
|
||||||
|
let opts = decommission_delete_marker_opts(&version, Some("version-id".to_string()), 7, Some(incarnation));
|
||||||
let replication = opts.delete_replication.expect("replication state should be preserved");
|
let replication = opts.delete_replication.expect("replication state should be preserved");
|
||||||
|
|
||||||
assert!(opts.versioned);
|
assert!(opts.versioned);
|
||||||
@@ -4357,11 +4406,25 @@ mod tests {
|
|||||||
assert_eq!(opts.src_pool_idx, 7);
|
assert_eq!(opts.src_pool_idx, 7);
|
||||||
assert_eq!(opts.version_id.as_deref(), Some("version-id"));
|
assert_eq!(opts.version_id.as_deref(), Some("version-id"));
|
||||||
assert_eq!(opts.mod_time, Some(mod_time));
|
assert_eq!(opts.mod_time, Some(mod_time));
|
||||||
|
assert_eq!(opts.expected_bucket_incarnation_id, Some(incarnation));
|
||||||
assert_eq!(replication.replica_status, ReplicationStatusType::Replica);
|
assert_eq!(replication.replica_status, ReplicationStatusType::Replica);
|
||||||
assert!(replication.delete_marker);
|
assert!(replication.delete_marker);
|
||||||
assert_eq!(replication.replicate_decision_str, "existing");
|
assert_eq!(replication.replicate_decision_str, "existing");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn decommission_delete_marker_opts_preserves_suspended_null_version() {
|
||||||
|
let version = rustfs_filemeta::FileInfo {
|
||||||
|
deleted: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let opts = decommission_delete_marker_opts(&version, None, 7, None);
|
||||||
|
|
||||||
|
assert!(!opts.versioned);
|
||||||
|
assert!(opts.version_suspended);
|
||||||
|
assert_eq!(opts.version_id.as_deref(), Some(uuid::Uuid::nil().to_string().as_str()));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_decommission_object_migration_read_opts_are_raw_data_movement() {
|
fn test_decommission_object_migration_read_opts_are_raw_data_movement() {
|
||||||
let opts = decommission_object_migration_read_opts(Some("vid-1".to_string()));
|
let opts = decommission_object_migration_read_opts(Some("vid-1".to_string()));
|
||||||
@@ -4383,7 +4446,8 @@ mod tests {
|
|||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
let opts = decommission_remote_tiered_opts(&version, Some("version-id".to_string()), 9);
|
let incarnation = uuid::Uuid::new_v4();
|
||||||
|
let opts = decommission_remote_tiered_opts(&version, Some("version-id".to_string()), 9, Some(incarnation));
|
||||||
|
|
||||||
assert!(opts.versioned);
|
assert!(opts.versioned);
|
||||||
assert!(opts.data_movement);
|
assert!(opts.data_movement);
|
||||||
@@ -4391,6 +4455,9 @@ mod tests {
|
|||||||
assert_eq!(opts.version_id.as_deref(), Some("version-id"));
|
assert_eq!(opts.version_id.as_deref(), Some("version-id"));
|
||||||
assert_eq!(opts.mod_time, Some(mod_time));
|
assert_eq!(opts.mod_time, Some(mod_time));
|
||||||
assert_eq!(opts.user_defined.get("x-amz-meta-key").map(String::as_str), Some("value"));
|
assert_eq!(opts.user_defined.get("x-amz-meta-key").map(String::as_str), Some("value"));
|
||||||
|
assert!(opts.include_part_checksums);
|
||||||
|
assert!(opts.http_preconditions.is_some());
|
||||||
|
assert_eq!(opts.expected_bucket_incarnation_id, Some(incarnation));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
+1555
-252
File diff suppressed because it is too large
Load Diff
@@ -137,6 +137,7 @@ pub(crate) const GET_METADATA_CACHE_REASON_NO_LOCK: &str = "no_lock";
|
|||||||
pub(crate) const GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED: &str = "not_found_or_expired";
|
pub(crate) const GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED: &str = "not_found_or_expired";
|
||||||
pub(crate) const GET_METADATA_CACHE_REASON_NOT_READ_DATA: &str = "not_read_data";
|
pub(crate) const GET_METADATA_CACHE_REASON_NOT_READ_DATA: &str = "not_read_data";
|
||||||
pub(crate) const GET_METADATA_CACHE_REASON_PART_NUMBER: &str = "part_number";
|
pub(crate) const GET_METADATA_CACHE_REASON_PART_NUMBER: &str = "part_number";
|
||||||
|
pub(crate) const GET_METADATA_CACHE_REASON_PART_CHECKSUMS: &str = "part_checksums";
|
||||||
pub(crate) const GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ: &str = "raw_data_movement_read";
|
pub(crate) const GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ: &str = "raw_data_movement_read";
|
||||||
pub(crate) const GET_METADATA_CACHE_REASON_STALE_PUBLICATION: &str = "stale_publication";
|
pub(crate) const GET_METADATA_CACHE_REASON_STALE_PUBLICATION: &str = "stale_publication";
|
||||||
pub(crate) const GET_METADATA_CACHE_REASON_USABLE: &str = "usable";
|
pub(crate) const GET_METADATA_CACHE_REASON_USABLE: &str = "usable";
|
||||||
@@ -480,6 +481,7 @@ mod tests {
|
|||||||
assert_eq!(GET_METADATA_CACHE_REASON_NO_LOCK, "no_lock");
|
assert_eq!(GET_METADATA_CACHE_REASON_NO_LOCK, "no_lock");
|
||||||
assert_eq!(GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED, "not_found_or_expired");
|
assert_eq!(GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED, "not_found_or_expired");
|
||||||
assert_eq!(GET_METADATA_CACHE_REASON_NOT_READ_DATA, "not_read_data");
|
assert_eq!(GET_METADATA_CACHE_REASON_NOT_READ_DATA, "not_read_data");
|
||||||
|
assert_eq!(GET_METADATA_CACHE_REASON_PART_CHECKSUMS, "part_checksums");
|
||||||
assert_eq!(GET_METADATA_CACHE_REASON_PART_NUMBER, "part_number");
|
assert_eq!(GET_METADATA_CACHE_REASON_PART_NUMBER, "part_number");
|
||||||
assert_eq!(GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, "raw_data_movement_read");
|
assert_eq!(GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, "raw_data_movement_read");
|
||||||
assert_eq!(GET_METADATA_CACHE_REASON_STALE_PUBLICATION, "stale_publication");
|
assert_eq!(GET_METADATA_CACHE_REASON_STALE_PUBLICATION, "stale_publication");
|
||||||
|
|||||||
@@ -9875,6 +9875,7 @@ impl DiskAPI for LocalDisk {
|
|||||||
FileInfoOpts {
|
FileInfoOpts {
|
||||||
data: read_data,
|
data: read_data,
|
||||||
include_free_versions: opts.incl_free_versions,
|
include_free_versions: opts.incl_free_versions,
|
||||||
|
include_part_checksums: false,
|
||||||
},
|
},
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
|
|||||||
@@ -260,6 +260,9 @@ pub struct ObjectOptions {
|
|||||||
|
|
||||||
pub data_movement: bool,
|
pub data_movement: bool,
|
||||||
pub raw_data_movement_read: bool,
|
pub raw_data_movement_read: bool,
|
||||||
|
/// Materialize the data-movement per-part checksum sidecar for APIs that
|
||||||
|
/// return part checksums. Ordinary object reads leave it encoded.
|
||||||
|
pub include_part_checksums: bool,
|
||||||
pub src_pool_idx: usize,
|
pub src_pool_idx: usize,
|
||||||
pub user_defined: HashMap<String, String>,
|
pub user_defined: HashMap<String, String>,
|
||||||
pub preserve_etag: Option<String>,
|
pub preserve_etag: Option<String>,
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ use super::meta::{
|
|||||||
clone_arc_by_index, ensure_valid_rebalance_pool_index, invalid_rebalance_pool_index_error,
|
clone_arc_by_index, ensure_valid_rebalance_pool_index, invalid_rebalance_pool_index_error,
|
||||||
rebalance_metadata_not_initialized_error, should_ignore_rebalance_data_usage_cache,
|
rebalance_metadata_not_initialized_error, should_ignore_rebalance_data_usage_cache,
|
||||||
};
|
};
|
||||||
use super::migration::migrate_entry_version;
|
use super::migration::{RebalanceMigrationBackend, migrate_entry_version};
|
||||||
use super::worker::{
|
use super::worker::{
|
||||||
RebalanceEntryCleanupResult, RebalanceEntryTask, load_rebalance_bucket_configs, rebalance_max_attempts,
|
RebalanceEntryCleanupResult, RebalanceEntryTask, load_rebalance_bucket_configs, rebalance_max_attempts,
|
||||||
resolve_rebalance_bucket_error, resolve_rebalance_entry_cleanup_delete_result, resolve_rebalance_file_info_versions_result,
|
resolve_rebalance_bucket_error, resolve_rebalance_entry_cleanup_delete_result, resolve_rebalance_file_info_versions_result,
|
||||||
@@ -144,6 +144,11 @@ impl ECStore {
|
|||||||
return Ok(RebalanceEntryOutcome::Completed);
|
return Ok(RebalanceEntryOutcome::Completed);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let bucket_incarnation_fence = match bucket_configs.bucket_incarnation_id {
|
||||||
|
Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?),
|
||||||
|
None => None,
|
||||||
|
};
|
||||||
|
|
||||||
let mut fivs =
|
let mut fivs =
|
||||||
resolve_rebalance_file_info_versions_result(entry.file_info_versions(&bucket), bucket.as_str(), entry.name.as_str())?;
|
resolve_rebalance_file_info_versions_result(entry.file_info_versions(&bucket), bucket.as_str(), entry.name.as_str())?;
|
||||||
|
|
||||||
@@ -203,9 +208,14 @@ impl ECStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let version_id = version.version_id.map(|v| v.to_string());
|
let version_id = version.version_id.map(|v| v.to_string());
|
||||||
|
let expected_bucket_incarnation_id = bucket_configs.bucket_incarnation_id;
|
||||||
let mut transfer = |src_pool_idx: usize, bucket: String, rd: GetObjectReader| {
|
let mut transfer = |src_pool_idx: usize, bucket: String, rd: GetObjectReader| {
|
||||||
let store = self.clone();
|
let store = self.clone();
|
||||||
async move { store.rebalance_object(src_pool_idx, bucket, rd).await }
|
async move {
|
||||||
|
store
|
||||||
|
.rebalance_object(src_pool_idx, bucket, rd, expected_bucket_incarnation_id)
|
||||||
|
.await
|
||||||
|
}
|
||||||
};
|
};
|
||||||
// Route delete-marker migration through the store layer so it lands on the
|
// Route delete-marker migration through the store layer so it lands on the
|
||||||
// cross-pool target (excluding the source pool), not back onto the source set.
|
// cross-pool target (excluding the source pool), not back onto the source set.
|
||||||
@@ -214,11 +224,12 @@ impl ECStore {
|
|||||||
async move { store.delete_object(&bucket, &object, opts).await }
|
async move { store.delete_object(&bucket, &object, opts).await }
|
||||||
};
|
};
|
||||||
let result = migrate_entry_version(
|
let result = migrate_entry_version(
|
||||||
set.as_ref(),
|
&RebalanceMigrationBackend::new(set.as_ref(), self.as_ref()),
|
||||||
bucket.clone(),
|
bucket.clone(),
|
||||||
pool_index,
|
pool_index,
|
||||||
version,
|
version,
|
||||||
version_id.clone(),
|
version_id.clone(),
|
||||||
|
expected_bucket_incarnation_id,
|
||||||
rebalance_max_attempts(),
|
rebalance_max_attempts(),
|
||||||
should_ignore_rebalance_data_usage_cache(bucket.as_str()),
|
should_ignore_rebalance_data_usage_cache(bucket.as_str()),
|
||||||
&mut transfer,
|
&mut transfer,
|
||||||
@@ -303,6 +314,9 @@ impl ECStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if should_cleanup_rebalance_source_entry(rebalanced, fivs.versions.len(), expired) {
|
if should_cleanup_rebalance_source_entry(rebalanced, fivs.versions.len(), expired) {
|
||||||
|
if bucket_incarnation_fence.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||||
|
return Err(Error::other("rebalance bucket incarnation fence was lost before source cleanup"));
|
||||||
|
}
|
||||||
let cleanup_result = self
|
let cleanup_result = self
|
||||||
.finish_rebalance_entry_after_cleanup(
|
.finish_rebalance_entry_after_cleanup(
|
||||||
pool_index,
|
pool_index,
|
||||||
@@ -315,6 +329,12 @@ impl ECStore {
|
|||||||
entry.name.as_str(),
|
entry.name.as_str(),
|
||||||
&fivs,
|
&fivs,
|
||||||
&cleanup_preflight_allowed_missing,
|
&cleanup_preflight_allowed_missing,
|
||||||
|
data_movement::SourceCleanupBucketFence {
|
||||||
|
expected_incarnation_id: bucket_configs.bucket_incarnation_id,
|
||||||
|
lifecycle_guard: bucket_incarnation_fence
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|guard| guard.namespace_lock_guard()),
|
||||||
|
},
|
||||||
"rebalance",
|
"rebalance",
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -389,8 +409,14 @@ impl ECStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tracing::instrument(skip(self, rd))]
|
#[tracing::instrument(skip(self, rd))]
|
||||||
async fn rebalance_object(self: Arc<Self>, pool_idx: usize, bucket: String, rd: GetObjectReader) -> Result<()> {
|
async fn rebalance_object(
|
||||||
data_movement::migrate_object(self, pool_idx, bucket, rd, "rebalance_object").await
|
self: Arc<Self>,
|
||||||
|
pool_idx: usize,
|
||||||
|
bucket: String,
|
||||||
|
rd: GetObjectReader,
|
||||||
|
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||||
|
) -> Result<()> {
|
||||||
|
data_movement::migrate_object(self, pool_idx, bucket, rd, expected_bucket_incarnation_id, "rebalance_object").await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn update_rebalance_last_error(&self, pool_idx: usize, message: String) -> Result<()> {
|
async fn update_rebalance_last_error(&self, pool_idx: usize, message: String) -> Result<()> {
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ use crate::error::{Error, Result, is_err_object_not_found, is_err_version_not_fo
|
|||||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
|
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
|
||||||
use crate::set_disk::SetDisks;
|
use crate::set_disk::SetDisks;
|
||||||
use crate::storage_api_contracts::{object::ObjectIO, range::HTTPRangeSpec};
|
use crate::storage_api_contracts::{object::ObjectIO, range::HTTPRangeSpec};
|
||||||
|
use crate::store::ECStore;
|
||||||
use http::HeaderMap;
|
use http::HeaderMap;
|
||||||
use rustfs_filemeta::FileInfo;
|
use rustfs_filemeta::FileInfo;
|
||||||
use rustfs_utils::path::encode_dir_object;
|
use rustfs_utils::path::encode_dir_object;
|
||||||
@@ -21,15 +22,23 @@ pub(crate) struct MigrationVersionResult {
|
|||||||
pub error: Option<Error>,
|
pub error: Option<Error>,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn rebalance_delete_marker_opts(version: &FileInfo, version_id: Option<String>, src_pool_idx: usize) -> ObjectOptions {
|
pub(super) fn rebalance_delete_marker_opts(
|
||||||
|
version: &FileInfo,
|
||||||
|
version_id: Option<String>,
|
||||||
|
src_pool_idx: usize,
|
||||||
|
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||||
|
) -> ObjectOptions {
|
||||||
|
let version_suspended = version.version_id.is_none() && version_id.is_none();
|
||||||
ObjectOptions {
|
ObjectOptions {
|
||||||
versioned: true,
|
versioned: !version_suspended,
|
||||||
version_id,
|
version_suspended,
|
||||||
|
version_id: version_id.or_else(|| version_suspended.then(|| uuid::Uuid::nil().to_string())),
|
||||||
mod_time: version.mod_time,
|
mod_time: version.mod_time,
|
||||||
src_pool_idx,
|
src_pool_idx,
|
||||||
data_movement: true,
|
data_movement: true,
|
||||||
delete_marker: true,
|
delete_marker: true,
|
||||||
skip_decommissioned: true,
|
skip_decommissioned: true,
|
||||||
|
expected_bucket_incarnation_id,
|
||||||
delete_replication: version
|
delete_replication: version
|
||||||
.replication_state_internal
|
.replication_state_internal
|
||||||
.as_ref()
|
.as_ref()
|
||||||
@@ -38,7 +47,12 @@ pub(super) fn rebalance_delete_marker_opts(version: &FileInfo, version_id: Optio
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn rebalance_remote_tiered_opts(version: &FileInfo, version_id: Option<String>, src_pool_idx: usize) -> ObjectOptions {
|
fn rebalance_remote_tiered_opts(
|
||||||
|
version: &FileInfo,
|
||||||
|
version_id: Option<String>,
|
||||||
|
src_pool_idx: usize,
|
||||||
|
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||||
|
) -> ObjectOptions {
|
||||||
ObjectOptions {
|
ObjectOptions {
|
||||||
versioned: version_id.is_some(),
|
versioned: version_id.is_some(),
|
||||||
version_id,
|
version_id,
|
||||||
@@ -46,6 +60,21 @@ fn rebalance_remote_tiered_opts(version: &FileInfo, version_id: Option<String>,
|
|||||||
user_defined: version.metadata.clone(),
|
user_defined: version.metadata.clone(),
|
||||||
src_pool_idx,
|
src_pool_idx,
|
||||||
data_movement: true,
|
data_movement: true,
|
||||||
|
include_part_checksums: true,
|
||||||
|
http_preconditions: Some(crate::data_movement::data_movement_target_precondition()),
|
||||||
|
expected_bucket_incarnation_id,
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn rebalance_object_migration_read_opts(version_id: Option<String>) -> ObjectOptions {
|
||||||
|
ObjectOptions {
|
||||||
|
version_id,
|
||||||
|
no_lock: true,
|
||||||
|
data_movement: true,
|
||||||
|
raw_data_movement_read: true,
|
||||||
|
skip_decommissioned: true,
|
||||||
|
skip_rebalancing: true,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -70,8 +99,19 @@ pub(crate) trait MigrationBackend: Send + Sync {
|
|||||||
) -> Result<()>;
|
) -> Result<()>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) struct RebalanceMigrationBackend<'a> {
|
||||||
|
source: &'a SetDisks,
|
||||||
|
store: &'a ECStore,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> RebalanceMigrationBackend<'a> {
|
||||||
|
pub(crate) fn new(source: &'a SetDisks, store: &'a ECStore) -> Self {
|
||||||
|
Self { source, store }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
#[async_trait::async_trait]
|
||||||
impl MigrationBackend for SetDisks {
|
impl MigrationBackend for RebalanceMigrationBackend<'_> {
|
||||||
async fn get_object_reader_for_migration(
|
async fn get_object_reader_for_migration(
|
||||||
&self,
|
&self,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
@@ -80,7 +120,7 @@ impl MigrationBackend for SetDisks {
|
|||||||
h: HeaderMap,
|
h: HeaderMap,
|
||||||
opts: &ObjectOptions,
|
opts: &ObjectOptions,
|
||||||
) -> Result<GetObjectReader> {
|
) -> Result<GetObjectReader> {
|
||||||
self.get_object_reader(bucket, object, range, h, opts).await
|
self.source.get_object_reader(bucket, object, range, h, opts).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn move_remote_version_for_migration(
|
async fn move_remote_version_for_migration(
|
||||||
@@ -90,7 +130,7 @@ impl MigrationBackend for SetDisks {
|
|||||||
fi: &FileInfo,
|
fi: &FileInfo,
|
||||||
opts: &ObjectOptions,
|
opts: &ObjectOptions,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
self.decommission_tiered_object(bucket, object, fi, opts).await
|
self.store.decommission_tiered_object(bucket, object, fi, opts).await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -101,6 +141,7 @@ pub(crate) async fn migrate_entry_version<Backend, F, Fut, D, DFut>(
|
|||||||
pool_index: usize,
|
pool_index: usize,
|
||||||
version: &FileInfo,
|
version: &FileInfo,
|
||||||
version_id: Option<String>,
|
version_id: Option<String>,
|
||||||
|
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||||
max_attempts: usize,
|
max_attempts: usize,
|
||||||
ignore_data_usage_cache: bool,
|
ignore_data_usage_cache: bool,
|
||||||
transfer: F,
|
transfer: F,
|
||||||
@@ -113,12 +154,13 @@ where
|
|||||||
D: FnMut(String, String, ObjectOptions) -> DFut + Send,
|
D: FnMut(String, String, ObjectOptions) -> DFut + Send,
|
||||||
DFut: Future<Output = Result<ObjectInfo>> + Send,
|
DFut: Future<Output = Result<ObjectInfo>> + Send,
|
||||||
{
|
{
|
||||||
migrate_entry_version_with_retry_wait(
|
migrate_entry_version_with_retry_wait_and_incarnation(
|
||||||
set,
|
set,
|
||||||
bucket,
|
bucket,
|
||||||
pool_index,
|
pool_index,
|
||||||
version,
|
version,
|
||||||
version_id,
|
version_id,
|
||||||
|
expected_bucket_incarnation_id,
|
||||||
max_attempts,
|
max_attempts,
|
||||||
ignore_data_usage_cache,
|
ignore_data_usage_cache,
|
||||||
transfer,
|
transfer,
|
||||||
@@ -137,6 +179,45 @@ pub(super) async fn migrate_entry_version_with_retry_wait<Backend, F, Fut, D, DF
|
|||||||
version_id: Option<String>,
|
version_id: Option<String>,
|
||||||
max_attempts: usize,
|
max_attempts: usize,
|
||||||
ignore_data_usage_cache: bool,
|
ignore_data_usage_cache: bool,
|
||||||
|
transfer: F,
|
||||||
|
delete_marker: D,
|
||||||
|
wait_retry: W,
|
||||||
|
) -> MigrationVersionResult
|
||||||
|
where
|
||||||
|
Backend: MigrationBackend + ?Sized,
|
||||||
|
F: FnMut(usize, String, GetObjectReader) -> Fut + Send,
|
||||||
|
Fut: Future<Output = Result<()>> + Send,
|
||||||
|
D: FnMut(String, String, ObjectOptions) -> DFut + Send,
|
||||||
|
DFut: Future<Output = Result<ObjectInfo>> + Send,
|
||||||
|
W: FnMut(Duration) -> WFut + Send,
|
||||||
|
WFut: Future<Output = ()> + Send,
|
||||||
|
{
|
||||||
|
migrate_entry_version_with_retry_wait_and_incarnation(
|
||||||
|
set,
|
||||||
|
bucket,
|
||||||
|
pool_index,
|
||||||
|
version,
|
||||||
|
version_id,
|
||||||
|
None,
|
||||||
|
max_attempts,
|
||||||
|
ignore_data_usage_cache,
|
||||||
|
transfer,
|
||||||
|
delete_marker,
|
||||||
|
wait_retry,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
async fn migrate_entry_version_with_retry_wait_and_incarnation<Backend, F, Fut, D, DFut, W, WFut>(
|
||||||
|
set: &Backend,
|
||||||
|
bucket: String,
|
||||||
|
pool_index: usize,
|
||||||
|
version: &FileInfo,
|
||||||
|
version_id: Option<String>,
|
||||||
|
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||||
|
max_attempts: usize,
|
||||||
|
ignore_data_usage_cache: bool,
|
||||||
mut transfer: F,
|
mut transfer: F,
|
||||||
mut delete_marker: D,
|
mut delete_marker: D,
|
||||||
mut wait_retry: W,
|
mut wait_retry: W,
|
||||||
@@ -169,7 +250,7 @@ where
|
|||||||
&bucket,
|
&bucket,
|
||||||
&version.name,
|
&version.name,
|
||||||
version,
|
version,
|
||||||
&rebalance_remote_tiered_opts(version, version_id, pool_index),
|
&rebalance_remote_tiered_opts(version, version_id, pool_index, expected_bucket_incarnation_id),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
@@ -212,7 +293,7 @@ where
|
|||||||
if let Err(err) = delete_marker(
|
if let Err(err) = delete_marker(
|
||||||
bucket.clone(),
|
bucket.clone(),
|
||||||
version.name.clone(),
|
version.name.clone(),
|
||||||
rebalance_delete_marker_opts(version, version_id, pool_index),
|
rebalance_delete_marker_opts(version, version_id, pool_index, expected_bucket_incarnation_id),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
@@ -255,11 +336,7 @@ where
|
|||||||
&encode_dir_object(&version.name),
|
&encode_dir_object(&version.name),
|
||||||
None,
|
None,
|
||||||
HeaderMap::new(),
|
HeaderMap::new(),
|
||||||
&ObjectOptions {
|
&rebalance_object_migration_read_opts(version_id.clone()),
|
||||||
version_id: version_id.clone(),
|
|
||||||
no_lock: true,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -113,6 +113,8 @@ struct LegacyRebalanceMeta {
|
|||||||
struct MigrationBackendSpy {
|
struct MigrationBackendSpy {
|
||||||
get_object_reader: Mutex<Option<core::result::Result<GetObjectReader, Error>>>,
|
get_object_reader: Mutex<Option<core::result::Result<GetObjectReader, Error>>>,
|
||||||
move_remote: Mutex<Option<core::result::Result<(), Error>>>,
|
move_remote: Mutex<Option<core::result::Result<(), Error>>>,
|
||||||
|
get_opts: Mutex<Vec<ObjectOptions>>,
|
||||||
|
move_remote_opts: Mutex<Vec<ObjectOptions>>,
|
||||||
get_calls: AtomicUsize,
|
get_calls: AtomicUsize,
|
||||||
move_remote_calls: AtomicUsize,
|
move_remote_calls: AtomicUsize,
|
||||||
}
|
}
|
||||||
@@ -125,6 +127,8 @@ impl MigrationBackendSpy {
|
|||||||
Self {
|
Self {
|
||||||
get_object_reader: Mutex::new(get_object_reader),
|
get_object_reader: Mutex::new(get_object_reader),
|
||||||
move_remote: Mutex::new(move_remote),
|
move_remote: Mutex::new(move_remote),
|
||||||
|
get_opts: Mutex::new(Vec::new()),
|
||||||
|
move_remote_opts: Mutex::new(Vec::new()),
|
||||||
get_calls: AtomicUsize::new(0),
|
get_calls: AtomicUsize::new(0),
|
||||||
move_remote_calls: AtomicUsize::new(0),
|
move_remote_calls: AtomicUsize::new(0),
|
||||||
}
|
}
|
||||||
@@ -138,6 +142,24 @@ impl MigrationBackendSpy {
|
|||||||
self.move_remote_calls.load(Ordering::SeqCst)
|
self.move_remote_calls.load(Ordering::SeqCst)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn last_get_opts(&self) -> ObjectOptions {
|
||||||
|
self.get_opts
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.last()
|
||||||
|
.cloned()
|
||||||
|
.expect("reader opts should be captured")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn last_move_remote_opts(&self) -> ObjectOptions {
|
||||||
|
self.move_remote_opts
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.last()
|
||||||
|
.cloned()
|
||||||
|
.expect("remote opts should be captured")
|
||||||
|
}
|
||||||
|
|
||||||
fn make_reader() -> GetObjectReader {
|
fn make_reader() -> GetObjectReader {
|
||||||
GetObjectReader {
|
GetObjectReader {
|
||||||
stream: Box::new(Cursor::new(vec![0_u8; 3])),
|
stream: Box::new(Cursor::new(vec![0_u8; 3])),
|
||||||
@@ -156,9 +178,10 @@ impl MigrationBackend for MigrationBackendSpy {
|
|||||||
_object: &str,
|
_object: &str,
|
||||||
_range: Option<HTTPRangeSpec>,
|
_range: Option<HTTPRangeSpec>,
|
||||||
_h: http::HeaderMap,
|
_h: http::HeaderMap,
|
||||||
_opts: &ObjectOptions,
|
opts: &ObjectOptions,
|
||||||
) -> Result<GetObjectReader> {
|
) -> Result<GetObjectReader> {
|
||||||
self.get_calls.fetch_add(1, Ordering::SeqCst);
|
self.get_calls.fetch_add(1, Ordering::SeqCst);
|
||||||
|
self.get_opts.lock().unwrap().push(opts.clone());
|
||||||
if let Some(result) = self.get_object_reader.lock().unwrap().take() {
|
if let Some(result) = self.get_object_reader.lock().unwrap().take() {
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
@@ -171,9 +194,10 @@ impl MigrationBackend for MigrationBackendSpy {
|
|||||||
_bucket: &str,
|
_bucket: &str,
|
||||||
_object: &str,
|
_object: &str,
|
||||||
_fi: &FileInfo,
|
_fi: &FileInfo,
|
||||||
_opts: &ObjectOptions,
|
opts: &ObjectOptions,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
self.move_remote_calls.fetch_add(1, Ordering::SeqCst);
|
self.move_remote_calls.fetch_add(1, Ordering::SeqCst);
|
||||||
|
self.move_remote_opts.lock().unwrap().push(opts.clone());
|
||||||
if let Some(result) = self.move_remote.lock().unwrap().take() {
|
if let Some(result) = self.move_remote.lock().unwrap().take() {
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
@@ -217,7 +241,8 @@ fn test_rebalance_delete_marker_opts_preserves_replication_state() {
|
|||||||
..version_deleted()
|
..version_deleted()
|
||||||
};
|
};
|
||||||
|
|
||||||
let opts = rebalance_delete_marker_opts(&version, Some("version-id".to_string()), 7);
|
let incarnation = uuid::Uuid::new_v4();
|
||||||
|
let opts = rebalance_delete_marker_opts(&version, Some("version-id".to_string()), 7, Some(incarnation));
|
||||||
let replication = opts.delete_replication.expect("replication state should be preserved");
|
let replication = opts.delete_replication.expect("replication state should be preserved");
|
||||||
|
|
||||||
assert!(opts.versioned);
|
assert!(opts.versioned);
|
||||||
@@ -227,11 +252,22 @@ fn test_rebalance_delete_marker_opts_preserves_replication_state() {
|
|||||||
assert_eq!(opts.src_pool_idx, 7);
|
assert_eq!(opts.src_pool_idx, 7);
|
||||||
assert_eq!(opts.version_id.as_deref(), Some("version-id"));
|
assert_eq!(opts.version_id.as_deref(), Some("version-id"));
|
||||||
assert_eq!(opts.mod_time, Some(mod_time));
|
assert_eq!(opts.mod_time, Some(mod_time));
|
||||||
|
assert_eq!(opts.expected_bucket_incarnation_id, Some(incarnation));
|
||||||
assert_eq!(replication.replica_status, ReplicationStatusType::Replica);
|
assert_eq!(replication.replica_status, ReplicationStatusType::Replica);
|
||||||
assert!(replication.delete_marker);
|
assert!(replication.delete_marker);
|
||||||
assert_eq!(replication.replicate_decision_str, "existing");
|
assert_eq!(replication.replicate_decision_str, "existing");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_rebalance_delete_marker_opts_preserves_suspended_null_version() {
|
||||||
|
let version = version_deleted();
|
||||||
|
let opts = rebalance_delete_marker_opts(&version, None, 7, None);
|
||||||
|
|
||||||
|
assert!(!opts.versioned);
|
||||||
|
assert!(opts.version_suspended);
|
||||||
|
assert_eq!(opts.version_id.as_deref(), Some(uuid::Uuid::nil().to_string().as_str()));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_migrate_entry_version_remote_version_is_moved_without_transfer() {
|
async fn test_migrate_entry_version_remote_version_is_moved_without_transfer() {
|
||||||
let backend = MigrationBackendSpy::new(None, Some(Ok(())));
|
let backend = MigrationBackendSpy::new(None, Some(Ok(())));
|
||||||
@@ -248,12 +284,14 @@ async fn test_migrate_entry_version_remote_version_is_moved_without_transfer() {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let incarnation = uuid::Uuid::new_v4();
|
||||||
let result = migrate_entry_version(
|
let result = migrate_entry_version(
|
||||||
&backend,
|
&backend,
|
||||||
"bucket".to_string(),
|
"bucket".to_string(),
|
||||||
0,
|
0,
|
||||||
&version,
|
&version,
|
||||||
version.version_id.map(|v| v.to_string()),
|
version.version_id.map(|v| v.to_string()),
|
||||||
|
Some(incarnation),
|
||||||
3,
|
3,
|
||||||
false,
|
false,
|
||||||
&mut transfer,
|
&mut transfer,
|
||||||
@@ -269,6 +307,10 @@ async fn test_migrate_entry_version_remote_version_is_moved_without_transfer() {
|
|||||||
assert_eq!(transfer_count.load(Ordering::SeqCst), 0);
|
assert_eq!(transfer_count.load(Ordering::SeqCst), 0);
|
||||||
assert_eq!(backend.move_remote_calls(), 1);
|
assert_eq!(backend.move_remote_calls(), 1);
|
||||||
assert_eq!(backend.get_calls(), 0);
|
assert_eq!(backend.get_calls(), 0);
|
||||||
|
let remote_opts = backend.last_move_remote_opts();
|
||||||
|
assert!(remote_opts.include_part_checksums);
|
||||||
|
assert!(remote_opts.http_preconditions.is_some());
|
||||||
|
assert_eq!(remote_opts.expected_bucket_incarnation_id, Some(incarnation));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -294,6 +336,7 @@ async fn test_migrate_entry_version_remote_not_found_is_cleanup_ignored() {
|
|||||||
0,
|
0,
|
||||||
&version,
|
&version,
|
||||||
version.version_id.map(|v| v.to_string()),
|
version.version_id.map(|v| v.to_string()),
|
||||||
|
None,
|
||||||
3,
|
3,
|
||||||
false,
|
false,
|
||||||
&mut transfer,
|
&mut transfer,
|
||||||
@@ -330,6 +373,7 @@ async fn test_migrate_entry_version_remote_overwrite_is_not_ignored() {
|
|||||||
0,
|
0,
|
||||||
&version,
|
&version,
|
||||||
Some("vid-1".to_string()),
|
Some("vid-1".to_string()),
|
||||||
|
None,
|
||||||
3,
|
3,
|
||||||
false,
|
false,
|
||||||
&mut transfer,
|
&mut transfer,
|
||||||
@@ -368,6 +412,7 @@ async fn test_migrate_entry_version_remote_failure_is_reported() {
|
|||||||
0,
|
0,
|
||||||
&version,
|
&version,
|
||||||
version.version_id.map(|v| v.to_string()),
|
version.version_id.map(|v| v.to_string()),
|
||||||
|
None,
|
||||||
3,
|
3,
|
||||||
false,
|
false,
|
||||||
&mut transfer,
|
&mut transfer,
|
||||||
@@ -410,6 +455,7 @@ async fn test_migrate_entry_version_deleted_version_routes_delete_through_store_
|
|||||||
1,
|
1,
|
||||||
&version,
|
&version,
|
||||||
version.version_id.map(|v| v.to_string()),
|
version.version_id.map(|v| v.to_string()),
|
||||||
|
None,
|
||||||
3,
|
3,
|
||||||
false,
|
false,
|
||||||
&mut transfer,
|
&mut transfer,
|
||||||
@@ -449,6 +495,7 @@ async fn test_migrate_entry_version_deleted_version_not_found_is_ignored() {
|
|||||||
1,
|
1,
|
||||||
&version,
|
&version,
|
||||||
version.version_id.map(|v| v.to_string()),
|
version.version_id.map(|v| v.to_string()),
|
||||||
|
None,
|
||||||
3,
|
3,
|
||||||
false,
|
false,
|
||||||
&mut transfer,
|
&mut transfer,
|
||||||
@@ -491,6 +538,7 @@ async fn test_migrate_entry_version_deleted_version_overwrite_is_not_ignored() {
|
|||||||
1,
|
1,
|
||||||
&version,
|
&version,
|
||||||
Some("vid-1".to_string()),
|
Some("vid-1".to_string()),
|
||||||
|
None,
|
||||||
3,
|
3,
|
||||||
false,
|
false,
|
||||||
&mut transfer,
|
&mut transfer,
|
||||||
@@ -520,6 +568,7 @@ async fn test_migrate_entry_version_reader_not_found_is_ignored() {
|
|||||||
1,
|
1,
|
||||||
&version,
|
&version,
|
||||||
version.version_id.map(|v| v.to_string()),
|
version.version_id.map(|v| v.to_string()),
|
||||||
|
None,
|
||||||
3,
|
3,
|
||||||
false,
|
false,
|
||||||
&mut transfer,
|
&mut transfer,
|
||||||
@@ -647,6 +696,7 @@ async fn test_migrate_entry_version_reader_fails_after_retries() {
|
|||||||
1,
|
1,
|
||||||
&version,
|
&version,
|
||||||
version.version_id.map(|v| v.to_string()),
|
version.version_id.map(|v| v.to_string()),
|
||||||
|
None,
|
||||||
3,
|
3,
|
||||||
false,
|
false,
|
||||||
&mut transfer,
|
&mut transfer,
|
||||||
@@ -685,6 +735,7 @@ async fn test_migrate_entry_version_zero_max_attempts_still_attempts_once() {
|
|||||||
1,
|
1,
|
||||||
&version,
|
&version,
|
||||||
version.version_id.map(|v| v.to_string()),
|
version.version_id.map(|v| v.to_string()),
|
||||||
|
None,
|
||||||
0,
|
0,
|
||||||
false,
|
false,
|
||||||
&mut transfer,
|
&mut transfer,
|
||||||
@@ -750,6 +801,13 @@ async fn test_migrate_entry_version_transfer_retries_before_success() {
|
|||||||
assert_eq!(backend.get_calls(), 2);
|
assert_eq!(backend.get_calls(), 2);
|
||||||
assert_eq!(transfer_count.load(Ordering::SeqCst), 2);
|
assert_eq!(transfer_count.load(Ordering::SeqCst), 2);
|
||||||
assert_eq!(wait_count.load(Ordering::SeqCst), 1);
|
assert_eq!(wait_count.load(Ordering::SeqCst), 1);
|
||||||
|
let read_opts = backend.last_get_opts();
|
||||||
|
assert_eq!(read_opts.version_id.as_deref(), version.version_id.map(|id| id.to_string()).as_deref());
|
||||||
|
assert!(read_opts.no_lock);
|
||||||
|
assert!(read_opts.data_movement);
|
||||||
|
assert!(read_opts.raw_data_movement_read);
|
||||||
|
assert!(read_opts.skip_decommissioned);
|
||||||
|
assert!(read_opts.skip_rebalancing);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -822,6 +880,7 @@ async fn test_migrate_entry_version_transfer_fails_after_retries() {
|
|||||||
1,
|
1,
|
||||||
&version,
|
&version,
|
||||||
version.version_id.map(|v| v.to_string()),
|
version.version_id.map(|v| v.to_string()),
|
||||||
|
None,
|
||||||
2,
|
2,
|
||||||
false,
|
false,
|
||||||
&mut transfer,
|
&mut transfer,
|
||||||
@@ -860,6 +919,7 @@ async fn test_migrate_entry_version_transfer_not_found_is_ignored() {
|
|||||||
1,
|
1,
|
||||||
&version,
|
&version,
|
||||||
version.version_id.map(|v| v.to_string()),
|
version.version_id.map(|v| v.to_string()),
|
||||||
|
None,
|
||||||
3,
|
3,
|
||||||
false,
|
false,
|
||||||
&mut transfer,
|
&mut transfer,
|
||||||
@@ -901,6 +961,7 @@ async fn test_migrate_entry_version_transfer_overwrite_is_not_ignored() {
|
|||||||
1,
|
1,
|
||||||
&version,
|
&version,
|
||||||
Some("vid-1".to_string()),
|
Some("vid-1".to_string()),
|
||||||
|
None,
|
||||||
3,
|
3,
|
||||||
false,
|
false,
|
||||||
&mut transfer,
|
&mut transfer,
|
||||||
@@ -943,6 +1004,7 @@ async fn test_migrate_entry_version_ignores_data_usage_cache_when_enabled() {
|
|||||||
1,
|
1,
|
||||||
&version,
|
&version,
|
||||||
version.version_id.map(|v| v.to_string()),
|
version.version_id.map(|v| v.to_string()),
|
||||||
|
None,
|
||||||
2,
|
2,
|
||||||
true,
|
true,
|
||||||
&mut transfer,
|
&mut transfer,
|
||||||
@@ -985,6 +1047,7 @@ async fn test_migrate_entry_version_data_usage_cache_moves_when_ignore_disabled(
|
|||||||
1,
|
1,
|
||||||
&version,
|
&version,
|
||||||
version.version_id.map(|v| v.to_string()),
|
version.version_id.map(|v| v.to_string()),
|
||||||
|
None,
|
||||||
2,
|
2,
|
||||||
false,
|
false,
|
||||||
&mut transfer,
|
&mut transfer,
|
||||||
@@ -2026,6 +2089,7 @@ async fn test_migrate_entry_version_transfer_failure_reports_write_target_stage(
|
|||||||
1,
|
1,
|
||||||
&version,
|
&version,
|
||||||
version.version_id.map(|v| v.to_string()),
|
version.version_id.map(|v| v.to_string()),
|
||||||
|
None,
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
&mut transfer,
|
&mut transfer,
|
||||||
@@ -2050,6 +2114,7 @@ async fn test_migrate_entry_version_reader_failure_reports_read_source_stage() {
|
|||||||
1,
|
1,
|
||||||
&version,
|
&version,
|
||||||
version.version_id.map(|v| v.to_string()),
|
version.version_id.map(|v| v.to_string()),
|
||||||
|
None,
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
&mut transfer,
|
&mut transfer,
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ pub type RStats = Vec<Arc<RebalanceStats>>;
|
|||||||
|
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
pub(super) struct RebalanceBucketConfigs {
|
pub(super) struct RebalanceBucketConfigs {
|
||||||
|
pub(super) bucket_incarnation_id: Option<uuid::Uuid>,
|
||||||
pub(super) lifecycle_config: Option<s3s::dto::BucketLifecycleConfiguration>,
|
pub(super) lifecycle_config: Option<s3s::dto::BucketLifecycleConfiguration>,
|
||||||
pub(super) object_lock_config: Option<s3s::dto::ObjectLockConfiguration>,
|
pub(super) object_lock_config: Option<s3s::dto::ObjectLockConfiguration>,
|
||||||
pub(super) replication_config: Option<(s3s::dto::ReplicationConfiguration, OffsetDateTime)>,
|
pub(super) replication_config: Option<(s3s::dto::ReplicationConfiguration, OffsetDateTime)>,
|
||||||
|
|||||||
@@ -406,6 +406,7 @@ pub(super) async fn load_rebalance_bucket_configs(api: &ECStore, bucket: &str) -
|
|||||||
|
|
||||||
let expiry_configs = crate::bucket::lifecycle::get_expiry_configs(api, bucket).await?;
|
let expiry_configs = crate::bucket::lifecycle::get_expiry_configs(api, bucket).await?;
|
||||||
Ok(RebalanceBucketConfigs {
|
Ok(RebalanceBucketConfigs {
|
||||||
|
bucket_incarnation_id: Some(api.bucket_incarnation_id_from_disk(bucket).await?),
|
||||||
lifecycle_config: expiry_configs.lifecycle.map(|config| (*config).clone()),
|
lifecycle_config: expiry_configs.lifecycle.map(|config| (*config).clone()),
|
||||||
object_lock_config: expiry_configs.object_lock.map(|config| (*config).clone()),
|
object_lock_config: expiry_configs.object_lock.map(|config| (*config).clone()),
|
||||||
replication_config: resolve_rebalance_optional_bucket_config_result(
|
replication_config: resolve_rebalance_optional_bucket_config_result(
|
||||||
|
|||||||
@@ -2443,13 +2443,14 @@ impl SetDisks {
|
|||||||
bucket: &str,
|
bucket: &str,
|
||||||
object: &str,
|
object: &str,
|
||||||
) -> Result<Option<rustfs_filemeta::FileInfoVersions>> {
|
) -> Result<Option<rustfs_filemeta::FileInfoVersions>> {
|
||||||
|
let disk_object = rustfs_utils::path::encode_dir_object(object);
|
||||||
let disks = self.get_disks_internal().await;
|
let disks = self.get_disks_internal().await;
|
||||||
if disks.is_empty() {
|
if disks.is_empty() {
|
||||||
return Err(to_object_err(StorageError::ErasureReadQuorum, vec![bucket, object]));
|
return Err(to_object_err(StorageError::ErasureReadQuorum, vec![bucket, object]));
|
||||||
}
|
}
|
||||||
|
|
||||||
let read_quorum = disks.len().div_ceil(2).max(1);
|
let read_quorum = disks.len().div_ceil(2).max(1);
|
||||||
let (raw_fileinfos, errs) = Self::read_all_raw_file_info(&disks, bucket, object, false).await;
|
let (raw_fileinfos, errs) = Self::read_all_raw_file_info(&disks, bucket, disk_object.as_str(), false).await;
|
||||||
|
|
||||||
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
|
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
|
||||||
let object_err = to_object_err(err.into(), vec![bucket, object]);
|
let object_err = to_object_err(err.into(), vec![bucket, object]);
|
||||||
@@ -2605,7 +2606,7 @@ impl SetDisks {
|
|||||||
//
|
//
|
||||||
// `into_fileinfo` with an empty version_id selects the first non-free version
|
// `into_fileinfo` with an empty version_id selects the first non-free version
|
||||||
// (see FileMeta::into_fileinfo); replicate that selection from the header here.
|
// (see FileMeta::into_fileinfo); replicate that selection from the header here.
|
||||||
let vid = match meta.into_fileinfo(bucket, object, "", true, incl_free_vers, true) {
|
let vid = match meta.into_fileinfo_without_part_checksums(bucket, object, "", true, incl_free_vers) {
|
||||||
Ok(finfo) if file_info_is_valid_for_metadata(&finfo) => finfo.version_id.unwrap_or(Uuid::nil()),
|
Ok(finfo) if file_info_is_valid_for_metadata(&finfo) => finfo.version_id.unwrap_or(Uuid::nil()),
|
||||||
_ => match meta
|
_ => match meta
|
||||||
.versions
|
.versions
|
||||||
@@ -2628,7 +2629,13 @@ impl SetDisks {
|
|||||||
|
|
||||||
for (idx, meta_op) in metadata_array.iter().enumerate() {
|
for (idx, meta_op) in metadata_array.iter().enumerate() {
|
||||||
if let Some(meta) = meta_op {
|
if let Some(meta) = meta_op {
|
||||||
match meta.into_fileinfo(bucket, object, vid.to_string().as_str(), read_data, incl_free_vers, true) {
|
match meta.into_fileinfo_without_part_checksums(
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
vid.to_string().as_str(),
|
||||||
|
read_data,
|
||||||
|
incl_free_vers,
|
||||||
|
) {
|
||||||
Ok(res) => match res.validate_for_metadata_read() {
|
Ok(res) => match res.validate_for_metadata_read() {
|
||||||
Ok(_) => meta_file_infos[idx] = res,
|
Ok(_) => meta_file_infos[idx] = res,
|
||||||
Err(err) => errs[idx] = Some(err.into()),
|
Err(err) => errs[idx] = Some(err.into()),
|
||||||
@@ -4533,26 +4540,29 @@ impl SetDisks {
|
|||||||
object: &str,
|
object: &str,
|
||||||
opts: &ObjectOptions,
|
opts: &ObjectOptions,
|
||||||
) -> Option<StorageError> {
|
) -> Option<StorageError> {
|
||||||
let mut opts = opts.clone();
|
let mut lookup_opts = opts.clone();
|
||||||
|
|
||||||
let http_preconditions = opts.http_preconditions?;
|
let http_preconditions = lookup_opts.http_preconditions?;
|
||||||
opts.http_preconditions = None;
|
lookup_opts.http_preconditions = None;
|
||||||
|
|
||||||
// Never claim a lock here, to avoid deadlock
|
// Never claim a lock here, to avoid deadlock
|
||||||
// - If no_lock is false, we must have obtained the lock out side of this function
|
// - If no_lock is false, we must have obtained the lock out side of this function
|
||||||
// - If no_lock is true, we should not obtain locks
|
// - If no_lock is true, we should not obtain locks
|
||||||
opts.no_lock = true;
|
lookup_opts.no_lock = true;
|
||||||
let oi = self.get_object_info(bucket, object, &opts).await;
|
let oi = self.get_object_info(bucket, object, &lookup_opts).await;
|
||||||
|
|
||||||
match oi {
|
match oi {
|
||||||
Ok(oi) => {
|
Ok(oi) => {
|
||||||
// If top level is a delete marker proceed to upload.
|
// Ordinary writes may proceed past a top-level delete marker;
|
||||||
|
// data movement must not replace an acknowledged deletion.
|
||||||
if oi.delete_marker {
|
if oi.delete_marker {
|
||||||
return None;
|
return opts.data_movement.then_some(StorageError::PreconditionFailed);
|
||||||
}
|
}
|
||||||
let if_none_match = http_preconditions.if_none_match_value().map(str::to_owned);
|
let if_none_match = http_preconditions.if_none_match_value().map(str::to_owned);
|
||||||
let if_match = http_preconditions.if_match_value().map(str::to_owned);
|
let if_match = http_preconditions.if_match_value().map(str::to_owned);
|
||||||
if should_prevent_write(&oi, if_none_match, if_match) {
|
if should_prevent_write(&oi, if_none_match, if_match)
|
||||||
|
&& !crate::data_movement::can_replace_stale_data_movement_target(&oi, opts)
|
||||||
|
{
|
||||||
return Some(StorageError::PreconditionFailed);
|
return Some(StorageError::PreconditionFailed);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -6449,6 +6459,32 @@ mod tests {
|
|||||||
assert_eq!(versions.versions[0].name, object);
|
assert_eq!(versions.versions[0].name, object);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn load_file_info_versions_exact_encodes_directory_key_but_returns_logical_name() {
|
||||||
|
let bucket = "exact-directory-versions-bucket";
|
||||||
|
let object = "prefix/directory/";
|
||||||
|
let disk_object = rustfs_utils::path::encode_dir_object(object);
|
||||||
|
let (_dir, disk) = read_multiple_test_disk(bucket, &[]).await;
|
||||||
|
let mut fi = metadata_test_fileinfo(object);
|
||||||
|
fi.version_id = Some(Uuid::new_v4());
|
||||||
|
fi.mod_time = Some(OffsetDateTime::now_utc());
|
||||||
|
disk.write_metadata(bucket, bucket, disk_object.as_str(), fi.clone())
|
||||||
|
.await
|
||||||
|
.expect("directory metadata should be written under the encoded key");
|
||||||
|
let set = io_primitives_test_set(vec![Some(disk)], 0).await;
|
||||||
|
|
||||||
|
let versions = set
|
||||||
|
.load_file_info_versions_exact(bucket, object)
|
||||||
|
.await
|
||||||
|
.expect("exact directory version load should succeed")
|
||||||
|
.expect("exact directory version load should find metadata");
|
||||||
|
|
||||||
|
assert_eq!(versions.name, object);
|
||||||
|
assert_eq!(versions.versions.len(), 1);
|
||||||
|
assert_eq!(versions.versions[0].name, object);
|
||||||
|
assert_eq!(versions.versions[0].version_id, fi.version_id);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn load_file_info_versions_exact_rejects_transitioned_duplicate_parts() {
|
async fn load_file_info_versions_exact_rejects_transitioned_duplicate_parts() {
|
||||||
let bucket = "exact-versions-bucket";
|
let bucket = "exact-versions-bucket";
|
||||||
|
|||||||
@@ -85,6 +85,15 @@ impl SetDisks {
|
|||||||
format!("{}/{}", Self::get_multipart_sha_dir(bucket, object), upload_uuid)
|
format!("{}/{}", Self::get_multipart_sha_dir(bucket, object), upload_uuid)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(super) fn get_multipart_upload_dir(bucket: &str, object: &str, upload_id: &str, data_movement: bool) -> String {
|
||||||
|
let upload_dir = Self::get_upload_id_dir(bucket, object, upload_id);
|
||||||
|
if data_movement {
|
||||||
|
format!("{DATA_MOVEMENT_MULTIPART_PREFIX}/{upload_dir}")
|
||||||
|
} else {
|
||||||
|
upload_dir
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) fn get_multipart_sha_dir(bucket: &str, object: &str) -> String {
|
pub(super) fn get_multipart_sha_dir(bucket: &str, object: &str) -> String {
|
||||||
let path = format!("{bucket}/{object}");
|
let path = format!("{bucket}/{object}");
|
||||||
let mut hasher = Sha256::new();
|
let mut hasher = Sha256::new();
|
||||||
@@ -466,6 +475,28 @@ impl SetDisks {
|
|||||||
Self::find_file_info_in_quorum(metas, &mod_time, &etag, quorum)
|
Self::find_file_info_in_quorum(metas, &mod_time, &etag, quorum)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn hydrate_selected_fileinfo_part_checksums(fi: &mut FileInfo) -> disk::error::Result<()> {
|
||||||
|
fi.hydrate_data_movement_part_checksums().map_err(DiskError::from)?;
|
||||||
|
for part in &fi.parts {
|
||||||
|
let Some(checksums) = part.checksums.as_ref() else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let mut algorithms = HashSet::with_capacity(checksums.len());
|
||||||
|
for (name, value) in checksums {
|
||||||
|
let Some(checksum) = rustfs_rio::Checksum::new_from_string(name, value) else {
|
||||||
|
return Err(DiskError::FileCorrupt);
|
||||||
|
};
|
||||||
|
if checksum.checksum_type.is(rustfs_rio::ChecksumType::MULTIPART) {
|
||||||
|
return Err(DiskError::FileCorrupt);
|
||||||
|
}
|
||||||
|
if !algorithms.insert(checksum.checksum_type.base().0) {
|
||||||
|
return Err(DiskError::FileCorrupt);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn update_hash_bytes(hasher: &mut Sha256, value: &[u8]) {
|
fn update_hash_bytes(hasher: &mut Sha256, value: &[u8]) {
|
||||||
hasher.update(value.len().to_le_bytes());
|
hasher.update(value.len().to_le_bytes());
|
||||||
hasher.update(value);
|
hasher.update(value);
|
||||||
|
|||||||
@@ -288,6 +288,7 @@ pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024;
|
|||||||
pub const MAX_PARTS_COUNT: usize = 10000;
|
pub const MAX_PARTS_COUNT: usize = 10000;
|
||||||
pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket";
|
pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket";
|
||||||
pub(crate) const RUSTFS_MULTIPART_OBJECT_KEY: &str = "x-rustfs-internal-multipart-object";
|
pub(crate) const RUSTFS_MULTIPART_OBJECT_KEY: &str = "x-rustfs-internal-multipart-object";
|
||||||
|
pub(crate) const DATA_MOVEMENT_MULTIPART_PREFIX: &str = "data-movement";
|
||||||
const ENV_ISSUE3031_DIAG_ENABLE: &str = "RUSTFS_ISSUE3031_DIAG_ENABLE";
|
const ENV_ISSUE3031_DIAG_ENABLE: &str = "RUSTFS_ISSUE3031_DIAG_ENABLE";
|
||||||
|
|
||||||
/// Validate disk metadata at a boundary that may legitimately return a delete
|
/// Validate disk metadata at a boundary that may legitimately return a delete
|
||||||
@@ -3780,8 +3781,10 @@ fn resolve_delete_version_state(opts: &ObjectOptions, goi: &ObjectInfo, version_
|
|||||||
if opts.version_id.is_some() {
|
if opts.version_id.is_some() {
|
||||||
// Decommission/rebalance may recreate a delete marker on a new pool before that
|
// Decommission/rebalance may recreate a delete marker on a new pool before that
|
||||||
// exact version exists there, so we must still treat it as a mark-delete write.
|
// exact version exists there, so we must still treat it as a mark-delete write.
|
||||||
if opts.data_movement && opts.delete_marker && !version_found {
|
let data_movement_missing_delete_marker = opts.data_movement && opts.delete_marker && !version_found;
|
||||||
|
if data_movement_missing_delete_marker {
|
||||||
mark_delete = true;
|
mark_delete = true;
|
||||||
|
delete_marker = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
let delete_marker_version_purge = version_found && goi.delete_marker && !opts.version_purge_status().is_empty();
|
let delete_marker_version_purge = version_found && goi.delete_marker && !opts.version_purge_status().is_empty();
|
||||||
@@ -3790,7 +3793,10 @@ fn resolve_delete_version_state(opts: &ObjectOptions, goi: &ObjectInfo, version_
|
|||||||
mark_delete = false;
|
mark_delete = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if opts.version_purge_status().is_empty() && opts.delete_marker_replication_status().is_empty() {
|
if !data_movement_missing_delete_marker
|
||||||
|
&& opts.version_purge_status().is_empty()
|
||||||
|
&& opts.delete_marker_replication_status().is_empty()
|
||||||
|
{
|
||||||
mark_delete = false;
|
mark_delete = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3832,6 +3838,19 @@ impl SetDisks {
|
|||||||
opts: &ObjectOptions,
|
opts: &ObjectOptions,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let storage_class_config = self.storage_class_config_snapshot();
|
let storage_class_config = self.storage_class_config_snapshot();
|
||||||
|
let bucket_lifecycle_guard = if let Some(expected_incarnation_id) = opts.expected_bucket_incarnation_id
|
||||||
|
&& opts.bucket_lifecycle_lock_fence.is_none()
|
||||||
|
&& !crate::bucket::utils::is_meta_bucketname(bucket)
|
||||||
|
{
|
||||||
|
Some(
|
||||||
|
metadata_sys::object_store_in(&self.ctx)
|
||||||
|
.await?
|
||||||
|
.acquire_bucket_incarnation_fence(bucket, expected_incarnation_id)
|
||||||
|
.await?,
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
let _lock_guard = if !opts.no_lock {
|
let _lock_guard = if !opts.no_lock {
|
||||||
Some(
|
Some(
|
||||||
self.new_ns_lock(bucket, object)
|
self.new_ns_lock(bucket, object)
|
||||||
@@ -3844,6 +3863,12 @@ impl SetDisks {
|
|||||||
None
|
None
|
||||||
};
|
};
|
||||||
|
|
||||||
|
if opts.http_preconditions.is_some()
|
||||||
|
&& let Some(err) = self.check_write_precondition(bucket, object, opts).await
|
||||||
|
{
|
||||||
|
return Err(err);
|
||||||
|
}
|
||||||
|
|
||||||
let disks = self.disks.read().await.clone();
|
let disks = self.disks.read().await.clone();
|
||||||
let storage_class = opts.user_defined.get(AMZ_STORAGE_CLASS).map(String::as_str);
|
let storage_class = opts.user_defined.get(AMZ_STORAGE_CLASS).map(String::as_str);
|
||||||
let layout = resolve_write_layout(
|
let layout = resolve_write_layout(
|
||||||
@@ -3856,6 +3881,20 @@ impl SetDisks {
|
|||||||
)?;
|
)?;
|
||||||
let fi = build_tiered_decommission_file_info(bucket, object, fi, layout);
|
let fi = build_tiered_decommission_file_info(bucket, object, fi, layout);
|
||||||
let write_quorum = layout.write_quorum;
|
let write_quorum = layout.write_quorum;
|
||||||
|
if opts
|
||||||
|
.bucket_lifecycle_lock_fence
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||||
|
|| bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||||
|
{
|
||||||
|
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||||
|
mode: "decommission_tiered_object_commit",
|
||||||
|
bucket: bucket.to_string(),
|
||||||
|
object: object.to_string(),
|
||||||
|
required: 1,
|
||||||
|
achieved: 0,
|
||||||
|
});
|
||||||
|
}
|
||||||
let parts_metadata = vec![fi.clone(); disks.len()];
|
let parts_metadata = vec![fi.clone(); disks.len()];
|
||||||
let (shuffle_disks, parts_metadata) = Self::shuffle_disks_and_parts_metadata(&disks, &parts_metadata, &fi);
|
let (shuffle_disks, parts_metadata) = Self::shuffle_disks_and_parts_metadata(&disks, &parts_metadata, &fi);
|
||||||
|
|
||||||
@@ -5283,6 +5322,22 @@ mod tests {
|
|||||||
assert!(delete_marker);
|
assert!(delete_marker);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resolve_delete_version_state_creates_missing_suspended_data_movement_marker() {
|
||||||
|
let opts = ObjectOptions {
|
||||||
|
version_suspended: true,
|
||||||
|
version_id: Some(Uuid::nil().to_string()),
|
||||||
|
data_movement: true,
|
||||||
|
delete_marker: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let (mark_delete, delete_marker) = resolve_delete_version_state(&opts, &ObjectInfo::default(), false);
|
||||||
|
|
||||||
|
assert!(mark_delete);
|
||||||
|
assert!(delete_marker);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn should_force_delete_marker_for_missing_version_rejects_data_movement_latest_delete() {
|
fn should_force_delete_marker_for_missing_version_rejects_data_movement_latest_delete() {
|
||||||
let opts = ObjectOptions {
|
let opts = ObjectOptions {
|
||||||
@@ -6966,6 +7021,7 @@ mod tests {
|
|||||||
rustfs_filemeta::FileInfoOpts {
|
rustfs_filemeta::FileInfoOpts {
|
||||||
data: false,
|
data: false,
|
||||||
include_free_versions: false,
|
include_free_versions: false,
|
||||||
|
include_part_checksums: true,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.expect("test file metadata should decode as file info")
|
.expect("test file metadata should decode as file info")
|
||||||
@@ -7094,20 +7150,92 @@ mod tests {
|
|||||||
fn test_latest_fileinfo_selection_preserves_degraded_read_quorum_without_competing_latest() {
|
fn test_latest_fileinfo_selection_preserves_degraded_read_quorum_without_competing_latest() {
|
||||||
let mod_time = OffsetDateTime::now_utc();
|
let mod_time = OffsetDateTime::now_utc();
|
||||||
let data_dir = Uuid::new_v4();
|
let data_dir = Uuid::new_v4();
|
||||||
let metas = vec![
|
let mut first = quorum_test_fileinfo(mod_time, data_dir, "part-etag-old", 1);
|
||||||
quorum_test_fileinfo(mod_time, data_dir, "part-etag-old", 1),
|
rustfs_utils::http::insert_str(
|
||||||
quorum_test_fileinfo(mod_time, data_dir, "part-etag-old", 2),
|
&mut first.metadata,
|
||||||
FileInfo::default(),
|
rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
|
||||||
FileInfo::default(),
|
r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string(),
|
||||||
];
|
);
|
||||||
|
let mut second = first.clone();
|
||||||
|
second.erasure.index = 2;
|
||||||
|
let metas = vec![first, second, FileInfo::default(), FileInfo::default()];
|
||||||
let errs = vec![None, None, Some(DiskError::DiskNotFound), Some(DiskError::DiskNotFound)];
|
let errs = vec![None, None, Some(DiskError::DiskNotFound), Some(DiskError::DiskNotFound)];
|
||||||
|
|
||||||
let (_, selected, selected_quorum) = SetDisks::select_valid_fileinfo(&vec![None; metas.len()], &metas, &errs, "", 2, 3)
|
let (_, mut selected, selected_quorum) =
|
||||||
.expect("read quorum should remain enough when no competing latest is visible");
|
SetDisks::select_valid_fileinfo(&vec![None; metas.len()], &metas, &errs, "", 2, 3)
|
||||||
|
.expect("read quorum should remain enough when no competing latest is visible");
|
||||||
|
|
||||||
assert_eq!(selected_quorum, 2);
|
assert_eq!(selected_quorum, 2);
|
||||||
assert_eq!(selected.data_dir, Some(data_dir));
|
assert_eq!(selected.data_dir, Some(data_dir));
|
||||||
assert_eq!(selected.parts[0].etag, "part-etag-old");
|
assert_eq!(selected.parts[0].etag, "part-etag-old");
|
||||||
|
assert!(selected.parts[0].checksums.is_none());
|
||||||
|
SetDisks::hydrate_selected_fileinfo_part_checksums(&mut selected)
|
||||||
|
.expect("requested part checksums should hydrate after winner selection");
|
||||||
|
assert_eq!(
|
||||||
|
selected.parts[0]
|
||||||
|
.checksums
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|checksums| checksums.get("CRC32C"))
|
||||||
|
.map(String::as_str),
|
||||||
|
Some("AAAAAA==")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_degraded_fileinfo_selection_rejects_malformed_part_checksum_metadata() {
|
||||||
|
let mod_time = OffsetDateTime::now_utc();
|
||||||
|
let data_dir = Uuid::new_v4();
|
||||||
|
let mut first = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 1);
|
||||||
|
rustfs_utils::http::insert_str(&mut first.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS, "not-json".to_string());
|
||||||
|
let mut second = first.clone();
|
||||||
|
second.erasure.index = 2;
|
||||||
|
let metas = vec![first, second, FileInfo::default(), FileInfo::default()];
|
||||||
|
let errs = vec![None, None, Some(DiskError::DiskNotFound), Some(DiskError::DiskNotFound)];
|
||||||
|
|
||||||
|
let (_, mut selected, _) = SetDisks::select_valid_fileinfo(&vec![None; metas.len()], &metas, &errs, "", 2, 3)
|
||||||
|
.expect("winner selection should defer sidecar decoding");
|
||||||
|
let err = SetDisks::hydrate_selected_fileinfo_part_checksums(&mut selected)
|
||||||
|
.expect_err("a malformed degraded winner must fail closed when checksums are requested");
|
||||||
|
|
||||||
|
assert_eq!(err, DiskError::FileCorrupt);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_pick_valid_fileinfo_rejects_malformed_part_checksum_metadata() {
|
||||||
|
let mod_time = OffsetDateTime::now_utc();
|
||||||
|
let data_dir = Uuid::new_v4();
|
||||||
|
let mut meta = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 1);
|
||||||
|
rustfs_utils::http::insert_str(&mut meta.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS, "not-json".to_string());
|
||||||
|
let mut second = meta.clone();
|
||||||
|
second.erasure.index = 2;
|
||||||
|
|
||||||
|
let mut selected = SetDisks::pick_valid_fileinfo(&[meta, second], Some(mod_time), None, 2)
|
||||||
|
.expect("winner selection should defer sidecar decoding");
|
||||||
|
let err = SetDisks::hydrate_selected_fileinfo_part_checksums(&mut selected)
|
||||||
|
.expect_err("a malformed winning part-checksum sidecar must fail closed when checksums are requested");
|
||||||
|
|
||||||
|
assert_eq!(err, DiskError::FileCorrupt);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_part_checksum_hydration_rejects_invalid_algorithm_and_value() {
|
||||||
|
let mod_time = OffsetDateTime::now_utc();
|
||||||
|
let data_dir = Uuid::new_v4();
|
||||||
|
for encoded in [
|
||||||
|
r#"[[1,[["UNKNOWN","AAAAAA=="]]]]"#,
|
||||||
|
r#"[[1,[["CRC32C","not-base64"]]]]"#,
|
||||||
|
r#"[[1,[["CRC32C","AA=="]]]]"#,
|
||||||
|
r#"[[1,[["CRC32C","AAAAAA==-0"]]]]"#,
|
||||||
|
r#"[[1,[["CRC32C","AAAAAA==-1"]]]]"#,
|
||||||
|
r#"[[1,[["CRC32C","AAAAAA=="],["crc32c","BBBBBB=="]]]]"#,
|
||||||
|
] {
|
||||||
|
let mut meta = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 1);
|
||||||
|
rustfs_utils::http::insert_str(&mut meta.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS, encoded.to_string());
|
||||||
|
|
||||||
|
let err = SetDisks::hydrate_selected_fileinfo_part_checksums(&mut meta)
|
||||||
|
.expect_err("invalid persisted part checksum metadata must fail closed");
|
||||||
|
assert_eq!(err, DiskError::FileCorrupt);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -10744,6 +10872,20 @@ mod tests {
|
|||||||
|
|
||||||
assert!(marker.delete_marker);
|
assert!(marker.delete_marker);
|
||||||
let marker_version = marker.version_id.expect("versioned delete marker should carry a version id");
|
let marker_version = marker.version_id.expect("versioned delete marker should carry a version id");
|
||||||
|
let create_only = ObjectOptions {
|
||||||
|
versioned: true,
|
||||||
|
data_movement: true,
|
||||||
|
http_preconditions: Some(HTTPPreconditions {
|
||||||
|
if_none_match: Some("*".to_string()),
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
set_disks.check_write_precondition(bucket, object, &create_only).await,
|
||||||
|
Some(StorageError::PreconditionFailed),
|
||||||
|
"data movement must not replace a target delete marker"
|
||||||
|
);
|
||||||
let err = match set_disks
|
let err = match set_disks
|
||||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
|
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
|
||||||
.await
|
.await
|
||||||
|
|||||||
@@ -542,7 +542,8 @@ impl SetDisks {
|
|||||||
|
|
||||||
let filter_by_etag = quorum_etag.is_some();
|
let filter_by_etag = quorum_etag.is_some();
|
||||||
match Self::pick_valid_fileinfo(&parts_metadata, quorum_mod_time, quorum_etag.clone(), read_quorum as usize) {
|
match Self::pick_valid_fileinfo(&parts_metadata, quorum_mod_time, quorum_etag.clone(), read_quorum as usize) {
|
||||||
Ok(latest_meta) => {
|
Ok(mut latest_meta) => {
|
||||||
|
Self::hydrate_selected_fileinfo_part_checksums(&mut latest_meta)?;
|
||||||
trace!(
|
trace!(
|
||||||
event = EVENT_SET_DISK_HEAL,
|
event = EVENT_SET_DISK_HEAL,
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
|||||||
@@ -33,6 +33,29 @@ use tokio::task::JoinSet;
|
|||||||
|
|
||||||
const MULTIPART_LIST_IO_CONCURRENCY: usize = 16;
|
const MULTIPART_LIST_IO_CONCURRENCY: usize = 16;
|
||||||
|
|
||||||
|
pub(crate) struct StaleMultipartCleanupGuard {
|
||||||
|
file_info: FileInfo,
|
||||||
|
upload_path: String,
|
||||||
|
write_quorum: usize,
|
||||||
|
lock_guard: ObjectLockDiagGuard,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StaleMultipartCleanupGuard {
|
||||||
|
pub(crate) fn file_info(&self) -> &FileInfo {
|
||||||
|
&self.file_info
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn is_lock_lost(&self) -> bool {
|
||||||
|
self.lock_guard.is_lock_lost()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn delete(self, set: &SetDisks) -> Result<()> {
|
||||||
|
fence_commit_on_lock_loss(Some(&self.lock_guard), "stale_multipart_cleanup", &self.upload_path)?;
|
||||||
|
set.delete_all_with_quorum(RUSTFS_META_MULTIPART_BUCKET, &self.upload_path, self.write_quorum)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||||
pub(crate) enum MultipartCommitPause {
|
pub(crate) enum MultipartCommitPause {
|
||||||
@@ -223,6 +246,20 @@ fn validate_multipart_bucket_incarnation(
|
|||||||
Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()))
|
Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn ensure_data_movement_upload_access(
|
||||||
|
fi: &FileInfo,
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
upload_id: &str,
|
||||||
|
opts: &ObjectOptions,
|
||||||
|
) -> Result<()> {
|
||||||
|
if rustfs_utils::http::contains_key_str(&fi.metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD) && !opts.data_movement
|
||||||
|
{
|
||||||
|
return Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
async fn ensure_multipart_bucket_incarnation(
|
async fn ensure_multipart_bucket_incarnation(
|
||||||
ctx: &crate::runtime::instance::InstanceContext,
|
ctx: &crate::runtime::instance::InstanceContext,
|
||||||
fi: &FileInfo,
|
fi: &FileInfo,
|
||||||
@@ -445,7 +482,7 @@ impl SetDisks {
|
|||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
|
|
||||||
let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id);
|
let upload_id_path = Self::get_multipart_upload_dir(bucket, object, upload_id, opts.data_movement);
|
||||||
self.acquire_read_lock_diag(op, RUSTFS_META_MULTIPART_BUCKET, &upload_id_path)
|
self.acquire_read_lock_diag(op, RUSTFS_META_MULTIPART_BUCKET, &upload_id_path)
|
||||||
.await
|
.await
|
||||||
.map(Some)
|
.map(Some)
|
||||||
@@ -463,7 +500,7 @@ impl SetDisks {
|
|||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
|
|
||||||
let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id);
|
let upload_id_path = Self::get_multipart_upload_dir(bucket, object, upload_id, opts.data_movement);
|
||||||
self.acquire_write_lock_diag(op, RUSTFS_META_MULTIPART_BUCKET, &upload_id_path)
|
self.acquire_write_lock_diag(op, RUSTFS_META_MULTIPART_BUCKET, &upload_id_path)
|
||||||
.await
|
.await
|
||||||
.map(Some)
|
.map(Some)
|
||||||
@@ -511,13 +548,49 @@ impl SetDisks {
|
|||||||
upload_id: &str,
|
upload_id: &str,
|
||||||
write: bool,
|
write: bool,
|
||||||
) -> Result<(FileInfo, Vec<FileInfo>)> {
|
) -> Result<(FileInfo, Vec<FileInfo>)> {
|
||||||
let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id);
|
self.check_upload_id_exists_for_data_movement(bucket, object, upload_id, write, false)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn check_upload_id_exists_with_opts(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
upload_id: &str,
|
||||||
|
write: bool,
|
||||||
|
opts: &ObjectOptions,
|
||||||
|
) -> Result<(FileInfo, Vec<FileInfo>)> {
|
||||||
|
self.check_upload_id_exists_for_data_movement(bucket, object, upload_id, write, opts.data_movement)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn check_upload_id_exists_for_data_movement(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
upload_id: &str,
|
||||||
|
write: bool,
|
||||||
|
data_movement: bool,
|
||||||
|
) -> Result<(FileInfo, Vec<FileInfo>)> {
|
||||||
|
let upload_id_path = Self::get_multipart_upload_dir(bucket, object, upload_id, data_movement);
|
||||||
|
self.check_multipart_upload_path_exists(bucket, object, upload_id, &upload_id_path, write)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn check_multipart_upload_path_exists(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
upload_id: &str,
|
||||||
|
upload_id_path: &str,
|
||||||
|
write: bool,
|
||||||
|
) -> Result<(FileInfo, Vec<FileInfo>)> {
|
||||||
let disks = self.disks.read().await;
|
let disks = self.disks.read().await;
|
||||||
|
|
||||||
let disks = disks.clone();
|
let disks = disks.clone();
|
||||||
|
|
||||||
let (parts_metadata, errs) =
|
let (parts_metadata, errs) =
|
||||||
Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_id_path, "", false, false, false)
|
Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, upload_id_path, "", false, false, false)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
let (read_quorum, write_quorum) = Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count)
|
let (read_quorum, write_quorum) = Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count)
|
||||||
@@ -562,6 +635,20 @@ impl SetDisks {
|
|||||||
Ok((fi, parts_metadata))
|
Ok((fi, parts_metadata))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn lock_stale_multipart_cleanup(&self, upload_path: &str) -> Result<StaleMultipartCleanupGuard> {
|
||||||
|
let lock_guard = self
|
||||||
|
.acquire_write_lock_diag("stale_multipart_cleanup", RUSTFS_META_MULTIPART_BUCKET, upload_path)
|
||||||
|
.await?;
|
||||||
|
let (file_info, _) = self.check_multipart_upload_path_exists("", "", "", upload_path, true).await?;
|
||||||
|
let write_quorum = file_info.write_quorum(self.default_write_quorum());
|
||||||
|
Ok(StaleMultipartCleanupGuard {
|
||||||
|
file_info,
|
||||||
|
upload_path: upload_path.to_string(),
|
||||||
|
write_quorum,
|
||||||
|
lock_guard,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
pub(crate) async fn list_multipart_uploads_for_incarnation(
|
pub(crate) async fn list_multipart_uploads_for_incarnation(
|
||||||
&self,
|
&self,
|
||||||
@@ -704,6 +791,12 @@ impl SetDisks {
|
|||||||
{
|
{
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
|
if rustfs_utils::http::contains_key_str(
|
||||||
|
&file_info.metadata,
|
||||||
|
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
|
||||||
|
) {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
|
||||||
let object = match (
|
let object = match (
|
||||||
file_info.metadata.get(RUSTFS_MULTIPART_BUCKET_KEY),
|
file_info.metadata.get(RUSTFS_MULTIPART_BUCKET_KEY),
|
||||||
@@ -852,9 +945,12 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
opts: &ObjectOptions,
|
opts: &ObjectOptions,
|
||||||
) -> Result<PartInfo> {
|
) -> Result<PartInfo> {
|
||||||
crate::hp_guard!("SetDisks::put_object_part");
|
crate::hp_guard!("SetDisks::put_object_part");
|
||||||
let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id);
|
let upload_id_path = Self::get_multipart_upload_dir(bucket, object, upload_id, opts.data_movement);
|
||||||
|
|
||||||
let (fi, _) = self.check_upload_id_exists(bucket, object, upload_id, true).await?;
|
let (fi, _) = self
|
||||||
|
.check_upload_id_exists_with_opts(bucket, object, upload_id, true, opts)
|
||||||
|
.await?;
|
||||||
|
ensure_data_movement_upload_access(&fi, bucket, object, upload_id, opts)?;
|
||||||
ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
|
ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
@@ -1120,7 +1216,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
.await?;
|
.await?;
|
||||||
(Some(upload_guard), Some(part_guard))
|
(Some(upload_guard), Some(part_guard))
|
||||||
};
|
};
|
||||||
let (commit_fi, _) = self.check_upload_id_exists(bucket, object, upload_id, false).await?;
|
let (commit_fi, _) = self
|
||||||
|
.check_upload_id_exists_with_opts(bucket, object, upload_id, false, opts)
|
||||||
|
.await?;
|
||||||
|
ensure_data_movement_upload_access(&commit_fi, bucket, object, upload_id, opts)?;
|
||||||
ensure_multipart_bucket_incarnation(
|
ensure_multipart_bucket_incarnation(
|
||||||
&self.ctx,
|
&self.ctx,
|
||||||
&commit_fi,
|
&commit_fi,
|
||||||
@@ -1196,11 +1295,14 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
let _upload_guard = self
|
let _upload_guard = self
|
||||||
.acquire_multipart_upload_read_lock("list_object_parts", bucket, object, upload_id, opts)
|
.acquire_multipart_upload_read_lock("list_object_parts", bucket, object, upload_id, opts)
|
||||||
.await?;
|
.await?;
|
||||||
let (fi, _) = self.check_upload_id_exists(bucket, object, upload_id, false).await?;
|
let (fi, _) = self
|
||||||
|
.check_upload_id_exists_with_opts(bucket, object, upload_id, false, opts)
|
||||||
|
.await?;
|
||||||
|
ensure_data_movement_upload_access(&fi, bucket, object, upload_id, opts)?;
|
||||||
ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
|
ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id);
|
let upload_id_path = Self::get_multipart_upload_dir(bucket, object, upload_id, opts.data_movement);
|
||||||
|
|
||||||
if max_parts > MAX_PARTS_COUNT {
|
if max_parts > MAX_PARTS_COUNT {
|
||||||
max_parts = MAX_PARTS_COUNT;
|
max_parts = MAX_PARTS_COUNT;
|
||||||
@@ -1384,6 +1486,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
let disks = disks.clone();
|
let disks = disks.clone();
|
||||||
|
|
||||||
let mut user_defined = opts.user_defined.clone();
|
let mut user_defined = opts.user_defined.clone();
|
||||||
|
rustfs_utils::http::remove_str(&mut user_defined, rustfs_utils::http::SUFFIX_PART_CHECKSUMS);
|
||||||
|
if !opts.data_movement {
|
||||||
|
rustfs_utils::http::remove_str(&mut user_defined, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD);
|
||||||
|
}
|
||||||
rustfs_utils::http::metadata_compat::remove_str(
|
rustfs_utils::http::metadata_compat::remove_str(
|
||||||
&mut user_defined,
|
&mut user_defined,
|
||||||
crate::object_api::ENCRYPTED_PART_LAYOUT_QUORUM_SUFFIX,
|
crate::object_api::ENCRYPTED_PART_LAYOUT_QUORUM_SUFFIX,
|
||||||
@@ -1489,7 +1595,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
|
|
||||||
let upload_id = runtime_sources::deployment_upload_id(&upload_uuid);
|
let upload_id = runtime_sources::deployment_upload_id(&upload_uuid);
|
||||||
|
|
||||||
let upload_path = Self::get_upload_id_dir(bucket, object, upload_uuid.as_str());
|
let upload_path = Self::get_multipart_upload_dir(bucket, object, upload_uuid.as_str(), opts.data_movement);
|
||||||
|
|
||||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||||
Self::write_unique_file_info(
|
Self::write_unique_file_info(
|
||||||
@@ -1524,9 +1630,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
.acquire_multipart_upload_read_lock("get_multipart_info", bucket, object, upload_id, opts)
|
.acquire_multipart_upload_read_lock("get_multipart_info", bucket, object, upload_id, opts)
|
||||||
.await?;
|
.await?;
|
||||||
let (mut fi, _) = self
|
let (mut fi, _) = self
|
||||||
.check_upload_id_exists(bucket, object, upload_id, false)
|
.check_upload_id_exists_with_opts(bucket, object, upload_id, false, opts)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| to_object_err(e, vec![bucket, object, upload_id]))?;
|
.map_err(|e| to_object_err(e, vec![bucket, object, upload_id]))?;
|
||||||
|
ensure_data_movement_upload_access(&fi, bucket, object, upload_id, opts)?;
|
||||||
ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
|
ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
|
||||||
.await?;
|
.await?;
|
||||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||||
@@ -1548,11 +1655,14 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
let _upload_guard = self
|
let _upload_guard = self
|
||||||
.acquire_multipart_upload_write_lock("abort_multipart_upload", bucket, object, upload_id, opts)
|
.acquire_multipart_upload_write_lock("abort_multipart_upload", bucket, object, upload_id, opts)
|
||||||
.await?;
|
.await?;
|
||||||
let (fi, _) = self.check_upload_id_exists(bucket, object, upload_id, true).await?;
|
let (fi, _) = self
|
||||||
|
.check_upload_id_exists_with_opts(bucket, object, upload_id, true, opts)
|
||||||
|
.await?;
|
||||||
|
ensure_data_movement_upload_access(&fi, bucket, object, upload_id, opts)?;
|
||||||
ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
|
ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
|
||||||
.await?;
|
.await?;
|
||||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||||
let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id);
|
let upload_id_path = Self::get_multipart_upload_dir(bucket, object, upload_id, opts.data_movement);
|
||||||
|
|
||||||
self.delete_all_with_quorum(
|
self.delete_all_with_quorum(
|
||||||
RUSTFS_META_MULTIPART_BUCKET,
|
RUSTFS_META_MULTIPART_BUCKET,
|
||||||
@@ -1574,7 +1684,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
crate::hp_guard!("SetDisks::complete_multipart_upload");
|
crate::hp_guard!("SetDisks::complete_multipart_upload");
|
||||||
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||||
|
|
||||||
let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id);
|
let upload_id_path = Self::get_multipart_upload_dir(bucket, object, upload_id, opts.data_movement);
|
||||||
let range_seek_rollout_enabled = crate::object_api::legacy_encrypted_range_seek_enabled() && !opts.no_lock;
|
let range_seek_rollout_enabled = crate::object_api::legacy_encrypted_range_seek_enabled() && !opts.no_lock;
|
||||||
let mut object_lock_guard = None;
|
let mut object_lock_guard = None;
|
||||||
|
|
||||||
@@ -1602,7 +1712,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
let expected_restore_operation_id = restore_commit_operation_id_from_metadata(&opts.user_defined)?;
|
let expected_restore_operation_id = restore_commit_operation_id_from_metadata(&opts.user_defined)?;
|
||||||
let (mut fi, files_metas) = self.check_upload_id_exists(bucket, object, upload_id, true).await?;
|
let (mut fi, files_metas) = self
|
||||||
|
.check_upload_id_exists_with_opts(bucket, object, upload_id, true, opts)
|
||||||
|
.await?;
|
||||||
|
ensure_data_movement_upload_access(&fi, bucket, object, upload_id, opts)?;
|
||||||
ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
|
ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
|
||||||
.await?;
|
.await?;
|
||||||
let has_layout_candidate = range_seek_rollout_enabled
|
let has_layout_candidate = range_seek_rollout_enabled
|
||||||
@@ -1771,7 +1884,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
|
|
||||||
// Build a lookup map for O(1) part resolution instead of O(n) find() in the loop
|
// Build a lookup map for O(1) part resolution instead of O(n) find() in the loop
|
||||||
// This optimizes from O(n^2) to O(n) when processing many parts
|
// This optimizes from O(n^2) to O(n) when processing many parts
|
||||||
use std::collections::HashMap;
|
use std::collections::{HashMap, HashSet};
|
||||||
let part_lookup: HashMap<usize, &ObjectPartInfo> = curr_fi.parts.iter().map(|part| (part.number, part)).collect();
|
let part_lookup: HashMap<usize, &ObjectPartInfo> = curr_fi.parts.iter().map(|part| (part.number, part)).collect();
|
||||||
|
|
||||||
for (i, p) in uploaded_parts.iter().enumerate() {
|
for (i, p) in uploaded_parts.iter().enumerate() {
|
||||||
@@ -1807,7 +1920,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
|
|
||||||
// TODO: crypto
|
// TODO: crypto
|
||||||
|
|
||||||
if (i < uploaded_parts.len() - 1) && !is_min_allowed_part_size(ext_part.actual_size) {
|
if (i < uploaded_parts.len() - 1)
|
||||||
|
&& !(opts.data_movement && ext_part.actual_size < 0)
|
||||||
|
&& !is_min_allowed_part_size(ext_part.actual_size)
|
||||||
|
{
|
||||||
error!(
|
error!(
|
||||||
"complete_multipart_upload part size too small: part {} size {} is less than minimum {}",
|
"complete_multipart_upload part size too small: part {} size {} is less than minimum {}",
|
||||||
p.part_num,
|
p.part_num,
|
||||||
@@ -1978,7 +2094,27 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
if opts.replication_request {
|
let data_movement_actual_size = if opts.data_movement {
|
||||||
|
rustfs_utils::http::get_consistent_str(&opts.user_defined, SUFFIX_ACTUAL_SIZE)
|
||||||
|
.map(|value| {
|
||||||
|
value
|
||||||
|
.parse::<i64>()
|
||||||
|
.ok()
|
||||||
|
.filter(|value| *value >= 0)
|
||||||
|
.ok_or_else(|| Error::other("data movement actual size metadata is invalid"))
|
||||||
|
})
|
||||||
|
.transpose()?
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
|
if let Some(actual_size) = data_movement_actual_size {
|
||||||
|
insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string());
|
||||||
|
if persist_encryption_original_size {
|
||||||
|
fi.metadata
|
||||||
|
.insert("x-rustfs-encryption-original-size".to_string(), actual_size.to_string());
|
||||||
|
}
|
||||||
|
} else if opts.replication_request {
|
||||||
if let Some(actual_size) = get_str(&opts.user_defined, SUFFIX_ACTUAL_OBJECT_SIZE_CAP) {
|
if let Some(actual_size) = get_str(&opts.user_defined, SUFFIX_ACTUAL_OBJECT_SIZE_CAP) {
|
||||||
insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.clone());
|
insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.clone());
|
||||||
if persist_encryption_original_size {
|
if persist_encryption_original_size {
|
||||||
@@ -1998,7 +2134,15 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
insert_str(&mut fi.metadata, SUFFIX_COMPRESSION_SIZE, object_size.to_string());
|
insert_str(&mut fi.metadata, SUFFIX_COMPRESSION_SIZE, object_size.to_string());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
rustfs_utils::http::remove_str(&mut fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS);
|
||||||
|
if let Some(part_checksums) =
|
||||||
|
rustfs_utils::http::get_consistent_str(&opts.user_defined, rustfs_utils::http::SUFFIX_PART_CHECKSUMS)
|
||||||
|
{
|
||||||
|
insert_str(&mut fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS, part_checksums.to_string());
|
||||||
|
}
|
||||||
|
|
||||||
if opts.data_movement {
|
if opts.data_movement {
|
||||||
|
rustfs_utils::http::remove_str(&mut fi.metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD);
|
||||||
fi.set_data_moved();
|
fi.set_data_moved();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2014,19 +2158,21 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let completed_part_numbers = fi.parts.iter().map(|part| part.number).collect::<HashSet<_>>();
|
||||||
|
let upload_data_dir = curr_fi.data_dir.unwrap_or(Uuid::nil()).to_string();
|
||||||
let mut parts = Vec::with_capacity(curr_fi.parts.len());
|
let mut parts = Vec::with_capacity(curr_fi.parts.len());
|
||||||
|
|
||||||
for p in curr_fi.parts.iter() {
|
for p in curr_fi.parts.iter() {
|
||||||
parts.push(path_join_buf(&[
|
parts.push(path_join_buf(&[
|
||||||
&upload_id_path,
|
&upload_id_path,
|
||||||
curr_fi.data_dir.unwrap_or(Uuid::nil()).to_string().as_str(),
|
upload_data_dir.as_str(),
|
||||||
format!("part.{}.meta", p.number).as_str(),
|
format!("part.{}.meta", p.number).as_str(),
|
||||||
]));
|
]));
|
||||||
|
|
||||||
if !fi.parts.iter().any(|v| v.number == p.number) {
|
if !completed_part_numbers.contains(&p.number) {
|
||||||
parts.push(path_join_buf(&[
|
parts.push(path_join_buf(&[
|
||||||
&upload_id_path,
|
&upload_id_path,
|
||||||
curr_fi.data_dir.unwrap_or(Uuid::nil()).to_string().as_str(),
|
upload_data_dir.as_str(),
|
||||||
format!("part.{}", p.number).as_str(),
|
format!("part.{}", p.number).as_str(),
|
||||||
]));
|
]));
|
||||||
}
|
}
|
||||||
@@ -2049,6 +2195,19 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
achieved: 0,
|
achieved: 0,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
if opts
|
||||||
|
.namespace_lock_fence
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||||
|
{
|
||||||
|
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||||
|
mode: "complete_multipart_upload_outer_lock",
|
||||||
|
bucket: bucket.to_string(),
|
||||||
|
object: object.to_string(),
|
||||||
|
required: 1,
|
||||||
|
achieved: 0,
|
||||||
|
});
|
||||||
|
}
|
||||||
if upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
if upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||||
mode: "complete_multipart_upload_commit",
|
mode: "complete_multipart_upload_commit",
|
||||||
@@ -2069,6 +2228,74 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
|
if opts.data_movement
|
||||||
|
&& opts.http_preconditions.is_some()
|
||||||
|
&& let Some(err) = self.check_write_precondition(bucket, object, opts).await
|
||||||
|
{
|
||||||
|
return Err(err);
|
||||||
|
}
|
||||||
|
if opts.data_movement && opts.http_preconditions.is_some() && !crate::bucket::utils::is_meta_bucketname(bucket) {
|
||||||
|
let current = self
|
||||||
|
.get_object_info(
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
&ObjectOptions {
|
||||||
|
version_id: opts.version_id.clone(),
|
||||||
|
no_lock: true,
|
||||||
|
metadata_cache_safe: false,
|
||||||
|
versioned: opts.versioned,
|
||||||
|
version_suspended: opts.version_suspended,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
match current {
|
||||||
|
Ok(existing) if crate::data_movement::can_replace_stale_data_movement_target(&existing, opts) => {
|
||||||
|
let object_lock_config = opts.object_lock_config_snapshot.as_deref().ok_or_else(|| {
|
||||||
|
Error::other("data movement completion is missing its Object Lock configuration snapshot")
|
||||||
|
})?;
|
||||||
|
if check_object_lock_for_deletion_with_state(object_lock_config.state(), &existing, false)?.is_some() {
|
||||||
|
return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object.to_string()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(_) => return Err(StorageError::PreconditionFailed),
|
||||||
|
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
|
||||||
|
Err(err) => return Err(err),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||||
|
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||||
|
mode: "complete_multipart_upload_commit",
|
||||||
|
bucket: bucket.to_string(),
|
||||||
|
object: object.to_string(),
|
||||||
|
required: 1,
|
||||||
|
achieved: 0,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if opts
|
||||||
|
.namespace_lock_fence
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||||
|
{
|
||||||
|
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||||
|
mode: "complete_multipart_upload_outer_lock",
|
||||||
|
bucket: bucket.to_string(),
|
||||||
|
object: object.to_string(),
|
||||||
|
required: 1,
|
||||||
|
achieved: 0,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||||
|
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||||
|
mode: "complete_multipart_upload_commit",
|
||||||
|
bucket: RUSTFS_META_MULTIPART_BUCKET.to_string(),
|
||||||
|
object: upload_id_path.clone(),
|
||||||
|
required: 1,
|
||||||
|
achieved: 0,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||||
|
|
||||||
let complete_tail_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
|
let complete_tail_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
|
||||||
|
|
||||||
// Crash-consistency injection: hard power loss after the upload is fully
|
// Crash-consistency injection: hard power loss after the upload is fully
|
||||||
@@ -2589,8 +2816,29 @@ mod tests {
|
|||||||
.new_multipart_upload(bucket, object, create_opts)
|
.new_multipart_upload(bucket, object, create_opts)
|
||||||
.await
|
.await
|
||||||
.expect("multipart upload should be created");
|
.expect("multipart upload should be created");
|
||||||
let part = put_test_part(set_disks, bucket, object, &upload.upload_id, 1, content, content.len() as i64).await;
|
let mut reader = PutObjReader::new(
|
||||||
(upload.upload_id, vec![part])
|
HashReader::from_stream(
|
||||||
|
Cursor::new(content.to_vec()),
|
||||||
|
content.len() as i64,
|
||||||
|
content.len() as i64,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.expect("hash reader should be constructed"),
|
||||||
|
);
|
||||||
|
let part = set_disks
|
||||||
|
.put_object_part(bucket, object, &upload.upload_id, 1, &mut reader, create_opts)
|
||||||
|
.await
|
||||||
|
.expect("uploading the part should succeed");
|
||||||
|
(
|
||||||
|
upload.upload_id,
|
||||||
|
vec![CompletePart {
|
||||||
|
part_num: part.part_num,
|
||||||
|
etag: part.etag,
|
||||||
|
..Default::default()
|
||||||
|
}],
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn put_test_part(
|
async fn put_test_part(
|
||||||
@@ -2742,6 +2990,353 @@ mod tests {
|
|||||||
assert!(matches!(overflow_err, StorageError::PartMissingOrCorrupt));
|
assert!(matches!(overflow_err, StorageError::PartMissingOrCorrupt));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn data_movement_upload_is_hidden_from_external_multipart_operations() {
|
||||||
|
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||||
|
let bucket = "data-movement-upload-complete-bucket";
|
||||||
|
make_bucket_on_all(&disk_stores, bucket).await;
|
||||||
|
let mut durable_metadata = HashMap::new();
|
||||||
|
rustfs_utils::http::insert_str(&mut durable_metadata, rustfs_utils::http::SUFFIX_DATA_MOVED, "true".to_string());
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
&mut durable_metadata,
|
||||||
|
rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
|
||||||
|
r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string(),
|
||||||
|
);
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
&mut durable_metadata,
|
||||||
|
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
|
||||||
|
"residual-final-marker".to_string(),
|
||||||
|
);
|
||||||
|
let ordinary_opts = ObjectOptions {
|
||||||
|
user_defined: durable_metadata,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let (upload_id, parts) =
|
||||||
|
stage_upload_with_create_opts(&set_disks, bucket, "ordinary-object", b"ordinary multipart body", &ordinary_opts)
|
||||||
|
.await;
|
||||||
|
set_disks
|
||||||
|
.clone()
|
||||||
|
.complete_multipart_upload(bucket, "ordinary-object", &upload_id, parts, &ordinary_opts)
|
||||||
|
.await
|
||||||
|
.expect("durable object metadata must not claim upload ownership");
|
||||||
|
let completed = set_disks
|
||||||
|
.get_object_info(
|
||||||
|
bucket,
|
||||||
|
"ordinary-object",
|
||||||
|
&ObjectOptions {
|
||||||
|
include_part_checksums: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("completed staging checksum metadata should remain readable");
|
||||||
|
assert_eq!(
|
||||||
|
completed.parts[0]
|
||||||
|
.checksums
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|checksums| checksums.get("CRC32C"))
|
||||||
|
.map(String::as_str),
|
||||||
|
Some("AAAAAA==")
|
||||||
|
);
|
||||||
|
|
||||||
|
let object = "owned-object";
|
||||||
|
let mut metadata = HashMap::new();
|
||||||
|
rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD, "true".to_string());
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
&mut metadata,
|
||||||
|
rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
|
||||||
|
r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string(),
|
||||||
|
);
|
||||||
|
let create_opts = ObjectOptions {
|
||||||
|
data_movement: true,
|
||||||
|
user_defined: metadata,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let upload = set_disks
|
||||||
|
.new_multipart_upload(bucket, object, &create_opts)
|
||||||
|
.await
|
||||||
|
.expect("data movement upload should be created");
|
||||||
|
let upload_id = upload.upload_id;
|
||||||
|
let ordinary_upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||||
|
let data_movement_upload_path = SetDisks::get_multipart_upload_dir(bucket, object, &upload_id, true);
|
||||||
|
assert_eq!(data_movement_upload_path, format!("data-movement/{ordinary_upload_path}"));
|
||||||
|
assert!(matches!(
|
||||||
|
set_disks.check_upload_id_exists(bucket, object, &upload_id, false).await,
|
||||||
|
Err(StorageError::InvalidUploadID(..))
|
||||||
|
));
|
||||||
|
set_disks
|
||||||
|
.check_upload_id_exists_with_opts(bucket, object, &upload_id, false, &create_opts)
|
||||||
|
.await
|
||||||
|
.expect("data movement lookup should find the isolated upload");
|
||||||
|
let mut part_reader = PutObjReader::from_vec(b"data movement multipart body".to_vec());
|
||||||
|
let uploaded_part = set_disks
|
||||||
|
.put_object_part(bucket, object, &upload_id, 1, &mut part_reader, &create_opts)
|
||||||
|
.await
|
||||||
|
.expect("data movement part upload should retain ownership of its upload");
|
||||||
|
let parts = vec![CompletePart {
|
||||||
|
part_num: uploaded_part.part_num,
|
||||||
|
etag: uploaded_part.etag,
|
||||||
|
..Default::default()
|
||||||
|
}];
|
||||||
|
|
||||||
|
let listed = set_disks
|
||||||
|
.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||||
|
.await
|
||||||
|
.expect("external multipart listing should succeed");
|
||||||
|
assert!(!listed.uploads.iter().any(|upload| upload.upload_id == upload_id));
|
||||||
|
|
||||||
|
let get_err = set_disks
|
||||||
|
.get_multipart_info(bucket, object, &upload_id, &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
.expect_err("external multipart metadata reads must not expose a data movement upload");
|
||||||
|
assert!(matches!(get_err, StorageError::InvalidUploadID(..)));
|
||||||
|
|
||||||
|
let list_parts_err = set_disks
|
||||||
|
.list_object_parts(bucket, object, &upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
.expect_err("external part listings must not expose a data movement upload");
|
||||||
|
assert!(matches!(list_parts_err, StorageError::InvalidUploadID(..)));
|
||||||
|
|
||||||
|
let mut external_part = PutObjReader::from_vec(b"external overwrite".to_vec());
|
||||||
|
let put_err = set_disks
|
||||||
|
.put_object_part(bucket, object, &upload_id, 1, &mut external_part, &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
.expect_err("external part uploads must not modify a data movement upload");
|
||||||
|
assert!(matches!(put_err, StorageError::InvalidUploadID(..)));
|
||||||
|
|
||||||
|
let abort_err = set_disks
|
||||||
|
.abort_multipart_upload(bucket, object, &upload_id, &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
.expect_err("external aborts must not remove a data movement upload");
|
||||||
|
assert!(matches!(abort_err, StorageError::InvalidUploadID(..)));
|
||||||
|
|
||||||
|
let external_err = set_disks
|
||||||
|
.clone()
|
||||||
|
.complete_multipart_upload(bucket, object, &upload_id, parts.clone(), &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
.expect_err("external completion must not finalize a data movement upload");
|
||||||
|
assert!(matches!(external_err, StorageError::InvalidUploadID(..)));
|
||||||
|
|
||||||
|
let internal_parts = set_disks
|
||||||
|
.list_object_parts(bucket, object, &upload_id, None, MAX_PARTS_COUNT, &create_opts)
|
||||||
|
.await
|
||||||
|
.expect("data movement part listing should retain ownership of its upload");
|
||||||
|
assert_eq!(internal_parts.parts.len(), 1);
|
||||||
|
|
||||||
|
set_disks
|
||||||
|
.clone()
|
||||||
|
.complete_multipart_upload(bucket, object, &upload_id, parts, &create_opts)
|
||||||
|
.await
|
||||||
|
.expect("data movement completion should retain ownership of its upload");
|
||||||
|
let completed = set_disks
|
||||||
|
.get_object_info(
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
&ObjectOptions {
|
||||||
|
include_part_checksums: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("completed data movement object should be readable");
|
||||||
|
assert!(!rustfs_utils::http::contains_key_str(
|
||||||
|
&completed.user_defined,
|
||||||
|
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD
|
||||||
|
));
|
||||||
|
for suffix in [
|
||||||
|
rustfs_utils::http::SUFFIX_DATA_MOVED,
|
||||||
|
rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
completed
|
||||||
|
.user_defined
|
||||||
|
.contains_key(&rustfs_utils::http::internal_key_rustfs(suffix))
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
completed
|
||||||
|
.user_defined
|
||||||
|
.contains_key(&format!("{}{suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
completed.parts[0]
|
||||||
|
.checksums
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|checksums| checksums.get("CRC32C"))
|
||||||
|
.map(String::as_str),
|
||||||
|
Some("AAAAAA==")
|
||||||
|
);
|
||||||
|
|
||||||
|
let abort_object = "owned-abort-object";
|
||||||
|
let abort_upload = set_disks
|
||||||
|
.new_multipart_upload(bucket, abort_object, &create_opts)
|
||||||
|
.await
|
||||||
|
.expect("data movement abort upload should be created");
|
||||||
|
let abort_err = set_disks
|
||||||
|
.abort_multipart_upload(bucket, abort_object, &abort_upload.upload_id, &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
.expect_err("external abort must not remove the second data movement upload");
|
||||||
|
assert!(matches!(abort_err, StorageError::InvalidUploadID(..)));
|
||||||
|
set_disks
|
||||||
|
.abort_multipart_upload(bucket, abort_object, &abort_upload.upload_id, &create_opts)
|
||||||
|
.await
|
||||||
|
.expect("data movement abort should retain ownership of its upload");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn stale_data_movement_replacement_fails_before_commit_on_outer_fence_loss() {
|
||||||
|
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||||
|
let bucket = "data-movement-stale-fence-bucket";
|
||||||
|
make_bucket_on_all(&disk_stores, bucket).await;
|
||||||
|
let old_time = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
|
||||||
|
let new_time = old_time + time::Duration::SECOND;
|
||||||
|
|
||||||
|
for (object, namespace_lock_fence, bucket_lifecycle_lock_fence) in [
|
||||||
|
("metadata-fence", Some(NamespaceLockFence::lost_for_test()), None),
|
||||||
|
("bucket-fence", None, Some(NamespaceLockFence::lost_for_test())),
|
||||||
|
] {
|
||||||
|
let version_id = Uuid::new_v4();
|
||||||
|
let old_body = format!("old-{object}").into_bytes();
|
||||||
|
let mut old_reader = PutObjReader::from_vec(old_body.clone());
|
||||||
|
set_disks
|
||||||
|
.put_object(
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
&mut old_reader,
|
||||||
|
&ObjectOptions {
|
||||||
|
data_movement: true,
|
||||||
|
versioned: true,
|
||||||
|
version_id: Some(version_id.to_string()),
|
||||||
|
mod_time: Some(old_time),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("seed old data movement target");
|
||||||
|
|
||||||
|
let replacement_body = format!("new-{object}").into_bytes();
|
||||||
|
let create_opts = ObjectOptions {
|
||||||
|
data_movement: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let (upload_id, parts) =
|
||||||
|
stage_upload_with_create_opts(&set_disks, bucket, object, &replacement_body, &create_opts).await;
|
||||||
|
let complete_opts = ObjectOptions {
|
||||||
|
data_movement: true,
|
||||||
|
versioned: true,
|
||||||
|
version_id: Some(version_id.to_string()),
|
||||||
|
mod_time: Some(new_time),
|
||||||
|
http_preconditions: Some(crate::data_movement::data_movement_target_precondition()),
|
||||||
|
namespace_lock_fence,
|
||||||
|
bucket_lifecycle_lock_fence,
|
||||||
|
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
|
||||||
|
ObjectLockConfigState::ConfirmedAbsent,
|
||||||
|
))),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let err = set_disks
|
||||||
|
.clone()
|
||||||
|
.complete_multipart_upload(bucket, object, &upload_id, parts, &complete_opts)
|
||||||
|
.await
|
||||||
|
.expect_err("a lost outer fence must abort stale target replacement");
|
||||||
|
assert!(matches!(err, StorageError::NamespaceLockQuorumUnavailable { .. }));
|
||||||
|
|
||||||
|
let mut preserved = set_disks
|
||||||
|
.get_object_reader(
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
None,
|
||||||
|
HeaderMap::new(),
|
||||||
|
&ObjectOptions {
|
||||||
|
versioned: true,
|
||||||
|
version_id: Some(version_id.to_string()),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("read target after rejected replacement");
|
||||||
|
let mut preserved_body = Vec::new();
|
||||||
|
preserved
|
||||||
|
.stream
|
||||||
|
.read_to_end(&mut preserved_body)
|
||||||
|
.await
|
||||||
|
.expect("drain target after rejected replacement");
|
||||||
|
assert_eq!(preserved_body, old_body);
|
||||||
|
set_disks
|
||||||
|
.check_upload_id_exists_with_opts(bucket, object, &upload_id, true, &complete_opts)
|
||||||
|
.await
|
||||||
|
.expect("fence loss must leave replacement staging retryable");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn data_movement_complete_accepts_unknown_compressed_part_actual_size() {
|
||||||
|
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||||
|
let bucket = "data-movement-unknown-actual-size-bucket";
|
||||||
|
let object = "object";
|
||||||
|
make_bucket_on_all(&disk_stores, bucket).await;
|
||||||
|
|
||||||
|
let mut metadata = HashMap::new();
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
&mut metadata,
|
||||||
|
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||||
|
crate::io_support::rio::compression_metadata_value(rustfs_utils::CompressionAlgorithm::default()),
|
||||||
|
);
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
&mut metadata,
|
||||||
|
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
|
||||||
|
"source-generation".to_string(),
|
||||||
|
);
|
||||||
|
let create_opts = ObjectOptions {
|
||||||
|
data_movement: true,
|
||||||
|
user_defined: metadata.clone(),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let upload = set_disks
|
||||||
|
.new_multipart_upload(bucket, object, &create_opts)
|
||||||
|
.await
|
||||||
|
.expect("data movement upload should be created");
|
||||||
|
|
||||||
|
let mut completed_parts = Vec::new();
|
||||||
|
for (number, actual_size) in [(1, -1), (2, 1)] {
|
||||||
|
let mut reader = PutObjReader::new(
|
||||||
|
HashReader::from_stream(Cursor::new(vec![number as u8]), 1, actual_size, None, None, false)
|
||||||
|
.expect("part reader should be constructed"),
|
||||||
|
);
|
||||||
|
let part = set_disks
|
||||||
|
.put_object_part(bucket, object, &upload.upload_id, number, &mut reader, &create_opts)
|
||||||
|
.await
|
||||||
|
.expect("data movement part should be written");
|
||||||
|
completed_parts.push(CompletePart {
|
||||||
|
part_num: number,
|
||||||
|
etag: part.etag,
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "2".to_string());
|
||||||
|
let completed = set_disks
|
||||||
|
.clone()
|
||||||
|
.complete_multipart_upload(
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
&upload.upload_id,
|
||||||
|
completed_parts,
|
||||||
|
&ObjectOptions {
|
||||||
|
data_movement: true,
|
||||||
|
user_defined: metadata,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("data movement completion should accept the persisted unknown-size sentinel");
|
||||||
|
|
||||||
|
assert_eq!(completed.parts[0].actual_size, -1);
|
||||||
|
assert_eq!(completed.get_actual_size().expect("completed object actual size"), 2);
|
||||||
|
}
|
||||||
|
|
||||||
async fn assert_complete_first_linearizes(bucket: &'static str, object: &'static str, create_opts: ObjectOptions) {
|
async fn assert_complete_first_linearizes(bucket: &'static str, object: &'static str, create_opts: ObjectOptions) {
|
||||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||||
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
|
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
|
||||||
|
|||||||
@@ -3292,7 +3292,11 @@ impl SetDisks {
|
|||||||
// Force the full quorum fanout (allow_early_stop=false): `disks` is the
|
// Force the full quorum fanout (allow_early_stop=false): `disks` is the
|
||||||
// write target below, and an early-stop subset would only carry read
|
// write target below, and an early-stop subset would only carry read
|
||||||
// quorum, failing write quorum on update_object_meta (backlog#872).
|
// quorum, failing write quorum on update_object_meta (backlog#872).
|
||||||
let (fi, _, disks) = self.get_object_fileinfo_gated(bucket, object, opts, false, false).await?;
|
let mut read_opts = opts.clone();
|
||||||
|
read_opts.include_part_checksums = true;
|
||||||
|
let (fi, _, disks) = self
|
||||||
|
.get_object_fileinfo_gated(bucket, object, &read_opts, false, false)
|
||||||
|
.await?;
|
||||||
let mut fi = fi.into_owned();
|
let mut fi = fi.into_owned();
|
||||||
|
|
||||||
fi.metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags.to_owned());
|
fi.metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags.to_owned());
|
||||||
@@ -3301,6 +3305,7 @@ impl SetDisks {
|
|||||||
fi.metadata.insert(key.clone(), value.clone());
|
fi.metadata.insert(key.clone(), value.clone());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
fi.acknowledge_data_movement();
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pause_object_tagging_commit(bucket, object).await;
|
pause_object_tagging_commit(bucket, object).await;
|
||||||
@@ -3503,11 +3508,26 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
fi.metadata = (*src_info.user_defined).clone();
|
let preserved_part_checksums = if (src_info.metadata_only || src_info.version_only)
|
||||||
|
&& rustfs_utils::http::contains_key_str(&fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS)
|
||||||
if let Some(etag) = &src_info.etag {
|
{
|
||||||
fi.metadata.insert("etag".to_owned(), etag.clone());
|
Self::hydrate_selected_fileinfo_part_checksums(&mut fi).map_err(|_| Error::FileCorrupt)?;
|
||||||
|
Some(
|
||||||
|
rustfs_utils::http::get_consistent_str(&fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS)
|
||||||
|
.ok_or(Error::FileCorrupt)?
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
let mut replacement_metadata = (*src_info.user_defined).clone();
|
||||||
|
if let Some(part_checksums) = preserved_part_checksums {
|
||||||
|
rustfs_utils::http::insert_str(&mut replacement_metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS, part_checksums);
|
||||||
}
|
}
|
||||||
|
if let Some(etag) = &src_info.etag {
|
||||||
|
replacement_metadata.insert("etag".to_owned(), etag.clone());
|
||||||
|
}
|
||||||
|
fi.metadata = replacement_metadata.clone();
|
||||||
|
|
||||||
let mod_time = OffsetDateTime::now_utc();
|
let mod_time = OffsetDateTime::now_utc();
|
||||||
fi.mod_time = Some(mod_time);
|
fi.mod_time = Some(mod_time);
|
||||||
@@ -3538,10 +3558,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
|||||||
|
|
||||||
for fi in metas.iter_mut() {
|
for fi in metas.iter_mut() {
|
||||||
if fi.has_valid_erasure_geometry() {
|
if fi.has_valid_erasure_geometry() {
|
||||||
fi.metadata = (*src_info.user_defined).clone();
|
fi.metadata.clone_from(&replacement_metadata);
|
||||||
if let Some(etag) = &src_info.etag {
|
|
||||||
fi.metadata.insert("etag".to_owned(), etag.clone());
|
|
||||||
}
|
|
||||||
fi.mod_time = Some(mod_time);
|
fi.mod_time = Some(mod_time);
|
||||||
fi.version_id = version_id;
|
fi.version_id = version_id;
|
||||||
fi.versioned = src_opts.versioned || src_opts.version_suspended;
|
fi.versioned = src_opts.versioned || src_opts.version_suspended;
|
||||||
@@ -4482,7 +4499,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fi.version_id = if let Some(vid) = opts.version_id.as_ref() {
|
fi.version_id = if let Some(vid) = opts.version_id.as_ref() {
|
||||||
Some(Uuid::parse_str(vid.as_str())?)
|
let vid = Uuid::parse_str(vid.as_str())?;
|
||||||
|
(!opts.version_suspended || !vid.is_nil()).then_some(vid)
|
||||||
|
} else if opts.version_suspended {
|
||||||
|
None
|
||||||
} else if opts.versioned {
|
} else if opts.versioned {
|
||||||
Some(Uuid::new_v4())
|
Some(Uuid::new_v4())
|
||||||
} else {
|
} else {
|
||||||
@@ -4507,7 +4527,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
|||||||
// Create a single object deletion request
|
// Create a single object deletion request
|
||||||
let mut dfi = FileInfo {
|
let mut dfi = FileInfo {
|
||||||
name: object.to_string(),
|
name: object.to_string(),
|
||||||
version_id: opts.version_id.as_ref().and_then(|v| Uuid::parse_str(v).ok()),
|
version_id: opts
|
||||||
|
.version_id
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|v| Uuid::parse_str(v).ok())
|
||||||
|
.filter(|vid| !opts.version_suspended || !vid.is_nil()),
|
||||||
mark_deleted: mark_delete,
|
mark_deleted: mark_delete,
|
||||||
deleted: delete_marker,
|
deleted: delete_marker,
|
||||||
mod_time: Some(mod_time),
|
mod_time: Some(mod_time),
|
||||||
@@ -4654,6 +4678,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fi.acknowledge_data_movement();
|
||||||
|
|
||||||
if opts.mod_time.is_some() {
|
if opts.mod_time.is_some() {
|
||||||
fi.mod_time = opts.mod_time;
|
fi.mod_time = opts.mod_time;
|
||||||
}
|
}
|
||||||
@@ -4720,7 +4746,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
|||||||
// _lock_guard = guard_opt;
|
// _lock_guard = guard_opt;
|
||||||
// }
|
// }
|
||||||
|
|
||||||
let (fi, meta_arr, online_disks) = self.get_object_fileinfo(bucket, object, opts, true, false).await?;
|
let mut transition_read_opts = opts.clone();
|
||||||
|
transition_read_opts.include_part_checksums = true;
|
||||||
|
let (fi, meta_arr, online_disks) = self
|
||||||
|
.get_object_fileinfo(bucket, object, &transition_read_opts, true, false)
|
||||||
|
.await?;
|
||||||
let mut fi = fi.into_owned();
|
let mut fi = fi.into_owned();
|
||||||
/*if err != nil {
|
/*if err != nil {
|
||||||
return Err(to_object_err(err, vec![bucket, object]));
|
return Err(to_object_err(err, vec![bucket, object]));
|
||||||
@@ -4778,6 +4808,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
|||||||
let transaction_id = transaction.transaction_id;
|
let transaction_id = transaction.transaction_id;
|
||||||
let dest_obj = transaction.remote_object.clone();
|
let dest_obj = transaction.remote_object.clone();
|
||||||
let mut transition_meta = (*oi.user_defined).clone();
|
let mut transition_meta = (*oi.user_defined).clone();
|
||||||
|
rustfs_utils::http::remove_str(&mut transition_meta, rustfs_utils::http::SUFFIX_PART_CHECKSUMS);
|
||||||
transition_meta.insert("name".to_string(), object.to_string());
|
transition_meta.insert("name".to_string(), object.to_string());
|
||||||
rustfs_utils::http::metadata_compat::insert_str(
|
rustfs_utils::http::metadata_compat::insert_str(
|
||||||
&mut transition_meta,
|
&mut transition_meta,
|
||||||
@@ -4938,6 +4969,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
|||||||
let mut commit_opts = opts.clone();
|
let mut commit_opts = opts.clone();
|
||||||
commit_opts.no_lock = true;
|
commit_opts.no_lock = true;
|
||||||
commit_opts.metadata_cache_safe = false;
|
commit_opts.metadata_cache_safe = false;
|
||||||
|
commit_opts.include_part_checksums = true;
|
||||||
let transition_lock_guard = if opts.no_lock {
|
let transition_lock_guard = if opts.no_lock {
|
||||||
None
|
None
|
||||||
} else {
|
} else {
|
||||||
@@ -5191,7 +5223,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
|||||||
achieved: 0,
|
achieved: 0,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
let fi = self.clone().get_object_fileinfo(bucket, object, opts, true, false).await;
|
let mut restore_read_opts = opts.clone();
|
||||||
|
restore_read_opts.include_part_checksums = true;
|
||||||
|
let fi = self
|
||||||
|
.clone()
|
||||||
|
.get_object_fileinfo(bucket, object, &restore_read_opts, true, false)
|
||||||
|
.await;
|
||||||
drop(bucket_lifecycle_guard);
|
drop(bucket_lifecycle_guard);
|
||||||
if let Err(err) = fi {
|
if let Err(err) = fi {
|
||||||
return set_restore_header_fn(&mut oi, Some(to_object_err(err, vec![bucket, object]))).await;
|
return set_restore_header_fn(&mut oi, Some(to_object_err(err, vec![bucket, object]))).await;
|
||||||
@@ -5211,7 +5248,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
|||||||
expected_operation_id.to_string(),
|
expected_operation_id.to_string(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
let restore_commit_metadata = if let Some(expected_operation_id) = expected_operation_id {
|
let mut restore_commit_metadata = if let Some(expected_operation_id) = expected_operation_id {
|
||||||
let mut metadata = HashMap::new();
|
let mut metadata = HashMap::new();
|
||||||
metadata.insert(X_AMZ_RESTORE.as_str().to_string(), "ongoing-request=\"false\"".to_string());
|
metadata.insert(X_AMZ_RESTORE.as_str().to_string(), "ongoing-request=\"false\"".to_string());
|
||||||
rustfs_utils::http::metadata_compat::insert_str(
|
rustfs_utils::http::metadata_compat::insert_str(
|
||||||
@@ -5223,6 +5260,15 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
|||||||
} else {
|
} else {
|
||||||
HashMap::new()
|
HashMap::new()
|
||||||
};
|
};
|
||||||
|
if let Some(part_checksums) =
|
||||||
|
rustfs_utils::http::get_consistent_str(&actual_fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS)
|
||||||
|
{
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
&mut restore_commit_metadata,
|
||||||
|
rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
|
||||||
|
part_checksums.to_string(),
|
||||||
|
);
|
||||||
|
}
|
||||||
// The restore copy-back re-writes this same object via put_object /
|
// The restore copy-back re-writes this same object via put_object /
|
||||||
// new_multipart_upload / complete_multipart_upload, each of which takes
|
// new_multipart_upload / complete_multipart_upload, each of which takes
|
||||||
// the object write lock in its commit phase. The caller
|
// the object write lock in its commit phase. The caller
|
||||||
@@ -6103,7 +6149,7 @@ mod get_object_downstream_close_accounting_tests {
|
|||||||
mod metadata_mutation_generation_tests {
|
mod metadata_mutation_generation_tests {
|
||||||
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
|
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::disk::DiskAPI as _;
|
use crate::disk::{DiskAPI as _, ReadOptions};
|
||||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
||||||
|
|
||||||
async fn put_and_prime(
|
async fn put_and_prime(
|
||||||
@@ -6140,6 +6186,31 @@ mod metadata_mutation_generation_tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn persist_part_checksum_sidecar(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, value: &str) {
|
||||||
|
let (fi, _, disks) = set_disks
|
||||||
|
.get_object_fileinfo(
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
&ObjectOptions {
|
||||||
|
no_lock: true,
|
||||||
|
metadata_cache_safe: false,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("object metadata should be readable before adding the checksum sidecar");
|
||||||
|
let mut fi = fi.into_owned();
|
||||||
|
let disks = disks.into_owned();
|
||||||
|
rustfs_utils::http::insert_str(&mut fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS, value.to_string());
|
||||||
|
set_disks
|
||||||
|
.update_object_meta(bucket, object, fi, &disks)
|
||||||
|
.await
|
||||||
|
.expect("checksum sidecar should be persisted");
|
||||||
|
set_disks.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial_test::serial(metadata_cache_invalidation_probe)]
|
#[serial_test::serial(metadata_cache_invalidation_probe)]
|
||||||
async fn metadata_semantic_mutation_generation_matrix_retires_cached_snapshot() {
|
async fn metadata_semantic_mutation_generation_matrix_retires_cached_snapshot() {
|
||||||
@@ -6226,6 +6297,112 @@ mod metadata_mutation_generation_tests {
|
|||||||
);
|
);
|
||||||
assert_retired(&set_disks, &metadata_key).await;
|
assert_retired(&set_disks, &metadata_key).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn metadata_only_copy_preserves_valid_part_checksums_and_rejects_conflicting_aliases() {
|
||||||
|
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||||
|
let bucket = "metadata-copy-part-checksums-bucket";
|
||||||
|
for disk in &disk_stores {
|
||||||
|
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||||
|
}
|
||||||
|
|
||||||
|
let valid_object = "valid-sidecar";
|
||||||
|
let (mut valid_source, _) = put_and_prime(&set_disks, bucket, valid_object, b"valid sidecar body").await;
|
||||||
|
persist_part_checksum_sidecar(&set_disks, bucket, valid_object, r#"[[1,[["CRC32C","AAAAAA=="]]]]"#).await;
|
||||||
|
valid_source.metadata_only = true;
|
||||||
|
Arc::make_mut(&mut valid_source.user_defined).insert("x-amz-meta-copy".to_string(), "updated".to_string());
|
||||||
|
set_disks
|
||||||
|
.copy_object(
|
||||||
|
bucket,
|
||||||
|
valid_object,
|
||||||
|
bucket,
|
||||||
|
valid_object,
|
||||||
|
&mut valid_source,
|
||||||
|
&ObjectOptions::default(),
|
||||||
|
&ObjectOptions::default(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("metadata-only copy should preserve a valid checksum sidecar");
|
||||||
|
let copied = set_disks
|
||||||
|
.get_object_info(
|
||||||
|
bucket,
|
||||||
|
valid_object,
|
||||||
|
&ObjectOptions {
|
||||||
|
include_part_checksums: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("copied object should retain readable part checksums");
|
||||||
|
assert_eq!(
|
||||||
|
copied.parts[0]
|
||||||
|
.checksums
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|checksums| checksums.get("CRC32C"))
|
||||||
|
.map(String::as_str),
|
||||||
|
Some("AAAAAA==")
|
||||||
|
);
|
||||||
|
|
||||||
|
let conflicting_object = "conflicting-sidecar";
|
||||||
|
let (mut conflicting_source, _) =
|
||||||
|
put_and_prime(&set_disks, bucket, conflicting_object, b"conflicting sidecar body").await;
|
||||||
|
let (fi, _, disks) = set_disks
|
||||||
|
.get_object_fileinfo(
|
||||||
|
bucket,
|
||||||
|
conflicting_object,
|
||||||
|
&ObjectOptions {
|
||||||
|
no_lock: true,
|
||||||
|
metadata_cache_safe: false,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("conflicting object metadata should be readable before corruption is injected");
|
||||||
|
let mut fi = fi.into_owned();
|
||||||
|
let disks = disks.into_owned();
|
||||||
|
let rustfs_key = format!(
|
||||||
|
"{}{}",
|
||||||
|
rustfs_utils::http::RUSTFS_INTERNAL_PREFIX,
|
||||||
|
rustfs_utils::http::SUFFIX_PART_CHECKSUMS
|
||||||
|
);
|
||||||
|
let minio_key = format!(
|
||||||
|
"{}{}",
|
||||||
|
rustfs_utils::http::MINIO_INTERNAL_PREFIX,
|
||||||
|
rustfs_utils::http::SUFFIX_PART_CHECKSUMS
|
||||||
|
);
|
||||||
|
fi.metadata
|
||||||
|
.insert(rustfs_key.clone(), r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string());
|
||||||
|
fi.metadata
|
||||||
|
.insert(minio_key.clone(), r#"[[1,[["CRC32C","AQAAAA=="]]]]"#.to_string());
|
||||||
|
set_disks
|
||||||
|
.update_object_meta(bucket, conflicting_object, fi, &disks)
|
||||||
|
.await
|
||||||
|
.expect("conflicting aliases should be persisted for the fail-closed regression");
|
||||||
|
set_disks
|
||||||
|
.invalidate_get_object_metadata_cache(bucket, conflicting_object)
|
||||||
|
.await;
|
||||||
|
conflicting_source.metadata_only = true;
|
||||||
|
let err = set_disks
|
||||||
|
.copy_object(
|
||||||
|
bucket,
|
||||||
|
conflicting_object,
|
||||||
|
bucket,
|
||||||
|
conflicting_object,
|
||||||
|
&mut conflicting_source,
|
||||||
|
&ObjectOptions::default(),
|
||||||
|
&ObjectOptions::default(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect_err("metadata-only copy must reject conflicting checksum aliases");
|
||||||
|
assert!(matches!(err, Error::FileCorrupt));
|
||||||
|
let raw_err = disk_stores[0]
|
||||||
|
.read_version("", bucket, conflicting_object, "", &ReadOptions::default())
|
||||||
|
.await
|
||||||
|
.expect_err("the rejected copy must leave the conflicting persisted aliases fail-closed");
|
||||||
|
assert!(matches!(raw_err, crate::disk::error::DiskError::FileCorrupt));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(all(test, feature = "test-util"))]
|
#[cfg(all(test, feature = "test-util"))]
|
||||||
@@ -6287,6 +6464,7 @@ mod transition_commit_failure_tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
#[serial_test::serial(restore_multipart_failure_point)]
|
#[serial_test::serial(restore_multipart_failure_point)]
|
||||||
async fn multipart_restore_aborts_every_post_create_failure() {
|
async fn multipart_restore_aborts_every_post_create_failure() {
|
||||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||||
@@ -6333,6 +6511,32 @@ mod transition_commit_failure_tests {
|
|||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.expect("source multipart upload should complete");
|
.expect("source multipart upload should complete");
|
||||||
|
let (source_fi, _, online_disks) = set_disks
|
||||||
|
.get_object_fileinfo(
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
&ObjectOptions {
|
||||||
|
no_lock: true,
|
||||||
|
metadata_cache_safe: false,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("source metadata should be readable before adding the checksum sidecar");
|
||||||
|
let mut source_fi = source_fi.into_owned();
|
||||||
|
let online_disks = online_disks.into_owned();
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
&mut source_fi.metadata,
|
||||||
|
rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
|
||||||
|
r#"[[1,[["CRC32C","AAAAAA=="]]],[2,[["CRC32C","AQAAAA=="]]]]"#.to_string(),
|
||||||
|
);
|
||||||
|
set_disks
|
||||||
|
.update_object_meta(bucket, object, source_fi, &online_disks)
|
||||||
|
.await
|
||||||
|
.expect("source checksum sidecar should be persisted before transition");
|
||||||
|
set_disks.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||||
register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||||
set_disks
|
set_disks
|
||||||
@@ -6416,9 +6620,32 @@ mod transition_commit_failure_tests {
|
|||||||
"successful multipart completion must disarm cleanup without aborting"
|
"successful multipart completion must disarm cleanup without aborting"
|
||||||
);
|
);
|
||||||
let restored = set_disks
|
let restored = set_disks
|
||||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
.get_object_info(
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
&ObjectOptions {
|
||||||
|
include_part_checksums: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
.expect("successful multipart restore must leave the committed object intact");
|
.expect("successful multipart restore must leave the committed object intact");
|
||||||
|
assert_eq!(
|
||||||
|
restored.parts[0]
|
||||||
|
.checksums
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|checksums| checksums.get("CRC32C"))
|
||||||
|
.map(String::as_str),
|
||||||
|
Some("AAAAAA==")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
restored.parts[1]
|
||||||
|
.checksums
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|checksums| checksums.get("CRC32C"))
|
||||||
|
.map(String::as_str),
|
||||||
|
Some("AQAAAA==")
|
||||||
|
);
|
||||||
let restore_header = restored
|
let restore_header = restored
|
||||||
.user_defined
|
.user_defined
|
||||||
.get(s3s::header::X_AMZ_RESTORE.as_str())
|
.get(s3s::header::X_AMZ_RESTORE.as_str())
|
||||||
@@ -7900,6 +8127,88 @@ mod transition_upload_integrity_tests {
|
|||||||
.expect("source object should be written")
|
.expect("source object should be written")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial_test::serial]
|
||||||
|
async fn data_movement_tiered_metadata_is_create_only_under_object_lock() {
|
||||||
|
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||||
|
let bucket = "tiered-data-movement-create-only";
|
||||||
|
let object = "object.bin";
|
||||||
|
let version_id = Uuid::new_v4();
|
||||||
|
for disk in &disk_stores {
|
||||||
|
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||||
|
}
|
||||||
|
let mut reader = PutObjReader::from_vec(b"existing target".to_vec());
|
||||||
|
set_disks
|
||||||
|
.put_object(
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
&mut reader,
|
||||||
|
&ObjectOptions {
|
||||||
|
versioned: true,
|
||||||
|
version_id: Some(version_id.to_string()),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("existing target should be written");
|
||||||
|
let conflicting = FileInfo {
|
||||||
|
volume: bucket.to_string(),
|
||||||
|
name: object.to_string(),
|
||||||
|
version_id: Some(version_id),
|
||||||
|
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND),
|
||||||
|
size: 12,
|
||||||
|
parts: vec![ObjectPartInfo {
|
||||||
|
number: 1,
|
||||||
|
size: 12,
|
||||||
|
actual_size: 12,
|
||||||
|
etag: "part-etag".to_string(),
|
||||||
|
..Default::default()
|
||||||
|
}],
|
||||||
|
transition_status: TRANSITION_COMPLETE.to_string(),
|
||||||
|
transition_tier: "WARM".to_string(),
|
||||||
|
transitioned_objname: "remote/object-a".to_string(),
|
||||||
|
transition_version: Some("remote-version-a".to_string()),
|
||||||
|
transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||||
|
fresh: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let err = set_disks
|
||||||
|
.decommission_tiered_object(
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
&conflicting,
|
||||||
|
&ObjectOptions {
|
||||||
|
versioned: true,
|
||||||
|
version_id: Some(version_id.to_string()),
|
||||||
|
mod_time: conflicting.mod_time,
|
||||||
|
data_movement: true,
|
||||||
|
http_preconditions: Some(crate::data_movement::data_movement_target_precondition()),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect_err("data movement must not overwrite an existing tiered version");
|
||||||
|
assert!(matches!(err, StorageError::PreconditionFailed));
|
||||||
|
|
||||||
|
let (stored, _, _) = set_disks
|
||||||
|
.get_object_fileinfo(
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
&ObjectOptions {
|
||||||
|
version_id: Some(version_id.to_string()),
|
||||||
|
no_lock: true,
|
||||||
|
metadata_cache_safe: false,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("the existing target should remain readable");
|
||||||
|
assert_ne!(stored.transition_status, TRANSITION_COMPLETE);
|
||||||
|
assert!(stored.transition_version.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
fn transition_options(original: &ObjectInfo, tier_name: String) -> ObjectOptions {
|
fn transition_options(original: &ObjectInfo, tier_name: String) -> ObjectOptions {
|
||||||
ObjectOptions {
|
ObjectOptions {
|
||||||
no_lock: true,
|
no_lock: true,
|
||||||
@@ -7954,6 +8263,31 @@ mod transition_upload_integrity_tests {
|
|||||||
let object = "object.bin";
|
let object = "object.bin";
|
||||||
let payload = b"transition remote object must be bound to its transaction id".repeat(1024);
|
let payload = b"transition remote object must be bound to its transaction id".repeat(1024);
|
||||||
let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await;
|
let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await;
|
||||||
|
let (source_fi, _, online_disks) = set_disks
|
||||||
|
.get_object_fileinfo(
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
&ObjectOptions {
|
||||||
|
no_lock: true,
|
||||||
|
metadata_cache_safe: false,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("source metadata should be readable");
|
||||||
|
let mut source_fi = source_fi.into_owned();
|
||||||
|
let online_disks = online_disks.into_owned();
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
&mut source_fi.metadata,
|
||||||
|
rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
|
||||||
|
r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string(),
|
||||||
|
);
|
||||||
|
set_disks
|
||||||
|
.update_object_meta(bucket, object, source_fi, &online_disks)
|
||||||
|
.await
|
||||||
|
.expect("source checksum sidecar should be persisted");
|
||||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||||
let remote_version = Uuid::new_v4().to_string();
|
let remote_version = Uuid::new_v4().to_string();
|
||||||
let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||||
@@ -7978,6 +8312,7 @@ mod transition_upload_integrity_tests {
|
|||||||
&ObjectOptions {
|
&ObjectOptions {
|
||||||
no_lock: true,
|
no_lock: true,
|
||||||
metadata_cache_safe: false,
|
metadata_cache_safe: false,
|
||||||
|
include_part_checksums: true,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
},
|
},
|
||||||
true,
|
true,
|
||||||
@@ -7991,6 +8326,22 @@ mod transition_upload_integrity_tests {
|
|||||||
fi.transition_version_id,
|
fi.transition_version_id,
|
||||||
Some(Uuid::parse_str(&remote_version).expect("test version id should parse"))
|
Some(Uuid::parse_str(&remote_version).expect("test version id should parse"))
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
fi.parts[0]
|
||||||
|
.checksums
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|checksums| checksums.get("CRC32C"))
|
||||||
|
.map(String::as_str),
|
||||||
|
Some("AAAAAA==")
|
||||||
|
);
|
||||||
|
let remote_metadata = backend
|
||||||
|
.metadata(remote_object)
|
||||||
|
.await
|
||||||
|
.expect("remote metadata should be stored");
|
||||||
|
assert!(
|
||||||
|
!rustfs_utils::http::contains_key_str(&remote_metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS),
|
||||||
|
"the internal checksum sidecar must not be uploaded as remote user metadata"
|
||||||
|
);
|
||||||
assert!(backend.contains(remote_object).await, "committed remote object should remain available");
|
assert!(backend.contains(remote_object).await, "committed remote object should remain available");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -8067,6 +8418,7 @@ mod transition_upload_integrity_tests {
|
|||||||
object,
|
object,
|
||||||
&expected,
|
&expected,
|
||||||
&[],
|
&[],
|
||||||
|
crate::data_movement::SourceCleanupBucketFence::default(),
|
||||||
"test_data_movement",
|
"test_data_movement",
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -8091,6 +8443,72 @@ mod transition_upload_integrity_tests {
|
|||||||
assert_local_source_intact(&set_disks, bucket, object, &payload).await;
|
assert_local_source_intact(&set_disks, bucket, object, &payload).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "current_thread", start_paused = true)]
|
||||||
|
#[serial_test::serial]
|
||||||
|
async fn data_movement_cleanup_aborts_after_bucket_fence_loss() {
|
||||||
|
let refresh_calls = Arc::new(AtomicUsize::new(0));
|
||||||
|
let lockers: Vec<Arc<dyn LockClient>> = (0..4)
|
||||||
|
.map(|_| Arc::new(LockLostRefreshClient::new(Arc::clone(&refresh_calls))) as Arc<dyn LockClient>)
|
||||||
|
.collect();
|
||||||
|
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||||
|
let bucket = "data-movement-cleanup-bucket-fence-lost";
|
||||||
|
let object = "object.bin";
|
||||||
|
let payload = b"lost bucket fence must preserve the source".repeat(1024);
|
||||||
|
write_source(&set_disks, &disk_stores, bucket, object, &payload).await;
|
||||||
|
let expected = set_disks
|
||||||
|
.load_file_info_versions_exact(bucket, object)
|
||||||
|
.await
|
||||||
|
.expect("source versions should be readable")
|
||||||
|
.expect("source versions should exist");
|
||||||
|
|
||||||
|
let distributed_setup = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||||
|
let bucket_guard = set_disks
|
||||||
|
.new_ns_lock(bucket, crate::storage_api_contracts::bucket::BUCKET_LIFECYCLE_LOCK_OBJECT)
|
||||||
|
.await
|
||||||
|
.expect("create bucket lifecycle lock")
|
||||||
|
.get_read_lock(get_lock_acquire_timeout())
|
||||||
|
.await
|
||||||
|
.expect("acquire bucket lifecycle read lock");
|
||||||
|
let local_cleanup_setup = SetupTypeGuard::switch_to(SetupType::Erasure).await;
|
||||||
|
let barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(bucket, object);
|
||||||
|
|
||||||
|
let cleanup_set = Arc::clone(&set_disks);
|
||||||
|
let cleanup = tokio::spawn(async move {
|
||||||
|
let result = crate::data_movement::cleanup_source_entry_if_unchanged(
|
||||||
|
cleanup_set,
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
&expected,
|
||||||
|
&[],
|
||||||
|
crate::data_movement::SourceCleanupBucketFence {
|
||||||
|
expected_incarnation_id: None,
|
||||||
|
lifecycle_guard: Some(&bucket_guard),
|
||||||
|
},
|
||||||
|
"test_data_movement",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
drop(bucket_guard);
|
||||||
|
result
|
||||||
|
});
|
||||||
|
barrier.wait_until_paused().await;
|
||||||
|
tokio::time::advance(Duration::from_secs(11)).await;
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
assert!(refresh_calls.load(Ordering::SeqCst) > 0, "bucket guard refresh must run before commit");
|
||||||
|
barrier.release();
|
||||||
|
|
||||||
|
let error = cleanup
|
||||||
|
.await
|
||||||
|
.expect("cleanup task should not panic")
|
||||||
|
.expect_err("cleanup must fail after its bucket lifecycle fence loses refresh quorum");
|
||||||
|
assert!(matches!(
|
||||||
|
error,
|
||||||
|
crate::data_movement::SourceCleanupError::Storage(StorageError::NamespaceLockQuorumUnavailable { .. })
|
||||||
|
));
|
||||||
|
assert_local_source_intact(&set_disks, bucket, object, &payload).await;
|
||||||
|
drop(local_cleanup_setup);
|
||||||
|
drop(distributed_setup);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial_test::serial]
|
#[serial_test::serial]
|
||||||
async fn partial_remote_acceptance_cleans_exact_candidate_and_preserves_source() {
|
async fn partial_remote_acceptance_cleans_exact_candidate_and_preserves_source() {
|
||||||
@@ -9925,6 +10343,33 @@ mod put_object_tags_early_stop_regression_tests {
|
|||||||
.await
|
.await
|
||||||
.expect("put_object should succeed");
|
.expect("put_object should succeed");
|
||||||
|
|
||||||
|
let (fi, _, disks) = set_disks
|
||||||
|
.get_object_fileinfo(
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
&ObjectOptions {
|
||||||
|
no_lock: true,
|
||||||
|
metadata_cache_safe: false,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("object metadata should be readable before adding the checksum sidecar");
|
||||||
|
let mut fi = fi.into_owned();
|
||||||
|
let disks = disks.into_owned();
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
&mut fi.metadata,
|
||||||
|
rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
|
||||||
|
r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string(),
|
||||||
|
);
|
||||||
|
set_disks
|
||||||
|
.update_object_meta(bucket, object, fi, &disks)
|
||||||
|
.await
|
||||||
|
.expect("checksum sidecar should be persisted before tagging");
|
||||||
|
set_disks.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||||
|
|
||||||
let tags = "unit=backlog881&stage=regression";
|
let tags = "unit=backlog881&stage=regression";
|
||||||
set_disks
|
set_disks
|
||||||
.put_object_tags(bucket, object, tags, &ObjectOptions::default())
|
.put_object_tags(bucket, object, tags, &ObjectOptions::default())
|
||||||
@@ -9944,6 +10389,11 @@ mod put_object_tags_early_stop_regression_tests {
|
|||||||
Some(tags),
|
Some(tags),
|
||||||
"disk {idx} must carry the tag written under early-stop (write set not shrunk to a read-quorum subset)"
|
"disk {idx} must carry the tag written under early-stop (write set not shrunk to a read-quorum subset)"
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
rustfs_utils::http::get_consistent_str(&fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS),
|
||||||
|
Some(r#"[[1,[["CRC32C","AAAAAA=="]]]]"#),
|
||||||
|
"disk {idx} must retain the checksum sidecar across the tag metadata update"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ use crate::diagnostics::get::{
|
|||||||
GET_METADATA_CACHE_REASON_DATA_MOVEMENT, GET_METADATA_CACHE_REASON_DELETE_MARKER, GET_METADATA_CACHE_REASON_DIST_ERASURE,
|
GET_METADATA_CACHE_REASON_DATA_MOVEMENT, GET_METADATA_CACHE_REASON_DELETE_MARKER, GET_METADATA_CACHE_REASON_DIST_ERASURE,
|
||||||
GET_METADATA_CACHE_REASON_INCL_FREE_VERSIONS, GET_METADATA_CACHE_REASON_INSUFFICIENT_CACHED_QUORUM,
|
GET_METADATA_CACHE_REASON_INCL_FREE_VERSIONS, GET_METADATA_CACHE_REASON_INSUFFICIENT_CACHED_QUORUM,
|
||||||
GET_METADATA_CACHE_REASON_META_BUCKET, GET_METADATA_CACHE_REASON_NO_LOCK, GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED,
|
GET_METADATA_CACHE_REASON_META_BUCKET, GET_METADATA_CACHE_REASON_NO_LOCK, GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED,
|
||||||
GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_NUMBER,
|
GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_CHECKSUMS, GET_METADATA_CACHE_REASON_PART_NUMBER,
|
||||||
GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_STALE_PUBLICATION,
|
GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_STALE_PUBLICATION,
|
||||||
GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID, GET_METADATA_CACHE_REASON_VERSION_SUSPENDED,
|
GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID, GET_METADATA_CACHE_REASON_VERSION_SUSPENDED,
|
||||||
GET_METADATA_CACHE_REASON_VERSIONED, GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA,
|
GET_METADATA_CACHE_REASON_VERSIONED, GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA,
|
||||||
@@ -340,7 +340,7 @@ impl SetDisks {
|
|||||||
// read_all_fileinfo_observed (see read_all_fileinfo_early_stop in
|
// read_all_fileinfo_observed (see read_all_fileinfo_early_stop in
|
||||||
// core/io_primitives.rs); unsafe requests and callers that opt out
|
// core/io_primitives.rs); unsafe requests and callers that opt out
|
||||||
// (allow_early_stop=false) fall back to full-wait.
|
// (allow_early_stop=false) fall back to full-wait.
|
||||||
let (parts_metadata, errs, metadata_fanout_diagnostics) = Self::read_all_fileinfo_observed(
|
let (mut parts_metadata, errs, metadata_fanout_diagnostics) = Self::read_all_fileinfo_observed(
|
||||||
&disks,
|
&disks,
|
||||||
"",
|
"",
|
||||||
bucket,
|
bucket,
|
||||||
@@ -394,8 +394,17 @@ impl SetDisks {
|
|||||||
return Err(to_object_err(err.into(), vec![bucket, object]));
|
return Err(to_object_err(err.into(), vec![bucket, object]));
|
||||||
}
|
}
|
||||||
|
|
||||||
let (op_online_disks, fi, fileinfo_selection_quorum) =
|
let (op_online_disks, mut fi, fileinfo_selection_quorum) =
|
||||||
Self::select_valid_fileinfo(&disks, &parts_metadata, &errs, vid.as_str(), read_quorum, write_quorum)?;
|
Self::select_valid_fileinfo(&disks, &parts_metadata, &errs, vid.as_str(), read_quorum, write_quorum)?;
|
||||||
|
let include_part_checksums =
|
||||||
|
opts.include_part_checksums || opts.part_number.is_some() || opts.data_movement || opts.raw_data_movement_read;
|
||||||
|
if include_part_checksums {
|
||||||
|
Self::hydrate_selected_fileinfo_part_checksums(&mut fi)?;
|
||||||
|
} else {
|
||||||
|
for metadata in std::iter::once(&mut fi).chain(parts_metadata.iter_mut()) {
|
||||||
|
rustfs_utils::http::remove_str(&mut metadata.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS);
|
||||||
|
}
|
||||||
|
}
|
||||||
metadata_fanout_diagnostics.record_quorum_candidate_latency(metadata_metrics_path, fileinfo_selection_quorum);
|
metadata_fanout_diagnostics.record_quorum_candidate_latency(metadata_metrics_path, fileinfo_selection_quorum);
|
||||||
if errs.iter().any(|err| err.is_some()) {
|
if errs.iter().any(|err| err.is_some()) {
|
||||||
let version_id = resolved_read_repair_version_id(&fi, opts.version_id.as_deref());
|
let version_id = resolved_read_repair_version_id(&fi, opts.version_id.as_deref());
|
||||||
@@ -1826,6 +1835,9 @@ fn get_object_metadata_cache_request_bypass_reason(bucket: &str, opts: &ObjectOp
|
|||||||
if opts.part_number.is_some() {
|
if opts.part_number.is_some() {
|
||||||
return Some(GET_METADATA_CACHE_REASON_PART_NUMBER);
|
return Some(GET_METADATA_CACHE_REASON_PART_NUMBER);
|
||||||
}
|
}
|
||||||
|
if opts.include_part_checksums {
|
||||||
|
return Some(GET_METADATA_CACHE_REASON_PART_CHECKSUMS);
|
||||||
|
}
|
||||||
if opts.data_movement {
|
if opts.data_movement {
|
||||||
return Some(GET_METADATA_CACHE_REASON_DATA_MOVEMENT);
|
return Some(GET_METADATA_CACHE_REASON_DATA_MOVEMENT);
|
||||||
}
|
}
|
||||||
@@ -2497,6 +2509,16 @@ mod metadata_cache_tests {
|
|||||||
Some(GET_METADATA_CACHE_REASON_PART_NUMBER)
|
Some(GET_METADATA_CACHE_REASON_PART_NUMBER)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
opts = ObjectOptions {
|
||||||
|
include_part_checksums: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert!(!is_get_object_metadata_cache_request_eligible("bucket", &opts, true));
|
||||||
|
assert_eq!(
|
||||||
|
get_object_metadata_cache_request_bypass_reason("bucket", &opts, true),
|
||||||
|
Some(GET_METADATA_CACHE_REASON_PART_CHECKSUMS)
|
||||||
|
);
|
||||||
|
|
||||||
opts = ObjectOptions {
|
opts = ObjectOptions {
|
||||||
data_movement: true,
|
data_movement: true,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
|
|||||||
@@ -75,6 +75,7 @@ impl SetDisks {
|
|||||||
version_id,
|
version_id,
|
||||||
versioned: opts.versioned,
|
versioned: opts.versioned,
|
||||||
version_suspended: opts.version_suspended,
|
version_suspended: opts.version_suspended,
|
||||||
|
include_part_checksums: true,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
let (fi, _, disks) = self
|
let (fi, _, disks) = self
|
||||||
@@ -142,6 +143,7 @@ impl SetDisks {
|
|||||||
version_id,
|
version_id,
|
||||||
versioned: opts.versioned,
|
versioned: opts.versioned,
|
||||||
version_suspended: opts.version_suspended,
|
version_suspended: opts.version_suspended,
|
||||||
|
include_part_checksums: true,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
let (fi, _, disks) = self
|
let (fi, _, disks) = self
|
||||||
|
|||||||
@@ -161,6 +161,10 @@ impl BucketIncarnationFenceGuard {
|
|||||||
pub(crate) fn is_lock_lost(&self) -> bool {
|
pub(crate) fn is_lock_lost(&self) -> bool {
|
||||||
self.inner.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
|
self.inner.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn namespace_lock_guard(&self) -> Option<&NamespaceLockGuard> {
|
||||||
|
self.inner.as_ref()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Drop for BucketIncarnationFenceGuard {
|
impl Drop for BucketIncarnationFenceGuard {
|
||||||
|
|||||||
+1493
-108
File diff suppressed because it is too large
Load Diff
@@ -66,6 +66,76 @@ fn ensure_multipart_bucket_lifecycle_guard_held(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
struct DataMovementMultipartCompletionBarrierState {
|
||||||
|
bucket: String,
|
||||||
|
arrived: tokio::sync::Notify,
|
||||||
|
release: tokio::sync::Notify,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) struct DataMovementMultipartCompletionBarrier {
|
||||||
|
state: Arc<DataMovementMultipartCompletionBarrierState>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
static DATA_MOVEMENT_MULTIPART_COMPLETION_BARRIER: std::sync::OnceLock<
|
||||||
|
std::sync::Mutex<Option<Arc<DataMovementMultipartCompletionBarrierState>>>,
|
||||||
|
> = std::sync::OnceLock::new();
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
impl DataMovementMultipartCompletionBarrier {
|
||||||
|
pub(crate) fn install(bucket: &str) -> Self {
|
||||||
|
let state = Arc::new(DataMovementMultipartCompletionBarrierState {
|
||||||
|
bucket: bucket.to_string(),
|
||||||
|
arrived: tokio::sync::Notify::new(),
|
||||||
|
release: tokio::sync::Notify::new(),
|
||||||
|
});
|
||||||
|
let mut slot = DATA_MOVEMENT_MULTIPART_COMPLETION_BARRIER
|
||||||
|
.get_or_init(|| std::sync::Mutex::new(None))
|
||||||
|
.lock()
|
||||||
|
.expect("data movement multipart completion barrier mutex should not poison");
|
||||||
|
assert!(slot.is_none(), "data movement multipart completion barrier must be unique");
|
||||||
|
*slot = Some(Arc::clone(&state));
|
||||||
|
Self { state }
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn wait_until_paused(&self) {
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(30), self.state.arrived.notified())
|
||||||
|
.await
|
||||||
|
.expect("data movement multipart operation should reach selected completion");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
impl Drop for DataMovementMultipartCompletionBarrier {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.state.release.notify_one();
|
||||||
|
let mut slot = DATA_MOVEMENT_MULTIPART_COMPLETION_BARRIER
|
||||||
|
.get_or_init(|| std::sync::Mutex::new(None))
|
||||||
|
.lock()
|
||||||
|
.expect("data movement multipart completion barrier mutex should not poison");
|
||||||
|
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
|
||||||
|
*slot = None;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
async fn pause_data_movement_multipart_before_selected_completion(bucket: &str) {
|
||||||
|
let barrier = DATA_MOVEMENT_MULTIPART_COMPLETION_BARRIER
|
||||||
|
.get_or_init(|| std::sync::Mutex::new(None))
|
||||||
|
.lock()
|
||||||
|
.expect("data movement multipart completion barrier mutex should not poison")
|
||||||
|
.as_ref()
|
||||||
|
.filter(|barrier| barrier.bucket == bucket)
|
||||||
|
.cloned();
|
||||||
|
if let Some(barrier) = barrier {
|
||||||
|
barrier.arrived.notify_one();
|
||||||
|
barrier.release.notified().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn list_pool_multipart_uploads_for_incarnation(
|
async fn list_pool_multipart_uploads_for_incarnation(
|
||||||
pool: &crate::core::sets::Sets,
|
pool: &crate::core::sets::Sets,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
@@ -332,7 +402,7 @@ impl ECStore {
|
|||||||
) -> Result<MultipartUploadResult> {
|
) -> Result<MultipartUploadResult> {
|
||||||
self.handle_new_multipart_upload_with_pool_idx(bucket, object, opts)
|
self.handle_new_multipart_upload_with_pool_idx(bucket, object, opts)
|
||||||
.await
|
.await
|
||||||
.map(|(res, _)| res)
|
.map(|(res, _, _)| res)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn handle_new_multipart_upload_with_pool_idx(
|
pub(crate) async fn handle_new_multipart_upload_with_pool_idx(
|
||||||
@@ -340,7 +410,7 @@ impl ECStore {
|
|||||||
bucket: &str,
|
bucket: &str,
|
||||||
object: &str,
|
object: &str,
|
||||||
opts: &ObjectOptions,
|
opts: &ObjectOptions,
|
||||||
) -> Result<(MultipartUploadResult, usize)> {
|
) -> Result<(MultipartUploadResult, usize, Option<Uuid>)> {
|
||||||
check_new_multipart_args(bucket, object)?;
|
check_new_multipart_args(bucket, object)?;
|
||||||
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||||
let opts = &opts;
|
let opts = &opts;
|
||||||
@@ -349,7 +419,20 @@ impl ECStore {
|
|||||||
return self.pools[0]
|
return self.pools[0]
|
||||||
.new_multipart_upload(bucket, object, opts)
|
.new_multipart_upload(bucket, object, opts)
|
||||||
.await
|
.await
|
||||||
.map(|res| (res, 0));
|
.map(|res| (res, 0, opts.expected_bucket_incarnation_id));
|
||||||
|
}
|
||||||
|
|
||||||
|
if opts.data_movement && opts.version_id.is_some() {
|
||||||
|
let idx = self.select_data_movement_pool_idx(bucket, object, -1, opts, false).await?;
|
||||||
|
if idx == opts.src_pool_idx {
|
||||||
|
return Err(StorageError::DataMovementOverwriteErr(
|
||||||
|
bucket.to_owned(),
|
||||||
|
object.to_owned(),
|
||||||
|
opts.version_id.clone().unwrap_or_default(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
|
||||||
|
return Ok((res, idx, opts.expected_bucket_incarnation_id));
|
||||||
}
|
}
|
||||||
|
|
||||||
for (idx, pool) in self.pools.iter().enumerate() {
|
for (idx, pool) in self.pools.iter().enumerate() {
|
||||||
@@ -372,7 +455,7 @@ impl ECStore {
|
|||||||
|
|
||||||
if !res.uploads.is_empty() {
|
if !res.uploads.is_empty() {
|
||||||
let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
|
let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
|
||||||
return Ok((res, idx));
|
return Ok((res, idx, opts.expected_bucket_incarnation_id));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let idx = self.get_pool_idx(bucket, object, -1).await?;
|
let idx = self.get_pool_idx(bucket, object, -1).await?;
|
||||||
@@ -385,7 +468,7 @@ impl ECStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
|
let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
|
||||||
Ok((res, idx))
|
Ok((res, idx, opts.expected_bucket_incarnation_id))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[instrument(skip(self))]
|
#[instrument(skip(self))]
|
||||||
@@ -456,6 +539,30 @@ impl ECStore {
|
|||||||
Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()))
|
Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn put_object_part_for_data_movement(
|
||||||
|
&self,
|
||||||
|
target_pool_idx: usize,
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
upload_id: &str,
|
||||||
|
data: &mut PutObjReader,
|
||||||
|
opts: &ObjectOptions,
|
||||||
|
) -> Result<PartInfo> {
|
||||||
|
let part_id = opts
|
||||||
|
.part_number
|
||||||
|
.ok_or_else(|| Error::other("targeted multipart upload requires a part number"))?;
|
||||||
|
check_put_object_part_args(bucket, object, upload_id)?;
|
||||||
|
if !opts.data_movement {
|
||||||
|
return Err(Error::other("targeted multipart upload requires data_movement options"));
|
||||||
|
}
|
||||||
|
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||||
|
let pool = self
|
||||||
|
.pools
|
||||||
|
.get(target_pool_idx)
|
||||||
|
.ok_or_else(|| Error::other(format!("data movement target pool {target_pool_idx} is out of range")))?;
|
||||||
|
pool.put_object_part(bucket, object, upload_id, part_id, data, &opts).await
|
||||||
|
}
|
||||||
|
|
||||||
#[instrument(skip(self))]
|
#[instrument(skip(self))]
|
||||||
pub(super) async fn handle_get_multipart_info(
|
pub(super) async fn handle_get_multipart_info(
|
||||||
&self,
|
&self,
|
||||||
@@ -530,6 +637,26 @@ impl ECStore {
|
|||||||
Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()))
|
Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn abort_multipart_upload_for_data_movement(
|
||||||
|
&self,
|
||||||
|
target_pool_idx: usize,
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
upload_id: &str,
|
||||||
|
opts: &ObjectOptions,
|
||||||
|
) -> Result<()> {
|
||||||
|
check_abort_multipart_args(bucket, object, upload_id)?;
|
||||||
|
if !opts.data_movement {
|
||||||
|
return Err(Error::other("targeted multipart abort requires data_movement options"));
|
||||||
|
}
|
||||||
|
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||||
|
let pool = self
|
||||||
|
.pools
|
||||||
|
.get(target_pool_idx)
|
||||||
|
.ok_or_else(|| Error::other(format!("data movement target pool {target_pool_idx} is out of range")))?;
|
||||||
|
pool.abort_multipart_upload(bucket, object, upload_id, &opts).await
|
||||||
|
}
|
||||||
|
|
||||||
#[instrument(skip(self))]
|
#[instrument(skip(self))]
|
||||||
pub(super) async fn handle_complete_multipart_upload(
|
pub(super) async fn handle_complete_multipart_upload(
|
||||||
self: Arc<Self>,
|
self: Arc<Self>,
|
||||||
@@ -574,6 +701,62 @@ impl ECStore {
|
|||||||
|
|
||||||
Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()))
|
Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn complete_multipart_upload_for_data_movement(
|
||||||
|
self: Arc<Self>,
|
||||||
|
target_pool_idx: usize,
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
upload_id: &str,
|
||||||
|
uploaded_parts: Vec<CompletePart>,
|
||||||
|
opts: &ObjectOptions,
|
||||||
|
) -> Result<ObjectInfo> {
|
||||||
|
check_complete_multipart_args(bucket, object, upload_id)?;
|
||||||
|
if !opts.data_movement {
|
||||||
|
return Err(Error::other("targeted multipart completion requires data_movement options"));
|
||||||
|
}
|
||||||
|
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||||
|
if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
|
||||||
|
let expected_incarnation_id = opts
|
||||||
|
.expected_bucket_incarnation_id
|
||||||
|
.ok_or_else(|| Error::other("data movement completion is missing its bucket incarnation"))?;
|
||||||
|
let lifecycle_fence = opts
|
||||||
|
.bucket_lifecycle_lock_fence
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| Error::other("data movement completion is missing its bucket lifecycle fence"))?;
|
||||||
|
let snapshot = match opts.object_lock_config_snapshot.as_ref() {
|
||||||
|
Some(snapshot) => Arc::clone(snapshot),
|
||||||
|
None => {
|
||||||
|
self.object_lock_config_snapshot_under_lifecycle_fence(bucket, lifecycle_fence)
|
||||||
|
.await?
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if !snapshot.is_valid_for_destructive_put(self.id, bucket, expected_incarnation_id) {
|
||||||
|
return Err(Error::other(
|
||||||
|
"data movement Object Lock snapshot does not match the target bucket generation",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
snapshot.add_lock_fences(&mut opts);
|
||||||
|
opts.object_lock_config_snapshot = Some(snapshot);
|
||||||
|
}
|
||||||
|
#[cfg(test)]
|
||||||
|
pause_data_movement_multipart_before_selected_completion(bucket).await;
|
||||||
|
let pool = self
|
||||||
|
.pools
|
||||||
|
.get(target_pool_idx)
|
||||||
|
.ok_or_else(|| Error::other(format!("data movement target pool {target_pool_idx} is out of range")))?
|
||||||
|
.clone();
|
||||||
|
let result = enqueue_transition_after_write(
|
||||||
|
pool.complete_multipart_upload(bucket, object, upload_id, uploaded_parts, &opts)
|
||||||
|
.await,
|
||||||
|
LcEventSrc::S3CompleteMultipartUpload,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
if result.is_ok() {
|
||||||
|
list_objects::observe_list_objects_mutation(self.as_ref(), bucket).await;
|
||||||
|
}
|
||||||
|
result
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Merges per-pool `ListMultipartUploads` pages into a single globally paginated
|
/// Merges per-pool `ListMultipartUploads` pages into a single globally paginated
|
||||||
|
|||||||
@@ -918,7 +918,7 @@ fn is_equivalent_data_movement_delete_marker(source: &ObjectInfo, target: &Objec
|
|||||||
&& is_data_movement_delete_marker(target)
|
&& is_data_movement_delete_marker(target)
|
||||||
&& source.version_id == target.version_id
|
&& source.version_id == target.version_id
|
||||||
&& source.mod_time == target.mod_time
|
&& source.mod_time == target.mod_time
|
||||||
&& source.user_defined == target.user_defined
|
&& is_equivalent_data_movement_delete_marker_metadata(&source.user_defined, &target.user_defined)
|
||||||
&& source.user_tags == target.user_tags
|
&& source.user_tags == target.user_tags
|
||||||
&& source.replication_status_internal == target.replication_status_internal
|
&& source.replication_status_internal == target.replication_status_internal
|
||||||
&& source.replication_status == target.replication_status
|
&& source.replication_status == target.replication_status
|
||||||
@@ -926,24 +926,185 @@ fn is_equivalent_data_movement_delete_marker(source: &ObjectInfo, target: &Objec
|
|||||||
&& source.version_purge_status == target.version_purge_status
|
&& source.version_purge_status == target.version_purge_status
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn is_equivalent_data_movement_delete_marker_metadata(
|
||||||
|
source: &HashMap<String, String>,
|
||||||
|
target: &HashMap<String, String>,
|
||||||
|
) -> bool {
|
||||||
|
matches!(
|
||||||
|
(
|
||||||
|
data_movement_delete_marker_metadata_identity(source),
|
||||||
|
data_movement_delete_marker_metadata_identity(target)
|
||||||
|
),
|
||||||
|
(Some(source), Some(target)) if source == target
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn data_movement_delete_marker_metadata_identity(metadata: &HashMap<String, String>) -> Option<HashMap<String, String>> {
|
||||||
|
let mut identity = HashMap::with_capacity(metadata.len());
|
||||||
|
let mut local_tier_free_version_id = None;
|
||||||
|
for (key, value) in metadata {
|
||||||
|
let Some(suffix) = rustfs_utils::http::strip_internal_prefix_preserving_case(key) else {
|
||||||
|
identity.insert(key.clone(), value.clone());
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
if suffix.eq_ignore_ascii_case(rustfs_utils::http::SUFFIX_TIER_FV_ID) {
|
||||||
|
let version_id = Uuid::parse_str(value).ok().filter(|version_id| !version_id.is_nil())?;
|
||||||
|
if local_tier_free_version_id.is_some_and(|expected| expected != version_id) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
local_tier_free_version_id = Some(version_id);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let canonical_suffix = [
|
||||||
|
rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP,
|
||||||
|
rustfs_utils::http::SUFFIX_REPLICA_STATUS,
|
||||||
|
rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,
|
||||||
|
rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
|
||||||
|
rustfs_utils::http::SUFFIX_PURGESTATUS,
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.find(|candidate| suffix.eq_ignore_ascii_case(candidate))
|
||||||
|
.map(str::to_string)
|
||||||
|
.or_else(|| {
|
||||||
|
[
|
||||||
|
rustfs_utils::http::SUFFIX_REPLICATION_RESET_ARN_PREFIX,
|
||||||
|
rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX,
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.find_map(|prefix| {
|
||||||
|
suffix
|
||||||
|
.get(..prefix.len())
|
||||||
|
.is_some_and(|candidate| candidate.eq_ignore_ascii_case(prefix))
|
||||||
|
.then(|| format!("{prefix}{}", &suffix[prefix.len()..]))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.unwrap_or_else(|| suffix.to_string());
|
||||||
|
let canonical_value = if canonical_suffix.eq_ignore_ascii_case(rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP)
|
||||||
|
|| canonical_suffix.eq_ignore_ascii_case(rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP)
|
||||||
|
{
|
||||||
|
rustfs_filemeta::parse_replication_timestamp(value)?
|
||||||
|
.unix_timestamp_nanos()
|
||||||
|
.to_string()
|
||||||
|
} else {
|
||||||
|
value.clone()
|
||||||
|
};
|
||||||
|
let canonical_key = format!("{}{canonical_suffix}", rustfs_utils::http::RUSTFS_INTERNAL_PREFIX);
|
||||||
|
if identity
|
||||||
|
.insert(canonical_key, canonical_value.clone())
|
||||||
|
.is_some_and(|existing| existing != canonical_value)
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (status_suffix, timestamp_suffix) in [
|
||||||
|
(rustfs_utils::http::SUFFIX_REPLICA_STATUS, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP),
|
||||||
|
(
|
||||||
|
rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
|
||||||
|
rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let status_key = format!("{}{status_suffix}", rustfs_utils::http::RUSTFS_INTERNAL_PREFIX);
|
||||||
|
let timestamp_key = format!("{}{timestamp_suffix}", rustfs_utils::http::RUSTFS_INTERNAL_PREFIX);
|
||||||
|
match (identity.contains_key(&status_key), identity.contains_key(×tamp_key)) {
|
||||||
|
(true, false) => {
|
||||||
|
identity.insert(timestamp_key, OffsetDateTime::UNIX_EPOCH.unix_timestamp_nanos().to_string());
|
||||||
|
}
|
||||||
|
(false, true) => return None,
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Some(identity)
|
||||||
|
}
|
||||||
|
|
||||||
fn is_data_movement_delete_marker(info: &ObjectInfo) -> bool {
|
fn is_data_movement_delete_marker(info: &ObjectInfo) -> bool {
|
||||||
info.delete_marker
|
info.delete_marker
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn is_expected_data_movement_delete_marker_source(source: &ObjectInfo, expected_mod_time: Option<OffsetDateTime>) -> bool {
|
||||||
|
is_data_movement_delete_marker(source)
|
||||||
|
&& source.mod_time.is_some()
|
||||||
|
&& source.mod_time == expected_mod_time
|
||||||
|
&& data_movement_delete_marker_metadata_identity(&source.user_defined).is_some()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn current_data_movement_delete_marker_opts(source: &ObjectInfo, opts: &ObjectOptions) -> Option<ObjectOptions> {
|
||||||
|
let replica_status = rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICA_STATUS);
|
||||||
|
let replica_timestamp = rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP);
|
||||||
|
let (replica_status, replica_timestamp) = match (replica_status, replica_timestamp) {
|
||||||
|
(None, None) => Default::default(),
|
||||||
|
(Some(status), timestamp) => {
|
||||||
|
let status = crate::bucket::replication::ReplicationStatusType::from(status.as_str());
|
||||||
|
if status.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let timestamp = match timestamp {
|
||||||
|
Some(timestamp) => rustfs_filemeta::parse_replication_timestamp(×tamp)?,
|
||||||
|
None => OffsetDateTime::UNIX_EPOCH,
|
||||||
|
};
|
||||||
|
(status, Some(timestamp))
|
||||||
|
}
|
||||||
|
(None, Some(_)) => return None,
|
||||||
|
};
|
||||||
|
let replication_status = rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_STATUS);
|
||||||
|
let replication_timestamp =
|
||||||
|
rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP);
|
||||||
|
let (replication_status, replication_timestamp, replication_targets) = match (replication_status, replication_timestamp) {
|
||||||
|
(None, None) => Default::default(),
|
||||||
|
(Some(status), timestamp) => {
|
||||||
|
let direct_status = crate::bucket::replication::ReplicationStatusType::from(status.as_str());
|
||||||
|
let targets = crate::bucket::replication::replication_statuses_map(status.as_str());
|
||||||
|
if direct_status.is_empty() && targets.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let timestamp = match timestamp {
|
||||||
|
Some(timestamp) => rustfs_filemeta::parse_replication_timestamp(×tamp)?,
|
||||||
|
None => OffsetDateTime::UNIX_EPOCH,
|
||||||
|
};
|
||||||
|
(Some(status), Some(timestamp), targets)
|
||||||
|
}
|
||||||
|
(None, Some(_)) => return None,
|
||||||
|
};
|
||||||
|
let mut state = source.replication_state();
|
||||||
|
if state.target_delete_marker_version_ids_corrupt {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
state.replica_status = replica_status;
|
||||||
|
state.replica_timestamp = replica_timestamp;
|
||||||
|
state.replication_status_internal = replication_status;
|
||||||
|
state.replication_timestamp = replication_timestamp;
|
||||||
|
state.targets = replication_targets;
|
||||||
|
state.replicate_decision_str = source.replication_decision.clone();
|
||||||
|
state.delete_marker = true;
|
||||||
|
|
||||||
|
let mut target_opts = opts.clone();
|
||||||
|
target_opts.mod_time = source.mod_time;
|
||||||
|
target_opts.delete_replication = Some(state);
|
||||||
|
Some(target_opts)
|
||||||
|
}
|
||||||
|
|
||||||
fn expected_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo) -> ObjectInfo {
|
fn expected_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo) -> ObjectInfo {
|
||||||
ObjectInfo::from_file_info(source, "", &source.name, source.version_id.is_some())
|
ObjectInfo::from_file_info(source, "", &source.name, source.version_id.is_some())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn is_equivalent_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo, target: &ObjectInfo) -> bool {
|
fn is_equivalent_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo, target: &ObjectInfo) -> bool {
|
||||||
let expected = expected_data_movement_tiered_object(source);
|
let expected = expected_data_movement_tiered_object(source);
|
||||||
|
let Some(source_actual_size) = effective_object_actual_size(&expected) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let Some(target_actual_size) = effective_object_actual_size(target) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
|
||||||
source.version_id == target.version_id
|
source.version_id == target.version_id
|
||||||
&& !target.delete_marker
|
&& !target.delete_marker
|
||||||
&& source.size == target.size
|
&& source.size == target.size
|
||||||
&& source.get_etag() == target.etag
|
&& source.get_etag() == target.etag
|
||||||
&& source.checksum == target.checksum
|
&& source.checksum == target.checksum
|
||||||
|
&& crate::data_movement::are_equivalent_data_movement_parts(&source.parts, &target.parts)
|
||||||
&& source.mod_time == target.mod_time
|
&& source.mod_time == target.mod_time
|
||||||
&& expected.user_defined == target.user_defined
|
&& crate::data_movement::is_equivalent_data_movement_metadata(&expected, target, source_actual_size, target_actual_size)
|
||||||
&& expected.user_tags == target.user_tags
|
&& expected.user_tags == target.user_tags
|
||||||
&& expected.expires == target.expires
|
&& expected.expires == target.expires
|
||||||
&& expected.storage_class == target.storage_class
|
&& expected.storage_class == target.storage_class
|
||||||
@@ -952,11 +1113,12 @@ fn is_equivalent_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo,
|
|||||||
&& expected.version_purge_status_internal == target.version_purge_status_internal
|
&& expected.version_purge_status_internal == target.version_purge_status_internal
|
||||||
&& expected.version_purge_status == target.version_purge_status
|
&& expected.version_purge_status == target.version_purge_status
|
||||||
&& expected.transitioned_object.status == target.transitioned_object.status
|
&& expected.transitioned_object.status == target.transitioned_object.status
|
||||||
|
&& expected.transition_version_state == target.transition_version_state
|
||||||
&& expected.transitioned_object.name == target.transitioned_object.name
|
&& expected.transitioned_object.name == target.transitioned_object.name
|
||||||
&& expected.transitioned_object.tier == target.transitioned_object.tier
|
&& expected.transitioned_object.tier == target.transitioned_object.tier
|
||||||
&& expected.transitioned_object.version_id == target.transitioned_object.version_id
|
&& expected.transitioned_object.version_id == target.transitioned_object.version_id
|
||||||
&& expected.transitioned_object.free_version == target.transitioned_object.free_version
|
&& expected.transitioned_object.free_version == target.transitioned_object.free_version
|
||||||
&& effective_object_actual_size(target) == Some(source.size)
|
&& source_actual_size == target_actual_size
|
||||||
}
|
}
|
||||||
|
|
||||||
fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
|
fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
|
||||||
@@ -1054,7 +1216,7 @@ impl ECStore {
|
|||||||
)))
|
)))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn object_lock_config_snapshot_under_lifecycle_fence(
|
pub(super) async fn object_lock_config_snapshot_under_lifecycle_fence(
|
||||||
&self,
|
&self,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
lifecycle_fence: &NamespaceLockFence,
|
lifecycle_fence: &NamespaceLockFence,
|
||||||
@@ -1454,7 +1616,8 @@ impl ECStore {
|
|||||||
target_pool_idx: usize,
|
target_pool_idx: usize,
|
||||||
opts: &ObjectOptions,
|
opts: &ObjectOptions,
|
||||||
) -> Result<Option<ObjectInfo>> {
|
) -> Result<Option<ObjectInfo>> {
|
||||||
let lookup_opts = version_aware_lookup_opts(opts, true);
|
let mut lookup_opts = version_aware_lookup_opts(opts, true);
|
||||||
|
lookup_opts.include_part_checksums = true;
|
||||||
|
|
||||||
let Some(pool) = self.pools.get(target_pool_idx) else {
|
let Some(pool) = self.pools.get(target_pool_idx) else {
|
||||||
return Err(Error::other(format!(
|
return Err(Error::other(format!(
|
||||||
@@ -1521,6 +1684,25 @@ impl ECStore {
|
|||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
check_put_object_args(bucket, object)?;
|
check_put_object_args(bucket, object)?;
|
||||||
|
|
||||||
|
let mut opts = opts.clone();
|
||||||
|
let bucket_incarnation_fence = if is_meta_bucketname(bucket) {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
let expected = opts
|
||||||
|
.expected_bucket_incarnation_id
|
||||||
|
.ok_or_else(|| Error::other("tiered data movement is missing its bucket incarnation snapshot"))?;
|
||||||
|
let guard = self.acquire_bucket_incarnation_fence(bucket, expected).await?;
|
||||||
|
if let Some(namespace_guard) = guard.namespace_lock_guard() {
|
||||||
|
opts.add_bucket_lifecycle_lock_guard(namespace_guard);
|
||||||
|
}
|
||||||
|
Some(guard)
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut fi = fi.clone();
|
||||||
|
if opts.data_movement {
|
||||||
|
crate::data_movement::prepare_tiered_data_movement_file_info(&mut fi)?;
|
||||||
|
}
|
||||||
|
|
||||||
let object = encode_dir_object(object);
|
let object = encode_dir_object(object);
|
||||||
|
|
||||||
if self.single_pool() {
|
if self.single_pool() {
|
||||||
@@ -1533,7 +1715,8 @@ impl ECStore {
|
|||||||
|
|
||||||
let idx = if opts.data_movement && opts.version_id.is_some() {
|
let idx = if opts.data_movement && opts.version_id.is_some() {
|
||||||
Self::resolve_decommission_target_pool_idx_result(
|
Self::resolve_decommission_target_pool_idx_result(
|
||||||
self.select_data_movement_pool_idx(bucket, &object, fi.size, opts, true).await,
|
self.select_data_movement_pool_idx(bucket, &object, fi.size, &opts, true)
|
||||||
|
.await,
|
||||||
bucket,
|
bucket,
|
||||||
&object,
|
&object,
|
||||||
)?
|
)?
|
||||||
@@ -1550,7 +1733,7 @@ impl ECStore {
|
|||||||
.await;
|
.await;
|
||||||
let target_pool_idx = resolve_data_movement_resume_target_pool(idx, resume_target_pool_idx, opts.src_pool_idx);
|
let target_pool_idx = resolve_data_movement_resume_target_pool(idx, resume_target_pool_idx, opts.src_pool_idx);
|
||||||
if self
|
if self
|
||||||
.has_equivalent_data_movement_tiered_object(bucket, &object, fi, opts, target_pool_idx)
|
.has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, target_pool_idx)
|
||||||
.await?
|
.await?
|
||||||
{
|
{
|
||||||
return Ok(());
|
return Ok(());
|
||||||
@@ -1563,14 +1746,27 @@ impl ECStore {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
Self::resolve_decommission_tiered_object_result(
|
let result = self.pools[idx]
|
||||||
self.pools[idx]
|
.get_disks_by_key(&object)
|
||||||
.get_disks_by_key(&object)
|
.decommission_tiered_object(bucket, &object, &fi, &opts)
|
||||||
.decommission_tiered_object(bucket, &object, fi, opts)
|
.await;
|
||||||
.await,
|
if matches!(result, Err(Error::PreconditionFailed)) {
|
||||||
bucket,
|
if self
|
||||||
&object,
|
.has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, idx)
|
||||||
)
|
.await?
|
||||||
|
{
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
return Err(StorageError::DataMovementOverwriteErr(
|
||||||
|
bucket.to_owned(),
|
||||||
|
object,
|
||||||
|
opts.version_id.clone().unwrap_or_default(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if bucket_incarnation_fence.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||||
|
return Err(Error::other("tiered data movement bucket incarnation fence was lost during target write"));
|
||||||
|
}
|
||||||
|
Self::resolve_decommission_tiered_object_result(result, bucket, &object)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[instrument(level = "debug", skip(self))]
|
#[instrument(level = "debug", skip(self))]
|
||||||
@@ -1607,15 +1803,7 @@ impl ECStore {
|
|||||||
Ok(Self::attach_read_lock_guard(reader, read_lock_guard))
|
Ok(Self::attach_read_lock_guard(reader, read_lock_guard))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[instrument(level = "debug", skip(self, data))]
|
async fn prepare_put_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<(String, ObjectOptions)> {
|
||||||
#[hotpath::measure(impl_type = "ECStore")]
|
|
||||||
pub(super) async fn handle_put_object(
|
|
||||||
&self,
|
|
||||||
bucket: &str,
|
|
||||||
object: &str,
|
|
||||||
data: &mut PutObjReader,
|
|
||||||
opts: &ObjectOptions,
|
|
||||||
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
|
|
||||||
check_put_object_args(bucket, object)?;
|
check_put_object_args(bucket, object)?;
|
||||||
|
|
||||||
let object = encode_dir_object(object);
|
let object = encode_dir_object(object);
|
||||||
@@ -1642,22 +1830,20 @@ impl ECStore {
|
|||||||
};
|
};
|
||||||
snapshot.add_lock_fences(&mut opts);
|
snapshot.add_lock_fences(&mut opts);
|
||||||
}
|
}
|
||||||
|
Ok((object, opts))
|
||||||
|
}
|
||||||
|
|
||||||
// Keep PUT atomic-read friendly: SetDisks takes the object write lock only
|
async fn select_put_object_pool_idx(&self, bucket: &str, object: &str, size: i64, opts: &ObjectOptions) -> Result<usize> {
|
||||||
// around precondition checks and the final rename/commit.
|
|
||||||
if self.single_pool() {
|
if self.single_pool() {
|
||||||
return self.pools[0]
|
return Ok(0);
|
||||||
.put_object_with_old_current_size(bucket, object.as_str(), data, &opts)
|
|
||||||
.await;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let idx = if opts.data_movement && opts.version_id.is_some() {
|
let idx = if opts.data_movement && opts.version_id.is_some() {
|
||||||
self.select_data_movement_pool_idx(bucket, &object, data.size(), &opts, false)
|
self.select_data_movement_pool_idx(bucket, object, size, opts, false).await?
|
||||||
.await?
|
|
||||||
} else if opts.no_lock {
|
} else if opts.no_lock {
|
||||||
self.get_pool_idx_no_lock(bucket, &object, data.size()).await?
|
self.get_pool_idx_no_lock(bucket, object, size).await?
|
||||||
} else {
|
} else {
|
||||||
self.get_pool_idx(bucket, &object, data.size()).await?
|
self.get_pool_idx(bucket, object, size).await?
|
||||||
};
|
};
|
||||||
|
|
||||||
if opts.data_movement && idx == opts.src_pool_idx {
|
if opts.data_movement && idx == opts.src_pool_idx {
|
||||||
@@ -1667,7 +1853,50 @@ impl ECStore {
|
|||||||
opts.version_id.clone().unwrap_or_default(),
|
opts.version_id.clone().unwrap_or_default(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
Ok(idx)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn put_object_for_data_movement(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
data: &mut PutObjReader,
|
||||||
|
opts: &ObjectOptions,
|
||||||
|
) -> Result<(usize, Result<ObjectInfo>)> {
|
||||||
|
if !opts.data_movement {
|
||||||
|
return Err(Error::other("data movement PUT requires data_movement options"));
|
||||||
|
}
|
||||||
|
let (object, opts) = self.prepare_put_object(bucket, object, opts).await?;
|
||||||
|
let idx = self
|
||||||
|
.select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts)
|
||||||
|
.await?;
|
||||||
|
let result = self.pools[idx]
|
||||||
|
.put_object_with_old_current_size(bucket, &object, data, &opts)
|
||||||
|
.await
|
||||||
|
.map(|(object_info, _)| object_info);
|
||||||
|
let result = enqueue_transition_after_write(result, LcEventSrc::S3PutObject).await;
|
||||||
|
if result.is_ok() {
|
||||||
|
list_objects::observe_list_objects_mutation(self, bucket).await;
|
||||||
|
}
|
||||||
|
Ok((idx, result))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(level = "debug", skip(self, data))]
|
||||||
|
#[hotpath::measure(impl_type = "ECStore")]
|
||||||
|
pub(super) async fn handle_put_object(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
data: &mut PutObjReader,
|
||||||
|
opts: &ObjectOptions,
|
||||||
|
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
|
||||||
|
let (object, opts) = self.prepare_put_object(bucket, object, opts).await?;
|
||||||
|
let idx = self
|
||||||
|
.select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
// Keep PUT atomic-read friendly: SetDisks takes the object write lock only
|
||||||
|
// around precondition checks and the final rename/commit.
|
||||||
self.pools[idx]
|
self.pools[idx]
|
||||||
.put_object_with_old_current_size(bucket, &object, data, &opts)
|
.put_object_with_old_current_size(bucket, &object, data, &opts)
|
||||||
.await
|
.await
|
||||||
@@ -2110,6 +2339,50 @@ impl ECStore {
|
|||||||
};
|
};
|
||||||
let target_pool_idx =
|
let target_pool_idx =
|
||||||
resolve_data_movement_resume_target_pool(selected_target_pool_idx, resume_target_pool_idx, opts.src_pool_idx);
|
resolve_data_movement_resume_target_pool(selected_target_pool_idx, resume_target_pool_idx, opts.src_pool_idx);
|
||||||
|
let mut delete_marker_target_opts = None;
|
||||||
|
|
||||||
|
if opts.delete_marker && should_check_data_movement_resume_target(opts.src_pool_idx, target_pool_idx) {
|
||||||
|
let source = self
|
||||||
|
.find_data_movement_target_info(bucket, object, opts.src_pool_idx, &opts)
|
||||||
|
.await?;
|
||||||
|
let Some(source) = source else {
|
||||||
|
return Err(StorageError::DataMovementOverwriteErr(
|
||||||
|
bucket.to_owned(),
|
||||||
|
object.to_owned(),
|
||||||
|
opts.version_id.unwrap_or_default(),
|
||||||
|
));
|
||||||
|
};
|
||||||
|
if !is_expected_data_movement_delete_marker_source(&source, opts.mod_time) {
|
||||||
|
return Err(StorageError::DataMovementOverwriteErr(
|
||||||
|
bucket.to_owned(),
|
||||||
|
object.to_owned(),
|
||||||
|
opts.version_id.unwrap_or_default(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let Some(target_opts) = current_data_movement_delete_marker_opts(&source, &opts) else {
|
||||||
|
return Err(StorageError::DataMovementOverwriteErr(
|
||||||
|
bucket.to_owned(),
|
||||||
|
object.to_owned(),
|
||||||
|
opts.version_id.unwrap_or_default(),
|
||||||
|
));
|
||||||
|
};
|
||||||
|
let target = self
|
||||||
|
.find_data_movement_target_info(bucket, object, target_pool_idx, &target_opts)
|
||||||
|
.await?;
|
||||||
|
if let Some(target) = target {
|
||||||
|
if is_equivalent_data_movement_delete_marker(&source, &target) {
|
||||||
|
let mut target = target;
|
||||||
|
target.name = decode_dir_object(object);
|
||||||
|
return Ok(target);
|
||||||
|
}
|
||||||
|
return Err(StorageError::DataMovementOverwriteErr(
|
||||||
|
bucket.to_owned(),
|
||||||
|
object.to_owned(),
|
||||||
|
opts.version_id.unwrap_or_default(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
delete_marker_target_opts = Some(target_opts);
|
||||||
|
}
|
||||||
|
|
||||||
if !should_check_data_movement_resume_target(opts.src_pool_idx, target_pool_idx) {
|
if !should_check_data_movement_resume_target(opts.src_pool_idx, target_pool_idx) {
|
||||||
if let Ok((source_pool_info, _)) = existing_pool_info
|
if let Ok((source_pool_info, _)) = existing_pool_info
|
||||||
@@ -2137,7 +2410,8 @@ impl ECStore {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, opts).await?;
|
let target_opts = delete_marker_target_opts.unwrap_or(opts);
|
||||||
|
let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, target_opts).await?;
|
||||||
obj.name = decode_dir_object(obj.name.as_str());
|
obj.name = decode_dir_object(obj.name.as_str());
|
||||||
return Ok(obj);
|
return Ok(obj);
|
||||||
}
|
}
|
||||||
@@ -3200,6 +3474,247 @@ mod tests {
|
|||||||
assert!(!is_equivalent_data_movement_delete_marker(&source, &mismatched));
|
assert!(!is_equivalent_data_movement_delete_marker(&source, &mismatched));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn equivalent_data_movement_delete_marker_accepts_distinct_local_free_version_ids() {
|
||||||
|
let mut source = ObjectInfo {
|
||||||
|
version_id: Some(Uuid::from_u128(1)),
|
||||||
|
delete_marker: true,
|
||||||
|
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
Arc::make_mut(&mut source.user_defined),
|
||||||
|
rustfs_utils::http::SUFFIX_TIER_FV_ID,
|
||||||
|
Uuid::from_u128(2).to_string(),
|
||||||
|
);
|
||||||
|
let mut target = source.clone();
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
Arc::make_mut(&mut target.user_defined),
|
||||||
|
rustfs_utils::http::SUFFIX_TIER_FV_ID,
|
||||||
|
Uuid::from_u128(3).to_string(),
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(is_equivalent_data_movement_delete_marker(&source, &target));
|
||||||
|
|
||||||
|
Arc::make_mut(&mut target.user_defined).insert(
|
||||||
|
format!("{}{}", rustfs_utils::http::MINIO_INTERNAL_PREFIX, rustfs_utils::http::SUFFIX_TIER_FV_ID),
|
||||||
|
Uuid::from_u128(4).to_string(),
|
||||||
|
);
|
||||||
|
assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn equivalent_data_movement_delete_marker_accepts_replication_alias_expansion() {
|
||||||
|
let key = format!(
|
||||||
|
"{}{}",
|
||||||
|
rustfs_utils::http::MINIO_INTERNAL_PREFIX,
|
||||||
|
rustfs_utils::http::SUFFIX_REPLICATION_STATUS
|
||||||
|
);
|
||||||
|
let timestamp_key = format!(
|
||||||
|
"{}{}",
|
||||||
|
rustfs_utils::http::MINIO_INTERNAL_PREFIX,
|
||||||
|
rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP
|
||||||
|
);
|
||||||
|
let source = ObjectInfo {
|
||||||
|
version_id: Some(Uuid::from_u128(1)),
|
||||||
|
delete_marker: true,
|
||||||
|
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||||
|
user_defined: Arc::new(HashMap::from([
|
||||||
|
(key.clone(), "arn=COMPLETED;".to_string()),
|
||||||
|
(timestamp_key, "1970-01-01T00:00:01Z".to_string()),
|
||||||
|
])),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let mut target = source.clone();
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
Arc::make_mut(&mut target.user_defined),
|
||||||
|
rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
|
||||||
|
"arn=COMPLETED;".to_string(),
|
||||||
|
);
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
Arc::make_mut(&mut target.user_defined),
|
||||||
|
rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,
|
||||||
|
(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND).to_string(),
|
||||||
|
);
|
||||||
|
assert!(is_equivalent_data_movement_delete_marker(&source, &target));
|
||||||
|
|
||||||
|
Arc::make_mut(&mut target.user_defined).insert(key, "arn=FAILED;".to_string());
|
||||||
|
assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn data_movement_delete_marker_source_requires_persisted_mod_time() {
|
||||||
|
let source = ObjectInfo {
|
||||||
|
delete_marker: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert!(!is_expected_data_movement_delete_marker_source(&source, None));
|
||||||
|
|
||||||
|
let source = ObjectInfo {
|
||||||
|
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||||
|
..source
|
||||||
|
};
|
||||||
|
assert!(is_expected_data_movement_delete_marker_source(&source, Some(OffsetDateTime::UNIX_EPOCH)));
|
||||||
|
assert!(!is_expected_data_movement_delete_marker_source(&source, None));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn data_movement_delete_marker_uses_current_source_replication_state() {
|
||||||
|
let expected_timestamp = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
|
||||||
|
let timestamp = expected_timestamp.to_string();
|
||||||
|
let mut metadata = HashMap::new();
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
&mut metadata,
|
||||||
|
rustfs_utils::http::SUFFIX_REPLICA_STATUS,
|
||||||
|
ReplicationStatusType::Replica.to_string(),
|
||||||
|
);
|
||||||
|
rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP, timestamp.clone());
|
||||||
|
rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP, timestamp);
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
&mut metadata,
|
||||||
|
rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
|
||||||
|
"arn=COMPLETED;".to_string(),
|
||||||
|
);
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
&mut metadata,
|
||||||
|
&format!(
|
||||||
|
"{}{}",
|
||||||
|
rustfs_utils::http::SUFFIX_REPLICATION_RESET_ARN_PREFIX,
|
||||||
|
"arn:minio:replication::TenantA:bucket"
|
||||||
|
),
|
||||||
|
"reset-id".to_string(),
|
||||||
|
);
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
&mut metadata,
|
||||||
|
&format!(
|
||||||
|
"{}{}",
|
||||||
|
rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX,
|
||||||
|
"arn:minio:replication::TenantA:bucket"
|
||||||
|
),
|
||||||
|
"target-version".to_string(),
|
||||||
|
);
|
||||||
|
let source = ObjectInfo {
|
||||||
|
delete_marker: true,
|
||||||
|
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||||
|
replication_status_internal: Some("arn=COMPLETED;".to_string()),
|
||||||
|
replication_decision: "arn=replicate;".to_string(),
|
||||||
|
user_defined: Arc::new(metadata),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let opts = ObjectOptions {
|
||||||
|
mod_time: source.mod_time,
|
||||||
|
delete_replication: Some(ReplicationState {
|
||||||
|
replication_status_internal: Some("arn=PENDING;".to_string()),
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let target_opts = current_data_movement_delete_marker_opts(&source, &opts).expect("valid current source state");
|
||||||
|
let state = target_opts.delete_replication.as_ref().expect("current replication state");
|
||||||
|
assert_eq!(state.replication_status_internal.as_deref(), Some("arn=COMPLETED;"));
|
||||||
|
assert_eq!(state.replica_status, crate::bucket::replication::ReplicationStatusType::Replica);
|
||||||
|
assert_eq!(state.replica_timestamp, Some(expected_timestamp));
|
||||||
|
assert_eq!(state.replication_timestamp, state.replica_timestamp);
|
||||||
|
assert_eq!(state.replicate_decision_str, "arn=replicate;");
|
||||||
|
assert_eq!(
|
||||||
|
state
|
||||||
|
.reset_statuses_map
|
||||||
|
.get("arn:minio:replication::TenantA:bucket")
|
||||||
|
.map(String::as_str),
|
||||||
|
Some("reset-id")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
state
|
||||||
|
.target_delete_marker_version_ids
|
||||||
|
.get("arn:minio:replication::TenantA:bucket")
|
||||||
|
.map(String::as_str),
|
||||||
|
Some("target-version")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn data_movement_delete_marker_rejects_corrupt_target_version_maps() {
|
||||||
|
let suffix = format!("{}not-an-arn", rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX);
|
||||||
|
let mut malformed = HashMap::new();
|
||||||
|
rustfs_utils::http::insert_str(&mut malformed, &suffix, "target-version".to_string());
|
||||||
|
let malformed_source = ObjectInfo {
|
||||||
|
delete_marker: true,
|
||||||
|
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||||
|
user_defined: Arc::new(malformed),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert!(current_data_movement_delete_marker_opts(&malformed_source, &ObjectOptions::default()).is_none());
|
||||||
|
|
||||||
|
let mut conflicted = HashMap::new();
|
||||||
|
let suffix = format!(
|
||||||
|
"{}arn:minio:replication::target:bucket",
|
||||||
|
rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX
|
||||||
|
);
|
||||||
|
rustfs_utils::http::insert_str(&mut conflicted, &suffix, "target-version-a".to_string());
|
||||||
|
conflicted.insert(
|
||||||
|
format!("{}{suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX),
|
||||||
|
"target-version-b".to_string(),
|
||||||
|
);
|
||||||
|
let conflicted_source = ObjectInfo {
|
||||||
|
user_defined: Arc::new(conflicted),
|
||||||
|
..malformed_source.clone()
|
||||||
|
};
|
||||||
|
assert!(current_data_movement_delete_marker_opts(&conflicted_source, &ObjectOptions::default()).is_none());
|
||||||
|
|
||||||
|
let mut over_cap = HashMap::new();
|
||||||
|
for index in 0..=1_000 {
|
||||||
|
let suffix = format!(
|
||||||
|
"{}arn:minio:replication::target:bucket-{index}",
|
||||||
|
rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX
|
||||||
|
);
|
||||||
|
rustfs_utils::http::insert_str(&mut over_cap, &suffix, format!("target-version-{index}"));
|
||||||
|
}
|
||||||
|
let over_cap_source = ObjectInfo {
|
||||||
|
user_defined: Arc::new(over_cap),
|
||||||
|
..malformed_source
|
||||||
|
};
|
||||||
|
assert!(current_data_movement_delete_marker_opts(&over_cap_source, &ObjectOptions::default()).is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn data_movement_delete_marker_normalizes_legacy_missing_replication_timestamps() {
|
||||||
|
let mut source_metadata = HashMap::new();
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
&mut source_metadata,
|
||||||
|
rustfs_utils::http::SUFFIX_REPLICA_STATUS,
|
||||||
|
ReplicationStatusType::Replica.to_string(),
|
||||||
|
);
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
&mut source_metadata,
|
||||||
|
rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
|
||||||
|
"arn=COMPLETED;".to_string(),
|
||||||
|
);
|
||||||
|
let source = ObjectInfo {
|
||||||
|
delete_marker: true,
|
||||||
|
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||||
|
replication_status_internal: Some("arn=COMPLETED;".to_string()),
|
||||||
|
user_defined: Arc::new(source_metadata),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let target_opts = current_data_movement_delete_marker_opts(&source, &ObjectOptions::default())
|
||||||
|
.expect("legacy status-only metadata should remain migratable");
|
||||||
|
let state = target_opts
|
||||||
|
.delete_replication
|
||||||
|
.expect("replication state should be reconstructed");
|
||||||
|
assert_eq!(state.replica_timestamp, Some(OffsetDateTime::UNIX_EPOCH));
|
||||||
|
assert_eq!(state.replication_timestamp, Some(OffsetDateTime::UNIX_EPOCH));
|
||||||
|
|
||||||
|
let mut target_metadata = (*source.user_defined).clone();
|
||||||
|
let epoch = OffsetDateTime::UNIX_EPOCH
|
||||||
|
.format(&time::format_description::well_known::Rfc3339)
|
||||||
|
.unwrap();
|
||||||
|
rustfs_utils::http::insert_str(&mut target_metadata, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP, epoch.clone());
|
||||||
|
rustfs_utils::http::insert_str(&mut target_metadata, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP, epoch);
|
||||||
|
assert!(is_equivalent_data_movement_delete_marker_metadata(&source.user_defined, &target_metadata));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn equivalent_data_movement_delete_marker_rejects_metadata_and_replication_mismatch() {
|
fn equivalent_data_movement_delete_marker_rejects_metadata_and_replication_mismatch() {
|
||||||
let version_id = Uuid::nil();
|
let version_id = Uuid::nil();
|
||||||
@@ -3344,6 +3859,67 @@ mod tests {
|
|||||||
assert!(is_equivalent_data_movement_tiered_object(&source, &target));
|
assert!(is_equivalent_data_movement_tiered_object(&source, &target));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn equivalent_data_movement_tiered_object_uses_logical_compressed_and_encrypted_sizes() {
|
||||||
|
let mut compressed = tiered_equivalence_source();
|
||||||
|
compressed.size = 600;
|
||||||
|
rustfs_utils::http::insert_str(&mut compressed.metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "S2".to_string());
|
||||||
|
rustfs_utils::http::insert_str(&mut compressed.metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1024".to_string());
|
||||||
|
let compressed_target = tiered_equivalence_target(&compressed);
|
||||||
|
assert!(is_equivalent_data_movement_tiered_object(&compressed, &compressed_target));
|
||||||
|
|
||||||
|
let mut encrypted = tiered_equivalence_source();
|
||||||
|
encrypted.size = 640;
|
||||||
|
encrypted.metadata.insert(
|
||||||
|
rustfs_utils::http::object_encryption_keys::INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(),
|
||||||
|
"key-id".to_string(),
|
||||||
|
);
|
||||||
|
encrypted.metadata.insert(
|
||||||
|
rustfs_utils::http::object_encryption_keys::INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER.to_string(),
|
||||||
|
"1024".to_string(),
|
||||||
|
);
|
||||||
|
let encrypted_target = tiered_equivalence_target(&encrypted);
|
||||||
|
assert!(is_equivalent_data_movement_tiered_object(&encrypted, &encrypted_target));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn equivalent_data_movement_tiered_object_accepts_transition_alias_expansion() {
|
||||||
|
let mut source = tiered_equivalence_source();
|
||||||
|
let suffix = rustfs_utils::http::SUFFIX_TRANSITION_TIER;
|
||||||
|
source.metadata.insert(
|
||||||
|
format!("{}{suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX),
|
||||||
|
source.transition_tier.clone(),
|
||||||
|
);
|
||||||
|
let mut target = tiered_equivalence_target(&source);
|
||||||
|
Arc::make_mut(&mut target.user_defined)
|
||||||
|
.insert(rustfs_utils::http::internal_key_rustfs(suffix), source.transition_tier.clone());
|
||||||
|
|
||||||
|
assert!(is_equivalent_data_movement_tiered_object(&source, &target));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn equivalent_data_movement_tiered_object_requires_hydrated_part_checksums() {
|
||||||
|
let mut source = tiered_equivalence_source();
|
||||||
|
source.parts = vec![rustfs_filemeta::ObjectPartInfo {
|
||||||
|
number: 1,
|
||||||
|
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND),
|
||||||
|
checksums: Some(HashMap::from([("CRC32C".to_string(), "AAAAAA==".to_string())])),
|
||||||
|
..Default::default()
|
||||||
|
}];
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
&mut source.metadata,
|
||||||
|
rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
|
||||||
|
r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string(),
|
||||||
|
);
|
||||||
|
let mut target = tiered_equivalence_target(&source);
|
||||||
|
Arc::make_mut(&mut target.parts)[0].mod_time = None;
|
||||||
|
assert!(is_equivalent_data_movement_tiered_object(&source, &target));
|
||||||
|
|
||||||
|
let mut missing = target;
|
||||||
|
Arc::make_mut(&mut missing.parts)[0].checksums = None;
|
||||||
|
assert!(!is_equivalent_data_movement_tiered_object(&source, &missing));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn equivalent_data_movement_tiered_object_rejects_transition_mismatch() {
|
fn equivalent_data_movement_tiered_object_rejects_transition_mismatch() {
|
||||||
let source = tiered_equivalence_source();
|
let source = tiered_equivalence_source();
|
||||||
@@ -3353,6 +3929,16 @@ mod tests {
|
|||||||
assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
|
assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn equivalent_data_movement_tiered_object_rejects_transition_version_state_mismatch() {
|
||||||
|
let mut source = tiered_equivalence_source();
|
||||||
|
source.transition_version_state = rustfs_filemeta::TransitionVersionState::Exact;
|
||||||
|
let mut target = tiered_equivalence_target(&source);
|
||||||
|
target.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown;
|
||||||
|
|
||||||
|
assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn equivalent_data_movement_tiered_object_rejects_user_metadata_mismatch() {
|
fn equivalent_data_movement_tiered_object_rejects_user_metadata_mismatch() {
|
||||||
let source = tiered_equivalence_source();
|
let source = tiered_equivalence_source();
|
||||||
|
|||||||
@@ -109,6 +109,7 @@ async fn run_legacy_bitrot_test_for_object(root: &std::path::Path, disk_name: &s
|
|||||||
FileInfoOpts {
|
FileInfoOpts {
|
||||||
data: true, // need inline data for inline objects
|
data: true, // need inline data for inline objects
|
||||||
include_free_versions: false,
|
include_free_versions: false,
|
||||||
|
include_part_checksums: true,
|
||||||
},
|
},
|
||||||
) {
|
) {
|
||||||
Ok(f) => f,
|
Ok(f) => f,
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ crc-fast = { workspace = true }
|
|||||||
rmp.workspace = true
|
rmp.workspace = true
|
||||||
rmp-serde.workspace = true
|
rmp-serde.workspace = true
|
||||||
serde = { workspace = true, features = ["derive"] }
|
serde = { workspace = true, features = ["derive"] }
|
||||||
|
serde_json.workspace = true
|
||||||
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||||
uuid = { workspace = true, features = ["v4", "fast-rng", "serde", "macro-diagnostics"] }
|
uuid = { workspace = true, features = ["v4", "fast-rng", "serde", "macro-diagnostics"] }
|
||||||
tokio = { workspace = true, features = ["io-util", "macros", "sync", "fs", "rt-multi-thread"] }
|
tokio = { workspace = true, features = ["io-util", "macros", "sync", "fs", "rt-multi-thread"] }
|
||||||
@@ -54,7 +55,6 @@ arc-swap.workspace = true
|
|||||||
criterion = { workspace = true, features = ["html_reports"] }
|
criterion = { workspace = true, features = ["html_reports"] }
|
||||||
tempfile = { workspace = true }
|
tempfile = { workspace = true }
|
||||||
proptest = "1"
|
proptest = "1"
|
||||||
serde_json.workspace = true
|
|
||||||
|
|
||||||
[[bench]]
|
[[bench]]
|
||||||
name = "xl_meta_bench"
|
name = "xl_meta_bench"
|
||||||
|
|||||||
@@ -52,6 +52,7 @@ fn main() {
|
|||||||
FileInfoOpts {
|
FileInfoOpts {
|
||||||
data: false,
|
data: false,
|
||||||
include_free_versions: true,
|
include_free_versions: true,
|
||||||
|
include_part_checksums: true,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.expect("decode file info");
|
.expect("decode file info");
|
||||||
|
|||||||
@@ -17,9 +17,9 @@ use bytes::Bytes;
|
|||||||
use rmp_serde::Serializer;
|
use rmp_serde::Serializer;
|
||||||
use rustfs_utils::HashAlgorithm;
|
use rustfs_utils::HashAlgorithm;
|
||||||
use rustfs_utils::http::{
|
use rustfs_utils::http::{
|
||||||
SUFFIX_COMPRESSION, SUFFIX_DATA_MOVED, SUFFIX_FREE_VERSION, SUFFIX_HEALING, SUFFIX_INLINE_DATA, SUFFIX_TIER_FV_ID,
|
AMZ_OBJECT_TAGGING, SUFFIX_COMPRESSION, SUFFIX_DATA_MOVED, SUFFIX_DATA_MOVED_TAGS, SUFFIX_FREE_VERSION, SUFFIX_HEALING,
|
||||||
SUFFIX_TIER_FV_MARKER, SUFFIX_TIER_SKIP_FV_ID, contains_key_str, get_str, has_internal_suffix, insert_str,
|
SUFFIX_INLINE_DATA, SUFFIX_TIER_FV_ID, SUFFIX_TIER_FV_MARKER, SUFFIX_TIER_SKIP_FV_ID, contains_key_str, get_str,
|
||||||
is_encryption_metadata_key, starts_with_ignore_ascii_case,
|
has_internal_suffix, insert_str, is_encryption_metadata_key, starts_with_ignore_ascii_case,
|
||||||
};
|
};
|
||||||
use s3s::dto::{RestoreStatus, Timestamp};
|
use s3s::dto::{RestoreStatus, Timestamp};
|
||||||
use s3s::header::X_AMZ_RESTORE;
|
use s3s::header::X_AMZ_RESTORE;
|
||||||
@@ -232,6 +232,17 @@ pub enum TransitionVersionState {
|
|||||||
Exact,
|
Exact,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl TransitionVersionState {
|
||||||
|
pub const fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Unknown => "unknown",
|
||||||
|
Self::KnownDisabled => "known-disabled",
|
||||||
|
Self::SuspendedNull => "suspended-null",
|
||||||
|
Self::Exact => "exact",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(PartialEq, Clone, Default)]
|
#[derive(PartialEq, Clone, Default)]
|
||||||
pub struct FileInfo {
|
pub struct FileInfo {
|
||||||
pub volume: String,
|
pub volume: String,
|
||||||
@@ -1151,9 +1162,16 @@ impl FileInfo {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn set_data_moved(&mut self) {
|
pub fn set_data_moved(&mut self) {
|
||||||
|
let tags_proof = format!("v1:{}", self.metadata.get(AMZ_OBJECT_TAGGING).map(String::as_str).unwrap_or_default());
|
||||||
|
insert_str(&mut self.metadata, SUFFIX_DATA_MOVED_TAGS, tags_proof);
|
||||||
insert_str(&mut self.metadata, SUFFIX_DATA_MOVED, "true".to_string());
|
insert_str(&mut self.metadata, SUFFIX_DATA_MOVED, "true".to_string());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn acknowledge_data_movement(&mut self) {
|
||||||
|
// Keep both empty aliases so mixed-version disks retain one metadata identity.
|
||||||
|
insert_str(&mut self.metadata, SUFFIX_DATA_MOVED, String::new());
|
||||||
|
}
|
||||||
|
|
||||||
pub fn inline_data(&self) -> bool {
|
pub fn inline_data(&self) -> bool {
|
||||||
contains_key_str(&self.metadata, SUFFIX_INLINE_DATA) && !self.is_remote()
|
contains_key_str(&self.metadata, SUFFIX_INLINE_DATA) && !self.is_remote()
|
||||||
}
|
}
|
||||||
|
|||||||
+156
-97
@@ -206,6 +206,42 @@ fn persist_reset_statuses(meta_sys: &mut HashMap<String, Vec<u8>>, reset_statuse
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn parse_replication_timestamp(value: &str) -> Option<OffsetDateTime> {
|
||||||
|
const DISPLAY_FORMAT: &[time::format_description::BorrowedFormatItem<'_>] = time::macros::format_description!(
|
||||||
|
"[year sign:automatic]-[month]-[day] [hour padding:none]:[minute]:[second].[subsecond] [offset_hour sign:mandatory]:[offset_minute]:[offset_second]"
|
||||||
|
);
|
||||||
|
OffsetDateTime::parse(value, &Rfc3339)
|
||||||
|
.or_else(|_| OffsetDateTime::parse(value, DISPLAY_FORMAT))
|
||||||
|
.ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn format_replication_timestamp(value: Option<OffsetDateTime>) -> String {
|
||||||
|
let value = value.unwrap_or(OffsetDateTime::UNIX_EPOCH);
|
||||||
|
value
|
||||||
|
.to_offset(time::UtcOffset::UTC)
|
||||||
|
.format(&Rfc3339)
|
||||||
|
.unwrap_or_else(|_| value.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn persist_delete_marker_replication_state(meta_sys: &mut HashMap<String, Vec<u8>>, state: &ReplicationState) {
|
||||||
|
if !state.replica_status.is_empty() {
|
||||||
|
insert_bytes(meta_sys, SUFFIX_REPLICA_STATUS, state.replica_status.as_str().as_bytes().to_vec());
|
||||||
|
insert_bytes(
|
||||||
|
meta_sys,
|
||||||
|
SUFFIX_REPLICA_TIMESTAMP,
|
||||||
|
format_replication_timestamp(state.replica_timestamp).into_bytes(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if let Some(status) = state.replication_status_internal.as_ref().filter(|status| !status.is_empty()) {
|
||||||
|
insert_bytes(meta_sys, SUFFIX_REPLICATION_STATUS, status.as_bytes().to_vec());
|
||||||
|
insert_bytes(
|
||||||
|
meta_sys,
|
||||||
|
SUFFIX_REPLICATION_TIMESTAMP,
|
||||||
|
format_replication_timestamp(state.replication_timestamp).into_bytes(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||||
pub struct FileMeta {
|
pub struct FileMeta {
|
||||||
pub versions: Vec<FileMetaShallowVersion>,
|
pub versions: Vec<FileMetaShallowVersion>,
|
||||||
@@ -213,6 +249,13 @@ pub struct FileMeta {
|
|||||||
pub meta_ver: u8,
|
pub meta_ver: u8,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct FileInfoDecodeOptions {
|
||||||
|
read_data: bool,
|
||||||
|
include_free_versions: bool,
|
||||||
|
all_parts: bool,
|
||||||
|
include_part_checksums: bool,
|
||||||
|
}
|
||||||
|
|
||||||
impl FileMeta {
|
impl FileMeta {
|
||||||
pub fn new() -> Self {
|
pub fn new() -> Self {
|
||||||
Self {
|
Self {
|
||||||
@@ -511,53 +554,8 @@ impl FileMeta {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if fi.deleted {
|
if fi.deleted {
|
||||||
if !fi.delete_marker_replication_status().is_empty()
|
if let (Some(delete_marker), Some(state)) = (ventry.delete_marker.as_mut(), fi.replication_state_internal.as_ref()) {
|
||||||
&& let Some(delete_marker) = ventry.delete_marker.as_mut()
|
persist_delete_marker_replication_state(&mut delete_marker.meta_sys, state);
|
||||||
{
|
|
||||||
if fi.delete_marker_replication_status() == ReplicationStatusType::Replica {
|
|
||||||
insert_bytes(
|
|
||||||
&mut delete_marker.meta_sys,
|
|
||||||
SUFFIX_REPLICA_STATUS,
|
|
||||||
fi.replication_state_internal
|
|
||||||
.as_ref()
|
|
||||||
.map(|v| v.replica_status.clone())
|
|
||||||
.unwrap_or_default()
|
|
||||||
.as_str()
|
|
||||||
.as_bytes()
|
|
||||||
.to_vec(),
|
|
||||||
);
|
|
||||||
insert_bytes(
|
|
||||||
&mut delete_marker.meta_sys,
|
|
||||||
SUFFIX_REPLICA_TIMESTAMP,
|
|
||||||
fi.replication_state_internal
|
|
||||||
.as_ref()
|
|
||||||
.map(|v| v.replica_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string())
|
|
||||||
.unwrap_or_default()
|
|
||||||
.as_bytes()
|
|
||||||
.to_vec(),
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
insert_bytes(
|
|
||||||
&mut delete_marker.meta_sys,
|
|
||||||
SUFFIX_REPLICATION_STATUS,
|
|
||||||
fi.replication_state_internal
|
|
||||||
.as_ref()
|
|
||||||
.map(|v| v.replication_status_internal.clone().unwrap_or_default())
|
|
||||||
.unwrap_or_default()
|
|
||||||
.as_bytes()
|
|
||||||
.to_vec(),
|
|
||||||
);
|
|
||||||
insert_bytes(
|
|
||||||
&mut delete_marker.meta_sys,
|
|
||||||
SUFFIX_REPLICATION_TIMESTAMP,
|
|
||||||
fi.replication_state_internal
|
|
||||||
.as_ref()
|
|
||||||
.map(|v| v.replication_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string())
|
|
||||||
.unwrap_or_default()
|
|
||||||
.as_bytes()
|
|
||||||
.to_vec(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if !fi.version_purge_status().is_empty()
|
if !fi.version_purge_status().is_empty()
|
||||||
@@ -609,51 +607,8 @@ impl FileMeta {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if let Some(delete_marker) = v.delete_marker.as_mut() {
|
if let Some(delete_marker) = v.delete_marker.as_mut() {
|
||||||
if !fi.delete_marker_replication_status().is_empty() {
|
if let Some(state) = fi.replication_state_internal.as_ref() {
|
||||||
if fi.delete_marker_replication_status() == ReplicationStatusType::Replica {
|
persist_delete_marker_replication_state(&mut delete_marker.meta_sys, state);
|
||||||
insert_bytes(
|
|
||||||
&mut delete_marker.meta_sys,
|
|
||||||
SUFFIX_REPLICA_STATUS,
|
|
||||||
fi.replication_state_internal
|
|
||||||
.as_ref()
|
|
||||||
.map(|v| v.replica_status.clone())
|
|
||||||
.unwrap_or_default()
|
|
||||||
.as_str()
|
|
||||||
.as_bytes()
|
|
||||||
.to_vec(),
|
|
||||||
);
|
|
||||||
insert_bytes(
|
|
||||||
&mut delete_marker.meta_sys,
|
|
||||||
SUFFIX_REPLICA_TIMESTAMP,
|
|
||||||
fi.replication_state_internal
|
|
||||||
.as_ref()
|
|
||||||
.map(|v| v.replica_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string())
|
|
||||||
.unwrap_or_default()
|
|
||||||
.as_bytes()
|
|
||||||
.to_vec(),
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
insert_bytes(
|
|
||||||
&mut delete_marker.meta_sys,
|
|
||||||
SUFFIX_REPLICATION_STATUS,
|
|
||||||
fi.replication_state_internal
|
|
||||||
.as_ref()
|
|
||||||
.map(|v| v.replication_status_internal.clone().unwrap_or_default())
|
|
||||||
.unwrap_or_default()
|
|
||||||
.as_bytes()
|
|
||||||
.to_vec(),
|
|
||||||
);
|
|
||||||
insert_bytes(
|
|
||||||
&mut delete_marker.meta_sys,
|
|
||||||
SUFFIX_REPLICATION_TIMESTAMP,
|
|
||||||
fi.replication_state_internal
|
|
||||||
.as_ref()
|
|
||||||
.map(|v| v.replication_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string())
|
|
||||||
.unwrap_or_default()
|
|
||||||
.as_bytes()
|
|
||||||
.to_vec(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(state) = fi.replication_state_internal.as_ref() {
|
if let Some(state) = fi.replication_state_internal.as_ref() {
|
||||||
@@ -773,6 +728,47 @@ impl FileMeta {
|
|||||||
read_data: bool,
|
read_data: bool,
|
||||||
include_free_versions: bool,
|
include_free_versions: bool,
|
||||||
all_parts: bool,
|
all_parts: bool,
|
||||||
|
) -> Result<FileInfo> {
|
||||||
|
self.to_fileinfo_with_part_checksums(
|
||||||
|
volume,
|
||||||
|
path,
|
||||||
|
version_id,
|
||||||
|
FileInfoDecodeOptions {
|
||||||
|
read_data,
|
||||||
|
include_free_versions,
|
||||||
|
all_parts,
|
||||||
|
include_part_checksums: true,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn into_fileinfo_without_part_checksums(
|
||||||
|
&self,
|
||||||
|
volume: &str,
|
||||||
|
path: &str,
|
||||||
|
version_id: &str,
|
||||||
|
read_data: bool,
|
||||||
|
include_free_versions: bool,
|
||||||
|
) -> Result<FileInfo> {
|
||||||
|
self.to_fileinfo_with_part_checksums(
|
||||||
|
volume,
|
||||||
|
path,
|
||||||
|
version_id,
|
||||||
|
FileInfoDecodeOptions {
|
||||||
|
read_data,
|
||||||
|
include_free_versions,
|
||||||
|
all_parts: true,
|
||||||
|
include_part_checksums: false,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn to_fileinfo_with_part_checksums(
|
||||||
|
&self,
|
||||||
|
volume: &str,
|
||||||
|
path: &str,
|
||||||
|
version_id: &str,
|
||||||
|
opts: FileInfoDecodeOptions,
|
||||||
) -> Result<FileInfo> {
|
) -> Result<FileInfo> {
|
||||||
let vid = {
|
let vid = {
|
||||||
if !version_id.is_empty() {
|
if !version_id.is_empty() {
|
||||||
@@ -795,7 +791,7 @@ impl FileMeta {
|
|||||||
|
|
||||||
if header.free_version() {
|
if header.free_version() {
|
||||||
non_free_versions -= 1;
|
non_free_versions -= 1;
|
||||||
if include_free_versions
|
if opts.include_free_versions
|
||||||
&& found_free_version.is_none()
|
&& found_free_version.is_none()
|
||||||
&& let Ok(found_free_fi) = ver.parse_version_meta()
|
&& let Ok(found_free_fi) = ver.parse_version_meta()
|
||||||
&& found_free_fi.version_type != VersionType::Invalid
|
&& found_free_fi.version_type != VersionType::Invalid
|
||||||
@@ -806,7 +802,8 @@ impl FileMeta {
|
|||||||
// Known side effect: if a disk holds only free versions and they are
|
// Known side effect: if a disk holds only free versions and they are
|
||||||
// corrupt, `into_fileinfo` falls through to `FileNotFound` (not
|
// corrupt, `into_fileinfo` falls through to `FileNotFound` (not
|
||||||
// `FileCorrupt`), so that disk is not enqueued for heal.
|
// `FileCorrupt`), so that disk is not enqueued for heal.
|
||||||
match found_free_fi.into_fileinfo(volume, path, all_parts) {
|
match found_free_fi.to_fileinfo_with_part_checksums(volume, path, opts.all_parts, opts.include_part_checksums)
|
||||||
|
{
|
||||||
Ok(mut free_fi) => {
|
Ok(mut free_fi) => {
|
||||||
free_fi.is_latest = true;
|
free_fi.is_latest = true;
|
||||||
found_free_version = Some(free_fi);
|
found_free_version = Some(free_fi);
|
||||||
@@ -834,14 +831,14 @@ impl FileMeta {
|
|||||||
|
|
||||||
found = true;
|
found = true;
|
||||||
|
|
||||||
let mut fi = ver.into_fileinfo(volume, path, all_parts)?;
|
let mut fi = ver.to_fileinfo_with_part_checksums(volume, path, opts.all_parts, opts.include_part_checksums)?;
|
||||||
fi.is_latest = is_latest;
|
fi.is_latest = is_latest;
|
||||||
|
|
||||||
if let Some(_d) = succ_mod_time {
|
if let Some(_d) = succ_mod_time {
|
||||||
fi.successor_mod_time = succ_mod_time;
|
fi.successor_mod_time = succ_mod_time;
|
||||||
}
|
}
|
||||||
|
|
||||||
if read_data && fi.inline_data() {
|
if opts.read_data && fi.inline_data() {
|
||||||
fi.data = self.find_inline_data_for_version(fi.version_id)?.map(bytes::Bytes::from);
|
fi.data = self.find_inline_data_for_version(fi.version_id)?.map(bytes::Bytes::from);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -850,7 +847,7 @@ impl FileMeta {
|
|||||||
|
|
||||||
if !found {
|
if !found {
|
||||||
if version_id.is_empty() {
|
if version_id.is_empty() {
|
||||||
if include_free_versions
|
if opts.include_free_versions
|
||||||
&& non_free_versions == 0
|
&& non_free_versions == 0
|
||||||
&& let Some(free_version) = found_free_version
|
&& let Some(free_version) = found_free_version
|
||||||
{
|
{
|
||||||
@@ -1537,6 +1534,68 @@ mod test {
|
|||||||
assert_eq!(meta_sys2.len(), 2, "must not create a double-prefixed key");
|
assert_eq!(meta_sys2.len(), 2, "must not create a double-prefixed key");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn persist_delete_marker_replication_state_keeps_replica_and_target_statuses() {
|
||||||
|
let replica_timestamp = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
|
||||||
|
let replication_timestamp = replica_timestamp + time::Duration::SECOND;
|
||||||
|
let state = ReplicationState {
|
||||||
|
replica_status: ReplicationStatusType::Replica,
|
||||||
|
replica_timestamp: Some(replica_timestamp),
|
||||||
|
replication_status_internal: Some("arn:target=COMPLETED;".to_string()),
|
||||||
|
replication_timestamp: Some(replication_timestamp),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let mut meta_sys = HashMap::new();
|
||||||
|
let replica_timestamp_string = replica_timestamp
|
||||||
|
.format(&Rfc3339)
|
||||||
|
.expect("timestamp should format as RFC3339");
|
||||||
|
let replication_timestamp_string = replication_timestamp
|
||||||
|
.format(&Rfc3339)
|
||||||
|
.expect("timestamp should format as RFC3339");
|
||||||
|
|
||||||
|
persist_delete_marker_replication_state(&mut meta_sys, &state);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
rustfs_utils::http::get_bytes(&meta_sys, SUFFIX_REPLICA_STATUS).as_deref(),
|
||||||
|
Some(b"REPLICA".as_slice())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
rustfs_utils::http::get_bytes(&meta_sys, SUFFIX_REPLICA_TIMESTAMP).as_deref(),
|
||||||
|
Some(replica_timestamp_string.as_bytes())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
rustfs_utils::http::get_bytes(&meta_sys, SUFFIX_REPLICATION_STATUS).as_deref(),
|
||||||
|
Some(b"arn:target=COMPLETED;".as_slice())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
rustfs_utils::http::get_bytes(&meta_sys, SUFFIX_REPLICATION_TIMESTAMP).as_deref(),
|
||||||
|
Some(replication_timestamp_string.as_bytes())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn persist_delete_marker_replication_timestamp_normalizes_second_offset_to_utc() {
|
||||||
|
let timestamp = OffsetDateTime::UNIX_EPOCH.to_offset(time::UtcOffset::from_hms(5, 30, 15).expect("valid offset"));
|
||||||
|
assert_eq!(parse_replication_timestamp(×tamp.to_string()), Some(timestamp));
|
||||||
|
let state = ReplicationState {
|
||||||
|
replica_status: ReplicationStatusType::Replica,
|
||||||
|
replica_timestamp: Some(timestamp),
|
||||||
|
replication_status_internal: Some("arn:target=COMPLETED;".to_string()),
|
||||||
|
replication_timestamp: Some(timestamp),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let mut meta_sys = HashMap::new();
|
||||||
|
|
||||||
|
persist_delete_marker_replication_state(&mut meta_sys, &state);
|
||||||
|
|
||||||
|
for suffix in [SUFFIX_REPLICA_TIMESTAMP, SUFFIX_REPLICATION_TIMESTAMP] {
|
||||||
|
let persisted = rustfs_utils::http::get_bytes(&meta_sys, suffix).expect("timestamp must be persisted");
|
||||||
|
let persisted = std::str::from_utf8(&persisted).expect("timestamp must be UTF-8");
|
||||||
|
assert_eq!(parse_replication_timestamp(persisted), Some(timestamp));
|
||||||
|
assert_ne!(persisted, OffsetDateTime::UNIX_EPOCH.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Regression test for rustfs/rustfs#2715: a corrupted version count in
|
/// Regression test for rustfs/rustfs#2715: a corrupted version count in
|
||||||
/// xl.meta must yield a decode error instead of sizing a huge allocation
|
/// xl.meta must yield a decode error instead of sizing a huge allocation
|
||||||
/// from the bogus count (which aborts the whole process).
|
/// from the bogus count (which aborts the whole process).
|
||||||
|
|||||||
@@ -29,11 +29,12 @@ use super::*;
|
|||||||
use crate::{ChecksumInfo, TransitionVersionState};
|
use crate::{ChecksumInfo, TransitionVersionState};
|
||||||
use rustfs_utils::HashAlgorithm;
|
use rustfs_utils::HashAlgorithm;
|
||||||
use rustfs_utils::http::{
|
use rustfs_utils::http::{
|
||||||
RUSTFS_INTERNAL_PREFIX, SUFFIX_CRC, SUFFIX_FREE_VERSION, SUFFIX_INLINE_DATA, SUFFIX_PURGESTATUS, SUFFIX_TIER_FV_ID,
|
RUSTFS_INTERNAL_PREFIX, SUFFIX_CRC, SUFFIX_FREE_VERSION, SUFFIX_INLINE_DATA, SUFFIX_PART_CHECKSUMS, SUFFIX_PURGESTATUS,
|
||||||
|
SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX, SUFFIX_REPLICATION_RESET_ARN_PREFIX, SUFFIX_TIER_FV_ID,
|
||||||
SUFFIX_TIER_FV_MARKER, SUFFIX_TRANSITION_STATUS, SUFFIX_TRANSITION_TIER, SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
SUFFIX_TIER_FV_MARKER, SUFFIX_TRANSITION_STATUS, SUFFIX_TRANSITION_TIER, SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||||
SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, contains_key_bytes,
|
SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, contains_key_bytes,
|
||||||
get_bytes, get_consistent_bytes, get_str, has_internal_suffix, insert_bytes, is_internal_key, remove_bytes,
|
get_bytes, get_consistent_bytes, get_str, has_internal_suffix, insert_bytes, is_internal_key, remove_bytes,
|
||||||
strip_internal_prefix, target_delete_marker_versions,
|
strip_internal_prefix, strip_internal_prefix_preserving_case, target_delete_marker_versions,
|
||||||
};
|
};
|
||||||
|
|
||||||
const MSGPACK_EXT8: u8 = 0xc7;
|
const MSGPACK_EXT8: u8 = 0xc7;
|
||||||
@@ -258,63 +259,181 @@ fn parse_legacy_uuid_bytes(bytes: &[u8], field: &str) -> Result<Option<Uuid>> {
|
|||||||
/// Legacy RustFS writes used 16 raw UUID bytes. New writes and MinIO-migrated
|
/// Legacy RustFS writes used 16 raw UUID bytes. New writes and MinIO-migrated
|
||||||
/// records use the provider's exact UTF-8 version text. Empty, nil UUID, and
|
/// records use the provider's exact UTF-8 version text. Empty, nil UUID, and
|
||||||
/// malformed bytes are not usable remote versions.
|
/// malformed bytes are not usable remote versions.
|
||||||
fn transitioned_version_from_meta_sys(meta_sys: &HashMap<String, Vec<u8>>) -> Result<Option<String>> {
|
fn transition_version_state_from_bytes(value: Option<&[u8]>) -> Result<TransitionVersionState> {
|
||||||
if !contains_key_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_ID) {
|
let Some(value) = value else {
|
||||||
return Ok(None);
|
return Ok(TransitionVersionState::Unknown);
|
||||||
}
|
|
||||||
let Some(value) = get_consistent_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_ID) else {
|
|
||||||
return Ok(None);
|
|
||||||
};
|
};
|
||||||
let value = value.to_vec();
|
match value {
|
||||||
|
b"known-disabled" => Ok(TransitionVersionState::KnownDisabled),
|
||||||
|
b"suspended-null" => Ok(TransitionVersionState::SuspendedNull),
|
||||||
|
b"exact" => Ok(TransitionVersionState::Exact),
|
||||||
|
b"unknown" => Ok(TransitionVersionState::Unknown),
|
||||||
|
_ => Err(Error::FileCorrupt),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn transitioned_version_from_bytes(value: Option<&[u8]>, state: TransitionVersionState) -> Option<String> {
|
||||||
|
let value = value?;
|
||||||
if value.is_empty() {
|
if value.is_empty() {
|
||||||
return Ok(None);
|
return None;
|
||||||
}
|
}
|
||||||
if let Ok(id) = Uuid::from_slice(&value) {
|
if state == TransitionVersionState::Unknown
|
||||||
return Ok((!id.is_nil()).then(|| id.to_string()));
|
&& let Ok(id) = Uuid::from_slice(value)
|
||||||
|
{
|
||||||
|
return (!id.is_nil()).then(|| id.to_string());
|
||||||
}
|
}
|
||||||
let Ok(value) = String::from_utf8(value) else {
|
let Ok(value) = std::str::from_utf8(value) else {
|
||||||
return Ok(None);
|
return None;
|
||||||
};
|
};
|
||||||
if value.is_empty()
|
if value.is_empty()
|
||||||
|| value.len() > MAX_TRANSITION_VERSION_LEN
|
|| value.len() > MAX_TRANSITION_VERSION_LEN
|
||||||
|| value.chars().any(char::is_control)
|
|| value.chars().any(char::is_control)
|
||||||
|| Uuid::parse_str(&value).is_ok_and(|id| id.is_nil())
|
|| Uuid::parse_str(value).is_ok_and(|id| id.is_nil())
|
||||||
{
|
{
|
||||||
Ok(None)
|
None
|
||||||
} else {
|
} else {
|
||||||
Ok(Some(value))
|
Some(value.to_string())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn transition_version_state_from_meta_sys(
|
fn validate_transition_version_state(state: TransitionVersionState, version: Option<&str>) -> Result<()> {
|
||||||
meta_sys: &HashMap<String, Vec<u8>>,
|
|
||||||
version: Option<&str>,
|
|
||||||
) -> Result<TransitionVersionState> {
|
|
||||||
if !contains_key_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE) {
|
|
||||||
return Ok(TransitionVersionState::Unknown);
|
|
||||||
}
|
|
||||||
let value = get_consistent_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE).ok_or(Error::FileCorrupt)?;
|
|
||||||
let state = match value {
|
|
||||||
b"known-disabled" => TransitionVersionState::KnownDisabled,
|
|
||||||
b"suspended-null" => TransitionVersionState::SuspendedNull,
|
|
||||||
b"exact" => TransitionVersionState::Exact,
|
|
||||||
b"unknown" => TransitionVersionState::Unknown,
|
|
||||||
_ => return Err(Error::FileCorrupt),
|
|
||||||
};
|
|
||||||
let valid = match state {
|
let valid = match state {
|
||||||
TransitionVersionState::Unknown | TransitionVersionState::KnownDisabled => version.is_none(),
|
TransitionVersionState::Unknown | TransitionVersionState::KnownDisabled => version.is_none(),
|
||||||
TransitionVersionState::SuspendedNull => version == Some("null"),
|
TransitionVersionState::SuspendedNull => version == Some("null"),
|
||||||
TransitionVersionState::Exact => version.is_some_and(|value| value != "null"),
|
TransitionVersionState::Exact => version.is_some_and(|value| value != "null"),
|
||||||
};
|
};
|
||||||
valid.then_some(state).ok_or(Error::FileCorrupt)
|
valid.then_some(()).ok_or(Error::FileCorrupt)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn transition_version_state_bytes(state: TransitionVersionState) -> &'static [u8] {
|
#[derive(Default)]
|
||||||
match state {
|
struct DerivedInternalMetadata<'a> {
|
||||||
TransitionVersionState::Unknown => b"unknown",
|
checksum: Option<&'a [u8]>,
|
||||||
TransitionVersionState::KnownDisabled => b"known-disabled",
|
part_checksums: Option<&'a [u8]>,
|
||||||
TransitionVersionState::SuspendedNull => b"suspended-null",
|
transition_status: Option<&'a [u8]>,
|
||||||
TransitionVersionState::Exact => b"exact",
|
transitioned_object: Option<&'a [u8]>,
|
||||||
|
transitioned_version: Option<&'a [u8]>,
|
||||||
|
transitioned_version_state: Option<&'a [u8]>,
|
||||||
|
transition_tier: Option<&'a [u8]>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> DerivedInternalMetadata<'a> {
|
||||||
|
fn from_meta_sys(meta_sys: &'a HashMap<String, Vec<u8>>) -> Result<Self> {
|
||||||
|
let mut canonical = Self::default();
|
||||||
|
let mut legacy = Self::default();
|
||||||
|
for (key, value) in meta_sys {
|
||||||
|
let Some(suffix) = rustfs_utils::http::strip_internal_prefix_preserving_case(key) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let (canonical_slot, legacy_slot, expected_suffix) = if suffix.eq_ignore_ascii_case(SUFFIX_CRC) {
|
||||||
|
(&mut canonical.checksum, &mut legacy.checksum, SUFFIX_CRC)
|
||||||
|
} else if suffix.eq_ignore_ascii_case(SUFFIX_PART_CHECKSUMS) {
|
||||||
|
(&mut canonical.part_checksums, &mut legacy.part_checksums, SUFFIX_PART_CHECKSUMS)
|
||||||
|
} else if suffix.eq_ignore_ascii_case(SUFFIX_TRANSITION_STATUS) {
|
||||||
|
(&mut canonical.transition_status, &mut legacy.transition_status, SUFFIX_TRANSITION_STATUS)
|
||||||
|
} else if suffix.eq_ignore_ascii_case(SUFFIX_TRANSITIONED_OBJECTNAME) {
|
||||||
|
(
|
||||||
|
&mut canonical.transitioned_object,
|
||||||
|
&mut legacy.transitioned_object,
|
||||||
|
SUFFIX_TRANSITIONED_OBJECTNAME,
|
||||||
|
)
|
||||||
|
} else if suffix.eq_ignore_ascii_case(SUFFIX_TRANSITIONED_VERSION_ID) {
|
||||||
|
(
|
||||||
|
&mut canonical.transitioned_version,
|
||||||
|
&mut legacy.transitioned_version,
|
||||||
|
SUFFIX_TRANSITIONED_VERSION_ID,
|
||||||
|
)
|
||||||
|
} else if suffix.eq_ignore_ascii_case(SUFFIX_TRANSITIONED_VERSION_STATE) {
|
||||||
|
(
|
||||||
|
&mut canonical.transitioned_version_state,
|
||||||
|
&mut legacy.transitioned_version_state,
|
||||||
|
SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||||
|
)
|
||||||
|
} else if suffix.eq_ignore_ascii_case(SUFFIX_TRANSITION_TIER) {
|
||||||
|
(&mut canonical.transition_tier, &mut legacy.transition_tier, SUFFIX_TRANSITION_TIER)
|
||||||
|
} else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let slot = if suffix == expected_suffix
|
||||||
|
&& (key.starts_with(RUSTFS_INTERNAL_PREFIX) || key.starts_with(rustfs_utils::http::MINIO_INTERNAL_PREFIX))
|
||||||
|
{
|
||||||
|
canonical_slot
|
||||||
|
} else {
|
||||||
|
legacy_slot
|
||||||
|
};
|
||||||
|
if slot.is_some_and(|current| current != value.as_slice()) {
|
||||||
|
return Err(Error::FileCorrupt);
|
||||||
|
}
|
||||||
|
*slot = Some(value.as_slice());
|
||||||
|
}
|
||||||
|
Ok(Self {
|
||||||
|
checksum: canonical.checksum.or(legacy.checksum),
|
||||||
|
part_checksums: canonical.part_checksums.or(legacy.part_checksums),
|
||||||
|
transition_status: canonical.transition_status.or(legacy.transition_status),
|
||||||
|
transitioned_object: canonical.transitioned_object.or(legacy.transitioned_object),
|
||||||
|
transitioned_version: canonical.transitioned_version.or(legacy.transitioned_version),
|
||||||
|
transitioned_version_state: canonical.transitioned_version_state.or(legacy.transitioned_version_state),
|
||||||
|
transition_tier: canonical.transition_tier.or(legacy.transition_tier),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct UniquePartChecksums(HashMap<String, String>);
|
||||||
|
|
||||||
|
impl<'de> serde::Deserialize<'de> for UniquePartChecksums {
|
||||||
|
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
|
||||||
|
where
|
||||||
|
D: serde::Deserializer<'de>,
|
||||||
|
{
|
||||||
|
struct UniquePartChecksumsVisitor;
|
||||||
|
|
||||||
|
impl<'de> serde::de::Visitor<'de> for UniquePartChecksumsVisitor {
|
||||||
|
type Value = UniquePartChecksums;
|
||||||
|
|
||||||
|
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
formatter.write_str("an array of unique checksum name and value pairs")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn visit_seq<A>(self, mut seq: A) -> std::result::Result<Self::Value, A::Error>
|
||||||
|
where
|
||||||
|
A: serde::de::SeqAccess<'de>,
|
||||||
|
{
|
||||||
|
let mut checksums = HashMap::with_capacity(seq.size_hint().unwrap_or_default());
|
||||||
|
while let Some((key, value)) = seq.next_element::<(String, String)>()? {
|
||||||
|
if checksums.insert(key, value).is_some() {
|
||||||
|
return Err(serde::de::Error::custom("duplicate part checksum name"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(UniquePartChecksums(checksums))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
deserializer.deserialize_seq(UniquePartChecksumsVisitor)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FileInfo {
|
||||||
|
pub fn hydrate_data_movement_part_checksums(&mut self) -> Result<()> {
|
||||||
|
let present = self
|
||||||
|
.metadata
|
||||||
|
.keys()
|
||||||
|
.any(|key| has_internal_suffix(key, SUFFIX_PART_CHECKSUMS));
|
||||||
|
if !present {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let encoded = rustfs_utils::http::get_consistent_str(&self.metadata, SUFFIX_PART_CHECKSUMS).ok_or(Error::FileCorrupt)?;
|
||||||
|
let persisted = serde_json::from_str::<Vec<(usize, UniquePartChecksums)>>(encoded).map_err(|_| Error::FileCorrupt)?;
|
||||||
|
let mut part_indices = HashMap::with_capacity(self.parts.len());
|
||||||
|
for (index, part) in self.parts.iter().enumerate() {
|
||||||
|
if part_indices.insert(part.number, index).is_some() {
|
||||||
|
return Err(Error::FileCorrupt);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (part_number, UniquePartChecksums(checksums)) in persisted {
|
||||||
|
let index = part_indices.remove(&part_number).ok_or(Error::FileCorrupt)?;
|
||||||
|
let part = self.parts.get_mut(index).ok_or(Error::FileCorrupt)?;
|
||||||
|
part.checksums = Some(checksums);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -322,21 +441,10 @@ fn set_transition_version_state(meta_sys: &mut HashMap<String, Vec<u8>>, state:
|
|||||||
if state == TransitionVersionState::Unknown {
|
if state == TransitionVersionState::Unknown {
|
||||||
remove_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE);
|
remove_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE);
|
||||||
} else {
|
} else {
|
||||||
insert_bytes(
|
insert_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE, state.as_str().as_bytes().to_vec());
|
||||||
meta_sys,
|
|
||||||
SUFFIX_TRANSITIONED_VERSION_STATE,
|
|
||||||
transition_version_state_bytes(state).to_vec(),
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn legacy_transitioned_version_id_from_meta_sys(meta_sys: &HashMap<String, Vec<u8>>) -> Option<Uuid> {
|
|
||||||
transitioned_version_from_meta_sys(meta_sys)
|
|
||||||
.ok()
|
|
||||||
.flatten()
|
|
||||||
.and_then(|value| Uuid::parse_str(&value).ok())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn transitioned_version_bytes(fi: &FileInfo) -> Option<Vec<u8>> {
|
fn transitioned_version_bytes(fi: &FileInfo) -> Option<Vec<u8>> {
|
||||||
fi.transition_version
|
fi.transition_version
|
||||||
.as_ref()
|
.as_ref()
|
||||||
@@ -448,6 +556,17 @@ impl FileMetaShallowVersion {
|
|||||||
pub fn into_fileinfo(&self, volume: &str, path: &str, all_parts: bool) -> Result<FileInfo> {
|
pub fn into_fileinfo(&self, volume: &str, path: &str, all_parts: bool) -> Result<FileInfo> {
|
||||||
self.parse_version_meta()?.into_fileinfo(volume, path, all_parts)
|
self.parse_version_meta()?.into_fileinfo(volume, path, all_parts)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(super) fn to_fileinfo_with_part_checksums(
|
||||||
|
&self,
|
||||||
|
volume: &str,
|
||||||
|
path: &str,
|
||||||
|
all_parts: bool,
|
||||||
|
include_part_checksums: bool,
|
||||||
|
) -> Result<FileInfo> {
|
||||||
|
self.parse_version_meta()?
|
||||||
|
.to_fileinfo_with_part_checksums(volume, path, all_parts, include_part_checksums)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl TryFrom<FileMetaVersion> for FileMetaShallowVersion {
|
impl TryFrom<FileMetaVersion> for FileMetaShallowVersion {
|
||||||
@@ -769,8 +888,16 @@ impl FileMetaVersion {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn into_fileinfo(&self, volume: &str, path: &str, all_parts: bool) -> Result<FileInfo> {
|
pub fn into_fileinfo(&self, volume: &str, path: &str, all_parts: bool) -> Result<FileInfo> {
|
||||||
// Only the Object arm carries part arrays and can fail the length guard; the
|
self.to_fileinfo_with_part_checksums(volume, path, all_parts, true)
|
||||||
// Legacy and Delete arms have no part arrays and stay infallible.
|
}
|
||||||
|
|
||||||
|
pub(super) fn to_fileinfo_with_part_checksums(
|
||||||
|
&self,
|
||||||
|
volume: &str,
|
||||||
|
path: &str,
|
||||||
|
all_parts: bool,
|
||||||
|
include_part_checksums: bool,
|
||||||
|
) -> Result<FileInfo> {
|
||||||
let mut fi = match self.version_type {
|
let mut fi = match self.version_type {
|
||||||
VersionType::Invalid | VersionType::Legacy => {
|
VersionType::Invalid | VersionType::Legacy => {
|
||||||
if let Some(ref legacy) = self.legacy_object {
|
if let Some(ref legacy) = self.legacy_object {
|
||||||
@@ -788,14 +915,14 @@ impl FileMetaVersion {
|
|||||||
self.object
|
self.object
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.unwrap_or(&default_object)
|
.unwrap_or(&default_object)
|
||||||
.into_fileinfo(volume, path, all_parts)?
|
.to_fileinfo_with_part_checksums(volume, path, all_parts, include_part_checksums)?
|
||||||
}
|
}
|
||||||
VersionType::Delete => {
|
VersionType::Delete => {
|
||||||
let default_marker = MetaDeleteMarker::default();
|
let default_marker = MetaDeleteMarker::default();
|
||||||
self.delete_marker
|
self.delete_marker
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.unwrap_or(&default_marker)
|
.unwrap_or(&default_marker)
|
||||||
.into_fileinfo(volume, path, all_parts)
|
.into_fileinfo(volume, path, all_parts)?
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
fi.uses_legacy_checksum = self.uses_legacy_checksum;
|
fi.uses_legacy_checksum = self.uses_legacy_checksum;
|
||||||
@@ -2390,7 +2517,18 @@ impl MetaObject {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn into_fileinfo(&self, volume: &str, path: &str, all_parts: bool) -> Result<FileInfo> {
|
pub fn into_fileinfo(&self, volume: &str, path: &str, all_parts: bool) -> Result<FileInfo> {
|
||||||
|
self.to_fileinfo_with_part_checksums(volume, path, all_parts, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn to_fileinfo_with_part_checksums(
|
||||||
|
&self,
|
||||||
|
volume: &str,
|
||||||
|
path: &str,
|
||||||
|
all_parts: bool,
|
||||||
|
include_part_checksums: bool,
|
||||||
|
) -> Result<FileInfo> {
|
||||||
let version_id = self.version_id.filter(|&vid| !vid.is_nil());
|
let version_id = self.version_id.filter(|&vid| !vid.is_nil());
|
||||||
|
let derived_metadata = DerivedInternalMetadata::from_meta_sys(&self.meta_sys)?;
|
||||||
|
|
||||||
let parts = if all_parts {
|
let parts = if all_parts {
|
||||||
let n = self.part_numbers.len();
|
let n = self.part_numbers.len();
|
||||||
@@ -2474,7 +2612,10 @@ impl MetaObject {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let checksum = get_bytes(&self.meta_sys, SUFFIX_CRC).map(Bytes::from);
|
let checksum = derived_metadata
|
||||||
|
.checksum
|
||||||
|
.filter(|checksum| !checksum.is_empty())
|
||||||
|
.map(Bytes::copy_from_slice);
|
||||||
|
|
||||||
let erasure = ErasureInfo {
|
let erasure = ErasureInfo {
|
||||||
algorithm: self.erasure_algorithm.to_string(),
|
algorithm: self.erasure_algorithm.to_string(),
|
||||||
@@ -2486,20 +2627,29 @@ impl MetaObject {
|
|||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
let transition_status = get_bytes(&self.meta_sys, SUFFIX_TRANSITION_STATUS)
|
let transition_status = derived_metadata
|
||||||
.map(|v| String::from_utf8_lossy(&v).to_string())
|
.transition_status
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(|v| String::from_utf8_lossy(v).to_string())
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
let transitioned_objname = get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_OBJECTNAME)
|
let transitioned_objname = derived_metadata
|
||||||
.map(|v| String::from_utf8_lossy(&v).to_string())
|
.transitioned_object
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(|v| String::from_utf8_lossy(v).to_string())
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
let transition_version = transitioned_version_from_meta_sys(&self.meta_sys)?;
|
let transition_version_state = transition_version_state_from_bytes(derived_metadata.transitioned_version_state)?;
|
||||||
let transition_version_state = transition_version_state_from_meta_sys(&self.meta_sys, transition_version.as_deref())?;
|
let transition_version = transitioned_version_from_bytes(derived_metadata.transitioned_version, transition_version_state);
|
||||||
|
if derived_metadata.transitioned_version_state.is_some() {
|
||||||
|
validate_transition_version_state(transition_version_state, transition_version.as_deref())?;
|
||||||
|
}
|
||||||
let transition_version_id = transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok());
|
let transition_version_id = transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok());
|
||||||
let transition_tier = get_bytes(&self.meta_sys, SUFFIX_TRANSITION_TIER)
|
let transition_tier = derived_metadata
|
||||||
.map(|v| String::from_utf8_lossy(&v).to_string())
|
.transition_tier
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(|v| String::from_utf8_lossy(v).to_string())
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
Ok(FileInfo {
|
let mut file_info = FileInfo {
|
||||||
version_id,
|
version_id,
|
||||||
erasure,
|
erasure,
|
||||||
data_dir: self.data_dir,
|
data_dir: self.data_dir,
|
||||||
@@ -2519,7 +2669,11 @@ impl MetaObject {
|
|||||||
transition_version_state,
|
transition_version_state,
|
||||||
transition_tier,
|
transition_tier,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
})
|
};
|
||||||
|
if all_parts && include_part_checksums {
|
||||||
|
file_info.hydrate_data_movement_part_checksums()?;
|
||||||
|
}
|
||||||
|
Ok(file_info)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn set_transition(&mut self, fi: &FileInfo) {
|
pub fn set_transition(&mut self, fi: &FileInfo) {
|
||||||
@@ -2710,39 +2864,31 @@ fn get_internal_replication_state(metadata: &HashMap<String, String>) -> Option<
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
let sub_key_opt = strip_internal_prefix(k);
|
if let Some(sub_key) = strip_internal_prefix_preserving_case(k) {
|
||||||
if let Some(ref sub_key) = sub_key_opt {
|
if sub_key.eq_ignore_ascii_case(SUFFIX_REPLICA_TIMESTAMP) {
|
||||||
match sub_key.as_str() {
|
has = true;
|
||||||
"replica-timestamp" => {
|
rs.replica_timestamp = Some(parse_replication_timestamp(v).unwrap_or(OffsetDateTime::UNIX_EPOCH));
|
||||||
has = true;
|
} else if sub_key.eq_ignore_ascii_case(SUFFIX_REPLICA_STATUS) {
|
||||||
rs.replica_timestamp = Some(OffsetDateTime::parse(v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH));
|
has = true;
|
||||||
}
|
rs.replica_status = ReplicationStatusType::from(v.as_str());
|
||||||
"replica-status" => {
|
} else if sub_key.eq_ignore_ascii_case(SUFFIX_REPLICATION_TIMESTAMP) {
|
||||||
has = true;
|
has = true;
|
||||||
rs.replica_status = ReplicationStatusType::from(v.as_str());
|
rs.replication_timestamp = Some(parse_replication_timestamp(v).unwrap_or(OffsetDateTime::UNIX_EPOCH))
|
||||||
}
|
} else if sub_key.eq_ignore_ascii_case(SUFFIX_REPLICATION_STATUS) {
|
||||||
"replication-timestamp" => {
|
has = true;
|
||||||
has = true;
|
rs.replication_status_internal = Some(v.clone());
|
||||||
rs.replication_timestamp = Some(OffsetDateTime::parse(v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH))
|
rs.targets = replication_statuses_map(v.as_str());
|
||||||
}
|
} else if let Some(arn) = rustfs_utils::http::internal_key_strip_suffix_prefix(k, SUFFIX_REPLICATION_RESET_ARN_PREFIX)
|
||||||
"replication-status" => {
|
{
|
||||||
has = true;
|
has = true;
|
||||||
rs.replication_status_internal = Some(v.clone());
|
// Store the canonical full-header key so the map matches
|
||||||
rs.targets = replication_statuses_map(v.as_str());
|
// the key `target_reset_header()` produces on the
|
||||||
}
|
// write/lookup side. Storing the bare ARN keyed the map
|
||||||
_ => {
|
// inconsistently (bare on read, full on write), which
|
||||||
if let Some(arn) = sub_key.strip_prefix("replication-reset-") {
|
// could drop reset state across merge/reflatten cycles
|
||||||
has = true;
|
// (backlog#799 B16).
|
||||||
// Store the canonical full-header key so the map matches
|
rs.reset_statuses_map
|
||||||
// the key `target_reset_header()` produces on the
|
.insert(crate::replication::target_reset_header(&arn), v.clone());
|
||||||
// write/lookup side. Storing the bare ARN keyed the map
|
|
||||||
// inconsistently (bare on read, full on write), which
|
|
||||||
// could drop reset state across merge/reflatten cycles
|
|
||||||
// (backlog#799 B16).
|
|
||||||
rs.reset_statuses_map
|
|
||||||
.insert(crate::replication::target_reset_header(arn), v.clone());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2786,7 +2932,7 @@ impl MetaDeleteMarker {
|
|||||||
contains_key_bytes(&self.meta_sys, SUFFIX_FREE_VERSION)
|
contains_key_bytes(&self.meta_sys, SUFFIX_FREE_VERSION)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn into_fileinfo(&self, volume: &str, path: &str, _all_parts: bool) -> FileInfo {
|
pub fn into_fileinfo(&self, volume: &str, path: &str, _all_parts: bool) -> Result<FileInfo> {
|
||||||
let metadata = self
|
let metadata = self
|
||||||
.meta_sys
|
.meta_sys
|
||||||
.clone()
|
.clone()
|
||||||
@@ -2808,22 +2954,27 @@ impl MetaDeleteMarker {
|
|||||||
|
|
||||||
if self.free_version() {
|
if self.free_version() {
|
||||||
fi.set_tier_free_version();
|
fi.set_tier_free_version();
|
||||||
fi.transition_tier = get_bytes(&self.meta_sys, SUFFIX_TRANSITION_TIER)
|
let derived_metadata = DerivedInternalMetadata::from_meta_sys(&self.meta_sys)?;
|
||||||
.map(|v| String::from_utf8_lossy(&v).to_string())
|
fi.transition_tier = derived_metadata
|
||||||
|
.transition_tier
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(|value| String::from_utf8_lossy(value).to_string())
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
fi.transitioned_objname = derived_metadata
|
||||||
fi.transitioned_objname = get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_OBJECTNAME)
|
.transitioned_object
|
||||||
.map(|v| String::from_utf8_lossy(&v).to_string())
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(|value| String::from_utf8_lossy(value).to_string())
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
fi.transition_version_state = transition_version_state_from_bytes(derived_metadata.transitioned_version_state)?;
|
||||||
fi.transition_version = transitioned_version_from_meta_sys(&self.meta_sys).ok().flatten();
|
fi.transition_version =
|
||||||
fi.transition_version_id = legacy_transitioned_version_id_from_meta_sys(&self.meta_sys);
|
transitioned_version_from_bytes(derived_metadata.transitioned_version, fi.transition_version_state);
|
||||||
fi.transition_version_state =
|
fi.transition_version_id = fi.transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok());
|
||||||
transition_version_state_from_meta_sys(&self.meta_sys, fi.transition_version.as_deref())
|
if derived_metadata.transitioned_version_state.is_some() {
|
||||||
.unwrap_or(TransitionVersionState::Unknown);
|
validate_transition_version_state(fi.transition_version_state, fi.transition_version.as_deref())?;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fi
|
Ok(fi)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn encode_to<W: std::io::Write>(&self, wr: &mut W) -> Result<()> {
|
pub fn encode_to<W: std::io::Write>(&self, wr: &mut W) -> Result<()> {
|
||||||
@@ -2941,6 +3092,13 @@ impl From<FileInfo> for MetaDeleteMarker {
|
|||||||
if !is_internal_key(key) || is_skip_meta_key(key) {
|
if !is_internal_key(key) || is_skip_meta_key(key) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if rustfs_utils::http::internal_key_strip_suffix_prefix(key, SUFFIX_REPLICATION_RESET_ARN_PREFIX).is_some()
|
||||||
|
|| rustfs_utils::http::internal_key_strip_suffix_prefix(key, SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX)
|
||||||
|
.is_some()
|
||||||
|
{
|
||||||
|
meta_sys.insert(key.clone(), metadata_value.as_bytes().to_vec());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
let Some(suffix) = strip_internal_prefix(key) else {
|
let Some(suffix) = strip_internal_prefix(key) else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
@@ -2982,6 +3140,11 @@ impl From<FileInfo> for MetaDeleteMarker {
|
|||||||
if !value.transition_tier.is_empty() {
|
if !value.transition_tier.is_empty() {
|
||||||
insert_bytes(&mut meta_sys, SUFFIX_TRANSITION_TIER, value.transition_tier.as_bytes().to_vec());
|
insert_bytes(&mut meta_sys, SUFFIX_TRANSITION_TIER, value.transition_tier.as_bytes().to_vec());
|
||||||
}
|
}
|
||||||
|
if let Some(state) = value.replication_state_internal.as_ref() {
|
||||||
|
persist_delete_marker_replication_state(&mut meta_sys, state);
|
||||||
|
persist_reset_statuses(&mut meta_sys, &state.reset_statuses_map);
|
||||||
|
persist_target_delete_marker_versions(&mut meta_sys, &state.target_delete_marker_version_ids, &value.metadata);
|
||||||
|
}
|
||||||
Self {
|
Self {
|
||||||
version_id: value.version_id,
|
version_id: value.version_id,
|
||||||
mod_time: value.mod_time,
|
mod_time: value.mod_time,
|
||||||
@@ -3278,6 +3441,7 @@ pub fn file_info_from_raw(
|
|||||||
FileInfoOpts {
|
FileInfoOpts {
|
||||||
data: read_data,
|
data: read_data,
|
||||||
include_free_versions,
|
include_free_versions,
|
||||||
|
include_part_checksums: true,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -3285,6 +3449,7 @@ pub fn file_info_from_raw(
|
|||||||
pub struct FileInfoOpts {
|
pub struct FileInfoOpts {
|
||||||
pub data: bool,
|
pub data: bool,
|
||||||
pub include_free_versions: bool,
|
pub include_free_versions: bool,
|
||||||
|
pub include_part_checksums: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn get_file_info(buf: &[u8], volume: &str, path: &str, version_id: &str, opts: FileInfoOpts) -> Result<FileInfo> {
|
pub fn get_file_info(buf: &[u8], volume: &str, path: &str, version_id: &str, opts: FileInfoOpts) -> Result<FileInfo> {
|
||||||
@@ -3309,7 +3474,11 @@ pub fn get_file_info(buf: &[u8], volume: &str, path: &str, version_id: &str, opt
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
let fi = meta.into_fileinfo(volume, path, version_id, opts.data, opts.include_free_versions, true)?;
|
let fi = if opts.include_part_checksums {
|
||||||
|
meta.into_fileinfo(volume, path, version_id, opts.data, opts.include_free_versions, true)?
|
||||||
|
} else {
|
||||||
|
meta.into_fileinfo_without_part_checksums(volume, path, version_id, opts.data, opts.include_free_versions)?
|
||||||
|
};
|
||||||
Ok(fi)
|
Ok(fi)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3555,6 +3724,56 @@ mod tests {
|
|||||||
assert!(!converted.meta_sys.contains_key("content-type"));
|
assert!(!converted.meta_sys.contains_key("content-type"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn delete_marker_conversion_does_not_lowercase_dynamic_replication_targets() {
|
||||||
|
let arn = "arn:rustfs:replication:us-east-1:TenantA:bucket";
|
||||||
|
let reset_suffix = format!("{SUFFIX_REPLICATION_RESET_ARN_PREFIX}{arn}");
|
||||||
|
let version_suffix = format!("{SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX}{arn}");
|
||||||
|
let mut marker = FileInfo::default();
|
||||||
|
marker.metadata.insert(
|
||||||
|
format!("{}{reset_suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX),
|
||||||
|
"2026-08-12T00:00:00Z;COMPLETED".to_string(),
|
||||||
|
);
|
||||||
|
marker.metadata.insert(
|
||||||
|
format!("{}{version_suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX),
|
||||||
|
"remote-version".to_string(),
|
||||||
|
);
|
||||||
|
marker.replication_state_internal = get_internal_replication_state(&marker.metadata);
|
||||||
|
|
||||||
|
let converted = MetaDeleteMarker::from(marker);
|
||||||
|
|
||||||
|
assert!(converted.meta_sys.keys().any(|key| key.ends_with(&reset_suffix)));
|
||||||
|
assert!(converted.meta_sys.keys().any(|key| key.ends_with(&version_suffix)));
|
||||||
|
assert!(!converted.meta_sys.keys().any(|key| key.contains("tenanta")));
|
||||||
|
|
||||||
|
let mut conflicting = FileInfo::default();
|
||||||
|
conflicting
|
||||||
|
.metadata
|
||||||
|
.insert(format!("{RUSTFS_INTERNAL_PREFIX}{version_suffix}"), "remote-version-a".to_string());
|
||||||
|
conflicting.metadata.insert(
|
||||||
|
format!("{}{version_suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX),
|
||||||
|
"remote-version-b".to_string(),
|
||||||
|
);
|
||||||
|
conflicting.replication_state_internal = get_internal_replication_state(&conflicting.metadata);
|
||||||
|
assert!(
|
||||||
|
conflicting
|
||||||
|
.replication_state_internal
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|state| state.target_delete_marker_version_ids_corrupt)
|
||||||
|
);
|
||||||
|
|
||||||
|
let roundtrip = MetaDeleteMarker::from(conflicting)
|
||||||
|
.into_fileinfo("bucket", "object", false)
|
||||||
|
.expect("dynamic replication aliases should remain decodable");
|
||||||
|
assert!(
|
||||||
|
roundtrip
|
||||||
|
.replication_state_internal
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|state| state.target_delete_marker_version_ids_corrupt),
|
||||||
|
"conflicting dynamic aliases must remain corrupt across persistence"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
enum LegacyDeleteVersionTypeFixture {
|
enum LegacyDeleteVersionTypeFixture {
|
||||||
#[serde(rename = "DeleteMarker")]
|
#[serde(rename = "DeleteMarker")]
|
||||||
@@ -3663,6 +3882,138 @@ mod tests {
|
|||||||
assert!(matches!(res, Err(Error::FileCorrupt)), "short part_sizes must map to FileCorrupt");
|
assert!(matches!(res, Err(Error::FileCorrupt)), "short part_sizes must map to FileCorrupt");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn into_fileinfo_rejects_conflicting_derived_internal_aliases() {
|
||||||
|
for suffix in [
|
||||||
|
SUFFIX_CRC,
|
||||||
|
SUFFIX_TRANSITION_STATUS,
|
||||||
|
SUFFIX_TRANSITIONED_OBJECTNAME,
|
||||||
|
SUFFIX_TRANSITIONED_VERSION_ID,
|
||||||
|
SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||||
|
SUFFIX_TRANSITION_TIER,
|
||||||
|
] {
|
||||||
|
let mut meta_sys = HashMap::from([(format!("{RUSTFS_INTERNAL_PREFIX}{suffix}"), vec![0xff, 1])]);
|
||||||
|
meta_sys.insert(format!("{}{suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX), vec![0xfe, 2]);
|
||||||
|
let object = MetaObject {
|
||||||
|
meta_sys,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
object
|
||||||
|
.into_fileinfo("bucket", "key", false)
|
||||||
|
.expect_err("conflicting aliases must fail closed"),
|
||||||
|
Error::FileCorrupt,
|
||||||
|
"suffix {suffix}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn into_fileinfo_recovers_noncanonical_binary_checksum_alias() {
|
||||||
|
let checksum = vec![0xff, 0x00, 0x80, 0x01];
|
||||||
|
let object = MetaObject {
|
||||||
|
meta_sys: HashMap::from([("X-Minio-Internal-crc".to_string(), checksum.clone())]),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let file_info = object
|
||||||
|
.into_fileinfo("bucket", "key", false)
|
||||||
|
.expect("a single legacy checksum alias should remain readable");
|
||||||
|
|
||||||
|
assert_eq!(file_info.checksum.as_deref(), Some(checksum.as_slice()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn into_fileinfo_prefers_canonical_rewrite_over_stale_mixed_case_alias() {
|
||||||
|
let checksum = vec![0xff, 0x00, 0x80, 0x01];
|
||||||
|
let mut object = MetaObject {
|
||||||
|
meta_sys: HashMap::from([("X-Minio-Internal-crc".to_string(), b"stale".to_vec())]),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
insert_bytes(&mut object.meta_sys, SUFFIX_CRC, checksum.clone());
|
||||||
|
|
||||||
|
let file_info = object
|
||||||
|
.into_fileinfo("bucket", "key", false)
|
||||||
|
.expect("canonical rewrites should supersede legacy mixed-case aliases");
|
||||||
|
|
||||||
|
assert_eq!(file_info.checksum.as_deref(), Some(checksum.as_slice()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn into_fileinfo_recovers_data_movement_part_checksums() {
|
||||||
|
let mut object = object_with_parts(vec![1], vec![16], vec![16]);
|
||||||
|
insert_bytes(&mut object.meta_sys, SUFFIX_PART_CHECKSUMS, br#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_vec());
|
||||||
|
|
||||||
|
let file_info = object
|
||||||
|
.into_fileinfo("bucket", "key", true)
|
||||||
|
.expect("data movement part checksums should decode");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
file_info.parts[0]
|
||||||
|
.checksums
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|checksums| checksums.get("CRC32C"))
|
||||||
|
.map(String::as_str),
|
||||||
|
Some("AAAAAA==")
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut deferred = object
|
||||||
|
.to_fileinfo_with_part_checksums("bucket", "key", true, false)
|
||||||
|
.expect("quorum candidates should retain raw checksum metadata");
|
||||||
|
assert!(deferred.parts[0].checksums.is_none());
|
||||||
|
deferred
|
||||||
|
.hydrate_data_movement_part_checksums()
|
||||||
|
.expect("the selected candidate should hydrate checksums once");
|
||||||
|
assert_eq!(deferred.parts[0].checksums, file_info.parts[0].checksums);
|
||||||
|
|
||||||
|
insert_bytes(&mut object.meta_sys, SUFFIX_PART_CHECKSUMS, b"not-json".to_vec());
|
||||||
|
assert_eq!(
|
||||||
|
object
|
||||||
|
.into_fileinfo("bucket", "key", true)
|
||||||
|
.expect_err("malformed data movement part checksums must fail closed"),
|
||||||
|
Error::FileCorrupt
|
||||||
|
);
|
||||||
|
|
||||||
|
for encoded in [
|
||||||
|
br#"[[1,[["CRC32C","AAAAAA=="]]],[1,[["CRC32C","BBBBBB=="]]]]"#.as_slice(),
|
||||||
|
br#"[[1,[["CRC32C","AAAAAA=="],["CRC32C","BBBBBB=="]]]]"#.as_slice(),
|
||||||
|
] {
|
||||||
|
insert_bytes(&mut object.meta_sys, SUFFIX_PART_CHECKSUMS, encoded.to_vec());
|
||||||
|
assert_eq!(
|
||||||
|
object
|
||||||
|
.into_fileinfo("bucket", "key", true)
|
||||||
|
.expect_err("duplicate part checksum keys must fail closed"),
|
||||||
|
Error::FileCorrupt
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
insert_bytes(&mut object.meta_sys, SUFFIX_PART_CHECKSUMS, br#"[[2,[["CRC32C","AAAAAA=="]]]]"#.to_vec());
|
||||||
|
assert_eq!(
|
||||||
|
object
|
||||||
|
.into_fileinfo("bucket", "key", true)
|
||||||
|
.expect_err("a sidecar for an unknown part must fail closed"),
|
||||||
|
Error::FileCorrupt
|
||||||
|
);
|
||||||
|
|
||||||
|
object.meta_sys = HashMap::from([
|
||||||
|
(
|
||||||
|
format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_PART_CHECKSUMS}"),
|
||||||
|
br#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_vec(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
format!("{}{}", rustfs_utils::http::MINIO_INTERNAL_PREFIX, SUFFIX_PART_CHECKSUMS),
|
||||||
|
br#"[[1,[["CRC32C","BBBBBB=="]]]]"#.to_vec(),
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
assert_eq!(
|
||||||
|
object
|
||||||
|
.into_fileinfo("bucket", "key", true)
|
||||||
|
.expect_err("conflicting sidecar aliases must fail closed"),
|
||||||
|
Error::FileCorrupt
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn into_fileinfo_rejects_short_part_actual_sizes_including_empty() {
|
fn into_fileinfo_rejects_short_part_actual_sizes_including_empty() {
|
||||||
let obj = object_with_parts(vec![1, 2], vec![10, 20], vec![]);
|
let obj = object_with_parts(vec![1, 2], vec![10, 20], vec![]);
|
||||||
@@ -4230,6 +4581,31 @@ mod tests {
|
|||||||
assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown);
|
assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn meta_object_transition_exact_rejects_legacy_raw_uuid_encoding() {
|
||||||
|
let mut sys = HashMap::new();
|
||||||
|
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, sample_version_id().as_bytes().to_vec());
|
||||||
|
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_STATE, b"exact".to_vec());
|
||||||
|
|
||||||
|
let err = make_meta_object_with_sys(sys)
|
||||||
|
.into_fileinfo("b", "k", false)
|
||||||
|
.expect_err("exact remote versions must use their UTF-8 provider representation");
|
||||||
|
|
||||||
|
assert_eq!(err, Error::FileCorrupt);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn meta_object_transition_version_id_mixed_case_alias_is_recovered() {
|
||||||
|
let id = sample_version_id();
|
||||||
|
let sys = HashMap::from([("X-Minio-Internal-transitioned-versionID".to_string(), id.as_bytes().to_vec())]);
|
||||||
|
let fi = make_meta_object_with_sys(sys)
|
||||||
|
.into_fileinfo("b", "k", false)
|
||||||
|
.expect("a legacy mixed-case transition version alias should decode");
|
||||||
|
|
||||||
|
assert_eq!(fi.transition_version_id, Some(id));
|
||||||
|
assert_eq!(fi.transition_version, Some(id.to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn meta_object_transition_version_id_opaque_text_is_preserved() {
|
fn meta_object_transition_version_id_opaque_text_is_preserved() {
|
||||||
let mut sys = HashMap::new();
|
let mut sys = HashMap::new();
|
||||||
@@ -4271,8 +4647,10 @@ mod tests {
|
|||||||
.map(Vec::as_slice),
|
.map(Vec::as_slice),
|
||||||
Some(b"exact".as_slice())
|
Some(b"exact".as_slice())
|
||||||
);
|
);
|
||||||
|
let persisted_version = get_consistent_bytes(&object.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
legacy_transitioned_version_id_from_meta_sys(&object.meta_sys),
|
transitioned_version_from_bytes(persisted_version, TransitionVersionState::Unknown)
|
||||||
|
.and_then(|value| Uuid::parse_str(&value).ok()),
|
||||||
Some(id),
|
Some(id),
|
||||||
"UUID exact writes must remain readable by the legacy UUID consumer"
|
"UUID exact writes must remain readable by the legacy UUID consumer"
|
||||||
);
|
);
|
||||||
@@ -4281,6 +4659,26 @@ mod tests {
|
|||||||
assert_eq!(decoded.transition_version.as_deref(), Some(expected_version.as_str()));
|
assert_eq!(decoded.transition_version.as_deref(), Some(expected_version.as_str()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn meta_object_transition_version_state_exact_preserves_sixteen_byte_opaque_text() {
|
||||||
|
let expected_version = "opaque.wasabi_01";
|
||||||
|
assert_eq!(expected_version.len(), 16);
|
||||||
|
let fi = FileInfo {
|
||||||
|
transition_status: "complete".to_string(),
|
||||||
|
transition_version: Some(expected_version.to_string()),
|
||||||
|
transition_version_state: TransitionVersionState::Exact,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let decoded = MetaObject::from(fi)
|
||||||
|
.into_fileinfo("b", "k", false)
|
||||||
|
.expect("exact opaque transition version should round trip");
|
||||||
|
|
||||||
|
assert_eq!(decoded.transition_version.as_deref(), Some(expected_version));
|
||||||
|
assert_eq!(decoded.transition_version_id, None);
|
||||||
|
assert_eq!(decoded.transition_version_state, TransitionVersionState::Exact);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn set_transition_known_disabled_removes_stale_version_dual_keys() {
|
fn set_transition_known_disabled_removes_stale_version_dual_keys() {
|
||||||
let mut meta_sys = HashMap::new();
|
let mut meta_sys = HashMap::new();
|
||||||
@@ -4375,7 +4773,8 @@ mod tests {
|
|||||||
mod_time: None,
|
mod_time: None,
|
||||||
meta_sys: sys,
|
meta_sys: sys,
|
||||||
}
|
}
|
||||||
.into_fileinfo("b", "k", false);
|
.into_fileinfo("b", "k", false)
|
||||||
|
.expect("nil tier version should remain an absent remote version");
|
||||||
assert_eq!(fi.transition_version_id, None);
|
assert_eq!(fi.transition_version_id, None);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -4390,7 +4789,8 @@ mod tests {
|
|||||||
mod_time: None,
|
mod_time: None,
|
||||||
meta_sys: sys,
|
meta_sys: sys,
|
||||||
}
|
}
|
||||||
.into_fileinfo("b", "k", false);
|
.into_fileinfo("b", "k", false)
|
||||||
|
.expect("legacy binary UUID tier version should decode");
|
||||||
assert_eq!(fi.transition_version_id, Some(id));
|
assert_eq!(fi.transition_version_id, Some(id));
|
||||||
assert_eq!(fi.transition_version, Some(id.to_string()));
|
assert_eq!(fi.transition_version, Some(id.to_string()));
|
||||||
}
|
}
|
||||||
@@ -4407,7 +4807,8 @@ mod tests {
|
|||||||
mod_time: Some(sample_mod_time()),
|
mod_time: Some(sample_mod_time()),
|
||||||
meta_sys: sys,
|
meta_sys: sys,
|
||||||
}
|
}
|
||||||
.into_fileinfo("b", "k", false);
|
.into_fileinfo("b", "k", false)
|
||||||
|
.expect("opaque tier version should remain readable");
|
||||||
|
|
||||||
assert_eq!(fi.transition_version_id, None);
|
assert_eq!(fi.transition_version_id, None);
|
||||||
assert_eq!(fi.transition_version.as_deref(), Some("opaque-generation-42"));
|
assert_eq!(fi.transition_version.as_deref(), Some("opaque-generation-42"));
|
||||||
@@ -4429,12 +4830,67 @@ mod tests {
|
|||||||
mod_time: Some(sample_mod_time()),
|
mod_time: Some(sample_mod_time()),
|
||||||
meta_sys: sys,
|
meta_sys: sys,
|
||||||
}
|
}
|
||||||
.into_fileinfo("b", "k", false);
|
.into_fileinfo("b", "k", false)
|
||||||
|
.expect("mixed-case tier aliases should decode");
|
||||||
|
|
||||||
assert_eq!(fi.transition_version_id, Some(id));
|
assert_eq!(fi.transition_version_id, Some(id));
|
||||||
assert_eq!(fi.transition_version, Some(id.to_string()));
|
assert_eq!(fi.transition_version, Some(id.to_string()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn delete_marker_free_version_recovers_mixed_case_transition_aliases() {
|
||||||
|
let id = sample_version_id();
|
||||||
|
let id_text = id.to_string();
|
||||||
|
let mut sys = HashMap::new();
|
||||||
|
insert_bytes(&mut sys, SUFFIX_FREE_VERSION, vec![]);
|
||||||
|
for (suffix, value) in [
|
||||||
|
(SUFFIX_TRANSITIONED_VERSION_ID, id_text.as_bytes()),
|
||||||
|
(SUFFIX_TRANSITIONED_VERSION_STATE, b"exact".as_slice()),
|
||||||
|
(SUFFIX_TRANSITION_TIER, b"WARM".as_slice()),
|
||||||
|
(SUFFIX_TRANSITIONED_OBJECTNAME, b"remote-object".as_slice()),
|
||||||
|
] {
|
||||||
|
sys.insert(format!("X-Minio-Internal-{suffix}"), value.to_vec());
|
||||||
|
}
|
||||||
|
|
||||||
|
let fi = MetaDeleteMarker {
|
||||||
|
version_id: Some(sample_version_id()),
|
||||||
|
mod_time: Some(sample_mod_time()),
|
||||||
|
meta_sys: sys,
|
||||||
|
}
|
||||||
|
.into_fileinfo("b", "k", false)
|
||||||
|
.expect("mixed-case tier aliases should decode");
|
||||||
|
|
||||||
|
assert_eq!(fi.transition_version_id, Some(id));
|
||||||
|
assert_eq!(fi.transition_version, Some(id.to_string()));
|
||||||
|
assert_eq!(fi.transition_version_state, TransitionVersionState::Exact);
|
||||||
|
assert_eq!(fi.transition_tier, "WARM");
|
||||||
|
assert_eq!(fi.transitioned_objname, "remote-object");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn delete_marker_free_version_rejects_conflicting_transition_aliases() {
|
||||||
|
let mut sys = HashMap::new();
|
||||||
|
insert_bytes(&mut sys, SUFFIX_FREE_VERSION, vec![]);
|
||||||
|
sys.insert(
|
||||||
|
format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_ID}"),
|
||||||
|
b"source-version".to_vec(),
|
||||||
|
);
|
||||||
|
sys.insert(
|
||||||
|
format!("{}{}", rustfs_utils::http::MINIO_INTERNAL_PREFIX, SUFFIX_TRANSITIONED_VERSION_ID),
|
||||||
|
b"target-version".to_vec(),
|
||||||
|
);
|
||||||
|
|
||||||
|
let err = MetaDeleteMarker {
|
||||||
|
version_id: Some(sample_version_id()),
|
||||||
|
mod_time: Some(sample_mod_time()),
|
||||||
|
meta_sys: sys,
|
||||||
|
}
|
||||||
|
.into_fileinfo("b", "k", false)
|
||||||
|
.expect_err("conflicting transition aliases must fail closed");
|
||||||
|
|
||||||
|
assert_eq!(err, Error::FileCorrupt);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn version_header_sorts_before_prefers_object_over_delete_marker_on_equal_mod_time() {
|
fn version_header_sorts_before_prefers_object_over_delete_marker_on_equal_mod_time() {
|
||||||
let object = FileMetaVersionHeader {
|
let object = FileMetaVersionHeader {
|
||||||
@@ -4758,7 +5214,7 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn target_delete_marker_version_metadata_is_forward_and_backward_compatible() {
|
fn target_delete_marker_version_metadata_is_forward_and_backward_compatible() {
|
||||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
let arn = "arn:rustfs:replication:us-east-1:TenantA:bucket";
|
||||||
let suffix = format!("{}{arn}", rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX);
|
let suffix = format!("{}{arn}", rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX);
|
||||||
let mut metadata = HashMap::from([(format!("{RUSTFS_INTERNAL_PREFIX}replication-status"), format!("{arn}=COMPLETED;"))]);
|
let mut metadata = HashMap::from([(format!("{RUSTFS_INTERNAL_PREFIX}replication-status"), format!("{arn}=COMPLETED;"))]);
|
||||||
|
|
||||||
@@ -4803,7 +5259,7 @@ mod tests {
|
|||||||
// must keep it keyed by `target_reset_header(arn)` (not the bare ARN) so
|
// must keep it keyed by `target_reset_header(arn)` (not the bare ARN) so
|
||||||
// `ReplicationState::target_state` finds it after a round trip
|
// `ReplicationState::target_state` finds it after a round trip
|
||||||
// (backlog#799 B16).
|
// (backlog#799 B16).
|
||||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
let arn = "arn:rustfs:replication:us-east-1:TenantA:bucket";
|
||||||
let ts = "2026-06-30T00:00:00Z;reset-1".to_string();
|
let ts = "2026-06-30T00:00:00Z;reset-1".to_string();
|
||||||
let key = crate::replication::target_reset_header(arn);
|
let key = crate::replication::target_reset_header(arn);
|
||||||
let mut metadata = HashMap::new();
|
let mut metadata = HashMap::new();
|
||||||
@@ -4822,6 +5278,33 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn get_internal_replication_state_accepts_rfc3339_and_rustfs_display_timestamps() {
|
||||||
|
let replica_timestamp = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
|
||||||
|
let replication_timestamp = replica_timestamp + time::Duration::SECOND;
|
||||||
|
let metadata = HashMap::from([
|
||||||
|
(format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_REPLICA_STATUS}"), "REPLICA".to_string()),
|
||||||
|
(
|
||||||
|
format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_REPLICA_TIMESTAMP}"),
|
||||||
|
replica_timestamp.to_string(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_REPLICATION_STATUS}"),
|
||||||
|
"arn:rustfs:replication:us-east-1:TenantA:bucket=COMPLETED;".to_string(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_REPLICATION_TIMESTAMP}"),
|
||||||
|
replication_timestamp
|
||||||
|
.format(&Rfc3339)
|
||||||
|
.expect("RFC3339 timestamp should format"),
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
|
||||||
|
let state = get_internal_replication_state(&metadata).expect("replication metadata should parse");
|
||||||
|
assert_eq!(state.replica_timestamp, Some(replica_timestamp));
|
||||||
|
assert_eq!(state.replication_timestamp, Some(replication_timestamp));
|
||||||
|
}
|
||||||
|
|
||||||
// ---- Header signature (backlog#861 / B12) ----
|
// ---- Header signature (backlog#861 / B12) ----
|
||||||
|
|
||||||
fn signed_object() -> MetaObject {
|
fn signed_object() -> MetaObject {
|
||||||
|
|||||||
@@ -157,7 +157,7 @@ impl MetaCacheEntry {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
let fi = fm.into_fileinfo(bucket, self.name.as_str(), "", false, false, true)?;
|
let fi = fm.into_fileinfo_without_part_checksums(bucket, self.name.as_str(), "", false, false)?;
|
||||||
return Ok(fi);
|
return Ok(fi);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -169,6 +169,7 @@ impl MetaCacheEntry {
|
|||||||
FileInfoOpts {
|
FileInfoOpts {
|
||||||
data: false,
|
data: false,
|
||||||
include_free_versions: false,
|
include_free_versions: false,
|
||||||
|
include_part_checksums: false,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -2157,4 +2158,38 @@ mod tests {
|
|||||||
let inner = caught.expect("file_info_versions must not panic");
|
let inner = caught.expect("file_info_versions must not panic");
|
||||||
assert!(matches!(inner, Err(Error::FileCorrupt)), "expected FileCorrupt");
|
assert!(matches!(inner, Err(Error::FileCorrupt)), "expected FileCorrupt");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn metacache_to_fileinfo_ignores_part_checksum_sidecar_regardless_of_cache_state() {
|
||||||
|
let mut meta = FileMeta::load(&create_real_xlmeta().expect("create real xl.meta")).expect("load real xl.meta");
|
||||||
|
let version_id = Uuid::parse_str("01234567-89ab-cdef-0123-456789abcdef").expect("valid fixture version id");
|
||||||
|
let (index, mut version) = meta.find_version(Some(version_id)).expect("find fixture object version");
|
||||||
|
rustfs_utils::http::insert_bytes(
|
||||||
|
&mut version.object.as_mut().expect("fixture object").meta_sys,
|
||||||
|
rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
|
||||||
|
b"not-json".to_vec(),
|
||||||
|
);
|
||||||
|
meta.versions[index] = FileMetaShallowVersion::try_from(version).expect("replace fixture object version");
|
||||||
|
let encoded = meta.marshal_msg().expect("marshal object metadata");
|
||||||
|
|
||||||
|
let uncached = MetaCacheEntry {
|
||||||
|
name: "object".to_string(),
|
||||||
|
metadata: encoded.clone(),
|
||||||
|
cached: None,
|
||||||
|
reusable: false,
|
||||||
|
}
|
||||||
|
.to_fileinfo("bucket")
|
||||||
|
.expect("uncached metacache conversion must stay lazy");
|
||||||
|
let cached = MetaCacheEntry {
|
||||||
|
name: "object".to_string(),
|
||||||
|
metadata: encoded,
|
||||||
|
cached: Some(meta),
|
||||||
|
reusable: false,
|
||||||
|
}
|
||||||
|
.to_fileinfo("bucket")
|
||||||
|
.expect("cached metacache conversion must stay lazy");
|
||||||
|
|
||||||
|
assert_eq!(cached, uncached);
|
||||||
|
assert!(cached.parts.iter().all(|part| part.checksums.is_none()));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -85,6 +85,7 @@ fn load_file_info(case_id: &str) -> FileInfo {
|
|||||||
FileInfoOpts {
|
FileInfoOpts {
|
||||||
data: false,
|
data: false,
|
||||||
include_free_versions: true,
|
include_free_versions: true,
|
||||||
|
include_part_checksums: true,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap_or_else(|err| panic!("decode {}: {err}", xl_meta.display()))
|
.unwrap_or_else(|err| panic!("decode {}: {err}", xl_meta.display()))
|
||||||
|
|||||||
@@ -30,6 +30,10 @@ pub const MINIO_INTERNAL_PREFIX: &str = "x-minio-internal-";
|
|||||||
// Key suffixes (lowercase, no prefix)
|
// Key suffixes (lowercase, no prefix)
|
||||||
pub const SUFFIX_INLINE_DATA: &str = "inline-data";
|
pub const SUFFIX_INLINE_DATA: &str = "inline-data";
|
||||||
pub const SUFFIX_DATA_MOVED: &str = "data-moved";
|
pub const SUFFIX_DATA_MOVED: &str = "data-moved";
|
||||||
|
/// Tags snapshot bound to a data-movement-owned target.
|
||||||
|
pub const SUFFIX_DATA_MOVED_TAGS: &str = "data-moved-tags";
|
||||||
|
/// Internal ownership marker for a data-movement multipart upload.
|
||||||
|
pub const SUFFIX_DATA_MOVEMENT_UPLOAD: &str = "data-movement-upload";
|
||||||
/// Transient flag for data movement
|
/// Transient flag for data movement
|
||||||
pub const SUFFIX_DATA_MOV: &str = "data-mov";
|
pub const SUFFIX_DATA_MOV: &str = "data-mov";
|
||||||
/// Transient flag for healing
|
/// Transient flag for healing
|
||||||
@@ -44,6 +48,8 @@ pub const SUFFIX_ACTUAL_OBJECT_SIZE: &str = "actual-object-size";
|
|||||||
/// Used by replication; key stored with capital A
|
/// Used by replication; key stored with capital A
|
||||||
pub const SUFFIX_ACTUAL_OBJECT_SIZE_CAP: &str = "Actual-Object-Size";
|
pub const SUFFIX_ACTUAL_OBJECT_SIZE_CAP: &str = "Actual-Object-Size";
|
||||||
pub const SUFFIX_CRC: &str = "crc";
|
pub const SUFFIX_CRC: &str = "crc";
|
||||||
|
/// JSON-encoded per-part S3 checksum maps retained across raw data movement.
|
||||||
|
pub const SUFFIX_PART_CHECKSUMS: &str = "part-checksums";
|
||||||
pub const SUFFIX_TRANSITION_STATUS: &str = "transition-status";
|
pub const SUFFIX_TRANSITION_STATUS: &str = "transition-status";
|
||||||
pub const SUFFIX_TRANSITIONED_OBJECTNAME: &str = "transitioned-object";
|
pub const SUFFIX_TRANSITIONED_OBJECTNAME: &str = "transitioned-object";
|
||||||
pub const SUFFIX_TRANSITIONED_VERSION_ID: &str = "transitioned-versionID";
|
pub const SUFFIX_TRANSITIONED_VERSION_ID: &str = "transitioned-versionID";
|
||||||
@@ -132,8 +138,10 @@ pub fn internal_key_starts_with(key: &str, suffix_prefix: &str) -> bool {
|
|||||||
/// For keys like x-rustfs-internal-replication-reset-{arn}, strips the internal prefix and suffix_prefix,
|
/// For keys like x-rustfs-internal-replication-reset-{arn}, strips the internal prefix and suffix_prefix,
|
||||||
/// returning the remainder (e.g. "arn1"). Returns None if key does not match.
|
/// returning the remainder (e.g. "arn1"). Returns None if key does not match.
|
||||||
pub fn internal_key_strip_suffix_prefix(key: &str, suffix_prefix: &str) -> Option<String> {
|
pub fn internal_key_strip_suffix_prefix(key: &str, suffix_prefix: &str) -> Option<String> {
|
||||||
let rest = strip_internal_prefix(key)?;
|
let rest = strip_internal_prefix_preserving_case(key)?;
|
||||||
rest.strip_prefix(suffix_prefix).map(|s| s.to_string())
|
rest.get(..suffix_prefix.len())
|
||||||
|
.is_some_and(|prefix| prefix.eq_ignore_ascii_case(suffix_prefix))
|
||||||
|
.then(|| rest[suffix_prefix.len()..].to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn both_keys(suffix: &str) -> (String, String) {
|
fn both_keys(suffix: &str) -> (String, String) {
|
||||||
@@ -529,6 +537,18 @@ mod tests {
|
|||||||
assert!(!has_internal_suffix(key, SUFFIX_COMPRESSION));
|
assert!(!has_internal_suffix(key, SUFFIX_COMPRESSION));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn internal_suffix_prefix_preserves_dynamic_identifier_case() {
|
||||||
|
assert_eq!(
|
||||||
|
internal_key_strip_suffix_prefix(
|
||||||
|
"X-Minio-Internal-Replication-Reset-arn:minio:replication::TenantA:bucket",
|
||||||
|
SUFFIX_REPLICATION_RESET_ARN_PREFIX,
|
||||||
|
)
|
||||||
|
.as_deref(),
|
||||||
|
Some("arn:minio:replication::TenantA:bucket")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_get_str_case_insensitive_fallback() {
|
fn test_get_str_case_insensitive_fallback() {
|
||||||
// Non-canonical mixed-case key must still be found via the case-insensitive scan.
|
// Non-canonical mixed-case key must still be found via the case-insensitive scan.
|
||||||
|
|||||||
@@ -5,9 +5,10 @@ availability, what to expect when several nodes are down at once (sequential
|
|||||||
cold start), and how to read the degraded-mode signals. Written for the
|
cold start), and how to read the degraded-mode signals. Written for the
|
||||||
failure pattern reported in rustfs/rustfs#4304.
|
failure pattern reported in rustfs/rustfs#4304.
|
||||||
|
|
||||||
> Upgrading the binary or container image never changes the on-disk data
|
> Upgrading the binary or container image does not change the on-disk data
|
||||||
> format. Replacing the executable and restarting is safe; no migration step
|
> format unless an explicitly enabled feature documents a version floor.
|
||||||
> runs on startup.
|
> Replacing the executable and restarting does not run a migration step on
|
||||||
|
> startup.
|
||||||
|
|
||||||
> [!WARNING]
|
> [!WARNING]
|
||||||
> The release that switches local SSE wrapped DEKs from the legacy
|
> The release that switches local SSE wrapped DEKs from the legacy
|
||||||
@@ -19,6 +20,17 @@ failure pattern reported in rustfs/rustfs#4304.
|
|||||||
> and then resume traffic. Downgrading or rolling back after new encrypted
|
> and then resume traffic. Downgrading or rolling back after new encrypted
|
||||||
> objects are written is not supported.
|
> objects are written is not supported.
|
||||||
|
|
||||||
|
> [!WARNING]
|
||||||
|
> `RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_WRITE` remains inactive unless
|
||||||
|
> `RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED` is also `true`. Enable
|
||||||
|
> both only after every node that can read or write object metadata supports
|
||||||
|
> the `part-checksums` sidecar and the fleet has adopted that version as its
|
||||||
|
> rollback floor. Leave either setting disabled throughout a mixed-version
|
||||||
|
> rolling upgrade. Once rebalance or decommission has migrated a legacy
|
||||||
|
> checksummed multipart object with both settings enabled, rolling back to an
|
||||||
|
> older build is not supported: older readers ignore the sidecar and can
|
||||||
|
> report an object checksum in place of the requested part checksum.
|
||||||
|
|
||||||
## TL;DR
|
## TL;DR
|
||||||
|
|
||||||
- **Rolling restart (no downtime):** restart **one node at a time**, and wait
|
- **Rolling restart (no downtime):** restart **one node at a time**, and wait
|
||||||
|
|||||||
@@ -77,8 +77,7 @@ where
|
|||||||
F: FnOnce() -> Fut + Send + 'static,
|
F: FnOnce() -> Fut + Send + 'static,
|
||||||
Fut: std::future::Future<Output = S3Result<T>> + Send + 'static,
|
Fut: std::future::Future<Output = S3Result<T>> + Send + 'static,
|
||||||
{
|
{
|
||||||
let store =
|
let store = current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init"))?;
|
||||||
current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?;
|
|
||||||
with_site_replication_state_lock_on(store, operation).await
|
with_site_replication_state_lock_on(store, operation).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6737,9 +6737,10 @@ impl DefaultObjectUsecase {
|
|||||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||||
};
|
};
|
||||||
|
|
||||||
let opts: ObjectOptions = get_opts(&bucket, &key, version_id.clone(), None, &req.headers)
|
let mut opts: ObjectOptions = get_opts(&bucket, &key, version_id.clone(), None, &req.headers)
|
||||||
.await
|
.await
|
||||||
.map_err(ApiError::from)?;
|
.map_err(ApiError::from)?;
|
||||||
|
opts.include_part_checksums = object_attributes_requested(&object_attributes, ObjectAttributes::OBJECT_PARTS);
|
||||||
|
|
||||||
let info = match store.get_object_info(&bucket, &key, &opts).await {
|
let info = match store.get_object_info(&bucket, &key, &opts).await {
|
||||||
Ok(info) => info,
|
Ok(info) => info,
|
||||||
|
|||||||
Reference in New Issue
Block a user