fix(replication): fence stale metadata status writeback (#7083)

Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
cxymds
2026-09-03 16:05:58 +08:00
committed by GitHub
parent 8023cf3e26
commit 86ebcb325c
29 changed files with 3219 additions and 229 deletions
+1 -1
View File
@@ -53,12 +53,12 @@ pub use replication_config_boundary::{
replication_target_arns, should_remove_replication_target, site_replication_rule_deployment_id,
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
};
pub(crate) use replication_filemeta_boundary::version_purge_statuses_map;
pub use replication_filemeta_boundary::{
MrfOpKind, MrfReplicateEntry, REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicationState,
ReplicationStatusType, ReplicationType, VersionPurgeStatusType, replication_state_to_filemeta,
replication_status_to_filemeta, replication_statuses_map, version_purge_status_to_filemeta,
};
pub(crate) use replication_filemeta_boundary::{ReplicationGenerationSnapshot, version_purge_statuses_map};
pub(crate) use replication_filemeta_boundary::{
replication_state_from_filemeta, replication_status_from_filemeta, version_purge_status_from_filemeta,
};
@@ -15,7 +15,7 @@
pub use rustfs_replication::{MrfOpKind, MrfReplicateEntry};
pub(crate) use rustfs_replication::{
REPLICATE_EXISTING, REPLICATE_HEAL_DELETE, ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction,
ReplicationWorkerOperation, ResyncDecision, get_replication_state, parse_replicate_decision,
ReplicationGenerationSnapshot, ReplicationWorkerOperation, ResyncDecision, get_replication_state, parse_replicate_decision,
replicate_decision_for_admitted_targets, target_reset_header, version_purge_statuses_map,
};
pub use rustfs_replication::{
@@ -575,7 +575,7 @@ pub(crate) async fn must_replicate(bucket: &str, object: &str, mopts: MustReplic
let mut sopts = opts.clone();
sopts.target_arn = arn.clone();
let replicate = cfg.replicate(&sopts) && mopts.metadata_target_is_eligible(&arn);
let replicate = metadata_target_should_replicate(&cfg, &sopts, &mopts, &arn);
let synchronous = if let Some(cli) = cli { cli.replicate_sync } else { false };
dsc.set(ReplicateTargetDecision::new(arn, replicate, synchronous));
@@ -584,6 +584,15 @@ pub(crate) async fn must_replicate(bucket: &str, object: &str, mopts: MustReplic
dsc
}
fn metadata_target_should_replicate(
cfg: &ReplicationConfiguration,
opts: &ObjectOpts,
mopts: &MustReplicateOptions,
arn: &str,
) -> bool {
cfg.replicate(opts) && mopts.metadata_target_is_eligible(arn)
}
#[cfg(test)]
mod tests {
use s3s::dto::{
@@ -613,6 +622,46 @@ mod tests {
}
}
#[test]
fn metadata_replication_requires_both_current_rule_match_and_historical_admission() {
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
let mut rule = replication_rule();
rule.destination.bucket = arn.to_string();
rule.filter = Some(ReplicationRuleFilter {
prefix: Some("admitted/".to_string()),
..Default::default()
});
let cfg = ReplicationConfiguration {
role: String::new(),
rules: vec![rule],
};
let mut metadata = HashMap::new();
rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_REPLICATION_STATUS, format!("{arn}=PENDING;"));
let admitted = MustReplicateOptions::new(&metadata, String::new(), ReplicationType::Metadata, false);
let matching = ObjectOpts {
name: "admitted/object".to_string(),
target_arn: arn.to_string(),
..Default::default()
};
assert!(metadata_target_should_replicate(&cfg, &matching, &admitted, arn));
let rule_mismatch = ObjectOpts {
name: "outside/object".to_string(),
target_arn: arn.to_string(),
..Default::default()
};
assert!(
!metadata_target_should_replicate(&cfg, &rule_mismatch, &admitted, arn),
"historical admission must not bypass the current replication rule"
);
let never_admitted = MustReplicateOptions::new(&HashMap::new(), String::new(), ReplicationType::Metadata, false);
assert!(
!metadata_target_should_replicate(&cfg, &matching, &never_admitted, arn),
"a current rule match must not create historical admission"
);
}
#[test]
fn replication_config_empty_and_replicate_follow_config() {
let empty = ReplicationConfig::default();
@@ -52,7 +52,6 @@ use super::runtime_boundary as runtime_sources;
use futures_util::stream::{self, StreamExt};
use metrics::{counter, histogram};
use rustfs_utils::hash::HashAlgorithm;
use rustfs_utils::http::{SUFFIX_REPLICATION_TIMESTAMP, get_str};
use std::collections::HashMap;
use std::collections::hash_map::Entry;
use std::sync::Arc;
@@ -939,7 +938,11 @@ async fn replay_mrf_object_entry<S: ReplicationStorage>(
Some(queue_replication_heal(&entry.bucket, oi, entry.retry_count.max(0) as u32).await)
} else {
let roi = admitted_mrf_replicate_object(oi, entry, entry.op.replication_type());
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
if replicate_object_with_outcome(roi, storage.clone())
.await
.1
.consumes_mrf_entry()
{
Some(ReplicationQueueAdmission::Queued)
} else {
Some(ReplicationQueueAdmission::Missed)
@@ -978,7 +981,11 @@ async fn replay_mrf_metadata_entry<S: ReplicationStorage>(
Some(queue_replication_metadata(&entry.bucket, oi, entry.retry_count.max(0) as u32).await)
} else {
let roi = admitted_mrf_replicate_object(oi, entry, ReplicationType::Metadata);
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
if replicate_object_with_outcome(roi, storage.clone())
.await
.1
.consumes_mrf_entry()
{
Some(ReplicationQueueAdmission::Queued)
} else {
Some(ReplicationQueueAdmission::Missed)
@@ -2978,8 +2985,11 @@ fn replicate_object_info_from_object_info(
) -> ReplicateObjectInfo {
let tgt_statuses = replication_statuses_map(&oi.replication_status_internal.clone().unwrap_or_default());
let purge_statuses = version_purge_statuses_map(&oi.version_purge_status_internal.clone().unwrap_or_default());
let tm = get_str(&oi.user_defined, SUFFIX_REPLICATION_TIMESTAMP)
.map(|v| OffsetDateTime::parse(&v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH));
let replication_generation = oi.replication_generation_snapshot();
let tm = replication_generation
.timestamp
.as_deref()
.map(|value| OffsetDateTime::parse(value, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH));
let mut rstate = oi.replication_state();
rstate.replicate_decision_str = dsc.to_string();
let asz = oi.get_actual_size_or_physical();
@@ -3006,6 +3016,7 @@ fn replicate_object_info_from_object_info(
target_statuses: tgt_statuses,
target_purge_statuses: purge_statuses,
replication_timestamp: tm,
replication_generation,
user_tags: (*oi.user_tags).clone(),
checksum,
retry_count: 0,
@@ -17,6 +17,8 @@ use super::replication_config_boundary::{ObjectOpts, ReplicationConfigurationExt
use super::replication_config_store::ReplicationConfigStore;
use super::replication_error_boundary::{Error, Result, is_err_object_not_found, is_err_version_not_found};
use super::replication_event_sink::{EventArgs, send_event, send_local_event};
#[cfg(test)]
use super::replication_filemeta_boundary::ReplicationGenerationSnapshot;
use super::replication_filemeta_boundary::{
REPLICATE_EXISTING, ReplicateDecision, ReplicateObjectInfo, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction,
ReplicationState, ReplicationStatusType, ReplicationType, VersionPurgeStatusType, get_replication_state,
@@ -47,12 +49,13 @@ use super::replication_resync_boundary::{
};
#[cfg(test)]
use super::replication_resync_boundary::{RESYNC_META_FORMAT, RESYNC_META_VERSION, WIRE_ZERO_TIME_UNIX};
#[cfg(test)]
use super::replication_storage_boundary::ReplicationDeletedObject;
use super::replication_storage_boundary::{
AdvancedGetOptions, EcstoreObjectOperations, GetObjectReader, HTTPPreconditions, HTTPRangeSpec, ObjectInfo, ObjectOptions,
ObjectToDelete, ReplicationObjectIO, ReplicationStorage, StatObjectOptions, StorageObjectInfoOrErr, WalkOptions,
ObjectToDelete, ReplicationObjectIO, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode, ReplicationStorage,
StatObjectOptions, StorageObjectInfoOrErr, WalkOptions,
};
#[cfg(test)]
use super::replication_storage_boundary::{NamespaceLockFence, NamespaceLockSignalTestFence, ReplicationDeletedObject};
use super::replication_target_boundary::{
ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions,
RemotePutObjectResponse, ReplicationTargetStore, S3ClientError, SsecPassthroughCapability, SsecPassthroughGate, TargetClient,
@@ -1625,6 +1628,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
let mut replication_state = oi.replication_state();
replication_state.replicate_decision_str = dsc.to_string();
let actual_size = oi.get_actual_size_or_physical();
let replication_generation = oi.replication_generation_snapshot();
Ok(ReplicateObjectInfo {
name: oi.name.clone(),
@@ -1647,6 +1651,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
target_statuses,
target_purge_statuses,
replication_timestamp: None,
replication_generation,
ssec: replication_object_is_ssec_encrypted(&user_defined),
user_tags: (*oi.user_tags).clone(),
checksum: oi.checksum.clone(),
@@ -2931,10 +2936,125 @@ pub async fn replicate_object<S: ReplicationStorage>(roi: ReplicateObjectInfo, s
replicate_object_with_outcome(roi, storage).await.0
}
enum ReplicationStatePersistOutcome {
Updated,
Superseded,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum ReplicationAttemptDisposition {
Persisted,
Superseded,
Retry,
}
impl ReplicationAttemptDisposition {
pub(crate) fn consumes_mrf_entry(self) -> bool {
matches!(self, Self::Persisted | Self::Superseded)
}
}
fn replication_attempt_preflight(roi: &ReplicateObjectInfo) -> Option<(ReplicationState, ReplicationAttemptDisposition)> {
roi.replication_generation
.invalid
.then(|| (roi.replication_state.clone().unwrap_or_default(), ReplicationAttemptDisposition::Retry))
}
#[derive(Debug, Default, PartialEq, Eq)]
struct ReplicationTerminalPublication {
emit_terminal_failure: bool,
emit_event: bool,
update_transition_stats: bool,
update_same_state_failure_stats: bool,
}
fn replication_terminal_publication(
suppress_terminal_publication: bool,
state_update_needed: bool,
attempt_status: ReplicationStatusType,
previous_internal_matches_attempt: bool,
) -> ReplicationTerminalPublication {
if suppress_terminal_publication {
return ReplicationTerminalPublication::default();
}
ReplicationTerminalPublication {
emit_terminal_failure: true,
emit_event: true,
update_transition_stats: state_update_needed,
update_same_state_failure_stats: attempt_status != ReplicationStatusType::Completed && previous_internal_matches_attempt,
}
}
fn replication_status_writeback_mode(state_update_needed: bool) -> ReplicationStatusWritebackMode {
if state_update_needed {
ReplicationStatusWritebackMode::Update
} else {
ReplicationStatusWritebackMode::ValidateOnly
}
}
/// Publish a worker's status with a storage-enforced compare-and-set token.
/// `put_object_metadata` checks the token while it owns the object write lock,
/// avoiding both a read/write race and any need to hold a hot object lock
/// across network I/O.
fn replication_status_writeback_options(
roi: &ReplicateObjectInfo,
replication_lock_guard: &rustfs_lock::NamespaceLockGuard,
new_replication_internal: Option<&String>,
mode: ReplicationStatusWritebackMode,
) -> ObjectOptions {
let mut eval_metadata = HashMap::new();
if let Some(status) = new_replication_internal {
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, status.clone());
}
let mut write_opts = ObjectOptions {
version_id: roi.version_id.map(|version_id| version_id.to_string()),
eval_metadata: Some(eval_metadata),
replication_status_writeback: Some(Box::new(ReplicationStatusWritebackCondition {
expected_generation: roi.replication_generation.clone(),
mode,
})),
..Default::default()
};
// The remote transfer runs under a renewable replication namespace lock.
// Carry that guard's loss signal into the storage commit so a worker whose
// lease expired while it was doing remote I/O cannot publish over a newer
// worker for the same generation.
write_opts.add_namespace_lock_guard(replication_lock_guard);
write_opts
}
async fn persist_replication_state_if_current<S: ReplicationStorage>(
roi: &ReplicateObjectInfo,
storage: &Arc<S>,
replication_lock_guard: &rustfs_lock::NamespaceLockGuard,
new_replication_internal: Option<&String>,
mode: ReplicationStatusWritebackMode,
object_info: &mut ObjectInfo,
) -> Result<ReplicationStatePersistOutcome> {
let write_opts = replication_status_writeback_options(roi, replication_lock_guard, new_replication_internal, mode);
match storage.put_object_metadata(&roi.bucket, &roi.name, &write_opts).await {
Ok(updated) => {
*object_info = updated;
Ok(ReplicationStatePersistOutcome::Updated)
}
Err(Error::PreconditionFailed) => Ok(ReplicationStatePersistOutcome::Superseded),
Err(error) => Err(error),
}
}
pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
roi: ReplicateObjectInfo,
storage: Arc<S>,
) -> (ReplicationState, bool) {
) -> (ReplicationState, ReplicationAttemptDisposition) {
// Conflicting compatibility aliases, empty opaque timestamps, and invalid
// mutation UUIDs are corruption, not evidence that another generation
// superseded this task. Fail before any target I/O and keep the MRF entry
// retryable instead of repeatedly transmitting and then acknowledging it.
if let Some(outcome) = replication_attempt_preflight(&roi) {
return outcome;
}
let bucket = roi.bucket.clone();
let object = roi.name.clone();
@@ -2963,10 +3083,10 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
user_agent: "Internal: [Replication]".to_string(),
..Default::default()
});
return (roi.replication_state.unwrap_or_default(), false);
return (roi.replication_state.unwrap_or_default(), ReplicationAttemptDisposition::Retry);
}
};
let _obj_lock_guard = match obj_ns_lock.get_write_lock(ReplicationLockTiming::acquire_timeout()).await {
let obj_lock_guard = match obj_ns_lock.get_write_lock(ReplicationLockTiming::acquire_timeout()).await {
Ok(g) => g,
Err(e) => {
debug!(
@@ -2986,7 +3106,7 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
user_agent: "Internal: [Replication]".to_string(),
..Default::default()
});
return (roi.replication_state.unwrap_or_default(), false);
return (roi.replication_state.unwrap_or_default(), ReplicationAttemptDisposition::Retry);
}
};
@@ -3057,72 +3177,98 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
}
let version_id = roi.version_id.map(|v| v.to_string());
note_replication_terminal_failure(&bucket, &object, version_id.as_deref(), &rinfos);
let previous_state = roi.replication_state.clone().unwrap_or_default();
let merged_state = get_replication_state(&rinfos, &previous_state, version_id);
let replication_status = merged_state.composite_replication_status();
let mut merged_state = get_replication_state(&rinfos, &previous_state, version_id.clone());
let mut replication_status = merged_state.composite_replication_status();
let new_replication_internal = merged_state.replication_status_internal.clone();
let mut object_info = roi.to_object_info();
let mut state_persisted = true;
let mut disposition = ReplicationAttemptDisposition::Persisted;
let mut suppress_terminal_publication = false;
let state_update_needed = roi.replication_status_internal != new_replication_internal || rinfos.replication_resynced();
let writeback_mode = replication_status_writeback_mode(state_update_needed);
if roi.replication_status_internal != new_replication_internal || rinfos.replication_resynced() {
let mut eval_metadata = HashMap::new();
if let Some(ref s) = new_replication_internal {
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, s.clone());
}
let popts = ObjectOptions {
version_id: roi.version_id.map(|v| v.to_string()),
eval_metadata: Some(eval_metadata),
..Default::default()
};
match storage.put_object_metadata(&bucket, &object, &popts).await {
Ok(u) => object_info = u,
Err(e) => {
state_persisted = false;
// Persisting the resynced replication status failed. Don't swallow
// it silently — the object's on-disk status now disagrees with the
// resync result and needs operator visibility (backlog#799 B23).
warn!(
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
object = %object,
error = %e,
"Failed to persist resynced replication status metadata"
);
}
}
if let Some(stats) = runtime_sources::replication_stats() {
for tgt in &rinfos.targets {
if tgt.replication_status != tgt.prev_replication_status {
stats
.update(&bucket, tgt, tgt.replication_status.clone(), tgt.prev_replication_status.clone())
.await;
match persist_replication_state_if_current(
&roi,
&storage,
&obj_lock_guard,
new_replication_internal.as_ref(),
writeback_mode,
&mut object_info,
)
.await
{
Ok(ReplicationStatePersistOutcome::Updated) => {}
Ok(ReplicationStatePersistOutcome::Superseded) => {
// A tag/retention/legal-hold mutation committed while this
// worker was in flight. Its PENDING state is authoritative and
// must remain discoverable by the queue/MRF/scanner after a
// crash or missed admission. Return that newer state to sync
// callers and leave its worker to publish the terminal status.
suppress_terminal_publication = true;
disposition = ReplicationAttemptDisposition::Superseded;
let read_opts = ObjectOptions {
version_id: roi.version_id.map(|version_id| version_id.to_string()),
..Default::default()
};
match storage.get_object_info(&bucket, &object, &read_opts).await {
Ok(current) => {
object_info = current;
merged_state = object_info.replication_state();
replication_status = merged_state.composite_replication_status();
}
Err(error) => {
// The CAS result is authoritative: a best-effort refetch
// failure must not turn a superseded task back into a
// retry that can publish the stale generation later.
debug!(
event = EVENT_RESYNC_STATUS_UPDATE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
object = %object,
error = %error,
reason = "source_snapshot_refetch_failed_after_superseded",
"Could not refresh source state after skipping stale replication status update"
);
}
}
debug!(
event = EVENT_RESYNC_STATUS_UPDATE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
object = %object,
reason = "source_replication_snapshot_superseded",
"Skipped stale replication status update"
);
}
Err(e) => {
disposition = ReplicationAttemptDisposition::Retry;
suppress_terminal_publication = true;
// Persisting the resynced replication status failed. Don't swallow
// it silently — the object's on-disk status now disagrees with the
// resync result and needs operator visibility (backlog#799 B23).
warn!(
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
object = %object,
error = %e,
"Failed to persist resynced replication status metadata"
);
}
}
let event_name = if replication_status == ReplicationStatusType::Completed {
EventName::ObjectReplicationComplete.to_string()
} else {
EventName::ObjectReplicationFailed.to_string()
};
let publication = replication_terminal_publication(
suppress_terminal_publication,
state_update_needed,
rinfos.replication_status(),
roi.replication_status_internal == rinfos.replication_status_internal(),
);
send_local_event(EventArgs {
event_name,
bucket_name: bucket.clone(),
object: object_info,
user_agent: "Internal: [Replication]".to_string(),
..Default::default()
});
if rinfos.replication_status() != ReplicationStatusType::Completed
&& roi.replication_status_internal == rinfos.replication_status_internal()
if publication.update_transition_stats
&& let Some(stats) = runtime_sources::replication_stats()
{
for tgt in &rinfos.targets {
@@ -3134,7 +3280,39 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
}
}
(merged_state, state_persisted)
if publication.emit_terminal_failure {
note_replication_terminal_failure(&bucket, &object, version_id.as_deref(), &rinfos);
}
let event_name = if replication_status == ReplicationStatusType::Completed {
EventName::ObjectReplicationComplete.to_string()
} else {
EventName::ObjectReplicationFailed.to_string()
};
if publication.emit_event {
send_local_event(EventArgs {
event_name,
bucket_name: bucket.clone(),
object: object_info,
user_agent: "Internal: [Replication]".to_string(),
..Default::default()
});
}
if publication.update_same_state_failure_stats
&& let Some(stats) = runtime_sources::replication_stats()
{
for tgt in &rinfos.targets {
if tgt.replication_status != tgt.prev_replication_status {
stats
.update(&bucket, tgt, tgt.replication_status.clone(), tgt.prev_replication_status.clone())
.await;
}
}
}
(merged_state, disposition)
}
/// Emit the operator-visible record of a replication attempt that ended FAILED.
@@ -4567,6 +4745,105 @@ async fn replicate_multipart_parts_and_complete<S: ReplicationObjectIO>(
mod tests {
use super::super::replication_filemeta_boundary::ReplicateTargetDecision;
#[test]
fn same_state_terminal_retry_uses_validate_only() {
assert_eq!(replication_status_writeback_mode(false), ReplicationStatusWritebackMode::ValidateOnly);
assert_eq!(replication_status_writeback_mode(true), ReplicationStatusWritebackMode::Update);
}
#[test]
fn superseded_attempt_has_no_terminal_publication_side_effects() {
assert!(ReplicationAttemptDisposition::Persisted.consumes_mrf_entry());
assert!(ReplicationAttemptDisposition::Superseded.consumes_mrf_entry());
assert!(!ReplicationAttemptDisposition::Retry.consumes_mrf_entry());
assert_eq!(
replication_terminal_publication(true, false, ReplicationStatusType::Failed, true),
ReplicationTerminalPublication::default()
);
assert_eq!(
replication_terminal_publication(true, true, ReplicationStatusType::Completed, false),
ReplicationTerminalPublication::default()
);
let current = replication_terminal_publication(false, false, ReplicationStatusType::Failed, true);
assert!(current.emit_terminal_failure);
assert!(current.emit_event);
assert!(!current.update_transition_stats);
assert!(current.update_same_state_failure_stats);
assert_eq!(
replication_terminal_publication(true, true, ReplicationStatusType::Failed, false),
ReplicationTerminalPublication::default(),
"a retryable status-persistence failure must not publish a terminal result"
);
}
#[test]
fn invalid_generation_retries_before_remote_replication() {
let mut preserved_state = ReplicationState::default();
preserved_state
.targets
.insert("arn:target".to_string(), ReplicationStatusType::Pending);
let invalid = ReplicateObjectInfo {
replication_generation: ReplicationGenerationSnapshot {
invalid: true,
..Default::default()
},
replication_state: Some(preserved_state.clone()),
..Default::default()
};
assert_eq!(
replication_attempt_preflight(&invalid),
Some((preserved_state, ReplicationAttemptDisposition::Retry))
);
assert!(replication_attempt_preflight(&ReplicateObjectInfo::default()).is_none());
}
#[tokio::test]
async fn terminal_writeback_carries_replication_lock_loss_fence() {
let lock = rustfs_lock::NamespaceLock::new(
"replication-status-writeback-fence".to_string(),
Arc::new(rustfs_lock::LocalClient::new()),
);
let guard = lock
.get_write_lock(
rustfs_lock::ObjectKey::new("bucket", "/[replicate]/object"),
"worker-a",
std::time::Duration::from_secs(2),
)
.await
.expect("replication lock should be acquired");
let signal = guard
.lock_lost_signal()
.expect("distributed guard must expose its loss signal");
let forced_lost = Arc::new(std::sync::atomic::AtomicBool::new(false));
let _test_fence = NamespaceLockSignalTestFence::install_with_loss_handle(&signal, Arc::clone(&forced_lost));
// Build the writeback after remote work has acquired the guard, then
// lose the lease before storage reaches its commit fence.
let opts = replication_status_writeback_options(
&ReplicateObjectInfo::default(),
&guard,
None,
ReplicationStatusWritebackMode::ValidateOnly,
);
forced_lost.store(true, std::sync::atomic::Ordering::Release);
assert!(
opts.namespace_lock_fence
.as_ref()
.is_some_and(NamespaceLockFence::is_lock_lost),
"terminal CAS must observe a replication lease lost after remote I/O"
);
assert!(!ReplicationAttemptDisposition::Retry.consumes_mrf_entry());
assert_eq!(
replication_terminal_publication(true, true, ReplicationStatusType::Failed, false),
ReplicationTerminalPublication::default(),
"a fenced writeback retry must not publish terminal events or statistics"
);
}
#[test]
fn unavailable_object_target_is_persisted_as_failed() {
let arn = "arn:object-target";
@@ -18,7 +18,11 @@ use tokio_util::sync::CancellationToken;
use super::replication_error_boundary::Error;
use super::replication_filemeta_boundary::{replication_state_from_filemeta, version_purge_status_from_filemeta};
pub(crate) type ReplicationObjectStore = crate::store::ECStore;
pub(crate) use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
pub(crate) use crate::object_api::{
GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode,
};
#[cfg(test)]
pub(crate) use crate::object_api::{NamespaceLockFence, NamespaceLockSignalTestFence};
pub(crate) use crate::storage_api_contracts::list::{
ListOperations, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions,
};
@@ -849,6 +849,11 @@ mod tests {
metadata.insert(MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), "DAREv2-HMAC-SHA256".to_string());
metadata.insert(MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER.to_string(), "sealed".to_string());
metadata.insert(MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER.to_string(), "true".to_string());
rustfs_utils::http::insert_str(
&mut metadata,
rustfs_utils::http::SUFFIX_REPLICATION_GENERATION,
Uuid::from_u128(1).to_string(),
);
let object_info = ObjectInfo {
user_defined: Arc::new(metadata),
@@ -894,6 +899,13 @@ mod tests {
assert!(!options.user_metadata.contains_key(AMZ_SERVER_SIDE_ENCRYPTION));
assert!(!options.user_metadata.contains_key(SSEC_ALGORITHM_HEADER));
assert!(!options.user_metadata.contains_key(INTERNAL_ENCRYPTION_IV_HEADER));
assert!(
options
.user_metadata
.keys()
.all(|key| !key.contains(rustfs_utils::http::SUFFIX_REPLICATION_GENERATION)),
"source-local mutation generation must never cross the replication wire"
);
assert!(
!options
.user_metadata
+3 -2
View File
@@ -18,8 +18,9 @@ pub mod object_api_utils;
use crate::bucket::metadata_sys::get_versioning_config;
use crate::bucket::replication::{
DeleteReplicationConfigSnapshot, ReplicateDecision, ReplicationState, ReplicationStatusType, VersionPurgeStatusType,
replication_status_from_filemeta, replication_statuses_map, version_purge_status_from_filemeta, version_purge_statuses_map,
DeleteReplicationConfigSnapshot, ReplicateDecision, ReplicationGenerationSnapshot, ReplicationState, ReplicationStatusType,
VersionPurgeStatusType, replication_status_from_filemeta, replication_statuses_map, version_purge_status_from_filemeta,
version_purge_statuses_map,
};
use crate::bucket::versioning::VersioningApi as _;
use crate::config::storageclass;
+195
View File
@@ -19,6 +19,7 @@ use crate::storage_api_contracts::{
HTTPPreconditions, ObjectLockRetentionOptions, ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState,
},
};
use sha2::{Digest, Sha256};
use std::io;
use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
use tokio::sync::{Mutex, Notify, OwnedRwLockReadGuard};
@@ -980,6 +981,13 @@ pub struct ObjectOptions {
pub lifecycle_audit_event: LcAuditEvent,
pub eval_metadata: Option<HashMap<String, String>>,
/// Internal compare-and-set condition for replication workers publishing
/// terminal status after remote I/O. Storage validates it while holding
/// the object write lock so an older worker cannot overwrite a newer
/// mutation's PENDING state. Keep the condition boxed because
/// `ObjectOptions` is passed by value through deep storage futures.
#[doc(hidden)]
pub replication_status_writeback: Option<Box<ReplicationStatusWritebackCondition>>,
pub object_lock_retention: Option<ObjectLockRetentionOptions>,
pub object_lock_delete: Option<crate::storage_api_contracts::object::ObjectLockDeleteOptions>,
/// Authoritative bucket Object Lock snapshot installed inside `ECStore`
@@ -1000,6 +1008,21 @@ pub struct ObjectOptions {
pub decommission_capacity_admission: Option<Arc<crate::store::ECStore>>,
}
#[derive(Clone, Debug, Default)]
#[doc(hidden)]
pub struct ReplicationStatusWritebackCondition {
pub(crate) expected_generation: ReplicationGenerationSnapshot,
pub(crate) mode: ReplicationStatusWritebackMode,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
#[doc(hidden)]
pub enum ReplicationStatusWritebackMode {
#[default]
Update,
ValidateOnly,
}
impl ObjectOptions {
pub(crate) fn with_capacity_expected_data_bytes(expected_data_bytes: Option<usize>) -> Self {
Self {
@@ -1086,6 +1109,7 @@ impl std::fmt::Debug for ObjectOptions {
|| !self.lifecycle_audit_event.event.storage_class.is_empty()),
)
.field("eval_metadata_count", &self.eval_metadata.as_ref().map(HashMap::len))
.field("replication_status_writeback", &self.replication_status_writeback.is_some())
.field("object_lock_retention", &self.object_lock_retention.is_some())
.field("object_lock_delete", &self.object_lock_delete)
.field("object_lock_config_snapshot", &self.object_lock_config_snapshot.is_some())
@@ -1281,6 +1305,32 @@ impl ObjectOptions {
}
}
fn replication_snapshot_internal_value(
metadata: &HashMap<String, String>,
suffix: &str,
) -> std::result::Result<Option<String>, ()> {
match rustfs_utils::http::get_consistent_str(metadata, suffix) {
Some(value) => Ok(Some(value.to_string())),
None if rustfs_utils::http::contains_key_str(metadata, suffix) => Err(()),
None => Ok(None),
}
}
fn update_replication_fingerprint_bytes(hasher: &mut Sha256, value: &[u8]) {
let len = u64::try_from(value.len()).unwrap_or(u64::MAX);
hasher.update(len.to_le_bytes());
hasher.update(value);
}
fn update_replication_fingerprint_optional_str(hasher: &mut Sha256, value: Option<&str>) {
if let Some(value) = value {
hasher.update([1]);
update_replication_fingerprint_bytes(hasher, value.as_bytes());
} else {
hasher.update([0]);
}
}
#[derive(Debug, Default)]
pub struct ObjectInfo {
pub bucket: String,
@@ -1369,6 +1419,144 @@ impl Clone for ObjectInfo {
}
impl ObjectInfo {
/// Capture the source mutation snapshot used by replication workers when
/// publishing terminal status. The semantic fingerprint is recomputed at
/// the storage CAS boundary, so an older writer that preserves an unknown
/// UUID and collides on the timestamp still cannot hide a payload change.
pub(crate) fn replication_generation_snapshot(&self) -> ReplicationGenerationSnapshot {
let timestamp = replication_snapshot_internal_value(&self.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP);
let mutation_id =
replication_snapshot_internal_value(&self.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_GENERATION);
let tagging_timestamp =
replication_snapshot_internal_value(&self.user_defined, rustfs_utils::http::SUFFIX_TAGGING_TIMESTAMP);
let retention_timestamp =
replication_snapshot_internal_value(&self.user_defined, rustfs_utils::http::SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP);
let legalhold_timestamp =
replication_snapshot_internal_value(&self.user_defined, rustfs_utils::http::SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP);
let invalid = timestamp.is_err()
|| mutation_id.is_err()
|| tagging_timestamp.is_err()
|| retention_timestamp.is_err()
|| legalhold_timestamp.is_err();
let timestamp = timestamp.unwrap_or_default();
let mutation_id = mutation_id.unwrap_or_default();
let tagging_timestamp = tagging_timestamp.unwrap_or_default();
let retention_timestamp = retention_timestamp.unwrap_or_default();
let legalhold_timestamp = legalhold_timestamp.unwrap_or_default();
let opaque_timestamp_is_invalid = [
timestamp.as_deref(),
tagging_timestamp.as_deref(),
retention_timestamp.as_deref(),
legalhold_timestamp.as_deref(),
]
.into_iter()
.flatten()
.any(str::is_empty);
let mutation_id_is_invalid = mutation_id.as_deref().is_some_and(|value| {
Uuid::parse_str(value)
.ok()
.filter(|generation| !generation.is_nil())
.is_none()
});
let invalid =
invalid || opaque_timestamp_is_invalid || mutation_id_is_invalid || (mutation_id.is_some() && timestamp.is_none());
let payload_fingerprint = (!invalid).then(|| {
self.replication_payload_fingerprint(
tagging_timestamp.as_deref(),
retention_timestamp.as_deref(),
legalhold_timestamp.as_deref(),
)
});
ReplicationGenerationSnapshot {
timestamp,
mutation_id,
payload_fingerprint,
invalid,
}
}
fn replication_payload_fingerprint(
&self,
tagging_timestamp: Option<&str>,
retention_timestamp: Option<&str>,
legalhold_timestamp: Option<&str>,
) -> [u8; 32] {
let mut hasher = Sha256::new();
hasher.update(b"rustfs-replication-payload-v1");
// A suspended bucket's current null version is represented as
// `Some(Uuid::nil())` at the request/queue boundary and as `None`
// when the same xl.meta is reread without versioning flags. They are
// one persisted identity, so do not let that representation detail
// make a worker permanently supersede its own terminal write-back.
if let Some(version_id) = self.version_id.filter(|version_id| !version_id.is_nil()) {
hasher.update([1]);
hasher.update(version_id.as_bytes());
} else {
hasher.update([0]);
}
if let Some(data_dir) = self.data_dir {
hasher.update([1]);
hasher.update(data_dir.as_bytes());
} else {
hasher.update([0]);
}
if let Some(mod_time) = self.mod_time {
hasher.update([1]);
hasher.update(mod_time.unix_timestamp_nanos().to_le_bytes());
} else {
hasher.update([0]);
}
update_replication_fingerprint_optional_str(&mut hasher, self.content_type.as_deref());
update_replication_fingerprint_optional_str(&mut hasher, self.content_encoding.as_deref());
update_replication_fingerprint_optional_str(&mut hasher, self.storage_class.as_deref());
if let Some(expires) = self.expires {
hasher.update([1]);
hasher.update(expires.unix_timestamp_nanos().to_le_bytes());
} else {
hasher.update([0]);
}
update_replication_fingerprint_bytes(&mut hasher, self.user_tags.as_bytes());
update_replication_fingerprint_optional_str(&mut hasher, tagging_timestamp);
update_replication_fingerprint_optional_str(&mut hasher, retention_timestamp);
update_replication_fingerprint_optional_str(&mut hasher, legalhold_timestamp);
let mut target_arns = self
.replication_status_internal
.as_deref()
.map(replication_statuses_map)
.unwrap_or_default()
.into_keys()
.collect::<Vec<_>>();
target_arns.sort_unstable();
hasher.update(u64::try_from(target_arns.len()).unwrap_or(u64::MAX).to_le_bytes());
for arn in target_arns {
update_replication_fingerprint_bytes(&mut hasher, arn.as_bytes());
}
update_replication_fingerprint_bytes(&mut hasher, self.replication_decision.as_bytes());
let mut user_metadata = self
.user_defined
.iter()
.filter(|(key, _)| {
!rustfs_utils::http::is_internal_key(key)
&& !key.eq_ignore_ascii_case(rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS)
})
.collect::<Vec<_>>();
user_metadata.sort_unstable_by(|left, right| left.0.cmp(right.0).then_with(|| left.1.cmp(right.1)));
hasher.update(u64::try_from(user_metadata.len()).unwrap_or(u64::MAX).to_le_bytes());
for (key, value) in user_metadata {
update_replication_fingerprint_bytes(&mut hasher, key.as_bytes());
update_replication_fingerprint_bytes(&mut hasher, value.as_bytes());
}
hasher.finalize().into()
}
pub fn is_compressed(&self) -> bool {
rustfs_utils::http::contains_key_str(&self.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION)
}
@@ -2001,6 +2189,13 @@ fn versions_after_marker(file_infos: &rustfs_filemeta::FileInfoVersions, marker:
mod tests {
use super::*;
#[test]
fn replication_status_writeback_condition_remains_indirected() {
fn assert_indirected(_: &Option<Box<ReplicationStatusWritebackCondition>>) {}
assert_indirected(&ObjectOptions::default().replication_status_writeback);
}
#[test]
fn object_lock_config_snapshot_is_bound_to_store_bucket_and_incarnation() {
let store_id = Uuid::new_v4();
@@ -55,6 +55,7 @@ use crate::api::config::storageclass;
#[cfg(test)]
use crate::bucket::metadata_sys::ObjectLockConfigState;
use crate::bucket::quota::reservation;
use crate::bucket::replication::ReplicationStatusType;
use crate::crash_inject::{self, CrashPoint};
use crate::disk::DiskAPI;
#[cfg(test)]
@@ -2366,6 +2367,43 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
ensure_data_movement_upload_access(&fi, bucket, object, upload_id, opts)?;
ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
.await?;
// The request layer computes replication admission immediately before
// completion. Apply that metadata only after the staged upload and its
// bucket incarnation are validated, while the object/upload commit
// locks are still held, so generation, PENDING targets, and the final
// object become visible as one commit.
if let Some(eval_metadata) = &opts.eval_metadata {
for suffix in [
rustfs_utils::http::SUFFIX_REPLICATION_GENERATION,
rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,
rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
] {
rustfs_utils::http::remove_str(&mut fi.metadata, suffix);
}
// A source-side completion creates a new local object identity;
// replica bookkeeping carried by an internal/legacy staged upload
// belongs to the source object and must not survive. Authorized
// inbound replication owns these fields and remains exempt. Older
// and heterogeneous replication clients can send an authorized
// REPLICA status on Complete without repeating the source-request
// marker, so either authenticated parser result is sufficient.
let authorized_inbound_replica =
opts.replication_request || opts.delete_marker_replication_status() == ReplicationStatusType::Replica;
if !authorized_inbound_replica {
fi.metadata
.retain(|key, _| !key.eq_ignore_ascii_case(rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS));
for suffix in [
rustfs_utils::http::SUFFIX_REPLICA_STATUS,
rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP,
] {
rustfs_utils::http::remove_str(&mut fi.metadata, suffix);
}
}
for (key, value) in eval_metadata {
fi.metadata.insert(key.clone(), value.clone());
}
fi.replication_state_internal = rustfs_filemeta::get_internal_replication_state(&fi.metadata);
}
let has_layout_candidate = range_seek_rollout_enabled
&& fi
.data_dir
@@ -2910,6 +2948,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
meta.mod_time = fi.mod_time;
meta.parts.clone_from(&fi.parts);
meta.metadata = fi.metadata.clone();
meta.replication_state_internal = fi.replication_state_internal.clone();
meta.versioned = opts.versioned || opts.version_suspended;
meta.version_id = fi.version_id;
meta.checksum = fi.checksum.clone();
@@ -8248,6 +8287,215 @@ mod tests {
assert_eq!(current.version_id, Some(version_id));
}
#[tokio::test]
#[serial]
async fn complete_multipart_upload_commits_replication_admission_metadata_atomically() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "multipart-replication-admission-bucket";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
let mut create_replication_metadata = HashMap::new();
rustfs_utils::http::insert_str(
&mut create_replication_metadata,
rustfs_utils::http::SUFFIX_REPLICATION_GENERATION,
Uuid::from_u128(1).to_string(),
);
rustfs_utils::http::insert_str(
&mut create_replication_metadata,
rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,
"create-time".to_string(),
);
rustfs_utils::http::insert_str(
&mut create_replication_metadata,
rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
"arn:create-target=PENDING;".to_string(),
);
rustfs_utils::http::insert_str(
&mut create_replication_metadata,
rustfs_utils::http::SUFFIX_REPLICA_STATUS,
"REPLICA".to_string(),
);
rustfs_utils::http::insert_str(
&mut create_replication_metadata,
rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP,
"foreign-replica-time".to_string(),
);
create_replication_metadata
.insert(rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS.to_string(), "REPLICA".to_string());
let (upload_id, parts) = stage_upload_with_create_opts(
&set_disks,
bucket,
object,
b"multipart replication body",
&ObjectOptions {
user_defined: create_replication_metadata.clone(),
..Default::default()
},
)
.await;
let generation = Uuid::from_u128(42).to_string();
let timestamp = "opaque-completion-time";
let status = "arn:complete-target=PENDING;";
let mut replication_metadata = HashMap::new();
rustfs_utils::http::insert_str(
&mut replication_metadata,
rustfs_utils::http::SUFFIX_REPLICATION_GENERATION,
generation.clone(),
);
rustfs_utils::http::insert_str(
&mut replication_metadata,
rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,
timestamp.to_string(),
);
rustfs_utils::http::insert_str(
&mut replication_metadata,
rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
status.to_string(),
);
let completed = set_disks
.clone()
.complete_multipart_upload(
bucket,
object,
&upload_id,
parts,
&ObjectOptions {
eval_metadata: Some(replication_metadata),
..Default::default()
},
)
.await
.expect("multipart completion with replication admission should succeed");
assert_eq!(
rustfs_utils::http::get_str(completed.user_defined.as_ref(), rustfs_utils::http::SUFFIX_REPLICATION_GENERATION,)
.as_deref(),
Some(generation.as_str())
);
assert_eq!(
rustfs_utils::http::get_str(completed.user_defined.as_ref(), rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,)
.as_deref(),
Some(timestamp)
);
assert_eq!(completed.replication_status_internal.as_deref(), Some(status));
for suffix in [
rustfs_utils::http::SUFFIX_REPLICA_STATUS,
rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP,
] {
assert!(
!rustfs_utils::http::contains_key_str(completed.user_defined.as_ref(), suffix),
"source completion must clear staged foreign {suffix}"
);
}
assert!(
completed
.user_defined
.iter()
.filter(|(key, _)| key.eq_ignore_ascii_case(rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS))
.all(|(_, value)| value != "REPLICA")
);
let disabled_object = "disabled";
let (upload_id, parts) = stage_upload_with_create_opts(
&set_disks,
bucket,
disabled_object,
b"multipart replication disabled body",
&ObjectOptions {
user_defined: create_replication_metadata,
..Default::default()
},
)
.await;
let completed = set_disks
.clone()
.complete_multipart_upload(
bucket,
disabled_object,
&upload_id,
parts,
&ObjectOptions {
eval_metadata: Some(HashMap::new()),
..Default::default()
},
)
.await
.expect("completion should clear stale create-time replication admission");
assert!(completed.replication_status_internal.is_none());
for suffix in [
rustfs_utils::http::SUFFIX_REPLICATION_GENERATION,
rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,
rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
rustfs_utils::http::SUFFIX_REPLICA_STATUS,
rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP,
] {
assert!(
!rustfs_utils::http::contains_key_str(completed.user_defined.as_ref(), suffix),
"disabled completion must clear stale {suffix}"
);
}
let inbound_object = "inbound";
let mut inbound_replica_metadata = HashMap::new();
rustfs_utils::http::insert_str(
&mut inbound_replica_metadata,
rustfs_utils::http::SUFFIX_REPLICA_STATUS,
"REPLICA".to_string(),
);
rustfs_utils::http::insert_str(
&mut inbound_replica_metadata,
rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP,
"authorized-inbound-time".to_string(),
);
inbound_replica_metadata.insert(rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS.to_string(), "REPLICA".to_string());
let (upload_id, parts) = stage_upload_with_create_opts(
&set_disks,
bucket,
inbound_object,
b"authorized inbound multipart body",
&ObjectOptions {
user_defined: inbound_replica_metadata,
..Default::default()
},
)
.await;
let mut inbound_complete_opts = ObjectOptions {
eval_metadata: Some(HashMap::new()),
..Default::default()
};
inbound_complete_opts.set_replica_status(ReplicationStatusType::Replica);
assert!(
!inbound_complete_opts.replication_request,
"the compatibility path must not depend on a source-request marker"
);
let inbound = set_disks
.clone()
.complete_multipart_upload(bucket, inbound_object, &upload_id, parts, &inbound_complete_opts)
.await
.expect("authorized REPLICA-only completion must preserve replica bookkeeping");
assert_eq!(
rustfs_utils::http::get_consistent_str(inbound.user_defined.as_ref(), rustfs_utils::http::SUFFIX_REPLICA_STATUS),
Some("REPLICA")
);
assert_eq!(
rustfs_utils::http::get_consistent_str(
inbound.user_defined.as_ref(),
rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP
),
Some("authorized-inbound-time")
);
assert_eq!(
inbound
.user_defined
.iter()
.find(|(key, _)| key.eq_ignore_ascii_case(rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS))
.map(|(_, value)| value.as_str()),
Some("REPLICA")
);
}
#[tokio::test]
#[serial]
async fn complete_multipart_upload_replaces_staged_nil_version_id() {
File diff suppressed because it is too large Load Diff