mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-05 12:57:42 +00:00
fix(replication): persist force-delete handoff state (#5641)
* fix(replication): persist force-delete handoff state * fix(arch): route force-delete config access through boundary * style: format force-delete imports --------- Co-authored-by: Zhengchao An <anzhengchao@gmail.com>
This commit is contained in:
@@ -188,11 +188,12 @@ pub mod bucket {
|
||||
ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
|
||||
ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
|
||||
ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
|
||||
VersionPurgeStatusType, delete_replication_state_from_config, delete_replication_version_id,
|
||||
get_global_replication_pool, get_global_replication_stats, init_background_replication,
|
||||
invalid_replication_config_status_field, read_durable_mrf_backlog, replication_state_to_filemeta,
|
||||
replication_status_to_filemeta, replication_statuses_map, replication_target_arns, resync_start_conflict_id,
|
||||
should_remove_replication_target, should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
VersionPurgeStatusType, commit_force_delete_intent, complete_force_delete_intent,
|
||||
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
|
||||
get_global_replication_stats, init_background_replication, invalid_replication_config_status_field,
|
||||
persist_force_delete_intent, read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta,
|
||||
replication_statuses_map, replication_target_arns, resync_start_conflict_id, should_remove_replication_target,
|
||||
should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
should_use_existing_delete_replication_source, unsupported_replication_config_field,
|
||||
validate_replication_config_target_arns, version_purge_status_to_filemeta,
|
||||
};
|
||||
|
||||
@@ -70,8 +70,9 @@ pub use replication_object_decision_boundary::{
|
||||
should_use_existing_delete_replication_source,
|
||||
};
|
||||
pub use replication_pool::{
|
||||
DurableMrfBacklog, DynReplicationPool, ReplicationPoolTrait, get_global_replication_pool, get_global_replication_stats,
|
||||
init_background_replication, read_durable_mrf_backlog, resync_start_conflict_id,
|
||||
DurableMrfBacklog, DynReplicationPool, ReplicationPoolTrait, commit_force_delete_intent, complete_force_delete_intent,
|
||||
get_global_replication_pool, get_global_replication_stats, init_background_replication, persist_force_delete_intent,
|
||||
read_durable_mrf_backlog, resync_start_conflict_id,
|
||||
};
|
||||
pub use replication_queue_boundary::{
|
||||
DeletedObjectReplicationInfo, ReplicationBatchAdmission, ReplicationHealQueueResult, ReplicationOperation,
|
||||
|
||||
@@ -30,10 +30,24 @@ impl ReplicationConfigStore {
|
||||
com::read_config(api, file).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
|
||||
where
|
||||
S: ReplicationObjectIO,
|
||||
{
|
||||
com::read_config_no_lock(api, file).await
|
||||
}
|
||||
|
||||
pub(crate) async fn save<S>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()>
|
||||
where
|
||||
S: ReplicationObjectIO,
|
||||
{
|
||||
com::save_config(api, file, data).await
|
||||
}
|
||||
|
||||
pub(crate) async fn save_no_lock<S>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()>
|
||||
where
|
||||
S: ReplicationObjectIO,
|
||||
{
|
||||
com::save_config_no_lock(api, file, data).await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,6 +32,7 @@ pub(crate) struct ReplicationMetadataStore;
|
||||
|
||||
impl ReplicationMetadataStore {
|
||||
pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
|
||||
pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
|
||||
|
||||
pub(crate) async fn replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
metadata_sys::get_replication_config(bucket).await
|
||||
@@ -109,5 +110,9 @@ mod tests {
|
||||
"buckets/bucket-a/.replication/resync.bin"
|
||||
);
|
||||
assert_eq!(ReplicationMetadataStore::MRF_REPLICATION_FILE, "config/replication/mrf.bin");
|
||||
assert_eq!(
|
||||
ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
|
||||
"config/replication/force-delete.bin"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,6 +89,13 @@ impl ReplicationObjectBridge {
|
||||
snapshot.has_active_rule(object)
|
||||
}
|
||||
|
||||
pub fn force_delete_target_set(
|
||||
snapshot: &DeleteReplicationConfigSnapshot,
|
||||
prefix: &str,
|
||||
) -> Option<(Vec<String>, time::OffsetDateTime)> {
|
||||
snapshot.force_delete_target_set(prefix)
|
||||
}
|
||||
|
||||
pub fn check_delete_with_snapshot(
|
||||
object: &ObjectToDelete,
|
||||
source: &ObjectInfo,
|
||||
|
||||
@@ -18,6 +18,7 @@ use crate::bucket::metadata::BucketMetadata;
|
||||
use rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS;
|
||||
use s3s::dto::{BucketVersioningStatus, ReplicationConfiguration, ReplicationRuleStatus, VersioningConfiguration};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use time::OffsetDateTime;
|
||||
use tracing::error;
|
||||
|
||||
use super::replication_config_boundary::{
|
||||
@@ -83,6 +84,15 @@ impl DeleteReplicationConfigSnapshot {
|
||||
.and_then(|metadata| metadata.replication_config.as_ref())
|
||||
}
|
||||
|
||||
pub(crate) fn force_delete_target_set(&self, prefix: &str) -> Option<(Vec<String>, OffsetDateTime)> {
|
||||
self.metadata.as_ref().and_then(|metadata| {
|
||||
metadata
|
||||
.replication_config
|
||||
.as_ref()
|
||||
.map(|config| (config.filter_force_delete_target_arns(prefix), metadata.replication_config_updated_at))
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn has_active_rule(&self, object: &str) -> bool {
|
||||
self.replication_config()
|
||||
.is_some_and(|config| config.has_active_rules(object, true))
|
||||
|
||||
@@ -136,6 +136,10 @@ static DURABLE_MRF_TARGET_BACKLOG: LazyLock<StdRwLock<Vec<DurableMrfTargetBacklo
|
||||
static MRF_BACKLOG_OBSERVABILITY: LazyLock<StdRwLock<MrfBacklogObservabilityTracker>> =
|
||||
LazyLock::new(|| StdRwLock::new(MrfBacklogObservabilityTracker::default()));
|
||||
|
||||
fn should_replay_force_delete_intent(entry: &MrfReplicateEntry) -> bool {
|
||||
entry.force_delete_id.is_some() && entry.force_delete_local_commit && !entry.target_arns.is_empty()
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
struct DurableMrfBacklogTracker {
|
||||
available: bool,
|
||||
@@ -302,6 +306,7 @@ where
|
||||
delete_marker: false,
|
||||
delete_marker_mtime: None,
|
||||
target_arns: Vec::new(),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
tracker.into_snapshot()
|
||||
@@ -394,6 +399,82 @@ pub async fn read_durable_mrf_backlog<S: ReplicationObjectIO>(storage: Arc<S>) -
|
||||
durable_mrf_backlog_from_read(ReplicationConfigStore::read(storage, ReplicationMetadataStore::MRF_REPLICATION_FILE).await)
|
||||
}
|
||||
|
||||
pub async fn persist_force_delete_intent<S: ReplicationStorage>(
|
||||
storage: Arc<S>,
|
||||
mut entry: MrfReplicateEntry,
|
||||
) -> Result<(), EcstoreError> {
|
||||
entry.force_delete_local_commit = false;
|
||||
let file = ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE;
|
||||
let lock = storage
|
||||
.new_ns_lock(ReplicationMetadataStore::rustfs_meta_bucket(), file)
|
||||
.await?;
|
||||
let _guard = lock.get_write_lock(ReplicationLockTiming::acquire_timeout()).await?;
|
||||
|
||||
let mut entries = match ReplicationConfigStore::read_no_lock(storage.clone(), file).await {
|
||||
Ok(data) => decode_mrf_file(&data)?,
|
||||
Err(EcstoreError::ConfigNotFound) => Vec::new(),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
if entries
|
||||
.iter()
|
||||
.any(|existing| existing.force_delete_id == entry.force_delete_id)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
entries.push(entry);
|
||||
let data = encode_mrf_file(&entries)?;
|
||||
ReplicationConfigStore::save_no_lock(storage, file, data).await
|
||||
}
|
||||
|
||||
pub async fn commit_force_delete_intent<S: ReplicationStorage>(
|
||||
storage: Arc<S>,
|
||||
operation_id: uuid::Uuid,
|
||||
) -> Result<(), EcstoreError> {
|
||||
let file = ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE;
|
||||
let lock = storage
|
||||
.new_ns_lock(ReplicationMetadataStore::rustfs_meta_bucket(), file)
|
||||
.await?;
|
||||
let _guard = lock.get_write_lock(ReplicationLockTiming::acquire_timeout()).await?;
|
||||
|
||||
let data = ReplicationConfigStore::read_no_lock(storage.clone(), file).await?;
|
||||
let mut entries = decode_mrf_file(&data)?;
|
||||
let Some(entry) = entries.iter_mut().find(|entry| entry.force_delete_id == Some(operation_id)) else {
|
||||
return Err(EcstoreError::ConfigNotFound);
|
||||
};
|
||||
if entry.force_delete_local_commit {
|
||||
return Ok(());
|
||||
}
|
||||
entry.force_delete_local_commit = true;
|
||||
ReplicationConfigStore::save_no_lock(storage, file, encode_mrf_file(&entries)?).await
|
||||
}
|
||||
|
||||
pub async fn complete_force_delete_intent<S: ReplicationStorage>(
|
||||
storage: Arc<S>,
|
||||
operation_id: uuid::Uuid,
|
||||
) -> Result<(), EcstoreError> {
|
||||
let file = ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE;
|
||||
let lock = storage
|
||||
.new_ns_lock(ReplicationMetadataStore::rustfs_meta_bucket(), file)
|
||||
.await?;
|
||||
let _guard = lock.get_write_lock(ReplicationLockTiming::acquire_timeout()).await?;
|
||||
|
||||
let data = match ReplicationConfigStore::read_no_lock(storage.clone(), file).await {
|
||||
Ok(data) => data,
|
||||
Err(EcstoreError::ConfigNotFound) => return Ok(()),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let mut entries = decode_mrf_file(&data)?;
|
||||
let original_len = entries.len();
|
||||
entries.retain(|entry| entry.force_delete_id != Some(operation_id));
|
||||
if entries.len() == original_len {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
ReplicationConfigStore::save_no_lock(storage, file, encode_mrf_file(&entries)?).await
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
|
||||
struct ResyncActiveConflictError {
|
||||
@@ -497,6 +578,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
// Start background tasks
|
||||
pool.start_mrf_processor().await;
|
||||
pool.start_force_delete_processor().await;
|
||||
pool.start_mrf_persister().await;
|
||||
|
||||
pool
|
||||
@@ -972,6 +1054,32 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
for entry in entries.iter() {
|
||||
match entry.op {
|
||||
MrfOpKind::Delete => {
|
||||
if should_replay_force_delete_intent(entry) {
|
||||
let Some(operation_id) = entry.force_delete_id else {
|
||||
continue;
|
||||
};
|
||||
schedule_replication_delete(DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: entry.object.clone(),
|
||||
force_delete: true,
|
||||
force_delete_id: Some(operation_id),
|
||||
force_delete_target_arns: entry.target_arns.clone(),
|
||||
force_delete_generation: entry.force_delete_generation,
|
||||
..Default::default()
|
||||
},
|
||||
bucket: entry.bucket.clone(),
|
||||
op_type: ReplicationType::Heal,
|
||||
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
queued_count += 1;
|
||||
continue;
|
||||
}
|
||||
if entry.force_delete_id.is_some() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Reconstruct a heal delete and re-queue it. We do NOT call
|
||||
// get_object_info here because the delete-marker or version may
|
||||
// already be absent from the local store — that is expected.
|
||||
@@ -1131,6 +1239,67 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
self.task_handles.lock().await.push(handle);
|
||||
}
|
||||
|
||||
async fn start_force_delete_processor(&self) {
|
||||
let storage = self.storage.clone();
|
||||
let handle =
|
||||
tokio::spawn(async move {
|
||||
let data =
|
||||
match ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE)
|
||||
.await
|
||||
{
|
||||
Ok(data) => data,
|
||||
Err(EcstoreError::ConfigNotFound) => return,
|
||||
Err(error) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to load durable force-delete intents"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let entries = match decode_mrf_file(&data) {
|
||||
Ok(entries) => entries,
|
||||
Err(error) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to decode durable force-delete intents"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
for entry in entries {
|
||||
if !should_replay_force_delete_intent(&entry) {
|
||||
continue;
|
||||
}
|
||||
let Some(operation_id) = entry.force_delete_id else {
|
||||
continue;
|
||||
};
|
||||
schedule_replication_delete(DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: entry.object,
|
||||
force_delete: true,
|
||||
force_delete_id: Some(operation_id),
|
||||
force_delete_target_arns: entry.target_arns,
|
||||
force_delete_generation: entry.force_delete_generation,
|
||||
..Default::default()
|
||||
},
|
||||
bucket: entry.bucket,
|
||||
op_type: ReplicationType::Heal,
|
||||
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
}
|
||||
});
|
||||
self.task_handles.lock().await.push(handle);
|
||||
}
|
||||
|
||||
/// Starts the MRF persister — ongoing background task.
|
||||
///
|
||||
/// Drains `mrf_save_rx` (entries that overflowed the normal worker channels) and
|
||||
@@ -2216,7 +2385,7 @@ mod tests {
|
||||
_h: Self::HeaderMap,
|
||||
_opts: &Self::ObjectOptions,
|
||||
) -> Result<Self::GetObjectReader, Self::Error> {
|
||||
if !object.ends_with("/.replication/resync.bin") {
|
||||
if !object.ends_with("/.replication/resync.bin") && !object.ends_with("config/replication/force-delete.bin") {
|
||||
return Err(EcstoreError::FileNotFound);
|
||||
}
|
||||
|
||||
@@ -3104,6 +3273,7 @@ mod tests {
|
||||
delete_marker: false,
|
||||
delete_marker_mtime: None,
|
||||
target_arns: Vec::new(),
|
||||
..Default::default()
|
||||
};
|
||||
let second = MrfReplicateEntry {
|
||||
object: "second".to_string(),
|
||||
@@ -3202,6 +3372,7 @@ mod tests {
|
||||
delete_marker: false,
|
||||
delete_marker_mtime: None,
|
||||
target_arns: Vec::new(),
|
||||
..Default::default()
|
||||
},
|
||||
"test",
|
||||
)
|
||||
@@ -3234,6 +3405,7 @@ mod tests {
|
||||
delete_marker: false,
|
||||
delete_marker_mtime: None,
|
||||
target_arns: Vec::new(),
|
||||
..Default::default()
|
||||
};
|
||||
observe_mrf_pending(&entry);
|
||||
|
||||
@@ -3289,6 +3461,7 @@ mod tests {
|
||||
delete_marker: false,
|
||||
delete_marker_mtime: None,
|
||||
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
stats.inc_q(&entry.bucket, entry.size, false, ReplicationType::Heal);
|
||||
@@ -3312,6 +3485,7 @@ mod tests {
|
||||
delete_marker: false,
|
||||
delete_marker_mtime: None,
|
||||
target_arns: Vec::new(),
|
||||
..Default::default()
|
||||
};
|
||||
let second = MrfReplicateEntry {
|
||||
object: "second".to_string(),
|
||||
@@ -3425,6 +3599,7 @@ mod tests {
|
||||
delete_marker: false,
|
||||
delete_marker_mtime: None,
|
||||
target_arns: Vec::new(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let encoded = encode_mrf_file(std::slice::from_ref(&entry)).expect("encode");
|
||||
@@ -3460,6 +3635,7 @@ mod tests {
|
||||
delete_marker: true,
|
||||
delete_marker_mtime: Some(mtime_nanos),
|
||||
target_arns: Vec::new(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let encoded = encode_mrf_file(std::slice::from_ref(&entry)).expect("encode");
|
||||
@@ -3495,6 +3671,7 @@ mod tests {
|
||||
delete_marker: false,
|
||||
delete_marker_mtime: None,
|
||||
target_arns: Vec::new(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let encoded = encode_mrf_file(&[entry]).expect("encode");
|
||||
@@ -3525,6 +3702,7 @@ mod tests {
|
||||
delete_marker: false,
|
||||
delete_marker_mtime: None,
|
||||
target_arns: Vec::new(),
|
||||
..Default::default()
|
||||
},
|
||||
MrfReplicateEntry {
|
||||
bucket: "b".to_string(),
|
||||
@@ -3538,6 +3716,7 @@ mod tests {
|
||||
delete_marker: true,
|
||||
delete_marker_mtime: None,
|
||||
target_arns: Vec::new(),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
|
||||
@@ -3569,6 +3748,7 @@ mod tests {
|
||||
delete_marker: false,
|
||||
delete_marker_mtime: None,
|
||||
target_arns: Vec::new(),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(obj_entry.op, MrfOpKind::Object);
|
||||
|
||||
@@ -3585,6 +3765,7 @@ mod tests {
|
||||
delete_marker: true,
|
||||
delete_marker_mtime: None,
|
||||
target_arns: Vec::new(),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(del_entry.op, MrfOpKind::Delete);
|
||||
|
||||
@@ -3602,6 +3783,7 @@ mod tests {
|
||||
delete_marker: false,
|
||||
delete_marker_mtime: None,
|
||||
target_arns: Vec::new(),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(legacy_entry.op, MrfOpKind::Object, "legacy default must be Object");
|
||||
}
|
||||
@@ -3668,6 +3850,7 @@ mod tests {
|
||||
delete_marker: false,
|
||||
delete_marker_mtime: None,
|
||||
target_arns: Vec::new(),
|
||||
..Default::default()
|
||||
}];
|
||||
let encoded = encode_mrf_file(&entries).expect("durable MRF backlog should encode");
|
||||
|
||||
@@ -3716,6 +3899,7 @@ mod tests {
|
||||
delete_marker: false,
|
||||
delete_marker_mtime: None,
|
||||
target_arns: vec!["arn:target-a".to_string(), "arn:target-b".to_string()],
|
||||
..Default::default()
|
||||
},
|
||||
MrfReplicateEntry {
|
||||
bucket: "b1".to_string(),
|
||||
@@ -3729,6 +3913,7 @@ mod tests {
|
||||
delete_marker: false,
|
||||
delete_marker_mtime: None,
|
||||
target_arns: vec!["arn:target-a".to_string()],
|
||||
..Default::default()
|
||||
},
|
||||
MrfReplicateEntry {
|
||||
bucket: "b1".to_string(),
|
||||
@@ -3742,6 +3927,7 @@ mod tests {
|
||||
delete_marker: false,
|
||||
delete_marker_mtime: None,
|
||||
target_arns: Vec::new(),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
|
||||
@@ -3797,10 +3983,89 @@ mod tests {
|
||||
delete_marker: false,
|
||||
delete_marker_mtime: None,
|
||||
target_arns: Vec::new(),
|
||||
..Default::default()
|
||||
}])
|
||||
.expect("invalid persisted entry should still encode for boundary testing");
|
||||
let invalid = durable_mrf_backlog_from_read(Ok(negative));
|
||||
assert!(!invalid.available);
|
||||
assert!(invalid.entries.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn force_delete_replay_requires_local_commit_and_keeps_persisted_targets() {
|
||||
let operation_id = Uuid::new_v4();
|
||||
let pending = MrfReplicateEntry {
|
||||
bucket: "source".to_string(),
|
||||
object: "logs/".to_string(),
|
||||
target_arns: vec!["arn:target:old-generation".to_string()],
|
||||
force_delete_id: Some(operation_id),
|
||||
force_delete_generation: Some(11),
|
||||
force_delete_local_commit: false,
|
||||
op: MrfOpKind::Delete,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!should_replay_force_delete_intent(&pending));
|
||||
|
||||
let mut committed = pending;
|
||||
committed.force_delete_local_commit = true;
|
||||
let recovered = decode_mrf_file(&encode_mrf_file(&[committed.clone()]).expect("force-delete intent should encode"))
|
||||
.expect("force-delete intent should decode");
|
||||
assert!(should_replay_force_delete_intent(&recovered[0]));
|
||||
assert_eq!(recovered[0].target_arns, vec!["arn:target:old-generation"]);
|
||||
assert_eq!(recovered[0].force_delete_generation, Some(11));
|
||||
|
||||
committed.target_arns.clear();
|
||||
assert!(!should_replay_force_delete_intent(&committed));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_delete_intent_append_commit_and_cleanup_are_idempotent() {
|
||||
let shared = empty_resync_shared_state();
|
||||
let storage = Arc::new(LoadResyncNodeStore::new("force-delete-journal", shared));
|
||||
let operation_id = Uuid::new_v4();
|
||||
let entry = MrfReplicateEntry {
|
||||
bucket: "source".to_string(),
|
||||
object: "logs/".to_string(),
|
||||
target_arns: vec!["arn:target:stable".to_string()],
|
||||
force_delete_id: Some(operation_id),
|
||||
force_delete_generation: Some(12),
|
||||
op: MrfOpKind::Delete,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
persist_force_delete_intent(storage.clone(), entry.clone())
|
||||
.await
|
||||
.expect("first journal append should succeed");
|
||||
persist_force_delete_intent(storage.clone(), entry)
|
||||
.await
|
||||
.expect("duplicate journal append should be a no-op");
|
||||
|
||||
let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE)
|
||||
.await
|
||||
.expect("journal should be readable");
|
||||
let entries = decode_mrf_file(&data).expect("journal should decode");
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert!(!entries[0].force_delete_local_commit);
|
||||
assert!(!should_replay_force_delete_intent(&entries[0]));
|
||||
|
||||
commit_force_delete_intent(storage.clone(), operation_id)
|
||||
.await
|
||||
.expect("commit marker should persist");
|
||||
commit_force_delete_intent(storage.clone(), operation_id)
|
||||
.await
|
||||
.expect("duplicate commit marker should be a no-op");
|
||||
let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE)
|
||||
.await
|
||||
.expect("committed journal should be readable");
|
||||
let entries = decode_mrf_file(&data).expect("committed journal should decode");
|
||||
assert!(should_replay_force_delete_intent(&entries[0]));
|
||||
assert_eq!(entries[0].target_arns, vec!["arn:target:stable"]);
|
||||
|
||||
complete_force_delete_intent(storage.clone(), operation_id)
|
||||
.await
|
||||
.expect("journal cleanup should succeed");
|
||||
complete_force_delete_intent(storage, operation_id)
|
||||
.await
|
||||
.expect("duplicate journal cleanup should be a no-op");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1642,54 +1642,62 @@ async fn replicate_delete_marker_purge_to_targets(bucket: &str, dobj: &DeletedOb
|
||||
async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &DeletedObjectReplicationInfo, storage: Arc<S>) {
|
||||
let bucket = &dobj.bucket;
|
||||
let object_name = &dobj.delete_object.object_name;
|
||||
let admitted_target_arns = dobj.admitted_target_arns();
|
||||
|
||||
let rcfg = match get_replication_config(bucket).await {
|
||||
Ok(Some(config)) => config,
|
||||
Ok(None) => {
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
reason = "replication_config_missing",
|
||||
"Skipping replication force-delete because replication config is missing"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
bucket: bucket.clone(),
|
||||
name: object_name.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
let legacy_target_arns = if admitted_target_arns.is_empty() {
|
||||
match get_replication_config(bucket).await {
|
||||
Ok(Some(config)) => config.filter_target_arns(&ObjectOpts {
|
||||
name: object_name.clone(),
|
||||
..Default::default()
|
||||
});
|
||||
return;
|
||||
}
|
||||
Err(err) => {
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
error = %err,
|
||||
reason = "replication_config_lookup_failed",
|
||||
"Skipping replication force-delete because replication config lookup failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
bucket: bucket.clone(),
|
||||
name: object_name.clone(),
|
||||
}),
|
||||
Ok(None) => {
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
reason = "replication_config_missing",
|
||||
"Skipping replication force-delete because replication config is missing"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
bucket: bucket.clone(),
|
||||
name: object_name.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return;
|
||||
});
|
||||
Vec::new()
|
||||
}
|
||||
Err(err) => {
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
error = %err,
|
||||
reason = "replication_config_lookup_failed",
|
||||
"Skipping replication force-delete because replication config lookup failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
bucket: bucket.clone(),
|
||||
name: object_name.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
let ns_lock = match storage
|
||||
@@ -1751,22 +1759,18 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
}
|
||||
};
|
||||
|
||||
let tgt_arns = {
|
||||
let admitted = dobj.admitted_target_arns();
|
||||
if admitted.is_empty() {
|
||||
rcfg.filter_target_arns(&ObjectOpts {
|
||||
name: object_name.clone(),
|
||||
..Default::default()
|
||||
})
|
||||
} else {
|
||||
admitted
|
||||
}
|
||||
let tgt_arns = if admitted_target_arns.is_empty() {
|
||||
legacy_target_arns
|
||||
} else {
|
||||
admitted_target_arns
|
||||
};
|
||||
|
||||
let mut join_set = JoinSet::new();
|
||||
let mut all_succeeded = true;
|
||||
|
||||
for arn in tgt_arns {
|
||||
let Some(tgt_client) = ReplicationTargetStore::remote_target_client(bucket, &arn).await else {
|
||||
all_succeeded = false;
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -1816,7 +1820,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
|
||||
if let Err(e) = tgt_client
|
||||
@@ -1845,24 +1849,46 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return false;
|
||||
}
|
||||
|
||||
true
|
||||
});
|
||||
}
|
||||
|
||||
while let Some(result) = join_set.join_next().await {
|
||||
if let Err(e) = result {
|
||||
error!(
|
||||
event = EVENT_RESYNC_TASK_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
object = %object_name,
|
||||
operation = "force_delete",
|
||||
error = %e,
|
||||
"Replication resync task failed"
|
||||
);
|
||||
match result {
|
||||
Ok(success) => all_succeeded &= success,
|
||||
Err(error) => {
|
||||
all_succeeded = false;
|
||||
error!(
|
||||
event = EVENT_RESYNC_TASK_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
object = %object_name,
|
||||
operation = "force_delete",
|
||||
error = %error,
|
||||
"Replication resync task failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if all_succeeded
|
||||
&& let Some(operation_id) = dobj.delete_object.force_delete_id
|
||||
&& let Err(error) = super::replication_pool::complete_force_delete_intent(storage, operation_id).await
|
||||
{
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
object = %object_name,
|
||||
operation_id = %operation_id,
|
||||
error = %error,
|
||||
"Force-delete replication completed but durable intent cleanup failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn target_delete_version_id(version_id: Uuid, version_purge: bool) -> Option<String> {
|
||||
|
||||
@@ -105,6 +105,9 @@ pub(crate) fn deleted_object_for_replication(delete_object: DeletedObject) -> Re
|
||||
replication_state: delete_object.replication_state.as_ref().map(replication_state_from_filemeta),
|
||||
found: delete_object.found,
|
||||
force_delete: delete_object.force_delete,
|
||||
force_delete_id: delete_object.force_delete_id,
|
||||
force_delete_target_arns: delete_object.force_delete_target_arns,
|
||||
force_delete_generation: delete_object.force_delete_generation,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ use s3s::dto::{
|
||||
ReplicationRuleStatus, ReplicationRules,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub const REPLICATION_CAPABILITY_CONTRACT_VERSION: u32 = 1;
|
||||
@@ -93,6 +93,7 @@ pub trait ReplicationConfigurationExt {
|
||||
fn get_destination(&self) -> Destination;
|
||||
fn has_active_rules(&self, prefix: &str, recursive: bool) -> bool;
|
||||
fn filter_target_arns(&self, obj: &ObjectOpts) -> Vec<String>;
|
||||
fn filter_force_delete_target_arns(&self, prefix: &str) -> Vec<String>;
|
||||
fn filter_target_replication_decisions(&self, obj: &ObjectOpts) -> Vec<(String, bool)> {
|
||||
self.filter_target_arns(obj)
|
||||
.into_iter()
|
||||
@@ -434,6 +435,51 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
|
||||
arns
|
||||
}
|
||||
|
||||
fn filter_force_delete_target_arns(&self, prefix: &str) -> Vec<String> {
|
||||
let role = self.role.trim();
|
||||
let mut selected = BTreeMap::<String, (&ReplicationRule, bool)>::new();
|
||||
|
||||
for rule in &self.rules {
|
||||
if rule.status == ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let rule_prefix = rule.prefix();
|
||||
if !prefix.starts_with(rule_prefix) && !rule_prefix.starts_with(prefix) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let target = if role.is_empty() {
|
||||
rule.destination.bucket.trim()
|
||||
} else {
|
||||
role
|
||||
};
|
||||
if target.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let delete_enabled =
|
||||
rule.delete_replication.as_ref().is_some_and(|delete| {
|
||||
delete.status == DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED)
|
||||
}) || rule.delete_marker_replication.as_ref().is_some_and(|delete_marker| {
|
||||
delete_marker.status
|
||||
== Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED))
|
||||
});
|
||||
|
||||
if selected
|
||||
.get(target)
|
||||
.is_none_or(|(current, _)| rule.priority > current.priority)
|
||||
{
|
||||
selected.insert(target.to_string(), (rule, delete_enabled));
|
||||
}
|
||||
}
|
||||
|
||||
selected
|
||||
.into_iter()
|
||||
.filter_map(|(target, (_, enabled))| enabled.then_some(target))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn filter_target_replication_decisions(&self, obj: &ObjectOpts) -> Vec<(String, bool)> {
|
||||
let rules = self.filter_actionable_rules(obj);
|
||||
let role = self.role.trim();
|
||||
@@ -1069,4 +1115,26 @@ mod tests {
|
||||
|
||||
assert_eq!(decisions, vec![(target_a.to_string(), false), (target_b.to_string(), true)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn force_delete_targets_use_overlapping_rules_and_highest_priority_switch() {
|
||||
let target_a = "arn:target:a";
|
||||
let target_b = "arn:target:b";
|
||||
let mut a_parent = delete_marker_rule("a-parent", target_a, "logs/", 1, true);
|
||||
a_parent.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
let a_child_disabled = delete_marker_rule("a-child", target_a, "logs/2026/", 5, false);
|
||||
let b_child = delete_marker_rule("b-child", target_b, "logs/2026/", 2, true);
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![a_parent, a_child_disabled, b_child],
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
config.filter_force_delete_target_arns("logs/2026/app.log"),
|
||||
vec![target_b.to_string()],
|
||||
"the child rule must win for target A while the overlapping child target B remains eligible"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,12 +33,15 @@ impl DeletedObjectReplicationInfo {
|
||||
return vec![self.target_arn.clone()];
|
||||
}
|
||||
|
||||
let mut target_arns = self
|
||||
.delete_object
|
||||
.replication_state
|
||||
.as_ref()
|
||||
.map(admitted_target_arns_from_replication_state)
|
||||
.unwrap_or_default();
|
||||
let mut target_arns = if !self.delete_object.force_delete_target_arns.is_empty() {
|
||||
self.delete_object.force_delete_target_arns.clone()
|
||||
} else {
|
||||
self.delete_object
|
||||
.replication_state
|
||||
.as_ref()
|
||||
.map(admitted_target_arns_from_replication_state)
|
||||
.unwrap_or_default()
|
||||
};
|
||||
target_arns.sort();
|
||||
target_arns.dedup();
|
||||
target_arns
|
||||
@@ -69,6 +72,9 @@ impl ReplicationWorkerOperation for DeletedObjectReplicationInfo {
|
||||
.delete_marker_mtime
|
||||
.and_then(|t| i64::try_from(t.unix_timestamp_nanos()).ok()),
|
||||
target_arns: self.admitted_target_arns(),
|
||||
force_delete_id: self.delete_object.force_delete_id,
|
||||
force_delete_generation: self.delete_object.force_delete_generation,
|
||||
force_delete_local_commit: self.delete_object.force_delete,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -231,6 +237,35 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deleted_object_replication_info_preserves_force_delete_handoff() {
|
||||
let operation_id = Uuid::new_v4();
|
||||
let info = DeletedObjectReplicationInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
delete_object: DeletedObject {
|
||||
object_name: "prefix/".to_string(),
|
||||
force_delete: true,
|
||||
force_delete_id: Some(operation_id),
|
||||
force_delete_target_arns: vec![
|
||||
"arn:target-b".to_string(),
|
||||
"arn:target-a".to_string(),
|
||||
"arn:target-b".to_string(),
|
||||
],
|
||||
force_delete_generation: Some(17),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let entry = info.to_mrf_entry();
|
||||
|
||||
assert!(entry.force_delete);
|
||||
assert_eq!(entry.force_delete_id, Some(operation_id));
|
||||
assert_eq!(entry.force_delete_generation, Some(17));
|
||||
assert!(entry.force_delete_local_commit);
|
||||
assert_eq!(entry.target_arns, vec!["arn:target-a", "arn:target-b"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_delete_replication_tracks_delete_marker_version_purge() {
|
||||
let dobj = DeletedObject {
|
||||
|
||||
@@ -572,7 +572,7 @@ impl MrfOpKind {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
|
||||
pub struct MrfReplicateEntry {
|
||||
#[serde(rename = "bucket")]
|
||||
pub bucket: String,
|
||||
@@ -619,6 +619,17 @@ pub struct MrfReplicateEntry {
|
||||
|
||||
#[serde(rename = "targetARNs", skip_serializing_if = "Vec::is_empty", default)]
|
||||
pub target_arns: Vec<String>,
|
||||
|
||||
// Force-delete entries use the target ARN list above as their immutable target set.
|
||||
// The id distinguishes a durable intent from legacy MRF delete entries.
|
||||
#[serde(rename = "forceDeleteID", skip_serializing_if = "Option::is_none", default)]
|
||||
pub force_delete_id: Option<Uuid>,
|
||||
#[serde(rename = "forceDeleteGeneration", skip_serializing_if = "Option::is_none", default)]
|
||||
pub force_delete_generation: Option<i64>,
|
||||
|
||||
// Replay is allowed only after the source-side recursive delete has committed.
|
||||
#[serde(rename = "forceDeleteLocalCommit", default)]
|
||||
pub force_delete_local_commit: bool,
|
||||
}
|
||||
|
||||
fn retry_count_to_mrf(retry_count: u32) -> i32 {
|
||||
@@ -865,6 +876,7 @@ impl ReplicationWorkerOperation for ReplicateObjectInfo {
|
||||
delete_marker: false,
|
||||
delete_marker_mtime: None,
|
||||
target_arns: self.admitted_target_arns(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -945,6 +957,7 @@ impl ReplicateObjectInfo {
|
||||
delete_marker: false,
|
||||
delete_marker_mtime: None,
|
||||
target_arns: self.admitted_target_arns(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -68,6 +68,9 @@ mod tests {
|
||||
retry_count: 1,
|
||||
size: 1024,
|
||||
op: MrfOpKind::Metadata,
|
||||
force_delete_id: None,
|
||||
force_delete_generation: None,
|
||||
force_delete_local_commit: false,
|
||||
force_delete: false,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
@@ -81,6 +84,9 @@ mod tests {
|
||||
retry_count: 2,
|
||||
size: 1024,
|
||||
op: MrfOpKind::Object,
|
||||
force_delete_id: None,
|
||||
force_delete_generation: None,
|
||||
force_delete_local_commit: false,
|
||||
force_delete: false,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
@@ -94,6 +100,9 @@ mod tests {
|
||||
retry_count: 0,
|
||||
size: 0,
|
||||
op: MrfOpKind::Delete,
|
||||
force_delete_id: None,
|
||||
force_delete_generation: None,
|
||||
force_delete_local_commit: false,
|
||||
force_delete: true,
|
||||
delete_marker_version_id: Some(del_vid),
|
||||
delete_marker: true,
|
||||
|
||||
@@ -49,6 +49,9 @@ pub struct DeletedObject {
|
||||
pub replication_state: Option<ReplicationState>,
|
||||
pub found: bool,
|
||||
pub force_delete: bool,
|
||||
pub force_delete_id: Option<Uuid>,
|
||||
pub force_delete_target_arns: Vec<String>,
|
||||
pub force_delete_generation: Option<i64>,
|
||||
}
|
||||
|
||||
impl DeletedObject {
|
||||
|
||||
@@ -213,6 +213,9 @@ pub struct DeletedObject {
|
||||
pub replication_state: Option<ReplicationState>,
|
||||
pub found: bool,
|
||||
pub force_delete: bool,
|
||||
pub force_delete_id: Option<Uuid>,
|
||||
pub force_delete_target_arns: Vec<String>,
|
||||
pub force_delete_generation: Option<i64>,
|
||||
}
|
||||
|
||||
impl DeletedObject {
|
||||
|
||||
@@ -1125,6 +1125,7 @@ mod tests {
|
||||
delete_marker: false,
|
||||
delete_marker_mtime: None,
|
||||
target_arns: vec!["arn-a".to_string(), "arn-durable-only".to_string()],
|
||||
..Default::default()
|
||||
},
|
||||
MrfReplicateEntry {
|
||||
bucket: "other-bucket".to_string(),
|
||||
@@ -1138,6 +1139,7 @@ mod tests {
|
||||
delete_marker: false,
|
||||
delete_marker_mtime: None,
|
||||
target_arns: Vec::new(),
|
||||
..Default::default()
|
||||
},
|
||||
],
|
||||
};
|
||||
@@ -1197,6 +1199,7 @@ mod tests {
|
||||
delete_marker: false,
|
||||
delete_marker_mtime: None,
|
||||
target_arns: Vec::new(),
|
||||
..Default::default()
|
||||
}],
|
||||
};
|
||||
|
||||
|
||||
@@ -41,11 +41,11 @@ use super::storage_api::object_usecase::bucket::{
|
||||
predict_lifecycle_expiration,
|
||||
quota::{QuotaCheckResult, QuotaError, QuotaOperation},
|
||||
replication::{
|
||||
DeleteReplicationConfigSnapshot, REPLICATE_INCOMING_DELETE, ReplicationStatusType, delete_replication_state_from_config,
|
||||
delete_replication_version_id, deleted_object_has_pending_replication_delete, has_active_delete_rule,
|
||||
load_delete_config_snapshot, must_replicate_object, schedule_object_replication, schedule_replication_delete,
|
||||
schedule_replication_deletes, set_deleted_object_replication_state, should_schedule_delete_replication,
|
||||
should_use_existing_delete_replication_info,
|
||||
DeleteReplicationConfigSnapshot, REPLICATE_INCOMING_DELETE, ReplicationStatusType, commit_force_delete_intent,
|
||||
delete_replication_state_from_config, delete_replication_version_id, deleted_object_has_pending_replication_delete,
|
||||
force_delete_target_set, has_active_delete_rule, load_delete_config_snapshot, must_replicate_object,
|
||||
persist_force_delete_intent, schedule_object_replication, schedule_replication_delete, schedule_replication_deletes,
|
||||
set_deleted_object_replication_state, should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
},
|
||||
tagging::decode_tags,
|
||||
validate_restore_request,
|
||||
@@ -7137,6 +7137,7 @@ impl DefaultObjectUsecase {
|
||||
opts.expected_current_version_id = expected_current_version_id.clone();
|
||||
|
||||
let replicate_force_delete = force_delete && !replica && has_active_delete_rule(&delete_config_snapshot, &key);
|
||||
let mut force_delete_intent = None;
|
||||
|
||||
// Check Object Lock retention before deletion
|
||||
// TODO: Future optimization (separate PR) - If performance becomes critical under high delete load:
|
||||
@@ -7176,6 +7177,17 @@ impl DefaultObjectUsecase {
|
||||
let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &key).await;
|
||||
}
|
||||
|
||||
if replicate_force_delete
|
||||
&& let Some((target_arns, generation)) = force_delete_target_set(&delete_config_snapshot, &key)
|
||||
&& !target_arns.is_empty()
|
||||
{
|
||||
let operation_id =
|
||||
persist_force_delete_intent(store.clone(), bucket.clone(), key.clone(), target_arns.clone(), generation)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
force_delete_intent = Some((operation_id, target_arns, generation));
|
||||
}
|
||||
|
||||
let obj_info = {
|
||||
match store
|
||||
.delete_object_with_tier_delete_journal(&bucket, &key, opts.clone())
|
||||
@@ -7183,6 +7195,18 @@ impl DefaultObjectUsecase {
|
||||
{
|
||||
Ok(obj) => obj,
|
||||
Err(err) => {
|
||||
if let Some((operation_id, _, _)) = force_delete_intent.as_ref()
|
||||
&& let Err(cleanup_error) =
|
||||
crate::storage::storage_api::complete_force_delete_intent(store.clone(), *operation_id).await
|
||||
{
|
||||
warn!(
|
||||
bucket = %bucket,
|
||||
object = %key,
|
||||
operation_id = %operation_id,
|
||||
error = %cleanup_error,
|
||||
"failed to remove uncommitted force-delete intent after local delete failure"
|
||||
);
|
||||
}
|
||||
if is_err_bucket_not_found(&err) {
|
||||
return Err(S3Error::with_message(S3ErrorCode::NoSuchBucket, "Bucket not found".to_string()));
|
||||
}
|
||||
@@ -7211,7 +7235,31 @@ impl DefaultObjectUsecase {
|
||||
}
|
||||
|
||||
if obj_info.name.is_empty() {
|
||||
if replicate_force_delete {
|
||||
if let Some((operation_id, target_arns, generation)) = force_delete_intent {
|
||||
if let Err(error) = commit_force_delete_intent(store.clone(), operation_id).await {
|
||||
warn!(
|
||||
bucket = %bucket,
|
||||
object = %key,
|
||||
operation_id = %operation_id,
|
||||
error = %error,
|
||||
"failed to mark force-delete intent committed after local delete"
|
||||
);
|
||||
}
|
||||
let generation = i64::try_from(generation.unix_timestamp_nanos()).unwrap_or(i64::MAX);
|
||||
schedule_replication_delete(
|
||||
StorageDeletedObject {
|
||||
object_name: key.clone(),
|
||||
force_delete: true,
|
||||
force_delete_id: Some(operation_id),
|
||||
force_delete_target_arns: target_arns,
|
||||
force_delete_generation: Some(generation),
|
||||
..Default::default()
|
||||
},
|
||||
bucket.clone(),
|
||||
REPLICATE_INCOMING_DELETE.to_string(),
|
||||
)
|
||||
.await;
|
||||
} else if replicate_force_delete {
|
||||
let mut delete_object = StorageDeletedObject {
|
||||
object_name: key.clone(),
|
||||
force_delete: true,
|
||||
|
||||
@@ -577,6 +577,44 @@ pub(crate) mod bucket {
|
||||
#[cfg(test)]
|
||||
pub(crate) use replication_contracts::replication_statuses_map;
|
||||
|
||||
pub(crate) async fn persist_force_delete_intent(
|
||||
store: Arc<crate::storage::storage_api::ECStore>,
|
||||
bucket: String,
|
||||
object: String,
|
||||
target_arns: Vec<String>,
|
||||
generation: time::OffsetDateTime,
|
||||
) -> crate::storage::storage_api::Result<Uuid> {
|
||||
let operation_id = Uuid::new_v4();
|
||||
crate::storage::storage_api::persist_force_delete_intent(
|
||||
store,
|
||||
replication_contracts::MrfReplicateEntry {
|
||||
bucket,
|
||||
object,
|
||||
version_id: None,
|
||||
retry_count: 0,
|
||||
size: 0,
|
||||
op: replication_contracts::MrfOpKind::Delete,
|
||||
force_delete: true,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
delete_marker_mtime: None,
|
||||
target_arns,
|
||||
force_delete_id: Some(operation_id),
|
||||
force_delete_generation: Some(i64::try_from(generation.unix_timestamp_nanos()).unwrap_or(i64::MAX)),
|
||||
force_delete_local_commit: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map(|_| operation_id)
|
||||
}
|
||||
|
||||
pub(crate) async fn commit_force_delete_intent(
|
||||
store: Arc<crate::storage::storage_api::ECStore>,
|
||||
operation_id: Uuid,
|
||||
) -> crate::storage::storage_api::Result<()> {
|
||||
crate::storage::storage_api::commit_force_delete_intent(store, operation_id).await
|
||||
}
|
||||
|
||||
/// Test-only counter of `must_replicate_object` invocations.
|
||||
///
|
||||
/// Used by white-box regression tests to assert that a single PUT
|
||||
@@ -614,6 +652,13 @@ pub(crate) mod bucket {
|
||||
ReplicationObjectBridge::has_active_delete_rule(snapshot, object)
|
||||
}
|
||||
|
||||
pub(crate) fn force_delete_target_set(
|
||||
snapshot: &DeleteReplicationConfigSnapshot,
|
||||
prefix: &str,
|
||||
) -> Option<(Vec<String>, time::OffsetDateTime)> {
|
||||
ReplicationObjectBridge::force_delete_target_set(snapshot, prefix)
|
||||
}
|
||||
|
||||
pub(crate) fn delete_replication_version_id(
|
||||
replication_source: &crate::storage::storage_api::StorageObjectInfo,
|
||||
deleted_delete_marker_version: bool,
|
||||
|
||||
@@ -1441,6 +1441,21 @@ pub(crate) async fn get_bucket_replication_config(
|
||||
ecstore_bucket::metadata_sys::get_replication_config(bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn persist_force_delete_intent(
|
||||
api: Arc<ECStore>,
|
||||
entry: ecstore_bucket::replication::MrfReplicateEntry,
|
||||
) -> Result<()> {
|
||||
ecstore_bucket::replication::persist_force_delete_intent(api, entry).await
|
||||
}
|
||||
|
||||
pub(crate) async fn commit_force_delete_intent(api: Arc<ECStore>, operation_id: uuid::Uuid) -> Result<()> {
|
||||
ecstore_bucket::replication::commit_force_delete_intent(api, operation_id).await
|
||||
}
|
||||
|
||||
pub(crate) async fn complete_force_delete_intent(api: Arc<ECStore>, operation_id: uuid::Uuid) -> Result<()> {
|
||||
ecstore_bucket::replication::complete_force_delete_intent(api, operation_id).await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_bucket_request_payment_config(
|
||||
bucket: &str,
|
||||
) -> Result<(s3s::dto::RequestPaymentConfiguration, time::OffsetDateTime)> {
|
||||
|
||||
Reference in New Issue
Block a user