mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-27 15:37:02 +00:00
fix(replication): persist force-delete handoff state (#5641)
* fix(replication): persist force-delete handoff state * fix(arch): route force-delete config access through boundary * style: format force-delete imports --------- Co-authored-by: Zhengchao An <anzhengchao@gmail.com>
This commit is contained in:
@@ -188,11 +188,12 @@ pub mod bucket {
|
|||||||
ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
|
ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
|
||||||
ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
|
ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
|
||||||
ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
|
ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
|
||||||
VersionPurgeStatusType, delete_replication_state_from_config, delete_replication_version_id,
|
VersionPurgeStatusType, commit_force_delete_intent, complete_force_delete_intent,
|
||||||
get_global_replication_pool, get_global_replication_stats, init_background_replication,
|
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
|
||||||
invalid_replication_config_status_field, read_durable_mrf_backlog, replication_state_to_filemeta,
|
get_global_replication_stats, init_background_replication, invalid_replication_config_status_field,
|
||||||
replication_status_to_filemeta, replication_statuses_map, replication_target_arns, resync_start_conflict_id,
|
persist_force_delete_intent, read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta,
|
||||||
should_remove_replication_target, should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
replication_statuses_map, replication_target_arns, resync_start_conflict_id, should_remove_replication_target,
|
||||||
|
should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||||
should_use_existing_delete_replication_source, unsupported_replication_config_field,
|
should_use_existing_delete_replication_source, unsupported_replication_config_field,
|
||||||
validate_replication_config_target_arns, version_purge_status_to_filemeta,
|
validate_replication_config_target_arns, version_purge_status_to_filemeta,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -70,8 +70,9 @@ pub use replication_object_decision_boundary::{
|
|||||||
should_use_existing_delete_replication_source,
|
should_use_existing_delete_replication_source,
|
||||||
};
|
};
|
||||||
pub use replication_pool::{
|
pub use replication_pool::{
|
||||||
DurableMrfBacklog, DynReplicationPool, ReplicationPoolTrait, get_global_replication_pool, get_global_replication_stats,
|
DurableMrfBacklog, DynReplicationPool, ReplicationPoolTrait, commit_force_delete_intent, complete_force_delete_intent,
|
||||||
init_background_replication, read_durable_mrf_backlog, resync_start_conflict_id,
|
get_global_replication_pool, get_global_replication_stats, init_background_replication, persist_force_delete_intent,
|
||||||
|
read_durable_mrf_backlog, resync_start_conflict_id,
|
||||||
};
|
};
|
||||||
pub use replication_queue_boundary::{
|
pub use replication_queue_boundary::{
|
||||||
DeletedObjectReplicationInfo, ReplicationBatchAdmission, ReplicationHealQueueResult, ReplicationOperation,
|
DeletedObjectReplicationInfo, ReplicationBatchAdmission, ReplicationHealQueueResult, ReplicationOperation,
|
||||||
|
|||||||
@@ -30,10 +30,24 @@ impl ReplicationConfigStore {
|
|||||||
com::read_config(api, file).await
|
com::read_config(api, file).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn read_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
|
||||||
|
where
|
||||||
|
S: ReplicationObjectIO,
|
||||||
|
{
|
||||||
|
com::read_config_no_lock(api, file).await
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn save<S>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()>
|
pub(crate) async fn save<S>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()>
|
||||||
where
|
where
|
||||||
S: ReplicationObjectIO,
|
S: ReplicationObjectIO,
|
||||||
{
|
{
|
||||||
com::save_config(api, file, data).await
|
com::save_config(api, file, data).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn save_no_lock<S>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()>
|
||||||
|
where
|
||||||
|
S: ReplicationObjectIO,
|
||||||
|
{
|
||||||
|
com::save_config_no_lock(api, file, data).await
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ pub(crate) struct ReplicationMetadataStore;
|
|||||||
|
|
||||||
impl ReplicationMetadataStore {
|
impl ReplicationMetadataStore {
|
||||||
pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
|
pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
|
||||||
|
pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
|
||||||
|
|
||||||
pub(crate) async fn replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
pub(crate) async fn replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||||
metadata_sys::get_replication_config(bucket).await
|
metadata_sys::get_replication_config(bucket).await
|
||||||
@@ -109,5 +110,9 @@ mod tests {
|
|||||||
"buckets/bucket-a/.replication/resync.bin"
|
"buckets/bucket-a/.replication/resync.bin"
|
||||||
);
|
);
|
||||||
assert_eq!(ReplicationMetadataStore::MRF_REPLICATION_FILE, "config/replication/mrf.bin");
|
assert_eq!(ReplicationMetadataStore::MRF_REPLICATION_FILE, "config/replication/mrf.bin");
|
||||||
|
assert_eq!(
|
||||||
|
ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
|
||||||
|
"config/replication/force-delete.bin"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -89,6 +89,13 @@ impl ReplicationObjectBridge {
|
|||||||
snapshot.has_active_rule(object)
|
snapshot.has_active_rule(object)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn force_delete_target_set(
|
||||||
|
snapshot: &DeleteReplicationConfigSnapshot,
|
||||||
|
prefix: &str,
|
||||||
|
) -> Option<(Vec<String>, time::OffsetDateTime)> {
|
||||||
|
snapshot.force_delete_target_set(prefix)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn check_delete_with_snapshot(
|
pub fn check_delete_with_snapshot(
|
||||||
object: &ObjectToDelete,
|
object: &ObjectToDelete,
|
||||||
source: &ObjectInfo,
|
source: &ObjectInfo,
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ use crate::bucket::metadata::BucketMetadata;
|
|||||||
use rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS;
|
use rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS;
|
||||||
use s3s::dto::{BucketVersioningStatus, ReplicationConfiguration, ReplicationRuleStatus, VersioningConfiguration};
|
use s3s::dto::{BucketVersioningStatus, ReplicationConfiguration, ReplicationRuleStatus, VersioningConfiguration};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
use time::OffsetDateTime;
|
||||||
use tracing::error;
|
use tracing::error;
|
||||||
|
|
||||||
use super::replication_config_boundary::{
|
use super::replication_config_boundary::{
|
||||||
@@ -83,6 +84,15 @@ impl DeleteReplicationConfigSnapshot {
|
|||||||
.and_then(|metadata| metadata.replication_config.as_ref())
|
.and_then(|metadata| metadata.replication_config.as_ref())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn force_delete_target_set(&self, prefix: &str) -> Option<(Vec<String>, OffsetDateTime)> {
|
||||||
|
self.metadata.as_ref().and_then(|metadata| {
|
||||||
|
metadata
|
||||||
|
.replication_config
|
||||||
|
.as_ref()
|
||||||
|
.map(|config| (config.filter_force_delete_target_arns(prefix), metadata.replication_config_updated_at))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn has_active_rule(&self, object: &str) -> bool {
|
pub(crate) fn has_active_rule(&self, object: &str) -> bool {
|
||||||
self.replication_config()
|
self.replication_config()
|
||||||
.is_some_and(|config| config.has_active_rules(object, true))
|
.is_some_and(|config| config.has_active_rules(object, true))
|
||||||
|
|||||||
@@ -136,6 +136,10 @@ static DURABLE_MRF_TARGET_BACKLOG: LazyLock<StdRwLock<Vec<DurableMrfTargetBacklo
|
|||||||
static MRF_BACKLOG_OBSERVABILITY: LazyLock<StdRwLock<MrfBacklogObservabilityTracker>> =
|
static MRF_BACKLOG_OBSERVABILITY: LazyLock<StdRwLock<MrfBacklogObservabilityTracker>> =
|
||||||
LazyLock::new(|| StdRwLock::new(MrfBacklogObservabilityTracker::default()));
|
LazyLock::new(|| StdRwLock::new(MrfBacklogObservabilityTracker::default()));
|
||||||
|
|
||||||
|
fn should_replay_force_delete_intent(entry: &MrfReplicateEntry) -> bool {
|
||||||
|
entry.force_delete_id.is_some() && entry.force_delete_local_commit && !entry.target_arns.is_empty()
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Default)]
|
#[derive(Debug, Clone, Default)]
|
||||||
struct DurableMrfBacklogTracker {
|
struct DurableMrfBacklogTracker {
|
||||||
available: bool,
|
available: bool,
|
||||||
@@ -302,6 +306,7 @@ where
|
|||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
delete_marker_mtime: None,
|
delete_marker_mtime: None,
|
||||||
target_arns: Vec::new(),
|
target_arns: Vec::new(),
|
||||||
|
..Default::default()
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
tracker.into_snapshot()
|
tracker.into_snapshot()
|
||||||
@@ -394,6 +399,82 @@ pub async fn read_durable_mrf_backlog<S: ReplicationObjectIO>(storage: Arc<S>) -
|
|||||||
durable_mrf_backlog_from_read(ReplicationConfigStore::read(storage, ReplicationMetadataStore::MRF_REPLICATION_FILE).await)
|
durable_mrf_backlog_from_read(ReplicationConfigStore::read(storage, ReplicationMetadataStore::MRF_REPLICATION_FILE).await)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn persist_force_delete_intent<S: ReplicationStorage>(
|
||||||
|
storage: Arc<S>,
|
||||||
|
mut entry: MrfReplicateEntry,
|
||||||
|
) -> Result<(), EcstoreError> {
|
||||||
|
entry.force_delete_local_commit = false;
|
||||||
|
let file = ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE;
|
||||||
|
let lock = storage
|
||||||
|
.new_ns_lock(ReplicationMetadataStore::rustfs_meta_bucket(), file)
|
||||||
|
.await?;
|
||||||
|
let _guard = lock.get_write_lock(ReplicationLockTiming::acquire_timeout()).await?;
|
||||||
|
|
||||||
|
let mut entries = match ReplicationConfigStore::read_no_lock(storage.clone(), file).await {
|
||||||
|
Ok(data) => decode_mrf_file(&data)?,
|
||||||
|
Err(EcstoreError::ConfigNotFound) => Vec::new(),
|
||||||
|
Err(err) => return Err(err),
|
||||||
|
};
|
||||||
|
|
||||||
|
if entries
|
||||||
|
.iter()
|
||||||
|
.any(|existing| existing.force_delete_id == entry.force_delete_id)
|
||||||
|
{
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
entries.push(entry);
|
||||||
|
let data = encode_mrf_file(&entries)?;
|
||||||
|
ReplicationConfigStore::save_no_lock(storage, file, data).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn commit_force_delete_intent<S: ReplicationStorage>(
|
||||||
|
storage: Arc<S>,
|
||||||
|
operation_id: uuid::Uuid,
|
||||||
|
) -> Result<(), EcstoreError> {
|
||||||
|
let file = ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE;
|
||||||
|
let lock = storage
|
||||||
|
.new_ns_lock(ReplicationMetadataStore::rustfs_meta_bucket(), file)
|
||||||
|
.await?;
|
||||||
|
let _guard = lock.get_write_lock(ReplicationLockTiming::acquire_timeout()).await?;
|
||||||
|
|
||||||
|
let data = ReplicationConfigStore::read_no_lock(storage.clone(), file).await?;
|
||||||
|
let mut entries = decode_mrf_file(&data)?;
|
||||||
|
let Some(entry) = entries.iter_mut().find(|entry| entry.force_delete_id == Some(operation_id)) else {
|
||||||
|
return Err(EcstoreError::ConfigNotFound);
|
||||||
|
};
|
||||||
|
if entry.force_delete_local_commit {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
entry.force_delete_local_commit = true;
|
||||||
|
ReplicationConfigStore::save_no_lock(storage, file, encode_mrf_file(&entries)?).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn complete_force_delete_intent<S: ReplicationStorage>(
|
||||||
|
storage: Arc<S>,
|
||||||
|
operation_id: uuid::Uuid,
|
||||||
|
) -> Result<(), EcstoreError> {
|
||||||
|
let file = ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE;
|
||||||
|
let lock = storage
|
||||||
|
.new_ns_lock(ReplicationMetadataStore::rustfs_meta_bucket(), file)
|
||||||
|
.await?;
|
||||||
|
let _guard = lock.get_write_lock(ReplicationLockTiming::acquire_timeout()).await?;
|
||||||
|
|
||||||
|
let data = match ReplicationConfigStore::read_no_lock(storage.clone(), file).await {
|
||||||
|
Ok(data) => data,
|
||||||
|
Err(EcstoreError::ConfigNotFound) => return Ok(()),
|
||||||
|
Err(err) => return Err(err),
|
||||||
|
};
|
||||||
|
let mut entries = decode_mrf_file(&data)?;
|
||||||
|
let original_len = entries.len();
|
||||||
|
entries.retain(|entry| entry.force_delete_id != Some(operation_id));
|
||||||
|
if entries.len() == original_len {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
ReplicationConfigStore::save_no_lock(storage, file, encode_mrf_file(&entries)?).await
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
#[derive(Debug, thiserror::Error)]
|
||||||
#[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
|
#[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
|
||||||
struct ResyncActiveConflictError {
|
struct ResyncActiveConflictError {
|
||||||
@@ -497,6 +578,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
|
|
||||||
// Start background tasks
|
// Start background tasks
|
||||||
pool.start_mrf_processor().await;
|
pool.start_mrf_processor().await;
|
||||||
|
pool.start_force_delete_processor().await;
|
||||||
pool.start_mrf_persister().await;
|
pool.start_mrf_persister().await;
|
||||||
|
|
||||||
pool
|
pool
|
||||||
@@ -972,6 +1054,32 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
for entry in entries.iter() {
|
for entry in entries.iter() {
|
||||||
match entry.op {
|
match entry.op {
|
||||||
MrfOpKind::Delete => {
|
MrfOpKind::Delete => {
|
||||||
|
if should_replay_force_delete_intent(entry) {
|
||||||
|
let Some(operation_id) = entry.force_delete_id else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
schedule_replication_delete(DeletedObjectReplicationInfo {
|
||||||
|
delete_object: ReplicationDeletedObject {
|
||||||
|
object_name: entry.object.clone(),
|
||||||
|
force_delete: true,
|
||||||
|
force_delete_id: Some(operation_id),
|
||||||
|
force_delete_target_arns: entry.target_arns.clone(),
|
||||||
|
force_delete_generation: entry.force_delete_generation,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
bucket: entry.bucket.clone(),
|
||||||
|
op_type: ReplicationType::Heal,
|
||||||
|
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
queued_count += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if entry.force_delete_id.is_some() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
// Reconstruct a heal delete and re-queue it. We do NOT call
|
// Reconstruct a heal delete and re-queue it. We do NOT call
|
||||||
// get_object_info here because the delete-marker or version may
|
// get_object_info here because the delete-marker or version may
|
||||||
// already be absent from the local store — that is expected.
|
// already be absent from the local store — that is expected.
|
||||||
@@ -1131,6 +1239,67 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
self.task_handles.lock().await.push(handle);
|
self.task_handles.lock().await.push(handle);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn start_force_delete_processor(&self) {
|
||||||
|
let storage = self.storage.clone();
|
||||||
|
let handle =
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let data =
|
||||||
|
match ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(data) => data,
|
||||||
|
Err(EcstoreError::ConfigNotFound) => return,
|
||||||
|
Err(error) => {
|
||||||
|
warn!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
error = %error,
|
||||||
|
"Failed to load durable force-delete intents"
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let entries = match decode_mrf_file(&data) {
|
||||||
|
Ok(entries) => entries,
|
||||||
|
Err(error) => {
|
||||||
|
warn!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
error = %error,
|
||||||
|
"Failed to decode durable force-delete intents"
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
for entry in entries {
|
||||||
|
if !should_replay_force_delete_intent(&entry) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Some(operation_id) = entry.force_delete_id else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
schedule_replication_delete(DeletedObjectReplicationInfo {
|
||||||
|
delete_object: ReplicationDeletedObject {
|
||||||
|
object_name: entry.object,
|
||||||
|
force_delete: true,
|
||||||
|
force_delete_id: Some(operation_id),
|
||||||
|
force_delete_target_arns: entry.target_arns,
|
||||||
|
force_delete_generation: entry.force_delete_generation,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
bucket: entry.bucket,
|
||||||
|
op_type: ReplicationType::Heal,
|
||||||
|
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
self.task_handles.lock().await.push(handle);
|
||||||
|
}
|
||||||
|
|
||||||
/// Starts the MRF persister — ongoing background task.
|
/// Starts the MRF persister — ongoing background task.
|
||||||
///
|
///
|
||||||
/// Drains `mrf_save_rx` (entries that overflowed the normal worker channels) and
|
/// Drains `mrf_save_rx` (entries that overflowed the normal worker channels) and
|
||||||
@@ -2216,7 +2385,7 @@ mod tests {
|
|||||||
_h: Self::HeaderMap,
|
_h: Self::HeaderMap,
|
||||||
_opts: &Self::ObjectOptions,
|
_opts: &Self::ObjectOptions,
|
||||||
) -> Result<Self::GetObjectReader, Self::Error> {
|
) -> Result<Self::GetObjectReader, Self::Error> {
|
||||||
if !object.ends_with("/.replication/resync.bin") {
|
if !object.ends_with("/.replication/resync.bin") && !object.ends_with("config/replication/force-delete.bin") {
|
||||||
return Err(EcstoreError::FileNotFound);
|
return Err(EcstoreError::FileNotFound);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3104,6 +3273,7 @@ mod tests {
|
|||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
delete_marker_mtime: None,
|
delete_marker_mtime: None,
|
||||||
target_arns: Vec::new(),
|
target_arns: Vec::new(),
|
||||||
|
..Default::default()
|
||||||
};
|
};
|
||||||
let second = MrfReplicateEntry {
|
let second = MrfReplicateEntry {
|
||||||
object: "second".to_string(),
|
object: "second".to_string(),
|
||||||
@@ -3202,6 +3372,7 @@ mod tests {
|
|||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
delete_marker_mtime: None,
|
delete_marker_mtime: None,
|
||||||
target_arns: Vec::new(),
|
target_arns: Vec::new(),
|
||||||
|
..Default::default()
|
||||||
},
|
},
|
||||||
"test",
|
"test",
|
||||||
)
|
)
|
||||||
@@ -3234,6 +3405,7 @@ mod tests {
|
|||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
delete_marker_mtime: None,
|
delete_marker_mtime: None,
|
||||||
target_arns: Vec::new(),
|
target_arns: Vec::new(),
|
||||||
|
..Default::default()
|
||||||
};
|
};
|
||||||
observe_mrf_pending(&entry);
|
observe_mrf_pending(&entry);
|
||||||
|
|
||||||
@@ -3289,6 +3461,7 @@ mod tests {
|
|||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
delete_marker_mtime: None,
|
delete_marker_mtime: None,
|
||||||
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
|
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
|
||||||
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
stats.inc_q(&entry.bucket, entry.size, false, ReplicationType::Heal);
|
stats.inc_q(&entry.bucket, entry.size, false, ReplicationType::Heal);
|
||||||
@@ -3312,6 +3485,7 @@ mod tests {
|
|||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
delete_marker_mtime: None,
|
delete_marker_mtime: None,
|
||||||
target_arns: Vec::new(),
|
target_arns: Vec::new(),
|
||||||
|
..Default::default()
|
||||||
};
|
};
|
||||||
let second = MrfReplicateEntry {
|
let second = MrfReplicateEntry {
|
||||||
object: "second".to_string(),
|
object: "second".to_string(),
|
||||||
@@ -3425,6 +3599,7 @@ mod tests {
|
|||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
delete_marker_mtime: None,
|
delete_marker_mtime: None,
|
||||||
target_arns: Vec::new(),
|
target_arns: Vec::new(),
|
||||||
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
let encoded = encode_mrf_file(std::slice::from_ref(&entry)).expect("encode");
|
let encoded = encode_mrf_file(std::slice::from_ref(&entry)).expect("encode");
|
||||||
@@ -3460,6 +3635,7 @@ mod tests {
|
|||||||
delete_marker: true,
|
delete_marker: true,
|
||||||
delete_marker_mtime: Some(mtime_nanos),
|
delete_marker_mtime: Some(mtime_nanos),
|
||||||
target_arns: Vec::new(),
|
target_arns: Vec::new(),
|
||||||
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
let encoded = encode_mrf_file(std::slice::from_ref(&entry)).expect("encode");
|
let encoded = encode_mrf_file(std::slice::from_ref(&entry)).expect("encode");
|
||||||
@@ -3495,6 +3671,7 @@ mod tests {
|
|||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
delete_marker_mtime: None,
|
delete_marker_mtime: None,
|
||||||
target_arns: Vec::new(),
|
target_arns: Vec::new(),
|
||||||
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
let encoded = encode_mrf_file(&[entry]).expect("encode");
|
let encoded = encode_mrf_file(&[entry]).expect("encode");
|
||||||
@@ -3525,6 +3702,7 @@ mod tests {
|
|||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
delete_marker_mtime: None,
|
delete_marker_mtime: None,
|
||||||
target_arns: Vec::new(),
|
target_arns: Vec::new(),
|
||||||
|
..Default::default()
|
||||||
},
|
},
|
||||||
MrfReplicateEntry {
|
MrfReplicateEntry {
|
||||||
bucket: "b".to_string(),
|
bucket: "b".to_string(),
|
||||||
@@ -3538,6 +3716,7 @@ mod tests {
|
|||||||
delete_marker: true,
|
delete_marker: true,
|
||||||
delete_marker_mtime: None,
|
delete_marker_mtime: None,
|
||||||
target_arns: Vec::new(),
|
target_arns: Vec::new(),
|
||||||
|
..Default::default()
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
@@ -3569,6 +3748,7 @@ mod tests {
|
|||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
delete_marker_mtime: None,
|
delete_marker_mtime: None,
|
||||||
target_arns: Vec::new(),
|
target_arns: Vec::new(),
|
||||||
|
..Default::default()
|
||||||
};
|
};
|
||||||
assert_eq!(obj_entry.op, MrfOpKind::Object);
|
assert_eq!(obj_entry.op, MrfOpKind::Object);
|
||||||
|
|
||||||
@@ -3585,6 +3765,7 @@ mod tests {
|
|||||||
delete_marker: true,
|
delete_marker: true,
|
||||||
delete_marker_mtime: None,
|
delete_marker_mtime: None,
|
||||||
target_arns: Vec::new(),
|
target_arns: Vec::new(),
|
||||||
|
..Default::default()
|
||||||
};
|
};
|
||||||
assert_eq!(del_entry.op, MrfOpKind::Delete);
|
assert_eq!(del_entry.op, MrfOpKind::Delete);
|
||||||
|
|
||||||
@@ -3602,6 +3783,7 @@ mod tests {
|
|||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
delete_marker_mtime: None,
|
delete_marker_mtime: None,
|
||||||
target_arns: Vec::new(),
|
target_arns: Vec::new(),
|
||||||
|
..Default::default()
|
||||||
};
|
};
|
||||||
assert_eq!(legacy_entry.op, MrfOpKind::Object, "legacy default must be Object");
|
assert_eq!(legacy_entry.op, MrfOpKind::Object, "legacy default must be Object");
|
||||||
}
|
}
|
||||||
@@ -3668,6 +3850,7 @@ mod tests {
|
|||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
delete_marker_mtime: None,
|
delete_marker_mtime: None,
|
||||||
target_arns: Vec::new(),
|
target_arns: Vec::new(),
|
||||||
|
..Default::default()
|
||||||
}];
|
}];
|
||||||
let encoded = encode_mrf_file(&entries).expect("durable MRF backlog should encode");
|
let encoded = encode_mrf_file(&entries).expect("durable MRF backlog should encode");
|
||||||
|
|
||||||
@@ -3716,6 +3899,7 @@ mod tests {
|
|||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
delete_marker_mtime: None,
|
delete_marker_mtime: None,
|
||||||
target_arns: vec!["arn:target-a".to_string(), "arn:target-b".to_string()],
|
target_arns: vec!["arn:target-a".to_string(), "arn:target-b".to_string()],
|
||||||
|
..Default::default()
|
||||||
},
|
},
|
||||||
MrfReplicateEntry {
|
MrfReplicateEntry {
|
||||||
bucket: "b1".to_string(),
|
bucket: "b1".to_string(),
|
||||||
@@ -3729,6 +3913,7 @@ mod tests {
|
|||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
delete_marker_mtime: None,
|
delete_marker_mtime: None,
|
||||||
target_arns: vec!["arn:target-a".to_string()],
|
target_arns: vec!["arn:target-a".to_string()],
|
||||||
|
..Default::default()
|
||||||
},
|
},
|
||||||
MrfReplicateEntry {
|
MrfReplicateEntry {
|
||||||
bucket: "b1".to_string(),
|
bucket: "b1".to_string(),
|
||||||
@@ -3742,6 +3927,7 @@ mod tests {
|
|||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
delete_marker_mtime: None,
|
delete_marker_mtime: None,
|
||||||
target_arns: Vec::new(),
|
target_arns: Vec::new(),
|
||||||
|
..Default::default()
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
@@ -3797,10 +3983,89 @@ mod tests {
|
|||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
delete_marker_mtime: None,
|
delete_marker_mtime: None,
|
||||||
target_arns: Vec::new(),
|
target_arns: Vec::new(),
|
||||||
|
..Default::default()
|
||||||
}])
|
}])
|
||||||
.expect("invalid persisted entry should still encode for boundary testing");
|
.expect("invalid persisted entry should still encode for boundary testing");
|
||||||
let invalid = durable_mrf_backlog_from_read(Ok(negative));
|
let invalid = durable_mrf_backlog_from_read(Ok(negative));
|
||||||
assert!(!invalid.available);
|
assert!(!invalid.available);
|
||||||
assert!(invalid.entries.is_empty());
|
assert!(invalid.entries.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn force_delete_replay_requires_local_commit_and_keeps_persisted_targets() {
|
||||||
|
let operation_id = Uuid::new_v4();
|
||||||
|
let pending = MrfReplicateEntry {
|
||||||
|
bucket: "source".to_string(),
|
||||||
|
object: "logs/".to_string(),
|
||||||
|
target_arns: vec!["arn:target:old-generation".to_string()],
|
||||||
|
force_delete_id: Some(operation_id),
|
||||||
|
force_delete_generation: Some(11),
|
||||||
|
force_delete_local_commit: false,
|
||||||
|
op: MrfOpKind::Delete,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert!(!should_replay_force_delete_intent(&pending));
|
||||||
|
|
||||||
|
let mut committed = pending;
|
||||||
|
committed.force_delete_local_commit = true;
|
||||||
|
let recovered = decode_mrf_file(&encode_mrf_file(&[committed.clone()]).expect("force-delete intent should encode"))
|
||||||
|
.expect("force-delete intent should decode");
|
||||||
|
assert!(should_replay_force_delete_intent(&recovered[0]));
|
||||||
|
assert_eq!(recovered[0].target_arns, vec!["arn:target:old-generation"]);
|
||||||
|
assert_eq!(recovered[0].force_delete_generation, Some(11));
|
||||||
|
|
||||||
|
committed.target_arns.clear();
|
||||||
|
assert!(!should_replay_force_delete_intent(&committed));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn force_delete_intent_append_commit_and_cleanup_are_idempotent() {
|
||||||
|
let shared = empty_resync_shared_state();
|
||||||
|
let storage = Arc::new(LoadResyncNodeStore::new("force-delete-journal", shared));
|
||||||
|
let operation_id = Uuid::new_v4();
|
||||||
|
let entry = MrfReplicateEntry {
|
||||||
|
bucket: "source".to_string(),
|
||||||
|
object: "logs/".to_string(),
|
||||||
|
target_arns: vec!["arn:target:stable".to_string()],
|
||||||
|
force_delete_id: Some(operation_id),
|
||||||
|
force_delete_generation: Some(12),
|
||||||
|
op: MrfOpKind::Delete,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
persist_force_delete_intent(storage.clone(), entry.clone())
|
||||||
|
.await
|
||||||
|
.expect("first journal append should succeed");
|
||||||
|
persist_force_delete_intent(storage.clone(), entry)
|
||||||
|
.await
|
||||||
|
.expect("duplicate journal append should be a no-op");
|
||||||
|
|
||||||
|
let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE)
|
||||||
|
.await
|
||||||
|
.expect("journal should be readable");
|
||||||
|
let entries = decode_mrf_file(&data).expect("journal should decode");
|
||||||
|
assert_eq!(entries.len(), 1);
|
||||||
|
assert!(!entries[0].force_delete_local_commit);
|
||||||
|
assert!(!should_replay_force_delete_intent(&entries[0]));
|
||||||
|
|
||||||
|
commit_force_delete_intent(storage.clone(), operation_id)
|
||||||
|
.await
|
||||||
|
.expect("commit marker should persist");
|
||||||
|
commit_force_delete_intent(storage.clone(), operation_id)
|
||||||
|
.await
|
||||||
|
.expect("duplicate commit marker should be a no-op");
|
||||||
|
let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE)
|
||||||
|
.await
|
||||||
|
.expect("committed journal should be readable");
|
||||||
|
let entries = decode_mrf_file(&data).expect("committed journal should decode");
|
||||||
|
assert!(should_replay_force_delete_intent(&entries[0]));
|
||||||
|
assert_eq!(entries[0].target_arns, vec!["arn:target:stable"]);
|
||||||
|
|
||||||
|
complete_force_delete_intent(storage.clone(), operation_id)
|
||||||
|
.await
|
||||||
|
.expect("journal cleanup should succeed");
|
||||||
|
complete_force_delete_intent(storage, operation_id)
|
||||||
|
.await
|
||||||
|
.expect("duplicate journal cleanup should be a no-op");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1642,54 +1642,62 @@ async fn replicate_delete_marker_purge_to_targets(bucket: &str, dobj: &DeletedOb
|
|||||||
async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &DeletedObjectReplicationInfo, storage: Arc<S>) {
|
async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &DeletedObjectReplicationInfo, storage: Arc<S>) {
|
||||||
let bucket = &dobj.bucket;
|
let bucket = &dobj.bucket;
|
||||||
let object_name = &dobj.delete_object.object_name;
|
let object_name = &dobj.delete_object.object_name;
|
||||||
|
let admitted_target_arns = dobj.admitted_target_arns();
|
||||||
|
|
||||||
let rcfg = match get_replication_config(bucket).await {
|
let legacy_target_arns = if admitted_target_arns.is_empty() {
|
||||||
Ok(Some(config)) => config,
|
match get_replication_config(bucket).await {
|
||||||
Ok(None) => {
|
Ok(Some(config)) => config.filter_target_arns(&ObjectOpts {
|
||||||
debug!(
|
name: object_name.clone(),
|
||||||
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
|
||||||
bucket = %bucket,
|
|
||||||
reason = "replication_config_missing",
|
|
||||||
"Skipping replication force-delete because replication config is missing"
|
|
||||||
);
|
|
||||||
send_local_event(EventArgs {
|
|
||||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
|
||||||
bucket_name: bucket.clone(),
|
|
||||||
object: ObjectInfo {
|
|
||||||
bucket: bucket.clone(),
|
|
||||||
name: object_name.clone(),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
user_agent: "Internal: [Replication]".to_string(),
|
|
||||||
..Default::default()
|
..Default::default()
|
||||||
});
|
}),
|
||||||
return;
|
Ok(None) => {
|
||||||
}
|
debug!(
|
||||||
Err(err) => {
|
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
|
||||||
debug!(
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
|
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
bucket = %bucket,
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
reason = "replication_config_missing",
|
||||||
bucket = %bucket,
|
"Skipping replication force-delete because replication config is missing"
|
||||||
error = %err,
|
);
|
||||||
reason = "replication_config_lookup_failed",
|
send_local_event(EventArgs {
|
||||||
"Skipping replication force-delete because replication config lookup failed"
|
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||||
);
|
bucket_name: bucket.clone(),
|
||||||
send_local_event(EventArgs {
|
object: ObjectInfo {
|
||||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
bucket: bucket.clone(),
|
||||||
bucket_name: bucket.clone(),
|
name: object_name.clone(),
|
||||||
object: ObjectInfo {
|
..Default::default()
|
||||||
bucket: bucket.clone(),
|
},
|
||||||
name: object_name.clone(),
|
user_agent: "Internal: [Replication]".to_string(),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
},
|
});
|
||||||
user_agent: "Internal: [Replication]".to_string(),
|
Vec::new()
|
||||||
..Default::default()
|
}
|
||||||
});
|
Err(err) => {
|
||||||
return;
|
debug!(
|
||||||
|
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||||
|
bucket = %bucket,
|
||||||
|
error = %err,
|
||||||
|
reason = "replication_config_lookup_failed",
|
||||||
|
"Skipping replication force-delete because replication config lookup failed"
|
||||||
|
);
|
||||||
|
send_local_event(EventArgs {
|
||||||
|
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||||
|
bucket_name: bucket.clone(),
|
||||||
|
object: ObjectInfo {
|
||||||
|
bucket: bucket.clone(),
|
||||||
|
name: object_name.clone(),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
user_agent: "Internal: [Replication]".to_string(),
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
Vec::new()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
Vec::new()
|
||||||
};
|
};
|
||||||
|
|
||||||
let ns_lock = match storage
|
let ns_lock = match storage
|
||||||
@@ -1751,22 +1759,18 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let tgt_arns = {
|
let tgt_arns = if admitted_target_arns.is_empty() {
|
||||||
let admitted = dobj.admitted_target_arns();
|
legacy_target_arns
|
||||||
if admitted.is_empty() {
|
} else {
|
||||||
rcfg.filter_target_arns(&ObjectOpts {
|
admitted_target_arns
|
||||||
name: object_name.clone(),
|
|
||||||
..Default::default()
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
admitted
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut join_set = JoinSet::new();
|
let mut join_set = JoinSet::new();
|
||||||
|
let mut all_succeeded = true;
|
||||||
|
|
||||||
for arn in tgt_arns {
|
for arn in tgt_arns {
|
||||||
let Some(tgt_client) = ReplicationTargetStore::remote_target_client(bucket, &arn).await else {
|
let Some(tgt_client) = ReplicationTargetStore::remote_target_client(bucket, &arn).await else {
|
||||||
|
all_succeeded = false;
|
||||||
debug!(
|
debug!(
|
||||||
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
|
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
@@ -1816,7 +1820,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
|||||||
user_agent: "Internal: [Replication]".to_string(),
|
user_agent: "Internal: [Replication]".to_string(),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
});
|
});
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Err(e) = tgt_client
|
if let Err(e) = tgt_client
|
||||||
@@ -1845,24 +1849,46 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
|||||||
user_agent: "Internal: [Replication]".to_string(),
|
user_agent: "Internal: [Replication]".to_string(),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
});
|
});
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
true
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
while let Some(result) = join_set.join_next().await {
|
while let Some(result) = join_set.join_next().await {
|
||||||
if let Err(e) = result {
|
match result {
|
||||||
error!(
|
Ok(success) => all_succeeded &= success,
|
||||||
event = EVENT_RESYNC_TASK_FAILED,
|
Err(error) => {
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
all_succeeded = false;
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
error!(
|
||||||
bucket = %bucket,
|
event = EVENT_RESYNC_TASK_FAILED,
|
||||||
object = %object_name,
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
operation = "force_delete",
|
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||||
error = %e,
|
bucket = %bucket,
|
||||||
"Replication resync task failed"
|
object = %object_name,
|
||||||
);
|
operation = "force_delete",
|
||||||
|
error = %error,
|
||||||
|
"Replication resync task failed"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if all_succeeded
|
||||||
|
&& let Some(operation_id) = dobj.delete_object.force_delete_id
|
||||||
|
&& let Err(error) = super::replication_pool::complete_force_delete_intent(storage, operation_id).await
|
||||||
|
{
|
||||||
|
warn!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||||
|
bucket = %bucket,
|
||||||
|
object = %object_name,
|
||||||
|
operation_id = %operation_id,
|
||||||
|
error = %error,
|
||||||
|
"Force-delete replication completed but durable intent cleanup failed"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn target_delete_version_id(version_id: Uuid, version_purge: bool) -> Option<String> {
|
fn target_delete_version_id(version_id: Uuid, version_purge: bool) -> Option<String> {
|
||||||
|
|||||||
@@ -105,6 +105,9 @@ pub(crate) fn deleted_object_for_replication(delete_object: DeletedObject) -> Re
|
|||||||
replication_state: delete_object.replication_state.as_ref().map(replication_state_from_filemeta),
|
replication_state: delete_object.replication_state.as_ref().map(replication_state_from_filemeta),
|
||||||
found: delete_object.found,
|
found: delete_object.found,
|
||||||
force_delete: delete_object.force_delete,
|
force_delete: delete_object.force_delete,
|
||||||
|
force_delete_id: delete_object.force_delete_id,
|
||||||
|
force_delete_target_arns: delete_object.force_delete_target_arns,
|
||||||
|
force_delete_generation: delete_object.force_delete_generation,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ use s3s::dto::{
|
|||||||
ReplicationRuleStatus, ReplicationRules,
|
ReplicationRuleStatus, ReplicationRules,
|
||||||
};
|
};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
pub const REPLICATION_CAPABILITY_CONTRACT_VERSION: u32 = 1;
|
pub const REPLICATION_CAPABILITY_CONTRACT_VERSION: u32 = 1;
|
||||||
@@ -93,6 +93,7 @@ pub trait ReplicationConfigurationExt {
|
|||||||
fn get_destination(&self) -> Destination;
|
fn get_destination(&self) -> Destination;
|
||||||
fn has_active_rules(&self, prefix: &str, recursive: bool) -> bool;
|
fn has_active_rules(&self, prefix: &str, recursive: bool) -> bool;
|
||||||
fn filter_target_arns(&self, obj: &ObjectOpts) -> Vec<String>;
|
fn filter_target_arns(&self, obj: &ObjectOpts) -> Vec<String>;
|
||||||
|
fn filter_force_delete_target_arns(&self, prefix: &str) -> Vec<String>;
|
||||||
fn filter_target_replication_decisions(&self, obj: &ObjectOpts) -> Vec<(String, bool)> {
|
fn filter_target_replication_decisions(&self, obj: &ObjectOpts) -> Vec<(String, bool)> {
|
||||||
self.filter_target_arns(obj)
|
self.filter_target_arns(obj)
|
||||||
.into_iter()
|
.into_iter()
|
||||||
@@ -434,6 +435,51 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
|
|||||||
arns
|
arns
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn filter_force_delete_target_arns(&self, prefix: &str) -> Vec<String> {
|
||||||
|
let role = self.role.trim();
|
||||||
|
let mut selected = BTreeMap::<String, (&ReplicationRule, bool)>::new();
|
||||||
|
|
||||||
|
for rule in &self.rules {
|
||||||
|
if rule.status == ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let rule_prefix = rule.prefix();
|
||||||
|
if !prefix.starts_with(rule_prefix) && !rule_prefix.starts_with(prefix) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let target = if role.is_empty() {
|
||||||
|
rule.destination.bucket.trim()
|
||||||
|
} else {
|
||||||
|
role
|
||||||
|
};
|
||||||
|
if target.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let delete_enabled =
|
||||||
|
rule.delete_replication.as_ref().is_some_and(|delete| {
|
||||||
|
delete.status == DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED)
|
||||||
|
}) || rule.delete_marker_replication.as_ref().is_some_and(|delete_marker| {
|
||||||
|
delete_marker.status
|
||||||
|
== Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED))
|
||||||
|
});
|
||||||
|
|
||||||
|
if selected
|
||||||
|
.get(target)
|
||||||
|
.is_none_or(|(current, _)| rule.priority > current.priority)
|
||||||
|
{
|
||||||
|
selected.insert(target.to_string(), (rule, delete_enabled));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
selected
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|(target, (_, enabled))| enabled.then_some(target))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
fn filter_target_replication_decisions(&self, obj: &ObjectOpts) -> Vec<(String, bool)> {
|
fn filter_target_replication_decisions(&self, obj: &ObjectOpts) -> Vec<(String, bool)> {
|
||||||
let rules = self.filter_actionable_rules(obj);
|
let rules = self.filter_actionable_rules(obj);
|
||||||
let role = self.role.trim();
|
let role = self.role.trim();
|
||||||
@@ -1069,4 +1115,26 @@ mod tests {
|
|||||||
|
|
||||||
assert_eq!(decisions, vec![(target_a.to_string(), false), (target_b.to_string(), true)]);
|
assert_eq!(decisions, vec![(target_a.to_string(), false), (target_b.to_string(), true)]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn force_delete_targets_use_overlapping_rules_and_highest_priority_switch() {
|
||||||
|
let target_a = "arn:target:a";
|
||||||
|
let target_b = "arn:target:b";
|
||||||
|
let mut a_parent = delete_marker_rule("a-parent", target_a, "logs/", 1, true);
|
||||||
|
a_parent.delete_replication = Some(DeleteReplication {
|
||||||
|
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||||
|
});
|
||||||
|
let a_child_disabled = delete_marker_rule("a-child", target_a, "logs/2026/", 5, false);
|
||||||
|
let b_child = delete_marker_rule("b-child", target_b, "logs/2026/", 2, true);
|
||||||
|
let config = ReplicationConfiguration {
|
||||||
|
role: String::new(),
|
||||||
|
rules: vec![a_parent, a_child_disabled, b_child],
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
config.filter_force_delete_target_arns("logs/2026/app.log"),
|
||||||
|
vec![target_b.to_string()],
|
||||||
|
"the child rule must win for target A while the overlapping child target B remains eligible"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -33,12 +33,15 @@ impl DeletedObjectReplicationInfo {
|
|||||||
return vec![self.target_arn.clone()];
|
return vec![self.target_arn.clone()];
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut target_arns = self
|
let mut target_arns = if !self.delete_object.force_delete_target_arns.is_empty() {
|
||||||
.delete_object
|
self.delete_object.force_delete_target_arns.clone()
|
||||||
.replication_state
|
} else {
|
||||||
.as_ref()
|
self.delete_object
|
||||||
.map(admitted_target_arns_from_replication_state)
|
.replication_state
|
||||||
.unwrap_or_default();
|
.as_ref()
|
||||||
|
.map(admitted_target_arns_from_replication_state)
|
||||||
|
.unwrap_or_default()
|
||||||
|
};
|
||||||
target_arns.sort();
|
target_arns.sort();
|
||||||
target_arns.dedup();
|
target_arns.dedup();
|
||||||
target_arns
|
target_arns
|
||||||
@@ -69,6 +72,9 @@ impl ReplicationWorkerOperation for DeletedObjectReplicationInfo {
|
|||||||
.delete_marker_mtime
|
.delete_marker_mtime
|
||||||
.and_then(|t| i64::try_from(t.unix_timestamp_nanos()).ok()),
|
.and_then(|t| i64::try_from(t.unix_timestamp_nanos()).ok()),
|
||||||
target_arns: self.admitted_target_arns(),
|
target_arns: self.admitted_target_arns(),
|
||||||
|
force_delete_id: self.delete_object.force_delete_id,
|
||||||
|
force_delete_generation: self.delete_object.force_delete_generation,
|
||||||
|
force_delete_local_commit: self.delete_object.force_delete,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -231,6 +237,35 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn deleted_object_replication_info_preserves_force_delete_handoff() {
|
||||||
|
let operation_id = Uuid::new_v4();
|
||||||
|
let info = DeletedObjectReplicationInfo {
|
||||||
|
bucket: "bucket".to_string(),
|
||||||
|
delete_object: DeletedObject {
|
||||||
|
object_name: "prefix/".to_string(),
|
||||||
|
force_delete: true,
|
||||||
|
force_delete_id: Some(operation_id),
|
||||||
|
force_delete_target_arns: vec![
|
||||||
|
"arn:target-b".to_string(),
|
||||||
|
"arn:target-a".to_string(),
|
||||||
|
"arn:target-b".to_string(),
|
||||||
|
],
|
||||||
|
force_delete_generation: Some(17),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let entry = info.to_mrf_entry();
|
||||||
|
|
||||||
|
assert!(entry.force_delete);
|
||||||
|
assert_eq!(entry.force_delete_id, Some(operation_id));
|
||||||
|
assert_eq!(entry.force_delete_generation, Some(17));
|
||||||
|
assert!(entry.force_delete_local_commit);
|
||||||
|
assert_eq!(entry.target_arns, vec!["arn:target-a", "arn:target-b"]);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn version_delete_replication_tracks_delete_marker_version_purge() {
|
fn version_delete_replication_tracks_delete_marker_version_purge() {
|
||||||
let dobj = DeletedObject {
|
let dobj = DeletedObject {
|
||||||
|
|||||||
@@ -572,7 +572,7 @@ impl MrfOpKind {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
|
||||||
pub struct MrfReplicateEntry {
|
pub struct MrfReplicateEntry {
|
||||||
#[serde(rename = "bucket")]
|
#[serde(rename = "bucket")]
|
||||||
pub bucket: String,
|
pub bucket: String,
|
||||||
@@ -619,6 +619,17 @@ pub struct MrfReplicateEntry {
|
|||||||
|
|
||||||
#[serde(rename = "targetARNs", skip_serializing_if = "Vec::is_empty", default)]
|
#[serde(rename = "targetARNs", skip_serializing_if = "Vec::is_empty", default)]
|
||||||
pub target_arns: Vec<String>,
|
pub target_arns: Vec<String>,
|
||||||
|
|
||||||
|
// Force-delete entries use the target ARN list above as their immutable target set.
|
||||||
|
// The id distinguishes a durable intent from legacy MRF delete entries.
|
||||||
|
#[serde(rename = "forceDeleteID", skip_serializing_if = "Option::is_none", default)]
|
||||||
|
pub force_delete_id: Option<Uuid>,
|
||||||
|
#[serde(rename = "forceDeleteGeneration", skip_serializing_if = "Option::is_none", default)]
|
||||||
|
pub force_delete_generation: Option<i64>,
|
||||||
|
|
||||||
|
// Replay is allowed only after the source-side recursive delete has committed.
|
||||||
|
#[serde(rename = "forceDeleteLocalCommit", default)]
|
||||||
|
pub force_delete_local_commit: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn retry_count_to_mrf(retry_count: u32) -> i32 {
|
fn retry_count_to_mrf(retry_count: u32) -> i32 {
|
||||||
@@ -865,6 +876,7 @@ impl ReplicationWorkerOperation for ReplicateObjectInfo {
|
|||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
delete_marker_mtime: None,
|
delete_marker_mtime: None,
|
||||||
target_arns: self.admitted_target_arns(),
|
target_arns: self.admitted_target_arns(),
|
||||||
|
..Default::default()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -945,6 +957,7 @@ impl ReplicateObjectInfo {
|
|||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
delete_marker_mtime: None,
|
delete_marker_mtime: None,
|
||||||
target_arns: self.admitted_target_arns(),
|
target_arns: self.admitted_target_arns(),
|
||||||
|
..Default::default()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -68,6 +68,9 @@ mod tests {
|
|||||||
retry_count: 1,
|
retry_count: 1,
|
||||||
size: 1024,
|
size: 1024,
|
||||||
op: MrfOpKind::Metadata,
|
op: MrfOpKind::Metadata,
|
||||||
|
force_delete_id: None,
|
||||||
|
force_delete_generation: None,
|
||||||
|
force_delete_local_commit: false,
|
||||||
force_delete: false,
|
force_delete: false,
|
||||||
delete_marker_version_id: None,
|
delete_marker_version_id: None,
|
||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
@@ -81,6 +84,9 @@ mod tests {
|
|||||||
retry_count: 2,
|
retry_count: 2,
|
||||||
size: 1024,
|
size: 1024,
|
||||||
op: MrfOpKind::Object,
|
op: MrfOpKind::Object,
|
||||||
|
force_delete_id: None,
|
||||||
|
force_delete_generation: None,
|
||||||
|
force_delete_local_commit: false,
|
||||||
force_delete: false,
|
force_delete: false,
|
||||||
delete_marker_version_id: None,
|
delete_marker_version_id: None,
|
||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
@@ -94,6 +100,9 @@ mod tests {
|
|||||||
retry_count: 0,
|
retry_count: 0,
|
||||||
size: 0,
|
size: 0,
|
||||||
op: MrfOpKind::Delete,
|
op: MrfOpKind::Delete,
|
||||||
|
force_delete_id: None,
|
||||||
|
force_delete_generation: None,
|
||||||
|
force_delete_local_commit: false,
|
||||||
force_delete: true,
|
force_delete: true,
|
||||||
delete_marker_version_id: Some(del_vid),
|
delete_marker_version_id: Some(del_vid),
|
||||||
delete_marker: true,
|
delete_marker: true,
|
||||||
|
|||||||
@@ -49,6 +49,9 @@ pub struct DeletedObject {
|
|||||||
pub replication_state: Option<ReplicationState>,
|
pub replication_state: Option<ReplicationState>,
|
||||||
pub found: bool,
|
pub found: bool,
|
||||||
pub force_delete: bool,
|
pub force_delete: bool,
|
||||||
|
pub force_delete_id: Option<Uuid>,
|
||||||
|
pub force_delete_target_arns: Vec<String>,
|
||||||
|
pub force_delete_generation: Option<i64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl DeletedObject {
|
impl DeletedObject {
|
||||||
|
|||||||
@@ -213,6 +213,9 @@ pub struct DeletedObject {
|
|||||||
pub replication_state: Option<ReplicationState>,
|
pub replication_state: Option<ReplicationState>,
|
||||||
pub found: bool,
|
pub found: bool,
|
||||||
pub force_delete: bool,
|
pub force_delete: bool,
|
||||||
|
pub force_delete_id: Option<Uuid>,
|
||||||
|
pub force_delete_target_arns: Vec<String>,
|
||||||
|
pub force_delete_generation: Option<i64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl DeletedObject {
|
impl DeletedObject {
|
||||||
|
|||||||
@@ -1125,6 +1125,7 @@ mod tests {
|
|||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
delete_marker_mtime: None,
|
delete_marker_mtime: None,
|
||||||
target_arns: vec!["arn-a".to_string(), "arn-durable-only".to_string()],
|
target_arns: vec!["arn-a".to_string(), "arn-durable-only".to_string()],
|
||||||
|
..Default::default()
|
||||||
},
|
},
|
||||||
MrfReplicateEntry {
|
MrfReplicateEntry {
|
||||||
bucket: "other-bucket".to_string(),
|
bucket: "other-bucket".to_string(),
|
||||||
@@ -1138,6 +1139,7 @@ mod tests {
|
|||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
delete_marker_mtime: None,
|
delete_marker_mtime: None,
|
||||||
target_arns: Vec::new(),
|
target_arns: Vec::new(),
|
||||||
|
..Default::default()
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
@@ -1197,6 +1199,7 @@ mod tests {
|
|||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
delete_marker_mtime: None,
|
delete_marker_mtime: None,
|
||||||
target_arns: Vec::new(),
|
target_arns: Vec::new(),
|
||||||
|
..Default::default()
|
||||||
}],
|
}],
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -41,11 +41,11 @@ use super::storage_api::object_usecase::bucket::{
|
|||||||
predict_lifecycle_expiration,
|
predict_lifecycle_expiration,
|
||||||
quota::{QuotaCheckResult, QuotaError, QuotaOperation},
|
quota::{QuotaCheckResult, QuotaError, QuotaOperation},
|
||||||
replication::{
|
replication::{
|
||||||
DeleteReplicationConfigSnapshot, REPLICATE_INCOMING_DELETE, ReplicationStatusType, delete_replication_state_from_config,
|
DeleteReplicationConfigSnapshot, REPLICATE_INCOMING_DELETE, ReplicationStatusType, commit_force_delete_intent,
|
||||||
delete_replication_version_id, deleted_object_has_pending_replication_delete, has_active_delete_rule,
|
delete_replication_state_from_config, delete_replication_version_id, deleted_object_has_pending_replication_delete,
|
||||||
load_delete_config_snapshot, must_replicate_object, schedule_object_replication, schedule_replication_delete,
|
force_delete_target_set, has_active_delete_rule, load_delete_config_snapshot, must_replicate_object,
|
||||||
schedule_replication_deletes, set_deleted_object_replication_state, should_schedule_delete_replication,
|
persist_force_delete_intent, schedule_object_replication, schedule_replication_delete, schedule_replication_deletes,
|
||||||
should_use_existing_delete_replication_info,
|
set_deleted_object_replication_state, should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||||
},
|
},
|
||||||
tagging::decode_tags,
|
tagging::decode_tags,
|
||||||
validate_restore_request,
|
validate_restore_request,
|
||||||
@@ -7137,6 +7137,7 @@ impl DefaultObjectUsecase {
|
|||||||
opts.expected_current_version_id = expected_current_version_id.clone();
|
opts.expected_current_version_id = expected_current_version_id.clone();
|
||||||
|
|
||||||
let replicate_force_delete = force_delete && !replica && has_active_delete_rule(&delete_config_snapshot, &key);
|
let replicate_force_delete = force_delete && !replica && has_active_delete_rule(&delete_config_snapshot, &key);
|
||||||
|
let mut force_delete_intent = None;
|
||||||
|
|
||||||
// Check Object Lock retention before deletion
|
// Check Object Lock retention before deletion
|
||||||
// TODO: Future optimization (separate PR) - If performance becomes critical under high delete load:
|
// TODO: Future optimization (separate PR) - If performance becomes critical under high delete load:
|
||||||
@@ -7176,6 +7177,17 @@ impl DefaultObjectUsecase {
|
|||||||
let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &key).await;
|
let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &key).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if replicate_force_delete
|
||||||
|
&& let Some((target_arns, generation)) = force_delete_target_set(&delete_config_snapshot, &key)
|
||||||
|
&& !target_arns.is_empty()
|
||||||
|
{
|
||||||
|
let operation_id =
|
||||||
|
persist_force_delete_intent(store.clone(), bucket.clone(), key.clone(), target_arns.clone(), generation)
|
||||||
|
.await
|
||||||
|
.map_err(ApiError::from)?;
|
||||||
|
force_delete_intent = Some((operation_id, target_arns, generation));
|
||||||
|
}
|
||||||
|
|
||||||
let obj_info = {
|
let obj_info = {
|
||||||
match store
|
match store
|
||||||
.delete_object_with_tier_delete_journal(&bucket, &key, opts.clone())
|
.delete_object_with_tier_delete_journal(&bucket, &key, opts.clone())
|
||||||
@@ -7183,6 +7195,18 @@ impl DefaultObjectUsecase {
|
|||||||
{
|
{
|
||||||
Ok(obj) => obj,
|
Ok(obj) => obj,
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
|
if let Some((operation_id, _, _)) = force_delete_intent.as_ref()
|
||||||
|
&& let Err(cleanup_error) =
|
||||||
|
crate::storage::storage_api::complete_force_delete_intent(store.clone(), *operation_id).await
|
||||||
|
{
|
||||||
|
warn!(
|
||||||
|
bucket = %bucket,
|
||||||
|
object = %key,
|
||||||
|
operation_id = %operation_id,
|
||||||
|
error = %cleanup_error,
|
||||||
|
"failed to remove uncommitted force-delete intent after local delete failure"
|
||||||
|
);
|
||||||
|
}
|
||||||
if is_err_bucket_not_found(&err) {
|
if is_err_bucket_not_found(&err) {
|
||||||
return Err(S3Error::with_message(S3ErrorCode::NoSuchBucket, "Bucket not found".to_string()));
|
return Err(S3Error::with_message(S3ErrorCode::NoSuchBucket, "Bucket not found".to_string()));
|
||||||
}
|
}
|
||||||
@@ -7211,7 +7235,31 @@ impl DefaultObjectUsecase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if obj_info.name.is_empty() {
|
if obj_info.name.is_empty() {
|
||||||
if replicate_force_delete {
|
if let Some((operation_id, target_arns, generation)) = force_delete_intent {
|
||||||
|
if let Err(error) = commit_force_delete_intent(store.clone(), operation_id).await {
|
||||||
|
warn!(
|
||||||
|
bucket = %bucket,
|
||||||
|
object = %key,
|
||||||
|
operation_id = %operation_id,
|
||||||
|
error = %error,
|
||||||
|
"failed to mark force-delete intent committed after local delete"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let generation = i64::try_from(generation.unix_timestamp_nanos()).unwrap_or(i64::MAX);
|
||||||
|
schedule_replication_delete(
|
||||||
|
StorageDeletedObject {
|
||||||
|
object_name: key.clone(),
|
||||||
|
force_delete: true,
|
||||||
|
force_delete_id: Some(operation_id),
|
||||||
|
force_delete_target_arns: target_arns,
|
||||||
|
force_delete_generation: Some(generation),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
bucket.clone(),
|
||||||
|
REPLICATE_INCOMING_DELETE.to_string(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
} else if replicate_force_delete {
|
||||||
let mut delete_object = StorageDeletedObject {
|
let mut delete_object = StorageDeletedObject {
|
||||||
object_name: key.clone(),
|
object_name: key.clone(),
|
||||||
force_delete: true,
|
force_delete: true,
|
||||||
|
|||||||
@@ -577,6 +577,44 @@ pub(crate) mod bucket {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use replication_contracts::replication_statuses_map;
|
pub(crate) use replication_contracts::replication_statuses_map;
|
||||||
|
|
||||||
|
pub(crate) async fn persist_force_delete_intent(
|
||||||
|
store: Arc<crate::storage::storage_api::ECStore>,
|
||||||
|
bucket: String,
|
||||||
|
object: String,
|
||||||
|
target_arns: Vec<String>,
|
||||||
|
generation: time::OffsetDateTime,
|
||||||
|
) -> crate::storage::storage_api::Result<Uuid> {
|
||||||
|
let operation_id = Uuid::new_v4();
|
||||||
|
crate::storage::storage_api::persist_force_delete_intent(
|
||||||
|
store,
|
||||||
|
replication_contracts::MrfReplicateEntry {
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
version_id: None,
|
||||||
|
retry_count: 0,
|
||||||
|
size: 0,
|
||||||
|
op: replication_contracts::MrfOpKind::Delete,
|
||||||
|
force_delete: true,
|
||||||
|
delete_marker_version_id: None,
|
||||||
|
delete_marker: false,
|
||||||
|
delete_marker_mtime: None,
|
||||||
|
target_arns,
|
||||||
|
force_delete_id: Some(operation_id),
|
||||||
|
force_delete_generation: Some(i64::try_from(generation.unix_timestamp_nanos()).unwrap_or(i64::MAX)),
|
||||||
|
force_delete_local_commit: false,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map(|_| operation_id)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn commit_force_delete_intent(
|
||||||
|
store: Arc<crate::storage::storage_api::ECStore>,
|
||||||
|
operation_id: Uuid,
|
||||||
|
) -> crate::storage::storage_api::Result<()> {
|
||||||
|
crate::storage::storage_api::commit_force_delete_intent(store, operation_id).await
|
||||||
|
}
|
||||||
|
|
||||||
/// Test-only counter of `must_replicate_object` invocations.
|
/// Test-only counter of `must_replicate_object` invocations.
|
||||||
///
|
///
|
||||||
/// Used by white-box regression tests to assert that a single PUT
|
/// Used by white-box regression tests to assert that a single PUT
|
||||||
@@ -614,6 +652,13 @@ pub(crate) mod bucket {
|
|||||||
ReplicationObjectBridge::has_active_delete_rule(snapshot, object)
|
ReplicationObjectBridge::has_active_delete_rule(snapshot, object)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn force_delete_target_set(
|
||||||
|
snapshot: &DeleteReplicationConfigSnapshot,
|
||||||
|
prefix: &str,
|
||||||
|
) -> Option<(Vec<String>, time::OffsetDateTime)> {
|
||||||
|
ReplicationObjectBridge::force_delete_target_set(snapshot, prefix)
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn delete_replication_version_id(
|
pub(crate) fn delete_replication_version_id(
|
||||||
replication_source: &crate::storage::storage_api::StorageObjectInfo,
|
replication_source: &crate::storage::storage_api::StorageObjectInfo,
|
||||||
deleted_delete_marker_version: bool,
|
deleted_delete_marker_version: bool,
|
||||||
|
|||||||
@@ -1441,6 +1441,21 @@ pub(crate) async fn get_bucket_replication_config(
|
|||||||
ecstore_bucket::metadata_sys::get_replication_config(bucket).await
|
ecstore_bucket::metadata_sys::get_replication_config(bucket).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn persist_force_delete_intent(
|
||||||
|
api: Arc<ECStore>,
|
||||||
|
entry: ecstore_bucket::replication::MrfReplicateEntry,
|
||||||
|
) -> Result<()> {
|
||||||
|
ecstore_bucket::replication::persist_force_delete_intent(api, entry).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn commit_force_delete_intent(api: Arc<ECStore>, operation_id: uuid::Uuid) -> Result<()> {
|
||||||
|
ecstore_bucket::replication::commit_force_delete_intent(api, operation_id).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn complete_force_delete_intent(api: Arc<ECStore>, operation_id: uuid::Uuid) -> Result<()> {
|
||||||
|
ecstore_bucket::replication::complete_force_delete_intent(api, operation_id).await
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn get_bucket_request_payment_config(
|
pub(crate) async fn get_bucket_request_payment_config(
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
) -> Result<(s3s::dto::RequestPaymentConfiguration, time::OffsetDateTime)> {
|
) -> Result<(s3s::dto::RequestPaymentConfiguration, time::OffsetDateTime)> {
|
||||||
|
|||||||
Reference in New Issue
Block a user