fix(replication): honor disabled version deletes

This commit is contained in:
overtrue
2026-07-30 13:37:01 +08:00
parent 67904a6c18
commit c568a54797
17 changed files with 1121 additions and 167 deletions
+295 -35
View File
@@ -18,7 +18,9 @@ use crate::rule::ReplicationRuleExt as _;
use s3s::dto::DeleteMarkerReplicationStatus;
use s3s::dto::DeleteReplicationStatus;
use s3s::dto::Destination;
use s3s::dto::{ExistingObjectReplicationStatus, ReplicationConfiguration, ReplicationRuleStatus, ReplicationRules};
use s3s::dto::{
ExistingObjectReplicationStatus, ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus, ReplicationRules,
};
use serde::{Deserialize, Serialize};
use std::collections::HashSet;
use uuid::Uuid;
@@ -45,6 +47,86 @@ pub trait ReplicationConfigurationExt {
fn filter_target_arns(&self, obj: &ObjectOpts) -> Vec<String>;
}
pub fn delete_replication_target_arns(config: &ReplicationConfiguration, object_name: &str, replica: bool) -> HashSet<String> {
let role = config.role.trim();
if !role.is_empty() && active_replication_rule_destination_arns(config).len() > 1 {
return HashSet::new();
}
let mut targets = HashSet::new();
let mut targets_with_unknown_tags = HashSet::new();
for rule in &config.rules {
if rule.status == ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED)
|| !object_name.starts_with(rule.prefix())
{
continue;
}
let arn = if role.is_empty() {
rule.destination.bucket.trim()
} else {
role
};
if arn.is_empty() {
continue;
}
targets.insert(arn.to_string());
if rule.filter.as_ref().is_some_and(|filter| {
filter.tag.is_some()
|| filter
.and
.as_ref()
.and_then(|and| and.tags.as_ref())
.is_some_and(|tags| !tags.is_empty())
}) {
targets_with_unknown_tags.insert(arn.to_string());
}
}
targets
.into_iter()
.filter(|arn| !targets_with_unknown_tags.contains(arn))
.filter(|arn| {
config.replicate(&ObjectOpts {
name: object_name.to_string(),
target_arn: arn.clone(),
version_id: Some(Uuid::nil()),
delete_marker: true,
op_type: ReplicationType::Delete,
replica,
..Default::default()
})
})
.collect()
}
fn rule_replicates(rule: &ReplicationRule, obj: &ObjectOpts) -> bool {
if let Some(status) = &rule.existing_object_replication
&& obj.existing_object
&& status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::DISABLED)
{
return false;
}
if obj.op_type == ReplicationType::Delete {
if !rule.metadata_replicate(obj) {
return false;
}
if obj.version_id.is_some() {
return rule
.delete_replication
.clone()
.is_some_and(|d| d.status == DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED));
}
return rule.delete_marker_replication.clone().is_some_and(|d| {
d.status == Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED))
});
}
rule.metadata_replicate(obj)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReplicationTargetValidationError {
RoleWithMultipleDestinations,
@@ -204,37 +286,7 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
continue;
}
if let Some(status) = &rule.existing_object_replication
&& obj.existing_object
&& status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::DISABLED)
{
return false;
}
if obj.op_type == ReplicationType::Delete {
if !rule.metadata_replicate(obj) {
return false;
}
if obj.version_id.is_some() {
if obj.delete_marker {
return rule.delete_marker_replication.clone().is_some_and(|d| {
d.status == Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED))
});
}
return rule
.delete_replication
.clone()
.is_some_and(|d| d.status == DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED));
} else {
return rule.delete_marker_replication.clone().is_some_and(|d| {
d.status == Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED))
});
}
}
// Regular object/metadata replication
return rule.metadata_replicate(obj);
return rule_replicates(rule, obj);
}
false
}
@@ -305,7 +357,10 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
#[cfg(test)]
mod tests {
use super::*;
use s3s::dto::{DeleteMarkerReplication, Destination, ExistingObjectReplication, ReplicationRule};
use s3s::dto::{
DeleteMarkerReplication, DeleteReplication, Destination, ExistingObjectReplication, ReplicationRule,
ReplicationRuleFilter, Tag,
};
fn replication_rule(id: &str, arn: &str) -> ReplicationRule {
ReplicationRule {
@@ -327,6 +382,37 @@ mod tests {
}
}
#[test]
fn delete_replication_target_arns_uses_highest_priority_matching_rule() {
let arn = "arn:target:a";
let mut lower_priority = replication_rule("lower", arn);
lower_priority.priority = Some(1);
lower_priority.delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
});
let mut higher_priority = replication_rule("higher", arn);
higher_priority.priority = Some(2);
higher_priority.delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
});
let mut config = ReplicationConfiguration {
role: String::new(),
rules: vec![lower_priority, higher_priority],
};
let targets = delete_replication_target_arns(&config, "object", false);
assert!(targets.is_empty(), "the higher-priority disabled rule must suppress the target");
config.rules[0].delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
});
config.rules[1].delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
});
assert_eq!(delete_replication_target_arns(&config, "object", false), HashSet::from([arn.to_string()]));
}
#[test]
fn filter_target_arns_uses_role_when_role_is_present() {
let config = ReplicationConfiguration {
@@ -337,15 +423,151 @@ mod tests {
],
};
let arns = config.filter_target_arns(&ObjectOpts {
let opts = ObjectOpts {
name: "object".to_string(),
op_type: ReplicationType::Object,
..Default::default()
});
};
let arns = config.filter_target_arns(&opts);
assert_eq!(arns, vec!["arn:legacy:target".to_string()]);
}
#[test]
fn delete_replication_target_arns_uses_role_when_role_is_present() {
let mut rule = replication_rule("rule", "arn:target:a");
rule.delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
});
let config = ReplicationConfiguration {
role: " arn:legacy:target ".to_string(),
rules: vec![rule],
};
assert_eq!(
delete_replication_target_arns(&config, "object", false),
HashSet::from(["arn:legacy:target".to_string()])
);
}
#[test]
fn delete_replication_target_arns_ignores_disjoint_prefix_rules() {
let arn = "arn:target:a";
let mut matching = replication_rule("matching", arn);
matching.prefix = None;
matching.filter = Some(ReplicationRuleFilter {
prefix: Some("logs/".to_string()),
..Default::default()
});
matching.delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
});
let mut unrelated = replication_rule("unrelated", arn);
unrelated.prefix = None;
unrelated.filter = Some(ReplicationRuleFilter {
prefix: Some("archive/".to_string()),
..Default::default()
});
unrelated.priority = Some(2);
unrelated.delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
});
let mut config = ReplicationConfiguration {
role: String::new(),
rules: vec![matching, unrelated],
};
assert!(delete_replication_target_arns(&config, "logs/object", false).is_empty());
config.rules[0].delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
});
config.rules[1].delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
});
assert_eq!(
delete_replication_target_arns(&config, "logs/object", false),
HashSet::from([arn.to_string()])
);
}
#[test]
fn delete_replication_target_arns_fails_closed_for_unknown_tag_rules() {
let arn = "arn:target:a";
let mut known = replication_rule("known", arn);
known.priority = Some(2);
known.delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
});
let mut unknown = replication_rule("unknown", arn);
unknown.priority = Some(1);
unknown.prefix = None;
unknown.filter = Some(ReplicationRuleFilter {
tag: Some(Tag {
key: Some("env".to_string()),
value: Some("prod".to_string()),
}),
..Default::default()
});
unknown.delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
});
let config = ReplicationConfiguration {
role: String::new(),
rules: vec![known, unknown],
};
assert!(delete_replication_target_arns(&config, "object", false).is_empty());
}
#[test]
fn delete_replication_target_arns_reuses_full_destination_rule_order() {
let arn = "arn:target:a";
let mut first = replication_rule("first", arn);
first.priority = Some(1);
first.destination.account = Some("account-a".to_string());
first.delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
});
let mut second = replication_rule("second", arn);
second.priority = Some(2);
second.destination.account = Some("account-b".to_string());
second.delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
});
let config = ReplicationConfiguration {
role: String::new(),
rules: vec![first, second],
};
let opts = ObjectOpts {
name: "object".to_string(),
target_arn: arn.to_string(),
version_id: Some(Uuid::new_v4()),
delete_marker: true,
op_type: ReplicationType::Delete,
..Default::default()
};
assert!(!config.replicate(&opts));
assert!(delete_replication_target_arns(&config, "object", false).is_empty());
}
#[test]
fn delete_replication_target_arns_rejects_role_with_multiple_destinations() {
let mut first = replication_rule("first", "arn:target:a");
first.delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
});
let mut second = replication_rule("second", "arn:target:b");
second.delete_replication = first.delete_replication.clone();
let config = ReplicationConfiguration {
role: "arn:legacy:target".to_string(),
rules: vec![first, second],
};
assert!(delete_replication_target_arns(&config, "object", false).is_empty());
}
#[test]
fn filter_target_arns_falls_back_to_role_when_destination_is_empty() {
let config = ReplicationConfiguration {
@@ -605,4 +827,42 @@ mod tests {
"highest-priority rule disables delete-marker replication, so the delete marker must not replicate"
);
}
#[test]
fn delete_marker_version_purge_requires_delete_replication() {
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
let mut config = ReplicationConfiguration {
role: String::new(),
rules: vec![delete_marker_rule("delete-markers-only", arn, "", 1, true)],
};
let opts = ObjectOpts {
name: "object.txt".to_string(),
op_type: ReplicationType::Delete,
delete_marker: true,
version_id: Some(Uuid::new_v4()),
..Default::default()
};
assert!(
!config.replicate(&opts),
"permanently deleting a delete-marker version must not use the delete-marker replication setting"
);
config.rules[0].delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
});
assert!(
!config.replicate(&opts),
"an explicitly disabled permanent-delete setting must not replicate"
);
config.rules[0].delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
});
assert!(
config.replicate(&opts),
"permanently deleting a delete-marker version should replicate when delete replication is enabled"
);
}
}
+31 -3
View File
@@ -15,7 +15,7 @@
use std::any::Any;
use crate::storage_api::DeletedObject;
use crate::{MrfOpKind, MrfReplicateEntry, ReplicationType, ReplicationWorkerOperation};
use crate::{MrfOpKind, MrfReplicateEntry, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation};
#[derive(Debug, Clone, Default)]
pub struct DeletedObjectReplicationInfo {
@@ -42,6 +42,11 @@ impl ReplicationWorkerOperation for DeletedObjectReplicationInfo {
op: MrfOpKind::Delete,
delete_marker_version_id: self.delete_object.delete_marker_version_id,
delete_marker: self.delete_object.delete_marker,
replica: self
.delete_object
.replication_state
.as_ref()
.is_some_and(|state| state.replica_status == ReplicationStatusType::Replica),
// Persist the original delete-marker mtime as Unix nanoseconds so replay after a
// restart stamps the replica with the source timestamp rather than the replay time
// (backlog#867). None when unknown; replay then falls back to the current time.
@@ -85,14 +90,22 @@ pub fn is_retryable_delete_replication_head_error(is_not_found: bool, code: Opti
!(is_not_found || matches!(code, Some("MethodNotAllowed" | "405")))
}
pub fn version_purge_target_missing(is_not_found: bool, code: Option<&str>, raw_status: Option<u16>) -> bool {
if matches!(code, Some("MethodNotAllowed" | "405")) || raw_status == Some(405) {
return false;
}
is_not_found || matches!(code, Some("NoSuchVersion" | "NoSuchKey" | "NotFound" | "404")) || raw_status == Some(404)
}
#[cfg(test)]
mod tests {
use super::{
DeletedObjectReplicationInfo, is_retryable_delete_replication_head_error, is_version_delete_replication,
should_retry_delete_marker_purge,
should_retry_delete_marker_purge, version_purge_target_missing,
};
use crate::storage_api::DeletedObject;
use crate::{MrfOpKind, ReplicationType, ReplicationWorkerOperation};
use crate::{MrfOpKind, ReplicationState, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation};
use uuid::Uuid;
#[test]
@@ -109,6 +122,10 @@ mod tests {
delete_marker_version_id: Some(delete_marker_version_id),
delete_marker: true,
delete_marker_mtime: Some(mtime),
replication_state: Some(ReplicationState {
replica_status: ReplicationStatusType::Replica,
..Default::default()
}),
..Default::default()
},
..Default::default()
@@ -122,6 +139,7 @@ mod tests {
assert_eq!(entry.delete_marker_version_id, Some(delete_marker_version_id));
assert_eq!(entry.op, MrfOpKind::Delete);
assert!(entry.delete_marker);
assert!(entry.replica);
// The original mtime must be persisted (as Unix nanos) so replay keeps the source
// timestamp instead of stamping the replica with the replay time (backlog#867).
assert_eq!(
@@ -212,4 +230,14 @@ mod tests {
assert!(!is_retryable_delete_replication_head_error(true, Some("NoSuchKey")));
assert!(is_retryable_delete_replication_head_error(false, Some("AccessDenied")));
}
#[test]
fn version_purge_target_missing_requires_not_found() {
assert!(version_purge_target_missing(true, Some("NoSuchVersion"), Some(404)));
assert!(version_purge_target_missing(false, Some("NoSuchVersion"), None));
assert!(version_purge_target_missing(false, None, Some(404)));
assert!(!version_purge_target_missing(false, None, None));
assert!(!version_purge_target_missing(true, Some("MethodNotAllowed"), Some(405)));
assert!(!version_purge_target_missing(true, Some("405"), None));
}
}
+7
View File
@@ -587,6 +587,11 @@ pub struct MrfReplicateEntry {
#[serde(rename = "deleteMarker", default)]
pub delete_marker: bool,
// For delete entries: whether the operation originated from a replica.
// Old files lack this field and therefore default to a local-source delete.
#[serde(rename = "replica", default)]
pub replica: bool,
// For delete entries: the original delete-marker mtime, persisted as Unix nanoseconds so
// replay stamps replicas with the source timestamp instead of the replay time. Old files
// lack this key; default=None means "unknown", and replay falls back to the current time
@@ -798,6 +803,7 @@ impl ReplicationWorkerOperation for ReplicateObjectInfo {
op: MrfOpKind::Object,
delete_marker_version_id: None,
delete_marker: false,
replica: false,
delete_marker_mtime: None,
}
}
@@ -852,6 +858,7 @@ impl ReplicateObjectInfo {
op: MrfOpKind::Object,
delete_marker_version_id: None,
delete_marker: false,
replica: false,
delete_marker_mtime: None,
}
}
+8 -6
View File
@@ -30,11 +30,12 @@ pub mod tagging;
pub use config::{
ObjectOpts, ReplicationConfigurationExt, ReplicationTargetValidationError, active_replication_rule_destination_arns,
replication_target_arns, should_remove_replication_target, validate_replication_config_target_arns,
delete_replication_target_arns, replication_target_arns, should_remove_replication_target,
validate_replication_config_target_arns,
};
pub use delete::{
DeletedObjectReplicationInfo, is_retryable_delete_replication_head_error, is_version_delete_replication,
should_retry_delete_marker_purge,
should_retry_delete_marker_purge, version_purge_target_missing,
};
pub use filemeta::{
REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL, REPLICATE_HEAL_DELETE, REPLICATE_INCOMING,
@@ -54,10 +55,11 @@ pub use object::{
replication_etags_match, target_is_newer_than_source_null_version,
};
pub use operation::{
MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteSource, ReplicationDeleteStateSource,
ReplicationResyncTargetObject, delete_replication_missing_source_decision, delete_replication_object_opts,
delete_replication_state_from_config, delete_replication_version_id, heal_uses_delete_replication_path, is_ssec_encrypted,
resync_target_for_object, should_schedule_delete_replication, should_use_existing_delete_replication_info,
MustReplicateOptions, ReplicationDeleteParts, ReplicationDeleteScheduleInput, ReplicationDeleteSource,
ReplicationDeleteStateSource, ReplicationResyncTargetObject, delete_replication_missing_source_decision,
delete_replication_object_opts, delete_replication_parts, delete_replication_state_from_config,
delete_replication_version_id, heal_uses_delete_replication_path, is_ssec_encrypted, resync_target_for_object,
should_schedule_delete_replication, should_use_existing_delete_replication_info,
should_use_existing_delete_replication_source,
};
pub use queue::{
+70 -1
View File
@@ -13,6 +13,7 @@
// limitations under the License.
use byteorder::{ByteOrder, LittleEndian};
use uuid::Uuid;
use crate::{Error, Result};
@@ -21,6 +22,31 @@ pub use crate::filemeta::{MrfOpKind, MrfReplicateEntry};
pub const MRF_META_FORMAT: u16 = 1;
pub const MRF_META_VERSION: u16 = 1;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct MrfDeleteParts {
pub version_id: Option<Uuid>,
pub delete_marker_version_id: Option<Uuid>,
pub delete_marker: bool,
}
impl MrfReplicateEntry {
pub fn delete_parts_for_replay(&self) -> Option<MrfDeleteParts> {
match (self.version_id, self.delete_marker_version_id) {
(Some(version_id), None) => Some(MrfDeleteParts {
version_id: Some(version_id),
delete_marker_version_id: None,
delete_marker: false,
}),
(None, Some(delete_marker_version_id)) => Some(MrfDeleteParts {
version_id: None,
delete_marker_version_id: Some(delete_marker_version_id),
delete_marker: self.delete_marker,
}),
_ => None,
}
}
}
pub fn encode_mrf_file(entries: &[MrfReplicateEntry]) -> Result<Vec<u8>> {
let payload = rmp_serde::to_vec_named(entries).map_err(|e| Error::Other(e.to_string()))?;
let mut data = Vec::with_capacity(4 + payload.len());
@@ -54,7 +80,6 @@ pub fn decode_mrf_file(data: &[u8]) -> Result<Vec<MrfReplicateEntry>> {
#[cfg(test)]
mod tests {
use super::*;
use uuid::Uuid;
#[test]
fn mrf_file_round_trips_object_and_delete_entries() {
@@ -70,6 +95,7 @@ mod tests {
op: MrfOpKind::Object,
delete_marker_version_id: None,
delete_marker: false,
replica: false,
delete_marker_mtime: None,
},
MrfReplicateEntry {
@@ -81,6 +107,7 @@ mod tests {
op: MrfOpKind::Delete,
delete_marker_version_id: Some(del_vid),
delete_marker: true,
replica: true,
delete_marker_mtime: Some(1_705_312_200_123_456_789),
},
];
@@ -95,6 +122,7 @@ mod tests {
assert_eq!(decoded[1].delete_marker_version_id, Some(del_vid));
assert_eq!(decoded[1].op, MrfOpKind::Delete);
assert!(decoded[1].delete_marker);
assert!(decoded[1].replica);
assert_eq!(
decoded[1].delete_marker_mtime,
Some(1_705_312_200_123_456_789),
@@ -102,6 +130,46 @@ mod tests {
);
}
#[test]
fn legacy_version_purge_replay_clears_delete_marker_creation() {
let entry = MrfReplicateEntry {
bucket: "bucket".to_string(),
object: "object".to_string(),
version_id: Some(Uuid::new_v4()),
retry_count: 0,
size: 0,
delete_marker: true,
op: MrfOpKind::Delete,
delete_marker_version_id: None,
replica: false,
delete_marker_mtime: None,
};
let encoded = encode_mrf_file(&[entry]).expect("legacy MRF entry should encode");
let decoded = decode_mrf_file(&encoded).expect("legacy MRF entry should decode");
assert_eq!(decoded[0].delete_parts_for_replay().map(|parts| parts.delete_marker), Some(false));
}
#[test]
fn invalid_delete_id_shapes_fail_closed_on_replay() {
for (version_id, delete_marker_version_id) in [(None, None), (Some(Uuid::new_v4()), Some(Uuid::new_v4()))] {
let entry = MrfReplicateEntry {
bucket: "bucket".to_string(),
object: "object".to_string(),
version_id,
retry_count: 0,
size: 0,
op: MrfOpKind::Delete,
delete_marker_version_id,
delete_marker: false,
replica: false,
delete_marker_mtime: None,
};
assert_eq!(entry.delete_parts_for_replay(), None);
}
}
#[test]
fn mrf_legacy_file_without_op_decodes_as_object() {
let mut payload = Vec::new();
@@ -129,6 +197,7 @@ mod tests {
assert_eq!(decoded[0].retry_count, 2);
assert_eq!(decoded[0].size, 100);
assert_eq!(decoded[0].op, MrfOpKind::Object);
assert!(!decoded[0].replica);
// Old files lack the deleteMarkerMtime key; it must default to None so replay keeps the
// pre-#867 fallback to the current time.
assert_eq!(decoded[0].delete_marker_mtime, None);
+117 -27
View File
@@ -151,6 +151,11 @@ pub fn delete_replication_state_from_config(
let pending_status = decision.pending_status();
let mut state = ReplicationState {
replica_status: if source.replica {
ReplicationStatusType::Replica
} else {
ReplicationStatusType::Empty
},
replicate_decision_str: decision.to_string(),
..Default::default()
};
@@ -174,11 +179,31 @@ pub struct ReplicationDeleteScheduleInput<'a> {
pub deleted_delete_marker_version: bool,
}
fn delete_version_purge_source_status(status: &ReplicationStatusType) -> bool {
status == &ReplicationStatusType::Replica
|| status == &ReplicationStatusType::Pending
|| status == &ReplicationStatusType::Completed
|| status == &ReplicationStatusType::Failed
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ReplicationDeleteParts {
pub delete_marker: bool,
pub version_id: Option<Uuid>,
pub delete_marker_version_id: Option<Uuid>,
}
pub fn delete_replication_parts(
source_delete_marker: bool,
source_version_id: Option<Uuid>,
version_purge: bool,
) -> Option<ReplicationDeleteParts> {
if version_purge {
return source_version_id.map(|version_id| ReplicationDeleteParts {
delete_marker: false,
version_id: Some(version_id),
delete_marker_version_id: None,
});
}
Some(ReplicationDeleteParts {
delete_marker: source_delete_marker,
version_id: None,
delete_marker_version_id: source_version_id,
})
}
pub fn should_schedule_delete_replication(input: ReplicationDeleteScheduleInput<'_>) -> bool {
@@ -186,14 +211,11 @@ pub fn should_schedule_delete_replication(input: ReplicationDeleteScheduleInput<
return false;
}
if input.version_id_requested && !input.deleted_delete_marker_version && !input.source_delete_marker {
return delete_version_purge_source_status(input.source_replication_status);
if input.version_id_requested {
return input.source_version_purge_status == &VersionPurgeStatusType::Pending;
}
input.source_replication_status == &ReplicationStatusType::Replica
|| input.source_replication_status == &ReplicationStatusType::Pending
|| input.source_version_purge_status == &VersionPurgeStatusType::Pending
|| (input.deleted_delete_marker_version && input.source_replication_status == &ReplicationStatusType::Completed)
input.source_replication_status == &ReplicationStatusType::Pending
}
pub fn delete_replication_version_id(
@@ -312,19 +334,20 @@ pub fn resync_target_for_object(
#[cfg(test)]
mod tests {
use super::{
MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteSource, ReplicationDeleteStateSource,
ReplicationResyncTargetObject, delete_replication_missing_source_decision, delete_replication_object_opts,
delete_replication_state_from_config, delete_replication_version_id, heal_uses_delete_replication_path,
is_ssec_encrypted, resync_target_for_object, should_schedule_delete_replication,
should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
MustReplicateOptions, ReplicationDeleteParts, ReplicationDeleteScheduleInput, ReplicationDeleteSource,
ReplicationDeleteStateSource, ReplicationResyncTargetObject, delete_replication_missing_source_decision,
delete_replication_object_opts, delete_replication_parts, delete_replication_state_from_config,
delete_replication_version_id, heal_uses_delete_replication_path, is_ssec_encrypted, resync_target_for_object,
should_schedule_delete_replication, should_use_existing_delete_replication_info,
should_use_existing_delete_replication_source,
};
use crate::http::{AMZ_BUCKET_REPLICATION_STATUS, SSEC_ALGORITHM_HEADER};
use crate::storage_api::ObjectToDelete;
use crate::{ReplicationStatusType, ReplicationType, VersionPurgeStatusType, target_reset_header};
use s3s::dto::{
DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ExistingObjectReplication,
ExistingObjectReplicationStatus, ReplicaModifications, ReplicaModificationsStatus, ReplicationConfiguration,
ReplicationRule, ReplicationRuleStatus, SourceSelectionCriteria,
DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication, DeleteReplicationStatus, Destination,
ExistingObjectReplication, ExistingObjectReplicationStatus, ReplicaModifications, ReplicaModificationsStatus,
ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus, SourceSelectionCriteria,
};
use std::collections::HashMap;
use time::{Duration, OffsetDateTime};
@@ -474,6 +497,7 @@ mod tests {
.expect("replica delete marker should be forwarded to downstream targets");
let pending = format!("{arn}=PENDING;");
assert_eq!(state.replica_status, ReplicationStatusType::Replica);
assert_eq!(state.replication_status_internal.as_deref(), Some(pending.as_str()));
assert_eq!(state.replicate_decision_str, format!("{arn}=true;false;{arn};"));
assert!(state.targets.contains_key(arn));
@@ -500,9 +524,13 @@ mod tests {
#[test]
fn delete_replication_state_tracks_delete_marker_version_purges() {
let arn = "arn:aws:s3:::target-bucket";
let mut rule = delete_replication_rule(arn, false);
rule.delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
});
let config = ReplicationConfiguration {
role: arn.to_string(),
rules: vec![delete_replication_rule(arn, false)],
rules: vec![rule],
};
let source = ReplicationDeleteStateSource {
name: "test/object.txt".to_string(),
@@ -513,9 +541,10 @@ mod tests {
};
let state = delete_replication_state_from_config(&config, &source)
.expect("delete-marker version purge should honor delete-marker replication rules");
.expect("delete-marker version purge should honor delete replication rules");
let pending = format!("{arn}=PENDING;");
assert_eq!(state.replica_status, ReplicationStatusType::Empty);
assert_eq!(state.version_purge_status_internal.as_deref(), Some(pending.as_str()));
assert_eq!(state.replicate_decision_str, format!("{arn}=true;false;{arn};"));
assert!(state.purge_targets.contains_key(arn));
@@ -534,13 +563,13 @@ mod tests {
}
#[test]
fn delete_replication_schedule_keeps_marker_and_version_purges() {
fn delete_replication_schedule_uses_pending_state_from_current_delete_rules() {
assert!(should_schedule_delete_replication(ReplicationDeleteScheduleInput {
replication_request: false,
version_id_requested: true,
source_delete_marker: true,
source_replication_status: &ReplicationStatusType::Completed,
source_version_purge_status: &VersionPurgeStatusType::Empty,
source_replication_status: &ReplicationStatusType::Empty,
source_version_purge_status: &VersionPurgeStatusType::Pending,
deleted_delete_marker_version: true,
}));
assert!(should_schedule_delete_replication(ReplicationDeleteScheduleInput {
@@ -548,19 +577,54 @@ mod tests {
version_id_requested: true,
source_delete_marker: false,
source_replication_status: &ReplicationStatusType::Completed,
source_version_purge_status: &VersionPurgeStatusType::Empty,
source_version_purge_status: &VersionPurgeStatusType::Pending,
deleted_delete_marker_version: false,
}));
assert!(should_schedule_delete_replication(ReplicationDeleteScheduleInput {
replication_request: false,
version_id_requested: false,
source_delete_marker: false,
source_replication_status: &ReplicationStatusType::Empty,
source_version_purge_status: &VersionPurgeStatusType::Pending,
source_replication_status: &ReplicationStatusType::Pending,
source_version_purge_status: &VersionPurgeStatusType::Empty,
deleted_delete_marker_version: false,
}));
}
#[test]
fn delete_replication_schedule_skips_non_pending_states() {
for replication_status in [
ReplicationStatusType::Empty,
ReplicationStatusType::Replica,
ReplicationStatusType::Completed,
ReplicationStatusType::CompletedLegacy,
ReplicationStatusType::Failed,
] {
assert!(!should_schedule_delete_replication(ReplicationDeleteScheduleInput {
replication_request: false,
version_id_requested: false,
source_delete_marker: false,
source_replication_status: &replication_status,
source_version_purge_status: &VersionPurgeStatusType::Empty,
deleted_delete_marker_version: false,
}));
}
for version_purge_status in [
VersionPurgeStatusType::Empty,
VersionPurgeStatusType::Complete,
VersionPurgeStatusType::Failed,
] {
assert!(!should_schedule_delete_replication(ReplicationDeleteScheduleInput {
replication_request: false,
version_id_requested: true,
source_delete_marker: true,
source_replication_status: &ReplicationStatusType::Completed,
source_version_purge_status: &version_purge_status,
deleted_delete_marker_version: true,
}));
}
}
#[test]
fn delete_replication_version_id_splits_marker_creation_and_purge() {
let version_id = Uuid::new_v4();
@@ -569,6 +633,32 @@ mod tests {
assert_eq!(delete_replication_version_id(true, Some(version_id), true), Some(version_id));
}
#[test]
fn delete_replication_parts_fail_closed_without_purge_version() {
assert_eq!(delete_replication_parts(true, None, true), None);
for version_id in [Uuid::nil(), Uuid::new_v4()] {
assert_eq!(
delete_replication_parts(true, Some(version_id), true),
Some(ReplicationDeleteParts {
delete_marker: false,
version_id: Some(version_id),
delete_marker_version_id: None,
})
);
}
let marker_version_id = Uuid::new_v4();
assert_eq!(
delete_replication_parts(true, Some(marker_version_id), false),
Some(ReplicationDeleteParts {
delete_marker: true,
version_id: None,
delete_marker_version_id: Some(marker_version_id),
})
);
}
#[test]
fn delete_replication_source_selection_prefers_existing_marker_source_only_for_replica_requests() {
assert!(should_use_existing_delete_replication_source(true, true, true));
+25 -13
View File
@@ -17,8 +17,8 @@ use std::any::Any;
use crate::storage_api::DeletedObject;
use crate::{
DeletedObjectReplicationInfo, MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_HEAL, REPLICATE_HEAL_DELETE,
ReplicateObjectInfo, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision,
VersionPurgeStatusType,
ReplicateObjectInfo, ReplicationDeleteParts, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation,
ResyncDecision, VersionPurgeStatusType, delete_replication_parts,
};
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
@@ -109,7 +109,11 @@ pub fn replication_heal_queue_action(roi: &mut ReplicateObjectInfo) -> Replicati
}
if roi.delete_marker || !roi.version_purge_status.is_empty() {
let delete_info = heal_deleted_object_replication_info(roi);
let Some(parts) = delete_replication_parts(roi.delete_marker, roi.version_id, !roi.version_purge_status.is_empty())
else {
return ReplicationHealQueueAction::Skip;
};
let delete_info = heal_deleted_object_replication_info(roi, parts);
if is_pending_or_failed_object_heal(roi) || is_pending_or_failed_version_purge(roi) {
return ReplicationHealQueueAction::QueueDelete(delete_info);
@@ -144,21 +148,18 @@ pub fn replication_heal_queue_action(roi: &mut ReplicateObjectInfo) -> Replicati
ReplicationHealQueueAction::Skip
}
fn heal_deleted_object_replication_info(roi: &ReplicateObjectInfo) -> DeletedObjectReplicationInfo {
let (version_id, delete_marker_version_id) = if roi.version_purge_status.is_empty() {
(None, roi.version_id)
} else {
(roi.version_id, None)
};
fn heal_deleted_object_replication_info(
roi: &ReplicateObjectInfo,
parts: ReplicationDeleteParts,
) -> DeletedObjectReplicationInfo {
DeletedObjectReplicationInfo {
delete_object: DeletedObject {
object_name: roi.name.clone(),
delete_marker_version_id,
version_id,
delete_marker_version_id: parts.delete_marker_version_id,
version_id: parts.version_id,
replication_state: roi.replication_state.clone(),
delete_marker_mtime: roi.mod_time,
delete_marker: roi.delete_marker,
delete_marker: parts.delete_marker,
..Default::default()
},
bucket: roi.bucket.clone(),
@@ -399,6 +400,7 @@ mod tests {
let version_id = Uuid::new_v4();
let mut roi = replicate_object_info(ReplicationStatusType::Completed);
roi.version_id = Some(version_id);
roi.delete_marker = true;
roi.version_purge_status = VersionPurgeStatusType::Pending;
let action = replication_heal_queue_action(&mut roi);
@@ -408,6 +410,16 @@ mod tests {
};
assert_eq!(delete_info.delete_object.version_id, Some(version_id));
assert_eq!(delete_info.delete_object.delete_marker_version_id, None);
assert!(!delete_info.delete_object.delete_marker);
}
#[test]
fn heal_queue_action_skips_version_purge_without_version_id() {
let mut roi = replicate_object_info(ReplicationStatusType::Completed);
roi.delete_marker = true;
roi.version_purge_status = VersionPurgeStatusType::Pending;
assert!(matches!(replication_heal_queue_action(&mut roi), ReplicationHealQueueAction::Skip));
}
#[test]