mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-20 19:42:17 +00:00
fix(replication): honor disabled version deletes
This commit is contained in:
@@ -18,7 +18,9 @@ use crate::rule::ReplicationRuleExt as _;
|
||||
use s3s::dto::DeleteMarkerReplicationStatus;
|
||||
use s3s::dto::DeleteReplicationStatus;
|
||||
use s3s::dto::Destination;
|
||||
use s3s::dto::{ExistingObjectReplicationStatus, ReplicationConfiguration, ReplicationRuleStatus, ReplicationRules};
|
||||
use s3s::dto::{
|
||||
ExistingObjectReplicationStatus, ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus, ReplicationRules,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashSet;
|
||||
use uuid::Uuid;
|
||||
@@ -45,6 +47,86 @@ pub trait ReplicationConfigurationExt {
|
||||
fn filter_target_arns(&self, obj: &ObjectOpts) -> Vec<String>;
|
||||
}
|
||||
|
||||
pub fn delete_replication_target_arns(config: &ReplicationConfiguration, object_name: &str, replica: bool) -> HashSet<String> {
|
||||
let role = config.role.trim();
|
||||
if !role.is_empty() && active_replication_rule_destination_arns(config).len() > 1 {
|
||||
return HashSet::new();
|
||||
}
|
||||
|
||||
let mut targets = HashSet::new();
|
||||
let mut targets_with_unknown_tags = HashSet::new();
|
||||
for rule in &config.rules {
|
||||
if rule.status == ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED)
|
||||
|| !object_name.starts_with(rule.prefix())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let arn = if role.is_empty() {
|
||||
rule.destination.bucket.trim()
|
||||
} else {
|
||||
role
|
||||
};
|
||||
if arn.is_empty() {
|
||||
continue;
|
||||
}
|
||||
targets.insert(arn.to_string());
|
||||
if rule.filter.as_ref().is_some_and(|filter| {
|
||||
filter.tag.is_some()
|
||||
|| filter
|
||||
.and
|
||||
.as_ref()
|
||||
.and_then(|and| and.tags.as_ref())
|
||||
.is_some_and(|tags| !tags.is_empty())
|
||||
}) {
|
||||
targets_with_unknown_tags.insert(arn.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
targets
|
||||
.into_iter()
|
||||
.filter(|arn| !targets_with_unknown_tags.contains(arn))
|
||||
.filter(|arn| {
|
||||
config.replicate(&ObjectOpts {
|
||||
name: object_name.to_string(),
|
||||
target_arn: arn.clone(),
|
||||
version_id: Some(Uuid::nil()),
|
||||
delete_marker: true,
|
||||
op_type: ReplicationType::Delete,
|
||||
replica,
|
||||
..Default::default()
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn rule_replicates(rule: &ReplicationRule, obj: &ObjectOpts) -> bool {
|
||||
if let Some(status) = &rule.existing_object_replication
|
||||
&& obj.existing_object
|
||||
&& status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::DISABLED)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if obj.op_type == ReplicationType::Delete {
|
||||
if !rule.metadata_replicate(obj) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if obj.version_id.is_some() {
|
||||
return rule
|
||||
.delete_replication
|
||||
.clone()
|
||||
.is_some_and(|d| d.status == DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED));
|
||||
}
|
||||
|
||||
return rule.delete_marker_replication.clone().is_some_and(|d| {
|
||||
d.status == Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED))
|
||||
});
|
||||
}
|
||||
|
||||
rule.metadata_replicate(obj)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ReplicationTargetValidationError {
|
||||
RoleWithMultipleDestinations,
|
||||
@@ -204,37 +286,7 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(status) = &rule.existing_object_replication
|
||||
&& obj.existing_object
|
||||
&& status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::DISABLED)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if obj.op_type == ReplicationType::Delete {
|
||||
if !rule.metadata_replicate(obj) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if obj.version_id.is_some() {
|
||||
if obj.delete_marker {
|
||||
return rule.delete_marker_replication.clone().is_some_and(|d| {
|
||||
d.status == Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED))
|
||||
});
|
||||
}
|
||||
return rule
|
||||
.delete_replication
|
||||
.clone()
|
||||
.is_some_and(|d| d.status == DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED));
|
||||
} else {
|
||||
return rule.delete_marker_replication.clone().is_some_and(|d| {
|
||||
d.status == Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED))
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Regular object/metadata replication
|
||||
return rule.metadata_replicate(obj);
|
||||
return rule_replicates(rule, obj);
|
||||
}
|
||||
false
|
||||
}
|
||||
@@ -305,7 +357,10 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use s3s::dto::{DeleteMarkerReplication, Destination, ExistingObjectReplication, ReplicationRule};
|
||||
use s3s::dto::{
|
||||
DeleteMarkerReplication, DeleteReplication, Destination, ExistingObjectReplication, ReplicationRule,
|
||||
ReplicationRuleFilter, Tag,
|
||||
};
|
||||
|
||||
fn replication_rule(id: &str, arn: &str) -> ReplicationRule {
|
||||
ReplicationRule {
|
||||
@@ -327,6 +382,37 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_target_arns_uses_highest_priority_matching_rule() {
|
||||
let arn = "arn:target:a";
|
||||
let mut lower_priority = replication_rule("lower", arn);
|
||||
lower_priority.priority = Some(1);
|
||||
lower_priority.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
let mut higher_priority = replication_rule("higher", arn);
|
||||
higher_priority.priority = Some(2);
|
||||
higher_priority.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
|
||||
});
|
||||
let mut config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![lower_priority, higher_priority],
|
||||
};
|
||||
|
||||
let targets = delete_replication_target_arns(&config, "object", false);
|
||||
|
||||
assert!(targets.is_empty(), "the higher-priority disabled rule must suppress the target");
|
||||
|
||||
config.rules[0].delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
|
||||
});
|
||||
config.rules[1].delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
assert_eq!(delete_replication_target_arns(&config, "object", false), HashSet::from([arn.to_string()]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filter_target_arns_uses_role_when_role_is_present() {
|
||||
let config = ReplicationConfiguration {
|
||||
@@ -337,15 +423,151 @@ mod tests {
|
||||
],
|
||||
};
|
||||
|
||||
let arns = config.filter_target_arns(&ObjectOpts {
|
||||
let opts = ObjectOpts {
|
||||
name: "object".to_string(),
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
});
|
||||
};
|
||||
let arns = config.filter_target_arns(&opts);
|
||||
|
||||
assert_eq!(arns, vec!["arn:legacy:target".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_target_arns_uses_role_when_role_is_present() {
|
||||
let mut rule = replication_rule("rule", "arn:target:a");
|
||||
rule.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
let config = ReplicationConfiguration {
|
||||
role: " arn:legacy:target ".to_string(),
|
||||
rules: vec![rule],
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
delete_replication_target_arns(&config, "object", false),
|
||||
HashSet::from(["arn:legacy:target".to_string()])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_target_arns_ignores_disjoint_prefix_rules() {
|
||||
let arn = "arn:target:a";
|
||||
let mut matching = replication_rule("matching", arn);
|
||||
matching.prefix = None;
|
||||
matching.filter = Some(ReplicationRuleFilter {
|
||||
prefix: Some("logs/".to_string()),
|
||||
..Default::default()
|
||||
});
|
||||
matching.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
|
||||
});
|
||||
let mut unrelated = replication_rule("unrelated", arn);
|
||||
unrelated.prefix = None;
|
||||
unrelated.filter = Some(ReplicationRuleFilter {
|
||||
prefix: Some("archive/".to_string()),
|
||||
..Default::default()
|
||||
});
|
||||
unrelated.priority = Some(2);
|
||||
unrelated.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
let mut config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![matching, unrelated],
|
||||
};
|
||||
|
||||
assert!(delete_replication_target_arns(&config, "logs/object", false).is_empty());
|
||||
|
||||
config.rules[0].delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
config.rules[1].delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
|
||||
});
|
||||
assert_eq!(
|
||||
delete_replication_target_arns(&config, "logs/object", false),
|
||||
HashSet::from([arn.to_string()])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_target_arns_fails_closed_for_unknown_tag_rules() {
|
||||
let arn = "arn:target:a";
|
||||
let mut known = replication_rule("known", arn);
|
||||
known.priority = Some(2);
|
||||
known.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
let mut unknown = replication_rule("unknown", arn);
|
||||
unknown.priority = Some(1);
|
||||
unknown.prefix = None;
|
||||
unknown.filter = Some(ReplicationRuleFilter {
|
||||
tag: Some(Tag {
|
||||
key: Some("env".to_string()),
|
||||
value: Some("prod".to_string()),
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
unknown.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![known, unknown],
|
||||
};
|
||||
|
||||
assert!(delete_replication_target_arns(&config, "object", false).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_target_arns_reuses_full_destination_rule_order() {
|
||||
let arn = "arn:target:a";
|
||||
let mut first = replication_rule("first", arn);
|
||||
first.priority = Some(1);
|
||||
first.destination.account = Some("account-a".to_string());
|
||||
first.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
|
||||
});
|
||||
let mut second = replication_rule("second", arn);
|
||||
second.priority = Some(2);
|
||||
second.destination.account = Some("account-b".to_string());
|
||||
second.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![first, second],
|
||||
};
|
||||
let opts = ObjectOpts {
|
||||
name: "object".to_string(),
|
||||
target_arn: arn.to_string(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
delete_marker: true,
|
||||
op_type: ReplicationType::Delete,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(!config.replicate(&opts));
|
||||
assert!(delete_replication_target_arns(&config, "object", false).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_target_arns_rejects_role_with_multiple_destinations() {
|
||||
let mut first = replication_rule("first", "arn:target:a");
|
||||
first.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
let mut second = replication_rule("second", "arn:target:b");
|
||||
second.delete_replication = first.delete_replication.clone();
|
||||
let config = ReplicationConfiguration {
|
||||
role: "arn:legacy:target".to_string(),
|
||||
rules: vec![first, second],
|
||||
};
|
||||
|
||||
assert!(delete_replication_target_arns(&config, "object", false).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filter_target_arns_falls_back_to_role_when_destination_is_empty() {
|
||||
let config = ReplicationConfiguration {
|
||||
@@ -605,4 +827,42 @@ mod tests {
|
||||
"highest-priority rule disables delete-marker replication, so the delete marker must not replicate"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_marker_version_purge_requires_delete_replication() {
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
let mut config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![delete_marker_rule("delete-markers-only", arn, "", 1, true)],
|
||||
};
|
||||
let opts = ObjectOpts {
|
||||
name: "object.txt".to_string(),
|
||||
op_type: ReplicationType::Delete,
|
||||
delete_marker: true,
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
!config.replicate(&opts),
|
||||
"permanently deleting a delete-marker version must not use the delete-marker replication setting"
|
||||
);
|
||||
|
||||
config.rules[0].delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
|
||||
});
|
||||
assert!(
|
||||
!config.replicate(&opts),
|
||||
"an explicitly disabled permanent-delete setting must not replicate"
|
||||
);
|
||||
|
||||
config.rules[0].delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
|
||||
assert!(
|
||||
config.replicate(&opts),
|
||||
"permanently deleting a delete-marker version should replicate when delete replication is enabled"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
use std::any::Any;
|
||||
|
||||
use crate::storage_api::DeletedObject;
|
||||
use crate::{MrfOpKind, MrfReplicateEntry, ReplicationType, ReplicationWorkerOperation};
|
||||
use crate::{MrfOpKind, MrfReplicateEntry, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation};
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct DeletedObjectReplicationInfo {
|
||||
@@ -42,6 +42,11 @@ impl ReplicationWorkerOperation for DeletedObjectReplicationInfo {
|
||||
op: MrfOpKind::Delete,
|
||||
delete_marker_version_id: self.delete_object.delete_marker_version_id,
|
||||
delete_marker: self.delete_object.delete_marker,
|
||||
replica: self
|
||||
.delete_object
|
||||
.replication_state
|
||||
.as_ref()
|
||||
.is_some_and(|state| state.replica_status == ReplicationStatusType::Replica),
|
||||
// Persist the original delete-marker mtime as Unix nanoseconds so replay after a
|
||||
// restart stamps the replica with the source timestamp rather than the replay time
|
||||
// (backlog#867). None when unknown; replay then falls back to the current time.
|
||||
@@ -85,14 +90,22 @@ pub fn is_retryable_delete_replication_head_error(is_not_found: bool, code: Opti
|
||||
!(is_not_found || matches!(code, Some("MethodNotAllowed" | "405")))
|
||||
}
|
||||
|
||||
pub fn version_purge_target_missing(is_not_found: bool, code: Option<&str>, raw_status: Option<u16>) -> bool {
|
||||
if matches!(code, Some("MethodNotAllowed" | "405")) || raw_status == Some(405) {
|
||||
return false;
|
||||
}
|
||||
|
||||
is_not_found || matches!(code, Some("NoSuchVersion" | "NoSuchKey" | "NotFound" | "404")) || raw_status == Some(404)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
DeletedObjectReplicationInfo, is_retryable_delete_replication_head_error, is_version_delete_replication,
|
||||
should_retry_delete_marker_purge,
|
||||
should_retry_delete_marker_purge, version_purge_target_missing,
|
||||
};
|
||||
use crate::storage_api::DeletedObject;
|
||||
use crate::{MrfOpKind, ReplicationType, ReplicationWorkerOperation};
|
||||
use crate::{MrfOpKind, ReplicationState, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[test]
|
||||
@@ -109,6 +122,10 @@ mod tests {
|
||||
delete_marker_version_id: Some(delete_marker_version_id),
|
||||
delete_marker: true,
|
||||
delete_marker_mtime: Some(mtime),
|
||||
replication_state: Some(ReplicationState {
|
||||
replica_status: ReplicationStatusType::Replica,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
@@ -122,6 +139,7 @@ mod tests {
|
||||
assert_eq!(entry.delete_marker_version_id, Some(delete_marker_version_id));
|
||||
assert_eq!(entry.op, MrfOpKind::Delete);
|
||||
assert!(entry.delete_marker);
|
||||
assert!(entry.replica);
|
||||
// The original mtime must be persisted (as Unix nanos) so replay keeps the source
|
||||
// timestamp instead of stamping the replica with the replay time (backlog#867).
|
||||
assert_eq!(
|
||||
@@ -212,4 +230,14 @@ mod tests {
|
||||
assert!(!is_retryable_delete_replication_head_error(true, Some("NoSuchKey")));
|
||||
assert!(is_retryable_delete_replication_head_error(false, Some("AccessDenied")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_purge_target_missing_requires_not_found() {
|
||||
assert!(version_purge_target_missing(true, Some("NoSuchVersion"), Some(404)));
|
||||
assert!(version_purge_target_missing(false, Some("NoSuchVersion"), None));
|
||||
assert!(version_purge_target_missing(false, None, Some(404)));
|
||||
assert!(!version_purge_target_missing(false, None, None));
|
||||
assert!(!version_purge_target_missing(true, Some("MethodNotAllowed"), Some(405)));
|
||||
assert!(!version_purge_target_missing(true, Some("405"), None));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -587,6 +587,11 @@ pub struct MrfReplicateEntry {
|
||||
#[serde(rename = "deleteMarker", default)]
|
||||
pub delete_marker: bool,
|
||||
|
||||
// For delete entries: whether the operation originated from a replica.
|
||||
// Old files lack this field and therefore default to a local-source delete.
|
||||
#[serde(rename = "replica", default)]
|
||||
pub replica: bool,
|
||||
|
||||
// For delete entries: the original delete-marker mtime, persisted as Unix nanoseconds so
|
||||
// replay stamps replicas with the source timestamp instead of the replay time. Old files
|
||||
// lack this key; default=None means "unknown", and replay falls back to the current time
|
||||
@@ -798,6 +803,7 @@ impl ReplicationWorkerOperation for ReplicateObjectInfo {
|
||||
op: MrfOpKind::Object,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
}
|
||||
}
|
||||
@@ -852,6 +858,7 @@ impl ReplicateObjectInfo {
|
||||
op: MrfOpKind::Object,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,11 +30,12 @@ pub mod tagging;
|
||||
|
||||
pub use config::{
|
||||
ObjectOpts, ReplicationConfigurationExt, ReplicationTargetValidationError, active_replication_rule_destination_arns,
|
||||
replication_target_arns, should_remove_replication_target, validate_replication_config_target_arns,
|
||||
delete_replication_target_arns, replication_target_arns, should_remove_replication_target,
|
||||
validate_replication_config_target_arns,
|
||||
};
|
||||
pub use delete::{
|
||||
DeletedObjectReplicationInfo, is_retryable_delete_replication_head_error, is_version_delete_replication,
|
||||
should_retry_delete_marker_purge,
|
||||
should_retry_delete_marker_purge, version_purge_target_missing,
|
||||
};
|
||||
pub use filemeta::{
|
||||
REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL, REPLICATE_HEAL_DELETE, REPLICATE_INCOMING,
|
||||
@@ -54,10 +55,11 @@ pub use object::{
|
||||
replication_etags_match, target_is_newer_than_source_null_version,
|
||||
};
|
||||
pub use operation::{
|
||||
MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteSource, ReplicationDeleteStateSource,
|
||||
ReplicationResyncTargetObject, delete_replication_missing_source_decision, delete_replication_object_opts,
|
||||
delete_replication_state_from_config, delete_replication_version_id, heal_uses_delete_replication_path, is_ssec_encrypted,
|
||||
resync_target_for_object, should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
MustReplicateOptions, ReplicationDeleteParts, ReplicationDeleteScheduleInput, ReplicationDeleteSource,
|
||||
ReplicationDeleteStateSource, ReplicationResyncTargetObject, delete_replication_missing_source_decision,
|
||||
delete_replication_object_opts, delete_replication_parts, delete_replication_state_from_config,
|
||||
delete_replication_version_id, heal_uses_delete_replication_path, is_ssec_encrypted, resync_target_for_object,
|
||||
should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
should_use_existing_delete_replication_source,
|
||||
};
|
||||
pub use queue::{
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use byteorder::{ByteOrder, LittleEndian};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{Error, Result};
|
||||
|
||||
@@ -21,6 +22,31 @@ pub use crate::filemeta::{MrfOpKind, MrfReplicateEntry};
|
||||
pub const MRF_META_FORMAT: u16 = 1;
|
||||
pub const MRF_META_VERSION: u16 = 1;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct MrfDeleteParts {
|
||||
pub version_id: Option<Uuid>,
|
||||
pub delete_marker_version_id: Option<Uuid>,
|
||||
pub delete_marker: bool,
|
||||
}
|
||||
|
||||
impl MrfReplicateEntry {
|
||||
pub fn delete_parts_for_replay(&self) -> Option<MrfDeleteParts> {
|
||||
match (self.version_id, self.delete_marker_version_id) {
|
||||
(Some(version_id), None) => Some(MrfDeleteParts {
|
||||
version_id: Some(version_id),
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
}),
|
||||
(None, Some(delete_marker_version_id)) => Some(MrfDeleteParts {
|
||||
version_id: None,
|
||||
delete_marker_version_id: Some(delete_marker_version_id),
|
||||
delete_marker: self.delete_marker,
|
||||
}),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn encode_mrf_file(entries: &[MrfReplicateEntry]) -> Result<Vec<u8>> {
|
||||
let payload = rmp_serde::to_vec_named(entries).map_err(|e| Error::Other(e.to_string()))?;
|
||||
let mut data = Vec::with_capacity(4 + payload.len());
|
||||
@@ -54,7 +80,6 @@ pub fn decode_mrf_file(data: &[u8]) -> Result<Vec<MrfReplicateEntry>> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[test]
|
||||
fn mrf_file_round_trips_object_and_delete_entries() {
|
||||
@@ -70,6 +95,7 @@ mod tests {
|
||||
op: MrfOpKind::Object,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
},
|
||||
MrfReplicateEntry {
|
||||
@@ -81,6 +107,7 @@ mod tests {
|
||||
op: MrfOpKind::Delete,
|
||||
delete_marker_version_id: Some(del_vid),
|
||||
delete_marker: true,
|
||||
replica: true,
|
||||
delete_marker_mtime: Some(1_705_312_200_123_456_789),
|
||||
},
|
||||
];
|
||||
@@ -95,6 +122,7 @@ mod tests {
|
||||
assert_eq!(decoded[1].delete_marker_version_id, Some(del_vid));
|
||||
assert_eq!(decoded[1].op, MrfOpKind::Delete);
|
||||
assert!(decoded[1].delete_marker);
|
||||
assert!(decoded[1].replica);
|
||||
assert_eq!(
|
||||
decoded[1].delete_marker_mtime,
|
||||
Some(1_705_312_200_123_456_789),
|
||||
@@ -102,6 +130,46 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_version_purge_replay_clears_delete_marker_creation() {
|
||||
let entry = MrfReplicateEntry {
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
retry_count: 0,
|
||||
size: 0,
|
||||
delete_marker: true,
|
||||
op: MrfOpKind::Delete,
|
||||
delete_marker_version_id: None,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
};
|
||||
let encoded = encode_mrf_file(&[entry]).expect("legacy MRF entry should encode");
|
||||
let decoded = decode_mrf_file(&encoded).expect("legacy MRF entry should decode");
|
||||
|
||||
assert_eq!(decoded[0].delete_parts_for_replay().map(|parts| parts.delete_marker), Some(false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_delete_id_shapes_fail_closed_on_replay() {
|
||||
for (version_id, delete_marker_version_id) in [(None, None), (Some(Uuid::new_v4()), Some(Uuid::new_v4()))] {
|
||||
let entry = MrfReplicateEntry {
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
version_id,
|
||||
retry_count: 0,
|
||||
size: 0,
|
||||
op: MrfOpKind::Delete,
|
||||
delete_marker_version_id,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
};
|
||||
|
||||
assert_eq!(entry.delete_parts_for_replay(), None);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_legacy_file_without_op_decodes_as_object() {
|
||||
let mut payload = Vec::new();
|
||||
@@ -129,6 +197,7 @@ mod tests {
|
||||
assert_eq!(decoded[0].retry_count, 2);
|
||||
assert_eq!(decoded[0].size, 100);
|
||||
assert_eq!(decoded[0].op, MrfOpKind::Object);
|
||||
assert!(!decoded[0].replica);
|
||||
// Old files lack the deleteMarkerMtime key; it must default to None so replay keeps the
|
||||
// pre-#867 fallback to the current time.
|
||||
assert_eq!(decoded[0].delete_marker_mtime, None);
|
||||
|
||||
@@ -151,6 +151,11 @@ pub fn delete_replication_state_from_config(
|
||||
|
||||
let pending_status = decision.pending_status();
|
||||
let mut state = ReplicationState {
|
||||
replica_status: if source.replica {
|
||||
ReplicationStatusType::Replica
|
||||
} else {
|
||||
ReplicationStatusType::Empty
|
||||
},
|
||||
replicate_decision_str: decision.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
@@ -174,11 +179,31 @@ pub struct ReplicationDeleteScheduleInput<'a> {
|
||||
pub deleted_delete_marker_version: bool,
|
||||
}
|
||||
|
||||
fn delete_version_purge_source_status(status: &ReplicationStatusType) -> bool {
|
||||
status == &ReplicationStatusType::Replica
|
||||
|| status == &ReplicationStatusType::Pending
|
||||
|| status == &ReplicationStatusType::Completed
|
||||
|| status == &ReplicationStatusType::Failed
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ReplicationDeleteParts {
|
||||
pub delete_marker: bool,
|
||||
pub version_id: Option<Uuid>,
|
||||
pub delete_marker_version_id: Option<Uuid>,
|
||||
}
|
||||
|
||||
pub fn delete_replication_parts(
|
||||
source_delete_marker: bool,
|
||||
source_version_id: Option<Uuid>,
|
||||
version_purge: bool,
|
||||
) -> Option<ReplicationDeleteParts> {
|
||||
if version_purge {
|
||||
return source_version_id.map(|version_id| ReplicationDeleteParts {
|
||||
delete_marker: false,
|
||||
version_id: Some(version_id),
|
||||
delete_marker_version_id: None,
|
||||
});
|
||||
}
|
||||
|
||||
Some(ReplicationDeleteParts {
|
||||
delete_marker: source_delete_marker,
|
||||
version_id: None,
|
||||
delete_marker_version_id: source_version_id,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn should_schedule_delete_replication(input: ReplicationDeleteScheduleInput<'_>) -> bool {
|
||||
@@ -186,14 +211,11 @@ pub fn should_schedule_delete_replication(input: ReplicationDeleteScheduleInput<
|
||||
return false;
|
||||
}
|
||||
|
||||
if input.version_id_requested && !input.deleted_delete_marker_version && !input.source_delete_marker {
|
||||
return delete_version_purge_source_status(input.source_replication_status);
|
||||
if input.version_id_requested {
|
||||
return input.source_version_purge_status == &VersionPurgeStatusType::Pending;
|
||||
}
|
||||
|
||||
input.source_replication_status == &ReplicationStatusType::Replica
|
||||
|| input.source_replication_status == &ReplicationStatusType::Pending
|
||||
|| input.source_version_purge_status == &VersionPurgeStatusType::Pending
|
||||
|| (input.deleted_delete_marker_version && input.source_replication_status == &ReplicationStatusType::Completed)
|
||||
input.source_replication_status == &ReplicationStatusType::Pending
|
||||
}
|
||||
|
||||
pub fn delete_replication_version_id(
|
||||
@@ -312,19 +334,20 @@ pub fn resync_target_for_object(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteSource, ReplicationDeleteStateSource,
|
||||
ReplicationResyncTargetObject, delete_replication_missing_source_decision, delete_replication_object_opts,
|
||||
delete_replication_state_from_config, delete_replication_version_id, heal_uses_delete_replication_path,
|
||||
is_ssec_encrypted, resync_target_for_object, should_schedule_delete_replication,
|
||||
should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
|
||||
MustReplicateOptions, ReplicationDeleteParts, ReplicationDeleteScheduleInput, ReplicationDeleteSource,
|
||||
ReplicationDeleteStateSource, ReplicationResyncTargetObject, delete_replication_missing_source_decision,
|
||||
delete_replication_object_opts, delete_replication_parts, delete_replication_state_from_config,
|
||||
delete_replication_version_id, heal_uses_delete_replication_path, is_ssec_encrypted, resync_target_for_object,
|
||||
should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
should_use_existing_delete_replication_source,
|
||||
};
|
||||
use crate::http::{AMZ_BUCKET_REPLICATION_STATUS, SSEC_ALGORITHM_HEADER};
|
||||
use crate::storage_api::ObjectToDelete;
|
||||
use crate::{ReplicationStatusType, ReplicationType, VersionPurgeStatusType, target_reset_header};
|
||||
use s3s::dto::{
|
||||
DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ExistingObjectReplication,
|
||||
ExistingObjectReplicationStatus, ReplicaModifications, ReplicaModificationsStatus, ReplicationConfiguration,
|
||||
ReplicationRule, ReplicationRuleStatus, SourceSelectionCriteria,
|
||||
DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication, DeleteReplicationStatus, Destination,
|
||||
ExistingObjectReplication, ExistingObjectReplicationStatus, ReplicaModifications, ReplicaModificationsStatus,
|
||||
ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus, SourceSelectionCriteria,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use time::{Duration, OffsetDateTime};
|
||||
@@ -474,6 +497,7 @@ mod tests {
|
||||
.expect("replica delete marker should be forwarded to downstream targets");
|
||||
let pending = format!("{arn}=PENDING;");
|
||||
|
||||
assert_eq!(state.replica_status, ReplicationStatusType::Replica);
|
||||
assert_eq!(state.replication_status_internal.as_deref(), Some(pending.as_str()));
|
||||
assert_eq!(state.replicate_decision_str, format!("{arn}=true;false;{arn};"));
|
||||
assert!(state.targets.contains_key(arn));
|
||||
@@ -500,9 +524,13 @@ mod tests {
|
||||
#[test]
|
||||
fn delete_replication_state_tracks_delete_marker_version_purges() {
|
||||
let arn = "arn:aws:s3:::target-bucket";
|
||||
let mut rule = delete_replication_rule(arn, false);
|
||||
rule.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
let config = ReplicationConfiguration {
|
||||
role: arn.to_string(),
|
||||
rules: vec![delete_replication_rule(arn, false)],
|
||||
rules: vec![rule],
|
||||
};
|
||||
let source = ReplicationDeleteStateSource {
|
||||
name: "test/object.txt".to_string(),
|
||||
@@ -513,9 +541,10 @@ mod tests {
|
||||
};
|
||||
|
||||
let state = delete_replication_state_from_config(&config, &source)
|
||||
.expect("delete-marker version purge should honor delete-marker replication rules");
|
||||
.expect("delete-marker version purge should honor delete replication rules");
|
||||
let pending = format!("{arn}=PENDING;");
|
||||
|
||||
assert_eq!(state.replica_status, ReplicationStatusType::Empty);
|
||||
assert_eq!(state.version_purge_status_internal.as_deref(), Some(pending.as_str()));
|
||||
assert_eq!(state.replicate_decision_str, format!("{arn}=true;false;{arn};"));
|
||||
assert!(state.purge_targets.contains_key(arn));
|
||||
@@ -534,13 +563,13 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_schedule_keeps_marker_and_version_purges() {
|
||||
fn delete_replication_schedule_uses_pending_state_from_current_delete_rules() {
|
||||
assert!(should_schedule_delete_replication(ReplicationDeleteScheduleInput {
|
||||
replication_request: false,
|
||||
version_id_requested: true,
|
||||
source_delete_marker: true,
|
||||
source_replication_status: &ReplicationStatusType::Completed,
|
||||
source_version_purge_status: &VersionPurgeStatusType::Empty,
|
||||
source_replication_status: &ReplicationStatusType::Empty,
|
||||
source_version_purge_status: &VersionPurgeStatusType::Pending,
|
||||
deleted_delete_marker_version: true,
|
||||
}));
|
||||
assert!(should_schedule_delete_replication(ReplicationDeleteScheduleInput {
|
||||
@@ -548,19 +577,54 @@ mod tests {
|
||||
version_id_requested: true,
|
||||
source_delete_marker: false,
|
||||
source_replication_status: &ReplicationStatusType::Completed,
|
||||
source_version_purge_status: &VersionPurgeStatusType::Empty,
|
||||
source_version_purge_status: &VersionPurgeStatusType::Pending,
|
||||
deleted_delete_marker_version: false,
|
||||
}));
|
||||
assert!(should_schedule_delete_replication(ReplicationDeleteScheduleInput {
|
||||
replication_request: false,
|
||||
version_id_requested: false,
|
||||
source_delete_marker: false,
|
||||
source_replication_status: &ReplicationStatusType::Empty,
|
||||
source_version_purge_status: &VersionPurgeStatusType::Pending,
|
||||
source_replication_status: &ReplicationStatusType::Pending,
|
||||
source_version_purge_status: &VersionPurgeStatusType::Empty,
|
||||
deleted_delete_marker_version: false,
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_schedule_skips_non_pending_states() {
|
||||
for replication_status in [
|
||||
ReplicationStatusType::Empty,
|
||||
ReplicationStatusType::Replica,
|
||||
ReplicationStatusType::Completed,
|
||||
ReplicationStatusType::CompletedLegacy,
|
||||
ReplicationStatusType::Failed,
|
||||
] {
|
||||
assert!(!should_schedule_delete_replication(ReplicationDeleteScheduleInput {
|
||||
replication_request: false,
|
||||
version_id_requested: false,
|
||||
source_delete_marker: false,
|
||||
source_replication_status: &replication_status,
|
||||
source_version_purge_status: &VersionPurgeStatusType::Empty,
|
||||
deleted_delete_marker_version: false,
|
||||
}));
|
||||
}
|
||||
|
||||
for version_purge_status in [
|
||||
VersionPurgeStatusType::Empty,
|
||||
VersionPurgeStatusType::Complete,
|
||||
VersionPurgeStatusType::Failed,
|
||||
] {
|
||||
assert!(!should_schedule_delete_replication(ReplicationDeleteScheduleInput {
|
||||
replication_request: false,
|
||||
version_id_requested: true,
|
||||
source_delete_marker: true,
|
||||
source_replication_status: &ReplicationStatusType::Completed,
|
||||
source_version_purge_status: &version_purge_status,
|
||||
deleted_delete_marker_version: true,
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_version_id_splits_marker_creation_and_purge() {
|
||||
let version_id = Uuid::new_v4();
|
||||
@@ -569,6 +633,32 @@ mod tests {
|
||||
assert_eq!(delete_replication_version_id(true, Some(version_id), true), Some(version_id));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_parts_fail_closed_without_purge_version() {
|
||||
assert_eq!(delete_replication_parts(true, None, true), None);
|
||||
|
||||
for version_id in [Uuid::nil(), Uuid::new_v4()] {
|
||||
assert_eq!(
|
||||
delete_replication_parts(true, Some(version_id), true),
|
||||
Some(ReplicationDeleteParts {
|
||||
delete_marker: false,
|
||||
version_id: Some(version_id),
|
||||
delete_marker_version_id: None,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
let marker_version_id = Uuid::new_v4();
|
||||
assert_eq!(
|
||||
delete_replication_parts(true, Some(marker_version_id), false),
|
||||
Some(ReplicationDeleteParts {
|
||||
delete_marker: true,
|
||||
version_id: None,
|
||||
delete_marker_version_id: Some(marker_version_id),
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_source_selection_prefers_existing_marker_source_only_for_replica_requests() {
|
||||
assert!(should_use_existing_delete_replication_source(true, true, true));
|
||||
|
||||
@@ -17,8 +17,8 @@ use std::any::Any;
|
||||
use crate::storage_api::DeletedObject;
|
||||
use crate::{
|
||||
DeletedObjectReplicationInfo, MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_HEAL, REPLICATE_HEAL_DELETE,
|
||||
ReplicateObjectInfo, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision,
|
||||
VersionPurgeStatusType,
|
||||
ReplicateObjectInfo, ReplicationDeleteParts, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation,
|
||||
ResyncDecision, VersionPurgeStatusType, delete_replication_parts,
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
@@ -109,7 +109,11 @@ pub fn replication_heal_queue_action(roi: &mut ReplicateObjectInfo) -> Replicati
|
||||
}
|
||||
|
||||
if roi.delete_marker || !roi.version_purge_status.is_empty() {
|
||||
let delete_info = heal_deleted_object_replication_info(roi);
|
||||
let Some(parts) = delete_replication_parts(roi.delete_marker, roi.version_id, !roi.version_purge_status.is_empty())
|
||||
else {
|
||||
return ReplicationHealQueueAction::Skip;
|
||||
};
|
||||
let delete_info = heal_deleted_object_replication_info(roi, parts);
|
||||
|
||||
if is_pending_or_failed_object_heal(roi) || is_pending_or_failed_version_purge(roi) {
|
||||
return ReplicationHealQueueAction::QueueDelete(delete_info);
|
||||
@@ -144,21 +148,18 @@ pub fn replication_heal_queue_action(roi: &mut ReplicateObjectInfo) -> Replicati
|
||||
ReplicationHealQueueAction::Skip
|
||||
}
|
||||
|
||||
fn heal_deleted_object_replication_info(roi: &ReplicateObjectInfo) -> DeletedObjectReplicationInfo {
|
||||
let (version_id, delete_marker_version_id) = if roi.version_purge_status.is_empty() {
|
||||
(None, roi.version_id)
|
||||
} else {
|
||||
(roi.version_id, None)
|
||||
};
|
||||
|
||||
fn heal_deleted_object_replication_info(
|
||||
roi: &ReplicateObjectInfo,
|
||||
parts: ReplicationDeleteParts,
|
||||
) -> DeletedObjectReplicationInfo {
|
||||
DeletedObjectReplicationInfo {
|
||||
delete_object: DeletedObject {
|
||||
object_name: roi.name.clone(),
|
||||
delete_marker_version_id,
|
||||
version_id,
|
||||
delete_marker_version_id: parts.delete_marker_version_id,
|
||||
version_id: parts.version_id,
|
||||
replication_state: roi.replication_state.clone(),
|
||||
delete_marker_mtime: roi.mod_time,
|
||||
delete_marker: roi.delete_marker,
|
||||
delete_marker: parts.delete_marker,
|
||||
..Default::default()
|
||||
},
|
||||
bucket: roi.bucket.clone(),
|
||||
@@ -399,6 +400,7 @@ mod tests {
|
||||
let version_id = Uuid::new_v4();
|
||||
let mut roi = replicate_object_info(ReplicationStatusType::Completed);
|
||||
roi.version_id = Some(version_id);
|
||||
roi.delete_marker = true;
|
||||
roi.version_purge_status = VersionPurgeStatusType::Pending;
|
||||
|
||||
let action = replication_heal_queue_action(&mut roi);
|
||||
@@ -408,6 +410,16 @@ mod tests {
|
||||
};
|
||||
assert_eq!(delete_info.delete_object.version_id, Some(version_id));
|
||||
assert_eq!(delete_info.delete_object.delete_marker_version_id, None);
|
||||
assert!(!delete_info.delete_object.delete_marker);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_queue_action_skips_version_purge_without_version_id() {
|
||||
let mut roi = replicate_object_info(ReplicationStatusType::Completed);
|
||||
roi.delete_marker = true;
|
||||
roi.version_purge_status = VersionPurgeStatusType::Pending;
|
||||
|
||||
assert!(matches!(replication_heal_queue_action(&mut roi), ReplicationHealQueueAction::Skip));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user