mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-18 18:46:17 +00:00
fix(replication): harden bucket replication correctness (#4116)
This commit is contained in:
@@ -82,9 +82,14 @@ use uuid::Uuid;
|
||||
|
||||
const DEFAULT_HEALTH_CHECK_DURATION: Duration = Duration::from_secs(5);
|
||||
const DEFAULT_HEALTH_CHECK_RELOAD_DURATION: Duration = Duration::from_secs(30 * 60);
|
||||
const REDACTED_CREDENTIAL: &str = "<redacted>";
|
||||
|
||||
pub static GLOBAL_BUCKET_TARGET_SYS: OnceLock<BucketTargetSys> = OnceLock::new();
|
||||
|
||||
fn replication_target_versioning_enabled(versioning: Option<&BucketVersioningStatus>) -> bool {
|
||||
matches!(versioning, Some(BucketVersioningStatus::Enabled))
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ArnTarget {
|
||||
pub client: Option<Arc<TargetClient>>,
|
||||
@@ -202,6 +207,7 @@ pub struct EpHealth {
|
||||
pub last_online: Option<OffsetDateTime>,
|
||||
pub last_hc_at: Option<OffsetDateTime>,
|
||||
pub offline_duration: Duration,
|
||||
pub offline_count: u64,
|
||||
pub latency: LatencyStat,
|
||||
}
|
||||
|
||||
@@ -214,11 +220,37 @@ impl Default for EpHealth {
|
||||
last_online: None,
|
||||
last_hc_at: None,
|
||||
offline_duration: Duration::from_secs(0),
|
||||
offline_count: 0,
|
||||
latency: LatencyStat::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn endpoint_health_key(url: &Url) -> String {
|
||||
let host = url.host_str().unwrap_or_default();
|
||||
match url.port() {
|
||||
Some(port) => format!("{host}:{port}"),
|
||||
None => host.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn update_endpoint_health(health: &mut EpHealth, online: bool, latency: Duration, now: OffsetDateTime) {
|
||||
let prev_online = health.online;
|
||||
health.online = online;
|
||||
health.last_hc_at = Some(now);
|
||||
health.latency.update(latency);
|
||||
|
||||
if online {
|
||||
health.last_online = Some(now);
|
||||
return;
|
||||
}
|
||||
|
||||
if prev_online {
|
||||
health.offline_count += 1;
|
||||
}
|
||||
health.offline_duration += latency;
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct BucketTargetSys {
|
||||
pub arn_remotes_map: Arc<RwLock<HashMap<String, ArnTarget>>>,
|
||||
@@ -246,9 +278,10 @@ impl BucketTargetSys {
|
||||
}
|
||||
|
||||
pub async fn is_offline(&self, url: &Url) -> bool {
|
||||
let key = endpoint_health_key(url);
|
||||
{
|
||||
let health_map = self.h_mutex.read().await;
|
||||
if let Some(health) = health_map.get(url.host_str().unwrap_or("")) {
|
||||
if let Some(health) = health_map.get(&key) {
|
||||
return !health.online;
|
||||
}
|
||||
}
|
||||
@@ -258,15 +291,16 @@ impl BucketTargetSys {
|
||||
}
|
||||
|
||||
pub async fn mark_offline(&self, url: &Url) {
|
||||
let key = endpoint_health_key(url);
|
||||
let mut health_map = self.h_mutex.write().await;
|
||||
if let Some(health) = health_map.get_mut(url.host_str().unwrap_or("")) {
|
||||
health.online = false;
|
||||
if let Some(health) = health_map.get_mut(&key) {
|
||||
update_endpoint_health(health, false, Duration::from_secs(0), OffsetDateTime::now_utc());
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn init_hc(&self, url: &Url) {
|
||||
let mut health_map = self.h_mutex.write().await;
|
||||
let host = url.host_str().unwrap_or("").to_string();
|
||||
let host = endpoint_health_key(url);
|
||||
health_map.insert(
|
||||
host.clone(),
|
||||
EpHealth {
|
||||
@@ -285,51 +319,35 @@ impl BucketTargetSys {
|
||||
|
||||
let endpoints = {
|
||||
let health_map = self.h_mutex.read().await;
|
||||
health_map.keys().cloned().collect::<Vec<_>>()
|
||||
health_map
|
||||
.iter()
|
||||
.map(|(endpoint, health)| (endpoint.clone(), health.scheme.clone()))
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
|
||||
for endpoint in endpoints {
|
||||
for (endpoint, scheme) in endpoints {
|
||||
// Perform health check
|
||||
let start = Instant::now();
|
||||
let online = self.check_endpoint_health(&endpoint).await;
|
||||
let online = self.check_endpoint_health(&endpoint, &scheme).await;
|
||||
let duration = start.elapsed();
|
||||
|
||||
{
|
||||
let mut health_map = self.h_mutex.write().await;
|
||||
if let Some(health) = health_map.get_mut(&endpoint) {
|
||||
let prev_online = health.online;
|
||||
health.online = online;
|
||||
health.last_hc_at = Some(OffsetDateTime::now_utc());
|
||||
health.latency.update(duration);
|
||||
|
||||
if online {
|
||||
health.last_online = Some(OffsetDateTime::now_utc());
|
||||
} else if prev_online {
|
||||
// Just went offline
|
||||
health.offline_duration += duration;
|
||||
}
|
||||
update_endpoint_health(health, online, duration, OffsetDateTime::now_utc());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn check_endpoint_health(&self, _endpoint: &str) -> bool {
|
||||
true
|
||||
// TODO: Health check
|
||||
|
||||
// // Simple health check implementation
|
||||
// // In a real implementation, you would make actual HTTP requests
|
||||
// match self
|
||||
// .hc_client
|
||||
// .get(format!("https://{}/rustfs/health/ready", endpoint))
|
||||
// .timeout(Duration::from_secs(3))
|
||||
// .send()
|
||||
// .await
|
||||
// {
|
||||
// Ok(response) => response.status().is_success(),
|
||||
// Err(_) => false,
|
||||
// }
|
||||
async fn check_endpoint_health(&self, endpoint: &str, scheme: &str) -> bool {
|
||||
let scheme = if scheme.is_empty() { "https" } else { scheme };
|
||||
let url = format!("{scheme}://{endpoint}/");
|
||||
match self.hc_client.head(url).timeout(Duration::from_secs(3)).send().await {
|
||||
Ok(response) => response.status().as_u16() < 500,
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn health_stats(&self) -> HashMap<String, EpHealth> {
|
||||
@@ -354,6 +372,7 @@ impl BucketTargetSys {
|
||||
avg: health.latency.avg,
|
||||
max: health.latency.peak,
|
||||
};
|
||||
target.offline_count = health.offline_count;
|
||||
}
|
||||
targets.push(target);
|
||||
}
|
||||
@@ -375,6 +394,7 @@ impl BucketTargetSys {
|
||||
avg: health.latency.avg,
|
||||
max: health.latency.peak,
|
||||
};
|
||||
target.offline_count = health.offline_count;
|
||||
}
|
||||
targets.push(target);
|
||||
}
|
||||
@@ -475,7 +495,7 @@ impl BucketTargetSys {
|
||||
error: e.to_string(),
|
||||
})?;
|
||||
|
||||
if versioning.is_none() {
|
||||
if !replication_target_versioning_enabled(versioning.as_ref()) {
|
||||
return Err(BucketTargetError::BucketRemoteTargetNotVersioned {
|
||||
bucket: target.target_bucket.to_string(),
|
||||
});
|
||||
@@ -1763,10 +1783,13 @@ impl fmt::Display for BucketTargetError {
|
||||
}
|
||||
BucketTargetError::RemoteTargetConnectionErr {
|
||||
bucket,
|
||||
access_key,
|
||||
access_key: _,
|
||||
error,
|
||||
} => {
|
||||
write!(f, "Connection error for bucket: {bucket}, access key: {access_key}, error: {error}")
|
||||
write!(
|
||||
f,
|
||||
"Connection error for bucket: {bucket}, access key: {REDACTED_CREDENTIAL}, error: {error}"
|
||||
)
|
||||
}
|
||||
BucketTargetError::BucketReplicationSourceNotVersioned { bucket } => {
|
||||
write!(f, "Replication source bucket not versioned: {bucket}")
|
||||
@@ -1795,6 +1818,76 @@ mod tests {
|
||||
use super::*;
|
||||
use rcgen::generate_simple_self_signed;
|
||||
|
||||
#[test]
|
||||
fn replication_target_versioning_enabled_requires_enabled_status() {
|
||||
let enabled = BucketVersioningStatus::Enabled;
|
||||
let suspended = BucketVersioningStatus::Suspended;
|
||||
|
||||
assert!(replication_target_versioning_enabled(Some(&enabled)));
|
||||
assert!(!replication_target_versioning_enabled(Some(&suspended)));
|
||||
assert!(!replication_target_versioning_enabled(None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_target_connection_error_display_redacts_access_key() {
|
||||
let err = BucketTargetError::RemoteTargetConnectionErr {
|
||||
bucket: "target".to_string(),
|
||||
access_key: "sensitive-access-key".to_string(),
|
||||
error: "connection refused".to_string(),
|
||||
};
|
||||
let message = err.to_string();
|
||||
|
||||
assert!(message.contains(REDACTED_CREDENTIAL));
|
||||
assert!(!message.contains("sensitive-access-key"));
|
||||
assert!(message.contains("connection refused"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn endpoint_health_key_preserves_explicit_port() {
|
||||
let url = Url::parse("https://remote.example:9443").expect("url should parse");
|
||||
|
||||
assert_eq!(endpoint_health_key(&url), "remote.example:9443");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn update_endpoint_health_counts_offline_transitions() {
|
||||
let mut health = EpHealth::default();
|
||||
let now = OffsetDateTime::now_utc();
|
||||
|
||||
update_endpoint_health(&mut health, false, Duration::from_millis(25), now);
|
||||
update_endpoint_health(&mut health, false, Duration::from_millis(25), now);
|
||||
update_endpoint_health(&mut health, true, Duration::from_millis(10), now);
|
||||
update_endpoint_health(&mut health, false, Duration::from_millis(25), now);
|
||||
|
||||
assert_eq!(health.offline_count, 2);
|
||||
assert_eq!(health.offline_duration, Duration::from_millis(75));
|
||||
assert_eq!(health.last_online, Some(now));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_targets_applies_health_stats_for_endpoint_with_port() {
|
||||
let sys = BucketTargetSys::default();
|
||||
let url = Url::parse("https://remote.example:9443").expect("url should parse");
|
||||
sys.init_hc(&url).await;
|
||||
sys.mark_offline(&url).await;
|
||||
|
||||
sys.targets_map.write().await.insert(
|
||||
"bucket".to_string(),
|
||||
vec![BucketTarget {
|
||||
endpoint: "remote.example:9443".to_string(),
|
||||
arn: "arn:rustfs:replication:us-east-1:bucket:id".to_string(),
|
||||
target_type: BucketTargetType::ReplicationService,
|
||||
..Default::default()
|
||||
}],
|
||||
);
|
||||
|
||||
let targets = sys.list_targets("", "").await;
|
||||
|
||||
assert_eq!(targets.len(), 1);
|
||||
assert!(!targets[0].online);
|
||||
assert_eq!(targets[0].offline_count, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_remove_object_headers_includes_internal_version_id_for_replication_delete() {
|
||||
let version_id = Uuid::new_v4().to_string();
|
||||
|
||||
@@ -49,9 +49,10 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
|
||||
/// Check whether any object-replication rules exist
|
||||
fn has_existing_object_replication(&self, arn: &str) -> (bool, bool) {
|
||||
let mut has_arn = false;
|
||||
let arn = arn.trim();
|
||||
|
||||
for rule in &self.rules {
|
||||
if rule.destination.bucket == arn || self.role == arn {
|
||||
if rule.destination.bucket.trim() == arn || self.role.trim() == arn {
|
||||
if !has_arn {
|
||||
has_arn = true;
|
||||
}
|
||||
@@ -77,7 +78,10 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
|
||||
continue;
|
||||
}
|
||||
|
||||
if !obj.target_arn.is_empty() && rule.destination.bucket != obj.target_arn && self.role != obj.target_arn {
|
||||
if !obj.target_arn.is_empty()
|
||||
&& rule.destination.bucket.trim() != obj.target_arn.trim()
|
||||
&& self.role.trim() != obj.target_arn.trim()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -215,6 +219,11 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
|
||||
|
||||
/// Filter target ARNs and return a slice of the distinct values in the config
|
||||
fn filter_target_arns(&self, obj: &ObjectOpts) -> Vec<String> {
|
||||
let role = self.role.trim();
|
||||
if !role.is_empty() {
|
||||
return vec![role.to_string()];
|
||||
}
|
||||
|
||||
let mut arns = Vec::new();
|
||||
let mut targets_map: HashSet<String> = HashSet::new();
|
||||
let rules = self.filter_actionable_rules(obj);
|
||||
@@ -224,16 +233,12 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
|
||||
continue;
|
||||
}
|
||||
|
||||
if !rule.destination.bucket.is_empty() && !targets_map.contains(&rule.destination.bucket) {
|
||||
targets_map.insert(rule.destination.bucket.clone());
|
||||
let arn = rule.destination.bucket.trim();
|
||||
if !arn.is_empty() && !targets_map.contains(arn) {
|
||||
targets_map.insert(arn.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
if targets_map.is_empty() && !self.role.is_empty() {
|
||||
arns.push(self.role.clone());
|
||||
return arns;
|
||||
}
|
||||
|
||||
for arn in targets_map {
|
||||
arns.push(arn);
|
||||
}
|
||||
@@ -267,9 +272,9 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filter_target_arns_keeps_multiple_destinations_when_role_is_present() {
|
||||
fn filter_target_arns_uses_role_when_role_is_present() {
|
||||
let config = ReplicationConfiguration {
|
||||
role: "arn:legacy:target".to_string(),
|
||||
role: " arn:legacy:target ".to_string(),
|
||||
rules: vec![
|
||||
replication_rule("rule-1", "arn:target:a"),
|
||||
replication_rule("rule-2", "arn:target:b"),
|
||||
@@ -282,9 +287,7 @@ mod tests {
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert_eq!(arns.len(), 2);
|
||||
assert!(arns.iter().any(|arn| arn == "arn:target:a"));
|
||||
assert!(arns.iter().any(|arn| arn == "arn:target:b"));
|
||||
assert_eq!(arns, vec!["arn:legacy:target".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1424,7 +1424,7 @@ pub fn resync_target(
|
||||
|
||||
if rs.is_none() {
|
||||
let reset_before_date = reset_before_date.unwrap_or(OffsetDateTime::UNIX_EPOCH);
|
||||
if !reset_id.is_empty() && mod_time < reset_before_date {
|
||||
if !reset_id.is_empty() && mod_time <= reset_before_date {
|
||||
dec.replicate = true;
|
||||
return dec;
|
||||
}
|
||||
@@ -1447,13 +1447,13 @@ pub fn resync_target(
|
||||
return dec;
|
||||
}
|
||||
|
||||
let new_reset = parts[0] == reset_id;
|
||||
let new_reset = parts[1] != reset_id;
|
||||
|
||||
if !new_reset && status == ReplicationStatusType::Completed {
|
||||
return dec;
|
||||
}
|
||||
|
||||
dec.replicate = new_reset && mod_time < reset_before_date;
|
||||
dec.replicate = new_reset && mod_time <= reset_before_date;
|
||||
|
||||
dec
|
||||
}
|
||||
@@ -3337,10 +3337,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
rinfo.replication_status = ReplicationStatusType::Completed;
|
||||
if replication_action == ReplicationAction::None {
|
||||
if self.op_type == ReplicationType::ExistingObject
|
||||
&& object_info.mod_time
|
||||
> oi.last_modified
|
||||
.map(|dt| OffsetDateTime::from_unix_timestamp(dt.secs()).unwrap_or(OffsetDateTime::UNIX_EPOCH))
|
||||
&& object_info.version_id.is_none()
|
||||
&& target_is_newer_than_source_null_version(&object_info, &oi)
|
||||
{
|
||||
warn!(
|
||||
event = EVENT_RESYNC_RUNTIME_SKIPPED,
|
||||
@@ -3350,7 +3347,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
object = %object,
|
||||
arn = %tgt_client.arn,
|
||||
endpoint = %tgt_client.to_url(),
|
||||
reason = "newer_target_version_exists",
|
||||
reason = "target_newer_than_source_null_version",
|
||||
"Skipping replication because newer target version exists"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
@@ -3558,7 +3555,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
replication_status: self.replication_status.clone(),
|
||||
version_purge_status_internal: self.version_purge_status_internal.clone(),
|
||||
version_purge_status: self.version_purge_status.clone(),
|
||||
delete_marker: true,
|
||||
delete_marker: self.delete_marker,
|
||||
checksum: self.checksum.clone(),
|
||||
..Default::default()
|
||||
}
|
||||
@@ -3983,14 +3980,15 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn get_replication_action(oi1: &ObjectInfo, oi2: &HeadObjectOutput, op_type: ReplicationType) -> ReplicationAction {
|
||||
if op_type == ReplicationType::ExistingObject
|
||||
&& oi1.mod_time
|
||||
> oi2
|
||||
.last_modified
|
||||
.map(|dt| OffsetDateTime::from_unix_timestamp(dt.secs()).unwrap_or(OffsetDateTime::UNIX_EPOCH))
|
||||
fn target_is_newer_than_source_null_version(oi1: &ObjectInfo, oi2: &HeadObjectOutput) -> bool {
|
||||
oi2.last_modified
|
||||
.map(|dt| OffsetDateTime::from_unix_timestamp(dt.secs()).unwrap_or(OffsetDateTime::UNIX_EPOCH))
|
||||
.is_some_and(|target_mod_time| target_mod_time > oi1.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH))
|
||||
&& oi1.version_id.is_none()
|
||||
{
|
||||
}
|
||||
|
||||
fn get_replication_action(oi1: &ObjectInfo, oi2: &HeadObjectOutput, op_type: ReplicationType) -> ReplicationAction {
|
||||
if op_type == ReplicationType::ExistingObject && target_is_newer_than_source_null_version(oi1, oi2) {
|
||||
return ReplicationAction::None;
|
||||
}
|
||||
|
||||
@@ -4093,8 +4091,9 @@ fn get_replication_action(oi1: &ObjectInfo, oi2: &HeadObjectOutput, op_type: Rep
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use aws_smithy_types::DateTime;
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
use time::{Duration, OffsetDateTime};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[test]
|
||||
@@ -4356,6 +4355,107 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_replication_action_existing_object_source_newer_null_version_requires_replication() {
|
||||
let source = ObjectInfo {
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + Duration::seconds(20)),
|
||||
version_id: None,
|
||||
..Default::default()
|
||||
};
|
||||
let target = HeadObjectOutput::builder().last_modified(DateTime::from_secs(10)).build();
|
||||
|
||||
assert_eq!(
|
||||
get_replication_action(&source, &target, ReplicationType::ExistingObject),
|
||||
ReplicationAction::All,
|
||||
"a newer source null version must not be skipped during existing-object replication"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_replication_action_existing_object_target_newer_null_version_skips() {
|
||||
let source = ObjectInfo {
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + Duration::seconds(10)),
|
||||
version_id: None,
|
||||
..Default::default()
|
||||
};
|
||||
let target = HeadObjectOutput::builder().last_modified(DateTime::from_secs(20)).build();
|
||||
|
||||
assert_eq!(
|
||||
get_replication_action(&source, &target, ReplicationType::ExistingObject),
|
||||
ReplicationAction::None,
|
||||
"a newer target null-version object should not be overwritten by existing-object replication"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resync_target_includes_object_at_reset_before_boundary() {
|
||||
let reset_before = OffsetDateTime::UNIX_EPOCH + Duration::seconds(30);
|
||||
let oi = ObjectInfo {
|
||||
mod_time: Some(reset_before),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let decision = resync_target(&oi, "arn:target", "reset-1", Some(reset_before), ReplicationStatusType::Completed);
|
||||
|
||||
assert!(
|
||||
decision.replicate,
|
||||
"objects whose mod_time equals reset_before must be included in the reset window"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resync_target_replicates_when_reset_id_changes() {
|
||||
let reset_before = OffsetDateTime::UNIX_EPOCH + Duration::seconds(30);
|
||||
let oi = ObjectInfo {
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + Duration::seconds(10)),
|
||||
user_defined: Arc::new(HashMap::from([(
|
||||
target_reset_header("arn:target"),
|
||||
"1970-01-01T00:00:20Z;old-reset".to_string(),
|
||||
)])),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let decision = resync_target(&oi, "arn:target", "new-reset", Some(reset_before), ReplicationStatusType::Completed);
|
||||
|
||||
assert!(decision.replicate, "a new reset id must resync objects marked by an older reset");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resync_target_skips_completed_object_for_same_reset_id() {
|
||||
let reset_before = OffsetDateTime::UNIX_EPOCH + Duration::seconds(30);
|
||||
let oi = ObjectInfo {
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + Duration::seconds(10)),
|
||||
user_defined: Arc::new(HashMap::from([(
|
||||
target_reset_header("arn:target"),
|
||||
"1970-01-01T00:00:20Z;same-reset".to_string(),
|
||||
)])),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let decision = resync_target(&oi, "arn:target", "same-reset", Some(reset_before), ReplicationStatusType::Completed);
|
||||
|
||||
assert!(!decision.replicate, "the same completed reset id must not resync again");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_replicate_object_info_to_object_info_preserves_delete_marker_flag() {
|
||||
let live = ReplicateObjectInfo {
|
||||
bucket: "source".to_string(),
|
||||
name: "object".to_string(),
|
||||
delete_marker: false,
|
||||
..Default::default()
|
||||
};
|
||||
let delete_marker = ReplicateObjectInfo {
|
||||
bucket: "source".to_string(),
|
||||
name: "object".to_string(),
|
||||
delete_marker: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(!live.to_object_info().delete_marker);
|
||||
assert!(delete_marker.to_object_info().delete_marker);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete_replication_object_opts_marks_replica_deletes() {
|
||||
let dobj = ObjectToDelete {
|
||||
|
||||
@@ -300,11 +300,18 @@ impl ReplicationState {
|
||||
|
||||
/// Returns replicatedInfos struct initialized with the previous state of replication
|
||||
pub fn target_state(&self, arn: &str) -> ReplicatedTargetInfo {
|
||||
let resync_timestamp = self
|
||||
.reset_statuses_map
|
||||
.get(&target_reset_header(arn))
|
||||
.or_else(|| self.reset_statuses_map.get(arn))
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
|
||||
ReplicatedTargetInfo {
|
||||
arn: arn.to_string(),
|
||||
prev_replication_status: self.targets.get(arn).cloned().unwrap_or_default(),
|
||||
version_purge_status: self.purge_targets.get(arn).cloned().unwrap_or_default(),
|
||||
resync_timestamp: self.reset_statuses_map.get(arn).cloned().unwrap_or_default(),
|
||||
resync_timestamp,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
@@ -862,6 +869,28 @@ pub fn version_purge_statuses_map(s: &str) -> HashMap<String, VersionPurgeStatus
|
||||
targets
|
||||
}
|
||||
|
||||
fn replication_statuses_string(targets: &HashMap<String, ReplicationStatusType>) -> Option<String> {
|
||||
let mut result = String::new();
|
||||
for (arn, status) in targets {
|
||||
if arn.is_empty() || status.is_empty() {
|
||||
continue;
|
||||
}
|
||||
result.push_str(&format!("{arn}={status};"));
|
||||
}
|
||||
if result.is_empty() { None } else { Some(result) }
|
||||
}
|
||||
|
||||
fn version_purge_statuses_string(targets: &HashMap<String, VersionPurgeStatusType>) -> Option<String> {
|
||||
let mut result = String::new();
|
||||
for (arn, status) in targets {
|
||||
if arn.is_empty() || status.is_empty() {
|
||||
continue;
|
||||
}
|
||||
result.push_str(&format!("{arn}={status};"));
|
||||
}
|
||||
if result.is_empty() { None } else { Some(result) }
|
||||
}
|
||||
|
||||
pub fn get_replication_state(rinfos: &ReplicatedInfos, prev_state: &ReplicationState, _vid: Option<String>) -> ReplicationState {
|
||||
let reset_status_map: Vec<(String, String)> = rinfos
|
||||
.targets
|
||||
@@ -870,8 +899,18 @@ pub fn get_replication_state(rinfos: &ReplicatedInfos, prev_state: &ReplicationS
|
||||
.map(|t| (target_reset_header(t.arn.as_str()), t.resync_timestamp.clone()))
|
||||
.collect();
|
||||
|
||||
let repl_statuses = rinfos.replication_status_internal();
|
||||
let vpurge_statuses = rinfos.version_purge_status_internal();
|
||||
let mut targets = prev_state.targets.clone();
|
||||
for (arn, status) in replication_statuses_map(&rinfos.replication_status_internal().unwrap_or_default()) {
|
||||
targets.insert(arn, status);
|
||||
}
|
||||
|
||||
let mut purge_targets = prev_state.purge_targets.clone();
|
||||
for (arn, status) in version_purge_statuses_map(&rinfos.version_purge_status_internal().unwrap_or_default()) {
|
||||
purge_targets.insert(arn, status);
|
||||
}
|
||||
|
||||
let repl_statuses = replication_statuses_string(&targets);
|
||||
let vpurge_statuses = version_purge_statuses_string(&purge_targets);
|
||||
|
||||
let mut reset_statuses_map = prev_state.reset_statuses_map.clone();
|
||||
for (key, value) in reset_status_map {
|
||||
@@ -883,10 +922,10 @@ pub fn get_replication_state(rinfos: &ReplicatedInfos, prev_state: &ReplicationS
|
||||
reset_statuses_map,
|
||||
replica_timestamp: prev_state.replica_timestamp,
|
||||
replica_status: prev_state.replica_status.clone(),
|
||||
targets: replication_statuses_map(&repl_statuses.clone().unwrap_or_default()),
|
||||
targets,
|
||||
replication_status_internal: repl_statuses,
|
||||
replication_timestamp: rinfos.replication_timestamp,
|
||||
purge_targets: version_purge_statuses_map(&vpurge_statuses.clone().unwrap_or_default()),
|
||||
purge_targets,
|
||||
version_purge_status_internal: vpurge_statuses,
|
||||
|
||||
..Default::default()
|
||||
@@ -934,3 +973,47 @@ impl Default for ResyncDecision {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn target_state_reads_resync_timestamp_from_target_reset_header_key() {
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
let timestamp = "2026-06-30T00:00:00Z;reset-1".to_string();
|
||||
let mut state = ReplicationState::default();
|
||||
state.reset_statuses_map.insert(target_reset_header(arn), timestamp.clone());
|
||||
|
||||
let target_state = state.target_state(arn);
|
||||
|
||||
assert_eq!(target_state.resync_timestamp, timestamp);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_replication_state_preserves_untouched_target_statuses() {
|
||||
let target_a = "arn:target:a".to_string();
|
||||
let target_b = "arn:target:b".to_string();
|
||||
let mut prev_state = ReplicationState::default();
|
||||
prev_state.targets.insert(target_a.clone(), ReplicationStatusType::Failed);
|
||||
prev_state.targets.insert(target_b.clone(), ReplicationStatusType::Completed);
|
||||
|
||||
let rinfos = ReplicatedInfos {
|
||||
replication_timestamp: None,
|
||||
targets: vec![ReplicatedTargetInfo {
|
||||
arn: target_a.clone(),
|
||||
replication_status: ReplicationStatusType::Completed,
|
||||
..Default::default()
|
||||
}],
|
||||
};
|
||||
|
||||
let state = get_replication_state(&rinfos, &prev_state, None);
|
||||
|
||||
assert_eq!(state.targets.get(&target_a), Some(&ReplicationStatusType::Completed));
|
||||
assert_eq!(state.targets.get(&target_b), Some(&ReplicationStatusType::Completed));
|
||||
assert_eq!(
|
||||
replication_statuses_map(&state.replication_status_internal.unwrap_or_default()).get(&target_b),
|
||||
Some(&ReplicationStatusType::Completed)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user