fix(replication): harden bucket replication correctness (#4116)

This commit is contained in:
cxymds
2026-07-01 09:21:30 +08:00
committed by GitHub
parent 722c0805ce
commit 6e5c58ca3f
8 changed files with 763 additions and 137 deletions
+130 -37
View File
@@ -82,9 +82,14 @@ use uuid::Uuid;
const DEFAULT_HEALTH_CHECK_DURATION: Duration = Duration::from_secs(5);
const DEFAULT_HEALTH_CHECK_RELOAD_DURATION: Duration = Duration::from_secs(30 * 60);
const REDACTED_CREDENTIAL: &str = "<redacted>";
pub static GLOBAL_BUCKET_TARGET_SYS: OnceLock<BucketTargetSys> = OnceLock::new();
fn replication_target_versioning_enabled(versioning: Option<&BucketVersioningStatus>) -> bool {
matches!(versioning, Some(BucketVersioningStatus::Enabled))
}
#[derive(Debug, Clone)]
pub struct ArnTarget {
pub client: Option<Arc<TargetClient>>,
@@ -202,6 +207,7 @@ pub struct EpHealth {
pub last_online: Option<OffsetDateTime>,
pub last_hc_at: Option<OffsetDateTime>,
pub offline_duration: Duration,
pub offline_count: u64,
pub latency: LatencyStat,
}
@@ -214,11 +220,37 @@ impl Default for EpHealth {
last_online: None,
last_hc_at: None,
offline_duration: Duration::from_secs(0),
offline_count: 0,
latency: LatencyStat::new(),
}
}
}
fn endpoint_health_key(url: &Url) -> String {
let host = url.host_str().unwrap_or_default();
match url.port() {
Some(port) => format!("{host}:{port}"),
None => host.to_string(),
}
}
fn update_endpoint_health(health: &mut EpHealth, online: bool, latency: Duration, now: OffsetDateTime) {
let prev_online = health.online;
health.online = online;
health.last_hc_at = Some(now);
health.latency.update(latency);
if online {
health.last_online = Some(now);
return;
}
if prev_online {
health.offline_count += 1;
}
health.offline_duration += latency;
}
#[derive(Debug, Default)]
pub struct BucketTargetSys {
pub arn_remotes_map: Arc<RwLock<HashMap<String, ArnTarget>>>,
@@ -246,9 +278,10 @@ impl BucketTargetSys {
}
pub async fn is_offline(&self, url: &Url) -> bool {
let key = endpoint_health_key(url);
{
let health_map = self.h_mutex.read().await;
if let Some(health) = health_map.get(url.host_str().unwrap_or("")) {
if let Some(health) = health_map.get(&key) {
return !health.online;
}
}
@@ -258,15 +291,16 @@ impl BucketTargetSys {
}
pub async fn mark_offline(&self, url: &Url) {
let key = endpoint_health_key(url);
let mut health_map = self.h_mutex.write().await;
if let Some(health) = health_map.get_mut(url.host_str().unwrap_or("")) {
health.online = false;
if let Some(health) = health_map.get_mut(&key) {
update_endpoint_health(health, false, Duration::from_secs(0), OffsetDateTime::now_utc());
}
}
pub async fn init_hc(&self, url: &Url) {
let mut health_map = self.h_mutex.write().await;
let host = url.host_str().unwrap_or("").to_string();
let host = endpoint_health_key(url);
health_map.insert(
host.clone(),
EpHealth {
@@ -285,51 +319,35 @@ impl BucketTargetSys {
let endpoints = {
let health_map = self.h_mutex.read().await;
health_map.keys().cloned().collect::<Vec<_>>()
health_map
.iter()
.map(|(endpoint, health)| (endpoint.clone(), health.scheme.clone()))
.collect::<Vec<_>>()
};
for endpoint in endpoints {
for (endpoint, scheme) in endpoints {
// Perform health check
let start = Instant::now();
let online = self.check_endpoint_health(&endpoint).await;
let online = self.check_endpoint_health(&endpoint, &scheme).await;
let duration = start.elapsed();
{
let mut health_map = self.h_mutex.write().await;
if let Some(health) = health_map.get_mut(&endpoint) {
let prev_online = health.online;
health.online = online;
health.last_hc_at = Some(OffsetDateTime::now_utc());
health.latency.update(duration);
if online {
health.last_online = Some(OffsetDateTime::now_utc());
} else if prev_online {
// Just went offline
health.offline_duration += duration;
}
update_endpoint_health(health, online, duration, OffsetDateTime::now_utc());
}
}
}
}
}
async fn check_endpoint_health(&self, _endpoint: &str) -> bool {
true
// TODO: Health check
// // Simple health check implementation
// // In a real implementation, you would make actual HTTP requests
// match self
// .hc_client
// .get(format!("https://{}/rustfs/health/ready", endpoint))
// .timeout(Duration::from_secs(3))
// .send()
// .await
// {
// Ok(response) => response.status().is_success(),
// Err(_) => false,
// }
async fn check_endpoint_health(&self, endpoint: &str, scheme: &str) -> bool {
let scheme = if scheme.is_empty() { "https" } else { scheme };
let url = format!("{scheme}://{endpoint}/");
match self.hc_client.head(url).timeout(Duration::from_secs(3)).send().await {
Ok(response) => response.status().as_u16() < 500,
Err(_) => false,
}
}
pub async fn health_stats(&self) -> HashMap<String, EpHealth> {
@@ -354,6 +372,7 @@ impl BucketTargetSys {
avg: health.latency.avg,
max: health.latency.peak,
};
target.offline_count = health.offline_count;
}
targets.push(target);
}
@@ -375,6 +394,7 @@ impl BucketTargetSys {
avg: health.latency.avg,
max: health.latency.peak,
};
target.offline_count = health.offline_count;
}
targets.push(target);
}
@@ -475,7 +495,7 @@ impl BucketTargetSys {
error: e.to_string(),
})?;
if versioning.is_none() {
if !replication_target_versioning_enabled(versioning.as_ref()) {
return Err(BucketTargetError::BucketRemoteTargetNotVersioned {
bucket: target.target_bucket.to_string(),
});
@@ -1763,10 +1783,13 @@ impl fmt::Display for BucketTargetError {
}
BucketTargetError::RemoteTargetConnectionErr {
bucket,
access_key,
access_key: _,
error,
} => {
write!(f, "Connection error for bucket: {bucket}, access key: {access_key}, error: {error}")
write!(
f,
"Connection error for bucket: {bucket}, access key: {REDACTED_CREDENTIAL}, error: {error}"
)
}
BucketTargetError::BucketReplicationSourceNotVersioned { bucket } => {
write!(f, "Replication source bucket not versioned: {bucket}")
@@ -1795,6 +1818,76 @@ mod tests {
use super::*;
use rcgen::generate_simple_self_signed;
#[test]
fn replication_target_versioning_enabled_requires_enabled_status() {
let enabled = BucketVersioningStatus::Enabled;
let suspended = BucketVersioningStatus::Suspended;
assert!(replication_target_versioning_enabled(Some(&enabled)));
assert!(!replication_target_versioning_enabled(Some(&suspended)));
assert!(!replication_target_versioning_enabled(None));
}
#[test]
fn remote_target_connection_error_display_redacts_access_key() {
let err = BucketTargetError::RemoteTargetConnectionErr {
bucket: "target".to_string(),
access_key: "sensitive-access-key".to_string(),
error: "connection refused".to_string(),
};
let message = err.to_string();
assert!(message.contains(REDACTED_CREDENTIAL));
assert!(!message.contains("sensitive-access-key"));
assert!(message.contains("connection refused"));
}
#[test]
fn endpoint_health_key_preserves_explicit_port() {
let url = Url::parse("https://remote.example:9443").expect("url should parse");
assert_eq!(endpoint_health_key(&url), "remote.example:9443");
}
#[test]
fn update_endpoint_health_counts_offline_transitions() {
let mut health = EpHealth::default();
let now = OffsetDateTime::now_utc();
update_endpoint_health(&mut health, false, Duration::from_millis(25), now);
update_endpoint_health(&mut health, false, Duration::from_millis(25), now);
update_endpoint_health(&mut health, true, Duration::from_millis(10), now);
update_endpoint_health(&mut health, false, Duration::from_millis(25), now);
assert_eq!(health.offline_count, 2);
assert_eq!(health.offline_duration, Duration::from_millis(75));
assert_eq!(health.last_online, Some(now));
}
#[tokio::test]
async fn list_targets_applies_health_stats_for_endpoint_with_port() {
let sys = BucketTargetSys::default();
let url = Url::parse("https://remote.example:9443").expect("url should parse");
sys.init_hc(&url).await;
sys.mark_offline(&url).await;
sys.targets_map.write().await.insert(
"bucket".to_string(),
vec![BucketTarget {
endpoint: "remote.example:9443".to_string(),
arn: "arn:rustfs:replication:us-east-1:bucket:id".to_string(),
target_type: BucketTargetType::ReplicationService,
..Default::default()
}],
);
let targets = sys.list_targets("", "").await;
assert_eq!(targets.len(), 1);
assert!(!targets[0].online);
assert_eq!(targets[0].offline_count, 1);
}
#[test]
fn build_remove_object_headers_includes_internal_version_id_for_replication_delete() {
let version_id = Uuid::new_v4().to_string();
+17 -14
View File
@@ -49,9 +49,10 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
/// Check whether any object-replication rules exist
fn has_existing_object_replication(&self, arn: &str) -> (bool, bool) {
let mut has_arn = false;
let arn = arn.trim();
for rule in &self.rules {
if rule.destination.bucket == arn || self.role == arn {
if rule.destination.bucket.trim() == arn || self.role.trim() == arn {
if !has_arn {
has_arn = true;
}
@@ -77,7 +78,10 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
continue;
}
if !obj.target_arn.is_empty() && rule.destination.bucket != obj.target_arn && self.role != obj.target_arn {
if !obj.target_arn.is_empty()
&& rule.destination.bucket.trim() != obj.target_arn.trim()
&& self.role.trim() != obj.target_arn.trim()
{
continue;
}
@@ -215,6 +219,11 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
/// Filter target ARNs and return a slice of the distinct values in the config
fn filter_target_arns(&self, obj: &ObjectOpts) -> Vec<String> {
let role = self.role.trim();
if !role.is_empty() {
return vec![role.to_string()];
}
let mut arns = Vec::new();
let mut targets_map: HashSet<String> = HashSet::new();
let rules = self.filter_actionable_rules(obj);
@@ -224,16 +233,12 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
continue;
}
if !rule.destination.bucket.is_empty() && !targets_map.contains(&rule.destination.bucket) {
targets_map.insert(rule.destination.bucket.clone());
let arn = rule.destination.bucket.trim();
if !arn.is_empty() && !targets_map.contains(arn) {
targets_map.insert(arn.to_string());
}
}
if targets_map.is_empty() && !self.role.is_empty() {
arns.push(self.role.clone());
return arns;
}
for arn in targets_map {
arns.push(arn);
}
@@ -267,9 +272,9 @@ mod tests {
}
#[test]
fn filter_target_arns_keeps_multiple_destinations_when_role_is_present() {
fn filter_target_arns_uses_role_when_role_is_present() {
let config = ReplicationConfiguration {
role: "arn:legacy:target".to_string(),
role: " arn:legacy:target ".to_string(),
rules: vec![
replication_rule("rule-1", "arn:target:a"),
replication_rule("rule-2", "arn:target:b"),
@@ -282,9 +287,7 @@ mod tests {
..Default::default()
});
assert_eq!(arns.len(), 2);
assert!(arns.iter().any(|arn| arn == "arn:target:a"));
assert!(arns.iter().any(|arn| arn == "arn:target:b"));
assert_eq!(arns, vec!["arn:legacy:target".to_string()]);
}
#[test]
@@ -1424,7 +1424,7 @@ pub fn resync_target(
if rs.is_none() {
let reset_before_date = reset_before_date.unwrap_or(OffsetDateTime::UNIX_EPOCH);
if !reset_id.is_empty() && mod_time < reset_before_date {
if !reset_id.is_empty() && mod_time <= reset_before_date {
dec.replicate = true;
return dec;
}
@@ -1447,13 +1447,13 @@ pub fn resync_target(
return dec;
}
let new_reset = parts[0] == reset_id;
let new_reset = parts[1] != reset_id;
if !new_reset && status == ReplicationStatusType::Completed {
return dec;
}
dec.replicate = new_reset && mod_time < reset_before_date;
dec.replicate = new_reset && mod_time <= reset_before_date;
dec
}
@@ -3337,10 +3337,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
rinfo.replication_status = ReplicationStatusType::Completed;
if replication_action == ReplicationAction::None {
if self.op_type == ReplicationType::ExistingObject
&& object_info.mod_time
> oi.last_modified
.map(|dt| OffsetDateTime::from_unix_timestamp(dt.secs()).unwrap_or(OffsetDateTime::UNIX_EPOCH))
&& object_info.version_id.is_none()
&& target_is_newer_than_source_null_version(&object_info, &oi)
{
warn!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
@@ -3350,7 +3347,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
object = %object,
arn = %tgt_client.arn,
endpoint = %tgt_client.to_url(),
reason = "newer_target_version_exists",
reason = "target_newer_than_source_null_version",
"Skipping replication because newer target version exists"
);
send_local_event(EventArgs {
@@ -3558,7 +3555,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
replication_status: self.replication_status.clone(),
version_purge_status_internal: self.version_purge_status_internal.clone(),
version_purge_status: self.version_purge_status.clone(),
delete_marker: true,
delete_marker: self.delete_marker,
checksum: self.checksum.clone(),
..Default::default()
}
@@ -3983,14 +3980,15 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
Ok(())
}
fn get_replication_action(oi1: &ObjectInfo, oi2: &HeadObjectOutput, op_type: ReplicationType) -> ReplicationAction {
if op_type == ReplicationType::ExistingObject
&& oi1.mod_time
> oi2
.last_modified
.map(|dt| OffsetDateTime::from_unix_timestamp(dt.secs()).unwrap_or(OffsetDateTime::UNIX_EPOCH))
fn target_is_newer_than_source_null_version(oi1: &ObjectInfo, oi2: &HeadObjectOutput) -> bool {
oi2.last_modified
.map(|dt| OffsetDateTime::from_unix_timestamp(dt.secs()).unwrap_or(OffsetDateTime::UNIX_EPOCH))
.is_some_and(|target_mod_time| target_mod_time > oi1.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH))
&& oi1.version_id.is_none()
{
}
fn get_replication_action(oi1: &ObjectInfo, oi2: &HeadObjectOutput, op_type: ReplicationType) -> ReplicationAction {
if op_type == ReplicationType::ExistingObject && target_is_newer_than_source_null_version(oi1, oi2) {
return ReplicationAction::None;
}
@@ -4093,8 +4091,9 @@ fn get_replication_action(oi1: &ObjectInfo, oi2: &HeadObjectOutput, op_type: Rep
#[cfg(test)]
mod tests {
use super::*;
use aws_smithy_types::DateTime;
use std::collections::HashMap;
use time::OffsetDateTime;
use time::{Duration, OffsetDateTime};
use uuid::Uuid;
#[test]
@@ -4356,6 +4355,107 @@ mod tests {
);
}
#[test]
fn test_get_replication_action_existing_object_source_newer_null_version_requires_replication() {
let source = ObjectInfo {
mod_time: Some(OffsetDateTime::UNIX_EPOCH + Duration::seconds(20)),
version_id: None,
..Default::default()
};
let target = HeadObjectOutput::builder().last_modified(DateTime::from_secs(10)).build();
assert_eq!(
get_replication_action(&source, &target, ReplicationType::ExistingObject),
ReplicationAction::All,
"a newer source null version must not be skipped during existing-object replication"
);
}
#[test]
fn test_get_replication_action_existing_object_target_newer_null_version_skips() {
let source = ObjectInfo {
mod_time: Some(OffsetDateTime::UNIX_EPOCH + Duration::seconds(10)),
version_id: None,
..Default::default()
};
let target = HeadObjectOutput::builder().last_modified(DateTime::from_secs(20)).build();
assert_eq!(
get_replication_action(&source, &target, ReplicationType::ExistingObject),
ReplicationAction::None,
"a newer target null-version object should not be overwritten by existing-object replication"
);
}
#[test]
fn test_resync_target_includes_object_at_reset_before_boundary() {
let reset_before = OffsetDateTime::UNIX_EPOCH + Duration::seconds(30);
let oi = ObjectInfo {
mod_time: Some(reset_before),
..Default::default()
};
let decision = resync_target(&oi, "arn:target", "reset-1", Some(reset_before), ReplicationStatusType::Completed);
assert!(
decision.replicate,
"objects whose mod_time equals reset_before must be included in the reset window"
);
}
#[test]
fn test_resync_target_replicates_when_reset_id_changes() {
let reset_before = OffsetDateTime::UNIX_EPOCH + Duration::seconds(30);
let oi = ObjectInfo {
mod_time: Some(OffsetDateTime::UNIX_EPOCH + Duration::seconds(10)),
user_defined: Arc::new(HashMap::from([(
target_reset_header("arn:target"),
"1970-01-01T00:00:20Z;old-reset".to_string(),
)])),
..Default::default()
};
let decision = resync_target(&oi, "arn:target", "new-reset", Some(reset_before), ReplicationStatusType::Completed);
assert!(decision.replicate, "a new reset id must resync objects marked by an older reset");
}
#[test]
fn test_resync_target_skips_completed_object_for_same_reset_id() {
let reset_before = OffsetDateTime::UNIX_EPOCH + Duration::seconds(30);
let oi = ObjectInfo {
mod_time: Some(OffsetDateTime::UNIX_EPOCH + Duration::seconds(10)),
user_defined: Arc::new(HashMap::from([(
target_reset_header("arn:target"),
"1970-01-01T00:00:20Z;same-reset".to_string(),
)])),
..Default::default()
};
let decision = resync_target(&oi, "arn:target", "same-reset", Some(reset_before), ReplicationStatusType::Completed);
assert!(!decision.replicate, "the same completed reset id must not resync again");
}
#[test]
fn test_replicate_object_info_to_object_info_preserves_delete_marker_flag() {
let live = ReplicateObjectInfo {
bucket: "source".to_string(),
name: "object".to_string(),
delete_marker: false,
..Default::default()
};
let delete_marker = ReplicateObjectInfo {
bucket: "source".to_string(),
name: "object".to_string(),
delete_marker: true,
..Default::default()
};
assert!(!live.to_object_info().delete_marker);
assert!(delete_marker.to_object_info().delete_marker);
}
#[test]
fn test_delete_replication_object_opts_marks_replica_deletes() {
let dobj = ObjectToDelete {
+88 -5
View File
@@ -300,11 +300,18 @@ impl ReplicationState {
/// Returns replicatedInfos struct initialized with the previous state of replication
pub fn target_state(&self, arn: &str) -> ReplicatedTargetInfo {
let resync_timestamp = self
.reset_statuses_map
.get(&target_reset_header(arn))
.or_else(|| self.reset_statuses_map.get(arn))
.cloned()
.unwrap_or_default();
ReplicatedTargetInfo {
arn: arn.to_string(),
prev_replication_status: self.targets.get(arn).cloned().unwrap_or_default(),
version_purge_status: self.purge_targets.get(arn).cloned().unwrap_or_default(),
resync_timestamp: self.reset_statuses_map.get(arn).cloned().unwrap_or_default(),
resync_timestamp,
..Default::default()
}
}
@@ -862,6 +869,28 @@ pub fn version_purge_statuses_map(s: &str) -> HashMap<String, VersionPurgeStatus
targets
}
fn replication_statuses_string(targets: &HashMap<String, ReplicationStatusType>) -> Option<String> {
let mut result = String::new();
for (arn, status) in targets {
if arn.is_empty() || status.is_empty() {
continue;
}
result.push_str(&format!("{arn}={status};"));
}
if result.is_empty() { None } else { Some(result) }
}
fn version_purge_statuses_string(targets: &HashMap<String, VersionPurgeStatusType>) -> Option<String> {
let mut result = String::new();
for (arn, status) in targets {
if arn.is_empty() || status.is_empty() {
continue;
}
result.push_str(&format!("{arn}={status};"));
}
if result.is_empty() { None } else { Some(result) }
}
pub fn get_replication_state(rinfos: &ReplicatedInfos, prev_state: &ReplicationState, _vid: Option<String>) -> ReplicationState {
let reset_status_map: Vec<(String, String)> = rinfos
.targets
@@ -870,8 +899,18 @@ pub fn get_replication_state(rinfos: &ReplicatedInfos, prev_state: &ReplicationS
.map(|t| (target_reset_header(t.arn.as_str()), t.resync_timestamp.clone()))
.collect();
let repl_statuses = rinfos.replication_status_internal();
let vpurge_statuses = rinfos.version_purge_status_internal();
let mut targets = prev_state.targets.clone();
for (arn, status) in replication_statuses_map(&rinfos.replication_status_internal().unwrap_or_default()) {
targets.insert(arn, status);
}
let mut purge_targets = prev_state.purge_targets.clone();
for (arn, status) in version_purge_statuses_map(&rinfos.version_purge_status_internal().unwrap_or_default()) {
purge_targets.insert(arn, status);
}
let repl_statuses = replication_statuses_string(&targets);
let vpurge_statuses = version_purge_statuses_string(&purge_targets);
let mut reset_statuses_map = prev_state.reset_statuses_map.clone();
for (key, value) in reset_status_map {
@@ -883,10 +922,10 @@ pub fn get_replication_state(rinfos: &ReplicatedInfos, prev_state: &ReplicationS
reset_statuses_map,
replica_timestamp: prev_state.replica_timestamp,
replica_status: prev_state.replica_status.clone(),
targets: replication_statuses_map(&repl_statuses.clone().unwrap_or_default()),
targets,
replication_status_internal: repl_statuses,
replication_timestamp: rinfos.replication_timestamp,
purge_targets: version_purge_statuses_map(&vpurge_statuses.clone().unwrap_or_default()),
purge_targets,
version_purge_status_internal: vpurge_statuses,
..Default::default()
@@ -934,3 +973,47 @@ impl Default for ResyncDecision {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn target_state_reads_resync_timestamp_from_target_reset_header_key() {
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
let timestamp = "2026-06-30T00:00:00Z;reset-1".to_string();
let mut state = ReplicationState::default();
state.reset_statuses_map.insert(target_reset_header(arn), timestamp.clone());
let target_state = state.target_state(arn);
assert_eq!(target_state.resync_timestamp, timestamp);
}
#[test]
fn get_replication_state_preserves_untouched_target_statuses() {
let target_a = "arn:target:a".to_string();
let target_b = "arn:target:b".to_string();
let mut prev_state = ReplicationState::default();
prev_state.targets.insert(target_a.clone(), ReplicationStatusType::Failed);
prev_state.targets.insert(target_b.clone(), ReplicationStatusType::Completed);
let rinfos = ReplicatedInfos {
replication_timestamp: None,
targets: vec![ReplicatedTargetInfo {
arn: target_a.clone(),
replication_status: ReplicationStatusType::Completed,
..Default::default()
}],
};
let state = get_replication_state(&rinfos, &prev_state, None);
assert_eq!(state.targets.get(&target_a), Some(&ReplicationStatusType::Completed));
assert_eq!(state.targets.get(&target_b), Some(&ReplicationStatusType::Completed));
assert_eq!(
replication_statuses_map(&state.replication_status_internal.unwrap_or_default()).get(&target_b),
Some(&ReplicationStatusType::Completed)
);
}
}