fix(replication): make sync delivery target-granular (#5630)

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -1823,12 +1823,20 @@ pub(crate) async fn schedule_replication<S: ReplicationStorage>(
     dsc: ReplicateDecision,
     op_type: ReplicationType,
 ) {
-    let synchronous = dsc.is_synchronous();
+    let (synchronous, asynchronous) = dsc.partition_by_sync();
-    let ri = replicate_object_info_from_object_info(oi, dsc, op_type);
+    let mut async_oi = oi;
 
-    if synchronous {
+    if synchronous.replicate_any() {
-        replicate_object(ri, o).await
+        let ri = replicate_object_info_from_object_info(async_oi.clone(), synchronous, op_type);
-    } else if let Some(pool) = runtime_sources::replication_pool() {
+        let state = replicate_object(ri, o.clone()).await;
+        async_oi.replication_status_internal = state.replication_status_internal;
+        async_oi.version_purge_status_internal = state.version_purge_status_internal;
+    }
+
+    if asynchronous.replicate_any()
+        && let Some(pool) = runtime_sources::replication_pool()
+    {
+        let ri = replicate_object_info_from_object_info(async_oi, asynchronous, op_type);
         let _ = pool.queue_replica_task(ri).await;
     }
 }

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -19,7 +19,7 @@ use super::replication_error_boundary::{Result, is_err_object_not_found, is_err_
 use super::replication_event_sink::{EventArgs, send_event, send_local_event};
 use super::replication_filemeta_boundary::{
     NULL_VERSION_ID, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedInfos,
-    ReplicatedTargetInfo, ReplicationAction, ReplicationStatusType, ReplicationType, VersionPurgeStatusType,
+    ReplicatedTargetInfo, ReplicationAction, ReplicationState, ReplicationStatusType, ReplicationType, VersionPurgeStatusType,
     get_replication_state, parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map,
 };
 use super::replication_lock_boundary::ReplicationLockTiming;
@@ -2001,61 +2001,14 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
     rinfo
 }
 
-pub async fn replicate_object<S: ReplicationStorage>(roi: ReplicateObjectInfo, storage: Arc<S>) {
+pub async fn replicate_object<S: ReplicationStorage>(roi: ReplicateObjectInfo, storage: Arc<S>) -> ReplicationState {
     let bucket = roi.bucket.clone();
     let object = roi.name.clone();
 
-    let cfg = match get_replication_config(&bucket).await {
+    // The admission decision is the target-granular contract. Re-evaluating the
-        Ok(Some(config)) => config,
+    // live config here could fan a synchronous request out to targets that were
-        Ok(None) => {
+    // not admitted, or promote an async target after a mixed-mode split.
-            debug!(
+    let tgt_arns = roi.dsc.replicate_target_arns();
-                event = EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED,
-                component = LOG_COMPONENT_ECSTORE,
-                subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
-                bucket = %bucket,
-                reason = "replication_config_missing",
-                "Skipping replication object because replication config is missing"
-            );
-            send_local_event(EventArgs {
-                event_name: EventName::ObjectReplicationNotTracked.to_string(),
-                bucket_name: bucket.clone(),
-                object: roi.to_object_info(),
-                user_agent: "Internal: [Replication]".to_string(),
-                ..Default::default()
-            });
-            return;
-        }
-        Err(err) => {
-            error!(
-                event = EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED,
-                component = LOG_COMPONENT_ECSTORE,
-                subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
-                bucket = %bucket,
-                reason = "replication_config_lookup_failed",
-                error = %err,
-                "Failed to look up replication config for object replication"
-            );
-            send_local_event(EventArgs {
-                event_name: EventName::ObjectReplicationNotTracked.to_string(),
-                bucket_name: bucket.clone(),
-                object: roi.to_object_info(),
-                user_agent: "Internal: [Replication]".to_string(),
-                ..Default::default()
-            });
-            return;
-        }
-    };
-
-    let tgt_arns = cfg.filter_target_arns(&ObjectOpts {
-        name: object.clone(),
-        user_tags: roi.user_tags.clone(),
-        ssec: roi.ssec,
-        op_type: roi.op_type,
-        // ExistingObject ops must respect per-rule ExistingObjectReplicationStatus.
-        // Heal ops intentionally bypass it (repairing a past failure is not an initial sync).
-        existing_object: roi.op_type == ReplicationType::ExistingObject,
-        ..Default::default()
-    });
 
     // Acquire a per-object namespace lock so that at most one worker (across all cluster
     // nodes and MRF retry goroutines) replicates this object version at a time.
@@ -2080,7 +2033,7 @@ pub async fn replicate_object<S: ReplicationStorage>(roi: ReplicateObjectInfo, s
                 user_agent: "Internal: [Replication]".to_string(),
                 ..Default::default()
             });
-            return;
+            return roi.replication_state.unwrap_or_default();
         }
     };
     let _obj_lock_guard = match obj_ns_lock.get_write_lock(ReplicationLockTiming::acquire_timeout()).await {
@@ -2103,7 +2056,7 @@ pub async fn replicate_object<S: ReplicationStorage>(roi: ReplicateObjectInfo, s
                 user_agent: "Internal: [Replication]".to_string(),
                 ..Default::default()
             });
-            return;
+            return roi.replication_state.unwrap_or_default();
         }
     };
 
@@ -2179,8 +2132,10 @@ pub async fn replicate_object<S: ReplicationStorage>(roi: ReplicateObjectInfo, s
         }
     }
 
-    let replication_status = rinfos.replication_status();
+    let previous_state = roi.replication_state.clone().unwrap_or_default();
-    let new_replication_internal = rinfos.replication_status_internal();
+    let merged_state = get_replication_state(&rinfos, &previous_state, roi.version_id.map(|v| v.to_string()));
+    let replication_status = merged_state.composite_replication_status();
+    let new_replication_internal = merged_state.replication_status_internal.clone();
     let mut object_info = roi.to_object_info();
 
     if roi.replication_status_internal != new_replication_internal || rinfos.replication_resynced() {
@@ -2249,6 +2204,8 @@ pub async fn replicate_object<S: ReplicationStorage>(roi: ReplicateObjectInfo, s
             }
         }
     }
+
+    merged_state
 }
 
 trait ReplicateObjectInfoExt {

crates/replication/src/filemeta.rs

@@ -660,6 +660,27 @@ impl ReplicateDecision {
         self.targets_map.values().any(|t| t.synchronous)
     }
 
+    /// Split admitted targets by their configured delivery mode.
+    ///
+    /// Non-replicating entries are intentionally omitted from both decisions.
+    /// Callers must not promote an async target merely because another target is
+    /// synchronous, and unsupported operation paths can keep both partitions
+    /// empty or explicitly async.
+    pub fn partition_by_sync(&self) -> (Self, Self) {
+        let mut synchronous = Self::new();
+        let mut asynchronous = Self::new();
+
+        for target in self.targets_map.values().filter(|target| target.replicate) {
+            if target.synchronous {
+                synchronous.set(target.clone());
+            } else {
+                asynchronous.set(target.clone());
+            }
+        }
+
+        (synchronous, asynchronous)
+    }
+
     /// Updates ReplicateDecision with target's replication decision
     pub fn set(&mut self, target: ReplicateTargetDecision) {
         self.targets_map.insert(target.arn.clone(), target);
@@ -1067,6 +1088,32 @@ mod tests {
         assert_eq!(entry.target_arns, vec!["arn:target-a".to_string()]);
     }
 
+    #[test]
+    fn partition_by_sync_keeps_mixed_targets_independent() {
+        let mut decision = ReplicateDecision::new();
+        decision.set(ReplicateTargetDecision::new("arn:sync".to_string(), true, true));
+        decision.set(ReplicateTargetDecision::new("arn:async".to_string(), true, false));
+        decision.set(ReplicateTargetDecision::new("arn:disabled".to_string(), false, true));
+
+        let (synchronous, asynchronous) = decision.partition_by_sync();
+
+        assert_eq!(synchronous.replicate_target_arns(), vec!["arn:sync".to_string()]);
+        assert_eq!(asynchronous.replicate_target_arns(), vec!["arn:async".to_string()]);
+        assert!(synchronous.is_synchronous());
+        assert!(!asynchronous.is_synchronous());
+    }
+
+    #[test]
+    fn partition_by_sync_does_not_promote_async_targets() {
+        let mut decision = ReplicateDecision::new();
+        decision.set(ReplicateTargetDecision::new("arn:async".to_string(), true, false));
+
+        let (synchronous, asynchronous) = decision.partition_by_sync();
+
+        assert!(!synchronous.replicate_any());
+        assert_eq!(asynchronous.replicate_target_arns(), vec!["arn:async".to_string()]);
+    }
+
     #[test]
     fn target_state_reads_resync_timestamp_from_target_reset_header_key() {
         let arn = "arn:rustfs:replication:us-east-1:target:bucket";