fix(replication): avoid re-replication loop in Active-Active replication (#1751)

Co-authored-by: loverustfs <hello@rustfs.com>

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -64,6 +64,8 @@ use uuid::Uuid;
 const DEFAULT_HEALTH_CHECK_DURATION: Duration = Duration::from_secs(5);
 const DEFAULT_HEALTH_CHECK_RELOAD_DURATION: Duration = Duration::from_secs(30 * 60);
 
+const REPLICATION_REQUEST_TRUE: HeaderValue = HeaderValue::from_static("true");
+
 pub static GLOBAL_BUCKET_TARGET_SYS: OnceLock<BucketTargetSys> = OnceLock::new();
 
 #[derive(Debug, Clone)]
@@ -999,10 +1001,14 @@ impl PutObjectOptions {
         if self.internal.source_mtime.unix_timestamp() != 0 {
             header.insert(
                 RUSTFS_BUCKET_SOURCE_MTIME,
-                HeaderValue::from_str(&self.internal.source_mtime.unix_timestamp().to_string()).expect("err"),
+                HeaderValue::from_str(&self.internal.source_mtime.format(&Rfc3339).unwrap_or_default()).expect("err"),
             );
         }
 
+        if self.internal.replication_request {
+            header.insert(RUSTFS_BUCKET_REPLICATION_REQUEST, REPLICATION_REQUEST_TRUE);
+        }
+
         header
     }
 
@@ -1211,6 +1217,9 @@ impl TargetClient {
         {
             headers.insert(RUSTFS_BUCKET_SOURCE_VERSION_ID, header_value);
         }
+        if opts.internal.replication_request {
+            headers.insert(RUSTFS_BUCKET_REPLICATION_REQUEST, REPLICATION_REQUEST_TRUE);
+        }
 
         match self
             .client

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -1684,10 +1684,6 @@ pub async fn replicate_object<S: StorageAPI>(roi: ReplicateObjectInfo, storage:
         if let Some(ref s) = new_replication_internal {
             eval_metadata.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}replication-status"), s.clone());
         }
-        eval_metadata.insert(
-            format!("{RESERVED_METADATA_PREFIX_LOWER}replication-timestamp"),
-            OffsetDateTime::now_utc().format(&Rfc3339).unwrap_or_default(),
-        );
         let popts = ObjectOptions {
             version_id: roi.version_id.map(|v| v.to_string()),
             eval_metadata: Some(eval_metadata),
@@ -1840,6 +1836,36 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
             return rinfo;
         }
 
+        let mut replication_action = replication_action;
+        match tgt_client
+            .head_object(&tgt_client.bucket, &object, self.version_id.map(|v| v.to_string()))
+            .await
+        {
+            Ok(oi) => {
+                replication_action = get_replication_action(&object_info, &oi, self.op_type);
+                if replication_action == ReplicationAction::None {
+                    rinfo.replication_status = ReplicationStatusType::Completed;
+                    rinfo.replication_resynced = true;
+                    rinfo.replication_action = ReplicationAction::None;
+                    rinfo.size = size;
+                    return rinfo;
+                }
+            }
+            Err(e) => {
+                if let Some(se) = e.as_service_error() {
+                    if !se.is_not_found() {
+                        rinfo.error = Some(e.to_string());
+                        warn!("replication head_object failed bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e);
+                        return rinfo;
+                    }
+                } else {
+                    rinfo.error = Some(e.to_string());
+                    warn!("replication head_object failed bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e);
+                    return rinfo;
+                }
+            }
+        }
+
         rinfo.replication_status = ReplicationStatusType::Completed;
         rinfo.replication_resynced = true;
         rinfo.size = size;

crates/ecstore/src/set_disk.rs

@@ -4786,7 +4786,9 @@ impl StorageAPI for SetDisks {
             }
         }
 
-        fi.mod_time = opts.mod_time;
+        if opts.mod_time.is_some() {
+            fi.mod_time = opts.mod_time;
+        }
         if let Some(ref version_id) = opts.version_id {
             fi.version_id = Uuid::parse_str(version_id).ok();
         }

crates/utils/src/http/headers.rs

@@ -175,7 +175,7 @@ pub const RUSTFS_REPLICATION_RESET_STATUS: &str = "X-Rustfs-Replication-Reset-St
 pub const RUSTFS_REPLICATION_ACTUAL_OBJECT_SIZE: &str = "X-Rustfs-Replication-Actual-Object-Size";
 
 pub const RUSTFS_BUCKET_SOURCE_VERSION_ID: &str = "X-Rustfs-Source-Version-Id";
-pub const RUSTFS_BUCKET_SOURCE_MTIME: &str = "X-Rustfs-Source-Mtime";
+pub const RUSTFS_BUCKET_SOURCE_MTIME: &str = "X-RustFS-Source-Mtime";
 pub const RUSTFS_BUCKET_SOURCE_ETAG: &str = "X-Rustfs-Source-Etag";
 pub const RUSTFS_BUCKET_REPLICATION_DELETE_MARKER: &str = "X-Rustfs-Source-DeleteMarker";
 pub const RUSTFS_BUCKET_REPLICATION_PROXY_REQUEST: &str = "X-Rustfs-Source-Proxy-Request";