feat(site-replication): persist durable resync lifecycle (#5125)

crates/e2e_test/src/replication_extension_test.rs

@@ -4048,17 +4048,23 @@ async fn test_site_replication_allows_private_ca_https_with_ca_cert_pem_real_dua
 
 #[tokio::test]
 #[serial]
-async fn test_site_replication_resync_start_cancel_restart_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
+async fn test_site_replication_resync_lifecycle_survives_real_server_restart() -> Result<(), Box<dyn Error + Send + Sync>> {
     init_logging();
+    let resync_process_env = [
+        ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", "true"),
+        // Verbose server logging can block startup when this focused test is run
+        // through a captured test process rather than nextest.
+        ("RUST_LOG", "error"),
+    ];
 
     let mut source_env = RustFSTestEnvironment::new().await?;
-    source_env
-        .start_rustfs_server_with_env(vec![], LOOPBACK_REPLICATION_TARGET_ENV)
-        .await?;
+    source_env.capture_log_path = Some(format!("{}/server.log", source_env.temp_dir));
+    source_env.start_rustfs_server_with_env(vec![], &resync_process_env).await?;
 
     let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.capture_log_path = Some(format!("{}/server.log", target_env.temp_dir));
     target_env
-        .start_rustfs_server_without_cleanup_with_env(LOOPBACK_REPLICATION_TARGET_ENV)
+        .start_rustfs_server_without_cleanup_with_env(&resync_process_env)
         .await?;
 
     let source_bucket = "site-repl-resync-src";
@@ -4106,12 +4112,12 @@ async fn test_site_replication_resync_start_cancel_restart_real_dual_node() -> R
     wait_for_bucket_on_target(&source_client, source_bucket).await?;
     let target_arn = wait_for_remote_target_arn(&source_env, source_bucket).await?;
 
-    for idx in 0..32 {
+    for idx in 0..96 {
         source_client
             .put_object()
             .bucket(source_bucket)
             .key(format!("resync-object-{idx:02}"))
-            .body(ByteStream::from(vec![b'x'; 256 * 1024]))
+            .body(ByteStream::from(vec![b'x'; 512 * 1024]))
             .send()
             .await?;
     }
@@ -4119,18 +4125,31 @@ async fn test_site_replication_resync_start_cancel_restart_real_dual_node() -> R
     let started = site_replication_resync_op(&source_env, "start", &remote_peer).await?;
     assert_eq!(started.status, "success", "unexpected start result: {:?}", started);
     assert!(
-        started
-            .buckets
-            .iter()
-            .any(|bucket| bucket.bucket == source_bucket && matches!(bucket.status.as_str(), "started" | "success")),
+        started.buckets.iter().any(|bucket| {
+            bucket.bucket == source_bucket && matches!(bucket.status.as_str(), "started" | "running" | "completed" | "success")
+        }),
         "source bucket start status missing: {:?}",
         started
     );
     assert!(!started.resync_id.is_empty(), "start response omitted the resync id: {:?}", started);
     let started_reset_id = started.resync_id.clone();
 
+    assert!(
+        matches!(started.state.as_str(), "pending" | "running"),
+        "the fixture must keep the first generation active long enough to test duplicate start: {:?}",
+        started
+    );
+    let duplicate_err = site_replication_resync_op(&source_env, "start", &remote_peer)
+        .await
+        .expect_err("duplicate start must be rejected while a generation is active");
+    assert!(
+        duplicate_err.to_string().contains("already active"),
+        "unexpected duplicate start error: {duplicate_err}"
+    );
+
     let canceled = site_replication_resync_op(&source_env, "cancel", &remote_peer).await?;
     assert_eq!(canceled.status, "success", "unexpected cancel result: {:?}", canceled);
+    assert_eq!(canceled.state, "canceled");
     assert!(
         canceled
             .buckets
@@ -4139,34 +4158,56 @@ async fn test_site_replication_resync_start_cancel_restart_real_dual_node() -> R
         "source bucket cancel status missing: {:?}",
         canceled
     );
+    let canceled_again = site_replication_resync_op(&source_env, "cancel", &remote_peer).await?;
+    assert_eq!(canceled_again.resync_id, canceled.resync_id, "repeated cancel must be idempotent");
+    assert_eq!(canceled_again.state, "canceled");
 
     let canceled_target =
         wait_for_replication_reset_target(&source_env, source_bucket, &target_arn, |target| target.status == "Canceled").await?;
-    assert_eq!(canceled_target.status, "Canceled");
     assert_eq!(canceled_target.reset_id, started_reset_id);
 
     let restarted = site_replication_resync_op(&source_env, "start", &remote_peer).await?;
     assert_eq!(restarted.status, "success", "unexpected restart result: {:?}", restarted);
+    assert_ne!(restarted.resync_id, started_reset_id);
     assert!(
+        matches!(restarted.state.as_str(), "pending" | "running"),
+        "the second generation must be active before the process restart: {:?}",
         restarted
-            .buckets
-            .iter()
-            .any(|bucket| bucket.bucket == source_bucket && matches!(bucket.status.as_str(), "started" | "success")),
-        "source bucket restart status missing: {:?}",
-        restarted
+    );
+    let restarted_reset_id = restarted.resync_id.clone();
+
+    source_env.restart_server_preserving_data(vec![], &resync_process_env).await?;
+    wait_for_site_replication_enabled(&source_env, 2).await?;
+
+    let after_restart = site_replication_resync_op(&source_env, "status", &remote_peer).await?;
+    assert_eq!(
+        after_restart.resync_id, restarted_reset_id,
+        "server restart changed the durable resync id"
+    );
+    assert_eq!(after_restart.generation, restarted.generation);
+    assert_eq!(after_restart.created_at, restarted.created_at);
+    assert!(
+        matches!(after_restart.state.as_str(), "pending" | "running" | "completed" | "failed"),
+        "unexpected recovered lifecycle state: {:?}",
+        after_restart
+    );
+    assert!(
+        after_restart.buckets.iter().any(|bucket| bucket.bucket == source_bucket),
+        "durable status lost the source bucket after restart: {:?}",
+        after_restart
     );
     let restart_snapshot = get_replication_reset_status(&source_env, source_bucket, &target_arn).await?;
     let restarted_target = wait_for_replication_reset_target(&source_env, source_bucket, &target_arn, |target| {
-        !target.reset_id.is_empty() && target.reset_id != started_reset_id
+        target.reset_id == restarted_reset_id
     })
     .await
     .map_err(|err| {
         format!(
             "restart ids: start={} restart={} snapshot={:?}; {err}",
-            started_reset_id, restarted.resync_id, restart_snapshot.targets
+            started_reset_id, restarted_reset_id, restart_snapshot.targets
         )
     })?;
-    assert_ne!(restarted_target.reset_id, started_reset_id);
+    assert_eq!(restarted_target.reset_id, restarted_reset_id);
 
     Ok(())
 }

crates/madmin/src/site_replication.rs

@@ -1156,10 +1156,50 @@ pub struct SRStateEditReq {
 pub struct ResyncBucketStatus {
     #[serde(default)]
     pub bucket: String,
+    #[serde(rename = "targetArn", default, skip_serializing_if = "String::is_empty")]
+    pub target_arn: String,
     #[serde(default)]
     pub status: String,
     #[serde(rename = "errorDetail", skip_serializing_if = "String::is_empty", default)]
     pub err_detail: String,
+    #[serde(
+        rename = "createdAt",
+        default,
+        with = "time::serde::rfc3339::option",
+        skip_serializing_if = "Option::is_none"
+    )]
+    pub created_at: Option<OffsetDateTime>,
+    #[serde(
+        rename = "startedAt",
+        default,
+        with = "time::serde::rfc3339::option",
+        skip_serializing_if = "Option::is_none"
+    )]
+    pub started_at: Option<OffsetDateTime>,
+    #[serde(
+        rename = "updatedAt",
+        default,
+        with = "time::serde::rfc3339::option",
+        skip_serializing_if = "Option::is_none"
+    )]
+    pub updated_at: Option<OffsetDateTime>,
+    #[serde(
+        rename = "completedAt",
+        default,
+        with = "time::serde::rfc3339::option",
+        skip_serializing_if = "Option::is_none"
+    )]
+    pub completed_at: Option<OffsetDateTime>,
+    #[serde(default, skip_serializing_if = "is_zero_u64")]
+    pub generation: u64,
+    #[serde(rename = "replicatedObjects", default, skip_serializing_if = "is_zero_u64")]
+    pub replicated_objects: u64,
+    #[serde(rename = "replicatedBytes", default, skip_serializing_if = "is_zero_u64")]
+    pub replicated_bytes: u64,
+    #[serde(rename = "failedObjects", default, skip_serializing_if = "is_zero_u64")]
+    pub failed_objects: u64,
+    #[serde(rename = "failedBytes", default, skip_serializing_if = "is_zero_u64")]
+    pub failed_bytes: u64,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
@@ -1170,10 +1210,74 @@ pub struct SRResyncOpStatus {
     pub resync_id: String,
     #[serde(default)]
     pub status: String,
+    #[serde(default, skip_serializing_if = "String::is_empty")]
+    pub state: String,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub buckets: Vec<ResyncBucketStatus>,
     #[serde(rename = "errorDetail", skip_serializing_if = "String::is_empty", default)]
     pub err_detail: String,
+    #[serde(
+        rename = "createdAt",
+        default,
+        with = "time::serde::rfc3339::option",
+        skip_serializing_if = "Option::is_none"
+    )]
+    pub created_at: Option<OffsetDateTime>,
+    #[serde(
+        rename = "startedAt",
+        default,
+        with = "time::serde::rfc3339::option",
+        skip_serializing_if = "Option::is_none"
+    )]
+    pub started_at: Option<OffsetDateTime>,
+    #[serde(
+        rename = "updatedAt",
+        default,
+        with = "time::serde::rfc3339::option",
+        skip_serializing_if = "Option::is_none"
+    )]
+    pub updated_at: Option<OffsetDateTime>,
+    #[serde(
+        rename = "completedAt",
+        default,
+        with = "time::serde::rfc3339::option",
+        skip_serializing_if = "Option::is_none"
+    )]
+    pub completed_at: Option<OffsetDateTime>,
+    #[serde(default, skip_serializing_if = "is_zero_u64")]
+    pub generation: u64,
+    #[serde(rename = "totalBuckets", default, skip_serializing_if = "is_zero_u64")]
+    pub total_buckets: u64,
+    #[serde(rename = "pendingBuckets", default, skip_serializing_if = "is_zero_u64")]
+    pub pending_buckets: u64,
+    #[serde(rename = "runningBuckets", default, skip_serializing_if = "is_zero_u64")]
+    pub running_buckets: u64,
+    #[serde(rename = "completedBuckets", default, skip_serializing_if = "is_zero_u64")]
+    pub completed_buckets: u64,
+    #[serde(rename = "failedBuckets", default, skip_serializing_if = "is_zero_u64")]
+    pub failed_buckets: u64,
+    #[serde(rename = "canceledBuckets", default, skip_serializing_if = "is_zero_u64")]
+    pub canceled_buckets: u64,
+    #[serde(rename = "replicatedObjects", default, skip_serializing_if = "is_zero_u64")]
+    pub replicated_objects: u64,
+    #[serde(rename = "replicatedBytes", default, skip_serializing_if = "is_zero_u64")]
+    pub replicated_bytes: u64,
+    #[serde(rename = "failedObjects", default, skip_serializing_if = "is_zero_u64")]
+    pub failed_objects: u64,
+    #[serde(rename = "failedBytes", default, skip_serializing_if = "is_zero_u64")]
+    pub failed_bytes: u64,
+    #[serde(default, skip_serializing_if = "is_false")]
+    pub truncated: bool,
+    #[serde(rename = "nextContinuationToken", default, skip_serializing_if = "String::is_empty")]
+    pub next_continuation_token: String,
+}
+
+fn is_zero_u64(value: &u64) -> bool {
+    *value == 0
+}
+
+fn is_false(value: &bool) -> bool {
+    !*value
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
@@ -1202,7 +1306,7 @@ pub struct SiteNetPerfResult {
 
 #[cfg(test)]
 mod tests {
-    use super::{PeerInfo, PeerSite};
+    use super::{PeerInfo, PeerSite, SRResyncOpStatus};
     use serde_json::{Value, json};
 
     const TEST_CA_CERT: &str = "-----BEGIN CERTIFICATE-----\ntest-ca\n-----END CERTIFICATE-----";
@@ -1302,4 +1406,94 @@ mod tests {
         assert!(peer_debug.contains("skip_tls_verify: false"));
         assert!(peer_debug.contains("has_custom_ca: true"));
     }
+
+    #[test]
+    fn resync_status_legacy_json_defaults_new_lifecycle_fields() {
+        let legacy_json = json!({
+            "op": "start",
+            "id": "resync-1",
+            "status": "success",
+            "buckets": [{
+                "bucket": "photos",
+                "status": "success"
+            }]
+        });
+
+        let status: SRResyncOpStatus =
+            serde_json::from_value(legacy_json.clone()).expect("legacy resync status should deserialize");
+
+        assert_eq!(status.generation, 0);
+        assert!(status.state.is_empty());
+        assert!(status.created_at.is_none());
+        assert_eq!(status.total_buckets, 0);
+        assert_eq!(status.replicated_objects, 0);
+        assert!(!status.truncated);
+        assert!(status.next_continuation_token.is_empty());
+        assert!(status.buckets[0].created_at.is_none());
+        assert!(status.buckets[0].target_arn.is_empty());
+        assert_eq!(status.buckets[0].generation, 0);
+        assert_eq!(status.buckets[0].replicated_bytes, 0);
+        assert_eq!(serde_json::to_value(status).expect("legacy resync status should serialize"), legacy_json);
+    }
+
+    #[test]
+    fn resync_status_lifecycle_fields_round_trip_with_exact_json_names() {
+        let status_json = json!({
+            "op": "status",
+            "id": "resync-2",
+            "status": "success",
+            "state": "running",
+            "createdAt": "2026-07-22T01:00:00Z",
+            "startedAt": "2026-07-22T01:00:01Z",
+            "updatedAt": "2026-07-22T01:01:00Z",
+            "completedAt": "2026-07-22T01:02:00Z",
+            "generation": 7,
+            "totalBuckets": 6,
+            "pendingBuckets": 1,
+            "runningBuckets": 1,
+            "completedBuckets": 1,
+            "failedBuckets": 1,
+            "canceledBuckets": 2,
+            "replicatedObjects": 12,
+            "replicatedBytes": 4096,
+            "failedObjects": 3,
+            "failedBytes": 512,
+            "truncated": true,
+            "nextContinuationToken": "bucket-page-2",
+            "buckets": [{
+                "bucket": "photos",
+                "targetArn": "arn:rustfs:replication::peer-a:photos",
+                "status": "failed",
+                "errorDetail": "target unavailable",
+                "createdAt": "2026-07-22T01:00:00Z",
+                "startedAt": "2026-07-22T01:00:01Z",
+                "updatedAt": "2026-07-22T01:01:00Z",
+                "completedAt": "2026-07-22T01:02:00Z",
+                "generation": 7,
+                "replicatedObjects": 12,
+                "replicatedBytes": 4096,
+                "failedObjects": 3,
+                "failedBytes": 512
+            }]
+        });
+
+        let status: SRResyncOpStatus =
+            serde_json::from_value(status_json.clone()).expect("expanded resync status should deserialize");
+
+        assert_eq!(status.generation, 7);
+        assert_eq!(status.state, "running");
+        assert_eq!(status.total_buckets, 6);
+        assert_eq!(status.completed_buckets, 1);
+        assert_eq!(status.replicated_bytes, 4096);
+        assert!(status.truncated);
+        assert_eq!(status.next_continuation_token, "bucket-page-2");
+        assert_eq!(status.buckets[0].generation, 7);
+        assert_eq!(status.buckets[0].target_arn, "arn:rustfs:replication::peer-a:photos");
+        assert_eq!(status.buckets[0].failed_objects, 3);
+        assert!(status.buckets[0].completed_at.is_some());
+        assert_eq!(
+            serde_json::to_value(status).expect("expanded resync status should serialize"),
+            status_json
+        );
+    }
 }

crates/replication/src/resync.rs

@@ -154,7 +154,7 @@ impl TargetReplicationResyncStatus {
     }
 
     fn marshal_wire_msg(&self, wr: &mut Vec<u8>) -> Result<()> {
-        rmp::encode::write_map_len(wr, 11)?;
+        rmp::encode::write_map_len(wr, 12)?;
         rmp::encode::write_str(wr, "st")?;
         write_msgp_time(wr, wire_time_or_default(self.start_time))?;
         rmp::encode::write_str(wr, "lst")?;
@@ -177,6 +177,8 @@ impl TargetReplicationResyncStatus {
         rmp::encode::write_str(wr, &self.bucket)?;
         rmp::encode::write_str(wr, "obj")?;
         rmp::encode::write_str(wr, &self.object)?;
+        rmp::encode::write_str(wr, "err")?;
+        rmp::encode::write_str(wr, self.error.as_deref().unwrap_or_default())?;
         Ok(())
     }
 
@@ -202,6 +204,10 @@ impl TargetReplicationResyncStatus {
                 "rrc" => out.replicated_count = rmp::decode::read_int(rd)?,
                 "bkt" => out.bucket = read_msgp_str(rd)?,
                 "obj" => out.object = read_msgp_str(rd)?,
+                "err" => {
+                    let error = read_msgp_str(rd)?;
+                    out.error = (!error.is_empty()).then_some(error);
+                }
                 _ => skip_msgp_value(rd)?,
             }
         }
@@ -623,6 +629,7 @@ mod tests {
                 bucket: "bucket-a".to_string(),
                 object: "object-a".to_string(),
                 replicated_count: 7,
+                error: Some("durable failure".to_string()),
                 ..Default::default()
             },
         );
@@ -634,6 +641,7 @@ mod tests {
         assert_eq!(got.targets_map["arn:replication:a"].resync_id, "rid-1");
         assert_eq!(got.targets_map["arn:replication:a"].resync_status, ResyncStatusType::ResyncStarted);
         assert_eq!(got.targets_map["arn:replication:a"].replicated_count, 7);
+        assert_eq!(got.targets_map["arn:replication:a"].error.as_deref(), Some("durable failure"));
     }
 
     #[test]