feat(admin): complete site replication support (#2346)

crates/e2e_test/src/replication_extension_test.rs

@@ -13,14 +13,38 @@
 // limitations under the License.
 
 use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
+use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
 use http::header::{CONTENT_TYPE, HOST};
 use reqwest::StatusCode;
+use rustfs_madmin::{
+    PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus, ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus,
+    SRStatusInfo, SiteReplicationInfo, SyncStatus,
+};
 use rustfs_signer::constants::UNSIGNED_PAYLOAD;
 use rustfs_signer::sign_v4;
 use s3s::Body;
 use serial_test::serial;
+use std::collections::BTreeMap;
 use std::error::Error;
+use time::Duration as TimeDuration;
+use tokio::time::{Duration, sleep};
+
+#[derive(Debug, Clone, serde::Deserialize)]
+struct ReplicationResetStatusResponse {
+    #[serde(rename = "Targets", default)]
+    targets: Vec<ReplicationResetStatusTarget>,
+}
+
+#[derive(Debug, Clone, serde::Deserialize)]
+struct ReplicationResetStatusTarget {
+    #[serde(rename = "Arn", default)]
+    arn: String,
+    #[serde(rename = "ResetID", default)]
+    reset_id: String,
+    #[serde(rename = "Status", default)]
+    status: String,
+}
 
 async fn signed_request(
     method: http::Method,
@@ -239,6 +263,265 @@ async fn list_replication_targets_request(
     signed_request(http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await
 }
 
+async fn site_replication_add(
+    env: &RustFSTestEnvironment,
+    sites: &[PeerSite],
+) -> Result<ReplicateAddStatus, Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/site-replication/add?replicateILMExpiry=false", env.url);
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(serde_json::to_vec(sites)?),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("site replication add failed: {status} {body}").into());
+    }
+
+    Ok(serde_json::from_slice(&response.bytes().await?)?)
+}
+
+async fn site_replication_info(env: &RustFSTestEnvironment) -> Result<SiteReplicationInfo, Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/site-replication/info", env.url);
+    let response = signed_request(http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("site replication info failed: {status} {body}").into());
+    }
+
+    Ok(serde_json::from_slice(&response.bytes().await?)?)
+}
+
+async fn site_replication_resync_op(
+    env: &RustFSTestEnvironment,
+    operation: &str,
+    peer: &PeerInfo,
+) -> Result<SRResyncOpStatus, Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/site-replication/resync/op?operation={operation}", env.url);
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(serde_json::to_vec(peer)?),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("site replication resync {operation} failed: {status} {body}").into());
+    }
+
+    Ok(serde_json::from_slice(&response.bytes().await?)?)
+}
+
+async fn site_replication_edit(
+    env: &RustFSTestEnvironment,
+    query: &str,
+    peer: &PeerInfo,
+) -> Result<ReplicateEditStatus, Box<dyn Error + Send + Sync>> {
+    let url = if query.is_empty() {
+        format!("{}/rustfs/admin/v3/site-replication/edit", env.url)
+    } else {
+        format!("{}/rustfs/admin/v3/site-replication/edit?{query}", env.url)
+    };
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(serde_json::to_vec(peer)?),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("site replication edit failed: {status} {body}").into());
+    }
+
+    Ok(serde_json::from_slice(&response.bytes().await?)?)
+}
+
+async fn site_replication_status(env: &RustFSTestEnvironment, query: &str) -> Result<SRStatusInfo, Box<dyn Error + Send + Sync>> {
+    let url = if query.is_empty() {
+        format!("{}/rustfs/admin/v3/site-replication/status", env.url)
+    } else {
+        format!("{}/rustfs/admin/v3/site-replication/status?{query}", env.url)
+    };
+    let response = signed_request(http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("site replication status failed: {status} {body}").into());
+    }
+
+    Ok(serde_json::from_slice(&response.bytes().await?)?)
+}
+
+async fn site_replication_remove(
+    env: &RustFSTestEnvironment,
+    req: &SRRemoveReq,
+) -> Result<ReplicateRemoveStatus, Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/site-replication/remove", env.url);
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(serde_json::to_vec(req)?),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("site replication remove failed: {status} {body}").into());
+    }
+
+    Ok(serde_json::from_slice(&response.bytes().await?)?)
+}
+
+async fn site_replication_state_edit(
+    env: &RustFSTestEnvironment,
+    body: &rustfs_madmin::SRStateEditReq,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/site-replication/state/edit", env.url);
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(serde_json::to_vec(body)?),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("site replication state edit failed: {status} {body}").into());
+    }
+
+    Ok(())
+}
+
+async fn get_replication_reset_status(
+    env: &RustFSTestEnvironment,
+    bucket: &str,
+    arn: &str,
+) -> Result<ReplicationResetStatusResponse, Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/{bucket}?replication-reset-status&arn={}", env.url, urlencoding::encode(arn));
+    let response = signed_request(http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("replication reset status failed: {status} {body}").into());
+    }
+
+    Ok(serde_json::from_slice(&response.bytes().await?)?)
+}
+
+async fn wait_for_site_replication_enabled(
+    env: &RustFSTestEnvironment,
+    expected_sites: usize,
+) -> Result<SiteReplicationInfo, Box<dyn Error + Send + Sync>> {
+    for _ in 0..40 {
+        let info = site_replication_info(env).await?;
+        if info.enabled && info.sites.len() == expected_sites {
+            return Ok(info);
+        }
+        sleep(Duration::from_millis(250)).await;
+    }
+
+    Err(format!("site replication did not reach {expected_sites} sites on {}", env.address).into())
+}
+
+async fn wait_for_site_replication_disabled(
+    env: &RustFSTestEnvironment,
+) -> Result<SiteReplicationInfo, Box<dyn Error + Send + Sync>> {
+    wait_for_site_replication_info(env, |info| !info.enabled && info.sites.is_empty()).await
+}
+
+async fn wait_for_site_replication_info<F>(
+    env: &RustFSTestEnvironment,
+    predicate: F,
+) -> Result<SiteReplicationInfo, Box<dyn Error + Send + Sync>>
+where
+    F: Fn(&SiteReplicationInfo) -> bool,
+{
+    for _ in 0..40 {
+        let info = site_replication_info(env).await?;
+        if predicate(&info) {
+            return Ok(info);
+        }
+        sleep(Duration::from_millis(250)).await;
+    }
+
+    Err(format!("site replication info did not reach expected state on {}", env.address).into())
+}
+
+async fn wait_for_site_replication_status<F>(
+    env: &RustFSTestEnvironment,
+    query: &str,
+    predicate: F,
+) -> Result<SRStatusInfo, Box<dyn Error + Send + Sync>>
+where
+    F: Fn(&SRStatusInfo) -> bool,
+{
+    for _ in 0..40 {
+        let status = site_replication_status(env, query).await?;
+        if predicate(&status) {
+            return Ok(status);
+        }
+        sleep(Duration::from_millis(250)).await;
+    }
+
+    Err(format!("site replication status did not reach expected state on {}", env.address).into())
+}
+
+async fn wait_for_replication_reset_target<F>(
+    env: &RustFSTestEnvironment,
+    bucket: &str,
+    arn: &str,
+    predicate: F,
+) -> Result<ReplicationResetStatusTarget, Box<dyn Error + Send + Sync>>
+where
+    F: Fn(&ReplicationResetStatusTarget) -> bool,
+{
+    let mut last_seen = None;
+    for _ in 0..40 {
+        let status = get_replication_reset_status(env, bucket, arn).await?;
+        if let Some(target) = status.targets.into_iter().find(|target| target.arn == arn) {
+            if predicate(&target) {
+                return Ok(target);
+            }
+            last_seen = Some(target);
+        }
+        sleep(Duration::from_millis(250)).await;
+    }
+
+    Err(format!(
+        "replication reset target {arn} for bucket {bucket} did not reach expected state; last seen: {:?}",
+        last_seen
+    )
+    .into())
+}
+
 async fn build_replication_pair(
     enable_target_versioning: bool,
 ) -> Result<(RustFSTestEnvironment, RustFSTestEnvironment, String), Box<dyn Error + Send + Sync>> {
@@ -800,3 +1083,410 @@ async fn test_remove_remote_target_rejects_target_used_by_replication() -> Resul
 
     Ok(())
 }
+
+#[tokio::test]
+#[serial]
+async fn test_site_replication_resync_start_cancel_restart_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_bucket = "site-repl-resync-src";
+    let target_bucket = "site-repl-resync-dst";
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+
+    source_client.create_bucket().bucket(source_bucket).send().await?;
+    target_client.create_bucket().bucket(target_bucket).send().await?;
+    enable_bucket_versioning(&source_env, source_bucket).await?;
+    enable_bucket_versioning(&target_env, target_bucket).await?;
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+
+    let source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?;
+    let remote_peer = source_info
+        .sites
+        .into_iter()
+        .find(|peer| peer.endpoint == target_env.url)
+        .ok_or("target peer missing from source site replication info")?;
+
+    let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
+    put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
+
+    for idx in 0..32 {
+        source_client
+            .put_object()
+            .bucket(source_bucket)
+            .key(format!("resync-object-{idx:02}"))
+            .body(ByteStream::from(vec![b'x'; 256 * 1024]))
+            .send()
+            .await?;
+    }
+
+    let started = site_replication_resync_op(&source_env, "start", &remote_peer).await?;
+    assert_eq!(started.status, "success", "unexpected start result: {:?}", started);
+    assert!(
+        started
+            .buckets
+            .iter()
+            .any(|bucket| bucket.bucket == source_bucket && matches!(bucket.status.as_str(), "started" | "success")),
+        "source bucket start status missing: {:?}",
+        started
+    );
+
+    let started_target =
+        wait_for_replication_reset_target(&source_env, source_bucket, &target_arn, |target| !target.reset_id.is_empty()).await?;
+    let started_reset_id = started_target.reset_id.clone();
+    assert!(
+        matches!(started_target.status.as_str(), "Pending" | "Started" | "InProgress" | "Completed"),
+        "unexpected start status: {:?}",
+        started_target
+    );
+
+    let canceled = site_replication_resync_op(&source_env, "cancel", &remote_peer).await?;
+    assert_eq!(canceled.status, "success", "unexpected cancel result: {:?}", canceled);
+    assert!(
+        canceled
+            .buckets
+            .iter()
+            .any(|bucket| bucket.bucket == source_bucket && matches!(bucket.status.as_str(), "canceled" | "success")),
+        "source bucket cancel status missing: {:?}",
+        canceled
+    );
+
+    let canceled_target =
+        wait_for_replication_reset_target(&source_env, source_bucket, &target_arn, |target| target.status == "Canceled").await?;
+    assert_eq!(canceled_target.status, "Canceled");
+    assert_eq!(canceled_target.reset_id, started_reset_id);
+
+    let restarted = site_replication_resync_op(&source_env, "start", &remote_peer).await?;
+    assert_eq!(restarted.status, "success", "unexpected restart result: {:?}", restarted);
+    assert!(
+        restarted
+            .buckets
+            .iter()
+            .any(|bucket| bucket.bucket == source_bucket && matches!(bucket.status.as_str(), "started" | "success")),
+        "source bucket restart status missing: {:?}",
+        restarted
+    );
+    let restart_snapshot = get_replication_reset_status(&source_env, source_bucket, &target_arn).await?;
+    let restarted_target = wait_for_replication_reset_target(&source_env, source_bucket, &target_arn, |target| {
+        !target.reset_id.is_empty() && target.reset_id != started_reset_id
+    })
+    .await
+    .map_err(|err| {
+        format!(
+            "restart ids: start={} restart={} snapshot={:?}; {err}",
+            started_reset_id, restarted.resync_id, restart_snapshot.targets
+        )
+    })?;
+    assert_ne!(restarted_target.reset_id, started_reset_id);
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_site_replication_edit_and_status_peer_state_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+
+    let source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?;
+    let mut remote_peer = source_info
+        .sites
+        .into_iter()
+        .find(|peer| peer.endpoint == target_env.url)
+        .ok_or("target peer missing from source site replication info")?;
+
+    remote_peer.sync_state = SyncStatus::Enable;
+    let edit_status = site_replication_edit(&source_env, "", &remote_peer).await?;
+    assert!(edit_status.success, "unexpected site edit result: {:?}", edit_status);
+
+    let source_after_sync = wait_for_site_replication_info(&source_env, |info| {
+        info.sites
+            .iter()
+            .any(|peer| peer.endpoint == target_env.url && peer.sync_state == SyncStatus::Enable)
+    })
+    .await?;
+    let target_after_sync = wait_for_site_replication_info(&target_env, |info| {
+        info.sites
+            .iter()
+            .any(|peer| peer.endpoint == target_env.url && peer.sync_state == SyncStatus::Enable)
+    })
+    .await?;
+    assert!(
+        source_after_sync
+            .sites
+            .iter()
+            .any(|peer| peer.endpoint == target_env.url && peer.sync_state == SyncStatus::Enable)
+    );
+    assert!(
+        target_after_sync
+            .sites
+            .iter()
+            .any(|peer| peer.endpoint == target_env.url && peer.sync_state == SyncStatus::Enable)
+    );
+
+    let ilm_edit_status = site_replication_edit(&source_env, "enableILMExpiryReplication=true", &PeerInfo::default()).await?;
+    assert!(ilm_edit_status.success, "unexpected ilm edit result: {:?}", ilm_edit_status);
+
+    let source_after_ilm = wait_for_site_replication_info(&source_env, |info| {
+        info.sites.len() == 2 && info.sites.iter().all(|peer| peer.replicate_ilm_expiry)
+    })
+    .await?;
+    let target_after_ilm = wait_for_site_replication_info(&target_env, |info| {
+        info.sites.len() == 2 && info.sites.iter().all(|peer| peer.replicate_ilm_expiry)
+    })
+    .await?;
+    assert!(source_after_ilm.sites.iter().all(|peer| peer.replicate_ilm_expiry));
+    assert!(target_after_ilm.sites.iter().all(|peer| peer.replicate_ilm_expiry));
+
+    let status_query = "peer-state=true";
+    let source_status = wait_for_site_replication_status(&source_env, status_query, |status| {
+        status.peer_states.len() == 2
+            && status
+                .peer_states
+                .values()
+                .all(|state| state.peers.len() == 2 && state.peers.values().all(|peer| peer.replicate_ilm_expiry))
+    })
+    .await?;
+    let target_status = wait_for_site_replication_status(&target_env, status_query, |status| {
+        status.peer_states.len() == 2
+            && status
+                .peer_states
+                .values()
+                .all(|state| state.peers.len() == 2 && state.peers.values().all(|peer| peer.replicate_ilm_expiry))
+    })
+    .await?;
+
+    assert_eq!(source_status.peer_states.len(), 2);
+    assert_eq!(target_status.peer_states.len(), 2);
+    assert!(source_status.peer_states.values().all(|state| state.peers.len() == 2));
+    assert!(target_status.peer_states.values().all(|state| state.peers.len() == 2));
+    assert!(
+        source_status
+            .peer_states
+            .values()
+            .all(|state| state.peers.values().all(|peer| peer.replicate_ilm_expiry))
+    );
+    assert!(
+        target_status
+            .peer_states
+            .values()
+            .all(|state| state.peers.values().all(|peer| peer.replicate_ilm_expiry))
+    );
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_site_replication_remove_all_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+
+    let _source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?;
+
+    let remove_status = site_replication_remove(
+        &source_env,
+        &SRRemoveReq {
+            remove_all: true,
+            ..Default::default()
+        },
+    )
+    .await?;
+    assert!(
+        !remove_status.status.is_empty() && remove_status.err_detail.is_empty(),
+        "unexpected site remove result: {:?}",
+        remove_status
+    );
+
+    let source_after_remove = wait_for_site_replication_disabled(&source_env).await?;
+    let target_after_remove = wait_for_site_replication_disabled(&target_env).await?;
+
+    assert!(!source_after_remove.enabled);
+    assert!(source_after_remove.sites.is_empty());
+    assert!(!target_after_remove.enabled);
+    assert!(target_after_remove.sites.is_empty());
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_site_replication_state_edit_fresh_and_stale_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+
+    let source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    let target_info = wait_for_site_replication_enabled(&target_env, 2).await?;
+    assert!(source_info.sites.iter().all(|peer| !peer.replicate_ilm_expiry));
+    assert!(target_info.sites.iter().all(|peer| !peer.replicate_ilm_expiry));
+
+    let target_status =
+        wait_for_site_replication_status(&target_env, "peer-state=true", |status| status.peer_states.len() == 2).await?;
+    let current_updated_at = target_status
+        .peer_states
+        .values()
+        .find_map(|state| state.updated_at)
+        .ok_or("missing target site replication updated_at")?;
+
+    let mut stale_peers = BTreeMap::new();
+    for peer in target_info.sites {
+        let mut peer = peer;
+        peer.replicate_ilm_expiry = true;
+        stale_peers.insert(peer.deployment_id.clone(), peer);
+    }
+    site_replication_state_edit(
+        &target_env,
+        &rustfs_madmin::SRStateEditReq {
+            peers: stale_peers,
+            updated_at: Some(current_updated_at - TimeDuration::seconds(1)),
+        },
+    )
+    .await?;
+
+    let target_after_stale = site_replication_info(&target_env).await?;
+    let source_after_stale = site_replication_info(&source_env).await?;
+    assert!(target_after_stale.sites.iter().all(|peer| !peer.replicate_ilm_expiry));
+    assert!(source_after_stale.sites.iter().all(|peer| !peer.replicate_ilm_expiry));
+
+    let mut fresh_peers = BTreeMap::new();
+    for peer in target_after_stale.sites {
+        let mut peer = peer;
+        peer.replicate_ilm_expiry = true;
+        fresh_peers.insert(peer.deployment_id.clone(), peer);
+    }
+    let fresh_updated_at = current_updated_at + TimeDuration::seconds(1);
+    site_replication_state_edit(
+        &target_env,
+        &rustfs_madmin::SRStateEditReq {
+            peers: fresh_peers,
+            updated_at: Some(fresh_updated_at),
+        },
+    )
+    .await?;
+
+    let target_after_fresh = wait_for_site_replication_info(&target_env, |info| {
+        info.sites.len() == 2 && info.sites.iter().all(|peer| peer.replicate_ilm_expiry)
+    })
+    .await?;
+    assert!(target_after_fresh.sites.iter().all(|peer| peer.replicate_ilm_expiry));
+
+    let target_status_after_fresh = wait_for_site_replication_status(&target_env, "peer-state=true", |status| {
+        status.peer_states.len() == 2
+            && status.peer_states.values().all(|state| {
+                state.updated_at == Some(fresh_updated_at) && state.peers.values().all(|peer| peer.replicate_ilm_expiry)
+            })
+    })
+    .await?;
+    assert!(target_status_after_fresh.peer_states.values().all(|state| {
+        state.updated_at == Some(fresh_updated_at) && state.peers.values().all(|peer| peer.replicate_ilm_expiry)
+    }));
+
+    let source_after_fresh = site_replication_info(&source_env).await?;
+    assert!(source_after_fresh.sites.iter().all(|peer| !peer.replicate_ilm_expiry));
+
+    Ok(())
+}