feat(admin): complete site replication support (#2346)

Cargo.lock

@@ -7648,6 +7648,7 @@ dependencies = [
  "serde_json",
  "serde_urlencoded",
  "serial_test",
+ "sha2 0.11.0-rc.5",
  "shadow-rs",
  "socket2",
  "starshard",

crates/e2e_test/src/replication_extension_test.rs

@@ -13,14 +13,38 @@
 // limitations under the License.
 
 use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
+use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
 use http::header::{CONTENT_TYPE, HOST};
 use reqwest::StatusCode;
+use rustfs_madmin::{
+    PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus, ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus,
+    SRStatusInfo, SiteReplicationInfo, SyncStatus,
+};
 use rustfs_signer::constants::UNSIGNED_PAYLOAD;
 use rustfs_signer::sign_v4;
 use s3s::Body;
 use serial_test::serial;
+use std::collections::BTreeMap;
 use std::error::Error;
+use time::Duration as TimeDuration;
+use tokio::time::{Duration, sleep};
+
+#[derive(Debug, Clone, serde::Deserialize)]
+struct ReplicationResetStatusResponse {
+    #[serde(rename = "Targets", default)]
+    targets: Vec<ReplicationResetStatusTarget>,
+}
+
+#[derive(Debug, Clone, serde::Deserialize)]
+struct ReplicationResetStatusTarget {
+    #[serde(rename = "Arn", default)]
+    arn: String,
+    #[serde(rename = "ResetID", default)]
+    reset_id: String,
+    #[serde(rename = "Status", default)]
+    status: String,
+}
 
 async fn signed_request(
     method: http::Method,
@@ -239,6 +263,265 @@ async fn list_replication_targets_request(
     signed_request(http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await
 }
 
+async fn site_replication_add(
+    env: &RustFSTestEnvironment,
+    sites: &[PeerSite],
+) -> Result<ReplicateAddStatus, Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/site-replication/add?replicateILMExpiry=false", env.url);
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(serde_json::to_vec(sites)?),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("site replication add failed: {status} {body}").into());
+    }
+
+    Ok(serde_json::from_slice(&response.bytes().await?)?)
+}
+
+async fn site_replication_info(env: &RustFSTestEnvironment) -> Result<SiteReplicationInfo, Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/site-replication/info", env.url);
+    let response = signed_request(http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("site replication info failed: {status} {body}").into());
+    }
+
+    Ok(serde_json::from_slice(&response.bytes().await?)?)
+}
+
+async fn site_replication_resync_op(
+    env: &RustFSTestEnvironment,
+    operation: &str,
+    peer: &PeerInfo,
+) -> Result<SRResyncOpStatus, Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/site-replication/resync/op?operation={operation}", env.url);
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(serde_json::to_vec(peer)?),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("site replication resync {operation} failed: {status} {body}").into());
+    }
+
+    Ok(serde_json::from_slice(&response.bytes().await?)?)
+}
+
+async fn site_replication_edit(
+    env: &RustFSTestEnvironment,
+    query: &str,
+    peer: &PeerInfo,
+) -> Result<ReplicateEditStatus, Box<dyn Error + Send + Sync>> {
+    let url = if query.is_empty() {
+        format!("{}/rustfs/admin/v3/site-replication/edit", env.url)
+    } else {
+        format!("{}/rustfs/admin/v3/site-replication/edit?{query}", env.url)
+    };
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(serde_json::to_vec(peer)?),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("site replication edit failed: {status} {body}").into());
+    }
+
+    Ok(serde_json::from_slice(&response.bytes().await?)?)
+}
+
+async fn site_replication_status(env: &RustFSTestEnvironment, query: &str) -> Result<SRStatusInfo, Box<dyn Error + Send + Sync>> {
+    let url = if query.is_empty() {
+        format!("{}/rustfs/admin/v3/site-replication/status", env.url)
+    } else {
+        format!("{}/rustfs/admin/v3/site-replication/status?{query}", env.url)
+    };
+    let response = signed_request(http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("site replication status failed: {status} {body}").into());
+    }
+
+    Ok(serde_json::from_slice(&response.bytes().await?)?)
+}
+
+async fn site_replication_remove(
+    env: &RustFSTestEnvironment,
+    req: &SRRemoveReq,
+) -> Result<ReplicateRemoveStatus, Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/site-replication/remove", env.url);
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(serde_json::to_vec(req)?),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("site replication remove failed: {status} {body}").into());
+    }
+
+    Ok(serde_json::from_slice(&response.bytes().await?)?)
+}
+
+async fn site_replication_state_edit(
+    env: &RustFSTestEnvironment,
+    body: &rustfs_madmin::SRStateEditReq,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/site-replication/state/edit", env.url);
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(serde_json::to_vec(body)?),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("site replication state edit failed: {status} {body}").into());
+    }
+
+    Ok(())
+}
+
+async fn get_replication_reset_status(
+    env: &RustFSTestEnvironment,
+    bucket: &str,
+    arn: &str,
+) -> Result<ReplicationResetStatusResponse, Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/{bucket}?replication-reset-status&arn={}", env.url, urlencoding::encode(arn));
+    let response = signed_request(http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("replication reset status failed: {status} {body}").into());
+    }
+
+    Ok(serde_json::from_slice(&response.bytes().await?)?)
+}
+
+async fn wait_for_site_replication_enabled(
+    env: &RustFSTestEnvironment,
+    expected_sites: usize,
+) -> Result<SiteReplicationInfo, Box<dyn Error + Send + Sync>> {
+    for _ in 0..40 {
+        let info = site_replication_info(env).await?;
+        if info.enabled && info.sites.len() == expected_sites {
+            return Ok(info);
+        }
+        sleep(Duration::from_millis(250)).await;
+    }
+
+    Err(format!("site replication did not reach {expected_sites} sites on {}", env.address).into())
+}
+
+async fn wait_for_site_replication_disabled(
+    env: &RustFSTestEnvironment,
+) -> Result<SiteReplicationInfo, Box<dyn Error + Send + Sync>> {
+    wait_for_site_replication_info(env, |info| !info.enabled && info.sites.is_empty()).await
+}
+
+async fn wait_for_site_replication_info<F>(
+    env: &RustFSTestEnvironment,
+    predicate: F,
+) -> Result<SiteReplicationInfo, Box<dyn Error + Send + Sync>>
+where
+    F: Fn(&SiteReplicationInfo) -> bool,
+{
+    for _ in 0..40 {
+        let info = site_replication_info(env).await?;
+        if predicate(&info) {
+            return Ok(info);
+        }
+        sleep(Duration::from_millis(250)).await;
+    }
+
+    Err(format!("site replication info did not reach expected state on {}", env.address).into())
+}
+
+async fn wait_for_site_replication_status<F>(
+    env: &RustFSTestEnvironment,
+    query: &str,
+    predicate: F,
+) -> Result<SRStatusInfo, Box<dyn Error + Send + Sync>>
+where
+    F: Fn(&SRStatusInfo) -> bool,
+{
+    for _ in 0..40 {
+        let status = site_replication_status(env, query).await?;
+        if predicate(&status) {
+            return Ok(status);
+        }
+        sleep(Duration::from_millis(250)).await;
+    }
+
+    Err(format!("site replication status did not reach expected state on {}", env.address).into())
+}
+
+async fn wait_for_replication_reset_target<F>(
+    env: &RustFSTestEnvironment,
+    bucket: &str,
+    arn: &str,
+    predicate: F,
+) -> Result<ReplicationResetStatusTarget, Box<dyn Error + Send + Sync>>
+where
+    F: Fn(&ReplicationResetStatusTarget) -> bool,
+{
+    let mut last_seen = None;
+    for _ in 0..40 {
+        let status = get_replication_reset_status(env, bucket, arn).await?;
+        if let Some(target) = status.targets.into_iter().find(|target| target.arn == arn) {
+            if predicate(&target) {
+                return Ok(target);
+            }
+            last_seen = Some(target);
+        }
+        sleep(Duration::from_millis(250)).await;
+    }
+
+    Err(format!(
+        "replication reset target {arn} for bucket {bucket} did not reach expected state; last seen: {:?}",
+        last_seen
+    )
+    .into())
+}
+
 async fn build_replication_pair(
     enable_target_versioning: bool,
 ) -> Result<(RustFSTestEnvironment, RustFSTestEnvironment, String), Box<dyn Error + Send + Sync>> {
@@ -800,3 +1083,410 @@ async fn test_remove_remote_target_rejects_target_used_by_replication() -> Resul
 
     Ok(())
 }
+
+#[tokio::test]
+#[serial]
+async fn test_site_replication_resync_start_cancel_restart_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_bucket = "site-repl-resync-src";
+    let target_bucket = "site-repl-resync-dst";
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+
+    source_client.create_bucket().bucket(source_bucket).send().await?;
+    target_client.create_bucket().bucket(target_bucket).send().await?;
+    enable_bucket_versioning(&source_env, source_bucket).await?;
+    enable_bucket_versioning(&target_env, target_bucket).await?;
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+
+    let source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?;
+    let remote_peer = source_info
+        .sites
+        .into_iter()
+        .find(|peer| peer.endpoint == target_env.url)
+        .ok_or("target peer missing from source site replication info")?;
+
+    let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
+    put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
+
+    for idx in 0..32 {
+        source_client
+            .put_object()
+            .bucket(source_bucket)
+            .key(format!("resync-object-{idx:02}"))
+            .body(ByteStream::from(vec![b'x'; 256 * 1024]))
+            .send()
+            .await?;
+    }
+
+    let started = site_replication_resync_op(&source_env, "start", &remote_peer).await?;
+    assert_eq!(started.status, "success", "unexpected start result: {:?}", started);
+    assert!(
+        started
+            .buckets
+            .iter()
+            .any(|bucket| bucket.bucket == source_bucket && matches!(bucket.status.as_str(), "started" | "success")),
+        "source bucket start status missing: {:?}",
+        started
+    );
+
+    let started_target =
+        wait_for_replication_reset_target(&source_env, source_bucket, &target_arn, |target| !target.reset_id.is_empty()).await?;
+    let started_reset_id = started_target.reset_id.clone();
+    assert!(
+        matches!(started_target.status.as_str(), "Pending" | "Started" | "InProgress" | "Completed"),
+        "unexpected start status: {:?}",
+        started_target
+    );
+
+    let canceled = site_replication_resync_op(&source_env, "cancel", &remote_peer).await?;
+    assert_eq!(canceled.status, "success", "unexpected cancel result: {:?}", canceled);
+    assert!(
+        canceled
+            .buckets
+            .iter()
+            .any(|bucket| bucket.bucket == source_bucket && matches!(bucket.status.as_str(), "canceled" | "success")),
+        "source bucket cancel status missing: {:?}",
+        canceled
+    );
+
+    let canceled_target =
+        wait_for_replication_reset_target(&source_env, source_bucket, &target_arn, |target| target.status == "Canceled").await?;
+    assert_eq!(canceled_target.status, "Canceled");
+    assert_eq!(canceled_target.reset_id, started_reset_id);
+
+    let restarted = site_replication_resync_op(&source_env, "start", &remote_peer).await?;
+    assert_eq!(restarted.status, "success", "unexpected restart result: {:?}", restarted);
+    assert!(
+        restarted
+            .buckets
+            .iter()
+            .any(|bucket| bucket.bucket == source_bucket && matches!(bucket.status.as_str(), "started" | "success")),
+        "source bucket restart status missing: {:?}",
+        restarted
+    );
+    let restart_snapshot = get_replication_reset_status(&source_env, source_bucket, &target_arn).await?;
+    let restarted_target = wait_for_replication_reset_target(&source_env, source_bucket, &target_arn, |target| {
+        !target.reset_id.is_empty() && target.reset_id != started_reset_id
+    })
+    .await
+    .map_err(|err| {
+        format!(
+            "restart ids: start={} restart={} snapshot={:?}; {err}",
+            started_reset_id, restarted.resync_id, restart_snapshot.targets
+        )
+    })?;
+    assert_ne!(restarted_target.reset_id, started_reset_id);
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_site_replication_edit_and_status_peer_state_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+
+    let source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?;
+    let mut remote_peer = source_info
+        .sites
+        .into_iter()
+        .find(|peer| peer.endpoint == target_env.url)
+        .ok_or("target peer missing from source site replication info")?;
+
+    remote_peer.sync_state = SyncStatus::Enable;
+    let edit_status = site_replication_edit(&source_env, "", &remote_peer).await?;
+    assert!(edit_status.success, "unexpected site edit result: {:?}", edit_status);
+
+    let source_after_sync = wait_for_site_replication_info(&source_env, |info| {
+        info.sites
+            .iter()
+            .any(|peer| peer.endpoint == target_env.url && peer.sync_state == SyncStatus::Enable)
+    })
+    .await?;
+    let target_after_sync = wait_for_site_replication_info(&target_env, |info| {
+        info.sites
+            .iter()
+            .any(|peer| peer.endpoint == target_env.url && peer.sync_state == SyncStatus::Enable)
+    })
+    .await?;
+    assert!(
+        source_after_sync
+            .sites
+            .iter()
+            .any(|peer| peer.endpoint == target_env.url && peer.sync_state == SyncStatus::Enable)
+    );
+    assert!(
+        target_after_sync
+            .sites
+            .iter()
+            .any(|peer| peer.endpoint == target_env.url && peer.sync_state == SyncStatus::Enable)
+    );
+
+    let ilm_edit_status = site_replication_edit(&source_env, "enableILMExpiryReplication=true", &PeerInfo::default()).await?;
+    assert!(ilm_edit_status.success, "unexpected ilm edit result: {:?}", ilm_edit_status);
+
+    let source_after_ilm = wait_for_site_replication_info(&source_env, |info| {
+        info.sites.len() == 2 && info.sites.iter().all(|peer| peer.replicate_ilm_expiry)
+    })
+    .await?;
+    let target_after_ilm = wait_for_site_replication_info(&target_env, |info| {
+        info.sites.len() == 2 && info.sites.iter().all(|peer| peer.replicate_ilm_expiry)
+    })
+    .await?;
+    assert!(source_after_ilm.sites.iter().all(|peer| peer.replicate_ilm_expiry));
+    assert!(target_after_ilm.sites.iter().all(|peer| peer.replicate_ilm_expiry));
+
+    let status_query = "peer-state=true";
+    let source_status = wait_for_site_replication_status(&source_env, status_query, |status| {
+        status.peer_states.len() == 2
+            && status
+                .peer_states
+                .values()
+                .all(|state| state.peers.len() == 2 && state.peers.values().all(|peer| peer.replicate_ilm_expiry))
+    })
+    .await?;
+    let target_status = wait_for_site_replication_status(&target_env, status_query, |status| {
+        status.peer_states.len() == 2
+            && status
+                .peer_states
+                .values()
+                .all(|state| state.peers.len() == 2 && state.peers.values().all(|peer| peer.replicate_ilm_expiry))
+    })
+    .await?;
+
+    assert_eq!(source_status.peer_states.len(), 2);
+    assert_eq!(target_status.peer_states.len(), 2);
+    assert!(source_status.peer_states.values().all(|state| state.peers.len() == 2));
+    assert!(target_status.peer_states.values().all(|state| state.peers.len() == 2));
+    assert!(
+        source_status
+            .peer_states
+            .values()
+            .all(|state| state.peers.values().all(|peer| peer.replicate_ilm_expiry))
+    );
+    assert!(
+        target_status
+            .peer_states
+            .values()
+            .all(|state| state.peers.values().all(|peer| peer.replicate_ilm_expiry))
+    );
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_site_replication_remove_all_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+
+    let _source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?;
+
+    let remove_status = site_replication_remove(
+        &source_env,
+        &SRRemoveReq {
+            remove_all: true,
+            ..Default::default()
+        },
+    )
+    .await?;
+    assert!(
+        !remove_status.status.is_empty() && remove_status.err_detail.is_empty(),
+        "unexpected site remove result: {:?}",
+        remove_status
+    );
+
+    let source_after_remove = wait_for_site_replication_disabled(&source_env).await?;
+    let target_after_remove = wait_for_site_replication_disabled(&target_env).await?;
+
+    assert!(!source_after_remove.enabled);
+    assert!(source_after_remove.sites.is_empty());
+    assert!(!target_after_remove.enabled);
+    assert!(target_after_remove.sites.is_empty());
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_site_replication_state_edit_fresh_and_stale_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+
+    let source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    let target_info = wait_for_site_replication_enabled(&target_env, 2).await?;
+    assert!(source_info.sites.iter().all(|peer| !peer.replicate_ilm_expiry));
+    assert!(target_info.sites.iter().all(|peer| !peer.replicate_ilm_expiry));
+
+    let target_status =
+        wait_for_site_replication_status(&target_env, "peer-state=true", |status| status.peer_states.len() == 2).await?;
+    let current_updated_at = target_status
+        .peer_states
+        .values()
+        .find_map(|state| state.updated_at)
+        .ok_or("missing target site replication updated_at")?;
+
+    let mut stale_peers = BTreeMap::new();
+    for peer in target_info.sites {
+        let mut peer = peer;
+        peer.replicate_ilm_expiry = true;
+        stale_peers.insert(peer.deployment_id.clone(), peer);
+    }
+    site_replication_state_edit(
+        &target_env,
+        &rustfs_madmin::SRStateEditReq {
+            peers: stale_peers,
+            updated_at: Some(current_updated_at - TimeDuration::seconds(1)),
+        },
+    )
+    .await?;
+
+    let target_after_stale = site_replication_info(&target_env).await?;
+    let source_after_stale = site_replication_info(&source_env).await?;
+    assert!(target_after_stale.sites.iter().all(|peer| !peer.replicate_ilm_expiry));
+    assert!(source_after_stale.sites.iter().all(|peer| !peer.replicate_ilm_expiry));
+
+    let mut fresh_peers = BTreeMap::new();
+    for peer in target_after_stale.sites {
+        let mut peer = peer;
+        peer.replicate_ilm_expiry = true;
+        fresh_peers.insert(peer.deployment_id.clone(), peer);
+    }
+    let fresh_updated_at = current_updated_at + TimeDuration::seconds(1);
+    site_replication_state_edit(
+        &target_env,
+        &rustfs_madmin::SRStateEditReq {
+            peers: fresh_peers,
+            updated_at: Some(fresh_updated_at),
+        },
+    )
+    .await?;
+
+    let target_after_fresh = wait_for_site_replication_info(&target_env, |info| {
+        info.sites.len() == 2 && info.sites.iter().all(|peer| peer.replicate_ilm_expiry)
+    })
+    .await?;
+    assert!(target_after_fresh.sites.iter().all(|peer| peer.replicate_ilm_expiry));
+
+    let target_status_after_fresh = wait_for_site_replication_status(&target_env, "peer-state=true", |status| {
+        status.peer_states.len() == 2
+            && status.peer_states.values().all(|state| {
+                state.updated_at == Some(fresh_updated_at) && state.peers.values().all(|peer| peer.replicate_ilm_expiry)
+            })
+    })
+    .await?;
+    assert!(target_status_after_fresh.peer_states.values().all(|state| {
+        state.updated_at == Some(fresh_updated_at) && state.peers.values().all(|peer| peer.replicate_ilm_expiry)
+    }));
+
+    let source_after_fresh = site_replication_info(&source_env).await?;
+    assert!(source_after_fresh.sites.iter().all(|peer| !peer.replicate_ilm_expiry));
+
+    Ok(())
+}

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -777,6 +777,14 @@ impl<S: StorageAPI> ReplicationPool<S> {
         Ok(status)
     }
 
+    pub async fn cancel_bucket_resync(&self, opts: ResyncOpts) -> Result<(), EcstoreError> {
+        self.resyncer.cancel(&opts).await;
+        self.resyncer
+            .mark_status(ResyncStatusType::ResyncCanceled, opts, self.storage.clone())
+            .await?;
+        Ok(())
+    }
+
     pub async fn start_bucket_resync(self: Arc<Self>, opts: ResyncOpts) -> Result<(), EcstoreError> {
         let now = OffsetDateTime::now_utc();
         let bucket_status = {
@@ -813,8 +821,14 @@ impl<S: StorageAPI> ReplicationPool<S> {
 
         let resyncer = self.resyncer.clone();
         let storage = self.storage.clone();
+        let cancel_token = CancellationToken::new();
+        resyncer.register_cancel_token(&opts, cancel_token.clone()).await;
         tokio::spawn(async move {
-            resyncer.resync_bucket(CancellationToken::new(), storage, false, opts).await;
+            resyncer
+                .clone()
+                .resync_bucket(cancel_token, storage, false, opts.clone())
+                .await;
+            resyncer.clear_cancel_token(&opts).await;
         });
 
         Ok(())
@@ -852,8 +866,12 @@ impl<S: StorageAPI> ReplicationPool<S> {
     }
 
     /// Load bucket replication resync statuses into memory
-    #[instrument(skip(cancellation_token))]
-    async fn load_resync(self: Arc<Self>, buckets: &[String], cancellation_token: CancellationToken) -> Result<(), EcstoreError> {
+    #[instrument(skip(_cancellation_token))]
+    async fn load_resync(
+        self: Arc<Self>,
+        buckets: &[String],
+        _cancellation_token: CancellationToken,
+    ) -> Result<(), EcstoreError> {
         // TODO: add leader_lock
         // Make sure only one node running resync on the cluster
         // Note: Leader lock implementation would be needed here
@@ -884,24 +902,20 @@ impl<S: StorageAPI> ReplicationPool<S> {
                         // Note: This would spawn a resync task in a real implementation
                         // For now, we just log the resync request
 
-                        let ctx = cancellation_token.clone();
+                        let ctx = CancellationToken::new();
                         let bucket_clone = bucket.clone();
                         let resync = self.resyncer.clone();
                         let storage = self.storage.clone();
+                        let opts = ResyncOpts {
+                            bucket: bucket_clone,
+                            arn,
+                            resync_id: stats.resync_id,
+                            resync_before: stats.resync_before_date,
+                        };
                         tokio::spawn(async move {
-                            resync
-                                .resync_bucket(
-                                    ctx,
-                                    storage,
-                                    true,
-                                    ResyncOpts {
-                                        bucket: bucket_clone,
-                                        arn,
-                                        resync_id: stats.resync_id,
-                                        resync_before: stats.resync_before_date,
-                                    },
-                                )
-                                .await;
+                            resync.register_cancel_token(&opts, ctx.clone()).await;
+                            resync.clone().resync_bucket(ctx, storage, true, opts.clone()).await;
+                            resync.clear_cancel_token(&opts).await;
                         });
                     }
                     _ => {}
@@ -949,6 +963,7 @@ pub trait ReplicationPoolTrait: std::fmt::Debug {
     async fn queue_replica_delete_task(&self, ri: DeletedObjectReplicationInfo);
     async fn resize(&self, priority: ReplicationPriority, max_workers: usize, max_l_workers: usize);
     async fn get_bucket_resync_status(&self, bucket: &str) -> Result<BucketReplicationResyncStatus, EcstoreError>;
+    async fn cancel_bucket_resync(&self, opts: ResyncOpts) -> Result<(), EcstoreError>;
     async fn start_bucket_resync(self: Arc<Self>, opts: ResyncOpts) -> Result<(), EcstoreError>;
     async fn init_resync(
         self: Arc<Self>,
@@ -976,6 +991,10 @@ impl<S: StorageAPI> ReplicationPoolTrait for ReplicationPool<S> {
         self.get_bucket_resync_status(bucket).await
     }
 
+    async fn cancel_bucket_resync(&self, opts: ResyncOpts) -> Result<(), EcstoreError> {
+        self.cancel_bucket_resync(opts).await
+    }
+
     async fn start_bucket_resync(self: Arc<Self>, opts: ResyncOpts) -> Result<(), EcstoreError> {
         self.start_bucket_resync(opts).await
     }

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -110,6 +110,10 @@ fn normalize_wire_time(value: Option<OffsetDateTime>) -> Option<OffsetDateTime>
     }
 }
 
+fn resync_state_accepts_update(state: &TargetReplicationResyncStatus, opts: &ResyncOpts) -> bool {
+    state.resync_id.is_empty() || opts.resync_id.is_empty() || state.resync_id == opts.resync_id
+}
+
 #[derive(Debug, Clone, Default)]
 pub struct ResyncOpts {
     pub bucket: String,
@@ -360,16 +364,13 @@ static RESYNC_WORKER_COUNT: usize = 10;
 pub struct ReplicationResyncer {
     pub status_map: Arc<RwLock<HashMap<String, BucketReplicationResyncStatus>>>,
     pub worker_size: usize,
-    pub resync_cancel_tx: CancellationToken,
-    pub resync_cancel_rx: CancellationToken,
+    pub cancel_tokens: Arc<RwLock<HashMap<String, CancellationToken>>>,
     pub worker_tx: tokio::sync::broadcast::Sender<()>,
     pub worker_rx: tokio::sync::broadcast::Receiver<()>,
 }
 
 impl ReplicationResyncer {
     pub async fn new() -> Self {
-        let resync_cancel_tx = CancellationToken::new();
-        let resync_cancel_rx = resync_cancel_tx.clone();
         let (worker_tx, worker_rx) = tokio::sync::broadcast::channel(RESYNC_WORKER_COUNT);
 
         for _ in 0..RESYNC_WORKER_COUNT {
@@ -381,16 +382,34 @@ impl ReplicationResyncer {
         Self {
             status_map: Arc::new(RwLock::new(HashMap::new())),
             worker_size: RESYNC_WORKER_COUNT,
-            resync_cancel_tx,
-            resync_cancel_rx,
+            cancel_tokens: Arc::new(RwLock::new(HashMap::new())),
             worker_tx,
             worker_rx,
         }
     }
 
+    fn cancel_key(opts: &ResyncOpts) -> String {
+        format!("{}:{}", opts.bucket, opts.arn)
+    }
+
+    pub async fn register_cancel_token(&self, opts: &ResyncOpts, token: CancellationToken) {
+        self.cancel_tokens.write().await.insert(Self::cancel_key(opts), token);
+    }
+
+    pub async fn clear_cancel_token(&self, opts: &ResyncOpts) {
+        self.cancel_tokens.write().await.remove(&Self::cancel_key(opts));
+    }
+
+    pub async fn cancel(&self, opts: &ResyncOpts) {
+        if let Some(token) = self.cancel_tokens.write().await.remove(&Self::cancel_key(opts)) {
+            token.cancel();
+        }
+    }
+
     pub async fn mark_status<S: StorageAPI>(&self, status: ResyncStatusType, opts: ResyncOpts, obj_layer: Arc<S>) -> Result<()> {
         let bucket_status = {
             let mut status_map = self.status_map.write().await;
+            let now = OffsetDateTime::now_utc();
 
             let bucket_status = if let Some(bucket_status) = status_map.get_mut(&opts.bucket) {
                 bucket_status
@@ -409,10 +428,33 @@ impl ReplicationResyncer {
                 bucket_status.targets_map.get_mut(&opts.arn).unwrap()
             };
 
-            state.resync_status = status;
-            state.last_update = Some(OffsetDateTime::now_utc());
+            if !resync_state_accepts_update(state, &opts) {
+                warn!(
+                    bucket = %opts.bucket,
+                    arn = %opts.arn,
+                    incoming_resync_id = %opts.resync_id,
+                    current_resync_id = %state.resync_id,
+                    "ignoring stale resync status update"
+                );
+                return Ok(());
+            }
 
-            bucket_status.last_update = Some(OffsetDateTime::now_utc());
+            if state.resync_id.is_empty() {
+                state.resync_id = opts.resync_id.clone();
+            }
+            if state.resync_before_date.is_none() {
+                state.resync_before_date = opts.resync_before;
+            }
+            if state.bucket.is_empty() {
+                state.bucket = opts.bucket.clone();
+            }
+            if status == ResyncStatusType::ResyncStarted && state.start_time.is_none() {
+                state.start_time = Some(now);
+            }
+            state.resync_status = status;
+            state.last_update = Some(now);
+
+            bucket_status.last_update = Some(now);
 
             bucket_status.clone()
         };
@@ -424,6 +466,7 @@ impl ReplicationResyncer {
 
     pub async fn inc_stats(&self, status: &TargetReplicationResyncStatus, opts: ResyncOpts) {
         let mut status_map = self.status_map.write().await;
+        let now = OffsetDateTime::now_utc();
 
         let bucket_status = if let Some(bucket_status) = status_map.get_mut(&opts.bucket) {
             bucket_status
@@ -442,13 +485,30 @@ impl ReplicationResyncer {
             bucket_status.targets_map.get_mut(&opts.arn).unwrap()
         };
 
+        if !resync_state_accepts_update(state, &opts) {
+            warn!(
+                bucket = %opts.bucket,
+                arn = %opts.arn,
+                incoming_resync_id = %opts.resync_id,
+                current_resync_id = %state.resync_id,
+                "ignoring stale resync stats update"
+            );
+            return;
+        }
+
+        if state.resync_id.is_empty() {
+            state.resync_id = opts.resync_id.clone();
+        }
+        if state.bucket.is_empty() {
+            state.bucket = opts.bucket.clone();
+        }
         state.object = status.object.clone();
         state.replicated_count += status.replicated_count;
         state.replicated_size += status.replicated_size;
         state.failed_count += status.failed_count;
         state.failed_size += status.failed_size;
-        state.last_update = Some(OffsetDateTime::now_utc());
-        bucket_status.last_update = Some(OffsetDateTime::now_utc());
+        state.last_update = Some(now);
+        bucket_status.last_update = Some(now);
     }
 
     pub async fn persist_to_disk<S: StorageAPI>(&self, cancel_token: CancellationToken, api: Arc<S>) {
@@ -640,7 +700,6 @@ impl ReplicationResyncer {
 
             let cancel_token = cancellation_token.clone();
             let target_client = target_client.clone();
-            let resync_cancel_rx = self.resync_cancel_rx.clone();
             let storage = storage.clone();
             let results_tx = results_tx.clone();
             let bucket_name = opts.bucket.clone();
@@ -714,10 +773,6 @@ impl ReplicationResyncer {
                         err,
                     );
 
-                    if resync_cancel_rx.is_cancelled() {
-                        return;
-                    }
-
                     if cancel_token.is_cancelled() {
                         return;
                     }
@@ -731,8 +786,6 @@ impl ReplicationResyncer {
             futures.push(f);
         }
 
-        let resync_cancel_rx = self.resync_cancel_rx.clone();
-
         while let Some(res) = rx.recv().await {
             if let Some(err) = res.err {
                 error!("Failed to get object info: {}", err);
@@ -741,14 +794,8 @@ impl ReplicationResyncer {
                 return;
             }
 
-            if resync_cancel_rx.is_cancelled() {
-                self.resync_bucket_mark_status(ResyncStatusType::ResyncCanceled, opts.clone(), storage.clone())
-                    .await;
-                return;
-            }
-
             if cancellation_token.is_cancelled() {
-                self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone())
+                self.resync_bucket_mark_status(ResyncStatusType::ResyncCanceled, opts.clone(), storage.clone())
                     .await;
                 return;
             }
@@ -770,14 +817,8 @@ impl ReplicationResyncer {
                 continue;
             }
 
-            if resync_cancel_rx.is_cancelled() {
-                self.resync_bucket_mark_status(ResyncStatusType::ResyncCanceled, opts.clone(), storage.clone())
-                    .await;
-                return;
-            }
-
             if cancellation_token.is_cancelled() {
-                self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone())
+                self.resync_bucket_mark_status(ResyncStatusType::ResyncCanceled, opts.clone(), storage.clone())
                     .await;
                 return;
             }
@@ -3430,4 +3471,54 @@ mod tests {
             "With no replication config, dsc may be empty; with config, replicate_any() would be true and queueing would occur"
         );
     }
+
+    #[tokio::test]
+    async fn test_cancel_marks_only_matching_bucket_target_token() {
+        let resyncer = ReplicationResyncer::new().await;
+        let opts_a = ResyncOpts {
+            bucket: "bucket-a".to_string(),
+            arn: "arn:replication::a".to_string(),
+            resync_id: "rid-a".to_string(),
+            resync_before: None,
+        };
+        let opts_b = ResyncOpts {
+            bucket: "bucket-b".to_string(),
+            arn: "arn:replication::b".to_string(),
+            resync_id: "rid-b".to_string(),
+            resync_before: None,
+        };
+        let token_a = CancellationToken::new();
+        let token_b = CancellationToken::new();
+        resyncer.register_cancel_token(&opts_a, token_a.clone()).await;
+        resyncer.register_cancel_token(&opts_b, token_b.clone()).await;
+
+        resyncer.cancel(&opts_a).await;
+
+        assert!(token_a.is_cancelled());
+        assert!(!token_b.is_cancelled());
+    }
+
+    #[test]
+    fn test_resync_state_accepts_update_only_for_matching_run() {
+        let current = TargetReplicationResyncStatus {
+            resync_id: "run-new".to_string(),
+            ..Default::default()
+        };
+        let matching = ResyncOpts {
+            bucket: "bucket".to_string(),
+            arn: "arn:replication::dest".to_string(),
+            resync_id: "run-new".to_string(),
+            resync_before: None,
+        };
+        let stale = ResyncOpts {
+            bucket: "bucket".to_string(),
+            arn: "arn:replication::dest".to_string(),
+            resync_id: "run-old".to_string(),
+            resync_before: None,
+        };
+
+        assert!(resync_state_accepts_update(&TargetReplicationResyncStatus::default(), &matching));
+        assert!(resync_state_accepts_update(&current, &matching));
+        assert!(!resync_state_accepts_update(&current, &stale));
+    }
 }

crates/madmin/src/group.rs

@@ -17,7 +17,7 @@ use serde::Deserializer;
 use serde::Serialize;
 use time::OffsetDateTime;
 
-#[derive(Debug, Serialize, Default, PartialEq, Eq)]
+#[derive(Debug, Clone, Serialize, Default, PartialEq, Eq)]
 #[serde(rename_all = "lowercase")]
 pub enum GroupStatus {
     #[default]
@@ -39,7 +39,7 @@ impl<'de> Deserialize<'de> for GroupStatus {
     }
 }
 
-#[derive(Debug, Serialize, Deserialize, Default)]
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
 pub struct GroupAddRemove {
     pub group: String,
     pub members: Vec<String>,

crates/madmin/src/lib.rs

@@ -20,6 +20,7 @@ pub mod metrics;
 pub mod net;
 pub mod policy;
 pub mod service_commands;
+pub mod site_replication;
 pub mod trace;
 pub mod user;
 pub mod utils;
@@ -27,4 +28,5 @@ pub mod utils;
 pub use group::*;
 pub use info_commands::*;
 pub use policy::*;
+pub use site_replication::*;
 pub use user::*;

crates/madmin/src/user.rs

@@ -21,7 +21,7 @@ use time::format_description::well_known::Rfc3339;
 
 use crate::BackendInfo;
 
-#[derive(Debug, Serialize, Deserialize, Default, PartialEq, Eq)]
+#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)]
 pub enum AccountStatus {
     #[serde(rename = "enabled")]
     Enabled,
@@ -94,7 +94,7 @@ pub struct UserInfo {
     pub updated_at: Option<OffsetDateTime>,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct AddOrUpdateUserReq {
     #[serde(rename = "secretKey")]
     pub secret_key: String,

rustfs/Cargo.toml

@@ -131,6 +131,7 @@ astral-tokio-tar = { workspace = true }
 atoi = { workspace = true }
 atomic_enum = { workspace = true }
 base64 = { workspace = true }
+sha2 = { workspace = true }
 base64-simd.workspace = true
 clap = { workspace = true }
 const-str = { workspace = true }

rustfs/src/admin/handlers/group.rs

@@ -15,6 +15,7 @@
 use crate::{
     admin::{
         auth::validate_admin_request,
+        handlers::site_replication::site_replication_iam_change_hook,
         router::{AdminOperation, Operation, S3Router},
         utils::has_space_be,
     },
@@ -27,7 +28,7 @@ use matchit::Params;
 use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
 use rustfs_credentials::get_global_action_cred;
 use rustfs_iam::error::{is_err_no_such_group, is_err_no_such_user};
-use rustfs_madmin::GroupAddRemove;
+use rustfs_madmin::{GroupAddRemove, GroupStatus, SITE_REPL_API_VERSION, SRGroupInfo, SRIAMItem};
 use rustfs_policy::policy::action::{Action, AdminAction};
 use s3s::{
     Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result,
@@ -222,7 +223,7 @@ impl Operation for DeleteGroup {
 
         let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) };
 
-        iam_store.remove_users_from_group(group, vec![]).await.map_err(|e| {
+        let updated_at = iam_store.remove_users_from_group(group, vec![]).await.map_err(|e| {
             warn!("delete group failed, e: {:?}", e);
             match e {
                 rustfs_iam::error::Error::GroupNotEmpty => {
@@ -241,6 +242,26 @@ impl Operation for DeleteGroup {
             }
         })?;
 
+        if let Err(err) = site_replication_iam_change_hook(SRIAMItem {
+            r#type: "group-info".to_string(),
+            group_info: Some(SRGroupInfo {
+                update_req: GroupAddRemove {
+                    group: group.to_string(),
+                    members: vec![],
+                    status: GroupStatus::Enabled,
+                    is_remove: true,
+                },
+                api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            }),
+            updated_at: Some(updated_at),
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            ..Default::default()
+        })
+        .await
+        {
+            warn!("site replication group delete hook failed, err: {err}");
+        }
+
         let mut header = HeaderMap::new();
         header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
         header.insert(CONTENT_LENGTH, "0".parse().unwrap());
@@ -287,26 +308,46 @@ impl Operation for SetGroupStatus {
 
         let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) };
 
-        if let Some(status) = query.status {
-            match status.as_str() {
-                "enabled" => {
-                    iam_store.set_group_status(&query.group, true).await.map_err(|e| {
-                        warn!("enable group failed, e: {:?}", e);
-                        S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
-                    })?;
-                }
-                "disabled" => {
-                    iam_store.set_group_status(&query.group, false).await.map_err(|e| {
-                        warn!("enable group failed, e: {:?}", e);
-                        S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
-                    })?;
-                }
+        let updated_at = if let Some(status) = query.status.as_deref() {
+            match status {
+                "enabled" => iam_store.set_group_status(&query.group, true).await.map_err(|e| {
+                    warn!("enable group failed, e: {:?}", e);
+                    S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
+                })?,
+                "disabled" => iam_store.set_group_status(&query.group, false).await.map_err(|e| {
+                    warn!("enable group failed, e: {:?}", e);
+                    S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
+                })?,
                 _ => {
                     return Err(s3_error!(InvalidArgument, "invalid status"));
                 }
             }
         } else {
             return Err(s3_error!(InvalidArgument, "status is required"));
+        };
+
+        if let Err(err) = site_replication_iam_change_hook(SRIAMItem {
+            r#type: "group-info".to_string(),
+            group_info: Some(SRGroupInfo {
+                update_req: GroupAddRemove {
+                    group: query.group.clone(),
+                    members: vec![],
+                    status: if matches!(query.status.as_deref(), Some("disabled")) {
+                        GroupStatus::Disabled
+                    } else {
+                        GroupStatus::Enabled
+                    },
+                    is_remove: false,
+                },
+                api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            }),
+            updated_at: Some(updated_at),
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            ..Default::default()
+        })
+        .await
+        {
+            warn!("site replication group status hook failed, err: {err}");
         }
 
         let mut header = HeaderMap::new();
@@ -387,15 +428,15 @@ impl Operation for UpdateGroupMembers {
             }
         }
 
-        if args.is_remove {
+        let updated_at = if args.is_remove {
             warn!("remove group members");
             iam_store
-                .remove_users_from_group(&args.group, args.members)
+                .remove_users_from_group(&args.group, args.members.clone())
                 .await
                 .map_err(|e| {
                     warn!("remove group members failed, e: {:?}", e);
                     S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
-                })?;
+                })?
         } else {
             warn!("add group members");
 
@@ -406,10 +447,28 @@ impl Operation for UpdateGroupMembers {
                 return Err(s3_error!(InvalidArgument, "not such group"));
             }
 
-            iam_store.add_users_to_group(&args.group, args.members).await.map_err(|e| {
-                warn!("add group members failed, e: {:?}", e);
-                S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
-            })?;
+            iam_store
+                .add_users_to_group(&args.group, args.members.clone())
+                .await
+                .map_err(|e| {
+                    warn!("add group members failed, e: {:?}", e);
+                    S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
+                })?
+        };
+
+        if let Err(err) = site_replication_iam_change_hook(SRIAMItem {
+            r#type: "group-info".to_string(),
+            group_info: Some(SRGroupInfo {
+                update_req: args,
+                api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            }),
+            updated_at: Some(updated_at),
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            ..Default::default()
+        })
+        .await
+        {
+            warn!("site replication group membership hook failed, err: {err}");
         }
 
         let mut header = HeaderMap::new();

rustfs/src/admin/handlers/mod.rs

@@ -33,6 +33,7 @@ pub mod quota;
 pub mod rebalance;
 pub mod replication;
 pub mod service_account;
+pub mod site_replication;
 pub mod sts;
 pub mod system;
 pub mod tier;
@@ -64,6 +65,9 @@ mod tests {
         let _set_remote_target_handler = replication::SetRemoteTargetHandler {};
         let _list_remote_target_handler = replication::ListRemoteTargetHandler {};
         let _remove_remote_target_handler = replication::RemoveRemoteTargetHandler {};
+        let _site_replication_add_handler = site_replication::SiteReplicationAddHandler {};
+        let _site_replication_info_handler = site_replication::SiteReplicationInfoHandler {};
+        let _site_replication_status_handler = site_replication::SiteReplicationStatusHandler {};
 
         // Just verify they can be created without panicking
         // Test passes if we reach this point without panicking

rustfs/src/admin/handlers/policies.rs

@@ -15,6 +15,7 @@
 use crate::{
     admin::{
         auth::validate_admin_request,
+        handlers::site_replication::site_replication_iam_change_hook,
         router::{AdminOperation, Operation, S3Router},
         utils::{encode_compatible_admin_payload, has_space_be, read_compatible_admin_body},
     },
@@ -28,7 +29,10 @@ use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
 use rustfs_credentials::get_global_action_cred;
 use rustfs_iam::error::is_err_no_such_user;
 use rustfs_iam::store::MappedPolicy;
-use rustfs_madmin::{GroupPolicyEntities, PolicyEntities, PolicyEntitiesResult, UserPolicyEntities};
+use rustfs_madmin::{
+    GroupPolicyEntities, PolicyEntities, PolicyEntitiesResult, SITE_REPL_API_VERSION, SRIAMItem, SRPolicyMapping,
+    UserPolicyEntities,
+};
 use rustfs_policy::policy::{
     Policy,
     action::{Action, AdminAction},
@@ -223,11 +227,26 @@ impl Operation for AddCannedPolicy {
         }
         let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) };
 
-        iam_store.set_policy(&query.name, policy).await.map_err(|e| {
+        let updated_at = iam_store.set_policy(&query.name, policy.clone()).await.map_err(|e| {
             warn!("set policy failed, e: {:?}", e);
             S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
         })?;
 
+        if let Err(err) = site_replication_iam_change_hook(SRIAMItem {
+            r#type: "policy".to_string(),
+            name: query.name.clone(),
+            policy: Some(
+                serde_json::to_value(&policy).map_err(|e| s3_error!(InternalError, "marshal policy failed, e: {:?}", e))?,
+            ),
+            updated_at: Some(updated_at),
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            ..Default::default()
+        })
+        .await
+        {
+            warn!(policy = %query.name, error = ?err, "site replication policy add hook failed");
+        }
+
         let mut header = HeaderMap::new();
         header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
         header.insert(CONTENT_LENGTH, "0".parse().unwrap());
@@ -337,6 +356,18 @@ impl Operation for RemoveCannedPolicy {
             S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
         })?;
 
+        if let Err(err) = site_replication_iam_change_hook(SRIAMItem {
+            r#type: "policy".to_string(),
+            name: query.name.clone(),
+            updated_at: Some(OffsetDateTime::now_utc()),
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            ..Default::default()
+        })
+        .await
+        {
+            warn!(policy = %query.name, error = ?err, "site replication policy delete hook failed");
+        }
+
         let mut header = HeaderMap::new();
         header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
         header.insert(CONTENT_LENGTH, "0".parse().unwrap());
@@ -428,7 +459,7 @@ impl Operation for SetPolicyForUserOrGroup {
             })?;
         }
 
-        iam_store
+        let updated_at = iam_store
             .policy_db_set(&query.user_or_group, rustfs_iam::store::UserType::Reg, query.is_group, &query.policy_name)
             .await
             .map_err(|e| {
@@ -436,6 +467,26 @@ impl Operation for SetPolicyForUserOrGroup {
                 S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
             })?;
 
+        if let Err(err) = site_replication_iam_change_hook(SRIAMItem {
+            r#type: "policy-mapping".to_string(),
+            policy_mapping: Some(SRPolicyMapping {
+                user_or_group: query.user_or_group.clone(),
+                user_type: rustfs_iam::store::UserType::Reg.to_u64(),
+                is_group: query.is_group,
+                policy: query.policy_name.clone(),
+                updated_at: Some(updated_at),
+                api_version: Some(SITE_REPL_API_VERSION.to_string()),
+                ..Default::default()
+            }),
+            updated_at: Some(updated_at),
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            ..Default::default()
+        })
+        .await
+        {
+            warn!(target = %query.user_or_group, error = ?err, "site replication policy mapping hook failed");
+        }
+
         let mut header = HeaderMap::new();
         header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
         header.insert(CONTENT_LENGTH, "0".parse().unwrap());
@@ -841,6 +892,26 @@ async fn handle_builtin_policy_association(req: S3Request<Body>, is_attach: bool
             S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
         })?;
 
+    if let Err(err) = site_replication_iam_change_hook(SRIAMItem {
+        r#type: "policy-mapping".to_string(),
+        policy_mapping: Some(SRPolicyMapping {
+            user_or_group: target_name.clone(),
+            user_type: rustfs_iam::store::UserType::Reg.to_u64(),
+            is_group,
+            policy: updated_policies.join(","),
+            updated_at: Some(updated_at),
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            ..Default::default()
+        }),
+        updated_at: Some(updated_at),
+        api_version: Some(SITE_REPL_API_VERSION.to_string()),
+        ..Default::default()
+    })
+    .await
+    {
+        warn!(target = %target_name, error = ?err, "site replication policy association hook failed");
+    }
+
     let policies_attached = if is_attach { changed_policies.clone() } else { Vec::new() };
     let policies_detached = if is_attach { Vec::new() } else { changed_policies };
 

rustfs/src/admin/handlers/service_account.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::admin::handlers::site_replication::site_replication_iam_change_hook;
 use crate::admin::utils::{encode_compatible_admin_payload, has_space_be, is_compat_admin_request, read_compatible_admin_body};
 use crate::auth::{constant_time_eq, get_condition_values, get_session_token};
 use crate::server::{ADMIN_PREFIX, RemoteAddr};
@@ -30,7 +31,8 @@ use rustfs_iam::sys::{NewServiceAccountOpts, UpdateServiceAccountOpts};
 use rustfs_madmin::{
     ACCESS_KEY_LIST_ALL, ACCESS_KEY_LIST_STS_ONLY, ACCESS_KEY_LIST_SVCACC_ONLY, ACCESS_KEY_LIST_USERS_ONLY, AddServiceAccountReq,
     AddServiceAccountResp, Credentials, InfoAccessKeyResp, InfoServiceAccountResp, LDAPSpecificAccessKeyInfo, ListAccessKeysResp,
-    ListServiceAccountsResp, OpenIDSpecificAccessKeyInfo, ServiceAccountInfo, TemporaryAccountInfoResp, UpdateServiceAccountReq,
+    ListServiceAccountsResp, OpenIDSpecificAccessKeyInfo, SITE_REPL_API_VERSION, SRIAMItem, SRSessionPolicy, SRSvcAccChange,
+    SRSvcAccCreate, SRSvcAccDelete, SRSvcAccUpdate, ServiceAccountInfo, TemporaryAccountInfoResp, UpdateServiceAccountReq,
 };
 use rustfs_policy::policy::action::{Action, AdminAction};
 use rustfs_policy::policy::{Args, Policy};
@@ -44,6 +46,15 @@ use time::OffsetDateTime;
 use tracing::{debug, warn};
 use url::form_urlencoded;
 
+fn sr_session_policy_from_value(value: Option<&serde_json::Value>) -> S3Result<SRSessionPolicy> {
+    let Some(value) = value else {
+        return Ok(SRSessionPolicy::default());
+    };
+
+    let raw = serde_json::to_string(value).map_err(|e| s3_error!(InvalidArgument, "marshal policy failed: {:?}", e))?;
+    SRSessionPolicy::from_json(&raw).map_err(|e| s3_error!(InvalidArgument, "marshal policy failed: {:?}", e))
+}
+
 fn compat_time_sentinel() -> OffsetDateTime {
     OffsetDateTime::UNIX_EPOCH
 }
@@ -294,6 +305,18 @@ impl Operation for AddServiceAccount {
             }
         }
 
+        let replication_claims = opts.claims.clone().unwrap_or_default();
+        let replication_policy = create_req
+            .policy
+            .as_ref()
+            .map(serde_json::to_string)
+            .transpose()
+            .map_err(|e| s3_error!(InvalidArgument, "marshal policy failed: {:?}", e))?;
+        let replication_groups = target_groups.clone().unwrap_or_default();
+        let replication_name = opts.name.clone().unwrap_or_default();
+        let replication_description = opts.description.clone().unwrap_or_default();
+        let replication_expiration = opts.expiration;
+
         let (new_cred, _) = iam_store
             .new_service_account(&target_user, target_groups, opts)
             .await
@@ -302,6 +325,39 @@ impl Operation for AddServiceAccount {
                 s3_error!(InternalError, "create service account failed, e: {:?}", e)
             })?;
 
+        if let Err(err) = site_replication_iam_change_hook(SRIAMItem {
+            r#type: "service-account".to_string(),
+            svc_acc_change: Some(SRSvcAccChange {
+                create: Some(SRSvcAccCreate {
+                    parent: target_user.clone(),
+                    access_key: new_cred.access_key.clone(),
+                    secret_key: new_cred.secret_key.clone(),
+                    groups: replication_groups,
+                    claims: replication_claims,
+                    session_policy: replication_policy
+                        .as_deref()
+                        .map(SRSessionPolicy::from_json)
+                        .transpose()
+                        .map_err(|e| s3_error!(InvalidArgument, "marshal policy failed: {:?}", e))?
+                        .unwrap_or_default(),
+                    status: String::new(),
+                    name: replication_name,
+                    description: replication_description,
+                    expiration: replication_expiration,
+                    api_version: Some(SITE_REPL_API_VERSION.to_string()),
+                }),
+                api_version: Some(SITE_REPL_API_VERSION.to_string()),
+                ..Default::default()
+            }),
+            updated_at: Some(OffsetDateTime::now_utc()),
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            ..Default::default()
+        })
+        .await
+        {
+            warn!(access_key = %new_cred.access_key, error = ?err, "site replication add service account hook failed");
+        }
+
         let resp = AddServiceAccountResp {
             credentials: Credentials {
                 access_key: &new_cred.access_key,
@@ -502,22 +558,54 @@ impl Operation for UpdateServiceAccount {
             return Err(s3_error!(AccessDenied, "access denied"));
         }
 
-        let sp = parse_update_service_account_policy(update_req.new_policy)?;
+        let new_secret_key = update_req.new_secret_key.clone();
+        let new_status = update_req.new_status.clone();
+        let new_name = update_req.new_name.clone();
+        let new_description = update_req.new_description.clone();
+        let new_expiration = update_req.new_expiration;
+        let new_policy = update_req.new_policy.clone();
+
+        let sp = parse_update_service_account_policy(new_policy.clone())?;
 
         let opts = UpdateServiceAccountOpts {
-            secret_key: update_req.new_secret_key,
-            status: update_req.new_status,
-            name: update_req.new_name,
-            description: update_req.new_description,
-            expiration: update_req.new_expiration,
+            secret_key: new_secret_key.clone(),
+            status: new_status.clone(),
+            name: new_name.clone(),
+            description: new_description.clone(),
+            expiration: new_expiration,
             session_policy: sp,
         };
 
-        let _ = iam_store
+        let updated_at = iam_store
             .update_service_account(&access_key, opts)
             .await
             .map_err(|e| map_service_account_lookup_error(e, "update service account failed"))?;
 
+        if let Err(err) = site_replication_iam_change_hook(SRIAMItem {
+            r#type: "service-account".to_string(),
+            svc_acc_change: Some(SRSvcAccChange {
+                update: Some(SRSvcAccUpdate {
+                    access_key: access_key.clone(),
+                    secret_key: new_secret_key.unwrap_or_default(),
+                    status: new_status.unwrap_or_default(),
+                    name: new_name.unwrap_or_default(),
+                    description: new_description.unwrap_or_default(),
+                    session_policy: sr_session_policy_from_value(new_policy.as_ref())?,
+                    expiration: new_expiration,
+                    api_version: Some(SITE_REPL_API_VERSION.to_string()),
+                }),
+                api_version: Some(SITE_REPL_API_VERSION.to_string()),
+                ..Default::default()
+            }),
+            updated_at: Some(updated_at),
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            ..Default::default()
+        })
+        .await
+        {
+            warn!(access_key = %access_key, error = ?err, "site replication update service account hook failed");
+        }
+
         let mut header = HeaderMap::new();
         header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
         header.insert(CONTENT_LENGTH, "0".parse().unwrap());
@@ -1205,6 +1293,25 @@ impl Operation for DeleteServiceAccount {
             s3_error!(InternalError, "delete service account failed")
         })?;
 
+        if let Err(err) = site_replication_iam_change_hook(SRIAMItem {
+            r#type: "service-account".to_string(),
+            svc_acc_change: Some(SRSvcAccChange {
+                delete: Some(SRSvcAccDelete {
+                    access_key: query.access_key.clone(),
+                    api_version: Some(SITE_REPL_API_VERSION.to_string()),
+                }),
+                api_version: Some(SITE_REPL_API_VERSION.to_string()),
+                ..Default::default()
+            }),
+            updated_at: Some(OffsetDateTime::now_utc()),
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            ..Default::default()
+        })
+        .await
+        {
+            warn!(access_key = %query.access_key, error = ?err, "site replication delete service account hook failed");
+        }
+
         let mut header = HeaderMap::new();
         header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
         header.insert(CONTENT_LENGTH, "0".parse().unwrap());

rustfs/src/admin/handlers/sts.rs

@@ -14,7 +14,10 @@
 
 use super::is_admin::IsAdminHandler;
 use crate::{
-    admin::router::{AdminOperation, Operation, S3Router},
+    admin::{
+        handlers::site_replication::site_replication_iam_change_hook,
+        router::{AdminOperation, Operation, S3Router},
+    },
     auth::{check_key_valid, get_session_token},
     server::ADMIN_PREFIX,
 };
@@ -23,8 +26,10 @@ use http::header::HeaderValue;
 use hyper::Method;
 use matchit::Params;
 use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
+use rustfs_credentials::get_global_action_cred;
 use rustfs_ecstore::bucket::utils::serialize;
 use rustfs_iam::{manager::get_token_signing_key, oidc::OidcClaims, sys::SESSION_POLICY_NAME};
+use rustfs_madmin::{SITE_REPL_API_VERSION, SRIAMItem, SRSTSCredential};
 use rustfs_policy::{auth::get_new_credentials_with_metadata, policy::Policy};
 use s3s::{
     Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result,
@@ -164,11 +169,31 @@ async fn handle_assume_role(
 
     debug!("AssumeRole get new_cred {:?}", &new_cred);
 
-    if let Err(_err) = iam_store.set_temp_user(&new_cred.access_key, &new_cred, None).await {
-        return Err(s3_error!(InternalError, "set_temp_user failed"));
-    }
+    let updated_at = iam_store
+        .set_temp_user(&new_cred.access_key, &new_cred, None)
+        .await
+        .map_err(|_| s3_error!(InternalError, "set_temp_user failed"))?;
 
-    // TODO: globalSiteReplicationSys
+    let root_access_key = get_global_action_cred().map(|cred| cred.access_key);
+    if root_access_key.as_deref() != Some(new_cred.parent_user.as_str())
+        && let Err(err) = site_replication_iam_change_hook(SRIAMItem {
+            r#type: "sts-credential".to_string(),
+            sts_credential: Some(SRSTSCredential {
+                access_key: new_cred.access_key.clone(),
+                secret_key: new_cred.secret_key.clone(),
+                session_token: new_cred.session_token.clone(),
+                parent_user: new_cred.parent_user.clone(),
+                api_version: Some(SITE_REPL_API_VERSION.to_string()),
+                ..Default::default()
+            }),
+            updated_at: Some(updated_at),
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            ..Default::default()
+        })
+        .await
+    {
+        warn!("site replication STS hook failed, err: {err}");
+    }
 
     let resp = AssumeRoleOutput {
         credentials: Some(Credentials {
@@ -358,11 +383,30 @@ pub async fn create_oidc_sts_credentials(
     // Store temp user in IAM
     let iam_store = rustfs_iam::get().map_err(|_| s3_error!(InternalError, "IAM not initialized"))?;
 
-    iam_store
+    let updated_at = iam_store
         .set_temp_user(&new_cred.access_key, &new_cred, None)
         .await
         .map_err(|_| s3_error!(InternalError, "failed to store temp user"))?;
 
+    if let Err(err) = site_replication_iam_change_hook(SRIAMItem {
+        r#type: "sts-credential".to_string(),
+        sts_credential: Some(SRSTSCredential {
+            access_key: new_cred.access_key.clone(),
+            secret_key: new_cred.secret_key.clone(),
+            session_token: new_cred.session_token.clone(),
+            parent_user: new_cred.parent_user.clone(),
+            parent_policy_mapping: policies.join(","),
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+        }),
+        updated_at: Some(updated_at),
+        api_version: Some(SITE_REPL_API_VERSION.to_string()),
+        ..Default::default()
+    })
+    .await
+    {
+        warn!("site replication OIDC STS hook failed, err: {err}");
+    }
+
     Ok(new_cred)
 }
 

rustfs/src/admin/handlers/user.rs

@@ -16,6 +16,7 @@ use super::{account_info, group, service_account, user_iam, user_lifecycle, user
 use crate::{
     admin::{
         auth::validate_admin_request,
+        handlers::site_replication::site_replication_iam_change_hook,
         router::{AdminOperation, Operation, S3Router},
         utils::{encode_compatible_admin_payload, has_space_be, read_compatible_admin_body},
     },
@@ -31,7 +32,8 @@ use rustfs_iam::{
     sys::{NewServiceAccountOpts, UpdateServiceAccountOpts},
 };
 use rustfs_madmin::{
-    AccountStatus, AddOrUpdateUserReq, IAMEntities, IAMErrEntities, IAMErrEntity, IAMErrPolicyEntity,
+    AccountStatus, AddOrUpdateUserReq, IAMEntities, IAMErrEntities, IAMErrEntity, IAMErrPolicyEntity, SITE_REPL_API_VERSION,
+    SRIAMItem, SRIAMUser,
     user::{ImportIAMResult, SRSessionPolicy, SRSvcAccCreate},
 };
 use rustfs_policy::policy::action::{Action, AdminAction};
@@ -225,11 +227,28 @@ impl Operation for AddUser {
         )
         .await?;
 
-        iam_store
+        let updated_at = iam_store
             .create_user(ak, &args)
             .await
             .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("create_user err {e}")))?;
 
+        if let Err(err) = site_replication_iam_change_hook(SRIAMItem {
+            r#type: "iam-user".to_string(),
+            iam_user: Some(SRIAMUser {
+                access_key: ak.to_string(),
+                is_delete_req: false,
+                user_req: Some(args.clone()),
+                api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            }),
+            updated_at: Some(updated_at),
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            ..Default::default()
+        })
+        .await
+        {
+            warn!(access_key = %ak, error = ?err, "site replication create user hook failed");
+        }
+
         let mut header = HeaderMap::new();
         header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
         header.insert(CONTENT_LENGTH, "0".parse().unwrap());
@@ -432,7 +451,22 @@ impl Operation for RemoveUser {
             .await
             .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("delete_user err {e}")))?;
 
-        // TODO: IAMChangeHook
+        if let Err(err) = site_replication_iam_change_hook(SRIAMItem {
+            r#type: "iam-user".to_string(),
+            iam_user: Some(SRIAMUser {
+                access_key: ak.to_string(),
+                is_delete_req: true,
+                user_req: None,
+                api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            }),
+            updated_at: Some(time::OffsetDateTime::now_utc()),
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            ..Default::default()
+        })
+        .await
+        {
+            warn!(access_key = %ak, error = ?err, "site replication delete user hook failed");
+        }
 
         let mut header = HeaderMap::new();
         header.insert(CONTENT_TYPE, "application/json".parse().unwrap());

rustfs/src/admin/mod.rs

@@ -16,6 +16,7 @@ mod auth;
 pub mod console;
 pub mod handlers;
 pub mod router;
+pub mod service;
 pub mod utils;
 
 #[cfg(test)]
@@ -24,7 +25,8 @@ mod console_test;
 mod route_registration_test;
 
 use handlers::{
-    bucket_meta, heal, health, kms, oidc, pools, profile_admin, quota, rebalance, replication, sts, system, tier, user,
+    bucket_meta, heal, health, kms, oidc, pools, profile_admin, quota, rebalance, replication, site_replication, sts, system,
+    tier, user,
 };
 use router::{AdminOperation, S3Router};
 use s3s::route::S3Route;
@@ -55,6 +57,7 @@ pub fn make_admin_route(console_enabled: bool) -> std::io::Result<impl S3Route>
     bucket_meta::register_bucket_meta_route(&mut r)?;
 
     replication::register_replication_route(&mut r)?;
+    site_replication::register_site_replication_route(&mut r)?;
     profile_admin::register_profiling_route(&mut r)?;
     kms::register_kms_route(&mut r)?;
     oidc::register_oidc_route(&mut r)?;

rustfs/src/admin/route_registration_test.rs

@@ -14,7 +14,8 @@
 
 use crate::admin::{
     handlers::{
-        bucket_meta, heal, health, kms, oidc, pools, profile_admin, quota, rebalance, replication, sts, system, tier, user,
+        bucket_meta, heal, health, kms, oidc, pools, profile_admin, quota, rebalance, replication, site_replication, sts, system,
+        tier, user,
     },
     router::{AdminOperation, S3Router},
 };
@@ -50,6 +51,7 @@ fn register_admin_routes(router: &mut S3Router<AdminOperation>) {
     quota::register_quota_route(router).expect("register quota route");
     bucket_meta::register_bucket_meta_route(router).expect("register bucket meta route");
     replication::register_replication_route(router).expect("register replication route");
+    site_replication::register_site_replication_route(router).expect("register site replication route");
     profile_admin::register_profiling_route(router).expect("register profile route");
     kms::register_kms_route(router).expect("register kms route");
     oidc::register_oidc_route(router).expect("register oidc route");
@@ -60,7 +62,6 @@ fn test_register_routes_cover_representative_admin_paths() {
     let mut router: S3Router<AdminOperation> = S3Router::new(false);
 
     register_admin_routes(&mut router);
-
     assert_route(&router, Method::GET, HEALTH_PREFIX);
     assert_route(&router, Method::HEAD, HEALTH_PREFIX);
     assert_route(&router, Method::GET, HEALTH_READY_PATH);
@@ -119,6 +120,23 @@ fn test_register_routes_cover_representative_admin_paths() {
     assert_route(&router, Method::PUT, &admin_path("/v3/import-bucket-metadata"));
     assert_route(&router, Method::GET, &admin_path("/v3/list-remote-targets"));
     assert_route(&router, Method::PUT, &admin_path("/v3/set-remote-target"));
+    assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/add"));
+    assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/remove"));
+    assert_route(&router, Method::GET, &admin_path("/v3/site-replication/info"));
+    assert_route(&router, Method::GET, &admin_path("/v3/site-replication/metainfo"));
+    assert_route(&router, Method::GET, &admin_path("/v3/site-replication/status"));
+    assert_route(&router, Method::POST, &admin_path("/v3/site-replication/devnull"));
+    assert_route(&router, Method::POST, &admin_path("/v3/site-replication/netperf"));
+    assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/peer/join"));
+    assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/peer/bucket-ops"));
+    assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/peer/iam-item"));
+    assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/peer/bucket-meta"));
+    assert_route(&router, Method::GET, &admin_path("/v3/site-replication/peer/idp-settings"));
+    assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/edit"));
+    assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/peer/edit"));
+    assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/peer/remove"));
+    assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/resync/op"));
+    assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/state/edit"));
     assert_route(&router, Method::GET, &admin_path("/debug/pprof/profile"));
 
     assert_route(&router, Method::POST, &admin_path("/v3/kms/create-key"));
@@ -178,6 +196,10 @@ fn test_admin_alias_paths_match_existing_admin_routes() {
         (Method::GET, compat_admin_alias_path("/v3/oidc/callback/default")),
         (Method::GET, compat_admin_alias_path("/v3/oidc/config")),
         (Method::PUT, compat_admin_alias_path("/v3/oidc/config/default")),
+        (Method::PUT, compat_admin_alias_path("/v3/site-replication/add")),
+        (Method::GET, compat_admin_alias_path("/v3/site-replication/info")),
+        (Method::GET, compat_admin_alias_path("/v3/site-replication/status")),
+        (Method::PUT, compat_admin_alias_path("/v3/site-replication/peer/join")),
         (Method::GET, compat_admin_alias_path("/export-bucket-metadata")),
         (Method::GET, compat_admin_alias_path("/v3/export-bucket-metadata")),
         (Method::PUT, compat_admin_alias_path("/import-bucket-metadata")),

rustfs/src/admin/service/mod.rs

@@ -0,0 +1,15 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod site_replication;

rustfs/src/admin/service/site_replication.rs

@@ -0,0 +1,43 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use rustfs_ecstore::config::com::read_config;
+use rustfs_ecstore::error::Error as StorageError;
+use rustfs_ecstore::new_object_layer_fn;
+use s3s::{S3Error, S3ErrorCode, S3Result};
+
+const SITE_REPLICATION_STATE_PATH: &str = "config/site-replication/state.json";
+
+/// Reload persisted site-replication state.
+///
+/// RustFS does not currently keep a separate in-memory cache for this state,
+/// so "reload" means validating that the persisted JSON is readable.
+pub async fn reload_site_replication_runtime_state() -> S3Result<()> {
+    let Some(store) = new_object_layer_fn() else {
+        return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
+    };
+
+    match read_config(store, SITE_REPLICATION_STATE_PATH).await {
+        Ok(data) => {
+            let _: serde_json::Value = serde_json::from_slice(&data)
+                .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("invalid site replication state: {e}")))?;
+            Ok(())
+        }
+        Err(StorageError::ConfigNotFound) => Ok(()),
+        Err(err) => Err(S3Error::with_message(
+            S3ErrorCode::InternalError,
+            format!("failed to load site replication state: {err}"),
+        )),
+    }
+}

rustfs/src/app/bucket_usecase.rs

@@ -14,6 +14,9 @@
 
 //! Bucket application use-case contracts.
 
+use crate::admin::handlers::site_replication::{
+    site_replication_bucket_meta_hook, site_replication_delete_bucket_hook, site_replication_make_bucket_hook,
+};
 use crate::app::context::{AppContext, default_notify_interface, get_global_app_context};
 use crate::auth::get_condition_values;
 use crate::error::ApiError;
@@ -49,6 +52,7 @@ use rustfs_ecstore::store_api::{
     BucketOperations, BucketOptions, DeleteBucketOptions, ListObjectVersionsInfo, ListObjectsV2Info, ListOperations,
     MakeBucketOptions, ObjectInfo,
 };
+use rustfs_madmin::{SITE_REPL_API_VERSION, SRBucketMeta};
 use rustfs_policy::policy::{
     action::{Action, S3Action},
     {BucketPolicy, BucketPolicyArgs, Effect, Validator},
@@ -81,6 +85,16 @@ fn to_internal_error(err: impl Display) -> S3Error {
     S3Error::with_message(S3ErrorCode::InternalError, format!("{err}"))
 }
 
+fn sr_bucket_meta_item(bucket: String, item_type: &str) -> SRBucketMeta {
+    SRBucketMeta {
+        bucket,
+        r#type: item_type.to_string(),
+        updated_at: Some(time::OffsetDateTime::now_utc()),
+        api_version: Some(SITE_REPL_API_VERSION.to_string()),
+        ..Default::default()
+    }
+}
+
 fn versioning_configuration_has_object_lock_incompatible_settings(config: &VersioningConfiguration) -> bool {
     config.suspended()
         || config.exclude_folders.unwrap_or(false)
@@ -538,6 +552,7 @@ impl DefaultBucketUsecase {
             object_lock_enabled_for_bucket,
             ..
         } = req.input;
+        let lock_enabled = object_lock_enabled_for_bucket.is_some_and(|v| v);
 
         let Some(store) = new_object_layer_fn() else {
             return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
@@ -548,7 +563,7 @@ impl DefaultBucketUsecase {
                 &bucket,
                 &MakeBucketOptions {
                     force_create: false,
-                    lock_enabled: object_lock_enabled_for_bucket.is_some_and(|v| v),
+                    lock_enabled,
                     ..Default::default()
                 },
             )
@@ -566,6 +581,10 @@ impl DefaultBucketUsecase {
             Err(e) => return Err(ApiError::from(e).into()),
         }
 
+        if let Err(err) = site_replication_make_bucket_hook(&bucket, lock_enabled).await {
+            warn!(bucket = %bucket, error = ?err, "site replication make bucket hook failed");
+        }
+
         let output = CreateBucketOutput::default();
         counter!("rustfs_create_bucket_total").increment(1);
         let result = Ok(S3Response::new(output));
@@ -637,6 +656,10 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
+        if let Err(err) = site_replication_delete_bucket_hook(&input.bucket, force).await {
+            warn!(bucket = %input.bucket, error = ?err, "site replication delete bucket hook failed");
+        }
+
         let result = Ok(S3Response::new(DeleteBucketOutput {}));
         let _ = helper.complete(&result);
         result
@@ -791,6 +814,11 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
+        let item = sr_bucket_meta_item(bucket.clone(), "sse-config");
+        if let Err(err) = site_replication_bucket_meta_hook(item).await {
+            warn!(bucket = %bucket, error = ?err, "site replication bucket encryption delete hook failed");
+        }
+
         Ok(S3Response::with_status(DeleteBucketEncryptionOutput::default(), StatusCode::NO_CONTENT))
     }
 
@@ -818,6 +846,11 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
+        let item = sr_bucket_meta_item(bucket.clone(), "cors-config");
+        if let Err(err) = site_replication_bucket_meta_hook(item).await {
+            warn!(bucket = %bucket, error = ?err, "site replication bucket cors delete hook failed");
+        }
+
         Ok(S3Response::new(DeleteBucketCorsOutput {}))
     }
 
@@ -845,6 +878,11 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
+        let item = sr_bucket_meta_item(bucket.clone(), "lc-config");
+        if let Err(err) = site_replication_bucket_meta_hook(item).await {
+            warn!(bucket = %bucket, error = ?err, "site replication bucket lifecycle delete hook failed");
+        }
+
         Ok(S3Response::new(DeleteBucketLifecycleOutput::default()))
     }
 
@@ -871,6 +909,11 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
+        let item = sr_bucket_meta_item(bucket.clone(), "policy");
+        if let Err(err) = site_replication_bucket_meta_hook(item).await {
+            warn!(bucket = %bucket, error = ?err, "site replication bucket policy delete hook failed");
+        }
+
         Ok(S3Response::new(DeleteBucketPolicyOutput {}))
     }
 
@@ -896,6 +939,11 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
+        let item = sr_bucket_meta_item(bucket.clone(), "replication-config");
+        if let Err(err) = site_replication_bucket_meta_hook(item).await {
+            warn!(bucket = %bucket, error = ?err, "site replication bucket replication-config delete hook failed");
+        }
+
         // TODO: remove targets
         info!(bucket = %bucket, "deleted bucket replication config");
 
@@ -917,6 +965,11 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
+        let item = sr_bucket_meta_item(bucket.clone(), "tags");
+        if let Err(err) = site_replication_bucket_meta_hook(item).await {
+            warn!(bucket = %bucket, error = ?err, "site replication bucket tagging delete hook failed");
+        }
+
         Ok(S3Response::new(tagging::build_delete_bucket_tagging_output()))
     }
 
@@ -1374,6 +1427,15 @@ impl DefaultBucketUsecase {
         metadata_sys::update(&bucket, BUCKET_SSECONFIG, data)
             .await
             .map_err(ApiError::from)?;
+
+        let mut item = sr_bucket_meta_item(bucket.clone(), "sse-config");
+        item.sse_config = Some(
+            serialize_config(&server_side_encryption_configuration)
+                .and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?,
+        );
+        if let Err(err) = site_replication_bucket_meta_hook(item).await {
+            warn!(bucket = %bucket, error = ?err, "site replication bucket encryption hook failed");
+        }
         Ok(S3Response::new(encryption::build_put_bucket_encryption_output()))
     }
 
@@ -1420,6 +1482,14 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
+        let mut item = sr_bucket_meta_item(bucket.clone(), "lc-config");
+        item.expiry_lc_config =
+            Some(serialize_config(&input_cfg).and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?);
+        item.expiry_updated_at = item.updated_at;
+        if let Err(err) = site_replication_bucket_meta_hook(item).await {
+            warn!(bucket = %bucket, error = ?err, "site replication bucket lifecycle hook failed");
+        }
+
         if lifecycle_has_transition_rules(&input_cfg)
             && let Some(store) = new_object_layer_fn()
         {
@@ -1564,6 +1634,12 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
+        let mut item = sr_bucket_meta_item(bucket.clone(), "policy");
+        item.policy = Some(serde_json::from_str(&policy).map_err(|e| s3_error!(InvalidArgument, "parse policy failed {:?}", e))?);
+        if let Err(err) = site_replication_bucket_meta_hook(item).await {
+            warn!(bucket = %bucket, error = ?err, "site replication bucket policy hook failed");
+        }
+
         Ok(S3Response::new(PutBucketPolicyOutput {}))
     }
 
@@ -1593,6 +1669,13 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
+        let mut item = sr_bucket_meta_item(bucket.clone(), "cors-config");
+        item.cors =
+            Some(serialize_config(&cors_configuration).and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?);
+        if let Err(err) = site_replication_bucket_meta_hook(item).await {
+            warn!(bucket = %bucket, error = ?err, "site replication bucket cors hook failed");
+        }
+
         Ok(S3Response::new(PutBucketCorsOutput::default()))
     }
 
@@ -1626,6 +1709,14 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
+        let mut item = sr_bucket_meta_item(bucket.clone(), "replication-config");
+        item.replication_config = Some(
+            serialize_config(&replication_configuration).and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?,
+        );
+        if let Err(err) = site_replication_bucket_meta_hook(item).await {
+            warn!(bucket = %bucket, error = ?err, "site replication bucket replication-config hook failed");
+        }
+
         Ok(S3Response::new(replication::build_put_bucket_replication_output()))
     }
 
@@ -1687,6 +1778,12 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
+        let mut item = sr_bucket_meta_item(bucket.clone(), "tags");
+        item.tags = Some(serialize_config(&tagging).and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?);
+        if let Err(err) = site_replication_bucket_meta_hook(item).await {
+            warn!(bucket = %bucket, error = ?err, "site replication bucket tagging hook failed");
+        }
+
         Ok(S3Response::new(tagging::build_put_bucket_tagging_output()))
     }
 
@@ -1713,6 +1810,14 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
 
+        let mut item = sr_bucket_meta_item(bucket.clone(), "version-config");
+        item.versioning = Some(
+            serialize_config(&versioning_configuration).and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?,
+        );
+        if let Err(err) = site_replication_bucket_meta_hook(item).await {
+            warn!(bucket = %bucket, error = ?err, "site replication bucket versioning hook failed");
+        }
+
         Ok(S3Response::new(PutBucketVersioningOutput {}))
     }
 

rustfs/src/storage/rpc/node_service.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::admin::service::site_replication::reload_site_replication_runtime_state;
 use bytes::Bytes;
 use futures::Stream;
 use futures_util::future::join_all;
@@ -757,7 +758,16 @@ impl Node for NodeService {
                 error_info: Some("errServerNotInitialized".to_string()),
             }));
         };
-        todo!()
+        match reload_site_replication_runtime_state().await {
+            Ok(()) => Ok(Response::new(ReloadSiteReplicationConfigResponse {
+                success: true,
+                error_info: None,
+            })),
+            Err(err) => Ok(Response::new(ReloadSiteReplicationConfigResponse {
+                success: false,
+                error_info: Some(err.to_string()),
+            })),
+        }
     }
 
     async fn signal_service(&self, request: Request<SignalServiceRequest>) -> Result<Response<SignalServiceResponse>, Status> {