diff --git a/Cargo.lock b/Cargo.lock index 07834d903..33842deb3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7648,6 +7648,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "serial_test", + "sha2 0.11.0-rc.5", "shadow-rs", "socket2", "starshard", diff --git a/crates/e2e_test/src/replication_extension_test.rs b/crates/e2e_test/src/replication_extension_test.rs index 3fb31735f..c41a82e64 100644 --- a/crates/e2e_test/src/replication_extension_test.rs +++ b/crates/e2e_test/src/replication_extension_test.rs @@ -13,14 +13,38 @@ // limitations under the License. use crate::common::{RustFSTestEnvironment, init_logging, local_http_client}; +use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration}; use http::header::{CONTENT_TYPE, HOST}; use reqwest::StatusCode; +use rustfs_madmin::{ + PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus, ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus, + SRStatusInfo, SiteReplicationInfo, SyncStatus, +}; use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; use serial_test::serial; +use std::collections::BTreeMap; use std::error::Error; +use time::Duration as TimeDuration; +use tokio::time::{Duration, sleep}; + +#[derive(Debug, Clone, serde::Deserialize)] +struct ReplicationResetStatusResponse { + #[serde(rename = "Targets", default)] + targets: Vec, +} + +#[derive(Debug, Clone, serde::Deserialize)] +struct ReplicationResetStatusTarget { + #[serde(rename = "Arn", default)] + arn: String, + #[serde(rename = "ResetID", default)] + reset_id: String, + #[serde(rename = "Status", default)] + status: String, +} async fn signed_request( method: http::Method, @@ -239,6 +263,265 @@ async fn list_replication_targets_request( signed_request(http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await } +async fn site_replication_add( + env: &RustFSTestEnvironment, + sites: &[PeerSite], +) -> Result> { + let url = format!("{}/rustfs/admin/v3/site-replication/add?replicateILMExpiry=false", env.url); + let response = signed_request( + http::Method::PUT, + &url, + &env.access_key, + &env.secret_key, + Some(serde_json::to_vec(sites)?), + Some("application/json"), + ) + .await?; + + if response.status() != StatusCode::OK { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(format!("site replication add failed: {status} {body}").into()); + } + + Ok(serde_json::from_slice(&response.bytes().await?)?) +} + +async fn site_replication_info(env: &RustFSTestEnvironment) -> Result> { + let url = format!("{}/rustfs/admin/v3/site-replication/info", env.url); + let response = signed_request(http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await?; + + if response.status() != StatusCode::OK { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(format!("site replication info failed: {status} {body}").into()); + } + + Ok(serde_json::from_slice(&response.bytes().await?)?) +} + +async fn site_replication_resync_op( + env: &RustFSTestEnvironment, + operation: &str, + peer: &PeerInfo, +) -> Result> { + let url = format!("{}/rustfs/admin/v3/site-replication/resync/op?operation={operation}", env.url); + let response = signed_request( + http::Method::PUT, + &url, + &env.access_key, + &env.secret_key, + Some(serde_json::to_vec(peer)?), + Some("application/json"), + ) + .await?; + + if response.status() != StatusCode::OK { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(format!("site replication resync {operation} failed: {status} {body}").into()); + } + + Ok(serde_json::from_slice(&response.bytes().await?)?) +} + +async fn site_replication_edit( + env: &RustFSTestEnvironment, + query: &str, + peer: &PeerInfo, +) -> Result> { + let url = if query.is_empty() { + format!("{}/rustfs/admin/v3/site-replication/edit", env.url) + } else { + format!("{}/rustfs/admin/v3/site-replication/edit?{query}", env.url) + }; + let response = signed_request( + http::Method::PUT, + &url, + &env.access_key, + &env.secret_key, + Some(serde_json::to_vec(peer)?), + Some("application/json"), + ) + .await?; + + if response.status() != StatusCode::OK { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(format!("site replication edit failed: {status} {body}").into()); + } + + Ok(serde_json::from_slice(&response.bytes().await?)?) +} + +async fn site_replication_status(env: &RustFSTestEnvironment, query: &str) -> Result> { + let url = if query.is_empty() { + format!("{}/rustfs/admin/v3/site-replication/status", env.url) + } else { + format!("{}/rustfs/admin/v3/site-replication/status?{query}", env.url) + }; + let response = signed_request(http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await?; + + if response.status() != StatusCode::OK { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(format!("site replication status failed: {status} {body}").into()); + } + + Ok(serde_json::from_slice(&response.bytes().await?)?) +} + +async fn site_replication_remove( + env: &RustFSTestEnvironment, + req: &SRRemoveReq, +) -> Result> { + let url = format!("{}/rustfs/admin/v3/site-replication/remove", env.url); + let response = signed_request( + http::Method::PUT, + &url, + &env.access_key, + &env.secret_key, + Some(serde_json::to_vec(req)?), + Some("application/json"), + ) + .await?; + + if response.status() != StatusCode::OK { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(format!("site replication remove failed: {status} {body}").into()); + } + + Ok(serde_json::from_slice(&response.bytes().await?)?) +} + +async fn site_replication_state_edit( + env: &RustFSTestEnvironment, + body: &rustfs_madmin::SRStateEditReq, +) -> Result<(), Box> { + let url = format!("{}/rustfs/admin/v3/site-replication/state/edit", env.url); + let response = signed_request( + http::Method::PUT, + &url, + &env.access_key, + &env.secret_key, + Some(serde_json::to_vec(body)?), + Some("application/json"), + ) + .await?; + + if response.status() != StatusCode::OK { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(format!("site replication state edit failed: {status} {body}").into()); + } + + Ok(()) +} + +async fn get_replication_reset_status( + env: &RustFSTestEnvironment, + bucket: &str, + arn: &str, +) -> Result> { + let url = format!("{}/{bucket}?replication-reset-status&arn={}", env.url, urlencoding::encode(arn)); + let response = signed_request(http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await?; + + if response.status() != StatusCode::OK { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(format!("replication reset status failed: {status} {body}").into()); + } + + Ok(serde_json::from_slice(&response.bytes().await?)?) +} + +async fn wait_for_site_replication_enabled( + env: &RustFSTestEnvironment, + expected_sites: usize, +) -> Result> { + for _ in 0..40 { + let info = site_replication_info(env).await?; + if info.enabled && info.sites.len() == expected_sites { + return Ok(info); + } + sleep(Duration::from_millis(250)).await; + } + + Err(format!("site replication did not reach {expected_sites} sites on {}", env.address).into()) +} + +async fn wait_for_site_replication_disabled( + env: &RustFSTestEnvironment, +) -> Result> { + wait_for_site_replication_info(env, |info| !info.enabled && info.sites.is_empty()).await +} + +async fn wait_for_site_replication_info( + env: &RustFSTestEnvironment, + predicate: F, +) -> Result> +where + F: Fn(&SiteReplicationInfo) -> bool, +{ + for _ in 0..40 { + let info = site_replication_info(env).await?; + if predicate(&info) { + return Ok(info); + } + sleep(Duration::from_millis(250)).await; + } + + Err(format!("site replication info did not reach expected state on {}", env.address).into()) +} + +async fn wait_for_site_replication_status( + env: &RustFSTestEnvironment, + query: &str, + predicate: F, +) -> Result> +where + F: Fn(&SRStatusInfo) -> bool, +{ + for _ in 0..40 { + let status = site_replication_status(env, query).await?; + if predicate(&status) { + return Ok(status); + } + sleep(Duration::from_millis(250)).await; + } + + Err(format!("site replication status did not reach expected state on {}", env.address).into()) +} + +async fn wait_for_replication_reset_target( + env: &RustFSTestEnvironment, + bucket: &str, + arn: &str, + predicate: F, +) -> Result> +where + F: Fn(&ReplicationResetStatusTarget) -> bool, +{ + let mut last_seen = None; + for _ in 0..40 { + let status = get_replication_reset_status(env, bucket, arn).await?; + if let Some(target) = status.targets.into_iter().find(|target| target.arn == arn) { + if predicate(&target) { + return Ok(target); + } + last_seen = Some(target); + } + sleep(Duration::from_millis(250)).await; + } + + Err(format!( + "replication reset target {arn} for bucket {bucket} did not reach expected state; last seen: {:?}", + last_seen + ) + .into()) +} + async fn build_replication_pair( enable_target_versioning: bool, ) -> Result<(RustFSTestEnvironment, RustFSTestEnvironment, String), Box> { @@ -800,3 +1083,410 @@ async fn test_remove_remote_target_rejects_target_used_by_replication() -> Resul Ok(()) } + +#[tokio::test] +#[serial] +async fn test_site_replication_resync_start_cancel_restart_real_dual_node() -> Result<(), Box> { + init_logging(); + + let mut source_env = RustFSTestEnvironment::new().await?; + source_env.start_rustfs_server(vec![]).await?; + + let mut target_env = RustFSTestEnvironment::new().await?; + target_env.start_rustfs_server_without_cleanup(vec![]).await?; + + let source_bucket = "site-repl-resync-src"; + let target_bucket = "site-repl-resync-dst"; + + let source_client = source_env.create_s3_client(); + let target_client = target_env.create_s3_client(); + + source_client.create_bucket().bucket(source_bucket).send().await?; + target_client.create_bucket().bucket(target_bucket).send().await?; + enable_bucket_versioning(&source_env, source_bucket).await?; + enable_bucket_versioning(&target_env, target_bucket).await?; + + let add_status = site_replication_add( + &source_env, + &[ + PeerSite { + name: "source-site".to_string(), + endpoint: source_env.url.clone(), + access_key: source_env.access_key.clone(), + secret_key: source_env.secret_key.clone(), + }, + PeerSite { + name: "target-site".to_string(), + endpoint: target_env.url.clone(), + access_key: target_env.access_key.clone(), + secret_key: target_env.secret_key.clone(), + }, + ], + ) + .await?; + assert!(add_status.success, "unexpected site add result: {:?}", add_status); + + let source_info = wait_for_site_replication_enabled(&source_env, 2).await?; + let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?; + let remote_peer = source_info + .sites + .into_iter() + .find(|peer| peer.endpoint == target_env.url) + .ok_or("target peer missing from source site replication info")?; + + let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?; + put_bucket_replication(&source_env, source_bucket, &target_arn).await?; + + for idx in 0..32 { + source_client + .put_object() + .bucket(source_bucket) + .key(format!("resync-object-{idx:02}")) + .body(ByteStream::from(vec![b'x'; 256 * 1024])) + .send() + .await?; + } + + let started = site_replication_resync_op(&source_env, "start", &remote_peer).await?; + assert_eq!(started.status, "success", "unexpected start result: {:?}", started); + assert!( + started + .buckets + .iter() + .any(|bucket| bucket.bucket == source_bucket && matches!(bucket.status.as_str(), "started" | "success")), + "source bucket start status missing: {:?}", + started + ); + + let started_target = + wait_for_replication_reset_target(&source_env, source_bucket, &target_arn, |target| !target.reset_id.is_empty()).await?; + let started_reset_id = started_target.reset_id.clone(); + assert!( + matches!(started_target.status.as_str(), "Pending" | "Started" | "InProgress" | "Completed"), + "unexpected start status: {:?}", + started_target + ); + + let canceled = site_replication_resync_op(&source_env, "cancel", &remote_peer).await?; + assert_eq!(canceled.status, "success", "unexpected cancel result: {:?}", canceled); + assert!( + canceled + .buckets + .iter() + .any(|bucket| bucket.bucket == source_bucket && matches!(bucket.status.as_str(), "canceled" | "success")), + "source bucket cancel status missing: {:?}", + canceled + ); + + let canceled_target = + wait_for_replication_reset_target(&source_env, source_bucket, &target_arn, |target| target.status == "Canceled").await?; + assert_eq!(canceled_target.status, "Canceled"); + assert_eq!(canceled_target.reset_id, started_reset_id); + + let restarted = site_replication_resync_op(&source_env, "start", &remote_peer).await?; + assert_eq!(restarted.status, "success", "unexpected restart result: {:?}", restarted); + assert!( + restarted + .buckets + .iter() + .any(|bucket| bucket.bucket == source_bucket && matches!(bucket.status.as_str(), "started" | "success")), + "source bucket restart status missing: {:?}", + restarted + ); + let restart_snapshot = get_replication_reset_status(&source_env, source_bucket, &target_arn).await?; + let restarted_target = wait_for_replication_reset_target(&source_env, source_bucket, &target_arn, |target| { + !target.reset_id.is_empty() && target.reset_id != started_reset_id + }) + .await + .map_err(|err| { + format!( + "restart ids: start={} restart={} snapshot={:?}; {err}", + started_reset_id, restarted.resync_id, restart_snapshot.targets + ) + })?; + assert_ne!(restarted_target.reset_id, started_reset_id); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_site_replication_edit_and_status_peer_state_real_dual_node() -> Result<(), Box> { + init_logging(); + + let mut source_env = RustFSTestEnvironment::new().await?; + source_env.start_rustfs_server(vec![]).await?; + + let mut target_env = RustFSTestEnvironment::new().await?; + target_env.start_rustfs_server_without_cleanup(vec![]).await?; + + let add_status = site_replication_add( + &source_env, + &[ + PeerSite { + name: "source-site".to_string(), + endpoint: source_env.url.clone(), + access_key: source_env.access_key.clone(), + secret_key: source_env.secret_key.clone(), + }, + PeerSite { + name: "target-site".to_string(), + endpoint: target_env.url.clone(), + access_key: target_env.access_key.clone(), + secret_key: target_env.secret_key.clone(), + }, + ], + ) + .await?; + assert!(add_status.success, "unexpected site add result: {:?}", add_status); + + let source_info = wait_for_site_replication_enabled(&source_env, 2).await?; + let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?; + let mut remote_peer = source_info + .sites + .into_iter() + .find(|peer| peer.endpoint == target_env.url) + .ok_or("target peer missing from source site replication info")?; + + remote_peer.sync_state = SyncStatus::Enable; + let edit_status = site_replication_edit(&source_env, "", &remote_peer).await?; + assert!(edit_status.success, "unexpected site edit result: {:?}", edit_status); + + let source_after_sync = wait_for_site_replication_info(&source_env, |info| { + info.sites + .iter() + .any(|peer| peer.endpoint == target_env.url && peer.sync_state == SyncStatus::Enable) + }) + .await?; + let target_after_sync = wait_for_site_replication_info(&target_env, |info| { + info.sites + .iter() + .any(|peer| peer.endpoint == target_env.url && peer.sync_state == SyncStatus::Enable) + }) + .await?; + assert!( + source_after_sync + .sites + .iter() + .any(|peer| peer.endpoint == target_env.url && peer.sync_state == SyncStatus::Enable) + ); + assert!( + target_after_sync + .sites + .iter() + .any(|peer| peer.endpoint == target_env.url && peer.sync_state == SyncStatus::Enable) + ); + + let ilm_edit_status = site_replication_edit(&source_env, "enableILMExpiryReplication=true", &PeerInfo::default()).await?; + assert!(ilm_edit_status.success, "unexpected ilm edit result: {:?}", ilm_edit_status); + + let source_after_ilm = wait_for_site_replication_info(&source_env, |info| { + info.sites.len() == 2 && info.sites.iter().all(|peer| peer.replicate_ilm_expiry) + }) + .await?; + let target_after_ilm = wait_for_site_replication_info(&target_env, |info| { + info.sites.len() == 2 && info.sites.iter().all(|peer| peer.replicate_ilm_expiry) + }) + .await?; + assert!(source_after_ilm.sites.iter().all(|peer| peer.replicate_ilm_expiry)); + assert!(target_after_ilm.sites.iter().all(|peer| peer.replicate_ilm_expiry)); + + let status_query = "peer-state=true"; + let source_status = wait_for_site_replication_status(&source_env, status_query, |status| { + status.peer_states.len() == 2 + && status + .peer_states + .values() + .all(|state| state.peers.len() == 2 && state.peers.values().all(|peer| peer.replicate_ilm_expiry)) + }) + .await?; + let target_status = wait_for_site_replication_status(&target_env, status_query, |status| { + status.peer_states.len() == 2 + && status + .peer_states + .values() + .all(|state| state.peers.len() == 2 && state.peers.values().all(|peer| peer.replicate_ilm_expiry)) + }) + .await?; + + assert_eq!(source_status.peer_states.len(), 2); + assert_eq!(target_status.peer_states.len(), 2); + assert!(source_status.peer_states.values().all(|state| state.peers.len() == 2)); + assert!(target_status.peer_states.values().all(|state| state.peers.len() == 2)); + assert!( + source_status + .peer_states + .values() + .all(|state| state.peers.values().all(|peer| peer.replicate_ilm_expiry)) + ); + assert!( + target_status + .peer_states + .values() + .all(|state| state.peers.values().all(|peer| peer.replicate_ilm_expiry)) + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_site_replication_remove_all_real_dual_node() -> Result<(), Box> { + init_logging(); + + let mut source_env = RustFSTestEnvironment::new().await?; + source_env.start_rustfs_server(vec![]).await?; + + let mut target_env = RustFSTestEnvironment::new().await?; + target_env.start_rustfs_server_without_cleanup(vec![]).await?; + + let add_status = site_replication_add( + &source_env, + &[ + PeerSite { + name: "source-site".to_string(), + endpoint: source_env.url.clone(), + access_key: source_env.access_key.clone(), + secret_key: source_env.secret_key.clone(), + }, + PeerSite { + name: "target-site".to_string(), + endpoint: target_env.url.clone(), + access_key: target_env.access_key.clone(), + secret_key: target_env.secret_key.clone(), + }, + ], + ) + .await?; + assert!(add_status.success, "unexpected site add result: {:?}", add_status); + + let _source_info = wait_for_site_replication_enabled(&source_env, 2).await?; + let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?; + + let remove_status = site_replication_remove( + &source_env, + &SRRemoveReq { + remove_all: true, + ..Default::default() + }, + ) + .await?; + assert!( + !remove_status.status.is_empty() && remove_status.err_detail.is_empty(), + "unexpected site remove result: {:?}", + remove_status + ); + + let source_after_remove = wait_for_site_replication_disabled(&source_env).await?; + let target_after_remove = wait_for_site_replication_disabled(&target_env).await?; + + assert!(!source_after_remove.enabled); + assert!(source_after_remove.sites.is_empty()); + assert!(!target_after_remove.enabled); + assert!(target_after_remove.sites.is_empty()); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_site_replication_state_edit_fresh_and_stale_real_dual_node() -> Result<(), Box> { + init_logging(); + + let mut source_env = RustFSTestEnvironment::new().await?; + source_env.start_rustfs_server(vec![]).await?; + + let mut target_env = RustFSTestEnvironment::new().await?; + target_env.start_rustfs_server_without_cleanup(vec![]).await?; + + let add_status = site_replication_add( + &source_env, + &[ + PeerSite { + name: "source-site".to_string(), + endpoint: source_env.url.clone(), + access_key: source_env.access_key.clone(), + secret_key: source_env.secret_key.clone(), + }, + PeerSite { + name: "target-site".to_string(), + endpoint: target_env.url.clone(), + access_key: target_env.access_key.clone(), + secret_key: target_env.secret_key.clone(), + }, + ], + ) + .await?; + assert!(add_status.success, "unexpected site add result: {:?}", add_status); + + let source_info = wait_for_site_replication_enabled(&source_env, 2).await?; + let target_info = wait_for_site_replication_enabled(&target_env, 2).await?; + assert!(source_info.sites.iter().all(|peer| !peer.replicate_ilm_expiry)); + assert!(target_info.sites.iter().all(|peer| !peer.replicate_ilm_expiry)); + + let target_status = + wait_for_site_replication_status(&target_env, "peer-state=true", |status| status.peer_states.len() == 2).await?; + let current_updated_at = target_status + .peer_states + .values() + .find_map(|state| state.updated_at) + .ok_or("missing target site replication updated_at")?; + + let mut stale_peers = BTreeMap::new(); + for peer in target_info.sites { + let mut peer = peer; + peer.replicate_ilm_expiry = true; + stale_peers.insert(peer.deployment_id.clone(), peer); + } + site_replication_state_edit( + &target_env, + &rustfs_madmin::SRStateEditReq { + peers: stale_peers, + updated_at: Some(current_updated_at - TimeDuration::seconds(1)), + }, + ) + .await?; + + let target_after_stale = site_replication_info(&target_env).await?; + let source_after_stale = site_replication_info(&source_env).await?; + assert!(target_after_stale.sites.iter().all(|peer| !peer.replicate_ilm_expiry)); + assert!(source_after_stale.sites.iter().all(|peer| !peer.replicate_ilm_expiry)); + + let mut fresh_peers = BTreeMap::new(); + for peer in target_after_stale.sites { + let mut peer = peer; + peer.replicate_ilm_expiry = true; + fresh_peers.insert(peer.deployment_id.clone(), peer); + } + let fresh_updated_at = current_updated_at + TimeDuration::seconds(1); + site_replication_state_edit( + &target_env, + &rustfs_madmin::SRStateEditReq { + peers: fresh_peers, + updated_at: Some(fresh_updated_at), + }, + ) + .await?; + + let target_after_fresh = wait_for_site_replication_info(&target_env, |info| { + info.sites.len() == 2 && info.sites.iter().all(|peer| peer.replicate_ilm_expiry) + }) + .await?; + assert!(target_after_fresh.sites.iter().all(|peer| peer.replicate_ilm_expiry)); + + let target_status_after_fresh = wait_for_site_replication_status(&target_env, "peer-state=true", |status| { + status.peer_states.len() == 2 + && status.peer_states.values().all(|state| { + state.updated_at == Some(fresh_updated_at) && state.peers.values().all(|peer| peer.replicate_ilm_expiry) + }) + }) + .await?; + assert!(target_status_after_fresh.peer_states.values().all(|state| { + state.updated_at == Some(fresh_updated_at) && state.peers.values().all(|peer| peer.replicate_ilm_expiry) + })); + + let source_after_fresh = site_replication_info(&source_env).await?; + assert!(source_after_fresh.sites.iter().all(|peer| !peer.replicate_ilm_expiry)); + + Ok(()) +} diff --git a/crates/ecstore/src/bucket/replication/replication_pool.rs b/crates/ecstore/src/bucket/replication/replication_pool.rs index 20422b5ec..8d2dcab22 100644 --- a/crates/ecstore/src/bucket/replication/replication_pool.rs +++ b/crates/ecstore/src/bucket/replication/replication_pool.rs @@ -777,6 +777,14 @@ impl ReplicationPool { Ok(status) } + pub async fn cancel_bucket_resync(&self, opts: ResyncOpts) -> Result<(), EcstoreError> { + self.resyncer.cancel(&opts).await; + self.resyncer + .mark_status(ResyncStatusType::ResyncCanceled, opts, self.storage.clone()) + .await?; + Ok(()) + } + pub async fn start_bucket_resync(self: Arc, opts: ResyncOpts) -> Result<(), EcstoreError> { let now = OffsetDateTime::now_utc(); let bucket_status = { @@ -813,8 +821,14 @@ impl ReplicationPool { let resyncer = self.resyncer.clone(); let storage = self.storage.clone(); + let cancel_token = CancellationToken::new(); + resyncer.register_cancel_token(&opts, cancel_token.clone()).await; tokio::spawn(async move { - resyncer.resync_bucket(CancellationToken::new(), storage, false, opts).await; + resyncer + .clone() + .resync_bucket(cancel_token, storage, false, opts.clone()) + .await; + resyncer.clear_cancel_token(&opts).await; }); Ok(()) @@ -852,8 +866,12 @@ impl ReplicationPool { } /// Load bucket replication resync statuses into memory - #[instrument(skip(cancellation_token))] - async fn load_resync(self: Arc, buckets: &[String], cancellation_token: CancellationToken) -> Result<(), EcstoreError> { + #[instrument(skip(_cancellation_token))] + async fn load_resync( + self: Arc, + buckets: &[String], + _cancellation_token: CancellationToken, + ) -> Result<(), EcstoreError> { // TODO: add leader_lock // Make sure only one node running resync on the cluster // Note: Leader lock implementation would be needed here @@ -884,24 +902,20 @@ impl ReplicationPool { // Note: This would spawn a resync task in a real implementation // For now, we just log the resync request - let ctx = cancellation_token.clone(); + let ctx = CancellationToken::new(); let bucket_clone = bucket.clone(); let resync = self.resyncer.clone(); let storage = self.storage.clone(); + let opts = ResyncOpts { + bucket: bucket_clone, + arn, + resync_id: stats.resync_id, + resync_before: stats.resync_before_date, + }; tokio::spawn(async move { - resync - .resync_bucket( - ctx, - storage, - true, - ResyncOpts { - bucket: bucket_clone, - arn, - resync_id: stats.resync_id, - resync_before: stats.resync_before_date, - }, - ) - .await; + resync.register_cancel_token(&opts, ctx.clone()).await; + resync.clone().resync_bucket(ctx, storage, true, opts.clone()).await; + resync.clear_cancel_token(&opts).await; }); } _ => {} @@ -949,6 +963,7 @@ pub trait ReplicationPoolTrait: std::fmt::Debug { async fn queue_replica_delete_task(&self, ri: DeletedObjectReplicationInfo); async fn resize(&self, priority: ReplicationPriority, max_workers: usize, max_l_workers: usize); async fn get_bucket_resync_status(&self, bucket: &str) -> Result; + async fn cancel_bucket_resync(&self, opts: ResyncOpts) -> Result<(), EcstoreError>; async fn start_bucket_resync(self: Arc, opts: ResyncOpts) -> Result<(), EcstoreError>; async fn init_resync( self: Arc, @@ -976,6 +991,10 @@ impl ReplicationPoolTrait for ReplicationPool { self.get_bucket_resync_status(bucket).await } + async fn cancel_bucket_resync(&self, opts: ResyncOpts) -> Result<(), EcstoreError> { + self.cancel_bucket_resync(opts).await + } + async fn start_bucket_resync(self: Arc, opts: ResyncOpts) -> Result<(), EcstoreError> { self.start_bucket_resync(opts).await } diff --git a/crates/ecstore/src/bucket/replication/replication_resyncer.rs b/crates/ecstore/src/bucket/replication/replication_resyncer.rs index 3eec82917..8379f1767 100644 --- a/crates/ecstore/src/bucket/replication/replication_resyncer.rs +++ b/crates/ecstore/src/bucket/replication/replication_resyncer.rs @@ -110,6 +110,10 @@ fn normalize_wire_time(value: Option) -> Option } } +fn resync_state_accepts_update(state: &TargetReplicationResyncStatus, opts: &ResyncOpts) -> bool { + state.resync_id.is_empty() || opts.resync_id.is_empty() || state.resync_id == opts.resync_id +} + #[derive(Debug, Clone, Default)] pub struct ResyncOpts { pub bucket: String, @@ -360,16 +364,13 @@ static RESYNC_WORKER_COUNT: usize = 10; pub struct ReplicationResyncer { pub status_map: Arc>>, pub worker_size: usize, - pub resync_cancel_tx: CancellationToken, - pub resync_cancel_rx: CancellationToken, + pub cancel_tokens: Arc>>, pub worker_tx: tokio::sync::broadcast::Sender<()>, pub worker_rx: tokio::sync::broadcast::Receiver<()>, } impl ReplicationResyncer { pub async fn new() -> Self { - let resync_cancel_tx = CancellationToken::new(); - let resync_cancel_rx = resync_cancel_tx.clone(); let (worker_tx, worker_rx) = tokio::sync::broadcast::channel(RESYNC_WORKER_COUNT); for _ in 0..RESYNC_WORKER_COUNT { @@ -381,16 +382,34 @@ impl ReplicationResyncer { Self { status_map: Arc::new(RwLock::new(HashMap::new())), worker_size: RESYNC_WORKER_COUNT, - resync_cancel_tx, - resync_cancel_rx, + cancel_tokens: Arc::new(RwLock::new(HashMap::new())), worker_tx, worker_rx, } } + fn cancel_key(opts: &ResyncOpts) -> String { + format!("{}:{}", opts.bucket, opts.arn) + } + + pub async fn register_cancel_token(&self, opts: &ResyncOpts, token: CancellationToken) { + self.cancel_tokens.write().await.insert(Self::cancel_key(opts), token); + } + + pub async fn clear_cancel_token(&self, opts: &ResyncOpts) { + self.cancel_tokens.write().await.remove(&Self::cancel_key(opts)); + } + + pub async fn cancel(&self, opts: &ResyncOpts) { + if let Some(token) = self.cancel_tokens.write().await.remove(&Self::cancel_key(opts)) { + token.cancel(); + } + } + pub async fn mark_status(&self, status: ResyncStatusType, opts: ResyncOpts, obj_layer: Arc) -> Result<()> { let bucket_status = { let mut status_map = self.status_map.write().await; + let now = OffsetDateTime::now_utc(); let bucket_status = if let Some(bucket_status) = status_map.get_mut(&opts.bucket) { bucket_status @@ -409,10 +428,33 @@ impl ReplicationResyncer { bucket_status.targets_map.get_mut(&opts.arn).unwrap() }; - state.resync_status = status; - state.last_update = Some(OffsetDateTime::now_utc()); + if !resync_state_accepts_update(state, &opts) { + warn!( + bucket = %opts.bucket, + arn = %opts.arn, + incoming_resync_id = %opts.resync_id, + current_resync_id = %state.resync_id, + "ignoring stale resync status update" + ); + return Ok(()); + } - bucket_status.last_update = Some(OffsetDateTime::now_utc()); + if state.resync_id.is_empty() { + state.resync_id = opts.resync_id.clone(); + } + if state.resync_before_date.is_none() { + state.resync_before_date = opts.resync_before; + } + if state.bucket.is_empty() { + state.bucket = opts.bucket.clone(); + } + if status == ResyncStatusType::ResyncStarted && state.start_time.is_none() { + state.start_time = Some(now); + } + state.resync_status = status; + state.last_update = Some(now); + + bucket_status.last_update = Some(now); bucket_status.clone() }; @@ -424,6 +466,7 @@ impl ReplicationResyncer { pub async fn inc_stats(&self, status: &TargetReplicationResyncStatus, opts: ResyncOpts) { let mut status_map = self.status_map.write().await; + let now = OffsetDateTime::now_utc(); let bucket_status = if let Some(bucket_status) = status_map.get_mut(&opts.bucket) { bucket_status @@ -442,13 +485,30 @@ impl ReplicationResyncer { bucket_status.targets_map.get_mut(&opts.arn).unwrap() }; + if !resync_state_accepts_update(state, &opts) { + warn!( + bucket = %opts.bucket, + arn = %opts.arn, + incoming_resync_id = %opts.resync_id, + current_resync_id = %state.resync_id, + "ignoring stale resync stats update" + ); + return; + } + + if state.resync_id.is_empty() { + state.resync_id = opts.resync_id.clone(); + } + if state.bucket.is_empty() { + state.bucket = opts.bucket.clone(); + } state.object = status.object.clone(); state.replicated_count += status.replicated_count; state.replicated_size += status.replicated_size; state.failed_count += status.failed_count; state.failed_size += status.failed_size; - state.last_update = Some(OffsetDateTime::now_utc()); - bucket_status.last_update = Some(OffsetDateTime::now_utc()); + state.last_update = Some(now); + bucket_status.last_update = Some(now); } pub async fn persist_to_disk(&self, cancel_token: CancellationToken, api: Arc) { @@ -640,7 +700,6 @@ impl ReplicationResyncer { let cancel_token = cancellation_token.clone(); let target_client = target_client.clone(); - let resync_cancel_rx = self.resync_cancel_rx.clone(); let storage = storage.clone(); let results_tx = results_tx.clone(); let bucket_name = opts.bucket.clone(); @@ -714,10 +773,6 @@ impl ReplicationResyncer { err, ); - if resync_cancel_rx.is_cancelled() { - return; - } - if cancel_token.is_cancelled() { return; } @@ -731,8 +786,6 @@ impl ReplicationResyncer { futures.push(f); } - let resync_cancel_rx = self.resync_cancel_rx.clone(); - while let Some(res) = rx.recv().await { if let Some(err) = res.err { error!("Failed to get object info: {}", err); @@ -741,14 +794,8 @@ impl ReplicationResyncer { return; } - if resync_cancel_rx.is_cancelled() { - self.resync_bucket_mark_status(ResyncStatusType::ResyncCanceled, opts.clone(), storage.clone()) - .await; - return; - } - if cancellation_token.is_cancelled() { - self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone()) + self.resync_bucket_mark_status(ResyncStatusType::ResyncCanceled, opts.clone(), storage.clone()) .await; return; } @@ -770,14 +817,8 @@ impl ReplicationResyncer { continue; } - if resync_cancel_rx.is_cancelled() { - self.resync_bucket_mark_status(ResyncStatusType::ResyncCanceled, opts.clone(), storage.clone()) - .await; - return; - } - if cancellation_token.is_cancelled() { - self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone()) + self.resync_bucket_mark_status(ResyncStatusType::ResyncCanceled, opts.clone(), storage.clone()) .await; return; } @@ -3430,4 +3471,54 @@ mod tests { "With no replication config, dsc may be empty; with config, replicate_any() would be true and queueing would occur" ); } + + #[tokio::test] + async fn test_cancel_marks_only_matching_bucket_target_token() { + let resyncer = ReplicationResyncer::new().await; + let opts_a = ResyncOpts { + bucket: "bucket-a".to_string(), + arn: "arn:replication::a".to_string(), + resync_id: "rid-a".to_string(), + resync_before: None, + }; + let opts_b = ResyncOpts { + bucket: "bucket-b".to_string(), + arn: "arn:replication::b".to_string(), + resync_id: "rid-b".to_string(), + resync_before: None, + }; + let token_a = CancellationToken::new(); + let token_b = CancellationToken::new(); + resyncer.register_cancel_token(&opts_a, token_a.clone()).await; + resyncer.register_cancel_token(&opts_b, token_b.clone()).await; + + resyncer.cancel(&opts_a).await; + + assert!(token_a.is_cancelled()); + assert!(!token_b.is_cancelled()); + } + + #[test] + fn test_resync_state_accepts_update_only_for_matching_run() { + let current = TargetReplicationResyncStatus { + resync_id: "run-new".to_string(), + ..Default::default() + }; + let matching = ResyncOpts { + bucket: "bucket".to_string(), + arn: "arn:replication::dest".to_string(), + resync_id: "run-new".to_string(), + resync_before: None, + }; + let stale = ResyncOpts { + bucket: "bucket".to_string(), + arn: "arn:replication::dest".to_string(), + resync_id: "run-old".to_string(), + resync_before: None, + }; + + assert!(resync_state_accepts_update(&TargetReplicationResyncStatus::default(), &matching)); + assert!(resync_state_accepts_update(¤t, &matching)); + assert!(!resync_state_accepts_update(¤t, &stale)); + } } diff --git a/crates/madmin/src/group.rs b/crates/madmin/src/group.rs index 3821af16b..e7da51a72 100644 --- a/crates/madmin/src/group.rs +++ b/crates/madmin/src/group.rs @@ -17,7 +17,7 @@ use serde::Deserializer; use serde::Serialize; use time::OffsetDateTime; -#[derive(Debug, Serialize, Default, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Default, PartialEq, Eq)] #[serde(rename_all = "lowercase")] pub enum GroupStatus { #[default] @@ -39,7 +39,7 @@ impl<'de> Deserialize<'de> for GroupStatus { } } -#[derive(Debug, Serialize, Deserialize, Default)] +#[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct GroupAddRemove { pub group: String, pub members: Vec, diff --git a/crates/madmin/src/lib.rs b/crates/madmin/src/lib.rs index 688152721..154663b11 100644 --- a/crates/madmin/src/lib.rs +++ b/crates/madmin/src/lib.rs @@ -20,6 +20,7 @@ pub mod metrics; pub mod net; pub mod policy; pub mod service_commands; +pub mod site_replication; pub mod trace; pub mod user; pub mod utils; @@ -27,4 +28,5 @@ pub mod utils; pub use group::*; pub use info_commands::*; pub use policy::*; +pub use site_replication::*; pub use user::*; diff --git a/crates/madmin/src/site_replication.rs b/crates/madmin/src/site_replication.rs new file mode 100644 index 000000000..68a13285b --- /dev/null +++ b/crates/madmin/src/site_replication.rs @@ -0,0 +1,1118 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{GroupAddRemove, GroupDesc, SRSvcAccCreate, UserInfo}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::{BTreeMap, HashMap}; +use time::OffsetDateTime; + +pub const SITE_REPL_API_VERSION: &str = "1"; + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct PeerSite { + #[serde(default)] + pub name: String, + #[serde(rename = "endpoints", default)] + pub endpoint: String, + #[serde(rename = "accessKey", default)] + pub access_key: String, + #[serde(rename = "secretKey", default)] + pub secret_key: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct ReplicateAddStatus { + #[serde(default)] + pub success: bool, + #[serde(default)] + pub status: String, + #[serde(rename = "errorDetail", skip_serializing_if = "String::is_empty", default)] + pub err_detail: String, + #[serde(rename = "initialSyncErrorMessage", skip_serializing_if = "String::is_empty", default)] + pub initial_sync_error_message: String, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SiteReplicationInfo { + #[serde(default)] + pub enabled: bool, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub name: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub sites: Vec, + #[serde(rename = "serviceAccountAccessKey", default, skip_serializing_if = "String::is_empty")] + pub service_account_access_key: String, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SRPeerJoinReq { + #[serde(rename = "svcAcctAccessKey", default)] + pub svc_acct_access_key: String, + #[serde(rename = "svcAcctSecretKey", default)] + pub svc_acct_secret_key: String, + #[serde(rename = "svcAcctParent", default)] + pub svc_acct_parent: String, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub peers: BTreeMap, + #[serde( + rename = "updatedAt", + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub updated_at: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct BucketBandwidth { + #[serde(rename = "bandwidthLimitPerBucket", default)] + pub limit: u64, + #[serde(default)] + pub set: bool, + #[serde( + rename = "updatedAt", + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub updated_at: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum SyncStatus { + #[serde(rename = "enable")] + Enable, + #[serde(rename = "disable")] + Disable, + #[default] + #[serde(other)] + Unknown, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct PeerInfo { + #[serde(default)] + pub endpoint: String, + #[serde(default)] + pub name: String, + #[serde(rename = "deploymentID", default)] + pub deployment_id: String, + #[serde(rename = "sync", default)] + pub sync_state: SyncStatus, + #[serde(rename = "defaultbandwidth", default)] + pub default_bandwidth: BucketBandwidth, + #[serde(rename = "replicate-ilm-expiry", default)] + pub replicate_ilm_expiry: bool, + #[serde(rename = "objectNamingMode", default, skip_serializing_if = "String::is_empty")] + pub object_naming_mode: String, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SRPolicyMapping { + #[serde(rename = "userOrGroup", default)] + pub user_or_group: String, + #[serde(rename = "userType", default)] + pub user_type: u64, + #[serde(rename = "isGroup", default)] + pub is_group: bool, + #[serde(default)] + pub policy: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub provider: String, + #[serde(rename = "configID", default, skip_serializing_if = "String::is_empty")] + pub config_id: String, + #[serde( + rename = "createdAt", + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub created_at: Option, + #[serde( + rename = "updatedAt", + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub updated_at: Option, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SRSTSCredential { + #[serde(rename = "accessKey", default)] + pub access_key: String, + #[serde(rename = "secretKey", default)] + pub secret_key: String, + #[serde(rename = "sessionToken", default)] + pub session_token: String, + #[serde(rename = "parentUser", default)] + pub parent_user: String, + #[serde(rename = "parentPolicyMapping", default, skip_serializing_if = "String::is_empty")] + pub parent_policy_mapping: String, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SRExternalUser { + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, + #[serde(default)] + pub name: String, + #[serde(rename = "isDeleteReq", default)] + pub is_delete_req: bool, + #[serde(rename = "openIDUser", skip_serializing_if = "Option::is_none")] + pub open_id_user: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SRLDAPUser { + #[serde(default)] + pub dn: String, + #[serde(default)] + pub username: String, + #[serde(rename = "validatedDN", default, skip_serializing_if = "String::is_empty")] + pub validated_dn: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub groups: Vec, + #[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] + pub expiry: Option, + #[serde(rename = "isDeleteReq", default)] + pub is_delete_req: bool, + #[serde(rename = "configName", default, skip_serializing_if = "String::is_empty")] + pub config_name: String, + #[serde( + rename = "updatedAt", + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub updated_at: Option, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct SRIAMUser { + #[serde(rename = "accessKey", default)] + pub access_key: String, + #[serde(rename = "isDeleteReq", default)] + pub is_delete_req: bool, + #[serde(rename = "userReq", skip_serializing_if = "Option::is_none")] + pub user_req: Option, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct SRGroupInfo { + #[serde(rename = "updateReq", default)] + pub update_req: GroupAddRemove, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SRSvcAccUpdate { + #[serde(rename = "accessKey", default)] + pub access_key: String, + #[serde(rename = "secretKey", default)] + pub secret_key: String, + #[serde(default)] + pub status: String, + #[serde(default)] + pub name: String, + #[serde(default)] + pub description: String, + #[serde(rename = "sessionPolicy", default)] + pub session_policy: crate::SRSessionPolicy, + #[serde( + rename = "expiration", + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub expiration: Option, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SRSvcAccDelete { + #[serde(rename = "accessKey", default)] + pub access_key: String, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SRSvcAccChange { + #[serde(rename = "crSvcAccCreate", skip_serializing_if = "Option::is_none")] + pub create: Option, + #[serde(rename = "crSvcAccUpdate", skip_serializing_if = "Option::is_none")] + pub update: Option, + #[serde(rename = "crSvcAccDelete", skip_serializing_if = "Option::is_none")] + pub delete: Option, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SRCredInfo { + #[serde(rename = "accessKey", default)] + pub access_key: String, + #[serde(rename = "iamUserType", default)] + pub iam_user_type: u64, + #[serde(rename = "isDeleteReq", default)] + pub is_delete_req: bool, + #[serde(rename = "userIdentityJSON", default, skip_serializing_if = "Option::is_none")] + pub user_identity_json: Option, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct SRIAMItem { + #[serde(default)] + pub r#type: String, + #[serde(default)] + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub policy: Option, + #[serde(rename = "policyMapping", skip_serializing_if = "Option::is_none")] + pub policy_mapping: Option, + #[serde(rename = "groupInfo", skip_serializing_if = "Option::is_none")] + pub group_info: Option, + #[serde(rename = "credentialChange", skip_serializing_if = "Option::is_none")] + pub credential_info: Option, + #[serde(rename = "serviceAccountChange", skip_serializing_if = "Option::is_none")] + pub svc_acc_change: Option, + #[serde(rename = "stsCredential", skip_serializing_if = "Option::is_none")] + pub sts_credential: Option, + #[serde(rename = "iamUser", skip_serializing_if = "Option::is_none")] + pub iam_user: Option, + #[serde(rename = "externalUser", skip_serializing_if = "Option::is_none")] + pub external_user: Option, + #[serde(rename = "ldapUser", skip_serializing_if = "Option::is_none")] + pub ldap_user: Option, + #[serde( + rename = "updatedAt", + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub updated_at: Option, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SRBucketMeta { + #[serde(default)] + pub r#type: String, + #[serde(default)] + pub bucket: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub policy: Option, + #[serde(rename = "versioningConfig", skip_serializing_if = "Option::is_none")] + pub versioning: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tags: Option, + #[serde(rename = "objectLockConfig", skip_serializing_if = "Option::is_none")] + pub object_lock_config: Option, + #[serde(rename = "sseConfig", skip_serializing_if = "Option::is_none")] + pub sse_config: Option, + #[serde(rename = "replicationConfig", skip_serializing_if = "Option::is_none")] + pub replication_config: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub quota: Option, + #[serde(rename = "expLCConfig", skip_serializing_if = "Option::is_none")] + pub expiry_lc_config: Option, + #[serde( + rename = "updatedAt", + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub updated_at: Option, + #[serde( + rename = "expiryUpdatedAt", + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub expiry_updated_at: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cors: Option, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SRBucketInfo { + #[serde(default)] + pub bucket: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub policy: Option, + #[serde(rename = "versioningConfig", skip_serializing_if = "Option::is_none")] + pub versioning: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tags: Option, + #[serde(rename = "objectLockConfig", skip_serializing_if = "Option::is_none")] + pub object_lock_config: Option, + #[serde(rename = "sseConfig", skip_serializing_if = "Option::is_none")] + pub sse_config: Option, + #[serde(rename = "replicationConfig", skip_serializing_if = "Option::is_none")] + pub replication_config: Option, + #[serde(rename = "quotaConfig", skip_serializing_if = "Option::is_none")] + pub quota_config: Option, + #[serde(rename = "expLCConfig", skip_serializing_if = "Option::is_none")] + pub expiry_lc_config: Option, + #[serde(rename = "corsConfig", skip_serializing_if = "Option::is_none")] + pub cors_config: Option, + #[serde( + rename = "policyTimestamp", + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub policy_updated_at: Option, + #[serde( + rename = "tagTimestamp", + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub tag_config_updated_at: Option, + #[serde( + rename = "olockTimestamp", + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub object_lock_config_updated_at: Option, + #[serde( + rename = "sseTimestamp", + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub sse_config_updated_at: Option, + #[serde( + rename = "versioningTimestamp", + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub versioning_config_updated_at: Option, + #[serde( + rename = "replicationConfigTimestamp", + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub replication_config_updated_at: Option, + #[serde( + rename = "quotaTimestamp", + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub quota_config_updated_at: Option, + #[serde( + rename = "expLCTimestamp", + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub expiry_lc_config_updated_at: Option, + #[serde( + rename = "bucketTimestamp", + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub created_at: Option, + #[serde( + rename = "bucketDeletedTimestamp", + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub deleted_at: Option, + #[serde( + rename = "corsTimestamp", + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub cors_config_updated_at: Option, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub location: String, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct OpenIDProviderSettings { + #[serde(rename = "ClaimName", default, skip_serializing_if = "String::is_empty")] + pub claim_name: String, + #[serde(rename = "ClaimUserinfoEnabled", default)] + pub claim_userinfo_enabled: bool, + #[serde(rename = "RolePolicy", default, skip_serializing_if = "String::is_empty")] + pub role_policy: String, + #[serde(rename = "ClientID", default, skip_serializing_if = "String::is_empty")] + pub client_id: String, + #[serde(rename = "HashedClientSecret", default, skip_serializing_if = "String::is_empty")] + pub hashed_client_secret: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct OpenIDSettings { + #[serde(rename = "Enabled", default)] + pub enabled: bool, + #[serde(rename = "Region", default, skip_serializing_if = "String::is_empty")] + pub region: String, + #[serde(rename = "Roles", default, skip_serializing_if = "BTreeMap::is_empty")] + pub roles: BTreeMap, + #[serde(rename = "ClaimProvider", default)] + pub claim_provider: OpenIDProviderSettings, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct LDAPSettings { + #[serde(rename = "IsLDAPEnabled", default)] + pub is_ldap_enabled: bool, + #[serde(rename = "LDAPUserDNSearchBase", default, skip_serializing_if = "String::is_empty")] + pub ldap_user_dn_search_base: String, + #[serde(rename = "LDAPUserDNSearchFilter", default, skip_serializing_if = "String::is_empty")] + pub ldap_user_dn_search_filter: String, + #[serde(rename = "LDAPGroupSearchBase", default, skip_serializing_if = "String::is_empty")] + pub ldap_group_search_base: String, + #[serde(rename = "LDAPGroupSearchFilter", default, skip_serializing_if = "String::is_empty")] + pub ldap_group_search_filter: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct LDAPProviderSettings { + #[serde(rename = "UserDNSearchBase", default, skip_serializing_if = "String::is_empty")] + pub user_dn_search_base: String, + #[serde(rename = "UserDNSearchFilter", default, skip_serializing_if = "String::is_empty")] + pub user_dn_search_filter: String, + #[serde(rename = "GroupSearchBase", default, skip_serializing_if = "String::is_empty")] + pub group_search_base: String, + #[serde(rename = "GroupSearchFilter", default, skip_serializing_if = "String::is_empty")] + pub group_search_filter: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct LDAPConfigSettings { + #[serde(rename = "Enabled", default)] + pub enabled: bool, + #[serde(rename = "Configs", default, skip_serializing_if = "BTreeMap::is_empty")] + pub configs: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct IDPSettings { + #[serde(rename = "LDAP", default)] + pub ldap: LDAPSettings, + #[serde(rename = "LDAPConfigs", default)] + pub ldap_configs: LDAPConfigSettings, + #[serde(rename = "OpenID", default)] + pub open_id: OpenIDSettings, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SRIAMPolicy { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub policy: Option, + #[serde( + rename = "updatedAt", + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub updated_at: Option, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct ILMExpiryRule { + #[serde(rename = "ilm-rule", default, skip_serializing_if = "String::is_empty")] + pub ilm_rule: String, + #[serde(default)] + pub bucket: String, + #[serde( + rename = "updatedAt", + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub updated_at: Option, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SRStateInfo { + #[serde(default, skip_serializing_if = "String::is_empty")] + pub name: String, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub peers: BTreeMap, + #[serde( + rename = "updatedAt", + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub updated_at: Option, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct SRInfo { + #[serde(default)] + pub enabled: bool, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub name: String, + #[serde(rename = "deploymentID", default, skip_serializing_if = "String::is_empty")] + pub deployment_id: String, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub buckets: BTreeMap, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub policies: BTreeMap, + #[serde(rename = "userPolicies", default, skip_serializing_if = "BTreeMap::is_empty")] + pub user_policies: BTreeMap, + #[serde(rename = "userInfoMap", default, skip_serializing_if = "BTreeMap::is_empty")] + pub user_info_map: BTreeMap, + #[serde(rename = "groupDescMap", default, skip_serializing_if = "BTreeMap::is_empty")] + pub group_desc_map: BTreeMap, + #[serde(rename = "groupPolicies", default, skip_serializing_if = "BTreeMap::is_empty")] + pub group_policies: BTreeMap, + #[serde(rename = "replicationCfg", default, skip_serializing_if = "BTreeMap::is_empty")] + pub replication_cfg: BTreeMap, + #[serde(rename = "ilmExpiryRules", default, skip_serializing_if = "BTreeMap::is_empty")] + pub ilm_expiry_rules: BTreeMap, + #[serde(default)] + pub state: SRStateInfo, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SRPolicyStatsSummary { + #[serde(rename = "DeploymentID", default, skip_serializing_if = "String::is_empty")] + pub deployment_id: String, + #[serde(rename = "PolicyMismatch", default)] + pub policy_mismatch: bool, + #[serde(rename = "HasPolicy", default)] + pub has_policy: bool, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SRUserStatsSummary { + #[serde(rename = "DeploymentID", default, skip_serializing_if = "String::is_empty")] + pub deployment_id: String, + #[serde(rename = "PolicyMismatch", default)] + pub policy_mismatch: bool, + #[serde(rename = "UserInfoMismatch", default)] + pub user_info_mismatch: bool, + #[serde(rename = "HasUser", default)] + pub has_user: bool, + #[serde(rename = "HasPolicyMapping", default)] + pub has_policy_mapping: bool, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SRGroupStatsSummary { + #[serde(rename = "DeploymentID", default, skip_serializing_if = "String::is_empty")] + pub deployment_id: String, + #[serde(rename = "PolicyMismatch", default)] + pub policy_mismatch: bool, + #[serde(rename = "HasGroup", default)] + pub has_group: bool, + #[serde(rename = "GroupDescMismatch", default)] + pub group_desc_mismatch: bool, + #[serde(rename = "HasPolicyMapping", default)] + pub has_policy_mapping: bool, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SRBucketStatsSummary { + #[serde(rename = "DeploymentID", default, skip_serializing_if = "String::is_empty")] + pub deployment_id: String, + #[serde(rename = "HasBucket", default)] + pub has_bucket: bool, + #[serde(rename = "BucketMarkedDeleted", default)] + pub bucket_marked_deleted: bool, + #[serde(rename = "TagMismatch", default)] + pub tag_mismatch: bool, + #[serde(rename = "VersioningConfigMismatch", default)] + pub versioning_config_mismatch: bool, + #[serde(rename = "OLockConfigMismatch", default)] + pub object_lock_config_mismatch: bool, + #[serde(rename = "PolicyMismatch", default)] + pub policy_mismatch: bool, + #[serde(rename = "SSEConfigMismatch", default)] + pub sse_config_mismatch: bool, + #[serde(rename = "ReplicationCfgMismatch", default)] + pub replication_cfg_mismatch: bool, + #[serde(rename = "QuotaCfgMismatch", default)] + pub quota_cfg_mismatch: bool, + #[serde(rename = "CorsCfgMismatch", default)] + pub cors_cfg_mismatch: bool, + #[serde(rename = "HasTagsSet", default)] + pub has_tags_set: bool, + #[serde(rename = "HasOLockConfigSet", default)] + pub has_object_lock_config_set: bool, + #[serde(rename = "HasPolicySet", default)] + pub has_policy_set: bool, + #[serde(rename = "HasSSECfgSet", default)] + pub has_sse_cfg_set: bool, + #[serde(rename = "HasReplicationCfg", default)] + pub has_replication_cfg: bool, + #[serde(rename = "HasQuotaCfgSet", default)] + pub has_quota_cfg_set: bool, + #[serde(rename = "HasCorsCfgSet", default)] + pub has_cors_cfg_set: bool, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SRILMExpiryStatsSummary { + #[serde(rename = "DeploymentID", default, skip_serializing_if = "String::is_empty")] + pub deployment_id: String, + #[serde(rename = "ILMExpiryRuleMismatch", default)] + pub ilm_expiry_rule_mismatch: bool, + #[serde(rename = "HasILMExpiryRules", default)] + pub has_ilm_expiry_rules: bool, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SRSiteSummary { + #[serde(rename = "ReplicatedBuckets", default)] + pub replicated_buckets: usize, + #[serde(rename = "ReplicatedTags", default)] + pub replicated_tags: usize, + #[serde(rename = "ReplicatedBucketPolicies", default)] + pub replicated_bucket_policies: usize, + #[serde(rename = "ReplicatedIAMPolicies", default)] + pub replicated_iam_policies: usize, + #[serde(rename = "ReplicatedUsers", default)] + pub replicated_users: usize, + #[serde(rename = "ReplicatedGroups", default)] + pub replicated_groups: usize, + #[serde(rename = "ReplicatedLockConfig", default)] + pub replicated_lock_config: usize, + #[serde(rename = "ReplicatedSSEConfig", default)] + pub replicated_sse_config: usize, + #[serde(rename = "ReplicatedVersioningConfig", default)] + pub replicated_versioning_config: usize, + #[serde(rename = "ReplicatedQuotaConfig", default)] + pub replicated_quota_config: usize, + #[serde(rename = "ReplicatedUserPolicyMappings", default)] + pub replicated_user_policy_mappings: usize, + #[serde(rename = "ReplicatedGroupPolicyMappings", default)] + pub replicated_group_policy_mappings: usize, + #[serde(rename = "ReplicatedILMExpiryRules", default)] + pub replicated_ilm_expiry_rules: usize, + #[serde(rename = "ReplicatedCorsConfig", default)] + pub replicated_cors_config: usize, + #[serde(rename = "TotalBucketsCount", default)] + pub total_buckets_count: usize, + #[serde(rename = "TotalTagsCount", default)] + pub total_tags_count: usize, + #[serde(rename = "TotalBucketPoliciesCount", default)] + pub total_bucket_policies_count: usize, + #[serde(rename = "TotalIAMPoliciesCount", default)] + pub total_iam_policies_count: usize, + #[serde(rename = "TotalLockConfigCount", default)] + pub total_lock_config_count: usize, + #[serde(rename = "TotalSSEConfigCount", default)] + pub total_sse_config_count: usize, + #[serde(rename = "TotalVersioningConfigCount", default)] + pub total_versioning_config_count: usize, + #[serde(rename = "TotalQuotaConfigCount", default)] + pub total_quota_config_count: usize, + #[serde(rename = "TotalUsersCount", default)] + pub total_users_count: usize, + #[serde(rename = "TotalGroupsCount", default)] + pub total_groups_count: usize, + #[serde(rename = "TotalUserPolicyMappingCount", default)] + pub total_user_policy_mapping_count: usize, + #[serde(rename = "TotalGroupPolicyMappingCount", default)] + pub total_group_policy_mapping_count: usize, + #[serde(rename = "TotalILMExpiryRulesCount", default)] + pub total_ilm_expiry_rules_count: usize, + #[serde(rename = "TotalCorsConfigCount", default)] + pub total_cors_config_count: usize, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct WorkerStat { + #[serde(rename = "curr", default)] + pub curr: i32, + #[serde(rename = "avg", default)] + pub avg: f64, + #[serde(rename = "max", default)] + pub max: i32, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct QStat { + #[serde(default)] + pub count: f64, + #[serde(default)] + pub bytes: f64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct InQueueMetric { + #[serde(default)] + pub curr: QStat, + #[serde(default)] + pub avg: QStat, + #[serde(default)] + pub max: QStat, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct InProgressMetric { + #[serde(default)] + pub curr: QStat, + #[serde(default)] + pub avg: QStat, + #[serde(default)] + pub max: QStat, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct Counter { + #[serde(rename = "last1hr", default)] + pub last_1hr: u64, + #[serde(rename = "last1m", default)] + pub last_1m: u64, + #[serde(default)] + pub total: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct ReplicationWindowedStats { + #[serde(default)] + pub curr: u64, + #[serde(rename = "avgRate", default)] + pub avg_rate: f64, + #[serde(rename = "peakRate", default)] + pub peak_rate: f64, + #[serde(default)] + pub total: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct ReplProxyMetric { + #[serde(rename = "putTaggingProxyTotal", default)] + pub put_tag_total: u64, + #[serde(rename = "getTaggingProxyTotal", default)] + pub get_tag_total: u64, + #[serde(rename = "removeTaggingProxyTotal", default)] + pub remove_tag_total: u64, + #[serde(rename = "getProxyTotal", default)] + pub get_total: u64, + #[serde(rename = "headProxyTotal", default)] + pub head_total: u64, + #[serde(rename = "putTaggingProxyFailed", default)] + pub put_tag_failed_total: u64, + #[serde(rename = "getTaggingProxyFailed", default)] + pub get_tag_failed_total: u64, + #[serde(rename = "removeTaggingProxyFailed", default)] + pub remove_tag_failed_total: u64, + #[serde(rename = "getProxyFailed", default)] + pub get_failed_total: u64, + #[serde(rename = "headProxyFailed", default)] + pub head_failed_total: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct LatencyStat { + #[serde(rename = "curr", default)] + pub curr_ns: i64, + #[serde(rename = "avg", default)] + pub average_ns: i64, + #[serde(rename = "max", default)] + pub max_ns: i64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct RStat { + #[serde(default)] + pub count: f64, + #[serde(default)] + pub bytes: i64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct TimedErrStats { + #[serde(rename = "lastMinute", default)] + pub last_minute: RStat, + #[serde(rename = "lastHour", default)] + pub last_hour: RStat, + #[serde(rename = "totals", default)] + pub totals: RStat, + #[serde(rename = "errCounts", default, skip_serializing_if = "BTreeMap::is_empty")] + pub err_counts: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct StatRecorder { + #[serde(default)] + pub total: i64, + #[serde(default)] + pub avg: i64, + #[serde(default)] + pub max: i64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct DowntimeInfo { + #[serde(default)] + pub duration: StatRecorder, + #[serde(default)] + pub count: StatRecorder, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SRMetric { + #[serde(rename = "deploymentID", default, skip_serializing_if = "String::is_empty")] + pub deployment_id: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub endpoint: String, + #[serde(rename = "totalDowntime", default)] + pub total_downtime_ns: i64, + #[serde( + rename = "lastOnline", + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub last_online: Option, + #[serde(rename = "isOnline", default)] + pub online: bool, + #[serde(default)] + pub latency: LatencyStat, + #[serde(rename = "replicatedSize", default)] + pub replicated_size: i64, + #[serde(rename = "replicatedCount", default)] + pub replicated_count: i64, + #[serde(default)] + pub failed: TimedErrStats, + #[serde(rename = "transferSummary", default, skip_serializing_if = "HashMap::is_empty")] + pub transfer_summary: HashMap, + #[serde(rename = "mrfStats", default, skip_serializing_if = "HashMap::is_empty")] + pub mrf_stats: HashMap, + #[serde(rename = "downtimeInfo", default)] + pub downtime_info: DowntimeInfo, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SRMetricsSummary { + #[serde(rename = "activeWorkers", default)] + pub active_workers: WorkerStat, + #[serde(rename = "replicaSize", default)] + pub replica_size: i64, + #[serde(rename = "replicaCount", default)] + pub replica_count: i64, + #[serde(default)] + pub queued: InQueueMetric, + #[serde(rename = "inProgress", default)] + pub in_progress: InProgressMetric, + #[serde(default)] + pub proxied: ReplProxyMetric, + #[serde(rename = "replMetrics", default, skip_serializing_if = "BTreeMap::is_empty")] + pub metrics: BTreeMap, + #[serde(default)] + pub uptime: i64, + #[serde(default)] + pub retries: Counter, + #[serde(default)] + pub errors: Counter, + #[serde(default)] + pub replicated: ReplicationWindowedStats, + #[serde(default)] + pub received: ReplicationWindowedStats, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SRStatusInfo { + #[serde(default)] + pub enabled: bool, + #[serde(rename = "MaxBuckets", default)] + pub max_buckets: usize, + #[serde(rename = "MaxUsers", default)] + pub max_users: usize, + #[serde(rename = "MaxGroups", default)] + pub max_groups: usize, + #[serde(rename = "MaxPolicies", default)] + pub max_policies: usize, + #[serde(rename = "MaxILMExpiryRules", default)] + pub max_ilm_expiry_rules: usize, + #[serde(rename = "Sites", default, skip_serializing_if = "BTreeMap::is_empty")] + pub sites: BTreeMap, + #[serde(rename = "StatsSummary", default, skip_serializing_if = "BTreeMap::is_empty")] + pub stats_summary: BTreeMap, + #[serde(rename = "BucketStats", default, skip_serializing_if = "BTreeMap::is_empty")] + pub bucket_stats: BTreeMap>, + #[serde(rename = "PolicyStats", default, skip_serializing_if = "BTreeMap::is_empty")] + pub policy_stats: BTreeMap>, + #[serde(rename = "UserStats", default, skip_serializing_if = "BTreeMap::is_empty")] + pub user_stats: BTreeMap>, + #[serde(rename = "GroupStats", default, skip_serializing_if = "BTreeMap::is_empty")] + pub group_stats: BTreeMap>, + #[serde(rename = "PeerStates", default, skip_serializing_if = "BTreeMap::is_empty")] + pub peer_states: BTreeMap, + #[serde(rename = "Metrics", default)] + pub metrics: SRMetricsSummary, + #[serde(rename = "ILMExpiryStats", default, skip_serializing_if = "BTreeMap::is_empty")] + pub ilm_expiry_stats: BTreeMap>, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct ReplicateEditStatus { + #[serde(default)] + pub success: bool, + #[serde(default)] + pub status: String, + #[serde(rename = "errorDetail", skip_serializing_if = "String::is_empty", default)] + pub err_detail: String, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct ReplicateRemoveStatus { + #[serde(default)] + pub status: String, + #[serde(rename = "errorDetail", skip_serializing_if = "String::is_empty", default)] + pub err_detail: String, + #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] + pub api_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SRRemoveReq { + #[serde(rename = "requestingDepID", default, skip_serializing_if = "String::is_empty")] + pub requesting_dep_id: String, + #[serde( + rename = "sites", + default, + deserialize_with = "deserialize_vec_null_default", + skip_serializing_if = "Vec::is_empty" + )] + pub site_names: Vec, + #[serde(rename = "all", default)] + pub remove_all: bool, +} + +fn deserialize_vec_null_default<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + Ok(Option::>::deserialize(deserializer)?.unwrap_or_default()) +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SRStateEditReq { + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub peers: BTreeMap, + #[serde( + rename = "updatedAt", + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub updated_at: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct ResyncBucketStatus { + #[serde(default)] + pub bucket: String, + #[serde(default)] + pub status: String, + #[serde(rename = "errorDetail", skip_serializing_if = "String::is_empty", default)] + pub err_detail: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SRResyncOpStatus { + #[serde(rename = "op", default)] + pub op_type: String, + #[serde(rename = "id", default)] + pub resync_id: String, + #[serde(default)] + pub status: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub buckets: Vec, + #[serde(rename = "errorDetail", skip_serializing_if = "String::is_empty", default)] + pub err_detail: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SiteNetPerfNodeResult { + #[serde(default, skip_serializing_if = "String::is_empty")] + pub endpoint: String, + #[serde(default)] + pub tx: u64, + #[serde(rename = "txTotalDuration", default)] + pub tx_total_duration_ns: i64, + #[serde(default)] + pub rx: u64, + #[serde(rename = "rxTotalDuration", default)] + pub rx_total_duration_ns: i64, + #[serde(rename = "totalConn", default)] + pub total_conn: u64, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub error: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SiteNetPerfResult { + #[serde(rename = "nodeResults", default, skip_serializing_if = "Vec::is_empty")] + pub node_results: Vec, +} diff --git a/crates/madmin/src/user.rs b/crates/madmin/src/user.rs index 50b23a862..3931639e1 100644 --- a/crates/madmin/src/user.rs +++ b/crates/madmin/src/user.rs @@ -21,7 +21,7 @@ use time::format_description::well_known::Rfc3339; use crate::BackendInfo; -#[derive(Debug, Serialize, Deserialize, Default, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] pub enum AccountStatus { #[serde(rename = "enabled")] Enabled, @@ -94,7 +94,7 @@ pub struct UserInfo { pub updated_at: Option, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct AddOrUpdateUserReq { #[serde(rename = "secretKey")] pub secret_key: String, diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index b4b77519c..30210f5e8 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -131,6 +131,7 @@ astral-tokio-tar = { workspace = true } atoi = { workspace = true } atomic_enum = { workspace = true } base64 = { workspace = true } +sha2 = { workspace = true } base64-simd.workspace = true clap = { workspace = true } const-str = { workspace = true } diff --git a/rustfs/src/admin/handlers/group.rs b/rustfs/src/admin/handlers/group.rs index f9b2044b6..63a59ee24 100644 --- a/rustfs/src/admin/handlers/group.rs +++ b/rustfs/src/admin/handlers/group.rs @@ -15,6 +15,7 @@ use crate::{ admin::{ auth::validate_admin_request, + handlers::site_replication::site_replication_iam_change_hook, router::{AdminOperation, Operation, S3Router}, utils::has_space_be, }, @@ -27,7 +28,7 @@ use matchit::Params; use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; use rustfs_credentials::get_global_action_cred; use rustfs_iam::error::{is_err_no_such_group, is_err_no_such_user}; -use rustfs_madmin::GroupAddRemove; +use rustfs_madmin::{GroupAddRemove, GroupStatus, SITE_REPL_API_VERSION, SRGroupInfo, SRIAMItem}; use rustfs_policy::policy::action::{Action, AdminAction}; use s3s::{ Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, @@ -222,7 +223,7 @@ impl Operation for DeleteGroup { let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; - iam_store.remove_users_from_group(group, vec![]).await.map_err(|e| { + let updated_at = iam_store.remove_users_from_group(group, vec![]).await.map_err(|e| { warn!("delete group failed, e: {:?}", e); match e { rustfs_iam::error::Error::GroupNotEmpty => { @@ -241,6 +242,26 @@ impl Operation for DeleteGroup { } })?; + if let Err(err) = site_replication_iam_change_hook(SRIAMItem { + r#type: "group-info".to_string(), + group_info: Some(SRGroupInfo { + update_req: GroupAddRemove { + group: group.to_string(), + members: vec![], + status: GroupStatus::Enabled, + is_remove: true, + }, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }), + updated_at: Some(updated_at), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }) + .await + { + warn!("site replication group delete hook failed, err: {err}"); + } + let mut header = HeaderMap::new(); header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); header.insert(CONTENT_LENGTH, "0".parse().unwrap()); @@ -287,26 +308,46 @@ impl Operation for SetGroupStatus { let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; - if let Some(status) = query.status { - match status.as_str() { - "enabled" => { - iam_store.set_group_status(&query.group, true).await.map_err(|e| { - warn!("enable group failed, e: {:?}", e); - S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) - })?; - } - "disabled" => { - iam_store.set_group_status(&query.group, false).await.map_err(|e| { - warn!("enable group failed, e: {:?}", e); - S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) - })?; - } + let updated_at = if let Some(status) = query.status.as_deref() { + match status { + "enabled" => iam_store.set_group_status(&query.group, true).await.map_err(|e| { + warn!("enable group failed, e: {:?}", e); + S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) + })?, + "disabled" => iam_store.set_group_status(&query.group, false).await.map_err(|e| { + warn!("enable group failed, e: {:?}", e); + S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) + })?, _ => { return Err(s3_error!(InvalidArgument, "invalid status")); } } } else { return Err(s3_error!(InvalidArgument, "status is required")); + }; + + if let Err(err) = site_replication_iam_change_hook(SRIAMItem { + r#type: "group-info".to_string(), + group_info: Some(SRGroupInfo { + update_req: GroupAddRemove { + group: query.group.clone(), + members: vec![], + status: if matches!(query.status.as_deref(), Some("disabled")) { + GroupStatus::Disabled + } else { + GroupStatus::Enabled + }, + is_remove: false, + }, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }), + updated_at: Some(updated_at), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }) + .await + { + warn!("site replication group status hook failed, err: {err}"); } let mut header = HeaderMap::new(); @@ -387,15 +428,15 @@ impl Operation for UpdateGroupMembers { } } - if args.is_remove { + let updated_at = if args.is_remove { warn!("remove group members"); iam_store - .remove_users_from_group(&args.group, args.members) + .remove_users_from_group(&args.group, args.members.clone()) .await .map_err(|e| { warn!("remove group members failed, e: {:?}", e); S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) - })?; + })? } else { warn!("add group members"); @@ -406,10 +447,28 @@ impl Operation for UpdateGroupMembers { return Err(s3_error!(InvalidArgument, "not such group")); } - iam_store.add_users_to_group(&args.group, args.members).await.map_err(|e| { - warn!("add group members failed, e: {:?}", e); - S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) - })?; + iam_store + .add_users_to_group(&args.group, args.members.clone()) + .await + .map_err(|e| { + warn!("add group members failed, e: {:?}", e); + S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) + })? + }; + + if let Err(err) = site_replication_iam_change_hook(SRIAMItem { + r#type: "group-info".to_string(), + group_info: Some(SRGroupInfo { + update_req: args, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }), + updated_at: Some(updated_at), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }) + .await + { + warn!("site replication group membership hook failed, err: {err}"); } let mut header = HeaderMap::new(); diff --git a/rustfs/src/admin/handlers/mod.rs b/rustfs/src/admin/handlers/mod.rs index af310e446..522cb0056 100644 --- a/rustfs/src/admin/handlers/mod.rs +++ b/rustfs/src/admin/handlers/mod.rs @@ -33,6 +33,7 @@ pub mod quota; pub mod rebalance; pub mod replication; pub mod service_account; +pub mod site_replication; pub mod sts; pub mod system; pub mod tier; @@ -64,6 +65,9 @@ mod tests { let _set_remote_target_handler = replication::SetRemoteTargetHandler {}; let _list_remote_target_handler = replication::ListRemoteTargetHandler {}; let _remove_remote_target_handler = replication::RemoveRemoteTargetHandler {}; + let _site_replication_add_handler = site_replication::SiteReplicationAddHandler {}; + let _site_replication_info_handler = site_replication::SiteReplicationInfoHandler {}; + let _site_replication_status_handler = site_replication::SiteReplicationStatusHandler {}; // Just verify they can be created without panicking // Test passes if we reach this point without panicking diff --git a/rustfs/src/admin/handlers/policies.rs b/rustfs/src/admin/handlers/policies.rs index c70769cc8..e55fdcb17 100644 --- a/rustfs/src/admin/handlers/policies.rs +++ b/rustfs/src/admin/handlers/policies.rs @@ -15,6 +15,7 @@ use crate::{ admin::{ auth::validate_admin_request, + handlers::site_replication::site_replication_iam_change_hook, router::{AdminOperation, Operation, S3Router}, utils::{encode_compatible_admin_payload, has_space_be, read_compatible_admin_body}, }, @@ -28,7 +29,10 @@ use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; use rustfs_credentials::get_global_action_cred; use rustfs_iam::error::is_err_no_such_user; use rustfs_iam::store::MappedPolicy; -use rustfs_madmin::{GroupPolicyEntities, PolicyEntities, PolicyEntitiesResult, UserPolicyEntities}; +use rustfs_madmin::{ + GroupPolicyEntities, PolicyEntities, PolicyEntitiesResult, SITE_REPL_API_VERSION, SRIAMItem, SRPolicyMapping, + UserPolicyEntities, +}; use rustfs_policy::policy::{ Policy, action::{Action, AdminAction}, @@ -223,11 +227,26 @@ impl Operation for AddCannedPolicy { } let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; - iam_store.set_policy(&query.name, policy).await.map_err(|e| { + let updated_at = iam_store.set_policy(&query.name, policy.clone()).await.map_err(|e| { warn!("set policy failed, e: {:?}", e); S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) })?; + if let Err(err) = site_replication_iam_change_hook(SRIAMItem { + r#type: "policy".to_string(), + name: query.name.clone(), + policy: Some( + serde_json::to_value(&policy).map_err(|e| s3_error!(InternalError, "marshal policy failed, e: {:?}", e))?, + ), + updated_at: Some(updated_at), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }) + .await + { + warn!(policy = %query.name, error = ?err, "site replication policy add hook failed"); + } + let mut header = HeaderMap::new(); header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); header.insert(CONTENT_LENGTH, "0".parse().unwrap()); @@ -337,6 +356,18 @@ impl Operation for RemoveCannedPolicy { S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) })?; + if let Err(err) = site_replication_iam_change_hook(SRIAMItem { + r#type: "policy".to_string(), + name: query.name.clone(), + updated_at: Some(OffsetDateTime::now_utc()), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }) + .await + { + warn!(policy = %query.name, error = ?err, "site replication policy delete hook failed"); + } + let mut header = HeaderMap::new(); header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); header.insert(CONTENT_LENGTH, "0".parse().unwrap()); @@ -428,7 +459,7 @@ impl Operation for SetPolicyForUserOrGroup { })?; } - iam_store + let updated_at = iam_store .policy_db_set(&query.user_or_group, rustfs_iam::store::UserType::Reg, query.is_group, &query.policy_name) .await .map_err(|e| { @@ -436,6 +467,26 @@ impl Operation for SetPolicyForUserOrGroup { S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) })?; + if let Err(err) = site_replication_iam_change_hook(SRIAMItem { + r#type: "policy-mapping".to_string(), + policy_mapping: Some(SRPolicyMapping { + user_or_group: query.user_or_group.clone(), + user_type: rustfs_iam::store::UserType::Reg.to_u64(), + is_group: query.is_group, + policy: query.policy_name.clone(), + updated_at: Some(updated_at), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }), + updated_at: Some(updated_at), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }) + .await + { + warn!(target = %query.user_or_group, error = ?err, "site replication policy mapping hook failed"); + } + let mut header = HeaderMap::new(); header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); header.insert(CONTENT_LENGTH, "0".parse().unwrap()); @@ -841,6 +892,26 @@ async fn handle_builtin_policy_association(req: S3Request, is_attach: bool S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) })?; + if let Err(err) = site_replication_iam_change_hook(SRIAMItem { + r#type: "policy-mapping".to_string(), + policy_mapping: Some(SRPolicyMapping { + user_or_group: target_name.clone(), + user_type: rustfs_iam::store::UserType::Reg.to_u64(), + is_group, + policy: updated_policies.join(","), + updated_at: Some(updated_at), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }), + updated_at: Some(updated_at), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }) + .await + { + warn!(target = %target_name, error = ?err, "site replication policy association hook failed"); + } + let policies_attached = if is_attach { changed_policies.clone() } else { Vec::new() }; let policies_detached = if is_attach { Vec::new() } else { changed_policies }; diff --git a/rustfs/src/admin/handlers/service_account.rs b/rustfs/src/admin/handlers/service_account.rs index ccdc13400..01e3821ad 100644 --- a/rustfs/src/admin/handlers/service_account.rs +++ b/rustfs/src/admin/handlers/service_account.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::admin::handlers::site_replication::site_replication_iam_change_hook; use crate::admin::utils::{encode_compatible_admin_payload, has_space_be, is_compat_admin_request, read_compatible_admin_body}; use crate::auth::{constant_time_eq, get_condition_values, get_session_token}; use crate::server::{ADMIN_PREFIX, RemoteAddr}; @@ -30,7 +31,8 @@ use rustfs_iam::sys::{NewServiceAccountOpts, UpdateServiceAccountOpts}; use rustfs_madmin::{ ACCESS_KEY_LIST_ALL, ACCESS_KEY_LIST_STS_ONLY, ACCESS_KEY_LIST_SVCACC_ONLY, ACCESS_KEY_LIST_USERS_ONLY, AddServiceAccountReq, AddServiceAccountResp, Credentials, InfoAccessKeyResp, InfoServiceAccountResp, LDAPSpecificAccessKeyInfo, ListAccessKeysResp, - ListServiceAccountsResp, OpenIDSpecificAccessKeyInfo, ServiceAccountInfo, TemporaryAccountInfoResp, UpdateServiceAccountReq, + ListServiceAccountsResp, OpenIDSpecificAccessKeyInfo, SITE_REPL_API_VERSION, SRIAMItem, SRSessionPolicy, SRSvcAccChange, + SRSvcAccCreate, SRSvcAccDelete, SRSvcAccUpdate, ServiceAccountInfo, TemporaryAccountInfoResp, UpdateServiceAccountReq, }; use rustfs_policy::policy::action::{Action, AdminAction}; use rustfs_policy::policy::{Args, Policy}; @@ -44,6 +46,15 @@ use time::OffsetDateTime; use tracing::{debug, warn}; use url::form_urlencoded; +fn sr_session_policy_from_value(value: Option<&serde_json::Value>) -> S3Result { + let Some(value) = value else { + return Ok(SRSessionPolicy::default()); + }; + + let raw = serde_json::to_string(value).map_err(|e| s3_error!(InvalidArgument, "marshal policy failed: {:?}", e))?; + SRSessionPolicy::from_json(&raw).map_err(|e| s3_error!(InvalidArgument, "marshal policy failed: {:?}", e)) +} + fn compat_time_sentinel() -> OffsetDateTime { OffsetDateTime::UNIX_EPOCH } @@ -294,6 +305,18 @@ impl Operation for AddServiceAccount { } } + let replication_claims = opts.claims.clone().unwrap_or_default(); + let replication_policy = create_req + .policy + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| s3_error!(InvalidArgument, "marshal policy failed: {:?}", e))?; + let replication_groups = target_groups.clone().unwrap_or_default(); + let replication_name = opts.name.clone().unwrap_or_default(); + let replication_description = opts.description.clone().unwrap_or_default(); + let replication_expiration = opts.expiration; + let (new_cred, _) = iam_store .new_service_account(&target_user, target_groups, opts) .await @@ -302,6 +325,39 @@ impl Operation for AddServiceAccount { s3_error!(InternalError, "create service account failed, e: {:?}", e) })?; + if let Err(err) = site_replication_iam_change_hook(SRIAMItem { + r#type: "service-account".to_string(), + svc_acc_change: Some(SRSvcAccChange { + create: Some(SRSvcAccCreate { + parent: target_user.clone(), + access_key: new_cred.access_key.clone(), + secret_key: new_cred.secret_key.clone(), + groups: replication_groups, + claims: replication_claims, + session_policy: replication_policy + .as_deref() + .map(SRSessionPolicy::from_json) + .transpose() + .map_err(|e| s3_error!(InvalidArgument, "marshal policy failed: {:?}", e))? + .unwrap_or_default(), + status: String::new(), + name: replication_name, + description: replication_description, + expiration: replication_expiration, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }), + updated_at: Some(OffsetDateTime::now_utc()), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }) + .await + { + warn!(access_key = %new_cred.access_key, error = ?err, "site replication add service account hook failed"); + } + let resp = AddServiceAccountResp { credentials: Credentials { access_key: &new_cred.access_key, @@ -502,22 +558,54 @@ impl Operation for UpdateServiceAccount { return Err(s3_error!(AccessDenied, "access denied")); } - let sp = parse_update_service_account_policy(update_req.new_policy)?; + let new_secret_key = update_req.new_secret_key.clone(); + let new_status = update_req.new_status.clone(); + let new_name = update_req.new_name.clone(); + let new_description = update_req.new_description.clone(); + let new_expiration = update_req.new_expiration; + let new_policy = update_req.new_policy.clone(); + + let sp = parse_update_service_account_policy(new_policy.clone())?; let opts = UpdateServiceAccountOpts { - secret_key: update_req.new_secret_key, - status: update_req.new_status, - name: update_req.new_name, - description: update_req.new_description, - expiration: update_req.new_expiration, + secret_key: new_secret_key.clone(), + status: new_status.clone(), + name: new_name.clone(), + description: new_description.clone(), + expiration: new_expiration, session_policy: sp, }; - let _ = iam_store + let updated_at = iam_store .update_service_account(&access_key, opts) .await .map_err(|e| map_service_account_lookup_error(e, "update service account failed"))?; + if let Err(err) = site_replication_iam_change_hook(SRIAMItem { + r#type: "service-account".to_string(), + svc_acc_change: Some(SRSvcAccChange { + update: Some(SRSvcAccUpdate { + access_key: access_key.clone(), + secret_key: new_secret_key.unwrap_or_default(), + status: new_status.unwrap_or_default(), + name: new_name.unwrap_or_default(), + description: new_description.unwrap_or_default(), + session_policy: sr_session_policy_from_value(new_policy.as_ref())?, + expiration: new_expiration, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }), + updated_at: Some(updated_at), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }) + .await + { + warn!(access_key = %access_key, error = ?err, "site replication update service account hook failed"); + } + let mut header = HeaderMap::new(); header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); header.insert(CONTENT_LENGTH, "0".parse().unwrap()); @@ -1205,6 +1293,25 @@ impl Operation for DeleteServiceAccount { s3_error!(InternalError, "delete service account failed") })?; + if let Err(err) = site_replication_iam_change_hook(SRIAMItem { + r#type: "service-account".to_string(), + svc_acc_change: Some(SRSvcAccChange { + delete: Some(SRSvcAccDelete { + access_key: query.access_key.clone(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }), + updated_at: Some(OffsetDateTime::now_utc()), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }) + .await + { + warn!(access_key = %query.access_key, error = ?err, "site replication delete service account hook failed"); + } + let mut header = HeaderMap::new(); header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); header.insert(CONTENT_LENGTH, "0".parse().unwrap()); diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs new file mode 100644 index 000000000..4319859c2 --- /dev/null +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -0,0 +1,2889 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::admin::auth::validate_admin_request; +use crate::admin::router::{AdminOperation, Operation, S3Router}; +use crate::admin::utils::{encode_compatible_admin_payload, read_compatible_admin_body}; +use crate::auth::{check_key_valid, get_session_token}; +use crate::error::ApiError; +use crate::server::{ADMIN_PREFIX, RemoteAddr}; +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use http::header::{CONTENT_TYPE, HOST}; +use http::{HeaderMap, HeaderValue, Uri}; +use hyper::{Method, StatusCode}; +use matchit::Params; +use rustfs_config::{DEFAULT_DELIMITER, MAX_ADMIN_REQUEST_BODY_SIZE}; +use rustfs_ecstore::bucket::bucket_target_sys::BucketTargetSys; +use rustfs_ecstore::bucket::metadata::{ + BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE, BUCKET_REPLICATION_CONFIG, + BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, OBJECT_LOCK_CONFIG, +}; +use rustfs_ecstore::bucket::metadata_sys; +use rustfs_ecstore::bucket::replication::GLOBAL_REPLICATION_STATS; +use rustfs_ecstore::bucket::replication::{ReplicationConfigurationExt, ResyncOpts, get_global_replication_pool}; +use rustfs_ecstore::bucket::target::{BucketTarget, BucketTargetType}; +use rustfs_ecstore::bucket::utils::serialize; +use rustfs_ecstore::config::com::{delete_config, read_config, save_config}; +use rustfs_ecstore::config::get_global_server_config; +use rustfs_ecstore::error::Error as StorageError; +use rustfs_ecstore::global::{get_global_deployment_id, get_global_endpoints_opt, get_global_region, global_rustfs_port}; +use rustfs_ecstore::new_object_layer_fn; +use rustfs_ecstore::store_api::{BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp}; +use rustfs_iam::store::{MappedPolicy, UserType}; +use rustfs_iam::sys::{NewServiceAccountOpts, UpdateServiceAccountOpts, get_claims_from_token_with_secret}; +use rustfs_iam::{get_global_iam_sys, get_oidc}; +use rustfs_madmin::{ + BucketBandwidth, GroupStatus, IDPSettings, InProgressMetric, InQueueMetric, LDAPConfigSettings, LDAPSettings, + OpenIDProviderSettings, PeerInfo, PeerSite, QStat, ReplProxyMetric, ReplicateAddStatus, ReplicateEditStatus, + ReplicateRemoveStatus, ResyncBucketStatus, SITE_REPL_API_VERSION, SRBucketInfo, SRBucketMeta, SRBucketStatsSummary, + SRGroupStatsSummary, SRIAMItem, SRIAMPolicy, SRILMExpiryStatsSummary, SRInfo, SRMetric, SRMetricsSummary, SRPeerJoinReq, + SRPolicyMapping, SRPolicyStatsSummary, SRRemoveReq, SRResyncOpStatus, SRSiteSummary, SRStateEditReq, SRStateInfo, + SRStatusInfo, SRUserStatsSummary, SiteReplicationInfo, SyncStatus, WorkerStat, +}; +use rustfs_policy::policy::{ + Policy, + action::{Action, AdminAction}, +}; +use rustfs_signer::constants::UNSIGNED_PAYLOAD; +use rustfs_signer::sign_v4; +use s3s::dto::{BucketVersioningStatus, VersioningConfiguration}; +use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; +use serde::Deserialize; +use serde::Serialize; +use serde::de::DeserializeOwned; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, HashMap, HashSet, hash_map::DefaultHasher}; +use std::hash::{Hash, Hasher}; +use std::sync::OnceLock; +use std::time::{Duration, Instant}; +use time::OffsetDateTime; +use url::{Url, form_urlencoded}; +use uuid::Uuid; + +const SITE_REPLICATION_STATE_PATH: &str = "config/site-replication/state.json"; +const SITE_REPL_ADD_SUCCESS: &str = "Requested sites were configured for replication successfully."; +const SITE_REPL_EDIT_SUCCESS: &str = "Requested site was updated successfully."; +const SITE_REPL_REMOVE_SUCCESS: &str = "Requested site(s) were removed from cluster replication successfully."; +const SITE_REPL_RESYNC_START: &str = "start"; +const SITE_REPL_RESYNC_CANCEL: &str = "cancel"; +const SITE_REPL_MIN_NETPERF_DURATION: Duration = Duration::from_secs(1); +const SITE_REPLICATION_PEER_REQUEST_TIMEOUT: Duration = Duration::from_secs(10); +const SITE_REPLICATION_PEER_CONNECT_TIMEOUT: Duration = Duration::from_secs(3); +const IDENTITY_LDAP_SUB_SYS: &str = "identity_ldap"; +const LEGACY_LDAP_SUB_SYS: &str = "ldapserverconfig"; +const SITE_REPLICATOR_SERVICE_ACCOUNT: &str = "site-replicator-0"; +const SITE_REPLICATION_PEER_JOIN_PATH: &str = "/rustfs/admin/v3/site-replication/peer/join"; +const SITE_REPLICATION_PEER_EDIT_PATH: &str = "/rustfs/admin/v3/site-replication/peer/edit"; +const SITE_REPLICATION_PEER_REMOVE_PATH: &str = "/rustfs/admin/v3/site-replication/peer/remove"; +static SITE_REPLICATION_PEER_CLIENT: OnceLock = OnceLock::new(); + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +struct SiteReplicationState { + name: String, + service_account_access_key: String, + service_account_secret_key: String, + service_account_parent: String, + peers: BTreeMap, + updated_at: Option, + resync_status: BTreeMap, +} + +const GO_GOB_SITE_NETPERF_SCHEMA: &[u8] = &[ + 0x7d, 0x7f, 0x03, 0x01, 0x01, 0x15, 0x53, 0x69, 0x74, 0x65, 0x4e, 0x65, 0x74, 0x50, 0x65, 0x72, 0x66, 0x4e, 0x6f, 0x64, 0x65, + 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x01, 0xff, 0x80, 0x00, 0x01, 0x07, 0x01, 0x08, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, + 0x74, 0x01, 0x0c, 0x00, 0x01, 0x02, 0x54, 0x58, 0x01, 0x06, 0x00, 0x01, 0x0f, 0x54, 0x58, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x44, + 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x01, 0x04, 0x00, 0x01, 0x02, 0x52, 0x58, 0x01, 0x06, 0x00, 0x01, 0x0f, 0x52, 0x58, + 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x01, 0x04, 0x00, 0x01, 0x09, 0x54, 0x6f, 0x74, + 0x61, 0x6c, 0x43, 0x6f, 0x6e, 0x6e, 0x01, 0x06, 0x00, 0x01, 0x05, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x01, 0x0c, 0x00, 0x00, 0x00, +]; + +#[derive(Debug, Clone)] +struct SiteNetPerfNodeResult { + endpoint: String, + tx: u64, + tx_total_duration_ns: i64, + rx: u64, + rx_total_duration_ns: i64, + total_conn: u64, + error: String, +} + +impl SiteReplicationState { + fn enabled(&self) -> bool { + self.peers.len() > 1 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +enum SREntityType { + #[default] + Unspecified, + Bucket, + Policy, + User, + Group, + IlmExpiryRule, +} + +#[derive(Debug, Clone, Default)] +struct SRStatusOptions { + buckets: bool, + policies: bool, + users: bool, + groups: bool, + metrics: bool, + peer_state: bool, + ilm_expiry_rules: bool, + entity: SREntityType, + entity_value: String, +} + +impl SRStatusOptions { + fn include_all_defaults(&self) -> bool { + !(self.buckets + || self.policies + || self.users + || self.groups + || self.metrics + || self.peer_state + || self.ilm_expiry_rules + || self.entity != SREntityType::Unspecified) + } +} + +pub fn register_site_replication_route(r: &mut S3Router) -> std::io::Result<()> { + for (method, path, operation) in [ + (Method::PUT, "/v3/site-replication/add", AdminOperation(&SiteReplicationAddHandler {})), + ( + Method::PUT, + "/v3/site-replication/remove", + AdminOperation(&SiteReplicationRemoveHandler {}), + ), + (Method::GET, "/v3/site-replication/info", AdminOperation(&SiteReplicationInfoHandler {})), + ( + Method::GET, + "/v3/site-replication/metainfo", + AdminOperation(&SiteReplicationMetaInfoHandler {}), + ), + ( + Method::GET, + "/v3/site-replication/status", + AdminOperation(&SiteReplicationStatusHandler {}), + ), + ( + Method::POST, + "/v3/site-replication/devnull", + AdminOperation(&SiteReplicationDevNullHandler {}), + ), + ( + Method::POST, + "/v3/site-replication/netperf", + AdminOperation(&SiteReplicationNetPerfHandler {}), + ), + (Method::PUT, "/v3/site-replication/peer/join", AdminOperation(&SRPeerJoinHandler {})), + ( + Method::PUT, + "/v3/site-replication/peer/bucket-ops", + AdminOperation(&SRPeerBucketOpsHandler {}), + ), + ( + Method::PUT, + "/v3/site-replication/peer/iam-item", + AdminOperation(&SRPeerReplicateIAMItemHandler {}), + ), + ( + Method::PUT, + "/v3/site-replication/peer/bucket-meta", + AdminOperation(&SRPeerReplicateBucketItemHandler {}), + ), + ( + Method::GET, + "/v3/site-replication/peer/idp-settings", + AdminOperation(&SRPeerGetIDPSettingsHandler {}), + ), + (Method::PUT, "/v3/site-replication/edit", AdminOperation(&SiteReplicationEditHandler {})), + (Method::PUT, "/v3/site-replication/peer/edit", AdminOperation(&SRPeerEditHandler {})), + (Method::PUT, "/v3/site-replication/peer/remove", AdminOperation(&SRPeerRemoveHandler {})), + ( + Method::PUT, + "/v3/site-replication/resync/op", + AdminOperation(&SiteReplicationResyncOpHandler {}), + ), + (Method::PUT, "/v3/site-replication/state/edit", AdminOperation(&SRStateEditHandler {})), + ] { + r.insert(method, format!("{ADMIN_PREFIX}{path}").as_str(), operation)?; + } + + Ok(()) +} + +async fn validate_site_replication_admin_request( + req: &S3Request, + action: AdminAction, +) -> S3Result { + let Some(input_cred) = req.credentials.as_ref() else { + return Err(s3_error!(InvalidRequest, "get cred failed")); + }; + + let (cred, owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + + let remote_addr = req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)); + validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await?; + + Ok(cred) +} + +fn json_response(value: &T) -> S3Result> { + let data = serde_json::to_vec(value) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to serialize response: {e}")))?; + let mut headers = HeaderMap::new(); + headers.insert(s3s::header::CONTENT_TYPE, HeaderValue::from_static("application/json")); + Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers)) +} + +fn go_gob_site_netperf_response(value: &SiteNetPerfNodeResult) -> S3Response<(StatusCode, Body)> { + let data = encode_go_gob_site_netperf_node_result(value); + S3Response::new((StatusCode::OK, Body::from(data))) +} + +fn encode_go_gob_site_netperf_node_result(value: &SiteNetPerfNodeResult) -> Vec { + let mut data = GO_GOB_SITE_NETPERF_SCHEMA.to_vec(); + let mut payload = Vec::new(); + write_go_gob_int(&mut payload, 64); + + let mut last_field = None; + encode_go_gob_string_field(&mut payload, &mut last_field, 0, &value.endpoint); + encode_go_gob_u64_field(&mut payload, &mut last_field, 1, value.tx); + encode_go_gob_i64_field(&mut payload, &mut last_field, 2, value.tx_total_duration_ns); + encode_go_gob_u64_field(&mut payload, &mut last_field, 3, value.rx); + encode_go_gob_i64_field(&mut payload, &mut last_field, 4, value.rx_total_duration_ns); + encode_go_gob_u64_field(&mut payload, &mut last_field, 5, value.total_conn); + encode_go_gob_string_field(&mut payload, &mut last_field, 6, &value.error); + payload.push(0); + + write_go_gob_uint(&mut data, payload.len() as u64); + data.extend(payload); + data +} + +fn encode_go_gob_string_field(out: &mut Vec, last_field: &mut Option, field: usize, value: &str) { + if value.is_empty() { + return; + } + write_go_gob_field_delta(out, last_field, field); + write_go_gob_uint(out, value.len() as u64); + out.extend_from_slice(value.as_bytes()); +} + +fn encode_go_gob_u64_field(out: &mut Vec, last_field: &mut Option, field: usize, value: u64) { + if value == 0 { + return; + } + write_go_gob_field_delta(out, last_field, field); + write_go_gob_uint(out, value); +} + +fn encode_go_gob_i64_field(out: &mut Vec, last_field: &mut Option, field: usize, value: i64) { + if value == 0 { + return; + } + write_go_gob_field_delta(out, last_field, field); + write_go_gob_int(out, value); +} + +fn write_go_gob_field_delta(out: &mut Vec, last_field: &mut Option, field: usize) { + let delta = match *last_field { + Some(previous) => field - previous, + None => field + 1, + }; + write_go_gob_uint(out, delta as u64); + *last_field = Some(field); +} + +fn write_go_gob_int(out: &mut Vec, value: i64) { + let encoded = if value < 0 { + ((!value as u64) << 1) | 1 + } else { + (value as u64) << 1 + }; + write_go_gob_uint(out, encoded); +} + +fn write_go_gob_uint(out: &mut Vec, value: u64) { + if value < 128 { + out.push(value as u8); + return; + } + + let bytes = value.to_be_bytes(); + let first_non_zero = bytes.iter().position(|byte| *byte != 0).unwrap_or(bytes.len() - 1); + let used = &bytes[first_non_zero..]; + out.push((0u8).wrapping_sub(used.len() as u8)); + out.extend_from_slice(used); +} + +fn empty_response(status: StatusCode) -> S3Response<(StatusCode, Body)> { + S3Response::new((status, Body::empty())) +} + +async fn read_plain_admin_body(mut input: Body) -> S3Result> { + let body = input + .store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE) + .await + .map_err(|e| s3_error!(InvalidRequest, "failed to read request body: {}", e))?; + Ok(body.to_vec()) +} + +async fn read_site_replication_json( + req: S3Request, + secret_key: &str, + compat_encrypted: bool, +) -> S3Result { + let body = if compat_encrypted { + read_compatible_admin_body(req.input, MAX_ADMIN_REQUEST_BODY_SIZE, req.uri.path(), secret_key).await? + } else { + read_plain_admin_body(req.input).await? + }; + + serde_json::from_slice(&body).map_err(|e| s3_error!(InvalidRequest, "invalid JSON: {}", e)) +} + +async fn load_site_replication_state() -> S3Result { + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + match read_config(store, SITE_REPLICATION_STATE_PATH).await { + Ok(data) => serde_json::from_slice(&data) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("invalid site replication state: {e}"))), + Err(StorageError::ConfigNotFound) => Ok(SiteReplicationState::default()), + Err(err) => Err(S3Error::with_message( + S3ErrorCode::InternalError, + format!("failed to load site replication state: {err}"), + )), + } +} + +async fn save_site_replication_state(state: &SiteReplicationState) -> S3Result<()> { + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + let data = serde_json::to_vec(state) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize state failed: {e}")))?; + save_config(store, SITE_REPLICATION_STATE_PATH, data) + .await + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("save state failed: {e}")))?; + Ok(()) +} + +async fn clear_site_replication_state() -> S3Result<()> { + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + match delete_config(store, SITE_REPLICATION_STATE_PATH).await { + Ok(()) | Err(StorageError::ConfigNotFound) => Ok(()), + Err(err) => Err(S3Error::with_message(S3ErrorCode::InternalError, format!("clear state failed: {err}"))), + } +} + +async fn persist_site_replication_state(state: &SiteReplicationState) -> S3Result<()> { + if state.peers.len() <= 1 { + clear_site_replication_state().await + } else { + save_site_replication_state(state).await + } +} + +fn site_replication_peer_client() -> &'static reqwest::Client { + SITE_REPLICATION_PEER_CLIENT.get_or_init(|| { + reqwest::Client::builder() + .timeout(SITE_REPLICATION_PEER_REQUEST_TIMEOUT) + .connect_timeout(SITE_REPLICATION_PEER_CONNECT_TIMEOUT) + .pool_idle_timeout(Some(Duration::from_secs(60))) + .build() + .unwrap_or_else(|_| reqwest::Client::new()) + }) +} + +fn query_pairs(uri: &Uri) -> HashMap { + uri.query() + .map(|query| { + form_urlencoded::parse(query.as_bytes()) + .into_owned() + .collect::>() + }) + .unwrap_or_default() +} + +fn query_flag(uri: &Uri, key: &str) -> bool { + query_pairs(uri).get(key).is_some_and(|value| value == "true") +} + +fn sr_entity_type(value: &str) -> SREntityType { + match value { + "bucket" => SREntityType::Bucket, + "policy" => SREntityType::Policy, + "user" => SREntityType::User, + "group" => SREntityType::Group, + "ilm-expiry-rule" => SREntityType::IlmExpiryRule, + _ => SREntityType::Unspecified, + } +} + +fn sr_status_options(uri: &Uri) -> SRStatusOptions { + let pairs = query_pairs(uri); + SRStatusOptions { + buckets: pairs.get("buckets").is_some_and(|value| value == "true"), + policies: pairs.get("policies").is_some_and(|value| value == "true"), + users: pairs.get("users").is_some_and(|value| value == "true"), + groups: pairs.get("groups").is_some_and(|value| value == "true"), + metrics: pairs.get("metrics").is_some_and(|value| value == "true"), + peer_state: pairs.get("peer-state").is_some_and(|value| value == "true"), + ilm_expiry_rules: pairs.get("ilm-expiry-rules").is_some_and(|value| value == "true"), + entity: pairs + .get("entity") + .map(String::as_str) + .map(sr_entity_type) + .unwrap_or(SREntityType::Unspecified), + entity_value: pairs.get("entityvalue").cloned().unwrap_or_default(), + } +} + +fn sr_add_replicate_ilm_expiry(uri: &Uri) -> bool { + query_flag(uri, "replicateILMExpiry") +} + +fn sr_edit_ilm_expiry_override(uri: &Uri) -> Option { + if query_flag(uri, "enableILMExpiryReplication") { + Some(true) + } else if query_flag(uri, "disableILMExpiryReplication") { + Some(false) + } else { + None + } +} + +fn hash_client_secret(secret: Option<&str>) -> String { + let Some(secret) = secret.filter(|secret| !secret.is_empty()) else { + return String::new(); + }; + + let mut hasher = Sha256::new(); + hasher.update(secret.as_bytes()); + URL_SAFE_NO_PAD.encode(hasher.finalize()) +} + +fn config_enabled(value: Option) -> bool { + matches!(value.as_deref(), Some("on" | "true" | "enabled")) +} + +fn ldap_settings_from_kvs(kvs: &rustfs_ecstore::config::KVS) -> (LDAPSettings, LDAPConfigSettings) { + let enabled = config_enabled(kvs.lookup("enable")); + let settings = LDAPSettings { + is_ldap_enabled: enabled, + ldap_user_dn_search_base: kvs.get("user_dn_search_base_dn"), + ldap_user_dn_search_filter: kvs.get("user_dn_search_filter"), + ldap_group_search_base: kvs.get("group_search_base_dn"), + ldap_group_search_filter: kvs.get("group_search_filter"), + }; + + let mut ldap_configs = LDAPConfigSettings { + enabled, + ..Default::default() + }; + + if !settings.ldap_user_dn_search_base.is_empty() + || !settings.ldap_user_dn_search_filter.is_empty() + || !settings.ldap_group_search_base.is_empty() + || !settings.ldap_group_search_filter.is_empty() + { + ldap_configs.configs.insert( + "default".to_string(), + rustfs_madmin::LDAPProviderSettings { + user_dn_search_base: settings.ldap_user_dn_search_base.clone(), + user_dn_search_filter: settings.ldap_user_dn_search_filter.clone(), + group_search_base: settings.ldap_group_search_base.clone(), + group_search_filter: settings.ldap_group_search_filter.clone(), + }, + ); + } + + (settings, ldap_configs) +} + +fn load_ldap_idp_settings() -> (LDAPSettings, LDAPConfigSettings) { + let Some(config) = get_global_server_config() else { + return (LDAPSettings::default(), LDAPConfigSettings::default()); + }; + + let ldap_kvs = config + .get_value(IDENTITY_LDAP_SUB_SYS, DEFAULT_DELIMITER) + .or_else(|| config.get_value(LEGACY_LDAP_SUB_SYS, DEFAULT_DELIMITER)); + + ldap_kvs + .as_ref() + .map(ldap_settings_from_kvs) + .unwrap_or_else(|| (LDAPSettings::default(), LDAPConfigSettings::default())) +} + +fn request_endpoint(uri: &Uri, headers: &HeaderMap) -> String { + let scheme = headers + .get("x-forwarded-proto") + .and_then(|value| value.to_str().ok()) + .filter(|value| !value.is_empty()) + .unwrap_or("http"); + + let host = headers + .get(http::header::HOST) + .and_then(|value| value.to_str().ok()) + .filter(|value| !value.is_empty()) + .map(str::to_string) + .or_else(|| { + get_global_endpoints_opt().and_then(|endpoints| { + endpoints + .as_ref() + .iter() + .flat_map(|pool| pool.endpoints.as_ref().iter()) + .find(|endpoint| endpoint.is_local) + .map(|endpoint| endpoint.host_port()) + }) + }) + .unwrap_or_else(|| format!("127.0.0.1:{}", global_rustfs_port())); + + if uri.scheme_str().is_some() { + return format!("{scheme}://{host}"); + } + + format!("{scheme}://{host}") +} + +fn current_local_runtime_endpoint() -> String { + request_endpoint(&Uri::from_static("/"), &HeaderMap::new()) +} + +fn infer_site_name(endpoint: &str) -> String { + endpoint + .trim_start_matches("http://") + .trim_start_matches("https://") + .split('/') + .next() + .unwrap_or_default() + .split(':') + .next() + .unwrap_or_default() + .to_string() +} + +fn deployment_id_for_endpoint(endpoint: &str) -> String { + let mut hasher = DefaultHasher::new(); + endpoint.hash(&mut hasher); + format!("{:016x}", hasher.finish()) +} + +fn qstat(count: i64, bytes: i64) -> QStat { + QStat { + count: count as f64, + bytes: bytes as f64, + } +} + +fn non_negative_u64(value: i64) -> u64 { + value.max(0) as u64 +} + +fn current_local_peer(req: &S3Request, state: &SiteReplicationState) -> PeerInfo { + let endpoint = request_endpoint(&req.uri, &req.headers); + let deployment_id = get_global_deployment_id().unwrap_or_else(|| deployment_id_for_endpoint(&endpoint)); + let stored_peer = state.peers.get(&deployment_id); + + PeerInfo { + endpoint: endpoint.clone(), + name: if state.name.is_empty() { + stored_peer + .map(|peer| peer.name.clone()) + .filter(|name| !name.is_empty()) + .unwrap_or_else(|| infer_site_name(&endpoint)) + } else { + state.name.clone() + }, + deployment_id, + sync_state: stored_peer.map(|peer| peer.sync_state.clone()).unwrap_or(SyncStatus::Unknown), + default_bandwidth: stored_peer.map(|peer| peer.default_bandwidth.clone()).unwrap_or_default(), + replicate_ilm_expiry: stored_peer.is_some_and(|peer| peer.replicate_ilm_expiry), + object_naming_mode: stored_peer.map(|peer| peer.object_naming_mode.clone()).unwrap_or_default(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + } +} + +fn current_local_runtime_peer(state: &SiteReplicationState) -> PeerInfo { + let endpoint = current_local_runtime_endpoint(); + let deployment_id = get_global_deployment_id().unwrap_or_else(|| deployment_id_for_endpoint(&endpoint)); + let stored_peer = state.peers.get(&deployment_id); + + PeerInfo { + endpoint: endpoint.clone(), + name: if state.name.is_empty() { + stored_peer + .map(|peer| peer.name.clone()) + .filter(|name| !name.is_empty()) + .unwrap_or_else(|| infer_site_name(&endpoint)) + } else { + state.name.clone() + }, + deployment_id, + sync_state: stored_peer.map(|peer| peer.sync_state.clone()).unwrap_or(SyncStatus::Unknown), + default_bandwidth: stored_peer.map(|peer| peer.default_bandwidth.clone()).unwrap_or_default(), + replicate_ilm_expiry: stored_peer.is_some_and(|peer| peer.replicate_ilm_expiry), + object_naming_mode: stored_peer.map(|peer| peer.object_naming_mode.clone()).unwrap_or_default(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + } +} + +fn canonical_endpoint(endpoint: &str) -> String { + let trimmed = endpoint.trim().trim_end_matches('/'); + let candidate = if trimmed.starts_with("http://") || trimmed.starts_with("https://") { + trimmed.to_string() + } else { + format!("http://{trimmed}") + }; + + Url::parse(&candidate) + .ok() + .map(|url| { + let scheme = url.scheme().to_ascii_lowercase(); + let host = url.host_str().unwrap_or_default().to_ascii_lowercase(); + let port = url.port_or_known_default(); + match port { + Some(port) => format!("{scheme}://{host}:{port}"), + None => format!("{scheme}://{host}"), + } + }) + .unwrap_or_else(|| trimmed.to_ascii_lowercase()) +} + +fn same_endpoint(left: &str, right: &str) -> bool { + canonical_endpoint(left) == canonical_endpoint(right) +} + +fn existing_peer_for_endpoint(state: &SiteReplicationState, endpoint: &str) -> Option { + state + .peers + .values() + .find(|peer| same_endpoint(&peer.endpoint, endpoint)) + .cloned() +} + +fn normalize_peer_info(mut peer: PeerInfo) -> PeerInfo { + if peer.deployment_id.is_empty() { + peer.deployment_id = deployment_id_for_endpoint(&peer.endpoint); + } + if peer.name.is_empty() { + peer.name = infer_site_name(&peer.endpoint); + } + if peer.api_version.is_none() { + peer.api_version = Some(SITE_REPL_API_VERSION.to_string()); + } + peer +} + +fn normalize_peer_site(site: PeerSite, replicate_ilm_expiry: bool) -> PeerInfo { + normalize_peer_info(PeerInfo { + endpoint: site.endpoint, + name: site.name, + deployment_id: String::new(), + sync_state: SyncStatus::Unknown, + default_bandwidth: BucketBandwidth::default(), + replicate_ilm_expiry, + object_naming_mode: String::new(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }) +} + +fn build_join_peers( + state: &SiteReplicationState, + local_peer: &PeerInfo, + sites: Vec, + replicate_ilm_expiry: bool, +) -> BTreeMap { + let mut peers = BTreeMap::new(); + let mut seen_endpoints = HashSet::new(); + + let mut normalized_local = local_peer.clone(); + normalized_local.replicate_ilm_expiry = replicate_ilm_expiry; + normalized_local = normalize_peer_info(normalized_local); + seen_endpoints.insert(canonical_endpoint(&normalized_local.endpoint)); + peers.insert(normalized_local.deployment_id.clone(), normalized_local); + + for site in sites { + let endpoint_key = canonical_endpoint(&site.endpoint); + if !seen_endpoints.insert(endpoint_key) { + continue; + } + + let mut peer = existing_peer_for_endpoint(state, &site.endpoint) + .unwrap_or_else(|| normalize_peer_site(site.clone(), replicate_ilm_expiry)); + peer.endpoint = site.endpoint; + if !site.name.is_empty() { + peer.name = site.name; + } + peer.replicate_ilm_expiry |= replicate_ilm_expiry; + peer = normalize_peer_info(peer); + peers.insert(peer.deployment_id.clone(), peer); + } + + peers +} + +fn normalize_join_peers_for_local(local_peer: &PeerInfo, peers: BTreeMap) -> BTreeMap { + let mut normalized = BTreeMap::new(); + + for (_, incoming_peer) in peers { + let mut peer = normalize_peer_info(incoming_peer); + if same_endpoint(&peer.endpoint, &local_peer.endpoint) { + peer.deployment_id = local_peer.deployment_id.clone(); + if peer.name.is_empty() { + peer.name = local_peer.name.clone(); + } + } + normalized.insert(peer.deployment_id.clone(), peer); + } + + if !normalized.contains_key(&local_peer.deployment_id) { + normalized.insert(local_peer.deployment_id.clone(), local_peer.clone()); + } + + normalized +} + +async fn ensure_site_replicator_service_account(parent_user: &str, state: &SiteReplicationState) -> S3Result<(String, String)> { + let Some(iam_sys) = get_global_iam_sys() else { + return Err(s3_error!(InvalidRequest, "iam not init")); + }; + + let access_key = SITE_REPLICATOR_SERVICE_ACCOUNT.to_string(); + let secret_key = + if state.service_account_access_key == SITE_REPLICATOR_SERVICE_ACCOUNT && !state.service_account_secret_key.is_empty() { + state.service_account_secret_key.clone() + } else { + rustfs_credentials::gen_secret_key(40) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("generate secret key failed: {e}")))? + }; + + if iam_sys.get_service_account(&access_key).await.is_ok() { + iam_sys + .update_service_account( + &access_key, + UpdateServiceAccountOpts { + session_policy: None, + secret_key: Some(secret_key.clone()), + name: None, + description: None, + expiration: None, + status: None, + }, + ) + .await + .map_err(ApiError::from)?; + } else { + iam_sys + .new_service_account( + parent_user, + None, + NewServiceAccountOpts { + session_policy: None, + access_key: access_key.clone(), + secret_key: secret_key.clone(), + name: None, + description: None, + expiration: None, + allow_site_replicator_account: true, + claims: None, + }, + ) + .await + .map_err(ApiError::from)?; + } + + Ok((access_key, secret_key)) +} + +async fn send_peer_admin_request( + endpoint: &str, + path: &str, + access_key: &str, + secret_key: &str, + body: &T, +) -> S3Result> { + let base = endpoint.trim_end_matches('/'); + let url = format!("{base}{path}"); + let uri = url + .parse::() + .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid peer endpoint: {e}")))?; + let authority = uri + .authority() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "peer endpoint missing authority".to_string()))? + .to_string(); + let payload = serde_json::to_vec(body) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize peer request failed: {e}")))?; + let (payload, content_type) = encode_compatible_admin_payload(path, secret_key, payload)?; + + let signed = sign_v4( + http::Request::builder() + .method(Method::PUT) + .uri(uri) + .header(HOST, authority) + .header("x-amz-content-sha256", UNSIGNED_PAYLOAD) + .header(CONTENT_TYPE, content_type) + .body(Body::empty()) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("build peer request failed: {e}")))?, + payload.len() as i64, + access_key, + secret_key, + "", + get_global_region() + .map(|region| region.to_string()) + .as_deref() + .unwrap_or("us-east-1"), + ); + + let mut req = site_replication_peer_client().request(reqwest::Method::PUT, &url); + for (name, value) in signed.headers() { + req = req.header(name, value); + } + + let response = req + .body(payload) + .send() + .await + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("peer request failed: {e}")))?; + + let status = response.status(); + let body = response + .bytes() + .await + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("read peer response failed: {e}")))?; + + if !status.is_success() { + let detail = String::from_utf8_lossy(&body).into_owned(); + return Err(S3Error::with_message( + S3ErrorCode::InternalError, + format!("peer request to {url} failed with {status}: {detail}"), + )); + } + + Ok(body.to_vec()) +} + +async fn runtime_site_replication_targets() -> S3Result> { + let state = load_site_replication_state().await?; + if !state.enabled() || state.service_account_access_key.is_empty() || state.service_account_secret_key.is_empty() { + return Ok(None); + } + + Ok(Some((state.clone(), current_local_runtime_peer(&state)))) +} + +async fn broadcast_site_replication_json(path: &str, body: &T) -> S3Result<()> { + let Some((state, local_peer)) = runtime_site_replication_targets().await? else { + return Ok(()); + }; + + for peer in state.peers.values() { + if peer.deployment_id == local_peer.deployment_id || same_endpoint(&peer.endpoint, &local_peer.endpoint) { + continue; + } + + send_peer_admin_request( + &peer.endpoint, + path, + &state.service_account_access_key, + &state.service_account_secret_key, + body, + ) + .await?; + } + + Ok(()) +} + +pub async fn site_replication_make_bucket_hook(bucket: &str, lock_enabled: bool) -> S3Result<()> { + let Some((_, _)) = runtime_site_replication_targets().await? else { + return Ok(()); + }; + + let created_at = new_object_layer_fn() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))? + .get_bucket_info(bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)? + .created + .unwrap_or_else(OffsetDateTime::now_utc) + .format(&time::format_description::well_known::Rfc3339) + .unwrap_or_default(); + + let path = { + let mut query = form_urlencoded::Serializer::new(String::new()); + query.append_pair("bucket", bucket); + query.append_pair("operation", "make-with-versioning"); + query.append_pair("createdAt", &created_at); + if lock_enabled { + query.append_pair("lockEnabled", "true"); + } + format!("/rustfs/admin/v3/site-replication/peer/bucket-ops?{}", query.finish()) + }; + broadcast_site_replication_json(&path, &serde_json::json!({})).await?; + + let configure_path = format!( + "/rustfs/admin/v3/site-replication/peer/bucket-ops?{}", + form_urlencoded::Serializer::new(String::new()) + .append_pair("bucket", bucket) + .append_pair("operation", "configure-replication") + .finish() + ); + broadcast_site_replication_json(&configure_path, &serde_json::json!({})).await +} + +pub async fn site_replication_delete_bucket_hook(bucket: &str, force_delete: bool) -> S3Result<()> { + let operation = if force_delete { + "force-delete-bucket" + } else { + "delete-bucket" + }; + let path = format!( + "/rustfs/admin/v3/site-replication/peer/bucket-ops?{}", + form_urlencoded::Serializer::new(String::new()) + .append_pair("bucket", bucket) + .append_pair("operation", operation) + .finish() + ); + broadcast_site_replication_json(&path, &serde_json::json!({})).await +} + +pub async fn site_replication_bucket_meta_hook(item: SRBucketMeta) -> S3Result<()> { + broadcast_site_replication_json("/rustfs/admin/v3/site-replication/peer/bucket-meta", &item).await +} + +pub async fn site_replication_iam_change_hook(item: SRIAMItem) -> S3Result<()> { + broadcast_site_replication_json("/rustfs/admin/v3/site-replication/peer/iam-item", &item).await +} + +fn raw_config_to_string(raw: &[u8]) -> Option { + if raw.is_empty() { + return None; + } + String::from_utf8(raw.to_vec()).ok() +} + +fn maybe_time(value: OffsetDateTime) -> Option { + (value != OffsetDateTime::UNIX_EPOCH).then_some(value) +} + +async fn build_sr_info(state: &SiteReplicationState, local_peer: &PeerInfo) -> S3Result { + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + let mut info = SRInfo { + enabled: state.enabled(), + name: local_peer.name.clone(), + deployment_id: local_peer.deployment_id.clone(), + state: SRStateInfo { + name: local_peer.name.clone(), + peers: state.peers.clone(), + updated_at: state.updated_at, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }; + + let buckets = store.list_bucket(&BucketOptions::default()).await.map_err(ApiError::from)?; + for bucket in buckets { + let metadata = metadata_sys::get(&bucket.name).await.ok(); + let mut entry = SRBucketInfo { + bucket: bucket.name.clone(), + created_at: bucket.created, + location: get_global_region().map(|region| region.to_string()).unwrap_or_default(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }; + + if let Some(metadata) = metadata { + entry.policy = raw_config_to_string(&metadata.policy_config_json).and_then(|raw| serde_json::from_str(&raw).ok()); + entry.versioning = raw_config_to_string(&metadata.versioning_config_xml); + entry.tags = raw_config_to_string(&metadata.tagging_config_xml); + entry.object_lock_config = raw_config_to_string(&metadata.object_lock_config_xml); + entry.sse_config = raw_config_to_string(&metadata.encryption_config_xml); + entry.replication_config = raw_config_to_string(&metadata.replication_config_xml); + entry.quota_config = raw_config_to_string(&metadata.quota_config_json); + entry.expiry_lc_config = raw_config_to_string(&metadata.lifecycle_config_xml); + entry.cors_config = raw_config_to_string(&metadata.cors_config_xml); + entry.policy_updated_at = maybe_time(metadata.policy_config_updated_at); + entry.tag_config_updated_at = maybe_time(metadata.tagging_config_updated_at); + entry.object_lock_config_updated_at = maybe_time(metadata.object_lock_config_updated_at); + entry.sse_config_updated_at = maybe_time(metadata.encryption_config_updated_at); + entry.versioning_config_updated_at = maybe_time(metadata.versioning_config_updated_at); + entry.replication_config_updated_at = maybe_time(metadata.replication_config_updated_at); + entry.quota_config_updated_at = maybe_time(metadata.quota_config_updated_at); + entry.expiry_lc_config_updated_at = maybe_time(metadata.lifecycle_config_updated_at); + entry.cors_config_updated_at = maybe_time(metadata.cors_config_updated_at); + } + + info.buckets.insert(bucket.name, entry); + } + + if let Some(iam_sys) = get_global_iam_sys() { + for (name, policy_doc) in iam_sys.list_policy_docs("").await.map_err(ApiError::from)? { + info.policies.insert( + name, + SRIAMPolicy { + policy: serde_json::to_value(policy_doc.policy).ok(), + updated_at: policy_doc.update_date, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }, + ); + } + + let users = iam_sys.list_users().await.map_err(ApiError::from)?; + for (name, user) in users { + info.user_info_map.insert(name, user); + } + + let groups = iam_sys.list_groups_load().await.map_err(ApiError::from)?; + for group in groups { + let desc = iam_sys.get_group_description(&group).await.map_err(ApiError::from)?; + info.group_desc_map.insert(group.clone(), desc); + } + + let mut user_policies = HashMap::::new(); + iam_sys + .load_mapped_policies(UserType::Reg, false, &mut user_policies) + .await + .map_err(ApiError::from)?; + for (name, mapping) in user_policies { + info.user_policies + .insert(name.clone(), mapped_policy_to_sr_mapping(name, false, UserType::Reg, mapping)); + } + + let mut group_policies = HashMap::::new(); + iam_sys + .load_mapped_policies(UserType::None, true, &mut group_policies) + .await + .map_err(ApiError::from)?; + for (name, mapping) in group_policies { + info.group_policies + .insert(name.clone(), mapped_policy_to_sr_mapping(name, true, UserType::None, mapping)); + } + } + + for (name, bucket_info) in &info.buckets { + if let Some(raw) = bucket_info + .replication_config + .as_ref() + .and_then(|value| serde_json::from_str::(value).ok()) + { + info.replication_cfg.insert(name.clone(), raw); + } + } + + Ok(info) +} + +fn mapped_policy_to_sr_mapping(name: String, is_group: bool, user_type: UserType, mapping: MappedPolicy) -> SRPolicyMapping { + SRPolicyMapping { + user_or_group: name, + user_type: user_type.to_u64(), + is_group, + policy: mapping.policies, + updated_at: Some(mapping.update_at), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + } +} + +fn filter_sr_info(mut info: SRInfo, opts: &SRStatusOptions) -> SRInfo { + if opts.include_all_defaults() { + return info; + } + + let include_buckets = + opts.buckets || opts.metrics || matches!(opts.entity, SREntityType::Bucket | SREntityType::IlmExpiryRule); + if !include_buckets { + info.buckets.clear(); + info.replication_cfg.clear(); + } else if opts.entity == SREntityType::Bucket && !opts.entity_value.is_empty() { + info.buckets.retain(|name, _| name == &opts.entity_value); + info.replication_cfg.retain(|name, _| name == &opts.entity_value); + } + + let include_policies = opts.policies || opts.entity == SREntityType::Policy; + if !include_policies { + info.policies.clear(); + } else if opts.entity == SREntityType::Policy && !opts.entity_value.is_empty() { + info.policies.retain(|name, _| name == &opts.entity_value); + } + + let include_users = opts.users || opts.entity == SREntityType::User; + if !include_users { + info.user_info_map.clear(); + info.user_policies.clear(); + } else if opts.entity == SREntityType::User && !opts.entity_value.is_empty() { + info.user_info_map.retain(|name, _| name == &opts.entity_value); + info.user_policies.retain(|name, _| name == &opts.entity_value); + } + + let include_groups = opts.groups || opts.entity == SREntityType::Group; + if !include_groups { + info.group_desc_map.clear(); + info.group_policies.clear(); + } else if opts.entity == SREntityType::Group && !opts.entity_value.is_empty() { + info.group_desc_map.retain(|name, _| name == &opts.entity_value); + info.group_policies.retain(|name, _| name == &opts.entity_value); + } + + let include_ilm_expiry = opts.ilm_expiry_rules || opts.entity == SREntityType::IlmExpiryRule; + if !include_ilm_expiry { + info.ilm_expiry_rules.clear(); + } else if opts.entity == SREntityType::IlmExpiryRule && !opts.entity_value.is_empty() { + info.ilm_expiry_rules.retain(|name, _| name == &opts.entity_value); + } + + info +} + +fn build_site_summary(info: &SRInfo) -> SRSiteSummary { + let replicated_buckets = info.buckets.len(); + let replicated_tags = info.buckets.values().filter(|bucket| bucket.tags.is_some()).count(); + let replicated_bucket_policies = info.buckets.values().filter(|bucket| bucket.policy.is_some()).count(); + let replicated_lock_config = info + .buckets + .values() + .filter(|bucket| bucket.object_lock_config.is_some()) + .count(); + let replicated_sse_config = info.buckets.values().filter(|bucket| bucket.sse_config.is_some()).count(); + let replicated_versioning_config = info.buckets.values().filter(|bucket| bucket.versioning.is_some()).count(); + let replicated_quota_config = info.buckets.values().filter(|bucket| bucket.quota_config.is_some()).count(); + let replicated_cors_config = info.buckets.values().filter(|bucket| bucket.cors_config.is_some()).count(); + + SRSiteSummary { + replicated_buckets, + replicated_tags, + replicated_bucket_policies, + replicated_iam_policies: info.policies.len(), + replicated_users: info.user_info_map.len(), + replicated_groups: info.group_desc_map.len(), + replicated_lock_config, + replicated_sse_config, + replicated_versioning_config, + replicated_quota_config, + replicated_user_policy_mappings: info.user_policies.len(), + replicated_group_policy_mappings: info.group_policies.len(), + replicated_ilm_expiry_rules: info.ilm_expiry_rules.len(), + replicated_cors_config, + total_buckets_count: info.buckets.len(), + total_tags_count: replicated_tags, + total_bucket_policies_count: replicated_bucket_policies, + total_iam_policies_count: info.policies.len(), + total_lock_config_count: replicated_lock_config, + total_sse_config_count: replicated_sse_config, + total_versioning_config_count: replicated_versioning_config, + total_quota_config_count: replicated_quota_config, + total_users_count: info.user_info_map.len(), + total_groups_count: info.group_desc_map.len(), + total_user_policy_mapping_count: info.user_policies.len(), + total_group_policy_mapping_count: info.group_policies.len(), + total_ilm_expiry_rules_count: info.ilm_expiry_rules.len(), + total_cors_config_count: replicated_cors_config, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + } +} + +async fn build_metrics_summary(local_peer: &PeerInfo) -> SRMetricsSummary { + let Some(stats) = GLOBAL_REPLICATION_STATS.get() else { + return SRMetricsSummary::default(); + }; + + let node = stats.get_sr_metrics_for_node().await; + let mut metrics = BTreeMap::new(); + metrics.insert( + local_peer.deployment_id.clone(), + SRMetric { + deployment_id: local_peer.deployment_id.clone(), + endpoint: local_peer.endpoint.clone(), + online: true, + replicated_size: node.replica_size, + replicated_count: node.replica_count, + last_online: Some(OffsetDateTime::now_utc()), + ..Default::default() + }, + ); + + SRMetricsSummary { + active_workers: WorkerStat { + curr: node.active_workers.curr, + avg: node.active_workers.avg, + max: node.active_workers.max, + }, + replica_size: node.replica_size, + replica_count: node.replica_count, + queued: InQueueMetric { + curr: qstat(node.queued.curr.count, node.queued.curr.bytes), + avg: qstat(node.queued.avg.count, node.queued.avg.bytes), + max: qstat(node.queued.max.count, node.queued.max.bytes), + }, + in_progress: InProgressMetric::default(), + proxied: ReplProxyMetric { + get_total: non_negative_u64(node.proxied.get_total), + head_total: non_negative_u64(node.proxied.head_total), + get_failed_total: non_negative_u64(node.proxied.get_failed), + head_failed_total: non_negative_u64(node.proxied.head_failed), + put_tag_total: non_negative_u64(node.proxied.put_total), + put_tag_failed_total: non_negative_u64(node.proxied.put_failed), + ..Default::default() + }, + metrics, + uptime: node.uptime, + ..Default::default() + } +} + +async fn build_status_info(state: &SiteReplicationState, local_peer: &PeerInfo, uri: &Uri) -> S3Result { + let opts = sr_status_options(uri); + let info = filter_sr_info(build_sr_info(state, local_peer).await?, &opts); + let metrics_requested = opts.metrics || opts.include_all_defaults() || opts.entity == SREntityType::Bucket; + + let mut status = SRStatusInfo { + enabled: state.enabled(), + max_buckets: info.buckets.len(), + max_users: info.user_info_map.len(), + max_groups: info.group_desc_map.len(), + max_policies: info.policies.len(), + max_ilm_expiry_rules: info.ilm_expiry_rules.len(), + sites: state.peers.clone(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }; + + for deployment_id in state.peers.keys() { + let summary = if deployment_id == &local_peer.deployment_id { + build_site_summary(&info) + } else { + SRSiteSummary { + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + } + }; + status.stats_summary.insert(deployment_id.clone(), summary); + + if deployment_id != &local_peer.deployment_id { + continue; + } + + if opts.include_all_defaults() || opts.buckets || opts.entity == SREntityType::Bucket { + for (bucket_name, bucket_info) in &info.buckets { + if opts.entity == SREntityType::Bucket && !opts.entity_value.is_empty() && bucket_name != &opts.entity_value { + continue; + } + status.bucket_stats.entry(bucket_name.clone()).or_default().insert( + deployment_id.clone(), + SRBucketStatsSummary { + deployment_id: deployment_id.clone(), + has_bucket: true, + has_tags_set: bucket_info.tags.is_some(), + has_object_lock_config_set: bucket_info.object_lock_config.is_some(), + has_policy_set: bucket_info.policy.is_some(), + has_sse_cfg_set: bucket_info.sse_config.is_some(), + has_replication_cfg: bucket_info.replication_config.is_some(), + has_quota_cfg_set: bucket_info.quota_config.is_some(), + has_cors_cfg_set: bucket_info.cors_config.is_some(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }, + ); + } + } + + if opts.include_all_defaults() || opts.policies || opts.entity == SREntityType::Policy { + for name in info.policies.keys() { + if opts.entity == SREntityType::Policy && !opts.entity_value.is_empty() && name != &opts.entity_value { + continue; + } + status.policy_stats.entry(name.clone()).or_default().insert( + deployment_id.clone(), + SRPolicyStatsSummary { + deployment_id: deployment_id.clone(), + has_policy: true, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }, + ); + } + } + + if opts.include_all_defaults() || opts.users || opts.entity == SREntityType::User { + for name in info.user_info_map.keys() { + if opts.entity == SREntityType::User && !opts.entity_value.is_empty() && name != &opts.entity_value { + continue; + } + status.user_stats.entry(name.clone()).or_default().insert( + deployment_id.clone(), + SRUserStatsSummary { + deployment_id: deployment_id.clone(), + has_user: true, + has_policy_mapping: info.user_policies.contains_key(name), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }, + ); + } + } + + if opts.include_all_defaults() || opts.groups || opts.entity == SREntityType::Group { + for name in info.group_desc_map.keys() { + if opts.entity == SREntityType::Group && !opts.entity_value.is_empty() && name != &opts.entity_value { + continue; + } + status.group_stats.entry(name.clone()).or_default().insert( + deployment_id.clone(), + SRGroupStatsSummary { + deployment_id: deployment_id.clone(), + has_group: true, + has_policy_mapping: info.group_policies.contains_key(name), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }, + ); + } + } + + if opts.include_all_defaults() || opts.ilm_expiry_rules || opts.entity == SREntityType::IlmExpiryRule { + for name in info.ilm_expiry_rules.keys() { + if opts.entity == SREntityType::IlmExpiryRule && !opts.entity_value.is_empty() && name != &opts.entity_value { + continue; + } + status.ilm_expiry_stats.entry(name.clone()).or_default().insert( + deployment_id.clone(), + SRILMExpiryStatsSummary { + deployment_id: deployment_id.clone(), + has_ilm_expiry_rules: true, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }, + ); + } + } + } + + if metrics_requested { + status.metrics = build_metrics_summary(local_peer).await; + } + + if opts.peer_state { + for (deployment_id, peer) in &state.peers { + status.peer_states.insert( + deployment_id.clone(), + SRStateInfo { + name: peer.name.clone(), + peers: state.peers.clone(), + updated_at: state.updated_at, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }, + ); + } + } + + Ok(status) +} + +fn merge_add_sites( + mut state: SiteReplicationState, + local_peer: PeerInfo, + sites: Vec, + service_account_access_key: String, + service_account_secret_key: String, + service_account_parent: String, + replicate_ilm_expiry: bool, +) -> SiteReplicationState { + state.name = local_peer.name.clone(); + state.service_account_access_key = service_account_access_key; + state.service_account_secret_key = service_account_secret_key; + state.service_account_parent = service_account_parent; + state.updated_at = Some(OffsetDateTime::now_utc()); + state.peers = build_join_peers(&state, &local_peer, sites, replicate_ilm_expiry); + state +} + +fn update_peer(mut state: SiteReplicationState, incoming: PeerInfo, ilm_expiry_override: Option) -> SiteReplicationState { + let mut peer = normalize_peer_info(incoming); + if let Some(enabled) = ilm_expiry_override { + peer.replicate_ilm_expiry = enabled; + } + state.updated_at = Some(OffsetDateTime::now_utc()); + state.peers.insert(peer.deployment_id.clone(), peer); + state +} + +fn edit_state(mut state: SiteReplicationState, incoming: PeerInfo, ilm_expiry_override: Option) -> SiteReplicationState { + if let Some(enabled) = ilm_expiry_override { + for peer in state.peers.values_mut() { + peer.replicate_ilm_expiry = enabled; + } + } + + if !incoming.deployment_id.is_empty() || !incoming.endpoint.is_empty() || !incoming.name.is_empty() { + state = update_peer(state, incoming, ilm_expiry_override); + } else { + state.updated_at = Some(OffsetDateTime::now_utc()); + } + + state +} + +fn remove_sites(mut state: SiteReplicationState, req: SRRemoveReq) -> SiteReplicationState { + if req.remove_all { + state.peers.clear(); + state.resync_status.clear(); + state.updated_at = Some(OffsetDateTime::now_utc()); + return state; + } + + let names: Vec = req.site_names.into_iter().collect(); + state.peers.retain(|_, peer| !names.iter().any(|name| name == &peer.name)); + state.updated_at = Some(OffsetDateTime::now_utc()); + state +} + +fn resync_status_for_state( + state: &mut SiteReplicationState, + op_type: &str, + peer: &PeerInfo, + bucket_names: Vec, +) -> SRResyncOpStatus { + let status = SRResyncOpStatus { + op_type: op_type.to_string(), + resync_id: Uuid::new_v4().to_string(), + status: "success".to_string(), + buckets: bucket_names + .into_iter() + .map(|bucket| ResyncBucketStatus { + bucket, + status: if op_type == SITE_REPL_RESYNC_CANCEL { + "canceled".to_string() + } else { + "started".to_string() + }, + ..Default::default() + }) + .collect(), + ..Default::default() + }; + state.resync_status.insert(peer.deployment_id.clone(), status.clone()); + status +} + +fn bucket_target_endpoint(target: &BucketTarget) -> String { + let scheme = if target.secure { "https" } else { "http" }; + canonical_endpoint(&format!("{scheme}://{}", target.endpoint)) +} + +fn bucket_target_matches_peer(target: &BucketTarget, peer: &PeerInfo) -> bool { + (!target.deployment_id.is_empty() && target.deployment_id == peer.deployment_id) + || bucket_target_endpoint(target) == canonical_endpoint(&peer.endpoint) +} + +async fn start_site_bucket_resync(bucket: &str, peer: &PeerInfo, resync_id: &str) -> ResyncBucketStatus { + let mut bucket_status = ResyncBucketStatus { + bucket: bucket.to_string(), + status: "started".to_string(), + ..Default::default() + }; + + let (config, _) = match metadata_sys::get_replication_config(bucket).await { + Ok(config) => config, + Err(err) => { + bucket_status.status = "failed".to_string(); + bucket_status.err_detail = err.to_string(); + return bucket_status; + } + }; + + let mut targets = match metadata_sys::list_bucket_targets(bucket).await { + Ok(targets) => targets, + Err(err) => { + bucket_status.status = "failed".to_string(); + bucket_status.err_detail = err.to_string(); + return bucket_status; + } + }; + + let reset_before = Some(OffsetDateTime::now_utc()); + let target_arn = { + let Some(target) = targets.targets.iter_mut().find(|target| { + target.target_type == BucketTargetType::ReplicationService && bucket_target_matches_peer(target, peer) + }) else { + bucket_status.status = "failed".to_string(); + bucket_status.err_detail = format!("no valid remote target found for peer {}", peer.deployment_id); + return bucket_status; + }; + + let (has_arn, existing_object_enabled) = config.has_existing_object_replication(&target.arn); + if !has_arn || !existing_object_enabled { + bucket_status.status = "failed".to_string(); + bucket_status.err_detail = "existing object replication is not enabled for the peer target".to_string(); + return bucket_status; + } + + target.reset_id = resync_id.to_string(); + target.reset_before_date = reset_before; + target.arn.clone() + }; + + let json_targets = match serde_json::to_vec(&targets) { + Ok(json_targets) => json_targets, + Err(err) => { + bucket_status.status = "failed".to_string(); + bucket_status.err_detail = err.to_string(); + return bucket_status; + } + }; + + if let Err(err) = metadata_sys::update(bucket, BUCKET_TARGETS_FILE, json_targets).await { + bucket_status.status = "failed".to_string(); + bucket_status.err_detail = err.to_string(); + return bucket_status; + } + BucketTargetSys::get().update_all_targets(bucket, Some(&targets)).await; + + let Some(pool) = get_global_replication_pool() else { + bucket_status.status = "failed".to_string(); + bucket_status.err_detail = "replication pool is not initialized".to_string(); + return bucket_status; + }; + + if let Err(err) = pool + .start_bucket_resync(ResyncOpts { + bucket: bucket.to_string(), + arn: target_arn, + resync_id: resync_id.to_string(), + resync_before: reset_before, + }) + .await + { + bucket_status.status = "failed".to_string(); + bucket_status.err_detail = err.to_string(); + } + + bucket_status +} + +async fn cancel_site_bucket_resync(bucket: &str, peer: &PeerInfo, resync_id: &str) -> ResyncBucketStatus { + let mut bucket_status = ResyncBucketStatus { + bucket: bucket.to_string(), + status: "canceled".to_string(), + ..Default::default() + }; + + let mut targets = match metadata_sys::list_bucket_targets(bucket).await { + Ok(targets) => targets, + Err(err) => { + bucket_status.status = "failed".to_string(); + bucket_status.err_detail = err.to_string(); + return bucket_status; + } + }; + + let Some(target) = targets.targets.iter_mut().find(|target| { + target.target_type == BucketTargetType::ReplicationService + && bucket_target_matches_peer(target, peer) + && target.reset_id == resync_id + }) else { + bucket_status.status = "failed".to_string(); + bucket_status.err_detail = format!("no in-progress resync target found for peer {}", peer.deployment_id); + return bucket_status; + }; + + target.reset_id.clear(); + target.reset_before_date = None; + let target_arn = target.arn.clone(); + + let json_targets = match serde_json::to_vec(&targets) { + Ok(json_targets) => json_targets, + Err(err) => { + bucket_status.status = "failed".to_string(); + bucket_status.err_detail = err.to_string(); + return bucket_status; + } + }; + + if let Err(err) = metadata_sys::update(bucket, BUCKET_TARGETS_FILE, json_targets).await { + bucket_status.status = "failed".to_string(); + bucket_status.err_detail = err.to_string(); + return bucket_status; + } + BucketTargetSys::get().update_all_targets(bucket, Some(&targets)).await; + + let Some(pool) = get_global_replication_pool() else { + bucket_status.status = "failed".to_string(); + bucket_status.err_detail = "replication pool is not initialized".to_string(); + return bucket_status; + }; + + if let Err(err) = pool + .cancel_bucket_resync(ResyncOpts { + bucket: bucket.to_string(), + arn: target_arn, + resync_id: resync_id.to_string(), + resync_before: None, + }) + .await + { + bucket_status.status = "failed".to_string(); + bucket_status.err_detail = err.to_string(); + } + + bucket_status +} + +fn apply_state_edit_req(mut state: SiteReplicationState, body: SRStateEditReq) -> SiteReplicationState { + let incoming_updated_at = body.updated_at.unwrap_or_else(OffsetDateTime::now_utc); + if state.updated_at.is_some_and(|current| incoming_updated_at <= current) { + return state; + } + + for (deployment_id, mut peer) in body.peers { + if peer.deployment_id.is_empty() { + peer.deployment_id = deployment_id.clone(); + } + if let Some(current_peer) = state.peers.get_mut(&deployment_id) { + current_peer.replicate_ilm_expiry = peer.replicate_ilm_expiry; + } else { + state.peers.insert(deployment_id, normalize_peer_info(peer)); + } + } + + state.updated_at = Some(incoming_updated_at); + state +} + +fn bucket_versioning_xml() -> S3Result> { + let config = VersioningConfiguration { + status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)), + ..Default::default() + }; + serialize(&config).map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize versioning failed: {e}"))) +} + +async fn apply_bucket_meta_item(item: SRBucketMeta) -> S3Result<()> { + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + store + .get_bucket_info(&item.bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + + let config_file = match item.r#type.as_str() { + "policy" => BUCKET_POLICY_CONFIG, + "tags" => BUCKET_TAGGING_CONFIG, + "version-config" => BUCKET_VERSIONING_CONFIG, + "object-lock-config" => OBJECT_LOCK_CONFIG, + "sse-config" => BUCKET_SSECONFIG, + "replication-config" => BUCKET_REPLICATION_CONFIG, + "quota-config" => BUCKET_QUOTA_CONFIG_FILE, + "lc-config" => BUCKET_LIFECYCLE_CONFIG, + "cors-config" => BUCKET_CORS_CONFIG, + _ => { + return Err(s3_error!( + NotImplemented, + "site replication bucket metadata type `{}` is not supported", + item.r#type + )); + } + }; + + let data = match item.r#type.as_str() { + "policy" => item + .policy + .map(|policy| serde_json::to_vec(&policy)) + .transpose() + .map_err(|e| s3_error!(InvalidRequest, "invalid bucket policy: {}", e))?, + "quota-config" => item + .quota + .map(|quota| serde_json::to_vec("a)) + .transpose() + .map_err(|e| s3_error!(InvalidRequest, "invalid bucket quota: {}", e))?, + "tags" => item.tags.map(String::into_bytes), + "version-config" => item.versioning.map(String::into_bytes), + "object-lock-config" => item.object_lock_config.map(String::into_bytes), + "sse-config" => item.sse_config.map(String::into_bytes), + "replication-config" => item.replication_config.map(String::into_bytes), + "lc-config" => item.expiry_lc_config.map(String::into_bytes), + "cors-config" => item + .cors + .map(|raw| BASE64_STANDARD.decode(raw.as_bytes()).unwrap_or_else(|_| raw.into_bytes())), + _ => unreachable!(), + }; + + if let Some(data) = data { + metadata_sys::update(&item.bucket, config_file, data) + .await + .map_err(ApiError::from)?; + } else { + metadata_sys::delete(&item.bucket, config_file) + .await + .map_err(ApiError::from)?; + } + Ok(()) +} + +async fn apply_iam_item(item: SRIAMItem) -> S3Result<()> { + let Some(iam_sys) = get_global_iam_sys() else { + return Err(s3_error!(InvalidRequest, "iam not init")); + }; + + match item.r#type.as_str() { + "policy" => { + if let Some(policy) = item.policy { + let policy: Policy = + serde_json::from_value(policy).map_err(|e| s3_error!(InvalidRequest, "invalid policy body: {}", e))?; + iam_sys.set_policy(&item.name, policy).await.map_err(ApiError::from)?; + } else { + iam_sys.delete_policy(&item.name, true).await.map_err(ApiError::from)?; + } + Ok(()) + } + "policy-mapping" => { + let Some(mapping) = item.policy_mapping else { + return Err(s3_error!(InvalidRequest, "policyMapping is required")); + }; + let user_type = UserType::from_u64(mapping.user_type).ok_or_else(|| s3_error!(InvalidRequest, "invalid userType"))?; + iam_sys + .policy_db_set(&mapping.user_or_group, user_type, mapping.is_group, &mapping.policy) + .await + .map_err(ApiError::from)?; + Ok(()) + } + "group-info" => { + let Some(group_info) = item.group_info else { + return Err(s3_error!(InvalidRequest, "groupInfo is required")); + }; + let update = group_info.update_req; + if update.is_remove { + iam_sys + .remove_users_from_group(&update.group, update.members) + .await + .map_err(ApiError::from)?; + return Ok(()); + } + + if update.members.is_empty() { + iam_sys + .set_group_status(&update.group, matches!(update.status, GroupStatus::Enabled)) + .await + .map_err(ApiError::from)?; + return Ok(()); + } + + iam_sys + .add_users_to_group(&update.group, update.members) + .await + .map_err(ApiError::from)?; + iam_sys + .set_group_status(&update.group, matches!(update.status, GroupStatus::Enabled)) + .await + .map_err(ApiError::from)?; + Ok(()) + } + "sts-credential" => { + let Some(sts_credential) = item.sts_credential else { + return Err(s3_error!(InvalidRequest, "stsCredential is required")); + }; + let Some(secret) = rustfs_iam::manager::get_token_signing_key() else { + return Err(s3_error!(InvalidRequest, "token signing key not initialized")); + }; + let claims = get_claims_from_token_with_secret(&sts_credential.session_token, &secret) + .map_err(|e| s3_error!(InvalidRequest, "invalid STS session token: {e}"))?; + let expiration = claims + .get("exp") + .and_then(claims_unix_timestamp) + .map(OffsetDateTime::from_unix_timestamp) + .transpose() + .map_err(|e| s3_error!(InvalidRequest, "invalid STS expiry: {e}"))?; + let cred = rustfs_credentials::Credentials { + access_key: sts_credential.access_key.clone(), + secret_key: sts_credential.secret_key.clone(), + session_token: sts_credential.session_token.clone(), + expiration, + status: "on".to_string(), + parent_user: sts_credential.parent_user.clone(), + claims: Some(claims), + ..Default::default() + }; + iam_sys + .set_temp_user( + &sts_credential.access_key, + &cred, + (!sts_credential.parent_policy_mapping.is_empty()).then_some(sts_credential.parent_policy_mapping.as_str()), + ) + .await + .map_err(ApiError::from)?; + Ok(()) + } + "iam-user" => { + let Some(user) = item.iam_user else { + return Err(s3_error!(InvalidRequest, "iamUser is required")); + }; + if user.is_delete_req { + iam_sys.delete_user(&user.access_key, true).await.map_err(ApiError::from)?; + } else { + let Some(user_req) = user.user_req else { + return Err(s3_error!(InvalidRequest, "userReq is required")); + }; + iam_sys + .create_user(&user.access_key, &user_req) + .await + .map_err(ApiError::from)?; + } + Ok(()) + } + "service-account" => { + let Some(change) = item.svc_acc_change else { + return Err(s3_error!(InvalidRequest, "serviceAccountChange is required")); + }; + if let Some(create) = change.create { + let session_policy = create.session_policy.as_str().and_then(|raw| serde_json::from_str(raw).ok()); + iam_sys + .new_service_account( + &create.parent, + Some(create.groups), + NewServiceAccountOpts { + session_policy, + access_key: create.access_key, + secret_key: create.secret_key, + name: (!create.name.is_empty()).then_some(create.name), + description: (!create.description.is_empty()).then_some(create.description), + expiration: create.expiration, + allow_site_replicator_account: true, + claims: Some(create.claims), + }, + ) + .await + .map_err(ApiError::from)?; + return Ok(()); + } + + if let Some(update) = change.update { + let session_policy = update.session_policy.as_str().and_then(|raw| serde_json::from_str(raw).ok()); + iam_sys + .update_service_account( + &update.access_key, + UpdateServiceAccountOpts { + session_policy, + secret_key: (!update.secret_key.is_empty()).then_some(update.secret_key), + name: (!update.name.is_empty()).then_some(update.name), + description: (!update.description.is_empty()).then_some(update.description), + expiration: update.expiration, + status: (!update.status.is_empty()).then_some(update.status), + }, + ) + .await + .map_err(ApiError::from)?; + return Ok(()); + } + + if let Some(delete) = change.delete { + iam_sys + .delete_service_account(&delete.access_key, true) + .await + .map_err(ApiError::from)?; + return Ok(()); + } + + Err(s3_error!(InvalidRequest, "serviceAccountChange is empty")) + } + _ => Err(s3_error!( + NotImplemented, + "site replication IAM item type `{}` is not supported", + item.r#type + )), + } +} + +fn claims_unix_timestamp(value: &Value) -> Option { + match value { + Value::Number(number) => number.as_i64(), + Value::String(raw) => raw.parse().ok(), + _ => None, + } +} + +pub struct SiteReplicationAddHandler {} + +#[async_trait::async_trait] +impl Operation for SiteReplicationAddHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + let cred = validate_site_replication_admin_request(&req, AdminAction::SiteReplicationAddAction).await?; + let replicate_ilm_expiry = sr_add_replicate_ilm_expiry(&req.uri); + let current_state = load_site_replication_state().await?; + let local_peer = current_local_peer(&req, ¤t_state); + let sites: Vec = read_site_replication_json(req, &cred.secret_key, true).await?; + let (service_account_access_key, service_account_secret_key) = + ensure_site_replicator_service_account(&cred.access_key, ¤t_state).await?; + let state = merge_add_sites( + current_state, + local_peer.clone(), + sites.clone(), + service_account_access_key.clone(), + service_account_secret_key.clone(), + cred.access_key.clone(), + replicate_ilm_expiry, + ); + let join_req = SRPeerJoinReq { + svc_acct_access_key: service_account_access_key, + svc_acct_secret_key: service_account_secret_key, + svc_acct_parent: String::new(), + peers: state.peers.clone(), + updated_at: state.updated_at, + }; + + let mut joined_endpoints = HashSet::new(); + for site in &sites { + let endpoint_key = canonical_endpoint(&site.endpoint); + if same_endpoint(&site.endpoint, &local_peer.endpoint) || !joined_endpoints.insert(endpoint_key) { + continue; + } + + let mut peer_join_req = join_req.clone(); + peer_join_req.svc_acct_parent = site.access_key.clone(); + send_peer_admin_request( + &site.endpoint, + SITE_REPLICATION_PEER_JOIN_PATH, + &site.access_key, + &site.secret_key, + &peer_join_req, + ) + .await?; + } + + persist_site_replication_state(&state).await?; + json_response(&ReplicateAddStatus { + success: true, + status: SITE_REPL_ADD_SUCCESS.to_string(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }) + } +} + +pub struct SiteReplicationRemoveHandler {} + +#[async_trait::async_trait] +impl Operation for SiteReplicationRemoveHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + validate_site_replication_admin_request(&req, AdminAction::SiteReplicationRemoveAction).await?; + let current_state = load_site_replication_state().await?; + let local_peer = current_local_peer(&req, ¤t_state); + let remove_req: SRRemoveReq = read_site_replication_json(req, "", false).await?; + + if !current_state.service_account_access_key.is_empty() && !current_state.service_account_secret_key.is_empty() { + for peer in current_state.peers.values() { + if same_endpoint(&peer.endpoint, &local_peer.endpoint) { + continue; + } + send_peer_admin_request( + &peer.endpoint, + SITE_REPLICATION_PEER_REMOVE_PATH, + ¤t_state.service_account_access_key, + ¤t_state.service_account_secret_key, + &SRRemoveReq { + requesting_dep_id: local_peer.deployment_id.clone(), + site_names: remove_req.site_names.clone(), + remove_all: remove_req.remove_all, + }, + ) + .await?; + } + } + + let state = remove_sites(current_state, remove_req); + persist_site_replication_state(&state).await?; + json_response(&ReplicateRemoveStatus { + status: SITE_REPL_REMOVE_SUCCESS.to_string(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }) + } +} + +pub struct SiteReplicationInfoHandler {} + +#[async_trait::async_trait] +impl Operation for SiteReplicationInfoHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + validate_site_replication_admin_request(&req, AdminAction::SiteReplicationInfoAction).await?; + let state = load_site_replication_state().await?; + let local_peer = current_local_peer(&req, &state); + let info = SiteReplicationInfo { + enabled: state.enabled(), + name: local_peer.name, + sites: state.peers.values().cloned().collect(), + service_account_access_key: state.service_account_access_key, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }; + json_response(&info) + } +} + +pub struct SiteReplicationMetaInfoHandler {} + +#[async_trait::async_trait] +impl Operation for SiteReplicationMetaInfoHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + validate_site_replication_admin_request(&req, AdminAction::SiteReplicationInfoAction).await?; + let state = load_site_replication_state().await?; + let local_peer = current_local_peer(&req, &state); + let opts = sr_status_options(&req.uri); + let info = filter_sr_info(build_sr_info(&state, &local_peer).await?, &opts); + json_response(&info) + } +} + +pub struct SiteReplicationStatusHandler {} + +#[async_trait::async_trait] +impl Operation for SiteReplicationStatusHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + validate_site_replication_admin_request(&req, AdminAction::SiteReplicationInfoAction).await?; + let state = load_site_replication_state().await?; + let local_peer = current_local_peer(&req, &state); + let status = build_status_info(&state, &local_peer, &req.uri).await?; + json_response(&status) + } +} + +pub struct SiteReplicationDevNullHandler {} + +#[async_trait::async_trait] +impl Operation for SiteReplicationDevNullHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + validate_site_replication_admin_request(&req, AdminAction::SiteReplicationInfoAction).await?; + let _ = read_plain_admin_body(req.input).await?; + Ok(empty_response(StatusCode::NO_CONTENT)) + } +} + +pub struct SiteReplicationNetPerfHandler {} + +#[async_trait::async_trait] +impl Operation for SiteReplicationNetPerfHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + validate_site_replication_admin_request(&req, AdminAction::SiteReplicationInfoAction).await?; + let duration = query_pairs(&req.uri) + .get("duration") + .and_then(|value| rustfs_madmin::utils::parse_duration(value).ok()) + .unwrap_or(SITE_REPL_MIN_NETPERF_DURATION) + .max(SITE_REPL_MIN_NETPERF_DURATION); + + let endpoint = request_endpoint(&req.uri, &req.headers); + let started_at = Instant::now(); + let body = read_plain_admin_body(req.input).await?; + let elapsed = started_at.elapsed().max(duration); + + Ok(go_gob_site_netperf_response(&SiteNetPerfNodeResult { + endpoint, + tx: body.len() as u64, + tx_total_duration_ns: elapsed.as_nanos() as i64, + rx: body.len() as u64, + rx_total_duration_ns: elapsed.as_nanos() as i64, + total_conn: 1, + error: String::new(), + })) + } +} + +pub struct SRPeerJoinHandler {} + +#[async_trait::async_trait] +impl Operation for SRPeerJoinHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + let cred = validate_site_replication_admin_request(&req, AdminAction::SiteReplicationAddAction).await?; + let mut state = load_site_replication_state().await?; + let local_peer = current_local_peer(&req, &state); + let join_req: SRPeerJoinReq = read_site_replication_json(req, &cred.secret_key, true).await?; + + if !join_req.svc_acct_access_key.is_empty() && !join_req.svc_acct_secret_key.is_empty() { + let Some(iam_sys) = get_global_iam_sys() else { + return Err(s3_error!(InvalidRequest, "iam not init")); + }; + + if iam_sys.get_service_account(&join_req.svc_acct_access_key).await.is_ok() { + iam_sys + .update_service_account( + &join_req.svc_acct_access_key, + UpdateServiceAccountOpts { + session_policy: None, + secret_key: Some(join_req.svc_acct_secret_key.clone()), + name: None, + description: None, + expiration: None, + status: None, + }, + ) + .await + .map_err(ApiError::from)?; + } else { + iam_sys + .new_service_account( + &join_req.svc_acct_parent, + None, + NewServiceAccountOpts { + session_policy: None, + access_key: join_req.svc_acct_access_key.clone(), + secret_key: join_req.svc_acct_secret_key.clone(), + name: None, + description: None, + expiration: None, + allow_site_replicator_account: join_req.svc_acct_access_key == SITE_REPLICATOR_SERVICE_ACCOUNT, + claims: None, + }, + ) + .await + .map_err(ApiError::from)?; + } + } + + state.service_account_access_key = join_req.svc_acct_access_key; + state.service_account_secret_key = join_req.svc_acct_secret_key; + state.service_account_parent = join_req.svc_acct_parent; + state.updated_at = join_req.updated_at.or_else(|| Some(OffsetDateTime::now_utc())); + state.peers = normalize_join_peers_for_local(&local_peer, join_req.peers); + state.name = state + .peers + .get(&local_peer.deployment_id) + .map(|peer| peer.name.clone()) + .filter(|name| !name.is_empty()) + .unwrap_or(local_peer.name); + persist_site_replication_state(&state).await?; + Ok(empty_response(StatusCode::OK)) + } +} + +pub struct SRPeerBucketOpsHandler {} + +#[async_trait::async_trait] +impl Operation for SRPeerBucketOpsHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + validate_site_replication_admin_request(&req, AdminAction::SiteReplicationOperationAction).await?; + let queries = query_pairs(&req.uri); + let bucket = queries + .get("bucket") + .filter(|bucket| !bucket.is_empty()) + .cloned() + .ok_or_else(|| s3_error!(InvalidRequest, "bucket is required"))?; + let operation = queries + .get("operation") + .filter(|value| !value.is_empty()) + .cloned() + .ok_or_else(|| s3_error!(InvalidRequest, "operation is required"))?; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + match operation.as_str() { + "make-with-versioning" => { + let created_at = queries + .get("createdAt") + .and_then(|value| OffsetDateTime::parse(value, &time::format_description::well_known::Rfc3339).ok()); + let lock_enabled = queries.get("lockEnabled").is_some_and(|value| value == "true"); + store + .make_bucket( + &bucket, + &MakeBucketOptions { + versioning_enabled: true, + lock_enabled, + created_at, + force_create: true, + ..Default::default() + }, + ) + .await + .map_err(ApiError::from)?; + metadata_sys::update(&bucket, BUCKET_VERSIONING_CONFIG, bucket_versioning_xml()?) + .await + .map_err(ApiError::from)?; + } + "configure-replication" => { + store + .get_bucket_info(&bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + } + "delete-bucket" => { + store + .delete_bucket( + &bucket, + &DeleteBucketOptions { + force: false, + srdelete_op: SRBucketDeleteOp::MarkDelete, + ..Default::default() + }, + ) + .await + .map_err(ApiError::from)?; + } + "force-delete-bucket" => { + store + .delete_bucket( + &bucket, + &DeleteBucketOptions { + force: true, + srdelete_op: SRBucketDeleteOp::Purge, + ..Default::default() + }, + ) + .await + .map_err(ApiError::from)?; + } + "purge-deleted-bucket" => { + let _ = store + .delete_bucket( + &bucket, + &DeleteBucketOptions { + force: true, + srdelete_op: SRBucketDeleteOp::Purge, + ..Default::default() + }, + ) + .await; + } + _ => return Err(s3_error!(InvalidRequest, "unsupported site replication bucket operation")), + } + + Ok(empty_response(StatusCode::OK)) + } +} + +pub struct SRPeerReplicateIAMItemHandler {} + +#[async_trait::async_trait] +impl Operation for SRPeerReplicateIAMItemHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + validate_site_replication_admin_request(&req, AdminAction::SiteReplicationOperationAction).await?; + let item: SRIAMItem = read_site_replication_json(req, "", false).await?; + apply_iam_item(item).await?; + Ok(empty_response(StatusCode::OK)) + } +} + +pub struct SRPeerReplicateBucketItemHandler {} + +#[async_trait::async_trait] +impl Operation for SRPeerReplicateBucketItemHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + validate_site_replication_admin_request(&req, AdminAction::SiteReplicationOperationAction).await?; + let item: SRBucketMeta = read_site_replication_json(req, "", false).await?; + apply_bucket_meta_item(item).await?; + Ok(empty_response(StatusCode::OK)) + } +} + +pub struct SRPeerGetIDPSettingsHandler {} + +#[async_trait::async_trait] +impl Operation for SRPeerGetIDPSettingsHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + validate_site_replication_admin_request(&req, AdminAction::SiteReplicationAddAction).await?; + + let mut settings = IDPSettings::default(); + if let Some(oidc) = get_oidc() { + let providers = oidc.list_providers(); + settings.open_id.enabled = !providers.is_empty(); + settings.open_id.region = get_global_region().map(|region| region.to_string()).unwrap_or_default(); + + for provider in providers { + let Some(config) = oidc.get_provider_config(&provider.provider_id) else { + continue; + }; + let provider_settings = OpenIDProviderSettings { + claim_name: config.claim_name.clone(), + claim_userinfo_enabled: false, + role_policy: config.role_policy.clone(), + client_id: config.client_id.clone(), + hashed_client_secret: hash_client_secret(config.client_secret.as_deref()), + }; + + let claim_provider_unset = settings.open_id.claim_provider.client_id.is_empty() + && settings.open_id.claim_provider.claim_name.is_empty() + && settings.open_id.claim_provider.role_policy.is_empty() + && settings.open_id.claim_provider.hashed_client_secret.is_empty(); + + if provider.provider_id == "default" || claim_provider_unset { + settings.open_id.claim_provider = provider_settings.clone(); + } else { + settings.open_id.roles.insert(provider.provider_id.clone(), provider_settings); + } + } + } + let (ldap, ldap_configs) = load_ldap_idp_settings(); + settings.ldap = ldap; + settings.ldap_configs = ldap_configs; + + json_response(&settings) + } +} + +pub struct SiteReplicationEditHandler {} + +#[async_trait::async_trait] +impl Operation for SiteReplicationEditHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + let cred = validate_site_replication_admin_request(&req, AdminAction::SiteReplicationAddAction).await?; + let ilm_expiry_override = sr_edit_ilm_expiry_override(&req.uri); + let incoming: PeerInfo = read_site_replication_json(req, &cred.secret_key, true).await?; + let current_state = load_site_replication_state().await?; + let state = edit_state(current_state.clone(), incoming.clone(), ilm_expiry_override); + + if !current_state.service_account_access_key.is_empty() && !current_state.service_account_secret_key.is_empty() { + let peers_to_send: Vec = if ilm_expiry_override.is_some() { + state.peers.values().cloned().collect() + } else { + vec![normalize_peer_info(incoming)] + }; + + for target in current_state.peers.values() { + let local_target = get_global_deployment_id() + .as_ref() + .is_some_and(|deployment_id| deployment_id == &target.deployment_id); + if local_target { + continue; + } + + for peer in &peers_to_send { + send_peer_admin_request( + &target.endpoint, + SITE_REPLICATION_PEER_EDIT_PATH, + ¤t_state.service_account_access_key, + ¤t_state.service_account_secret_key, + peer, + ) + .await?; + } + } + } + + save_site_replication_state(&state).await?; + json_response(&ReplicateEditStatus { + success: true, + status: SITE_REPL_EDIT_SUCCESS.to_string(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }) + } +} + +pub struct SRPeerEditHandler {} + +#[async_trait::async_trait] +impl Operation for SRPeerEditHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + validate_site_replication_admin_request(&req, AdminAction::SiteReplicationAddAction).await?; + let ilm_expiry_override = sr_edit_ilm_expiry_override(&req.uri); + let state = load_site_replication_state().await?; + let local_peer = current_local_peer(&req, &state); + let mut incoming: PeerInfo = read_site_replication_json(req, "", false).await?; + if same_endpoint(&incoming.endpoint, &local_peer.endpoint) { + incoming.deployment_id = local_peer.deployment_id.clone(); + if incoming.name.is_empty() { + incoming.name = local_peer.name.clone(); + } + } + let state = update_peer(state, incoming, ilm_expiry_override); + save_site_replication_state(&state).await?; + Ok(empty_response(StatusCode::OK)) + } +} + +pub struct SRPeerRemoveHandler {} + +#[async_trait::async_trait] +impl Operation for SRPeerRemoveHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + validate_site_replication_admin_request(&req, AdminAction::SiteReplicationRemoveAction).await?; + let remove_req: SRRemoveReq = read_site_replication_json(req, "", false).await?; + let state = remove_sites(load_site_replication_state().await?, remove_req); + persist_site_replication_state(&state).await?; + Ok(empty_response(StatusCode::OK)) + } +} + +pub struct SiteReplicationResyncOpHandler {} + +#[async_trait::async_trait] +impl Operation for SiteReplicationResyncOpHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + validate_site_replication_admin_request(&req, AdminAction::SiteReplicationResyncAction).await?; + let operation = query_pairs(&req.uri).get("operation").cloned().unwrap_or_default(); + let peer: PeerInfo = read_site_replication_json(req, "", false).await?; + let mut state = load_site_replication_state().await?; + let local_peer = current_local_runtime_peer(&state); + let peer = normalize_peer_info(peer); + if peer.deployment_id == local_peer.deployment_id { + return Err(s3_error!(InvalidRequest, "invalid peer specified - cannot resync to self")); + } + if !state.peers.contains_key(&peer.deployment_id) { + return Err(s3_error!(InvalidRequest, "site replication peer not found")); + } + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + let buckets = store.list_bucket(&BucketOptions::default()).await.map_err(ApiError::from)?; + let bucket_names: Vec = buckets.into_iter().map(|bucket| bucket.name).collect(); + + let status = match operation.as_str() { + SITE_REPL_RESYNC_START => { + let mut status = resync_status_for_state(&mut state, &operation, &peer, vec![]); + let mut bucket_statuses = Vec::new(); + for bucket in bucket_names { + bucket_statuses.push(start_site_bucket_resync(&bucket, &peer, &status.resync_id).await); + } + let failures = bucket_statuses.iter().filter(|bucket| bucket.status == "failed").count(); + if failures == bucket_statuses.len() && !bucket_statuses.is_empty() { + status.status = "failed".to_string(); + status.err_detail = "all buckets resync failed".to_string(); + } else if failures > 0 { + status.err_detail = "partial failure in starting site resync".to_string(); + } + status.buckets = bucket_statuses; + state.resync_status.insert(peer.deployment_id.clone(), status.clone()); + status + } + SITE_REPL_RESYNC_CANCEL => { + let Some(existing_status) = state.resync_status.get(&peer.deployment_id).cloned() else { + return Err(s3_error!(InvalidRequest, "no resync in progress")); + }; + if existing_status.resync_id.is_empty() { + return Err(s3_error!(InvalidRequest, "no resync in progress")); + } + let mut status = SRResyncOpStatus { + op_type: operation.clone(), + resync_id: existing_status.resync_id.clone(), + status: "success".to_string(), + ..Default::default() + }; + let mut bucket_statuses = Vec::new(); + for bucket in bucket_names { + bucket_statuses.push(cancel_site_bucket_resync(&bucket, &peer, &existing_status.resync_id).await); + } + let failures = bucket_statuses.iter().filter(|bucket| bucket.status == "failed").count(); + if failures == bucket_statuses.len() && !bucket_statuses.is_empty() { + status.status = "failed".to_string(); + status.err_detail = "all buckets resync cancel failed".to_string(); + } else if failures > 0 { + status.err_detail = "partial failure in canceling site resync".to_string(); + } + status.buckets = bucket_statuses; + state.resync_status.insert(peer.deployment_id.clone(), status.clone()); + status + } + _ => return Err(s3_error!(InvalidRequest, "unsupported resync operation")), + }; + save_site_replication_state(&state).await?; + json_response(&status) + } +} + +pub struct SRStateEditHandler {} + +#[async_trait::async_trait] +impl Operation for SRStateEditHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + validate_site_replication_admin_request(&req, AdminAction::SiteReplicationOperationAction).await?; + let body: SRStateEditReq = read_site_replication_json(req, "", false).await?; + let state = apply_state_edit_req(load_site_replication_state().await?, body); + save_site_replication_state(&state).await?; + Ok(empty_response(StatusCode::OK)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use http::Uri; + + fn peer(name: &str, endpoint: &str) -> PeerInfo { + PeerInfo { + name: name.to_string(), + endpoint: endpoint.to_string(), + deployment_id: String::new(), + sync_state: SyncStatus::Unknown, + default_bandwidth: BucketBandwidth::default(), + replicate_ilm_expiry: false, + object_naming_mode: String::new(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + } + } + + #[test] + fn test_sr_status_options_parse_minio_query_flags() { + let uri: Uri = "/rustfs/admin/v3/site-replication/status?buckets=true&policies=true&users=true&groups=true&metrics=true&peer-state=true&ilm-expiry-rules=true&entity=bucket&entityvalue=photos" + .parse() + .unwrap(); + + let opts = sr_status_options(&uri); + + assert!(opts.buckets); + assert!(opts.policies); + assert!(opts.users); + assert!(opts.groups); + assert!(opts.metrics); + assert!(opts.peer_state); + assert!(opts.ilm_expiry_rules); + assert_eq!(opts.entity, SREntityType::Bucket); + assert_eq!(opts.entity_value, "photos"); + } + + #[test] + fn test_query_flag_parses_lock_enabled() { + let uri: Uri = + "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning&lockEnabled=true" + .parse() + .unwrap(); + + assert!(query_flag(&uri, "lockEnabled")); + assert!(!query_flag(&uri, "missing")); + } + + #[test] + fn test_merge_add_sites_propagates_replicate_ilm_expiry() { + let state = merge_add_sites( + SiteReplicationState::default(), + peer("local", "https://local.example.com"), + vec![PeerSite { + name: "remote".to_string(), + endpoint: "https://remote.example.com".to_string(), + access_key: "remote-ak".to_string(), + secret_key: "remote-sk".to_string(), + }], + "svc-ak".to_string(), + "svc-sk".to_string(), + "root".to_string(), + true, + ); + + assert!(state.peers.values().all(|peer| peer.replicate_ilm_expiry)); + } + + #[test] + fn test_merge_add_sites_deduplicates_local_site_from_input() { + let local_peer = PeerInfo { + deployment_id: "local-dep".to_string(), + ..peer("local", "https://local.example.com") + }; + let state = merge_add_sites( + SiteReplicationState::default(), + local_peer, + vec![ + PeerSite { + name: "local".to_string(), + endpoint: "https://local.example.com/".to_string(), + access_key: "local-ak".to_string(), + secret_key: "local-sk".to_string(), + }, + PeerSite { + name: "remote".to_string(), + endpoint: "https://remote.example.com".to_string(), + access_key: "remote-ak".to_string(), + secret_key: "remote-sk".to_string(), + }, + ], + "svc-ak".to_string(), + "svc-sk".to_string(), + "root".to_string(), + true, + ); + + assert_eq!(state.peers.len(), 2); + assert!(state.peers.contains_key("local-dep")); + } + + #[test] + fn test_normalize_join_peers_rewrites_local_endpoint_to_real_deployment_id() { + let local_peer = PeerInfo { + deployment_id: "real-local".to_string(), + ..peer("local", "https://local.example.com") + }; + let peers = BTreeMap::from([ + ( + "hash-local".to_string(), + PeerInfo { + deployment_id: "hash-local".to_string(), + ..peer("local", "https://local.example.com/") + }, + ), + ( + "hash-remote".to_string(), + PeerInfo { + deployment_id: "hash-remote".to_string(), + ..peer("remote", "https://remote.example.com") + }, + ), + ]); + + let normalized = normalize_join_peers_for_local(&local_peer, peers); + + assert!(normalized.contains_key("real-local")); + assert!(!normalized.contains_key("hash-local")); + assert!(normalized.contains_key("hash-remote")); + } + + #[test] + fn test_site_replication_state_requires_remote_peer_to_be_enabled() { + let mut state = SiteReplicationState::default(); + state.peers.insert( + "local".to_string(), + PeerInfo { + deployment_id: "local".to_string(), + ..peer("local", "https://local.example.com") + }, + ); + + assert!(!state.enabled()); + } + + #[test] + fn test_sr_remove_req_accepts_null_sites() { + let req: SRRemoveReq = serde_json::from_str(r#"{"all":true,"sites":null}"#).expect("parse remove req"); + + assert!(req.remove_all); + assert!(req.site_names.is_empty()); + } + + #[test] + fn test_update_peer_respects_ilm_expiry_override() { + let peer = peer("remote", "https://remote.example.com"); + + let state = update_peer(SiteReplicationState::default(), peer, Some(true)); + + assert!(state.peers.values().next().unwrap().replicate_ilm_expiry); + } + + #[test] + fn test_edit_state_updates_ilm_expiry_for_all_peers() { + let mut state = SiteReplicationState::default(); + state.peers.insert( + "local".to_string(), + PeerInfo { + deployment_id: "local".to_string(), + ..peer("local", "https://local.example.com") + }, + ); + state.peers.insert( + "remote".to_string(), + PeerInfo { + deployment_id: "remote".to_string(), + ..peer("remote", "https://remote.example.com") + }, + ); + + let edited = edit_state(state, PeerInfo::default(), Some(true)); + + assert!(edited.peers.values().all(|peer| peer.replicate_ilm_expiry)); + } + + #[test] + fn test_bucket_target_matches_peer_by_deployment_id() { + let target = BucketTarget { + deployment_id: "remote-dep".to_string(), + endpoint: "other-host:9000".to_string(), + target_type: BucketTargetType::ReplicationService, + ..Default::default() + }; + let mut remote = peer("remote", "https://remote.example.com"); + remote.deployment_id = "remote-dep".to_string(); + + assert!(bucket_target_matches_peer(&target, &remote)); + } + + #[test] + fn test_bucket_target_matches_peer_by_endpoint() { + let target = BucketTarget { + endpoint: "remote.example.com:443".to_string(), + secure: true, + target_type: BucketTargetType::ReplicationService, + ..Default::default() + }; + let remote = peer("remote", "https://remote.example.com/"); + + assert!(bucket_target_matches_peer(&target, &remote)); + } + + #[test] + fn test_apply_state_edit_req_only_updates_ilm_expiry_flags() { + let mut state = SiteReplicationState::default(); + let mut remote = peer("remote", "https://remote.example.com"); + remote.deployment_id = "remote".to_string(); + remote.object_naming_mode = "uuid".to_string(); + state.peers.insert(remote.deployment_id.clone(), remote); + state.updated_at = Some(OffsetDateTime::UNIX_EPOCH); + + let edited = apply_state_edit_req( + state, + SRStateEditReq { + peers: BTreeMap::from([( + "remote".to_string(), + PeerInfo { + deployment_id: "remote".to_string(), + replicate_ilm_expiry: true, + object_naming_mode: "should-not-overwrite".to_string(), + ..peer("remote", "https://remote.example.com") + }, + )]), + updated_at: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(10)), + }, + ); + + assert!(edited.peers["remote"].replicate_ilm_expiry); + assert_eq!(edited.peers["remote"].object_naming_mode, "uuid"); + } + + #[test] + fn test_apply_state_edit_req_ignores_stale_updates() { + let mut state = SiteReplicationState::default(); + let mut remote = peer("remote", "https://remote.example.com"); + remote.deployment_id = "remote".to_string(); + state.peers.insert(remote.deployment_id.clone(), remote); + state.updated_at = Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(20)); + + let edited = apply_state_edit_req( + state.clone(), + SRStateEditReq { + peers: BTreeMap::from([( + "remote".to_string(), + PeerInfo { + deployment_id: "remote".to_string(), + replicate_ilm_expiry: true, + ..peer("remote", "https://remote.example.com") + }, + )]), + updated_at: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(10)), + }, + ); + + assert_eq!(edited.updated_at, state.updated_at); + assert!(!edited.peers["remote"].replicate_ilm_expiry); + } + + #[test] + fn test_filter_sr_info_keeps_only_requested_entity() { + let mut info = SRInfo::default(); + info.buckets.insert("photos".to_string(), SRBucketInfo::default()); + info.buckets.insert("logs".to_string(), SRBucketInfo::default()); + info.policies.insert("readonly".to_string(), SRIAMPolicy::default()); + + let filtered = filter_sr_info( + info, + &SRStatusOptions { + entity: SREntityType::Bucket, + entity_value: "photos".to_string(), + ..Default::default() + }, + ); + + assert!(filtered.buckets.contains_key("photos")); + assert!(!filtered.buckets.contains_key("logs")); + assert!(filtered.policies.is_empty()); + } + + #[test] + fn test_hash_client_secret_matches_minio_style_base64url_sha256() { + assert_eq!(hash_client_secret(Some("secret")), "K7gNU3sdo-OL0wNhqoVWhr3g6s1xYv72ol_pe_Unols"); + } + + #[test] + fn test_ldap_settings_from_kvs_reads_minio_style_keys() { + let kvs = rustfs_ecstore::config::KVS(vec![ + rustfs_ecstore::config::KV { + key: "enable".to_string(), + value: "on".to_string(), + hidden_if_empty: false, + }, + rustfs_ecstore::config::KV { + key: "user_dn_search_base_dn".to_string(), + value: "ou=people,dc=example,dc=com".to_string(), + hidden_if_empty: false, + }, + rustfs_ecstore::config::KV { + key: "user_dn_search_filter".to_string(), + value: "(uid=%s)".to_string(), + hidden_if_empty: false, + }, + rustfs_ecstore::config::KV { + key: "group_search_base_dn".to_string(), + value: "ou=groups,dc=example,dc=com".to_string(), + hidden_if_empty: false, + }, + rustfs_ecstore::config::KV { + key: "group_search_filter".to_string(), + value: "(&(objectclass=groupOfNames)(member=%s))".to_string(), + hidden_if_empty: false, + }, + ]); + + let (ldap, ldap_configs) = ldap_settings_from_kvs(&kvs); + + assert!(ldap.is_ldap_enabled); + assert_eq!(ldap.ldap_user_dn_search_base, "ou=people,dc=example,dc=com"); + assert_eq!(ldap.ldap_user_dn_search_filter, "(uid=%s)"); + assert_eq!(ldap.ldap_group_search_base, "ou=groups,dc=example,dc=com"); + assert_eq!(ldap.ldap_group_search_filter, "(&(objectclass=groupOfNames)(member=%s))"); + assert!(ldap_configs.enabled); + assert!(ldap_configs.configs.contains_key("default")); + } + + #[test] + fn test_gob_site_netperf_node_result_matches_go_encoding() { + let data = encode_go_gob_site_netperf_node_result(&SiteNetPerfNodeResult { + endpoint: "https://peer.example.com".to_string(), + tx: 123, + tx_total_duration_ns: 456, + rx: 789, + rx_total_duration_ns: 321, + total_conn: 3, + error: String::new(), + }); + + let expected: &[u8] = &[ + 0x7d, 0x7f, 0x03, 0x01, 0x01, 0x15, 0x53, 0x69, 0x74, 0x65, 0x4e, 0x65, 0x74, 0x50, 0x65, 0x72, 0x66, 0x4e, 0x6f, + 0x64, 0x65, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x01, 0xff, 0x80, 0x00, 0x01, 0x07, 0x01, 0x08, 0x45, 0x6e, 0x64, + 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x01, 0x0c, 0x00, 0x01, 0x02, 0x54, 0x58, 0x01, 0x06, 0x00, 0x01, 0x0f, 0x54, 0x58, + 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x01, 0x04, 0x00, 0x01, 0x02, 0x52, + 0x58, 0x01, 0x06, 0x00, 0x01, 0x0f, 0x52, 0x58, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x01, 0x04, 0x00, 0x01, 0x09, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x43, 0x6f, 0x6e, 0x6e, 0x01, 0x06, 0x00, + 0x01, 0x05, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x01, 0x0c, 0x00, 0x00, 0x00, 0x2d, 0xff, 0x80, 0x01, 0x18, 0x68, 0x74, + 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x70, 0x65, 0x65, 0x72, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x2e, + 0x63, 0x6f, 0x6d, 0x01, 0x7b, 0x01, 0xfe, 0x03, 0x90, 0x01, 0xfe, 0x03, 0x15, 0x01, 0xfe, 0x02, 0x82, 0x01, 0x03, + 0x00, + ]; + + assert_eq!(data, expected); + } +} diff --git a/rustfs/src/admin/handlers/sts.rs b/rustfs/src/admin/handlers/sts.rs index c36917568..447d10a18 100644 --- a/rustfs/src/admin/handlers/sts.rs +++ b/rustfs/src/admin/handlers/sts.rs @@ -14,7 +14,10 @@ use super::is_admin::IsAdminHandler; use crate::{ - admin::router::{AdminOperation, Operation, S3Router}, + admin::{ + handlers::site_replication::site_replication_iam_change_hook, + router::{AdminOperation, Operation, S3Router}, + }, auth::{check_key_valid, get_session_token}, server::ADMIN_PREFIX, }; @@ -23,8 +26,10 @@ use http::header::HeaderValue; use hyper::Method; use matchit::Params; use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; +use rustfs_credentials::get_global_action_cred; use rustfs_ecstore::bucket::utils::serialize; use rustfs_iam::{manager::get_token_signing_key, oidc::OidcClaims, sys::SESSION_POLICY_NAME}; +use rustfs_madmin::{SITE_REPL_API_VERSION, SRIAMItem, SRSTSCredential}; use rustfs_policy::{auth::get_new_credentials_with_metadata, policy::Policy}; use s3s::{ Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, @@ -164,11 +169,31 @@ async fn handle_assume_role( debug!("AssumeRole get new_cred {:?}", &new_cred); - if let Err(_err) = iam_store.set_temp_user(&new_cred.access_key, &new_cred, None).await { - return Err(s3_error!(InternalError, "set_temp_user failed")); - } + let updated_at = iam_store + .set_temp_user(&new_cred.access_key, &new_cred, None) + .await + .map_err(|_| s3_error!(InternalError, "set_temp_user failed"))?; - // TODO: globalSiteReplicationSys + let root_access_key = get_global_action_cred().map(|cred| cred.access_key); + if root_access_key.as_deref() != Some(new_cred.parent_user.as_str()) + && let Err(err) = site_replication_iam_change_hook(SRIAMItem { + r#type: "sts-credential".to_string(), + sts_credential: Some(SRSTSCredential { + access_key: new_cred.access_key.clone(), + secret_key: new_cred.secret_key.clone(), + session_token: new_cred.session_token.clone(), + parent_user: new_cred.parent_user.clone(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }), + updated_at: Some(updated_at), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }) + .await + { + warn!("site replication STS hook failed, err: {err}"); + } let resp = AssumeRoleOutput { credentials: Some(Credentials { @@ -358,11 +383,30 @@ pub async fn create_oidc_sts_credentials( // Store temp user in IAM let iam_store = rustfs_iam::get().map_err(|_| s3_error!(InternalError, "IAM not initialized"))?; - iam_store + let updated_at = iam_store .set_temp_user(&new_cred.access_key, &new_cred, None) .await .map_err(|_| s3_error!(InternalError, "failed to store temp user"))?; + if let Err(err) = site_replication_iam_change_hook(SRIAMItem { + r#type: "sts-credential".to_string(), + sts_credential: Some(SRSTSCredential { + access_key: new_cred.access_key.clone(), + secret_key: new_cred.secret_key.clone(), + session_token: new_cred.session_token.clone(), + parent_user: new_cred.parent_user.clone(), + parent_policy_mapping: policies.join(","), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }), + updated_at: Some(updated_at), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }) + .await + { + warn!("site replication OIDC STS hook failed, err: {err}"); + } + Ok(new_cred) } diff --git a/rustfs/src/admin/handlers/user.rs b/rustfs/src/admin/handlers/user.rs index bd0af3f8c..2f9a2bd7a 100644 --- a/rustfs/src/admin/handlers/user.rs +++ b/rustfs/src/admin/handlers/user.rs @@ -16,6 +16,7 @@ use super::{account_info, group, service_account, user_iam, user_lifecycle, user use crate::{ admin::{ auth::validate_admin_request, + handlers::site_replication::site_replication_iam_change_hook, router::{AdminOperation, Operation, S3Router}, utils::{encode_compatible_admin_payload, has_space_be, read_compatible_admin_body}, }, @@ -31,7 +32,8 @@ use rustfs_iam::{ sys::{NewServiceAccountOpts, UpdateServiceAccountOpts}, }; use rustfs_madmin::{ - AccountStatus, AddOrUpdateUserReq, IAMEntities, IAMErrEntities, IAMErrEntity, IAMErrPolicyEntity, + AccountStatus, AddOrUpdateUserReq, IAMEntities, IAMErrEntities, IAMErrEntity, IAMErrPolicyEntity, SITE_REPL_API_VERSION, + SRIAMItem, SRIAMUser, user::{ImportIAMResult, SRSessionPolicy, SRSvcAccCreate}, }; use rustfs_policy::policy::action::{Action, AdminAction}; @@ -225,11 +227,28 @@ impl Operation for AddUser { ) .await?; - iam_store + let updated_at = iam_store .create_user(ak, &args) .await .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("create_user err {e}")))?; + if let Err(err) = site_replication_iam_change_hook(SRIAMItem { + r#type: "iam-user".to_string(), + iam_user: Some(SRIAMUser { + access_key: ak.to_string(), + is_delete_req: false, + user_req: Some(args.clone()), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }), + updated_at: Some(updated_at), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }) + .await + { + warn!(access_key = %ak, error = ?err, "site replication create user hook failed"); + } + let mut header = HeaderMap::new(); header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); header.insert(CONTENT_LENGTH, "0".parse().unwrap()); @@ -432,7 +451,22 @@ impl Operation for RemoveUser { .await .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("delete_user err {e}")))?; - // TODO: IAMChangeHook + if let Err(err) = site_replication_iam_change_hook(SRIAMItem { + r#type: "iam-user".to_string(), + iam_user: Some(SRIAMUser { + access_key: ak.to_string(), + is_delete_req: true, + user_req: None, + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }), + updated_at: Some(time::OffsetDateTime::now_utc()), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }) + .await + { + warn!(access_key = %ak, error = ?err, "site replication delete user hook failed"); + } let mut header = HeaderMap::new(); header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); diff --git a/rustfs/src/admin/mod.rs b/rustfs/src/admin/mod.rs index d2bbaa5a5..bf54583ab 100644 --- a/rustfs/src/admin/mod.rs +++ b/rustfs/src/admin/mod.rs @@ -16,6 +16,7 @@ mod auth; pub mod console; pub mod handlers; pub mod router; +pub mod service; pub mod utils; #[cfg(test)] @@ -24,7 +25,8 @@ mod console_test; mod route_registration_test; use handlers::{ - bucket_meta, heal, health, kms, oidc, pools, profile_admin, quota, rebalance, replication, sts, system, tier, user, + bucket_meta, heal, health, kms, oidc, pools, profile_admin, quota, rebalance, replication, site_replication, sts, system, + tier, user, }; use router::{AdminOperation, S3Router}; use s3s::route::S3Route; @@ -55,6 +57,7 @@ pub fn make_admin_route(console_enabled: bool) -> std::io::Result bucket_meta::register_bucket_meta_route(&mut r)?; replication::register_replication_route(&mut r)?; + site_replication::register_site_replication_route(&mut r)?; profile_admin::register_profiling_route(&mut r)?; kms::register_kms_route(&mut r)?; oidc::register_oidc_route(&mut r)?; diff --git a/rustfs/src/admin/route_registration_test.rs b/rustfs/src/admin/route_registration_test.rs index 4ebdf8e7a..d58556e8e 100644 --- a/rustfs/src/admin/route_registration_test.rs +++ b/rustfs/src/admin/route_registration_test.rs @@ -14,7 +14,8 @@ use crate::admin::{ handlers::{ - bucket_meta, heal, health, kms, oidc, pools, profile_admin, quota, rebalance, replication, sts, system, tier, user, + bucket_meta, heal, health, kms, oidc, pools, profile_admin, quota, rebalance, replication, site_replication, sts, system, + tier, user, }, router::{AdminOperation, S3Router}, }; @@ -50,6 +51,7 @@ fn register_admin_routes(router: &mut S3Router) { quota::register_quota_route(router).expect("register quota route"); bucket_meta::register_bucket_meta_route(router).expect("register bucket meta route"); replication::register_replication_route(router).expect("register replication route"); + site_replication::register_site_replication_route(router).expect("register site replication route"); profile_admin::register_profiling_route(router).expect("register profile route"); kms::register_kms_route(router).expect("register kms route"); oidc::register_oidc_route(router).expect("register oidc route"); @@ -60,7 +62,6 @@ fn test_register_routes_cover_representative_admin_paths() { let mut router: S3Router = S3Router::new(false); register_admin_routes(&mut router); - assert_route(&router, Method::GET, HEALTH_PREFIX); assert_route(&router, Method::HEAD, HEALTH_PREFIX); assert_route(&router, Method::GET, HEALTH_READY_PATH); @@ -119,6 +120,23 @@ fn test_register_routes_cover_representative_admin_paths() { assert_route(&router, Method::PUT, &admin_path("/v3/import-bucket-metadata")); assert_route(&router, Method::GET, &admin_path("/v3/list-remote-targets")); assert_route(&router, Method::PUT, &admin_path("/v3/set-remote-target")); + assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/add")); + assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/remove")); + assert_route(&router, Method::GET, &admin_path("/v3/site-replication/info")); + assert_route(&router, Method::GET, &admin_path("/v3/site-replication/metainfo")); + assert_route(&router, Method::GET, &admin_path("/v3/site-replication/status")); + assert_route(&router, Method::POST, &admin_path("/v3/site-replication/devnull")); + assert_route(&router, Method::POST, &admin_path("/v3/site-replication/netperf")); + assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/peer/join")); + assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/peer/bucket-ops")); + assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/peer/iam-item")); + assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/peer/bucket-meta")); + assert_route(&router, Method::GET, &admin_path("/v3/site-replication/peer/idp-settings")); + assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/edit")); + assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/peer/edit")); + assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/peer/remove")); + assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/resync/op")); + assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/state/edit")); assert_route(&router, Method::GET, &admin_path("/debug/pprof/profile")); assert_route(&router, Method::POST, &admin_path("/v3/kms/create-key")); @@ -178,6 +196,10 @@ fn test_admin_alias_paths_match_existing_admin_routes() { (Method::GET, compat_admin_alias_path("/v3/oidc/callback/default")), (Method::GET, compat_admin_alias_path("/v3/oidc/config")), (Method::PUT, compat_admin_alias_path("/v3/oidc/config/default")), + (Method::PUT, compat_admin_alias_path("/v3/site-replication/add")), + (Method::GET, compat_admin_alias_path("/v3/site-replication/info")), + (Method::GET, compat_admin_alias_path("/v3/site-replication/status")), + (Method::PUT, compat_admin_alias_path("/v3/site-replication/peer/join")), (Method::GET, compat_admin_alias_path("/export-bucket-metadata")), (Method::GET, compat_admin_alias_path("/v3/export-bucket-metadata")), (Method::PUT, compat_admin_alias_path("/import-bucket-metadata")), diff --git a/rustfs/src/admin/service/mod.rs b/rustfs/src/admin/service/mod.rs new file mode 100644 index 000000000..4fc81d80f --- /dev/null +++ b/rustfs/src/admin/service/mod.rs @@ -0,0 +1,15 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod site_replication; diff --git a/rustfs/src/admin/service/site_replication.rs b/rustfs/src/admin/service/site_replication.rs new file mode 100644 index 000000000..b8e3c78b8 --- /dev/null +++ b/rustfs/src/admin/service/site_replication.rs @@ -0,0 +1,43 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use rustfs_ecstore::config::com::read_config; +use rustfs_ecstore::error::Error as StorageError; +use rustfs_ecstore::new_object_layer_fn; +use s3s::{S3Error, S3ErrorCode, S3Result}; + +const SITE_REPLICATION_STATE_PATH: &str = "config/site-replication/state.json"; + +/// Reload persisted site-replication state. +/// +/// RustFS does not currently keep a separate in-memory cache for this state, +/// so "reload" means validating that the persisted JSON is readable. +pub async fn reload_site_replication_runtime_state() -> S3Result<()> { + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + match read_config(store, SITE_REPLICATION_STATE_PATH).await { + Ok(data) => { + let _: serde_json::Value = serde_json::from_slice(&data) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("invalid site replication state: {e}")))?; + Ok(()) + } + Err(StorageError::ConfigNotFound) => Ok(()), + Err(err) => Err(S3Error::with_message( + S3ErrorCode::InternalError, + format!("failed to load site replication state: {err}"), + )), + } +} diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index 323563fbe..64e963f82 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -14,6 +14,9 @@ //! Bucket application use-case contracts. +use crate::admin::handlers::site_replication::{ + site_replication_bucket_meta_hook, site_replication_delete_bucket_hook, site_replication_make_bucket_hook, +}; use crate::app::context::{AppContext, default_notify_interface, get_global_app_context}; use crate::auth::get_condition_values; use crate::error::ApiError; @@ -49,6 +52,7 @@ use rustfs_ecstore::store_api::{ BucketOperations, BucketOptions, DeleteBucketOptions, ListObjectVersionsInfo, ListObjectsV2Info, ListOperations, MakeBucketOptions, ObjectInfo, }; +use rustfs_madmin::{SITE_REPL_API_VERSION, SRBucketMeta}; use rustfs_policy::policy::{ action::{Action, S3Action}, {BucketPolicy, BucketPolicyArgs, Effect, Validator}, @@ -81,6 +85,16 @@ fn to_internal_error(err: impl Display) -> S3Error { S3Error::with_message(S3ErrorCode::InternalError, format!("{err}")) } +fn sr_bucket_meta_item(bucket: String, item_type: &str) -> SRBucketMeta { + SRBucketMeta { + bucket, + r#type: item_type.to_string(), + updated_at: Some(time::OffsetDateTime::now_utc()), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + } +} + fn versioning_configuration_has_object_lock_incompatible_settings(config: &VersioningConfiguration) -> bool { config.suspended() || config.exclude_folders.unwrap_or(false) @@ -538,6 +552,7 @@ impl DefaultBucketUsecase { object_lock_enabled_for_bucket, .. } = req.input; + let lock_enabled = object_lock_enabled_for_bucket.is_some_and(|v| v); let Some(store) = new_object_layer_fn() else { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); @@ -548,7 +563,7 @@ impl DefaultBucketUsecase { &bucket, &MakeBucketOptions { force_create: false, - lock_enabled: object_lock_enabled_for_bucket.is_some_and(|v| v), + lock_enabled, ..Default::default() }, ) @@ -566,6 +581,10 @@ impl DefaultBucketUsecase { Err(e) => return Err(ApiError::from(e).into()), } + if let Err(err) = site_replication_make_bucket_hook(&bucket, lock_enabled).await { + warn!(bucket = %bucket, error = ?err, "site replication make bucket hook failed"); + } + let output = CreateBucketOutput::default(); counter!("rustfs_create_bucket_total").increment(1); let result = Ok(S3Response::new(output)); @@ -637,6 +656,10 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; + if let Err(err) = site_replication_delete_bucket_hook(&input.bucket, force).await { + warn!(bucket = %input.bucket, error = ?err, "site replication delete bucket hook failed"); + } + let result = Ok(S3Response::new(DeleteBucketOutput {})); let _ = helper.complete(&result); result @@ -791,6 +814,11 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; + let item = sr_bucket_meta_item(bucket.clone(), "sse-config"); + if let Err(err) = site_replication_bucket_meta_hook(item).await { + warn!(bucket = %bucket, error = ?err, "site replication bucket encryption delete hook failed"); + } + Ok(S3Response::with_status(DeleteBucketEncryptionOutput::default(), StatusCode::NO_CONTENT)) } @@ -818,6 +846,11 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; + let item = sr_bucket_meta_item(bucket.clone(), "cors-config"); + if let Err(err) = site_replication_bucket_meta_hook(item).await { + warn!(bucket = %bucket, error = ?err, "site replication bucket cors delete hook failed"); + } + Ok(S3Response::new(DeleteBucketCorsOutput {})) } @@ -845,6 +878,11 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; + let item = sr_bucket_meta_item(bucket.clone(), "lc-config"); + if let Err(err) = site_replication_bucket_meta_hook(item).await { + warn!(bucket = %bucket, error = ?err, "site replication bucket lifecycle delete hook failed"); + } + Ok(S3Response::new(DeleteBucketLifecycleOutput::default())) } @@ -871,6 +909,11 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; + let item = sr_bucket_meta_item(bucket.clone(), "policy"); + if let Err(err) = site_replication_bucket_meta_hook(item).await { + warn!(bucket = %bucket, error = ?err, "site replication bucket policy delete hook failed"); + } + Ok(S3Response::new(DeleteBucketPolicyOutput {})) } @@ -896,6 +939,11 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; + let item = sr_bucket_meta_item(bucket.clone(), "replication-config"); + if let Err(err) = site_replication_bucket_meta_hook(item).await { + warn!(bucket = %bucket, error = ?err, "site replication bucket replication-config delete hook failed"); + } + // TODO: remove targets info!(bucket = %bucket, "deleted bucket replication config"); @@ -917,6 +965,11 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; + let item = sr_bucket_meta_item(bucket.clone(), "tags"); + if let Err(err) = site_replication_bucket_meta_hook(item).await { + warn!(bucket = %bucket, error = ?err, "site replication bucket tagging delete hook failed"); + } + Ok(S3Response::new(tagging::build_delete_bucket_tagging_output())) } @@ -1374,6 +1427,15 @@ impl DefaultBucketUsecase { metadata_sys::update(&bucket, BUCKET_SSECONFIG, data) .await .map_err(ApiError::from)?; + + let mut item = sr_bucket_meta_item(bucket.clone(), "sse-config"); + item.sse_config = Some( + serialize_config(&server_side_encryption_configuration) + .and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?, + ); + if let Err(err) = site_replication_bucket_meta_hook(item).await { + warn!(bucket = %bucket, error = ?err, "site replication bucket encryption hook failed"); + } Ok(S3Response::new(encryption::build_put_bucket_encryption_output())) } @@ -1420,6 +1482,14 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; + let mut item = sr_bucket_meta_item(bucket.clone(), "lc-config"); + item.expiry_lc_config = + Some(serialize_config(&input_cfg).and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?); + item.expiry_updated_at = item.updated_at; + if let Err(err) = site_replication_bucket_meta_hook(item).await { + warn!(bucket = %bucket, error = ?err, "site replication bucket lifecycle hook failed"); + } + if lifecycle_has_transition_rules(&input_cfg) && let Some(store) = new_object_layer_fn() { @@ -1564,6 +1634,12 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; + let mut item = sr_bucket_meta_item(bucket.clone(), "policy"); + item.policy = Some(serde_json::from_str(&policy).map_err(|e| s3_error!(InvalidArgument, "parse policy failed {:?}", e))?); + if let Err(err) = site_replication_bucket_meta_hook(item).await { + warn!(bucket = %bucket, error = ?err, "site replication bucket policy hook failed"); + } + Ok(S3Response::new(PutBucketPolicyOutput {})) } @@ -1593,6 +1669,13 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; + let mut item = sr_bucket_meta_item(bucket.clone(), "cors-config"); + item.cors = + Some(serialize_config(&cors_configuration).and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?); + if let Err(err) = site_replication_bucket_meta_hook(item).await { + warn!(bucket = %bucket, error = ?err, "site replication bucket cors hook failed"); + } + Ok(S3Response::new(PutBucketCorsOutput::default())) } @@ -1626,6 +1709,14 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; + let mut item = sr_bucket_meta_item(bucket.clone(), "replication-config"); + item.replication_config = Some( + serialize_config(&replication_configuration).and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?, + ); + if let Err(err) = site_replication_bucket_meta_hook(item).await { + warn!(bucket = %bucket, error = ?err, "site replication bucket replication-config hook failed"); + } + Ok(S3Response::new(replication::build_put_bucket_replication_output())) } @@ -1687,6 +1778,12 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; + let mut item = sr_bucket_meta_item(bucket.clone(), "tags"); + item.tags = Some(serialize_config(&tagging).and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?); + if let Err(err) = site_replication_bucket_meta_hook(item).await { + warn!(bucket = %bucket, error = ?err, "site replication bucket tagging hook failed"); + } + Ok(S3Response::new(tagging::build_put_bucket_tagging_output())) } @@ -1713,6 +1810,14 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; + let mut item = sr_bucket_meta_item(bucket.clone(), "version-config"); + item.versioning = Some( + serialize_config(&versioning_configuration).and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?, + ); + if let Err(err) = site_replication_bucket_meta_hook(item).await { + warn!(bucket = %bucket, error = ?err, "site replication bucket versioning hook failed"); + } + Ok(S3Response::new(PutBucketVersioningOutput {})) } diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 360f65ed4..c0989a9e1 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::admin::service::site_replication::reload_site_replication_runtime_state; use bytes::Bytes; use futures::Stream; use futures_util::future::join_all; @@ -757,7 +758,16 @@ impl Node for NodeService { error_info: Some("errServerNotInitialized".to_string()), })); }; - todo!() + match reload_site_replication_runtime_state().await { + Ok(()) => Ok(Response::new(ReloadSiteReplicationConfigResponse { + success: true, + error_info: None, + })), + Err(err) => Ok(Response::new(ReloadSiteReplicationConfigResponse { + success: false, + error_info: Some(err.to_string()), + })), + } } async fn signal_service(&self, request: Request) -> Result, Status> {