diff --git a/rustfs/src/app/bucket_usecase/replication.rs b/rustfs/src/app/bucket_usecase/replication.rs index b9bc97d4d5..e99033273a 100644 --- a/rustfs/src/app/bucket_usecase/replication.rs +++ b/rustfs/src/app/bucket_usecase/replication.rs @@ -15,11 +15,8 @@ //! Bucket replication configuration use-case operations. use super::*; -use crate::app::storage_api::s3::dto::{ - DeleteBucketReplicationInput, DeleteBucketReplicationOutput, PutBucketReplicationInput, PutBucketReplicationOutput, -}; -use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; use rustfs_gateway_types::persistence::parse_replication_dto; +use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result, s3_error}; fn validate_replication_config_targets(targets: &BucketTargets, config: &PersistedReplicationConfiguration) -> S3Result<()> { let configured_arns = targets @@ -168,6 +165,15 @@ async fn write_replication_targets_after_config_delete( removed: usize, expected_incarnation_id: Option, ) -> S3Result<()> { + #[cfg(test)] + if let Some(fail) = tests::TARGET_CLEANUP_FAILURE + .try_with(std::cell::RefCell::take) + .ok() + .flatten() + { + fail.await?; + } + let json_targets = serde_json::to_vec(&targets).map_err(to_internal_error)?; update_bucket_config_for_incarnation(bucket, BUCKET_TARGETS_FILE, json_targets, expected_incarnation_id) .await @@ -210,38 +216,56 @@ async fn restore_replication_config_after_target_cleanup_failure( } impl DefaultBucketUsecase { - /// `site_peers` is the set of remote site-replication peer deployment ids - /// (empty when site replication is disabled). The interface layer reads it - /// from the persisted state and fails closed on a read error, so this - /// usecase stays a pure function of its inputs (layer rule: app never - /// imports interface). - pub async fn execute_delete_bucket_replication( + /// Removes operator rules and targets unique to those rules. + /// + /// Preparation runs inside the audit scope and resolves the peer contract + /// before any bucket lookup or mutation. + pub(crate) async fn execute_delete_bucket_replication( &self, - req: S3Request, + envelope: RequestEnvelope, + bucket: &str, + prepare: impl Future, OperatorRuleContract)>> + Send, + ) -> S3Result<()> { + // Preserve the success status recorded by the legacy audit for an empty response. + RequestEnvelope::audit_only_operation( + envelope, + S3Operation::DeleteBucketReplication, + StatusCode::OK, + |envelope| async move { + let (site_peers, contract) = prepare.await?; + self.delete_bucket_replication(envelope, bucket, site_peers, contract).await + }, + ) + .await + } + + async fn delete_bucket_replication( + &self, + envelope: RequestEnvelope, + bucket: &str, site_peers: HashSet, contract: OperatorRuleContract, - ) -> S3Result> { - let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; - let request_context = req.extensions.get::().cloned(); - let DeleteBucketReplicationInput { bucket, .. } = req.input; + ) -> S3Result<()> { + let expected_incarnation_id = bucket_config_mutation_incarnation(&envelope, bucket)?; + let request_context = envelope.request_context().cloned(); let Some(store) = self.object_store() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init")); }; store - .get_bucket_info(&bucket, &BucketOptions::default()) + .get_bucket_info(bucket, &BucketOptions::default()) .await .map_err(ApiError::from)?; - let targets_guard = lock_bucket_targets_metadata(&bucket).await; - let replication_config = match metadata_sys::get_replication_config(&bucket).await { + let targets_guard = lock_bucket_targets_metadata(bucket).await; + let replication_config = match metadata_sys::get_replication_config(bucket).await { Ok((config, _)) => Some(config), Err(StorageError::ConfigNotFound) => None, Err(err) => return Err(ApiError::from(err).into()), }; let (remaining_config, updated_targets) = if let Some(config) = replication_config.as_ref() { let (remaining, removable_arns) = split_replication_config_for_user_delete(config.clone(), &site_peers, contract); - let targets = replication_targets_without_arns(&bucket, &removable_arns).await?; + let targets = replication_targets_without_arns(bucket, &removable_arns).await?; (remaining, targets) } else { (None, None) @@ -252,23 +276,23 @@ impl DefaultBucketUsecase { // S3 delete (issue #1948); only the operator-authored rules go. Some(remaining) => { let data = serialize_config(&remaining)?; - update_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id) + update_bucket_config_for_incarnation(bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id) .await .map_err(ApiError::from)?; } None => { - delete_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, expected_incarnation_id) + delete_bucket_config_for_incarnation(bucket, BUCKET_REPLICATION_CONFIG, expected_incarnation_id) .await .map_err(ApiError::from)?; } } if let Some((targets, removed)) = updated_targets && let Err(err) = - write_replication_targets_after_config_delete(&bucket, &targets, removed, expected_incarnation_id).await + write_replication_targets_after_config_delete(bucket, &targets, removed, expected_incarnation_id).await { if let Some(config) = replication_config.as_ref() { return Err(restore_replication_config_after_target_cleanup_failure( - &bucket, + bucket, config, err, expected_incarnation_id, @@ -279,16 +303,16 @@ impl DefaultBucketUsecase { } drop(targets_guard); - notify_bucket_metadata_reload(bucket.clone(), "delete bucket replication", request_context, true).await; + notify_bucket_metadata_reload(bucket.to_owned(), "delete bucket replication", request_context, true).await; - let item = sr_bucket_meta_item(bucket.clone(), "replication-config"); + let item = sr_bucket_meta_item(bucket.to_owned(), "replication-config"); if let Err(err) = site_replication_bucket_meta_hook(item).await { warn!(bucket = %bucket, error = ?err, "site replication bucket replication-config delete hook failed"); } info!(bucket = %bucket, "deleted bucket replication config"); - Ok(S3Response::new(DeleteBucketReplicationOutput::default())) + Ok(()) } /// The stored replication configuration as the gateway's answer holds it. @@ -335,36 +359,52 @@ impl DefaultBucketUsecase { }) } - /// See [`Self::execute_delete_bucket_replication`] for `site_peers` and - /// `contract`. `replication_configuration` is the request body in the - /// engine's shape: the s3s edge (`storage::ecfs`) converts it, and this use - /// case never reads `req.input.replication_configuration`. - pub async fn execute_put_bucket_replication( + /// Replaces operator rules after the interface resolves the peer contract. + pub(crate) async fn execute_put_bucket_replication( &self, - req: S3Request, + envelope: RequestEnvelope, + bucket: &str, + prepare: impl Future, OperatorRuleContract)>> + Send, + ) -> S3Result<()> { + RequestEnvelope::audit_only_operation( + envelope, + S3Operation::PutBucketReplication, + StatusCode::OK, + |envelope| async move { + let (replication_configuration, site_peers, contract) = prepare.await?; + self.put_bucket_replication(envelope, bucket, replication_configuration, site_peers, contract) + .await + }, + ) + .await + } + + async fn put_bucket_replication( + &self, + envelope: RequestEnvelope, + bucket: &str, replication_configuration: PersistedReplicationConfiguration, site_peers: HashSet, contract: OperatorRuleContract, - ) -> S3Result> { - let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; - let request_context = req.extensions.get::().cloned(); - let PutBucketReplicationInput { bucket, .. } = req.input; + ) -> S3Result<()> { + let expected_incarnation_id = bucket_config_mutation_incarnation(&envelope, bucket)?; + let request_context = envelope.request_context().cloned(); info!(bucket = %bucket, "updating bucket replication config"); validate_replication_config_capabilities(&replication_configuration)?; let Some(store) = self.object_store() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init")); }; store - .get_bucket_info(&bucket, &BucketOptions::default()) + .get_bucket_info(bucket, &BucketOptions::default()) .await .map_err(ApiError::from)?; - let targets_guard = lock_bucket_targets_metadata(&bucket).await; - validate_bucket_replication_update(&bucket, &replication_configuration).await?; - let existing_config = match metadata_sys::get_replication_config(&bucket).await { + let targets_guard = lock_bucket_targets_metadata(bucket).await; + validate_bucket_replication_update(bucket, &replication_configuration).await?; + let existing_config = match metadata_sys::get_replication_config(bucket).await { Ok((config, _)) => Some(config), Err(StorageError::ConfigNotFound) => None, Err(err) => return Err(ApiError::from(err).into()), @@ -372,14 +412,14 @@ impl DefaultBucketUsecase { let replication_configuration = merge_user_replication_config_update(replication_configuration, existing_config, &site_peers, contract); let data = serialize_config(&replication_configuration)?; - update_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id) + update_bucket_config_for_incarnation(bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id) .await .map_err(ApiError::from)?; drop(targets_guard); - notify_bucket_metadata_reload(bucket.clone(), "put bucket replication", request_context, true).await; + notify_bucket_metadata_reload(bucket.to_owned(), "put bucket replication", request_context, true).await; - let mut item = sr_bucket_meta_item(bucket.clone(), "replication-config"); + let mut item = sr_bucket_meta_item(bucket.to_owned(), "replication-config"); item.replication_config = Some( serialize_config(&replication_configuration).and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?, ); @@ -387,19 +427,29 @@ impl DefaultBucketUsecase { warn!(bucket = %bucket, error = ?err, "site replication bucket replication-config hook failed"); } - Ok(S3Response::new(PutBucketReplicationOutput::default())) + Ok(()) } } #[cfg(test)] mod tests { use super::*; - use crate::app::bucket_usecase::test_support::build_request; + use crate::app::bucket_usecase::test_support::{build_request, owned_envelope}; + use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient}; + use crate::app::runtime_sources::ServerContextSlot; + use crate::app::storage_api::owned_parity::{ReqInfo, authorize_bucket_config_mutation}; + use crate::site_replication::hooks::{BUCKET_META_HOOK_CALLS, BucketMetaHookCall}; + use futures::{FutureExt, future::BoxFuture}; use http::Method; + use rustfs_credentials::Credentials; use rustfs_gateway_types::dto::Status; use rustfs_gateway_types::persistence::{ PersistedEncryptionConfiguration, PersistedReplicationDestination, PersistedReplicationRule, }; + use rustfs_policy::policy::action::{Action, S3Action}; + use serial_test::serial; + use std::cell::RefCell; + use std::sync::Mutex; fn replication_rule_for_target(arn: &str) -> PersistedReplicationRule { PersistedReplicationRule { @@ -834,16 +884,11 @@ mod tests { #[tokio::test] async fn execute_delete_bucket_replication_returns_internal_error_when_store_uninitialized() { - let input = DeleteBucketReplicationInput::builder() - .bucket("test-bucket".to_string()) - .build() - .unwrap(); - - let req = build_request(input, Method::DELETE); let usecase = DefaultBucketUsecase::without_context(); - let err = usecase - .execute_delete_bucket_replication(req, HashSet::new(), OperatorRuleContract::Derived) + .execute_delete_bucket_replication(owned_envelope(Method::DELETE), "test-bucket", async { + Ok((HashSet::new(), OperatorRuleContract::Derived)) + }) .await .unwrap_err(); assert_eq!(err.code(), &S3ErrorCode::InternalError); @@ -869,17 +914,11 @@ mod tests { role: "arn:aws:iam::123456789012:role/test".to_string(), rules: vec![replication_rule_for_target("arn:rustfs:replication:us-east-1:target:bucket")], }; - let input = PutBucketReplicationInput::builder() - .bucket("test-bucket".to_string()) - .replication_configuration(Default::default()) - .build() - .unwrap(); - - let req = build_request(input, Method::PUT); let usecase = DefaultBucketUsecase::without_context(); - let err = usecase - .execute_put_bucket_replication(req, configuration, HashSet::new(), OperatorRuleContract::Derived) + .execute_put_bucket_replication(owned_envelope(Method::PUT), "test-bucket", async { + Ok((configuration, HashSet::new(), OperatorRuleContract::Derived)) + }) .await .unwrap_err(); assert_eq!(err.code(), &S3ErrorCode::InternalError); @@ -893,19 +932,10 @@ mod tests { role: String::new(), rules: vec![rule], }; - let input = PutBucketReplicationInput::builder() - .bucket("test-bucket".to_string()) - .replication_configuration(Default::default()) - .build() - .unwrap(); - let err = DefaultBucketUsecase::without_context() - .execute_put_bucket_replication( - build_request(input, Method::PUT), - configuration, - HashSet::new(), - OperatorRuleContract::Derived, - ) + .execute_put_bucket_replication(owned_envelope(Method::PUT), "test-bucket", async { + Ok((configuration, HashSet::new(), OperatorRuleContract::Derived)) + }) .await .expect_err("unsupported fields must be rejected before store access"); @@ -915,4 +945,316 @@ mod tests { Some("replication field Destination.Account is not supported by this RustFS version") ); } + + const PEER_ARN: &str = "arn:rustfs:replication::peer-dep:bucket"; + const USER_ARN: &str = "arn:rustfs:replication:us-east-1:user:bucket"; + const SITE_RULE: &str = "site-repl-peer-dep"; + + tokio::task_local! { + pub(super) static TARGET_CLEANUP_FAILURE: RefCell>>>; + } + + fn put_configuration() -> PersistedReplicationConfiguration { + PersistedReplicationConfiguration { + role: String::new(), + rules: vec![ + replication_rule_with_id(USER_ARN, "new-operator", 3), + replication_rule_with_id(PEER_ARN, "shared-peer-target", 4), + ], + } + } + + async fn seed_replication(store: &ECStore, bucket: &str, operator: &str) -> (Vec, Vec) { + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket for replication"); + metadata_sys::update( + bucket, + BUCKET_VERSIONING_CONFIG, + b"Enabled".to_vec(), + ) + .await + .expect("enable versioning"); + let config = PersistedReplicationConfiguration { + role: String::new(), + rules: vec![ + replication_rule_with_id(USER_ARN, operator, 1), + replication_rule_with_id(PEER_ARN, SITE_RULE, 2), + ], + }; + let targets = BucketTargets { + targets: [PEER_ARN, USER_ARN] + .into_iter() + .map(|arn| BucketTarget { + source_bucket: bucket.to_owned(), + arn: arn.to_owned(), + target_type: BucketTargetType::ReplicationService, + ..Default::default() + }) + .collect(), + }; + let bytes = serialize(&config).expect("seed configuration bytes"); + let targets = serde_json::to_vec(&targets).expect("seed target bytes"); + metadata_sys::update(bucket, BUCKET_TARGETS_FILE, targets.clone()) + .await + .expect("seed target metadata"); + metadata_sys::update(bucket, BUCKET_REPLICATION_CONFIG, bytes.clone()) + .await + .expect("seed replication configuration"); + (bytes, targets) + } + + async fn persisted_replication(bucket: &str) -> (Vec, Vec) { + let stored = metadata_sys::get_global_bucket_metadata_sys() + .expect("initialized metadata system") + .read() + .await + .get_config_from_disk(bucket) + .await + .expect("disk metadata"); + (stored.replication_config_xml, stored.bucket_targets_config_json) + } + + async fn authorize_replication(context: &Arc, bucket: &str, method: Method) -> RequestEnvelope { + let server = ServerContextSlot::new(); + assert!(server.install(Arc::clone(context))); + let mut request = build_request((), method); + request.uri = format!("/{bucket}?replication").parse().expect("replication URI"); + request.extensions.insert(Arc::clone(&server)); + request.extensions.insert(ReqInfo { + cred: Some(Credentials { + access_key: "replication-owner".to_owned(), + ..Default::default() + }), + is_owner: true, + ..Default::default() + }); + let mut envelope = RequestEnvelope::from_s3s(&request); + authorize_bucket_config_mutation( + &server, + &mut envelope, + bucket, + Action::S3Action(S3Action::PutReplicationConfigurationAction), + ) + .await + .expect("snapshot authorized bucket"); + envelope + } + + async fn edit_replication( + usecase: &DefaultBucketUsecase, + envelope: RequestEnvelope, + bucket: &str, + put: bool, + ) -> S3Result<()> { + let peers = HashSet::from(["peer-dep".to_owned()]); + if put { + usecase + .execute_put_bucket_replication(envelope, bucket, async { + Ok((put_configuration(), peers, OperatorRuleContract::Derived)) + }) + .await + } else { + usecase + .execute_delete_bucket_replication(envelope, bucket, async { Ok((peers, OperatorRuleContract::Derived)) }) + .await + } + } + + #[derive(Debug, PartialEq, Eq)] + struct Reload { + completed: bool, + maintenance: bool, + target_lock_available: bool, + hook_calls: usize, + } + + async fn observe_replication_edit( + bucket: &str, + edit: impl Future>, + ) -> (S3Result<()>, Vec, Vec) { + let reloads = Arc::new(Mutex::new(Vec::new())); + let record = Arc::clone(&reloads); + let target_bucket = bucket.to_owned(); + let (result, hooks) = BUCKET_META_HOOK_CALLS + .scope( + RefCell::default(), + BUCKET_METADATA_RELOAD_PROBE.scope( + ( + bucket.to_owned(), + Box::new(move |completed, _, maintenance| { + record.lock().expect("reloads").push(Reload { + completed, + maintenance, + target_lock_available: lock_bucket_targets_metadata(&target_bucket).now_or_never().is_some(), + hook_calls: BUCKET_META_HOOK_CALLS.with(|calls| calls.borrow().len()), + }); + }), + ), + async { + let result = edit.await; + (result, BUCKET_META_HOOK_CALLS.with(RefCell::take)) + }, + ), + ) + .await; + let reloads = std::mem::take(&mut *reloads.lock().expect("reloads")); + (result, reloads, hooks) + } + + #[test] + #[serial] + fn owned_replication_preserves_peer_rules_and_finishes_reload_before_site_hook() { + run_large_stack_test("owned-replication-peer-rules", || async { + let context = shared_gating_ambient().await; + let bucket = format!("replication-peer-{}", uuid::Uuid::new_v4().simple()); + let (original, _) = seed_replication(&context.object_store(), &bucket, "old-operator").await; + let usecase = DefaultBucketUsecase::with_context(Some(Arc::clone(&context))); + for (put, method) in [(true, Method::PUT), (false, Method::DELETE)] { + let envelope = authorize_replication(&context, &bucket, method).await; + let (result, reloads, hooks) = + observe_replication_edit(&bucket, edit_replication(&usecase, envelope, &bucket, put)).await; + result.expect("replication edit"); + assert_eq!( + reloads, + [ + Reload { + completed: false, + maintenance: true, + target_lock_available: true, + hook_calls: 0 + }, + Reload { + completed: true, + maintenance: true, + target_lock_available: true, + hook_calls: 0 + }, + ] + ); + + let (xml, target_bytes) = persisted_replication(&bucket).await; + assert_ne!(xml, original); + let config = rustfs_gateway_types::persistence::parse_replication(&xml).expect("stored XML"); + let ids: HashSet<_> = config.rules.iter().filter_map(|rule| rule.id.as_deref()).collect(); + let targets: BucketTargets = serde_json::from_slice(&target_bytes).expect("stored targets"); + let arns: HashSet<_> = targets.targets.iter().map(|target| target.arn.as_str()).collect(); + if put { + assert_eq!(ids, HashSet::from([SITE_RULE, "new-operator", "shared-peer-target"])); + assert_eq!(arns, HashSet::from([PEER_ARN, USER_ARN])); + } else { + assert_eq!(ids, HashSet::from([SITE_RULE])); + assert_eq!(arns, HashSet::from([PEER_ARN])); + } + assert_eq!(hooks.len(), 1); + assert_eq!( + (hooks[0].bucket.as_str(), hooks[0].item_type.as_str()), + (bucket.as_str(), "replication-config") + ); + assert_eq!(hooks[0].replication_config.as_deref().map(str::as_bytes), put.then_some(xml.as_slice())); + } + }); + } + + #[test] + #[serial] + fn owned_replication_stale_edits_leave_replacement_bytes_and_skip_callbacks() { + run_large_stack_test("owned-replication-stale", || async { + let context = shared_gating_ambient().await; + let store = context.object_store(); + let usecase = DefaultBucketUsecase::with_context(Some(Arc::clone(&context))); + for (method, preserve_peer) in [(Method::PUT, true), (Method::DELETE, true), (Method::DELETE, false)] { + let bucket = format!("replication-stale-{}", uuid::Uuid::new_v4().simple()); + seed_replication(&store, &bucket, "old-operator").await; + let put = method == Method::PUT; + let envelope = authorize_replication(&context, &bucket, method).await; + store + .delete_bucket(&bucket, &DeleteBucketOptions::default()) + .await + .expect("delete authorized bucket"); + let replacement = seed_replication(&store, &bucket, "replacement").await; + let edit = async { + if preserve_peer { + edit_replication(&usecase, envelope, &bucket, put).await + } else { + usecase + .execute_delete_bucket_replication(envelope, &bucket, async { + Ok((HashSet::new(), OperatorRuleContract::Derived)) + }) + .await + } + }; + let (result, reloads, hooks) = observe_replication_edit(&bucket, edit).await; + assert_eq!( + ( + result.err().map(|err| err.code().clone()), + persisted_replication(&bucket).await, + reloads.len(), + hooks.len() + ), + (Some(S3ErrorCode::NoSuchBucket), replacement, 0, 0), + "stale {}", + if put { "Put" } else { "Delete" } + ); + } + }); + } + + #[test] + #[serial] + fn owned_replication_cleanup_error_restores_original_and_never_overwrites_a_replacement() { + run_large_stack_test("owned-replication-cleanup", || async { + let context = shared_gating_ambient().await; + let store = context.object_store(); + let usecase = DefaultBucketUsecase::with_context(Some(Arc::clone(&context))); + for (replace, fail_target_write) in [(false, true), (true, true), (true, false)] { + let bucket = format!("replication-cleanup-{}", uuid::Uuid::new_v4().simple()); + let original = seed_replication(&store, &bucket, "old-operator").await; + let envelope = authorize_replication(&context, &bucket, Method::DELETE).await; + let expected_storage = Arc::new(Mutex::new(original)); + let output = Arc::clone(&expected_storage); + let replacement_bucket = bucket.clone(); + let old_store = Arc::clone(&store); + let fail = Box::pin(async move { + let (after_delete, _) = persisted_replication(&replacement_bucket).await; + assert!(!String::from_utf8_lossy(&after_delete).contains("old-operator")); + assert!(lock_bucket_targets_metadata(&replacement_bucket).now_or_never().is_none()); + if replace { + old_store + .delete_bucket(&replacement_bucket, &DeleteBucketOptions::default()) + .await + .expect("delete original before restoration"); + let replacement = seed_replication(&old_store, &replacement_bucket, "replacement").await; + *output.lock().expect("replacement bytes") = replacement; + } + if fail_target_write { + Err(s3_error!(InternalError, "replication target cleanup failed")) + } else { + Ok(()) + } + }); + let (result, reloads, hooks) = TARGET_CLEANUP_FAILURE + .scope( + RefCell::new(Some(fail)), + observe_replication_edit(&bucket, edit_replication(&usecase, envelope, &bucket, false)), + ) + .await; + let expected = if fail_target_write { + S3ErrorCode::InternalError + } else { + S3ErrorCode::NoSuchBucket + }; + let stored = persisted_replication(&bucket).await; + assert_eq!( + (result.as_ref().err().map(|error| error.code().clone()), stored), + (Some(expected), expected_storage.lock().expect("expected bytes").clone()) + ); + if fail_target_write { + assert_eq!(result.unwrap_err().message(), Some("replication target cleanup failed")); + } + assert!(reloads.is_empty() && hooks.is_empty()); + } + }); + } } diff --git a/rustfs/src/server/gateway_handlers/bucket_replication_lifecycle.rs b/rustfs/src/server/gateway_handlers/bucket_replication_lifecycle.rs index 5fed603fb1..2d7e23b506 100644 --- a/rustfs/src/server/gateway_handlers/bucket_replication_lifecycle.rs +++ b/rustfs/src/server/gateway_handlers/bucket_replication_lifecycle.rs @@ -20,8 +20,11 @@ use super::{OpRow, Served, row}; use crate::app::RequestEnvelope; use crate::app::bucket_usecase::DefaultBucketUsecase; use crate::runtime_sources::AppContext; -use crate::storage_api::server::http::access::authorize_bucket_request; +use crate::storage_api::server::http::access::{ + access_owned::bucket_replication_lifecycle::prepare_replication_config_edit, authorize_bucket_request, +}; use rustfs_gateway::{Req, Resp, dto}; +use rustfs_gateway_types::persistence::{parse_replication, serialize_replication_dto}; use rustfs_policy::policy::action::{Action, S3Action}; use rustfs_s3_ops::S3Operation; use rustfs_s3_types::S3Result; @@ -32,6 +35,8 @@ pub(super) const ROWS: &[OpRow] = &[ row::(), row::(), row::(), + row::(), + row::(), row::(), ]; @@ -67,6 +72,47 @@ impl Served for dto::GetBucketReplication { } } +impl Served for dto::PutBucketReplication { + const METRIC: S3Operation = S3Operation::PutBucketReplication; + + async fn authorize(envelope: &mut RequestEnvelope, request: &Req) -> S3Result<()> { + let action = Action::S3Action(S3Action::PutReplicationConfigurationAction); + authorize_bucket_config_mutation(envelope, request.input().bucket.as_str(), action).await + } + + async fn execute(context: Option>, envelope: RequestEnvelope, request: Req) -> S3Result> { + let input = request.into_input(); + let prepare = prepare_replication_config_edit(&envelope); + DefaultBucketUsecase::with_context(context) + .execute_put_bucket_replication(envelope, input.bucket.as_str(), async move { + let (site_peers, contract) = prepare.await?; + let bytes = serialize_replication_dto(&input.replication_configuration) + .map_err(|err| rustfs_s3_types::s3_error!(MalformedXML, "{err}"))?; + let configuration = parse_replication(&bytes).map_err(|err| rustfs_s3_types::s3_error!(MalformedXML, "{err}"))?; + Ok((configuration, site_peers, contract)) + }) + .await?; + Ok(Resp::new(dto::PutBucketReplicationOutput::default())) + } +} + +impl Served for dto::DeleteBucketReplication { + const METRIC: S3Operation = S3Operation::DeleteBucketReplication; + + async fn authorize(envelope: &mut RequestEnvelope, request: &Req) -> S3Result<()> { + let action = Action::S3Action(S3Action::PutReplicationConfigurationAction); + authorize_bucket_config_mutation(envelope, request.input().bucket.as_str(), action).await + } + + async fn execute(context: Option>, envelope: RequestEnvelope, request: Req) -> S3Result> { + let prepare = prepare_replication_config_edit(&envelope); + DefaultBucketUsecase::with_context(context) + .execute_delete_bucket_replication(envelope, request.input().bucket.as_str(), prepare) + .await?; + Ok(Resp::new(dto::DeleteBucketReplicationOutput::default())) + } +} + impl Served for dto::GetObjectLockConfiguration { const METRIC: S3Operation = S3Operation::GetObjectLockConfiguration; diff --git a/rustfs/src/site_replication/hooks.rs b/rustfs/src/site_replication/hooks.rs index 4ff248ea72..f4dc8c12cb 100644 --- a/rustfs/src/site_replication/hooks.rs +++ b/rustfs/src/site_replication/hooks.rs @@ -1007,6 +1007,7 @@ pub(crate) struct BucketMetaHookCall { pub(crate) bucket: String, pub(crate) item_type: String, pub(crate) scanner_generation: Option, + pub(crate) replication_config: Option, pub(crate) scanner_maintenance_generation: u64, } @@ -1021,6 +1022,7 @@ pub async fn site_replication_bucket_meta_hook(mut item: SRBucketMeta) -> S3Resu calls.borrow_mut().push(BucketMetaHookCall { bucket: item.bucket.clone(), item_type: item.r#type.clone(), + replication_config: item.replication_config.clone(), scanner_generation: rustfs_scanner::scanner_dirty_usage_bucket_generation(&item.bucket), scanner_maintenance_generation: rustfs_scanner::scanner_maintenance_generation(), }); diff --git a/rustfs/src/storage/access_owned/bucket_replication_lifecycle.rs b/rustfs/src/storage/access_owned/bucket_replication_lifecycle.rs index 4e80975e28..7dce046239 100644 --- a/rustfs/src/storage/access_owned/bucket_replication_lifecycle.rs +++ b/rustfs/src/storage/access_owned/bucket_replication_lifecycle.rs @@ -12,11 +12,82 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Authorization shapes that only migration batch A3 (rustfs/backlog#2765) uses: bucket lifecycle, replication and object lock configuration. +//! Prepares external S3 replication edits after IAM authorization. //! -//! Responsible for: the owned authorization function of each such operation, generic over -//! `AuthorizationRequest`, so the legacy `S3Access` hook and `Served::authorize` call the same code. -//! Not responsible for: shapes more than one batch shares (those stay in `storage/access.rs`), the -//! policy evaluation itself (`authorize_request`), or the operation's use case. -//! Upstream: the legacy hook in `storage/access.rs` and the batch's `server/gateway_handlers/bucket_replication_lifecycle.rs`. -//! Downstream: the shared shapes in `storage/access.rs`. +//! The interface resolves owner privileges and the site contract. The bucket +//! use case receives those facts without consulting site membership itself. + +use crate::storage::access::{AuthorizationRequest, req_info_ref}; +use crate::storage::storage_api::ecstore_bucket::replication::OperatorRuleContract; +use http::StatusCode; +use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result}; +use std::collections::HashSet; +use std::future::Future; + +#[cfg(test)] +pub(crate) static SITE_REPLICATION_GATE_TEST_OVERRIDE: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0); + +#[cfg(test)] +pub(crate) const SITE_REPLICATION_GATE_FORCE_DISABLED: u8 = 1; + +#[cfg(test)] +pub(crate) const SITE_REPLICATION_GATE_FORCE_ENABLED: u8 = 2; + +async fn site_replication_gate_enabled() -> S3Result { + #[cfg(test)] + match SITE_REPLICATION_GATE_TEST_OVERRIDE.load(std::sync::atomic::Ordering::SeqCst) { + SITE_REPLICATION_GATE_FORCE_DISABLED => return Ok(false), + SITE_REPLICATION_GATE_FORCE_ENABLED => return Ok(true), + _ => {} + } + crate::admin::handlers::site_replication::site_replication_enabled().await +} + +/// Remote site-replication peer deployment ids and the operator-rule contract +/// the peers support, handed to the bucket usecase so an S3 replication-config +/// edit keeps exactly the reconciler-owned rules and merges the way every +/// peer will (issue #1948). Read here, in the interface layer, because the +/// usecase must not import the admin handlers (layer guard); a state-read +/// failure propagates so the edit fails closed. +async fn site_replication_edit_context() -> S3Result<(HashSet, OperatorRuleContract)> { + // While the gate override is in effect the test exercises the deny/allow + // branch, not the peer set; there is no persisted state to read. + #[cfg(test)] + if SITE_REPLICATION_GATE_TEST_OVERRIDE.load(std::sync::atomic::Ordering::SeqCst) != 0 { + return Ok((HashSet::new(), OperatorRuleContract::Derived)); + } + crate::admin::handlers::site_replication::site_replication_edit_context().await +} + +/// MinIO `ErrReplicationDenyEditError`. +fn replication_deny_edit_error() -> S3Error { + let mut err = S3Error::with_message( + S3ErrorCode::Custom("XMinioReplicationDenyEdit".into()), + "Sub-User is not allowed to edit Replication configuration", + ); + err.set_status_code(StatusCode::BAD_REQUEST); + err +} + +/// Site-replication gate for S3 replication-config edits (issue #1948). +/// +/// On a site-replication deployment the bucket's replication config carries +/// the operator-managed `site-repl-*` rules that keep every peer in sync, and +/// a successful edit is broadcast to all peers — so a user holding only +/// bucket-scoped `s3:PutReplicationConfiguration` could rewrite or erase +/// replication net-wide. MinIO parity (`ErrReplicationDenyEditError`): only +/// owner credentials (root or root-parented) may edit. Runs after the policy +/// authorization in the access layer and only on the external S3 path — the +/// reconciler and peer bucket-meta ingestion never route through these +/// handlers. +pub(crate) fn prepare_replication_config_edit( + request: &R, +) -> impl Future, OperatorRuleContract)>> + Send + use { + let is_owner = req_info_ref(request).map(|info| info.is_owner).map_err(S3Error::from); + async move { + if !is_owner? && site_replication_gate_enabled().await? { + return Err(replication_deny_edit_error()); + } + site_replication_edit_context().await + } +} diff --git a/rustfs/src/storage/legacy_edge/bucket_replication_lifecycle.rs b/rustfs/src/storage/legacy_edge/bucket_replication_lifecycle.rs index d085902a58..f5e2068ab4 100644 --- a/rustfs/src/storage/legacy_edge/bucket_replication_lifecycle.rs +++ b/rustfs/src/storage/legacy_edge/bucket_replication_lifecycle.rs @@ -21,9 +21,9 @@ use crate::admin::handlers::site_replication::site_replication_bucket_meta_hook; use crate::app::RequestEnvelope; -use crate::app::storage_api::object_usecase::bucket::replication::OperatorRuleContract; use crate::error::ApiError; use crate::storage::StorageVersioningConfigExt as _; +use crate::storage::access::access_owned::bucket_replication_lifecycle::prepare_replication_config_edit; use crate::storage::access::bucket_config_mutation_incarnation; use crate::storage::ecfs::*; use crate::storage::helper::audit_only_operation; @@ -38,7 +38,6 @@ use crate::storage::{ BUCKET_VERSIONING_CONFIG, BucketVersioningSys, OBJECT_LOCK_CONFIG, StorageError, get_bucket_object_lock_config, update_bucket_metadata_config_if_incarnation, }; -use http::StatusCode; use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::{ get_bucket_lifecycle_configuration, get_bucket_replication, get_object_lock_configuration, }; @@ -49,102 +48,8 @@ use tracing::{instrument, warn}; const LOG_SUBSYSTEM_OBJECT_LOCK: &str = "object_lock"; -#[cfg(test)] -pub(super) static SITE_REPLICATION_GATE_TEST_OVERRIDE: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0); - -#[cfg(test)] -pub(super) const SITE_REPLICATION_GATE_FORCE_DISABLED: u8 = 1; - -#[cfg(test)] -pub(super) const SITE_REPLICATION_GATE_FORCE_ENABLED: u8 = 2; - -async fn site_replication_gate_enabled() -> S3Result { - #[cfg(test)] - match SITE_REPLICATION_GATE_TEST_OVERRIDE.load(std::sync::atomic::Ordering::SeqCst) { - SITE_REPLICATION_GATE_FORCE_DISABLED => return Ok(false), - SITE_REPLICATION_GATE_FORCE_ENABLED => return Ok(true), - _ => {} - } - crate::admin::handlers::site_replication::site_replication_enabled() - .await - .map_err(Into::into) -} - -/// Remote site-replication peer deployment ids and the operator-rule contract -/// the peers support, handed to the bucket usecase so an S3 replication-config -/// edit keeps exactly the reconciler-owned rules and merges the way every -/// peer will (issue #1948). Read here, in the interface layer, because the -/// usecase must not import the admin handlers (layer guard); a state-read -/// failure propagates so the edit fails closed. -async fn site_replication_edit_context() -> S3Result<(std::collections::HashSet, OperatorRuleContract)> { - // While the gate override is in effect the test exercises the deny/allow - // branch, not the peer set; there is no persisted state to read. - #[cfg(test)] - if SITE_REPLICATION_GATE_TEST_OVERRIDE.load(std::sync::atomic::Ordering::SeqCst) != 0 { - return Ok((std::collections::HashSet::new(), OperatorRuleContract::Derived)); - } - crate::admin::handlers::site_replication::site_replication_edit_context() - .await - .map_err(Into::into) -} - -/// MinIO `ErrReplicationDenyEditError`. -fn replication_deny_edit_error() -> S3Error { - let mut err = S3Error::with_message( - S3ErrorCode::Custom("XMinioReplicationDenyEdit".into()), - "Sub-User is not allowed to edit Replication configuration", - ); - err.set_status_code(StatusCode::BAD_REQUEST); - err -} - -/// Site-replication gate for S3 replication-config edits (issue #1948). -/// -/// On a site-replication deployment the bucket's replication config carries -/// the operator-managed `site-repl-*` rules that keep every peer in sync, and -/// a successful edit is broadcast to all peers — so a user holding only -/// bucket-scoped `s3:PutReplicationConfiguration` could rewrite or erase -/// replication net-wide. MinIO parity (`ErrReplicationDenyEditError`): only -/// owner credentials (root or root-parented) may edit. Runs after the policy -/// authorization in the access layer and only on the external S3 path — the -/// reconciler and peer bucket-meta ingestion never route through these -/// handlers. -async fn deny_replication_config_edit_for_non_owner(req: &S3Request) -> S3Result<()> { - if crate::storage::access::req_info_ref(req)?.is_owner { - return Ok(()); - } - if site_replication_gate_enabled().await? { - return Err(replication_deny_edit_error()); - } - Ok(()) -} - -/// Handler bodies that [`S3`] runs through `audit_only_operation`. +/// Handler body that runs through `audit_only_operation`. impl FS { - async fn delete_bucket_replication_inner( - &self, - req: S3Request, - ) -> S3Result> { - deny_replication_config_edit_for_non_owner(&req).await?; - let (site_peers, contract) = site_replication_edit_context().await?; - let usecase = s3_api::bucket_usecase_for(self); - usecase.execute_delete_bucket_replication(req, site_peers, contract).await - } - - async fn put_bucket_replication_inner( - &self, - mut req: S3Request, - ) -> S3Result> { - deny_replication_config_edit_for_non_owner(&req).await?; - let (site_peers, contract) = site_replication_edit_context().await?; - let replication_configuration = - replication_configuration_from_s3s(std::mem::take(&mut req.input.replication_configuration)); - let usecase = s3_api::bucket_usecase_for(self); - usecase - .execute_put_bucket_replication(req, replication_configuration, site_peers, contract) - .await - } - async fn put_object_lock_configuration_inner( &self, req: S3Request, @@ -274,7 +179,12 @@ impl FS { &self, req: S3Request, ) -> S3Result> { - audit_only_operation(req, S3Operation::DeleteBucketReplication, |req| self.delete_bucket_replication_inner(req)).await + let envelope = RequestEnvelope::from_s3s(&req); + let prepare = prepare_replication_config_edit(&envelope); + s3_api::bucket_usecase_for(self) + .execute_delete_bucket_replication(envelope, &req.input.bucket, prepare) + .await?; + Ok(S3Response::new(DeleteBucketReplicationOutput::default())) } #[instrument(name = "get_bucket_lifecycle_configuration", level = "debug", skip(self, req))] @@ -357,7 +267,20 @@ impl FS { &self, req: S3Request, ) -> S3Result> { - audit_only_operation(req, S3Operation::PutBucketReplication, |req| self.put_bucket_replication_inner(req)).await + let envelope = RequestEnvelope::from_s3s(&req); + let prepare = prepare_replication_config_edit(&envelope); + let PutBucketReplicationInput { + bucket, + replication_configuration, + .. + } = req.input; + s3_api::bucket_usecase_for(self) + .execute_put_bucket_replication(envelope, &bucket, async move { + let (site_peers, contract) = prepare.await?; + Ok((replication_configuration_from_s3s(replication_configuration), site_peers, contract)) + }) + .await?; + Ok(S3Response::new(PutBucketReplicationOutput::default())) } #[instrument(name = "put_object_lock_configuration", level = "debug", skip(self, req))] diff --git a/rustfs/src/storage/legacy_edge/tests/bucket_replication_lifecycle.rs b/rustfs/src/storage/legacy_edge/tests/bucket_replication_lifecycle.rs index 57529b08d0..d2d1412786 100644 --- a/rustfs/src/storage/legacy_edge/tests/bucket_replication_lifecycle.rs +++ b/rustfs/src/storage/legacy_edge/tests/bucket_replication_lifecycle.rs @@ -15,11 +15,11 @@ //! Tests of the `bucket_replication_lifecycle` legacy-edge handlers (batch A3), moved unchanged //! from `ecfs.rs` and `ecfs_test.rs`. -use super::super::bucket_replication_lifecycle::{ - SITE_REPLICATION_GATE_FORCE_DISABLED, SITE_REPLICATION_GATE_FORCE_ENABLED, SITE_REPLICATION_GATE_TEST_OVERRIDE, -}; use super::{build_request, store_uninitialized_premise_holds}; use crate::storage::access::ReqInfo; +use crate::storage::access::access_owned::bucket_replication_lifecycle::{ + SITE_REPLICATION_GATE_FORCE_DISABLED, SITE_REPLICATION_GATE_FORCE_ENABLED, SITE_REPLICATION_GATE_TEST_OVERRIDE, +}; use crate::storage::ecfs::*; use http::Method; use http::StatusCode; diff --git a/rustfs/tests/embedded_gateway_stack_test/bucket_replication_lifecycle.rs b/rustfs/tests/embedded_gateway_stack_test/bucket_replication_lifecycle.rs index 8a268374a5..5bf165b29c 100644 --- a/rustfs/tests/embedded_gateway_stack_test/bucket_replication_lifecycle.rs +++ b/rustfs/tests/embedded_gateway_stack_test/bucket_replication_lifecycle.rs @@ -24,11 +24,16 @@ use super::{ outcome, s3_client, with_stack, }; use aws_sdk_s3::Client; +use aws_sdk_s3::client::customize::CustomizableOperation; use aws_sdk_s3::error::ProvideErrorMetadata as _; +use aws_sdk_s3::operation::put_bucket_replication::builders::PutBucketReplicationFluentBuilder; +use aws_sdk_s3::operation::put_bucket_replication::{PutBucketReplicationError, PutBucketReplicationOutput}; use futures::future::BoxFuture; use rustfs::app::context::get_global_app_context; use rustfs::embedded::RustFSServer; -use rustfs_ecstore::api::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_REPLICATION_CONFIG, OBJECT_LOCK_CONFIG}; +use rustfs_ecstore::api::bucket::metadata::{ + BUCKET_LIFECYCLE_CONFIG, BUCKET_REPLICATION_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, OBJECT_LOCK_CONFIG, +}; use rustfs_ecstore::api::bucket::metadata_sys; use rustfs_iam::store::UserType; use rustfs_policy::policy::Policy; @@ -74,6 +79,37 @@ pub(crate) const MATRIX: &[MatrixRow] = &[ has_error_body: true, call: stored::<1>, }, + MatrixRow { + operation: "PutBucketReplication", + metric: S3Operation::PutBucketReplication, + has_error_body: true, + call: |client, _| { + Box::pin(async move { + seed_replication_target(REPLICATION_MATRIX_BUCKET).await; + outcome( + put_replication(client, REPLICATION_MATRIX_BUCKET, REPLICATION_WRITE) + .send() + .await, + ) + }) + }, + }, + MatrixRow { + operation: "DeleteBucketReplication", + metric: S3Operation::DeleteBucketReplication, + has_error_body: true, + call: |client, _| { + Box::pin(async move { + outcome( + client + .delete_bucket_replication() + .bucket(REPLICATION_MATRIX_BUCKET) + .send() + .await, + ) + }) + }, + }, MatrixRow { operation: "GetObjectLockConfiguration", metric: S3Operation::GetObjectLockConfiguration, @@ -628,3 +664,245 @@ fn gateway_lifecycle_put_rejects_submillisecond_date() { server.shutdown().await; }); } + +pub(super) const REPLICATION_MATRIX_BUCKET: &str = "replication-matrix"; +const REPLICATION_ARN: &str = "arn:rustfs:replication:us-east-1:operator:replica"; +const REPLICATION_WRITE: &str = concat!( + "Disabled", + "Enabledarn:rustfs:replication:us-east-1:operator:replica", + "STANDARDEnabled", + "logs/&/teamstorage", + "operator7Enabled", + "Enabled", +); +const REPLICATION_SIMPLE: &str = concat!( + "arn:rustfs:replication:us-east-1:operator:replica", + "simpleEnabled", +); +const REPLICATION_UNSUPPORTED: &str = concat!( + "123456789012", + "arn:rustfs:replication:us-east-1:operator:replicaEnabled", +); + +async fn seed_replication_target(bucket: &str) { + use rustfs_ecstore::api::bucket::target::{BucketTarget, BucketTargetType, BucketTargets}; + + metadata_sys::update( + bucket, + BUCKET_VERSIONING_CONFIG, + b"Enabled".to_vec(), + ) + .await + .expect("enable replication source versioning"); + let targets = BucketTargets { + targets: vec![BucketTarget { + source_bucket: bucket.to_owned(), + target_bucket: "replica".to_owned(), + arn: REPLICATION_ARN.to_owned(), + target_type: BucketTargetType::ReplicationService, + ..Default::default() + }], + }; + metadata_sys::update(bucket, BUCKET_TARGETS_FILE, serde_json::to_vec(&targets).expect("target bytes")) + .await + .expect("write replication targets"); +} + +fn put_replication( + client: &Client, + bucket: &str, + document: &'static str, +) -> CustomizableOperation { + use aws_sdk_s3::types::{Destination, ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus}; + let configuration = ReplicationConfiguration::builder() + .role("") + .rules( + ReplicationRule::builder() + .destination( + Destination::builder() + .bucket(REPLICATION_ARN) + .build() + .expect("placeholder destination"), + ) + .status(ReplicationRuleStatus::Enabled) + .build() + .expect("placeholder rule"), + ) + .build() + .expect("placeholder configuration"); + client + .put_bucket_replication() + .bucket(bucket) + .replication_configuration(configuration) + .customize() + .mutate_request(move |request| send_document(request, document)) +} + +async fn owned_replication_wire_and_disk(server: RustFSServer) { + const BUCKET: &str = "replication-edit-granted"; + const OTHER: &str = "replication-edit-other"; + const WRITER: &str = "replicationeditwriter"; + const READER: &str = "replicationeditreader"; + for bucket in [BUCKET, OTHER] { + create_bucket_in_the_store(bucket).await; + } + create_user_allowed_on_one_bucket(WRITER, "s3:PutReplicationConfiguration", BUCKET).await; + create_user_allowed_on_one_bucket(READER, "s3:GetReplicationConfiguration", BUCKET).await; + let endpoint = server.endpoint(); + let root = s3_client(&endpoint, server.access_key(), server.secret_key()); + let writer = s3_client(&endpoint, WRITER, USER_SECRET_KEY); + let reader = s3_client(&endpoint, READER, USER_SECRET_KEY); + + let unsupported = put_replication(&root, BUCKET, REPLICATION_UNSUPPORTED) + .send() + .await + .expect_err("unsupported destination"); + let disabled = put_replication(&root, BUCKET, REPLICATION_SIMPLE) + .send() + .await + .expect_err("versioning disabled"); + for (err, message) in [ + ( + unsupported, + "replication field Destination.Account is not supported by this RustFS version", + ), + (disabled, "bucket versioning must be enabled before replication can be configured"), + ] { + let refusal = err.as_service_error().expect("service refusal"); + assert_eq!((refusal.code(), refusal.message()), (Some("InvalidRequest"), Some(message))); + } + + seed_replication_target(BUCKET).await; + seed_replication_target(OTHER).await; + for (client, bucket) in [(&reader, BUCKET), (&writer, OTHER)] { + assert_eq!( + answer(outcome(put_replication(client, bucket, REPLICATION_SIMPLE).send().await)), + refused("AccessDenied", 403) + ); + } + for bucket in [BUCKET, OTHER] { + let stored = metadata_sys::get_config_from_disk(bucket) + .await + .expect("disk metadata before Put"); + assert!(stored.replication_config_xml.is_empty()); + } + let stale = put_replication( + &writer, + BUCKET, + "arn:rustfs:replication:us-east-1:absent:replicaEnabled", + ) + .send() + .await + .expect_err("stale target"); + let stale = stale.as_service_error().expect("stale target refusal"); + assert_eq!( + (stale.code(), stale.message()), + (Some("InvalidRequest"), Some("replication config has a stale target")) + ); + + assert_eq!(answer(outcome(put_replication(&writer, BUCKET, REPLICATION_WRITE).send().await)), Ok(())); + let actual = metadata_sys::get_config_from_disk(BUCKET) + .await + .expect("disk metadata after Put"); + assert_eq!(actual.replication_config_xml, REPLICATION_WRITE.as_bytes()); + + // Keep actual bytes to observe attempted deletes under both wrong policies. + metadata_sys::update(OTHER, BUCKET_REPLICATION_CONFIG, REPLICATION_WRITE.as_bytes().to_vec()) + .await + .expect("persist other bucket's rules"); + for (client, bucket) in [(&reader, BUCKET), (&writer, OTHER)] { + assert_eq!( + answer(outcome(client.delete_bucket_replication().bucket(bucket).send().await)), + refused("AccessDenied", 403) + ); + let stored = metadata_sys::get_config_from_disk(bucket) + .await + .expect("disk metadata after refused Delete"); + assert_eq!(stored.replication_config_xml, REPLICATION_WRITE.as_bytes()); + } + + for _ in 0..2 { + let status = StatusCapture::default(); + let deleted = writer + .delete_bucket_replication() + .bucket(BUCKET) + .customize() + .interceptor(status.clone()) + .send() + .await; + assert_eq!((answer(outcome(deleted)), status.take()), (Ok(()), Some(204))); + let stored = metadata_sys::get_config_from_disk(BUCKET) + .await + .expect("disk metadata after Delete"); + assert!(stored.replication_config_xml.is_empty()); + let targets: rustfs_ecstore::api::bucket::target::BucketTargets = + serde_json::from_slice(&stored.bucket_targets_config_json).expect("disk targets after Delete"); + assert!(targets.targets.is_empty()); + } + + // Real persisted state is consulted after IAM authorization on both edges. + let store = get_global_app_context().expect("embedded server context").object_store(); + let state_path = "config/site-replication/state.json"; + let state = br#"{ + "name":"local","service_account_access_key":"","service_account_parent":"", + "peers":{"local":{"deploymentID":"local","endpoint":"http://127.0.0.1:9001"},"remote":{"deploymentID":"remote","endpoint":"http://127.0.0.1:9002"}}, + "updated_at":null,"resync_status":{} + }"#; + rustfs_ecstore::api::config::com::save_config(store.clone(), state_path, state.to_vec()) + .await + .expect("enable site replication without peer credentials"); + let before = metadata_sys::get_config_from_disk(BUCKET) + .await + .expect("before site-protected edits"); + let gate = refused("XMinioReplicationDenyEdit", 400); + assert_eq!(answer(outcome(put_replication(&writer, BUCKET, REPLICATION_SIMPLE).send().await)), gate); + assert_eq!(answer(outcome(writer.delete_bucket_replication().bucket(BUCKET).send().await)), gate); + // IAM still denies an unrelated principal before the site gate. + assert_eq!( + answer(outcome(reader.delete_bucket_replication().bucket(BUCKET).send().await)), + refused("AccessDenied", 403) + ); + + rustfs_ecstore::api::config::com::save_config(store.clone(), state_path, b"{".to_vec()) + .await + .expect("store corrupt site replication state"); + for client in [&root, &writer] { + let err = put_replication(client, BUCKET, REPLICATION_SIMPLE) + .send() + .await + .expect_err("corrupt site replication state"); + let error = err.as_service_error().expect("state read failure"); + assert_eq!(error.code(), Some("InternalError")); + assert!( + error + .message() + .is_some_and(|message| message.starts_with("invalid site replication state:")) + ); + assert_eq!( + answer(outcome(client.delete_bucket_replication().bucket(BUCKET).send().await)), + refused("InternalError", 500) + ); + } + let after = metadata_sys::get_config_from_disk(BUCKET) + .await + .expect("after site-protected edits"); + assert_eq!( + (after.replication_config_xml, after.bucket_targets_config_json), + (before.replication_config_xml, before.bucket_targets_config_json) + ); + rustfs_ecstore::api::config::com::delete_config(store, state_path) + .await + .expect("remove test site state"); + + server.shutdown().await; +} + +#[test] +fn owned_replication_gateway_writes_and_removes_the_config_and_targets() { + with_stack(GATEWAY, owned_replication_wire_and_disk); +} + +#[test] +fn owned_replication_legacy_writes_and_removes_the_config_and_targets() { + with_stack(LEGACY, owned_replication_wire_and_disk); +} diff --git a/rustfs/tests/embedded_gateway_stack_test/main.rs b/rustfs/tests/embedded_gateway_stack_test/main.rs index ddb9ccf0b4..50c5a3760b 100644 --- a/rustfs/tests/embedded_gateway_stack_test/main.rs +++ b/rustfs/tests/embedded_gateway_stack_test/main.rs @@ -356,6 +356,7 @@ async fn every_served_operation_serves_root_and_refuses_the_callers_it_cannot_ad create_bucket_in_the_store(bucket_configs::ENCRYPTION_MATRIX_BUCKET).await; bucket_configs::store_read_configs(MATRIX_BUCKET, bucket_configs::ENCRYPTION_MATRIX_BUCKET).await; bucket_core::seed_website(MATRIX_BUCKET).await; + create_bucket_in_the_store(bucket_replication_lifecycle::REPLICATION_MATRIX_BUCKET).await; create_bucket_in_the_store(object_meta::MATRIX_BUCKET).await; create_bucket_in_the_store(object_misc::MATRIX_BUCKET).await; create_bucket_in_the_store(object_read::MATRIX_BUCKET).await; @@ -400,6 +401,8 @@ async fn every_served_operation_serves_root_and_refuses_the_callers_it_cannot_ad | S3Operation::PutBucketRequestPayment | S3Operation::PutBucketWebsite | S3Operation::DeleteBucketWebsite + | S3Operation::PutBucketReplication + | S3Operation::DeleteBucketReplication | S3Operation::PutBucketEncryption | S3Operation::PutBucketTagging | S3Operation::DeleteBucketEncryption