mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-16 04:52:36 +00:00
feat(gateway): serve bucket replication configuration writes
Route PutBucketReplication and DeleteBucketReplication through the owned bucket use case. Share site checks, metadata fencing, target cleanup recovery, and audit completion with the legacy edge. Signed requests exercise 28 refusals and six accepted edits across both stacks. The direct storage tests cover six failure paths and two successful edits. Twenty-four faults turned their expected disk, authorization, locking, callback, audit, or metric assertions red. The restored gateway-stack build passed 132 selected tests on75b293f09, including both HTTP stacks. Both Clippy variants and the mutation cohort passed onfb3e2818c. The fixed legacy CLI matched all six response records. The standalone default runtime suite, multi-site delivery, and CI were unverified.
This commit is contained in:
@@ -15,11 +15,8 @@
|
||||
//! Bucket replication configuration use-case operations.
|
||||
|
||||
use super::*;
|
||||
use crate::app::storage_api::s3::dto::{
|
||||
DeleteBucketReplicationInput, DeleteBucketReplicationOutput, PutBucketReplicationInput, PutBucketReplicationOutput,
|
||||
};
|
||||
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
||||
use rustfs_gateway_types::persistence::parse_replication_dto;
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result, s3_error};
|
||||
|
||||
fn validate_replication_config_targets(targets: &BucketTargets, config: &PersistedReplicationConfiguration) -> S3Result<()> {
|
||||
let configured_arns = targets
|
||||
@@ -168,6 +165,15 @@ async fn write_replication_targets_after_config_delete(
|
||||
removed: usize,
|
||||
expected_incarnation_id: Option<uuid::Uuid>,
|
||||
) -> S3Result<()> {
|
||||
#[cfg(test)]
|
||||
if let Some(fail) = tests::TARGET_CLEANUP_FAILURE
|
||||
.try_with(std::cell::RefCell::take)
|
||||
.ok()
|
||||
.flatten()
|
||||
{
|
||||
fail.await?;
|
||||
}
|
||||
|
||||
let json_targets = serde_json::to_vec(&targets).map_err(to_internal_error)?;
|
||||
update_bucket_config_for_incarnation(bucket, BUCKET_TARGETS_FILE, json_targets, expected_incarnation_id)
|
||||
.await
|
||||
@@ -210,38 +216,56 @@ async fn restore_replication_config_after_target_cleanup_failure(
|
||||
}
|
||||
|
||||
impl DefaultBucketUsecase {
|
||||
/// `site_peers` is the set of remote site-replication peer deployment ids
|
||||
/// (empty when site replication is disabled). The interface layer reads it
|
||||
/// from the persisted state and fails closed on a read error, so this
|
||||
/// usecase stays a pure function of its inputs (layer rule: app never
|
||||
/// imports interface).
|
||||
pub async fn execute_delete_bucket_replication(
|
||||
/// Removes operator rules and targets unique to those rules.
|
||||
///
|
||||
/// Preparation runs inside the audit scope and resolves the peer contract
|
||||
/// before any bucket lookup or mutation.
|
||||
pub(crate) async fn execute_delete_bucket_replication(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketReplicationInput>,
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
prepare: impl Future<Output = S3Result<(HashSet<String>, OperatorRuleContract)>> + Send,
|
||||
) -> S3Result<()> {
|
||||
// Preserve the success status recorded by the legacy audit for an empty response.
|
||||
RequestEnvelope::audit_only_operation(
|
||||
envelope,
|
||||
S3Operation::DeleteBucketReplication,
|
||||
StatusCode::OK,
|
||||
|envelope| async move {
|
||||
let (site_peers, contract) = prepare.await?;
|
||||
self.delete_bucket_replication(envelope, bucket, site_peers, contract).await
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete_bucket_replication(
|
||||
&self,
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
site_peers: HashSet<String>,
|
||||
contract: OperatorRuleContract,
|
||||
) -> S3Result<S3Response<DeleteBucketReplicationOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
|
||||
let DeleteBucketReplicationInput { bucket, .. } = req.input;
|
||||
) -> S3Result<()> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&envelope, bucket)?;
|
||||
let request_context = envelope.request_context().cloned();
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init"));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
let targets_guard = lock_bucket_targets_metadata(&bucket).await;
|
||||
let replication_config = match metadata_sys::get_replication_config(&bucket).await {
|
||||
let targets_guard = lock_bucket_targets_metadata(bucket).await;
|
||||
let replication_config = match metadata_sys::get_replication_config(bucket).await {
|
||||
Ok((config, _)) => Some(config),
|
||||
Err(StorageError::ConfigNotFound) => None,
|
||||
Err(err) => return Err(ApiError::from(err).into()),
|
||||
};
|
||||
let (remaining_config, updated_targets) = if let Some(config) = replication_config.as_ref() {
|
||||
let (remaining, removable_arns) = split_replication_config_for_user_delete(config.clone(), &site_peers, contract);
|
||||
let targets = replication_targets_without_arns(&bucket, &removable_arns).await?;
|
||||
let targets = replication_targets_without_arns(bucket, &removable_arns).await?;
|
||||
(remaining, targets)
|
||||
} else {
|
||||
(None, None)
|
||||
@@ -252,23 +276,23 @@ impl DefaultBucketUsecase {
|
||||
// S3 delete (issue #1948); only the operator-authored rules go.
|
||||
Some(remaining) => {
|
||||
let data = serialize_config(&remaining)?;
|
||||
update_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id)
|
||||
update_bucket_config_for_incarnation(bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
}
|
||||
None => {
|
||||
delete_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, expected_incarnation_id)
|
||||
delete_bucket_config_for_incarnation(bucket, BUCKET_REPLICATION_CONFIG, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
}
|
||||
}
|
||||
if let Some((targets, removed)) = updated_targets
|
||||
&& let Err(err) =
|
||||
write_replication_targets_after_config_delete(&bucket, &targets, removed, expected_incarnation_id).await
|
||||
write_replication_targets_after_config_delete(bucket, &targets, removed, expected_incarnation_id).await
|
||||
{
|
||||
if let Some(config) = replication_config.as_ref() {
|
||||
return Err(restore_replication_config_after_target_cleanup_failure(
|
||||
&bucket,
|
||||
bucket,
|
||||
config,
|
||||
err,
|
||||
expected_incarnation_id,
|
||||
@@ -279,16 +303,16 @@ impl DefaultBucketUsecase {
|
||||
}
|
||||
drop(targets_guard);
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "delete bucket replication", request_context, true).await;
|
||||
notify_bucket_metadata_reload(bucket.to_owned(), "delete bucket replication", request_context, true).await;
|
||||
|
||||
let item = sr_bucket_meta_item(bucket.clone(), "replication-config");
|
||||
let item = sr_bucket_meta_item(bucket.to_owned(), "replication-config");
|
||||
if let Err(err) = site_replication_bucket_meta_hook(item).await {
|
||||
warn!(bucket = %bucket, error = ?err, "site replication bucket replication-config delete hook failed");
|
||||
}
|
||||
|
||||
info!(bucket = %bucket, "deleted bucket replication config");
|
||||
|
||||
Ok(S3Response::new(DeleteBucketReplicationOutput::default()))
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The stored replication configuration as the gateway's answer holds it.
|
||||
@@ -335,36 +359,52 @@ impl DefaultBucketUsecase {
|
||||
})
|
||||
}
|
||||
|
||||
/// See [`Self::execute_delete_bucket_replication`] for `site_peers` and
|
||||
/// `contract`. `replication_configuration` is the request body in the
|
||||
/// engine's shape: the s3s edge (`storage::ecfs`) converts it, and this use
|
||||
/// case never reads `req.input.replication_configuration`.
|
||||
pub async fn execute_put_bucket_replication(
|
||||
/// Replaces operator rules after the interface resolves the peer contract.
|
||||
pub(crate) async fn execute_put_bucket_replication(
|
||||
&self,
|
||||
req: S3Request<PutBucketReplicationInput>,
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
prepare: impl Future<Output = S3Result<(PersistedReplicationConfiguration, HashSet<String>, OperatorRuleContract)>> + Send,
|
||||
) -> S3Result<()> {
|
||||
RequestEnvelope::audit_only_operation(
|
||||
envelope,
|
||||
S3Operation::PutBucketReplication,
|
||||
StatusCode::OK,
|
||||
|envelope| async move {
|
||||
let (replication_configuration, site_peers, contract) = prepare.await?;
|
||||
self.put_bucket_replication(envelope, bucket, replication_configuration, site_peers, contract)
|
||||
.await
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn put_bucket_replication(
|
||||
&self,
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
replication_configuration: PersistedReplicationConfiguration,
|
||||
site_peers: HashSet<String>,
|
||||
contract: OperatorRuleContract,
|
||||
) -> S3Result<S3Response<PutBucketReplicationOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
|
||||
let PutBucketReplicationInput { bucket, .. } = req.input;
|
||||
) -> S3Result<()> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&envelope, bucket)?;
|
||||
let request_context = envelope.request_context().cloned();
|
||||
info!(bucket = %bucket, "updating bucket replication config");
|
||||
|
||||
validate_replication_config_capabilities(&replication_configuration)?;
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init"));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let targets_guard = lock_bucket_targets_metadata(&bucket).await;
|
||||
validate_bucket_replication_update(&bucket, &replication_configuration).await?;
|
||||
let existing_config = match metadata_sys::get_replication_config(&bucket).await {
|
||||
let targets_guard = lock_bucket_targets_metadata(bucket).await;
|
||||
validate_bucket_replication_update(bucket, &replication_configuration).await?;
|
||||
let existing_config = match metadata_sys::get_replication_config(bucket).await {
|
||||
Ok((config, _)) => Some(config),
|
||||
Err(StorageError::ConfigNotFound) => None,
|
||||
Err(err) => return Err(ApiError::from(err).into()),
|
||||
@@ -372,14 +412,14 @@ impl DefaultBucketUsecase {
|
||||
let replication_configuration =
|
||||
merge_user_replication_config_update(replication_configuration, existing_config, &site_peers, contract);
|
||||
let data = serialize_config(&replication_configuration)?;
|
||||
update_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id)
|
||||
update_bucket_config_for_incarnation(bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
drop(targets_guard);
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put bucket replication", request_context, true).await;
|
||||
notify_bucket_metadata_reload(bucket.to_owned(), "put bucket replication", request_context, true).await;
|
||||
|
||||
let mut item = sr_bucket_meta_item(bucket.clone(), "replication-config");
|
||||
let mut item = sr_bucket_meta_item(bucket.to_owned(), "replication-config");
|
||||
item.replication_config = Some(
|
||||
serialize_config(&replication_configuration).and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?,
|
||||
);
|
||||
@@ -387,19 +427,29 @@ impl DefaultBucketUsecase {
|
||||
warn!(bucket = %bucket, error = ?err, "site replication bucket replication-config hook failed");
|
||||
}
|
||||
|
||||
Ok(S3Response::new(PutBucketReplicationOutput::default()))
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use crate::app::bucket_usecase::test_support::{build_request, owned_envelope};
|
||||
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient};
|
||||
use crate::app::runtime_sources::ServerContextSlot;
|
||||
use crate::app::storage_api::owned_parity::{ReqInfo, authorize_bucket_config_mutation};
|
||||
use crate::site_replication::hooks::{BUCKET_META_HOOK_CALLS, BucketMetaHookCall};
|
||||
use futures::{FutureExt, future::BoxFuture};
|
||||
use http::Method;
|
||||
use rustfs_credentials::Credentials;
|
||||
use rustfs_gateway_types::dto::Status;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedEncryptionConfiguration, PersistedReplicationDestination, PersistedReplicationRule,
|
||||
};
|
||||
use rustfs_policy::policy::action::{Action, S3Action};
|
||||
use serial_test::serial;
|
||||
use std::cell::RefCell;
|
||||
use std::sync::Mutex;
|
||||
|
||||
fn replication_rule_for_target(arn: &str) -> PersistedReplicationRule {
|
||||
PersistedReplicationRule {
|
||||
@@ -834,16 +884,11 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_delete_bucket_replication_returns_internal_error_when_store_uninitialized() {
|
||||
let input = DeleteBucketReplicationInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::DELETE);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_delete_bucket_replication(req, HashSet::new(), OperatorRuleContract::Derived)
|
||||
.execute_delete_bucket_replication(owned_envelope(Method::DELETE), "test-bucket", async {
|
||||
Ok((HashSet::new(), OperatorRuleContract::Derived))
|
||||
})
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
@@ -869,17 +914,11 @@ mod tests {
|
||||
role: "arn:aws:iam::123456789012:role/test".to_string(),
|
||||
rules: vec![replication_rule_for_target("arn:rustfs:replication:us-east-1:target:bucket")],
|
||||
};
|
||||
let input = PutBucketReplicationInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.replication_configuration(Default::default())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::PUT);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_put_bucket_replication(req, configuration, HashSet::new(), OperatorRuleContract::Derived)
|
||||
.execute_put_bucket_replication(owned_envelope(Method::PUT), "test-bucket", async {
|
||||
Ok((configuration, HashSet::new(), OperatorRuleContract::Derived))
|
||||
})
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
@@ -893,19 +932,10 @@ mod tests {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
};
|
||||
let input = PutBucketReplicationInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.replication_configuration(Default::default())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let err = DefaultBucketUsecase::without_context()
|
||||
.execute_put_bucket_replication(
|
||||
build_request(input, Method::PUT),
|
||||
configuration,
|
||||
HashSet::new(),
|
||||
OperatorRuleContract::Derived,
|
||||
)
|
||||
.execute_put_bucket_replication(owned_envelope(Method::PUT), "test-bucket", async {
|
||||
Ok((configuration, HashSet::new(), OperatorRuleContract::Derived))
|
||||
})
|
||||
.await
|
||||
.expect_err("unsupported fields must be rejected before store access");
|
||||
|
||||
@@ -915,4 +945,316 @@ mod tests {
|
||||
Some("replication field Destination.Account is not supported by this RustFS version")
|
||||
);
|
||||
}
|
||||
|
||||
const PEER_ARN: &str = "arn:rustfs:replication::peer-dep:bucket";
|
||||
const USER_ARN: &str = "arn:rustfs:replication:us-east-1:user:bucket";
|
||||
const SITE_RULE: &str = "site-repl-peer-dep";
|
||||
|
||||
tokio::task_local! {
|
||||
pub(super) static TARGET_CLEANUP_FAILURE: RefCell<Option<BoxFuture<'static, S3Result<()>>>>;
|
||||
}
|
||||
|
||||
fn put_configuration() -> PersistedReplicationConfiguration {
|
||||
PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![
|
||||
replication_rule_with_id(USER_ARN, "new-operator", 3),
|
||||
replication_rule_with_id(PEER_ARN, "shared-peer-target", 4),
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
async fn seed_replication(store: &ECStore, bucket: &str, operator: &str) -> (Vec<u8>, Vec<u8>) {
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket for replication");
|
||||
metadata_sys::update(
|
||||
bucket,
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("enable versioning");
|
||||
let config = PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![
|
||||
replication_rule_with_id(USER_ARN, operator, 1),
|
||||
replication_rule_with_id(PEER_ARN, SITE_RULE, 2),
|
||||
],
|
||||
};
|
||||
let targets = BucketTargets {
|
||||
targets: [PEER_ARN, USER_ARN]
|
||||
.into_iter()
|
||||
.map(|arn| BucketTarget {
|
||||
source_bucket: bucket.to_owned(),
|
||||
arn: arn.to_owned(),
|
||||
target_type: BucketTargetType::ReplicationService,
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
let bytes = serialize(&config).expect("seed configuration bytes");
|
||||
let targets = serde_json::to_vec(&targets).expect("seed target bytes");
|
||||
metadata_sys::update(bucket, BUCKET_TARGETS_FILE, targets.clone())
|
||||
.await
|
||||
.expect("seed target metadata");
|
||||
metadata_sys::update(bucket, BUCKET_REPLICATION_CONFIG, bytes.clone())
|
||||
.await
|
||||
.expect("seed replication configuration");
|
||||
(bytes, targets)
|
||||
}
|
||||
|
||||
async fn persisted_replication(bucket: &str) -> (Vec<u8>, Vec<u8>) {
|
||||
let stored = metadata_sys::get_global_bucket_metadata_sys()
|
||||
.expect("initialized metadata system")
|
||||
.read()
|
||||
.await
|
||||
.get_config_from_disk(bucket)
|
||||
.await
|
||||
.expect("disk metadata");
|
||||
(stored.replication_config_xml, stored.bucket_targets_config_json)
|
||||
}
|
||||
|
||||
async fn authorize_replication(context: &Arc<AppContext>, bucket: &str, method: Method) -> RequestEnvelope {
|
||||
let server = ServerContextSlot::new();
|
||||
assert!(server.install(Arc::clone(context)));
|
||||
let mut request = build_request((), method);
|
||||
request.uri = format!("/{bucket}?replication").parse().expect("replication URI");
|
||||
request.extensions.insert(Arc::clone(&server));
|
||||
request.extensions.insert(ReqInfo {
|
||||
cred: Some(Credentials {
|
||||
access_key: "replication-owner".to_owned(),
|
||||
..Default::default()
|
||||
}),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
let mut envelope = RequestEnvelope::from_s3s(&request);
|
||||
authorize_bucket_config_mutation(
|
||||
&server,
|
||||
&mut envelope,
|
||||
bucket,
|
||||
Action::S3Action(S3Action::PutReplicationConfigurationAction),
|
||||
)
|
||||
.await
|
||||
.expect("snapshot authorized bucket");
|
||||
envelope
|
||||
}
|
||||
|
||||
async fn edit_replication(
|
||||
usecase: &DefaultBucketUsecase,
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
put: bool,
|
||||
) -> S3Result<()> {
|
||||
let peers = HashSet::from(["peer-dep".to_owned()]);
|
||||
if put {
|
||||
usecase
|
||||
.execute_put_bucket_replication(envelope, bucket, async {
|
||||
Ok((put_configuration(), peers, OperatorRuleContract::Derived))
|
||||
})
|
||||
.await
|
||||
} else {
|
||||
usecase
|
||||
.execute_delete_bucket_replication(envelope, bucket, async { Ok((peers, OperatorRuleContract::Derived)) })
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
struct Reload {
|
||||
completed: bool,
|
||||
maintenance: bool,
|
||||
target_lock_available: bool,
|
||||
hook_calls: usize,
|
||||
}
|
||||
|
||||
async fn observe_replication_edit(
|
||||
bucket: &str,
|
||||
edit: impl Future<Output = S3Result<()>>,
|
||||
) -> (S3Result<()>, Vec<Reload>, Vec<BucketMetaHookCall>) {
|
||||
let reloads = Arc::new(Mutex::new(Vec::new()));
|
||||
let record = Arc::clone(&reloads);
|
||||
let target_bucket = bucket.to_owned();
|
||||
let (result, hooks) = BUCKET_META_HOOK_CALLS
|
||||
.scope(
|
||||
RefCell::default(),
|
||||
BUCKET_METADATA_RELOAD_PROBE.scope(
|
||||
(
|
||||
bucket.to_owned(),
|
||||
Box::new(move |completed, _, maintenance| {
|
||||
record.lock().expect("reloads").push(Reload {
|
||||
completed,
|
||||
maintenance,
|
||||
target_lock_available: lock_bucket_targets_metadata(&target_bucket).now_or_never().is_some(),
|
||||
hook_calls: BUCKET_META_HOOK_CALLS.with(|calls| calls.borrow().len()),
|
||||
});
|
||||
}),
|
||||
),
|
||||
async {
|
||||
let result = edit.await;
|
||||
(result, BUCKET_META_HOOK_CALLS.with(RefCell::take))
|
||||
},
|
||||
),
|
||||
)
|
||||
.await;
|
||||
let reloads = std::mem::take(&mut *reloads.lock().expect("reloads"));
|
||||
(result, reloads, hooks)
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn owned_replication_preserves_peer_rules_and_finishes_reload_before_site_hook() {
|
||||
run_large_stack_test("owned-replication-peer-rules", || async {
|
||||
let context = shared_gating_ambient().await;
|
||||
let bucket = format!("replication-peer-{}", uuid::Uuid::new_v4().simple());
|
||||
let (original, _) = seed_replication(&context.object_store(), &bucket, "old-operator").await;
|
||||
let usecase = DefaultBucketUsecase::with_context(Some(Arc::clone(&context)));
|
||||
for (put, method) in [(true, Method::PUT), (false, Method::DELETE)] {
|
||||
let envelope = authorize_replication(&context, &bucket, method).await;
|
||||
let (result, reloads, hooks) =
|
||||
observe_replication_edit(&bucket, edit_replication(&usecase, envelope, &bucket, put)).await;
|
||||
result.expect("replication edit");
|
||||
assert_eq!(
|
||||
reloads,
|
||||
[
|
||||
Reload {
|
||||
completed: false,
|
||||
maintenance: true,
|
||||
target_lock_available: true,
|
||||
hook_calls: 0
|
||||
},
|
||||
Reload {
|
||||
completed: true,
|
||||
maintenance: true,
|
||||
target_lock_available: true,
|
||||
hook_calls: 0
|
||||
},
|
||||
]
|
||||
);
|
||||
|
||||
let (xml, target_bytes) = persisted_replication(&bucket).await;
|
||||
assert_ne!(xml, original);
|
||||
let config = rustfs_gateway_types::persistence::parse_replication(&xml).expect("stored XML");
|
||||
let ids: HashSet<_> = config.rules.iter().filter_map(|rule| rule.id.as_deref()).collect();
|
||||
let targets: BucketTargets = serde_json::from_slice(&target_bytes).expect("stored targets");
|
||||
let arns: HashSet<_> = targets.targets.iter().map(|target| target.arn.as_str()).collect();
|
||||
if put {
|
||||
assert_eq!(ids, HashSet::from([SITE_RULE, "new-operator", "shared-peer-target"]));
|
||||
assert_eq!(arns, HashSet::from([PEER_ARN, USER_ARN]));
|
||||
} else {
|
||||
assert_eq!(ids, HashSet::from([SITE_RULE]));
|
||||
assert_eq!(arns, HashSet::from([PEER_ARN]));
|
||||
}
|
||||
assert_eq!(hooks.len(), 1);
|
||||
assert_eq!(
|
||||
(hooks[0].bucket.as_str(), hooks[0].item_type.as_str()),
|
||||
(bucket.as_str(), "replication-config")
|
||||
);
|
||||
assert_eq!(hooks[0].replication_config.as_deref().map(str::as_bytes), put.then_some(xml.as_slice()));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn owned_replication_stale_edits_leave_replacement_bytes_and_skip_callbacks() {
|
||||
run_large_stack_test("owned-replication-stale", || async {
|
||||
let context = shared_gating_ambient().await;
|
||||
let store = context.object_store();
|
||||
let usecase = DefaultBucketUsecase::with_context(Some(Arc::clone(&context)));
|
||||
for (method, preserve_peer) in [(Method::PUT, true), (Method::DELETE, true), (Method::DELETE, false)] {
|
||||
let bucket = format!("replication-stale-{}", uuid::Uuid::new_v4().simple());
|
||||
seed_replication(&store, &bucket, "old-operator").await;
|
||||
let put = method == Method::PUT;
|
||||
let envelope = authorize_replication(&context, &bucket, method).await;
|
||||
store
|
||||
.delete_bucket(&bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete authorized bucket");
|
||||
let replacement = seed_replication(&store, &bucket, "replacement").await;
|
||||
let edit = async {
|
||||
if preserve_peer {
|
||||
edit_replication(&usecase, envelope, &bucket, put).await
|
||||
} else {
|
||||
usecase
|
||||
.execute_delete_bucket_replication(envelope, &bucket, async {
|
||||
Ok((HashSet::new(), OperatorRuleContract::Derived))
|
||||
})
|
||||
.await
|
||||
}
|
||||
};
|
||||
let (result, reloads, hooks) = observe_replication_edit(&bucket, edit).await;
|
||||
assert_eq!(
|
||||
(
|
||||
result.err().map(|err| err.code().clone()),
|
||||
persisted_replication(&bucket).await,
|
||||
reloads.len(),
|
||||
hooks.len()
|
||||
),
|
||||
(Some(S3ErrorCode::NoSuchBucket), replacement, 0, 0),
|
||||
"stale {}",
|
||||
if put { "Put" } else { "Delete" }
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn owned_replication_cleanup_error_restores_original_and_never_overwrites_a_replacement() {
|
||||
run_large_stack_test("owned-replication-cleanup", || async {
|
||||
let context = shared_gating_ambient().await;
|
||||
let store = context.object_store();
|
||||
let usecase = DefaultBucketUsecase::with_context(Some(Arc::clone(&context)));
|
||||
for (replace, fail_target_write) in [(false, true), (true, true), (true, false)] {
|
||||
let bucket = format!("replication-cleanup-{}", uuid::Uuid::new_v4().simple());
|
||||
let original = seed_replication(&store, &bucket, "old-operator").await;
|
||||
let envelope = authorize_replication(&context, &bucket, Method::DELETE).await;
|
||||
let expected_storage = Arc::new(Mutex::new(original));
|
||||
let output = Arc::clone(&expected_storage);
|
||||
let replacement_bucket = bucket.clone();
|
||||
let old_store = Arc::clone(&store);
|
||||
let fail = Box::pin(async move {
|
||||
let (after_delete, _) = persisted_replication(&replacement_bucket).await;
|
||||
assert!(!String::from_utf8_lossy(&after_delete).contains("old-operator"));
|
||||
assert!(lock_bucket_targets_metadata(&replacement_bucket).now_or_never().is_none());
|
||||
if replace {
|
||||
old_store
|
||||
.delete_bucket(&replacement_bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete original before restoration");
|
||||
let replacement = seed_replication(&old_store, &replacement_bucket, "replacement").await;
|
||||
*output.lock().expect("replacement bytes") = replacement;
|
||||
}
|
||||
if fail_target_write {
|
||||
Err(s3_error!(InternalError, "replication target cleanup failed"))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
});
|
||||
let (result, reloads, hooks) = TARGET_CLEANUP_FAILURE
|
||||
.scope(
|
||||
RefCell::new(Some(fail)),
|
||||
observe_replication_edit(&bucket, edit_replication(&usecase, envelope, &bucket, false)),
|
||||
)
|
||||
.await;
|
||||
let expected = if fail_target_write {
|
||||
S3ErrorCode::InternalError
|
||||
} else {
|
||||
S3ErrorCode::NoSuchBucket
|
||||
};
|
||||
let stored = persisted_replication(&bucket).await;
|
||||
assert_eq!(
|
||||
(result.as_ref().err().map(|error| error.code().clone()), stored),
|
||||
(Some(expected), expected_storage.lock().expect("expected bytes").clone())
|
||||
);
|
||||
if fail_target_write {
|
||||
assert_eq!(result.unwrap_err().message(), Some("replication target cleanup failed"));
|
||||
}
|
||||
assert!(reloads.is_empty() && hooks.is_empty());
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,8 +20,11 @@ use super::{OpRow, Served, row};
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::app::bucket_usecase::DefaultBucketUsecase;
|
||||
use crate::runtime_sources::AppContext;
|
||||
use crate::storage_api::server::http::access::authorize_bucket_request;
|
||||
use crate::storage_api::server::http::access::{
|
||||
access_owned::bucket_replication_lifecycle::prepare_replication_config_edit, authorize_bucket_request,
|
||||
};
|
||||
use rustfs_gateway::{Req, Resp, dto};
|
||||
use rustfs_gateway_types::persistence::{parse_replication, serialize_replication_dto};
|
||||
use rustfs_policy::policy::action::{Action, S3Action};
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_s3_types::S3Result;
|
||||
@@ -32,6 +35,8 @@ pub(super) const ROWS: &[OpRow] = &[
|
||||
row::<dto::PutBucketLifecycleConfiguration>(),
|
||||
row::<dto::DeleteBucketLifecycle>(),
|
||||
row::<dto::GetBucketReplication>(),
|
||||
row::<dto::PutBucketReplication>(),
|
||||
row::<dto::DeleteBucketReplication>(),
|
||||
row::<dto::GetObjectLockConfiguration>(),
|
||||
];
|
||||
|
||||
@@ -67,6 +72,47 @@ impl Served for dto::GetBucketReplication {
|
||||
}
|
||||
}
|
||||
|
||||
impl Served for dto::PutBucketReplication {
|
||||
const METRIC: S3Operation = S3Operation::PutBucketReplication;
|
||||
|
||||
async fn authorize(envelope: &mut RequestEnvelope, request: &Req<Self>) -> S3Result<()> {
|
||||
let action = Action::S3Action(S3Action::PutReplicationConfigurationAction);
|
||||
authorize_bucket_config_mutation(envelope, request.input().bucket.as_str(), action).await
|
||||
}
|
||||
|
||||
async fn execute(context: Option<Arc<AppContext>>, envelope: RequestEnvelope, request: Req<Self>) -> S3Result<Resp<Self>> {
|
||||
let input = request.into_input();
|
||||
let prepare = prepare_replication_config_edit(&envelope);
|
||||
DefaultBucketUsecase::with_context(context)
|
||||
.execute_put_bucket_replication(envelope, input.bucket.as_str(), async move {
|
||||
let (site_peers, contract) = prepare.await?;
|
||||
let bytes = serialize_replication_dto(&input.replication_configuration)
|
||||
.map_err(|err| rustfs_s3_types::s3_error!(MalformedXML, "{err}"))?;
|
||||
let configuration = parse_replication(&bytes).map_err(|err| rustfs_s3_types::s3_error!(MalformedXML, "{err}"))?;
|
||||
Ok((configuration, site_peers, contract))
|
||||
})
|
||||
.await?;
|
||||
Ok(Resp::new(dto::PutBucketReplicationOutput::default()))
|
||||
}
|
||||
}
|
||||
|
||||
impl Served for dto::DeleteBucketReplication {
|
||||
const METRIC: S3Operation = S3Operation::DeleteBucketReplication;
|
||||
|
||||
async fn authorize(envelope: &mut RequestEnvelope, request: &Req<Self>) -> S3Result<()> {
|
||||
let action = Action::S3Action(S3Action::PutReplicationConfigurationAction);
|
||||
authorize_bucket_config_mutation(envelope, request.input().bucket.as_str(), action).await
|
||||
}
|
||||
|
||||
async fn execute(context: Option<Arc<AppContext>>, envelope: RequestEnvelope, request: Req<Self>) -> S3Result<Resp<Self>> {
|
||||
let prepare = prepare_replication_config_edit(&envelope);
|
||||
DefaultBucketUsecase::with_context(context)
|
||||
.execute_delete_bucket_replication(envelope, request.input().bucket.as_str(), prepare)
|
||||
.await?;
|
||||
Ok(Resp::new(dto::DeleteBucketReplicationOutput::default()))
|
||||
}
|
||||
}
|
||||
|
||||
impl Served for dto::GetObjectLockConfiguration {
|
||||
const METRIC: S3Operation = S3Operation::GetObjectLockConfiguration;
|
||||
|
||||
|
||||
@@ -1007,6 +1007,7 @@ pub(crate) struct BucketMetaHookCall {
|
||||
pub(crate) bucket: String,
|
||||
pub(crate) item_type: String,
|
||||
pub(crate) scanner_generation: Option<u64>,
|
||||
pub(crate) replication_config: Option<String>,
|
||||
pub(crate) scanner_maintenance_generation: u64,
|
||||
}
|
||||
|
||||
@@ -1021,6 +1022,7 @@ pub async fn site_replication_bucket_meta_hook(mut item: SRBucketMeta) -> S3Resu
|
||||
calls.borrow_mut().push(BucketMetaHookCall {
|
||||
bucket: item.bucket.clone(),
|
||||
item_type: item.r#type.clone(),
|
||||
replication_config: item.replication_config.clone(),
|
||||
scanner_generation: rustfs_scanner::scanner_dirty_usage_bucket_generation(&item.bucket),
|
||||
scanner_maintenance_generation: rustfs_scanner::scanner_maintenance_generation(),
|
||||
});
|
||||
|
||||
@@ -12,11 +12,82 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Authorization shapes that only migration batch A3 (rustfs/backlog#2765) uses: bucket lifecycle, replication and object lock configuration.
|
||||
//! Prepares external S3 replication edits after IAM authorization.
|
||||
//!
|
||||
//! Responsible for: the owned authorization function of each such operation, generic over
|
||||
//! `AuthorizationRequest`, so the legacy `S3Access` hook and `Served::authorize` call the same code.
|
||||
//! Not responsible for: shapes more than one batch shares (those stay in `storage/access.rs`), the
|
||||
//! policy evaluation itself (`authorize_request`), or the operation's use case.
|
||||
//! Upstream: the legacy hook in `storage/access.rs` and the batch's `server/gateway_handlers/bucket_replication_lifecycle.rs`.
|
||||
//! Downstream: the shared shapes in `storage/access.rs`.
|
||||
//! The interface resolves owner privileges and the site contract. The bucket
|
||||
//! use case receives those facts without consulting site membership itself.
|
||||
|
||||
use crate::storage::access::{AuthorizationRequest, req_info_ref};
|
||||
use crate::storage::storage_api::ecstore_bucket::replication::OperatorRuleContract;
|
||||
use http::StatusCode;
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
|
||||
use std::collections::HashSet;
|
||||
use std::future::Future;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) static SITE_REPLICATION_GATE_TEST_OVERRIDE: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0);
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) const SITE_REPLICATION_GATE_FORCE_DISABLED: u8 = 1;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) const SITE_REPLICATION_GATE_FORCE_ENABLED: u8 = 2;
|
||||
|
||||
async fn site_replication_gate_enabled() -> S3Result<bool> {
|
||||
#[cfg(test)]
|
||||
match SITE_REPLICATION_GATE_TEST_OVERRIDE.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
SITE_REPLICATION_GATE_FORCE_DISABLED => return Ok(false),
|
||||
SITE_REPLICATION_GATE_FORCE_ENABLED => return Ok(true),
|
||||
_ => {}
|
||||
}
|
||||
crate::admin::handlers::site_replication::site_replication_enabled().await
|
||||
}
|
||||
|
||||
/// Remote site-replication peer deployment ids and the operator-rule contract
|
||||
/// the peers support, handed to the bucket usecase so an S3 replication-config
|
||||
/// edit keeps exactly the reconciler-owned rules and merges the way every
|
||||
/// peer will (issue #1948). Read here, in the interface layer, because the
|
||||
/// usecase must not import the admin handlers (layer guard); a state-read
|
||||
/// failure propagates so the edit fails closed.
|
||||
async fn site_replication_edit_context() -> S3Result<(HashSet<String>, OperatorRuleContract)> {
|
||||
// While the gate override is in effect the test exercises the deny/allow
|
||||
// branch, not the peer set; there is no persisted state to read.
|
||||
#[cfg(test)]
|
||||
if SITE_REPLICATION_GATE_TEST_OVERRIDE.load(std::sync::atomic::Ordering::SeqCst) != 0 {
|
||||
return Ok((HashSet::new(), OperatorRuleContract::Derived));
|
||||
}
|
||||
crate::admin::handlers::site_replication::site_replication_edit_context().await
|
||||
}
|
||||
|
||||
/// MinIO `ErrReplicationDenyEditError`.
|
||||
fn replication_deny_edit_error() -> S3Error {
|
||||
let mut err = S3Error::with_message(
|
||||
S3ErrorCode::Custom("XMinioReplicationDenyEdit".into()),
|
||||
"Sub-User is not allowed to edit Replication configuration",
|
||||
);
|
||||
err.set_status_code(StatusCode::BAD_REQUEST);
|
||||
err
|
||||
}
|
||||
|
||||
/// Site-replication gate for S3 replication-config edits (issue #1948).
|
||||
///
|
||||
/// On a site-replication deployment the bucket's replication config carries
|
||||
/// the operator-managed `site-repl-*` rules that keep every peer in sync, and
|
||||
/// a successful edit is broadcast to all peers — so a user holding only
|
||||
/// bucket-scoped `s3:PutReplicationConfiguration` could rewrite or erase
|
||||
/// replication net-wide. MinIO parity (`ErrReplicationDenyEditError`): only
|
||||
/// owner credentials (root or root-parented) may edit. Runs after the policy
|
||||
/// authorization in the access layer and only on the external S3 path — the
|
||||
/// reconciler and peer bucket-meta ingestion never route through these
|
||||
/// handlers.
|
||||
pub(crate) fn prepare_replication_config_edit<R: AuthorizationRequest + ?Sized>(
|
||||
request: &R,
|
||||
) -> impl Future<Output = S3Result<(HashSet<String>, OperatorRuleContract)>> + Send + use<R> {
|
||||
let is_owner = req_info_ref(request).map(|info| info.is_owner).map_err(S3Error::from);
|
||||
async move {
|
||||
if !is_owner? && site_replication_gate_enabled().await? {
|
||||
return Err(replication_deny_edit_error());
|
||||
}
|
||||
site_replication_edit_context().await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,9 +21,9 @@
|
||||
|
||||
use crate::admin::handlers::site_replication::site_replication_bucket_meta_hook;
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::app::storage_api::object_usecase::bucket::replication::OperatorRuleContract;
|
||||
use crate::error::ApiError;
|
||||
use crate::storage::StorageVersioningConfigExt as _;
|
||||
use crate::storage::access::access_owned::bucket_replication_lifecycle::prepare_replication_config_edit;
|
||||
use crate::storage::access::bucket_config_mutation_incarnation;
|
||||
use crate::storage::ecfs::*;
|
||||
use crate::storage::helper::audit_only_operation;
|
||||
@@ -38,7 +38,6 @@ use crate::storage::{
|
||||
BUCKET_VERSIONING_CONFIG, BucketVersioningSys, OBJECT_LOCK_CONFIG, StorageError, get_bucket_object_lock_config,
|
||||
update_bucket_metadata_config_if_incarnation,
|
||||
};
|
||||
use http::StatusCode;
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::{
|
||||
get_bucket_lifecycle_configuration, get_bucket_replication, get_object_lock_configuration,
|
||||
};
|
||||
@@ -49,102 +48,8 @@ use tracing::{instrument, warn};
|
||||
|
||||
const LOG_SUBSYSTEM_OBJECT_LOCK: &str = "object_lock";
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) static SITE_REPLICATION_GATE_TEST_OVERRIDE: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0);
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) const SITE_REPLICATION_GATE_FORCE_DISABLED: u8 = 1;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) const SITE_REPLICATION_GATE_FORCE_ENABLED: u8 = 2;
|
||||
|
||||
async fn site_replication_gate_enabled() -> S3Result<bool> {
|
||||
#[cfg(test)]
|
||||
match SITE_REPLICATION_GATE_TEST_OVERRIDE.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
SITE_REPLICATION_GATE_FORCE_DISABLED => return Ok(false),
|
||||
SITE_REPLICATION_GATE_FORCE_ENABLED => return Ok(true),
|
||||
_ => {}
|
||||
}
|
||||
crate::admin::handlers::site_replication::site_replication_enabled()
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Remote site-replication peer deployment ids and the operator-rule contract
|
||||
/// the peers support, handed to the bucket usecase so an S3 replication-config
|
||||
/// edit keeps exactly the reconciler-owned rules and merges the way every
|
||||
/// peer will (issue #1948). Read here, in the interface layer, because the
|
||||
/// usecase must not import the admin handlers (layer guard); a state-read
|
||||
/// failure propagates so the edit fails closed.
|
||||
async fn site_replication_edit_context() -> S3Result<(std::collections::HashSet<String>, OperatorRuleContract)> {
|
||||
// While the gate override is in effect the test exercises the deny/allow
|
||||
// branch, not the peer set; there is no persisted state to read.
|
||||
#[cfg(test)]
|
||||
if SITE_REPLICATION_GATE_TEST_OVERRIDE.load(std::sync::atomic::Ordering::SeqCst) != 0 {
|
||||
return Ok((std::collections::HashSet::new(), OperatorRuleContract::Derived));
|
||||
}
|
||||
crate::admin::handlers::site_replication::site_replication_edit_context()
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
/// MinIO `ErrReplicationDenyEditError`.
|
||||
fn replication_deny_edit_error() -> S3Error {
|
||||
let mut err = S3Error::with_message(
|
||||
S3ErrorCode::Custom("XMinioReplicationDenyEdit".into()),
|
||||
"Sub-User is not allowed to edit Replication configuration",
|
||||
);
|
||||
err.set_status_code(StatusCode::BAD_REQUEST);
|
||||
err
|
||||
}
|
||||
|
||||
/// Site-replication gate for S3 replication-config edits (issue #1948).
|
||||
///
|
||||
/// On a site-replication deployment the bucket's replication config carries
|
||||
/// the operator-managed `site-repl-*` rules that keep every peer in sync, and
|
||||
/// a successful edit is broadcast to all peers — so a user holding only
|
||||
/// bucket-scoped `s3:PutReplicationConfiguration` could rewrite or erase
|
||||
/// replication net-wide. MinIO parity (`ErrReplicationDenyEditError`): only
|
||||
/// owner credentials (root or root-parented) may edit. Runs after the policy
|
||||
/// authorization in the access layer and only on the external S3 path — the
|
||||
/// reconciler and peer bucket-meta ingestion never route through these
|
||||
/// handlers.
|
||||
async fn deny_replication_config_edit_for_non_owner<T>(req: &S3Request<T>) -> S3Result<()> {
|
||||
if crate::storage::access::req_info_ref(req)?.is_owner {
|
||||
return Ok(());
|
||||
}
|
||||
if site_replication_gate_enabled().await? {
|
||||
return Err(replication_deny_edit_error());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Handler bodies that [`S3`] runs through `audit_only_operation`.
|
||||
/// Handler body that runs through `audit_only_operation`.
|
||||
impl FS {
|
||||
async fn delete_bucket_replication_inner(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketReplicationInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketReplicationOutput>> {
|
||||
deny_replication_config_edit_for_non_owner(&req).await?;
|
||||
let (site_peers, contract) = site_replication_edit_context().await?;
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_delete_bucket_replication(req, site_peers, contract).await
|
||||
}
|
||||
|
||||
async fn put_bucket_replication_inner(
|
||||
&self,
|
||||
mut req: S3Request<PutBucketReplicationInput>,
|
||||
) -> S3Result<S3Response<PutBucketReplicationOutput>> {
|
||||
deny_replication_config_edit_for_non_owner(&req).await?;
|
||||
let (site_peers, contract) = site_replication_edit_context().await?;
|
||||
let replication_configuration =
|
||||
replication_configuration_from_s3s(std::mem::take(&mut req.input.replication_configuration));
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase
|
||||
.execute_put_bucket_replication(req, replication_configuration, site_peers, contract)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn put_object_lock_configuration_inner(
|
||||
&self,
|
||||
req: S3Request<PutObjectLockConfigurationInput>,
|
||||
@@ -274,7 +179,12 @@ impl FS {
|
||||
&self,
|
||||
req: S3Request<DeleteBucketReplicationInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketReplicationOutput>> {
|
||||
audit_only_operation(req, S3Operation::DeleteBucketReplication, |req| self.delete_bucket_replication_inner(req)).await
|
||||
let envelope = RequestEnvelope::from_s3s(&req);
|
||||
let prepare = prepare_replication_config_edit(&envelope);
|
||||
s3_api::bucket_usecase_for(self)
|
||||
.execute_delete_bucket_replication(envelope, &req.input.bucket, prepare)
|
||||
.await?;
|
||||
Ok(S3Response::new(DeleteBucketReplicationOutput::default()))
|
||||
}
|
||||
|
||||
#[instrument(name = "get_bucket_lifecycle_configuration", level = "debug", skip(self, req))]
|
||||
@@ -357,7 +267,20 @@ impl FS {
|
||||
&self,
|
||||
req: S3Request<PutBucketReplicationInput>,
|
||||
) -> S3Result<S3Response<PutBucketReplicationOutput>> {
|
||||
audit_only_operation(req, S3Operation::PutBucketReplication, |req| self.put_bucket_replication_inner(req)).await
|
||||
let envelope = RequestEnvelope::from_s3s(&req);
|
||||
let prepare = prepare_replication_config_edit(&envelope);
|
||||
let PutBucketReplicationInput {
|
||||
bucket,
|
||||
replication_configuration,
|
||||
..
|
||||
} = req.input;
|
||||
s3_api::bucket_usecase_for(self)
|
||||
.execute_put_bucket_replication(envelope, &bucket, async move {
|
||||
let (site_peers, contract) = prepare.await?;
|
||||
Ok((replication_configuration_from_s3s(replication_configuration), site_peers, contract))
|
||||
})
|
||||
.await?;
|
||||
Ok(S3Response::new(PutBucketReplicationOutput::default()))
|
||||
}
|
||||
|
||||
#[instrument(name = "put_object_lock_configuration", level = "debug", skip(self, req))]
|
||||
|
||||
@@ -15,11 +15,11 @@
|
||||
//! Tests of the `bucket_replication_lifecycle` legacy-edge handlers (batch A3), moved unchanged
|
||||
//! from `ecfs.rs` and `ecfs_test.rs`.
|
||||
|
||||
use super::super::bucket_replication_lifecycle::{
|
||||
SITE_REPLICATION_GATE_FORCE_DISABLED, SITE_REPLICATION_GATE_FORCE_ENABLED, SITE_REPLICATION_GATE_TEST_OVERRIDE,
|
||||
};
|
||||
use super::{build_request, store_uninitialized_premise_holds};
|
||||
use crate::storage::access::ReqInfo;
|
||||
use crate::storage::access::access_owned::bucket_replication_lifecycle::{
|
||||
SITE_REPLICATION_GATE_FORCE_DISABLED, SITE_REPLICATION_GATE_FORCE_ENABLED, SITE_REPLICATION_GATE_TEST_OVERRIDE,
|
||||
};
|
||||
use crate::storage::ecfs::*;
|
||||
use http::Method;
|
||||
use http::StatusCode;
|
||||
|
||||
@@ -24,11 +24,16 @@ use super::{
|
||||
outcome, s3_client, with_stack,
|
||||
};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::client::customize::CustomizableOperation;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata as _;
|
||||
use aws_sdk_s3::operation::put_bucket_replication::builders::PutBucketReplicationFluentBuilder;
|
||||
use aws_sdk_s3::operation::put_bucket_replication::{PutBucketReplicationError, PutBucketReplicationOutput};
|
||||
use futures::future::BoxFuture;
|
||||
use rustfs::app::context::get_global_app_context;
|
||||
use rustfs::embedded::RustFSServer;
|
||||
use rustfs_ecstore::api::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_REPLICATION_CONFIG, OBJECT_LOCK_CONFIG};
|
||||
use rustfs_ecstore::api::bucket::metadata::{
|
||||
BUCKET_LIFECYCLE_CONFIG, BUCKET_REPLICATION_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, OBJECT_LOCK_CONFIG,
|
||||
};
|
||||
use rustfs_ecstore::api::bucket::metadata_sys;
|
||||
use rustfs_iam::store::UserType;
|
||||
use rustfs_policy::policy::Policy;
|
||||
@@ -74,6 +79,37 @@ pub(crate) const MATRIX: &[MatrixRow] = &[
|
||||
has_error_body: true,
|
||||
call: stored::<1>,
|
||||
},
|
||||
MatrixRow {
|
||||
operation: "PutBucketReplication",
|
||||
metric: S3Operation::PutBucketReplication,
|
||||
has_error_body: true,
|
||||
call: |client, _| {
|
||||
Box::pin(async move {
|
||||
seed_replication_target(REPLICATION_MATRIX_BUCKET).await;
|
||||
outcome(
|
||||
put_replication(client, REPLICATION_MATRIX_BUCKET, REPLICATION_WRITE)
|
||||
.send()
|
||||
.await,
|
||||
)
|
||||
})
|
||||
},
|
||||
},
|
||||
MatrixRow {
|
||||
operation: "DeleteBucketReplication",
|
||||
metric: S3Operation::DeleteBucketReplication,
|
||||
has_error_body: true,
|
||||
call: |client, _| {
|
||||
Box::pin(async move {
|
||||
outcome(
|
||||
client
|
||||
.delete_bucket_replication()
|
||||
.bucket(REPLICATION_MATRIX_BUCKET)
|
||||
.send()
|
||||
.await,
|
||||
)
|
||||
})
|
||||
},
|
||||
},
|
||||
MatrixRow {
|
||||
operation: "GetObjectLockConfiguration",
|
||||
metric: S3Operation::GetObjectLockConfiguration,
|
||||
@@ -628,3 +664,245 @@ fn gateway_lifecycle_put_rejects_submillisecond_date() {
|
||||
server.shutdown().await;
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) const REPLICATION_MATRIX_BUCKET: &str = "replication-matrix";
|
||||
const REPLICATION_ARN: &str = "arn:rustfs:replication:us-east-1:operator:replica";
|
||||
const REPLICATION_WRITE: &str = concat!(
|
||||
"<ReplicationConfiguration><Role></Role><Rule><DeleteMarkerReplication><Status>Disabled</Status></DeleteMarkerReplication>",
|
||||
"<DeleteReplication><Status>Enabled</Status></DeleteReplication><Destination><Bucket>arn:rustfs:replication:us-east-1:operator:replica</Bucket>",
|
||||
"<StorageClass>STANDARD</StorageClass></Destination><ExistingObjectReplication><Status>Enabled</Status></ExistingObjectReplication>",
|
||||
"<Filter><And><Prefix>logs/&/</Prefix><Tag><Key>team</Key><Value>storage</Value></Tag></And></Filter>",
|
||||
"<ID>operator</ID><Priority>7</Priority><SourceSelectionCriteria><ReplicaModifications><Status>Enabled</Status>",
|
||||
"</ReplicaModifications></SourceSelectionCriteria><Status>Enabled</Status></Rule></ReplicationConfiguration>",
|
||||
);
|
||||
const REPLICATION_SIMPLE: &str = concat!(
|
||||
"<ReplicationConfiguration><Role></Role><Rule><Destination><Bucket>arn:rustfs:replication:us-east-1:operator:replica</Bucket>",
|
||||
"</Destination><ID>simple</ID><Status>Enabled</Status></Rule></ReplicationConfiguration>",
|
||||
);
|
||||
const REPLICATION_UNSUPPORTED: &str = concat!(
|
||||
"<ReplicationConfiguration><Role></Role><Rule><Destination><Account>123456789012</Account>",
|
||||
"<Bucket>arn:rustfs:replication:us-east-1:operator:replica</Bucket></Destination><Status>Enabled</Status></Rule></ReplicationConfiguration>",
|
||||
);
|
||||
|
||||
async fn seed_replication_target(bucket: &str) {
|
||||
use rustfs_ecstore::api::bucket::target::{BucketTarget, BucketTargetType, BucketTargets};
|
||||
|
||||
metadata_sys::update(
|
||||
bucket,
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("enable replication source versioning");
|
||||
let targets = BucketTargets {
|
||||
targets: vec![BucketTarget {
|
||||
source_bucket: bucket.to_owned(),
|
||||
target_bucket: "replica".to_owned(),
|
||||
arn: REPLICATION_ARN.to_owned(),
|
||||
target_type: BucketTargetType::ReplicationService,
|
||||
..Default::default()
|
||||
}],
|
||||
};
|
||||
metadata_sys::update(bucket, BUCKET_TARGETS_FILE, serde_json::to_vec(&targets).expect("target bytes"))
|
||||
.await
|
||||
.expect("write replication targets");
|
||||
}
|
||||
|
||||
fn put_replication(
|
||||
client: &Client,
|
||||
bucket: &str,
|
||||
document: &'static str,
|
||||
) -> CustomizableOperation<PutBucketReplicationOutput, PutBucketReplicationError, PutBucketReplicationFluentBuilder> {
|
||||
use aws_sdk_s3::types::{Destination, ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus};
|
||||
let configuration = ReplicationConfiguration::builder()
|
||||
.role("")
|
||||
.rules(
|
||||
ReplicationRule::builder()
|
||||
.destination(
|
||||
Destination::builder()
|
||||
.bucket(REPLICATION_ARN)
|
||||
.build()
|
||||
.expect("placeholder destination"),
|
||||
)
|
||||
.status(ReplicationRuleStatus::Enabled)
|
||||
.build()
|
||||
.expect("placeholder rule"),
|
||||
)
|
||||
.build()
|
||||
.expect("placeholder configuration");
|
||||
client
|
||||
.put_bucket_replication()
|
||||
.bucket(bucket)
|
||||
.replication_configuration(configuration)
|
||||
.customize()
|
||||
.mutate_request(move |request| send_document(request, document))
|
||||
}
|
||||
|
||||
async fn owned_replication_wire_and_disk(server: RustFSServer) {
|
||||
const BUCKET: &str = "replication-edit-granted";
|
||||
const OTHER: &str = "replication-edit-other";
|
||||
const WRITER: &str = "replicationeditwriter";
|
||||
const READER: &str = "replicationeditreader";
|
||||
for bucket in [BUCKET, OTHER] {
|
||||
create_bucket_in_the_store(bucket).await;
|
||||
}
|
||||
create_user_allowed_on_one_bucket(WRITER, "s3:PutReplicationConfiguration", BUCKET).await;
|
||||
create_user_allowed_on_one_bucket(READER, "s3:GetReplicationConfiguration", BUCKET).await;
|
||||
let endpoint = server.endpoint();
|
||||
let root = s3_client(&endpoint, server.access_key(), server.secret_key());
|
||||
let writer = s3_client(&endpoint, WRITER, USER_SECRET_KEY);
|
||||
let reader = s3_client(&endpoint, READER, USER_SECRET_KEY);
|
||||
|
||||
let unsupported = put_replication(&root, BUCKET, REPLICATION_UNSUPPORTED)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("unsupported destination");
|
||||
let disabled = put_replication(&root, BUCKET, REPLICATION_SIMPLE)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("versioning disabled");
|
||||
for (err, message) in [
|
||||
(
|
||||
unsupported,
|
||||
"replication field Destination.Account is not supported by this RustFS version",
|
||||
),
|
||||
(disabled, "bucket versioning must be enabled before replication can be configured"),
|
||||
] {
|
||||
let refusal = err.as_service_error().expect("service refusal");
|
||||
assert_eq!((refusal.code(), refusal.message()), (Some("InvalidRequest"), Some(message)));
|
||||
}
|
||||
|
||||
seed_replication_target(BUCKET).await;
|
||||
seed_replication_target(OTHER).await;
|
||||
for (client, bucket) in [(&reader, BUCKET), (&writer, OTHER)] {
|
||||
assert_eq!(
|
||||
answer(outcome(put_replication(client, bucket, REPLICATION_SIMPLE).send().await)),
|
||||
refused("AccessDenied", 403)
|
||||
);
|
||||
}
|
||||
for bucket in [BUCKET, OTHER] {
|
||||
let stored = metadata_sys::get_config_from_disk(bucket)
|
||||
.await
|
||||
.expect("disk metadata before Put");
|
||||
assert!(stored.replication_config_xml.is_empty());
|
||||
}
|
||||
let stale = put_replication(
|
||||
&writer,
|
||||
BUCKET,
|
||||
"<ReplicationConfiguration><Role></Role><Rule><Destination><Bucket>arn:rustfs:replication:us-east-1:absent:replica</Bucket></Destination><Status>Enabled</Status></Rule></ReplicationConfiguration>",
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("stale target");
|
||||
let stale = stale.as_service_error().expect("stale target refusal");
|
||||
assert_eq!(
|
||||
(stale.code(), stale.message()),
|
||||
(Some("InvalidRequest"), Some("replication config has a stale target"))
|
||||
);
|
||||
|
||||
assert_eq!(answer(outcome(put_replication(&writer, BUCKET, REPLICATION_WRITE).send().await)), Ok(()));
|
||||
let actual = metadata_sys::get_config_from_disk(BUCKET)
|
||||
.await
|
||||
.expect("disk metadata after Put");
|
||||
assert_eq!(actual.replication_config_xml, REPLICATION_WRITE.as_bytes());
|
||||
|
||||
// Keep actual bytes to observe attempted deletes under both wrong policies.
|
||||
metadata_sys::update(OTHER, BUCKET_REPLICATION_CONFIG, REPLICATION_WRITE.as_bytes().to_vec())
|
||||
.await
|
||||
.expect("persist other bucket's rules");
|
||||
for (client, bucket) in [(&reader, BUCKET), (&writer, OTHER)] {
|
||||
assert_eq!(
|
||||
answer(outcome(client.delete_bucket_replication().bucket(bucket).send().await)),
|
||||
refused("AccessDenied", 403)
|
||||
);
|
||||
let stored = metadata_sys::get_config_from_disk(bucket)
|
||||
.await
|
||||
.expect("disk metadata after refused Delete");
|
||||
assert_eq!(stored.replication_config_xml, REPLICATION_WRITE.as_bytes());
|
||||
}
|
||||
|
||||
for _ in 0..2 {
|
||||
let status = StatusCapture::default();
|
||||
let deleted = writer
|
||||
.delete_bucket_replication()
|
||||
.bucket(BUCKET)
|
||||
.customize()
|
||||
.interceptor(status.clone())
|
||||
.send()
|
||||
.await;
|
||||
assert_eq!((answer(outcome(deleted)), status.take()), (Ok(()), Some(204)));
|
||||
let stored = metadata_sys::get_config_from_disk(BUCKET)
|
||||
.await
|
||||
.expect("disk metadata after Delete");
|
||||
assert!(stored.replication_config_xml.is_empty());
|
||||
let targets: rustfs_ecstore::api::bucket::target::BucketTargets =
|
||||
serde_json::from_slice(&stored.bucket_targets_config_json).expect("disk targets after Delete");
|
||||
assert!(targets.targets.is_empty());
|
||||
}
|
||||
|
||||
// Real persisted state is consulted after IAM authorization on both edges.
|
||||
let store = get_global_app_context().expect("embedded server context").object_store();
|
||||
let state_path = "config/site-replication/state.json";
|
||||
let state = br#"{
|
||||
"name":"local","service_account_access_key":"","service_account_parent":"",
|
||||
"peers":{"local":{"deploymentID":"local","endpoint":"http://127.0.0.1:9001"},"remote":{"deploymentID":"remote","endpoint":"http://127.0.0.1:9002"}},
|
||||
"updated_at":null,"resync_status":{}
|
||||
}"#;
|
||||
rustfs_ecstore::api::config::com::save_config(store.clone(), state_path, state.to_vec())
|
||||
.await
|
||||
.expect("enable site replication without peer credentials");
|
||||
let before = metadata_sys::get_config_from_disk(BUCKET)
|
||||
.await
|
||||
.expect("before site-protected edits");
|
||||
let gate = refused("XMinioReplicationDenyEdit", 400);
|
||||
assert_eq!(answer(outcome(put_replication(&writer, BUCKET, REPLICATION_SIMPLE).send().await)), gate);
|
||||
assert_eq!(answer(outcome(writer.delete_bucket_replication().bucket(BUCKET).send().await)), gate);
|
||||
// IAM still denies an unrelated principal before the site gate.
|
||||
assert_eq!(
|
||||
answer(outcome(reader.delete_bucket_replication().bucket(BUCKET).send().await)),
|
||||
refused("AccessDenied", 403)
|
||||
);
|
||||
|
||||
rustfs_ecstore::api::config::com::save_config(store.clone(), state_path, b"{".to_vec())
|
||||
.await
|
||||
.expect("store corrupt site replication state");
|
||||
for client in [&root, &writer] {
|
||||
let err = put_replication(client, BUCKET, REPLICATION_SIMPLE)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("corrupt site replication state");
|
||||
let error = err.as_service_error().expect("state read failure");
|
||||
assert_eq!(error.code(), Some("InternalError"));
|
||||
assert!(
|
||||
error
|
||||
.message()
|
||||
.is_some_and(|message| message.starts_with("invalid site replication state:"))
|
||||
);
|
||||
assert_eq!(
|
||||
answer(outcome(client.delete_bucket_replication().bucket(BUCKET).send().await)),
|
||||
refused("InternalError", 500)
|
||||
);
|
||||
}
|
||||
let after = metadata_sys::get_config_from_disk(BUCKET)
|
||||
.await
|
||||
.expect("after site-protected edits");
|
||||
assert_eq!(
|
||||
(after.replication_config_xml, after.bucket_targets_config_json),
|
||||
(before.replication_config_xml, before.bucket_targets_config_json)
|
||||
);
|
||||
rustfs_ecstore::api::config::com::delete_config(store, state_path)
|
||||
.await
|
||||
.expect("remove test site state");
|
||||
|
||||
server.shutdown().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn owned_replication_gateway_writes_and_removes_the_config_and_targets() {
|
||||
with_stack(GATEWAY, owned_replication_wire_and_disk);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn owned_replication_legacy_writes_and_removes_the_config_and_targets() {
|
||||
with_stack(LEGACY, owned_replication_wire_and_disk);
|
||||
}
|
||||
|
||||
@@ -356,6 +356,7 @@ async fn every_served_operation_serves_root_and_refuses_the_callers_it_cannot_ad
|
||||
create_bucket_in_the_store(bucket_configs::ENCRYPTION_MATRIX_BUCKET).await;
|
||||
bucket_configs::store_read_configs(MATRIX_BUCKET, bucket_configs::ENCRYPTION_MATRIX_BUCKET).await;
|
||||
bucket_core::seed_website(MATRIX_BUCKET).await;
|
||||
create_bucket_in_the_store(bucket_replication_lifecycle::REPLICATION_MATRIX_BUCKET).await;
|
||||
create_bucket_in_the_store(object_meta::MATRIX_BUCKET).await;
|
||||
create_bucket_in_the_store(object_misc::MATRIX_BUCKET).await;
|
||||
create_bucket_in_the_store(object_read::MATRIX_BUCKET).await;
|
||||
@@ -400,6 +401,8 @@ async fn every_served_operation_serves_root_and_refuses_the_callers_it_cannot_ad
|
||||
| S3Operation::PutBucketRequestPayment
|
||||
| S3Operation::PutBucketWebsite
|
||||
| S3Operation::DeleteBucketWebsite
|
||||
| S3Operation::PutBucketReplication
|
||||
| S3Operation::DeleteBucketReplication
|
||||
| S3Operation::PutBucketEncryption
|
||||
| S3Operation::PutBucketTagging
|
||||
| S3Operation::DeleteBucketEncryption
|
||||
|
||||
Reference in New Issue
Block a user