Compare commits

..

2 Commits

Author SHA1 Message Date
cxymds 8c90f212a3 fix(replication): declare native error macro at removal boundary 2026-11-17 20:20:31 +08:00
cxymds b96f913375 fix(replication): durably decommission unreachable sites 2026-11-17 20:09:08 +08:00
19 changed files with 1902 additions and 149 deletions
+2 -2
View File
@@ -253,7 +253,7 @@ threads-required = "num-test-threads"
# peers complete durable IO. Reserve capacity from unrelated test processes;
# the shared fixture's durable_iam_state_ prefix keeps the whole family together.
[[profile.default.overrides]]
filter = 'package(rustfs) & test(/^admin::handlers::site_replication::tests::durable_iam_state_/)'
filter = 'package(rustfs) & test(/^admin::handlers::site_replication::(tests::durable_iam_state_|removal_tests::decommission_durable_)/)'
threads-required = "num-test-threads"
# Real target-repair and rebalance-stop fixtures commit durable metadata while
@@ -582,7 +582,7 @@ threads-required = "num-test-threads"
# Keep the same IAM/state fixture capacity reservation in CI without changing the
# production lock deadline, internal concurrency, assertions, or retry policy.
[[profile.ci.overrides]]
filter = 'package(rustfs) & test(/^admin::handlers::site_replication::tests::durable_iam_state_/)'
filter = 'package(rustfs) & test(/^admin::handlers::site_replication::(tests::durable_iam_state_|removal_tests::decommission_durable_)/)'
threads-required = "num-test-threads"
# Match the real admin fixture capacity reservation without retries or changes
@@ -2251,6 +2251,228 @@ async fn site_replication_remove(
Ok(serde_json::from_slice(&response.bytes().await?)?)
}
async fn decommission_site_v2(
env: &RustFSTestEnvironment,
request: &rustfs_madmin::SRRemoveV2Req,
) -> Result<rustfs_madmin::SRRemoveV2Status, Box<dyn Error + Send + Sync>> {
let response = signed_request(
http::Method::PUT,
&format!("{}/rustfs/admin/v3/site-replication/remove-v2", env.url),
&env.access_key,
&env.secret_key,
Some(serde_json::to_vec(request)?),
Some("application/json"),
)
.await?;
if response.status() != StatusCode::OK {
return Err(format!("decommission failed: {} {}", response.status(), response.text().await?).into());
}
Ok(response.json().await?)
}
#[tokio::test]
async fn test_site_decommission_dead_peer_recovers_pending_remove_and_bucket_delete() -> Result<(), Box<dyn Error + Send + Sync>>
{
init_logging();
let mut source = RustFSTestEnvironment::new().await?;
let mut target = RustFSTestEnvironment::new().await?;
let mut env = replication_fast_env();
env.extend_from_slice(FAST_SCANNER_ENV);
env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
source.start_rustfs_server_with_env(vec![], &env).await?;
target.start_rustfs_server_without_cleanup_with_env(&env).await?;
let added = site_replication_add(
&source,
&[
PeerSite {
name: "survivor".to_string(),
endpoint: source.url.clone(),
access_key: source.access_key.clone(),
secret_key: source.secret_key.clone(),
..Default::default()
},
PeerSite {
name: "retired".to_string(),
endpoint: target.url.clone(),
access_key: target.access_key.clone(),
secret_key: target.secret_key.clone(),
..Default::default()
},
],
)
.await?;
assert!(added.success, "{added:?}");
let source_client = source.create_s3_client();
let target_client = target.create_s3_client();
let bucket = "decommission-pending-delete";
source_client.create_bucket().bucket(bucket).send().await?;
wait_for_bucket_on_target(&target_client, bucket).await?;
let version = source_client
.put_object()
.bucket(bucket)
.key("data")
.body(ByteStream::from_static(b"decommission data survives until purge"))
.send()
.await?
.version_id
.ok_or("site bucket must be versioned")?;
wait_for_object_on_target(&target_client, bucket, "data").await?;
target.stop_server();
source_client
.delete_object()
.bucket(bucket)
.key("data")
.version_id(&version)
.send()
.await?;
let blocked = source_client
.delete_bucket()
.bucket(bucket)
.send()
.await
.expect_err("pending version must block delete");
assert_eq!(blocked.code(), Some("BucketNotEmpty"));
let legacy = site_replication_remove(
&source,
&SRRemoveReq {
site_names: vec!["retired".to_string()],
..Default::default()
},
)
.await?;
assert_eq!(legacy.status, "Partial");
assert!(site_replication_info(&source).await?.pending_operation.is_some());
let mut request = rustfs_madmin::SRRemoveV2Req {
mode: "dry-run".to_string(),
sites: vec!["retired".to_string()],
..Default::default()
};
let preview = decommission_site_v2(&source, &request).await?;
assert_eq!(preview.status, "planned");
request.mode = "execute".to_string();
request.operation_id = Some(preview.operation_id.clone());
request.preflight_token = Some(preview.preflight_token);
let removed = decommission_site_v2(&source, &request).await?;
assert_eq!(removed.status, "success", "{removed:?}");
assert_site_replication_bucket_detached(&source, bucket).await?;
assert!(site_replication_info(&source).await?.pending_operation.is_none());
source.restart_server_preserving_data(vec![], &env).await?;
let replay = decommission_site_v2(&source, &request).await?;
assert_eq!(replay.status, "success");
assert_eq!(replay.operation_id, removed.operation_id);
let deadline = tokio::time::Instant::now() + Duration::from_secs(120);
loop {
match source_client.delete_bucket().bucket(bucket).send().await {
Ok(_) => break,
Err(err) if err.code() == Some("BucketNotEmpty") && tokio::time::Instant::now() < deadline => {
tokio::time::sleep(Duration::from_millis(250)).await;
}
Err(err) => return Err(err.into()),
}
}
assert!(source_client.head_bucket().bucket(bucket).send().await.is_err());
Ok(())
}
#[tokio::test]
async fn test_site_decommission_preserves_two_live_sites_and_bidirectional_replication()
-> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
let mut a = RustFSTestEnvironment::new().await?;
let mut b = RustFSTestEnvironment::new().await?;
let mut c = RustFSTestEnvironment::new().await?;
let mut env = replication_fast_env();
env.extend_from_slice(FAST_SCANNER_ENV);
env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
a.start_rustfs_server_with_env(vec![], &env).await?;
b.start_rustfs_server_without_cleanup_with_env(&env).await?;
c.start_rustfs_server_without_cleanup_with_env(&env).await?;
let added = site_replication_add(
&a,
&[
PeerSite {
name: "a".to_string(),
endpoint: a.url.clone(),
access_key: a.access_key.clone(),
secret_key: a.secret_key.clone(),
..Default::default()
},
PeerSite {
name: "b".to_string(),
endpoint: b.url.clone(),
access_key: b.access_key.clone(),
secret_key: b.secret_key.clone(),
..Default::default()
},
PeerSite {
name: "c".to_string(),
endpoint: c.url.clone(),
access_key: c.access_key.clone(),
secret_key: c.secret_key.clone(),
..Default::default()
},
],
)
.await?;
assert!(added.success, "{added:?}");
let ca = a.create_s3_client();
let cb = b.create_s3_client();
let cc = c.create_s3_client();
let bucket = "decommission-preserve-live-peers";
ca.create_bucket().bucket(bucket).send().await?;
wait_for_bucket_on_target(&cb, bucket).await?;
wait_for_bucket_on_target(&cc, bucket).await?;
ca.put_object()
.bucket(bucket)
.key("baseline")
.body(ByteStream::from_static(b"baseline"))
.send()
.await?;
wait_for_object_on_target(&cc, bucket, "baseline").await?;
b.stop_server();
let mut request = rustfs_madmin::SRRemoveV2Req {
mode: "dry-run".to_string(),
sites: vec!["b".to_string()],
..Default::default()
};
let preview = decommission_site_v2(&a, &request).await?;
request.mode = "execute".to_string();
request.operation_id = Some(preview.operation_id);
request.preflight_token = Some(preview.preflight_token);
let result = decommission_site_v2(&a, &request).await?;
assert_eq!(result.status, "success", "{result:?}");
let ia = site_replication_info(&a).await?;
let ic = site_replication_info(&c).await?;
assert!(ia.enabled && ic.enabled);
assert!(ia.pending_operation.is_none() && ic.pending_operation.is_none());
assert_eq!(
ia.sites
.iter()
.map(|peer| peer.deployment_id.clone())
.collect::<std::collections::BTreeSet<_>>(),
ic.sites.iter().map(|peer| peer.deployment_id.clone()).collect()
);
assert!(!ia.sites.iter().any(|peer| peer.name == "b"));
ca.put_object()
.bucket(bucket)
.key("a-to-c")
.body(ByteStream::from_static(b"a to c survives"))
.send()
.await?;
assert_eq!(wait_for_object_on_target(&cc, bucket, "a-to-c").await?, b"a to c survives");
cc.put_object()
.bucket(bucket)
.key("c-to-a")
.body(ByteStream::from_static(b"c to a survives"))
.send()
.await?;
assert_eq!(wait_for_object_on_target(&ca, bucket, "c-to-a").await?, b"c to a survives");
c.restart_server_preserving_data(vec![], &env).await?;
assert!(site_replication_info(&c).await?.pending_operation.is_none());
assert_eq!(decommission_site_v2(&a, &request).await?.status, "success");
Ok(())
}
async fn site_replication_state_edit(
env: &RustFSTestEnvironment,
body: &rustfs_madmin::SRStateEditReq,
+32
View File
@@ -1237,6 +1237,38 @@ pub struct SRRemoveReq {
pub remove_all: bool,
}
/// Explicit decommissioning request. The v2 route never falls back to legacy removal.
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct SRRemoveV2Req {
pub mode: String,
#[serde(default)]
pub sites: Vec<String>,
#[serde(default)]
pub all: bool,
#[serde(default)]
pub decommissioned_sites: Vec<String>,
#[serde(default)]
pub operation_id: Option<String>,
#[serde(default)]
pub preflight_token: Option<String>,
}
/// Decommissioning progress; completion describes configuration cleanup, not an empty bucket.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct SRRemoveV2Status {
pub contract_version: u32,
pub operation_id: String,
pub status: String,
pub preflight_token: String,
pub removed_deployment_ids: Vec<String>,
pub decommissioned_deployment_ids: Vec<String>,
pub pending_peers: Vec<String>,
pub local_cleanup_complete: bool,
pub errors: Vec<String>,
}
fn deserialize_vec_null_default<'de, D>(deserializer: D) -> Result<Vec<String>, D::Error>
where
D: serde::Deserializer<'de>,
+21 -6
View File
@@ -74,12 +74,27 @@ or the remote endpoint.
### `remove`
The original topology and each peer acknowledgement are persisted before the
operation finalizes. While peers remain in `pendingPeers`, restore access to
them and wait for reconciliation. If a peer is permanently gone, a new remove
request may remove all currently active unacknowledged peers; RustFS permits
that request and then finalizes against the remaining topology. Removing the
local site or all sites is also an explicit completion path.
The original topology and each peer acknowledgement are persisted before the operation finalizes. Ordinary removal requires every original remote peer to acknowledge, including the removed peer. Repeating ordinary removal, including with `--all`, resumes the recorded operation and does not bypass that acknowledgement requirement.
For a permanently retired peer, use durable decommissioning after upgrading every node in the surviving sites to support the v2 removal protocol:
```bash
rc admin replicate remove primary --site "Site 2" --force --yes
```
The CLI first requests a preview and binds execution to its operation ID, exact deployment identities, deletion scope, and current topology. Only the selected retired site is exempt from acknowledgement. Other participating sites must durably clean their local bucket replication rules and targets before acknowledging. There is no fallback to legacy removal when the server does not support the v2 route.
To dissolve the group, explicitly name every permanently retired remote site whose acknowledgement may be skipped:
```bash
rc admin replicate remove primary --all --force --decommissioned-site "Site 2" --yes
```
An existing `pending_remove` can be converted only when the request preserves its original deletion scope; the server resolves removed identities from the recorded original topology. A `Partial` response remains pending across restarts and the heavyweight reconciler retries it. Resume it with the same selectors and `--operation-id <id>`. Never erase the pending state object manually.
Decommissioning cleans site-derived rules, unreferenced site-derived targets, control-plane retries, and the removed targets' durable resync intents. Independent operator bucket replication rules and ILM targets remain. A successful response means configuration cleanup completed; it does not delete ordinary data, historical versions, or Object Lock protections. The scanner and deletion worker subsequently release versions already pending deletion when their removed target is no longer configured. Other active targets must still confirm their part of a pending purge. Use ordinary S3 version cleanup and bucket deletion afterwards.
Completed v2 removal identities and the topology timestamp remain durable to fence delayed requests. Do not downgrade surviving nodes while a v2 removal is pending; older code does not understand its journal. A previously retired deployment may rejoin only through an explicit add after the removal completes. Decommissioning does not revoke shared site-replication credentials from a reachable former peer; isolate that peer before decommissioning and use the existing credential rotation procedure if credential revocation is required.
Do not re-add a site merely to hide this marker. First compare the topology on
all reachable peers. If the same operation ID makes no progress for more than
+259 -110
View File
@@ -103,6 +103,11 @@ use uuid::Uuid;
// paths keep resolving while this file keeps only the HTTP handlers.
pub(crate) use crate::site_replication::*;
mod removal;
#[cfg(test)]
mod removal_tests;
use removal::{SiteRemovalV2CapabilitiesHandler, SiteRemovalV2Handler, SiteRemovalV2PeerHandler, resume_site_removal_v2};
// Serializes peer-join admission (staleness check -> IAM upsert -> state
// commit) across every node of this site; see admit_peer_join. Never an
// actual object — only a namespace-lock key, like the repair execution lock.
@@ -494,6 +499,17 @@ pub fn register_site_replication_route(r: &mut S3Router<AdminOperation>) -> std:
"/v3/site-replication/remove",
AdminOperation(&SiteReplicationRemoveHandler {}),
),
(Method::PUT, "/v3/site-replication/remove-v2", AdminOperation(&SiteRemovalV2Handler)),
(
Method::GET,
"/v3/site-replication/remove-v2/capabilities",
AdminOperation(&SiteRemovalV2CapabilitiesHandler),
),
(
Method::PUT,
"/v3/site-replication/peer/remove-v2",
AdminOperation(&SiteRemovalV2PeerHandler),
),
(Method::GET, "/v3/site-replication/info", AdminOperation(&SiteReplicationInfoHandler {})),
(
Method::GET,
@@ -1926,8 +1942,7 @@ fn reconcile_site_replication_retry_drain() -> std::pin::Pin<Box<dyn std::future
}
match load_site_replication_state().await {
Ok(state) => {
if state.pending_endpoint_refresh.is_some() || state.pending_rotation.is_some() || state.pending_remove.is_some()
{
if state.pending_endpoint_refresh.is_some() || state.pending_rotation.is_some() || state.removal_pending() {
return;
}
}
@@ -1987,6 +2002,14 @@ fn reconcile_site_replication_wiring() -> std::pin::Pin<Box<dyn std::future::Fut
// up the round (rustfs/rustfs#5963). The reconcilers below
// still skip this round either way: the topology is only
// settled once the removal clears, and the next tick sees it.
if let Some(pending) = state.pending_remove_v2.as_ref() {
if let Err(err) = resume_site_removal_v2(pending).await {
warn!(event = EVENT_ADMIN_SITE_REPLICATION_STATE, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, result = "decommission_resume_failed",
error = %err, "site replication decommission resume failed");
}
return;
}
if let Some(pending_remove) = state.pending_remove.clone() {
resume_pending_remove(&state, &pending_remove).await;
return;
@@ -3041,7 +3064,16 @@ fn merge_add_sites(
state.name = local_peer.name.clone();
state.service_account_access_key = service_account_access_key;
state.service_account_parent = service_account_parent;
state.updated_at = Some(OffsetDateTime::now_utc());
let now = OffsetDateTime::now_utc();
state.updated_at = Some(if !state.remove_v2_receipts.is_empty() {
state
.updated_at
.filter(|previous| *previous >= now)
.and_then(|previous| previous.checked_add(time::Duration::nanoseconds(1)))
.unwrap_or(now)
} else {
now
});
state.peers = build_join_peers(&state, &local_peer, sites, replicate_ilm_expiry);
// Every join must carry the verified identities, including peers that
// have not joined yet. Fixing only the coordinator after each reply
@@ -3240,7 +3272,8 @@ fn ensure_edit_precondition(
expected_pending_id: Option<&String>,
stage: &str,
) -> S3Result<()> {
if state.updated_at != expected_updated_at
if state.pending_remove_v2.is_some()
|| state.updated_at != expected_updated_at
|| pending_endpoint_refresh(state).as_ref().map(|pending| &pending.id) != expected_pending_id
{
return Err(s3_error!(InvalidRequest, "site replication state changed during {stage}"));
@@ -3933,6 +3966,21 @@ fn status_peer_error(peer: &PeerInfo, detail: String) -> SRPeerError {
}
fn pending_operation_for_state(state: &SiteReplicationState, local_peer: &PeerInfo) -> Option<SRPendingOperation> {
if let Some(pending) = &state.pending_remove_v2 {
let acked = if pending.commit_decided {
&pending.committed_peers
} else {
&pending.cleaned_peers
};
return Some(SRPendingOperation {
operation: "decommission".to_string(),
id: pending.plan.id.clone(),
pending_peers: required_removal_peers(&pending.plan).difference(acked).cloned().collect(),
acked_peers: acked.iter().cloned().collect(),
updated_at: Some(pending.plan.topology_at),
api_version: Some(SITE_REPL_API_VERSION.to_string()),
});
}
if let Some(pending) = state.pending_remove.as_ref() {
let pending_peers = pending_remote_peer_ids(&pending.original_peers, local_peer)
.into_iter()
@@ -4534,13 +4582,18 @@ async fn drive_pending_remove(pending_remove: &PendingRemove, local_peer: &PeerI
for peer in
pending_peers_awaiting_notification(&pending_remove.original_peers, local_peer, &pending_remove.acked_deployment_ids)
{
if let Err(err) = PeerAdminRequest::put(
&runtime_peer_connection(peer)?,
SITE_REPLICATION_PEER_REMOVE_PATH,
&pending_remove.service_account_access_key,
)
.send_with_secret_candidates(&secret_candidates, &pending_remove.req)
.await
let remove_path = format!(
"{SITE_REPLICATION_PEER_REMOVE_PATH}?remove-id={}&remove-at={}",
pending_remove.id,
pending_remove
.updated_at
.map(|stamp| stamp.unix_timestamp_nanos())
.unwrap_or_default()
);
if let Err(err) =
PeerAdminRequest::put(&runtime_peer_connection(peer)?, &remove_path, &pending_remove.service_account_access_key)
.send_with_secret_candidates(&secret_candidates, &pending_remove.req)
.await
{
let err_detail = summarize_peer_error_detail(&format!("{}: {err}", peer.endpoint));
warn!(
@@ -5092,6 +5145,7 @@ fn prune_removed_site_replication_rules(
return (Some(config), 0);
}
let original_role = config.role.clone();
if replication_target_arn_deployment_id(&config.role)
.map(|deployment_id| removed_deployment_ids.contains(&deployment_id))
.unwrap_or(false)
@@ -5105,7 +5159,9 @@ fn prune_removed_site_replication_rules(
.map(|deployment_id| !removed_deployment_ids.contains(&deployment_id))
.unwrap_or(true)
});
let removed = original_len.saturating_sub(config.rules.len());
let removed = original_len
.saturating_sub(config.rules.len())
.saturating_add(usize::from(config.role != original_role));
if removed == 0 {
return (Some(config), 0);
@@ -5164,61 +5220,86 @@ async fn ensure_site_replication_bucket_setup_for_incarnation(bucket: &str, inca
}
async fn cleanup_removed_site_replication_bucket(bucket: &str, removed_deployment_ids: &HashSet<String>) -> S3Result<usize> {
let expected_incarnation_id = metadata_sys::capture_bucket_metadata_incarnation(bucket)
let incarnation = metadata_sys::capture_bucket_metadata_incarnation(bucket)
.await
.map_err(ApiError::from)?;
let _targets_guard = lock_bucket_targets_metadata(bucket).await;
let mut removed = 0usize;
let config = match metadata_sys::get_replication_config(bucket).await {
Ok((config, _)) => Some(config),
Err(StorageError::ConfigNotFound) => None,
Err(err) => return Err(ApiError::from(err).into()),
let targets_guard = lock_bucket_targets_metadata(bucket).await;
let transaction = metadata_sys::acquire_bucket_metadata_transaction_lock_for_incarnation(bucket, incarnation)
.await
.map_err(ApiError::from)?;
let metadata = metadata_sys::get_config_from_disk(bucket).await.map_err(ApiError::from)?;
if metadata.bucket_targets_unreadable()
|| (!metadata.replication_config_xml.is_empty() && metadata.replication_config.is_none())
{
return Err(s3_error!(
InternalError,
"unreadable bucket replication metadata cannot be cleared by decommissioning"
));
}
let config = metadata.replication_config;
let removed_arns = removed_site_replication_rule_arns(config.as_ref(), removed_deployment_ids);
let (mut targets, removed_targets) = prune_removed_site_replication_bucket_targets(
metadata.bucket_target_config.unwrap_or_default(),
bucket,
removed_deployment_ids,
&removed_arns,
);
let (updated_config, removed_rules) = match config {
Some(config) => prune_removed_site_replication_rules(config, removed_deployment_ids),
None => (None, 0),
};
let removed_rule_arns = removed_site_replication_rule_arns(config.as_ref(), removed_deployment_ids);
match metadata_sys::list_bucket_targets(bucket).await {
Ok(targets) => {
let (updated_targets, removed_targets) =
prune_removed_site_replication_bucket_targets(targets, bucket, removed_deployment_ids, &removed_rule_arns);
if removed_targets > 0 {
let json_targets = serde_json::to_vec(&updated_targets).map_err(|e| {
S3Error::with_message(S3ErrorCode::InternalError, format!("serialize bucket targets failed: {e}"))
})?;
metadata_sys::update_if_incarnation(bucket, BUCKET_TARGETS_FILE, json_targets, expected_incarnation_id)
.await
.map_err(ApiError::from)?;
removed = removed.saturating_add(removed_targets);
}
}
Err(StorageError::ConfigNotFound) => {}
Err(err) => return Err(ApiError::from(err).into()),
}
if let Some(config) = config {
// A target shared with an operator rule remains operator-owned.
if let Some(remaining) = &updated_config {
let surviving_arns = remaining.filter_all_replication_target_arns();
for target in metadata_sys::list_bucket_targets(bucket)
.await
.map_err(ApiError::from)?
.targets
{
let (updated_config, removed_rules) = prune_removed_site_replication_rules(config, removed_deployment_ids);
if removed_rules > 0 {
if let Some(updated_config) = updated_config {
let data = serialize(&updated_config).map_err(|e| {
S3Error::with_message(S3ErrorCode::InternalError, format!("serialize replication failed: {e}"))
})?;
metadata_sys::update_if_incarnation(bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id)
.await
.map_err(ApiError::from)?;
} else {
metadata_sys::delete_if_incarnation(bucket, BUCKET_REPLICATION_CONFIG, expected_incarnation_id)
.await
.map_err(ApiError::from)?;
}
removed = removed.saturating_add(removed_rules);
if surviving_arns.contains(&target.arn) && !targets.targets.iter().any(|kept| kept.arn == target.arn) {
targets.targets.push(target);
}
}
}
// A shared target's operator resync remains live. Persist cancellation before
// either config write so a restart cannot resume a site-only target intent.
let surviving_arns = updated_config
.as_ref()
.map(|config| config.filter_all_replication_target_arns())
.unwrap_or_default();
if let Some(pool) = current_replication_pool_handle() {
for arn in removed_arns.iter().filter(|arn| !surviving_arns.contains(arn)) {
pool.cancel_bucket_resync_for_removed_target(bucket, arn)
.await
.map_err(ApiError::from)?;
}
}
if removed_targets > 0 {
let data = serde_json::to_vec(&targets).map_err(|_| s3_error!(InternalError, "cannot encode bucket targets"))?;
metadata_sys::update_bucket_targets_under_transaction_lock(&transaction, bucket, data)
.await
.map_err(ApiError::from)?;
}
if removed_rules > 0 {
if let Some(config) = updated_config {
let data = serialize(&config).map_err(|_| s3_error!(InternalError, "cannot encode bucket replication rules"))?;
metadata_sys::update_under_transaction_lock(&transaction, bucket, BUCKET_REPLICATION_CONFIG, data)
.await
.map_err(ApiError::from)?;
} else {
metadata_sys::delete_under_transaction_lock(&transaction, bucket, BUCKET_REPLICATION_CONFIG)
.await
.map_err(ApiError::from)?;
}
}
drop(transaction);
drop(targets_guard);
let removed = removed_targets.saturating_add(removed_rules);
if removed > 0 {
reload_bucket_metadata_on_peers(bucket, "site_replication_bucket_cleanup", true).await;
}
// Also dirty an idempotent retry after a previous process persisted configuration.
rustfs_scanner::record_scanner_maintenance_change(bucket);
Ok(removed)
}
@@ -5227,9 +5308,8 @@ async fn cleanup_removed_site_replication_buckets(removed_deployment_ids: &HashS
return Ok(0);
}
let Some(store) = current_object_store_handle() else {
return Ok(0);
};
let store =
current_object_store_handle().ok_or_else(|| s3_error!(InternalError, "object store is unavailable during cleanup"))?;
let buckets = store.list_bucket(&BucketOptions::default()).await.map_err(ApiError::from)?;
let mut removed = 0usize;
@@ -5239,6 +5319,13 @@ async fn cleanup_removed_site_replication_buckets(removed_deployment_ids: &HashS
removed = removed.saturating_add(bucket_removed);
}
Err(err) => {
if store
.get_bucket_info(&bucket.name, &BucketOptions::default())
.await
.is_err_and(|error| is_err_bucket_not_found(&error))
{
continue;
}
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
@@ -6898,6 +6985,9 @@ fn adopt_add_commit_state(state: &mut SiteReplicationState, next_state: SiteRepl
resync_status: _,
pending_rotation: _,
pending_remove: _,
pending_remove_v2: _,
remove_v2_receipts: _,
removal_v2_fence: _,
pending_endpoint_refresh: _,
retry_queue: _,
iam_deletion_replays: _,
@@ -6974,7 +7064,9 @@ impl Operation for SiteReplicationAddHandler {
let admission_store = current_object_store_handle()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?;
let list_store = admission_store.clone();
let lifecycle_store = admission_store.clone();
let (state, edit_generation, local_peer, service_account_secret_key, mut initial_sync_errors, _add_guard) =
with_config_object_write_lock(lifecycle_store, SITE_REMOVE_V2_ADMISSION_LOCK.to_string(), move |_guard| async move {
with_site_replication_bucket_mutation_admission_lock(admission_store, move || async move {
// The writer starts before the local bucket snapshot and stays
// held through every peer join and the topology commit. A
@@ -6983,7 +7075,10 @@ impl Operation for SiteReplicationAddHandler {
// use their internal path and do not acquire this public-
// mutation admission lock.
let current_state = load_site_replication_state().await?;
if pending_endpoint_refresh(&current_state).is_some() {
if current_state.pending_remove_v2.is_some() {
return Err(s3_error!(InvalidRequest, "durable decommission is pending"));
}
if pending_endpoint_refresh(&current_state).is_some() {
return Err(s3_error!(InvalidRequest, "endpoint target refresh is pending"));
}
let local_peer = local_peer_at_endpoint(local_endpoint, &current_state);
@@ -7131,7 +7226,7 @@ impl Operation for SiteReplicationAddHandler {
.collect::<HashSet<_>>();
ensure_add_bucket_set_matches_preflight(&expected_buckets, &present)?;
let (state, edit_generation) = update_site_replication_state(move |state| {
if state.updated_at != expected_updated_at || pending_endpoint_refresh(state).is_some() {
if state.updated_at != expected_updated_at || pending_endpoint_refresh(state).is_some() || state.pending_remove_v2.is_some() {
return Err(s3_error!(
InvalidRequest,
"site replication state changed during peer join; the peers may already be joined — re-run replicate add"
@@ -7151,7 +7246,8 @@ impl Operation for SiteReplicationAddHandler {
add_in_progress_guard,
))
})
.await?;
.await
}).await.map_err(|err| S3Error::from(ApiError::from(err)))??;
// The finalize fan-out delivers peer-edit payloads, so it carries the
// generation allocated in the commit above: the receiving site orders
@@ -7220,51 +7316,59 @@ impl Operation for SiteReplicationRemoveHandler {
// would starve every bucket-operation hook in the meantime).
let local_endpoint = site_replication_local_endpoint(req.uri(), req.headers());
let remove_req: SRRemoveReq = read_site_replication_json(req, "", false).await?;
let (pending_remove, local_peer) = {
let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.write().await;
update_site_replication_state_when_changed(move |state| {
let local_peer = local_peer_at_endpoint(local_endpoint, state);
if let Some(pending) = pending_endpoint_refresh(state)
&& !pending_endpoint_refresh_allows_remove(state, &pending, &local_peer, &remove_req)
{
return Err(s3_error!(InvalidRequest, "endpoint target refresh is pending"));
}
if state.pending_rotation.is_some() {
return Err(s3_error!(InvalidRequest, "service account rotation is pending"));
}
let admission_store =
current_object_store_handle().ok_or_else(|| s3_error!(InternalError, "object store is unavailable"))?;
let (pending_remove, local_peer) =
with_config_object_write_lock(admission_store, SITE_REMOVE_V2_ADMISSION_LOCK.to_string(), move |_guard| async move {
let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.write().await;
update_site_replication_state_when_changed(move |state| {
let local_peer = local_peer_at_endpoint(local_endpoint, state);
if let Some(pending) = pending_endpoint_refresh(state)
&& !pending_endpoint_refresh_allows_remove(state, &pending, &local_peer, &remove_req)
{
return Err(s3_error!(InvalidRequest, "endpoint target refresh is pending"));
}
if state.pending_rotation.is_some() {
return Err(s3_error!(InvalidRequest, "service account rotation is pending"));
}
// Resuming: the peers were already told about this pending
// removal, so re-persisting the same record buys nothing.
if let Some(pending) = state.pending_remove.clone() {
return Ok(StateCommit::Unchanged((pending, local_peer)));
}
if state.pending_remove_v2.is_some() {
return Err(s3_error!(InvalidRequest, "durable decommission is pending; resume remove-v2"));
}
// Resuming: the peers were already told about this pending
// removal, so re-persisting the same record buys nothing.
if let Some(pending) = state.pending_remove.clone() {
return Ok(StateCommit::Unchanged((pending, local_peer)));
}
validate_remove_sites_req(state, &remove_req)?;
let service_account_access_key = state.service_account_access_key.clone();
let secret_candidates = legacy_site_replicator_state_secret(state).into_iter().collect();
let original_peers = state.peers.clone();
let mut peer_remove_req = remove_req.clone();
peer_remove_req.requesting_dep_id = local_peer.deployment_id.clone();
*state = remove_sites(std::mem::take(state), remove_req);
// A permitted remove can include the endpoint being edited.
// Drop that obsolete journal before recording the removal;
// otherwise its resume path would commit the edited peer back
// into the topology on the next heavyweight tick.
discard_endpoint_refresh_if_target_was_removed(state);
let pending = PendingRemove {
id: Uuid::new_v4().to_string(),
req: peer_remove_req,
service_account_access_key,
secret_candidates,
original_peers,
acked_deployment_ids: BTreeSet::new(),
updated_at: state.updated_at,
};
state.pending_remove = Some(pending.clone());
Ok(StateCommit::Changed((pending, local_peer)))
validate_remove_sites_req(state, &remove_req)?;
let service_account_access_key = state.service_account_access_key.clone();
let secret_candidates = legacy_site_replicator_state_secret(state).into_iter().collect();
let original_peers = state.peers.clone();
let mut peer_remove_req = remove_req.clone();
peer_remove_req.requesting_dep_id = local_peer.deployment_id.clone();
*state = remove_sites(std::mem::take(state), remove_req);
// A permitted remove can include the endpoint being edited.
// Drop that obsolete journal before recording the removal;
// otherwise its resume path would commit the edited peer back
// into the topology on the next heavyweight tick.
discard_endpoint_refresh_if_target_was_removed(state);
let pending = PendingRemove {
id: Uuid::new_v4().to_string(),
req: peer_remove_req,
service_account_access_key,
secret_candidates,
original_peers,
acked_deployment_ids: BTreeSet::new(),
updated_at: state.updated_at,
};
state.pending_remove = Some(pending.clone());
Ok(StateCommit::Changed((pending, local_peer)))
})
.await
})
.await?
};
.await
.map_err(|err| S3Error::from(ApiError::from(err)))??;
let (mut peer_errors, complete) = drive_pending_remove(&pending_remove, &local_peer).await?;
if !complete && peer_errors.is_empty() {
@@ -7392,7 +7496,7 @@ fn ensure_initial_sync_join_current(state: &SiteReplicationState, join_req: &SRP
|| state.updated_at != join_req.updated_at
|| state.service_account_access_key.is_empty()
|| state.service_account_access_key != join_req.svc_acct_access_key
|| state.pending_remove.is_some()
|| state.removal_pending()
|| state.pending_rotation.is_some()
|| pending_endpoint_refresh(state).is_some()
{
@@ -7475,6 +7579,15 @@ where
move |_write_guard| async move {
let fresh = load_site_replication_state().await?;
let fresh_local_peer = local_peer_at_endpoint(local_endpoint.clone(), &fresh);
if fresh.pending_remove_v2.is_some() {
return Err(s3_error!(InvalidRequest, "durable decommission is pending"));
}
if !fresh.remove_v2_receipts.is_empty()
&& !fresh.peers.contains_key(&fresh_local_peer.deployment_id)
&& !join_req.peers.contains_key(&fresh_local_peer.deployment_id)
{
return Err(s3_error!(PreconditionFailed, "join does not include this deployment"));
}
if join_request_is_superseded(&fresh, join_req.updated_at) {
let peer = fresh
.peers
@@ -7488,6 +7601,9 @@ where
let incoming_updated_at = join_req.updated_at;
update_site_replication_state_when_changed(move |state| {
if state.pending_remove_v2.is_some() {
return Err(s3_error!(InvalidRequest, "durable decommission is pending"));
}
let local_peer = local_peer_at_endpoint(local_endpoint, state);
if join_request_is_superseded(state, incoming_updated_at) {
let peer = state.peers.get(&local_peer.deployment_id).cloned().unwrap_or(local_peer);
@@ -7696,7 +7812,7 @@ impl Operation for SRPeerBucketOpsHandler {
.filter(|value| !value.is_empty())
.cloned()
.ok_or_else(|| s3_error!(InvalidRequest, "operation is required"))?;
if state.pending_remove.is_some()
if state.removal_pending()
|| (!state.enabled()
&& !bootstrap_peer_bucket_operation_allowed(
&bucket,
@@ -7852,7 +7968,7 @@ impl Operation for SiteReplicationEditHandler {
if !incoming.deployment_id.is_empty() || !incoming.endpoint.is_empty() || !incoming.name.is_empty() {
validate_proposed_peer(&incoming)?;
}
if current_state.pending_rotation.is_some() || current_state.pending_remove.is_some() {
if current_state.pending_rotation.is_some() || current_state.removal_pending() {
return Err(s3_error!(InvalidRequest, "another site replication operation is pending"));
}
let persisted_pending = pending_endpoint_refresh(&current_state);
@@ -8187,6 +8303,15 @@ impl Operation for SRPeerEditHandler {
let outcome = update_site_replication_state_when_changed(move |state| {
let mut incoming = incoming;
let local_peer = local_peer_at_endpoint(commit_endpoint, state);
if state.pending_remove_v2.is_some() {
return Err(s3_error!(InvalidRequest, "durable decommission is pending"));
}
if !state.remove_v2_receipts.is_empty()
&& !state.peers.contains_key(&incoming.deployment_id)
&& !same_identity_endpoint(&incoming.endpoint, &local_peer.endpoint)
{
return Err(s3_error!(PreconditionFailed, "removed deployments must rejoin through explicit site add"));
}
// The fence is self-reported — the shared service account means
// the sender cannot be identified — so it is honoured only after
// the admissibility check, against the same state it will gate.
@@ -8208,7 +8333,7 @@ impl Operation for SRPeerEditHandler {
{
return Ok(StateCommit::Unchanged(PeerEditOutcome::Acked));
}
if endpoint_refresh_requested && (state.pending_rotation.is_some() || state.pending_remove.is_some()) {
if endpoint_refresh_requested && (state.pending_rotation.is_some() || state.removal_pending()) {
return Ok(StateCommit::Unchanged(PeerEditOutcome::Rejected(
"another site replication operation is pending",
)));
@@ -8344,10 +8469,25 @@ pub struct SRPeerRemoveHandler {}
impl Operation for SRPeerRemoveHandler {
async fn call(&self, req: AdminRequest, _params: Params<'_, '_>) -> S3Result<AdminResponse> {
validate_site_replication_admin_request(&req, AdminAction::SiteReplicationRemoveAction).await?;
let query = query_pairs(req.uri());
let remove_id_valid = query.get("remove-id").is_some_and(|id| Uuid::parse_str(id).is_ok());
let remove_at = query
.get("remove-at")
.and_then(|stamp| stamp.parse::<i128>().ok())
.and_then(|stamp| OffsetDateTime::from_unix_timestamp_nanos(stamp).ok());
let remove_req: SRRemoveReq = read_site_replication_json(req, "", false).await?;
let _lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await?;
let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.write().await;
let removed_deployment_ids = update_site_replication_state(move |state| {
if state.pending_remove_v2.is_some() {
return Err(s3_error!(InvalidRequest, "legacy peer removal cannot cross a durable decommission"));
}
if let Some(fence) = state.removal_v2_fence
&& (!remove_id_valid
|| remove_at.is_none_or(|stamp| stamp <= fence || state.updated_at.is_some_and(|current| stamp < current)))
{
return Err(s3_error!(PreconditionFailed, "legacy removal predates the current topology"));
}
if pending_endpoint_refresh(state).is_some() {
return Err(s3_error!(InvalidRequest, "endpoint target refresh is pending"));
}
@@ -8357,6 +8497,9 @@ impl Operation for SRPeerRemoveHandler {
let removed_deployment_ids = removed_deployment_ids_for_remove_req(state, &remove_req);
*state = remove_sites(std::mem::take(state), remove_req);
if state.removal_v2_fence.is_some() {
state.updated_at = remove_at;
}
Ok(removed_deployment_ids)
})
.await?;
@@ -8552,6 +8695,12 @@ impl Operation for SRStateEditHandler {
reject_site_replicator_on_public_admin(&cred)?;
let body: SRStateEditReq = read_site_replication_json(req, "", false).await?;
update_site_replication_state(move |state| {
if state.pending_remove_v2.is_some() {
return Err(s3_error!(InvalidRequest, "durable decommission is pending"));
}
if !state.remove_v2_receipts.is_empty() && body.peers.keys().any(|id| !state.peers.contains_key(id)) {
return Err(s3_error!(PreconditionFailed, "removed deployments must rejoin through explicit site add"));
}
*state = apply_state_edit_req(std::mem::take(state), body);
Ok(())
})
@@ -8699,7 +8848,7 @@ impl Operation for SRRotateServiceAccountHandler {
if pending_endpoint_refresh(state).is_some() {
return Err(s3_error!(InvalidRequest, "endpoint target refresh is pending"));
}
if state.pending_remove.is_some() {
if state.removal_pending() {
return Err(s3_error!(InvalidRequest, "site replication remove is pending"));
}
let local_peer = local_peer_at_endpoint(local_endpoint, state);
@@ -0,0 +1,444 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
use crate::runtime_sources::current_endpoints_handle;
use rustfs_madmin::SRRemoveV2Req;
use rustfs_s3_types::s3_error;
pub(super) struct SiteRemovalV2Handler;
pub(super) struct SiteRemovalV2CapabilitiesHandler;
pub(super) struct SiteRemovalV2PeerHandler;
#[cfg(test)]
pub(super) static FAIL_NEXT_LOCAL_REMOVAL_CLEANUP: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
async fn with_removal_admission<T, F, Fut>(
store: std::sync::Arc<crate::storage_api::site_replication::ECStore>,
operation: F,
) -> S3Result<T>
where
T: Send + 'static,
F: FnOnce() -> Fut + Send + 'static,
Fut: std::future::Future<Output = S3Result<T>> + Send + 'static,
{
with_config_object_write_lock(store, SITE_REMOVE_V2_ADMISSION_LOCK.to_string(), move |guard| async move {
if guard.is_lock_lost() { return Err(s3_error!(PreconditionFailed, "decommission admission lock was lost")); }
let signal = guard.lock_lost_signal();
tokio::select! {
biased;
_ = async { if let Some(signal) = signal { signal.notified().await; } else { std::future::pending::<()>().await; } } => {
Err(s3_error!(PreconditionFailed, "decommission admission lock was lost"))
}
result = operation() => {
if guard.is_lock_lost() { Err(s3_error!(PreconditionFailed, "decommission admission lock was lost")) }
else { result }
}
}
}).await.map_err(|err| S3Error::from(ApiError::from(err)))?
}
#[async_trait::async_trait]
impl Operation for SiteRemovalV2CapabilitiesHandler {
async fn call(&self, req: AdminRequest, _params: Params<'_, '_>) -> S3Result<AdminResponse> {
let cred = validate_site_replication_admin_request(&req, AdminAction::SiteReplicationRemoveAction).await?;
if !req
.uri()
.query()
.is_some_and(|query| form_urlencoded::parse(query.as_bytes()).any(|(key, value)| key == "local" && value == "true"))
{
require_local_removal_nodes(&cred).await?;
}
owned_json_response(StatusCode::OK, &serde_json::json!({"contractVersion": 2, "durableCleanup": true}))
}
}
/// Probe every node directly; a load balancer hitting one upgraded node cannot
/// authorize a mixed-version site's destructive operation.
async fn require_local_removal_nodes(cred: &rustfs_credentials::Credentials) -> S3Result<()> {
let Some(endpoints) = current_endpoints_handle() else {
return Err(s3_error!(InternalError, "site topology is unavailable"));
};
let (nodes, local_host) = endpoints.peers();
let state = load_site_replication_state().await?;
let local = current_local_runtime_peer(&state);
let scheme = if is_https_endpoint(&local.endpoint) { "https" } else { "http" };
for node in nodes.into_iter().filter(|node| *node != local_host) {
let peer = PeerInfo {
endpoint: format!("{scheme}://{node}"),
..local.clone()
};
let connection = runtime_peer_connection(&peer)?;
let body = PeerAdminRequest::get(
&connection,
"/rustfs/admin/v3/site-replication/remove-v2/capabilities?local=true",
&cred.access_key,
)
.send_get(&cred.secret_key)
.await?;
let capability: Value =
serde_json::from_slice(&body).map_err(|_| s3_error!(NotImplemented, "a node does not advertise durable removal"))?;
if capability["contractVersion"] != 2 || capability["durableCleanup"] != true {
return Err(s3_error!(NotImplemented, "all nodes must support durable removal v2"));
}
}
Ok(())
}
#[async_trait::async_trait]
impl Operation for SiteRemovalV2Handler {
async fn call(&self, req: AdminRequest, _params: Params<'_, '_>) -> S3Result<AdminResponse> {
let cred = validate_site_replication_admin_request(&req, AdminAction::SiteReplicationRemoveAction).await?;
reject_site_replicator_on_public_admin(&cred)?;
let endpoint = site_replication_local_endpoint(req.uri(), req.headers());
let input: SRRemoveV2Req = read_site_replication_json(req, "", false).await?;
if input.mode == "dry-run" && input.preflight_token.is_some() {
return Err(s3_error!(InvalidRequest, "dry-run does not accept preflightToken"));
}
if input.mode == "execute" {
require_local_removal_nodes(&cred).await?;
}
let signing_key =
current_token_signing_key().ok_or_else(|| s3_error!(InternalError, "token signing key is unavailable"))?;
let _lifecycle = SiteReplicationLifecycleGuard::acquire().await?;
let store = current_object_store_handle().ok_or_else(|| s3_error!(InternalError, "object store is unavailable"))?;
let result = with_removal_admission(store, move || async move {
let state = load_site_replication_state().await?;
let local = local_peer_at_endpoint(endpoint, &state);
let plan = plan_site_removal(&state, &local, &input)?;
let token = site_removal_preflight_token(&plan, signing_key.as_bytes())?;
let complete = state.remove_v2_receipts.contains_key(&plan.id);
if input.mode == "dry-run" {
return Ok(site_removal_status(&plan, state.pending_remove_v2.as_ref(), token, complete, Vec::new()));
}
if input.operation_id.as_deref() != Some(plan.id.as_str())
|| input
.preflight_token
.as_deref()
.is_none_or(|supplied| !crate::auth::constant_time_eq(supplied, &token))
{
return Err(s3_error!(
PreconditionFailed,
"execute requires the current operationId and preflightToken"
));
}
if complete {
return Ok(site_removal_status(&plan, None, token, true, Vec::new()));
}
if state.pending_remove_v2.is_none() {
probe_removal_peers(&plan, &state).await?;
let planned = plan.clone();
let local_id = local.deployment_id;
update_site_replication_state(move |state| admit_site_removal(state, planned, &local_id, true)).await?;
}
let errors = drive_site_removal_v2_locked(&plan.id).await?;
let final_state = load_site_replication_state().await?;
Ok(site_removal_status(
&plan,
final_state.pending_remove_v2.as_ref(),
token,
final_state.remove_v2_receipts.contains_key(&plan.id),
errors,
))
})
.await?;
owned_json_response(StatusCode::OK, &result)
}
}
async fn probe_removal_peers(plan: &SiteRemovalPlan, state: &SiteReplicationState) -> S3Result<()> {
let access_key = state
.pending_remove
.as_ref()
.map(|pending| pending.service_account_access_key.as_str())
.unwrap_or(state.service_account_access_key.as_str());
// A two-site decommission has no remote participant and requires no peer secret.
let required = required_removal_peers(plan);
if required.is_empty() {
return Ok(());
}
let secret = site_replicator_service_account_secret(access_key).await?;
for id in required {
let peer = plan
.original_peers
.get(&id)
.ok_or_else(|| s3_error!(InvalidRequest, "required peer is missing"))?;
let connection = runtime_peer_connection(peer)?;
let body = PeerAdminRequest::get(&connection, SITE_REMOVE_V2_CAPABILITY_PATH, access_key)
.send_get(&secret)
.await?;
let value: Value =
serde_json::from_slice(&body).map_err(|_| s3_error!(NotImplemented, "peer does not advertise durable removal"))?;
if value["contractVersion"] != 2 || value["durableCleanup"] != true {
return Err(s3_error!(NotImplemented, "every participating peer must support durable removal v2"));
}
}
Ok(())
}
#[async_trait::async_trait]
impl Operation for SiteRemovalV2PeerHandler {
async fn call(&self, req: AdminRequest, _params: Params<'_, '_>) -> S3Result<AdminResponse> {
validate_site_replication_admin_request(&req, AdminAction::SiteReplicationRemoveAction).await?;
let input: SiteRemovalPeerRequest = read_site_replication_json(req, "", false).await?;
if !matches!(input.phase.as_str(), "prepare" | "commit") {
return Err(s3_error!(InvalidRequest, "phase must be prepare or commit"));
}
let _lifecycle = SiteReplicationLifecycleGuard::acquire().await?;
let store = current_object_store_handle().ok_or_else(|| s3_error!(InternalError, "object store is unavailable"))?;
let result = with_removal_admission(store, move || async move {
let state = load_site_replication_state().await?;
let local = current_local_runtime_peer(&state);
validate_removal_plan(&input.plan, &local.deployment_id)?;
let digest = site_removal_digest(&input.plan)?;
if let Some(receipt) = state.remove_v2_receipts.get(&input.plan.id) {
if receipt.digest != digest {
return Err(s3_error!(PreconditionFailed, "operationId was used for another removal"));
}
return peer_removal_response(&input.plan, true, true);
}
if let Some(pending) = &state.pending_remove_v2 {
if pending.coordinator || site_removal_digest(&pending.plan)? != digest {
return Err(s3_error!(PreconditionFailed, "another site removal is pending"));
}
} else {
if input.phase == "commit" {
return Err(s3_error!(PreconditionFailed, "site removal was not prepared"));
}
let plan = input.plan.clone();
let local_id = local.deployment_id;
update_site_replication_state(move |state| admit_site_removal(state, plan, &local_id, false)).await?;
}
cleanup_site_removal_v2_local(&input.plan.id).await?;
if input.phase == "commit" {
let id = input.plan.id.clone();
update_site_replication_state(move |state| {
let pending = state
.pending_remove_v2
.as_mut()
.filter(|pending| pending.plan.id == id)
.ok_or_else(|| s3_error!(PreconditionFailed, "site removal changed"))?;
pending.commit_decided = true;
finish_site_removal(state, &id)
})
.await?;
}
peer_removal_response(&input.plan, true, input.phase == "commit")
})
.await?;
owned_json_response(StatusCode::OK, &result)
}
}
fn peer_removal_response(plan: &SiteRemovalPlan, cleaned: bool, committed: bool) -> S3Result<SiteRemovalPeerResponse> {
Ok(SiteRemovalPeerResponse {
contract_version: 2,
operation_id: plan.id.clone(),
digest: site_removal_digest(plan)?,
cleanup_complete: cleaned,
committed,
})
}
/// Called under the distributed lifecycle admission lock. Peer IO never holds the state lock.
pub(super) async fn drive_site_removal_v2_locked(id: &str) -> S3Result<Vec<String>> {
let state = load_site_replication_state().await?;
let Some(pending) = state.pending_remove_v2.as_ref().filter(|pending| pending.plan.id == id) else {
return Ok(Vec::new());
};
if !pending.coordinator {
cleanup_site_removal_v2_local(id).await?;
return Ok(Vec::new());
}
let mut errors = Vec::new();
if let Err(err) = cleanup_site_removal_v2_local(id).await {
errors.push(summarize_peer_error_detail(&format!("local cleanup failed: {err}")));
}
for phase in ["prepare", "commit"] {
let state = load_site_replication_state().await?;
let Some(pending) = state.pending_remove_v2.as_ref().filter(|pending| pending.plan.id == id) else {
break;
};
if phase == "commit" && !pending.commit_decided {
break;
}
let peers = required_removal_peers(&pending.plan);
let acked = if phase == "prepare" {
&pending.cleaned_peers
} else {
&pending.committed_peers
};
let mut candidates = pending.secret_candidates.clone();
if let Ok(secret) = site_replicator_service_account_secret(&pending.access_key).await {
push_unique_secret_candidate(&mut candidates, secret);
}
if candidates != pending.secret_candidates {
let id = id.to_string();
let saved = candidates.clone();
update_site_replication_state(move |state| {
let pending = state
.pending_remove_v2
.as_mut()
.filter(|pending| pending.plan.id == id)
.ok_or_else(|| s3_error!(PreconditionFailed, "site removal changed before secret recording"))?;
pending.secret_candidates = saved;
Ok(())
})
.await?;
}
for peer_id in peers.difference(acked) {
let peer = pending
.plan
.original_peers
.get(peer_id)
.ok_or_else(|| s3_error!(InvalidRequest, "required peer is missing"))?;
let request = SiteRemovalPeerRequest {
phase: phase.to_string(),
plan: pending.plan.clone(),
};
let result = async {
let connection = runtime_peer_connection(peer)?;
let body = PeerAdminRequest::put(&connection, SITE_PEER_REMOVE_V2_PATH, &pending.access_key)
.send_with_secret_candidates(&candidates, &request)
.await?;
let response: SiteRemovalPeerResponse = serde_json::from_slice(&body)
.map_err(|_| s3_error!(InternalError, "invalid durable removal acknowledgement"))?;
if response.contract_version != 2
|| response.operation_id != id
|| response.digest != site_removal_digest(&pending.plan)?
|| !response.cleanup_complete
|| (phase == "commit" && !response.committed)
{
return Err(s3_error!(InternalError, "durable removal acknowledgement does not match the operation"));
}
Ok::<(), S3Error>(())
}
.await;
match result {
Ok(()) => {
let id = id.to_string();
let peer_id = peer_id.clone();
let commit = phase == "commit";
update_site_replication_state_when_changed(move |state| {
let pending = state
.pending_remove_v2
.as_mut()
.filter(|pending| pending.plan.id == id)
.ok_or_else(|| s3_error!(PreconditionFailed, "site removal changed before acknowledgement"))?;
let inserted = if commit {
pending.committed_peers.insert(peer_id)
} else {
pending.cleaned_peers.insert(peer_id)
};
Ok(if inserted {
StateCommit::Changed(())
} else {
StateCommit::Unchanged(())
})
})
.await?;
}
Err(err) => errors.push(summarize_peer_error_detail(&format!("peer {} {phase} failed: {err}", peer.name))),
}
}
if phase == "prepare" {
let id = id.to_string();
update_site_replication_state_when_changed(move |state| {
let pending = state
.pending_remove_v2
.as_mut()
.filter(|pending| pending.plan.id == id)
.ok_or_else(|| s3_error!(PreconditionFailed, "site removal changed before decision"))?;
if pending.commit_decided
|| !pending.local_cleanup_complete
|| !required_removal_peers(&pending.plan).is_subset(&pending.cleaned_peers)
{
return Ok(StateCommit::Unchanged(()));
}
pending.commit_decided = true;
Ok(StateCommit::Changed(()))
})
.await?;
}
}
let id = id.to_string();
update_site_replication_state_when_changed(move |state| {
let Some(pending) = state.pending_remove_v2.as_ref().filter(|pending| pending.plan.id == id) else {
return Ok(StateCommit::Unchanged(()));
};
if !pending.commit_decided || !required_removal_peers(&pending.plan).is_subset(&pending.committed_peers) {
return Ok(StateCommit::Unchanged(()));
}
finish_site_removal(state, &id)?;
Ok(StateCommit::Changed(()))
})
.await?;
Ok(errors)
}
async fn cleanup_site_removal_v2_local(id: &str) -> S3Result<()> {
let state = load_site_replication_state().await?;
let pending = state
.pending_remove_v2
.as_ref()
.filter(|pending| pending.plan.id == id)
.ok_or_else(|| s3_error!(PreconditionFailed, "site removal changed before cleanup"))?;
if pending.local_cleanup_complete {
return Ok(());
}
let local = current_local_runtime_peer(&state);
let removed: HashSet<String> = if pending.plan.removed.contains(&local.deployment_id) {
pending
.plan
.original_peers
.keys()
.filter(|id| **id != local.deployment_id)
.cloned()
.collect()
} else {
pending.plan.removed.iter().cloned().collect()
};
let _bucket_operations = SITE_REPLICATION_BUCKET_OP_LOCK.write().await;
#[cfg(test)]
if FAIL_NEXT_LOCAL_REMOVAL_CLEANUP.swap(false, std::sync::atomic::Ordering::SeqCst) {
return Err(s3_error!(InternalError, "injected local cleanup I/O failure"));
}
cleanup_removed_site_replication_buckets(&removed).await?;
let id = id.to_string();
update_site_replication_state(move |state| {
let pending = state
.pending_remove_v2
.as_mut()
.filter(|pending| pending.plan.id == id)
.ok_or_else(|| s3_error!(PreconditionFailed, "site removal changed after cleanup"))?;
pending.local_cleanup_complete = true;
Ok(())
})
.await
}
pub(super) async fn resume_site_removal_v2(pending: &PendingRemoveV2) -> S3Result<()> {
let store = current_object_store_handle().ok_or_else(|| s3_error!(InternalError, "object store is unavailable"))?;
let id = pending.plan.id.clone();
with_removal_admission(store, move || async move {
let errors = drive_site_removal_v2_locked(&id).await?;
if !errors.is_empty() {
warn!(event = EVENT_ADMIN_SITE_REPLICATION_STATE, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, result = "decommission_still_pending", operation_id = %id,
error_count = errors.len(), "site replication decommission is pending");
}
Ok::<(), S3Error>(())
})
.await
}
@@ -0,0 +1,180 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
use rustfs_madmin::SRRemoveV2Req;
#[test]
fn decommission_role_only_cleanup_removes_legacy_role() {
let config = PersistedReplicationConfiguration {
role: "arn:rustfs:replication::b:bucket".to_string(),
rules: vec![build_site_replication_rule("arn:rustfs:replication::c:bucket", 1, "operator")],
};
let (remaining, removed) = prune_removed_site_replication_rules(config, &HashSet::from(["b".to_string()]));
assert_eq!(removed, 1, "a role-only mutation must be persisted");
assert!(remaining.expect("operator rule survives").role.is_empty());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial_test::serial]
async fn decommission_durable_cleanup_receipt_survives_state_reload() {
use crate::admin::runtime_sources::{AppContext, publish_test_app_context};
use std::sync::Arc;
let env = rustfs_test_utils::TestECStoreEnv::builder()
.prefix("site_decommission")
.disk_count(1)
.init_bucket_metadata(false)
.build()
.await;
let iam = rustfs_iam::build_iam_sys(Arc::clone(&env.ecstore)).await.expect("IAM");
publish_test_app_context(Arc::new(AppContext::with_default_interfaces(
env.ecstore.clone(),
iam,
Arc::new(rustfs_kms::KmsServiceManager::new()),
)));
let local = current_local_runtime_peer(&SiteReplicationState::default());
let remote = PeerInfo {
deployment_id: "retired-site".to_string(),
name: "retired".to_string(),
endpoint: "http://127.0.0.1:1".to_string(),
..Default::default()
};
let seed = SiteReplicationState {
name: local.name.clone(),
peers: BTreeMap::from([
(local.deployment_id.clone(), local.clone()),
(remote.deployment_id.clone(), remote),
]),
..Default::default()
};
save_site_replication_state(&seed).await.expect("seed durable topology");
let loaded = load_site_replication_state()
.await
.expect("read canonical persisted topology");
let plan = plan_site_removal(
&loaded,
&local,
&SRRemoveV2Req {
mode: "dry-run".to_string(),
sites: vec!["retired".to_string()],
..Default::default()
},
)
.expect("plan");
let admitted = plan.clone();
let local_id = local.deployment_id;
update_site_replication_state(move |state| admit_site_removal(state, admitted, &local_id, true))
.await
.expect("admit");
let restored = load_site_replication_state().await.expect("reload pending");
assert!(restored.pending_remove_v2.is_some());
removal::resume_site_removal_v2(restored.pending_remove_v2.as_ref().expect("pending"))
.await
.expect("resume without remote");
let restored = load_site_replication_state().await.expect("reload committed");
assert!(restored.pending_remove_v2.is_none());
assert!(
restored.remove_v2_receipts.contains_key(&plan.id),
"disabled replication must retain its completion fence"
);
assert!(!restored.enabled());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial_test::serial]
async fn decommission_durable_cleanup_failure_retains_journal_until_retry_succeeds() {
use crate::admin::runtime_sources::{AppContext, publish_test_app_context};
use std::sync::Arc;
let env = rustfs_test_utils::TestECStoreEnv::builder()
.prefix("site_decommission_retry")
.disk_count(1)
.init_bucket_metadata(true)
.build()
.await;
let iam = rustfs_iam::build_iam_sys(Arc::clone(&env.ecstore)).await.expect("IAM");
publish_test_app_context(Arc::new(AppContext::with_default_interfaces(
env.ecstore.clone(),
iam,
Arc::new(rustfs_kms::KmsServiceManager::new()),
)));
let bucket = "decommission-cleanup-retry";
env.ecstore
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket");
removal::FAIL_NEXT_LOCAL_REMOVAL_CLEANUP.store(true, std::sync::atomic::Ordering::SeqCst);
let local = current_local_runtime_peer(&SiteReplicationState::default());
let remote = PeerInfo {
deployment_id: "retired-site".to_string(),
name: "retired".to_string(),
endpoint: "http://127.0.0.1:1".to_string(),
..Default::default()
};
let seed = SiteReplicationState {
name: local.name.clone(),
peers: BTreeMap::from([
(local.deployment_id.clone(), local.clone()),
(remote.deployment_id.clone(), remote),
]),
..Default::default()
};
save_site_replication_state(&seed).await.expect("seed");
let loaded = load_site_replication_state().await.expect("read canonical topology");
let stale_runtime = SiteReplicationRuntime {
state: loaded.clone(),
local_peer: local.clone(),
service_account_secret_key: "test-secret".to_string(),
};
let plan = plan_site_removal(
&loaded,
&local,
&SRRemoveV2Req {
mode: "dry-run".to_string(),
sites: vec!["retired".to_string()],
..Default::default()
},
)
.expect("plan");
let admitted = plan.clone();
let local_id = local.deployment_id;
update_site_replication_state(move |state| admit_site_removal(state, admitted, &local_id, true))
.await
.expect("admit");
let incarnation = metadata_sys::capture_bucket_metadata_incarnation(bucket)
.await
.expect("incarnation");
assert!(
ensure_site_replication_bucket_setup_with_runtime_for_incarnation(bucket, &stale_runtime, incarnation)
.await
.is_err(),
"an old topology snapshot must not restore the retired target"
);
let errors = removal::drive_site_removal_v2_locked(&plan.id).await.expect("partial driver");
assert!(!errors.is_empty(), "cleanup failure must remain partial");
let restored = load_site_replication_state().await.expect("reload");
assert!(
!restored
.pending_remove_v2
.as_ref()
.expect("journal must survive failure")
.local_cleanup_complete
);
assert!(!restored.remove_v2_receipts.contains_key(&plan.id));
removal::resume_site_removal_v2(restored.pending_remove_v2.as_ref().expect("pending"))
.await
.expect("retry");
let restored = load_site_replication_state().await.expect("reload complete");
assert!(restored.pending_remove_v2.is_none());
assert!(restored.remove_v2_receipts.contains_key(&plan.id));
}
+18
View File
@@ -827,6 +827,24 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
SITE_REPLICATION_REMOVE,
RouteRiskLevel::High,
),
admin(
HttpMethod::Put,
"/rustfs/admin/v3/site-replication/remove-v2",
SITE_REPLICATION_REMOVE,
RouteRiskLevel::High,
),
admin(
HttpMethod::Get,
"/rustfs/admin/v3/site-replication/remove-v2/capabilities",
SITE_REPLICATION_REMOVE,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Put,
"/rustfs/admin/v3/site-replication/peer/remove-v2",
SITE_REPLICATION_REMOVE,
RouteRiskLevel::High,
),
admin(
HttpMethod::Get,
"/rustfs/admin/v3/site-replication/info",
@@ -431,6 +431,9 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
admin_route(Method::DELETE, "/v3/cancel-job"),
admin_route(Method::PUT, "/v3/site-replication/add"),
admin_route(Method::PUT, "/v3/site-replication/remove"),
admin_route(Method::PUT, "/v3/site-replication/remove-v2"),
admin_route(Method::GET, "/v3/site-replication/remove-v2/capabilities"),
admin_route(Method::PUT, "/v3/site-replication/peer/remove-v2"),
admin_route(Method::GET, "/v3/site-replication/info"),
admin_route(Method::GET, "/v3/site-replication/metainfo"),
admin_route(Method::GET, "/v3/site-replication/status"),
@@ -1484,6 +1487,9 @@ fn test_register_routes_cover_representative_admin_paths() {
assert_route(&router, Method::DELETE, &admin_path("/v3/cancel-job"));
assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/add"));
assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/remove"));
assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/remove-v2"));
assert_route(&router, Method::GET, &admin_path("/v3/site-replication/remove-v2/capabilities"));
assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/peer/remove-v2"));
assert_route(&router, Method::GET, &admin_path("/v3/site-replication/info"));
assert_route(&router, Method::GET, &admin_path("/v3/site-replication/metainfo"));
assert_route(&router, Method::GET, &admin_path("/v3/site-replication/status"));
+38 -7
View File
@@ -224,7 +224,7 @@ impl DefaultBucketUsecase {
&self,
envelope: RequestEnvelope,
bucket: &str,
prepare: impl Future<Output = S3Result<(HashSet<String>, OperatorRuleContract)>> + Send,
prepare: impl Future<Output = S3Result<(HashSet<String>, OperatorRuleContract)>> + Send + 'static,
) -> S3Result<()> {
// Preserve the success status recorded by the legacy audit for an empty response.
RequestEnvelope::audit_only_operation(
@@ -232,8 +232,23 @@ impl DefaultBucketUsecase {
S3Operation::DeleteBucketReplication,
StatusCode::OK,
|envelope| async move {
let (site_peers, contract) = prepare.await?;
self.delete_bucket_replication(envelope, bucket, site_peers, contract).await
let store = self
.object_store()
.ok_or_else(|| s3_error!(InternalError, "object store is unavailable"))?;
let usecase = self.clone();
let bucket = bucket.to_string();
crate::storage_api::site_replication::with_config_object_read_lock(
store,
crate::site_replication::SITE_REMOVE_V2_ADMISSION_LOCK.to_string(),
move || async move {
let (site_peers, contract) = prepare.await?;
usecase
.delete_bucket_replication(envelope, &bucket, site_peers, contract)
.await
},
)
.await
.map_err(|err| S3Error::from(ApiError::from(err)))?
},
)
.await
@@ -364,16 +379,32 @@ impl DefaultBucketUsecase {
&self,
envelope: RequestEnvelope,
bucket: &str,
prepare: impl Future<Output = S3Result<(PersistedReplicationConfiguration, HashSet<String>, OperatorRuleContract)>> + Send,
prepare: impl Future<Output = S3Result<(PersistedReplicationConfiguration, HashSet<String>, OperatorRuleContract)>>
+ Send
+ 'static,
) -> S3Result<()> {
RequestEnvelope::audit_only_operation(
envelope,
S3Operation::PutBucketReplication,
StatusCode::OK,
|envelope| async move {
let (replication_configuration, site_peers, contract) = prepare.await?;
self.put_bucket_replication(envelope, bucket, replication_configuration, site_peers, contract)
.await
let store = self
.object_store()
.ok_or_else(|| s3_error!(InternalError, "object store is unavailable"))?;
let usecase = self.clone();
let bucket = bucket.to_string();
crate::storage_api::site_replication::with_config_object_read_lock(
store,
crate::site_replication::SITE_REMOVE_V2_ADMISSION_LOCK.to_string(),
move || async move {
let (replication_configuration, site_peers, contract) = prepare.await?;
usecase
.put_bucket_replication(envelope, &bucket, replication_configuration, site_peers, contract)
.await
},
)
.await
.map_err(|err| S3Error::from(ApiError::from(err)))?
},
)
.await
+44 -8
View File
@@ -1741,6 +1741,28 @@ pub(crate) async fn ensure_site_replication_bucket_targets_with_runtime(
config: Option<&PersistedReplicationConfiguration>,
service_account_secret_key: &str,
expected_incarnation_id: Uuid,
) -> S3Result<bool> {
let transaction = metadata_sys::acquire_bucket_metadata_transaction_lock_for_incarnation(bucket, expected_incarnation_id)
.await
.map_err(ApiError::from)?;
ensure_site_replication_bucket_targets_under_transaction(
bucket,
state,
local_peer,
config,
service_account_secret_key,
&transaction,
)
.await
}
async fn ensure_site_replication_bucket_targets_under_transaction(
bucket: &str,
state: &SiteReplicationState,
local_peer: &PeerInfo,
config: Option<&PersistedReplicationConfiguration>,
service_account_secret_key: &str,
transaction: &metadata_sys::BucketMetadataMutationGuard,
) -> S3Result<bool> {
let existing = match metadata_sys::list_bucket_targets(bucket).await {
Ok(targets) => targets,
@@ -1764,7 +1786,7 @@ pub(crate) async fn ensure_site_replication_bucket_targets_with_runtime(
if json_targets == existing_json {
return Ok(false);
}
metadata_sys::update_if_incarnation(bucket, BUCKET_TARGETS_FILE, json_targets, expected_incarnation_id)
metadata_sys::update_under_transaction_lock(transaction, bucket, BUCKET_TARGETS_FILE, json_targets)
.await
.map_err(ApiError::from)?;
Ok(true)
@@ -1781,12 +1803,12 @@ pub(crate) async fn bucket_replication_config_for_target_refresh(
}
/// Returns whether the replication configuration was rewritten.
pub(crate) async fn ensure_site_replication_bucket_replication_config_with_runtime(
async fn ensure_site_replication_bucket_replication_config_under_transaction(
bucket: &str,
state: &SiteReplicationState,
local_peer: &PeerInfo,
service_account_secret_key: &str,
expected_incarnation_id: Uuid,
transaction: &metadata_sys::BucketMetadataMutationGuard,
) -> S3Result<bool> {
let existing = match metadata_sys::get_replication_config(bucket).await {
Ok((existing, _)) => Some(existing),
@@ -1833,7 +1855,7 @@ pub(crate) async fn ensure_site_replication_bucket_replication_config_with_runti
let data = serialize(&PersistedReplicationConfiguration { role, rules })
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize replication failed: {e}")))?;
metadata_sys::update_if_incarnation(bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id)
metadata_sys::update_under_transaction_lock(transaction, bucket, BUCKET_REPLICATION_CONFIG, data)
.await
.map_err(ApiError::from)?;
@@ -1855,25 +1877,39 @@ pub(crate) async fn ensure_site_replication_bucket_setup_with_runtime_for_incarn
runtime: &SiteReplicationRuntime,
expected_incarnation_id: Uuid,
) -> S3Result<()> {
// Bucket transaction -> short state read -> metadata writes. A removal
// commits topology first and then waits on this same transaction for cleanup.
// An old runtime either finishes before cleanup or observes the new topology.
let targets_guard = lock_bucket_targets_metadata(bucket).await;
let transaction = metadata_sys::acquire_bucket_metadata_transaction_lock_for_incarnation(bucket, expected_incarnation_id)
.await
.map_err(ApiError::from)?;
let current = load_site_replication_state().await?;
if current.pending_remove_v2.is_some()
|| current.updated_at != runtime.state.updated_at
|| current.peers.keys().ne(runtime.state.peers.keys())
{
return Err(s3_error!(PreconditionFailed, "site replication topology changed before bucket setup"));
}
let config = bucket_replication_config_for_target_refresh(bucket).await?;
let targets_written = ensure_site_replication_bucket_targets_with_runtime(
let targets_written = ensure_site_replication_bucket_targets_under_transaction(
bucket,
&runtime.state,
&runtime.local_peer,
config.as_ref(),
&runtime.service_account_secret_key,
expected_incarnation_id,
&transaction,
)
.await?;
let config_written = ensure_site_replication_bucket_replication_config_with_runtime(
let config_written = ensure_site_replication_bucket_replication_config_under_transaction(
bucket,
&runtime.state,
&runtime.local_peer,
&runtime.service_account_secret_key,
expected_incarnation_id,
&transaction,
)
.await?;
drop(transaction);
drop(targets_guard);
if targets_written || config_written {
reload_bucket_metadata_on_peers(bucket, "site_replication_bucket_setup", config_written).await;
+2
View File
@@ -29,6 +29,7 @@ pub(crate) mod identity;
pub(crate) mod state_lock;
pub(crate) mod hooks;
pub(crate) mod removal;
pub(crate) mod repair;
pub(crate) mod retry;
pub(crate) mod state;
@@ -38,6 +39,7 @@ pub(crate) mod transport;
mod tests;
pub(crate) use self::hooks::*;
pub(crate) use self::removal::*;
pub(crate) use self::repair::*;
pub(crate) use self::retry::*;
pub(crate) use self::state::*;
+571
View File
@@ -0,0 +1,571 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Durable decommissioning contract shared by admin admission and peer delivery.
use super::*;
use rustfs_madmin::{SRRemoveV2Req, SRRemoveV2Status};
use rustfs_s3_types::s3_error;
use sha2::{Digest, Sha256};
pub(crate) const SITE_PEER_REMOVE_V2_PATH: &str = "/rustfs/admin/v3/site-replication/peer/remove-v2";
pub(crate) const SITE_REMOVE_V2_CAPABILITY_PATH: &str = "/rustfs/admin/v3/site-replication/remove-v2/capabilities";
/// This lock precedes bucket admission and the state transaction. Join admission uses the same lock.
pub(crate) const SITE_REMOVE_V2_ADMISSION_LOCK: &str = "config/site-replication/join-admission.lock";
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub(crate) struct SiteRemovalPlan {
pub(crate) id: String,
pub(crate) coordinator: String,
pub(crate) original_peers: BTreeMap<String, PeerInfo>,
pub(crate) removed: BTreeSet<String>,
pub(crate) decommissioned: BTreeSet<String>,
#[serde(with = "time::serde::rfc3339")]
pub(crate) topology_at: OffsetDateTime,
pub(crate) base_fingerprint: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct PendingRemoveV2 {
pub(crate) plan: SiteRemovalPlan,
pub(crate) coordinator: bool,
pub(crate) access_key: String,
#[serde(default)]
pub(crate) secret_candidates: Vec<String>,
#[serde(default)]
pub(crate) cleaned_peers: BTreeSet<String>,
#[serde(default)]
pub(crate) committed_peers: BTreeSet<String>,
#[serde(default)]
pub(crate) local_cleanup_complete: bool,
#[serde(default)]
pub(crate) commit_decided: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct SiteRemovalReceipt {
pub(crate) digest: String,
pub(crate) plan: SiteRemovalPlan,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub(crate) struct SiteRemovalPeerRequest {
pub(crate) phase: String,
pub(crate) plan: SiteRemovalPlan,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub(crate) struct SiteRemovalPeerResponse {
pub(crate) contract_version: u32,
pub(crate) operation_id: String,
pub(crate) digest: String,
pub(crate) cleanup_complete: bool,
pub(crate) committed: bool,
}
pub(crate) fn site_removal_digest(plan: &SiteRemovalPlan) -> S3Result<String> {
let data = serde_json::to_vec(plan).map_err(|_| s3_error!(InternalError, "cannot encode site removal plan"))?;
Ok(URL_SAFE_NO_PAD.encode_to_string(Sha256::digest(data)))
}
pub(crate) fn site_removal_preflight_token(plan: &SiteRemovalPlan, key: &[u8]) -> S3Result<String> {
if key.is_empty() {
return Err(s3_error!(InternalError, "empty decommission preflight signing key"));
}
let mut mac = <Hmac<Sha256> as hmac::digest::KeyInit>::new_from_slice(key)
.map_err(|_| s3_error!(InternalError, "invalid preflight signing key"))?;
mac.update(b"rustfs:site-replication:decommission-preflight:v2\0");
// Execution chooses the commit timestamp; the preview binds the identity, scope and base.
let data = serde_json::to_vec(&(&plan.id, &plan.coordinator, &plan.removed, &plan.decommissioned, &plan.base_fingerprint))
.map_err(|_| s3_error!(InternalError, "cannot encode site removal preflight"))?;
mac.update(&data);
Ok(URL_SAFE_NO_PAD.encode_to_string(mac.finalize().into_bytes()))
}
fn removal_base_fingerprint(state: &SiteReplicationState) -> S3Result<String> {
// ACKs and candidate secrets may change during preflight without changing its scope.
let legacy = state
.pending_remove
.as_ref()
.map(|pending| (&pending.id, &pending.req, &pending.original_peers));
let data = serde_json::to_vec(&(&state.peers, state.updated_at, legacy))
.map_err(|_| s3_error!(InternalError, "cannot encode site removal precondition"))?;
Ok(URL_SAFE_NO_PAD.encode_to_string(Sha256::digest(data)))
}
fn resolve_removal_names(peers: &BTreeMap<String, PeerInfo>, names: &[String]) -> S3Result<BTreeSet<String>> {
let mut ids = BTreeSet::new();
for name in names {
if name.is_empty() || name.trim() != name {
return Err(s3_error!(
InvalidRequest,
"site names must be nonempty and have no surrounding whitespace"
));
}
let matches = peers.values().filter(|peer| peer.name == *name).collect::<Vec<_>>();
let [peer] = matches.as_slice() else {
return Err(s3_error!(
InvalidRequest,
"each requested site must identify exactly one configured deployment"
));
};
if !ids.insert(peer.deployment_id.clone()) {
return Err(s3_error!(InvalidRequest, "duplicate site name"));
}
}
Ok(ids)
}
pub(crate) fn plan_site_removal(
state: &SiteReplicationState,
local: &PeerInfo,
req: &SRRemoveV2Req,
) -> S3Result<SiteRemovalPlan> {
if !matches!(req.mode.as_str(), "dry-run" | "execute") {
return Err(s3_error!(InvalidRequest, "mode must be dry-run or execute"));
}
if let Some(pending) = &state.pending_remove_v2 {
if !pending.coordinator {
return Err(s3_error!(InvalidRequest, "site removal is coordinated by another site"));
}
validate_removal_request_matches(&pending.plan, req)?;
return Ok(pending.plan.clone());
}
if let Some(id) = &req.operation_id {
if let Some(receipt) = state.remove_v2_receipts.get(id) {
validate_removal_request_matches(&receipt.plan, req)?;
return Ok(receipt.plan.clone());
}
if Uuid::parse_str(id).is_err() {
return Err(s3_error!(InvalidRequest, "operationId must be a UUID"));
}
}
if state.pending_rotation.is_some() || state.pending_endpoint_refresh.is_some() {
return Err(s3_error!(InvalidRequest, "another site replication operation is pending"));
}
let original_peers = state
.pending_remove
.as_ref()
.map(|pending| &pending.original_peers)
.unwrap_or(&state.peers);
if original_peers.len() < 2 || !original_peers.contains_key(&local.deployment_id) {
return Err(s3_error!(InvalidRequest, "site replication topology is not configured"));
}
let removed = if req.all {
if !req.sites.is_empty() {
return Err(s3_error!(InvalidRequest, "sites must be empty when all=true"));
}
original_peers.keys().cloned().collect()
} else {
resolve_removal_names(original_peers, &req.sites)?
};
if removed.is_empty() {
return Err(s3_error!(InvalidRequest, "at least one site must be removed"));
}
let decommissioned_names = if req.all { &req.decommissioned_sites } else { &req.sites };
if !req.all && !req.decommissioned_sites.is_empty() {
return Err(s3_error!(InvalidRequest, "decommissionedSites is only valid with all=true"));
}
let decommissioned = resolve_removal_names(original_peers, decommissioned_names)?;
if decommissioned.is_empty() || decommissioned.contains(&local.deployment_id) || !decommissioned.is_subset(&removed) {
return Err(s3_error!(InvalidRequest, "explicit decommissioned remote sites are required"));
}
if let Some(pending) = &state.pending_remove {
let legacy_removed = if pending.req.remove_all || pending.req.site_names.contains(&state.name) {
original_peers.keys().cloned().collect()
} else {
resolve_removal_names(original_peers, &pending.req.site_names)?
};
if legacy_removed != removed {
return Err(s3_error!(InvalidRequest, "force removal must preserve the pending removal's site set"));
}
}
let now = OffsetDateTime::now_utc();
let topology_at = state
.updated_at
.into_iter()
.chain(state.removal_v2_fence)
.max()
.filter(|stamp| *stamp >= now)
.map(|stamp| {
stamp
.checked_add(time::Duration::nanoseconds(1))
.ok_or_else(|| s3_error!(InvalidRequest, "topology timestamp exhausted"))
})
.transpose()?
.unwrap_or(now);
let base_fingerprint = removal_base_fingerprint(state)?;
let identity = serde_json::to_vec(&(&local.deployment_id, &removed, &decommissioned, &base_fingerprint))
.map_err(|_| s3_error!(InternalError, "cannot encode decommission identity"))?;
let id = Uuid::new_v5(&Uuid::NAMESPACE_OID, &identity).to_string();
if req.operation_id.as_ref().is_some_and(|requested| *requested != id) {
return Err(s3_error!(PreconditionFailed, "operationId does not match the current removal preview"));
}
Ok(SiteRemovalPlan {
id,
coordinator: local.deployment_id.clone(),
original_peers: original_peers.clone(),
removed,
decommissioned,
topology_at,
base_fingerprint,
})
}
fn validate_removal_request_matches(plan: &SiteRemovalPlan, req: &SRRemoveV2Req) -> S3Result<()> {
let removed = if req.all {
if !req.sites.is_empty() {
return Err(s3_error!(InvalidRequest, "sites must be empty when all=true"));
}
plan.original_peers.keys().cloned().collect()
} else {
resolve_removal_names(&plan.original_peers, &req.sites)?
};
let names = if req.all { &req.decommissioned_sites } else { &req.sites };
if (!req.all && !req.decommissioned_sites.is_empty())
|| removed != plan.removed
|| resolve_removal_names(&plan.original_peers, names)? != plan.decommissioned
|| req.operation_id.as_ref().is_some_and(|id| *id != plan.id)
{
return Err(s3_error!(InvalidRequest, "request does not match the recorded site removal"));
}
Ok(())
}
pub(crate) fn required_removal_peers(plan: &SiteRemovalPlan) -> BTreeSet<String> {
plan.original_peers
.keys()
.filter(|id| **id != plan.coordinator && !plan.decommissioned.contains(*id))
.cloned()
.collect()
}
pub(crate) fn validate_removal_plan(plan: &SiteRemovalPlan, local_id: &str) -> S3Result<()> {
if Uuid::parse_str(&plan.id).is_err()
|| !plan.original_peers.contains_key(&plan.coordinator)
|| !plan.original_peers.contains_key(local_id)
|| plan.decommissioned.contains(local_id)
|| plan.decommissioned.contains(&plan.coordinator)
|| plan.removed.is_empty()
|| plan.decommissioned.is_empty()
|| !plan.decommissioned.is_subset(&plan.removed)
|| plan.removed.iter().any(|id| !plan.original_peers.contains_key(id))
|| plan
.original_peers
.iter()
.any(|(id, peer)| id.is_empty() || *id != peer.deployment_id)
{
return Err(s3_error!(InvalidRequest, "invalid site removal plan"));
}
Ok(())
}
pub(crate) fn admit_site_removal(
state: &mut SiteReplicationState,
plan: SiteRemovalPlan,
local_id: &str,
coordinator: bool,
) -> S3Result<()> {
validate_removal_plan(&plan, local_id)?;
if state.pending_remove_v2.is_some() || state.remove_v2_receipts.contains_key(&plan.id) {
return Err(s3_error!(InvalidRequest, "site removal is already recorded"));
}
if state.pending_rotation.is_some() || state.pending_endpoint_refresh.is_some() {
return Err(s3_error!(InvalidRequest, "another site replication operation is pending"));
}
if coordinator {
if removal_base_fingerprint(state)? != plan.base_fingerprint {
return Err(s3_error!(PreconditionFailed, "site removal preflight is stale"));
}
} else {
if state.updated_at.is_some_and(|stamp| stamp >= plan.topology_at)
|| state.removal_v2_fence.is_some_and(|stamp| stamp >= plan.topology_at)
{
return Err(s3_error!(PreconditionFailed, "site removal was superseded by a newer topology"));
}
// A legacy remove --all may already have cleared this participant's state.
// The authenticated retry can still clean its old derived bucket targets.
if state.peers.iter().any(|(id, peer)| {
plan.original_peers
.get(id)
.is_none_or(|original| !same_identity_endpoint(&peer.endpoint, &original.endpoint))
}) || (!state.peers.is_empty()
&& plan
.original_peers
.keys()
.any(|id| !plan.removed.contains(id) && !state.peers.contains_key(id)))
{
return Err(s3_error!(PreconditionFailed, "peer topology does not match the removal plan"));
}
}
let access_key = state
.pending_remove
.as_ref()
.map(|pending| pending.service_account_access_key.clone())
.unwrap_or_else(|| state.service_account_access_key.clone());
let secret_candidates = state
.pending_remove
.as_ref()
.map(|pending| pending.secret_candidates.clone())
.unwrap_or_default();
state.peers.retain(|id, _| !plan.removed.contains(id));
if plan.removed.contains(local_id) {
state.peers.clear();
}
state.resync_status.retain(|id, _| state.peers.contains_key(id));
state
.retry_queue
.retain(|event| state.peers.contains_key(&event.peer_deployment_id));
state
.iam_deletion_replays
.retain(|event| state.peers.contains_key(&event.peer_deployment_id));
state.updated_at = Some(plan.topology_at);
state.removal_v2_fence = Some(
state
.removal_v2_fence
.map_or(plan.topology_at, |stamp| stamp.max(plan.topology_at)),
);
state.pending_remove = None;
state.pending_remove_v2 = Some(PendingRemoveV2 {
plan,
coordinator,
access_key,
secret_candidates,
cleaned_peers: BTreeSet::new(),
committed_peers: BTreeSet::new(),
local_cleanup_complete: false,
commit_decided: false,
});
Ok(())
}
pub(crate) fn finish_site_removal(state: &mut SiteReplicationState, id: &str) -> S3Result<()> {
let Some(pending) = state.pending_remove_v2.as_ref().filter(|pending| pending.plan.id == id) else {
return Err(s3_error!(PreconditionFailed, "site removal changed before completion"));
};
if !pending.local_cleanup_complete
|| !pending.commit_decided
|| (pending.coordinator && !required_removal_peers(&pending.plan).is_subset(&pending.committed_peers))
{
return Err(s3_error!(InvalidRequest, "site removal cleanup is not committed"));
}
let receipt = SiteRemovalReceipt {
digest: site_removal_digest(&pending.plan)?,
plan: pending.plan.clone(),
};
state.remove_v2_receipts.insert(id.to_string(), receipt);
state.pending_remove_v2 = None;
// The retained topology timestamp fences an evicted operation; old requests fail closed.
while state.remove_v2_receipts.len() > 32 {
let oldest = state
.remove_v2_receipts
.iter()
.min_by_key(|(_, receipt)| receipt.plan.topology_at)
.map(|(key, _)| key.clone());
if let Some(oldest) = oldest {
state.remove_v2_receipts.remove(&oldest);
}
}
Ok(())
}
pub(crate) fn site_removal_status(
plan: &SiteRemovalPlan,
pending: Option<&PendingRemoveV2>,
token: String,
complete: bool,
errors: Vec<String>,
) -> SRRemoveV2Status {
let pending_peers = pending
.map(|pending| {
let acked = if pending.commit_decided {
&pending.committed_peers
} else {
&pending.cleaned_peers
};
required_removal_peers(plan).difference(acked).cloned().collect()
})
.unwrap_or_default();
SRRemoveV2Status {
contract_version: 2,
operation_id: plan.id.clone(),
status: if complete {
"success"
} else if pending.is_some() {
"Partial"
} else {
"planned"
}
.to_string(),
preflight_token: token,
removed_deployment_ids: plan.removed.iter().cloned().collect(),
decommissioned_deployment_ids: plan.decommissioned.iter().cloned().collect(),
pending_peers,
local_cleanup_complete: complete || pending.is_some_and(|pending| pending.local_cleanup_complete),
errors,
}
}
#[cfg(test)]
mod tests {
use super::*;
fn fixture() -> (SiteReplicationState, PeerInfo, SRRemoveV2Req) {
let peers: BTreeMap<String, PeerInfo> = ["a", "b", "c"]
.into_iter()
.map(|id| {
(
id.to_string(),
PeerInfo {
deployment_id: id.to_string(),
name: id.to_string(),
endpoint: format!("https://{id}.example.test"),
..Default::default()
},
)
})
.collect();
let local = peers["a"].clone();
(
SiteReplicationState {
name: "a".to_string(),
peers,
..Default::default()
},
local,
SRRemoveV2Req {
mode: "dry-run".to_string(),
sites: vec!["b".to_string()],
..Default::default()
},
)
}
#[test]
fn decommission_skips_only_the_selected_site_and_waits_for_commit_ack() {
let (mut state, local, req) = fixture();
let plan = plan_site_removal(&state, &local, &req).expect("plan retirement");
assert_eq!(required_removal_peers(&plan), BTreeSet::from(["c".to_string()]));
admit_site_removal(&mut state, plan.clone(), "a", true).expect("admit");
assert!(!state.peers.contains_key("b"));
assert!(finish_site_removal(&mut state, &plan.id).is_err(), "cleanup must precede completion");
let pending = state.pending_remove_v2.as_mut().expect("pending");
pending.local_cleanup_complete = true;
pending.cleaned_peers.insert("c".to_string());
pending.commit_decided = true;
assert!(finish_site_removal(&mut state, &plan.id).is_err(), "decision delivery must be confirmed");
state
.pending_remove_v2
.as_mut()
.expect("pending")
.committed_peers
.insert("c".to_string());
finish_site_removal(&mut state, &plan.id).expect("finish");
assert!(state.pending_remove_v2.is_none());
assert_eq!(state.remove_v2_receipts[&plan.id].digest, site_removal_digest(&plan).expect("digest"));
}
#[test]
fn decommission_recovers_legacy_removed_identity_and_preserves_scope() {
let (mut state, local, mut req) = fixture();
let original_peers = state.peers.clone();
state.peers.remove("b");
state.pending_remove = Some(PendingRemove {
id: "legacy".to_string(),
original_peers,
req: SRRemoveReq {
site_names: vec!["b".to_string()],
..Default::default()
},
..Default::default()
});
let plan = plan_site_removal(&state, &local, &req).expect("recover removed peer");
assert!(plan.decommissioned.contains("b"));
req.sites = vec!["c".to_string()];
assert!(plan_site_removal(&state, &local, &req).is_err(), "cannot change the legacy pending scope");
admit_site_removal(&mut state, plan, "a", true).expect("upgrade journal");
assert!(state.pending_remove.is_none());
let restored = parse_site_replication_state(&serde_json::to_vec(&state).expect("encode")).expect("restore");
assert!(restored.pending_remove_v2.is_some(), "restart must retain the recovery obligation");
}
#[test]
fn decommission_preflight_binds_topology_but_not_background_ack_updates() {
let (mut state, local, mut req) = fixture();
let plan = plan_site_removal(&state, &local, &req).expect("plan");
req.operation_id = Some(plan.id.clone());
let replanned = plan_site_removal(&state, &local, &req).expect("execute plan");
assert_eq!(
site_removal_preflight_token(&plan, b"key").expect("token"),
site_removal_preflight_token(&replanned, b"key").expect("token")
);
state.peers.get_mut("c").expect("peer").endpoint = "https://new-c.example.test".to_string();
assert!(
admit_site_removal(&mut state, plan, "a", true).is_err(),
"stale preflight must fail closed"
);
}
#[test]
fn decommission_rejects_unknown_duplicate_local_and_inconsistent_selectors() {
let (state, local, mut req) = fixture();
for sites in [vec!["unknown"], vec!["b", "b"], vec!["a"], vec![" b"], vec![]] {
req.sites = sites.into_iter().map(str::to_string).collect();
assert!(plan_site_removal(&state, &local, &req).is_err());
}
req.sites.clear();
req.all = true;
assert!(plan_site_removal(&state, &local, &req).is_err(), "all does not waive every ACK");
req.decommissioned_sites = vec!["b".to_string()];
let plan = plan_site_removal(&state, &local, &req).expect("explicit all");
assert_eq!(required_removal_peers(&plan), BTreeSet::from(["c".to_string()]));
}
#[test]
fn decommission_peer_rejects_stale_replay_after_rejoin_and_keeps_receipt_on_restart() {
let (state, local, req) = fixture();
let plan = plan_site_removal(&state, &local, &req).expect("plan");
let mut peer_state = state.clone();
admit_site_removal(&mut peer_state, plan.clone(), "c", false).expect("peer prepare");
let pending = peer_state.pending_remove_v2.as_mut().expect("pending");
pending.local_cleanup_complete = true;
pending.commit_decided = true;
finish_site_removal(&mut peer_state, &plan.id).expect("peer commit");
peer_state.peers = state.peers;
peer_state.updated_at = Some(plan.topology_at + time::Duration::seconds(1));
assert!(admit_site_removal(&mut peer_state, plan.clone(), "c", false).is_err());
let loaded = parse_site_replication_state(&serde_json::to_vec(&peer_state).expect("encode")).expect("load");
assert!(loaded.remove_v2_receipts.contains_key(&plan.id));
assert!(loaded.peers.contains_key("b"), "a late request must not tear down the rejoined site");
}
#[test]
fn decommission_fence_survives_receipt_eviction_and_clock_rollback() {
let (state, local, req) = fixture();
let plan = plan_site_removal(&state, &local, &req).expect("plan");
let mut peer_state = state.clone();
admit_site_removal(&mut peer_state, plan.clone(), "c", false).expect("admit");
peer_state.pending_remove_v2 = None;
peer_state.remove_v2_receipts.clear();
peer_state.updated_at = None;
peer_state.peers = state.peers;
assert!(
admit_site_removal(&mut peer_state, plan, "c", false).is_err(),
"persistent fence must reject stale removal"
);
}
}
+14 -9
View File
@@ -711,22 +711,27 @@ pub(crate) async fn execute_site_replication_repair(
) -> S3Result<S3Response<(StatusCode, Body)>> {
let store =
current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?;
with_config_object_write_lock(
store,
SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(),
move |_write_guard| async move { execute_site_replication_repair_locked(request).await },
)
let admission_store = store.clone();
with_config_object_read_lock(admission_store, SITE_REMOVE_V2_ADMISSION_LOCK.to_string(), move || async move {
with_config_object_write_lock(
store,
SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(),
move |_write_guard| async move { execute_site_replication_repair_locked(request).await },
)
.await
.map_err(|_| {
S3Error::with_message(S3ErrorCode::ClientTokenConflict, "another site replication repair is active".to_string())
})?
})
.await
.map_err(|_| {
S3Error::with_message(S3ErrorCode::ClientTokenConflict, "another site replication repair is active".to_string())
})?
.map_err(|_| s3_error!(ClientTokenConflict, "site decommission admission is unavailable"))?
}
pub(crate) async fn execute_site_replication_repair_locked(
request: SiteReplicationRepairExecutionRequest,
) -> S3Result<S3Response<(StatusCode, Body)>> {
let state = load_site_replication_state().await?;
if !state.enabled() || state.service_account_access_key.is_empty() {
if state.pending_remove_v2.is_some() || !state.enabled() || state.service_account_access_key.is_empty() {
return Err(s3_error!(InvalidRequest, "site replication is not configured"));
}
let info = build_sr_info(&state, &request.local_peer).await?;
+4 -4
View File
@@ -1681,7 +1681,7 @@ async fn drain_site_replication_retry_queue_lightweight_inner() -> S3Result<()>
};
log_site_replication_retry_liabilities(&runtime.state);
if runtime.state.pending_endpoint_refresh.is_some()
|| runtime.state.pending_remove.is_some()
|| runtime.state.removal_pending()
|| runtime.state.pending_rotation.is_some()
{
return Ok(());
@@ -1708,7 +1708,7 @@ async fn drain_site_replication_retry_queue_lightweight_inner() -> S3Result<()>
return Ok(());
};
if runtime.state.pending_endpoint_refresh.is_some()
|| runtime.state.pending_remove.is_some()
|| runtime.state.removal_pending()
|| runtime.state.pending_rotation.is_some()
{
return Ok(());
@@ -1742,7 +1742,7 @@ pub(crate) async fn drain_site_replication_retry_queue_inner() -> S3Result<()> {
return Ok(());
};
if runtime.state.pending_endpoint_refresh.is_some()
|| runtime.state.pending_remove.is_some()
|| runtime.state.removal_pending()
|| runtime.state.pending_rotation.is_some()
{
// The tick-level gate ran before the reconcilers; a multi-step flow
@@ -1777,7 +1777,7 @@ pub(crate) async fn drain_site_replication_retry_queue_inner() -> S3Result<()> {
return Ok(());
};
if runtime.state.pending_endpoint_refresh.is_some()
|| runtime.state.pending_remove.is_some()
|| runtime.state.removal_pending()
|| runtime.state.pending_rotation.is_some()
{
return Ok(());
+41 -1
View File
@@ -39,6 +39,13 @@ pub(crate) struct SiteReplicationState {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub(crate) pending_remove: Option<PendingRemove>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub(crate) pending_remove_v2: Option<PendingRemoveV2>,
/// Completed removal identities survive disabled replication and fence delayed retries.
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub(crate) remove_v2_receipts: BTreeMap<String, SiteRemovalReceipt>,
#[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")]
pub(crate) removal_v2_fence: Option<OffsetDateTime>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub(crate) pending_endpoint_refresh: Option<PendingEndpointRefresh>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub(crate) retry_queue: Vec<SiteReplicationRetryEvent>,
@@ -212,12 +219,34 @@ impl SiteReplicationState {
pub(crate) fn enabled(&self) -> bool {
self.peers.len() > 1
}
pub(crate) fn removal_pending(&self) -> bool {
self.pending_remove.is_some() || self.pending_remove_v2.is_some()
}
}
pub(crate) fn parse_site_replication_state(data: &[u8]) -> S3Result<SiteReplicationState> {
let mut state: SiteReplicationState = serde_json::from_slice(data)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("invalid site replication state: {e}")))?;
state.peers = normalize_peer_map_by_identity(state.peers);
if let Some(pending) = &state.pending_remove_v2 {
validate_removal_plan(&pending.plan, &pending.plan.coordinator)
.map_err(|_| s3_error!(InternalError, "invalid persisted decommission plan"))?;
if state.pending_remove.is_some()
|| !pending.cleaned_peers.is_subset(&required_removal_peers(&pending.plan))
|| !pending.committed_peers.is_subset(&pending.cleaned_peers)
|| (pending.coordinator
&& pending.commit_decided
&& (!pending.local_cleanup_complete || !required_removal_peers(&pending.plan).is_subset(&pending.cleaned_peers)))
{
return Err(s3_error!(InternalError, "inconsistent persisted decommission progress"));
}
}
for (id, receipt) in &state.remove_v2_receipts {
if *id != receipt.plan.id || site_removal_digest(&receipt.plan)? != receipt.digest {
return Err(s3_error!(InternalError, "invalid persisted decommission receipt"));
}
}
// A peer-edit high-water mark only fences a CURRENT peer. A site that
// leaves drops below two peers, which clears its own state object and
// restarts its generation counter — a mark left over from the previous
@@ -277,6 +306,12 @@ pub(crate) async fn site_replication_enabled() -> S3Result<bool> {
/// exactly the reconciler-owned rules (issue #1948); a state-read failure
/// propagates so the edit fails closed.
pub(crate) async fn site_replication_edit_context() -> S3Result<(HashSet<String>, OperatorRuleContract)> {
if load_site_replication_state().await?.pending_remove_v2.is_some() {
return Err(s3_error!(
InvalidRequest,
"bucket replication edits are blocked while decommissioning is pending"
));
}
let Some(runtime) = runtime_site_replication_targets().await? else {
// Enabled without a service account is a state this site cannot
// broadcast from either; the peers are still the reconciler's.
@@ -401,7 +436,12 @@ pub(crate) async fn persist_site_replication_state_no_lock(
guard: &crate::storage_api::site_replication::WriteCommitGuard,
) -> S3Result<()> {
state.peers = normalize_peer_map_by_identity(state.peers);
if state.peers.len() <= 1 && state.pending_rotation.is_none() && state.pending_remove.is_none() {
if state.peers.len() <= 1
&& state.pending_rotation.is_none()
&& !state.removal_pending()
&& state.remove_v2_receipts.is_empty()
&& state.removal_v2_fence.is_none()
{
match delete_config_no_lock(store, SITE_REPLICATION_STATE_PATH).await {
Ok(()) | Err(StorageError::ConfigNotFound) => Ok(()),
Err(err) => Err(S3Error::with_message(S3ErrorCode::InternalError, format!("clear state failed: {err}"))),
@@ -82,7 +82,7 @@ fn replication_deny_edit_error() -> S3Error {
/// handlers.
pub(crate) fn prepare_replication_config_edit<R: AuthorizationRequest + ?Sized>(
request: &R,
) -> impl Future<Output = S3Result<(HashSet<String>, OperatorRuleContract)>> + Send + use<R> {
) -> impl Future<Output = S3Result<(HashSet<String>, OperatorRuleContract)>> + Send + 'static {
let is_owner = req_info_ref(request).map(|info| info.is_owner).map_err(S3Error::from);
async move {
if !is_owner? && site_replication_gate_enabled().await? {
+2 -1
View File
@@ -391,8 +391,9 @@ pub(crate) mod site_replication {
pub(crate) mod metadata_sys {
pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata_sys::{
BucketMetadataMutationGuard, acquire_bucket_metadata_transaction_lock_for_incarnation,
capture_bucket_metadata_incarnation, get, get_replication_config, get_versioning_config, list_bucket_targets,
update_if_incarnation,
update_if_incarnation, update_under_transaction_lock,
};
}
+1
View File
@@ -30,6 +30,7 @@ checked_files=(
"rustfs/src/admin/handlers/kms_keys.rs"
"rustfs/src/admin/handlers/kms_key_lifecycle.rs"
"rustfs/src/admin/handlers/site_replication.rs"
"rustfs/src/admin/handlers/site_replication/removal.rs"
"rustfs/src/site_replication/mod.rs"
"rustfs/src/site_replication/identity.rs"
"rustfs/src/site_replication/state_lock.rs"