gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-08 14:23:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(replication): harden bucket replication correctness (#4116)

Browse Source
This commit is contained in:
cxymds
2026-07-01 09:21:30 +08:00
committed by GitHub
parent 722c0805ce
commit 6e5c58ca3f
8 changed files with 763 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/admin/handlers/replication.rs
+210 -6
View File
@@ -20,7 +20,7 @@ use crate::admin::storage_api::bucket::metadata::BUCKET_TARGETS_FILE;
use crate::admin::storage_api::bucket::metadata_sys;
use crate::admin::storage_api::bucket::metadata_sys::get_replication_config;
use crate::admin::storage_api::bucket::replication::BucketStats;
use crate::admin::storage_api::bucket::target::BucketTarget;
use crate::admin::storage_api::bucket::target::{BucketTarget, BucketTargetType, Credentials as TargetCredentials, LatencyStat};
use crate::admin::storage_api::bucket::target_sys::{BucketTargetError, BucketTargetSys};
use crate::admin::storage_api::contract::bucket::{BucketOperations, BucketOptions};
use crate::admin::storage_api::error::StorageError;
@@ -36,7 +36,10 @@ use rustfs_credentials::Credentials;
use rustfs_policy::policy::action::{Action, AdminAction};
use s3s::header::CONTENT_TYPE;
use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
use serde::Deserialize;
use std::collections::HashMap;
use std::time::Duration;
use time::OffsetDateTime;
use tracing::{debug, error, info, warn};
use url::Host;
@@ -68,6 +71,139 @@ fn map_bucket_target_error(err: BucketTargetError) -> S3Error {
}
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
struct RemoteTargetCredentialsRequest {
#[serde(rename = "accessKey")]
access_key: String,
#[serde(rename = "secretKey")]
secret_key: String,
session_token: Option<String>,
expiration: Option<chrono::DateTime<chrono::Utc>>,
}
impl From<RemoteTargetCredentialsRequest> for TargetCredentials {
fn from(value: RemoteTargetCredentialsRequest) -> Self {
Self {
access_key: value.access_key,
secret_key: value.secret_key,
session_token: value.session_token,
expiration: value.expiration,
}
}
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
struct RemoteTargetRequest {
#[serde(rename = "sourcebucket", default)]
source_bucket: String,
endpoint: String,
credentials: RemoteTargetCredentialsRequest,
#[serde(rename = "targetbucket")]
target_bucket: String,
#[serde(default)]
secure: bool,
#[serde(default)]
path: String,
#[serde(default)]
api: String,
#[serde(default)]
arn: String,
#[serde(rename = "type")]
target_type: BucketTargetType,
#[serde(default)]
region: String,
#[serde(alias = "bandwidth", default)]
bandwidth_limit: i64,
#[serde(rename = "replicationSync", default)]
replication_sync: bool,
#[serde(default)]
storage_class: String,
#[serde(rename = "skipTlsVerify", default)]
skip_tls_verify: bool,
#[serde(rename = "caCertPem", default)]
ca_cert_pem: String,
#[serde(rename = "healthCheckDuration", default)]
health_check_duration: u64,
#[serde(rename = "disableProxy", default)]
disable_proxy: bool,
#[serde(rename = "resetBeforeDate", with = "time::serde::rfc3339::option", default)]
reset_before_date: Option<OffsetDateTime>,
#[serde(default)]
reset_id: String,
#[serde(rename = "totalDowntime", default)]
total_downtime: u64,
#[serde(rename = "lastOnline", with = "time::serde::rfc3339::option", default)]
last_online: Option<OffsetDateTime>,
#[serde(rename = "isOnline", default)]
online: bool,
#[serde(default)]
latency: LatencyStat,
#[serde(default)]
deployment_id: String,
#[serde(default)]
edge: bool,
#[serde(rename = "edgeSyncBeforeExpiry", default)]
edge_sync_before_expiry: bool,
#[serde(rename = "offlineCount", default)]
offline_count: u64,
}
impl RemoteTargetRequest {
fn into_bucket_target(self) -> S3Result<BucketTarget> {
if self.endpoint.trim().is_empty() {
return Err(s3_error!(InvalidRequest, "endpoint is required"));
}
if self.target_bucket.trim().is_empty() {
return Err(s3_error!(InvalidRequest, "targetbucket is required"));
}
if !self.target_type.is_valid() {
return Err(s3_error!(InvalidRequest, "type is invalid"));
}
if self.credentials.access_key.trim().is_empty() {
return Err(s3_error!(InvalidRequest, "credentials.accessKey is required"));
}
if self.credentials.secret_key.trim().is_empty() {
return Err(s3_error!(InvalidRequest, "credentials.secretKey is required"));
}
Ok(BucketTarget {
source_bucket: self.source_bucket,
endpoint: self.endpoint,
credentials: Some(self.credentials.into()),
target_bucket: self.target_bucket,
secure: self.secure,
path: self.path,
api: self.api,
arn: self.arn,
target_type: self.target_type,
region: self.region,
bandwidth_limit: self.bandwidth_limit,
replication_sync: self.replication_sync,
storage_class: self.storage_class,
skip_tls_verify: self.skip_tls_verify,
ca_cert_pem: self.ca_cert_pem,
health_check_duration: Duration::from_secs(self.health_check_duration),
disable_proxy: self.disable_proxy,
reset_before_date: self.reset_before_date,
reset_id: self.reset_id,
total_downtime: Duration::from_secs(self.total_downtime),
last_online: self.last_online,
online: self.online,
latency: self.latency,
deployment_id: self.deployment_id,
edge: self.edge,
edge_sync_before_expiry: self.edge_sync_before_expiry,
offline_count: self.offline_count,
})
}
}
fn validate_remote_target_tls_settings(remote_target: &BucketTarget) -> S3Result<()> {
let has_custom_ca = !remote_target.ca_cert_pem.trim().is_empty();
@@ -227,10 +363,12 @@ impl Operation for SetRemoteTargetHandler {
}
};
let mut remote_target: BucketTarget = serde_json::from_slice(&body).map_err(|e| {
error!("Failed to parse BucketTarget from body: {}", e);
ApiError::other(e)
})?;
let mut remote_target = serde_json::from_slice::<RemoteTargetRequest>(&body)
.map_err(|e| {
error!("Failed to parse remote target request body: {}", e);
S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid remote target request: {e}"))
})?
.into_bucket_target()?;
validate_remote_target_tls_settings(&remote_target)?;
let Ok(target_url) = remote_target.url() else {
@@ -448,10 +586,23 @@ impl Operation for RemoveRemoteTargetHandler {
#[cfg(test)]
mod tests {
use super::{extract_query_params, validate_remote_target_tls_settings};
use super::{RemoteTargetRequest, extract_query_params, validate_remote_target_tls_settings};
use crate::admin::storage_api::bucket::target::BucketTarget;
use http::Uri;
fn valid_remote_target_request() -> serde_json::Value {
serde_json::json!({
"endpoint": "192.168.1.10:9000",
"credentials": {
"accessKey": "access",
"secretKey": "secret"
},
"targetbucket": "target",
"secure": true,
"type": "replication"
})
}
#[test]
fn test_extract_query_params_decodes_percent_encoded_values() {
let uri: Uri = "/rustfs/admin/v3/list-remote-targets?bucket=foo%2Fbar&flag=a+b"
@@ -512,4 +663,57 @@ mod tests {
})
.expect("HTTPS targets should allow skipTlsVerify when no custom CA is configured");
}
#[test]
fn remote_target_request_rejects_unknown_fields() {
let mut request = valid_remote_target_request();
request["unexpected"] = serde_json::json!(true);
let err = serde_json::from_value::<RemoteTargetRequest>(request)
.expect_err("remote target request should reject unknown fields");
assert!(err.to_string().contains("unknown field"));
}
#[test]
fn remote_target_request_rejects_missing_credentials() {
let mut request = valid_remote_target_request();
request
.as_object_mut()
.expect("request should be an object")
.remove("credentials");
let err =
serde_json::from_value::<RemoteTargetRequest>(request).expect_err("remote target request should require credentials");
assert!(err.to_string().contains("missing field"));
}
#[test]
fn remote_target_request_rejects_empty_secret_key() {
let mut request = valid_remote_target_request();
request["credentials"]["secretKey"] = serde_json::json!("");
let request: RemoteTargetRequest =
serde_json::from_value(request).expect("request should deserialize before semantic validation");
let err = match request.into_bucket_target() {
Ok(_) => panic!("empty secret key should fail semantic validation"),
Err(err) => err,
};
assert!(err.to_string().contains("credentials.secretKey is required"));
}
#[test]
fn remote_target_request_converts_to_bucket_target() {
let target = serde_json::from_value::<RemoteTargetRequest>(valid_remote_target_request())
.expect("request should deserialize")
.into_bucket_target()
.expect("request should pass semantic validation");
assert_eq!(target.endpoint, "192.168.1.10:9000");
assert_eq!(target.target_bucket, "target");
assert!(target.secure);
assert_eq!(target.credentials.expect("credentials should be present").access_key, "access");
}
}
rustfs/src/admin/router.rs
+43 -22
View File
@@ -1532,15 +1532,19 @@ async fn authorize_replication_extension_request(req: &mut S3Request<Body>, ext_
}
})?;
let action = match ext_req.route {
ReplicationExtRoute::MetricsV1 | ReplicationExtRoute::MetricsV2 | ReplicationExtRoute::Check => {
authorize_request(req, replication_extension_policy_action(ext_req.route)).await
}
fn replication_extension_policy_action(route: ReplicationExtRoute) -> Action {
match route {
ReplicationExtRoute::MetricsV1 | ReplicationExtRoute::MetricsV2 => {
Action::S3Action(S3Action::GetReplicationConfigurationAction)
}
ReplicationExtRoute::Check => Action::S3Action(S3Action::PutReplicationConfigurationAction),
ReplicationExtRoute::ResetStart | ReplicationExtRoute::ResetStatus => {
Action::S3Action(S3Action::ResetBucketReplicationStateAction)
}
};
authorize_request(req, action).await
}
}
fn parse_reset_start_target(uri: &Uri) -> S3Result<ReplicationResetStartRequest> {
@@ -1586,17 +1590,17 @@ fn collect_resettable_replication_target_arns(config: &s3s::dto::ReplicationConf
continue;
}
let arn = if config.role.is_empty() {
rule.destination.bucket.clone()
let arn = if config.role.trim().is_empty() {
rule.destination.bucket.trim().to_string()
} else {
config.role.clone()
config.role.trim().to_string()
};
if seen.insert(arn.clone()) {
arns.push(arn);
}
if !config.role.is_empty() {
if !config.role.trim().is_empty() {
break;
}
}
@@ -1775,26 +1779,27 @@ fn validate_replication_check_config_targets(
.map(|target| target.arn.as_str())
.collect::<HashSet<_>>();
let role = config.role.trim();
if !role.is_empty() {
if !configured_arns.contains(role) {
return Err(s3_error!(InvalidRequest, "replication config has stale target {role}"));
}
return Ok(());
}
for rule in &config.rules {
if rule.status == s3s::dto::ReplicationRuleStatus::from_static(s3s::dto::ReplicationRuleStatus::DISABLED) {
continue;
}
let configured_arn = if config.role.is_empty() {
rule.destination.bucket.as_str()
} else {
config.role.as_str()
};
if configured_arns.contains(configured_arn) {
continue;
let configured_arn = rule.destination.bucket.trim();
if !configured_arn.is_empty() && !configured_arns.contains(configured_arn) {
let rule_id = rule.id.as_deref().unwrap_or("<unknown>");
return Err(s3_error!(
InvalidRequest,
"replication rule {rule_id} references stale target {configured_arn}"
));
}
return Err(s3_error!(
InvalidRequest,
"replication config with rule ID {} has a stale target",
rule.id.clone().unwrap_or_default()
));
}
Ok(())
@@ -2630,6 +2635,22 @@ mod tests {
assert!(parse_replication_extension_request(&Method::PUT, &wrong_method_status).is_none());
}
#[test]
fn replication_extension_policy_action_uses_write_permission_for_active_check() {
assert_eq!(
replication_extension_policy_action(ReplicationExtRoute::MetricsV1),
Action::S3Action(S3Action::GetReplicationConfigurationAction)
);
assert_eq!(
replication_extension_policy_action(ReplicationExtRoute::Check),
Action::S3Action(S3Action::PutReplicationConfigurationAction)
);
assert_eq!(
replication_extension_policy_action(ReplicationExtRoute::ResetStart),
Action::S3Action(S3Action::ResetBucketReplicationStateAction)
);
}
#[test]
fn parse_reset_start_target_defaults_reset_before_and_supports_older_than() {
let no_window: Uri = "/demo-bucket?replication-reset".parse().expect("uri should parse");
rustfs/src/admin/storage_api.rs
+1
View File
@@ -319,6 +319,7 @@ pub(crate) mod target {
pub(crate) type BucketTargetType = super::ecstore_bucket::target::BucketTargetType;
pub(crate) type BucketTargets = super::ecstore_bucket::target::BucketTargets;
pub(crate) type Credentials = super::ecstore_bucket::target::Credentials;
pub(crate) type LatencyStat = super::ecstore_bucket::target::LatencyStat;
}
pub(crate) mod ecstore_utils {
rustfs/src/app/bucket_usecase.rs
+157 -36
View File
@@ -246,15 +246,14 @@ fn notify_bucket_metadata_reload(
});
}
fn replication_target_arns(config: &ReplicationConfiguration) -> HashSet<String> {
fn active_replication_rule_destination_arns(config: &ReplicationConfiguration) -> HashSet<String> {
let mut arns = HashSet::new();
if !config.role.trim().is_empty() {
arns.insert(config.role.clone());
return arns;
}
for rule in &config.rules {
if rule.status == ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED) {
continue;
}
let arn = rule.destination.bucket.trim();
if !arn.is_empty() {
arns.insert(arn.to_string());
@@ -264,6 +263,17 @@ fn replication_target_arns(config: &ReplicationConfiguration) -> HashSet<String>
arns
}
fn replication_target_arns(config: &ReplicationConfiguration) -> HashSet<String> {
let role = config.role.trim();
if !role.is_empty() {
let mut arns = HashSet::new();
arns.insert(role.to_string());
return arns;
}
active_replication_rule_destination_arns(config)
}
fn validate_replication_config_targets(targets: &BucketTargets, config: &ReplicationConfiguration) -> S3Result<()> {
let configured_arns = targets
.targets
@@ -272,28 +282,21 @@ fn validate_replication_config_targets(targets: &BucketTargets, config: &Replica
.map(|target| target.arn.as_str())
.collect::<HashSet<_>>();
for rule in &config.rules {
if rule.status == ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED) {
continue;
}
let configured_arn = if config.role.trim().is_empty() {
rule.destination.bucket.trim()
} else {
config.role.trim()
};
if !configured_arn.is_empty() && configured_arns.contains(configured_arn) {
continue;
}
let role = config.role.trim();
let destination_arns = active_replication_rule_destination_arns(config);
if !role.is_empty() && destination_arns.len() > 1 {
return Err(s3_error!(
InvalidRequest,
"replication config with rule ID {} has a stale target",
rule.id.clone().unwrap_or_default()
"replication config with Role cannot define multiple destination targets"
));
}
for configured_arn in replication_target_arns(config) {
if !configured_arns.contains(configured_arn.as_str()) {
return Err(s3_error!(InvalidRequest, "replication config has a stale target"));
}
}
Ok(())
}
@@ -317,41 +320,81 @@ async fn validate_bucket_replication_update(bucket: &str, config: &ReplicationCo
validate_replication_config_targets(&targets, config)
}
async fn remove_replication_targets_for_config(bucket: &str, config: &ReplicationConfiguration) -> S3Result<()> {
async fn replication_targets_without_config_targets(
bucket: &str,
config: &ReplicationConfiguration,
) -> S3Result<Option<(BucketTargets, usize)>> {
let target_arns = replication_target_arns(config);
if target_arns.is_empty() {
return Ok(());
return Ok(None);
}
let mut targets = match metadata_sys::get_bucket_targets_config(bucket).await {
Ok(targets) => targets,
Err(StorageError::ConfigNotFound) => {
BucketTargetSys::get().update_all_targets(bucket, None).await;
return Ok(());
return Ok(None);
}
Err(err) => return Err(ApiError::from(err).into()),
};
let removed = remove_replication_targets_from_config_targets(&mut targets, &target_arns);
if removed == 0 {
return Ok(None);
}
Ok(Some((targets, removed)))
}
fn remove_replication_targets_from_config_targets(targets: &mut BucketTargets, target_arns: &HashSet<String>) -> usize {
let original_len = targets.targets.len();
targets.targets.retain(|target| {
target.target_type != BucketTargetType::ReplicationService || !target_arns.contains(target.arn.as_str())
});
if targets.targets.len() == original_len {
return Ok(());
}
original_len - targets.targets.len()
}
let removed = original_len - targets.targets.len();
async fn write_replication_targets_after_config_delete(bucket: &str, targets: &BucketTargets, removed: usize) -> S3Result<()> {
let json_targets = serde_json::to_vec(&targets).map_err(to_internal_error)?;
metadata_sys::update(bucket, BUCKET_TARGETS_FILE, json_targets)
.await
.map_err(ApiError::from)?;
BucketTargetSys::get().update_all_targets(bucket, Some(&targets)).await;
BucketTargetSys::get().update_all_targets(bucket, Some(targets)).await;
info!(bucket = %bucket, removed, "removed replication remote targets referenced by deleted bucket replication config");
Ok(())
}
async fn restore_replication_config_after_target_cleanup_failure(
bucket: &str,
config: &ReplicationConfiguration,
cleanup_err: S3Error,
) -> S3Error {
match serialize(config) {
Ok(data) => {
if let Err(restore_err) = metadata_sys::update(bucket, BUCKET_REPLICATION_CONFIG, data).await {
error!(
bucket = %bucket,
error = ?restore_err,
cleanup_error = ?cleanup_err,
"failed to restore bucket replication config after target cleanup failure"
);
}
}
Err(restore_err) => {
error!(
bucket = %bucket,
error = ?restore_err,
cleanup_error = ?cleanup_err,
"failed to serialize bucket replication config for restore after target cleanup failure"
);
}
}
cleanup_err
}
fn versioning_configuration_has_object_lock_incompatible_settings(config: &VersioningConfiguration) -> bool {
config.suspended()
|| config.exclude_folders.unwrap_or(false)
@@ -1129,14 +1172,22 @@ impl DefaultBucketUsecase {
Err(StorageError::ConfigNotFound) => None,
Err(err) => return Err(ApiError::from(err).into()),
};
let updated_targets = if let Some(config) = replication_config.as_ref() {
replication_targets_without_config_targets(&bucket, config).await?
} else {
None
};
metadata_sys::delete(&bucket, BUCKET_REPLICATION_CONFIG)
.await
.map_err(ApiError::from)?;
if let Some(config) = replication_config.as_ref()
&& let Err(err) = remove_replication_targets_for_config(&bucket, config).await
if let Some((targets, removed)) = updated_targets
&& let Err(err) = write_replication_targets_after_config_delete(&bucket, &targets, removed).await
{
warn!(bucket = %bucket, error = ?err, "failed to remove replication targets referenced by deleted bucket replication config");
if let Some(config) = replication_config.as_ref() {
return Err(restore_replication_config_after_target_cleanup_failure(&bucket, config, err).await);
}
return Err(err);
}
let item = sr_bucket_meta_item(bucket.clone(), "replication-config");
@@ -2319,7 +2370,7 @@ mod tests {
let role = "arn:rustfs:replication:us-east-1:source:bucket";
let destination = "arn:rustfs:replication:us-east-1:target:bucket";
let config = ReplicationConfiguration {
role: role.to_string(),
role: format!(" {role} "),
rules: vec![replication_rule_for_target(destination)],
};
@@ -2386,13 +2437,44 @@ mod tests {
let arn = "arn:rustfs:replication:us-east-1:role-target:bucket";
let targets = replication_targets_with_arn(&[arn]);
let config = ReplicationConfiguration {
role: arn.to_string(),
role: format!(" {arn} "),
rules: vec![replication_rule_for_target("arn:rustfs:replication:us-east-1:ignored:bucket")],
};
validate_replication_config_targets(&targets, &config).expect("matching role ARN should pass validation");
}
#[test]
fn validate_replication_config_targets_rejects_role_with_multiple_destinations() {
let role = "arn:rustfs:replication:us-east-1:role-target:bucket";
let targets = replication_targets_with_arn(&[role]);
let config = ReplicationConfiguration {
role: role.to_string(),
rules: vec![
replication_rule_for_target("arn:rustfs:replication:us-east-1:target-a:bucket"),
replication_rule_for_target("arn:rustfs:replication:us-east-1:target-b:bucket"),
],
};
let err = validate_replication_config_targets(&targets, &config)
.expect_err("role plus multiple destinations should be rejected");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
}
#[test]
fn validate_replication_config_targets_trims_destination_arns() {
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
let targets = replication_targets_with_arn(&[arn]);
let config = ReplicationConfiguration {
role: String::new(),
rules: vec![replication_rule_for_target(
" arn:rustfs:replication:us-east-1:target:bucket ",
)],
};
validate_replication_config_targets(&targets, &config).expect("trimmed destination ARN should match configured target");
}
#[test]
fn validate_replication_config_targets_ignores_disabled_rules() {
let targets = replication_targets_with_arn(&[]);
@@ -2406,6 +2488,45 @@ mod tests {
validate_replication_config_targets(&targets, &config).expect("disabled rules should not require live targets");
}
#[test]
fn remove_replication_targets_from_config_targets_only_removes_referenced_replication_targets() {
let removed_arn = "arn:rustfs:replication:us-east-1:removed:bucket";
let kept_replication_arn = "arn:rustfs:replication:us-east-1:kept:bucket";
let kept_ilm_arn = "arn:rustfs:ilm:us-east-1:kept:bucket";
let mut targets = BucketTargets {
targets: vec![
BucketTarget {
arn: removed_arn.to_string(),
target_type: BucketTargetType::ReplicationService,
..Default::default()
},
BucketTarget {
arn: kept_replication_arn.to_string(),
target_type: BucketTargetType::ReplicationService,
..Default::default()
},
BucketTarget {
arn: kept_ilm_arn.to_string(),
target_type: BucketTargetType::IlmService,
..Default::default()
},
],
};
let target_arns = HashSet::from([removed_arn.to_string(), kept_ilm_arn.to_string()]);
let removed = remove_replication_targets_from_config_targets(&mut targets, &target_arns);
assert_eq!(removed, 1);
let remaining_arns = targets
.targets
.iter()
.map(|target| target.arn.as_str())
.collect::<HashSet<_>>();
assert!(!remaining_arns.contains(removed_arn));
assert!(remaining_arns.contains(kept_replication_arn));
assert!(remaining_arns.contains(kept_ilm_arn));
}
#[test]
fn versioning_configuration_has_object_lock_incompatible_settings_rejects_suspended() {
let config = VersioningConfiguration {