mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-29 16:37:07 +00:00
fix(replication): harden live delete admission (#5599)
This commit is contained in:
@@ -1995,6 +1995,49 @@ mod tests {
|
||||
use super::*;
|
||||
use rcgen::generate_simple_self_signed;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct RecordingHttpConnector {
|
||||
request_uris: Arc<std::sync::Mutex<Vec<String>>>,
|
||||
}
|
||||
|
||||
impl SmithyHttpConnector for RecordingHttpConnector {
|
||||
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
|
||||
self.request_uris
|
||||
.lock()
|
||||
.expect("recorded request lock should not be poisoned")
|
||||
.push(request.uri().to_string());
|
||||
HttpConnectorFuture::ready(Ok(HttpResponse::new(
|
||||
aws_smithy_runtime_api::http::StatusCode::try_from(204_u16).expect("204 should be a valid response status"),
|
||||
SdkBody::empty(),
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
fn recording_target_client() -> (TargetClient, Arc<std::sync::Mutex<Vec<String>>>) {
|
||||
let request_uris = Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||
let connector = SharedHttpConnector::new(RecordingHttpConnector {
|
||||
request_uris: Arc::clone(&request_uris),
|
||||
});
|
||||
let http_client = http_client_fn(move |_settings, _components| connector.clone());
|
||||
let client = s3_client_with_http_client(443, http_client);
|
||||
(
|
||||
TargetClient {
|
||||
endpoint: "https://localhost:443".to_string(),
|
||||
credentials: None,
|
||||
bucket: "target-bucket".to_string(),
|
||||
storage_class: String::new(),
|
||||
disable_proxy: false,
|
||||
arn: "arn:rustfs:replication:us-east-1:target:bucket".to_string(),
|
||||
reset_id: String::new(),
|
||||
secure: true,
|
||||
health_check_duration: Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(client),
|
||||
},
|
||||
request_uris,
|
||||
)
|
||||
}
|
||||
|
||||
fn spawn_single_request_https_server(cert: &rcgen::CertifiedKey<rcgen::KeyPair>) -> (u16, std::thread::JoinHandle<()>) {
|
||||
use std::io::{Read, Write};
|
||||
|
||||
@@ -2285,6 +2328,32 @@ mod tests {
|
||||
assert_eq!(got.as_deref(), Some(vid.as_str()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remove_object_writes_null_purge_and_omits_marker_creation_version_queries() {
|
||||
let (client, request_uris) = recording_target_client();
|
||||
client
|
||||
.remove_object("target-bucket", "object", Some("null".to_string()), remove_opts(true, false))
|
||||
.await
|
||||
.expect("explicit null version purge should reach the target client");
|
||||
client
|
||||
.remove_object("target-bucket", "object", Some(Uuid::new_v4().to_string()), remove_opts(true, true))
|
||||
.await
|
||||
.expect("delete marker creation should reach the target client");
|
||||
|
||||
let request_uris = request_uris.lock().expect("recorded request lock should not be poisoned");
|
||||
assert_eq!(request_uris.len(), 2);
|
||||
assert!(
|
||||
request_uris[0].contains("versionId=null"),
|
||||
"an explicit null purge must be emitted as a target versionId query: {}",
|
||||
request_uris[0]
|
||||
);
|
||||
assert!(
|
||||
!request_uris[1].contains("versionId="),
|
||||
"delete marker creation must omit the target versionId query: {}",
|
||||
request_uris[1]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_object_headers_include_non_empty_source_etag_only() {
|
||||
let mut opts = PutObjectOptions::default();
|
||||
|
||||
@@ -16,6 +16,7 @@ use super::msgp_decode::{read_msgp_ext8_time, skip_msgp_value, write_msgp_time};
|
||||
use super::object_lock::ObjectLockApi;
|
||||
use super::versioning::VersioningApi;
|
||||
use super::{quota::BucketQuota, target::BucketTargets};
|
||||
use crate::bucket::replication::invalid_replication_config_status_field;
|
||||
use crate::bucket::utils::deserialize;
|
||||
use crate::config::com::{read_config, save_config};
|
||||
use crate::disk::BUCKET_META_PREFIX;
|
||||
@@ -25,9 +26,9 @@ use crate::store::ECStore;
|
||||
use byteorder::{BigEndian, ByteOrder, LittleEndian};
|
||||
use rustfs_policy::policy::BucketPolicy;
|
||||
use s3s::dto::{
|
||||
AccelerateConfiguration, BucketLifecycleConfiguration, BucketLoggingStatus, CORSConfiguration, NotificationConfiguration,
|
||||
ObjectLockConfiguration, PublicAccessBlockConfiguration, ReplicationConfiguration, RequestPaymentConfiguration,
|
||||
ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration, WebsiteConfiguration,
|
||||
AccelerateConfiguration, BucketLifecycleConfiguration, BucketLoggingStatus, BucketVersioningStatus, CORSConfiguration,
|
||||
NotificationConfiguration, ObjectLockConfiguration, PublicAccessBlockConfiguration, ReplicationConfiguration,
|
||||
RequestPaymentConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration, WebsiteConfiguration,
|
||||
};
|
||||
use serde::Serializer;
|
||||
use sha2::{Digest, Sha256};
|
||||
@@ -751,11 +752,33 @@ impl BucketMetadata {
|
||||
self.object_lock_config_updated_at = updated;
|
||||
}
|
||||
BUCKET_VERSIONING_CONFIG => {
|
||||
let config = if data.is_empty() {
|
||||
None
|
||||
} else {
|
||||
let config = deserialize::<VersioningConfiguration>(&data)?;
|
||||
if config.status.as_ref().is_some_and(|status| {
|
||||
!matches!(status.as_str(), BucketVersioningStatus::ENABLED | BucketVersioningStatus::SUSPENDED)
|
||||
}) {
|
||||
return Err(Error::other("bucket versioning configuration has an invalid status"));
|
||||
}
|
||||
Some(config)
|
||||
};
|
||||
self.versioning_config_xml = data;
|
||||
self.versioning_config = config;
|
||||
self.versioning_config_updated_at = updated;
|
||||
}
|
||||
BUCKET_REPLICATION_CONFIG => {
|
||||
let config = if data.is_empty() {
|
||||
None
|
||||
} else {
|
||||
let config = deserialize::<ReplicationConfiguration>(&data)?;
|
||||
if let Some(field) = invalid_replication_config_status_field(&config) {
|
||||
return Err(Error::other(format!("replication field {field} has an invalid status")));
|
||||
}
|
||||
Some(config)
|
||||
};
|
||||
self.replication_config_xml = data;
|
||||
self.replication_config = config;
|
||||
self.replication_config_updated_at = updated;
|
||||
}
|
||||
BUCKET_TARGETS_FILE => {
|
||||
@@ -825,7 +848,6 @@ impl BucketMetadata {
|
||||
/// ambient (first) one. [`BucketMetadata::save`] keeps the ambient default.
|
||||
pub async fn save_with_store(&mut self, store: std::sync::Arc<crate::store::ECStore>) -> Result<()> {
|
||||
self.parse_all_configs()?;
|
||||
|
||||
let mut buf: Vec<u8> = vec![0; 4];
|
||||
|
||||
LittleEndian::write_u16(&mut buf[0..2], BUCKET_METADATA_FORMAT);
|
||||
@@ -906,6 +928,7 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.versioning_config = None;
|
||||
if !self.versioning_config_xml.is_empty()
|
||||
&& let Err(e) =
|
||||
deserialize::<VersioningConfiguration>(&self.versioning_config_xml).map(|c| self.versioning_config = Some(c))
|
||||
@@ -960,6 +983,7 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
self.replication_config = None;
|
||||
if !self.replication_config_xml.is_empty()
|
||||
&& let Err(e) =
|
||||
deserialize::<ReplicationConfiguration>(&self.replication_config_xml).map(|c| self.replication_config = Some(c))
|
||||
@@ -1345,6 +1369,66 @@ mod test {
|
||||
assert!(bm.tagging_config.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_admission_configs_update_parsed_state_atomically() {
|
||||
let mut bm = BucketMetadata::new("test-bucket");
|
||||
let versioning_xml = b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>";
|
||||
let replication_xml = b"<ReplicationConfiguration><Role>arn:aws:s3:::target-bucket</Role><Rule><ID>rule1</ID><Status>Enabled</Status><Prefix></Prefix><Destination><Bucket>arn:aws:s3:::target-bucket</Bucket></Destination></Rule></ReplicationConfiguration>";
|
||||
|
||||
bm.update_config(BUCKET_VERSIONING_CONFIG, versioning_xml.to_vec())
|
||||
.expect("valid versioning config should update parsed state");
|
||||
bm.update_config(BUCKET_REPLICATION_CONFIG, replication_xml.to_vec())
|
||||
.expect("valid replication config should update parsed state");
|
||||
|
||||
assert!(bm.versioning_config.as_ref().is_some_and(VersioningConfiguration::enabled));
|
||||
assert_eq!(
|
||||
bm.replication_config.as_ref().map(|config| config.role.as_str()),
|
||||
Some("arn:aws:s3:::target-bucket")
|
||||
);
|
||||
|
||||
assert!(
|
||||
bm.update_config(BUCKET_VERSIONING_CONFIG, b"<VersioningConfiguration>".to_vec())
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
bm.update_config(BUCKET_REPLICATION_CONFIG, b"<ReplicationConfiguration>".to_vec())
|
||||
.is_err()
|
||||
);
|
||||
|
||||
assert_eq!(bm.versioning_config_xml, versioning_xml);
|
||||
assert_eq!(bm.replication_config_xml, replication_xml);
|
||||
assert!(bm.versioning_config.as_ref().is_some_and(VersioningConfiguration::enabled));
|
||||
assert_eq!(
|
||||
bm.replication_config.as_ref().map(|config| config.role.as_str()),
|
||||
Some("arn:aws:s3:::target-bucket")
|
||||
);
|
||||
|
||||
assert!(
|
||||
bm.update_config(
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
b"<VersioningConfiguration><Status>Enabld</Status></VersioningConfiguration>".to_vec(),
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
bm.update_config(
|
||||
BUCKET_REPLICATION_CONFIG,
|
||||
b"<ReplicationConfiguration><Role>arn:aws:s3:::target-bucket</Role><Rule><ID>rule1</ID><Status>Enabld</Status><Prefix></Prefix><Destination><Bucket>arn:aws:s3:::target-bucket</Bucket></Destination></Rule></ReplicationConfiguration>".to_vec(),
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(bm.versioning_config_xml, versioning_xml);
|
||||
assert_eq!(bm.replication_config_xml, replication_xml);
|
||||
|
||||
bm.versioning_config_xml = b"<VersioningConfiguration>".to_vec();
|
||||
bm.replication_config_xml = b"<ReplicationConfiguration>".to_vec();
|
||||
bm.parse_all_configs()
|
||||
.expect("bulk config parsing reports malformed fields through cleared typed state");
|
||||
|
||||
assert!(bm.versioning_config.is_none());
|
||||
assert!(bm.replication_config.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn marshal_msg_complete_example() {
|
||||
// Create a complete BucketMetadata with various configurations
|
||||
|
||||
@@ -1027,7 +1027,6 @@ impl BucketMetadataSys {
|
||||
/// server's metadata never leaks into the ambient (first) instance.
|
||||
pub(crate) async fn persist_and_set(&self, bm: BucketMetadata) -> Result<()> {
|
||||
let mut bm = bm;
|
||||
|
||||
bm.save_with_store(self.api.clone()).await?;
|
||||
|
||||
self.set(bm.name.clone(), Arc::new(bm)).await;
|
||||
@@ -1221,7 +1220,9 @@ impl BucketMetadataSys {
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(config) = &bm.versioning_config {
|
||||
if !bm.versioning_config_xml.is_empty() && bm.versioning_config.is_none() {
|
||||
Err(Error::other("persisted bucket versioning configuration is invalid"))
|
||||
} else if let Some(config) = &bm.versioning_config {
|
||||
Ok((config.clone(), bm.versioning_config_updated_at))
|
||||
} else {
|
||||
Ok((VersioningConfiguration::default(), bm.versioning_config_updated_at))
|
||||
@@ -1407,7 +1408,9 @@ impl BucketMetadataSys {
|
||||
pub async fn get_replication_config(&self, bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
let (bm, _) = self.get_config(bucket).await?;
|
||||
|
||||
if let Some(config) = &bm.replication_config {
|
||||
if !bm.replication_config_xml.is_empty() && bm.replication_config.is_none() {
|
||||
Err(Error::other("persisted bucket replication configuration is invalid"))
|
||||
} else if let Some(config) = &bm.replication_config {
|
||||
Ok((config.clone(), bm.replication_config_updated_at))
|
||||
} else {
|
||||
Err(Error::ConfigNotFound)
|
||||
@@ -1481,6 +1484,28 @@ mod tests {
|
||||
use serial_test::serial;
|
||||
use tokio::time::timeout;
|
||||
|
||||
#[tokio::test]
|
||||
async fn malformed_delete_configs_are_not_treated_as_absent() {
|
||||
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let sys = BucketMetadataSys::new(ecstore);
|
||||
let bucket = "malformed-delete-config";
|
||||
let mut metadata = BucketMetadata::new(bucket);
|
||||
metadata.versioning_config_xml = b"<VersioningConfiguration>".to_vec();
|
||||
metadata.versioning_config = None;
|
||||
metadata.replication_config_xml = b"<ReplicationConfiguration>".to_vec();
|
||||
metadata.replication_config = None;
|
||||
sys.set(bucket.to_string(), Arc::new(metadata)).await;
|
||||
|
||||
assert!(
|
||||
sys.get_versioning_config(bucket).await.is_err(),
|
||||
"malformed versioning metadata must block destructive requests"
|
||||
);
|
||||
assert!(
|
||||
sys.get_replication_config(bucket).await.is_err(),
|
||||
"malformed replication metadata must not be reported as ConfigNotFound"
|
||||
);
|
||||
}
|
||||
|
||||
/// Concurrent cache misses for one bucket must collapse into a single disk
|
||||
/// load.
|
||||
///
|
||||
|
||||
@@ -45,8 +45,9 @@ mod runtime_boundary;
|
||||
|
||||
pub use datatypes::ResyncStatusType;
|
||||
pub use replication_config_boundary::{
|
||||
ObjectOpts, ReplicationConfigurationExt, ReplicationTargetValidationError, replication_target_arns,
|
||||
should_remove_replication_target, unsupported_replication_config_field, validate_replication_config_target_arns,
|
||||
ObjectOpts, ReplicationConfigurationExt, ReplicationTargetValidationError, invalid_replication_config_status_field,
|
||||
replication_target_arns, should_remove_replication_target, unsupported_replication_config_field,
|
||||
validate_replication_config_target_arns,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use replication_filemeta_boundary::ReplicateTargetDecision;
|
||||
@@ -62,7 +63,7 @@ pub(crate) use replication_filemeta_boundary::{
|
||||
pub(crate) use replication_lifecycle_bridge::{ReplicationLifecycleBridge, ReplicationLifecycleConfig};
|
||||
pub(crate) use replication_migration_bridge::ReplicationMigrationBridge;
|
||||
pub use replication_object_bridge::ReplicationObjectBridge;
|
||||
pub use replication_object_config::ReplicationConfig;
|
||||
pub use replication_object_config::{DeleteReplicationConfigSnapshot, ReplicationConfig};
|
||||
pub use replication_object_decision_boundary::{
|
||||
MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, delete_replication_state_from_config,
|
||||
delete_replication_version_id, should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub use rustfs_replication::{
|
||||
ObjectOpts, ReplicationConfigurationExt, ReplicationTargetValidationError, replication_target_arns,
|
||||
should_remove_replication_target, unsupported_replication_config_field, validate_replication_config_target_arns,
|
||||
ObjectOpts, ReplicationConfigurationExt, ReplicationRuleExt, ReplicationTargetValidationError,
|
||||
invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
|
||||
unsupported_replication_config_field, validate_replication_config_target_arns,
|
||||
};
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) use rustfs_filemeta::NULL_VERSION_ID;
|
||||
pub use rustfs_replication::{MrfOpKind, MrfReplicateEntry};
|
||||
pub(crate) use rustfs_replication::{
|
||||
REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL_DELETE, ReplicateTargetDecision, ReplicatedInfos,
|
||||
|
||||
@@ -12,14 +12,19 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::bucket::metadata_sys;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::bucket::{metadata::BucketMetadata, metadata_sys};
|
||||
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use super::replication_error_boundary::{Error, Result};
|
||||
|
||||
pub(crate) type ReplicationInstanceContext = InstanceContext;
|
||||
|
||||
const REPLICATION_DIR: &str = ".replication";
|
||||
const RESYNC_FILE_NAME: &str = "resync.bin";
|
||||
|
||||
@@ -45,6 +50,20 @@ impl ReplicationMetadataStore {
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_metadata(bucket: &str) -> Result<Arc<BucketMetadata>> {
|
||||
let sys = metadata_sys::get_bucket_metadata_sys()?;
|
||||
let sys = sys.read().await;
|
||||
Ok(sys.get_config(bucket).await?.0)
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_metadata_in(ctx: &ReplicationInstanceContext, bucket: &str) -> Result<Arc<BucketMetadata>> {
|
||||
let sys = ctx
|
||||
.bucket_metadata_sys()
|
||||
.ok_or_else(|| Error::other("request instance bucket metadata system is not initialized"))?;
|
||||
let sys = sys.read().await;
|
||||
Ok(sys.get_config(bucket).await?.0)
|
||||
}
|
||||
|
||||
pub(crate) fn rustfs_meta_bucket() -> &'static str {
|
||||
RUSTFS_META_BUCKET
|
||||
}
|
||||
|
||||
@@ -16,14 +16,17 @@ use std::{collections::HashMap, sync::Arc};
|
||||
|
||||
use super::replication_error_boundary::Result;
|
||||
use super::replication_filemeta_boundary::{ReplicateDecision, ReplicationStatusType, ReplicationType};
|
||||
use super::replication_metadata_boundary::ReplicationInstanceContext;
|
||||
use super::replication_object_config::{
|
||||
check_replicate_delete, check_replicate_delete_strict, get_must_replicate_options, must_replicate,
|
||||
DeleteReplicationConfigSnapshot, check_replicate_delete, check_replicate_delete_strict, check_replicate_delete_with_snapshot,
|
||||
get_must_replicate_options, load_delete_replication_config_in, load_delete_request_config_in, must_replicate,
|
||||
};
|
||||
use super::replication_object_decision_boundary::MustReplicateOptions;
|
||||
use super::replication_pool::{schedule_replication, schedule_replication_delete};
|
||||
use super::replication_queue_boundary::DeletedObjectReplicationInfo;
|
||||
use super::replication_storage_boundary::{
|
||||
DeletedObject, ObjectInfo, ObjectOptions, ObjectToDelete, ReplicationStorage, deleted_object_for_replication,
|
||||
DeletedObject, ObjectInfo, ObjectOptions, ObjectToDelete, ReplicationObjectStore, ReplicationStorage,
|
||||
deleted_object_for_replication,
|
||||
};
|
||||
|
||||
pub struct ReplicationObjectBridge;
|
||||
@@ -63,6 +66,39 @@ impl ReplicationObjectBridge {
|
||||
check_replicate_delete_strict(bucket, object, source, opts, get_error).await
|
||||
}
|
||||
|
||||
pub async fn delete_request_config(api: &ReplicationObjectStore, bucket: &str) -> Result<DeleteReplicationConfigSnapshot> {
|
||||
load_delete_request_config_in(&api.ctx, bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_request_config_in(
|
||||
ctx: &ReplicationInstanceContext,
|
||||
bucket: &str,
|
||||
) -> Result<DeleteReplicationConfigSnapshot> {
|
||||
load_delete_request_config_in(ctx, bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_config_snapshot_in(
|
||||
ctx: &ReplicationInstanceContext,
|
||||
bucket: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<DeleteReplicationConfigSnapshot> {
|
||||
load_delete_replication_config_in(ctx, bucket, opts).await
|
||||
}
|
||||
|
||||
pub fn has_active_delete_rule(snapshot: &DeleteReplicationConfigSnapshot, object: &str) -> bool {
|
||||
snapshot.has_active_rule(object)
|
||||
}
|
||||
|
||||
pub fn check_delete_with_snapshot(
|
||||
object: &ObjectToDelete,
|
||||
source: &ObjectInfo,
|
||||
opts: &ObjectOptions,
|
||||
source_error: bool,
|
||||
snapshot: &DeleteReplicationConfigSnapshot,
|
||||
) -> ReplicateDecision {
|
||||
check_replicate_delete_with_snapshot(object, source, opts, source_error, snapshot)
|
||||
}
|
||||
|
||||
pub async fn schedule_object<S: ReplicationStorage>(
|
||||
object: ObjectInfo,
|
||||
storage: Arc<S>,
|
||||
|
||||
@@ -12,23 +12,26 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::{collections::HashMap, fmt, sync::Arc};
|
||||
|
||||
use crate::bucket::metadata::BucketMetadata;
|
||||
use rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS;
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
use s3s::dto::{BucketVersioningStatus, ReplicationConfiguration, ReplicationRuleStatus, VersioningConfiguration};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tracing::error;
|
||||
|
||||
use super::replication_config_boundary::{ObjectOpts, ReplicationConfigurationExt as _};
|
||||
use super::replication_config_boundary::{
|
||||
ObjectOpts, ReplicationConfigurationExt as _, ReplicationRuleExt as _, invalid_replication_config_status_field,
|
||||
};
|
||||
use super::replication_error_boundary::Result;
|
||||
use super::replication_filemeta_boundary::{
|
||||
ReplicateDecision, ReplicateTargetDecision, ReplicationStatusType, ReplicationType, ResyncDecision,
|
||||
};
|
||||
use super::replication_logging::{EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REPLICATION_RESYNC};
|
||||
use super::replication_metadata_boundary::ReplicationMetadataStore;
|
||||
use super::replication_metadata_boundary::{ReplicationInstanceContext, ReplicationMetadataStore};
|
||||
use super::replication_object_decision_boundary::{
|
||||
MustReplicateOptions, ReplicationDeleteSource, ReplicationResyncTargetObject, delete_replication_missing_source_decision,
|
||||
delete_replication_object_opts, resync_target_for_object,
|
||||
delete_replication_object_opts, heal_uses_delete_replication_path, resync_target_for_object,
|
||||
};
|
||||
use super::replication_storage_boundary::{ObjectInfo, ObjectOptions, ObjectToDelete, object_to_delete_for_replication};
|
||||
use super::replication_target_boundary::{BucketTargets, ReplicationTargetStore};
|
||||
@@ -36,7 +39,197 @@ use super::replication_versioning_boundary::ReplicationVersioningStore;
|
||||
use super::runtime_boundary as runtime_sources;
|
||||
|
||||
pub(crate) async fn get_replication_config(bucket: &str) -> Result<Option<ReplicationConfiguration>> {
|
||||
ReplicationMetadataStore::optional_replication_config(bucket).await
|
||||
let config = ReplicationMetadataStore::optional_replication_config(bucket).await?;
|
||||
validate_delete_replication_config(&VersioningConfiguration::default(), config.as_ref())?;
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct DeleteReplicationConfigSnapshot {
|
||||
metadata: Option<Arc<BucketMetadata>>,
|
||||
versioning: VersioningConfiguration,
|
||||
}
|
||||
|
||||
impl fmt::Debug for DeleteReplicationConfigSnapshot {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("DeleteReplicationConfigSnapshot")
|
||||
.field("has_replication_config", &self.replication_config().is_some())
|
||||
.field("versioning_status", &self.versioning.status)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl DeleteReplicationConfigSnapshot {
|
||||
#[cfg(test)]
|
||||
pub(crate) fn from_configs_for_test(
|
||||
versioning: VersioningConfiguration,
|
||||
replication: Option<ReplicationConfiguration>,
|
||||
) -> Self {
|
||||
let metadata = replication.map(|config| {
|
||||
let mut metadata = BucketMetadata::new("test-bucket");
|
||||
metadata.replication_config = Some(config);
|
||||
Arc::new(metadata)
|
||||
});
|
||||
Self { metadata, versioning }
|
||||
}
|
||||
|
||||
pub fn versioning_config(&self) -> &VersioningConfiguration {
|
||||
&self.versioning
|
||||
}
|
||||
|
||||
pub fn replication_config(&self) -> Option<&ReplicationConfiguration> {
|
||||
self.metadata
|
||||
.as_ref()
|
||||
.and_then(|metadata| metadata.replication_config.as_ref())
|
||||
}
|
||||
|
||||
pub(crate) fn has_active_rule(&self, object: &str) -> bool {
|
||||
self.replication_config()
|
||||
.is_some_and(|config| config.has_active_rules(object, true))
|
||||
}
|
||||
|
||||
pub(crate) fn active_delete_marker_rules_require_tags(&self, object: &str) -> bool {
|
||||
self.replication_config().is_some_and(|config| {
|
||||
config.rules.iter().any(|rule| {
|
||||
if rule.status == ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED) {
|
||||
return false;
|
||||
}
|
||||
if !object.starts_with(rule.prefix()) {
|
||||
return false;
|
||||
}
|
||||
rule.filter.as_ref().is_some_and(|filter| {
|
||||
filter.tag.is_some()
|
||||
|| filter
|
||||
.and
|
||||
.as_ref()
|
||||
.and_then(|and| and.tags.as_ref())
|
||||
.is_some_and(|tags| !tags.is_empty())
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_delete_replication_config(
|
||||
versioning: &VersioningConfiguration,
|
||||
config: Option<&ReplicationConfiguration>,
|
||||
) -> Result<()> {
|
||||
if versioning
|
||||
.status
|
||||
.as_ref()
|
||||
.is_some_and(|status| !matches!(status.as_str(), BucketVersioningStatus::ENABLED | BucketVersioningStatus::SUSPENDED))
|
||||
{
|
||||
return Err(super::replication_error_boundary::Error::other(
|
||||
"bucket versioning configuration has an invalid status",
|
||||
));
|
||||
}
|
||||
|
||||
if let Some(config) = config {
|
||||
if let Some(field) = invalid_replication_config_status_field(config) {
|
||||
return Err(super::replication_error_boundary::Error::other(format!(
|
||||
"replication field {field} has an invalid status"
|
||||
)));
|
||||
}
|
||||
|
||||
let role = config.role.trim();
|
||||
let mut role_destination = None;
|
||||
for rule in &config.rules {
|
||||
if rule.status.as_str() == ReplicationRuleStatus::ENABLED {
|
||||
let destination = rule.destination.bucket.trim();
|
||||
if role.is_empty() && destination.is_empty() {
|
||||
return Err(super::replication_error_boundary::Error::other(
|
||||
"enabled replication rule has no destination ARN",
|
||||
));
|
||||
}
|
||||
if !role.is_empty() && !destination.is_empty() {
|
||||
match role_destination {
|
||||
Some(existing) if existing != destination => {
|
||||
return Err(super::replication_error_boundary::Error::other(
|
||||
"replication role cannot address multiple active destinations",
|
||||
));
|
||||
}
|
||||
None => role_destination = Some(destination),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn replication_config_from_metadata(metadata: &BucketMetadata) -> Result<Option<&ReplicationConfiguration>> {
|
||||
if !metadata.replication_config_xml.is_empty() && metadata.replication_config.is_none() {
|
||||
return Err(super::replication_error_boundary::Error::other(
|
||||
"persisted bucket replication configuration is invalid",
|
||||
));
|
||||
}
|
||||
Ok(metadata.replication_config.as_ref())
|
||||
}
|
||||
|
||||
fn delete_request_snapshot_from_metadata(metadata: Arc<BucketMetadata>) -> Result<DeleteReplicationConfigSnapshot> {
|
||||
if !metadata.versioning_config_xml.is_empty() && metadata.versioning_config.is_none() {
|
||||
return Err(super::replication_error_boundary::Error::other(
|
||||
"persisted bucket versioning configuration is invalid",
|
||||
));
|
||||
}
|
||||
|
||||
let versioning = metadata.versioning_config.clone().unwrap_or_default();
|
||||
let has_config = {
|
||||
let config = replication_config_from_metadata(&metadata)?;
|
||||
if versioning.status.is_none() && config.is_some() {
|
||||
return Err(super::replication_error_boundary::Error::other(
|
||||
"bucket replication configuration requires versioning",
|
||||
));
|
||||
}
|
||||
validate_delete_replication_config(&versioning, config)?;
|
||||
config.is_some()
|
||||
};
|
||||
Ok(DeleteReplicationConfigSnapshot {
|
||||
metadata: has_config.then_some(metadata),
|
||||
versioning,
|
||||
})
|
||||
}
|
||||
|
||||
fn delete_snapshot_from_metadata(metadata: Arc<BucketMetadata>) -> Result<DeleteReplicationConfigSnapshot> {
|
||||
let has_config = {
|
||||
let config = replication_config_from_metadata(&metadata)?;
|
||||
validate_delete_replication_config(&VersioningConfiguration::default(), config)?;
|
||||
config.is_some()
|
||||
};
|
||||
Ok(DeleteReplicationConfigSnapshot {
|
||||
metadata: has_config.then_some(metadata),
|
||||
versioning: VersioningConfiguration::default(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn load_delete_request_config_in(
|
||||
ctx: &ReplicationInstanceContext,
|
||||
bucket: &str,
|
||||
) -> Result<DeleteReplicationConfigSnapshot> {
|
||||
delete_request_snapshot_from_metadata(ReplicationMetadataStore::delete_metadata_in(ctx, bucket).await?)
|
||||
}
|
||||
|
||||
pub(crate) async fn load_delete_replication_config(
|
||||
bucket: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<DeleteReplicationConfigSnapshot> {
|
||||
if opts.replication_request || (!opts.versioned && !opts.version_suspended) {
|
||||
return Ok(DeleteReplicationConfigSnapshot::default());
|
||||
}
|
||||
delete_snapshot_from_metadata(ReplicationMetadataStore::delete_metadata(bucket).await?)
|
||||
}
|
||||
|
||||
pub(crate) async fn load_delete_replication_config_in(
|
||||
ctx: &ReplicationInstanceContext,
|
||||
bucket: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<DeleteReplicationConfigSnapshot> {
|
||||
if opts.replication_request || (!opts.versioned && !opts.version_suspended) {
|
||||
return Ok(DeleteReplicationConfigSnapshot::default());
|
||||
}
|
||||
delete_snapshot_from_metadata(ReplicationMetadataStore::delete_metadata_in(ctx, bucket).await?)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
@@ -54,10 +247,31 @@ impl ReplicationConfig {
|
||||
self.config.is_none()
|
||||
}
|
||||
|
||||
pub(crate) fn validate(&self) -> Result<()> {
|
||||
validate_delete_replication_config(&VersioningConfiguration::default(), self.config.as_ref())
|
||||
}
|
||||
|
||||
pub fn replicate(&self, obj: &ObjectOpts) -> bool {
|
||||
self.config.as_ref().is_some_and(|config| config.replicate(obj))
|
||||
}
|
||||
|
||||
pub(crate) fn check_delete_for_heal(
|
||||
&self,
|
||||
object: &ObjectToDelete,
|
||||
source: &ObjectInfo,
|
||||
opts: &ObjectOptions,
|
||||
) -> ReplicateDecision {
|
||||
check_replicate_delete_with_config(
|
||||
object,
|
||||
source,
|
||||
opts,
|
||||
false,
|
||||
self.config.as_ref(),
|
||||
source.delete_marker && source.version_purge_status.is_empty(),
|
||||
true,
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn resync(
|
||||
&self,
|
||||
oi: ObjectInfo,
|
||||
@@ -70,30 +284,34 @@ impl ReplicationConfig {
|
||||
|
||||
let mut dsc = dsc;
|
||||
|
||||
if oi.delete_marker {
|
||||
if heal_uses_delete_replication_path(oi.delete_marker, &oi.version_purge_status) {
|
||||
if !dsc.targets_map.is_empty() {
|
||||
return self.resync_internal(oi, dsc, status);
|
||||
}
|
||||
let opts = ObjectOpts {
|
||||
name: oi.name.clone(),
|
||||
version_id: oi.version_id,
|
||||
delete_marker: true,
|
||||
version_id: if oi.version_purge_status.is_empty() {
|
||||
None
|
||||
} else {
|
||||
oi.version_id
|
||||
},
|
||||
delete_marker: oi.delete_marker,
|
||||
op_type: ReplicationType::Delete,
|
||||
existing_object: true,
|
||||
..Default::default()
|
||||
};
|
||||
let arns = self
|
||||
let targets = self
|
||||
.config
|
||||
.as_ref()
|
||||
.map(|config| config.filter_target_arns(&opts))
|
||||
.map(|config| config.filter_target_replication_decisions(&opts))
|
||||
.unwrap_or_default();
|
||||
|
||||
if arns.is_empty() {
|
||||
if targets.is_empty() {
|
||||
return ResyncDecision::default();
|
||||
}
|
||||
|
||||
for arn in arns {
|
||||
let mut opts = opts.clone();
|
||||
opts.target_arn = arn;
|
||||
|
||||
dsc.set(ReplicateTargetDecision::new(opts.target_arn.clone(), self.replicate(&opts), false));
|
||||
for (arn, replicate) in targets {
|
||||
dsc.set(ReplicateTargetDecision::new(arn, replicate, false));
|
||||
}
|
||||
|
||||
return self.resync_internal(oi, dsc, status);
|
||||
@@ -169,8 +387,10 @@ pub(crate) async fn check_replicate_delete(
|
||||
del_opts: &ObjectOptions,
|
||||
gerr: Option<String>,
|
||||
) -> ReplicateDecision {
|
||||
match check_replicate_delete_strict(bucket, dobj, oi, del_opts, gerr).await {
|
||||
Ok(decision) => decision,
|
||||
match load_delete_replication_config(bucket, del_opts).await {
|
||||
Ok(snapshot) => {
|
||||
check_replicate_delete_with_config(dobj, oi, del_opts, gerr.is_some(), snapshot.replication_config(), false, false)
|
||||
}
|
||||
Err(err) => {
|
||||
error!(
|
||||
event = EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED,
|
||||
@@ -193,66 +413,101 @@ pub(crate) async fn check_replicate_delete_strict(
|
||||
del_opts: &ObjectOptions,
|
||||
gerr: Option<String>,
|
||||
) -> Result<ReplicateDecision> {
|
||||
let rcfg = match get_replication_config(bucket).await {
|
||||
Ok(Some(config)) => config,
|
||||
Ok(None) => return Ok(ReplicateDecision::default()),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
if del_opts.replication_request {
|
||||
let Some(config) = get_replication_config(bucket).await? else {
|
||||
return Ok(ReplicateDecision::default());
|
||||
};
|
||||
let mut decision = check_replicate_delete_with_config(dobj, oi, del_opts, gerr.is_some(), Some(&config), false, false);
|
||||
if gerr.is_some() {
|
||||
return Ok(decision);
|
||||
}
|
||||
|
||||
for target in decision.targets_map.values_mut() {
|
||||
if let Some(client) = ReplicationTargetStore::remote_target_client(bucket, &target.arn).await {
|
||||
target.synchronous = client.replicate_sync;
|
||||
} else {
|
||||
target.replicate = false;
|
||||
target.synchronous = false;
|
||||
}
|
||||
}
|
||||
Ok(decision)
|
||||
}
|
||||
|
||||
pub(crate) fn check_replicate_delete_with_snapshot(
|
||||
dobj: &ObjectToDelete,
|
||||
oi: &ObjectInfo,
|
||||
del_opts: &ObjectOptions,
|
||||
source_error: bool,
|
||||
snapshot: &DeleteReplicationConfigSnapshot,
|
||||
) -> ReplicateDecision {
|
||||
check_replicate_delete_with_config(dobj, oi, del_opts, source_error, snapshot.replication_config(), false, false)
|
||||
}
|
||||
|
||||
fn check_replicate_delete_with_config(
|
||||
dobj: &ObjectToDelete,
|
||||
oi: &ObjectInfo,
|
||||
del_opts: &ObjectOptions,
|
||||
source_error: bool,
|
||||
config: Option<&ReplicationConfiguration>,
|
||||
existing_delete_marker: bool,
|
||||
trust_persisted_replica_status: bool,
|
||||
) -> ReplicateDecision {
|
||||
if del_opts.replication_request {
|
||||
return ReplicateDecision::default();
|
||||
}
|
||||
|
||||
if !del_opts.versioned && !del_opts.version_suspended {
|
||||
return Ok(ReplicateDecision::default());
|
||||
return ReplicateDecision::default();
|
||||
}
|
||||
|
||||
let Some(rcfg) = config else {
|
||||
return ReplicateDecision::default();
|
||||
};
|
||||
|
||||
let replication_delete = object_to_delete_for_replication(dobj);
|
||||
let opts = delete_replication_object_opts(
|
||||
let missing_source_marker = source_error && dobj.version_id.is_none();
|
||||
let mut opts = delete_replication_object_opts(
|
||||
&replication_delete,
|
||||
&ReplicationDeleteSource {
|
||||
user_defined: oi.user_defined.as_ref(),
|
||||
user_tags: oi.user_tags.as_str(),
|
||||
delete_marker: oi.delete_marker,
|
||||
replication_status: oi.replication_status.clone(),
|
||||
delete_marker: oi.delete_marker || missing_source_marker,
|
||||
replication_status: if trust_persisted_replica_status {
|
||||
oi.replication_status.clone()
|
||||
} else {
|
||||
ReplicationStatusType::Empty
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
let tgt_arns = rcfg.filter_target_arns(&opts);
|
||||
let mut dsc = ReplicateDecision::new();
|
||||
|
||||
if tgt_arns.is_empty() {
|
||||
return Ok(dsc);
|
||||
if existing_delete_marker {
|
||||
opts.version_id = None;
|
||||
}
|
||||
|
||||
for tgt_arn in tgt_arns {
|
||||
let mut opts = opts.clone();
|
||||
opts.target_arn = tgt_arn.clone();
|
||||
let replicate = rcfg.replicate(&opts);
|
||||
let sync = false;
|
||||
let target_decisions = rcfg.filter_target_replication_decisions(&opts);
|
||||
let mut dsc = ReplicateDecision::new();
|
||||
|
||||
if gerr.is_some() {
|
||||
if let Some(replicate) = delete_replication_missing_source_decision(
|
||||
oi.delete_marker,
|
||||
if target_decisions.is_empty() {
|
||||
return dsc;
|
||||
}
|
||||
|
||||
for (tgt_arn, replicate) in target_decisions {
|
||||
let effective_replicate = if source_error {
|
||||
delete_replication_missing_source_decision(
|
||||
oi.delete_marker || missing_source_marker,
|
||||
oi.target_replication_status(&tgt_arn),
|
||||
replicate,
|
||||
&oi.version_purge_status,
|
||||
) {
|
||||
dsc.set(ReplicateTargetDecision::new(tgt_arn, replicate, sync));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
let tgt = ReplicationTargetStore::remote_target_client(bucket, &tgt_arn).await;
|
||||
let tgt_dsc = if let Some(tgt) = tgt {
|
||||
ReplicateTargetDecision::new(tgt_arn, replicate, tgt.replicate_sync)
|
||||
)
|
||||
} else {
|
||||
ReplicateTargetDecision::new(tgt_arn, false, false)
|
||||
Some(replicate)
|
||||
};
|
||||
dsc.set(tgt_dsc);
|
||||
let Some(effective_replicate) = effective_replicate else {
|
||||
continue;
|
||||
};
|
||||
|
||||
dsc.set(ReplicateTargetDecision::new(tgt_arn, effective_replicate, false));
|
||||
}
|
||||
|
||||
Ok(dsc)
|
||||
dsc
|
||||
}
|
||||
|
||||
pub(crate) async fn must_replicate(bucket: &str, object: &str, mopts: MustReplicateOptions) -> ReplicateDecision {
|
||||
@@ -312,8 +567,13 @@ pub(crate) async fn must_replicate(bucket: &str, object: &str, mopts: MustReplic
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use s3s::dto::{Destination, ReplicationRule, ReplicationRuleStatus};
|
||||
use s3s::dto::{
|
||||
DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication, DeleteReplicationStatus, Destination,
|
||||
ReplicaModifications, ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, SourceSelectionCriteria, Tag,
|
||||
};
|
||||
|
||||
use super::super::replication_filemeta_boundary::VersionPurgeStatusType;
|
||||
use super::super::replication_target_boundary::BucketTarget;
|
||||
use super::*;
|
||||
|
||||
fn replication_rule() -> ReplicationRule {
|
||||
@@ -373,4 +633,379 @@ mod tests {
|
||||
assert!(options.is_replication_request());
|
||||
assert_eq!(options.user_tags(), "env=prod");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_snapshot_rejects_enabled_rules_without_a_destination() {
|
||||
let mut rule = replication_rule();
|
||||
rule.destination.bucket.clear();
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
};
|
||||
|
||||
let err = validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config))
|
||||
.expect_err("an enabled rule without a destination must fail closed");
|
||||
|
||||
assert!(err.to_string().contains("destination ARN"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_snapshot_rejects_role_with_multiple_destinations() {
|
||||
let first = replication_rule();
|
||||
let mut second = replication_rule();
|
||||
second.destination.bucket = "arn:aws:s3:::other-target".to_string();
|
||||
let config = ReplicationConfiguration {
|
||||
role: "arn:aws:s3:::role-target".to_string(),
|
||||
rules: vec![first, second],
|
||||
};
|
||||
|
||||
assert!(validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config)).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_snapshot_rejects_unknown_status_values() {
|
||||
let invalid_versioning = VersioningConfiguration {
|
||||
status: Some("Enabld".to_string().into()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(validate_delete_replication_config(&invalid_versioning, None).is_err());
|
||||
|
||||
let mut invalid_rule = replication_rule();
|
||||
invalid_rule.status = "Enabld".to_string().into();
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![invalid_rule],
|
||||
};
|
||||
assert!(validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config)).is_err());
|
||||
|
||||
let mut invalid_delete = replication_rule();
|
||||
invalid_delete.delete_replication = Some(DeleteReplication {
|
||||
status: "Enabld".to_string().into(),
|
||||
});
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![invalid_delete],
|
||||
};
|
||||
assert!(validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config)).is_err());
|
||||
|
||||
let mut invalid_delete_marker = replication_rule();
|
||||
invalid_delete_marker.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some("Enabld".to_string().into()),
|
||||
});
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![invalid_delete_marker],
|
||||
};
|
||||
assert!(validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config)).is_err());
|
||||
|
||||
let mut invalid_replica_modifications = replication_rule();
|
||||
invalid_replica_modifications.source_selection_criteria = Some(SourceSelectionCriteria {
|
||||
replica_modifications: Some(ReplicaModifications {
|
||||
status: "Enabld".to_string().into(),
|
||||
}),
|
||||
sse_kms_encrypted_objects: None,
|
||||
});
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![invalid_replica_modifications],
|
||||
};
|
||||
assert!(validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config)).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_snapshot_borrows_cached_replication_config() {
|
||||
let mut metadata = BucketMetadata::new("bucket");
|
||||
metadata.versioning_config_xml = b"configured".to_vec();
|
||||
metadata.versioning_config = Some(VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
..Default::default()
|
||||
});
|
||||
metadata.replication_config_xml = b"configured".to_vec();
|
||||
metadata.replication_config = Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![replication_rule()],
|
||||
});
|
||||
let metadata = Arc::new(metadata);
|
||||
let cached_config = metadata.replication_config.as_ref().expect("cached config") as *const _;
|
||||
|
||||
let snapshot = delete_request_snapshot_from_metadata(Arc::clone(&metadata)).expect("valid snapshot");
|
||||
|
||||
assert_eq!(snapshot.replication_config().expect("snapshot config") as *const _, cached_config);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_snapshot_debug_redacts_bucket_target_credentials() {
|
||||
let secret = "snapshot-secret-must-not-be-formatted";
|
||||
let mut metadata = BucketMetadata::new("bucket");
|
||||
metadata.bucket_targets_config_json = format!(r#"{{"secretKey":"{secret}"}}"#).into_bytes();
|
||||
let opts = ObjectOptions {
|
||||
delete_replication_config_snapshot: Some(Arc::new(DeleteReplicationConfigSnapshot {
|
||||
metadata: Some(Arc::new(metadata)),
|
||||
versioning: VersioningConfiguration::default(),
|
||||
})),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let debug = format!("{opts:?}");
|
||||
assert!(!debug.contains(secret), "delete tracing must not expose replication target credentials");
|
||||
assert!(debug.contains("has_replication_config"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_snapshot_rejects_replication_without_versioning_status() {
|
||||
let mut malformed = BucketMetadata::new("bucket");
|
||||
malformed.replication_config_xml = b"<ReplicationConfiguration>".to_vec();
|
||||
assert!(
|
||||
delete_request_snapshot_from_metadata(Arc::new(malformed)).is_err(),
|
||||
"malformed replication metadata must fail closed even when versioning has no status"
|
||||
);
|
||||
|
||||
let mut inconsistent = BucketMetadata::new("bucket");
|
||||
inconsistent.replication_config = Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![replication_rule()],
|
||||
});
|
||||
assert!(
|
||||
delete_request_snapshot_from_metadata(Arc::new(inconsistent)).is_err(),
|
||||
"replication metadata without an enabled or suspended versioning state must fail closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_source_marker_creation_is_still_admitted() {
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
let mut rule = replication_rule();
|
||||
rule.destination.bucket = arn.to_string();
|
||||
rule.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
|
||||
});
|
||||
let mut metadata = BucketMetadata::new("bucket");
|
||||
metadata.replication_config = Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
});
|
||||
let snapshot = DeleteReplicationConfigSnapshot {
|
||||
metadata: Some(Arc::new(metadata)),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let decision = check_replicate_delete_with_snapshot(
|
||||
&ObjectToDelete {
|
||||
object_name: "object".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
&ObjectInfo::default(),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
true,
|
||||
&snapshot,
|
||||
);
|
||||
|
||||
assert!(decision.replicate_any());
|
||||
assert!(decision.targets_map.get(arn).is_some_and(|target| target.replicate));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_marker_source_read_is_required_only_for_tag_filtered_rules() {
|
||||
let mut prefix_rule = replication_rule();
|
||||
prefix_rule.prefix = Some("logs/".to_string());
|
||||
prefix_rule.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
|
||||
});
|
||||
|
||||
let prefix_snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
..Default::default()
|
||||
},
|
||||
Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![prefix_rule],
|
||||
}),
|
||||
);
|
||||
|
||||
let mut tag_rule = replication_rule();
|
||||
tag_rule.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
|
||||
});
|
||||
tag_rule.filter = Some(ReplicationRuleFilter {
|
||||
tag: Some(Tag {
|
||||
key: Some("class".to_string()),
|
||||
value: Some("audit".to_string()),
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
let tag_snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
..Default::default()
|
||||
},
|
||||
Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![tag_rule],
|
||||
}),
|
||||
);
|
||||
|
||||
assert!(!prefix_snapshot.active_delete_marker_rules_require_tags("logs/2026/app.log"));
|
||||
assert!(tag_snapshot.active_delete_marker_rules_require_tags("logs/2026/app.log"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_uses_delete_switch_for_pending_purges_and_marker_switch_for_stored_markers() {
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
let mut rule = replication_rule();
|
||||
rule.destination.bucket = arn.to_string();
|
||||
rule.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
rule.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::DISABLED)),
|
||||
});
|
||||
let config = ReplicationConfig::new(
|
||||
Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
}),
|
||||
None,
|
||||
);
|
||||
let object = ObjectToDelete {
|
||||
object_name: "object".to_string(),
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
..Default::default()
|
||||
};
|
||||
let opts = ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let purge = ObjectInfo {
|
||||
version_id: object.version_id,
|
||||
version_purge_status: VersionPurgeStatusType::Pending,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(config.check_delete_for_heal(&object, &purge, &opts).replicate_any());
|
||||
|
||||
let marker = ObjectInfo {
|
||||
delete_marker: true,
|
||||
version_id: object.version_id,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!config.check_delete_for_heal(&object, &marker, &opts).replicate_any());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn live_delete_does_not_trust_persisted_replica_status() {
|
||||
let mut rule = replication_rule();
|
||||
rule.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
rule.source_selection_criteria = Some(SourceSelectionCriteria {
|
||||
replica_modifications: Some(ReplicaModifications {
|
||||
status: s3s::dto::ReplicaModificationsStatus::from_static(s3s::dto::ReplicaModificationsStatus::DISABLED),
|
||||
}),
|
||||
sse_kms_encrypted_objects: None,
|
||||
});
|
||||
let replication = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
};
|
||||
let snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
..Default::default()
|
||||
},
|
||||
Some(replication.clone()),
|
||||
);
|
||||
let object = ObjectToDelete {
|
||||
object_name: "object".to_string(),
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
..Default::default()
|
||||
};
|
||||
let source = ObjectInfo {
|
||||
replication_status: ReplicationStatusType::Replica,
|
||||
..Default::default()
|
||||
};
|
||||
let opts = ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
check_replicate_delete_with_snapshot(&object, &source, &opts, false, &snapshot).replicate_any(),
|
||||
"an ordinary authenticated delete must not inherit replica identity from object metadata"
|
||||
);
|
||||
assert!(
|
||||
!ReplicationConfig::new(Some(replication), None)
|
||||
.check_delete_for_heal(&object, &source, &opts)
|
||||
.replicate_any(),
|
||||
"heal must still honor the persisted replica identity"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resync_keeps_marker_version_purges_separate_from_marker_creation() {
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
let mut rule = replication_rule();
|
||||
rule.destination.bucket = arn.to_string();
|
||||
rule.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
rule.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::DISABLED)),
|
||||
});
|
||||
let config = ReplicationConfig::new(
|
||||
Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
}),
|
||||
Some(BucketTargets {
|
||||
targets: vec![BucketTarget {
|
||||
arn: arn.to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
}),
|
||||
);
|
||||
let marker = ObjectInfo {
|
||||
name: "object".to_string(),
|
||||
delete_marker: true,
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let purge = config
|
||||
.resync(
|
||||
ObjectInfo {
|
||||
version_purge_status: VersionPurgeStatusType::Pending,
|
||||
..marker.clone()
|
||||
},
|
||||
ReplicateDecision::default(),
|
||||
&HashMap::new(),
|
||||
)
|
||||
.await;
|
||||
assert!(purge.targets.get(arn).is_some_and(|target| target.replicate));
|
||||
|
||||
let mut object_purge_decision = ReplicateDecision::default();
|
||||
object_purge_decision.set(ReplicateTargetDecision::new(arn.to_string(), true, false));
|
||||
let object_purge = config
|
||||
.resync(
|
||||
ObjectInfo {
|
||||
name: "object".to_string(),
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
version_purge_status: VersionPurgeStatusType::Pending,
|
||||
..Default::default()
|
||||
},
|
||||
object_purge_decision,
|
||||
&HashMap::new(),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
object_purge.targets.get(arn).is_some_and(|target| target.replicate),
|
||||
"non-marker PENDING purges must stay on the delete resync path"
|
||||
);
|
||||
|
||||
let stored_marker = config.resync(marker, ReplicateDecision::default(), &HashMap::new()).await;
|
||||
assert!(!stored_marker.targets.contains_key(arn));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1967,7 +1967,24 @@ pub(crate) async fn queue_replication_heal_internal(
|
||||
};
|
||||
}
|
||||
|
||||
roi = get_heal_replicate_object_info(&oi, &rcfg).await;
|
||||
roi = match get_heal_replicate_object_info(&oi, &rcfg).await {
|
||||
Ok(roi) => roi,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
bucket = %oi.bucket,
|
||||
object = %oi.name,
|
||||
error = %err,
|
||||
"Failed to classify object for replication heal"
|
||||
);
|
||||
return ReplicationHealQueueResult {
|
||||
object_info: roi,
|
||||
admission: ReplicationQueueAdmission::Missed,
|
||||
};
|
||||
}
|
||||
};
|
||||
roi.retry_count = retry_count;
|
||||
|
||||
match replication_heal_queue_action(&mut roi) {
|
||||
@@ -2861,6 +2878,55 @@ mod tests {
|
||||
assert_eq!(admission, ReplicationQueueAdmission::Missed);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_queue_marks_missing_versioning_state_as_missed() {
|
||||
use super::super::replication_target_boundary::BucketTargets;
|
||||
use s3s::dto::{
|
||||
DeleteReplication, DeleteReplicationStatus, Destination, ReplicationConfiguration, ReplicationRule,
|
||||
ReplicationRuleStatus,
|
||||
};
|
||||
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
let result = queue_replication_heal_internal(
|
||||
"missing-versioning-state",
|
||||
ObjectInfo {
|
||||
bucket: "missing-versioning-state".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
version_purge_status: super::super::replication_filemeta_boundary::VersionPurgeStatusType::Pending,
|
||||
mod_time: Some(OffsetDateTime::now_utc()),
|
||||
..Default::default()
|
||||
},
|
||||
ReplicationConfig::new(
|
||||
Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![ReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
}),
|
||||
destination: Destination {
|
||||
bucket: arn.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
existing_object_replication: None,
|
||||
filter: None,
|
||||
id: Some("delete".to_string()),
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
}],
|
||||
}),
|
||||
Some(BucketTargets::default()),
|
||||
),
|
||||
0,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(result.admission, ReplicationQueueAdmission::Missed);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queue_replica_task_counts_mrf_pending_backlog_when_worker_queue_is_full() {
|
||||
let shared = empty_resync_shared_state();
|
||||
|
||||
@@ -18,16 +18,16 @@ use super::replication_config_store::ReplicationConfigStore;
|
||||
use super::replication_error_boundary::{Result, is_err_object_not_found, is_err_version_not_found};
|
||||
use super::replication_event_sink::{EventArgs, send_event, send_local_event};
|
||||
use super::replication_filemeta_boundary::{
|
||||
REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedInfos, ReplicatedTargetInfo,
|
||||
ReplicationAction, ReplicationStatusType, ReplicationType, VersionPurgeStatusType, get_replication_state,
|
||||
parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map,
|
||||
NULL_VERSION_ID, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedInfos,
|
||||
ReplicatedTargetInfo, ReplicationAction, ReplicationStatusType, ReplicationType, VersionPurgeStatusType,
|
||||
get_replication_state, parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map,
|
||||
};
|
||||
use super::replication_lock_boundary::ReplicationLockTiming;
|
||||
use super::replication_logging::{EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REPLICATION_RESYNC};
|
||||
use super::replication_metadata_boundary::ReplicationMetadataStore;
|
||||
#[cfg(test)]
|
||||
use super::replication_msgp_boundary::ReplicationMsgpCodec;
|
||||
use super::replication_object_config::{ReplicationConfig, check_replicate_delete, get_replication_config, must_replicate};
|
||||
use super::replication_object_config::{ReplicationConfig, get_replication_config, must_replicate};
|
||||
use super::replication_object_decision_boundary::{
|
||||
MustReplicateOptions, ReplicationMultipartPartInput, heal_uses_delete_replication_path,
|
||||
is_retryable_delete_replication_head_error, is_version_delete_replication, replication_etags_match,
|
||||
@@ -642,6 +642,21 @@ impl ReplicationResyncer {
|
||||
};
|
||||
|
||||
let rcfg = ReplicationConfig::new(cfg.clone(), Some(targets));
|
||||
if let Err(err) = rcfg.validate() {
|
||||
error!(
|
||||
event = EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %opts.bucket,
|
||||
arn = %opts.arn,
|
||||
error = %err,
|
||||
reason = "replication_config_invalid",
|
||||
"Replication resync config is invalid"
|
||||
);
|
||||
self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone())
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
|
||||
let target_arns = if let Some(cfg) = cfg {
|
||||
cfg.filter_target_arns(&ObjectOpts {
|
||||
@@ -982,7 +997,36 @@ impl ReplicationResyncer {
|
||||
}
|
||||
last_checkpoint = None;
|
||||
|
||||
let roi = get_heal_replicate_object_info(&object, &rcfg).await;
|
||||
let roi = match get_heal_replicate_object_info(&object, &rcfg).await {
|
||||
Ok(roi) => roi,
|
||||
Err(err) => {
|
||||
error!(
|
||||
event = EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %opts.bucket,
|
||||
arn = %opts.arn,
|
||||
object = %object.name,
|
||||
error = %err,
|
||||
"Failed to classify object for replication resync"
|
||||
);
|
||||
let worker_failed = finish_resync_workers(worker_txs, results_tx, futures, false).await;
|
||||
if worker_failed {
|
||||
error!(
|
||||
event = EVENT_RESYNC_TASK_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %opts.bucket,
|
||||
arn = %opts.arn,
|
||||
reason = "worker_join_failed_after_classification_error",
|
||||
"Replication resync worker cleanup observed task failure"
|
||||
);
|
||||
}
|
||||
self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone())
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
if !roi.existing_obj_resync.must_resync() {
|
||||
continue;
|
||||
}
|
||||
@@ -1037,18 +1081,25 @@ impl ReplicationResyncer {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationConfig) -> ReplicateObjectInfo {
|
||||
pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationConfig) -> Result<ReplicateObjectInfo> {
|
||||
let mut oi = oi.clone();
|
||||
let mut user_defined = (*oi.user_defined).clone();
|
||||
let delete_path = heal_uses_delete_replication_path(oi.delete_marker, &oi.version_purge_status);
|
||||
let stored_delete_decision = if delete_path && !oi.replication_decision.is_empty() {
|
||||
Some(parse_replicate_decision(&oi.bucket, &oi.replication_decision)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let has_stored_delete_decision = stored_delete_decision.is_some();
|
||||
|
||||
if let Some(rc) = rcfg.config.as_ref()
|
||||
&& !rc.role.is_empty()
|
||||
{
|
||||
if !oi.version_purge_status.is_empty() {
|
||||
if oi.version_purge_status_internal.is_none() && !oi.version_purge_status.is_empty() {
|
||||
oi.version_purge_status_internal = Some(format!("{}={};", rc.role, oi.version_purge_status.as_str()));
|
||||
}
|
||||
|
||||
if !oi.replication_status.is_empty() {
|
||||
if oi.replication_status_internal.is_none() && !oi.replication_status.is_empty() {
|
||||
oi.replication_status_internal = Some(format!("{}={};", rc.role, oi.replication_status.as_str()));
|
||||
}
|
||||
|
||||
@@ -1064,23 +1115,31 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
|
||||
}
|
||||
}
|
||||
|
||||
let dsc = if heal_uses_delete_replication_path(oi.delete_marker, &oi.version_purge_status) {
|
||||
check_replicate_delete(
|
||||
oi.bucket.as_str(),
|
||||
&ObjectToDelete {
|
||||
object_name: oi.name.clone(),
|
||||
version_id: oi.version_id,
|
||||
..Default::default()
|
||||
},
|
||||
&oi,
|
||||
&ObjectOptions {
|
||||
versioned: ReplicationVersioningStore::prefix_enabled(&oi.bucket, &oi.name).await,
|
||||
version_suspended: ReplicationVersioningStore::prefix_suspended(&oi.bucket, &oi.name).await,
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await
|
||||
let delete_state = if delete_path && !has_stored_delete_decision {
|
||||
ReplicationVersioningStore::prefix_state(&oi.bucket, &oi.name).await?
|
||||
} else {
|
||||
(false, false)
|
||||
};
|
||||
let dsc = if let Some(decision) = stored_delete_decision {
|
||||
decision
|
||||
} else if delete_path {
|
||||
if !delete_state.0 && !delete_state.1 {
|
||||
ReplicateDecision::default()
|
||||
} else {
|
||||
rcfg.check_delete_for_heal(
|
||||
&ObjectToDelete {
|
||||
object_name: oi.name.clone(),
|
||||
version_id: oi.version_id,
|
||||
..Default::default()
|
||||
},
|
||||
&oi,
|
||||
&ObjectOptions {
|
||||
versioned: delete_state.0,
|
||||
version_suspended: delete_state.1,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
}
|
||||
} else {
|
||||
must_replicate(
|
||||
oi.bucket.as_str(),
|
||||
@@ -1092,12 +1151,16 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
|
||||
|
||||
let target_statuses = replication_statuses_map(&oi.replication_status_internal.clone().unwrap_or_default());
|
||||
let target_purge_statuses = version_purge_statuses_map(&oi.version_purge_status_internal.clone().unwrap_or_default());
|
||||
let existing_obj_resync = rcfg.resync(oi.clone(), dsc.clone(), &target_statuses).await;
|
||||
let existing_obj_resync = if delete_path && !has_stored_delete_decision && !delete_state.0 && !delete_state.1 {
|
||||
Default::default()
|
||||
} else {
|
||||
rcfg.resync(oi.clone(), dsc.clone(), &target_statuses).await
|
||||
};
|
||||
let mut replication_state = oi.replication_state();
|
||||
replication_state.replicate_decision_str = dsc.to_string();
|
||||
let actual_size = oi.get_actual_size().unwrap_or_default();
|
||||
|
||||
ReplicateObjectInfo {
|
||||
Ok(ReplicateObjectInfo {
|
||||
name: oi.name.clone(),
|
||||
size: oi.size,
|
||||
actual_size,
|
||||
@@ -1122,7 +1185,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
|
||||
user_tags: (*oi.user_tags).clone(),
|
||||
checksum: oi.checksum.clone(),
|
||||
retry_count: 0,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn save_resync_status<S: ReplicationObjectIO>(
|
||||
@@ -1566,7 +1629,7 @@ async fn replicate_delete_marker_purge_to_targets(bucket: &str, dobj: &DeletedOb
|
||||
.remove_object(
|
||||
&tgt_client.bucket,
|
||||
&dobj.delete_object.object_name,
|
||||
Some(delete_marker_version_id.to_string()),
|
||||
target_delete_version_id(delete_marker_version_id, true),
|
||||
replication_delete_marker_purge_remove_options(dobj.delete_object.delete_marker_mtime),
|
||||
)
|
||||
.await;
|
||||
@@ -1796,6 +1859,14 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
}
|
||||
}
|
||||
|
||||
fn target_delete_version_id(version_id: Uuid, version_purge: bool) -> Option<String> {
|
||||
if version_id.is_nil() {
|
||||
version_purge.then(|| NULL_VERSION_ID.to_string())
|
||||
} else {
|
||||
Some(version_id.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
|
||||
let version_id = if let Some(version_id) = &dobj.delete_object.delete_marker_version_id {
|
||||
version_id.to_owned()
|
||||
@@ -1835,11 +1906,7 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
|
||||
return rinfo;
|
||||
}
|
||||
|
||||
let version_id = if version_id.is_nil() {
|
||||
None
|
||||
} else {
|
||||
Some(version_id.to_string())
|
||||
};
|
||||
let version_id = target_delete_version_id(version_id, is_version_purge);
|
||||
|
||||
if dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
|
||||
match head_object_with_proxy_stats(
|
||||
@@ -3088,7 +3155,12 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::replication_target_boundary::{BucketTarget, BucketTargets};
|
||||
use super::*;
|
||||
use s3s::dto::{
|
||||
BucketVersioningStatus, DeleteReplication, DeleteReplicationStatus, Destination, ExcludedPrefix, ReplicationRule,
|
||||
ReplicationRuleStatus, VersioningConfiguration,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
@@ -3530,7 +3602,9 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
let rcfg = ReplicationConfig::new(None, None);
|
||||
let roi = get_heal_replicate_object_info(&oi, &rcfg).await;
|
||||
let roi = get_heal_replicate_object_info(&oi, &rcfg)
|
||||
.await
|
||||
.expect("non-delete heal classification should succeed");
|
||||
|
||||
assert_eq!(roi.replication_status, ReplicationStatusType::Failed);
|
||||
assert_eq!(roi.op_type, ReplicationType::Heal);
|
||||
@@ -3555,7 +3629,9 @@ mod tests {
|
||||
};
|
||||
let rcfg = ReplicationConfig::new(None, None);
|
||||
|
||||
let roi = get_heal_replicate_object_info(&oi, &rcfg).await;
|
||||
let roi = get_heal_replicate_object_info(&oi, &rcfg)
|
||||
.await
|
||||
.expect("non-delete heal classification should succeed");
|
||||
|
||||
assert!(roi.ssec);
|
||||
assert_eq!(roi.checksum, Some(checksum));
|
||||
@@ -3571,6 +3647,7 @@ mod tests {
|
||||
version_purge_status: VersionPurgeStatusType::Pending,
|
||||
version_id: Some(Uuid::nil()),
|
||||
mod_time: Some(OffsetDateTime::now_utc()),
|
||||
replication_decision: format!("{role}=true;false;{role};"),
|
||||
..Default::default()
|
||||
};
|
||||
let rcfg = ReplicationConfig::new(
|
||||
@@ -3580,13 +3657,176 @@ mod tests {
|
||||
}),
|
||||
None,
|
||||
);
|
||||
let roi = get_heal_replicate_object_info(&oi, &rcfg).await;
|
||||
let roi = get_heal_replicate_object_info(&oi, &rcfg)
|
||||
.await
|
||||
.expect("stored purge admission should classify without a live versioning lookup");
|
||||
|
||||
assert_eq!(roi.replication_status_internal, None);
|
||||
assert_eq!(roi.version_purge_status_internal.as_deref(), Some(format!("{role}=PENDING;").as_str()));
|
||||
assert_eq!(roi.target_purge_statuses.get(role), Some(&VersionPurgeStatusType::Pending));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_pending_purge_reads_one_versioning_generation() {
|
||||
let bucket = format!("heal-versioning-snapshot-{}", Uuid::new_v4());
|
||||
let object = "archive/object";
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
ReplicationVersioningStore::install_prefix_state_test_config(
|
||||
&bucket,
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
excluded_prefixes: Some(vec![ExcludedPrefix {
|
||||
prefix: Some("archive/".to_string()),
|
||||
}]),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let rcfg = ReplicationConfig::new(
|
||||
Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![ReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
}),
|
||||
destination: Destination {
|
||||
bucket: arn.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
existing_object_replication: None,
|
||||
filter: None,
|
||||
id: Some("delete".to_string()),
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
}],
|
||||
}),
|
||||
Some(BucketTargets {
|
||||
targets: vec![BucketTarget {
|
||||
arn: arn.to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
}),
|
||||
);
|
||||
let oi = ObjectInfo {
|
||||
bucket,
|
||||
name: object.to_string(),
|
||||
version_id: Some(Uuid::nil()),
|
||||
version_purge_status: VersionPurgeStatusType::Pending,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let roi = get_heal_replicate_object_info(&oi, &rcfg)
|
||||
.await
|
||||
.expect("pending null purge classification should succeed");
|
||||
|
||||
assert!(roi.dsc.targets_map.get(arn).is_some_and(|target| target.replicate));
|
||||
assert!(
|
||||
roi.existing_obj_resync
|
||||
.targets
|
||||
.get(arn)
|
||||
.is_some_and(|target| target.replicate)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_pending_purge_preserves_the_persisted_admission_decision() {
|
||||
let admitted_arn = "arn:rustfs:replication:us-east-1:target:admitted";
|
||||
let current_role = "arn:rustfs:replication:us-east-1:target:current";
|
||||
let rcfg = ReplicationConfig::new(
|
||||
Some(ReplicationConfiguration {
|
||||
role: current_role.to_string(),
|
||||
rules: vec![ReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
|
||||
}),
|
||||
destination: Destination {
|
||||
bucket: current_role.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
existing_object_replication: None,
|
||||
filter: None,
|
||||
id: Some("delete".to_string()),
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
}],
|
||||
}),
|
||||
Some(BucketTargets {
|
||||
targets: vec![
|
||||
BucketTarget {
|
||||
arn: admitted_arn.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
BucketTarget {
|
||||
arn: current_role.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
let oi = ObjectInfo {
|
||||
bucket: "heal-persisted-delete-decision".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
version_purge_status: VersionPurgeStatusType::Pending,
|
||||
version_purge_status_internal: Some(format!("{admitted_arn}=PENDING;")),
|
||||
replication_decision: format!("{admitted_arn}=true;false;{admitted_arn};"),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let roi = get_heal_replicate_object_info(&oi, &rcfg)
|
||||
.await
|
||||
.expect("persisted delete admission should survive live rule disablement");
|
||||
|
||||
assert_eq!(
|
||||
roi.version_purge_status_internal.as_deref(),
|
||||
Some(format!("{admitted_arn}=PENDING;").as_str())
|
||||
);
|
||||
assert!(roi.dsc.targets_map.get(admitted_arn).is_some_and(|target| target.replicate));
|
||||
assert!(!roi.dsc.targets_map.contains_key(current_role));
|
||||
assert!(
|
||||
roi.existing_obj_resync
|
||||
.targets
|
||||
.get(admitted_arn)
|
||||
.is_some_and(|target| target.replicate)
|
||||
);
|
||||
assert!(!roi.existing_obj_resync.targets.contains_key(current_role));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_rejects_semantically_invalid_replication_config() {
|
||||
let rcfg = ReplicationConfig::new(
|
||||
Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![ReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: None,
|
||||
destination: Destination {
|
||||
bucket: "arn:rustfs:replication:us-east-1:target:bucket".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
existing_object_replication: None,
|
||||
filter: None,
|
||||
id: Some("invalid".to_string()),
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: ReplicationRuleStatus::from_static("Enabld"),
|
||||
}],
|
||||
}),
|
||||
Some(BucketTargets::default()),
|
||||
);
|
||||
let err = rcfg
|
||||
.validate()
|
||||
.expect_err("invalid string-backed statuses must fail before heal classification loop");
|
||||
|
||||
assert!(err.to_string().contains("Rule.Status"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cancel_marks_only_matching_bucket_target_token() {
|
||||
let resyncer = ReplicationResyncer::new().await;
|
||||
@@ -3753,4 +3993,13 @@ mod tests {
|
||||
assert_eq!(resync_status_duration(ResyncStatusType::ResyncStarted, Some(start), end), None);
|
||||
assert_eq!(resync_status_duration(ResyncStatusType::ResyncFailed, None, end), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn target_delete_version_id_preserves_explicit_null_purges() {
|
||||
let version_id = Uuid::new_v4();
|
||||
|
||||
assert_eq!(target_delete_version_id(version_id, true), Some(version_id.to_string()));
|
||||
assert_eq!(target_delete_version_id(Uuid::nil(), true).as_deref(), Some(NULL_VERSION_ID));
|
||||
assert_eq!(target_delete_version_id(Uuid::nil(), false), None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,6 +35,8 @@ use time::format_description::well_known::Rfc3339;
|
||||
pub(crate) use crate::bucket::bucket_target_sys::{
|
||||
AdvancedPutOptions, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::bucket::target::BucketTarget;
|
||||
pub(crate) use crate::bucket::target::BucketTargets;
|
||||
|
||||
use super::replication_config_store::ReplicationConfigStore;
|
||||
|
||||
@@ -12,11 +12,28 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||
use super::replication_error_boundary::Result;
|
||||
use crate::bucket::{versioning::VersioningApi as _, versioning_sys::BucketVersioningSys};
|
||||
#[cfg(test)]
|
||||
use s3s::dto::VersioningConfiguration;
|
||||
#[cfg(test)]
|
||||
use std::{collections::HashMap, sync::LazyLock, sync::Mutex};
|
||||
|
||||
#[cfg(test)]
|
||||
static PREFIX_STATE_TEST_CONFIGS: LazyLock<Mutex<HashMap<String, VersioningConfiguration>>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
pub(crate) struct ReplicationVersioningStore;
|
||||
|
||||
impl ReplicationVersioningStore {
|
||||
#[cfg(test)]
|
||||
pub(crate) fn install_prefix_state_test_config(bucket: &str, config: VersioningConfiguration) {
|
||||
PREFIX_STATE_TEST_CONFIGS
|
||||
.lock()
|
||||
.expect("replication versioning test config lock should not be poisoned")
|
||||
.insert(bucket.to_string(), config);
|
||||
}
|
||||
|
||||
pub(crate) async fn prefix_enabled(bucket: &str, prefix: &str) -> bool {
|
||||
BucketVersioningSys::prefix_enabled(bucket, prefix).await
|
||||
}
|
||||
@@ -24,4 +41,18 @@ impl ReplicationVersioningStore {
|
||||
pub(crate) async fn prefix_suspended(bucket: &str, prefix: &str) -> bool {
|
||||
BucketVersioningSys::prefix_suspended(bucket, prefix).await
|
||||
}
|
||||
|
||||
pub(crate) async fn prefix_state(bucket: &str, prefix: &str) -> Result<(bool, bool)> {
|
||||
#[cfg(test)]
|
||||
if let Some(config) = PREFIX_STATE_TEST_CONFIGS
|
||||
.lock()
|
||||
.expect("replication versioning test config lock should not be poisoned")
|
||||
.remove(bucket)
|
||||
{
|
||||
return Ok((config.prefix_enabled(prefix), config.prefix_suspended(prefix)));
|
||||
}
|
||||
|
||||
let config = BucketVersioningSys::get(bucket).await?;
|
||||
Ok((config.prefix_enabled(prefix), config.prefix_suspended(prefix)))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,6 +19,9 @@ pub trait VersioningApi {
|
||||
fn enabled(&self) -> bool;
|
||||
fn prefix_enabled(&self, prefix: &str) -> bool;
|
||||
fn prefix_suspended(&self, prefix: &str) -> bool;
|
||||
fn delete_state(&self, prefix: &str) -> (bool, bool) {
|
||||
(self.prefix_enabled(prefix), self.suspended())
|
||||
}
|
||||
fn versioned(&self, prefix: &str) -> bool;
|
||||
fn suspended(&self) -> bool;
|
||||
}
|
||||
@@ -92,3 +95,60 @@ impl VersioningApi for VersioningConfiguration {
|
||||
self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use s3s::dto::{BucketVersioningStatus, ExcludedPrefix};
|
||||
|
||||
use super::*;
|
||||
|
||||
struct LegacyVersioning;
|
||||
|
||||
impl VersioningApi for LegacyVersioning {
|
||||
fn enabled(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn prefix_enabled(&self, _prefix: &str) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn prefix_suspended(&self, _prefix: &str) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn versioned(&self, _prefix: &str) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn suspended(&self) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_state_has_a_backward_compatible_default() {
|
||||
assert_eq!(LegacyVersioning.delete_state("object"), (false, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_state_treats_excluded_prefixes_as_unversioned() {
|
||||
let config = VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
excluded_prefixes: Some(vec![ExcludedPrefix {
|
||||
prefix: Some("archive/".to_string()),
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(config.delete_state("archive/object"), (false, false));
|
||||
assert!(config.prefix_suspended("archive/object"));
|
||||
assert_eq!(config.delete_state("live/object"), (true, false));
|
||||
|
||||
let suspended = VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(suspended.delete_state("archive/object"), (false, true));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user