fix(replication): prevent target state loss across buckets (#2704)

crates/e2e_test/src/replication_extension_test.rs

@@ -21,6 +21,7 @@ use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
 use aws_sdk_s3::{Client, Config};
 use http::header::{CONTENT_TYPE, HOST};
 use reqwest::StatusCode;
+use rustfs_ecstore::bucket::bucket_target_sys::BucketTargetSys;
 use rustfs_madmin::{
     AddServiceAccountReq, ListServiceAccountsResp, PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus,
     ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus, SRStatusInfo, SiteReplicationInfo, SyncStatus,
@@ -1648,6 +1649,90 @@ async fn test_single_bucket_replication_fans_out_to_multiple_targets() -> Result
     Ok(())
 }
 
+#[tokio::test]
+#[serial]
+async fn test_sequential_bucket_replication_succeeds_for_multiple_buckets() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+
+    for idx in 1..=5 {
+        let source_bucket = format!("replication-multi-src-{idx}");
+        let target_bucket = format!("replication-multi-dst-{idx}");
+        let object_key = format!("probe-{idx}.txt");
+        let body = format!("payload-{idx}");
+
+        source_client.create_bucket().bucket(&source_bucket).send().await?;
+        target_client.create_bucket().bucket(&target_bucket).send().await?;
+        enable_bucket_versioning(&source_env, &source_bucket).await?;
+        enable_bucket_versioning(&target_env, &target_bucket).await?;
+
+        let target_arn = set_replication_target(&source_env, &source_bucket, &target_env, &target_bucket).await?;
+        put_bucket_replication(&source_env, &source_bucket, &target_arn).await?;
+
+        source_client
+            .put_object()
+            .bucket(&source_bucket)
+            .key(&object_key)
+            .body(ByteStream::from(body.clone().into_bytes()))
+            .send()
+            .await?;
+
+        wait_for_replicated_object(&target_client, &target_bucket, &object_key, &body).await?;
+    }
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_replication_recovers_after_runtime_target_cache_is_cleared() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_bucket = "replication-refresh-src";
+    let target_bucket = "replication-refresh-dst";
+    let object_key = "probe-refresh.txt";
+    let body = "payload-refresh";
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+
+    source_client.create_bucket().bucket(source_bucket).send().await?;
+    target_client.create_bucket().bucket(target_bucket).send().await?;
+    enable_bucket_versioning(&source_env, source_bucket).await?;
+    enable_bucket_versioning(&target_env, target_bucket).await?;
+
+    let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
+    put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
+
+    BucketTargetSys::get().delete(source_bucket).await;
+
+    source_client
+        .put_object()
+        .bucket(source_bucket)
+        .key(object_key)
+        .body(ByteStream::from(body.as_bytes().to_vec()))
+        .send()
+        .await?;
+
+    wait_for_replicated_object(&target_client, target_bucket, object_key, body).await?;
+
+    Ok(())
+}
+
 #[tokio::test]
 #[serial]
 async fn test_site_replication_resync_start_cancel_restart_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {