refactor(replication): add config store boundary (#4073)

crates/ecstore/src/bucket/replication/mod.rs

@@ -14,6 +14,7 @@
 
 mod config;
 pub mod datatypes;
+mod replication_config_store;
 mod replication_event_sink;
 mod replication_metadata_boundary;
 mod replication_pool;

crates/ecstore/src/bucket/replication/replication_config_store.rs

@@ -0,0 +1,50 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::config::com;
+use crate::error::Result;
+use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
+use crate::storage_api_contracts::{object::ObjectIO, range::HTTPRangeSpec};
+use http::HeaderMap;
+use std::sync::Arc;
+
+pub(crate) async fn read<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
+where
+    S: ObjectIO<
+            Error = crate::error::Error,
+            RangeSpec = HTTPRangeSpec,
+            HeaderMap = HeaderMap,
+            ObjectOptions = ObjectOptions,
+            ObjectInfo = ObjectInfo,
+            GetObjectReader = GetObjectReader,
+            PutObjectReader = PutObjReader,
+        >,
+{
+    com::read_config(api, file).await
+}
+
+pub(crate) async fn save<S>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()>
+where
+    S: ObjectIO<
+            Error = crate::error::Error,
+            RangeSpec = HTTPRangeSpec,
+            HeaderMap = HeaderMap,
+            ObjectOptions = ObjectOptions,
+            ObjectInfo = ObjectInfo,
+            GetObjectReader = GetObjectReader,
+            PutObjectReader = PutObjReader,
+        >,
+{
+    com::save_config(api, file, data).await
+}

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use super::replication_config_store as config_store;
 use super::replication_metadata_boundary as metadata_boundary;
 use super::replication_target_boundary as target_boundary;
 use super::runtime_boundary as runtime_sources;
@@ -25,7 +26,6 @@ use crate::bucket::replication::replication_resyncer::{
     decode_resync_file, encode_mrf_file, get_heal_replicate_object_info, save_resync_status,
 };
 use crate::bucket::replication::replication_state::ReplicationStats;
-use crate::config::com::{read_config, save_config};
 use crate::disk::BUCKET_META_PREFIX;
 use crate::error::Error as EcstoreError;
 use crate::object_api::{ObjectInfo, ObjectOptions};
@@ -802,7 +802,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
         let storage = self.storage.clone();
 
         let handle = tokio::spawn(async move {
-            let data = match read_config(storage.clone(), MRF_REPLICATION_FILE).await {
+            let data = match config_store::read(storage.clone(), MRF_REPLICATION_FILE).await {
                 Ok(d) => d,
                 Err(EcstoreError::ConfigNotFound) => return, // no file yet — normal on first start
                 Err(e) => {
@@ -826,7 +826,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
                         "Failed to decode MRF recovery file — discarding corrupt data"
                     );
                     // Overwrite the corrupt file so we don't fail again on next restart.
-                    let _ = save_config(storage, MRF_REPLICATION_FILE, encode_mrf_file(&[]).unwrap_or_default()).await;
+                    let _ = config_store::save(storage, MRF_REPLICATION_FILE, encode_mrf_file(&[]).unwrap_or_default()).await;
                     return;
                 }
             };
@@ -885,7 +885,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
             // Clear AFTER all entries are processed so a crash mid-replay causes at-most-twice
             // delivery (idempotent) rather than entry loss.
-            if let Err(e) = save_config(storage, MRF_REPLICATION_FILE, encode_mrf_file(&[]).unwrap_or_default()).await {
+            if let Err(e) = config_store::save(storage, MRF_REPLICATION_FILE, encode_mrf_file(&[]).unwrap_or_default()).await {
                 warn!(
                     component = LOG_COMPONENT_ECSTORE,
                     subsystem = LOG_SUBSYSTEM_REPLICATION,
@@ -1257,7 +1257,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 async fn flush_mrf_to_disk<S: EcstoreObjectIO>(entries: &[MrfReplicateEntry], storage: &Arc<S>) -> bool {
     match encode_mrf_file(entries) {
         Ok(data) => {
-            if let Err(e) = save_config(storage.clone(), MRF_REPLICATION_FILE, data).await {
+            if let Err(e) = config_store::save(storage.clone(), MRF_REPLICATION_FILE, data).await {
                 warn!(
                     component = LOG_COMPONENT_ECSTORE,
                     subsystem = LOG_SUBSYSTEM_REPLICATION,
@@ -1292,7 +1292,7 @@ async fn load_bucket_resync_metadata<S: EcstoreObjectIO>(
     let resync_dir_path = format!("{BUCKET_META_PREFIX}/{bucket}/{REPLICATION_DIR}");
     let resync_file_path = format!("{resync_dir_path}/{RESYNC_FILE_NAME}");
 
-    let data = match read_config(obj_api, &resync_file_path).await {
+    let data = match config_store::read(obj_api, &resync_file_path).await {
         Ok(data) => data,
         Err(EcstoreError::ConfigNotFound) => return Ok(brs),
         Err(err) => return Err(err),

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use super::replication_config_store as config_store;
 use super::replication_event_sink::{EventArgs, send_event};
 use super::replication_metadata_boundary as metadata_boundary;
 use super::replication_target_boundary as target_boundary;
@@ -27,7 +28,6 @@ use crate::bucket::replication::{ObjectOpts, ReplicationConfigurationExt as _};
 use crate::bucket::tagging::decode_tags_to_map;
 use crate::bucket::target::BucketTargets;
 use crate::client::api_get_options::{AdvancedGetOptions, StatObjectOptions};
-use crate::config::com::save_config;
 use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
 use crate::error::{Error, Result, is_err_object_not_found, is_err_version_not_found};
 use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
@@ -1312,7 +1312,7 @@ pub(crate) async fn save_resync_status<S: EcstoreObjectIO>(
     let data = encode_resync_file(status)?;
 
     let config_file = path_join_buf(&[BUCKET_META_PREFIX, bucket, REPLICATION_DIR, RESYNC_FILE_NAME]);
-    save_config(api, &config_file, data).await?;
+    config_store::save(api, &config_file, data).await?;
 
     Ok(())
 }

scripts/check_architecture_migration_rules.sh

@@ -183,6 +183,7 @@ EXTERNAL_TEST_ECSTORE_COMPAT_BYPASS_HITS_FILE="${TMP_DIR}/external_test_ecstore_
 FUZZ_ECSTORE_COMPAT_BYPASS_HITS_FILE="${TMP_DIR}/fuzz_ecstore_compat_bypass_hits.txt"
 EXTERNAL_ECSTORE_API_BOUNDARY_HITS_FILE="${TMP_DIR}/external_ecstore_api_boundary_hits.txt"
 REPLICATION_FACADE_BYPASS_HITS_FILE="${TMP_DIR}/replication_facade_bypass_hits.txt"
+REPLICATION_CONFIG_STORE_BYPASS_HITS_FILE="${TMP_DIR}/replication_config_store_bypass_hits.txt"
 REPLICATION_EVENT_SINK_BYPASS_HITS_FILE="${TMP_DIR}/replication_event_sink_bypass_hits.txt"
 REPLICATION_METADATA_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_metadata_boundary_bypass_hits.txt"
 REPLICATION_TARGET_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_target_boundary_bypass_hits.txt"
@@ -2354,6 +2355,18 @@ if [[ -s "$REPLICATION_EVENT_SINK_BYPASS_HITS_FILE" ]]; then
   report_failure "replication event notification access must stay behind replication event sink: $(paste -sd '; ' "$REPLICATION_EVENT_SINK_BYPASS_HITS_FILE")"
 fi
 
+(
+  cd "$ROOT_DIR"
+  rg -n --with-filename 'crate::config::com::\{[^}]*\b(read_config|save_config)\b|crate::config::com::(read_config|save_config)|\b(read_config|save_config)\(' \
+    crates/ecstore/src/bucket/replication \
+    --glob '*.rs' |
+    rg -v '^crates/ecstore/src/bucket/replication/replication_config_store\.rs:' || true
+) >"$REPLICATION_CONFIG_STORE_BYPASS_HITS_FILE"
+
+if [[ -s "$REPLICATION_CONFIG_STORE_BYPASS_HITS_FILE" ]]; then
+  report_failure "replication config persistence must stay behind replication config store: $(paste -sd '; ' "$REPLICATION_CONFIG_STORE_BYPASS_HITS_FILE")"
+fi
+
 (
   cd "$ROOT_DIR"
   rg -n --with-filename 'BucketTargetSys::get\(\)' \