refactor(ecstore): use local replication paths (#4130)

This commit is contained in:
Zhengchao An
2026-07-01 15:08:26 +08:00
committed by GitHub
parent 7cb42529fc
commit 8fd532ebb7
8 changed files with 89 additions and 19 deletions
@@ -46,7 +46,9 @@ and lifecycle/heal scheduling paths.
only after call sites prove a narrower object read/write/delete/walk shape.
4. Preserve `rustfs_ecstore::api::bucket::replication` compatibility until
lifecycle, scanner, OBS, heal, and tests compile through replacement paths.
5. Move at most one contract boundary per code-bearing PR and verify it with
5. Keep imports between modules in this directory relative to the local
replication module, not `crate::bucket::replication::*` self paths.
6. Move at most one contract boundary per code-bearing PR and verify it with
focused replication tests before broad gates.
## First Code-Bearing Step
@@ -14,7 +14,7 @@
use super::replication_filemeta_boundary::ReplicationType;
use super::replication_tagging_boundary as tagging_boundary;
use crate::bucket::replication::ReplicationRuleExt as _;
use super::rule::ReplicationRuleExt as _;
use s3s::dto::DeleteMarkerReplicationStatus;
use s3s::dto::DeleteReplicationStatus;
use s3s::dto::Destination;
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::datatypes::ResyncStatusType;
use super::replication_config_store as config_store;
use super::replication_error_boundary::Error as EcstoreError;
use super::replication_filemeta_boundary::{
@@ -20,19 +21,15 @@ use super::replication_filemeta_boundary::{
ResyncDecision, VersionPurgeStatusType, replication_statuses_map, version_purge_statuses_map,
};
use super::replication_metadata_boundary as metadata_boundary;
use super::replication_resyncer::{
BucketReplicationResyncStatus, DeletedObjectReplicationInfo, ReplicationConfig, ReplicationResyncer, ResyncOpts,
TargetReplicationResyncStatus, decode_mrf_file, decode_resync_file, encode_mrf_file, get_heal_replicate_object_info,
replicate_delete, replicate_object, save_resync_status,
};
use super::replication_state::ReplicationStats;
use super::replication_storage_boundary::{DeletedObject, ObjectInfo, ObjectOptions, ReplicationObjectIO, ReplicationStorage};
use super::replication_target_boundary as target_boundary;
use super::runtime_boundary as runtime_sources;
use crate::bucket::replication::ResyncOpts;
use crate::bucket::replication::ResyncStatusType;
use crate::bucket::replication::replicate_delete;
use crate::bucket::replication::replicate_object;
use crate::bucket::replication::replication_resyncer::{
BucketReplicationResyncStatus, DeletedObjectReplicationInfo, ReplicationConfig, ReplicationResyncer,
TargetReplicationResyncStatus, decode_mrf_file, decode_resync_file, encode_mrf_file, get_heal_replicate_object_info,
save_resync_status,
};
use crate::bucket::replication::replication_state::ReplicationStats;
use lazy_static::lazy_static;
use rustfs_utils::http::{SUFFIX_REPLICATION_TIMESTAMP, get_str};
use std::any::Any;
@@ -1692,8 +1689,8 @@ async fn queue_replicate_deletes_wrapper(
#[cfg(test)]
mod tests {
use super::super::replication_resyncer::{decode_mrf_file, encode_mrf_file};
use super::*;
use crate::bucket::replication::replication_resyncer::{decode_mrf_file, encode_mrf_file};
use uuid::Uuid;
#[test]
@@ -12,6 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::config::{ObjectOpts, ReplicationConfigurationExt as _};
use super::datatypes::ResyncStatusType;
use super::replication_bandwidth_boundary;
use super::replication_config_store as config_store;
use super::replication_error_boundary::{Error, Result, is_err_object_not_found, is_err_version_not_found};
@@ -36,8 +38,6 @@ use super::replication_target_boundary::{
};
use super::replication_versioning_boundary as versioning_boundary;
use super::runtime_boundary as runtime_sources;
use crate::bucket::replication::ResyncStatusType;
use crate::bucket::replication::{ObjectOpts, ReplicationConfigurationExt as _};
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
use aws_sdk_s3::operation::head_object::{HeadObjectError, HeadObjectOutput};
use aws_sdk_s3::primitives::ByteStream;
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::bucket::replication::ObjectOpts;
use super::config::ObjectOpts;
use s3s::dto::ReplicaModificationsStatus;
use s3s::dto::ReplicationRule;
@@ -14,9 +14,9 @@
use std::sync::Arc;
use super::replication_pool::DynReplicationPool;
use super::replication_state::ReplicationStats;
use crate::bucket::bandwidth::monitor::Monitor;
use crate::bucket::replication::replication_pool::DynReplicationPool;
use crate::bucket::replication::replication_state::ReplicationStats;
use crate::runtime::sources;
use crate::store::ECStore;
@@ -115,7 +115,9 @@ Current coupling:
- lifecycle and heal paths schedule replication work through the current ECStore
module;
- global replication pool/stat initialization still lives with ECStore runtime
compatibility state.
compatibility state;
- modules inside `bucket/replication` use local relative paths rather than the
ECStore owner path for replication self-imports.
Required contracts before crate movement:
@@ -204,6 +204,7 @@ REPLICATION_ERROR_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_error_bounda
REPLICATION_EVENT_SINK_BYPASS_HITS_FILE="${TMP_DIR}/replication_event_sink_bypass_hits.txt"
REPLICATION_EVENT_HOST_BYPASS_HITS_FILE="${TMP_DIR}/replication_event_host_bypass_hits.txt"
REPLICATION_FILEMETA_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_filemeta_boundary_bypass_hits.txt"
REPLICATION_LOCAL_PATH_BYPASS_HITS_FILE="${TMP_DIR}/replication_local_path_bypass_hits.txt"
REPLICATION_LOCK_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_lock_boundary_bypass_hits.txt"
REPLICATION_METADATA_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_metadata_boundary_bypass_hits.txt"
REPLICATION_MSGP_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_msgp_boundary_bypass_hits.txt"
@@ -2501,6 +2502,74 @@ if [[ -s "$REPLICATION_FACADE_BYPASS_HITS_FILE" ]]; then
report_failure "replication facade imports must stay in local storage_api boundaries: $(paste -sd '; ' "$REPLICATION_FACADE_BYPASS_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
find crates/ecstore/src/bucket/replication -type f -name '*.rs' -print0 |
xargs -0 perl -0ne '
sub blank {
my ($text) = @_;
$text =~ s/[^\n]/ /g;
return $text;
}
sub blank_block_comments {
my ($text) = @_;
my $out = $text;
my $len = length($text);
my $i = 0;
while ($i < $len) {
if ($i + 1 < $len && substr($text, $i, 2) eq "/*") {
my $start = $i;
my $depth = 0;
while ($i < $len) {
if ($i + 1 < $len && substr($text, $i, 2) eq "/*") {
$depth++;
$i += 2;
next;
}
if ($i + 1 < $len && substr($text, $i, 2) eq "*/") {
$depth--;
$i += 2;
last if $depth == 0;
next;
}
$i++;
}
substr($out, $start, $i - $start) = blank(substr($text, $start, $i - $start));
next;
}
$i++;
}
return $out;
}
my $source = $_;
my $hash = chr(35);
$source =~ s{b?r(${hash}*)".*?"\1}{blank($&)}egs;
$source =~ s{b?"(?:\\.|[^"\\])*"}{blank($&)}egs;
$source =~ s{'\''(?:\\.|[^'\''\\])+'\''}{blank($&)}egs;
$source = blank_block_comments($source);
$source =~ s{//[^\n]*}{blank($&)}eg;
while ($source =~ /(?:crate::bucket::replication\b|crate::bucket::\{[^;]*\breplication\b|crate::\{[^;]*\bbucket::replication\b|crate::\{[^;]*\bbucket::\{[^;]*\breplication\b)/sg) {
my $prefix = substr($source, 0, $-[0]);
my $line = ($prefix =~ tr/\n//) + 1;
my $match = $&;
$match =~ s/\s+/ /g;
print "$ARGV:$line:$match\n";
}
' || true
) >"$REPLICATION_LOCAL_PATH_BYPASS_HITS_FILE"
if [[ -s "$REPLICATION_LOCAL_PATH_BYPASS_HITS_FILE" ]]; then
report_failure "replication modules must use local relative paths instead of crate-qualified replication self paths: $(paste -sd '; ' "$REPLICATION_LOCAL_PATH_BYPASS_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
rg -n --with-filename 'crate::runtime::sources(\s+as\s+runtime_sources|::)' \