gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-28 09:08:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(ecstore): hide replication runtime types (#4131)

Browse Source
This commit is contained in:
Zhengchao An
2026-07-01 15:39:02 +08:00
committed by GitHub
parent 8fd532ebb7
commit b0f491549a
6 changed files with 101 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/bucket/replication/README.md
+5 -3
View File
@@ -13,7 +13,7 @@ and lifecycle/heal scheduling paths.
| `datatypes.rs` | Replication status and operation DTOs. | Publicly re-exported through the ECStore replication facade. |
| `replication_pool.rs` | Replication queue, worker pool, MRF persistence, bucket stats, and delete/object scheduling. | Depends on bucket target sys, bucket metadata sys, metadata paths, and file metadata replication contracts through local boundaries, config storage, storage contracts through the replication storage boundary, runtime sources, and notification state. |
| `replication_resyncer.rs` | Object replication, delete replication, resync, MRF encode/decode, target calls, and multipart target upload paths. | Depends on target calls and target config types through the replication target boundary, metadata paths and metadata systems through the replication metadata boundary, file metadata replication contracts through the filemeta boundary, error contracts through the error boundary, versioning systems, storage contracts through the replication storage boundary, config-derived storage class labels through the config store, runtime sources, notification events and local event host selection through the event sink, bandwidth reader wrapping, and SetDisks lock timing. |
| `replication_state.rs` | Replication queue/stat state and worker accounting. | Reads runtime sources, file metadata replication contracts, and error contracts through local boundaries, and owns shared replication pool/stat state. |
| `replication_state.rs` | Replication queue/stat state and worker accounting. | Reads runtime sources, file metadata replication contracts, error contracts, and bucket monitor handles through local boundaries, and owns shared replication pool/stat state. |
| `rule.rs` | Rule evaluation helpers for object replication options. | Depends on ECStore replication object option types. |
| `mod.rs` | Compatibility re-export facade for the current ECStore owner. | Must stay stable until downstream scanner, lifecycle, heal, and metrics paths compile through replacement contracts. |
@@ -28,7 +28,7 @@ and lifecycle/heal scheduling paths.
| `ReplicationFileMeta` | Replication status, decisions, MRF entries, resync decisions, and target reset helpers. | `rustfs_filemeta` replication contracts are concentrated in `replication_filemeta_boundary.rs`; `FileInfo` remains in the storage boundary for storage trait bindings and walk options. |
| `ReplicationErrorBoundary` | ECStore error/result contracts and replication-specific error classifiers. | `crate::error` imports are concentrated in `replication_error_boundary.rs`. |
| `ReplicationTargetStore` | Bucket target listing, target client lookup, target offline checks, target config types, and target operation option types. | Bucket target sys access, `BucketTargets`, and target operation types are concentrated in `replication_target_boundary.rs`. |
| `ReplicationRuntime` | Worker pool, queue sizing, stats, bucket monitor, local node identity, cancellation, and admission state. | Direct runtime source/global access and shared replication pool/stat state. |
| `ReplicationRuntime` | Worker pool, queue sizing, stats, bucket monitor, local node identity, cancellation, and admission state. | Direct runtime source/global access and shared replication pool/stat state; ECStore object store and bucket monitor implementation types stay behind local storage/bandwidth boundaries. |
| `ReplicationBandwidthLimiter` | Target reader wrapping for replication bandwidth accounting and throttling. | Direct bucket bandwidth reader imports from resyncer paths. |
| `ReplicationEventSink` | Notification and audit events for skipped, failed, pending, and completed replication operations. | Event notification service calls and local event host selection are concentrated in `replication_event_sink.rs`. |
| `ReplicationTagFilter` | Decode object tag strings for rule and metadata replication decisions. | Direct bucket tagging helper imports from replication workers. |
@@ -48,7 +48,9 @@ and lifecycle/heal scheduling paths.
lifecycle, scanner, OBS, heal, and tests compile through replacement paths.
5. Keep imports between modules in this directory relative to the local
replication module, not `crate::bucket::replication::*` self paths.
6. Move at most one contract boundary per code-bearing PR and verify it with
6. Keep runtime source access from importing ECStore object store or bucket
monitor implementation types directly; use local boundary-owned aliases.
7. Move at most one contract boundary per code-bearing PR and verify it with
focused replication tests before broad gates.
## First Code-Bearing Step
crates/ecstore/src/bucket/replication/replication_bandwidth_boundary.rs
+3 -1
View File
@@ -19,9 +19,11 @@ use tokio::io::AsyncRead;
use crate::bucket::bandwidth::monitor::Monitor;
use crate::bucket::bandwidth::reader::{BucketOptions, MonitorReaderOptions, MonitoredReader};
pub(crate) type ReplicationBucketMonitor = Monitor;
pub(crate) fn wrap_reader(
stream: Box<dyn AsyncRead + Unpin + Send + Sync>,
monitor: Arc<Monitor>,
monitor: Arc<ReplicationBucketMonitor>,
bucket: &str,
arn: &str,
header_size: usize,
crates/ecstore/src/bucket/replication/replication_storage_boundary.rs
+1
View File
@@ -16,6 +16,7 @@ use rustfs_filemeta::FileInfo;
use tokio_util::sync::CancellationToken;
use super::replication_error_boundary::Error;
pub(crate) type ReplicationObjectStore = crate::store::ECStore;
pub(crate) use crate::client::api_get_options::{AdvancedGetOptions, StatObjectOptions};
pub(crate) use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
pub(crate) use crate::storage_api_contracts::list::{
crates/ecstore/src/bucket/replication/runtime_boundary.rs
+4 -4
View File
@@ -14,13 +14,13 @@
use std::sync::Arc;
use super::replication_bandwidth_boundary::ReplicationBucketMonitor;
use super::replication_pool::DynReplicationPool;
use super::replication_state::ReplicationStats;
use crate::bucket::bandwidth::monitor::Monitor;
use super::replication_storage_boundary::ReplicationObjectStore;
use crate::runtime::sources;
use crate::store::ECStore;
pub(crate) fn object_store_handle() -> Option<Arc<ECStore>> {
pub(crate) fn object_store_handle() -> Option<Arc<ReplicationObjectStore>> {
sources::object_store_handle()
}
@@ -40,6 +40,6 @@ pub(crate) fn replication_runtime_initialized() -> bool {
sources::replication_runtime_initialized()
}
pub(crate) fn bucket_monitor() -> Option<Arc<Monitor>> {
pub(crate) fn bucket_monitor() -> Option<Arc<ReplicationBucketMonitor>> {
sources::bucket_monitor()
}
docs/architecture/ecstore-module-split-plan.md
+5 -2
View File
@@ -117,7 +117,9 @@ Current coupling:
- global replication pool/stat initialization still lives with ECStore runtime
compatibility state;
- modules inside `bucket/replication` use local relative paths rather than the
ECStore owner path for replication self-imports.
ECStore owner path for replication self-imports;
- replication runtime source access uses storage/bandwidth boundary aliases for
ECStore object store and bucket monitor implementation types.
Required contracts before crate movement:
@@ -154,7 +156,8 @@ Required contracts before crate movement:
are concentrated in
`crates/ecstore/src/bucket/replication/replication_target_boundary.rs`.
- `ReplicationRuntime`: pool, stats, worker admission, bucket monitor, local
node identity, cancellation, and queue sizing.
node identity, cancellation, and queue sizing. Concrete ECStore object store
and bucket monitor types stay behind local storage/bandwidth boundaries.
- `ReplicationBandwidthLimiter`: target reader wrapping for replication
bandwidth accounting and throttling.
- `ReplicationEventSink`: notification/audit events for skipped, failed, and
scripts/check_architecture_migration_rules.sh
+83
View File
@@ -208,6 +208,7 @@ REPLICATION_LOCAL_PATH_BYPASS_HITS_FILE="${TMP_DIR}/replication_local_path_bypas
REPLICATION_LOCK_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_lock_boundary_bypass_hits.txt"
REPLICATION_METADATA_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_metadata_boundary_bypass_hits.txt"
REPLICATION_MSGP_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_msgp_boundary_bypass_hits.txt"
REPLICATION_RUNTIME_TYPE_BYPASS_HITS_FILE="${TMP_DIR}/replication_runtime_type_bypass_hits.txt"
REPLICATION_STORAGE_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_storage_boundary_bypass_hits.txt"
REPLICATION_TARGET_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_target_boundary_bypass_hits.txt"
REPLICATION_TAGGING_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_tagging_boundary_bypass_hits.txt"
@@ -2570,6 +2571,88 @@ if [[ -s "$REPLICATION_LOCAL_PATH_BYPASS_HITS_FILE" ]]; then
report_failure "replication modules must use local relative paths instead of crate-qualified replication self paths: $(paste -sd '; ' "$REPLICATION_LOCAL_PATH_BYPASS_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
find crates/ecstore/src/bucket/replication -type f -name '*.rs' -print0 |
xargs -0 perl -0ne '
next if $ARGV =~ m{replication_(storage|bandwidth)_boundary\.rs$};
sub blank {
my ($text) = @_;
$text =~ s/[^\n]/ /g;
return $text;
}
sub blank_block_comments {
my ($text) = @_;
my $out = $text;
my $len = length($text);
my $i = 0;
while ($i < $len) {
if ($i + 1 < $len && substr($text, $i, 2) eq "/*") {
my $start = $i;
my $depth = 0;
while ($i < $len) {
if ($i + 1 < $len && substr($text, $i, 2) eq "/*") {
$depth++;
$i += 2;
next;
}
if ($i + 1 < $len && substr($text, $i, 2) eq "*/") {
$depth--;
$i += 2;
last if $depth == 0;
next;
}
$i++;
}
substr($out, $start, $i - $start) = blank(substr($text, $start, $i - $start));
next;
}
$i++;
}
return $out;
}
my $source = $_;
my $hash = chr(35);
$source =~ s{b?r(${hash}*)".*?"\1}{blank($&)}egs;
$source =~ s{b?"(?:\\.|[^"\\])*"}{blank($&)}egs;
$source =~ s{'\''(?:\\.|[^'\''\\])+'\''}{blank($&)}egs;
$source = blank_block_comments($source);
$source =~ s{//[^\n]*}{blank($&)}eg;
for my $statement (split /;/, $source) {
my $normalized = $statement;
$normalized =~ s/\s+//g;
my $hits_ecstore =
$normalized =~ /\bECStore\b/
&& $normalized =~ /crate::(?:store(?:::|::\{)|\{.*store(?:::|::\{))/s;
my $hits_monitor =
$normalized =~ /\bMonitor\b/
&& $normalized =~ /crate::(?:bucket(?:::|::\{)|\{.*bucket(?:::|::\{))/s
&& $normalized =~ /bucket(?:::|::\{).*bandwidth(?:::|::\{).*monitor(?:::|::\{)/s;
if ($hits_ecstore || $hits_monitor) {
my $prefix = substr($source, 0, index($source, $statement));
my $line = ($prefix =~ tr/\n//) + 1;
$normalized =~ s/\s+/ /g;
print "$ARGV:$line:$normalized\n";
}
}
' || true
) >"$REPLICATION_RUNTIME_TYPE_BYPASS_HITS_FILE"
if [[ -s "$REPLICATION_RUNTIME_TYPE_BYPASS_HITS_FILE" ]]; then
report_failure "replication runtime concrete types must stay behind local storage or bandwidth boundaries: $(paste -sd '; ' "$REPLICATION_RUNTIME_TYPE_BYPASS_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
rg -n --with-filename 'crate::runtime::sources(\s+as\s+runtime_sources|::)' \