mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-27 16:48:58 +00:00
refactor(ecstore): use local replication paths (#4130)
This commit is contained in:
@@ -46,7 +46,9 @@ and lifecycle/heal scheduling paths.
|
||||
only after call sites prove a narrower object read/write/delete/walk shape.
|
||||
4. Preserve `rustfs_ecstore::api::bucket::replication` compatibility until
|
||||
lifecycle, scanner, OBS, heal, and tests compile through replacement paths.
|
||||
5. Move at most one contract boundary per code-bearing PR and verify it with
|
||||
5. Keep imports between modules in this directory relative to the local
|
||||
replication module, not `crate::bucket::replication::*` self paths.
|
||||
6. Move at most one contract boundary per code-bearing PR and verify it with
|
||||
focused replication tests before broad gates.
|
||||
|
||||
## First Code-Bearing Step
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use super::replication_filemeta_boundary::ReplicationType;
|
||||
use super::replication_tagging_boundary as tagging_boundary;
|
||||
use crate::bucket::replication::ReplicationRuleExt as _;
|
||||
use super::rule::ReplicationRuleExt as _;
|
||||
use s3s::dto::DeleteMarkerReplicationStatus;
|
||||
use s3s::dto::DeleteReplicationStatus;
|
||||
use s3s::dto::Destination;
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::datatypes::ResyncStatusType;
|
||||
use super::replication_config_store as config_store;
|
||||
use super::replication_error_boundary::Error as EcstoreError;
|
||||
use super::replication_filemeta_boundary::{
|
||||
@@ -20,19 +21,15 @@ use super::replication_filemeta_boundary::{
|
||||
ResyncDecision, VersionPurgeStatusType, replication_statuses_map, version_purge_statuses_map,
|
||||
};
|
||||
use super::replication_metadata_boundary as metadata_boundary;
|
||||
use super::replication_resyncer::{
|
||||
BucketReplicationResyncStatus, DeletedObjectReplicationInfo, ReplicationConfig, ReplicationResyncer, ResyncOpts,
|
||||
TargetReplicationResyncStatus, decode_mrf_file, decode_resync_file, encode_mrf_file, get_heal_replicate_object_info,
|
||||
replicate_delete, replicate_object, save_resync_status,
|
||||
};
|
||||
use super::replication_state::ReplicationStats;
|
||||
use super::replication_storage_boundary::{DeletedObject, ObjectInfo, ObjectOptions, ReplicationObjectIO, ReplicationStorage};
|
||||
use super::replication_target_boundary as target_boundary;
|
||||
use super::runtime_boundary as runtime_sources;
|
||||
use crate::bucket::replication::ResyncOpts;
|
||||
use crate::bucket::replication::ResyncStatusType;
|
||||
use crate::bucket::replication::replicate_delete;
|
||||
use crate::bucket::replication::replicate_object;
|
||||
use crate::bucket::replication::replication_resyncer::{
|
||||
BucketReplicationResyncStatus, DeletedObjectReplicationInfo, ReplicationConfig, ReplicationResyncer,
|
||||
TargetReplicationResyncStatus, decode_mrf_file, decode_resync_file, encode_mrf_file, get_heal_replicate_object_info,
|
||||
save_resync_status,
|
||||
};
|
||||
use crate::bucket::replication::replication_state::ReplicationStats;
|
||||
use lazy_static::lazy_static;
|
||||
use rustfs_utils::http::{SUFFIX_REPLICATION_TIMESTAMP, get_str};
|
||||
use std::any::Any;
|
||||
@@ -1692,8 +1689,8 @@ async fn queue_replicate_deletes_wrapper(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::replication_resyncer::{decode_mrf_file, encode_mrf_file};
|
||||
use super::*;
|
||||
use crate::bucket::replication::replication_resyncer::{decode_mrf_file, encode_mrf_file};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -12,6 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::config::{ObjectOpts, ReplicationConfigurationExt as _};
|
||||
use super::datatypes::ResyncStatusType;
|
||||
use super::replication_bandwidth_boundary;
|
||||
use super::replication_config_store as config_store;
|
||||
use super::replication_error_boundary::{Error, Result, is_err_object_not_found, is_err_version_not_found};
|
||||
@@ -36,8 +38,6 @@ use super::replication_target_boundary::{
|
||||
};
|
||||
use super::replication_versioning_boundary as versioning_boundary;
|
||||
use super::runtime_boundary as runtime_sources;
|
||||
use crate::bucket::replication::ResyncStatusType;
|
||||
use crate::bucket::replication::{ObjectOpts, ReplicationConfigurationExt as _};
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::operation::head_object::{HeadObjectError, HeadObjectOutput};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::bucket::replication::ObjectOpts;
|
||||
use super::config::ObjectOpts;
|
||||
use s3s::dto::ReplicaModificationsStatus;
|
||||
use s3s::dto::ReplicationRule;
|
||||
|
||||
|
||||
@@ -14,9 +14,9 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::replication_pool::DynReplicationPool;
|
||||
use super::replication_state::ReplicationStats;
|
||||
use crate::bucket::bandwidth::monitor::Monitor;
|
||||
use crate::bucket::replication::replication_pool::DynReplicationPool;
|
||||
use crate::bucket::replication::replication_state::ReplicationStats;
|
||||
use crate::runtime::sources;
|
||||
use crate::store::ECStore;
|
||||
|
||||
|
||||
@@ -115,7 +115,9 @@ Current coupling:
|
||||
- lifecycle and heal paths schedule replication work through the current ECStore
|
||||
module;
|
||||
- global replication pool/stat initialization still lives with ECStore runtime
|
||||
compatibility state.
|
||||
compatibility state;
|
||||
- modules inside `bucket/replication` use local relative paths rather than the
|
||||
ECStore owner path for replication self-imports.
|
||||
|
||||
Required contracts before crate movement:
|
||||
|
||||
|
||||
@@ -204,6 +204,7 @@ REPLICATION_ERROR_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_error_bounda
|
||||
REPLICATION_EVENT_SINK_BYPASS_HITS_FILE="${TMP_DIR}/replication_event_sink_bypass_hits.txt"
|
||||
REPLICATION_EVENT_HOST_BYPASS_HITS_FILE="${TMP_DIR}/replication_event_host_bypass_hits.txt"
|
||||
REPLICATION_FILEMETA_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_filemeta_boundary_bypass_hits.txt"
|
||||
REPLICATION_LOCAL_PATH_BYPASS_HITS_FILE="${TMP_DIR}/replication_local_path_bypass_hits.txt"
|
||||
REPLICATION_LOCK_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_lock_boundary_bypass_hits.txt"
|
||||
REPLICATION_METADATA_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_metadata_boundary_bypass_hits.txt"
|
||||
REPLICATION_MSGP_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_msgp_boundary_bypass_hits.txt"
|
||||
@@ -2501,6 +2502,74 @@ if [[ -s "$REPLICATION_FACADE_BYPASS_HITS_FILE" ]]; then
|
||||
report_failure "replication facade imports must stay in local storage_api boundaries: $(paste -sd '; ' "$REPLICATION_FACADE_BYPASS_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
find crates/ecstore/src/bucket/replication -type f -name '*.rs' -print0 |
|
||||
xargs -0 perl -0ne '
|
||||
sub blank {
|
||||
my ($text) = @_;
|
||||
$text =~ s/[^\n]/ /g;
|
||||
return $text;
|
||||
}
|
||||
|
||||
sub blank_block_comments {
|
||||
my ($text) = @_;
|
||||
my $out = $text;
|
||||
my $len = length($text);
|
||||
my $i = 0;
|
||||
|
||||
while ($i < $len) {
|
||||
if ($i + 1 < $len && substr($text, $i, 2) eq "/*") {
|
||||
my $start = $i;
|
||||
my $depth = 0;
|
||||
|
||||
while ($i < $len) {
|
||||
if ($i + 1 < $len && substr($text, $i, 2) eq "/*") {
|
||||
$depth++;
|
||||
$i += 2;
|
||||
next;
|
||||
}
|
||||
if ($i + 1 < $len && substr($text, $i, 2) eq "*/") {
|
||||
$depth--;
|
||||
$i += 2;
|
||||
last if $depth == 0;
|
||||
next;
|
||||
}
|
||||
$i++;
|
||||
}
|
||||
|
||||
substr($out, $start, $i - $start) = blank(substr($text, $start, $i - $start));
|
||||
next;
|
||||
}
|
||||
|
||||
$i++;
|
||||
}
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
my $source = $_;
|
||||
my $hash = chr(35);
|
||||
$source =~ s{b?r(${hash}*)".*?"\1}{blank($&)}egs;
|
||||
$source =~ s{b?"(?:\\.|[^"\\])*"}{blank($&)}egs;
|
||||
$source =~ s{'\''(?:\\.|[^'\''\\])+'\''}{blank($&)}egs;
|
||||
$source = blank_block_comments($source);
|
||||
$source =~ s{//[^\n]*}{blank($&)}eg;
|
||||
|
||||
while ($source =~ /(?:crate::bucket::replication\b|crate::bucket::\{[^;]*\breplication\b|crate::\{[^;]*\bbucket::replication\b|crate::\{[^;]*\bbucket::\{[^;]*\breplication\b)/sg) {
|
||||
my $prefix = substr($source, 0, $-[0]);
|
||||
my $line = ($prefix =~ tr/\n//) + 1;
|
||||
my $match = $&;
|
||||
$match =~ s/\s+/ /g;
|
||||
print "$ARGV:$line:$match\n";
|
||||
}
|
||||
' || true
|
||||
) >"$REPLICATION_LOCAL_PATH_BYPASS_HITS_FILE"
|
||||
|
||||
if [[ -s "$REPLICATION_LOCAL_PATH_BYPASS_HITS_FILE" ]]; then
|
||||
report_failure "replication modules must use local relative paths instead of crate-qualified replication self paths: $(paste -sd '; ' "$REPLICATION_LOCAL_PATH_BYPASS_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'crate::runtime::sources(\s+as\s+runtime_sources|::)' \
|
||||
|
||||
Reference in New Issue
Block a user