gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 16:28:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(replication): add tagging boundary (#4081)

Browse Source
This commit is contained in:
Zhengchao An
2026-06-30 03:39:17 +08:00
committed by GitHub
parent 4f335889c1
commit fb7bcfb541
6 changed files with 55 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/bucket/replication/README.md
+2 -1
View File
@@ -9,7 +9,7 @@ and lifecycle/heal scheduling paths.
| Module | Current role | Split blocker |
|---|---|---|
| `config.rs` | Replication config helpers, rule matching, and tag filtering. | Uses ECStore bucket tagging helpers and S3 DTOs directly. |
| `config.rs` | Replication config helpers, rule matching, and tag filtering. | Uses replication-local tagging boundary and S3 DTOs directly. |
| `datatypes.rs` | Replication status and operation DTOs. | Publicly re-exported through the ECStore replication facade. |
| `replication_pool.rs` | Replication queue, worker pool, MRF persistence, bucket stats, and delete/object scheduling. | Depends on bucket target sys, bucket metadata sys, config storage, object API, runtime sources, notification state, and storage-api object IO contracts. |
| `replication_resyncer.rs` | Object replication, delete replication, resync, MRF encode/decode, target calls, and multipart target upload paths. | Depends on bucket target clients, metadata/versioning systems, ECStore object readers/writers, disk paths, runtime sources, notification events, and SetDisks lock timing. |
@@ -25,6 +25,7 @@ and lifecycle/heal scheduling paths.
| `ReplicationMetadataStore` | Replication config, bucket targets, MRF/resync state, target reset headers, and status persistence. | Direct bucket target sys, metadata sys, versioning sys, config storage, and file metadata imports. |
| `ReplicationRuntime` | Worker pool, queue sizing, stats, bucket monitor, local node identity, cancellation, and admission state. | Direct runtime source/global access and shared replication pool/stat state. |
| `ReplicationEventSink` | Notification and audit events for skipped, failed, pending, and completed replication operations. | Direct event notification service calls from worker code. |
| `ReplicationTagFilter` | Decode object tag strings for rule and metadata replication decisions. | Direct bucket tagging helper imports from replication workers. |
| `ReplicationLifecycleBridge` | Lifecycle-originated delete and version-purge scheduling. | Direct coupling between lifecycle worker paths and replication delete scheduling. |
## Migration Rules
crates/ecstore/src/bucket/replication/config.rs
+2 -2
View File
@@ -12,8 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::replication_tagging_boundary as tagging_boundary;
use crate::bucket::replication::ReplicationRuleExt as _;
use crate::bucket::tagging::decode_tags_to_map;
use rustfs_filemeta::ReplicationType;
use s3s::dto::DeleteMarkerReplicationStatus;
use s3s::dto::DeleteReplicationStatus;
@@ -98,7 +98,7 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
}
if let Some(filter) = &rule.filter {
let object_tags = decode_tags_to_map(&obj.user_tags);
let object_tags = tagging_boundary::decode_tags_to_map(&obj.user_tags);
if filter.test_tags(&object_tags) {
rules.push(rule.clone());
}
crates/ecstore/src/bucket/replication/mod.rs
+1
View File
@@ -22,6 +22,7 @@ mod replication_msgp_boundary;
mod replication_pool;
mod replication_resyncer;
mod replication_state;
mod replication_tagging_boundary;
mod replication_target_boundary;
mod replication_versioning_boundary;
mod rule;
crates/ecstore/src/bucket/replication/replication_resyncer.rs
+4 -4
View File
@@ -17,6 +17,7 @@ use super::replication_event_sink::{EventArgs, send_event};
use super::replication_lock_boundary as lock_boundary;
use super::replication_metadata_boundary as metadata_boundary;
use super::replication_msgp_boundary::{read_msgp_ext8_time, skip_msgp_value, write_msgp_time};
use super::replication_tagging_boundary as tagging_boundary;
use super::replication_target_boundary as target_boundary;
use super::replication_versioning_boundary as versioning_boundary;
use super::runtime_boundary as runtime_sources;
@@ -26,7 +27,6 @@ use crate::bucket::bucket_target_sys::{
};
use crate::bucket::replication::ResyncStatusType;
use crate::bucket::replication::{ObjectOpts, ReplicationConfigurationExt as _};
use crate::bucket::tagging::decode_tags_to_map;
use crate::bucket::target::BucketTargets;
use crate::client::api_get_options::{AdvancedGetOptions, StatObjectOptions};
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
@@ -3870,7 +3870,7 @@ fn put_replication_opts(sc: &str, object_info: &ObjectInfo) -> Result<(PutObject
};
if !object_info.user_tags.is_empty() {
let tags = decode_tags_to_map(&object_info.user_tags);
let tags = tagging_boundary::decode_tags_to_map(&object_info.user_tags);
if !tags.is_empty() {
put_op.user_tags = tags;
@@ -4140,8 +4140,8 @@ fn get_replication_action(oi1: &ObjectInfo, oi2: &HeadObjectOutput, op_type: Rep
}
}
let oi1_tags = decode_tags_to_map(&oi1.user_tags);
let oi2_tags = decode_tags_to_map(metadata.get(AMZ_OBJECT_TAGGING).cloned().unwrap_or_default().as_str());
let oi1_tags = tagging_boundary::decode_tags_to_map(&oi1.user_tags);
let oi2_tags = tagging_boundary::decode_tags_to_map(metadata.get(AMZ_OBJECT_TAGGING).cloned().unwrap_or_default().as_str());
if (oi2.tag_count.unwrap_or_default() > 0 && oi1_tags != oi2_tags)
|| oi2.tag_count.unwrap_or_default() != oi1_tags.len() as i32
crates/ecstore/src/bucket/replication/replication_tagging_boundary.rs
+33
View File
@@ -0,0 +1,33 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::HashMap;
pub(crate) fn decode_tags_to_map(tags: &str) -> HashMap<String, String> {
crate::bucket::tagging::decode_tags_to_map(tags)
}
#[cfg(test)]
mod tests {
use super::decode_tags_to_map;
#[test]
fn decode_tags_to_map_preserves_bucket_tagging_parser_behavior() {
let tags = decode_tags_to_map("env=prod&encoded=a%2Fb&=ignored");
assert_eq!(tags.get("env").map(String::as_str), Some("prod"));
assert_eq!(tags.get("encoded").map(String::as_str), Some("a/b"));
assert!(!tags.contains_key(""));
}
}
scripts/check_architecture_migration_rules.sh
+13
View File
@@ -190,6 +190,7 @@ REPLICATION_LOCK_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_lock_boundary
REPLICATION_METADATA_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_metadata_boundary_bypass_hits.txt"
REPLICATION_MSGP_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_msgp_boundary_bypass_hits.txt"
REPLICATION_TARGET_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_target_boundary_bypass_hits.txt"
REPLICATION_TAGGING_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_tagging_boundary_bypass_hits.txt"
REPLICATION_VERSIONING_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_versioning_boundary_bypass_hits.txt"
REPLICATION_RUNTIME_SOURCE_BYPASS_HITS_FILE="${TMP_DIR}/replication_runtime_source_bypass_hits.txt"
GLOBAL_REPLICATION_STATE_BYPASS_HITS_FILE="${TMP_DIR}/global_replication_state_bypass_hits.txt"
@@ -2408,6 +2409,18 @@ if [[ -s "$REPLICATION_MSGP_BOUNDARY_BYPASS_HITS_FILE" ]]; then
report_failure "replication msgp helpers must stay behind replication msgp boundary: $(paste -sd '; ' "$REPLICATION_MSGP_BOUNDARY_BYPASS_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
rg -n --with-filename 'crate::bucket::tagging::decode_tags_to_map|use crate::bucket::tagging' \
crates/ecstore/src/bucket/replication \
--glob '*.rs' |
rg -v '^crates/ecstore/src/bucket/replication/replication_tagging_boundary\.rs:' || true
) >"$REPLICATION_TAGGING_BOUNDARY_BYPASS_HITS_FILE"
if [[ -s "$REPLICATION_TAGGING_BOUNDARY_BYPASS_HITS_FILE" ]]; then
report_failure "replication tag decoding must stay behind replication tagging boundary: $(paste -sd '; ' "$REPLICATION_TAGGING_BOUNDARY_BYPASS_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
rg -n --with-filename 'BucketTargetSys::get\(\)' \