From 123552d32bc6a78a4700c58772e95a347e5b620f Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 4 Jul 2026 08:00:37 +0800 Subject: [PATCH] refactor(replication): own utility wire contracts (#4255) --- Cargo.lock | 1 - crates/replication/Cargo.toml | 1 - crates/replication/src/filemeta.rs | 2 +- crates/replication/src/http.rs | 147 ++++++++++++++++++ crates/replication/src/lib.rs | 1 + crates/replication/src/multipart.rs | 8 +- crates/replication/src/object.rs | 9 +- crates/replication/src/operation.rs | 8 +- .../architecture/ecstore-module-split-plan.md | 8 + scripts/check_architecture_migration_rules.sh | 14 ++ 10 files changed, 183 insertions(+), 16 deletions(-) create mode 100644 crates/replication/src/http.rs diff --git a/Cargo.lock b/Cargo.lock index 5ef41185b..9b1dbcfbf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9621,7 +9621,6 @@ dependencies = [ "regex", "rmp", "rmp-serde", - "rustfs-utils", "s3s", "serde", "time", diff --git a/crates/replication/Cargo.toml b/crates/replication/Cargo.toml index 40076cdd6..be759ceea 100644 --- a/crates/replication/Cargo.toml +++ b/crates/replication/Cargo.toml @@ -31,7 +31,6 @@ byteorder.workspace = true regex.workspace = true rmp.workspace = true rmp-serde.workspace = true -rustfs-utils = { workspace = true, features = ["http", "path", "string"] } s3s.workspace = true serde.workspace = true time.workspace = true diff --git a/crates/replication/src/filemeta.rs b/crates/replication/src/filemeta.rs index 14121110a..d7edcd6c5 100644 --- a/crates/replication/src/filemeta.rs +++ b/crates/replication/src/filemeta.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::http::internal_key_rustfs; use bytes::Bytes; use core::fmt; use regex::Regex; -use rustfs_utils::http::internal_key_rustfs; use serde::{Deserialize, Serialize}; use std::any::Any; use std::collections::HashMap; diff --git a/crates/replication/src/http.rs b/crates/replication/src/http.rs new file mode 100644 index 000000000..d11d62ee0 --- /dev/null +++ b/crates/replication/src/http.rs @@ -0,0 +1,147 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; + +const RUSTFS_INTERNAL_PREFIX: &str = "x-rustfs-internal-"; +const MINIO_INTERNAL_PREFIX: &str = "x-minio-internal-"; +const RUSTFS_HEADER_PREFIX: &str = "x-rustfs-"; +const MINIO_HEADER_PREFIX: &str = "x-minio-"; + +pub(crate) const AMZ_BUCKET_REPLICATION_STATUS: &str = "X-Amz-Replication-Status"; +pub(crate) const AMZ_OBJECT_LOCK_LEGAL_HOLD: &str = "X-Amz-Object-Lock-Legal-Hold"; +pub(crate) const AMZ_OBJECT_LOCK_MODE: &str = "X-Amz-Object-Lock-Mode"; +pub(crate) const AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE: &str = "X-Amz-Object-Lock-Retain-Until-Date"; +pub(crate) const AMZ_OBJECT_TAGGING: &str = "X-Amz-Tagging"; +pub(crate) const AMZ_WEBSITE_REDIRECT_LOCATION: &str = "x-amz-website-redirect-location"; +pub(crate) const CACHE_CONTROL: &str = "Cache-Control"; +pub(crate) const CONTENT_DISPOSITION: &str = "Content-Disposition"; +pub(crate) const CONTENT_ENCODING: &str = "Content-Encoding"; +pub(crate) const CONTENT_LANGUAGE: &str = "Content-Language"; +pub(crate) const EXPIRES: &str = "Expires"; +pub(crate) const SSEC_ALGORITHM_HEADER: &str = "x-amz-server-side-encryption-customer-algorithm"; +pub(crate) const SSEC_KEY_HEADER: &str = "x-amz-server-side-encryption-customer-key"; +pub(crate) const SSEC_KEY_MD5_HEADER: &str = "x-amz-server-side-encryption-customer-key-md5"; +pub(crate) const SUFFIX_ACTUAL_SIZE: &str = "actual-size"; +pub(crate) const SUFFIX_REPLICATION_RESET_STATUS: &str = "replication-reset-status"; + +fn internal_keys(suffix: &str) -> (String, String) { + (format!("{RUSTFS_INTERNAL_PREFIX}{suffix}"), format!("{MINIO_INTERNAL_PREFIX}{suffix}")) +} + +fn rustfs_header_key(suffix: &str) -> String { + format!("{RUSTFS_HEADER_PREFIX}{suffix}") +} + +fn minio_header_key(suffix: &str) -> String { + format!("{MINIO_HEADER_PREFIX}{suffix}") +} + +pub(crate) fn internal_key_rustfs(suffix: &str) -> String { + format!("{RUSTFS_INTERNAL_PREFIX}{suffix}") +} + +pub(crate) fn get_internal_metadata(map: &HashMap, suffix: &str) -> Option { + let (rustfs_key, minio_key) = internal_keys(suffix); + map.get(&rustfs_key) + .cloned() + .or_else(|| map.get(&minio_key).cloned()) + .or_else(|| { + map.iter() + .find(|(key, _)| key.eq_ignore_ascii_case(&rustfs_key) || key.eq_ignore_ascii_case(&minio_key)) + .map(|(_, value)| value.clone()) + }) +} + +pub(crate) fn get_header_metadata(map: &HashMap, suffix: &str) -> Option { + let rustfs_key = rustfs_header_key(suffix); + let minio_key = minio_header_key(suffix); + map.get(&rustfs_key).cloned().or_else(|| map.get(&minio_key).cloned()) +} + +pub(crate) fn has_prefix_fold(s: &str, prefix: &str) -> bool { + if s.starts_with(prefix) { + return true; + } + + s.get(..prefix.len()) + .is_some_and(|s_prefix| s_prefix.eq_ignore_ascii_case(prefix)) +} + +pub(crate) fn trim_etag(etag: &str) -> String { + etag.trim_matches('"').to_string() +} + +#[cfg(test)] +pub(crate) fn insert_internal_metadata(map: &mut HashMap, suffix: &str, value: String) { + let (rustfs_key, minio_key) = internal_keys(suffix); + map.insert(rustfs_key, value.clone()); + map.insert(minio_key, value); +} + +#[cfg(test)] +mod tests { + use super::{ + SUFFIX_ACTUAL_SIZE, SUFFIX_REPLICATION_RESET_STATUS, get_header_metadata, get_internal_metadata, has_prefix_fold, + insert_internal_metadata, internal_key_rustfs, trim_etag, + }; + use std::collections::HashMap; + + #[test] + fn internal_metadata_prefers_rustfs_and_falls_back_to_minio() { + let mut metadata = HashMap::from([("x-minio-internal-actual-size".to_string(), "10".to_string())]); + + assert_eq!(get_internal_metadata(&metadata, SUFFIX_ACTUAL_SIZE).as_deref(), Some("10")); + + metadata.insert("x-rustfs-internal-actual-size".to_string(), "11".to_string()); + assert_eq!(get_internal_metadata(&metadata, SUFFIX_ACTUAL_SIZE).as_deref(), Some("11")); + } + + #[test] + fn internal_metadata_keeps_case_insensitive_lookup_compatibility() { + let metadata = HashMap::from([("X-RustFS-Internal-Actual-Size".to_string(), "12".to_string())]); + + assert_eq!(get_internal_metadata(&metadata, SUFFIX_ACTUAL_SIZE).as_deref(), Some("12")); + } + + #[test] + fn internal_metadata_insert_writes_rustfs_and_minio_keys() { + let mut metadata = HashMap::new(); + insert_internal_metadata(&mut metadata, SUFFIX_ACTUAL_SIZE, "13".to_string()); + + assert_eq!(metadata.get("x-rustfs-internal-actual-size").map(String::as_str), Some("13")); + assert_eq!(metadata.get("x-minio-internal-actual-size").map(String::as_str), Some("13")); + } + + #[test] + fn header_metadata_prefers_rustfs_then_minio() { + let mut metadata = HashMap::from([("x-minio-replication-reset-status".to_string(), "old".to_string())]); + + assert_eq!(get_header_metadata(&metadata, SUFFIX_REPLICATION_RESET_STATUS).as_deref(), Some("old")); + + metadata.insert("x-rustfs-replication-reset-status".to_string(), "new".to_string()); + assert_eq!(get_header_metadata(&metadata, SUFFIX_REPLICATION_RESET_STATUS).as_deref(), Some("new")); + } + + #[test] + fn helper_contracts_match_replication_wire_rules() { + assert_eq!( + internal_key_rustfs("replication-reset-arn:target"), + "x-rustfs-internal-replication-reset-arn:target" + ); + assert_eq!(trim_etag("\"abc\""), "abc"); + assert!(has_prefix_fold("X-Amz-Meta-Foo", "x-amz-meta-")); + assert!(!has_prefix_fold("X-Amz-Meta-Foo", "amz-meta")); + } +} diff --git a/crates/replication/src/lib.rs b/crates/replication/src/lib.rs index 7cf1d4797..89e646c39 100644 --- a/crates/replication/src/lib.rs +++ b/crates/replication/src/lib.rs @@ -15,6 +15,7 @@ pub mod config; pub mod delete; mod filemeta; +mod http; pub mod mrf; pub mod multipart; pub mod object; diff --git a/crates/replication/src/multipart.rs b/crates/replication/src/multipart.rs index e3c81a370..05c9e9d85 100644 --- a/crates/replication/src/multipart.rs +++ b/crates/replication/src/multipart.rs @@ -15,7 +15,7 @@ use std::collections::HashMap; use std::fmt; -use rustfs_utils::http::{SUFFIX_ACTUAL_SIZE, get_str}; +use crate::http::{SUFFIX_ACTUAL_SIZE, get_internal_metadata}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ReplicationMultipartPartInput { @@ -106,7 +106,7 @@ pub fn replication_multipart_part_plan( } pub fn replication_multipart_complete_actual_size(user_defined: &HashMap) -> String { - get_str(user_defined, SUFFIX_ACTUAL_SIZE).unwrap_or_default() + get_internal_metadata(user_defined, SUFFIX_ACTUAL_SIZE).unwrap_or_default() } #[cfg(test)] @@ -115,7 +115,7 @@ mod tests { ReplicationMultipartPartInput, ReplicationMultipartPartPlan, ReplicationMultipartPlanError, ReplicationMultipartRange, replication_multipart_complete_actual_size, replication_multipart_part_plan, }; - use rustfs_utils::http::{SUFFIX_ACTUAL_SIZE, insert_str}; + use crate::http::{SUFFIX_ACTUAL_SIZE, insert_internal_metadata}; use std::collections::HashMap; #[test] @@ -214,7 +214,7 @@ mod tests { #[test] fn multipart_complete_actual_size_reads_compatible_metadata() { let mut user_defined = HashMap::new(); - insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "123".to_string()); + insert_internal_metadata(&mut user_defined, SUFFIX_ACTUAL_SIZE, "123".to_string()); assert_eq!(replication_multipart_complete_actual_size(&user_defined), "123"); assert!(replication_multipart_complete_actual_size(&HashMap::new()).is_empty()); diff --git a/crates/replication/src/object.rs b/crates/replication/src/object.rs index 44f7d8cf6..572d97d97 100644 --- a/crates/replication/src/object.rs +++ b/crates/replication/src/object.rs @@ -13,13 +13,12 @@ // limitations under the License. use crate::filemeta::{ReplicationAction, ReplicationType}; -use crate::tagging::ReplicationTagFilter; -use rustfs_utils::http::{ +use crate::http::{ AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_TAGGING, AMZ_WEBSITE_REDIRECT_LOCATION, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, EXPIRES, + has_prefix_fold, trim_etag, }; -use rustfs_utils::path::trim_etag; -use rustfs_utils::string::strings_has_prefix_fold; +use crate::tagging::ReplicationTagFilter; use std::collections::HashMap; use time::OffsetDateTime; @@ -150,7 +149,7 @@ fn comparable_metadata(metadata: Option<&HashMap>) -> HashMap"$REPLICATION_CRATE_UTILS_BYPASS_HITS_FILE" + +if [[ -s "$REPLICATION_CRATE_UTILS_BYPASS_HITS_FILE" ]]; then + report_failure "replication crate HTTP/helper contracts must not import or depend on rustfs-utils: $(paste -sd '; ' "$REPLICATION_CRATE_UTILS_BYPASS_HITS_FILE")" +fi + ( cd "$ROOT_DIR" replication_config_rule_status=0