From 1735dcde9ca598e909bd9197d1426862b9e3e6fc Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 23 Jun 2026 23:23:16 +0800 Subject: [PATCH] refactor: centralize ecstore data-plane runtime sources (#3797) --- crates/ecstore/src/erasure_coding/encode.rs | 7 ++- crates/ecstore/src/lib.rs | 1 + crates/ecstore/src/object_api/readers.rs | 4 +- crates/ecstore/src/runtime_sources.rs | 44 ++++++++++++++++ crates/ecstore/src/set_disk.rs | 42 ++++------------ crates/ecstore/src/set_disk/heal.rs | 11 ++-- docs/architecture/migration-progress.md | 56 ++++++++++++++++++--- 7 files changed, 113 insertions(+), 52 deletions(-) create mode 100644 crates/ecstore/src/runtime_sources.rs diff --git a/crates/ecstore/src/erasure_coding/encode.rs b/crates/ecstore/src/erasure_coding/encode.rs index 0d4ee1886..91508698a 100644 --- a/crates/ecstore/src/erasure_coding/encode.rs +++ b/crates/ecstore/src/erasure_coding/encode.rs @@ -18,6 +18,7 @@ use crate::disk::error_reduce::{ }; use crate::erasure_coding::BitrotWriterWrapper; use crate::erasure_coding::Erasure; +use crate::runtime_sources; use bytes::Bytes; use futures::StreamExt; use futures::stream::FuturesUnordered; @@ -138,8 +139,7 @@ impl<'a> MultiWriter<'a> { if let Some(write_err) = reduce_write_quorum_errs(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum) { let summary = build_write_quorum_failure_summary(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum); let summary_text = format_write_quorum_failure(&summary); - rustfs_io_metrics::internode_metrics::global_internode_metrics() - .record_erasure_write_quorum_failure("write", quorum_dominant_error_metric_label(&summary)); + runtime_sources::record_erasure_write_quorum_failure("write", quorum_dominant_error_metric_label(&summary)); error!("reduce_write_quorum_errs: {:?}, {}, errs={:?}", write_err, summary_text, self.errs); return Err(std::io::Error::other(format!("Failed to write data: {summary_text}"))); } @@ -193,8 +193,7 @@ impl<'a> MultiWriter<'a> { if let Some(write_err) = reduce_write_quorum_errs(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum) { let summary = build_write_quorum_failure_summary(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum); let summary_text = format_write_quorum_failure(&summary); - rustfs_io_metrics::internode_metrics::global_internode_metrics() - .record_erasure_write_quorum_failure("shutdown", quorum_dominant_error_metric_label(&summary)); + runtime_sources::record_erasure_write_quorum_failure("shutdown", quorum_dominant_error_metric_label(&summary)); error!( "reduce_write_quorum_errs during shutdown: {:?}, {}, errs={:?}", write_err, summary_text, self.errs diff --git a/crates/ecstore/src/lib.rs b/crates/ecstore/src/lib.rs index 3e8fa724f..b26764a56 100644 --- a/crates/ecstore/src/lib.rs +++ b/crates/ecstore/src/lib.rs @@ -40,6 +40,7 @@ mod pools; mod rebalance; mod rio; mod rpc; +mod runtime_sources; mod set_disk; mod sets; mod storage_api_contracts; diff --git a/crates/ecstore/src/object_api/readers.rs b/crates/ecstore/src/object_api/readers.rs index f94e695ff..1412713ec 100644 --- a/crates/ecstore/src/object_api/readers.rs +++ b/crates/ecstore/src/object_api/readers.rs @@ -11,7 +11,7 @@ use chacha20poly1305::ChaCha20Poly1305; #[cfg(feature = "rio-v2")] use hmac::{Hmac, Mac}; use md5::{Digest, Md5}; -use rustfs_kms::{service_manager::get_global_encryption_service, types::ObjectEncryptionContext}; +use rustfs_kms::types::ObjectEncryptionContext; use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER}; use rustfs_utils::path::path_join_buf; #[cfg(feature = "rio-v2")] @@ -1308,7 +1308,7 @@ async fn resolve_managed_material(bucket: &str, object: &str, metadata: &HashMap let kms_context: Option> = None; let object_context = build_object_encryption_context(bucket, object, kms_context.as_ref()); - let decrypted_key = if let Some(service) = get_global_encryption_service().await { + let decrypted_key = if let Some(service) = crate::runtime_sources::object_encryption_service().await { #[cfg(feature = "rio-v2")] let data_key = if is_legacy_rustfs_managed_metadata(&normalized_metadata) { service.decrypt_legacy_data_key(&encrypted_dek).await diff --git a/crates/ecstore/src/runtime_sources.rs b/crates/ecstore/src/runtime_sources.rs new file mode 100644 index 000000000..5ed470386 --- /dev/null +++ b/crates/ecstore/src/runtime_sources.rs @@ -0,0 +1,44 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use crate::{config::get_global_storage_class, global::get_global_deployment_id}; +use rustfs_io_metrics::internode_metrics::global_internode_metrics; +use rustfs_kms::{ObjectEncryptionService, get_global_encryption_service}; + +pub(crate) fn record_erasure_write_quorum_failure(stage: &'static str, dominant_error: &'static str) { + global_internode_metrics().record_erasure_write_quorum_failure(stage, dominant_error); +} + +pub(crate) async fn object_encryption_service() -> Option> { + get_global_encryption_service().await +} + +pub(crate) fn storage_class_parity(storage_class: Option<&str>) -> Option { + get_global_storage_class().and_then(|sc| sc.get_parity_for_sc(storage_class.unwrap_or_default())) +} + +pub(crate) fn storage_class_should_inline(shard_size: i64, versioned: bool) -> bool { + get_global_storage_class().is_some_and(|sc| sc.should_inline(shard_size, versioned)) +} + +pub(crate) fn deployment_upload_id(upload_id: &str) -> String { + base64_simd::URL_SAFE_NO_PAD + .encode_to_string(format!("{}.{}", get_global_deployment_id().unwrap_or_default(), upload_id).as_bytes()) +} + +pub(crate) fn global_lock_manager() -> Arc { + rustfs_lock::get_global_lock_manager() +} diff --git a/crates/ecstore/src/set_disk.rs b/crates/ecstore/src/set_disk.rs index 9b4a9c15b..f40fa2758 100644 --- a/crates/ecstore/src/set_disk.rs +++ b/crates/ecstore/src/set_disk.rs @@ -38,13 +38,14 @@ use crate::error::{GenericError, ObjectApiError, is_err_object_not_found}; use crate::global::{GLOBAL_LocalNodeName, GLOBAL_TierConfigMgr}; use crate::object_api::ObjectOptions; use crate::rpc::heal_bucket_local_on_disks; +use crate::runtime_sources; use crate::store_utils::is_reserved_or_invalid_bucket; use crate::{ bucket::lifecycle::bucket_lifecycle_ops::{ LifecycleOps, gen_transition_objname, get_transitioned_object_reader, put_restore_opts, }, cache_value::metacache_set::{ListPathRawOptions, list_path_raw}, - config::{get_global_storage_class, storageclass}, + config::storageclass, disk::{ CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskOption, DiskStore, FileInfoVersions, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, @@ -53,7 +54,7 @@ use crate::{ error::{StorageError, to_object_err}, // event::name::EventName, event_notification::{EventArgs, send_event}, - global::{GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id, is_dist_erasure}, + global::{GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, is_dist_erasure}, object_api::{GetObjectReader, ObjectInfo, PutObjReader}, store_init::{get_format_erasure_in_quorum, load_format_erasure, load_format_erasure_all, save_format_file}, }; @@ -396,9 +397,7 @@ fn build_tiered_decommission_file_info( default_parity_count: usize, storage_class: Option<&str>, ) -> (FileInfo, usize) { - let parity_drives = get_global_storage_class() - .and_then(|sc| sc.get_parity_for_sc(storage_class.unwrap_or_default())) - .unwrap_or(default_parity_count); + let parity_drives = runtime_sources::storage_class_parity(storage_class).unwrap_or(default_parity_count); let data_drives = disk_count - parity_drives; let mut write_quorum = data_drives; if data_drives == parity_drives { @@ -562,7 +561,7 @@ impl SetDisks { set_endpoints, disk_health_cache: Arc::new(RwLock::new(Vec::new())), lockers, - local_lock_manager: rustfs_lock::get_global_lock_manager(), + local_lock_manager: runtime_sources::global_lock_manager(), }) } @@ -1028,13 +1027,7 @@ impl rustfs_storage_api::ObjectIO for SetDisks { user_defined.insert(key.clone(), value.clone()); } } - let sc_parity_drives = { - if let Some(sc) = get_global_storage_class() { - sc.get_parity_for_sc(user_defined.get(AMZ_STORAGE_CLASS).cloned().unwrap_or_default().as_str()) - } else { - None - } - }; + let sc_parity_drives = runtime_sources::storage_class_parity(user_defined.get(AMZ_STORAGE_CLASS).map(String::as_str)); let mut parity_drives = sc_parity_drives.unwrap_or(self.default_parity_count); if opts.max_parity { @@ -1082,13 +1075,8 @@ impl rustfs_storage_api::ObjectIO for SetDisks { let result: Result = async { let erasure = erasure_coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size); - let is_inline_buffer = { - if let Some(sc) = get_global_storage_class() { - sc.should_inline(erasure.shard_file_size(data.size()), opts.versioned) - } else { - false - } - }; + let is_inline_buffer = + runtime_sources::storage_class_should_inline(erasure.shard_file_size(data.size()), opts.versioned); let shard_file_size = erasure.shard_file_size(data.size()); let shard_size = erasure.shard_size(); @@ -3729,8 +3717,7 @@ impl rustfs_storage_api::MultipartOperations for SetDisks { uploads.push(MultipartInfo { bucket: bucket.to_owned(), object: object.to_owned(), - upload_id: base64_simd::URL_SAFE_NO_PAD - .encode_to_string(format!("{}.{}", get_global_deployment_id().unwrap_or_default(), upload_id).as_bytes()), + upload_id: runtime_sources::deployment_upload_id(&upload_id), initiated: Some(start_time), ..Default::default() }); @@ -3820,13 +3807,7 @@ impl rustfs_storage_api::MultipartOperations for SetDisks { let _ = user_defined.remove(AMZ_STORAGE_CLASS); } - let sc_parity_drives = { - if let Some(sc) = get_global_storage_class() { - sc.get_parity_for_sc(user_defined.get(AMZ_STORAGE_CLASS).cloned().unwrap_or_default().as_str()) - } else { - None - } - }; + let sc_parity_drives = runtime_sources::storage_class_parity(user_defined.get(AMZ_STORAGE_CLASS).map(String::as_str)); let mut parity_drives = sc_parity_drives.unwrap_or(self.default_parity_count); if opts.max_parity { @@ -3897,8 +3878,7 @@ impl rustfs_storage_api::MultipartOperations for SetDisks { let upload_uuid = format!("{}x{}", Uuid::new_v4(), mod_time.unix_timestamp_nanos()); - let upload_id = base64_simd::URL_SAFE_NO_PAD - .encode_to_string(format!("{}.{}", get_global_deployment_id().unwrap_or_default(), upload_uuid).as_bytes()); + let upload_id = runtime_sources::deployment_upload_id(&upload_uuid); let upload_path = Self::get_upload_id_dir(bucket, object, upload_uuid.as_str()); diff --git a/crates/ecstore/src/set_disk/heal.rs b/crates/ecstore/src/set_disk/heal.rs index 9efdad716..67ce3a155 100644 --- a/crates/ecstore/src/set_disk/heal.rs +++ b/crates/ecstore/src/set_disk/heal.rs @@ -414,13 +414,10 @@ impl SetDisks { } } - let is_inline_buffer = { - if let Some(sc) = get_global_storage_class() { - sc.should_inline(erasure.shard_file_size(latest_meta.size), false) - } else { - false - } - }; + let is_inline_buffer = runtime_sources::storage_class_should_inline( + erasure.shard_file_size(latest_meta.size), + false, + ); // create writers for all disk positions, but only for outdated disks for (index, disk_op) in out_dated_disks.iter().enumerate() { if let Some(outdated_disk) = disk_op { diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index 849439664..d83be5ccb 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,11 +5,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-network-client-runtime-sources` -- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185`. -- Based on: latest `origin/main` after PR #3795 merged API-184; this branch - batches network client runtime source boundary cleanup on top of that - baseline. +- Branch: `overtrue/arch-ecstore-data-plane-runtime-sources` +- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186`. +- Based on: latest `origin/main` after PR #3796 merged API-185. - PR type for this branch: `consumer-migration` - Runtime behavior changes: none. - Rust code changes: route replication pool, outbound TLS generation, runtime @@ -18,8 +16,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block rules/event dispatch, admin OIDC/token-signing reads, IAM root credential consumers, IAM OIDC config reads, scanner runtime-config reads, OBS metrics runtime source reads, RIO HTTP reader TLS/metrics runtime source reads, and - gRPC/transition network client TLS/metrics runtime source reads through - AppContext-first or owner-crate resolver boundaries. + gRPC/transition network client TLS/metrics runtime source reads, plus ECStore + data-plane KMS/storage-class/deployment-id/lock-manager/erasure metric reads, + through AppContext-first or owner-crate resolver boundaries. - CI/script changes: lock completed owner and test/fuzz boundaries against bare/glob imports, scattered raw ECStore facade subpaths, and startup runtime/root-server/table/S3/app shared/app bucket/app ECStore/admin facade @@ -28,7 +27,7 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block and storage owner thin bridge regressions, plus app context and notify event-bridge thin module regressions; accept the reviewed AppContext resolver reverse dependencies in the layer baseline. -- Docs changes: record the API-136 through API-185 owner facade cleanup. +- Docs changes: record the API-136 through API-186 owner facade cleanup. ## Phase 0 Tasks @@ -4685,6 +4684,22 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block client runtime source scan, Rust risk scan, branch freshness check, pre-commit quality gate, and three-expert review. +- [x] `API-186` Centralize ECStore data-plane runtime source reads. + - Do: add an ECStore runtime-source boundary for erasure quorum failure + metrics, object-read managed KMS service lookup, storage-class parity and + inline decisions, deployment-prefixed multipart upload ids, and the local + lock-manager handle. + - Acceptance: ECStore data-plane write/read/multipart paths no longer import + those global runtime sources directly outside the owner runtime-source + module. + - Must preserve: erasure quorum metric stage/error labels, managed-KMS + decrypt fallback behavior, storage-class parity and inline decisions, + multipart upload id encoding, and local lock-manager initialization. + - Verification: ECStore compile coverage, focused unit tests, formatting, + migration guard, layer guard, diff hygiene, residual data-plane runtime + source scan, Rust risk scan, branch freshness check, pre-commit quality + gate, and three-expert review. + ## Next PRs 1. `consumer-migration`: continue reducing direct global reads behind AppContext resolver boundaries. @@ -4796,11 +4811,36 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block | Quality/architecture | pass | API-185 keeps protos and ECStore network client TLS/metrics globals behind owner runtime-source modules without widening public APIs. | | Migration preservation | pass | gRPC TLS cache invalidation, transition-client TLS generation reporting, dial metrics, retry metrics, and read/write byte/error counters keep existing behavior. | | Testing/verification | pass | Protos/ECStore compile/tests, formatting, residual network client runtime source scan, targeted guard checks, and pre-commit passed for API-185. | +| Quality/architecture | pass | API-186 keeps ECStore data-plane runtime globals behind an ECStore-owned runtime-source module without widening public APIs. | +| Migration preservation | pass | Erasure quorum metric labels, managed-KMS fallback, storage-class decisions, multipart upload id encoding, and lock-manager initialization keep existing behavior. | +| Testing/verification | pass | ECStore compile/focused tests, formatting, residual data-plane runtime source scan, targeted guard checks, and pre-commit passed for API-186. | ## Verification Notes Passed before push: +- Issue #660 API-186 current slice: + - `cargo check -p rustfs-ecstore --tests`: passed. + - `cargo test -p rustfs-ecstore --lib erasure_coding -- --test-threads=1`: + passed. + - `cargo test -p rustfs-ecstore --lib set_disk -- --test-threads=1`: + passed. + - `cargo test -p rustfs-ecstore --lib readers -- --test-threads=1`: + passed. + - `cargo fmt --all`: passed. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_layer_dependencies.sh`: passed. + - ECStore data-plane runtime source scan: passed; direct storage-class, + deployment id, KMS encryption service, lock-manager, and erasure internode + metric global reads are isolated to the ECStore runtime source module. + - Rust risk scan: passed; diff adds no new `expect`, `panic`, `todo`, + `unimplemented`, or `unsafe`. + - Branch freshness check: rebased onto latest `origin/main` after PR #3796 + merged API-185. + - `make pre-commit`: passed. + - Issue #660 API-185 current slice: - `cargo check -p rustfs-protos -p rustfs-ecstore --tests`: passed. - `cargo test -p rustfs-protos --lib`: passed.