gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-28 17:18:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: centralize ecstore data-plane runtime sources (#3797)

Browse Source
This commit is contained in:
Zhengchao An
2026-06-23 23:23:16 +08:00
committed by GitHub
parent e59e1852ec
commit 1735dcde9c
7 changed files with 113 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/erasure_coding/encode.rs
+3 -4
View File
@@ -18,6 +18,7 @@ use crate::disk::error_reduce::{
};
use crate::erasure_coding::BitrotWriterWrapper;
use crate::erasure_coding::Erasure;
use crate::runtime_sources;
use bytes::Bytes;
use futures::StreamExt;
use futures::stream::FuturesUnordered;
@@ -138,8 +139,7 @@ impl<'a> MultiWriter<'a> {
if let Some(write_err) = reduce_write_quorum_errs(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum) {
let summary = build_write_quorum_failure_summary(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum);
let summary_text = format_write_quorum_failure(&summary);
rustfs_io_metrics::internode_metrics::global_internode_metrics()
.record_erasure_write_quorum_failure("write", quorum_dominant_error_metric_label(&summary));
runtime_sources::record_erasure_write_quorum_failure("write", quorum_dominant_error_metric_label(&summary));
error!("reduce_write_quorum_errs: {:?}, {}, errs={:?}", write_err, summary_text, self.errs);
return Err(std::io::Error::other(format!("Failed to write data: {summary_text}")));
}
@@ -193,8 +193,7 @@ impl<'a> MultiWriter<'a> {
if let Some(write_err) = reduce_write_quorum_errs(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum) {
let summary = build_write_quorum_failure_summary(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum);
let summary_text = format_write_quorum_failure(&summary);
rustfs_io_metrics::internode_metrics::global_internode_metrics()
.record_erasure_write_quorum_failure("shutdown", quorum_dominant_error_metric_label(&summary));
runtime_sources::record_erasure_write_quorum_failure("shutdown", quorum_dominant_error_metric_label(&summary));
error!(
"reduce_write_quorum_errs during shutdown: {:?}, {}, errs={:?}",
write_err, summary_text, self.errs
crates/ecstore/src/lib.rs
+1
View File
@@ -40,6 +40,7 @@ mod pools;
mod rebalance;
mod rio;
mod rpc;
mod runtime_sources;
mod set_disk;
mod sets;
mod storage_api_contracts;
crates/ecstore/src/object_api/readers.rs
+2 -2
View File
@@ -11,7 +11,7 @@ use chacha20poly1305::ChaCha20Poly1305;
#[cfg(feature = "rio-v2")]
use hmac::{Hmac, Mac};
use md5::{Digest, Md5};
use rustfs_kms::{service_manager::get_global_encryption_service, types::ObjectEncryptionContext};
use rustfs_kms::types::ObjectEncryptionContext;
use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER};
use rustfs_utils::path::path_join_buf;
#[cfg(feature = "rio-v2")]
@@ -1308,7 +1308,7 @@ async fn resolve_managed_material(bucket: &str, object: &str, metadata: &HashMap
let kms_context: Option<HashMap<String, String>> = None;
let object_context = build_object_encryption_context(bucket, object, kms_context.as_ref());
let decrypted_key = if let Some(service) = get_global_encryption_service().await {
let decrypted_key = if let Some(service) = crate::runtime_sources::object_encryption_service().await {
#[cfg(feature = "rio-v2")]
let data_key = if is_legacy_rustfs_managed_metadata(&normalized_metadata) {
service.decrypt_legacy_data_key(&encrypted_dek).await
crates/ecstore/src/runtime_sources.rs
+44
View File
@@ -0,0 +1,44 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use crate::{config::get_global_storage_class, global::get_global_deployment_id};
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use rustfs_kms::{ObjectEncryptionService, get_global_encryption_service};
pub(crate) fn record_erasure_write_quorum_failure(stage: &'static str, dominant_error: &'static str) {
global_internode_metrics().record_erasure_write_quorum_failure(stage, dominant_error);
}
pub(crate) async fn object_encryption_service() -> Option<Arc<ObjectEncryptionService>> {
get_global_encryption_service().await
}
pub(crate) fn storage_class_parity(storage_class: Option<&str>) -> Option<usize> {
get_global_storage_class().and_then(|sc| sc.get_parity_for_sc(storage_class.unwrap_or_default()))
}
pub(crate) fn storage_class_should_inline(shard_size: i64, versioned: bool) -> bool {
get_global_storage_class().is_some_and(|sc| sc.should_inline(shard_size, versioned))
}
pub(crate) fn deployment_upload_id(upload_id: &str) -> String {
base64_simd::URL_SAFE_NO_PAD
.encode_to_string(format!("{}.{}", get_global_deployment_id().unwrap_or_default(), upload_id).as_bytes())
}
pub(crate) fn global_lock_manager() -> Arc<rustfs_lock::GlobalLockManager> {
rustfs_lock::get_global_lock_manager()
}
crates/ecstore/src/set_disk.rs
+11 -31
View File
@@ -38,13 +38,14 @@ use crate::error::{GenericError, ObjectApiError, is_err_object_not_found};
use crate::global::{GLOBAL_LocalNodeName, GLOBAL_TierConfigMgr};
use crate::object_api::ObjectOptions;
use crate::rpc::heal_bucket_local_on_disks;
use crate::runtime_sources;
use crate::store_utils::is_reserved_or_invalid_bucket;
use crate::{
bucket::lifecycle::bucket_lifecycle_ops::{
LifecycleOps, gen_transition_objname, get_transitioned_object_reader, put_restore_opts,
},
cache_value::metacache_set::{ListPathRawOptions, list_path_raw},
config::{get_global_storage_class, storageclass},
config::storageclass,
disk::{
CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskOption, DiskStore, FileInfoVersions,
RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions,
@@ -53,7 +54,7 @@ use crate::{
error::{StorageError, to_object_err},
// event::name::EventName,
event_notification::{EventArgs, send_event},
global::{GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id, is_dist_erasure},
global::{GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, is_dist_erasure},
object_api::{GetObjectReader, ObjectInfo, PutObjReader},
store_init::{get_format_erasure_in_quorum, load_format_erasure, load_format_erasure_all, save_format_file},
};
@@ -396,9 +397,7 @@ fn build_tiered_decommission_file_info(
default_parity_count: usize,
storage_class: Option<&str>,
) -> (FileInfo, usize) {
let parity_drives = get_global_storage_class()
.and_then(|sc| sc.get_parity_for_sc(storage_class.unwrap_or_default()))
.unwrap_or(default_parity_count);
let parity_drives = runtime_sources::storage_class_parity(storage_class).unwrap_or(default_parity_count);
let data_drives = disk_count - parity_drives;
let mut write_quorum = data_drives;
if data_drives == parity_drives {
@@ -562,7 +561,7 @@ impl SetDisks {
set_endpoints,
disk_health_cache: Arc::new(RwLock::new(Vec::new())),
lockers,
local_lock_manager: rustfs_lock::get_global_lock_manager(),
local_lock_manager: runtime_sources::global_lock_manager(),
})
}
@@ -1028,13 +1027,7 @@ impl rustfs_storage_api::ObjectIO for SetDisks {
user_defined.insert(key.clone(), value.clone());
}
}
let sc_parity_drives = {
if let Some(sc) = get_global_storage_class() {
sc.get_parity_for_sc(user_defined.get(AMZ_STORAGE_CLASS).cloned().unwrap_or_default().as_str())
} else {
None
}
};
let sc_parity_drives = runtime_sources::storage_class_parity(user_defined.get(AMZ_STORAGE_CLASS).map(String::as_str));
let mut parity_drives = sc_parity_drives.unwrap_or(self.default_parity_count);
if opts.max_parity {
@@ -1082,13 +1075,8 @@ impl rustfs_storage_api::ObjectIO for SetDisks {
let result: Result<ObjectInfo> = async {
let erasure = erasure_coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
let is_inline_buffer = {
if let Some(sc) = get_global_storage_class() {
sc.should_inline(erasure.shard_file_size(data.size()), opts.versioned)
} else {
false
}
};
let is_inline_buffer =
runtime_sources::storage_class_should_inline(erasure.shard_file_size(data.size()), opts.versioned);
let shard_file_size = erasure.shard_file_size(data.size());
let shard_size = erasure.shard_size();
@@ -3729,8 +3717,7 @@ impl rustfs_storage_api::MultipartOperations for SetDisks {
uploads.push(MultipartInfo {
bucket: bucket.to_owned(),
object: object.to_owned(),
upload_id: base64_simd::URL_SAFE_NO_PAD
.encode_to_string(format!("{}.{}", get_global_deployment_id().unwrap_or_default(), upload_id).as_bytes()),
upload_id: runtime_sources::deployment_upload_id(&upload_id),
initiated: Some(start_time),
..Default::default()
});
@@ -3820,13 +3807,7 @@ impl rustfs_storage_api::MultipartOperations for SetDisks {
let _ = user_defined.remove(AMZ_STORAGE_CLASS);
}
let sc_parity_drives = {
if let Some(sc) = get_global_storage_class() {
sc.get_parity_for_sc(user_defined.get(AMZ_STORAGE_CLASS).cloned().unwrap_or_default().as_str())
} else {
None
}
};
let sc_parity_drives = runtime_sources::storage_class_parity(user_defined.get(AMZ_STORAGE_CLASS).map(String::as_str));
let mut parity_drives = sc_parity_drives.unwrap_or(self.default_parity_count);
if opts.max_parity {
@@ -3897,8 +3878,7 @@ impl rustfs_storage_api::MultipartOperations for SetDisks {
let upload_uuid = format!("{}x{}", Uuid::new_v4(), mod_time.unix_timestamp_nanos());
let upload_id = base64_simd::URL_SAFE_NO_PAD
.encode_to_string(format!("{}.{}", get_global_deployment_id().unwrap_or_default(), upload_uuid).as_bytes());
let upload_id = runtime_sources::deployment_upload_id(&upload_uuid);
let upload_path = Self::get_upload_id_dir(bucket, object, upload_uuid.as_str());
crates/ecstore/src/set_disk/heal.rs
+4 -7
View File
@@ -414,13 +414,10 @@ impl SetDisks {
}
}
let is_inline_buffer = {
if let Some(sc) = get_global_storage_class() {
sc.should_inline(erasure.shard_file_size(latest_meta.size), false)
} else {
false
}
};
let is_inline_buffer = runtime_sources::storage_class_should_inline(
erasure.shard_file_size(latest_meta.size),
false,
);
// create writers for all disk positions, but only for outdated disks
for (index, disk_op) in out_dated_disks.iter().enumerate() {
if let Some(outdated_disk) = disk_op {
docs/architecture/migration-progress.md
+48 -8
View File
@@ -5,11 +5,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
## Current Context
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
- Branch: `overtrue/arch-network-client-runtime-sources`
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185`.
- Based on: latest `origin/main` after PR #3795 merged API-184; this branch
batches network client runtime source boundary cleanup on top of that
baseline.
- Branch: `overtrue/arch-ecstore-data-plane-runtime-sources`
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186`.
- Based on: latest `origin/main` after PR #3796 merged API-185.
- PR type for this branch: `consumer-migration`
- Runtime behavior changes: none.
- Rust code changes: route replication pool, outbound TLS generation, runtime
@@ -18,8 +16,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
rules/event dispatch, admin OIDC/token-signing reads, IAM root credential
consumers, IAM OIDC config reads, scanner runtime-config reads, OBS metrics
runtime source reads, RIO HTTP reader TLS/metrics runtime source reads, and
gRPC/transition network client TLS/metrics runtime source reads through
AppContext-first or owner-crate resolver boundaries.
gRPC/transition network client TLS/metrics runtime source reads, plus ECStore
data-plane KMS/storage-class/deployment-id/lock-manager/erasure metric reads,
through AppContext-first or owner-crate resolver boundaries.
- CI/script changes: lock completed owner and test/fuzz boundaries against
bare/glob imports, scattered raw ECStore facade subpaths, and startup
runtime/root-server/table/S3/app shared/app bucket/app ECStore/admin facade
@@ -28,7 +27,7 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
and storage owner thin bridge regressions, plus app context and notify
event-bridge thin module regressions; accept the reviewed AppContext resolver
reverse dependencies in the layer baseline.
- Docs changes: record the API-136 through API-185 owner facade cleanup.
- Docs changes: record the API-136 through API-186 owner facade cleanup.
## Phase 0 Tasks
@@ -4685,6 +4684,22 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
client runtime source scan, Rust risk scan, branch freshness check,
pre-commit quality gate, and three-expert review.
- [x] `API-186` Centralize ECStore data-plane runtime source reads.
- Do: add an ECStore runtime-source boundary for erasure quorum failure
metrics, object-read managed KMS service lookup, storage-class parity and
inline decisions, deployment-prefixed multipart upload ids, and the local
lock-manager handle.
- Acceptance: ECStore data-plane write/read/multipart paths no longer import
those global runtime sources directly outside the owner runtime-source
module.
- Must preserve: erasure quorum metric stage/error labels, managed-KMS
decrypt fallback behavior, storage-class parity and inline decisions,
multipart upload id encoding, and local lock-manager initialization.
- Verification: ECStore compile coverage, focused unit tests, formatting,
migration guard, layer guard, diff hygiene, residual data-plane runtime
source scan, Rust risk scan, branch freshness check, pre-commit quality
gate, and three-expert review.
## Next PRs
1. `consumer-migration`: continue reducing direct global reads behind AppContext resolver boundaries.
@@ -4796,11 +4811,36 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
| Quality/architecture | pass | API-185 keeps protos and ECStore network client TLS/metrics globals behind owner runtime-source modules without widening public APIs. |
| Migration preservation | pass | gRPC TLS cache invalidation, transition-client TLS generation reporting, dial metrics, retry metrics, and read/write byte/error counters keep existing behavior. |
| Testing/verification | pass | Protos/ECStore compile/tests, formatting, residual network client runtime source scan, targeted guard checks, and pre-commit passed for API-185. |
| Quality/architecture | pass | API-186 keeps ECStore data-plane runtime globals behind an ECStore-owned runtime-source module without widening public APIs. |
| Migration preservation | pass | Erasure quorum metric labels, managed-KMS fallback, storage-class decisions, multipart upload id encoding, and lock-manager initialization keep existing behavior. |
| Testing/verification | pass | ECStore compile/focused tests, formatting, residual data-plane runtime source scan, targeted guard checks, and pre-commit passed for API-186. |
## Verification Notes
Passed before push:
- Issue #660 API-186 current slice:
- `cargo check -p rustfs-ecstore --tests`: passed.
- `cargo test -p rustfs-ecstore --lib erasure_coding -- --test-threads=1`:
passed.
- `cargo test -p rustfs-ecstore --lib set_disk -- --test-threads=1`:
passed.
- `cargo test -p rustfs-ecstore --lib readers -- --test-threads=1`:
passed.
- `cargo fmt --all`: passed.
- `cargo fmt --all --check`: passed.
- `git diff --check`: passed.
- `./scripts/check_architecture_migration_rules.sh`: passed.
- `./scripts/check_layer_dependencies.sh`: passed.
- ECStore data-plane runtime source scan: passed; direct storage-class,
deployment id, KMS encryption service, lock-manager, and erasure internode
metric global reads are isolated to the ECStore runtime source module.
- Rust risk scan: passed; diff adds no new `expect`, `panic`, `todo`,
`unimplemented`, or `unsafe`.
- Branch freshness check: rebased onto latest `origin/main` after PR #3796
merged API-185.
- `make pre-commit`: passed.
- Issue #660 API-185 current slice:
- `cargo check -p rustfs-protos -p rustfs-ecstore --tests`: passed.
- `cargo test -p rustfs-protos --lib`: passed.