refactor: add object store resolver for standalone crates (#3437)

This commit is contained in:
安正超
2026-06-14 18:02:39 +08:00
committed by GitHub
parent 59f99a30e2
commit 84f7027e90
12 changed files with 105 additions and 67 deletions
+15
View File
@@ -171,6 +171,21 @@ pub fn new_object_layer_fn() -> Option<Arc<ECStore>> {
GLOBAL_OBJECT_API.get().cloned()
}
pub type ObjectStoreResolver = dyn Fn() -> Option<Arc<ECStore>> + Send + Sync + 'static;
static GLOBAL_OBJECT_STORE_RESOLVER: OnceLock<Arc<ObjectStoreResolver>> = OnceLock::new();
pub fn set_object_store_resolver(resolver: Arc<ObjectStoreResolver>) -> bool {
GLOBAL_OBJECT_STORE_RESOLVER.set(resolver).is_ok()
}
pub fn resolve_object_store_handle() -> Option<Arc<ECStore>> {
GLOBAL_OBJECT_STORE_RESOLVER
.get()
.and_then(|resolver| resolver())
.or_else(new_object_layer_fn)
}
/// Set the global object layer
///
/// # Arguments
+1 -1
View File
@@ -54,10 +54,10 @@ mod pools_test;
mod store_test;
pub mod tier;
pub use global::new_object_layer_fn;
pub use global::set_global_endpoints;
pub use global::update_erasure_type;
pub use global::{get_global_lock_client, get_global_lock_clients, set_global_lock_client, set_global_lock_clients};
pub use global::{new_object_layer_fn, resolve_object_store_handle, set_object_store_resolver};
pub use global::GLOBAL_Endpoints;
pub use store_api::StorageAPI;
+1 -1
View File
@@ -316,7 +316,7 @@ impl NotifyConfigManager {
where
F: FnMut(&mut Config) -> bool,
{
let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
let Some(store) = rustfs_ecstore::global::resolve_object_store_handle() else {
return Err(NotificationError::StorageNotAvailable(
"Failed to save target configuration: server storage not initialized".to_string(),
));
+7 -7
View File
@@ -34,8 +34,8 @@ use rustfs_ecstore::bucket::metadata_sys::get_quota_config;
use rustfs_ecstore::bucket::replication::GLOBAL_REPLICATION_STATS;
use rustfs_ecstore::data_usage::load_data_usage_from_backend;
use rustfs_ecstore::global::get_global_bucket_monitor;
use rustfs_ecstore::new_object_layer_fn;
use rustfs_ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free};
use rustfs_ecstore::resolve_object_store_handle;
use rustfs_ecstore::store_api::BucketOperations;
use rustfs_iam::{get_global_iam_sys, oidc::oidc_plugin_authn_metrics_snapshot};
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
@@ -152,7 +152,7 @@ pub struct ProcessMetricBundle {
/// Collect cluster and cluster-health statistics from a single storage snapshot.
pub async fn collect_cluster_and_health_stats() -> (ClusterStats, ClusterHealthStats) {
let Some(store) = new_object_layer_fn() else {
let Some(store) = resolve_object_store_handle() else {
return (ClusterStats::default(), ClusterHealthStats::default());
};
@@ -242,7 +242,7 @@ pub async fn collect_cluster_health_stats() -> ClusterHealthStats {
/// Collect bucket statistics from the storage layer.
pub async fn collect_bucket_stats() -> Vec<BucketStats> {
let Some(store) = new_object_layer_fn() else {
let Some(store) = resolve_object_store_handle() else {
return Vec::new();
};
@@ -523,7 +523,7 @@ pub fn collect_system_memory_stats() -> MemoryStats {
/// Collect node disk stats and drive stats from a single storage snapshot.
pub async fn collect_disk_and_system_drive_stats() -> (Vec<DiskStats>, Vec<DriveDetailedStats>, DriveCountStats) {
let Some(store) = new_object_layer_fn() else {
let Some(store) = resolve_object_store_handle() else {
return (Vec::new(), Vec::new(), DriveCountStats::default());
};
@@ -711,7 +711,7 @@ pub fn collect_internode_network_stats() -> Option<NetworkStats> {
/// Collect cluster config metrics from backend parity configuration.
pub async fn collect_cluster_config_stats() -> Option<ClusterConfigStats> {
let store = new_object_layer_fn()?;
let store = resolve_object_store_handle()?;
let backend = StorageAdminApi::backend_info(store.as_ref()).await;
Some(ClusterConfigStats {
@@ -722,7 +722,7 @@ pub async fn collect_cluster_config_stats() -> Option<ClusterConfigStats> {
/// Collect cluster erasure set metrics from storage and backend topology info.
pub async fn collect_erasure_set_stats() -> Vec<ErasureSetStats> {
let Some(store) = new_object_layer_fn() else {
let Some(store) = resolve_object_store_handle() else {
return Vec::new();
};
@@ -801,7 +801,7 @@ pub async fn collect_iam_stats() -> Option<IamStats> {
/// builds cluster totals plus per-bucket distributions from the returned
/// histograms. It does not trigger an inline object-data rescan.
pub async fn collect_cluster_usage_metric_stats() -> Option<(ClusterUsageStats, Vec<BucketUsageStats>)> {
let store = new_object_layer_fn()?;
let store = resolve_object_store_handle()?;
let data_usage = load_data_usage_from_backend(store.clone()).await.ok()?;
let mut buckets = Vec::with_capacity(data_usage.buckets_usage.len());
+2 -2
View File
@@ -16,7 +16,7 @@
use super::{SwiftError, SwiftResult};
use rustfs_credentials::Credentials;
use rustfs_ecstore::new_object_layer_fn;
use rustfs_ecstore::resolve_object_store_handle;
use rustfs_ecstore::store_api::BucketOperations;
use rustfs_storage_api::MakeBucketOptions;
use s3s::dto::{Tag, Tagging};
@@ -160,7 +160,7 @@ pub async fn update_account_metadata(
) -> SwiftResult<()> {
let bucket_name = get_account_metadata_bucket_name(account);
let Some(store) = new_object_layer_fn() else {
let Some(store) = resolve_object_store_handle() else {
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
};
+10 -10
View File
@@ -20,7 +20,7 @@ use super::account::validate_account_access;
use super::types::Container;
use super::{SwiftError, SwiftResult};
use rustfs_credentials::Credentials;
use rustfs_ecstore::new_object_layer_fn;
use rustfs_ecstore::resolve_object_store_handle;
use rustfs_ecstore::store_api::{BucketOperations, ListOperations};
use rustfs_storage_api::{BucketInfo, BucketOptions, DeleteBucketOptions, MakeBucketOptions};
use s3s::dto::{Tag, Tagging};
@@ -242,7 +242,7 @@ pub async fn list_containers(account: &str, credentials: &Credentials) -> SwiftR
let mapper = ContainerMapper::default();
// Get storage layer
let Some(store) = new_object_layer_fn() else {
let Some(store) = resolve_object_store_handle() else {
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
};
@@ -300,7 +300,7 @@ pub async fn create_container(account: &str, container: &str, credentials: &Cred
let bucket_name = mapper.swift_to_s3_bucket(container, &project_id);
// Get storage layer
let Some(store) = new_object_layer_fn() else {
let Some(store) = resolve_object_store_handle() else {
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
};
@@ -394,7 +394,7 @@ pub async fn get_container_metadata(account: &str, container: &str, credentials:
let bucket_name = mapper.swift_to_s3_bucket(container, &project_id);
// Get storage layer
let Some(store) = new_object_layer_fn() else {
let Some(store) = resolve_object_store_handle() else {
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
};
@@ -471,7 +471,7 @@ pub async fn update_container_metadata(
let bucket_name = mapper.swift_to_s3_bucket(container, &project_id);
// Get storage layer
let Some(store) = new_object_layer_fn() else {
let Some(store) = resolve_object_store_handle() else {
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
};
@@ -570,7 +570,7 @@ pub async fn delete_container(account: &str, container: &str, credentials: &Cred
let bucket_name = mapper.swift_to_s3_bucket(container, &project_id);
// Get storage layer
let Some(store) = new_object_layer_fn() else {
let Some(store) = resolve_object_store_handle() else {
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
};
@@ -658,7 +658,7 @@ pub async fn list_objects(
let bucket = mapper.swift_to_s3_bucket(container, &project_id);
// Get storage layer
let Some(store) = new_object_layer_fn() else {
let Some(store) = resolve_object_store_handle() else {
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
};
@@ -768,7 +768,7 @@ pub async fn enable_versioning(
let archive_bucket_name = mapper.swift_to_s3_bucket(archive_container, &project_id);
// Get storage layer
let Some(store) = new_object_layer_fn() else {
let Some(store) = resolve_object_store_handle() else {
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
};
@@ -864,7 +864,7 @@ pub async fn disable_versioning(account: &str, container: &str, credentials: &Cr
let bucket_name = mapper.swift_to_s3_bucket(container, &project_id);
// Verify container exists
let Some(_store) = new_object_layer_fn() else {
let Some(_store) = resolve_object_store_handle() else {
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
};
@@ -1015,7 +1015,7 @@ pub async fn set_container_acl(
let bucket_name = mapper.swift_to_s3_bucket(container, &project_id);
// Get storage layer
let Some(store) = new_object_layer_fn() else {
let Some(store) = resolve_object_store_handle() else {
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
};
+8 -8
View File
@@ -54,7 +54,7 @@ use super::container::ContainerMapper;
use super::{SwiftError, SwiftResult};
use axum::http::HeaderMap;
use rustfs_credentials::Credentials;
use rustfs_ecstore::new_object_layer_fn;
use rustfs_ecstore::resolve_object_store_handle;
use rustfs_ecstore::store_api::{BucketOperations, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader};
use rustfs_rio::HashReader;
use rustfs_storage_api::BucketOptions;
@@ -375,7 +375,7 @@ where
}
// 12. Get storage layer
let Some(store) = new_object_layer_fn() else {
let Some(store) = resolve_object_store_handle() else {
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
};
@@ -455,7 +455,7 @@ where
validate_metadata(metadata)?;
// Get storage layer
let Some(store) = new_object_layer_fn() else {
let Some(store) = resolve_object_store_handle() else {
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
};
@@ -543,7 +543,7 @@ pub async fn get_object(
let bucket = mapper.swift_to_s3_bucket(container, &project_id);
// 5. Get storage layer
let Some(store) = new_object_layer_fn() else {
let Some(store) = resolve_object_store_handle() else {
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
};
@@ -603,7 +603,7 @@ pub async fn head_object(
let bucket = mapper.swift_to_s3_bucket(container, &project_id);
// 5. Get storage layer
let Some(store) = new_object_layer_fn() else {
let Some(store) = resolve_object_store_handle() else {
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
};
@@ -661,7 +661,7 @@ pub async fn delete_object(account: &str, container: &str, object: &str, credent
let bucket = mapper.swift_to_s3_bucket(container, &project_id);
// 5. Get storage layer
let Some(store) = new_object_layer_fn() else {
let Some(store) = resolve_object_store_handle() else {
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
};
@@ -724,7 +724,7 @@ pub async fn update_object_metadata(
let bucket = mapper.swift_to_s3_bucket(container, &project_id);
// 5. Get storage layer
let Some(store) = new_object_layer_fn() else {
let Some(store) = resolve_object_store_handle() else {
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
};
@@ -848,7 +848,7 @@ pub async fn copy_object(
let dst_bucket = mapper.swift_to_s3_bucket(dst_container, &dst_project_id);
// 6. Get storage layer
let Some(store) = new_object_layer_fn() else {
let Some(store) = resolve_object_store_handle() else {
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
};
+4 -4
View File
@@ -56,7 +56,7 @@ use super::container::ContainerMapper;
use super::object::{ObjectKeyMapper, head_object};
use super::{SwiftError, SwiftResult};
use rustfs_credentials::Credentials;
use rustfs_ecstore::new_object_layer_fn;
use rustfs_ecstore::resolve_object_store_handle;
use rustfs_ecstore::store_api::{ListOperations, ObjectOperations, ObjectOptions};
use std::time::{SystemTime, UNIX_EPOCH};
use tracing::{debug, error};
@@ -197,7 +197,7 @@ pub async fn archive_current_version(
let version_key = ObjectKeyMapper::swift_to_s3_key(&version_name)?;
// Get storage layer
let Some(store) = new_object_layer_fn() else {
let Some(store) = resolve_object_store_handle() else {
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
};
@@ -340,7 +340,7 @@ pub async fn restore_previous_version(
let version_key = ObjectKeyMapper::swift_to_s3_key(newest_version)?;
// Get storage layer
let Some(store) = new_object_layer_fn() else {
let Some(store) = resolve_object_store_handle() else {
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
};
@@ -476,7 +476,7 @@ pub async fn list_object_versions(
let archive_bucket = mapper.swift_to_s3_bucket(archive_container, &project_id);
// Get storage layer
let Some(store) = new_object_layer_fn() else {
let Some(store) = resolve_object_store_handle() else {
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
};
+2 -2
View File
@@ -26,7 +26,7 @@ use object_store::{
use pin_project_lite::pin_project;
use rustfs_common::DEFAULT_DELIMITER;
use rustfs_ecstore::error::{StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found};
use rustfs_ecstore::new_object_layer_fn;
use rustfs_ecstore::resolve_object_store_handle;
use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
use rustfs_ecstore::store::ECStore;
use rustfs_ecstore::store_api::{GetObjectReader, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectOptions};
@@ -107,7 +107,7 @@ pub struct InvalidScanRange;
impl EcObjectStore {
pub fn new(input: Arc<SelectObjectContentInput>) -> S3Result<Self> {
let Some(store) = new_object_layer_fn() else {
let Some(store) = resolve_object_store_handle() else {
return Err(s3_error!(InternalError, "ec store not inited"));
};
+2 -2
View File
@@ -39,7 +39,7 @@ use rustfs_ecstore::disk::error::DiskError;
use rustfs_ecstore::disk::{Disk, DiskAPI};
use rustfs_ecstore::error::{Error, StorageError};
use rustfs_ecstore::global::GLOBAL_TierConfigMgr;
use rustfs_ecstore::new_object_layer_fn;
use rustfs_ecstore::resolve_object_store_handle;
use rustfs_ecstore::set_disk::SetDisks;
use rustfs_ecstore::store_api::{BucketOperations, ObjectIO, ObjectInfo};
use rustfs_ecstore::{error::Result, store::ECStore};
@@ -1333,7 +1333,7 @@ impl ScannerIODisk for Disk {
// TODO: object lock
let Some(ecstore) = new_object_layer_fn() else {
let Some(ecstore) = resolve_object_store_handle() else {
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
+48 -28
View File
@@ -5,17 +5,19 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
## Current Context
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
- Branch: `overtrue/arch-admin-zip-object-store-context`
- Baseline: `upstream/main` at `5e68cf8a296f0453fd050d137830032329685d12`
- Branch: `overtrue/arch-object-store-resolver-crates`
- Baseline: `upstream/main` at `59f99a30e2dee0794658cb1a2793cf48ed62e5ef`
- PR type for this branch: `consumer-migration`
- Runtime behavior changes: no external behavior change expected; admin object
ZIP download object-store lookups prefer the AppContext-owned object store
and keep the existing global fallback.
- Rust code changes: migrate rustfs/src/admin/handlers/object_zip_download.rs
object-store lookups to the shared resolver without touching infra-layer
server/storage modules or standalone crates.
- Runtime behavior changes: no external behavior change expected; standalone
protocol, observability, scanner, notify, and S3 Select object-store lookups
prefer the AppContext-owned object store when the application context is
installed and keep the existing global fallback.
- Rust code changes: add an ECStore-owned object-store resolver hook, install
it from AppContext initialization, and migrate the standalone crate consumer
group without touching server/storage infra modules or ECStore internal
background consumers.
- CI/script changes: none.
- Docs changes: record `CTX-007` consumer migration scope and verification.
- Docs changes: record `CTX-008` consumer migration scope and verification.
## Phase 0 Tasks
@@ -192,6 +194,18 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
and streaming behavior, and existing storage error paths.
- Verification: formatting, compile check, migration guards, diff hygiene,
Rust risk scan, and full `make pre-commit`.
- [x] `CTX-008` Migrate standalone crate object-store consumers.
- Do: add an ECStore-owned resolver hook for AppContext-first object-store
lookup and migrate Swift, S3 Select, scanner, notify, and observability
object-store consumers to that resolver.
- Acceptance: standalone crates can prefer the AppContext-owned object store
without depending on the `rustfs` application crate and preserve the
existing global object-layer fallback.
- Must preserve: Swift protocol behavior, S3 Select object reads, scanner
cache/scan behavior, notification config persistence, observability stats
collection, and existing storage error paths.
- Verification: formatting, compile checks, migration guards, diff hygiene,
Rust risk scan, and full `make pre-commit`.
## Phase 1 Security Governance Tasks
@@ -498,9 +512,8 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
## Next PRs
1. `consumer-migration`: design the crate-local object-store resolver path for
crates/protocols, crates/obs, crates/scanner, and crates/notify without
introducing reverse dependencies on rustfs.
1. `consumer-migration`: migrate the remaining server/storage infra object-store
consumers after confirming the ECStore-owned resolver is the right boundary.
2. `pure-move`: start `R-009` boot wrapper with the IAM degraded readiness
contract covered.
@@ -508,37 +521,44 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
| Expert | Status | Notes |
|---|---|---|
| Quality/architecture | pass | Single `consumer-migration` slice; admin ZIP code may depend on app context, and the layer guard reports no new reverse dependencies. |
| Migration preservation | pass | AppContext object-store resolution is preferred when present, while existing global object-layer fallback and ZIP error paths are preserved. |
| Testing/verification | pass | Formatting, compile check, diff hygiene, migration/layer guards, added-line Rust risk scan, and full `make pre-commit` passed. |
| Quality/architecture | passed | Single consumer-migration slice; ECStore owns the resolver hook, standalone crates do not depend on `rustfs`, and the layer guard reports no new reverse dependencies. |
| Migration preservation | passed | AppContext object-store resolution is preferred after context initialization, while the old global object-layer fallback and existing storage error paths are preserved. |
| Testing/verification | passed | Formatting, compile checks, Swift feature check, diff hygiene, migration/layer guards, added-line Rust risk scan, and full `make pre-commit` passed. |
## Verification Notes
Passed on `5e68cf8a296f0453fd050d137830032329685d12`:
Passed on `59f99a30e2dee0794658cb1a2793cf48ed62e5ef`:
- `cargo fmt --all`.
- `cargo fmt --all --check`.
- `cargo check -p rustfs-ecstore -p rustfs-s3select-api -p rustfs-scanner -p rustfs-notify -p rustfs-obs`.
- `cargo check -p rustfs-protocols --features swift`.
- `cargo check -p rustfs --lib`.
- `git diff --check`.
- `./scripts/check_architecture_migration_rules.sh`.
- `./scripts/check_layer_dependencies.sh`.
- Rust risk scan for changed Rust files; full-file matches were existing
tests and S3 result signatures, and the added-line scan returned no
- Rust risk scan for changed Rust files; full-file matches were existing tests,
numeric casts, and relaxed counters, and the added-line scan returned no
`unwrap`/`expect`, numeric cast, string error, boxed error, print macro, or
relaxed-ordering match.
- `make pre-commit`; all checks passed, including nextest 5939 passed and
111 skipped, plus doctests.
- `make pre-commit`; all checks passed, including nextest 5941 passed and 111
skipped, plus doctests.
Notes:
- This slice migrates the admin object ZIP download object-store consumer
group.
- Server/storage infra consumers still need a crate-local resolver path before
migration.
- This slice adds an ECStore-owned object-store resolver hook for standalone
crates.
- Swift, S3 Select, scanner, notify, and observability consumers now use the
resolver without depending on the `rustfs` application crate.
- Server/storage infra consumers and ECStore internal background consumers
remain on the old global accessor for a follow-up slice.
- Global object-layer fallback remains available until the planned cleanup
phase.
## Handoff Notes
- CTX-007 is complete.
- Keep infra-layer server/storage consumers on the global accessor until a
crate-local resolver path is designed; preserve global fallback until the
planned cleanup phase.
- CTX-008 is complete.
- Remaining server/storage infra consumers can migrate to the ECStore-owned
resolver in the next consumer-migration slice if the layer guard still passes.
- ECStore internal background consumers should be reviewed separately before
changing their global accessor behavior.
+5 -2
View File
@@ -21,7 +21,7 @@ use super::interfaces::{
BucketMetadataInterface, BufferConfigInterface, EndpointsInterface, IamInterface, KmsInterface, KmsRuntimeInterface,
NotifyInterface, RegionInterface, ServerConfigInterface, TierConfigInterface,
};
use rustfs_ecstore::store::ECStore;
use rustfs_ecstore::{set_object_store_resolver, store::ECStore};
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
use rustfs_kms::KmsServiceManager;
use std::sync::{Arc, OnceLock};
@@ -151,7 +151,10 @@ static APP_CONTEXT_SINGLETON: OnceLock<Arc<AppContext>> = OnceLock::new();
/// Initialize global application context once and return the canonical instance.
pub fn init_global_app_context(context: AppContext) -> Arc<AppContext> {
APP_CONTEXT_SINGLETON.get_or_init(|| Arc::new(context)).clone()
let context = APP_CONTEXT_SINGLETON.get_or_init(|| Arc::new(context)).clone();
let resolver_context = context.clone();
let _ = set_object_store_resolver(Arc::new(move || Some(resolver_context.object_store())));
context
}
/// Get global application context if it has been initialized.