refactor(startup): thread explicit InstanceContext through the storage startup path (#4611)

This commit is contained in:
Zhengchao An
2026-07-09 16:53:41 +08:00
committed by GitHub
parent 8bcffd8a04
commit 91a5c87132
17 changed files with 526 additions and 140 deletions
+4 -2
View File
@@ -326,6 +326,7 @@ pub mod global {
}
pub mod runtime {
pub use crate::runtime::instance::{InstanceContext, bootstrap_ctx};
pub use crate::runtime::sources::{
boot_time, bucket_monitor, deployment_id, endpoint_pools, expiry_state_handle, first_cluster_node_is_local,
global_lock_client, global_lock_clients, global_tier_config_mgr, local_disk_map_read, object_store_handle, region,
@@ -389,8 +390,9 @@ pub mod store_list {
pub mod storage {
pub use crate::store::HealWalkVersion;
pub use crate::store::{
ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks, init_lock_clients,
prewarm_local_disk_id_map,
ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
prewarm_local_disk_id_map_with_instance_ctx,
};
}
+81 -7
View File
@@ -102,6 +102,21 @@ impl Sets {
fm: &FormatV3,
pool_idx: usize,
parity_count: usize,
) -> Result<Arc<Self>> {
Self::new_with_instance_ctx(disks, endpoints, fm, pool_idx, parity_count, bootstrap_ctx()).await
}
/// Build the pool's sets bound to an explicit instance context (Phase 5
/// follow-up, backlog#1052). The legacy [`Sets::new`] entry adopts the
/// process bootstrap context; a store constructed around its own context
/// passes it here so the whole object graph shares one cell.
pub async fn new_with_instance_ctx(
disks: Vec<Option<DiskStore>>,
endpoints: &PoolEndpoints,
fm: &FormatV3,
pool_idx: usize,
parity_count: usize,
instance_ctx: Arc<InstanceContext>,
) -> Result<Arc<Self>> {
let set_count = fm.erasure.sets.len();
let set_drive_count = fm.erasure.sets[0].len();
@@ -127,8 +142,8 @@ impl Sets {
continue;
}
if disk.as_ref().unwrap().is_local() && runtime_sources::setup_is_dist_erasure().await {
let local_disk = runtime_sources::local_disk_set_drive(pool_idx, i, j).await;
if disk.as_ref().unwrap().is_local() && instance_ctx.is_dist_erasure().await {
let local_disk = runtime_sources::local_disk_set_drive(&instance_ctx, pool_idx, i, j).await;
if local_disk.is_none() {
warn!("sets new set_drive {}-{} local_disk is none", i, j);
@@ -163,7 +178,7 @@ impl Sets {
.as_ref()
.map(|registry| registry.clients_for_endpoints(&set_endpoints))
.unwrap_or_default();
let set_disks = SetDisks::new(
let set_disks = SetDisks::new_with_instance_ctx(
runtime_sources::local_node_name().await,
Arc::new(RwLock::new(set_drive)),
set_drive_count,
@@ -173,6 +188,7 @@ impl Sets {
set_endpoints,
fm.clone(),
lockers,
instance_ctx.clone(),
)
.await;
@@ -193,10 +209,7 @@ impl Sets {
default_parity_count: parity_count,
distribution_algo: fm.erasure.distribution_algo.clone(),
exit_signal: Some(tx),
// Single-instance: same bootstrap context the owning ECStore adopts
// (constructed before the store, so sourced here directly). Slice 8
// threads a per-instance context in for true multi-instance.
ctx: bootstrap_ctx(),
ctx: instance_ctx,
});
let asets = sets.clone();
@@ -1462,4 +1475,65 @@ mod tests {
"unformatted disk must be Missing on its real index, not on a placeholder"
);
}
fn instance_ctx_test_pool_endpoints() -> (FormatV3, PoolEndpoints) {
let format = FormatV3::new(1, 2);
let endpoints = vec![
Endpoint::try_from("http://127.0.0.1:9000/data0").expect("first endpoint should parse"),
Endpoint::try_from("http://127.0.0.1:9001/data1").expect("second endpoint should parse"),
];
let pool_endpoints = PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 2,
endpoints: Endpoints::from(endpoints),
cmd_line: "instance-ctx-adoption-test".to_string(),
platform: "test".to_string(),
};
(format, pool_endpoints)
}
// Phase 5 follow-up (backlog#1052): a pool built through the ctx-explicit
// constructor carries the caller's context through Sets AND every SetDisks,
// so nothing in the object graph silently binds to the process bootstrap.
#[tokio::test]
async fn sets_new_with_instance_ctx_threads_context_through_graph() {
let (format, pool_endpoints) = instance_ctx_test_pool_endpoints();
let instance_ctx = Arc::new(InstanceContext::new());
let sets = Sets::new_with_instance_ctx(vec![None, None], &pool_endpoints, &format, 0, 1, instance_ctx.clone())
.await
.expect("sets should build with empty disks");
assert!(
Arc::ptr_eq(sets.instance_ctx(), &instance_ctx),
"Sets must adopt the explicitly passed instance context"
);
for set_disks in &sets.disk_set {
assert!(
Arc::ptr_eq(set_disks.instance_ctx(), &instance_ctx),
"every SetDisks must adopt the explicitly passed instance context"
);
}
assert!(
!Arc::ptr_eq(sets.instance_ctx(), &bootstrap_ctx()),
"a fresh context must not alias the process bootstrap context"
);
}
// The legacy constructor keeps single-instance behavior byte-for-byte: it
// still adopts the process bootstrap context.
#[tokio::test]
async fn sets_new_legacy_adopts_bootstrap_context() {
let (format, pool_endpoints) = instance_ctx_test_pool_endpoints();
let sets = Sets::new(vec![None, None], &pool_endpoints, &format, 0, 1)
.await
.expect("sets should build with empty disks");
assert!(
Arc::ptr_eq(sets.instance_ctx(), &bootstrap_ctx()),
"legacy Sets::new must keep adopting the process bootstrap context"
);
}
}
+21 -17
View File
@@ -20,6 +20,7 @@ use std::{
use crate::bucket::bandwidth::monitor::Monitor;
use crate::disk::endpoint::Endpoint;
use crate::runtime::instance::InstanceContext;
use crate::{
bucket::lifecycle::bucket_lifecycle_ops::{ExpiryState, TransitionState},
bucket::metadata_sys::{BucketMetadataSys, get_global_bucket_metadata_sys},
@@ -33,8 +34,8 @@ use crate::{
TypeLocalDiskSetDrives, get_background_services_cancel_token, get_global_bucket_monitor, get_global_deployment_id,
get_global_endpoints, get_global_endpoints_opt, get_global_lock_client, get_global_lock_clients, get_global_region,
get_global_tier_config_mgr, global_rustfs_port, init_global_bucket_monitor, is_dist_erasure, is_erasure, is_erasure_sd,
is_first_cluster_node_local, resolve_object_store_handle, set_global_deployment_id, set_global_lock_client,
set_global_lock_clients, set_object_layer, update_erasure_type,
is_first_cluster_node_local, resolve_object_store_handle, set_global_lock_client, set_global_lock_clients,
set_object_layer, update_erasure_type,
},
services::batch_processor::{GlobalBatchProcessors, get_global_processors},
services::event_notification::EventNotifier,
@@ -278,12 +279,6 @@ pub(crate) fn replication_runtime_initialized() -> bool {
crate::runtime::global::current_ctx().replication_initialized()
}
pub(crate) fn ensure_deployment_id(deployment_id: Uuid) {
if get_global_deployment_id().is_none() {
set_global_deployment_id(deployment_id);
}
}
pub fn global_lock_client() -> Option<Arc<dyn LockClient>> {
get_global_lock_client()
}
@@ -419,8 +414,8 @@ pub(crate) async fn clear_local_disk_id_map_for_test() {
local_disk_id_map_handle().write().await.clear();
}
pub(crate) async fn record_local_disk_id(disk_id: Uuid, endpoint: String) {
local_disk_id_map_handle().write().await.insert(disk_id, endpoint);
pub(crate) async fn record_local_disk_id(instance_ctx: &Arc<InstanceContext>, disk_id: Uuid, endpoint: String) {
instance_ctx.local_disk_id_map().write().await.insert(disk_id, endpoint);
}
pub(crate) async fn replace_local_disk_id(previous: Option<Uuid>, current: Option<Uuid>, endpoint: String) {
@@ -434,8 +429,8 @@ pub(crate) async fn replace_local_disk_id(previous: Option<Uuid>, current: Optio
}
}
pub(crate) async fn record_local_disks(disks: Vec<DiskStore>) {
let map = local_disk_map_handle();
pub(crate) async fn record_local_disks(instance_ctx: &Arc<InstanceContext>, disks: Vec<DiskStore>) {
let map = instance_ctx.local_disk_map();
let mut global_local_disk_map = map.write().await;
for disk in disks {
let path = disk.endpoint().to_string();
@@ -443,8 +438,13 @@ pub(crate) async fn record_local_disks(disks: Vec<DiskStore>) {
}
}
pub(crate) async fn local_disk_set_drive(pool_idx: usize, set_idx: usize, disk_idx: usize) -> Option<DiskStore> {
local_disk_set_drives_handle().read().await[pool_idx][set_idx][disk_idx].clone()
pub(crate) async fn local_disk_set_drive(
instance_ctx: &Arc<InstanceContext>,
pool_idx: usize,
set_idx: usize,
disk_idx: usize,
) -> Option<DiskStore> {
instance_ctx.local_disk_set_drives().read().await[pool_idx][set_idx][disk_idx].clone()
}
pub(crate) async fn local_disk_for_endpoint(endpoint: &Endpoint) -> Option<DiskStore> {
@@ -488,8 +488,12 @@ pub(crate) async fn local_disk_entries() -> Vec<Option<DiskStore>> {
local_disk_map_handle().read().await.values().cloned().collect()
}
pub(crate) async fn initialize_local_disk_maps(endpoint_pools: EndpointServerPools, opt: &DiskOption) -> Result<()> {
let set_drives = local_disk_set_drives_handle();
pub(crate) async fn initialize_local_disk_maps(
instance_ctx: &Arc<InstanceContext>,
endpoint_pools: EndpointServerPools,
opt: &DiskOption,
) -> Result<()> {
let set_drives = instance_ctx.local_disk_set_drives();
let mut global_set_drives = set_drives.write().await;
for pool_eps in endpoint_pools.as_ref().iter() {
let mut set_count_drives = Vec::with_capacity(pool_eps.set_count);
@@ -500,7 +504,7 @@ pub(crate) async fn initialize_local_disk_maps(endpoint_pools: EndpointServerPoo
global_set_drives.push(set_count_drives);
}
let map = local_disk_map_handle();
let map = instance_ctx.local_disk_map();
let mut global_local_disk_map = map.write().await;
for pool_eps in endpoint_pools.as_ref().iter() {
+33 -4
View File
@@ -1744,10 +1744,39 @@ impl SetDisks {
format: FormatV3,
lockers: Vec<Arc<dyn LockClient>>,
) -> Arc<Self> {
// Single-instance sources the process bootstrap context (the one the
// owning ECStore adopts). Slice 8 threads a per-instance context in for
// true multi-instance.
let ctx = bootstrap_ctx();
Self::new_with_instance_ctx(
locker_owner,
disks,
set_drive_count,
default_parity_count,
set_index,
pool_index,
set_endpoints,
format,
lockers,
bootstrap_ctx(),
)
.await
}
/// Build a set bound to an explicit instance context (Phase 5 follow-up,
/// backlog#1052). The legacy [`SetDisks::new`] entry adopts the process
/// bootstrap context; a store constructed around its own context threads it
/// down here so the whole object graph shares one cell.
#[allow(clippy::too_many_arguments)]
pub async fn new_with_instance_ctx(
locker_owner: String,
disks: Arc<RwLock<Vec<Option<DiskStore>>>>,
set_drive_count: usize,
default_parity_count: usize,
set_index: usize,
pool_index: usize,
set_endpoints: Vec<Endpoint>,
format: FormatV3,
lockers: Vec<Arc<dyn LockClient>>,
instance_ctx: Arc<InstanceContext>,
) -> Arc<Self> {
let ctx = instance_ctx;
Arc::new(SetDisks {
locker_owner,
disks,
+112 -9
View File
@@ -15,6 +15,7 @@
use super::*;
use crate::core::pools::local_decommission_queue_prefix;
use crate::error::is_err_decommission_running;
use crate::runtime::instance::InstanceContext;
use crate::runtime::sources as runtime_sources;
use crate::storage_api_contracts::object::EcstoreObjectIO;
use tracing::{debug, error, info, warn};
@@ -165,6 +166,24 @@ impl ECStore {
#[allow(clippy::new_ret_no_self)]
#[instrument(level = "debug", skip(endpoint_pools))]
pub async fn new(address: SocketAddr, endpoint_pools: EndpointServerPools, ctx: CancellationToken) -> Result<Arc<Self>> {
Self::new_with_instance_ctx(address, endpoint_pools, ctx, crate::runtime::instance::bootstrap_ctx()).await
}
/// Build a store around an explicit instance context (Phase 5 follow-up,
/// backlog#1052). The legacy [`ECStore::new`] entry adopts the process
/// bootstrap context, keeping single-instance startup byte-for-byte
/// unchanged; a caller that owns its own context (a future second embedded
/// server) passes it here so every construction-time write — pool sets,
/// local-disk registry, deployment id — lands on that context instead of
/// the shared bootstrap one.
#[allow(clippy::new_ret_no_self)]
#[instrument(level = "debug", skip(endpoint_pools, instance_ctx))]
pub async fn new_with_instance_ctx(
address: SocketAddr,
endpoint_pools: EndpointServerPools,
ctx: CancellationToken,
instance_ctx: Arc<InstanceContext>,
) -> Result<Arc<Self>> {
// let layouts = DisksLayout::from_volumes(endpoints.as_slice())?;
let mut deployment_id = None;
@@ -308,15 +327,16 @@ impl ECStore {
}
}
let sets = Sets::new(disks.clone(), pool_eps, &fm, i, common_parity_drives).await?;
let sets =
Sets::new_with_instance_ctx(disks.clone(), pool_eps, &fm, i, common_parity_drives, instance_ctx.clone()).await?;
pools.push(sets);
disk_map.insert(i, disks);
}
// Replace the local disk
if !runtime_sources::setup_is_dist_erasure().await {
runtime_sources::record_local_disks(local_disks).await;
if !instance_ctx.is_dist_erasure().await {
runtime_sources::record_local_disks(&instance_ctx, local_disks).await;
}
let peer_sys = S3PeerSys::new(&endpoint_pools);
@@ -334,14 +354,17 @@ impl ECStore {
decommission_cancelers,
start_gate: Mutex::new(()),
pool_meta_save_gate: Mutex::new(()),
// Adopt the process bootstrap context so startup writes (erasure
// type recorded before this point) and later reads share one cell.
ctx: crate::runtime::instance::bootstrap_ctx(),
// Adopt the caller's context (the process bootstrap one on the
// legacy path) so startup writes (erasure type recorded before
// this point) and later reads share one cell.
ctx: instance_ctx.clone(),
});
// Only set it when the global deployment ID is not yet configured
if let Some(dep_id) = deployment_id {
runtime_sources::ensure_deployment_id(dep_id);
// Only set it when this instance's deployment ID is not yet configured
if let Some(dep_id) = deployment_id
&& instance_ctx.deployment_id().is_none()
{
instance_ctx.set_deployment_id(dep_id);
}
let wait_sec = 5;
@@ -759,4 +782,84 @@ mod tests {
"the expanded pool should be initialized by its own first local endpoint"
);
}
// Phase 5 follow-up (backlog#1052): building a real store through the
// ctx-explicit constructor lands every construction-time write — object
// graph adoption, local-disk registry, deployment id — on the passed
// context, not on the process bootstrap one. This is the storage-layer
// seam a future second embedded server needs to stay isolated.
#[tokio::test]
async fn new_with_instance_ctx_threads_context_through_store_graph() {
use crate::runtime::instance::InstanceContext;
let temp_dir = tempfile::tempdir().expect("create temp store dir");
let disk_paths: Vec<_> = (1..=4).map(|i| temp_dir.path().join(format!("disk{i}"))).collect();
for path in &disk_paths {
tokio::fs::create_dir_all(path).await.expect("create disk dir");
}
let mut endpoints = Vec::new();
for (i, path) in disk_paths.iter().enumerate() {
let mut endpoint = Endpoint::try_from(path.to_str().expect("disk path should be utf-8")).expect("local endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(i);
endpoints.push(endpoint);
}
let endpoint_pools = EndpointServerPools(vec![PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 4,
endpoints: Endpoints::from(endpoints),
cmd_line: "instance-ctx-store-graph-test".to_string(),
platform: "test".to_string(),
}]);
let instance_ctx = Arc::new(InstanceContext::new());
crate::store::init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
.await
.expect("register local disks into the fresh context");
let store = crate::store::ECStore::new_with_instance_ctx(
"127.0.0.1:0".parse().expect("test address"),
endpoint_pools,
CancellationToken::new(),
instance_ctx.clone(),
)
.await
.expect("store should build around the fresh context");
assert!(
Arc::ptr_eq(&store.ctx, &instance_ctx),
"the store must adopt the explicitly passed instance context"
);
for sets in &store.pools {
assert!(
Arc::ptr_eq(sets.instance_ctx(), &instance_ctx),
"every pool's Sets must carry the passed instance context"
);
}
assert_eq!(
instance_ctx.deployment_id(),
Some(store.id),
"the deployment id must land on the passed context and mirror the store id"
);
let registered: Vec<String> = instance_ctx.local_disk_map().read().await.keys().cloned().collect();
assert_eq!(registered.len(), 4, "the passed context must register all four local disks");
let bootstrap = crate::runtime::instance::bootstrap_ctx();
assert_ne!(
bootstrap.deployment_id(),
Some(store.id),
"the bootstrap context must not absorb the fresh store's deployment id"
);
let bootstrap_map = bootstrap.local_disk_map();
let bootstrap_map = bootstrap_map.read().await;
for key in &registered {
assert!(
!bootstrap_map.contains_key(key),
"the bootstrap context must not absorb the fresh store's disks"
);
}
}
}
+3 -2
View File
@@ -164,8 +164,9 @@ pub(crate) mod utils;
use peer::init_local_peer;
pub use peer::{
all_local_disk, all_local_disk_path, find_local_disk_by_ref, get_disk_infos, init_local_disks, init_lock_clients,
prewarm_local_disk_id_map,
all_local_disk, all_local_disk_path, find_local_disk_by_ref, get_disk_infos, init_local_disks,
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
prewarm_local_disk_id_map_with_instance_ctx,
};
pub struct ECStore {
+94 -4
View File
@@ -13,6 +13,7 @@
// limitations under the License.
use super::*;
use crate::runtime::instance::InstanceContext;
use crate::runtime::sources as runtime_sources;
use tracing::{debug, error};
@@ -22,8 +23,12 @@ const EVENT_LOCAL_DISK_ID_PREWARM_SKIPPED: &str = "local_disk_id_prewarm_skipped
const EVENT_LOCK_CLIENT_INITIALIZATION_FAILED: &str = "lock_client_initialization_failed";
async fn remember_local_disk_id(disk: &DiskStore) -> Option<Uuid> {
remember_local_disk_id_with_instance_ctx(&crate::runtime::global::current_ctx(), disk).await
}
async fn remember_local_disk_id_with_instance_ctx(instance_ctx: &Arc<InstanceContext>, disk: &DiskStore) -> Option<Uuid> {
let disk_id = disk.get_disk_id().await.ok().flatten()?;
runtime_sources::record_local_disk_id(disk_id, disk.endpoint().to_string()).await;
runtime_sources::record_local_disk_id(instance_ctx, disk_id, disk.endpoint().to_string()).await;
Some(disk_id)
}
@@ -69,7 +74,21 @@ pub async fn all_local_disk() -> Vec<DiskStore> {
}
pub async fn prewarm_local_disk_id_map() {
for disk in all_local_disk().await {
prewarm_local_disk_id_map_with_instance_ctx(&crate::runtime::global::current_ctx()).await
}
/// Prewarm the disk-id map of an explicit instance context (Phase 5 follow-up,
/// backlog#1052): startup passes the context whose disk map it just populated
/// instead of resolving the process-level default.
pub async fn prewarm_local_disk_id_map_with_instance_ctx(instance_ctx: &Arc<InstanceContext>) {
let disks: Vec<DiskStore> = instance_ctx
.local_disk_map()
.read()
.await
.values()
.filter_map(|v| v.as_ref().cloned())
.collect();
for disk in disks {
if let Err(err) = disk.get_disk_id().await {
debug!(
event = EVENT_LOCAL_DISK_ID_PREWARM_SKIPPED,
@@ -82,17 +101,28 @@ pub async fn prewarm_local_disk_id_map() {
continue;
}
let _ = remember_local_disk_id(&disk).await;
let _ = remember_local_disk_id_with_instance_ctx(instance_ctx, &disk).await;
}
}
pub async fn init_local_disks(endpoint_pools: EndpointServerPools) -> Result<()> {
init_local_disks_with_instance_ctx(&crate::runtime::global::current_ctx(), endpoint_pools).await
}
/// Register the pools' local disks into an explicit instance context (Phase 5
/// follow-up, backlog#1052). The legacy [`init_local_disks`] entry resolves the
/// process-level default context; startup paths that own a context pass it here
/// so a future second instance's disks cannot leak into the first one's registry.
pub async fn init_local_disks_with_instance_ctx(
instance_ctx: &Arc<InstanceContext>,
endpoint_pools: EndpointServerPools,
) -> Result<()> {
let opt = &DiskOption {
cleanup: true,
health_check: true,
};
runtime_sources::initialize_local_disk_maps(endpoint_pools, opt).await
runtime_sources::initialize_local_disk_maps(instance_ctx, endpoint_pools, opt).await
}
pub fn init_lock_clients(endpoint_pools: EndpointServerPools) {
@@ -182,3 +212,63 @@ pub async fn get_disk_infos(disks: &[Option<DiskStore>]) -> Vec<Option<DiskInfo>
res
}
#[cfg(test)]
mod tests {
use super::*;
use crate::layout::endpoints::{Endpoints, PoolEndpoints};
fn single_local_disk_pools(dir: &std::path::Path) -> EndpointServerPools {
let mut endpoint = Endpoint::try_from(dir.to_str().expect("temp dir path should be utf-8")).expect("local endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
EndpointServerPools(vec![PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 1,
endpoints: Endpoints::from(vec![endpoint]),
cmd_line: "instance-ctx-disk-registry-test".to_string(),
platform: "test".to_string(),
}])
}
// Phase 5 follow-up (backlog#1052): registering local disks through the
// ctx-explicit entry writes the passed context's registry only — the
// process bootstrap context (and any other instance) stays clean, so a
// future second server's disks cannot leak into the first one's registry.
#[tokio::test]
async fn init_local_disks_with_instance_ctx_isolates_disk_registry() {
let temp_dir = tempfile::tempdir().expect("create temp disk dir");
let endpoint_pools = single_local_disk_pools(temp_dir.path());
let instance_ctx = Arc::new(InstanceContext::new());
init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools)
.await
.expect("local disks should register into the passed context");
let registered: Vec<String> = instance_ctx.local_disk_map().read().await.keys().cloned().collect();
assert_eq!(registered.len(), 1, "the passed context must hold exactly the one local disk");
assert_eq!(
instance_ctx.local_disk_set_drives().read().await.len(),
1,
"the passed context must hold the pool/set/drive layout"
);
let bootstrap = crate::runtime::instance::bootstrap_ctx();
let bootstrap_map = bootstrap.local_disk_map();
let bootstrap_map = bootstrap_map.read().await;
let sibling = InstanceContext::new();
for key in &registered {
assert!(
!bootstrap_map.contains_key(key),
"bootstrap context must not absorb a disk registered into an explicit context"
);
assert!(
!sibling.local_disk_map().read().await.contains_key(key),
"a sibling context must not observe another instance's disks"
);
}
}
}
+2 -1
View File
@@ -44,7 +44,8 @@
//!
//! Only **one `RustFSServer`** may exist per process because the underlying
//! storage engine uses process-global singletons (`OnceLock`). Attempting to
//! start a second server will return an error.
//! start a second server will return an error. Removing this limitation is
//! tracked in [backlog#1052](https://github.com/rustfs/backlog/issues/1052).
use crate::server::ShutdownHandle;
use crate::startup_embedded::{EmbeddedStartedServer, EmbeddedStartupArgs, EmbeddedStartupError, run_embedded_startup};
+12 -2
View File
@@ -24,6 +24,7 @@ use crate::{
startup_services::init_embedded_startup_runtime_services,
startup_shutdown::signal_embedded_startup_shutdown,
startup_storage::{init_embedded_startup_storage_foundation, init_embedded_startup_storage_runtime},
storage_api::startup::storage::bootstrap_instance_ctx,
};
use std::{io, net::SocketAddr, path::PathBuf};
use tokio_util::sync::CancellationToken;
@@ -108,6 +109,12 @@ pub(crate) async fn run_embedded_startup(args: EmbeddedStartupArgs) -> Result<Em
// case callers can retry.
let mut startup_guard = EmbeddedStartupGuard::new();
// The embedded server still shares the process bootstrap context (see the
// EMBEDDED_SERVER_STARTED guard); once the request path and app subsystems
// are per-server too (backlog#1052 S2S4), each server constructs its own
// context here instead.
let instance_ctx = bootstrap_instance_ctx();
let EmbeddedStartupConfig {
config,
identity,
@@ -120,13 +127,15 @@ pub(crate) async fn run_embedded_startup(args: EmbeddedStartupArgs) -> Result<Em
.await
.map_err(init_error)?;
let listen_context = init_embedded_startup_listen_context(&config).await.map_err(init_error)?;
let listen_context = init_embedded_startup_listen_context(&config, &instance_ctx)
.await
.map_err(init_error)?;
startup_guard
.mark_global_init_started()
.map_err(|_| EmbeddedStartupError::AlreadyStarted)?;
let endpoint_pools = init_embedded_startup_storage_foundation(&listen_context.server_address, &config.volumes)
let endpoint_pools = init_embedded_startup_storage_foundation(&listen_context.server_address, &config.volumes, &instance_ctx)
.await
.map_err(init_error)?;
@@ -140,6 +149,7 @@ pub(crate) async fn run_embedded_startup(args: EmbeddedStartupArgs) -> Result<Em
&endpoint_pools,
listen_context.readiness.clone(),
cancel_token.clone(),
instance_ctx,
)
.await
{
+9 -3
View File
@@ -19,6 +19,7 @@ use crate::{
startup_server::{StartupHttpServers, StartupListenContext, init_startup_http_servers, init_startup_listen_context},
startup_services::init_startup_runtime_services,
startup_storage::{StartupStorageRuntime, init_startup_storage_foundation, init_startup_storage_runtime},
storage_api::startup::storage::bootstrap_instance_ctx,
};
use std::io::{Error, Result};
use tracing::{error, instrument};
@@ -103,13 +104,18 @@ async fn async_main() -> Result<()> {
#[instrument(skip(config))]
async fn run(config: Config) -> Result<()> {
// Single-instance startup threads the process bootstrap context through
// the storage path explicitly (Phase 5 follow-up, backlog#1052); a future
// multi-instance server constructs its own context here instead.
let instance_ctx = bootstrap_instance_ctx();
let StartupListenContext {
readiness,
server_addr,
server_address,
} = init_startup_listen_context(&config).await?;
} = init_startup_listen_context(&config, &instance_ctx).await?;
let endpoint_pools = init_startup_storage_foundation(&server_address, &config.volumes).await?;
let endpoint_pools = init_startup_storage_foundation(&server_address, &config.volumes, &instance_ctx).await?;
let StartupHttpServers {
state_manager,
s3_shutdown_tx,
@@ -119,7 +125,7 @@ async fn run(config: Config) -> Result<()> {
let StartupStorageRuntime {
store,
shutdown_token: ctx,
} = init_startup_storage_runtime(server_addr, &endpoint_pools, readiness.clone()).await?;
} = init_startup_storage_runtime(server_addr, &endpoint_pools, readiness.clone(), instance_ctx).await?;
let service_runtime = init_startup_runtime_services(
&config,
+11 -8
View File
@@ -45,14 +45,17 @@ const EVENT_EMBEDDED_SERVER_STATE: &str = "embedded_server_state";
///
/// Phase 5 (backlog#939) moved per-instance runtime state (erasure setup,
/// region, deployment id, endpoints, service handles, disk registry, cancel
/// token) into `ECStore`'s `InstanceContext`, so two `ECStore` object graphs
/// can now stay isolated. This guard is intentionally **retained**: the startup
/// path still publishes into the process-level *bootstrap* context (write-once
/// region/endpoints/deployment id) and the single `GLOBAL_OBJECT_API` handle,
/// so a second startup would fail-fast on that shared state. Lifting the guard
/// requires threading a per-instance context through storage startup (so each
/// server constructs its own context instead of sharing the bootstrap); until
/// then, rejecting the second start is safer than the panic it would become.
/// token) into `ECStore`'s `InstanceContext`, and backlog#1052 S1 threads an
/// explicit context through the storage startup path, so a second server's
/// *storage-layer* state could now stay isolated. This guard is still
/// intentionally **retained**: embedded startup shares the bootstrap context
/// (see `run_embedded_startup`), the request path resolves the store per
/// request through the process-level `GLOBAL_OBJECT_API`/`AppContext`
/// singletons (a second listener would serve the first instance's data), and
/// IAM/bucket-metadata/config/credentials remain process singletons. Lifting
/// the guard is staged in backlog#1052 (S2 per-server dispatch, S3 app
/// subsystems, S4 node-identity globals, then S5 removes this guard); until
/// then, rejecting the second start is safer than the failure it would become.
static EMBEDDED_SERVER_STARTED: AtomicBool = AtomicBool::new(false);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+3 -3
View File
@@ -16,7 +16,7 @@ use crate::config::RustFSBufferConfig;
use crate::runtime_sources::{
current_outbound_tls_generation as runtime_current_outbound_tls_generation, current_replication_pool_handle,
};
use crate::storage_api::startup::runtime_sources::{DynReplicationPool, set_global_region, set_global_rustfs_port};
use crate::storage_api::startup::runtime_sources::{DynReplicationPool, InstanceContext, set_global_rustfs_port};
use rustfs_kms::KmsServiceManager;
use rustfs_obs::{GlobalError as ObservabilityError, OtelGuard};
use rustfs_tls_runtime::{OutboundTlsMaterial, TlsGeneration};
@@ -30,8 +30,8 @@ pub(crate) fn init_action_credentials(
rustfs_credentials::init_global_action_credentials(Some(access_key), Some(secret_key))
}
pub(crate) fn publish_region(region: s3s::region::Region) {
set_global_region(region);
pub(crate) fn publish_region(instance_ctx: &Arc<InstanceContext>, region: s3s::region::Region) {
instance_ctx.set_region(region);
}
pub(crate) fn publish_server_port(port: u16) {
+11 -4
View File
@@ -17,6 +17,7 @@ use crate::{
config::Config,
server::{ServiceState, ServiceStateManager, ShutdownHandle, start_http_server},
startup_runtime_sources,
storage_api::startup::runtime_sources::InstanceContext,
};
use rustfs_common::GlobalReadiness;
use rustfs_utils::net::parse_and_resolve_address;
@@ -76,14 +77,17 @@ pub(crate) struct StartupHttpServers {
pub(crate) console_shutdown_tx: Option<ShutdownHandle>,
}
pub(crate) async fn init_startup_listen_context(config: &Config) -> Result<StartupListenContext> {
pub(crate) async fn init_startup_listen_context(
config: &Config,
instance_ctx: &Arc<InstanceContext>,
) -> Result<StartupListenContext> {
log_sanitized_server_config(config);
let readiness = Arc::new(GlobalReadiness::new());
if let Some(region_str) = &config.region {
region_str
.parse::<s3s::region::Region>()
.map(startup_runtime_sources::publish_region)
.map(|region| startup_runtime_sources::publish_region(instance_ctx, region))
.map_err(|err| Error::other(format!("invalid region '{}': {}", region_str, err)))?;
}
@@ -191,7 +195,10 @@ pub(crate) fn find_embedded_available_port() -> Result<u16> {
Err(last_err.unwrap_or_else(|| Error::other("failed to reserve an embedded TCP port")))
}
pub(crate) async fn init_embedded_startup_listen_context(config: &Config) -> Result<EmbeddedStartupListenContext> {
pub(crate) async fn init_embedded_startup_listen_context(
config: &Config,
instance_ctx: &Arc<InstanceContext>,
) -> Result<EmbeddedStartupListenContext> {
let readiness = Arc::new(GlobalReadiness::new());
let server_addr =
@@ -210,7 +217,7 @@ pub(crate) async fn init_embedded_startup_listen_context(config: &Config) -> Res
if let Some(region_str) = &config.region {
region_str
.parse::<s3s::region::Region>()
.map(startup_runtime_sources::publish_region)
.map(|region| startup_runtime_sources::publish_region(instance_ctx, region))
.map_err(|err| Error::other(format!("invalid region '{region_str}': {err}")))?;
}
+76 -27
View File
@@ -14,9 +14,9 @@
use crate::startup_fs_guard::enforce_unsupported_fs_policy;
use crate::storage_api::startup::storage::{
ECStore, EndpointServerPools, global_config_init_error_is_deterministic, init_background_replication, init_ecstore_config,
init_global_config_sys, init_local_disks, init_lock_clients, prewarm_local_disk_id_map, set_global_endpoints,
try_migrate_server_config, update_erasure_type,
ECStore, EndpointServerPools, InstanceContext, global_config_init_error_is_deterministic, init_background_replication,
init_ecstore_config, init_global_config_sys, init_local_disks_with_instance_ctx, init_lock_clients,
prewarm_local_disk_id_map_with_instance_ctx, try_migrate_server_config,
};
use rustfs_common::{GlobalReadiness, SystemStage};
use std::{
@@ -44,7 +44,11 @@ pub(crate) struct StartupStorageRuntime {
pub(crate) shutdown_token: CancellationToken,
}
pub(crate) async fn init_startup_storage_foundation(server_address: &str, volumes: &[String]) -> Result<EndpointServerPools> {
pub(crate) async fn init_startup_storage_foundation(
server_address: &str,
volumes: &[String],
instance_ctx: &Arc<InstanceContext>,
) -> Result<EndpointServerPools> {
info!(
target: "rustfs::main::run",
event = EVENT_ENDPOINT_PARSING_STARTED,
@@ -71,8 +75,8 @@ pub(crate) async fn init_startup_storage_foundation(server_address: &str, volume
.map_err(Error::other)?;
enforce_unsupported_fs_policy(&endpoint_pools)?;
set_global_endpoints(endpoint_pools.as_ref().clone());
update_erasure_type(setup_type).await;
instance_ctx.set_endpoints(endpoint_pools.clone());
instance_ctx.update_erasure_type(setup_type).await;
debug!(
target: "rustfs::main::run",
@@ -83,7 +87,7 @@ pub(crate) async fn init_startup_storage_foundation(server_address: &str, volume
state = "starting",
"starting local disk initialization"
);
init_local_disks(endpoint_pools.clone())
init_local_disks_with_instance_ctx(instance_ctx, endpoint_pools.clone())
.await
.inspect_err(|err| {
error!(
@@ -98,7 +102,7 @@ pub(crate) async fn init_startup_storage_foundation(server_address: &str, volume
);
})
.map_err(Error::other)?;
prewarm_local_disk_id_map().await;
prewarm_local_disk_id_map_with_instance_ctx(instance_ctx).await;
init_lock_clients(endpoint_pools.clone());
log_storage_pool_layout(&endpoint_pools);
@@ -109,16 +113,17 @@ pub(crate) async fn init_startup_storage_foundation(server_address: &str, volume
pub(crate) async fn init_embedded_startup_storage_foundation(
server_address: &str,
volumes: &[String],
instance_ctx: &Arc<InstanceContext>,
) -> Result<EndpointServerPools> {
let (endpoint_pools, setup_type) = EndpointServerPools::from_volumes(server_address, volumes.to_vec())
.await
.map_err(|err| Error::other(format!("endpoints: {err}")))?;
enforce_unsupported_fs_policy(&endpoint_pools).map_err(|err| Error::other(format!("unsupported fs guard: {err}")))?;
set_global_endpoints(endpoint_pools.as_ref().clone());
update_erasure_type(setup_type).await;
instance_ctx.set_endpoints(endpoint_pools.clone());
instance_ctx.update_erasure_type(setup_type).await;
init_local_disks(endpoint_pools.clone())
init_local_disks_with_instance_ctx(instance_ctx, endpoint_pools.clone())
.await
.map_err(|err| Error::other(format!("local disks: {err}")))?;
init_lock_clients(endpoint_pools.clone());
@@ -130,6 +135,7 @@ pub(crate) async fn init_startup_storage_runtime(
server_addr: SocketAddr,
endpoint_pools: &EndpointServerPools,
readiness: Arc<GlobalReadiness>,
instance_ctx: Arc<InstanceContext>,
) -> Result<StartupStorageRuntime> {
let ctx = CancellationToken::new();
@@ -142,7 +148,7 @@ pub(crate) async fn init_startup_storage_runtime(
state = "starting",
"starting ECStore initialization"
);
let store = ECStore::new(server_addr, endpoint_pools.clone(), ctx.clone())
let store = ECStore::new_with_instance_ctx(server_addr, endpoint_pools.clone(), ctx.clone(), instance_ctx)
.await
.inspect_err(|err| {
error!(
@@ -172,21 +178,23 @@ pub(crate) async fn init_embedded_startup_storage_runtime(
endpoint_pools: &EndpointServerPools,
readiness: Arc<GlobalReadiness>,
shutdown_token: CancellationToken,
instance_ctx: Arc<InstanceContext>,
) -> Result<StartupStorageRuntime> {
let store = match ECStore::new(server_addr, endpoint_pools.clone(), shutdown_token.clone()).await {
Ok(store) => store,
Err(err) => {
error!(
component = LOG_COMPONENT_EMBEDDED,
subsystem = LOG_SUBSYSTEM_EMBEDDED,
event = EVENT_EMBEDDED_STORAGE_INIT_FAILED,
stage = "ecstore_new",
error = ?err,
"Embedded storage initialization failed"
);
return Err(Error::other(format!("ECStore: {err}")));
}
};
let store =
match ECStore::new_with_instance_ctx(server_addr, endpoint_pools.clone(), shutdown_token.clone(), instance_ctx).await {
Ok(store) => store,
Err(err) => {
error!(
component = LOG_COMPONENT_EMBEDDED,
subsystem = LOG_SUBSYSTEM_EMBEDDED,
event = EVENT_EMBEDDED_STORAGE_INIT_FAILED,
stage = "ecstore_new",
error = ?err,
"Embedded storage initialization failed"
);
return Err(Error::other(format!("ECStore: {err}")));
}
};
init_embedded_startup_storage_global_config(store.clone()).await?;
readiness.mark_stage(SystemStage::StorageReady);
@@ -324,7 +332,9 @@ fn global_config_retry_exhausted(retry_count: usize) -> bool {
#[cfg(test)]
mod tests {
use super::{global_config_retry_exhausted, storage_pool_has_host_failure_risk};
use super::{global_config_retry_exhausted, init_embedded_startup_storage_foundation, storage_pool_has_host_failure_risk};
use crate::storage_api::startup::storage::{InstanceContext, bootstrap_instance_ctx};
use std::sync::Arc;
#[test]
fn reports_host_failure_risk_only_for_multi_drive_sets() {
@@ -333,6 +343,45 @@ mod tests {
assert!(storage_pool_has_host_failure_risk(2));
}
// Phase 5 follow-up (backlog#1052): the embedded storage foundation writes
// its topology (endpoints, erasure kind) into the explicitly passed
// instance context, leaving the process bootstrap context untouched — the
// seam a future second embedded server needs to avoid the write-once
// panics on shared startup state.
#[tokio::test]
async fn embedded_foundation_writes_land_on_passed_instance_ctx() {
let temp_dir = tempfile::tempdir().expect("create temp volume");
let volume = temp_dir.path().display().to_string();
let instance_ctx = Arc::new(InstanceContext::new());
init_embedded_startup_storage_foundation("127.0.0.1:29123", &[volume], &instance_ctx)
.await
.expect("embedded storage foundation should initialize");
assert!(
instance_ctx.endpoints().is_some(),
"the passed context must hold the parsed endpoint topology"
);
assert!(
instance_ctx.is_erasure_sd().await,
"a single local volume must record single-drive erasure on the passed context"
);
let bootstrap = bootstrap_instance_ctx();
assert!(
!Arc::ptr_eq(&instance_ctx, &bootstrap),
"the test context must be distinct from the process bootstrap context"
);
assert!(
bootstrap.endpoints().is_none(),
"the bootstrap context must not absorb topology written to an explicit context"
);
assert!(
!bootstrap.is_erasure_sd().await,
"the bootstrap context must not absorb the erasure kind written to an explicit context"
);
}
#[test]
fn global_config_retry_limit_matches_startup_policy() {
assert!(!global_config_retry_exhausted(15));
+21 -21
View File
@@ -51,28 +51,28 @@ pub(crate) use storage_api::{
DEFAULT_READ_BUFFER_SIZE, DailyAllTierStats, DeleteOptions, DiskError, DiskInfo, DiskInfoOptions, DiskResult, DiskStore,
DynReader, DynReplicationPool, ECStore, Endpoint, EndpointServerPools, Error, EventArgs, ExpiryState, FS, FileInfoVersions,
FileReader, FileWriter, GetObjectReader, HashReader, LocalPeerS3Client, MetricType, NotificationSys, OBJECT_LOCK_CONFIG,
ObjectInfo, ObjectLockBlockReason, ObjectOptions, ObjectPartInfo, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PolicySys,
PoolEndpoints, PutObjReader, QuotaError, RUSTFS_META_BUCKET, RawFileInfo, ReadMultipleReq, ReadMultipleResp, ReadOptions,
RenameDataResp, ReplicationStats, ReplicationStatusType, Result, SERVICE_SIGNAL_REFRESH_CONFIG,
SERVICE_SIGNAL_RELOAD_DYNAMIC, SetupType, StorageDeletedObject, StorageDiskRpcExt, StorageError, StorageGetObjectReader,
StorageObjectInfo, StorageObjectOptions, StorageObjectToDelete, StoragePeerS3ClientExt, StoragePutObjReader,
StorageReplicationConfigExt, StorageVersioningConfigExt, TONIC_RPC_PREFIX, TierConfigMgr, UpdateMetadataOpts, VolumeInfo,
WalkDirOptions, WorkloadAdmissionSnapshotProviderRef, WriteEncryption, WritePlan, access_consumer, add_object_lock_years,
all_local_disk, all_local_disk_path, check_retention_for_modification, collect_local_metrics, compression_metadata_value,
contract, decode_tags, decode_tags_to_map, delete_bucket_metadata_config, disk_drive_path, disk_endpoint, ecfs_consumer,
ecfs_extend_consumer, ecstore_admin, ecstore_bucket, ecstore_capacity, ecstore_client, ecstore_cluster, ecstore_compression,
ecstore_config, ecstore_data_usage, ecstore_disk, ecstore_error, ecstore_event, ecstore_layout, ecstore_metrics,
ecstore_notification, ecstore_rebalance, ecstore_rio, ecstore_rpc, ecstore_set_disk, ecstore_storage, ecstore_tier,
encode_tags, find_local_disk_by_ref, get_bucket_accelerate_config, get_bucket_cors_config, get_bucket_logging_config,
get_bucket_metadata, get_bucket_notification_config, get_bucket_object_lock_config, get_bucket_policy_raw,
get_bucket_replication_config, get_bucket_request_payment_config, get_bucket_sse_config, get_bucket_website_config,
get_local_server_property, get_lock_acquire_timeout, get_public_access_block_config, head_prefix_consumer, helper_consumer,
init_background_replication, init_bucket_metadata_sys, init_ecstore_config, init_local_disks, init_lock_clients,
ObjectInfo, ObjectLockBlockReason, ObjectOptions, ObjectPartInfo, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PolicySys, PutObjReader,
QuotaError, RUSTFS_META_BUCKET, RawFileInfo, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
ReplicationStats, ReplicationStatusType, Result, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
StorageDeletedObject, StorageDiskRpcExt, StorageError, StorageGetObjectReader, StorageObjectInfo, StorageObjectOptions,
StorageObjectToDelete, StoragePeerS3ClientExt, StoragePutObjReader, StorageReplicationConfigExt, StorageVersioningConfigExt,
TONIC_RPC_PREFIX, TierConfigMgr, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, WorkloadAdmissionSnapshotProviderRef,
WriteEncryption, WritePlan, access_consumer, add_object_lock_years, all_local_disk, all_local_disk_path,
check_retention_for_modification, collect_local_metrics, compression_metadata_value, contract, decode_tags,
decode_tags_to_map, delete_bucket_metadata_config, disk_drive_path, disk_endpoint, ecfs_consumer, ecfs_extend_consumer,
ecstore_admin, ecstore_bucket, ecstore_capacity, ecstore_client, ecstore_cluster, ecstore_compression, ecstore_config,
ecstore_data_usage, ecstore_disk, ecstore_error, ecstore_event, ecstore_layout, ecstore_metrics, ecstore_notification,
ecstore_rebalance, ecstore_rio, ecstore_rpc, ecstore_set_disk, ecstore_storage, ecstore_tier, encode_tags,
find_local_disk_by_ref, get_bucket_accelerate_config, get_bucket_cors_config, get_bucket_logging_config, get_bucket_metadata,
get_bucket_notification_config, get_bucket_object_lock_config, get_bucket_policy_raw, get_bucket_replication_config,
get_bucket_request_payment_config, get_bucket_sse_config, get_bucket_website_config, get_local_server_property,
get_lock_acquire_timeout, get_public_access_block_config, head_prefix_consumer, helper_consumer, init_background_replication,
init_bucket_metadata_sys, init_ecstore_config, init_local_disks_with_instance_ctx, init_lock_clients,
is_all_buckets_not_found, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, is_valid_storage_class,
load_bucket_metadata, options_consumer, prewarm_local_disk_id_map, read_config, record_replication_proxy, rpc_consumer,
runtime_sources_consumer, s3_api_consumer, save_config, serialize, set_bucket_metadata, table_catalog_path_hash, to_s3s_etag,
topology_snapshot_from_endpoint_pools_with_capabilities, try_migrate_bucket_metadata, try_migrate_iam_config,
try_migrate_server_config, update_bucket_metadata_config, verify_rpc_signature, wrap_reader,
load_bucket_metadata, options_consumer, prewarm_local_disk_id_map_with_instance_ctx, read_config, record_replication_proxy,
rpc_consumer, runtime_sources_consumer, s3_api_consumer, save_config, serialize, set_bucket_metadata,
table_catalog_path_hash, to_s3s_etag, topology_snapshot_from_endpoint_pools_with_capabilities, try_migrate_bucket_metadata,
try_migrate_iam_config, try_migrate_server_config, update_bucket_metadata_config, verify_rpc_signature, wrap_reader,
};
#[cfg(test)]
+28 -22
View File
@@ -389,15 +389,14 @@ pub(crate) mod ecstore_event {
pub(crate) mod ecstore_global {
pub(crate) use rustfs_ecstore::api::global::{
set_global_endpoints, set_global_region, set_global_rustfs_port, set_object_store_resolver, shutdown_background_services,
update_erasure_type,
set_global_rustfs_port, set_object_store_resolver, shutdown_background_services,
};
}
pub(crate) mod ecstore_runtime {
pub(crate) use rustfs_ecstore::api::runtime::{
boot_time, bucket_monitor, deployment_id, endpoint_pools, global_lock_client, global_lock_clients,
global_tier_config_mgr, region, rustfs_port, setup_is_dist_erasure,
InstanceContext, boot_time, bootstrap_ctx, bucket_monitor, deployment_id, endpoint_pools, global_lock_client,
global_lock_clients, global_tier_config_mgr, region, rustfs_port, setup_is_dist_erasure,
};
}
@@ -450,9 +449,11 @@ pub(crate) mod ecstore_set_disk {
}
pub(crate) mod ecstore_storage {
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::storage::init_local_disks;
pub(crate) use rustfs_ecstore::api::storage::{
ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks, init_lock_clients,
prewarm_local_disk_id_map,
ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks_with_instance_ctx,
init_lock_clients, prewarm_local_disk_id_map_with_instance_ctx,
};
}
@@ -516,13 +517,15 @@ pub(crate) type FileReader = ecstore_disk::FileReader;
pub(crate) type FileWriter = ecstore_disk::FileWriter;
pub(crate) type FS = super::ecfs::FS;
pub(crate) type HashReader = ecstore_rio::HashReader;
pub(crate) type InstanceContext = ecstore_runtime::InstanceContext;
pub(crate) type LocalPeerS3Client = ecstore_rpc::LocalPeerS3Client;
pub(crate) type MetricType = ecstore_metrics::MetricType;
pub(crate) type ObjectPartInfo = rustfs_filemeta::ObjectPartInfo;
pub(crate) type ObjectLockBlockReason = ecstore_bucket::object_lock::objectlock_sys::ObjectLockBlockReason;
pub(crate) type ObjectStoreResolver = dyn Fn() -> Option<Arc<ECStore>> + Send + Sync + 'static;
pub(crate) type PolicySys = ecstore_bucket::policy_sys::PolicySys;
#[cfg(test)]
pub(crate) type PoolEndpoints = ecstore_layout::PoolEndpoints;
pub(crate) type PolicySys = ecstore_bucket::policy_sys::PolicySys;
pub(crate) type WorkloadAdmissionSnapshotProviderRef = rustfs_ecstore::WorkloadAdmissionSnapshotProviderRef;
pub(crate) type QuotaError = ecstore_bucket::quota::QuotaError;
pub(crate) type RawFileInfo = rustfs_filemeta::RawFileInfo;
@@ -535,7 +538,6 @@ pub(crate) type OldCurrentSize = ecstore_disk::OldCurrentSize;
pub(crate) type RenameDataResp = ecstore_disk::RenameDataResp;
pub(crate) type ReplicationStatusType = ecstore_bucket::replication::ReplicationStatusType;
pub(crate) type ReplicationStats = StorageReplicationStatsHandle;
pub(crate) type SetupType = ecstore_layout::SetupType;
pub(crate) type StorageError = ecstore_error::StorageError;
pub(crate) type TierConfigMgr = ecstore_tier::TierConfigMgr;
pub(crate) use ecstore_disk::validate_batch_read_version_item_count;
@@ -757,10 +759,26 @@ pub(crate) fn global_config_init_error_is_deterministic(err: &Error) -> bool {
ecstore_config::com::is_server_config_corrupt_error(err)
}
pub(crate) async fn init_local_disks_with_instance_ctx(
instance_ctx: &Arc<InstanceContext>,
endpoint_pools: EndpointServerPools,
) -> Result<()> {
ecstore_storage::init_local_disks_with_instance_ctx(instance_ctx, endpoint_pools).await
}
/// Test-only legacy entry: registers disks into the ambient (process-default)
/// context, matching what pre-#1052 production startup did.
#[cfg(test)]
pub(crate) async fn init_local_disks(endpoint_pools: EndpointServerPools) -> Result<()> {
ecstore_storage::init_local_disks(endpoint_pools).await
}
/// The process-level bootstrap instance context that single-instance startup
/// threads through the storage foundation (Phase 5 follow-up, backlog#1052).
pub(crate) fn bootstrap_instance_ctx() -> Arc<InstanceContext> {
ecstore_runtime::bootstrap_ctx()
}
pub(crate) fn init_lock_clients(endpoint_pools: EndpointServerPools) {
ecstore_storage::init_lock_clients(endpoint_pools);
}
@@ -773,8 +791,8 @@ pub(crate) async fn read_config(api: Arc<ECStore>, file: &str) -> Result<Vec<u8>
ecstore_config::com::read_config(api, file).await
}
pub(crate) async fn prewarm_local_disk_id_map() {
ecstore_storage::prewarm_local_disk_id_map().await;
pub(crate) async fn prewarm_local_disk_id_map_with_instance_ctx(instance_ctx: &Arc<InstanceContext>) {
ecstore_storage::prewarm_local_disk_id_map_with_instance_ctx(instance_ctx).await;
}
pub(crate) fn replication_queue_current_count() -> Option<i64> {
@@ -793,14 +811,6 @@ pub(crate) fn shutdown_background_monitors() {
rustfs_ecstore::shutdown_background_monitors();
}
pub(crate) fn set_global_endpoints(endpoints: Vec<PoolEndpoints>) {
ecstore_global::set_global_endpoints(endpoints);
}
pub(crate) fn set_global_region(region: s3s::region::Region) {
ecstore_global::set_global_region(region);
}
pub(crate) fn set_global_rustfs_port(value: u16) {
ecstore_global::set_global_rustfs_port(value);
}
@@ -815,10 +825,6 @@ pub(crate) async fn try_migrate_server_config(store: Arc<ECStore>) {
ecstore_config::try_migrate_server_config(store, Some(decrypt_fn)).await;
}
pub(crate) async fn update_erasure_type(setup_type: SetupType) {
ecstore_global::update_erasure_type(setup_type).await;
}
pub(crate) trait StorageDiskRpcExt {
async fn disk_info(&self, opts: &DiskInfoOptions) -> DiskResult<DiskInfo>;
async fn delete_volume(&self, volume: &str, force_delete: bool) -> DiskResult<()>;
+5 -4
View File
@@ -211,7 +211,7 @@ pub(crate) mod startup {
}
pub(crate) mod runtime_sources {
pub(crate) use crate::storage::storage_api::{DynReplicationPool, set_global_region, set_global_rustfs_port};
pub(crate) use crate::storage::storage_api::{DynReplicationPool, InstanceContext, set_global_rustfs_port};
}
pub(crate) mod services {
@@ -226,9 +226,10 @@ pub(crate) mod startup {
pub(crate) mod storage {
pub(crate) use crate::storage::storage_api::{
ECStore, EndpointServerPools, global_config_init_error_is_deterministic, init_background_replication,
init_compression_total_memory_from_backend, init_ecstore_config, init_global_config_sys, init_local_disks,
init_lock_clients, prewarm_local_disk_id_map, set_global_endpoints, try_migrate_server_config, update_erasure_type,
ECStore, EndpointServerPools, InstanceContext, bootstrap_instance_ctx, global_config_init_error_is_deterministic,
init_background_replication, init_compression_total_memory_from_backend, init_ecstore_config, init_global_config_sys,
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map_with_instance_ctx,
try_migrate_server_config,
};
}
}