diff --git a/crates/ecstore/src/runtime/global.rs b/crates/ecstore/src/runtime/global.rs index f8a6da4a6..0387c7257 100644 --- a/crates/ecstore/src/runtime/global.rs +++ b/crates/ecstore/src/runtime/global.rs @@ -18,7 +18,6 @@ use crate::{ bucket::lifecycle::bucket_lifecycle_ops::LifecycleSys, disk::DiskStore, layout::endpoints::{EndpointServerPools, PoolEndpoints, SetupType}, - services::event_notification::EventNotifier, services::tier::tier::TierConfigMgr, store::ECStore, }; @@ -47,21 +46,18 @@ pub const DISK_RESERVE_FRACTION: f64 = 0.15; // GLOBAL_ROOT_DISK_THRESHOLD, GLOBAL_LIFECYCLE_SYS, GLOBAL_EVENT_NOTIFIER, etc. // Tier B (keep as static): GLOBAL_RUSTFS_PORT, env var caches, etc. // -// Phase 5 (backlog#939): the erasure setup type and the S3 region moved into -// the per-instance `InstanceContext` (see `super::instance`); their facades -// below now forward to the current instance's context. +// Phase 5 (backlog#939): identity/runtime state (erasure setup, S3 region, +// deployment id, endpoint topology, tier config manager, ...) is moving into +// the per-instance `InstanceContext` (see `super::instance`); the facades below +// forward to the current instance's context. lazy_static! { static ref GLOBAL_RUSTFS_PORT: OnceLock = OnceLock::new(); - static ref GLOBAL_DEPLOYMENT_ID: OnceLock = OnceLock::new(); pub static ref GLOBAL_OBJECT_API: OnceLock> = OnceLock::new(); pub static ref GLOBAL_LOCAL_DISK_MAP: Arc>>> = Arc::new(RwLock::new(HashMap::new())); pub static ref GLOBAL_LOCAL_DISK_ID_MAP: Arc>> = Arc::new(RwLock::new(HashMap::new())); pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc> = Arc::new(RwLock::new(Vec::new())); - pub static ref GLOBAL_ENDPOINTS: OnceLock = OnceLock::new(); pub static ref GLOBAL_ROOT_DISK_THRESHOLD: RwLock = RwLock::new(0); - pub static ref GLOBAL_TIER_CONFIG_MGR: Arc> = TierConfigMgr::new(); pub static ref GLOBAL_LIFECYCLE_SYS: Arc = LifecycleSys::new(); - pub static ref GLOBAL_EVENT_NOTIFIER: Arc> = EventNotifier::new(); pub static ref GLOBAL_BOOT_TIME: OnceCell = OnceCell::new(); pub static ref GLOBAL_LOCAL_NODE_NAME_FALLBACK: String = "127.0.0.1:9000".to_string(); pub static ref GLOBAL_LOCAL_NODE_NAME_HEX_FALLBACK: String = @@ -125,9 +121,7 @@ pub fn set_global_rustfs_port(value: u16) { /// * None /// pub fn set_global_deployment_id(id: Uuid) { - GLOBAL_DEPLOYMENT_ID - .set(id) - .expect("GLOBAL_DEPLOYMENT_ID should be initialized once during startup"); + current_ctx().set_deployment_id(id); } /// Get the global deployment id @@ -136,7 +130,7 @@ pub fn set_global_deployment_id(id: Uuid) { /// * `Option` - The global deployment id as a string, if set /// pub fn get_global_deployment_id() -> Option { - GLOBAL_DEPLOYMENT_ID.get().map(|v| v.to_string()) + current_ctx().deployment_id().map(|v| v.to_string()) } /// Set the global endpoints /// @@ -147,9 +141,7 @@ pub fn get_global_deployment_id() -> Option { /// * None /// pub fn set_global_endpoints(eps: Vec) { - GLOBAL_ENDPOINTS - .set(EndpointServerPools::from(eps)) - .expect("GLOBAL_ENDPOINTS should be initialized once during storage startup") + current_ctx().set_endpoints(EndpointServerPools::from(eps)); } /// Get the global endpoints @@ -158,15 +150,11 @@ pub fn set_global_endpoints(eps: Vec) { /// * `EndpointServerPools` - The global endpoints /// pub fn get_global_endpoints() -> EndpointServerPools { - if let Some(eps) = GLOBAL_ENDPOINTS.get() { - eps.clone() - } else { - EndpointServerPools::default() - } + current_ctx().endpoints().unwrap_or_default() } pub fn get_global_endpoints_opt() -> Option { - GLOBAL_ENDPOINTS.get().cloned() + current_ctx().endpoints() } #[cfg(test)] @@ -181,7 +169,7 @@ pub async fn is_first_cluster_node_local() -> bool { } pub fn get_global_tier_config_mgr() -> Arc> { - GLOBAL_TIER_CONFIG_MGR.clone() + current_ctx().tier_config_mgr() } /// Create a new object layer instance diff --git a/crates/ecstore/src/runtime/instance.rs b/crates/ecstore/src/runtime/instance.rs index ef3611005..e0c823f94 100644 --- a/crates/ecstore/src/runtime/instance.rs +++ b/crates/ecstore/src/runtime/instance.rs @@ -39,18 +39,20 @@ //! startup writes and post-construction reads hit the same cell and //! single-instance behavior is byte-for-byte unchanged. -use crate::layout::endpoints::SetupType; +use crate::layout::endpoints::{EndpointServerPools, SetupType}; +use crate::services::event_notification::EventNotifier; +use crate::services::tier::tier::TierConfigMgr; use rustfs_lock::{GlobalLockManager, get_global_lock_manager}; use s3s::region::Region; use std::sync::{Arc, OnceLock}; use tokio::sync::RwLock; +use uuid::Uuid; /// Runtime state owned by a single `ECStore` instance. /// /// This is intentionally minimal in the first migration slice; subsequent /// slices move additional identity/runtime state (topology, disk registry, /// service handles, cancellation token) into this struct. -#[derive(Debug)] pub struct InstanceContext { /// The deployment's erasure setup type. /// @@ -72,6 +74,28 @@ pub struct InstanceContext { /// Write-once like the process global it replaces: startup sets it exactly /// once and a duplicate write fails fast (see [`InstanceContext::set_region`]). region: OnceLock, + /// This instance's deployment id (Phase 5 Slice 5, backlog#939). + /// + /// Write-once identity, mirrored by `ECStore::id` (both come from the same + /// startup value). Replaces the process-global deployment-id static. + deployment_id: OnceLock, + /// This instance's endpoint topology (Phase 5 Slice 6, backlog#939). + /// + /// Write-once: storage startup publishes the server pools exactly once. + /// Replaces the process-global endpoints static. + endpoints: OnceLock, + /// This instance's tier configuration manager (Phase 5 Slice 7, backlog#939). + /// + /// Lazily materialized on first access so `InstanceContext::new()` stays + /// cheap: single-instance creates one shared manager on first use — exactly + /// the "create-once, then share" semantics of the eager process static it + /// replaces — while a genuinely independent instance gets its own. + tier_config_mgr: OnceLock>>, + /// This instance's event notifier (Phase 5 Slice 8, backlog#939). + /// + /// Lazily materialized like the tier config manager; holds the instance's + /// notification target list. Replaces the eager process static. + event_notifier: OnceLock>>, } impl InstanceContext { @@ -92,6 +116,10 @@ impl InstanceContext { erasure_kind: RwLock::new(SetupType::Unknown), lock_manager, region: OnceLock::new(), + deployment_id: OnceLock::new(), + endpoints: OnceLock::new(), + tier_config_mgr: OnceLock::new(), + event_notifier: OnceLock::new(), } } @@ -115,6 +143,48 @@ impl InstanceContext { self.region.get().cloned() } + /// Set this instance's deployment id. + /// + /// Write-once: panics on a second write, preserving the startup fail-fast + /// contract of the process global it replaces. + pub fn set_deployment_id(&self, id: Uuid) { + self.deployment_id + .set(id) + .expect("instance deployment id should be initialized once during startup"); + } + + /// This instance's deployment id, if it has been set. + pub fn deployment_id(&self) -> Option { + self.deployment_id.get().copied() + } + + /// Set this instance's endpoint topology. + /// + /// Write-once: panics on a second write, preserving the startup fail-fast + /// contract of the process global it replaces. + pub fn set_endpoints(&self, endpoints: EndpointServerPools) { + self.endpoints + .set(endpoints) + .expect("instance endpoints should be initialized once during storage startup"); + } + + /// This instance's endpoint topology, if it has been set. + pub fn endpoints(&self) -> Option { + self.endpoints.get().cloned() + } + + /// This instance's tier configuration manager, materializing it on first + /// access. Shared for the lifetime of the instance. + pub fn tier_config_mgr(&self) -> Arc> { + self.tier_config_mgr.get_or_init(TierConfigMgr::new).clone() + } + + /// This instance's event notifier, materializing it on first access. + /// Shared for the lifetime of the instance. + pub fn event_notifier(&self) -> Arc> { + self.event_notifier.get_or_init(EventNotifier::new).clone() + } + /// Update this instance's erasure setup type. pub async fn update_erasure_type(&self, setup_type: SetupType) { *self.erasure_kind.write().await = setup_type; @@ -147,6 +217,21 @@ impl Default for InstanceContext { } } +// Manual Debug: service handles (e.g. the tier config manager) are not Debug, +// so summarize their materialization state rather than their contents. +impl std::fmt::Debug for InstanceContext { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("InstanceContext") + .field("erasure_kind", &self.erasure_kind) + .field("region", &self.region) + .field("deployment_id", &self.deployment_id) + .field("endpoints", &self.endpoints) + .field("tier_config_mgr_set", &self.tier_config_mgr.get().is_some()) + .field("event_notifier_set", &self.event_notifier.get().is_some()) + .finish_non_exhaustive() + } +} + /// Process-level bootstrap instance context. /// /// Storage startup writes erasure state before any `ECStore` exists (see @@ -285,4 +370,123 @@ mod tests { ctx.set_region(region("us-east-1")); ctx.set_region(region("eu-west-1")); } + + // A fresh context has no deployment id until set; set-then-get round-trips. + #[tokio::test] + async fn deployment_id_set_and_get() { + let ctx = InstanceContext::new(); + assert_eq!(ctx.deployment_id(), None); + let id = Uuid::new_v4(); + ctx.set_deployment_id(id); + assert_eq!(ctx.deployment_id(), Some(id)); + } + + // Two contexts hold independent deployment ids. + #[tokio::test] + async fn distinct_contexts_have_distinct_deployment_id() { + let ctx_a = InstanceContext::new(); + let ctx_b = InstanceContext::new(); + let id_a = Uuid::new_v4(); + let id_b = Uuid::new_v4(); + ctx_a.set_deployment_id(id_a); + ctx_b.set_deployment_id(id_b); + assert_eq!(ctx_a.deployment_id(), Some(id_a)); + assert_eq!(ctx_b.deployment_id(), Some(id_b)); + } + + // The deployment id is write-once: a second set fails fast. + #[tokio::test] + #[should_panic(expected = "initialized once")] + async fn set_deployment_id_twice_panics() { + let ctx = InstanceContext::new(); + ctx.set_deployment_id(Uuid::new_v4()); + ctx.set_deployment_id(Uuid::new_v4()); + } + + fn endpoints_with_pools(n: usize) -> EndpointServerPools { + use crate::layout::endpoint::Endpoint; + use crate::layout::endpoints::{Endpoints, PoolEndpoints}; + let pool = PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: 1, + endpoints: Endpoints::from(vec![Endpoint::try_from("http://127.0.0.1:9000/data0").expect("valid endpoint")]), + cmd_line: "test".to_string(), + platform: "test".to_string(), + }; + EndpointServerPools((0..n).map(|_| pool.clone()).collect()) + } + + // A fresh context has no endpoints until set; set-then-get round-trips. + #[tokio::test] + async fn endpoints_set_and_get() { + let ctx = InstanceContext::new(); + assert!(ctx.endpoints().is_none()); + ctx.set_endpoints(endpoints_with_pools(1)); + assert_eq!(ctx.endpoints().expect("endpoints set").0.len(), 1); + } + + // Two contexts hold independent endpoint topologies. + #[tokio::test] + async fn distinct_contexts_have_distinct_endpoints() { + let ctx_a = InstanceContext::new(); + let ctx_b = InstanceContext::new(); + ctx_a.set_endpoints(endpoints_with_pools(1)); + ctx_b.set_endpoints(endpoints_with_pools(2)); + assert_eq!(ctx_a.endpoints().expect("a").0.len(), 1); + assert_eq!(ctx_b.endpoints().expect("b").0.len(), 2); + } + + // Endpoints are write-once: a second set fails fast. + #[tokio::test] + #[should_panic(expected = "initialized once")] + async fn set_endpoints_twice_panics() { + let ctx = InstanceContext::new(); + ctx.set_endpoints(endpoints_with_pools(1)); + ctx.set_endpoints(endpoints_with_pools(2)); + } + + // The tier config manager is materialized once and shared for the + // instance's lifetime — the "create-once, then share" contract of the + // eager process static it replaced. + #[tokio::test] + async fn tier_config_mgr_is_stable_within_an_instance() { + let ctx = InstanceContext::new(); + assert!( + Arc::ptr_eq(&ctx.tier_config_mgr(), &ctx.tier_config_mgr()), + "repeated access must return the same manager" + ); + } + + // Two contexts own distinct tier config managers. + #[tokio::test] + async fn distinct_contexts_have_distinct_tier_config_mgr() { + let ctx_a = InstanceContext::new(); + let ctx_b = InstanceContext::new(); + assert!( + !Arc::ptr_eq(&ctx_a.tier_config_mgr(), &ctx_b.tier_config_mgr()), + "fresh contexts must not share a tier config manager" + ); + } + + // The event notifier is materialized once and shared within an instance. + #[tokio::test] + async fn event_notifier_is_stable_within_an_instance() { + let ctx = InstanceContext::new(); + assert!( + Arc::ptr_eq(&ctx.event_notifier(), &ctx.event_notifier()), + "repeated access must return the same notifier" + ); + } + + // Two contexts own distinct event notifiers. + #[tokio::test] + async fn distinct_contexts_have_distinct_event_notifier() { + let ctx_a = InstanceContext::new(); + let ctx_b = InstanceContext::new(); + assert!( + !Arc::ptr_eq(&ctx_a.event_notifier(), &ctx_b.event_notifier()), + "fresh contexts must not share an event notifier" + ); + } } diff --git a/crates/ecstore/src/runtime/sources.rs b/crates/ecstore/src/runtime/sources.rs index 6a8654093..406bd6e2b 100644 --- a/crates/ecstore/src/runtime/sources.rs +++ b/crates/ecstore/src/runtime/sources.rs @@ -32,13 +32,13 @@ use crate::{ error::Result, layout::endpoints::{EndpointServerPools, SetupType}, runtime::global::{ - GLOBAL_BOOT_TIME, GLOBAL_EVENT_NOTIFIER, GLOBAL_LIFECYCLE_SYS, GLOBAL_LOCAL_DISK_ID_MAP, GLOBAL_LOCAL_DISK_MAP, - GLOBAL_LOCAL_DISK_SET_DRIVES, GLOBAL_LOCAL_NODE_NAME_FALLBACK, GLOBAL_ROOT_DISK_THRESHOLD, GLOBAL_TIER_CONFIG_MGR, - TypeLocalDiskSetDrives, get_background_services_cancel_token, get_global_bucket_monitor, get_global_deployment_id, - get_global_endpoints, get_global_endpoints_opt, get_global_lock_client, get_global_lock_clients, get_global_region, - get_global_tier_config_mgr, global_rustfs_port, init_global_bucket_monitor, is_dist_erasure, is_erasure, is_erasure_sd, - is_first_cluster_node_local, resolve_object_store_handle, set_global_deployment_id, set_global_lock_client, - set_global_lock_clients, set_object_layer, update_erasure_type, + GLOBAL_BOOT_TIME, GLOBAL_LIFECYCLE_SYS, GLOBAL_LOCAL_DISK_ID_MAP, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, + GLOBAL_LOCAL_NODE_NAME_FALLBACK, GLOBAL_ROOT_DISK_THRESHOLD, TypeLocalDiskSetDrives, + get_background_services_cancel_token, get_global_bucket_monitor, get_global_deployment_id, get_global_endpoints, + get_global_endpoints_opt, get_global_lock_client, get_global_lock_clients, get_global_region, get_global_tier_config_mgr, + global_rustfs_port, init_global_bucket_monitor, is_dist_erasure, is_erasure, is_erasure_sd, is_first_cluster_node_local, + resolve_object_store_handle, set_global_deployment_id, set_global_lock_client, set_global_lock_clients, set_object_layer, + update_erasure_type, }, services::batch_processor::{GlobalBatchProcessors, get_global_processors}, services::event_notification::EventNotifier, @@ -387,7 +387,7 @@ pub(crate) fn local_disk_set_drives_handle() -> Arc Arc> { - GLOBAL_TIER_CONFIG_MGR.clone() + get_global_tier_config_mgr() } pub fn expiry_state_handle() -> Arc> { @@ -399,7 +399,7 @@ pub fn transition_state_handle() -> Arc { } pub(crate) fn event_notifier_handle() -> Arc> { - GLOBAL_EVENT_NOTIFIER.clone() + crate::runtime::global::current_ctx().event_notifier() } pub(crate) async fn local_disk_by_path(path: &str) -> Option { @@ -521,7 +521,7 @@ pub(crate) async fn initialize_local_disk_maps(endpoint_pools: EndpointServerPoo } pub(crate) async fn init_tier_config_mgr(store: Arc) -> Result<()> { - GLOBAL_TIER_CONFIG_MGR.write().await.init(store).await + get_global_tier_config_mgr().write().await.init(store).await } #[cfg(test)]