diff --git a/crates/config/src/constants/object.rs b/crates/config/src/constants/object.rs index 42797dda0..9502253cf 100644 --- a/crates/config/src/constants/object.rs +++ b/crates/config/src/constants/object.rs @@ -181,6 +181,22 @@ pub const DEFAULT_POOL_META_V2_FLEET_CONFIRMED: bool = false; const _: () = assert!(!DEFAULT_POOL_META_V2_WRITE); const _: () = assert!(!DEFAULT_POOL_META_V2_FLEET_CONFIRMED); +/// Request writing pool metadata version 3 with durable generations. +/// +/// Existing deployments remain on their observed version until +/// [`ENV_POOL_META_V3_FLEET_CONFIRMED`] is also enabled. Fresh deployments may +/// initialize directly at version 3 because they have no legacy readers. +pub const ENV_POOL_META_V3_WRITE: &str = "RUSTFS_POOL_META_V3_WRITE"; +pub const DEFAULT_POOL_META_V3_WRITE: bool = false; + +/// Operator-attested confirmation that every pool metadata reader and writer +/// understands the version 3 generation and recovery protocol. +pub const ENV_POOL_META_V3_FLEET_CONFIRMED: &str = "RUSTFS_POOL_META_V3_FLEET_CONFIRMED"; +pub const DEFAULT_POOL_META_V3_FLEET_CONFIRMED: bool = false; + +const _: () = assert!(!DEFAULT_POOL_META_V3_WRITE); +const _: () = assert!(!DEFAULT_POOL_META_V3_FLEET_CONFIRMED); + // ============================================================================= // Concurrent Request Fix - Timeout and Backpressure Configuration // ============================================================================= @@ -755,4 +771,10 @@ mod remote_version_state_tests { assert_eq!(super::ENV_POOL_META_V2_WRITE, "RUSTFS_POOL_META_V2_WRITE"); assert_eq!(super::ENV_POOL_META_V2_FLEET_CONFIRMED, "RUSTFS_POOL_META_V2_FLEET_CONFIRMED"); } + + #[test] + fn pool_meta_v3_gate_uses_stable_environment_names() { + assert_eq!(super::ENV_POOL_META_V3_WRITE, "RUSTFS_POOL_META_V3_WRITE"); + assert_eq!(super::ENV_POOL_META_V3_FLEET_CONFIRMED, "RUSTFS_POOL_META_V3_FLEET_CONFIRMED"); + } } diff --git a/crates/ecstore/src/config/com.rs b/crates/ecstore/src/config/com.rs index 472c87e63..98409f513 100644 --- a/crates/ecstore/src/config/com.rs +++ b/crates/ecstore/src/config/com.rs @@ -704,7 +704,12 @@ where save_config_with_opts_inner(api, file, data, opts, false).await.map(|_| ()) } -async fn save_config_with_opts_and_metadata(api: Arc, file: &str, data: Vec, opts: &ObjectOptions) -> Result +pub(crate) async fn save_config_with_opts_and_metadata( + api: Arc, + file: &str, + data: Vec, + opts: &ObjectOptions, +) -> Result where S: ObjectIO< Error = Error, diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index 576407b2e..0f188db18 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -34,7 +34,7 @@ use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw}; use crate::config::com::{ CONFIG_PREFIX, delete_config, read_config_limited_preserve_empty, read_config_limited_preserve_empty_with_metadata, read_config_no_lock_preserve_empty_with_metadata, read_config_preserve_empty, save_config_with_opts, - save_config_with_opts_quiet, + save_config_with_opts_and_metadata, }; use crate::data_movement; use crate::data_movement::backpressure::{self, DataMovementOperation}; @@ -131,9 +131,20 @@ const DECOMMISSION_DURABLE_ILM_RECEIPT_CAS_ATTEMPTS: usize = 3; const DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); pub const POOL_META_NAME: &str = "pool.bin"; +pub(crate) const POOL_META_IDENTITY_NAME: &str = "pool.bin.identity"; pub const POOL_META_FORMAT: u16 = 1; const POOL_META_V1_VERSION: u16 = 1; pub const POOL_META_VERSION: u16 = 2; +const POOL_META_GENERATION_VERSION: u16 = 3; +const POOL_META_IDENTITY_FORMAT: u16 = 1; +const POOL_META_IDENTITY_VERSION: u16 = 1; +const POOL_META_INITIAL_EPOCH: u64 = 1; +const POOL_META_CAS_MAX_ATTEMPTS: usize = 3; +const METRIC_POOL_META_STALE_WRITE_REJECTIONS_TOTAL: &str = "rustfs_pool_meta_stale_write_rejections_total"; + +fn record_pool_meta_stale_write_rejection(reason: &'static str) { + metrics::counter!(METRIC_POOL_META_STALE_WRITE_REJECTIONS_TOTAL, "reason" => reason).increment(1); +} fn pool_meta_v2_writer_enabled_for(requested: bool, fleet_confirmed: bool) -> bool { requested && fleet_confirmed @@ -149,6 +160,20 @@ fn pool_meta_v2_writer_enabled() -> bool { ) } +fn pool_meta_v3_writer_enabled_for(requested: bool, fleet_confirmed: bool) -> bool { + requested && fleet_confirmed +} + +fn pool_meta_v3_writer_enabled() -> bool { + pool_meta_v3_writer_enabled_for( + rustfs_utils::get_env_bool(rustfs_config::ENV_POOL_META_V3_WRITE, rustfs_config::DEFAULT_POOL_META_V3_WRITE), + rustfs_utils::get_env_bool( + rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, + rustfs_config::DEFAULT_POOL_META_V3_FLEET_CONFIRMED, + ), + ) +} + #[derive(Clone, Debug)] pub struct DecommissionCanceler { operation: Arc, @@ -1705,6 +1730,44 @@ fn merge_pool_meta_updates_for_save( if current_pool.id != idx || persisted_pool.id != idx || current_pool.cmd_line != persisted_pool.cmd_line { return Err(Error::other(format!("{operation}: pool metadata layout changed for pool {idx}"))); } + if current_pool.decommission.is_none() + && persisted_pool.decommission.as_ref().is_some_and(|info| { + info.has_decommission_state() && is_decommission_active(info.complete, info.failed, info.canceled) + }) + { + record_pool_meta_stale_write_rejection("nonterminal_decommission_clear"); + return Err(Error::other(format!( + "{operation}: stale pool metadata update rejected for pool {idx}; persisted active or queued decommission cannot be cleared" + ))); + } + if current_pool.last_update < persisted_pool.last_update { + record_pool_meta_stale_write_rejection("older_pool_revision"); + return Err(Error::other(format!( + "{operation}: stale pool metadata update rejected for pool {idx}; persisted update is newer" + ))); + } + if let (Some(persisted_info), Some(current_info)) = + (persisted_pool.decommission.as_ref(), current_pool.decommission.as_ref()) + { + let persisted_terminal = (persisted_info.complete, persisted_info.failed, persisted_info.canceled); + let current_terminal = (current_info.complete, current_info.failed, current_info.canceled); + if persisted_terminal != (false, false, false) && persisted_terminal != current_terminal { + record_pool_meta_stale_write_rejection("terminal_state_regression"); + return Err(Error::other(format!( + "{operation}: stale pool metadata update rejected for pool {idx}; terminal state cannot be replaced" + ))); + } + if current_info.items_decommissioned < persisted_info.items_decommissioned + || current_info.items_decommission_failed < persisted_info.items_decommission_failed + || current_info.bytes_done < persisted_info.bytes_done + || current_info.bytes_failed < persisted_info.bytes_failed + { + record_pool_meta_stale_write_rejection("progress_regression"); + return Err(Error::other(format!( + "{operation}: stale pool metadata update rejected for pool {idx}; durable progress cannot decrease" + ))); + } + } *persisted_pool = current_pool.clone(); } @@ -1712,6 +1775,7 @@ fn merge_pool_meta_updates_for_save( } fn publish_pool_meta_updates(current: &mut PoolMeta, saved: &PoolMeta, indices: &[usize]) { + current.version = current.version.max(saved.version); for &idx in indices { let Some(saved_pool) = saved.pools.get(idx) else { continue; @@ -1852,6 +1916,7 @@ struct PoolActivationDurableSaveBarrierState { pool_key: usize, arrived: tokio::sync::Notify, release: tokio::sync::Notify, + force_fence_loss: AtomicBool, } #[cfg(test)] @@ -1876,6 +1941,7 @@ impl PoolActivationDurableSaveBarrier { pool_key: pool_activation_test_pool_key(pool), arrived: tokio::sync::Notify::new(), release: tokio::sync::Notify::new(), + force_fence_loss: AtomicBool::new(false), }); let mut barrier = POOL_ACTIVATION_DURABLE_SAVE_BARRIER .get_or_init(|| std::sync::Mutex::new(None)) @@ -1893,6 +1959,11 @@ impl PoolActivationDurableSaveBarrier { } pub(crate) fn release_after_fence_loss(&self) { + self.state.force_fence_loss.store(true, Ordering::Release); + self.state.release.notify_one(); + } + + pub(crate) fn release_without_fence_loss(&self) { self.state.release.notify_one(); } } @@ -1928,7 +1999,9 @@ pub(crate) async fn pause_pool_activation_after_durable_save(pool: &Arc, f if let Some(barrier) = barrier { barrier.arrived.notify_one(); barrier.release.notified().await; - fence.force_lost_for_test(); + if barrier.force_fence_loss.load(Ordering::Acquire) { + fence.force_lost_for_test(); + } } } @@ -2709,6 +2782,38 @@ pub struct PoolMeta { pub dont_save: bool, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct PoolMetaRevision { + version: u16, + cluster_id: Option, + epoch: u64, + generation: u64, + transaction_id: Option, +} + +impl PoolMetaRevision { + fn legacy(version: u16) -> Self { + Self { + version, + cluster_id: None, + epoch: 0, + generation: 0, + transaction_id: None, + } + } + + fn is_generation_protocol(self) -> bool { + self.version == POOL_META_GENERATION_VERSION + } +} + +#[derive(Debug, Clone)] +struct PoolMetaCommittedCandidate { + canonical: Vec, + meta: PoolMeta, + revision: PoolMetaRevision, +} + #[derive(Debug)] enum PoolMetaReplica { Missing, @@ -2716,12 +2821,37 @@ enum PoolMetaReplica { raw: Vec, canonical: Vec, meta: PoolMeta, + revision: PoolMetaRevision, + committed: bool, + previous: Option>, }, Corrupt(String), Incompatible(String), Unreadable(String), } +#[derive(Debug, Clone)] +enum PoolMetaCasToken { + Missing, + Existing(String), + Unsafe, +} + +#[derive(Debug)] +struct PoolMetaReplicaRead { + replica: PoolMetaReplica, + cas: PoolMetaCasToken, +} + +impl From for PoolMetaReplicaRead { + fn from(replica: PoolMetaReplica) -> Self { + Self { + replica, + cas: PoolMetaCasToken::Unsafe, + } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct PoolMetaReplicaState { pub(crate) needs_repair: bool, @@ -2739,12 +2869,51 @@ impl PoolMetaReplicaState { } } -#[derive(Debug, Clone, Copy, Default)] +#[derive(Debug, Clone, Default)] pub(crate) struct PoolMetaWriteState { write_blocked: bool, + aborted_transaction: Arc, + expected_cluster_id: Option, + cluster_epoch: Option, + pool_meta_absent: bool, + fresh_bootstrap_proven: bool, + identity_initialized: Option, + identity_fresh_bootstrap_nonce: Option, + identity_needs_repair: bool, } impl PoolMetaWriteState { + pub(crate) fn for_startup(cluster_id: uuid::Uuid, fresh_bootstrap_proven: bool) -> Self { + Self { + expected_cluster_id: Some(cluster_id), + fresh_bootstrap_proven, + ..Default::default() + } + } + + pub(crate) fn fresh_bootstrap_proven(&self) -> bool { + self.fresh_bootstrap_proven + } + + pub(crate) fn identity_is_pending(&self) -> bool { + self.identity_initialized == Some(false) + } + + #[cfg(test)] + pub(crate) fn aborted_transaction_latch_for_test(&self) -> Arc { + Arc::clone(&self.aborted_transaction) + } + + #[cfg(any(test, feature = "test-util"))] + fn for_test_bootstrap() -> Self { + Self { + fresh_bootstrap_proven: true, + identity_initialized: Some(false), + identity_fresh_bootstrap_nonce: Some(uuid::Uuid::new_v4()), + ..Default::default() + } + } + pub(crate) fn observe_replicas(&mut self, replica_state: PoolMetaReplicaState) { self.write_blocked |= !replica_state.repair_write_safe; } @@ -2753,12 +2922,104 @@ impl PoolMetaWriteState { self.write_blocked = true; } - fn restore_writes(&mut self, was_write_blocked: bool) { - self.write_blocked = was_write_blocked; + pub(crate) fn block_writes_after_fence_loss(&mut self) { + self.block_writes(); } - pub(crate) fn ensure_write_safe(self, operation: &str) -> Result<()> { - if !self.write_blocked { + fn arm_transaction(&self) -> PoolMetaTransactionArm { + PoolMetaTransactionArm { + aborted_transaction: Arc::clone(&self.aborted_transaction), + armed: true, + } + } + + fn observe_selection(&mut self, selection: &PoolMetaSelection) -> Result<()> { + self.pool_meta_absent = selection.absent; + if let Some(expected_cluster_id) = self.expected_cluster_id + && let Some((cluster_id, _)) = selection.generation_identity + && cluster_id != expected_cluster_id + { + self.block_writes(); + return Err(Error::other(format!( + "pool metadata incompatible: cluster identity {cluster_id} does not match deployment {expected_cluster_id}" + ))); + } + if let Some(identity_epoch) = self.cluster_epoch + && let Some((_, metadata_epoch)) = selection.generation_identity + && metadata_epoch != identity_epoch + { + self.block_writes(); + return Err(Error::other(format!( + "pool metadata recovery required: committed epoch {} does not match cluster identity epoch {identity_epoch}", + metadata_epoch + ))); + } + if self.cluster_epoch.is_none() + && let Some((_, metadata_epoch)) = selection.generation_identity + { + self.cluster_epoch = Some(metadata_epoch); + } + Ok(()) + } + + fn observe_identity(&mut self, selection: &PoolMetaIdentitySelection) -> Result<()> { + self.identity_needs_repair = selection.needs_repair; + self.identity_initialized = selection.identity.map(|identity| identity.initialized); + self.identity_fresh_bootstrap_nonce = selection.identity.and_then(|identity| identity.fresh_bootstrap_nonce); + if let Some(identity) = selection.identity { + if identity.initialized { + self.fresh_bootstrap_proven = false; + } + if let Some(metadata_epoch) = self.cluster_epoch + && metadata_epoch != identity.epoch + { + self.block_writes(); + return Err(Error::other(format!( + "pool metadata recovery required: metadata epoch {metadata_epoch} does not match cluster identity epoch {}", + identity.epoch + ))); + } + self.cluster_epoch = Some(identity.epoch); + } + if !selection.repair_write_safe { + self.block_writes(); + } + Ok(()) + } + + pub(crate) fn ensure_missing_metadata_can_initialize(&mut self) -> Result<()> { + if !self.pool_meta_absent { + return Ok(()); + } + match self.identity_initialized { + Some(false) if self.fresh_bootstrap_proven && self.identity_fresh_bootstrap_nonce.is_some() => Ok(()), + Some(false) => { + self.block_writes(); + Err(Error::other( + "pool metadata recovery required: pending cluster identity exists but this startup has no verified fresh-bootstrap proof", + )) + } + Some(true) => { + self.block_writes(); + Err(Error::other( + "pool metadata recovery required: initialized cluster identity exists but every pool.bin replica is missing", + )) + } + None => { + self.block_writes(); + Err(Error::other( + "pool metadata recovery required: no durable bootstrap identity or pool.bin replica is available", + )) + } + } + } + + pub(crate) fn identity_requires_repair(&self) -> bool { + self.expected_cluster_id.is_some() && (self.identity_needs_repair || self.identity_initialized != Some(true)) + } + + pub(crate) fn ensure_write_safe(&self, operation: &str) -> Result<()> { + if !self.write_blocked && !self.aborted_transaction.load(Ordering::SeqCst) { return Ok(()); } Err(Error::other(format!( @@ -2767,10 +3028,36 @@ impl PoolMetaWriteState { } } +#[derive(Debug)] +struct PoolMetaTransactionArm { + aborted_transaction: Arc, + armed: bool, +} + +impl PoolMetaTransactionArm { + fn disarm(&mut self) { + self.armed = false; + } +} + +impl Drop for PoolMetaTransactionArm { + fn drop(&mut self) { + if self.armed { + self.aborted_transaction.store(true, Ordering::SeqCst); + } + } +} + #[derive(Debug)] struct PoolMetaSelection { meta: PoolMeta, + revision: PoolMetaRevision, + canonical: Option>, replica_state: PoolMetaReplicaState, + cas_tokens: Vec, + absent: bool, + generation_protocol_observed: bool, + generation_identity: Option<(uuid::Uuid, u64)>, } fn classify_pool_meta_tuple_decode_error(kind: &str, err: rmp_serde::decode::Error) -> PoolMetaReplica { @@ -2787,6 +3074,159 @@ fn classify_pool_meta_tuple_decode_error(kind: &str, err: rmp_serde::decode::Err } } +fn parse_pool_meta_uuid(value: &str, field: &str) -> Result { + let parsed = uuid::Uuid::parse_str(value) + .map_err(|err| Error::other(format!("pool metadata corrupt: invalid {field} `{value}`: {err}")))?; + if parsed.is_nil() || parsed == uuid::Uuid::max() { + return Err(Error::other(format!("pool metadata corrupt: {field} must be a non-reserved UUID"))); + } + Ok(parsed) +} + +fn pool_meta_generation_revision( + cluster_id: &str, + epoch: u64, + generation: u64, + transaction_id: &str, +) -> Result { + if epoch == 0 || generation == 0 { + return Err(Error::other( + "pool metadata corrupt: version 3 epoch and generation must both be non-zero", + )); + } + Ok(PoolMetaRevision { + version: POOL_META_GENERATION_VERSION, + cluster_id: Some(parse_pool_meta_uuid(cluster_id, "cluster identity")?), + epoch, + generation, + transaction_id: Some(parse_pool_meta_uuid(transaction_id, "transaction id")?), + }) +} + +fn pool_meta_from_v3_statuses(version: u16, pools: Vec) -> Result { + if !matches!(version, POOL_META_V1_VERSION | POOL_META_VERSION | POOL_META_GENERATION_VERSION) { + return Err(Error::other(format!( + "pool metadata corrupt: version 3 previous snapshot has unsupported source version {version}" + ))); + } + Ok(PoolMeta { + version, + pools: pools.into_iter().map(TryInto::try_into).collect::>>()?, + dont_save: false, + }) +} + +fn pool_meta_previous_candidate(value: PersistedPoolMetaV3Previous) -> Result { + let revision = match value.version { + POOL_META_V1_VERSION | POOL_META_VERSION => { + if value.cluster_id.is_some() || value.epoch != 0 || value.generation != 0 || value.transaction_id.is_some() { + return Err(Error::other( + "pool metadata corrupt: legacy previous snapshot carries version 3 revision fields", + )); + } + PoolMetaRevision::legacy(value.version) + } + POOL_META_GENERATION_VERSION => pool_meta_generation_revision( + value + .cluster_id + .as_deref() + .ok_or_else(|| Error::other("pool metadata corrupt: version 3 previous snapshot has no cluster identity"))?, + value.epoch, + value.generation, + value + .transaction_id + .as_deref() + .ok_or_else(|| Error::other("pool metadata corrupt: version 3 previous snapshot has no transaction id"))?, + )?, + version => { + return Err(Error::other(format!( + "pool metadata corrupt: previous snapshot has unsupported version {version}" + ))); + } + }; + let meta = pool_meta_from_v3_statuses(value.version, value.pools)?; + let canonical = if revision.is_generation_protocol() { + encode_pool_meta_v3_envelope(&meta, revision, true, None)? + } else { + meta.encode_config_data_for_v2_gate(true)? + }; + Ok(PoolMetaCommittedCandidate { + canonical, + meta, + revision, + }) +} + +fn decode_pool_meta_v3(data: Vec) -> PoolMetaReplica { + let persisted = match rmp_serde::from_slice::(&data[4..]) { + Ok(persisted) => persisted, + Err(err) => return classify_pool_meta_tuple_decode_error("v3", err), + }; + if persisted.version != POOL_META_GENERATION_VERSION { + return PoolMetaReplica::Corrupt(format!( + "v3 payload has version {}, expected {POOL_META_GENERATION_VERSION}", + persisted.version + )); + } + let revision = match pool_meta_generation_revision( + &persisted.cluster_id, + persisted.epoch, + persisted.generation, + &persisted.transaction_id, + ) { + Ok(revision) => revision, + Err(err) => return PoolMetaReplica::Corrupt(err.to_string()), + }; + let meta = match pool_meta_from_v3_statuses(POOL_META_GENERATION_VERSION, persisted.pools) { + Ok(meta) => meta, + Err(err) => return PoolMetaReplica::Corrupt(err.to_string()), + }; + let previous = match persisted.previous.map(pool_meta_previous_candidate).transpose() { + Ok(previous) => previous, + Err(err) => return PoolMetaReplica::Corrupt(err.to_string()), + }; + if persisted.committed && previous.is_some() { + return PoolMetaReplica::Corrupt("committed v3 payload unexpectedly retains a previous snapshot".to_string()); + } + if !persisted.committed { + match previous.as_ref() { + Some(previous) if previous.revision.is_generation_protocol() => { + if previous.revision.cluster_id != revision.cluster_id + || previous.revision.epoch != revision.epoch + || previous.revision.generation.checked_add(1) != Some(revision.generation) + { + return PoolMetaReplica::Corrupt( + "pending v3 payload does not advance exactly one generation from its previous snapshot".to_string(), + ); + } + } + Some(_) if revision.generation != 1 => { + return PoolMetaReplica::Corrupt( + "first v3 generation must be generation 1 when migrating a legacy snapshot".to_string(), + ); + } + None if revision.generation != 1 => { + return PoolMetaReplica::Corrupt( + "pending v3 payload without a previous snapshot must be the initial generation".to_string(), + ); + } + _ => {} + } + } + let canonical = match encode_pool_meta_v3_envelope(&meta, revision, true, None) { + Ok(canonical) => canonical, + Err(err) => return PoolMetaReplica::Corrupt(err.to_string()), + }; + PoolMetaReplica::Valid { + raw: data, + canonical, + meta, + revision, + committed: persisted.committed, + previous: previous.map(Box::new), + } +} + fn decode_pool_meta_replica(data: Vec) -> PoolMetaReplica { if data.len() <= 4 { return PoolMetaReplica::Corrupt("metadata payload is empty or truncated".to_string()); @@ -2797,9 +3237,12 @@ fn decode_pool_meta_replica(data: Vec) -> PoolMetaReplica { return PoolMetaReplica::Incompatible(format!("unsupported format {format}")); } let version = LittleEndian::read_u16(&data[2..4]); - if !matches!(version, POOL_META_V1_VERSION | POOL_META_VERSION) { + if !matches!(version, POOL_META_V1_VERSION | POOL_META_VERSION | POOL_META_GENERATION_VERSION) { return PoolMetaReplica::Incompatible(format!("unsupported version {version}")); } + if version == POOL_META_GENERATION_VERSION { + return decode_pool_meta_v3(data); + } let payload = &data[4..]; let meta = match (version, rmp::decode::read_array_len(&mut &payload[..])) { @@ -2851,47 +3294,73 @@ fn decode_pool_meta_replica(data: Vec) -> PoolMetaReplica { raw: data, canonical, meta, + revision: PoolMetaRevision::legacy(version), + committed: true, + previous: None, }, Err(err) => PoolMetaReplica::Corrupt(err.to_string()), } } -async fn read_pool_meta_replica(pool: Arc, no_lock: bool) -> PoolMetaReplica +#[cfg(test)] +pub(crate) fn pool_meta_v3_commit_state_for_test(data: Vec) -> Result<(u64, bool)> { + match decode_pool_meta_replica(data) { + PoolMetaReplica::Valid { revision, committed, .. } if revision.is_generation_protocol() => { + Ok((revision.generation, committed)) + } + _ => Err(Error::other("test pool metadata is not a valid V3 replica")), + } +} + +async fn read_pool_meta_replica(pool: Arc, no_lock: bool) -> PoolMetaReplicaRead where S: EcstoreObjectIO, { let result = if no_lock { read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_NAME) .await - .map(|(data, _)| data) + .map(|(data, object_info)| (data, object_info.etag)) } else { - read_config_preserve_empty(pool, POOL_META_NAME).await + read_config_preserve_empty(pool, POOL_META_NAME) + .await + .map(|data| (data, None)) }; match result { - Ok(data) => decode_pool_meta_replica(data), - Err(Error::ConfigNotFound) => PoolMetaReplica::Missing, - Err(err) => PoolMetaReplica::Unreadable(err.to_string()), + Ok((data, etag)) => PoolMetaReplicaRead { + replica: decode_pool_meta_replica(data), + cas: etag + .filter(|etag| !etag.trim().is_empty()) + .map(PoolMetaCasToken::Existing) + .unwrap_or(PoolMetaCasToken::Unsafe), + }, + Err(Error::ConfigNotFound) => PoolMetaReplicaRead { + replica: PoolMetaReplica::Missing, + cas: PoolMetaCasToken::Missing, + }, + Err(err) => PoolMetaReplicaRead { + replica: PoolMetaReplica::Unreadable(err.to_string()), + cas: PoolMetaCasToken::Unsafe, + }, } } -fn select_pool_meta_replica(replicas: Vec) -> Result { - if replicas.is_empty() { +fn select_pool_meta_replica_reads(reads: Vec) -> Result { + if reads.is_empty() { return Err(Error::other("pool metadata recovery required: no storage pools available")); } - // Pool metadata has no durable generation. Pool zero remains the canonical - // commit record because every writer saves it first and legacy startup read - // it alone. Divergent backups are ambiguous when pool zero is unavailable - // and require an operator-selected recovery source. - let mut selected: Option<(usize, Vec, Vec, PoolMeta)> = None; + let cas_tokens = reads.iter().map(|read| read.cas.clone()).collect(); + let mut committed = Vec::<(usize, Vec, PoolMetaCommittedCandidate)>::new(); let mut needs_repair = false; let mut repair_write_safe = true; let mut missing = 0usize; let mut unusable = Vec::new(); let mut observed_version = POOL_META_V1_VERSION; + let mut generation_protocol_observed = false; + let mut generation_identity = None; - for (idx, replica) in replicas.into_iter().enumerate() { - match replica { + for (idx, read) in reads.into_iter().enumerate() { + match read.replica { PoolMetaReplica::Missing => { missing += 1; needs_repair = true; @@ -2910,87 +3379,212 @@ fn select_pool_meta_replica(replicas: Vec) -> Result { - observed_version = observed_version.max(meta.version); - if let Some((selected_idx, selected_raw, selected_canonical, selected_meta)) = selected.as_ref() { - if selected_canonical != &canonical { - if selected_meta.version == meta.version && *selected_idx == 0 { - // Pool zero is the durable commit record: every - // pool metadata writer commits it before replicas, - // and pre-replica-recovery startup read it alone. - // This ordering is safe only within one format - // version because V1 cannot preserve V2-only data. - needs_repair = true; - } else { - return Err(Error::other(format!( - "pool metadata recovery required: valid replicas in pools {selected_idx} and {idx} diverge; restore one matching pool.bin snapshot before restart" - ))); - } - } else { - needs_repair |= selected_raw != &raw; + PoolMetaReplica::Valid { + raw, + canonical, + meta, + revision, + committed: is_committed, + previous, + } => { + generation_protocol_observed |= revision.is_generation_protocol(); + if revision.is_generation_protocol() { + let identity = ( + revision + .cluster_id + .expect("validated V3 revision should carry a cluster identity"), + revision.epoch, + ); + if generation_identity.is_some_and(|current| current != identity) { + return Err(Error::other( + "pool metadata recovery required: V3 replicas disagree on cluster identity or epoch", + )); } + generation_identity = Some(identity); + } + if is_committed { + observed_version = observed_version.max(meta.version); + committed.push(( + idx, + raw, + PoolMetaCommittedCandidate { + canonical, + meta, + revision, + }, + )); + } else if let Some(previous) = previous { + observed_version = observed_version.max(previous.meta.version); + needs_repair = true; + committed.push((idx, raw, *previous)); } else { - selected = Some((idx, raw, canonical, meta)); + needs_repair = true; + unusable.push(format!("pool {idx} contains an uncommitted initial generation")); } } } } - if let Some((_, _, _, mut meta)) = selected { - meta.version = observed_version; - return Ok(PoolMetaSelection { - meta, - replica_state: PoolMetaReplicaState { - needs_repair, - repair_write_safe, - }, - }); - } - if missing > 0 && unusable.is_empty() { + if committed.is_empty() && missing > 0 && unusable.is_empty() { return Ok(PoolMetaSelection { meta: PoolMeta::default(), + revision: PoolMetaRevision::legacy(0), + canonical: None, replica_state: PoolMetaReplicaState { needs_repair: false, repair_write_safe: true, }, + cas_tokens, + absent: true, + generation_protocol_observed, + generation_identity, + }); + } + if committed.is_empty() { + return Err(Error::other(format!( + "pool metadata recovery required: no valid committed replica is available ({})", + unusable.join("; ") + ))); + } + + if committed + .iter() + .any(|(_, _, candidate)| candidate.revision.is_generation_protocol()) + { + let highest = committed + .iter() + .filter(|(_, _, candidate)| candidate.revision.is_generation_protocol()) + .map(|(_, _, candidate)| candidate.revision.generation) + .max() + .ok_or_else(|| Error::other("pool metadata recovery required: no committed V3 generation is available"))?; + let mut selected: Option<(usize, &PoolMetaCommittedCandidate)> = None; + for (idx, _, candidate) in &committed { + if !candidate.revision.is_generation_protocol() || candidate.revision.generation != highest { + needs_repair = true; + continue; + } + if let Some((selected_idx, selected_candidate)) = selected { + if selected_candidate.canonical != candidate.canonical + || selected_candidate.revision.transaction_id != candidate.revision.transaction_id + { + return Err(Error::other(format!( + "pool metadata recovery required: committed generation {highest} diverges between pools {selected_idx} and {idx}" + ))); + } + } else { + selected = Some((*idx, candidate)); + } + } + let (_, selected) = selected + .ok_or_else(|| Error::other("pool metadata recovery required: highest V3 generation has no valid snapshot"))?; + for (_, raw, candidate) in &committed { + needs_repair |= candidate.canonical != selected.canonical || raw != &selected.canonical; + } + return Ok(PoolMetaSelection { + meta: selected.meta.clone(), + revision: selected.revision, + canonical: Some(selected.canonical.clone()), + replica_state: PoolMetaReplicaState { + needs_repair, + repair_write_safe, + }, + cas_tokens, + absent: false, + generation_protocol_observed, + generation_identity, }); } - Err(Error::other(format!( - "pool metadata recovery required: no valid replica is available ({})", - unusable.join("; ") - ))) + // Legacy V1/V2 has no durable generation. Pool zero remains the commit + // record; divergent backups without it are ambiguous and fail closed. + let mut selected: Option<(usize, Vec, PoolMetaCommittedCandidate)> = None; + for (idx, raw, candidate) in committed { + if let Some((selected_idx, selected_raw, selected_candidate)) = selected.as_ref() { + if selected_candidate.canonical != candidate.canonical { + if selected_candidate.meta.version == candidate.meta.version && *selected_idx == 0 { + needs_repair = true; + } else { + return Err(Error::other(format!( + "pool metadata recovery required: valid replicas in pools {selected_idx} and {idx} diverge; restore one matching pool.bin snapshot before restart" + ))); + } + } else { + needs_repair |= selected_raw != &raw; + } + } else { + selected = Some((idx, raw, candidate)); + } + } + let (_, _, mut selected) = + selected.ok_or_else(|| Error::other("pool metadata recovery required: no valid legacy replica is available"))?; + selected.meta.version = observed_version; + selected.revision.version = observed_version; + Ok(PoolMetaSelection { + meta: selected.meta, + revision: selected.revision, + canonical: Some(selected.canonical), + replica_state: PoolMetaReplicaState { + needs_repair, + repair_write_safe, + }, + cas_tokens, + absent: false, + generation_protocol_observed, + generation_identity, + }) } -async fn read_pool_meta_replicas(pools: Vec>, no_lock: bool) -> Vec +#[cfg(test)] +fn select_pool_meta_replica(replicas: Vec) -> Result { + select_pool_meta_replica_reads( + replicas + .into_iter() + .map(|replica| PoolMetaReplicaRead { + replica, + cas: PoolMetaCasToken::Unsafe, + }) + .collect(), + ) +} + +async fn read_pool_meta_replicas(pools: Vec>, no_lock: bool) -> Vec where S: EcstoreObjectIO, { join_all(pools.into_iter().map(|pool| read_pool_meta_replica(pool, no_lock))).await } -fn select_pool_meta_replicas_observing( - write_state: &mut PoolMetaWriteState, - replicas: Vec, -) -> Result { +fn select_pool_meta_replicas_observing(write_state: &mut PoolMetaWriteState, replicas: Vec) -> Result +where + R: Into, +{ + let replicas = replicas.into_iter().map(Into::into).collect::>(); if replicas .iter() - .any(|replica| matches!(replica, PoolMetaReplica::Unreadable(_))) + .any(|replica| matches!(&replica.replica, PoolMetaReplica::Unreadable(_))) { write_state.block_writes(); } - let selection = select_pool_meta_replica(replicas); - if selection.is_err() { - write_state.block_writes(); + match select_pool_meta_replica_reads(replicas) { + Ok(selection) => { + if let Err(err) = write_state.observe_selection(&selection) { + write_state.block_writes(); + return Err(err); + } + Ok(selection) + } + Err(err) => { + write_state.block_writes(); + Err(err) + } } - selection } async fn load_pool_meta_replicas(pools: Vec>, no_lock: bool) -> Result where S: EcstoreObjectIO, { - select_pool_meta_replica(read_pool_meta_replicas(pools, no_lock).await) + select_pool_meta_replica_reads(read_pool_meta_replicas(pools, no_lock).await) } async fn load_pool_meta_replicas_observing( @@ -3005,6 +3599,451 @@ where select_pool_meta_replicas_observing(write_state, replicas) } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PersistedPoolMetaV3 { + version: u16, + cluster_id: String, + epoch: u64, + generation: u64, + transaction_id: String, + committed: bool, + pools: Vec, + previous: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PersistedPoolMetaV3Previous { + version: u16, + cluster_id: Option, + epoch: u64, + generation: u64, + transaction_id: Option, + pools: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PersistedPoolMetaIdentity { + version: u16, + cluster_id: uuid::Uuid, + epoch: u64, + initialized: bool, + fresh_bootstrap_nonce: Option, +} + +#[derive(Debug)] +enum PoolMetaIdentityReplica { + Missing, + Valid(PersistedPoolMetaIdentity), + Corrupt(String), + Incompatible(String), + Unreadable(String), +} + +#[derive(Debug)] +struct PoolMetaIdentityRead { + replica: PoolMetaIdentityReplica, + cas: PoolMetaCasToken, +} + +#[derive(Debug)] +struct PoolMetaIdentitySelection { + identity: Option, + needs_repair: bool, + repair_write_safe: bool, + cas_tokens: Vec, +} + +fn encode_pool_meta_identity(identity: PersistedPoolMetaIdentity) -> Result> { + let mut data = Vec::new(); + data.write_u16::(POOL_META_IDENTITY_FORMAT)?; + data.write_u16::(POOL_META_IDENTITY_VERSION)?; + identity.serialize(&mut Serializer::new(&mut data))?; + Ok(data) +} + +fn decode_pool_meta_identity(data: &[u8]) -> PoolMetaIdentityReplica { + if data.len() <= 4 { + return PoolMetaIdentityReplica::Corrupt("identity payload is empty or truncated".to_string()); + } + let format = LittleEndian::read_u16(&data[0..2]); + let version = LittleEndian::read_u16(&data[2..4]); + if format != POOL_META_IDENTITY_FORMAT || version != POOL_META_IDENTITY_VERSION { + return PoolMetaIdentityReplica::Incompatible(format!("unsupported identity format {format} version {version}")); + } + let identity = match rmp_serde::from_slice::(&data[4..]) { + Ok(identity) => identity, + Err(err) => return PoolMetaIdentityReplica::Corrupt(format!("identity payload is not decodable: {err}")), + }; + let invalid_bootstrap_nonce = identity + .fresh_bootstrap_nonce + .is_some_and(|nonce| nonce.is_nil() || nonce == uuid::Uuid::max()); + if identity.version != POOL_META_IDENTITY_VERSION + || identity.cluster_id.is_nil() + || identity.cluster_id == uuid::Uuid::max() + || identity.epoch == 0 + || invalid_bootstrap_nonce + || identity.initialized == identity.fresh_bootstrap_nonce.is_some() + { + return PoolMetaIdentityReplica::Corrupt( + "identity payload contains an invalid version, UUID, epoch, or fresh-bootstrap proof".to_string(), + ); + } + PoolMetaIdentityReplica::Valid(identity) +} + +#[cfg(test)] +pub(crate) fn pool_meta_identity_initialized_for_test(data: &[u8]) -> Result { + match decode_pool_meta_identity(data) { + PoolMetaIdentityReplica::Valid(identity) => Ok(identity.initialized), + _ => Err(Error::other("test pool metadata identity is not valid")), + } +} + +#[cfg(test)] +pub(crate) fn initialized_pool_meta_identity_for_test(cluster_id: uuid::Uuid, epoch: u64) -> Result> { + encode_pool_meta_identity(PersistedPoolMetaIdentity { + version: POOL_META_IDENTITY_VERSION, + cluster_id, + epoch, + initialized: true, + fresh_bootstrap_nonce: None, + }) +} + +async fn read_pool_meta_identity_replica(pool: Arc) -> PoolMetaIdentityRead +where + S: EcstoreObjectIO, +{ + match read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_IDENTITY_NAME).await { + Ok((data, object_info)) => PoolMetaIdentityRead { + replica: decode_pool_meta_identity(&data), + cas: object_info + .etag + .filter(|etag| !etag.trim().is_empty()) + .map(PoolMetaCasToken::Existing) + .unwrap_or(PoolMetaCasToken::Unsafe), + }, + Err(Error::ConfigNotFound) => PoolMetaIdentityRead { + replica: PoolMetaIdentityReplica::Missing, + cas: PoolMetaCasToken::Missing, + }, + Err(err) => PoolMetaIdentityRead { + replica: PoolMetaIdentityReplica::Unreadable(err.to_string()), + cas: PoolMetaCasToken::Unsafe, + }, + } +} + +fn select_pool_meta_identity( + reads: Vec, + expected_cluster_id: uuid::Uuid, +) -> Result { + let cas_tokens = reads.iter().map(|read| read.cas.clone()).collect(); + let mut selected: Option = None; + let mut needs_repair = false; + let mut repair_write_safe = true; + let mut unusable = Vec::new(); + for (idx, read) in reads.into_iter().enumerate() { + match read.replica { + PoolMetaIdentityReplica::Missing => needs_repair = true, + PoolMetaIdentityReplica::Corrupt(reason) => { + needs_repair = true; + unusable.push(format!("pool {idx} identity is corrupt: {reason}")); + } + PoolMetaIdentityReplica::Unreadable(reason) => { + needs_repair = true; + repair_write_safe = false; + unusable.push(format!("pool {idx} identity is unreadable: {reason}")); + } + PoolMetaIdentityReplica::Incompatible(reason) => { + return Err(Error::other(format!("pool metadata incompatible: pool {idx} identity uses {reason}"))); + } + PoolMetaIdentityReplica::Valid(identity) => { + if identity.cluster_id != expected_cluster_id { + return Err(Error::other(format!( + "pool metadata incompatible: pool {idx} identity {} does not match deployment {expected_cluster_id}", + identity.cluster_id + ))); + } + if let Some(current) = selected { + if current.cluster_id != identity.cluster_id || current.epoch != identity.epoch { + return Err(Error::other( + "pool metadata recovery required: identity replicas disagree on cluster identity or epoch", + )); + } + if current.initialized != identity.initialized { + needs_repair = true; + selected = Some(if current.initialized { current } else { identity }); + } else if !current.initialized && current.fresh_bootstrap_nonce != identity.fresh_bootstrap_nonce { + return Err(Error::other( + "pool metadata recovery required: pending identity replicas disagree on fresh-bootstrap proof", + )); + } + } else { + selected = Some(identity); + } + } + } + } + if selected.is_none() && !unusable.is_empty() { + return Err(Error::other(format!( + "pool metadata recovery required: no valid cluster identity replica is available ({})", + unusable.join("; ") + ))); + } + Ok(PoolMetaIdentitySelection { + identity: selected, + needs_repair, + repair_write_safe, + cas_tokens, + }) +} + +async fn load_pool_meta_identity(pools: Vec>, expected_cluster_id: uuid::Uuid) -> Result +where + S: EcstoreObjectIO, +{ + let reads = join_all(pools.into_iter().map(read_pool_meta_identity_replica)).await; + select_pool_meta_identity(reads, expected_cluster_id) +} + +async fn load_pool_meta_identity_selection_observing( + pools: Vec>, + write_state: &mut PoolMetaWriteState, + expected_cluster_id: uuid::Uuid, +) -> Result +where + S: EcstoreObjectIO, +{ + let selection = match load_pool_meta_identity(pools, expected_cluster_id).await { + Ok(selection) => selection, + Err(err) => { + write_state.block_writes(); + return Err(err); + } + }; + if let Err(err) = write_state.observe_identity(&selection) { + write_state.block_writes(); + return Err(err); + } + Ok(selection) +} + +fn persisted_pool_meta_v3_previous(candidate: &PoolMetaCommittedCandidate) -> PersistedPoolMetaV3Previous { + PersistedPoolMetaV3Previous { + version: candidate.revision.version, + cluster_id: candidate.revision.cluster_id.map(|id| id.to_string()), + epoch: candidate.revision.epoch, + generation: candidate.revision.generation, + transaction_id: candidate.revision.transaction_id.map(|id| id.to_string()), + pools: candidate.meta.pools.iter().map(Into::into).collect(), + } +} + +fn encode_pool_meta_v3_envelope( + meta: &PoolMeta, + revision: PoolMetaRevision, + committed: bool, + previous: Option<&PoolMetaCommittedCandidate>, +) -> Result> { + if meta.dont_save { + return Ok(Vec::new()); + } + let cluster_id = revision + .cluster_id + .ok_or_else(|| Error::other("pool metadata V3 save failed: cluster identity is not initialized"))?; + let transaction_id = revision + .transaction_id + .ok_or_else(|| Error::other("pool metadata V3 save failed: transaction id is not initialized"))?; + if revision.version != POOL_META_GENERATION_VERSION || revision.epoch == 0 || revision.generation == 0 { + return Err(Error::other("pool metadata V3 save failed: invalid durable revision")); + } + let persisted = PersistedPoolMetaV3 { + version: POOL_META_GENERATION_VERSION, + cluster_id: cluster_id.to_string(), + epoch: revision.epoch, + generation: revision.generation, + transaction_id: transaction_id.to_string(), + committed, + pools: meta.pools.iter().map(Into::into).collect(), + previous: previous.map(persisted_pool_meta_v3_previous), + }; + let mut data = Vec::new(); + data.write_u16::(POOL_META_FORMAT)?; + data.write_u16::(POOL_META_GENERATION_VERSION)?; + persisted.serialize(&mut Serializer::new(&mut data))?; + Ok(data) +} + +enum PoolMetaPersistenceFence<'a> { + Distributed(Option>), + Activation(&'a PoolRebalanceActivationFence), +} + +impl PoolMetaPersistenceFence<'_> { + fn ensure_held(&self) -> Result<()> { + match self { + Self::Distributed(Some(signal)) if signal.is_lost() => { + Err(Error::other("pool metadata distributed fence was lost before a replica write")) + } + Self::Activation(fence) => fence.ensure_held(), + _ => Ok(()), + } + } + + fn add_to_options(&self, opts: &mut ObjectOptions) { + match self { + Self::Distributed(Some(signal)) => opts.add_namespace_lock_lost_signal(Arc::clone(signal)), + Self::Activation(fence) => fence.add_namespace_lock_fence(opts), + Self::Distributed(None) => {} + } + } + + fn is_activation(&self) -> bool { + matches!(self, Self::Activation(_)) + } +} + +fn pool_meta_cas_preconditions(token: &PoolMetaCasToken, object: &str) -> Result { + match token { + PoolMetaCasToken::Missing => Ok(HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }), + PoolMetaCasToken::Existing(etag) => Ok(HTTPPreconditions { + if_match: Some(etag.clone()), + ..Default::default() + }), + PoolMetaCasToken::Unsafe => Err(Error::other(format!( + "pool metadata recovery required: {object} replica has no safe conditional-write revision" + ))), + } +} + +async fn save_pool_meta_object_cas( + pool: Arc, + object: &str, + data: Vec, + token: &PoolMetaCasToken, + fence: &PoolMetaPersistenceFence<'_>, + phase: &'static str, +) -> Result +where + S: EcstoreObjectIO, +{ + fence.ensure_held()?; + let mut opts = ObjectOptions { + max_parity: true, + no_lock: true, + http_preconditions: Some(pool_meta_cas_preconditions(token, object)?), + ..Default::default() + }; + fence.add_to_options(&mut opts); + let result = save_config_with_opts_and_metadata(pool, object, data, &opts).await; + if matches!(&result, Err(Error::PreconditionFailed)) { + record_pool_meta_stale_write_rejection(phase); + } + let object_info = result?; + fence.ensure_held()?; + Ok(object_info) +} + +async fn persist_pool_meta_identity( + pools: Vec>, + write_state: &mut PoolMetaWriteState, + initialized: bool, + fence: &PoolMetaPersistenceFence<'_>, +) -> Result<()> +where + S: EcstoreObjectIO, +{ + let Some(cluster_id) = write_state.expected_cluster_id else { + return Ok(()); + }; + for attempt in 0..POOL_META_CAS_MAX_ATTEMPTS { + let selection = load_pool_meta_identity_selection_observing(pools.clone(), write_state, cluster_id).await?; + if !selection.repair_write_safe { + write_state.block_writes(); + return Err(Error::other( + "pool metadata recovery required: cluster identity has an unreadable replica", + )); + } + let identity = match selection.identity { + Some(identity) if identity.initialized || initialized => PersistedPoolMetaIdentity { + initialized: true, + fresh_bootstrap_nonce: None, + ..identity + }, + Some(identity) => identity, + None if !initialized && !write_state.fresh_bootstrap_proven() => { + write_state.block_writes(); + return Err(Error::other( + "pool metadata recovery required: cannot create a pending cluster identity without verified fresh-bootstrap proof", + )); + } + None => PersistedPoolMetaIdentity { + version: POOL_META_IDENTITY_VERSION, + cluster_id, + epoch: write_state.cluster_epoch.unwrap_or(POOL_META_INITIAL_EPOCH), + initialized, + fresh_bootstrap_nonce: (!initialized).then(uuid::Uuid::new_v4), + }, + }; + if selection.identity == Some(identity) && !selection.needs_repair { + return Ok(()); + } + let data = encode_pool_meta_identity(identity)?; + let mut conflict = false; + for (pool, token) in pools.iter().cloned().zip(&selection.cas_tokens) { + match save_pool_meta_object_cas(pool, POOL_META_IDENTITY_NAME, data.clone(), token, fence, "identity_cas").await { + Ok(_) => {} + Err(Error::PreconditionFailed) => { + conflict = true; + break; + } + Err(err) => return Err(err), + } + } + if conflict { + if attempt + 1 < POOL_META_CAS_MAX_ATTEMPTS { + continue; + } + return Err(Error::PreconditionFailed); + } + let confirmed = load_pool_meta_identity_selection_observing(pools.clone(), write_state, cluster_id).await?; + if confirmed.identity == Some(identity) && !confirmed.needs_repair { + return Ok(()); + } + } + Err(Error::PreconditionFailed) +} + +pub(crate) async fn load_pool_meta_identity_observing(pools: Vec>, write_state: &mut PoolMetaWriteState) -> Result<()> +where + S: EcstoreObjectIO, +{ + let Some(cluster_id) = write_state.expected_cluster_id else { + return Ok(()); + }; + load_pool_meta_identity_selection_observing(pools, write_state, cluster_id) + .await + .map(|_| ()) +} + +pub(crate) async fn persist_pool_meta_identity_for_startup( + pools: Vec>, + write_state: &mut PoolMetaWriteState, + initialized: bool, +) -> Result<()> +where + S: EcstoreObjectIO, +{ + persist_pool_meta_identity(pools, write_state, initialized, &PoolMetaPersistenceFence::Distributed(None)).await +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct PersistedPoolMeta { @@ -3465,6 +4504,23 @@ impl From<&PoolDecommissionInfo> for PersistedPoolDecommissionInfoV1 { } } +#[derive(Debug)] +struct PoolMetaSaveOutcome { + transaction_arm: PoolMetaTransactionArm, + committed: PoolMeta, +} + +impl PoolMetaSaveOutcome { + fn disarm(mut self) { + self.transaction_arm.disarm(); + } + + fn into_committed(mut self) -> PoolMeta { + self.transaction_arm.disarm(); + self.committed + } +} + impl PoolMeta { fn decode_pool_meta_payload(version: u16, payload: &[u8]) -> Result { match version { @@ -3488,7 +4544,9 @@ impl PoolMeta { pub fn new(pools: &[Arc], prev_meta: &PoolMeta) -> Self { let mut new_meta = Self { - version: if prev_meta.version == POOL_META_VERSION || pool_meta_v2_writer_enabled() { + version: if prev_meta.version == POOL_META_GENERATION_VERSION || pool_meta_v3_writer_enabled() { + POOL_META_GENERATION_VERSION + } else if prev_meta.version == POOL_META_VERSION || pool_meta_v2_writer_enabled() { POOL_META_VERSION } else { POOL_META_V1_VERSION @@ -3626,13 +4684,26 @@ impl PoolMeta { return Err(Error::other(format!("pool metadata load failed: unknown format {format}"))); } let version = LittleEndian::read_u16(&data[2..4]); - if !matches!(version, POOL_META_V1_VERSION | POOL_META_VERSION) { + if !matches!(version, POOL_META_V1_VERSION | POOL_META_VERSION | POOL_META_GENERATION_VERSION) { return Err(Error::other(format!("pool metadata load failed: unknown version {version}"))); } + if version == POOL_META_GENERATION_VERSION { + let PoolMetaReplica::Valid { + meta, committed: true, .. + } = decode_pool_meta_replica(data) + else { + return Err(Error::other( + "pool metadata load failed: V3 payload is corrupt, incompatible, or not committed", + )); + }; + *self = meta; + return Ok(()); + } + *self = Self::decode_pool_meta_payload(version, &data[4..])?; - if !matches!(self.version, POOL_META_V1_VERSION | POOL_META_VERSION) { + if !matches!(self.version, POOL_META_V1_VERSION | POOL_META_VERSION | POOL_META_GENERATION_VERSION) { return Err(Error::other(format!( "pool metadata load failed: unexpected decoded version {}", self.version @@ -3673,6 +4744,7 @@ impl PoolMeta { Ok(selection.replica_state) } + #[cfg(test)] fn encode_config_data(&self) -> Result> { self.encode_config_data_for_v2_gate(pool_meta_v2_writer_enabled()) } @@ -3681,6 +4753,11 @@ impl PoolMeta { if self.dont_save { return Ok(Vec::new()); } + if self.version == POOL_META_GENERATION_VERSION { + return Err(Error::other( + "pool metadata V3 save requires cluster identity and generation transaction context", + )); + } if !matches!(self.version, 0 | POOL_META_V1_VERSION | POOL_META_VERSION) { return Err(Error::other(format!( "pool metadata save failed: unexpected runtime version {}", @@ -3730,56 +4807,51 @@ impl PoolMeta { .ok_or_else(|| Error::other("pool metadata save failed: no storage pools available"))?; let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?; let pool_meta_guard = pool_meta_lock.get_write_lock(get_lock_acquire_timeout()).await?; - let selection = load_pool_meta_replicas(pools.clone(), true).await?; - selection.replica_state.ensure_write_safe("pool metadata save failed")?; - self.save_no_lock_with_fence(pools, pool_meta_guard.lock_lost_signal()).await + let mut write_state = PoolMetaWriteState::default(); + let indices = (0..self.pools.len()).collect::>(); + let outcome = self + .save_no_lock_armed_scoped(pools, &mut write_state, pool_meta_guard.lock_lost_signal(), Some(&indices)) + .await?; + outcome.disarm(); + Ok(()) } /// Startup has a single elected local writer, so it must not depend on namespace locks here. + #[cfg(any(test, feature = "test-util"))] pub(crate) async fn save_for_startup(&self, pools: Vec>) -> Result<()> where S: EcstoreObjectIO, { - self.save_no_lock(pools).await + let mut write_state = PoolMetaWriteState::for_test_bootstrap(); + self.save_for_startup_observing(pools, &mut write_state).await.map(|_| ()) } - async fn save_no_lock(&self, pools: Vec>) -> Result<()> + pub(crate) async fn save_for_startup_observing( + &self, + pools: Vec>, + write_state: &mut PoolMetaWriteState, + ) -> Result where S: EcstoreObjectIO, { - self.save_no_lock_with_fence(pools, None).await + let outcome = self.save_no_lock_armed_scoped(pools, write_state, None, None).await?; + Ok(outcome.into_committed()) } async fn save_no_lock_with_fence( &self, pools: Vec>, lock_lost: Option>, + indices: &[usize], ) -> Result<()> where S: EcstoreObjectIO, { - let data = self.encode_config_data()?; - if data.is_empty() { - return Ok(()); - } - for pool in pools { - if lock_lost.as_ref().is_some_and(|signal| signal.is_lost()) { - return Err(Error::other("pool metadata distributed fence was lost before a replica write")); - } - let mut opts = ObjectOptions { - max_parity: true, - no_lock: true, - ..Default::default() - }; - if let Some(signal) = lock_lost.as_ref() { - opts.add_namespace_lock_lost_signal(signal.clone()); - } - save_config_with_opts(pool, POOL_META_NAME, data.clone(), &opts).await?; - if lock_lost.as_ref().is_some_and(|signal| signal.is_lost()) { - return Err(Error::other("pool metadata distributed fence was lost during a replica write")); - } - } - + let mut write_state = PoolMetaWriteState::default(); + let outcome = self + .save_no_lock_armed_scoped(pools, &mut write_state, lock_lost, Some(indices)) + .await?; + outcome.disarm(); Ok(()) } @@ -3787,66 +4859,22 @@ impl PoolMeta { &self, pools: Vec>, write_state: &mut PoolMetaWriteState, - activation_fence: PoolRebalanceActivationFence, - ) -> Result + activation_fence: &PoolRebalanceActivationFence, + indices: &[usize], + ) -> Result where S: EcstoreObjectIO, { - let was_write_blocked = write_state.write_blocked; - // Arm before the first replica write. If this future is dropped, the - // mutex guard is released with the sticky write gate still blocked. - write_state.block_writes(); - let data = self.encode_config_data()?; - if data.is_empty() { - return Ok(was_write_blocked); - } - let mut pools = pools.into_iter(); - let Some(canonical_pool) = pools.next() else { - return Ok(was_write_blocked); - }; - let mut opts = ObjectOptions { - max_parity: true, - no_lock: true, - ..Default::default() - }; - activation_fence.add_namespace_lock_fence(&mut opts); - activation_fence.ensure_held()?; - #[cfg(test)] - let barrier_pool = canonical_pool.clone(); - save_config_with_opts(canonical_pool, POOL_META_NAME, data.clone(), &opts).await?; - #[cfg(test)] - pause_pool_activation_after_durable_save(&barrier_pool, &activation_fence).await; - - // Pool zero is canonical. Once its save succeeds, later writes only - // replicate committed state and must not reuse the admission fence. - // Failed replicas remain repairable by the next full pool metadata save. - drop(activation_fence); - for (pool_index, pool) in pools.enumerate() { - if let Err(err) = save_config_with_opts_quiet( - pool, - POOL_META_NAME, - data.clone(), - &ObjectOptions { - max_parity: true, - no_lock: true, - ..Default::default() - }, - ) - .await - { - warn!( - event = EVENT_DECOMMISSION_STATE, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_POOLS, - pool_index = pool_index + 1, - state = "activation_replica_repair_pending", - error = %err, - "Decommission activation replica repair pending" - ); - } - } - - Ok(was_write_blocked) + write_state.ensure_write_safe("pool metadata activation save failed")?; + let transaction_arm = write_state.arm_transaction(); + let fence = PoolMetaPersistenceFence::Activation(activation_fence); + let committed = self + .save_no_lock_transaction(pools, write_state, &fence, Some(indices)) + .await?; + Ok(PoolMetaSaveOutcome { + transaction_arm, + committed, + }) } async fn save_no_lock_armed( @@ -3854,16 +4882,283 @@ impl PoolMeta { pools: Vec>, write_state: &mut PoolMetaWriteState, lock_lost: Option>, - ) -> Result + indices: &[usize], + ) -> Result where S: EcstoreObjectIO, { - let was_write_blocked = write_state.write_blocked; - // Arm before the first replica write. If this future is dropped, the - // mutex guard is released with the sticky write gate still blocked. - write_state.block_writes(); - self.save_no_lock_with_fence(pools, lock_lost).await?; - Ok(was_write_blocked) + self.save_no_lock_armed_scoped(pools, write_state, lock_lost, Some(indices)) + .await + } + + async fn save_no_lock_armed_scoped( + &self, + pools: Vec>, + write_state: &mut PoolMetaWriteState, + lock_lost: Option>, + indices: Option<&[usize]>, + ) -> Result + where + S: EcstoreObjectIO, + { + write_state.ensure_write_safe("pool metadata save failed")?; + // The arm does not block its own transaction. If this future or its + // returned outcome is dropped before publication, Drop latches the + // sticky recovery gate. + let transaction_arm = write_state.arm_transaction(); + let fence = PoolMetaPersistenceFence::Distributed(lock_lost); + let committed = self.save_no_lock_transaction(pools, write_state, &fence, indices).await?; + Ok(PoolMetaSaveOutcome { + transaction_arm, + committed, + }) + } + + async fn save_no_lock_transaction( + &self, + pools: Vec>, + write_state: &mut PoolMetaWriteState, + fence: &PoolMetaPersistenceFence<'_>, + indices: Option<&[usize]>, + ) -> Result + where + S: EcstoreObjectIO, + { + if pools.is_empty() { + return Err(Error::other("pool metadata save failed: no storage pools available")); + } + if self.dont_save { + return Ok(self.clone()); + } + for attempt in 0..POOL_META_CAS_MAX_ATTEMPTS { + match self + .save_no_lock_transaction_once(pools.clone(), write_state, fence, indices) + .await + { + Ok(committed) => return Ok(committed), + Err(Error::PreconditionFailed) if attempt + 1 < POOL_META_CAS_MAX_ATTEMPTS => continue, + Err(err) => return Err(err), + } + } + Err(Error::PreconditionFailed) + } + + async fn save_no_lock_transaction_once( + &self, + pools: Vec>, + write_state: &mut PoolMetaWriteState, + fence: &PoolMetaPersistenceFence<'_>, + indices: Option<&[usize]>, + ) -> Result + where + S: EcstoreObjectIO, + { + fence.ensure_held()?; + let selection = load_pool_meta_replicas_observing(pools.clone(), true, write_state).await?; + write_state.observe_replicas(selection.replica_state); + write_state.ensure_write_safe("pool metadata save failed")?; + let mut bootstrap_generation_required = false; + if let Some(cluster_id) = write_state.expected_cluster_id { + let identity = load_pool_meta_identity_selection_observing(pools.clone(), write_state, cluster_id).await?; + bootstrap_generation_required = selection.absent || write_state.identity_initialized == Some(false); + if !identity.repair_write_safe { + write_state.block_writes(); + return Err(Error::other( + "pool metadata recovery required: cluster identity has an unreadable replica", + )); + } + if let Some(identity) = identity.identity + && selection.revision.is_generation_protocol() + && identity.epoch != selection.revision.epoch + { + write_state.block_writes(); + return Err(Error::other(format!( + "pool metadata recovery required: committed epoch {} does not match cluster identity epoch {}", + selection.revision.epoch, identity.epoch + ))); + } + if !selection.absent && write_state.identity_requires_repair() { + let initialized = write_state.identity_initialized != Some(false) || selection.revision.is_generation_protocol(); + persist_pool_meta_identity(pools.clone(), write_state, initialized, fence).await?; + } + } + // Startup is the only path allowed to create an all-missing metadata + // set. Runtime callers without an identity context must fail closed. + write_state.ensure_missing_metadata_can_initialize()?; + + let mut committed = if let Some(indices) = indices { + if selection.meta.pools.is_empty() { + self.clone() + } else { + let mut requested = self.clone(); + requested.version = selection.meta.version; + let mut latest = selection.meta.clone(); + merge_pool_meta_updates_for_save(&mut latest, &requested, indices, "pool metadata save failed")?; + latest + } + } else { + self.clone() + }; + let target_version = if selection.generation_protocol_observed + || selection.revision.is_generation_protocol() + || pool_meta_v3_writer_enabled() + || bootstrap_generation_required + { + POOL_META_GENERATION_VERSION + } else if selection.meta.version == POOL_META_VERSION + || self.version == POOL_META_VERSION + || pool_meta_v2_writer_enabled() + { + POOL_META_VERSION + } else { + POOL_META_V1_VERSION + }; + committed.version = target_version; + + if target_version != POOL_META_GENERATION_VERSION { + let data = committed.encode_config_data_for_v2_gate(target_version == POOL_META_VERSION)?; + let mut canonical_saved = false; + for (pool_index, (pool, token)) in pools.iter().cloned().zip(&selection.cas_tokens).enumerate() { + let result = + save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, data.clone(), token, fence, "legacy_cas").await; + if result.is_ok() && pool_index == 0 { + canonical_saved = true; + #[cfg(test)] + if let PoolMetaPersistenceFence::Activation(activation_fence) = fence { + pause_pool_activation_after_durable_save(&pool, activation_fence).await; + } + } + if let Err(err) = result { + if canonical_saved && fence.is_activation() { + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index, + state = "activation_replica_repair_pending", + error = %err, + "Decommission activation replica repair pending" + ); + continue; + } + return Err(err); + } + } + let confirmed = if fence.is_activation() { + load_pool_meta_replicas(pools, true).await? + } else { + let confirmed = load_pool_meta_replicas_observing(pools, true, write_state).await?; + write_state.observe_replicas(confirmed.replica_state); + confirmed + }; + let expected = committed.encode_config_data_for_v2_gate(true)?; + if confirmed.canonical.as_ref() != Some(&expected) { + record_pool_meta_stale_write_rejection("legacy_verify"); + return Err(Error::PreconditionFailed); + } + return Ok(confirmed.meta); + } + + let cluster_id = selection + .revision + .cluster_id + .or(selection.generation_identity.map(|(cluster_id, _)| cluster_id)) + .or(write_state.expected_cluster_id) + .ok_or_else(|| Error::other("pool metadata V3 save failed: cluster identity is not initialized"))?; + let epoch = if selection.revision.is_generation_protocol() { + selection.revision.epoch + } else { + selection + .generation_identity + .map(|(_, epoch)| epoch) + .or(write_state.cluster_epoch) + .unwrap_or(POOL_META_INITIAL_EPOCH) + }; + let generation = if selection.revision.is_generation_protocol() { + selection + .revision + .generation + .checked_add(1) + .ok_or_else(|| Error::other("pool metadata V3 generation exhausted"))? + } else { + 1 + }; + let revision = PoolMetaRevision { + version: POOL_META_GENERATION_VERSION, + cluster_id: Some(cluster_id), + epoch, + generation, + transaction_id: Some(uuid::Uuid::new_v4()), + }; + let previous = if let Some(canonical) = selection.canonical.clone() { + PoolMetaCommittedCandidate { + canonical, + meta: selection.meta.clone(), + revision: selection.revision, + } + } else { + let empty = PoolMeta { + version: POOL_META_V1_VERSION, + ..Default::default() + }; + PoolMetaCommittedCandidate { + canonical: empty.encode_config_data_for_v2_gate(true)?, + meta: empty, + revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION), + } + }; + let pending = encode_pool_meta_v3_envelope(&committed, revision, false, Some(&previous))?; + let durable = encode_pool_meta_v3_envelope(&committed, revision, true, None)?; + let mut pending_tokens = Vec::with_capacity(pools.len()); + for (pool, token) in pools.iter().cloned().zip(&selection.cas_tokens) { + let object_info = + save_pool_meta_object_cas(pool, POOL_META_NAME, pending.clone(), token, fence, "prepare_cas").await?; + let etag = object_info + .etag + .filter(|etag| !etag.trim().is_empty()) + .ok_or_else(|| Error::other("pool metadata V3 prepare succeeded without a conditional-write revision"))?; + pending_tokens.push(PoolMetaCasToken::Existing(etag)); + } + + let mut commit_error = None; + let mut commit_succeeded = false; + #[cfg(test)] + let mut first_pool = true; + for (pool, token) in pools.iter().cloned().zip(&pending_tokens) { + match save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, durable.clone(), token, fence, "commit_cas").await { + Ok(_) => { + commit_succeeded = true; + #[cfg(test)] + if first_pool && let PoolMetaPersistenceFence::Activation(activation_fence) = fence { + pause_pool_activation_after_durable_save(&pool, activation_fence).await; + } + } + Err(err) => { + commit_error.get_or_insert(err); + } + } + #[cfg(test)] + { + first_pool = false; + } + } + let confirmed = if fence.is_activation() { + load_pool_meta_replicas(pools.clone(), true).await? + } else { + let confirmed = load_pool_meta_replicas_observing(pools.clone(), true, write_state).await?; + write_state.observe_replicas(confirmed.replica_state); + confirmed + }; + if confirmed.revision == revision && confirmed.canonical.as_ref() == Some(&durable) { + persist_pool_meta_identity(pools, write_state, true, fence).await?; + return Ok(confirmed.meta); + } + if !commit_succeeded { + return Err(commit_error.unwrap_or(Error::PreconditionFailed)); + } + Err(commit_error.unwrap_or_else(|| { + Error::other("pool metadata recovery required: committed V3 transaction was not selected after write") + })) } #[cfg(test)] @@ -3871,8 +5166,9 @@ impl PoolMeta { where S: EcstoreObjectIO, { - let was_write_blocked = self.save_no_lock_armed(pools, write_state, None).await?; - write_state.restore_writes(was_write_blocked); + let indices = (0..self.pools.len()).collect::>(); + let outcome = self.save_no_lock_armed(pools, write_state, None, &indices).await?; + outcome.disarm(); Ok(()) } @@ -4747,6 +6043,7 @@ impl ECStore { operation: &str, ) -> Result<(rustfs_lock::NamespaceLockGuard, PoolMeta)> { write_state.ensure_write_safe(operation)?; + load_pool_meta_identity_observing(self.pools.clone(), write_state).await?; let pool = self .pools .first() @@ -4767,6 +6064,31 @@ impl ECStore { self.pool_meta_save_gate.lock().await.ensure_write_safe(operation) } + async fn load_runtime_pool_meta_observing(&self, write_state: &mut PoolMetaWriteState, operation: &str) -> Result { + write_state.ensure_write_safe(operation)?; + load_pool_meta_identity_observing(self.pools.clone(), write_state).await?; + let mut pool_meta = PoolMeta::default(); + let replica_state = pool_meta + .load_no_lock_from_replicas_observing(self.pools.clone(), write_state) + .await?; + write_state.observe_replicas(replica_state); + write_state.ensure_missing_metadata_can_initialize()?; + write_state.ensure_write_safe(operation)?; + Ok(pool_meta) + } + + pub(crate) async fn load_runtime_pool_meta_under_activation_fence( + &self, + write_state: &mut PoolMetaWriteState, + activation_fence: &PoolRebalanceActivationFence, + operation: &str, + ) -> Result { + activation_fence.ensure_held()?; + let pool_meta = self.load_runtime_pool_meta_observing(write_state, operation).await?; + activation_fence.ensure_held()?; + Ok(pool_meta) + } + pub(crate) async fn load_runtime_pool_meta(&self, operation: &str) -> Result { let mut write_state = self.pool_meta_save_gate.lock().await; write_state.ensure_write_safe(operation)?; @@ -4777,13 +6099,7 @@ impl ECStore { .ok_or_else(|| Error::other(format!("{operation}: no storage pools available")))?; let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?; let _pool_meta_guard = pool_meta_lock.get_read_lock(get_lock_acquire_timeout()).await?; - let mut pool_meta = PoolMeta::default(); - let replica_state = pool_meta - .load_no_lock_from_replicas_observing(self.pools.clone(), &mut write_state) - .await?; - write_state.observe_replicas(replica_state); - write_state.ensure_write_safe(operation)?; - Ok(pool_meta) + self.load_runtime_pool_meta_observing(&mut write_state, operation).await } async fn run_guarded_decommission_side_effect( @@ -4829,18 +6145,23 @@ impl ECStore { let pool_meta = self.pool_meta.read().await; merge_pool_meta_updates_for_save(&mut snapshot, &pool_meta, indices, "pool metadata save failed")?; } - let was_write_blocked = snapshot - .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal()) + let outcome = snapshot + .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), indices) .await?; let mut pool_meta = self.pool_meta.write().await; ensure_pool_meta_write_fence(&pool_meta_guard, "pool metadata save failed")?; - publish_pool_meta_updates(&mut pool_meta, &snapshot, indices); + publish_pool_meta_updates(&mut pool_meta, &outcome.committed, indices); ensure_pool_meta_write_fence(&pool_meta_guard, "pool metadata save failed")?; drop(pool_meta); - save_guard.restore_writes(was_write_blocked); + outcome.disarm(); Ok(()) } + #[cfg(test)] + pub(crate) async fn save_current_pool_meta_for_test(&self, indices: &[usize]) -> Result<()> { + self.save_current_pool_meta(indices).await + } + async fn persist_decommission_unresolved_entry( &self, idx: usize, @@ -4885,11 +6206,11 @@ impl ECStore { (snapshot, checkpoint) }; - let was_write_blocked = match snapshot - .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal()) + let outcome = match snapshot + .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &[idx]) .await { - Ok(was_write_blocked) => was_write_blocked, + Ok(outcome) => outcome, Err(err) => { let retry_after = OffsetDateTime::now_utc() + DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF; let mut pool_meta = self.pool_meta.write().await; @@ -4900,10 +6221,11 @@ impl ECStore { let mut pool_meta = self.pool_meta.write().await; ensure_pool_meta_write_fence(&pool_meta_guard, "decommission progress save failed")?; + pool_meta.version = pool_meta.version.max(outcome.committed.version); let committed = pool_meta.commit_decommission_progress_checkpoint(idx, checkpoint); ensure_pool_meta_write_fence(&pool_meta_guard, "decommission progress save failed")?; drop(pool_meta); - save_guard.restore_writes(was_write_blocked); + outcome.disarm(); Ok(committed) } @@ -4929,18 +6251,19 @@ impl ECStore { return Ok(false); } - let was_write_blocked = snapshot - .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal()) + let outcome = snapshot + .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &[idx]) .await .map_err(|err| { Error::other(format!("decommission metadata save failed for pool {idx} bucket {}: {err}", bucket.name)) })?; let mut pool_meta = self.pool_meta.write().await; ensure_pool_meta_write_fence(&pool_meta_guard, "decommission bucket completion save failed")?; + pool_meta.version = pool_meta.version.max(outcome.committed.version); pool_meta.mark_decommission_progress_saved(); ensure_pool_meta_write_fence(&pool_meta_guard, "decommission bucket completion save failed")?; drop(pool_meta); - save_guard.restore_writes(was_write_blocked); + outcome.disarm(); Ok(true) } @@ -5010,14 +6333,16 @@ impl ECStore { } activation_fence.ensure_held()?; - let was_write_blocked = latest_pool_meta - .save_no_lock_with_activation_fence(self.pools.clone(), &mut save_guard, activation_fence) + let outcome = latest_pool_meta + .save_no_lock_with_activation_fence(self.pools.clone(), &mut save_guard, &activation_fence, indices) .await?; + activation_fence.ensure_held()?; { let mut pool_meta = self.pool_meta.write().await; - publish_pool_meta_updates(&mut pool_meta, &latest_pool_meta, indices); + publish_pool_meta_updates(&mut pool_meta, &outcome.committed, indices); } - save_guard.restore_writes(was_write_blocked); + activation_fence.ensure_held()?; + outcome.disarm(); Ok(previous_pool_meta) } @@ -5034,15 +6359,15 @@ impl ECStore { .acquire_pool_meta_write_guard(&mut save_guard, "decommission start rollback failed") .await?; rollback_start_decommission_pool_meta(&mut snapshot, previous_pool_meta, indices); - let was_write_blocked = snapshot - .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal()) + let outcome = snapshot + .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), indices) .await?; let mut pool_meta = self.pool_meta.write().await; ensure_pool_meta_write_fence(&pool_meta_guard, "decommission start rollback failed")?; - publish_pool_meta_updates(&mut pool_meta, &snapshot, indices); + publish_pool_meta_updates(&mut pool_meta, &outcome.committed, indices); ensure_pool_meta_write_fence(&pool_meta_guard, "decommission start rollback failed")?; drop(pool_meta); - save_guard.restore_writes(was_write_blocked); + outcome.disarm(); self.ctx.advance_data_movement_operation_epoch(); Ok(()) } @@ -5426,7 +6751,7 @@ impl ECStore { tokio::spawn(async move { store .decommission_cancel_transaction(idx, owner, true, move |snapshot, pool_meta_fence| async move { - snapshot.save_no_lock_with_fence(pools, pool_meta_fence).await + snapshot.save_no_lock_with_fence(pools, pool_meta_fence, &[idx]).await }) .await }) @@ -5559,12 +6884,12 @@ impl ECStore { } drop(pool_meta); - let (was_write_blocked, save_error) = if changed { + let (save_outcome, save_error) = if changed { match snapshot - .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal()) + .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &[idx]) .await { - Ok(was_write_blocked) => (Some(was_write_blocked), None), + Ok(outcome) => (Some(outcome), None), Err(err) => (None, Some(err)), } } else { @@ -5572,8 +6897,11 @@ impl ECStore { }; let generation = self.active_decommission_generation(idx).await?; ensure_pool_meta_write_fence(&pool_meta_guard, "decommission promotion failed")?; - if let Some(was_write_blocked) = was_write_blocked { - save_guard.restore_writes(was_write_blocked); + if let Some(outcome) = save_outcome { + let mut pool_meta = self.pool_meta.write().await; + pool_meta.version = pool_meta.version.max(outcome.committed.version); + drop(pool_meta); + outcome.disarm(); } (changed, generation, save_error) }; @@ -9502,7 +10830,7 @@ mod tests { #[tokio::test] #[serial_test::serial] - async fn decommission_activation_replicates_commit_after_post_save_fence_loss() { + async fn decommission_activation_fence_loss_after_durable_save_blocks_publication() { let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await; let barrier = PoolActivationDurableSaveBarrier::install(&store.pools[0]); let start_store = Arc::clone(&store); @@ -9525,27 +10853,33 @@ mod tests { barrier.wait_until_paused().await; barrier.release_after_fence_loss(); - tokio::time::timeout(std::time::Duration::from_secs(30), start_task) + let err = tokio::time::timeout(std::time::Duration::from_secs(30), start_task) .await - .expect("decommission activation should finish after its canonical commit") + .expect("decommission activation should stop after losing its fence") .expect("decommission activation task should not panic") - .expect("post-commit fence loss must not report the committed activation as failed"); + .expect_err("post-durable-save fence loss must reject in-memory publication"); + assert!(err.to_string().contains("activation lock lost")); let local = store.pool_meta.read().await; - assert!(pool_meta_has_active_decommission(&local)); + assert!( + !pool_meta_has_active_decommission(&local), + "the activation must not publish after its durable fence is lost" + ); drop(local); + store + .ensure_pool_meta_side_effects_safe("post-durable-save activation fence loss") + .await + .expect_err("the undisarmed transaction must latch the sticky pool metadata gate"); - for pool in &store.pools { - let mut persisted = PoolMeta::default(); - persisted - .load_no_lock_from_replicas(vec![pool.clone()]) - .await - .expect("every pool should retain readable committed decommission metadata"); - assert!( - pool_meta_has_active_decommission(&persisted), - "every pool must adopt the canonical committed activation" - ); - } + let mut persisted = PoolMeta::default(); + persisted + .load_no_lock_from_replicas(vec![store.pools[0].clone()]) + .await + .expect("the durable replica should remain readable for recovery"); + assert!( + pool_meta_has_active_decommission(&persisted), + "the test must lose the fence only after one durable replica commit" + ); } #[tokio::test] @@ -9579,7 +10913,7 @@ mod tests { *disks = vec![None; saved.len()]; replica_disks.push(saved); } - barrier.release_after_fence_loss(); + barrier.release_without_fence_loss(); tokio::time::timeout(std::time::Duration::from_secs(30), start_task) .await @@ -9761,7 +11095,10 @@ mod tests { ..Default::default() }); let data = pool_meta_v1_replica_test_data(&source); - let PoolMetaReplica::Valid { raw, canonical, meta } = decode_pool_meta_replica(data) else { + let PoolMetaReplica::Valid { + raw, canonical, meta, .. + } = decode_pool_meta_replica(data) + else { panic!("v1 pool metadata should remain readable"); }; @@ -9779,7 +11116,10 @@ mod tests { #[test] fn pool_meta_legacy_v1_replica_migrates_to_v2_canonical_form() { let data = pool_meta_legacy_v1_replica_test_data("pool-0"); - let PoolMetaReplica::Valid { raw, canonical, meta } = decode_pool_meta_replica(data) else { + let PoolMetaReplica::Valid { + raw, canonical, meta, .. + } = decode_pool_meta_replica(data) + else { panic!("legacy v1 pool metadata should remain readable"); }; @@ -9850,6 +11190,238 @@ mod tests { assert!(pool_meta_v2_writer_enabled_for(true, true)); } + #[test] + fn pool_meta_v3_writer_requires_both_gates() { + assert!(!pool_meta_v3_writer_enabled_for(false, false)); + assert!(!pool_meta_v3_writer_enabled_for(true, false)); + assert!(!pool_meta_v3_writer_enabled_for(false, true)); + assert!(pool_meta_v3_writer_enabled_for(true, true)); + } + + #[test] + fn pool_meta_stale_write_rejection_metric_is_countable() { + let recorder = metrics_util::debugging::DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + metrics::with_local_recorder(&recorder, || { + record_pool_meta_stale_write_rejection("prepare_cas"); + record_pool_meta_stale_write_rejection("prepare_cas"); + }); + + let total = snapshotter + .snapshot() + .into_vec() + .into_iter() + .filter(|(composite, _, _, _)| composite.key().name() == METRIC_POOL_META_STALE_WRITE_REJECTIONS_TOTAL) + .filter_map(|(_, _, _, value)| match value { + metrics_util::debugging::DebugValue::Counter(value) => Some(value), + _ => None, + }) + .sum::(); + assert_eq!(total, 2); + } + + fn pool_meta_v3_test_revision(cluster_id: uuid::Uuid, generation: u64, transaction_id: uuid::Uuid) -> PoolMetaRevision { + PoolMetaRevision { + version: POOL_META_GENERATION_VERSION, + cluster_id: Some(cluster_id), + epoch: POOL_META_INITIAL_EPOCH, + generation, + transaction_id: Some(transaction_id), + } + } + + fn pool_meta_legacy_candidate(meta: PoolMeta) -> PoolMetaCommittedCandidate { + PoolMetaCommittedCandidate { + canonical: meta + .encode_config_data_for_v2_gate(true) + .expect("legacy pool metadata should encode"), + revision: PoolMetaRevision::legacy(meta.version), + meta, + } + } + + #[test] + fn pool_meta_v3_pending_prepare_recovers_previous_snapshot_after_restart() { + let previous = pool_meta_replica_test_meta("pool-before"); + let mut next = pool_meta_replica_test_meta("pool-after"); + next.version = POOL_META_GENERATION_VERSION; + let revision = pool_meta_v3_test_revision(uuid::Uuid::new_v4(), 1, uuid::Uuid::new_v4()); + let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&pool_meta_legacy_candidate(previous.clone()))) + .expect("pending generation should encode"); + + let selected = select_pool_meta_replica(vec![ + decode_pool_meta_replica(pending), + decode_pool_meta_replica( + previous + .encode_config_data_for_v2_gate(true) + .expect("previous snapshot should encode"), + ), + ]) + .expect("a prepare-only transaction should expose its committed predecessor"); + + assert_eq!(selected.meta.pools[0].cmd_line, "pool-before"); + assert_eq!(selected.revision.version, POOL_META_VERSION); + assert!(selected.replica_state.needs_repair); + } + + #[test] + fn pool_meta_v3_partial_commit_selects_new_generation_after_restart() { + let previous = pool_meta_replica_test_meta("pool-before"); + let mut next = pool_meta_replica_test_meta("pool-after"); + next.version = POOL_META_GENERATION_VERSION; + let revision = pool_meta_v3_test_revision(uuid::Uuid::new_v4(), 1, uuid::Uuid::new_v4()); + let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&pool_meta_legacy_candidate(previous))) + .expect("pending generation should encode"); + let committed = encode_pool_meta_v3_envelope(&next, revision, true, None).expect("committed generation should encode"); + + let selected = select_pool_meta_replica(vec![decode_pool_meta_replica(committed), decode_pool_meta_replica(pending)]) + .expect("one committed replica should make the cross-pool transaction durable"); + + assert_eq!(selected.meta.pools[0].cmd_line, "pool-after"); + assert_eq!(selected.revision, revision); + assert!(selected.replica_state.needs_repair); + } + + #[test] + fn pool_meta_v3_uses_valid_committed_backup_but_rejects_same_generation_fork() { + let cluster_id = uuid::Uuid::new_v4(); + let mut next = pool_meta_replica_test_meta("pool-after"); + next.version = POOL_META_GENERATION_VERSION; + let revision = pool_meta_v3_test_revision(cluster_id, 7, uuid::Uuid::new_v4()); + let committed = encode_pool_meta_v3_envelope(&next, revision, true, None).expect("committed generation should encode"); + let selected = select_pool_meta_replica(vec![ + PoolMetaReplica::Corrupt("truncated".to_string()), + decode_pool_meta_replica(committed.clone()), + ]) + .expect("a corrupt first copy should fall back to a valid committed generation"); + assert_eq!(selected.revision, revision); + assert!(selected.replica_state.needs_repair); + + let fork_revision = pool_meta_v3_test_revision(cluster_id, 7, uuid::Uuid::new_v4()); + let fork = encode_pool_meta_v3_envelope(&next, fork_revision, true, None).expect("fork generation should encode"); + let err = select_pool_meta_replica(vec![decode_pool_meta_replica(committed), decode_pool_meta_replica(fork)]) + .expect_err("same-generation transactions must never be selected by pool order"); + assert!(err.to_string().contains("committed generation 7 diverges")); + } + + #[test] + fn pool_meta_v3_migration_keeps_legacy_committed_until_commit_record_exists() { + let previous = pool_meta_replica_test_meta("pool-before"); + let mut next = pool_meta_replica_test_meta("pool-after"); + next.version = POOL_META_GENERATION_VERSION; + let revision = pool_meta_v3_test_revision(uuid::Uuid::new_v4(), 1, uuid::Uuid::new_v4()); + let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&pool_meta_legacy_candidate(previous.clone()))) + .expect("pending migration should encode"); + let legacy = previous + .encode_config_data_for_v2_gate(true) + .expect("legacy snapshot should encode"); + + let prepared = select_pool_meta_replica(vec![decode_pool_meta_replica(pending), decode_pool_meta_replica(legacy)]) + .expect("prepared migration should retain the legacy commit"); + assert_eq!(prepared.meta.version, POOL_META_VERSION); + assert_eq!(prepared.meta.pools[0].cmd_line, "pool-before"); + + let committed = encode_pool_meta_v3_envelope(&next, revision, true, None).expect("committed migration should encode"); + let migrated = select_pool_meta_replica(vec![decode_pool_meta_replica(committed)]) + .expect("committed migration should establish the V3 floor"); + assert_eq!(migrated.meta.version, POOL_META_GENERATION_VERSION); + assert_eq!(migrated.meta.pools[0].cmd_line, "pool-after"); + } + + #[test] + fn pool_meta_v3_unknown_fields_remain_incompatible_not_silently_ignored() { + #[derive(Serialize)] + struct FuturePoolMetaV3 { + version: u16, + cluster_id: String, + epoch: u64, + generation: u64, + transaction_id: String, + committed: bool, + pools: Vec, + previous: Option, + future_guard: u64, + } + + let mut data = Vec::new(); + data.write_u16::(POOL_META_FORMAT) + .expect("pool metadata format should encode"); + data.write_u16::(POOL_META_GENERATION_VERSION) + .expect("pool metadata version should encode"); + FuturePoolMetaV3 { + version: POOL_META_GENERATION_VERSION, + cluster_id: uuid::Uuid::new_v4().to_string(), + epoch: POOL_META_INITIAL_EPOCH, + generation: 1, + transaction_id: uuid::Uuid::new_v4().to_string(), + committed: true, + pools: Vec::new(), + previous: None, + future_guard: 1, + } + .serialize(&mut Serializer::new(&mut data)) + .expect("future V3 payload should encode"); + + assert!(matches!(decode_pool_meta_replica(data), PoolMetaReplica::Incompatible(_))); + } + + #[test] + fn pool_meta_identity_classifies_corrupt_incompatible_and_divergent_replicas() { + let cluster_id = uuid::Uuid::new_v4(); + let identity = PersistedPoolMetaIdentity { + version: POOL_META_IDENTITY_VERSION, + cluster_id, + epoch: POOL_META_INITIAL_EPOCH, + initialized: true, + fresh_bootstrap_nonce: None, + }; + let selected = select_pool_meta_identity( + vec![ + PoolMetaIdentityRead { + replica: PoolMetaIdentityReplica::Corrupt("truncated".to_string()), + cas: PoolMetaCasToken::Existing("corrupt".to_string()), + }, + PoolMetaIdentityRead { + replica: PoolMetaIdentityReplica::Valid(identity), + cas: PoolMetaCasToken::Existing("valid".to_string()), + }, + ], + cluster_id, + ) + .expect("a verified identity backup should survive a corrupt first replica"); + assert_eq!(selected.identity, Some(identity)); + assert!(selected.needs_repair); + + let incompatible = select_pool_meta_identity( + vec![PoolMetaIdentityRead { + replica: PoolMetaIdentityReplica::Incompatible("future version".to_string()), + cas: PoolMetaCasToken::Existing("future".to_string()), + }], + cluster_id, + ) + .expect_err("an incompatible identity must fail closed"); + assert!(incompatible.to_string().contains("incompatible")); + + let divergent = select_pool_meta_identity( + vec![ + PoolMetaIdentityRead { + replica: PoolMetaIdentityReplica::Valid(identity), + cas: PoolMetaCasToken::Existing("epoch-1".to_string()), + }, + PoolMetaIdentityRead { + replica: PoolMetaIdentityReplica::Valid(PersistedPoolMetaIdentity { + epoch: POOL_META_INITIAL_EPOCH + 1, + ..identity + }), + cas: PoolMetaCasToken::Existing("epoch-2".to_string()), + }, + ], + cluster_id, + ) + .expect_err("identity epoch divergence must require recovery"); + assert!(divergent.to_string().contains("recovery required")); + } + #[test] fn pool_meta_v2_floor_is_sticky_after_observation() { let meta = pool_meta_replica_test_meta("pool-0"); @@ -9917,7 +11489,7 @@ mod tests { PoolMetaReplica::Unreadable("read quorum unavailable".to_string()), ]) .expect_err("an unreadable replica must not be treated as a new deployment"); - assert!(err.to_string().contains("no valid replica is available")); + assert!(err.to_string().contains("no valid committed replica is available")); assert!(err.to_string().contains("pool 1 is unreadable")); } @@ -11183,19 +12755,19 @@ mod pools_tests { DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD, DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT, DecomBucketInfo, DecommissionCanceler, DecommissionDurableIlmReceipt, DecommissionEntryEnqueueResult, DecommissionStartPoolState, DecommissionTerminalState, DecommissionUnresolvedEntry, - ListCallback, POOL_META_NAME, POOL_META_V1_VERSION, POOL_META_VERSION, PoolDecommissionInfo, PoolMeta, PoolSpaceInfo, - PoolStatus, QueuedDecommissionEntry, REBAL_META_NAME, acquire_pool_rebalance_activation_locks, - apply_decommission_status_space_info, await_decommission_worker, bind_decommission_cancelers, - bind_missing_decommission_cancelers, cancel_decommission_canceler, clamp_decommission_entry_concurrency, - classify_decommission_terminal_state, count_decommission_item, decommission_cancel_signal_result, - decommission_durable_ilm_receipt_path, decommission_durable_ilm_receipt_run_prefix, - decommission_durable_ilm_receipt_run_token, decommission_entry_queue_capacity, decommission_item_size, - decommission_meta_bucket_options, decommission_retry_backoff_delay, decommission_start_pool_state, - decommission_unresolved_listing_error, dedup_indices, default_decommission_bucket_concurrency, - default_decommission_entry_concurrency, drain_decommission_entry_queue, enqueue_decommission_entry, - ensure_decommission_cancel_allowed, ensure_decommission_clear_allowed, ensure_decommission_generation, - ensure_decommission_listing_disks_available, ensure_decommission_not_rebalancing, ensure_decommission_start_allowed, - ensure_decommission_start_keeps_active_pool, ensure_decommission_start_local_leader, + ListCallback, POOL_META_IDENTITY_NAME, POOL_META_NAME, POOL_META_V1_VERSION, POOL_META_VERSION, PoolDecommissionInfo, + PoolMeta, PoolMetaCasToken, PoolMetaPersistenceFence, PoolSpaceInfo, PoolStatus, QueuedDecommissionEntry, + REBAL_META_NAME, acquire_pool_rebalance_activation_locks, apply_decommission_status_space_info, + await_decommission_worker, bind_decommission_cancelers, bind_missing_decommission_cancelers, + cancel_decommission_canceler, clamp_decommission_entry_concurrency, classify_decommission_terminal_state, + count_decommission_item, decommission_cancel_signal_result, decommission_durable_ilm_receipt_path, + decommission_durable_ilm_receipt_run_prefix, decommission_durable_ilm_receipt_run_token, + decommission_entry_queue_capacity, decommission_item_size, decommission_meta_bucket_options, + decommission_retry_backoff_delay, decommission_start_pool_state, decommission_unresolved_listing_error, dedup_indices, + default_decommission_bucket_concurrency, default_decommission_entry_concurrency, drain_decommission_entry_queue, + enqueue_decommission_entry, ensure_decommission_cancel_allowed, ensure_decommission_clear_allowed, + ensure_decommission_generation, ensure_decommission_listing_disks_available, ensure_decommission_not_rebalancing, + ensure_decommission_start_allowed, ensure_decommission_start_keeps_active_pool, ensure_decommission_start_local_leader, ensure_decommission_start_pool_states, ensure_decommission_start_rebalance_meta_allowed, ensure_decommission_start_target_capacity, ensure_decommission_terminal_operation_supported, ensure_decommission_unresolved_verification_disk_count, ensure_local_decommission_pool_leaders, @@ -11204,18 +12776,19 @@ mod pools_tests { load_decommission_entry_versions, local_decommission_queue_prefix, mark_decommission_bucket_done, merge_decommission_durable_ilm_receipts, merge_pool_meta_updates_for_save, merge_pool_status_refresh, missing_decommission_worker_prefix, observe_decommission_terminal_reload_result, pool_meta_has_active_decommission, - publish_pool_meta_updates, reconcile_decommission_meta_buckets, reconcile_decommission_unresolved_entries_for_completion, - record_decommission_unresolved_entry, require_decommission_store, reserve_decommission_start_cancelers, - resolve_decommission_bucket_state, resolve_decommission_check_after_list_result, - resolve_decommission_entry_cleanup_delete_result, resolve_decommission_entry_exact_versions, - resolve_decommission_entry_reload_result, resolve_decommission_listing_worker_result, - resolve_decommission_optional_bucket_config_result, resolve_decommission_pool_meta_reload_result, - resolve_decommission_preflight_heal_result, resolve_decommission_progress_save_result, - resolve_decommission_terminal_mark_after_error_result, resolve_decommission_terminal_mark_result, - resolve_decommission_update_after_result, resolve_start_decommission_pool_meta_reload_result, - resumable_decommission_queue_indices, rollback_start_decommission_pool_meta, run_decommission_buckets_bounded, - run_decommission_listing_with_retry, run_decommission_listing_with_retry_and_drain, run_decommission_phases, - run_decommission_side_effect, should_cleanup_decommission_source_entry, should_continue_decommission_queue, + publish_pool_meta_updates, read_pool_meta_replica, reconcile_decommission_meta_buckets, + reconcile_decommission_unresolved_entries_for_completion, record_decommission_unresolved_entry, + require_decommission_store, reserve_decommission_start_cancelers, resolve_decommission_bucket_state, + resolve_decommission_check_after_list_result, resolve_decommission_entry_cleanup_delete_result, + resolve_decommission_entry_exact_versions, resolve_decommission_entry_reload_result, + resolve_decommission_listing_worker_result, resolve_decommission_optional_bucket_config_result, + resolve_decommission_pool_meta_reload_result, resolve_decommission_preflight_heal_result, + resolve_decommission_progress_save_result, resolve_decommission_terminal_mark_after_error_result, + resolve_decommission_terminal_mark_result, resolve_decommission_update_after_result, + resolve_start_decommission_pool_meta_reload_result, resumable_decommission_queue_indices, + rollback_start_decommission_pool_meta, run_decommission_buckets_bounded, run_decommission_listing_with_retry, + run_decommission_listing_with_retry_and_drain, run_decommission_phases, run_decommission_side_effect, + save_pool_meta_object_cas, should_cleanup_decommission_source_entry, should_continue_decommission_queue, should_count_decommission_version_complete, should_fail_decommission_pool_after_exhausted_source_changed, should_preserve_decommission_canceled_state, should_reject_decommission_cancel_as_terminal, should_retry_decommission_cancel_reload, should_retry_decommission_listing, should_skip_canceled_decommission_routine, @@ -11238,6 +12811,7 @@ mod pools_tests { use crate::runtime::instance::InstanceContext; use crate::services::rebalance::{RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats}; use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions}; + use crate::storage_api_contracts::object::HTTPPreconditions; use crate::storage_api_contracts::{object::ObjectIO, range::HTTPRangeSpec}; use crate::store::ECStore; use byteorder::{ByteOrder, LittleEndian}; @@ -11246,6 +12820,7 @@ mod pools_tests { use rustfs_lock::{GlobalLockManager, LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey}; use rustfs_rio::Index; use std::future::Future; + use std::io::Cursor; use std::sync::{ Arc, Mutex as StdMutex, atomic::{AtomicBool, AtomicUsize, Ordering}, @@ -11253,6 +12828,7 @@ mod pools_tests { use std::task::{Context, Poll}; use std::time::Duration as StdDuration; use time::{Duration, OffsetDateTime}; + use tokio::io::AsyncReadExt; use tokio::sync::Semaphore; use tokio_util::sync::CancellationToken; @@ -11300,7 +12876,7 @@ mod pools_tests { rebalance_meta: tokio::sync::RwLock::new(None), decommission_cancelers: tokio::sync::RwLock::new(cancelers), start_gate: tokio::sync::Mutex::new(()), - pool_meta_save_gate: tokio::sync::Mutex::default(), + pool_meta_save_gate: tokio::sync::Mutex::new(super::PoolMetaWriteState::for_test_bootstrap()), ctx, bucket_fence_registry: Arc::default(), }) @@ -11309,9 +12885,13 @@ mod pools_tests { #[derive(Debug)] struct PartialPoolMetaWriteStorage { fail_write: bool, + fail_after_first_write: bool, pending_write: bool, write_started: tokio::sync::Notify, wrote: AtomicBool, + revision: AtomicUsize, + stored: StdMutex, String)>>, + identity: StdMutex, String)>>, } #[async_trait::async_trait] @@ -11326,48 +12906,241 @@ mod pools_tests { async fn get_object_reader( &self, - _bucket: &str, - _object: &str, + bucket: &str, + object: &str, _range: Option, _h: Self::HeaderMap, _opts: &Self::ObjectOptions, ) -> std::result::Result { - Err(Error::FileNotFound) + let stored = if object == POOL_META_IDENTITY_NAME { + &self.identity + } else { + &self.stored + }; + let Some((data, etag)) = stored + .lock() + .expect("pool metadata test storage should not be poisoned") + .clone() + else { + return Err(Error::FileNotFound); + }; + Ok(crate::object_api::GetObjectReader { + stream: Box::new(Cursor::new(data.clone())), + object_info: crate::object_api::ObjectInfo { + bucket: bucket.to_string(), + name: object.to_string(), + size: data.len() as i64, + etag: Some(etag), + ..Default::default() + }, + buffered_body: None, + body_source: Default::default(), + }) } async fn put_object( &self, _bucket: &str, - _object: &str, - _data: &mut Self::PutObjectReader, - _opts: &Self::ObjectOptions, + object: &str, + data: &mut Self::PutObjectReader, + opts: &Self::ObjectOptions, ) -> std::result::Result { self.write_started.notify_one(); if self.pending_write { std::future::pending().await } - if self.fail_write { + if object == POOL_META_NAME + && (self.fail_write || (self.fail_after_first_write && self.revision.load(Ordering::SeqCst) > 0)) + { return Err(Error::Timeout); } + let stored = if object == POOL_META_IDENTITY_NAME { + &self.identity + } else { + &self.stored + }; + let current_etag = stored + .lock() + .expect("pool metadata test storage should not be poisoned") + .as_ref() + .map(|(_, etag)| etag.clone()); + if opts + .http_preconditions + .as_ref() + .and_then(HTTPPreconditions::if_none_match_value) + == Some("*") + && current_etag.is_some() + { + return Err(Error::PreconditionFailed); + } + if let Some(expected) = opts.http_preconditions.as_ref().and_then(HTTPPreconditions::if_match_value) + && current_etag.as_deref() != Some(expected) + { + return Err(Error::PreconditionFailed); + } + let mut payload = Vec::new(); + data.stream.read_to_end(&mut payload).await?; + let etag = format!("pool-meta-test-{}", self.revision.fetch_add(1, Ordering::SeqCst) + 1); + *stored.lock().expect("pool metadata test storage should not be poisoned") = Some((payload, etag.clone())); self.wrote.store(true, Ordering::SeqCst); - Ok(crate::object_api::ObjectInfo::default()) + Ok(crate::object_api::ObjectInfo { + etag: Some(etag), + ..Default::default() + }) } } + #[tokio::test] + async fn test_pool_meta_cas_deterministically_rejects_stale_writer() { + let storage = Arc::new(PartialPoolMetaWriteStorage { + fail_write: false, + fail_after_first_write: false, + pending_write: false, + write_started: tokio::sync::Notify::new(), + wrote: AtomicBool::new(false), + revision: AtomicUsize::new(0), + stored: StdMutex::new(None), + identity: StdMutex::new(None), + }); + let stale_token = read_pool_meta_replica(storage.clone(), true).await.cas; + assert!(matches!(&stale_token, PoolMetaCasToken::Missing)); + let winner = PoolMeta { + version: POOL_META_VERSION, + pools: vec![decommission_test_pool_status(0, None)], + ..Default::default() + } + .encode_config_data_for_test() + .expect("winning pool metadata should encode"); + let stale = PoolMeta { + version: POOL_META_VERSION, + pools: vec![decommission_test_pool_status( + 0, + Some(PoolDecommissionInfo { + complete: true, + ..Default::default() + }), + )], + ..Default::default() + } + .encode_config_data_for_test() + .expect("stale pool metadata should encode"); + let fence = PoolMetaPersistenceFence::Distributed(None); + + save_pool_meta_object_cas(storage.clone(), POOL_META_NAME, winner.clone(), &stale_token, &fence, "prepare_cas") + .await + .expect("the first writer should create pool metadata"); + let err = save_pool_meta_object_cas(storage.clone(), POOL_META_NAME, stale, &stale_token, &fence, "prepare_cas") + .await + .expect_err("the second writer must not reuse the stale missing-object revision"); + + assert_eq!(err, Error::PreconditionFailed); + let stored = storage + .stored + .lock() + .expect("pool metadata test storage should not be poisoned") + .as_ref() + .map(|(data, _)| data.clone()) + .expect("the winning pool metadata should remain stored"); + assert_eq!(stored, winner); + } + + #[tokio::test] + async fn test_pool_meta_v3_single_replica_commit_failure_recovers_on_restart() { + let committed_replica = Arc::new(PartialPoolMetaWriteStorage { + fail_write: false, + fail_after_first_write: false, + pending_write: false, + write_started: tokio::sync::Notify::new(), + wrote: AtomicBool::new(false), + revision: AtomicUsize::new(0), + stored: StdMutex::new(None), + identity: StdMutex::new(None), + }); + let pending_replica = Arc::new(PartialPoolMetaWriteStorage { + fail_write: false, + fail_after_first_write: true, + pending_write: false, + write_started: tokio::sync::Notify::new(), + wrote: AtomicBool::new(false), + revision: AtomicUsize::new(0), + stored: StdMutex::new(None), + identity: StdMutex::new(None), + }); + let cluster_id = uuid::Uuid::new_v4(); + let fresh_bootstrap_nonce = uuid::Uuid::new_v4(); + let identity = super::encode_pool_meta_identity(super::PersistedPoolMetaIdentity { + version: super::POOL_META_IDENTITY_VERSION, + cluster_id, + epoch: super::POOL_META_INITIAL_EPOCH, + initialized: false, + fresh_bootstrap_nonce: Some(fresh_bootstrap_nonce), + }) + .expect("pending bootstrap identity should encode"); + *committed_replica + .identity + .lock() + .expect("identity storage should not be poisoned") = Some((identity.clone(), "identity-0".to_string())); + *pending_replica + .identity + .lock() + .expect("identity storage should not be poisoned") = Some((identity, "identity-0".to_string())); + let mut write_state = super::PoolMetaWriteState::for_startup(cluster_id, true); + let snapshot = PoolMeta { + version: super::POOL_META_GENERATION_VERSION, + pools: vec![decommission_test_pool_status( + 0, + Some(PoolDecommissionInfo { + queued: true, + ..Default::default() + }), + )], + ..Default::default() + }; + + snapshot + .save_no_lock_observing(vec![committed_replica.clone(), pending_replica.clone()], &mut write_state) + .await + .expect("one committed replica should durably complete the V3 transaction"); + + let mut restarted = PoolMeta::default(); + let replica_state = restarted + .load_no_lock_from_replicas(vec![committed_replica, pending_replica.clone()]) + .await + .expect("restart should select the committed generation over a pending replica"); + assert_eq!(restarted.version, super::POOL_META_GENERATION_VERSION); + assert!(restarted.pools[0].decommission.as_ref().is_some_and(|info| info.queued)); + assert!(replica_state.needs_repair); + + let mut pending_only = PoolMeta::default(); + pending_only + .load_no_lock_from_replicas(vec![pending_replica]) + .await + .expect("a prepare-only replica should expose the embedded predecessor"); + assert!(pending_only.pools.is_empty()); + } + #[tokio::test] async fn test_partial_pool_meta_save_failure_blocks_following_side_effect() { let store = decommission_worker_test_store(PoolMeta::default(), Vec::new()); let committed = Arc::new(PartialPoolMetaWriteStorage { fail_write: false, + fail_after_first_write: false, pending_write: false, write_started: tokio::sync::Notify::new(), wrote: AtomicBool::new(false), + revision: AtomicUsize::new(0), + stored: StdMutex::new(None), + identity: StdMutex::new(None), }); let failed = Arc::new(PartialPoolMetaWriteStorage { fail_write: true, + fail_after_first_write: false, pending_write: false, write_started: tokio::sync::Notify::new(), wrote: AtomicBool::new(false), + revision: AtomicUsize::new(0), + stored: StdMutex::new(None), + identity: StdMutex::new(None), }); let snapshot = PoolMeta { version: super::POOL_META_VERSION, @@ -11403,15 +13176,23 @@ mod pools_tests { let store = decommission_worker_test_store(PoolMeta::default(), Vec::new()); let committed = Arc::new(PartialPoolMetaWriteStorage { fail_write: false, + fail_after_first_write: false, pending_write: false, write_started: tokio::sync::Notify::new(), wrote: AtomicBool::new(false), + revision: AtomicUsize::new(0), + stored: StdMutex::new(None), + identity: StdMutex::new(None), }); let pending = Arc::new(PartialPoolMetaWriteStorage { fail_write: false, + fail_after_first_write: false, pending_write: true, write_started: tokio::sync::Notify::new(), wrote: AtomicBool::new(false), + revision: AtomicUsize::new(0), + stored: StdMutex::new(None), + identity: StdMutex::new(None), }); let snapshot = PoolMeta { version: super::POOL_META_VERSION, @@ -11464,9 +13245,13 @@ mod pools_tests { let store = decommission_worker_test_store(PoolMeta::default(), Vec::new()); let committed = Arc::new(PartialPoolMetaWriteStorage { fail_write: false, + fail_after_first_write: false, pending_write: false, write_started: tokio::sync::Notify::new(), wrote: AtomicBool::new(false), + revision: AtomicUsize::new(0), + stored: StdMutex::new(None), + identity: StdMutex::new(None), }); let snapshot = PoolMeta { version: super::POOL_META_VERSION, @@ -11481,12 +13266,12 @@ mod pools_tests { let task_publish_release = publish_release.clone(); let save_task = tokio::spawn(async move { let mut save_guard = save_store.pool_meta_save_gate.lock().await; - let was_write_blocked = snapshot - .save_no_lock_armed(vec![save_committed], &mut save_guard, None) + let outcome = snapshot + .save_no_lock_armed(vec![save_committed], &mut save_guard, None, &[0]) .await?; task_publish_started.notify_one(); task_publish_release.notified().await; - save_guard.restore_writes(was_write_blocked); + outcome.disarm(); Ok::<(), Error>(()) }); @@ -11532,9 +13317,13 @@ mod pools_tests { let storage = Arc::new(PartialPoolMetaWriteStorage { fail_write: false, + fail_after_first_write: false, pending_write: false, write_started: tokio::sync::Notify::new(), wrote: AtomicBool::new(false), + revision: AtomicUsize::new(0), + stored: StdMutex::new(None), + identity: StdMutex::new(None), }); let snapshot = PoolMeta { version: super::POOL_META_VERSION, @@ -11543,7 +13332,7 @@ mod pools_tests { }; let mut write_state = super::PoolMetaWriteState::default(); let err = snapshot - .save_no_lock_armed(vec![storage.clone()], &mut write_state, guard.lock_lost_signal()) + .save_no_lock_armed(vec![storage.clone()], &mut write_state, guard.lock_lost_signal(), &[0]) .await .expect_err("a writer must not persist after losing the distributed pool metadata fence"); @@ -11573,9 +13362,13 @@ mod pools_tests { .expect("the publishing writer should acquire the pool metadata fence"); let storage = Arc::new(PartialPoolMetaWriteStorage { fail_write: false, + fail_after_first_write: false, pending_write: false, write_started: tokio::sync::Notify::new(), wrote: AtomicBool::new(false), + revision: AtomicUsize::new(0), + stored: StdMutex::new(None), + identity: StdMutex::new(None), }); let mut saved = PoolMeta { version: super::POOL_META_VERSION, @@ -11585,9 +13378,9 @@ mod pools_tests { saved.pools[0].last_update = OffsetDateTime::UNIX_EPOCH + Duration::seconds(1); let mut current = saved.clone(); current.pools[0].last_update = OffsetDateTime::UNIX_EPOCH; - let mut write_state = super::PoolMetaWriteState::default(); - let was_write_blocked = saved - .save_no_lock_armed(vec![storage], &mut write_state, guard.lock_lost_signal()) + let mut write_state = super::PoolMetaWriteState::for_test_bootstrap(); + let outcome = saved + .save_no_lock_armed(vec![storage], &mut write_state, guard.lock_lost_signal(), &[0]) .await .expect("the replica save should finish while the fence is valid"); @@ -11601,7 +13394,7 @@ mod pools_tests { .expect_err("the post-publication fence check must observe the loss"); assert_eq!(current.pools[0].last_update, saved.pools[0].last_update); - assert!(!was_write_blocked); + drop(outcome); write_state .ensure_write_safe("lost pool metadata publish fence") .expect_err("the sticky write gate must remain armed after post-publication fence loss"); @@ -12105,6 +13898,42 @@ mod pools_tests { assert!(persisted.pools[1].decommission.as_ref().is_some_and(|info| info.failed)); } + #[test] + fn test_pool_meta_save_merge_rejects_stale_terminal_resurrection() { + let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid"); + let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid"); + let mut persisted = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: newer, + decommission: Some(PoolDecommissionInfo { + canceled: true, + ..Default::default() + }), + }], + ..Default::default() + }; + let stale = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: older, + decommission: Some(PoolDecommissionInfo { + start_time: Some(older), + ..Default::default() + }), + }], + ..Default::default() + }; + + let err = merge_pool_meta_updates_for_save(&mut persisted, &stale, &[0], "stale writer") + .expect_err("a stale active snapshot must not resurrect a canceled pool"); + + assert!(err.to_string().contains("stale pool metadata update rejected")); + assert!(persisted.pools[0].decommission.as_ref().is_some_and(|info| info.canceled)); + } + #[test] fn test_pool_meta_publish_preserves_untouched_runtime_progress() { let mut current = PoolMeta { @@ -13447,20 +15276,18 @@ mod pools_tests { .expect("test bucket should be created"); let generation = OffsetDateTime::now_utc(); - let active_meta = { + { let mut pool_meta = store.pool_meta.write().await; - pool_meta.version = POOL_META_VERSION; pool_meta.dont_save = false; pool_meta.pools[0].decommission = Some(PoolDecommissionInfo { start_time: Some(generation), ..Default::default() }); - pool_meta.clone() - }; - active_meta - .save(store.pools.clone()) + } + store + .save_current_pool_meta_for_test(&[0]) .await - .expect("active V2 decommission metadata should be persisted before the final sweep"); + .expect("active decommission metadata should be persisted before the final sweep"); for (disk_index, dir) in dirs.iter().enumerate() { let object_dir = dir.path().join(bucket).join(object); diff --git a/crates/ecstore/src/core/sets.rs b/crates/ecstore/src/core/sets.rs index d4c6703a3..648b7cf0d 100644 --- a/crates/ecstore/src/core/sets.rs +++ b/crates/ecstore/src/core/sets.rs @@ -106,6 +106,89 @@ impl Drop for Sets { } } +#[cfg(test)] +struct HealFormatAfterSaveBarrierState { + pool_key: usize, + disk_index: usize, + arrived: tokio::sync::Notify, + release: tokio::sync::Notify, +} + +#[cfg(test)] +static HEAL_FORMAT_AFTER_SAVE_BARRIER: std::sync::OnceLock>>> = + std::sync::OnceLock::new(); + +#[cfg(test)] +pub(crate) struct HealFormatAfterSaveBarrier { + state: Arc, +} + +#[cfg(test)] +impl HealFormatAfterSaveBarrier { + pub(crate) fn install(pool: &Arc, disk_index: usize) -> Self { + let state = Arc::new(HealFormatAfterSaveBarrierState { + pool_key: Arc::as_ptr(pool) as usize, + disk_index, + arrived: tokio::sync::Notify::new(), + release: tokio::sync::Notify::new(), + }); + let mut barrier = HEAL_FORMAT_AFTER_SAVE_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("heal format after-save barrier should not be poisoned"); + assert!(barrier.is_none(), "heal format after-save barrier must be unique"); + *barrier = Some(Arc::clone(&state)); + Self { state } + } + + pub(crate) async fn wait_until_paused(&self) { + tokio::time::timeout(std::time::Duration::from_secs(30), self.state.arrived.notified()) + .await + .expect("format heal should reach the after-save barrier"); + } + + pub(crate) fn release(&self) { + self.state.release.notify_one(); + } +} + +#[cfg(test)] +impl Drop for HealFormatAfterSaveBarrier { + fn drop(&mut self) { + self.state.release.notify_one(); + let mut barrier = HEAL_FORMAT_AFTER_SAVE_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("heal format after-save barrier should not be poisoned"); + if barrier.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { + *barrier = None; + } + } +} + +#[cfg(test)] +async fn pause_heal_format_after_save(pool: &Sets, disk_index: usize) { + let pool_key = std::ptr::from_ref(pool) as usize; + let barrier = { + let mut barrier = HEAL_FORMAT_AFTER_SAVE_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("heal format after-save barrier should not be poisoned"); + if barrier + .as_ref() + .is_some_and(|state| state.pool_key == pool_key && state.disk_index == disk_index) + { + barrier.take() + } else { + None + } + }; + if let Some(barrier) = barrier { + barrier.arrived.notify_one(); + barrier.release.notified().await; + } +} + impl Sets { #[tracing::instrument(level = "debug", skip(disks, endpoints, fm, pool_idx, parity_count))] pub async fn new( @@ -989,9 +1072,13 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for Sets { } impl Sets { - pub(crate) async fn heal_format_with_fence(&self, dry_run: bool, fence_lost: F) -> Result<(HealResultItem, Option)> + pub(crate) async fn heal_format_with_fence( + &self, + dry_run: bool, + mut fence_lost: F, + ) -> Result<(HealResultItem, Option)> where - F: Fn() -> bool + Send + Sync, + F: FnMut() -> bool + Send, { let (disks, init_errs) = init_storage_disks_with_errors( &self.endpoints.endpoints, @@ -1074,6 +1161,11 @@ impl Sets { } return Ok((res, Some(err.into()))); } + #[cfg(test)] + pause_heal_format_after_save(self, index).await; + if fence_lost() { + return Ok((res, Some(StorageError::SlowDown))); + } if let Some(saved_format) = fm.as_ref() { res.after.drives[index].uuid = saved_format.erasure.this.to_string(); res.after.drives[index].state = DriveState::Ok.to_string(); @@ -1081,8 +1173,14 @@ impl Sets { } } + if fence_lost() { + return Ok((res, Some(StorageError::SlowDown))); + } for (index, fm) in tmp_new_formats.iter().enumerate() { if let Some(fm) = fm { + if fence_lost() { + return Ok((res, Some(StorageError::SlowDown))); + } let (m, n) = match ref_format.find_disk_index_by_disk_id(fm.erasure.this) { Ok((m, n)) => (m, n), Err(_) => continue, @@ -1094,6 +1192,9 @@ impl Sets { } if let Some(Some(disk)) = disks.get(index) { + if fence_lost() { + return Ok((res, Some(StorageError::SlowDown))); + } self.disk_set[m].renew_disk(&disk.endpoint()).await; } } diff --git a/crates/ecstore/src/services/rebalance/control.rs b/crates/ecstore/src/services/rebalance/control.rs index 516b3c94b..a4a65ae86 100644 --- a/crates/ecstore/src/services/rebalance/control.rs +++ b/crates/ecstore/src/services/rebalance/control.rs @@ -540,9 +540,12 @@ impl ECStore { #[cfg(test)] crate::core::pools::observe_pool_activation_start_attempt(crate::core::pools::PoolActivationStartKind::Rebalance); let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?; + let mut pool_meta_guard = self.pool_meta_save_gate.lock().await; + pool_meta_guard.ensure_write_safe(stage)?; let activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), fleet_proof).await?; - let mut pool_meta = PoolMeta::default(); - pool_meta.load_no_lock_from_replicas(self.pools.clone()).await?; + let pool_meta = self + .load_runtime_pool_meta_under_activation_fence(&mut pool_meta_guard, &activation_fence, stage) + .await?; ensure_rebalance_activation_pool_meta_allowed(&pool_meta)?; merge_and_save_rebalance_meta_no_lock( @@ -568,9 +571,12 @@ impl ECStore { S: EcstoreObjectIO + StorageNamespaceLocking, { let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?; + let mut pool_meta_guard = self.pool_meta_save_gate.lock().await; + pool_meta_guard.ensure_write_safe("rebalance worker activation")?; let activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), fleet_proof).await?; - let mut pool_meta = PoolMeta::default(); - pool_meta.load_no_lock_from_replicas(self.pools.clone()).await?; + let pool_meta = self + .load_runtime_pool_meta_under_activation_fence(&mut pool_meta_guard, &activation_fence, "rebalance worker activation") + .await?; ensure_rebalance_activation_pool_meta_allowed(&pool_meta)?; let mut persisted = RebalanceMeta::new(); @@ -1301,10 +1307,40 @@ impl ECStore { #[cfg(test)] mod tests { use super::*; - use crate::core::pools::{PoolActivationDurableSaveBarrier, PoolActivationStartKind, PoolActivationStartProbe}; + use crate::config::com::delete_config; + use crate::core::pools::{ + POOL_META_NAME, PoolActivationDurableSaveBarrier, PoolActivationStartKind, PoolActivationStartProbe, PoolMetaWriteState, + persist_pool_meta_identity_for_startup, + }; use crate::object_api::NamespaceLockFence; use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause, hermetic_set_disks_isolated}; + async fn persist_initialized_identity_then_remove_pool_meta(store: &Arc) { + let mut write_state = PoolMetaWriteState::for_startup(store.id, false); + persist_pool_meta_identity_for_startup(store.pools.clone(), &mut write_state, true) + .await + .expect("initialized pool metadata identity should persist"); + *store.pool_meta_save_gate.lock().await = write_state; + for pool in &store.pools { + delete_config(pool.clone(), POOL_META_NAME) + .await + .expect("every pool metadata replica should be removed"); + } + } + + async fn assert_activation_locks_released(store: &Arc) { + let fleet_proof = acquire_pool_activation_fleet_proof(&store.ctx) + .await + .expect("fleet proof should remain available"); + tokio::time::timeout( + std::time::Duration::from_secs(5), + acquire_pool_rebalance_activation_locks(store.pools[0].clone(), fleet_proof), + ) + .await + .expect("a rejected activation must release namespace fences promptly") + .expect("a rejected activation must release both namespace fences"); + } + #[tokio::test] async fn rebalance_stop_wait_probe_matches_run_id() { let probe = RebalanceStopWaitProbe::install("rebalance-stop-current"); @@ -1356,6 +1392,90 @@ mod tests { assert!(cancel.is_cancelled()); } + #[tokio::test] + #[serial_test::serial] + async fn rebalance_activation_rejects_initialized_cluster_with_all_pool_meta_missing() { + let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await; + persist_initialized_identity_then_remove_pool_meta(&store).await; + set_rebalance_disk_stats_override_for_test( + store.id, + vec![ + DiskStat { + total_space: 100, + available_space: 0, + }, + DiskStat { + total_space: 100, + available_space: 100, + }, + ], + ); + + let err = store + .init_rebalance_start(vec!["missing-pool-meta".to_string()]) + .await + .expect_err("rebalance activation must fail closed when every pool.bin is missing"); + assert!(err.to_string().contains("initialized cluster identity exists")); + store + .ensure_pool_meta_side_effects_safe("rebalance activation after missing pool metadata") + .await + .expect_err("the missing metadata observation must latch the shared runtime gate"); + + let mut persisted = RebalanceMeta::new(); + assert!( + matches!(persisted.load(store.pools[0].clone()).await, Err(Error::ConfigNotFound)), + "rejected activation must not create rebalance metadata" + ); + assert_activation_locks_released(&store).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn rebalance_worker_rejects_initialized_cluster_with_all_pool_meta_missing() { + let rebalance_id = "missing-pool-meta-worker"; + let active = RebalanceMeta { + id: rebalance_id.to_string(), + percent_free_goal: 0.5, + pool_stats: vec![ + RebalanceStats { + participating: true, + init_capacity: 100, + info: RebalanceInfo { + status: RebalStatus::Started, + ..Default::default() + }, + ..Default::default() + }, + RebalanceStats { + participating: true, + init_capacity: 100, + info: RebalanceInfo { + status: RebalStatus::Started, + ..Default::default() + }, + ..Default::default() + }, + ], + ..Default::default() + }; + let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(Some(active)).await; + persist_initialized_identity_then_remove_pool_meta(&store).await; + + let err = match store + .fence_rebalance_worker_activation(store.pools[0].clone(), rebalance_id) + .await + { + Ok(_) => panic!("worker activation must not return a fence when every pool.bin is missing"), + Err(err) => err, + }; + assert!(err.to_string().contains("initialized cluster identity exists")); + store + .ensure_pool_meta_side_effects_safe("rebalance worker after missing pool metadata") + .await + .expect_err("worker validation must latch the shared runtime gate"); + assert_activation_locks_released(&store).await; + } + #[tokio::test] #[serial_test::serial] async fn rebalance_activation_adopts_commit_after_post_save_fence_loss() { diff --git a/crates/ecstore/src/services/rebalance/mod.rs b/crates/ecstore/src/services/rebalance/mod.rs index 7c55c4ea7..4701b5c7f 100644 --- a/crates/ecstore/src/services/rebalance/mod.rs +++ b/crates/ecstore/src/services/rebalance/mod.rs @@ -63,6 +63,12 @@ pub async fn test_store_with_persisted_rebalance_meta( ) -> (Vec, std::sync::Arc) { let ctx = std::sync::Arc::new(crate::runtime::instance::InstanceContext::new()); let (temp_dirs, pool) = crate::core::sets::make_local_two_set_sets_with_ctx(ctx.clone()).await; + let pools = vec![pool.clone()]; + let pool_meta = crate::core::pools::PoolMeta::new(&pools, &crate::core::pools::PoolMeta::default()); + pool_meta + .save_for_startup(pools.clone()) + .await + .expect("rebalance test pool metadata should be persisted"); meta.save(pool.clone()) .await .expect("rebalance test metadata should be persisted"); @@ -70,9 +76,9 @@ pub async fn test_store_with_persisted_rebalance_meta( let store = std::sync::Arc::new(crate::store::ECStore { id: uuid::Uuid::new_v4(), disk_map: std::collections::HashMap::new(), - pools: vec![pool], + pools, peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()), - pool_meta: tokio::sync::RwLock::new(crate::core::pools::PoolMeta::default()), + pool_meta: tokio::sync::RwLock::new(pool_meta), rebalance_meta: tokio::sync::RwLock::new(Some(meta)), decommission_cancelers: tokio::sync::RwLock::new(vec![None]), start_gate: tokio::sync::Mutex::new(()), @@ -139,7 +145,7 @@ async fn test_two_pool_stores_with_contexts( } let pool_meta = PoolMeta::new(&pools, &PoolMeta::default()); pool_meta - .save(pools.clone()) + .save_for_startup(pools.clone()) .await .expect("baseline pool metadata should be persisted"); if let Some(meta) = rebalance_meta.as_ref() { diff --git a/crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs b/crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs index e1b691f07..e0e50f4f8 100644 --- a/crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs +++ b/crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs @@ -2793,7 +2793,10 @@ async fn test_rebalance_start_save_failure_retries_persisted_completed_state() { .start_rebalance_under_gate() .await .expect_err("the injected first activation save must fail through the real start path"); - assert!(err.to_string().contains("injected rebalance activation save failure")); + assert!( + err.to_string().contains("injected rebalance activation save failure"), + "unexpected activation error: {err}" + ); { let local = store.rebalance_meta.read().await; let local = local.as_ref().expect("local rebalance metadata should remain present"); @@ -2854,7 +2857,10 @@ async fn test_rebalance_start_save_failure_retries_persisted_stopped_state() { .start_rebalance_under_gate() .await .expect_err("the injected first stopped-state save must fail through the real start path"); - assert!(err.to_string().contains("injected rebalance activation save failure")); + assert!( + err.to_string().contains("injected rebalance activation save failure"), + "unexpected activation error: {err}" + ); let mut after_failure = RebalanceMeta::new(); after_failure .load(store.pools[0].clone()) diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index 65fcfdbd6..62ada21f1 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -2034,11 +2034,24 @@ impl SetDisks { } impl SetDisks { + #[cfg(test)] pub(crate) async fn heal_replacement_format( &self, dry_run: bool, targets: &[String], ) -> Result<(HealResultItem, Option)> { + self.heal_replacement_format_with_fence(dry_run, targets, || false).await + } + + pub(crate) async fn heal_replacement_format_with_fence( + &self, + dry_run: bool, + targets: &[String], + fence_lost: F, + ) -> Result<(HealResultItem, Option)> + where + F: FnMut() -> bool + Send, + { if targets.is_empty() { return Err(Error::other("replacement format requires at least one target")); } @@ -2054,14 +2067,18 @@ impl SetDisks { target_slots.push(slot); } - self.heal_format_for_slots(dry_run, Some(&target_slots)).await + self.heal_format_for_slots(dry_run, Some(&target_slots), fence_lost).await } - async fn heal_format_for_slots( + async fn heal_format_for_slots( &self, dry_run: bool, target_slots: Option<&[usize]>, - ) -> Result<(HealResultItem, Option)> { + mut fence_lost: F, + ) -> Result<(HealResultItem, Option)> + where + F: FnMut() -> bool + Send, + { let disks = self.disks.read().await.clone(); let (formats, errs) = load_format_erasure_all(&disks, true).await; if errs.iter().any(|err| { @@ -2131,8 +2148,14 @@ impl SetDisks { let mut new_format = ref_format.clone(); new_format.erasure.this = ref_format.erasure.sets[self.set_index][disk_idx]; + if fence_lost() { + return Ok((result, Some(StorageError::SlowDown))); + } match save_format_file(&disks[disk_idx], &Some(new_format.clone())).await { Ok(()) => { + if fence_lost() { + return Ok((result, Some(StorageError::SlowDown))); + } result.after.drives[disk_idx].uuid = new_format.erasure.this.to_string(); result.after.drives[disk_idx].state = DriveState::Ok.to_string(); } @@ -2158,7 +2181,7 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks { #[tracing::instrument(skip(self))] async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option)> { - self.heal_format_for_slots(dry_run, None).await + self.heal_format_for_slots(dry_run, None, || false).await } #[tracing::instrument(skip(self))] diff --git a/crates/ecstore/src/store/heal.rs b/crates/ecstore/src/store/heal.rs index 364bb87da..da18546eb 100644 --- a/crates/ecstore/src/store/heal.rs +++ b/crates/ecstore/src/store/heal.rs @@ -13,7 +13,7 @@ // limitations under the License. use super::*; -use crate::core::pools::POOL_META_NAME; +use crate::core::pools::{POOL_META_NAME, load_pool_meta_identity_observing}; use crate::services::rebalance::{REBAL_META_NAME, RebalStatus}; use crate::set_disk::get_lock_acquire_timeout; use crate::storage_api_contracts::heal::HealOperations as _; @@ -91,7 +91,13 @@ fn heal_format_fence_lost_error() -> Error { impl ECStore { async fn acquire_heal_format_fence( &self, - ) -> Result<(NamespaceLockGuard, NamespaceLockGuard, PoolMeta, Option)> { + ) -> Result<( + tokio::sync::MutexGuard<'_, PoolMetaWriteState>, + NamespaceLockGuard, + NamespaceLockGuard, + PoolMeta, + Option, + )> { let metadata_pool = self .pools .first() @@ -111,13 +117,14 @@ impl ECStore { return Err(heal_format_fence_lost_error()); } + load_pool_meta_identity_observing(self.pools.clone(), &mut write_state).await?; let mut pool_meta = PoolMeta::default(); let replica_state = pool_meta .load_no_lock_from_replicas_observing(self.pools.clone(), &mut write_state) .await?; write_state.observe_replicas(replica_state); + write_state.ensure_missing_metadata_can_initialize()?; write_state.ensure_write_safe("heal format fence failed")?; - drop(write_state); if pool_meta.pools.len() != self.pools.len() || pool_meta.pools.iter().enumerate().any(|(pool_idx, pool)| { pool.id != pool_idx || pool.cmd_line.is_empty() || pool.cmd_line != self.pools[pool_idx].endpoints.cmd_line @@ -149,11 +156,12 @@ impl ECStore { return Err(heal_format_fence_lost_error()); } + write_state.ensure_write_safe("heal format fence failed")?; if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() { return Err(heal_format_fence_lost_error()); } - Ok((pool_guard, rebalance_guard, pool_meta, rebalance_meta)) + Ok((write_state, pool_guard, rebalance_guard, pool_meta, rebalance_meta)) } fn get_pools_for_heal_object(&self, opts: &HealOpts) -> Result>> { @@ -234,7 +242,8 @@ impl ECStore { let mut count_completed = 0; let mut first_error = None; for (pool_idx, pool) in self.pools.iter().enumerate() { - let (pool_guard, rebalance_guard, pool_meta, rebalance_meta) = self.acquire_heal_format_fence().await?; + let (mut write_state, pool_guard, rebalance_guard, pool_meta, rebalance_meta) = + self.acquire_heal_format_fence().await?; if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() { first_error.get_or_insert(heal_format_fence_lost_error()); break; @@ -249,7 +258,15 @@ impl ECStore { continue; } - let fence_lost = || pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost(); + let fence_lost = || { + let lost = pool_guard.is_lock_lost() + || rebalance_guard.is_lock_lost() + || write_state.ensure_write_safe("heal format write fence failed").is_err(); + if lost { + write_state.block_writes_after_fence_loss(); + } + lost + }; let (mut result, err) = pool.heal_format_with_fence(dry_run, fence_lost).await?; if let Some(err) = err { match err { @@ -268,7 +285,11 @@ impl ECStore { // A lease can be lost after the final write; fail closed before // reporting the pool as successfully healed. - if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() { + let fence_lost = pool_guard.is_lock_lost() + || rebalance_guard.is_lock_lost() + || write_state.ensure_write_safe("heal format publication fence failed").is_err(); + if fence_lost { + write_state.block_writes_after_fence_loss(); first_error.get_or_insert(heal_format_fence_lost_error()); break; } @@ -321,7 +342,33 @@ impl ECStore { ) })?; - set.heal_replacement_format(dry_run, targets).await + let (mut write_state, pool_guard, rebalance_guard, pool_meta, rebalance_meta) = self.acquire_heal_format_fence().await?; + if let Some(skip) = classify_heal_format_pool(pool_index, &pool.endpoints.cmd_line, &pool_meta, rebalance_meta.as_ref()) { + return Ok((HealResultItem::default(), Some(heal_format_pool_skip_error(skip)))); + } + + let fence_lost = || { + let lost = pool_guard.is_lock_lost() + || rebalance_guard.is_lock_lost() + || write_state + .ensure_write_safe("replacement format write fence failed") + .is_err(); + if lost { + write_state.block_writes_after_fence_loss(); + } + lost + }; + let result = set.heal_replacement_format_with_fence(dry_run, targets, fence_lost).await?; + let fence_lost = pool_guard.is_lock_lost() + || rebalance_guard.is_lock_lost() + || write_state + .ensure_write_safe("replacement format publication fence failed") + .is_err(); + if fence_lost { + write_state.block_writes_after_fence_loss(); + return Ok((result.0, Some(heal_format_fence_lost_error()))); + } + Ok(result) } #[instrument(skip(self, targets), fields(pool_index, set_index, target_count = targets.len()))] @@ -545,9 +592,13 @@ mod tests { use super::*; use crate::bucket::metadata_sys; use crate::cluster::rpc::PeerS3Client; - use crate::core::pools::{PoolDecommissionInfo, PoolMetaReplicaState, PoolStatus}; + use crate::config::com::{delete_config, read_config_no_lock_preserve_empty_with_metadata, save_config}; + use crate::core::pools::{ + POOL_META_IDENTITY_NAME, PoolDecommissionInfo, PoolMetaReplicaState, PoolStatus, initialized_pool_meta_identity_for_test, + }; + use crate::core::sets::HealFormatAfterSaveBarrier; use crate::disk::error::Result as DiskResult; - use crate::disk::{DeleteOptions, DiskOption, format::FormatV3, new_disk}; + use crate::disk::{DeleteOptions, DiskOption, FORMAT_CONFIG_FILE, format::FormatV3, new_disk}; use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}; use crate::runtime::instance::InstanceContext; use crate::services::rebalance::{RebalanceInfo, RebalanceStats}; @@ -557,6 +608,7 @@ mod tests { use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations}; use crate::store::init_format::{load_format_erasure, save_format_file}; use crate::store::init_local_disks_with_instance_ctx; + use rustfs_common::heal_channel::DriveState; use tokio_util::sync::CancellationToken; #[derive(Debug)] @@ -933,6 +985,197 @@ mod tests { (temp_dir, store, shutdown) } + fn heal_test_format_path(temp_dir: &tempfile::TempDir, pool_index: usize, disk_index: usize) -> std::path::PathBuf { + temp_dir + .path() + .join(format!("pool{pool_index}-disk{disk_index}")) + .join(crate::disk::RUSTFS_META_BUCKET) + .join(FORMAT_CONFIG_FILE) + } + + async fn remove_heal_test_format( + temp_dir: &tempfile::TempDir, + store: &ECStore, + pool_index: usize, + disk_index: usize, + ) -> String { + let target = store.pools[pool_index].endpoints.endpoints.as_ref()[disk_index].to_string(); + let format_path = heal_test_format_path(temp_dir, pool_index, disk_index); + tokio::fs::remove_file(&format_path) + .await + .expect("replacement target format should be removable"); + assert!( + !tokio::fs::try_exists(&format_path) + .await + .expect("replacement target format path should be inspectable") + ); + target + } + + async fn assert_heal_test_format_missing(temp_dir: &tempfile::TempDir, pool_index: usize, disk_index: usize) { + assert!( + !tokio::fs::try_exists(heal_test_format_path(temp_dir, pool_index, disk_index)) + .await + .expect("replacement target format path should be inspectable"), + "format heal must not write the replacement target" + ); + } + + #[tokio::test] + #[serial_test::serial] + async fn full_format_heal_preblocked_pool_metadata_never_writes_format() { + let (temp_dir, store, shutdown) = multi_pool_heal_store().await; + remove_heal_test_format(&temp_dir, &store, 0, 3).await; + store.pool_meta_save_gate.lock().await.observe_replicas(PoolMetaReplicaState { + needs_repair: true, + repair_write_safe: false, + }); + + let err = store + .handle_heal_format(false) + .await + .expect_err("a preblocked pool metadata state must reject full format heal"); + assert!( + err.to_string() + .contains("restart after all replicas are readable and consistent") + ); + assert_heal_test_format_missing(&temp_dir, 0, 3).await; + store + .ensure_pool_meta_side_effects_safe("preblocked format heal side effect") + .await + .expect_err("the preblocked state must remain sticky"); + shutdown.cancel(); + } + + #[tokio::test] + #[serial_test::serial] + async fn full_format_heal_future_identity_never_writes_format_and_latches() { + let (temp_dir, store, shutdown) = multi_pool_heal_store().await; + remove_heal_test_format(&temp_dir, &store, 0, 3).await; + let (mut future_identity, _) = + read_config_no_lock_preserve_empty_with_metadata(store.pools[1].clone(), POOL_META_IDENTITY_NAME) + .await + .expect("current identity should be readable"); + let future_version = u16::from_le_bytes([future_identity[2], future_identity[3]]) + .checked_add(1) + .expect("identity version should have a future value"); + future_identity[2..4].copy_from_slice(&future_version.to_le_bytes()); + save_config(store.pools[1].clone(), POOL_META_IDENTITY_NAME, future_identity) + .await + .expect("future identity should be persisted"); + + let err = store + .handle_heal_format(false) + .await + .expect_err("a future identity must reject full format heal"); + assert!(err.to_string().contains("pool metadata incompatible")); + assert_heal_test_format_missing(&temp_dir, 0, 3).await; + store + .ensure_pool_meta_side_effects_safe("future identity format heal side effect") + .await + .expect_err("the future identity rejection must latch the write gate"); + shutdown.cancel(); + } + + #[tokio::test] + #[serial_test::serial] + async fn replacement_format_heal_epoch_conflict_never_writes_format_and_latches() { + let (temp_dir, store, shutdown) = multi_pool_heal_store().await; + let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await; + let conflicting_identity = + initialized_pool_meta_identity_for_test(store.id, 2).expect("conflicting identity should encode"); + save_config(store.pools[1].clone(), POOL_META_IDENTITY_NAME, conflicting_identity) + .await + .expect("conflicting identity should be persisted"); + + let err = store + .heal_replacement_format(false, 0, 0, &[target]) + .await + .expect_err("an identity epoch conflict must reject replacement format heal"); + assert!( + err.to_string() + .contains("identity replicas disagree on cluster identity or epoch") + ); + assert_heal_test_format_missing(&temp_dir, 0, 3).await; + store + .ensure_pool_meta_side_effects_safe("epoch conflict replacement format side effect") + .await + .expect_err("the identity epoch conflict must latch the write gate"); + shutdown.cancel(); + } + + #[tokio::test] + #[serial_test::serial] + async fn replacement_format_heal_initialized_identity_without_pool_meta_never_writes_and_latches() { + let (temp_dir, store, shutdown) = multi_pool_heal_store().await; + let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await; + for pool in &store.pools { + delete_config(pool.clone(), POOL_META_NAME) + .await + .expect("pool metadata replica should be removable"); + } + + let err = store + .heal_replacement_format(false, 0, 0, &[target]) + .await + .expect_err("initialized identity without pool metadata must reject replacement format heal"); + assert!(err.to_string().contains("initialized cluster identity exists")); + assert_heal_test_format_missing(&temp_dir, 0, 3).await; + store + .ensure_pool_meta_side_effects_safe("missing pool metadata replacement format side effect") + .await + .expect_err("missing initialized pool metadata must latch the write gate"); + shutdown.cancel(); + } + + #[tokio::test] + #[serial_test::serial] + async fn full_format_heal_lost_after_last_save_does_not_publish_or_renew_and_latches() { + let (temp_dir, store, shutdown) = multi_pool_heal_store().await; + let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await; + store.pools[0].disk_set[0].disks.write().await[3] = None; + let barrier = HealFormatAfterSaveBarrier::install(&store.pools[0], 3); + let recovery_latch = store.pool_meta_save_gate.lock().await.aborted_transaction_latch_for_test(); + let mut heal = tokio::spawn({ + let store = Arc::clone(&store); + async move { store.handle_heal_format(false).await } + }); + + barrier.wait_until_paused().await; + let saved = tokio::fs::read(heal_test_format_path(&temp_dir, 0, 3)) + .await + .expect("the last replacement format must be durable before fence loss"); + let saved = FormatV3::try_from(saved.as_slice()).expect("the durable replacement format should decode"); + assert_eq!(saved.erasure.this, store.pools[0].format.erasure.sets[0][3]); + recovery_latch.store(true, std::sync::atomic::Ordering::SeqCst); + barrier.release(); + + let (result, err) = tokio::time::timeout(std::time::Duration::from_secs(30), &mut heal) + .await + .expect("format heal should stop after the lost fence") + .expect("format heal task should not panic") + .expect("format heal should return its fenced result"); + assert!(matches!(err, Some(StorageError::SlowDown))); + assert!( + result + .after + .drives + .iter() + .any(|drive| drive.endpoint == target && drive.state == DriveState::Missing.to_string()), + "the lost fence must prevent the durable format from being published in the heal result" + ); + assert!( + store.pools[0].disk_set[0].disks.read().await[3].is_none(), + "the lost fence must prevent renew_disk from attaching the replacement" + ); + recovery_latch.store(false, std::sync::atomic::Ordering::SeqCst); + store + .ensure_pool_meta_side_effects_safe("post-save format fence loss side effect") + .await + .expect_err("post-save format fence loss must remain sticky"); + shutdown.cancel(); + } + #[tokio::test] async fn heal_object_pool_scope_selects_only_requested_pool() { let store = minimal_heal_store().await; @@ -1573,13 +1816,18 @@ mod tests { .handle_heal_format(false) .await .expect_err("missing pool metadata must fail closed before format writes"); - assert!(matches!(err, StorageError::SlowDown)); + assert!(err.to_string().contains("no durable bootstrap identity or pool.bin replica")); + store + .ensure_pool_meta_side_effects_safe("missing format-heal metadata side effect") + .await + .expect_err("missing metadata must latch the format-heal write gate"); let pool_meta = PoolMeta::new(&store.pools, &PoolMeta::default()); pool_meta .save(store.pools.clone()) .await .expect("pool metadata should be persisted before format heal"); + *store.pool_meta_save_gate.lock().await = PoolMetaWriteState::default(); let (result, err) = store .handle_heal_format(false) diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 2d6453508..65292980a 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -14,7 +14,8 @@ use super::*; use crate::core::pools::{ - PoolMetaReplicaState, PoolMetaWriteState, local_decommission_queue_prefix, pool_meta_has_active_decommission, + PoolMetaReplicaState, PoolMetaWriteState, load_pool_meta_identity_observing, local_decommission_queue_prefix, + persist_pool_meta_identity_for_startup, pool_meta_has_active_decommission, }; use crate::error::is_err_decommission_running; use crate::runtime::instance::InstanceContext; @@ -137,47 +138,84 @@ async fn load_pool_meta_for_startup( where S: EcstoreObjectIO, { + load_pool_meta_identity_observing(pools.clone(), write_state) + .await + .map_err(|err| Error::other(format!("store init failed during load_pool_meta_identity: {err}")))?; let mut meta = PoolMeta::default(); let replica_state = meta .load_no_lock_from_replicas_observing(pools, write_state) .await .map_err(|err| Error::other(format!("store init failed during load_pool_meta: {err}")))?; write_state.observe_replicas(replica_state); + write_state + .ensure_missing_metadata_can_initialize() + .map_err(|err| Error::other(format!("store init failed during classify_pool_meta_absence: {err}")))?; Ok((meta, replica_state)) } -async fn save_validated_pool_meta_for_startup(meta: &PoolMeta, pools: Vec>) -> Result<()> +async fn establish_pool_meta_bootstrap_identity_if_proven( + pools: Vec>, + write_state: &mut PoolMetaWriteState, + elected_writer: bool, +) -> Result<()> where S: EcstoreObjectIO, { - resolve_store_init_stage_result(meta.save_for_startup(pools).await, "save_validated_pool_meta") + if elected_writer && write_state.fresh_bootstrap_proven() { + persist_pool_meta_identity_for_startup(pools, write_state, false).await?; + } + Ok(()) +} + +async fn save_validated_pool_meta_for_startup( + meta: &PoolMeta, + pools: Vec>, + write_state: &mut PoolMetaWriteState, +) -> Result +where + S: EcstoreObjectIO, +{ + meta.save_for_startup_observing(pools, write_state) + .await + .map_err(|err| Error::other(format!("store init failed during save_validated_pool_meta: {err}"))) } async fn persist_pool_meta_for_startup_if_safe( meta: &PoolMeta, pools: Vec>, replica_state: PoolMetaReplicaState, - write_state: PoolMetaWriteState, + write_state: &mut PoolMetaWriteState, topology_update: bool, elected_writer: bool, -) -> Result<()> +) -> Result where S: EcstoreObjectIO, { if !elected_writer { - return Ok(()); + return Ok(meta.clone()); } let should_write = topology_update || (replica_state.needs_repair && replica_state.repair_write_safe); if topology_update { replica_state.ensure_write_safe("store init failed during save_validated_pool_meta")?; } - if should_write { + if should_write || write_state.identity_requires_repair() { write_state.ensure_write_safe("store init failed during save_validated_pool_meta")?; } + let mut committed = meta.clone(); if should_write { - save_validated_pool_meta_for_startup(meta, pools).await?; + if write_state.fresh_bootstrap_proven() || write_state.identity_is_pending() { + persist_pool_meta_identity_for_startup(pools.clone(), write_state, false) + .await + .map_err(|err| Error::other(format!("store init failed during prepare_pool_meta_identity: {err}")))?; + } + committed = save_validated_pool_meta_for_startup(meta, pools.clone(), write_state).await?; } - Ok(()) + if should_write || write_state.identity_requires_repair() { + persist_pool_meta_identity_for_startup(pools, write_state, true) + .await + .map_err(|err| Error::other(format!("store init failed during commit_pool_meta_identity: {err}")))?; + } + Ok(committed) } async fn resume_local_decommission_after_init(store: Arc, rx: CancellationToken, pool_indices: Vec) { @@ -286,6 +324,7 @@ impl ECStore { preflight_startup_rpc_secret(&endpoint_pools)?; let mut deployment_id = None; + let mut fresh_bootstrap_proven = true; // let (endpoint_pools, _) = EndpointServerPools::create_server_endpoints(address.as_str(), &layouts)?; @@ -345,7 +384,7 @@ impl ECStore { check_disk_fatal_errs(&errs)?; - let fm = { + let loaded_format = { let mut times = 0; let mut interval = 1; loop { @@ -401,6 +440,8 @@ impl ECStore { } } }?; + fresh_bootstrap_proven &= loaded_format.fresh_bootstrap_proven; + let fm = loaded_format.format; // Format loading succeeded, enable health monitoring on all disks for disk in disks.iter().flatten() { @@ -436,13 +477,14 @@ impl ECStore { runtime_sources::record_local_disks(&instance_ctx, local_disks).await; } + let deployment_id = deployment_id.ok_or_else(|| Error::other("store init failed: deployment id is not initialized"))?; let peer_sys = S3PeerSys::new_with_instance_ctx(&endpoint_pools, instance_ctx.clone()); let mut pool_meta = PoolMeta::new(&pools, &PoolMeta::default()); pool_meta.dont_save = true; let decommission_cancelers = RwLock::new(vec![None; pools.len()]); let ec = Arc::new(ECStore { - id: deployment_id.ok_or_else(|| Error::other("store init failed: deployment id is not initialized"))?, + id: deployment_id, disk_map, pools, peer_sys, @@ -450,7 +492,7 @@ impl ECStore { rebalance_meta: RwLock::new(None), decommission_cancelers, start_gate: Mutex::new(()), - pool_meta_save_gate: Mutex::default(), + pool_meta_save_gate: Mutex::new(PoolMetaWriteState::for_startup(deployment_id, fresh_bootstrap_proven)), // Adopt the caller's context (the process bootstrap one on the // legacy path) so startup writes (erasure type recorded before // this point) and later reads share one cell. @@ -459,10 +501,8 @@ impl ECStore { }); // Only set it when this instance's deployment ID is not yet configured - if let Some(dep_id) = deployment_id - && instance_ctx.deployment_id().is_none() - { - instance_ctx.set_deployment_id(dep_id); + if instance_ctx.deployment_id().is_none() { + instance_ctx.set_deployment_id(deployment_id); } let wait_sec = 5; @@ -494,15 +534,21 @@ impl ECStore { pub async fn init(self: &Arc, rx: CancellationToken) -> Result<()> { runtime_sources::ensure_boot_time().await; + let should_persist_pool_meta = self + .pools + .first() + .is_some_and(|pool| pool_first_endpoint_is_local(&pool.endpoints)); let (meta, pool_meta_replica_state) = { let mut write_state = self.pool_meta_save_gate.lock().await; + establish_pool_meta_bootstrap_identity_if_proven(self.pools.clone(), &mut write_state, should_persist_pool_meta) + .await + .map_err(|err| Error::other(format!("store init failed during establish_pool_meta_bootstrap_identity: {err}")))?; load_pool_meta_for_startup(self.pools.clone(), &mut write_state).await? }; let update = meta.validate(self.pools.clone())?; let endpoints = runtime_sources::endpoint_pools_or_default(); - let should_persist_pool_meta = runtime_sources::first_cluster_node_is_local().await; - let installed_pool_meta = if update { + let mut installed_pool_meta = if update { PoolMeta::new(&self.pools, &meta) } else { meta.clone() @@ -510,12 +556,12 @@ impl ECStore { // Only one local node should persist validated pool metadata here; otherwise // distributed startup can race on the same lock and replay the prior init bug. { - let write_state = self.pool_meta_save_gate.lock().await; - persist_pool_meta_for_startup_if_safe( + let mut write_state = self.pool_meta_save_gate.lock().await; + installed_pool_meta = persist_pool_meta_for_startup_if_safe( &installed_pool_meta, self.pools.clone(), pool_meta_replica_state, - *write_state, + &mut write_state, update, should_persist_pool_meta, ) @@ -615,11 +661,11 @@ impl ECStore { #[cfg(test)] mod tests { use super::{ - LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES, PoolMetaWriteState, load_pool_meta_for_startup, - persist_pool_meta_for_startup_if_safe, pool_first_endpoint_is_local, pool_meta_has_active_decommission, - preflight_startup_rpc_secret_with, resolve_startup_pool_defaults_with, resolve_store_init_stage_result, - save_validated_pool_meta_for_startup, should_auto_start_rebalance_after_init, should_retry_format_load, - should_retry_local_decommission_resume, wait_for_local_decommission_resume_delay, + LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES, PoolMetaWriteState, establish_pool_meta_bootstrap_identity_if_proven, + load_pool_meta_for_startup, persist_pool_meta_for_startup_if_safe, pool_first_endpoint_is_local, + pool_meta_has_active_decommission, preflight_startup_rpc_secret_with, resolve_startup_pool_defaults_with, + resolve_store_init_stage_result, save_validated_pool_meta_for_startup, should_auto_start_rebalance_after_init, + should_retry_format_load, should_retry_local_decommission_resume, wait_for_local_decommission_resume_delay, }; #[cfg(feature = "test-util")] use crate::disk::DiskAPI; @@ -677,7 +723,10 @@ mod tests { }; use crate::{ bucket::replication::{ReplicationState, ReplicationStatusType, replication_statuses_map}, - core::pools::{POOL_META_VERSION, PoolDecommissionInfo, PoolMeta, PoolStatus}, + core::pools::{ + POOL_META_IDENTITY_NAME, POOL_META_NAME, POOL_META_VERSION, PoolDecommissionInfo, PoolMeta, PoolStatus, + pool_meta_identity_initialized_for_test, pool_meta_v3_commit_state_for_test, + }, disk::endpoint::Endpoint, error::{Error, Result, StorageError}, io_support::rio::{WritePlan, compression_metadata_value}, @@ -718,6 +767,7 @@ mod tests { use time::OffsetDateTime; use tokio::io::AsyncReadExt; use tokio_util::sync::CancellationToken; + use uuid::Uuid; fn startup_pool_meta_payload(meta: &PoolMeta) -> Vec { meta.encode_config_data_for_test().expect("pool metadata should encode") @@ -731,10 +781,19 @@ mod tests { wrote_without_lock: AtomicBool, wrote_with_max_parity: AtomicBool, written_payload: Mutex>>, + revision: AtomicUsize, + pool_meta_write_attempts: AtomicUsize, + fail_pool_meta_write_at: AtomicUsize, + pool_meta_written_versions: Mutex>, + objects: Mutex, String)>>, } impl StartupPoolMetaStorage { fn new(read_payload: Vec) -> Self { + let mut objects = HashMap::new(); + if !read_payload.is_empty() { + objects.insert(POOL_META_NAME.to_string(), (read_payload.clone(), "startup-pool-meta-0".to_string())); + } Self { read_payload, read_error: false, @@ -742,6 +801,11 @@ mod tests { wrote_without_lock: AtomicBool::new(false), wrote_with_max_parity: AtomicBool::new(false), written_payload: Mutex::new(None), + revision: AtomicUsize::new(0), + pool_meta_write_attempts: AtomicUsize::new(0), + fail_pool_meta_write_at: AtomicUsize::new(0), + pool_meta_written_versions: Mutex::new(Vec::new()), + objects: Mutex::new(objects), } } @@ -753,15 +817,21 @@ mod tests { wrote_without_lock: AtomicBool::new(false), wrote_with_max_parity: AtomicBool::new(false), written_payload: Mutex::new(None), + revision: AtomicUsize::new(0), + pool_meta_write_attempts: AtomicUsize::new(0), + fail_pool_meta_write_at: AtomicUsize::new(0), + pool_meta_written_versions: Mutex::new(Vec::new()), + objects: Mutex::new(HashMap::new()), } } - fn object_info(&self, bucket: &str, object: &str, size: usize) -> ObjectInfo { + fn object_info(&self, bucket: &str, object: &str, size: usize, etag: String) -> ObjectInfo { ObjectInfo { bucket: bucket.to_string(), name: object.to_string(), size: size as i64, actual_size: size as i64, + etag: Some(etag), ..Default::default() } } @@ -790,13 +860,19 @@ mod tests { if self.read_error { return Err(Error::other("pool metadata read quorum unavailable")); } - if self.read_payload.is_empty() { + let Some((payload, etag)) = self + .objects + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(object) + .cloned() + else { return Err(Error::FileNotFound); - } + }; Ok(GetObjectReader { - stream: Box::new(Cursor::new(self.read_payload.clone())), - object_info: self.object_info(bucket, object, self.read_payload.len()), + stream: Box::new(Cursor::new(payload.clone())), + object_info: self.object_info(bucket, object, payload.len(), etag), buffered_body: None, body_source: Default::default(), }) @@ -812,11 +888,53 @@ mod tests { assert!(opts.no_lock, "store init pool metadata save must not require namespace locks"); self.wrote_without_lock.store(true, Ordering::SeqCst); self.wrote_with_max_parity.store(opts.max_parity, Ordering::SeqCst); + if object == POOL_META_NAME { + let attempt = self.pool_meta_write_attempts.fetch_add(1, Ordering::SeqCst) + 1; + if self.fail_pool_meta_write_at.load(Ordering::SeqCst) == attempt { + return Err(Error::Timeout); + } + } + let current_etag = self + .objects + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(object) + .map(|(_, etag)| etag.clone()); + if opts + .http_preconditions + .as_ref() + .and_then(|preconditions| preconditions.if_none_match_value()) + == Some("*") + && current_etag.is_some() + { + return Err(Error::PreconditionFailed); + } + if let Some(expected) = opts + .http_preconditions + .as_ref() + .and_then(|preconditions| preconditions.if_match_value()) + && current_etag.as_deref() != Some(expected) + { + return Err(Error::PreconditionFailed); + } let mut payload = Vec::new(); data.stream.read_to_end(&mut payload).await?; let size = payload.len(); - *self.written_payload.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(payload); - Ok(self.object_info(bucket, object, size)) + if object == POOL_META_NAME { + *self.written_payload.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(payload.clone()); + if payload.len() >= 4 { + self.pool_meta_written_versions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(u16::from_le_bytes([payload[2], payload[3]])); + } + } + let etag = format!("startup-pool-meta-{}", self.revision.fetch_add(1, Ordering::SeqCst) + 1); + self.objects + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert(object.to_string(), (payload, etag.clone())); + Ok(self.object_info(bucket, object, size, etag)) } } @@ -834,9 +952,12 @@ mod tests { } #[tokio::test] - async fn test_store_init_pool_meta_io_bypasses_namespace_lock_surface() { + async fn test_fresh_bootstrap_pool_meta_save_reaches_v3_cas_without_blocking_itself() { let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new())); - let mut write_state = PoolMetaWriteState::default(); + let mut write_state = PoolMetaWriteState::for_startup(Uuid::new_v4(), true); + establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut write_state, true) + .await + .expect("the elected fresh bootstrap should persist its identity before loading pool metadata"); let (loaded, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut write_state) .await @@ -851,11 +972,303 @@ mod tests { pools: Vec::new(), dont_save: false, }; - save_validated_pool_meta_for_startup(&meta, vec![storage.clone()]) + save_validated_pool_meta_for_startup(&meta, vec![storage.clone()], &mut write_state) .await .expect("startup pool metadata save should bypass locks"); assert!(storage.wrote_without_lock.load(Ordering::SeqCst)); assert!(storage.wrote_with_max_parity.load(Ordering::SeqCst)); + assert_eq!( + storage.pool_meta_write_attempts.load(Ordering::SeqCst), + 2, + "fresh bootstrap should reach both V3 prepare and commit CAS writes" + ); + assert_eq!( + *storage + .pool_meta_written_versions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner), + vec![3, 3] + ); + write_state + .ensure_write_safe("fresh bootstrap publication") + .expect("successful startup publication must disarm the transaction guard"); + } + + #[tokio::test] + async fn test_unproven_pending_identity_cannot_authorize_all_missing_pool_meta() { + let deployment_id = Uuid::new_v4(); + let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new())); + let mut proven_bootstrap = PoolMetaWriteState::for_startup(deployment_id, true); + establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut proven_bootstrap, true) + .await + .expect("fresh topology proof should persist a nonce-bound pending identity"); + + let identity = storage + .objects + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(POOL_META_IDENTITY_NAME) + .map(|(payload, _)| payload.clone()) + .expect("pending identity should be durable"); + assert!(!pool_meta_identity_initialized_for_test(&identity).expect("decode pending identity")); + + let mut unproven_restart = PoolMetaWriteState::for_startup(deployment_id, false); + let err = load_pool_meta_for_startup(vec![storage.clone()], &mut unproven_restart) + .await + .expect_err("a pending identity alone must not authorize an all-missing restart"); + assert!(err.to_string().contains("no verified fresh-bootstrap proof")); + unproven_restart + .ensure_write_safe("unproven pending identity restart") + .expect_err("the rejected restart must latch the write gate"); + assert!( + !storage + .objects + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .contains_key(POOL_META_NAME), + "classification must not create pool metadata" + ); + } + + #[tokio::test] + async fn test_nonfresh_identity_repair_crash_never_persists_pending_bootstrap_authority() { + let deployment_id = Uuid::new_v4(); + let initial = init_test_pool_meta(None); + let storage = Arc::new(StartupPoolMetaStorage::new(startup_pool_meta_payload(&initial))); + let mut write_state = PoolMetaWriteState::for_startup(deployment_id, false); + let (loaded, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut write_state) + .await + .expect("an existing pool metadata snapshot may repair its missing identity"); + + storage.fail_pool_meta_write_at.store(1, Ordering::SeqCst); + persist_pool_meta_for_startup_if_safe(&loaded, vec![storage.clone()], replica_state, &mut write_state, true, true) + .await + .expect_err("the injected crash boundary should stop before the pool metadata replacement"); + + let identity = storage + .objects + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(POOL_META_IDENTITY_NAME) + .map(|(payload, _)| payload.clone()) + .expect("identity repair should be durable before the failed topology write"); + assert!( + pool_meta_identity_initialized_for_test(&identity).expect("decode repaired identity"), + "an existing cluster must never leave pending bootstrap authority at this crash boundary" + ); + + storage + .objects + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(POOL_META_NAME); + let mut restarted = PoolMetaWriteState::for_startup(deployment_id, false); + let err = load_pool_meta_for_startup(vec![storage], &mut restarted) + .await + .expect_err("wiping pool metadata after the crash must require recovery"); + assert!(err.to_string().contains("initialized cluster identity exists")); + } + + #[tokio::test] + #[serial_test::serial(pool_meta_version_env)] + async fn test_existing_legacy_identity_and_replica_repair_respects_disabled_v3_gates() { + temp_env::async_with_vars( + [ + (rustfs_config::ENV_POOL_META_V3_WRITE, None::<&str>), + (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, None::<&str>), + ], + async { + let deployment_id = Uuid::new_v4(); + let corrupt = Arc::new(StartupPoolMetaStorage::new(vec![0, 1, 2])); + let legacy = init_test_pool_meta(None); + let valid = Arc::new(StartupPoolMetaStorage::new(startup_pool_meta_payload(&legacy))); + let mut write_state = PoolMetaWriteState::for_startup(deployment_id, false); + let (loaded, replica_state) = load_pool_meta_for_startup(vec![corrupt.clone(), valid.clone()], &mut write_state) + .await + .expect("existing V2 metadata should remain readable while its identity is missing"); + assert!(replica_state.needs_repair); + + let committed = persist_pool_meta_for_startup_if_safe( + &loaded, + vec![corrupt.clone(), valid.clone()], + replica_state, + &mut write_state, + true, + true, + ) + .await + .expect("legacy replica and identity repair should succeed without crossing the V3 gate"); + assert_eq!(committed.version, POOL_META_VERSION); + for storage in [corrupt, valid] { + assert_eq!( + *storage + .pool_meta_written_versions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner), + vec![POOL_META_VERSION], + "legacy repair must write exactly one V2 snapshot" + ); + let identity = storage + .objects + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(POOL_META_IDENTITY_NAME) + .map(|(payload, _)| payload.clone()) + .expect("identity repair should persist on every pool"); + assert!(pool_meta_identity_initialized_for_test(&identity).expect("decode repaired identity")); + } + }, + ) + .await; + } + + #[tokio::test] + async fn test_store_init_distinguishes_fresh_deployment_from_wiped_lagging_node() { + let deployment_id = Uuid::new_v4(); + let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new())); + let mut fresh_state = PoolMetaWriteState::for_startup(deployment_id, true); + establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut fresh_state, true) + .await + .expect("fresh topology proof should become a durable pending identity"); + let (_, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut fresh_state) + .await + .expect("new formats with no identity may initialize pool metadata exactly once"); + let committed = persist_pool_meta_for_startup_if_safe( + &init_test_pool_meta(None), + vec![storage.clone()], + replica_state, + &mut fresh_state, + true, + true, + ) + .await + .expect("fresh deployment should durably commit identity and pool metadata"); + assert_eq!(committed.pools[0].cmd_line, "pool-0"); + { + let objects = storage.objects.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + assert!(objects.contains_key(POOL_META_NAME)); + assert!(objects.contains_key(POOL_META_IDENTITY_NAME)); + } + + storage + .objects + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(POOL_META_NAME); + let mut wiped_pool_state = PoolMetaWriteState::for_startup(deployment_id, false); + let err = load_pool_meta_for_startup(vec![storage.clone()], &mut wiped_pool_state) + .await + .expect_err("an initialized identity must prevent an empty pool metadata rebuild"); + assert!(err.to_string().contains("initialized cluster identity exists")); + + storage + .objects + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(POOL_META_IDENTITY_NAME); + let mut wiped_all_state = PoolMetaWriteState::for_startup(deployment_id, false); + let err = load_pool_meta_for_startup(vec![storage], &mut wiped_all_state) + .await + .expect_err("existing formats without identity or pool metadata must require recovery"); + assert!(err.to_string().contains("no durable bootstrap identity")); + + let distributed_node = Arc::new(StartupPoolMetaStorage::new(Vec::new())); + let mut non_elected_state = PoolMetaWriteState::for_startup(Uuid::new_v4(), true); + establish_pool_meta_bootstrap_identity_if_proven(vec![distributed_node.clone()], &mut non_elected_state, false) + .await + .expect("a non-elected distributed node must not create bootstrap authority"); + let err = load_pool_meta_for_startup(vec![distributed_node], &mut non_elected_state) + .await + .expect_err("a distributed node without durable identity or pool metadata must require recovery"); + assert!(err.to_string().contains("no durable bootstrap identity")); + } + + #[tokio::test] + async fn test_store_init_resumes_pending_v3_bootstrap_without_legacy_overwrite() { + let deployment_id = Uuid::new_v4(); + let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new())); + let mut first_bootstrap = PoolMetaWriteState::for_startup(deployment_id, true); + establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut first_bootstrap, true) + .await + .expect("fresh bootstrap should persist a pending identity"); + let (_, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut first_bootstrap) + .await + .expect("the durable pending identity should authorize the initial pool metadata write"); + + storage.fail_pool_meta_write_at.store(2, Ordering::SeqCst); + persist_pool_meta_for_startup_if_safe( + &init_test_pool_meta(None), + vec![storage.clone()], + replica_state, + &mut first_bootstrap, + true, + true, + ) + .await + .expect_err("the injected crash boundary should leave only the V3 prepare record"); + + let (pending_pool, pending_identity) = { + let objects = storage.objects.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + ( + objects + .get(POOL_META_NAME) + .map(|(payload, _)| payload.clone()) + .expect("the V3 prepare record should be durable"), + objects + .get(POOL_META_IDENTITY_NAME) + .map(|(payload, _)| payload.clone()) + .expect("the bootstrap identity should be durable"), + ) + }; + assert_eq!(pool_meta_v3_commit_state_for_test(pending_pool).expect("decode pending V3"), (1, false)); + assert!(!pool_meta_identity_initialized_for_test(&pending_identity).expect("decode pending identity")); + + storage.fail_pool_meta_write_at.store(0, Ordering::SeqCst); + let mut restarted = PoolMetaWriteState::for_startup(deployment_id, false); + let (loaded, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut restarted) + .await + .expect("restart should recover the predecessor embedded in the pending V3 record"); + assert!(loaded.pools.is_empty()); + assert!(replica_state.needs_repair); + let committed = persist_pool_meta_for_startup_if_safe( + &init_test_pool_meta(None), + vec![storage.clone()], + replica_state, + &mut restarted, + true, + true, + ) + .await + .expect("restart should finish generation 1 before promoting the identity"); + assert_eq!(committed.version, 3); + + let (committed_pool, committed_identity) = { + let objects = storage.objects.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + ( + objects + .get(POOL_META_NAME) + .map(|(payload, _)| payload.clone()) + .expect("the committed V3 record should be durable"), + objects + .get(POOL_META_IDENTITY_NAME) + .map(|(payload, _)| payload.clone()) + .expect("the committed identity should be durable"), + ) + }; + assert_eq!( + pool_meta_v3_commit_state_for_test(committed_pool).expect("decode committed V3"), + (1, true) + ); + assert!(pool_meta_identity_initialized_for_test(&committed_identity).expect("decode committed identity")); + assert!( + storage + .pool_meta_written_versions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .iter() + .all(|version| *version == 3), + "bootstrap recovery must never overwrite a pending V3 record with legacy metadata" + ); } #[tokio::test] @@ -880,7 +1293,7 @@ mod tests { &loaded, vec![corrupt.clone(), backup.clone()], replica_state, - write_state, + &mut write_state, false, true, ) @@ -919,7 +1332,7 @@ mod tests { &loaded, vec![valid.clone(), unreadable.clone()], replica_state, - write_state, + &mut write_state, false, true, ) @@ -932,7 +1345,7 @@ mod tests { &loaded, vec![valid.clone(), unreadable.clone()], replica_state, - write_state, + &mut write_state, true, true, ) @@ -968,9 +1381,10 @@ mod tests { .expect("a later startup retry may read the repaired replica"); assert!(replica_state.repair_write_safe); - let err = persist_pool_meta_for_startup_if_safe(&loaded, vec![repaired.clone()], replica_state, write_state, true, true) - .await - .expect_err("the same store instance must not write after observing unreadable replicas"); + let err = + persist_pool_meta_for_startup_if_safe(&loaded, vec![repaired.clone()], replica_state, &mut write_state, true, true) + .await + .expect_err("the same store instance must not write after observing unreadable replicas"); assert!( err.to_string() .contains("restart after all replicas are readable and consistent") diff --git a/crates/ecstore/src/store/init_format.rs b/crates/ecstore/src/store/init_format.rs index 17246cb15..22cdf518a 100644 --- a/crates/ecstore/src/store/init_format.rs +++ b/crates/ecstore/src/store/init_format.rs @@ -77,7 +77,14 @@ pub async fn connect_load_init_formats( deployment_id: Option, ) -> Result { let instance_ctx = crate::runtime::global::current_ctx(); - connect_load_init_formats_with_instance_ctx(&instance_ctx, first_disk, disks, set_count, set_drive_count, deployment_id).await + connect_load_init_formats_with_instance_ctx(&instance_ctx, first_disk, disks, set_count, set_drive_count, deployment_id) + .await + .map(|loaded| loaded.format) +} + +pub(crate) struct LoadedFormat { + pub(crate) format: FormatV3, + pub(crate) fresh_bootstrap_proven: bool, } pub(crate) async fn connect_load_init_formats_with_instance_ctx( @@ -87,17 +94,18 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx( set_count: usize, set_drive_count: usize, deployment_id: Option, -) -> Result { +) -> Result { let (formats, errs) = load_format_erasure_all(disks, false).await; check_disk_fatal_errs(&errs)?; - // Treat transient network errors (connection refused, timeout, etc.) as - // equivalent to UnformattedDisk for the bootstrap decision. During - // fresh-cluster startup a remote peer that cannot be reached is - // indistinguishable from an unformatted disk — the peer may simply not - // have started its gRPC server yet. + // Transient network errors still follow the first-node wait path so peers + // can come online, but they are never fresh-cluster authority below. let all_unformatted = errs.iter().all(is_unformatted_or_transient_network); + // Fresh-cluster authority requires a response from every configured disk. + // A transiently unreachable peer may belong to an existing cluster and + // must never be treated as proof that the topology is new. + let fresh_bootstrap_proven = should_init_erasure_disks(&errs); let formats_present = formats.iter().flatten().count(); let mut format_quorum = (formats_present > 0).then(|| select_format_erasure_in_quorum(&formats, 0)); if format_quorum.as_ref().is_none_or(Result::is_err) @@ -123,7 +131,10 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx( Ok(LegacyFormatOutcome::Migrated { format, quorum_members }) => { info!("Migrated format from MinIO config"); retain_format_quorum_members(instance_ctx, disks, &format, &quorum_members, set_drive_count).await?; - return Ok(*format); + return Ok(LoadedFormat { + format: *format, + fresh_bootstrap_proven: false, + }); } Ok(LegacyFormatOutcome::Incompatible) => { error!( @@ -138,9 +149,12 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx( Ok(LegacyFormatOutcome::None) => {} Err(e) => return Err(e), } - if all_unformatted { + if fresh_bootstrap_proven { let fm = init_format_erasure(instance_ctx, disks, set_count, set_drive_count, deployment_id).await?; - return Ok(fm); + return Ok(LoadedFormat { + format: fm, + fresh_bootstrap_proven: true, + }); } } @@ -166,7 +180,10 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx( check_format_erasure_value_for_topology(&fm, formats.len(), set_drive_count)?; retain_format_quorum_members(instance_ctx, disks, &fm, &quorum_members, set_drive_count).await?; - Ok(fm) + Ok(LoadedFormat { + format: fm, + fresh_bootstrap_proven: false, + }) } async fn retain_format_quorum_members( @@ -258,11 +275,8 @@ pub fn should_init_erasure_disks(errs: &[Option]) -> bool { count_errs(errs, &DiskError::UnformattedDisk) == errs.len() } -/// Returns `true` if the error represents a disk that is either unformatted -/// or unreachable due to a transient network failure. During fresh-cluster -/// bootstrap a remote peer that cannot be reached is indistinguishable from -/// an unformatted disk — the peer may simply not have started its gRPC -/// server yet. +/// Returns `true` for errors that stay on the first-node wait path. This is not +/// fresh-cluster proof; only [`should_init_erasure_disks`] grants that. fn is_unformatted_or_transient_network(err: &Option) -> bool { matches!(err, Some(DiskError::UnformattedDisk)) || err.as_ref().is_some_and(is_network_like_disk_error) } @@ -1294,9 +1308,14 @@ mod tests { async fn fresh_format_load_initializes_all_disks() { let (_temp_dir, mut disks) = local_disks(3).await; - let format = connect_load_init_formats(true, &mut disks, 1, 3, None) + let loaded = connect_load_init_formats_with_instance_ctx(¤t_ctx(), true, &mut disks, 1, 3, None) .await .expect("fresh disks should receive a storage format"); + assert!( + loaded.fresh_bootstrap_proven, + "every configured disk explicitly reporting unformatted should establish fresh topology proof" + ); + let format = loaded.format; let (formats, errors) = load_format_erasure_all(&disks, false).await; assert!(errors.iter().all(Option::is_none), "every disk should load its fresh format: {errors:?}"); @@ -1640,6 +1659,14 @@ mod tests { assert!(!is_unformatted_or_transient_network(&Some(DiskError::FileNotFound))); assert!(!is_unformatted_or_transient_network(&Some(DiskError::CorruptedFormat))); assert!(!is_unformatted_or_transient_network(&Some(DiskError::DiskFull))); + assert!(should_init_erasure_disks(&[ + Some(DiskError::UnformattedDisk), + Some(DiskError::UnformattedDisk), + ])); + assert!( + !should_init_erasure_disks(&[Some(DiskError::UnformattedDisk), Some(DiskError::Timeout)]), + "an unreachable peer is not fresh-topology proof" + ); } } diff --git a/crates/ecstore/src/store/rebalance.rs b/crates/ecstore/src/store/rebalance.rs index 946b4460c..4dd777e15 100644 --- a/crates/ecstore/src/store/rebalance.rs +++ b/crates/ecstore/src/store/rebalance.rs @@ -931,8 +931,12 @@ fn lifecycle_delete_all_test_failure(phase: crate::object_api::LifecycleDeleteAl mod tests { use super::*; use crate::bucket::replication::{ReplicationStatusType, VersionPurgeStatusType}; + use crate::config::com::{read_config_no_lock_preserve_empty_with_metadata, save_config}; use crate::config::storageclass::{CLASS_RRS, CLASS_STANDARD, lookup_config_for_pools_without_env}; - use crate::core::pools::{POOL_META_NAME, POOL_META_VERSION, PoolDecommissionInfo, PoolStatus}; + use crate::core::pools::{ + POOL_META_IDENTITY_NAME, POOL_META_NAME, POOL_META_VERSION, PoolDecommissionInfo, PoolStatus, + initialized_pool_meta_identity_for_test, pool_meta_v3_commit_state_for_test, + }; use crate::disk::error::DiskError; use crate::layout::endpoint::Endpoint; use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}; @@ -2375,6 +2379,135 @@ mod tests { shutdown.cancel(); } + #[tokio::test] + #[serial_test::serial] + async fn runtime_save_current_pool_meta_reaches_v3_cas_and_disarms_transaction() { + let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-runtime-cas", &[2]).await; + let bootstrapped = store + .load_runtime_pool_meta("verify fresh bootstrap V3 CAS save") + .await + .expect("fresh startup should durably publish pool metadata"); + assert_eq!(bootstrapped.version, 3); + let (bootstrap_payload, _) = read_config_no_lock_preserve_empty_with_metadata(store.pools[0].clone(), POOL_META_NAME) + .await + .expect("fresh bootstrap V3 replica should be readable"); + assert_eq!( + pool_meta_v3_commit_state_for_test(bootstrap_payload).expect("fresh bootstrap V3 envelope should decode"), + (1, true) + ); + + let saved_at = OffsetDateTime::now_utc() + TimeDuration::seconds(30); + { + let mut pool_meta = store.pool_meta.write().await; + pool_meta.pools[0].last_update = saved_at; + } + + store + .save_current_pool_meta_for_test(&[0]) + .await + .expect("runtime pool metadata save should reach V3 conditional writes"); + store + .ensure_pool_meta_side_effects_safe("runtime save publication") + .await + .expect("successful runtime publication must disarm the transaction guard"); + let persisted = store + .load_runtime_pool_meta("verify runtime V3 CAS save") + .await + .expect("runtime load should observe the committed save"); + assert_eq!(persisted.version, 3); + assert_eq!(persisted.pools[0].last_update, saved_at); + let (runtime_payload, _) = read_config_no_lock_preserve_empty_with_metadata(store.pools[0].clone(), POOL_META_NAME) + .await + .expect("runtime V3 replica should be readable"); + assert_eq!( + pool_meta_v3_commit_state_for_test(runtime_payload).expect("runtime V3 envelope should decode"), + (2, true) + ); + + shutdown.cancel(); + } + + #[tokio::test] + #[serial_test::serial] + async fn stale_clear_decommission_cannot_erase_active_or_queued_replacement() { + let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-stale-clear", &[2, 2]).await; + let replacement_time = OffsetDateTime::UNIX_EPOCH + TimeDuration::seconds(100); + let cases = [ + ( + "failed-active", + PoolDecommissionInfo { + failed: true, + ..Default::default() + }, + PoolDecommissionInfo { + start_time: Some(replacement_time), + ..Default::default() + }, + ), + ( + "canceled-queued", + PoolDecommissionInfo { + canceled: true, + ..Default::default() + }, + PoolDecommissionInfo { + queued: true, + ..Default::default() + }, + ), + ]; + + for (case, stale_terminal, replacement_info) in cases { + let mut replacement = store + .load_runtime_pool_meta("prepare stale clear replacement") + .await + .expect("the current durable pool metadata should load"); + replacement.pools[0].last_update = replacement_time; + replacement.pools[0].decommission = Some(replacement_info.clone()); + persist_reload_snapshot(&store, &replacement).await; + + let mut stale_local = replacement.clone(); + stale_local.pools[0].last_update = OffsetDateTime::UNIX_EPOCH; + stale_local.pools[0].decommission = Some(stale_terminal.clone()); + *store.pool_meta.write().await = stale_local; + + let err = store + .clear_decommission(0) + .await + .expect_err("a stale terminal clear must not erase a durable replacement"); + assert!( + err.to_string() + .contains("persisted active or queued decommission cannot be cleared"), + "unexpected {case} rejection: {err}" + ); + + let durable = store + .load_runtime_pool_meta("verify stale clear replacement") + .await + .expect("the durable replacement should remain readable"); + let durable_info = durable.pools[0] + .decommission + .as_ref() + .expect("the durable replacement must not be cleared"); + assert_eq!(durable.pools[0].last_update, replacement_time, "{case} replacement revision changed"); + assert_eq!( + durable_info.start_time, replacement_info.start_time, + "{case} replacement generation changed" + ); + assert_eq!(durable_info.queued, replacement_info.queued, "{case} replacement queue state changed"); + + let local = store.pool_meta.read().await; + let local_info = local.pools[0] + .decommission + .as_ref() + .expect("the failed clear must roll back the stale local terminal state"); + assert_eq!(local_info.failed, stale_terminal.failed, "{case} failed state was not rolled back"); + assert_eq!(local_info.canceled, stale_terminal.canceled, "{case} canceled state was not rolled back"); + } + + shutdown.cancel(); + } + #[tokio::test] #[serial_test::serial] async fn peer_pool_meta_reload_does_not_rollback_newer_local_states() { @@ -2565,7 +2698,7 @@ mod tests { #[tokio::test] #[serial_test::serial] - async fn peer_pool_meta_reload_fails_closed_when_persisted_metadata_is_missing() { + async fn peer_pool_meta_reload_latches_recovery_when_initialized_metadata_is_missing() { let (temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-missing", &[2]).await; let kept_time = OffsetDateTime::now_utc(); @@ -2612,11 +2745,15 @@ mod tests { panic!("no pool.bin found under {:?}", temp_dir.path()); } - let merged_newer = store + let err = store .reload_pool_meta() .await - .expect("reload with missing metadata should fail closed, not error"); - assert!(!merged_newer, "missing persisted metadata must not count as merged state"); + .expect_err("an initialized cluster with every pool.bin missing must require recovery"); + assert!(err.to_string().contains("initialized cluster identity exists")); + store + .ensure_pool_meta_side_effects_safe("test reload side effect") + .await + .expect_err("the recovery-required reload must latch the runtime side-effect gate"); let pool_meta = store.pool_meta.read().await; let info = pool_meta.pools[0] @@ -2628,4 +2765,60 @@ mod tests { shutdown.cancel(); } + + #[tokio::test] + #[serial_test::serial] + async fn peer_pool_meta_reload_latches_recovery_after_identity_epoch_conflict() { + let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-identity-conflict", &[2, 2]).await; + let conflicting_identity = + initialized_pool_meta_identity_for_test(store.id, 2).expect("conflicting initialized identity should encode"); + save_config(store.pools[1].clone(), POOL_META_IDENTITY_NAME, conflicting_identity) + .await + .expect("second pool identity should be replaced for the conflict scenario"); + + let err = store + .reload_pool_meta() + .await + .expect_err("divergent identity epochs must reject runtime reload"); + assert!( + err.to_string() + .contains("identity replicas disagree on cluster identity or epoch") + ); + store + .ensure_pool_meta_side_effects_safe("identity epoch conflict side effect") + .await + .expect_err("identity epoch conflict must latch the runtime side-effect gate"); + + shutdown.cancel(); + } + + #[tokio::test] + #[serial_test::serial] + async fn peer_pool_meta_reload_latches_recovery_after_future_identity_version() { + let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-future-identity", &[2, 2]).await; + let (mut future_identity, _) = + read_config_no_lock_preserve_empty_with_metadata(store.pools[1].clone(), POOL_META_IDENTITY_NAME) + .await + .expect("current identity should be readable"); + let current_version = u16::from_le_bytes([future_identity[2], future_identity[3]]); + let future_version = current_version + .checked_add(1) + .expect("identity version should have a future value"); + future_identity[2..4].copy_from_slice(&future_version.to_le_bytes()); + save_config(store.pools[1].clone(), POOL_META_IDENTITY_NAME, future_identity) + .await + .expect("second pool identity should be replaced with a future version"); + + let err = store + .reload_pool_meta() + .await + .expect_err("a future identity version must reject runtime reload"); + assert!(err.to_string().contains("pool metadata incompatible")); + store + .ensure_pool_meta_side_effects_safe("future identity version side effect") + .await + .expect_err("future identity version must latch the runtime side-effect gate"); + + shutdown.cancel(); + } } diff --git a/docs/operations/pool-metadata-recovery.md b/docs/operations/pool-metadata-recovery.md new file mode 100644 index 000000000..9b45d116d --- /dev/null +++ b/docs/operations/pool-metadata-recovery.md @@ -0,0 +1,70 @@ +# Pool metadata upgrade and recovery + +`pool.bin` is cluster state. Do not delete or copy it independently on a live +node. Version 3 adds a deployment identity, epoch, durable generation, and a +recoverable prepare/commit record on every pool. + +## Compatibility matrix + +| Reader or writer | V1 | V2 | V3 | +| --- | --- | --- | --- | +| Legacy V1 binary | read/write | reject | reject | +| V2-capable binary | read/write while mixed | read/write after the V2 fleet gate | reject | +| V3-capable binary | read/migrate | read/migrate | read/write; never downgrade | + +Leave `RUSTFS_POOL_META_V3_WRITE` or +`RUSTFS_POOL_META_V3_FLEET_CONFIRMED` disabled while any running process lacks +V3 support. Both must be `true` before an existing cluster migrates. A fresh +deployment can initialize directly at V3. Once a committed V3 generation is +observed, rollback to a V1/V2-only binary is not supported. +Repairing a missing identity on an existing V1/V2 snapshot does not cross the +V3 gate; the identity is committed as initialized while `pool.bin` stays on its +observed legacy version. + +Unknown fields are not ignored. An unsupported version or field layout is +reported as **incompatible** and is never overwritten. A truncated or invalid +payload is **corrupt** and may be repaired only from a verified committed +replica. Conflicting identities, epochs, or transactions at the same generation +are **recovery required** and need an operator-selected source. + +## Partial writes + +A V3 update first conditionally writes a pending generation containing the last +committed snapshot, then conditionally replaces it with the committed record. +During initial bootstrap, `pool.bin.identity` remains `initialized=false` and +carries a unique fresh-bootstrap nonce until that committed V3 record is +verified. Restarting from an initial prepare record finishes generation 1; it +never rewrites the record as V1 or V2. +On restart: + +- prepare-only replicas expose their previous committed snapshot; +- one committed replica makes that transaction authoritative; +- remaining pending or older replicas are repairable by the next fenced save; +- two different committed transactions at one generation stop startup. + +Do not hand-edit a pending record or select a replica only because it is in pool +zero. Preserve all copies when escalating recovery. + +## Disk replacement and metadata erasure + +1. Keep a quorum of nodes online and verify the cluster is ready. +2. Stop the lagging node before replacing or erasing its metadata drive. +3. Restore storage formats and the `pool.bin.identity` marker from the same + deployment before rejoining it. +4. Start the node and wait for it to load the verified committed generation and + repair its replicas before touching another node. + +An initialized identity with every `pool.bin` missing is recovery required. +Existing storage formats with neither identity nor `pool.bin` are also recovery +required. Format creation alone is not fresh-cluster proof. Only the elected +first topology node may create a durable `initialized=false` bootstrap identity +with a fresh-bootstrap nonce, and only after every configured disk explicitly +responds that it is unformatted. +An unreachable peer, a non-elected distributed node, or an existing format is +not sufficient proof. All-missing `pool.bin` replicas are accepted only by the +same startup that proved the fresh topology and persisted that pending identity. +When every `pool.bin` is missing, a later startup must recover even if the +pending identity survived. This prevents a wiped or lagging node from rebuilding +empty state and overwriting the cluster. Runtime reload, rebalance activation, +and rebalance worker admission all fail closed and latch the same recovery gate +until the node is restarted with readable metadata. diff --git a/docs/operations/rolling-restart.md b/docs/operations/rolling-restart.md index fc263d432..253327b97 100644 --- a/docs/operations/rolling-restart.md +++ b/docs/operations/rolling-restart.md @@ -40,6 +40,14 @@ failure pattern reported in rustfs/rustfs#4304. > binaries or rollback builds cannot read it. Unresolved decommission entries > fail closed instead of being written in the version 1 format. +> [!WARNING] +> Pool metadata version 3 remains inactive on an existing cluster unless both +> `RUSTFS_POOL_META_V3_WRITE=true` and +> `RUSTFS_POOL_META_V3_FLEET_CONFIRMED=true`. V3 adds durable generations and a +> recoverable cross-pool commit protocol. Once committed, V1/V2-only binaries +> cannot rejoin. Follow [Pool metadata upgrade and recovery](pool-metadata-recovery.md) +> for the compatibility matrix and disk-replacement order. + ## TL;DR - **Rolling restart (no downtime):** restart **one node at a time**, and wait