mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-08 04:58:12 +00:00
fix(storage): harden ODM and scanner publication (#7187)
* fix(storage): harden ODM and scanner publication * fix(app): simplify absent SSE configuration matching * test(heal): settle PUT rename tails before disk-wipe fixtures * fix(ecstore): remove duplicate local rename implementation Keep the canonical commit module after concurrent storage changes merged. The control-write and rollback changes are already present there. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * fix(ci): satisfy new clippy lints * style(scanner): order merged test imports * fix(scanner): invalidate bucket work after namespace completion * fix(scanner): fence cached snapshots by scan execution --------- Co-authored-by: houseme <housemecn@gmail.com> Co-authored-by: heihutu <heihutu@gmail.com> Co-authored-by: zhi22915 <qiuzgang@gmail.com>
This commit is contained in:
@@ -78,6 +78,21 @@ pub(crate) struct ScannerPublicationLeaseEntry {
|
||||
pub(crate) _operation_guard: OwnedRwLockReadGuard<()>,
|
||||
}
|
||||
|
||||
pub(crate) struct NamespaceCommitGuard {
|
||||
ctx: Arc<InstanceContext>,
|
||||
counted: bool,
|
||||
}
|
||||
|
||||
impl Drop for NamespaceCommitGuard {
|
||||
fn drop(&mut self) {
|
||||
if self.counted {
|
||||
// Publish the new generation before a zero-pending publication probe.
|
||||
self.ctx.advance_namespace_commit_generation();
|
||||
self.ctx.namespace_commits.fetch_sub(1, Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Runtime state owned by a single `ECStore` instance.
|
||||
///
|
||||
/// This is intentionally minimal in the first migration slice; subsequent
|
||||
@@ -209,9 +224,13 @@ pub struct InstanceContext {
|
||||
/// Last storage-owned movement snapshot observed under the operation
|
||||
/// gate. SetDisks cache writers fail closed until ECStore refreshes it.
|
||||
scanner_publication_state: AtomicU8,
|
||||
namespace_commits: AtomicU64,
|
||||
namespace_commit_generation: AtomicU64,
|
||||
/// Resolves object-encryption material at the application boundary.
|
||||
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
|
||||
tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
|
||||
#[cfg(test)]
|
||||
suppress_tier_delete_journal_recovery: bool,
|
||||
transition_transaction_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
|
||||
tier_delete_journal_recovery_wakeup: tokio::sync::Notify,
|
||||
}
|
||||
@@ -256,8 +275,12 @@ impl InstanceContext {
|
||||
data_movement_generation_exhausted: AtomicBool::new(false),
|
||||
data_movement_generation_notify: Arc::new(Notify::new()),
|
||||
scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
|
||||
namespace_commits: AtomicU64::new(0),
|
||||
namespace_commit_generation: AtomicU64::new(0),
|
||||
object_encryption_resolver: OnceLock::new(),
|
||||
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
|
||||
#[cfg(test)]
|
||||
suppress_tier_delete_journal_recovery: false,
|
||||
transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
|
||||
tier_delete_journal_recovery_wakeup: tokio::sync::Notify::new(),
|
||||
}
|
||||
@@ -385,6 +408,36 @@ impl InstanceContext {
|
||||
&& self.scanner_publication_state.load(Ordering::Acquire) == SCANNER_PUBLICATION_STATE_ALLOWED
|
||||
}
|
||||
|
||||
pub(crate) fn begin_namespace_commit(self: &Arc<Self>) -> Arc<NamespaceCommitGuard> {
|
||||
let counted = self
|
||||
.namespace_commits
|
||||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| count.checked_add(1))
|
||||
.is_ok();
|
||||
if counted {
|
||||
self.advance_namespace_commit_generation();
|
||||
} else {
|
||||
self.namespace_commit_generation.store(u64::MAX, Ordering::Release);
|
||||
}
|
||||
Arc::new(NamespaceCommitGuard {
|
||||
ctx: Arc::clone(self),
|
||||
counted,
|
||||
})
|
||||
}
|
||||
|
||||
fn advance_namespace_commit_generation(&self) {
|
||||
let _ = self
|
||||
.namespace_commit_generation
|
||||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |generation| Some(generation.saturating_add(1)));
|
||||
}
|
||||
|
||||
pub(crate) fn namespace_commit_generation(&self) -> u64 {
|
||||
self.namespace_commit_generation.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) fn namespace_commits_pending(&self) -> bool {
|
||||
self.namespace_commits.load(Ordering::Acquire) != 0 || self.namespace_commit_generation() == u64::MAX
|
||||
}
|
||||
|
||||
pub(crate) fn set_scanner_publication_state(&self, blocked: bool) {
|
||||
self.scanner_publication_state.store(
|
||||
if blocked {
|
||||
@@ -640,12 +693,21 @@ impl InstanceContext {
|
||||
}
|
||||
|
||||
pub(crate) fn mark_tier_delete_journal_recovery_started(&self, store_id: Uuid) -> bool {
|
||||
#[cfg(test)]
|
||||
if self.suppress_tier_delete_journal_recovery {
|
||||
return false;
|
||||
}
|
||||
self.tier_delete_journal_recovery_stores
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.insert(store_id)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn suppress_tier_delete_journal_recovery_for_test(&mut self) {
|
||||
self.suppress_tier_delete_journal_recovery = true;
|
||||
}
|
||||
|
||||
pub(crate) fn mark_transition_transaction_recovery_started(&self, store_id: Uuid) -> bool {
|
||||
self.transition_transaction_recovery_stores
|
||||
.lock()
|
||||
@@ -756,6 +818,50 @@ pub fn bootstrap_ctx() -> Arc<InstanceContext> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn namespace_commit_guards_are_instance_local_and_count_until_last_owner() {
|
||||
let first = Arc::new(InstanceContext::new());
|
||||
let other = Arc::new(InstanceContext::new());
|
||||
first.set_scanner_publication_state(false);
|
||||
other.set_scanner_publication_state(false);
|
||||
assert!(first.scanner_publication_state_allowed());
|
||||
let one = first.begin_namespace_commit();
|
||||
let shared_owner = Arc::clone(&one);
|
||||
let two = first.begin_namespace_commit();
|
||||
assert!(first.namespace_commits_pending());
|
||||
assert!(first.scanner_publication_state_allowed(), "pending writes must not block scan admission");
|
||||
assert_eq!(first.namespace_commit_generation(), 2);
|
||||
assert!(!other.namespace_commits_pending());
|
||||
assert_eq!(other.namespace_commit_generation(), 0);
|
||||
assert!(other.scanner_publication_state_allowed());
|
||||
drop(one);
|
||||
assert_eq!(first.namespace_commit_generation(), 2);
|
||||
drop(shared_owner);
|
||||
assert!(first.namespace_commits_pending());
|
||||
assert_eq!(first.namespace_commit_generation(), 3);
|
||||
drop(two);
|
||||
assert!(!first.namespace_commits_pending());
|
||||
assert_eq!(first.namespace_commit_generation(), 4);
|
||||
assert!(first.scanner_publication_state_allowed());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn namespace_commit_counter_exhaustion_keeps_publication_blocked() {
|
||||
for (count, generation) in [(0, u64::MAX - 1), (u64::MAX, 0)] {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
ctx.set_scanner_publication_state(false);
|
||||
ctx.namespace_commits.store(count, Ordering::Release);
|
||||
ctx.namespace_commit_generation.store(generation, Ordering::Release);
|
||||
let guard = ctx.begin_namespace_commit();
|
||||
assert!(ctx.namespace_commits_pending());
|
||||
assert_eq!(ctx.namespace_commit_generation(), u64::MAX);
|
||||
drop(guard);
|
||||
assert!(ctx.namespace_commits_pending());
|
||||
assert_eq!(ctx.namespace_commit_generation(), u64::MAX);
|
||||
assert_eq!(ctx.namespace_commits.load(Ordering::Acquire), count);
|
||||
}
|
||||
}
|
||||
|
||||
// The SetupType inputs must derive the exact (is_erasure,
|
||||
// is_dist_erasure, is_erasure_sd) triples that the original three
|
||||
// process-global erasure bools produced via update_erasure_type().
|
||||
@@ -1073,6 +1179,12 @@ mod tests {
|
||||
assert!(!ctx_a.mark_tier_delete_journal_recovery_started(store_a));
|
||||
assert!(ctx_a.mark_tier_delete_journal_recovery_started(store_b));
|
||||
assert!(ctx_b.mark_tier_delete_journal_recovery_started(store_a));
|
||||
|
||||
let mut manual_ctx = InstanceContext::new();
|
||||
manual_ctx.suppress_tier_delete_journal_recovery_for_test();
|
||||
assert!(!manual_ctx.mark_tier_delete_journal_recovery_started(store_a));
|
||||
assert!(!manual_ctx.mark_tier_delete_journal_recovery_started(store_b));
|
||||
assert!(ctx_b.mark_tier_delete_journal_recovery_started(store_b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user