gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-30 16:59:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ecstore): fence rebalance and decommission activation (#6400)

Browse Source 
* fix(ecstore): fence rebalance and decommission activation

* fix(ecstore): fence lost activation locks

* fix(ecstore): bind rebalance workers to activation id

* fix(ecstore): close rebalance activation races

* test(ecstore): exercise lost rebalance commit fence

* fix(ecstore): repair rebalance fence test wiring

* test(ecstore): reuse rebalance metadata fixture

* fix(ecstore): satisfy rebalance activation clippy checks

* fix(ecstore): fence stale rebalance workers

* fix(ecstore): commit rebalance activation after persistence

* fix(ecstore): fence rebalance commits and unblock stop

* test(ecstore): exercise real rebalance fences

* fix(rebalance): cancel admin stop before activation wait

* fix(ecstore): fence multipart staging on rebalance lock loss

* fix(ecstore): adopt activations after durable commit

* fix(rebalance): preserve committed activation recovery

* fix(rebalance): make prepared stop terminal-safe

* fix(ecstore): repair rebalance test imports

* fix(ecstore): repair rebalance entry runtime failures

* test(ecstore): fix activation fence synchronization

* test(ecstore): scope rebalance disk trait import

* test(ecstore): observe decommission lock attempt

* fix(ecstore): align activation fence test imports

* fix(ecstore): remove duplicate activation test import

* fix(ecstore): resolve CI clippy failures

* fix: satisfy activation merge lint gates

---------

Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
Zhengchao An
2026-08-24 14:03:21 +08:00
committed by GitHub
parent 40bf9f0425
commit 73cd1b5be2
27 changed files with 4270 additions and 328 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/services/notification_sys.rs
+34 -4
View File
@@ -51,7 +51,7 @@ const TIER_CONFIG_RELOAD_RETRY_CAP: Duration = Duration::from_secs(5);
const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10);
const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 1;
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2;
/// Cached result from the last successful admin call to a peer.
struct PeerAdminCache {
@@ -210,6 +210,24 @@ pub fn cross_pool_fence_fleet_proof_matches(proof: &CrossPoolFenceFleetProofToke
fleet_capability_proof_matches(cross_pool_fence_fleet_proof_slot(), &proof.0)
}
#[cfg(test)]
pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() {
let topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY
.get()
.cloned()
.unwrap_or_else(|| "pool-activation-test-topology".to_string());
let _ = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology.clone());
let mut state = cross_pool_fence_fleet_proof_slot()
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.topology_conflict = false;
state.proof = Some(FleetCapabilityProof {
topology_fingerprint: topology,
peer_epochs: Arc::new(BTreeMap::new()),
expires_at: Instant::now() + Duration::from_secs(60 * 60),
});
}
#[cfg(any(test, feature = "test-util"))]
pub fn rotate_cross_pool_fence_fleet_proof_for_test() -> bool {
let mut state = cross_pool_fence_fleet_proof_slot()
@@ -352,7 +370,7 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
capability = "cross_pool_fence_v1",
capability = "cross_pool_fence_v2",
state = "failed_closed",
error = %err,
"notification capability probe"
@@ -1121,9 +1139,21 @@ impl NotificationSys {
}
}
match store.stop_rebalance_for_id(expected_rebalance_id).await {
let local_rebalance_id = match expected_rebalance_id {
Some(expected_id) => Some(expected_id.to_owned()),
None => store.current_rebalance_id().await,
};
match store.stop_rebalance_for_id(local_rebalance_id.as_deref()).await {
Ok(_) => {
if let Err(err) = store.save_rebalance_stats(usize::MAX, RebalSaveOpt::StoppedAt).await {
let save_result = match local_rebalance_id.as_deref() {
Some(expected_id) => {
store
.save_rebalance_stats_for_id(usize::MAX, RebalSaveOpt::StoppedAt, expected_id)
.await
}
None => Ok(()),
};
if let Err(err) = save_result {
error!(
event = EVENT_NOTIFICATION_PEER_PROPAGATION,
component = LOG_COMPONENT_ECSTORE,