mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-20 19:42:17 +00:00
fix(heal): recover renewed disk health checks (#3366)
* fix(heal): recover renewed disk health checks * test(heal): cover replaced remote disk rebuild --------- Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com> Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
@@ -649,6 +649,11 @@ impl LocalDiskWrapper {
|
||||
self.health.reset_for_store_init_retry(&self.disk.endpoint());
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn health_check_enabled_for_test(&self) -> bool {
|
||||
self.health_check
|
||||
}
|
||||
|
||||
/// Enable health monitoring after disk creation.
|
||||
/// Used to defer health checks until after startup format loading completes.
|
||||
pub fn enable_health_check(&self) {
|
||||
|
||||
@@ -435,6 +435,14 @@ impl Disk {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn health_check_enabled_for_test(&self) -> bool {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.health_check_enabled_for_test(),
|
||||
Disk::Remote(remote_disk) => remote_disk.health_check_enabled_for_test(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_capacity_probe(&self, total: u64, used: u64, free: u64) {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.record_capacity_probe(total, used, free),
|
||||
|
||||
@@ -221,6 +221,11 @@ impl RemoteDisk {
|
||||
self.health.reset_for_store_init_retry(&self.endpoint);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn health_check_enabled_for_test(&self) -> bool {
|
||||
self.health_check
|
||||
}
|
||||
|
||||
fn spawn_recovery_monitor_if_needed(&self) {
|
||||
if !self.health_check {
|
||||
return;
|
||||
|
||||
@@ -300,6 +300,7 @@ impl SetDisks {
|
||||
// Check that the endpoint matches
|
||||
|
||||
let _ = new_disk.set_disk_id(Some(fm.erasure.this)).await;
|
||||
new_disk.enable_health_check();
|
||||
|
||||
if new_disk.is_local() {
|
||||
let mut global_local_disk_map = GLOBAL_LOCAL_DISK_MAP.write().await;
|
||||
@@ -337,7 +338,14 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
pub(super) async fn connect_endpoint(ep: &Endpoint) -> disk::error::Result<(DiskStore, FormatV3)> {
|
||||
let disk = new_disk(ep, &DiskOption::default()).await?;
|
||||
let disk = new_disk(
|
||||
ep,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: true,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
let fm = load_format_erasure(&disk, false).await?;
|
||||
|
||||
@@ -604,4 +612,43 @@ mod tests {
|
||||
|
||||
drop(temp_dirs);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn renew_disk_enables_health_monitoring_for_recovered_disk() {
|
||||
let disk_count = 4;
|
||||
let format = FormatV3::new(1, disk_count);
|
||||
|
||||
let mut temp_dirs = Vec::with_capacity(disk_count);
|
||||
let mut endpoints = Vec::with_capacity(disk_count);
|
||||
|
||||
for disk_idx in 0..disk_count {
|
||||
let (temp_dir, endpoint, _) = make_formatted_local_disk(disk_idx, &format).await;
|
||||
temp_dirs.push(temp_dir);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
|
||||
let set_disks = SetDisks::new(
|
||||
"test-owner".to_string(),
|
||||
Arc::new(RwLock::new(vec![None; disk_count])),
|
||||
disk_count,
|
||||
disk_count / 2,
|
||||
0,
|
||||
0,
|
||||
endpoints.clone(),
|
||||
format,
|
||||
Vec::new(),
|
||||
)
|
||||
.await;
|
||||
|
||||
set_disks.renew_disk(&endpoints[0]).await;
|
||||
|
||||
let disks = set_disks.get_disks_internal().await;
|
||||
let renewed_disk = disks[0].as_ref().expect("renew_disk should attach the recovered disk");
|
||||
assert!(
|
||||
renewed_disk.health_check_enabled_for_test(),
|
||||
"renewed disks must keep health monitoring enabled so later faulty marks can recover"
|
||||
);
|
||||
|
||||
drop(temp_dirs);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user