fix(heal): recover renewed disk health checks (#3366)

* fix(heal): recover renewed disk health checks

* test(heal): cover replaced remote disk rebuild

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
Henry Guo
2026-06-12 07:05:52 +08:00
committed by GitHub
parent c3055f9335
commit 51ef87ed19
7 changed files with 247 additions and 3 deletions
+5
View File
@@ -649,6 +649,11 @@ impl LocalDiskWrapper {
self.health.reset_for_store_init_retry(&self.disk.endpoint());
}
#[cfg(test)]
pub fn health_check_enabled_for_test(&self) -> bool {
self.health_check
}
/// Enable health monitoring after disk creation.
/// Used to defer health checks until after startup format loading completes.
pub fn enable_health_check(&self) {
+8
View File
@@ -435,6 +435,14 @@ impl Disk {
}
}
#[cfg(test)]
pub fn health_check_enabled_for_test(&self) -> bool {
match self {
Disk::Local(local_disk) => local_disk.health_check_enabled_for_test(),
Disk::Remote(remote_disk) => remote_disk.health_check_enabled_for_test(),
}
}
pub fn record_capacity_probe(&self, total: u64, used: u64, free: u64) {
match self {
Disk::Local(local_disk) => local_disk.record_capacity_probe(total, used, free),
+5
View File
@@ -221,6 +221,11 @@ impl RemoteDisk {
self.health.reset_for_store_init_retry(&self.endpoint);
}
#[cfg(test)]
pub fn health_check_enabled_for_test(&self) -> bool {
self.health_check
}
fn spawn_recovery_monitor_if_needed(&self) {
if !self.health_check {
return;
+48 -1
View File
@@ -300,6 +300,7 @@ impl SetDisks {
// Check that the endpoint matches
let _ = new_disk.set_disk_id(Some(fm.erasure.this)).await;
new_disk.enable_health_check();
if new_disk.is_local() {
let mut global_local_disk_map = GLOBAL_LOCAL_DISK_MAP.write().await;
@@ -337,7 +338,14 @@ impl SetDisks {
}
pub(super) async fn connect_endpoint(ep: &Endpoint) -> disk::error::Result<(DiskStore, FormatV3)> {
let disk = new_disk(ep, &DiskOption::default()).await?;
let disk = new_disk(
ep,
&DiskOption {
cleanup: false,
health_check: true,
},
)
.await?;
let fm = load_format_erasure(&disk, false).await?;
@@ -604,4 +612,43 @@ mod tests {
drop(temp_dirs);
}
#[tokio::test]
async fn renew_disk_enables_health_monitoring_for_recovered_disk() {
let disk_count = 4;
let format = FormatV3::new(1, disk_count);
let mut temp_dirs = Vec::with_capacity(disk_count);
let mut endpoints = Vec::with_capacity(disk_count);
for disk_idx in 0..disk_count {
let (temp_dir, endpoint, _) = make_formatted_local_disk(disk_idx, &format).await;
temp_dirs.push(temp_dir);
endpoints.push(endpoint);
}
let set_disks = SetDisks::new(
"test-owner".to_string(),
Arc::new(RwLock::new(vec![None; disk_count])),
disk_count,
disk_count / 2,
0,
0,
endpoints.clone(),
format,
Vec::new(),
)
.await;
set_disks.renew_disk(&endpoints[0]).await;
let disks = set_disks.get_disks_internal().await;
let renewed_disk = disks[0].as_ref().expect("renew_disk should attach the recovered disk");
assert!(
renewed_disk.health_check_enabled_for_test(),
"renewed disks must keep health monitoring enabled so later faulty marks can recover"
);
drop(temp_dirs);
}
}