gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-20 19:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ecstore): harden issue3031 multipart validation path (#3106)

Browse Source 
* fix(ecstore): harden issue3031 multipart validation path

- clear stale multipart part destinations before rename fan-out
- add repeated part overwrite regression coverage
- reduce remote disk startup false-fault escalation to suspect-first
- refine remote locker diagnostics and lower scanner leader-lock log noise
- add a dedicated 4-node issue3031 docker validation script

* refactor(admin): inline console version json macro

- drop the unused serde_json::json import in admin console
- call serde_json::json! inline in version_handler
- keep the console version response behavior unchanged

* fix(remote-disk): recover suspect health on probe success

- record probe success during remote disk health checks so suspect drives recover
- use async_with_vars for the remote disk health probe test
- make the missing-listener test assert the state transition more robustly
This commit is contained in:
houseme
2026-05-28 22:26:31 +08:00
committed by GitHub
parent 8d20e89bf8
commit 088c4bda43
7 changed files with 717 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/rpc/remote_disk.rs
+83 -23
View File
@@ -175,6 +175,10 @@ impl RemoteDisk {
});
}
fn mark_suspect_or_offline(&self, reason: &'static str) -> bool {
self.health.mark_failure(&self.endpoint, reason)
}
/// Enable health monitoring after disk creation.
/// Used to defer health checks until after startup format loading completes,
/// so that remote peers have time to come online.
@@ -202,7 +206,10 @@ impl RemoteDisk {
let mut interval = time::interval(get_drive_active_check_interval());
// Perform basic connectivity check
if Self::perform_connectivity_check(&addr).await.is_err() && health.mark_offline(&endpoint, "connectivity_probe_failed") {
let initial_probe_ok = Self::perform_connectivity_check(&addr).await.is_ok();
if initial_probe_ok {
health.record_operation_success(&endpoint, "connectivity_probe_success");
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
warn!("Remote disk health check failed for {}: marking as faulty", addr);
// Start recovery monitoring
@@ -245,7 +252,9 @@ impl RemoteDisk {
}
// Perform basic connectivity check
if Self::perform_connectivity_check(&addr).await.is_err() && health.mark_offline(&endpoint, "connectivity_probe_failed") {
if Self::perform_connectivity_check(&addr).await.is_ok() {
health.record_operation_success(&endpoint, "connectivity_probe_success");
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
warn!("Remote disk health check failed for {}: marking as faulty", addr);
// Start recovery monitoring
@@ -286,7 +295,7 @@ impl RemoteDisk {
return;
}
} else {
health.mark_offline(&endpoint, "connectivity_probe_failed");
health.mark_failure(&endpoint, "connectivity_probe_failed");
}
}
}
@@ -440,7 +449,11 @@ impl RemoteDisk {
}
async fn mark_faulty_and_evict(&self, reason: &'static str) {
if self.health.mark_offline(&self.endpoint, reason) {
let previous_state = self.runtime_state();
let became_offline = self.mark_suspect_or_offline(reason);
let state = self.runtime_state();
if state != previous_state {
self.spawn_recovery_monitor_if_needed();
counter!(
"rustfs_drive_faulty_mark_total",
@@ -448,10 +461,17 @@ impl RemoteDisk {
"reason" => reason.to_string()
)
.increment(1);
warn!(
"Remote disk marked faulty after timeout: endpoint={}, addr={}, reason={}",
self.endpoint, self.addr, reason
);
if became_offline || state == RuntimeDriveHealthState::Offline {
warn!(
"Remote disk marked faulty after timeout: endpoint={}, addr={}, reason={}",
self.endpoint, self.addr, reason
);
} else {
warn!(
"Remote disk marked suspect after timeout: endpoint={}, addr={}, reason={}, state={:?}",
self.endpoint, self.addr, reason, state
);
}
counter!(
"rustfs_drive_connection_evict_total",
"endpoint" => self.endpoint.to_string(),
@@ -1951,20 +1971,39 @@ mod tests {
disk_idx: 0,
};
let disk_option = DiskOption {
cleanup: false,
health_check: true,
};
temp_env::async_with_vars(
[
(rustfs_config::ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, Some("1")),
(rustfs_config::ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS, Some("1")),
],
async {
let disk_option = DiskOption {
cleanup: false,
health_check: true,
};
let remote_disk = RemoteDisk::new(&endpoint, &disk_option, Arc::new(TcpHttpInternodeDataTransport))
.await
.unwrap();
remote_disk.enable_health_check();
let remote_disk = RemoteDisk::new(&endpoint, &disk_option, Arc::new(TcpHttpInternodeDataTransport))
.await
.unwrap();
remote_disk.enable_health_check();
// wait for health check connect timeout
tokio::time::sleep(Duration::from_secs(6)).await;
assert!(!remote_disk.is_online().await);
// Wait out the initial success-grace window so the active probe loop
// actually attempts a connectivity check. Under the new
// suspect-first semantics we only need to prove that the drive
// transitions away from a clean Online state at least once.
tokio::time::sleep(SKIP_IF_SUCCESS_BEFORE + Duration::from_secs(2)).await;
assert!(
remote_disk.offline_duration_secs().is_some(),
"missing listener should transition the drive through suspect/offline tracking"
);
assert_ne!(
remote_disk.runtime_state(),
RuntimeDriveHealthState::Online,
"missing listener should not remain in a clean Online state after probing"
);
},
)
.await;
}
#[tokio::test]
@@ -2284,7 +2323,12 @@ mod tests {
.expect_err("timeout should fail");
assert!(err.to_string().contains("timeout"));
assert!(!remote_disk.is_online().await, "remote disk should be marked faulty after timeout");
assert!(remote_disk.is_online().await, "first timeout should keep the remote disk online");
assert_eq!(
remote_disk.runtime_state(),
RuntimeDriveHealthState::Suspect,
"first timeout should move the remote disk into suspect state"
);
}
#[tokio::test]
@@ -2450,7 +2494,15 @@ mod tests {
},
std::io::ErrorKind::TimedOut
);
assert!(!remote_disk.is_online().await, "timeout-like errors should mark remote disk faulty");
assert!(
remote_disk.is_online().await,
"first timeout-like error should keep the remote disk online"
);
assert_eq!(
remote_disk.runtime_state(),
RuntimeDriveHealthState::Suspect,
"first timeout-like error should move the remote disk into suspect state"
);
assert!(
!GLOBAL_CONN_MAP.read().await.contains_key(&addr),
"timeout-like errors should evict cached connection"
@@ -2503,7 +2555,15 @@ mod tests {
},
std::io::ErrorKind::ConnectionRefused
);
assert!(!remote_disk.is_online().await, "network-like errors should mark remote disk faulty");
assert!(
remote_disk.is_online().await,
"first network-like error should keep the remote disk online"
);
assert_eq!(
remote_disk.runtime_state(),
RuntimeDriveHealthState::Suspect,
"first network-like error should move the remote disk into suspect state"
);
assert!(
!GLOBAL_CONN_MAP.read().await.contains_key(&addr),
"network-like errors should evict cached connection"