gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-25 13:36:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(logging): unify governance runtime events (#3367)

Browse Source
This commit is contained in:
houseme
2026-06-11 22:26:02 +08:00
committed by GitHub
parent b5676dcc8e
commit 82af181dcf
17 changed files with 1357 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/disk/disk_store.rs
+88 -13
View File
@@ -45,6 +45,11 @@ use uuid::Uuid;
/// Disk health status constants
const DISK_HEALTH_OK: u32 = 0;
const DISK_HEALTH_FAULTY: u32 = 1;
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_DISK: &str = "disk";
const EVENT_DISK_HEALTH_CHECK_FAILED: &str = "disk_health_check_failed";
const EVENT_DISK_RECOVERY_PROBE_STATE: &str = "disk_recovery_probe_state";
const EVENT_DISK_TIMEOUT_POLICY_FALLBACK: &str = "disk_timeout_policy_fallback";
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum TimeoutHealthAction {
@@ -118,10 +123,14 @@ fn resolve_drive_timeout_profile_from_env() -> DriveTimeoutProfile {
return profile;
}
warn!(
event = EVENT_DISK_TIMEOUT_POLICY_FALLBACK,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK,
env = rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE,
value = %raw,
default = rustfs_config::DEFAULT_DRIVE_TIMEOUT_PROFILE,
"Invalid drive timeout profile; falling back to default"
reason = "invalid_timeout_profile",
"Disk timeout policy fell back to default"
);
DriveTimeoutProfile::parse(rustfs_config::DEFAULT_DRIVE_TIMEOUT_PROFILE).unwrap_or(DriveTimeoutProfile::Default)
}
@@ -211,10 +220,14 @@ fn resolve_drive_timeout_health_policy_from_env() -> TimeoutHealthPolicy {
return policy;
}
warn!(
event = EVENT_DISK_TIMEOUT_POLICY_FALLBACK,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK,
env = rustfs_config::ENV_DRIVE_TIMEOUT_HEALTH_ACTION,
value = %raw,
default = rustfs_config::DEFAULT_DRIVE_TIMEOUT_HEALTH_ACTION,
"Invalid drive timeout health action policy; falling back to default"
reason = "invalid_health_action_policy",
"Disk timeout policy fell back to default"
);
TimeoutHealthPolicy::parse(rustfs_config::DEFAULT_DRIVE_TIMEOUT_HEALTH_ACTION).unwrap_or(TimeoutHealthPolicy::MarkFailure)
}
@@ -719,7 +732,14 @@ impl LocalDiskWrapper {
&& health.mark_failure(&disk.endpoint(), "active_health_check_failed")
{
// Health check failed, disk is considered faulty
warn!("health check: failed, disk is considered faulty");
warn!(
event = EVENT_DISK_HEALTH_CHECK_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK,
endpoint = %disk.endpoint(),
reason = "faulty_disk",
"Disk health check marked disk faulty"
);
health.increment_waiting(); // Balance the increment from failed operation
@@ -756,9 +776,14 @@ impl LocalDiskWrapper {
// Verify data integrity
if read_data.len() != test_data.len() {
warn!(
"health check: test file data length mismatch: expected {} bytes, got {}",
test_data.len(),
read_data.len()
event = EVENT_DISK_HEALTH_CHECK_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK,
endpoint = %disk.endpoint(),
reason = "data_length_mismatch",
expected_bytes = test_data.len(),
actual_bytes = read_data.len(),
"Disk health check detected data length mismatch"
);
if check_faulty_only {
return Ok(());
@@ -787,7 +812,15 @@ impl LocalDiskWrapper {
Ok(result) => match result {
Ok(()) => Ok(()),
Err(e) => {
warn!("health check: failed: {:?}", e);
warn!(
event = EVENT_DISK_HEALTH_CHECK_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK,
endpoint = %disk.endpoint(),
reason = "operation_failed",
error = ?e,
"Disk health check failed"
);
if e == DiskError::FaultyDisk {
return Err(e);
@@ -798,7 +831,15 @@ impl LocalDiskWrapper {
},
Err(_) => {
// Timeout occurred
warn!("health check: timeout after {:?}", timeout_duration);
warn!(
event = EVENT_DISK_HEALTH_CHECK_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK,
endpoint = %disk.endpoint(),
reason = "timeout",
timeout_secs = timeout_duration.as_secs(),
"Disk health check timed out"
);
Err(DiskError::FaultyDisk)
}
}
@@ -835,17 +876,40 @@ impl LocalDiskWrapper {
Ok(_) => {
let state_before = health.runtime_state();
let is_online = health.mark_recovery_success(&disk.endpoint(), "recovery_probe_success");
info!("Disk {} recovery probe succeeded; state={:?}", disk.to_string(), state_before);
info!(
event = EVENT_DISK_RECOVERY_PROBE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK,
endpoint = %disk.endpoint(),
state = "probe_succeeded",
previous_state = ?state_before,
"Disk recovery probe state changed"
);
if !is_online {
continue;
}
info!("Disk {} is back online", disk.to_string());
info!(
event = EVENT_DISK_RECOVERY_PROBE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK,
endpoint = %disk.endpoint(),
state = "online",
"Disk recovery probe restored disk online"
);
health.decrement_waiting();
return;
}
Err(e) => {
health.mark_failure(&disk.endpoint(), "recovery_probe_failed");
warn!("Disk {} still faulty: {:?}", disk.to_string(), e);
warn!(
event = EVENT_DISK_RECOVERY_PROBE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK,
endpoint = %disk.endpoint(),
state = "still_faulty",
error = ?e,
"Disk recovery probe detected disk still faulty"
);
}
}
}
@@ -946,7 +1010,14 @@ impl LocalDiskWrapper {
{
// Check if disk is faulty
if self.health.is_faulty() {
warn!("local disk {} health is faulty, returning error", self.to_string());
warn!(
event = EVENT_DISK_HEALTH_CHECK_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK,
endpoint = %self.endpoint(),
reason = "disk_marked_faulty",
"Disk health check rejected operation because disk is marked faulty"
);
return Err(DiskError::FaultyDisk);
}
@@ -996,10 +1067,14 @@ impl LocalDiskWrapper {
)
.increment(1);
warn!(
event = EVENT_DISK_HEALTH_CHECK_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK,
endpoint = %self.endpoint(),
op,
timeout_ms = timeout_duration.as_millis(),
"Local disk operation timed out"
reason = "operation_timeout",
"Disk operation timed out"
);
Err(DiskError::Timeout)
}