gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-30 00:47:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(logging): normalize admin telemetry and error messages (#3430)

Browse Source
This commit is contained in:
houseme
2026-06-14 13:27:10 +08:00
committed by GitHub
parent dc82efbab4
commit e8012bd1ba
70 changed files with 4807 additions and 1445 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/heal/src/heal/channel.rs
+17 -17
View File
@@ -69,7 +69,7 @@ impl HealChannelProcessor {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_CHANNEL,
state = "started",
"Heal channel processor state updated"
"Heal channel started"
);
loop {
@@ -85,7 +85,7 @@ impl HealChannelProcessor {
subsystem = LOG_SUBSYSTEM_CHANNEL,
state = "process_failed",
error = %e,
"Heal channel request processing failed"
"Heal channel processing failed"
);
}
}
@@ -96,7 +96,7 @@ impl HealChannelProcessor {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_CHANNEL,
state = "receiver_closed",
"Heal channel processor state updated"
"Heal channel receiver closed"
);
break;
}
@@ -105,7 +105,7 @@ impl HealChannelProcessor {
response = self.response_receiver.recv() => {
if let Some(response) = response {
// Handle response if needed
info!(
debug!(
target: "rustfs::heal::channel",
event = EVENT_HEAL_CHANNEL_RESPONSE,
component = LOG_COMPONENT_HEAL,
@@ -113,7 +113,7 @@ impl HealChannelProcessor {
request_id = %response.request_id,
success = response.success,
state = "received_local",
"Heal channel response observed"
"Heal response observed"
);
}
}
@@ -126,7 +126,7 @@ impl HealChannelProcessor {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_CHANNEL,
state = "stopped",
"Heal channel processor state updated"
"Heal channel stopped"
);
Ok(())
}
@@ -154,7 +154,7 @@ impl HealChannelProcessor {
request: HealChannelRequest,
response_tx: oneshot::Sender<std::result::Result<HealAdmissionResult, String>>,
) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::channel",
event = EVENT_HEAL_CHANNEL_REQUEST,
component = LOG_COMPONENT_HEAL,
@@ -163,7 +163,7 @@ impl HealChannelProcessor {
bucket = %request.bucket,
object_prefix = %request.object_prefix.as_deref().unwrap_or(""),
state = "start_received",
"Heal channel start request received"
"Heal start received"
);
// Convert channel request to heal request
@@ -185,7 +185,7 @@ impl HealChannelProcessor {
// Submit to heal manager
match self.heal_manager.submit_heal_request(heal_request).await {
Ok(admission) => {
info!(
debug!(
target: "rustfs::heal::channel",
event = EVENT_HEAL_CHANNEL_REQUEST,
component = LOG_COMPONENT_HEAL,
@@ -193,7 +193,7 @@ impl HealChannelProcessor {
request_id = %request.id,
admission = admission.result_label(),
state = "admission_decided",
"Heal channel admission decision completed"
"Heal admission decided"
);
let _ = response_tx.send(Ok(admission));
@@ -225,7 +225,7 @@ impl HealChannelProcessor {
request_id = %request.id,
state = "submit_failed",
error = %error_text,
"Heal channel start request failed"
"Heal start submission failed"
);
let _ = response_tx.send(Err(error_text.clone()));
@@ -251,7 +251,7 @@ impl HealChannelProcessor {
client_token: String,
response_tx: oneshot::Sender<std::result::Result<HealChannelResponse, String>>,
) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::channel",
event = EVENT_HEAL_CHANNEL_REQUEST,
component = LOG_COMPONENT_HEAL,
@@ -259,7 +259,7 @@ impl HealChannelProcessor {
request_id = %client_token,
heal_path = %heal_path,
state = "query_received",
"Heal channel query request received"
"Heal query received"
);
let (summary, detail, items) = match self.heal_manager.get_task_report_for_path(&heal_path, &client_token).await {
@@ -332,7 +332,7 @@ impl HealChannelProcessor {
client_token: String,
response_tx: oneshot::Sender<std::result::Result<HealChannelResponse, String>>,
) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::channel",
event = EVENT_HEAL_CHANNEL_REQUEST,
component = LOG_COMPONENT_HEAL,
@@ -340,7 +340,7 @@ impl HealChannelProcessor {
request_id = %client_token,
heal_path = %heal_path,
state = "cancel_received",
"Heal channel cancel request received"
"Heal cancel received"
);
let request_id = if client_token.is_empty() {
@@ -451,7 +451,7 @@ impl HealChannelProcessor {
request_id = %response.request_id,
state = "enqueue_local_failed",
error = %e,
"Heal channel response enqueue failed"
"Heal response local enqueue failed"
);
}
// Always attempt to broadcast, even if local send failed
@@ -464,7 +464,7 @@ impl HealChannelProcessor {
subsystem = LOG_SUBSYSTEM_CHANNEL,
state = "broadcast_failed",
error = %e,
"Heal channel response broadcast failed"
"Heal response broadcast failed"
);
}
}
crates/heal/src/heal/erasure_healer.rs
+27 -27
View File
@@ -27,7 +27,7 @@ use std::sync::{
atomic::{AtomicUsize, Ordering},
};
use tokio::sync::{RwLock, Semaphore};
use tracing::{error, info, warn};
use tracing::{debug, error, info, warn};
const LOG_COMPONENT_HEAL: &str = "heal";
const LOG_SUBSYSTEM_ERASURE_HEALER: &str = "erasure_healer";
@@ -99,7 +99,7 @@ impl ErasureSetHealer {
/// execute erasure set heal with resume
#[tracing::instrument(skip(self, buckets), fields(set_disk_id = %set_disk_id, bucket_count = buckets.len()))]
pub async fn heal_erasure_set(&self, buckets: &[String], set_disk_id: &str) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
@@ -162,7 +162,7 @@ impl ErasureSetHealer {
Ok(manager) => {
let state = manager.get_state().await;
if state.set_disk_id == set_disk_id && ResumeUtils::can_resume_task(&self.disk, &task_id).await {
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
@@ -170,7 +170,7 @@ impl ErasureSetHealer {
task_id,
set_disk_id,
state = "resume_found",
"Erasure set resume state selected"
"Erasure set resume selected"
);
return Ok(task_id);
}
@@ -193,7 +193,7 @@ impl ErasureSetHealer {
// create new task id
let task_id = format!("{}_{}", set_disk_id, ResumeUtils::generate_task_id());
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
@@ -215,7 +215,7 @@ impl ErasureSetHealer {
) -> Result<(ResumeManager, CheckpointManager)> {
// check if resume state exists
if ResumeManager::has_resume_state(&self.disk, task_id).await {
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
@@ -235,7 +235,7 @@ impl ErasureSetHealer {
Ok((resume_manager, checkpoint_manager))
} else {
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
@@ -243,7 +243,7 @@ impl ErasureSetHealer {
task_id,
set_disk_id,
state = "creating_new",
"Erasure set resume state creating"
"Erasure set resume created"
);
let resume_manager = ResumeManager::new(
@@ -273,7 +273,7 @@ impl ErasureSetHealer {
let state = resume_manager.get_state().await;
let checkpoint = checkpoint_manager.get_checkpoint().await;
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
@@ -282,7 +282,7 @@ impl ErasureSetHealer {
current_bucket_index = checkpoint.current_bucket_index,
current_object_index = checkpoint.current_object_index,
state = "resuming",
"Erasure set heal resumed from checkpoint"
"Erasure set resumed"
);
// 2. initialize progress
@@ -349,7 +349,7 @@ impl ErasureSetHealer {
match bucket_result {
Ok(_) => {
resume_manager.complete_bucket(bucket).await?;
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_BUCKET_STATE,
component = LOG_COMPONENT_HEAL,
@@ -357,7 +357,7 @@ impl ErasureSetHealer {
set_disk_id,
bucket,
state = "completed",
"Erasure set bucket heal completed"
"Erasure set bucket completed"
);
}
Err(e) => {
@@ -383,14 +383,14 @@ impl ErasureSetHealer {
// 5. mark task completed
resume_manager.mark_completed().await?;
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
set_disk_id,
state = "completed",
"Erasure set heal completed"
"Erasure set completed"
);
Ok(())
}
@@ -411,7 +411,7 @@ impl ErasureSetHealer {
resume_manager: &ResumeManager,
checkpoint_manager: &CheckpointManager,
) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_BUCKET_STATE,
component = LOG_COMPONENT_HEAL,
@@ -421,7 +421,7 @@ impl ErasureSetHealer {
bucket_index,
current_object_index = *current_object_index,
state = "started",
"Erasure set bucket heal started"
"Erasure set bucket started"
);
// 1. get bucket info
@@ -584,7 +584,7 @@ impl ErasureSetHealer {
Ok(true) => {
*successful_objects += 1;
checkpoint_manager.add_processed_object(object.clone()).await?;
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
component = LOG_COMPONENT_HEAL,
@@ -593,13 +593,13 @@ impl ErasureSetHealer {
bucket,
object = %object,
state = "healed",
"Erasure set object heal completed"
"Erasure set object healed"
);
}
Ok(false) => {
checkpoint_manager.add_processed_object(object.clone()).await?;
*successful_objects += 1;
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
component = LOG_COMPONENT_HEAL,
@@ -608,7 +608,7 @@ impl ErasureSetHealer {
bucket,
object = %object,
state = "missing_treated_as_ok",
"Erasure set object heal skipped because object disappeared"
"Erasure set missing object treated as ok"
);
}
Err(Error::TaskCancelled) => {
@@ -743,14 +743,14 @@ impl ErasureSetHealer {
bucket: &str,
progress: &Arc<RwLock<HealProgress>>,
) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_BUCKET_STATE,
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
bucket,
state = "started",
"Erasure set bucket heal started"
"Erasure set bucket started"
);
// 1. get bucket info
@@ -847,7 +847,7 @@ impl ErasureSetHealer {
p.set_current_object(Some(format!("completed bucket: {bucket}")));
}
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_BUCKET_STATE,
component = LOG_COMPONENT_HEAL,
@@ -857,7 +857,7 @@ impl ErasureSetHealer {
total_failed,
total_scanned,
state = "completed",
"Erasure set bucket heal completed"
"Erasure set bucket completed"
);
Ok(())
@@ -890,7 +890,7 @@ impl ErasureSetHealer {
match storage.heal_object(&bucket, &object, None, &heal_opts).await {
Ok((_result, None)) => {
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
component = LOG_COMPONENT_HEAL,
@@ -898,7 +898,7 @@ impl ErasureSetHealer {
bucket,
object = %object,
state = "healed",
"Erasure set object heal completed"
"Erasure set object healed"
);
Ok(())
}
@@ -956,7 +956,7 @@ impl ErasureSetHealer {
success_count,
total,
state = "summary",
"Erasure set heal summary recorded"
"Erasure set summary recorded"
);
if failure_count > 0 {
crates/heal/src/heal/manager.rs
+56 -36
View File
@@ -558,7 +558,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER,
state = "already_running",
"Heal manager state unchanged"
"Heal manager already running"
);
return Ok(());
}
@@ -571,7 +571,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER,
state = "starting",
"Heal manager state updated"
"Heal manager starting"
);
// start scheduler
@@ -586,7 +586,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER,
state = "running",
"Heal manager state updated"
"Heal manager started"
);
Ok(())
}
@@ -599,7 +599,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER,
state = "stopping",
"Heal manager state updated"
"Heal manager stopping"
);
// cancel all tasks
@@ -617,7 +617,7 @@ impl HealManager {
state = "task_cancel_failed",
task_id = %task.id,
error = %e,
"Heal manager failed to cancel active task"
"Heal active task cancellation failed"
);
}
}
@@ -636,7 +636,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER,
state = "stopped",
"Heal manager state updated"
"Heal manager stopped"
);
Ok(())
}
@@ -659,7 +659,7 @@ impl HealManager {
match admission {
HealAdmissionResult::Merged => {
info!(
debug!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_QUEUE_ADMISSION,
component = LOG_COMPONENT_HEAL,
@@ -667,7 +667,7 @@ impl HealManager {
request_id = %request.id,
priority = ?request.priority,
result = "merged_duplicate",
"Heal queue admission decided"
"Heal queue request merged"
);
}
HealAdmissionResult::Dropped(reason) => {
@@ -680,7 +680,7 @@ impl HealManager {
priority = ?request.priority,
reason = reason.as_str(),
result = "dropped_duplicate",
"Heal queue admission decided"
"Heal queue request dropped"
);
}
HealAdmissionResult::Accepted | HealAdmissionResult::Full => {}
@@ -707,7 +707,7 @@ impl HealManager {
queue_len,
queue_capacity,
result = "accepted_by_displacement",
"Heal queue admission decided"
"Heal queue request accepted by displacement"
);
drop(queue);
if config.event_driven_scheduler_enable {
@@ -726,7 +726,7 @@ impl HealManager {
queue_len,
queue_capacity,
result = "full_no_displacement_candidate",
"Heal queue admission decided"
"Heal queue request rejected without displacement"
);
return Ok(HealAdmissionResult::Full);
}
@@ -745,7 +745,7 @@ impl HealManager {
queue_capacity,
reason = reason.as_str(),
result = "dropped_full",
"Heal queue admission decided"
"Heal queue request dropped"
);
}
HealAdmissionResult::Full => {
@@ -759,7 +759,7 @@ impl HealManager {
queue_len,
queue_capacity,
result = "rejected_full",
"Heal queue admission decided"
"Heal queue request rejected"
);
}
HealAdmissionResult::Accepted | HealAdmissionResult::Merged => {}
@@ -779,7 +779,7 @@ impl HealManager {
queue_capacity,
queue_usage_pct = (queue_len * 100) / queue_capacity,
result = "queue_pressure_high",
"Heal queue pressure increased"
"Heal queue pressure high"
);
}
@@ -793,7 +793,7 @@ impl HealManager {
// Log queue statistics periodically (when adding high/urgent priority items)
if matches!(priority, HealPriority::High | HealPriority::Urgent) {
let stats = queue.get_priority_stats();
info!(
debug!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_QUEUE_ADMISSION,
component = LOG_COMPONENT_HEAL,
@@ -812,7 +812,7 @@ impl HealManager {
drop(queue);
info!(
debug!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_QUEUE_ADMISSION,
component = LOG_COMPONENT_HEAL,
@@ -821,7 +821,7 @@ impl HealManager {
priority = ?priority,
queue_len = queue_len + 1,
result = "accepted",
"Heal queue admission decided"
"Heal queue request accepted"
);
if config.event_driven_scheduler_enable {
self.notify.notify_one();
@@ -1119,7 +1119,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER,
state = "shutdown",
"Heal scheduler state updated"
"Heal scheduler stopped"
);
break;
}
@@ -1158,13 +1158,17 @@ impl HealManager {
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
state = "started",
interval = ?duration,
"Heal auto disk scanner state updated"
"Heal auto disk scanner started"
);
tokio::spawn(async move {
let mut interval = interval(duration);
loop {
let mut candidate_count = 0usize;
let mut skipped_duplicate_count = 0usize;
let mut skipped_invalid_count = 0usize;
let mut enqueued_count = 0usize;
tokio::select! {
_ = cancel_token.cancelled() => {
info!(
@@ -1173,7 +1177,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
state = "shutdown",
"Heal auto disk scanner state updated"
"Heal auto disk scanner stopped"
);
break;
}
@@ -1185,14 +1189,15 @@ impl HealManager {
// detect unformatted disk via get_disk_id()
match disk.get_disk_id().await {
Err(DiskError::UnformattedDisk) => {
info!(
candidate_count += 1;
debug!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_AUTO_SCAN_DISK,
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
endpoint = %disk.endpoint(),
disk_state = "unformatted",
"Heal auto disk scanner detected candidate disk"
"Heal auto-scan candidate detected"
);
endpoints.push(disk.endpoint());
}
@@ -1205,7 +1210,7 @@ impl HealManager {
endpoint = %disk.endpoint(),
disk_state = "check_failed",
error = ?e,
"Heal auto disk scanner failed to inspect disk"
"Heal auto-scan disk inspection failed"
);
}
Ok(_) => {
@@ -1222,7 +1227,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
state = "idle",
"Heal auto disk scanner found no candidate disks"
"Heal auto disk scanner idle"
);
continue;
}
@@ -1238,7 +1243,7 @@ impl HealManager {
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
state = "bucket_list_failed",
error = %e,
"Heal auto disk scanner failed to list buckets"
"Heal auto-scan bucket listing failed"
);
continue;
}
@@ -1256,8 +1261,9 @@ impl HealManager {
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
endpoint = %ep,
result = "skipped_invalid_set_disk_id",
"Heal auto disk scanner skipped enqueue"
"Heal auto-scan enqueue skipped"
);
skipped_invalid_count += 1;
continue;
};
// skip if already queued or healing
@@ -1283,7 +1289,8 @@ impl HealManager {
}
if skip {
info!(
skipped_duplicate_count += 1;
debug!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
component = LOG_COMPONENT_HEAL,
@@ -1291,7 +1298,7 @@ impl HealManager {
endpoint = %ep,
set_disk_id,
result = "skipped_duplicate",
"Heal auto disk scanner skipped enqueue"
"Heal auto-scan duplicate skipped"
);
continue;
}
@@ -1312,7 +1319,8 @@ impl HealManager {
if config.event_driven_scheduler_enable {
notify.notify_one();
}
info!(
enqueued_count += 1;
debug!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
component = LOG_COMPONENT_HEAL,
@@ -1321,10 +1329,22 @@ impl HealManager {
set_disk_id,
bucket_count = buckets.len(),
result = "enqueued",
"Heal auto disk scanner enqueued task"
"Heal auto-scan task enqueued"
);
}
}
info!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_AUTO_SCAN_STATE,
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
state = "cycle_completed",
candidate_count,
enqueued_count,
skipped_duplicate_count,
skipped_invalid_count,
"Heal auto-scan cycle completed"
);
}
}
}
@@ -1405,7 +1425,7 @@ impl HealManager {
// start heal task
tokio::spawn(async move {
info!(
debug!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_SCHEDULER_STATE,
component = LOG_COMPONENT_HEAL,
@@ -1415,12 +1435,12 @@ impl HealManager {
heal_type = %task_type_label_for_spawn,
set = %task_set_label_for_spawn,
state = "task_started",
"Heal scheduler started task"
"Heal scheduler task started"
);
let result = task.execute().await;
match result {
Ok(_) => {
info!(
debug!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_SCHEDULER_STATE,
component = LOG_COMPONENT_HEAL,
@@ -1429,7 +1449,7 @@ impl HealManager {
heal_type = %task_type_label_for_spawn,
set = %task_set_label_for_spawn,
state = "task_completed",
"Heal scheduler finished task"
"Heal scheduler task completed"
);
}
Err(e) => {
@@ -1443,7 +1463,7 @@ impl HealManager {
set = %task_set_label_for_spawn,
state = "task_failed",
error = %e,
"Heal scheduler observed task failure"
"Heal scheduler task failed"
);
}
}
@@ -1500,7 +1520,7 @@ impl HealManager {
queue_len = remaining,
active_tasks = active_heals_guard.len(),
state = "backlog_high",
"Heal queue backlog summary recorded"
"Heal queue backlog high"
);
}
}
crates/heal/src/heal/resume.rs
+5 -5
View File
@@ -20,7 +20,7 @@ use std::path::Path;
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tokio::sync::RwLock;
use tracing::{debug, info, warn};
use tracing::{debug, warn};
use uuid::Uuid;
const LOG_COMPONENT_HEAL: &str = "heal";
@@ -299,7 +299,7 @@ impl ResumeManager {
let _ = self.disk.delete(RUSTFS_META_BUCKET, path_str, Default::default()).await;
}
info!(
debug!(
target: "rustfs::heal::resume",
event = EVENT_HEAL_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
@@ -524,7 +524,7 @@ impl CheckpointManager {
let _ = self.disk.delete(RUSTFS_META_BUCKET, path_str, Default::default()).await;
}
info!(
debug!(
target: "rustfs::heal::resume",
event = EVENT_HEAL_CHECKPOINT_STATE,
component = LOG_COMPONENT_HEAL,
@@ -649,7 +649,7 @@ impl ResumeUtils {
let age_hours = (current_time - state.last_update) / 3600;
if age_hours > max_age_hours {
info!(
debug!(
target: "rustfs::heal::resume",
event = EVENT_HEAL_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
@@ -657,7 +657,7 @@ impl ResumeUtils {
task_id,
age_hours,
state = "expired_cleanup_started",
"Heal resume state cleanup started"
"Heal resume cleanup started"
);
if let Err(e) = resume_manager.cleanup().await {
warn!(
crates/heal/src/heal/storage.rs
+33 -33
View File
@@ -24,7 +24,7 @@ use rustfs_ecstore::{
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_storage_api::{BucketInfo, DiskSetSelector, StorageAdminApi};
use std::sync::Arc;
use tracing::{debug, error, info, warn};
use tracing::{debug, error, warn};
const LOG_COMPONENT_HEAL: &str = "heal";
const LOG_SUBSYSTEM_STORAGE: &str = "storage";
@@ -206,7 +206,7 @@ impl HealStorageAPI for ECStoreHealStorage {
bucket,
object,
result = "not_found",
"Heal storage request finished"
"Heal storage object metadata missing"
);
Ok(None)
} else {
@@ -333,7 +333,7 @@ impl HealStorageAPI for ECStoreHealStorage {
.await
{
Ok(_) => {
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_OBJECT_IO,
component = LOG_COMPONENT_HEAL,
@@ -342,7 +342,7 @@ impl HealStorageAPI for ECStoreHealStorage {
bucket,
object,
result = "ok",
"Heal storage request finished"
"Heal storage object write completed"
);
Ok(())
}
@@ -378,7 +378,7 @@ impl HealStorageAPI for ECStoreHealStorage {
match self.ecstore.delete_object(bucket, object, Default::default()).await {
Ok(_) => {
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_OBJECT_IO,
component = LOG_COMPONENT_HEAL,
@@ -387,7 +387,7 @@ impl HealStorageAPI for ECStoreHealStorage {
bucket,
object,
result = "ok",
"Heal storage request finished"
"Heal storage object delete completed"
);
Ok(())
}
@@ -447,7 +447,7 @@ impl HealStorageAPI for ECStoreHealStorage {
let mut stream = reader.stream;
match tokio::io::copy(&mut stream, &mut tokio::io::sink()).await {
Ok(_) => {
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_OBJECT_VERIFY,
component = LOG_COMPONENT_HEAL,
@@ -455,7 +455,7 @@ impl HealStorageAPI for ECStoreHealStorage {
bucket,
object,
state = "ok",
"Heal storage object verification finished"
"Heal storage object verified"
);
Ok(true)
}
@@ -544,7 +544,7 @@ impl HealStorageAPI for ECStoreHealStorage {
// After healing, try to read the object data
match self.get_object_data(bucket, object).await? {
Some(data) => {
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_REPAIR_OP,
component = LOG_COMPONENT_HEAL,
@@ -554,7 +554,7 @@ impl HealStorageAPI for ECStoreHealStorage {
object,
bytes = data.len(),
state = "ok",
"Heal storage repair finished"
"Heal storage EC decode rebuild completed"
);
Ok(data)
}
@@ -608,7 +608,7 @@ impl HealStorageAPI for ECStoreHealStorage {
// TODO: implement disk status check using ecstore
// For now, return Ok status
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_ADMIN_OP,
component = LOG_COMPONENT_HEAL,
@@ -617,7 +617,7 @@ impl HealStorageAPI for ECStoreHealStorage {
endpoint = ?endpoint,
result = "ok",
disk_status = "ok",
"Heal storage admin operation finished"
"Heal storage disk status resolved"
);
Ok(DiskStatus::Ok)
}
@@ -640,7 +640,7 @@ impl HealStorageAPI for ECStoreHealStorage {
if error.is_some() {
return Err(Error::other(format!("Format failed: {error:?}")));
}
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_ADMIN_OP,
component = LOG_COMPONENT_HEAL,
@@ -648,7 +648,7 @@ impl HealStorageAPI for ECStoreHealStorage {
operation = "format_disk",
endpoint = ?endpoint,
result = "ok",
"Heal storage admin operation finished"
"Heal storage disk format completed"
);
Ok(())
}
@@ -726,7 +726,7 @@ impl HealStorageAPI for ECStoreHealStorage {
match self.heal_bucket(bucket, &heal_opts).await {
Ok(_) => {
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_REPAIR_OP,
component = LOG_COMPONENT_HEAL,
@@ -734,7 +734,7 @@ impl HealStorageAPI for ECStoreHealStorage {
operation = "heal_bucket_metadata",
bucket,
result = "ok",
"Heal storage repair finished"
"Heal storage bucket metadata repaired"
);
Ok(())
}
@@ -816,7 +816,7 @@ impl HealStorageAPI for ECStoreHealStorage {
bucket,
object,
result = "not_found",
"Heal storage request finished"
"Heal storage object absence confirmed"
);
Ok(false)
} else if is_transient_object_exists_error(&e) {
@@ -923,7 +923,7 @@ impl HealStorageAPI for ECStoreHealStorage {
match self.ecstore.heal_object(bucket, object, version_id_str, opts).await {
Ok((result, ecstore_error)) => {
let error = ecstore_error.map(Error::other);
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_REPAIR_OP,
component = LOG_COMPONENT_HEAL,
@@ -935,7 +935,7 @@ impl HealStorageAPI for ECStoreHealStorage {
drives_after = result.after.drives.len(),
has_error = error.is_some(),
result = "ok",
"Heal storage repair finished"
"Heal storage object repair completed"
);
Ok((result, error))
}
@@ -974,7 +974,7 @@ impl HealStorageAPI for ECStoreHealStorage {
match self.ecstore.heal_bucket(bucket, opts).await {
Ok(result) => {
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_REPAIR_OP,
component = LOG_COMPONENT_HEAL,
@@ -983,7 +983,7 @@ impl HealStorageAPI for ECStoreHealStorage {
bucket,
drives_after = result.after.drives.len(),
result = "ok",
"Heal storage repair finished"
"Heal storage bucket repair completed"
);
Ok(result)
}
@@ -1019,7 +1019,7 @@ impl HealStorageAPI for ECStoreHealStorage {
match self.ecstore.heal_format(dry_run).await {
Ok((result, ecstore_error)) => {
let error = ecstore_error.map(Error::other);
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_REPAIR_OP,
component = LOG_COMPONENT_HEAL,
@@ -1028,7 +1028,7 @@ impl HealStorageAPI for ECStoreHealStorage {
drives_after = result.after.drives.len(),
has_error = error.is_some(),
result = "ok",
"Heal storage repair finished"
"Heal storage format repair completed"
);
Ok((result, error))
}
@@ -1103,7 +1103,7 @@ impl HealStorageAPI for ECStoreHealStorage {
}
}
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_ADMIN_OP,
component = LOG_COMPONENT_HEAL,
@@ -1113,7 +1113,7 @@ impl HealStorageAPI for ECStoreHealStorage {
prefix,
object_count = all_objects.len(),
result = "ok",
"Heal storage admin operation finished"
"Heal storage object listing completed"
);
Ok(all_objects)
}
@@ -1175,12 +1175,12 @@ impl HealStorageAPI for ECStoreHealStorage {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_STORAGE,
operation = "list_objects_for_heal_page",
bucket,
prefix,
object_count = page_count,
is_truncated = list_info.is_truncated,
state = "page_loaded",
"Heal storage admin operation finished"
bucket,
prefix,
object_count = page_count,
is_truncated = list_info.is_truncated,
state = "page_loaded",
"Heal storage object listing page loaded"
);
Ok((page_objects, list_info.next_continuation_token, list_info.is_truncated))
@@ -1210,7 +1210,7 @@ impl HealStorageAPI for ECStoreHealStorage {
// Find the first available disk
if let Some(disk_store) = disks.into_iter().flatten().next() {
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_ADMIN_OP,
component = LOG_COMPONENT_HEAL,
@@ -1219,7 +1219,7 @@ impl HealStorageAPI for ECStoreHealStorage {
set_disk_id,
result = "ok",
disk = ?disk_store,
"Heal storage admin operation finished"
"Heal storage resume disk resolved"
);
return Ok(disk_store);
}
crates/heal/src/heal/task.rs
+51 -51
View File
@@ -445,7 +445,7 @@ impl HealTask {
heal_type = self.heal_type.log_kind(),
state = "started",
queue_delay = ?queue_delay,
"Heal task state updated"
"Heal task started"
);
let result = match &self.heal_type {
@@ -484,7 +484,7 @@ impl HealTask {
task_id = %self.id,
heal_type = self.heal_type.log_kind(),
state = "completed",
"Heal task state updated"
"Heal task completed"
);
}
Err(Error::TaskCancelled) => {
@@ -498,7 +498,7 @@ impl HealTask {
task_id = %self.id,
heal_type = self.heal_type.log_kind(),
state = "cancelled",
"Heal task state updated"
"Heal task cancelled"
);
}
Err(Error::TaskTimeout) => {
@@ -512,7 +512,7 @@ impl HealTask {
task_id = %self.id,
heal_type = self.heal_type.log_kind(),
state = "timed_out",
"Heal task state updated"
"Heal task timed out"
);
}
Err(e) => {
@@ -527,7 +527,7 @@ impl HealTask {
heal_type = self.heal_type.log_kind(),
state = "failed",
error = %e,
"Heal task state updated"
"Heal task failed"
);
}
}
@@ -539,7 +539,7 @@ impl HealTask {
self.cancel_token.cancel();
let mut status = self.status.write().await;
*status = HealTaskStatus::Cancelled;
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_TASK_STATE,
component = LOG_COMPONENT_HEAL,
@@ -548,7 +548,7 @@ impl HealTask {
heal_type = self.heal_type.log_kind(),
state = "cancelled",
source = "manual",
"Heal task state updated"
"Heal task cancellation requested"
);
Ok(())
}
@@ -572,7 +572,7 @@ impl HealTask {
// specific heal implementation method
#[tracing::instrument(skip(self), fields(bucket = %bucket, object = %object, version_id = ?version_id))]
async fn heal_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -582,7 +582,7 @@ impl HealTask {
object,
version_id = ?version_id,
stage = "start",
"Heal object workflow started"
"Heal object started"
);
// update progress
@@ -625,7 +625,7 @@ impl HealTask {
"Heal target object is missing"
);
if self.options.recreate_missing {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -634,7 +634,7 @@ impl HealTask {
bucket,
object,
stage = "recreate_missing",
"Heal object recovery started"
"Heal object recreate requested"
);
return self.recreate_missing_object(bucket, object, version_id).await;
} else {
@@ -690,7 +690,7 @@ impl HealTask {
// Check if this is a "File not found" error during delete operations
let error_msg = format!("{e}");
if error_msg.contains("File not found") || error_msg.contains("not found") {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -699,7 +699,7 @@ impl HealTask {
bucket,
object,
result = "treated_as_deleted",
"Heal object finished after target disappeared during repair"
"Heal missing object treated as deleted"
);
{
let mut progress = self.progress.write().await;
@@ -723,7 +723,7 @@ impl HealTask {
// If heal failed and remove_corrupted is enabled, delete the corrupted object
if self.options.remove_corrupted {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_CLEANUP,
component = LOG_COMPONENT_HEAL,
@@ -737,7 +737,7 @@ impl HealTask {
);
if !self.options.dry_run {
self.await_with_control(self.storage.delete_object(bucket, object)).await?;
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_CLEANUP,
component = LOG_COMPONENT_HEAL,
@@ -747,7 +747,7 @@ impl HealTask {
object,
action = "delete_corrupted_object",
result = "deleted",
"Heal object cleanup completed"
"Heal corrupted object deleted"
);
}
}
@@ -775,7 +775,7 @@ impl HealTask {
"Heal object stage entered"
);
let object_size = result.object_size as u64;
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -786,7 +786,7 @@ impl HealTask {
object_size = object_size,
drives_healed = result.after.drives.len(),
result = "ok",
"Heal object completed"
"Heal object repaired"
);
{
@@ -806,7 +806,7 @@ impl HealTask {
// Check if this is a "File not found" error during delete operations
let error_msg = format!("{e}");
if error_msg.contains("File not found") || error_msg.contains("not found") {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -815,7 +815,7 @@ impl HealTask {
bucket,
object,
result = "treated_as_deleted",
"Heal object finished after target disappeared during repair"
"Heal missing object treated as deleted"
);
{
let mut progress = self.progress.write().await;
@@ -839,7 +839,7 @@ impl HealTask {
// If heal failed and remove_corrupted is enabled, delete the corrupted object
if self.options.remove_corrupted {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_CLEANUP,
component = LOG_COMPONENT_HEAL,
@@ -853,7 +853,7 @@ impl HealTask {
);
if !self.options.dry_run {
self.await_with_control(self.storage.delete_object(bucket, object)).await?;
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_CLEANUP,
component = LOG_COMPONENT_HEAL,
@@ -863,7 +863,7 @@ impl HealTask {
object,
action = "delete_corrupted_object",
result = "deleted",
"Heal object cleanup completed"
"Heal corrupted object deleted"
);
}
}
@@ -882,7 +882,7 @@ impl HealTask {
/// Recreate missing object (for EC decode scenarios)
async fn recreate_missing_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -892,7 +892,7 @@ impl HealTask {
object,
version_id = ?version_id,
stage = "recreate_missing",
"Heal object recovery started"
"Heal object recreate started"
);
// Use ecstore's heal_object with recreate option
@@ -932,7 +932,7 @@ impl HealTask {
}
let object_size = result.object_size as u64;
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -942,7 +942,7 @@ impl HealTask {
object,
object_size,
result = "recreated",
"Heal object recovery finished"
"Heal object recreated"
);
{
@@ -975,7 +975,7 @@ impl HealTask {
}
async fn heal_bucket(&self, bucket: &str) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_BUCKET_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -984,7 +984,7 @@ impl HealTask {
bucket,
stage = "start",
recursive = self.options.recursive,
"Heal bucket workflow started"
"Heal bucket started"
);
// update progress
@@ -1060,7 +1060,7 @@ impl HealTask {
match heal_result {
Ok(result) => {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_BUCKET_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -1269,7 +1269,7 @@ impl HealTask {
});
}
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_BUCKET_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -1287,7 +1287,7 @@ impl HealTask {
}
async fn heal_metadata(&self, bucket: &str, object: &str) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_METADATA_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -1296,7 +1296,7 @@ impl HealTask {
bucket,
object,
stage = "start",
"Heal metadata workflow started"
"Heal metadata started"
);
// update progress
@@ -1400,7 +1400,7 @@ impl HealTask {
});
}
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_METADATA_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -1410,7 +1410,7 @@ impl HealTask {
object,
drives_healed = result.after.drives.len(),
result = "ok",
"Heal metadata completed"
"Heal metadata repaired"
);
{
@@ -1447,7 +1447,7 @@ impl HealTask {
}
async fn heal_mrf(&self, meta_path: &str) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_MRF_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -1455,7 +1455,7 @@ impl HealTask {
task_id = %self.id,
meta_path,
stage = "start",
"Heal MRF workflow started"
"Heal MRF started"
);
// update progress
@@ -1530,7 +1530,7 @@ impl HealTask {
});
}
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_MRF_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -1541,7 +1541,7 @@ impl HealTask {
object = %object,
drives_healed = result.after.drives.len(),
result = "ok",
"Heal MRF completed"
"Heal MRF repaired"
);
{
@@ -1579,7 +1579,7 @@ impl HealTask {
}
async fn heal_ec_decode(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_EC_DECODE_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -1589,7 +1589,7 @@ impl HealTask {
object,
version_id = ?version_id,
stage = "start",
"Heal EC decode workflow started"
"Heal EC decode started"
);
// update progress
@@ -1694,7 +1694,7 @@ impl HealTask {
}
let object_size = result.object_size as u64;
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_EC_DECODE_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -1705,7 +1705,7 @@ impl HealTask {
object_size,
drives_healed = result.after.drives.len(),
result = "ok",
"Heal EC decode completed"
"Heal EC decode repaired"
);
{
@@ -1742,7 +1742,7 @@ impl HealTask {
}
async fn heal_erasure_set(&self, buckets: Vec<String>, set_disk_id: String) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_ERASURE_SET_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -1751,7 +1751,7 @@ impl HealTask {
set_disk_id,
bucket_count = buckets.len(),
stage = "start",
"Heal erasure set workflow started"
"Heal erasure set started"
);
// update progress
@@ -1762,7 +1762,7 @@ impl HealTask {
}
let buckets = if buckets.is_empty() {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_ERASURE_SET_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -1770,7 +1770,7 @@ impl HealTask {
task_id = %self.id,
set_disk_id,
stage = "list_buckets",
"Heal erasure set resolved bucket list from storage"
"Heal erasure set bucket list resolved"
);
let bucket_infos = self.await_with_control(self.storage.list_buckets()).await?;
bucket_infos.into_iter().map(|info| info.name).collect()
@@ -1814,7 +1814,7 @@ impl HealTask {
});
}
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_ERASURE_SET_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -1823,7 +1823,7 @@ impl HealTask {
set_disk_id,
drives_healed = result.after.drives.len(),
result = "format_ok",
"Heal erasure set format repair completed"
"Heal erasure set format repaired"
);
}
Err(Error::TaskCancelled) => return Err(Error::TaskCancelled),
@@ -1970,7 +1970,7 @@ impl HealTask {
match result {
Ok(_) => {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_ERASURE_SET_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -1979,7 +1979,7 @@ impl HealTask {
set_disk_id,
bucket_count = buckets.len(),
result = "ok",
"Heal erasure set completed"
"Heal erasure set repaired"
);
Ok(())
}