gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-01 09:48:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(logging): normalize admin telemetry and error messages (#3430)

Browse Source
This commit is contained in:
houseme
2026-06-14 13:27:10 +08:00
committed by GitHub
parent dc82efbab4
commit e8012bd1ba
70 changed files with 4807 additions and 1445 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/pools.rs
+323 -43
View File
@@ -71,6 +71,12 @@ use time::{Duration, OffsetDateTime};
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn};
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_POOLS: &str = "pools";
const EVENT_DECOMMISSION_STATE: &str = "decommission_state";
const EVENT_DECOMMISSION_BUCKET: &str = "decommission_bucket";
const EVENT_DECOMMISSION_ENTRY: &str = "decommission_entry";
pub const POOL_META_NAME: &str = "pool.bin";
pub const POOL_META_FORMAT: u16 = 1;
pub const POOL_META_VERSION: u16 = 1;
@@ -1280,7 +1286,15 @@ impl ECStore {
take_decommission_canceler(cancelers.as_mut_slice(), idx)
};
if !cancel_decommission_canceler(canceler) {
warn!("decommission_cancel: no active canceler found for pool {}", idx);
warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "cancel_skipped",
reason = "no_active_canceler",
"Decommission cancel skipped"
);
}
if should_reload_pool_meta && let Some(notification_sys) = get_global_notification_sys() {
@@ -1338,7 +1352,15 @@ impl ECStore {
let store = store.clone();
tokio::spawn(async move {
if let Err(err) = store.do_decommission_in_routine(canceler, idx).await {
error!("decommission: routine failed for idx {}: {err}", idx);
error!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "routine_failed",
error = %err,
"Decommission routine failed"
);
}
});
}
@@ -1350,7 +1372,14 @@ impl ECStore {
pub async fn decommission(&self, rx: CancellationToken, indices: Vec<usize>) -> Result<()> {
let indices = dedup_indices(&indices);
warn!("decommission: {:?}", indices);
info!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_indices = ?indices,
state = "requested",
"Decommission requested"
);
validate_start_decommission_request(&indices, self.single_pool())?;
ensure_decommission_not_rebalancing(self.is_rebalance_conflicting_with_decommission().await)?;
@@ -1363,8 +1392,13 @@ impl ECStore {
for idx in indices {
if let Err(cancel_err) = self.decommission_cancel(idx).await {
error!(
"decommission: failed to rollback decommission state for idx {} after spawn error: {:?}",
idx, cancel_err
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "rollback_failed",
error = ?cancel_err,
"Decommission rollback failed after spawn error"
);
if rollback_err.is_none() {
rollback_err = Some(Error::other(format!("decommission rollback failed for idx {idx}: {cancel_err}")));
@@ -1390,10 +1424,28 @@ impl ECStore {
lock_retention: Option<DefaultRetention>,
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
) -> Result<()> {
warn!("decommission_entry: {} {}", &bucket, &entry.name);
debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %entry.name,
state = "started",
"Decommission entry started"
);
wk.give().await;
if entry.is_dir() {
warn!("decommission_entry: skip dir {}", &entry.name);
debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %entry.name,
state = "skipped_directory",
"Decommission entry skipped directory"
);
return Ok(());
}
@@ -1426,7 +1478,16 @@ impl ECStore {
if should_skip_decommission_delete_marker(version, remaining_versions, replication_config.is_some()) {
//
decommissioned += 1;
info!("decommission_pool: DELETE marked object with no other non-current versions will be skipped");
debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %version.name,
state = "skipped_delete_marker",
"Decommission delete marker skipped"
);
continue;
}
@@ -1447,8 +1508,16 @@ impl ECStore {
{
if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) {
warn!(
"decommission_pool: ignore delete-marker copy for {}/{} version {:?}: {:?}",
&bucket, &version.name, &version_id, &err
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %version.name,
version_id = ?version_id,
state = "ignored_delete_marker_copy",
error = ?err,
"Decommission delete marker copy ignored"
);
ignore = true;
cleanup_ignored = true;
@@ -1463,7 +1532,16 @@ impl ECStore {
if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) {
decommissioned += 1;
}
info!("decommission_pool: ignore {}", &version.name);
debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %version.name,
state = "ignored",
"Decommission entry ignored"
);
continue;
}
@@ -1483,9 +1561,17 @@ impl ECStore {
decommissioned += 1;
}
info!(
"decommission_pool: DecomCopyDeleteMarker {} {} {:?} {:?}",
&bucket, &version.name, &version_id, error
debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %version.name,
version_id = ?version_id,
result = ?error,
state = "delete_marker_copied",
"Decommission delete marker copied"
);
continue;
}
@@ -1571,8 +1657,15 @@ impl ECStore {
}
warn!(
"decommission_pool: decommission_object done {}/{} {}",
&bucket_name, &object_name, &version.name
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket_name,
object = %object_name,
version = %version.name,
state = "object_migrated",
"Decommission object migrated"
);
failure = false;
@@ -1583,7 +1676,16 @@ impl ECStore {
if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) {
decommissioned += 1;
}
info!("decommission_pool: ignore {}", &version.name);
debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %version.name,
state = "ignored",
"Decommission entry ignored"
);
continue;
}
@@ -1624,12 +1726,17 @@ impl ECStore {
resolve_decommission_entry_cleanup_delete_result(cleanup_result, bucket.as_str(), entry.name.as_str())?
} else if decommissioned != fivs.versions.len() || expired > 0 {
warn!(
"decommission_pool: source object retained for {}/{} because only {}/{} versions were decommissioned and {} expired by lifecycle",
&bucket,
&entry.name,
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %entry.name,
decommissioned,
fivs.versions.len(),
expired
total_versions = fivs.versions.len(),
expired,
state = "source_retained",
"Decommission source object retained"
);
}
@@ -1667,7 +1774,16 @@ impl ECStore {
}
}
warn!("decommission_pool: decommission_entry done {} {}", &bucket, &entry.name);
debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %entry.name,
state = "completed",
"Decommission entry completed"
);
Ok(())
}
@@ -1705,7 +1821,16 @@ impl ECStore {
for (set_idx, set) in pool.disk_set.iter().enumerate() {
wk.clone().take().await;
warn!("decommission_pool: decommission_pool {} {}", set_idx, &bi.name);
debug!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
set_index = set_idx,
bucket = %bi.name,
state = "listing_worker_started",
"Decommission listing worker started"
);
let decommission_entry: ListCallback = Arc::new({
let this = Arc::clone(self);
@@ -1753,23 +1878,69 @@ impl ECStore {
let worker = tokio::spawn(async move {
loop {
if rx_clone.is_cancelled() {
warn!("decommission_pool: cancel {}", set_id);
debug!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
set_index = set_id,
bucket = %bi.name,
state = "listing_worker_cancelled",
"Decommission listing worker cancelled"
);
break;
}
warn!("decommission_pool: list_objects_to_decommission {} {}", set_id, &bi.name);
debug!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
set_index = set_id,
bucket = %bi.name,
state = "listing_started",
"Decommission listing started"
);
match set
.list_objects_to_decommission(rx_clone.clone(), bi.clone(), decommission_entry.clone())
.await
{
Ok(_) => {
warn!("decommission_pool: list_objects_to_decommission {} done", set_id);
debug!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
set_index = set_id,
bucket = %bi.name,
state = "listing_completed",
"Decommission listing completed"
);
break;
}
Err(err) => {
error!("decommission_pool: list_objects_to_decommission {} err {:?}", set_id, &err);
error!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
set_index = set_id,
bucket = %bi.name,
state = "listing_failed",
error = ?err,
"Decommission listing failed"
);
if is_err_bucket_not_found(&err) {
warn!("decommission_pool: list_objects_to_decommission {} volume not found", set_id);
warn!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
set_index = set_id,
bucket = %bi.name,
state = "listing_bucket_missing",
"Decommission listing bucket missing"
);
break;
}
@@ -1783,7 +1954,15 @@ impl ECStore {
listing_workers.push((set_id, worker));
}
warn!("decommission_pool: decommission_pool wait {} {}", idx, &bi.name);
debug!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bi.name,
state = "waiting_for_workers",
"Decommission waiting for workers"
);
let mut listing_worker_error = None;
for (set_id, worker) in listing_workers {
@@ -1807,11 +1986,28 @@ impl ECStore {
}
if let Err(err) = decommission_cancel_signal_result(rx.is_cancelled()) {
warn!("decommission_pool: canceled after wait {} {}", idx, &bi.name);
warn!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bi.name,
state = "cancelled_after_wait",
error = %err,
"Decommission bucket cancelled after wait"
);
return Err(err);
}
warn!("decommission_pool: decommission_pool done {} {}", idx, &bi.name);
debug!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bi.name,
state = "completed",
"Decommission bucket completed"
);
Ok(())
}
@@ -1821,7 +2017,14 @@ impl ECStore {
defer!(|| async {
let mut cancelers = self.decommission_cancelers.write().await;
if take_decommission_canceler(cancelers.as_mut_slice(), idx).is_none() {
warn!("decommission: canceler already cleared for pool {}", idx);
warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "canceler_already_cleared",
"Decommission canceler already cleared"
);
}
});
@@ -1830,7 +2033,14 @@ impl ECStore {
let (final_state, canceled, cmd_line) = {
let pool_meta = self.pool_meta.read().await;
let Some(pool) = pool_meta.pools.get(idx) else {
error!("decommission: pool metadata missing for idx {}", idx);
error!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "pool_metadata_missing",
"Decommission pool metadata missing"
);
return Err(Error::other(format!(
"failed to resolve decommission final state: pool metadata missing for idx {idx}"
)));
@@ -1849,29 +2059,75 @@ impl ECStore {
};
if let Err(err) = result {
error!("decom err {:?}", &err);
error!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "background_failed",
error = ?err,
"Decommission background routine failed"
);
if is_err_operation_canceled(&err) || should_preserve_decommission_canceled_state(canceled, rx.is_cancelled()) {
warn!("decommission: canceled for pool {}, preserving canceled state", cmd_line);
warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
cmd_line = %cmd_line,
state = "cancelled_preserved",
"Decommission cancelled; preserving canceled state"
);
return Ok(());
}
resolve_decommission_terminal_mark_after_error_result(self.decommission_failed(idx).await, idx, &err)?;
warn!("decommission: decommission_failed {}", idx);
warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "marked_failed",
"Decommission marked failed"
);
return Ok(());
}
warn!("decommission: decommission_in_background complete {}", idx);
debug!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "background_complete",
"Decommission background routine completed"
);
if should_preserve_decommission_canceled_state(canceled, rx.is_cancelled()) {
warn!("decommission: canceled for pool {}, skipping terminal state overwrite", cmd_line);
warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
cmd_line = %cmd_line,
state = "terminal_state_preserved",
"Decommission terminal state preserved after cancellation"
);
return Ok(());
}
match final_state {
DecommissionFinalState::Complete => {
warn!("Decommissioning complete for pool {}, verifying for any pending objects", cmd_line);
debug!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
cmd_line = %cmd_line,
state = "verifying_completion",
"Decommission completion verification started"
);
if let Err(err) = self.check_after_decommission(idx).await {
resolve_decommission_terminal_mark_result(self.decommission_failed(idx).await, "failed", &cmd_line)?;
return Err(Error::other(format!(
@@ -1879,16 +2135,40 @@ impl ECStore {
)));
}
warn!("Decommissioning complete for pool {}, marking completed state", cmd_line);
info!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
cmd_line = %cmd_line,
state = "marking_completed",
"Decommission marking completed state"
);
resolve_decommission_terminal_mark_result(self.complete_decommission(idx).await, "completed", &cmd_line)?;
}
DecommissionFinalState::Failed => {
warn!("Decommissioning finished with failed items for pool {}, marking failed state", cmd_line);
warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
cmd_line = %cmd_line,
state = "marking_failed",
"Decommission marking failed state"
);
resolve_decommission_terminal_mark_result(self.decommission_failed(idx).await, "failed", &cmd_line)?;
}
}
warn!("Decommissioning complete for pool {}", cmd_line);
info!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
cmd_line = %cmd_line,
state = "completed",
"Decommission completed"
);
Ok(())
}