gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-25 13:36:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(logging): unify governance runtime events (#3367)

Browse Source
This commit is contained in:
houseme
2026-06-11 22:26:02 +08:00
committed by GitHub
parent b5676dcc8e
commit 82af181dcf
17 changed files with 1357 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+83 -17
View File
@@ -87,6 +87,9 @@ const EVENT_LIFECYCLE_WORKER_STATE: &str = "lifecycle_worker_state";
const EVENT_LIFECYCLE_TRANSITION_COMPENSATION: &str = "lifecycle_transition_compensation";
const EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP: &str = "lifecycle_stale_multipart_cleanup";
const EVENT_LIFECYCLE_SCAN_SKIPPED: &str = "lifecycle_scan_skipped";
const EVENT_LIFECYCLE_TIER_AUDIT: &str = "lifecycle_tier_audit";
const EVENT_LIFECYCLE_TIER_OPERATION_FAILED: &str = "lifecycle_tier_operation_failed";
const EVENT_LIFECYCLE_DELETE_FAILED: &str = "lifecycle_delete_failed";
pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
pub type TraceFn =
@@ -956,12 +959,16 @@ impl TransitionState {
pub async fn init(api: Arc<ECStore>) {
let (configured, absolute_max, n) = resolve_transition_worker_count();
info!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
configured_transition_workers = configured,
absolute_max_workers = absolute_max,
effective_transition_workers = n,
transition_queue_capacity = GLOBAL_TransitionState.transition_queue_capacity,
transition_queue_send_timeout_ms = GLOBAL_TransitionState.transition_queue_send_timeout.as_millis() as u64,
"transition worker count resolved"
state = "configured",
"Lifecycle worker state resolved"
);
//let mut transition_state = GLOBAL_TransitionState.write().await;
@@ -1032,14 +1039,24 @@ impl TransitionState {
..Default::default()
};
if let Err(err) = transition_object(api.clone(), &task.obj_info, LcAuditEvent::new(task.event.clone(), task.src.clone())).await {
global_metrics().record_scanner_transition_failed(1);
if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) && !err.to_string().contains("use of closed network connection") {
error!("Transition to {} failed for {}/{} version:{} with {}",
task.event.storage_class, task.obj_info.bucket, task.obj_info.name, task.obj_info.version_id.map(|v| v.to_string()).unwrap_or_default(), err.to_string());
}
// Send s3:ObjectTransition:Failed event
send_event(EventArgs {
if let Err(err) = transition_object(api.clone(), &task.obj_info, LcAuditEvent::new(task.event.clone(), task.src.clone())).await {
global_metrics().record_scanner_transition_failed(1);
if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) && !err.to_string().contains("use of closed network connection") {
error!(
event = EVENT_LIFECYCLE_TIER_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %task.obj_info.bucket,
object = %task.obj_info.name,
version_id = %task.obj_info.version_id.map(|v| v.to_string()).unwrap_or_default(),
tier = %task.event.storage_class,
operation = "transition_object",
error = %err,
"Lifecycle tier operation failed"
);
}
// Send s3:ObjectTransition:Failed event
send_event(EventArgs {
event_name: EventName::ObjectTransitionFailed.to_string(),
bucket_name: obj_info_for_event.bucket.clone(),
object: obj_info_for_event,
@@ -1142,13 +1159,17 @@ impl TransitionState {
GLOBAL_TransitionState.record_scanner_transition_state();
info!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
requested_transition_workers = requested,
effective_transition_workers = n,
absolute_max_workers = absolute_max,
previous_transition_workers = previous_num_workers,
current_transition_workers = current_workers,
pruned_finished_transition_workers = pruned_finished_workers,
"transition workers updated"
state = "resized",
"Lifecycle worker state updated"
);
}
}
@@ -1279,7 +1300,15 @@ async fn read_stale_multipart_candidate(
) {
Ok(file_info) => (Some(file_info.metadata), file_info.mod_time),
Err(err) => {
warn!(path = %metadata_path, error = ?err, "failed to parse multipart metadata during stale cleanup");
warn!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
path = %metadata_path,
error = ?err,
reason = "multipart_metadata_parse_failed",
"Skipped multipart metadata parse during stale cleanup"
);
(None, None)
}
};
@@ -1865,14 +1894,35 @@ pub async fn expire_transitioned_object(
)
.await;
if let Err(e) = &ret {
error!("Failed to delete remote transitioned object {}: {:?}", oi.transitioned_object.name, e);
error!(
event = EVENT_LIFECYCLE_TIER_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %oi.bucket,
object = %oi.name,
tier = %oi.transitioned_object.tier,
tier_object = %oi.transitioned_object.name,
tier_version_id = %oi.transitioned_object.version_id,
operation = "delete_remote_transitioned_object",
error = ?e,
"Lifecycle tier operation failed"
);
}
mark_delete_opts_skip_decommissioned_on_remote_success(&mut opts, ret.is_ok());
let dobj = match api.delete_object(&oi.bucket, &oi.name, opts).await {
Ok(obj) => obj,
Err(e) => {
error!("Failed to delete transitioned object {}/{}: {:?}", oi.bucket, oi.name, e);
error!(
event = EVENT_LIFECYCLE_DELETE_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %oi.bucket,
object = %oi.name,
operation = "delete_transitioned_object",
error = ?e,
"Lifecycle delete failed"
);
// Return the original object info if deletion fails
oi.clone()
}
@@ -1959,10 +2009,13 @@ pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn {
let tier = tier.clone();
Box::pin(async move {
info!(
event = EVENT_LIFECYCLE_TIER_AUDIT,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
tier = %tier,
bytes = bytes,
"ILM tier transition audit: completed transition of {} bytes to tier '{}'",
bytes, tier
state = "transition_completed",
"Lifecycle tier transition audit completed"
);
})
})
@@ -2009,13 +2062,17 @@ pub async fn get_transitioned_object_reader(
.await
.map_err(|e| {
tracing::error!(
event = EVENT_LIFECYCLE_TIER_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %bucket,
object = %object,
tier = %oi.transitioned_object.tier,
tier_object = %oi.transitioned_object.name,
tier_version_id = %oi.transitioned_object.version_id,
error = %e,
"tier GET failed"
operation = "tier_get",
"Lifecycle tier operation failed"
);
e
})?;
@@ -2332,7 +2389,16 @@ pub async fn apply_expiry_on_non_transitioned_objects(
let mut dobj = match api.delete_object(&oi.bucket, &encode_dir_object(&oi.name), opts).await {
Ok(dobj) => dobj,
Err(e) => {
error!("delete_object error: {:?}", e);
error!(
event = EVENT_LIFECYCLE_DELETE_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %oi.bucket,
object = %oi.name,
operation = "delete_object",
error = ?e,
"Lifecycle delete failed"
);
return false;
}
};
crates/ecstore/src/bucket/lifecycle/core.rs
+12 -2
View File
@@ -23,13 +23,16 @@ use std::collections::HashMap;
use std::sync::Arc;
use time::macros::offset;
use time::{self, Duration, OffsetDateTime};
use tracing::{debug, info};
use tracing::debug;
use uuid::Uuid;
use crate::store_api::ObjectInfo;
pub const TRANSITION_COMPLETE: &str = "complete";
pub const TRANSITION_PENDING: &str = "pending";
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
const EVENT_LIFECYCLE_EXPIRY_COMPUTED: &str = "lifecycle_expiry_computed";
const ERR_LIFECYCLE_NO_RULE: &str = "Lifecycle configuration should have at least one rule";
const ERR_LIFECYCLE_DUPLICATE_ID: &str = "Rule ID must be unique. Found same ID for more than one rule";
const _ERR_XML_NOT_WELL_FORMED: &str =
@@ -800,7 +803,14 @@ impl LifecycleCalculate for Transition {
pub fn expected_expiry_time(mod_time: OffsetDateTime, days: i32) -> OffsetDateTime {
if days == 0 {
info!("expected_expiry_time: days=0, returning UNIX_EPOCH for immediate expiry");
debug!(
event = EVENT_LIFECYCLE_EXPIRY_COMPUTED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
days,
result = "unix_epoch",
"Computed immediate lifecycle expiry time"
);
return OffsetDateTime::UNIX_EPOCH; // Return epoch time to ensure immediate expiry
}
let t = mod_time
crates/ecstore/src/bucket/lifecycle/evaluator.rs
+12 -1
View File
@@ -23,6 +23,10 @@ use crate::bucket::object_lock::objectlock_sys::is_object_locked_by_metadata;
use crate::bucket::replication::ReplicationConfig;
use rustfs_common::metrics::IlmAction;
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
const EVENT_LIFECYCLE_VERSION_SCAN_SKIPPED: &str = "lifecycle_version_scan_skipped";
/// Evaluator - evaluates lifecycle policy on objects for the given lifecycle
/// configuration, lock retention configuration and replication configuration.
pub struct Evaluator {
@@ -112,7 +116,14 @@ impl Evaluator {
// events after DeleteAllVersionsAction*
events[i] = event;
info!("eval_inner: skipping remaining versions' lifecycle events after DeleteAllVersionsAction*");
info!(
event = EVENT_LIFECYCLE_VERSION_SCAN_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
reason = "delete_all_versions_action",
action = ?events[i].action,
"Skipped remaining lifecycle version scan"
);
break 'top_loop;
}