gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-25 05:26:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(logging): unify governance runtime events (#3367)

Browse Source
This commit is contained in:
houseme
2026-06-11 22:26:02 +08:00
committed by GitHub
parent b5676dcc8e
commit 82af181dcf
17 changed files with 1357 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+83 -17
View File
@@ -87,6 +87,9 @@ const EVENT_LIFECYCLE_WORKER_STATE: &str = "lifecycle_worker_state";
const EVENT_LIFECYCLE_TRANSITION_COMPENSATION: &str = "lifecycle_transition_compensation";
const EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP: &str = "lifecycle_stale_multipart_cleanup";
const EVENT_LIFECYCLE_SCAN_SKIPPED: &str = "lifecycle_scan_skipped";
const EVENT_LIFECYCLE_TIER_AUDIT: &str = "lifecycle_tier_audit";
const EVENT_LIFECYCLE_TIER_OPERATION_FAILED: &str = "lifecycle_tier_operation_failed";
const EVENT_LIFECYCLE_DELETE_FAILED: &str = "lifecycle_delete_failed";
pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
pub type TraceFn =
@@ -956,12 +959,16 @@ impl TransitionState {
pub async fn init(api: Arc<ECStore>) {
let (configured, absolute_max, n) = resolve_transition_worker_count();
info!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
configured_transition_workers = configured,
absolute_max_workers = absolute_max,
effective_transition_workers = n,
transition_queue_capacity = GLOBAL_TransitionState.transition_queue_capacity,
transition_queue_send_timeout_ms = GLOBAL_TransitionState.transition_queue_send_timeout.as_millis() as u64,
"transition worker count resolved"
state = "configured",
"Lifecycle worker state resolved"
);
//let mut transition_state = GLOBAL_TransitionState.write().await;
@@ -1032,14 +1039,24 @@ impl TransitionState {
..Default::default()
};
if let Err(err) = transition_object(api.clone(), &task.obj_info, LcAuditEvent::new(task.event.clone(), task.src.clone())).await {
global_metrics().record_scanner_transition_failed(1);
if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) && !err.to_string().contains("use of closed network connection") {
error!("Transition to {} failed for {}/{} version:{} with {}",
task.event.storage_class, task.obj_info.bucket, task.obj_info.name, task.obj_info.version_id.map(|v| v.to_string()).unwrap_or_default(), err.to_string());
}
// Send s3:ObjectTransition:Failed event
send_event(EventArgs {
if let Err(err) = transition_object(api.clone(), &task.obj_info, LcAuditEvent::new(task.event.clone(), task.src.clone())).await {
global_metrics().record_scanner_transition_failed(1);
if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) && !err.to_string().contains("use of closed network connection") {
error!(
event = EVENT_LIFECYCLE_TIER_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %task.obj_info.bucket,
object = %task.obj_info.name,
version_id = %task.obj_info.version_id.map(|v| v.to_string()).unwrap_or_default(),
tier = %task.event.storage_class,
operation = "transition_object",
error = %err,
"Lifecycle tier operation failed"
);
}
// Send s3:ObjectTransition:Failed event
send_event(EventArgs {
event_name: EventName::ObjectTransitionFailed.to_string(),
bucket_name: obj_info_for_event.bucket.clone(),
object: obj_info_for_event,
@@ -1142,13 +1159,17 @@ impl TransitionState {
GLOBAL_TransitionState.record_scanner_transition_state();
info!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
requested_transition_workers = requested,
effective_transition_workers = n,
absolute_max_workers = absolute_max,
previous_transition_workers = previous_num_workers,
current_transition_workers = current_workers,
pruned_finished_transition_workers = pruned_finished_workers,
"transition workers updated"
state = "resized",
"Lifecycle worker state updated"
);
}
}
@@ -1279,7 +1300,15 @@ async fn read_stale_multipart_candidate(
) {
Ok(file_info) => (Some(file_info.metadata), file_info.mod_time),
Err(err) => {
warn!(path = %metadata_path, error = ?err, "failed to parse multipart metadata during stale cleanup");
warn!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
path = %metadata_path,
error = ?err,
reason = "multipart_metadata_parse_failed",
"Skipped multipart metadata parse during stale cleanup"
);
(None, None)
}
};
@@ -1865,14 +1894,35 @@ pub async fn expire_transitioned_object(
)
.await;
if let Err(e) = &ret {
error!("Failed to delete remote transitioned object {}: {:?}", oi.transitioned_object.name, e);
error!(
event = EVENT_LIFECYCLE_TIER_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %oi.bucket,
object = %oi.name,
tier = %oi.transitioned_object.tier,
tier_object = %oi.transitioned_object.name,
tier_version_id = %oi.transitioned_object.version_id,
operation = "delete_remote_transitioned_object",
error = ?e,
"Lifecycle tier operation failed"
);
}
mark_delete_opts_skip_decommissioned_on_remote_success(&mut opts, ret.is_ok());
let dobj = match api.delete_object(&oi.bucket, &oi.name, opts).await {
Ok(obj) => obj,
Err(e) => {
error!("Failed to delete transitioned object {}/{}: {:?}", oi.bucket, oi.name, e);
error!(
event = EVENT_LIFECYCLE_DELETE_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %oi.bucket,
object = %oi.name,
operation = "delete_transitioned_object",
error = ?e,
"Lifecycle delete failed"
);
// Return the original object info if deletion fails
oi.clone()
}
@@ -1959,10 +2009,13 @@ pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn {
let tier = tier.clone();
Box::pin(async move {
info!(
event = EVENT_LIFECYCLE_TIER_AUDIT,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
tier = %tier,
bytes = bytes,
"ILM tier transition audit: completed transition of {} bytes to tier '{}'",
bytes, tier
state = "transition_completed",
"Lifecycle tier transition audit completed"
);
})
})
@@ -2009,13 +2062,17 @@ pub async fn get_transitioned_object_reader(
.await
.map_err(|e| {
tracing::error!(
event = EVENT_LIFECYCLE_TIER_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %bucket,
object = %object,
tier = %oi.transitioned_object.tier,
tier_object = %oi.transitioned_object.name,
tier_version_id = %oi.transitioned_object.version_id,
error = %e,
"tier GET failed"
operation = "tier_get",
"Lifecycle tier operation failed"
);
e
})?;
@@ -2332,7 +2389,16 @@ pub async fn apply_expiry_on_non_transitioned_objects(
let mut dobj = match api.delete_object(&oi.bucket, &encode_dir_object(&oi.name), opts).await {
Ok(dobj) => dobj,
Err(e) => {
error!("delete_object error: {:?}", e);
error!(
event = EVENT_LIFECYCLE_DELETE_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %oi.bucket,
object = %oi.name,
operation = "delete_object",
error = ?e,
"Lifecycle delete failed"
);
return false;
}
};
crates/ecstore/src/bucket/lifecycle/core.rs
+12 -2
View File
@@ -23,13 +23,16 @@ use std::collections::HashMap;
use std::sync::Arc;
use time::macros::offset;
use time::{self, Duration, OffsetDateTime};
use tracing::{debug, info};
use tracing::debug;
use uuid::Uuid;
use crate::store_api::ObjectInfo;
pub const TRANSITION_COMPLETE: &str = "complete";
pub const TRANSITION_PENDING: &str = "pending";
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
const EVENT_LIFECYCLE_EXPIRY_COMPUTED: &str = "lifecycle_expiry_computed";
const ERR_LIFECYCLE_NO_RULE: &str = "Lifecycle configuration should have at least one rule";
const ERR_LIFECYCLE_DUPLICATE_ID: &str = "Rule ID must be unique. Found same ID for more than one rule";
const _ERR_XML_NOT_WELL_FORMED: &str =
@@ -800,7 +803,14 @@ impl LifecycleCalculate for Transition {
pub fn expected_expiry_time(mod_time: OffsetDateTime, days: i32) -> OffsetDateTime {
if days == 0 {
info!("expected_expiry_time: days=0, returning UNIX_EPOCH for immediate expiry");
debug!(
event = EVENT_LIFECYCLE_EXPIRY_COMPUTED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
days,
result = "unix_epoch",
"Computed immediate lifecycle expiry time"
);
return OffsetDateTime::UNIX_EPOCH; // Return epoch time to ensure immediate expiry
}
let t = mod_time
crates/ecstore/src/bucket/lifecycle/evaluator.rs
+12 -1
View File
@@ -23,6 +23,10 @@ use crate::bucket::object_lock::objectlock_sys::is_object_locked_by_metadata;
use crate::bucket::replication::ReplicationConfig;
use rustfs_common::metrics::IlmAction;
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
const EVENT_LIFECYCLE_VERSION_SCAN_SKIPPED: &str = "lifecycle_version_scan_skipped";
/// Evaluator - evaluates lifecycle policy on objects for the given lifecycle
/// configuration, lock retention configuration and replication configuration.
pub struct Evaluator {
@@ -112,7 +116,14 @@ impl Evaluator {
// events after DeleteAllVersionsAction*
events[i] = event;
info!("eval_inner: skipping remaining versions' lifecycle events after DeleteAllVersionsAction*");
info!(
event = EVENT_LIFECYCLE_VERSION_SCAN_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
reason = "delete_all_versions_action",
action = ?events[i].action,
"Skipped remaining lifecycle version scan"
);
break 'top_loop;
}
crates/ecstore/src/bucket/replication/replication_resyncer.rs
+282 -42
View File
@@ -96,6 +96,10 @@ const EVENT_RESYNC_OBJECT_PROCESSED: &str = "replication_resync_object_processed
const EVENT_RESYNC_RUNTIME_SKIPPED: &str = "replication_resync_runtime_skipped";
const EVENT_REPLICATION_DELETE_SKIPPED: &str = "replication_delete_skipped";
const EVENT_REPLICATION_FORCE_DELETE_SKIPPED: &str = "replication_force_delete_skipped";
const EVENT_RESYNC_WORKER_SIGNAL_FAILED: &str = "replication_resync_worker_signal_failed";
const EVENT_RESYNC_TASK_FAILED: &str = "replication_resync_task_failed";
const EVENT_RESYNC_TARGET_OPERATION_FAILED: &str = "replication_resync_target_operation_failed";
const EVENT_RESYNC_RUNTIME_CHANNEL_FAILED: &str = "replication_resync_runtime_channel_failed";
pub(crate) const REPLICATION_DIR: &str = ".replication";
pub(crate) const RESYNC_FILE_NAME: &str = "resync.bin";
@@ -463,7 +467,14 @@ impl ReplicationResyncer {
for _ in 0..RESYNC_WORKER_COUNT {
if let Err(err) = worker_tx.send(()) {
error!("Failed to send worker message: {}", err);
error!(
event = EVENT_RESYNC_WORKER_SIGNAL_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
reason = "worker_bootstrap_signal_failed",
error = %err,
"Failed to signal replication resync worker"
);
}
}
@@ -639,7 +650,15 @@ impl ReplicationResyncer {
if update {
if let Err(err) = save_resync_status(bucket, status, api.clone()).await {
error!("Failed to save resync status: {}", err);
error!(
event = EVENT_RESYNC_STATUS_UPDATE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
reason = "persist_failed",
error = %err,
"Failed to persist resync status"
);
} else {
last_update_times.insert(bucket.clone(), status.last_update.unwrap());
}
@@ -654,10 +673,28 @@ impl ReplicationResyncer {
async fn resync_bucket_mark_status<S: ObjectIO>(&self, status: ResyncStatusType, opts: ResyncOpts, storage: Arc<S>) {
if let Err(err) = self.mark_status(status, opts.clone(), storage.clone()).await {
error!("Failed to mark resync status: {}", err);
error!(
event = EVENT_RESYNC_STATUS_UPDATE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %opts.bucket,
arn = %opts.arn,
reason = "mark_status_failed",
error = %err,
"Failed to update resync status"
);
}
if let Err(err) = self.worker_tx.send(()) {
error!("Failed to send worker message: {}", err);
error!(
event = EVENT_RESYNC_WORKER_SIGNAL_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %opts.bucket,
arn = %opts.arn,
reason = "worker_release_signal_failed",
error = %err,
"Failed to signal replication resync worker"
);
}
// TODO: Metrics
}
@@ -683,7 +720,16 @@ impl ReplicationResyncer {
let cfg = match get_replication_config(&opts.bucket).await {
Ok(cfg) => cfg,
Err(err) => {
error!("Failed to get replication config: {}", err);
error!(
event = EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %opts.bucket,
arn = %opts.arn,
reason = "replication_config_lookup_failed",
error = %err,
"Failed to look up replication config during resync"
);
self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone())
.await;
return;
@@ -722,8 +768,13 @@ impl ReplicationResyncer {
if target_arns.len() != 1 {
error!(
"replication resync failed for {} - arn specified {} is missing in the replication config",
opts.bucket, opts.arn
event = EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %opts.bucket,
arn = %opts.arn,
reason = "target_arn_missing_from_replication_config",
"Replication resync target ARN missing from replication config"
);
self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone())
.await;
@@ -735,8 +786,13 @@ impl ReplicationResyncer {
.await
else {
error!(
"replication resync failed for {} - arn specified {} is missing in the bucket targets",
opts.bucket, opts.arn
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %opts.bucket,
arn = %opts.arn,
reason = "target_client_missing",
"Replication resync target client missing from bucket targets"
);
self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone())
.await;
@@ -748,7 +804,16 @@ impl ReplicationResyncer {
.mark_status(ResyncStatusType::ResyncStarted, opts.clone(), storage.clone())
.await
{
error!("Failed to mark resync status: {}", e);
error!(
event = EVENT_RESYNC_STATUS_UPDATE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %opts.bucket,
arn = %opts.arn,
reason = "mark_started_failed",
error = %e,
"Failed to update resync status"
);
}
let (tx, mut rx) = tokio::sync::mpsc::channel(100);
@@ -758,7 +823,16 @@ impl ReplicationResyncer {
.walk(cancellation_token.clone(), &opts.bucket, "", tx.clone(), WalkOptions::default())
.await
{
error!("Failed to walk bucket {}: {}", opts.bucket, err);
error!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %opts.bucket,
arn = %opts.arn,
reason = "walk_failed",
error = %err,
"Replication resync bucket walk failed"
);
self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone())
.await;
return;
@@ -890,7 +964,15 @@ impl ReplicationResyncer {
}
if let Err(err) = results_tx.send(st) {
error!("Failed to send resync status: {}", err);
error!(
event = EVENT_RESYNC_RUNTIME_CHANNEL_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket_name,
reason = "status_channel_send_failed",
error = %err,
"Failed to send resync status"
);
}
}
});
@@ -900,7 +982,16 @@ impl ReplicationResyncer {
while let Some(res) = rx.recv().await {
if let Some(err) = res.err {
error!("Failed to get object info: {}", err);
error!(
event = EVENT_RESYNC_RUNTIME_CHANNEL_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %opts.bucket,
arn = %opts.arn,
reason = "object_info_failed",
error = %err,
"Failed to receive resync object info"
);
self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone())
.await;
return;
@@ -938,7 +1029,16 @@ impl ReplicationResyncer {
let worker_idx = sip_hash(&roi.name, RESYNC_WORKER_COUNT, &DEFAULT_SIP_HASH_KEY);
if let Err(err) = worker_txs[worker_idx].send(roi).await {
error!("Failed to send object info to worker: {}", err);
error!(
event = EVENT_RESYNC_RUNTIME_CHANNEL_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %opts.bucket,
arn = %opts.arn,
reason = "worker_queue_send_failed",
error = %err,
"Failed to send resync object to worker"
);
self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone())
.await;
return;
@@ -1368,7 +1468,15 @@ pub async fn check_replicate_delete(
return ReplicateDecision::default();
}
Err(err) => {
error!("Failed to get replication config for bucket {}: {}", bucket, err);
error!(
event = EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
reason = "replication_config_lookup_failed",
error = %err,
"Failed to look up replication config for delete replication"
);
return ReplicateDecision::default();
}
};
@@ -1853,7 +1961,16 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
rinfos.targets.push(tgt_info);
}
Err(e) => {
error!("replicate_delete task failed: {}", e);
error!(
event = EVENT_RESYNC_TASK_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
object = %dobj.delete_object.object_name,
operation = "replicate_delete",
error = %e,
"Replication resync task failed"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -1967,7 +2084,17 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
});
}
Err(e) => {
error!("failed to delete object for bucket:{} arn:{} error:{}", bucket, dobj.target_arn, e);
error!(
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %dobj.target_arn,
object = %dobj.delete_object.object_name,
operation = "apply_replication_delete_state",
error = %e,
"Replication target operation failed"
);
send_event(EventArgs {
event_name,
bucket_name: bucket.clone(),
@@ -2105,8 +2232,14 @@ async fn replicate_force_delete_to_targets<S: StorageAPI>(dobj: &DeletedObjectRe
Ok(ns_lock) => ns_lock,
Err(e) => {
warn!(
"replicate force-delete: failed to get ns lock bucket:{} object:{} error:{}",
bucket, object_name, e
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
object = %object_name,
reason = "ns_lock_create_failed",
error = %e,
"Skipping replication force-delete"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
@@ -2128,8 +2261,14 @@ async fn replicate_force_delete_to_targets<S: StorageAPI>(dobj: &DeletedObjectRe
Ok(guard) => guard,
Err(e) => {
warn!(
"replicate force-delete: failed to get write lock bucket:{} object:{} error:{}",
bucket, object_name, e
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
object = %object_name,
reason = "write_lock_failed",
error = %e,
"Skipping replication force-delete"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
@@ -2189,7 +2328,16 @@ async fn replicate_force_delete_to_targets<S: StorageAPI>(dobj: &DeletedObjectRe
join_set.spawn(async move {
if BucketTargetSys::get().is_offline(&tgt_client.to_url()).await {
error!("replicate force-delete: target offline bucket:{} arn:{}", bucket, tgt_client.arn);
error!(
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
reason = "target_offline",
endpoint = %tgt_client.to_url(),
"Skipping replication force-delete"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationFailed.to_string(),
bucket_name: bucket.clone(),
@@ -2223,8 +2371,15 @@ async fn replicate_force_delete_to_targets<S: StorageAPI>(dobj: &DeletedObjectRe
.await
{
error!(
"replicate force-delete failed bucket:{} object:{} arn:{} error:{}",
bucket, object_name, tgt_client.arn, e
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
object = %object_name,
arn = %tgt_client.arn,
operation = "force_delete_remove_object",
error = %e,
"Replication target operation failed"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationFailed.to_string(),
@@ -2244,7 +2399,16 @@ async fn replicate_force_delete_to_targets<S: StorageAPI>(dobj: &DeletedObjectRe
while let Some(result) = join_set.join_next().await {
if let Err(e) = result {
error!("replicate force-delete task panicked: {}", e);
error!(
event = EVENT_RESYNC_TASK_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
object = %object_name,
operation = "force_delete",
error = %e,
"Replication resync task failed"
);
}
}
}
@@ -2369,13 +2533,17 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
}
Err(e) => {
warn!(
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = tgt_client.bucket,
object = dobj.delete_object.object_name,
version_id = ?version_id,
delete_marker = dobj.delete_object.delete_marker,
is_version_purge,
error = %e,
"replicate_delete_to_target failed"
operation = "replicate_delete_to_target",
"Replication target operation failed"
);
rinfo.error = Some(e.to_string());
if !is_version_purge {
@@ -2423,7 +2591,15 @@ pub async fn replicate_object<S: StorageAPI>(roi: ReplicateObjectInfo, storage:
return;
}
Err(err) => {
error!("Failed to get replication config for bucket {}: {}", bucket, err);
error!(
event = EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
reason = "replication_config_lookup_failed",
error = %err,
"Failed to look up replication config for object replication"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -2491,7 +2667,16 @@ pub async fn replicate_object<S: StorageAPI>(roi: ReplicateObjectInfo, storage:
rinfos.targets.push(tgt_info);
}
Err(e) => {
error!("replicate_object task failed: {}", e);
error!(
event = EVENT_RESYNC_TASK_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
object = %object,
operation = "replicate_object",
error = %e,
"Replication resync task failed"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -2749,15 +2934,30 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
Err(e2) => {
rinfo.error = Some(e2.to_string());
warn!(
"replication head_object fallback failed bucket:{} arn:{} error:{}",
bucket, tgt_client.arn, e2
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
operation = "head_object_fallback",
error = %e2,
"Replication target operation failed"
);
return rinfo;
}
}
} else {
rinfo.error = Some(e.to_string());
warn!("replication head_object failed bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e);
warn!(
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
operation = "head_object",
error = %e,
"Replication target operation failed"
);
return rinfo;
}
}
@@ -2772,8 +2972,14 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
Ok((put_opts, is_mp)) => (put_opts, is_mp),
Err(e) => {
warn!(
"failed to get put replication opts for bucket:{} arn:{} error:{}",
bucket, tgt_client.arn, e
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
operation = "build_put_options",
error = %e,
"Replication target operation failed"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
@@ -2823,8 +3029,16 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
rinfo.replication_status = ReplicationStatusType::Failed;
rinfo.error = Some(err.to_string());
warn!(
"replication put_object failed src_bucket={} dest_bucket={} object={} err={:?}",
bucket, tgt_client.bucket, object, err
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
target_bucket = %tgt_client.bucket,
arn = %tgt_client.arn,
object = %object,
operation = "put_object",
error = ?err,
"Replication target operation failed"
);
// TODO: check offline
@@ -3022,10 +3236,15 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
&& object_info.version_id.is_none()
{
warn!(
"unable to replicate {}/{} Newer version exists on target {}",
bucket,
object,
tgt_client.to_url()
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
object = %object,
arn = %tgt_client.arn,
endpoint = %tgt_client.to_url(),
reason = "newer_target_version_exists",
"Skipping replication because newer target version exists"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
@@ -3139,8 +3358,14 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
Err(e) => {
rinfo.error = Some(e.to_string());
warn!(
"failed to get put replication opts for bucket:{} arn:{} error:{}",
bucket, tgt_client.arn, e
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
operation = "build_put_options",
error = %e,
"Replication target operation failed"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
@@ -3191,6 +3416,17 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
} {
rinfo.replication_status = ReplicationStatusType::Failed;
rinfo.error = Some(err.to_string());
warn!(
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
object = %object,
operation = "put_object",
error = ?err,
"Replication target operation failed"
);
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
// TODO: check offline
@@ -3270,7 +3506,11 @@ fn wrap_with_bandwidth_monitor_with_header(
} else {
WARNED_MONITOR_UNINIT.call_once(|| {
warn!(
"Global bucket monitor uninitialized; proceeding with unthrottled replication (bandwidth limits will be ignored)"
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
reason = "bucket_monitor_uninitialized",
"Skipping replication bandwidth monitor because global bucket monitor is uninitialized"
)
});
stream