refactor(logging): normalize admin telemetry and error messages (#3430)

This commit is contained in:
houseme
2026-06-14 13:27:10 +08:00
committed by GitHub
parent dc82efbab4
commit e8012bd1ba
70 changed files with 4807 additions and 1445 deletions
+3 -3
View File
@@ -160,7 +160,7 @@ impl AuditMetrics {
component = LOG_COMPONENT_AUDIT,
subsystem = LOG_SUBSYSTEM_OBSERVABILITY,
state = "config_reloaded",
"Audit observability state updated"
"audit observability state"
);
}
@@ -173,7 +173,7 @@ impl AuditMetrics {
component = LOG_COMPONENT_AUDIT,
subsystem = LOG_SUBSYSTEM_OBSERVABILITY,
state = "system_started",
"Audit observability state updated"
"audit observability state"
);
}
@@ -249,7 +249,7 @@ impl AuditMetrics {
component = LOG_COMPONENT_AUDIT,
subsystem = LOG_SUBSYSTEM_OBSERVABILITY,
state = "metrics_reset",
"Audit observability state updated"
"audit observability state"
);
}
+13 -13
View File
@@ -266,7 +266,7 @@ impl AuditRuntimeView {
subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target_id,
state = "enabled",
"Changed audit target state"
"audit target state"
);
Ok(())
} else {
@@ -283,7 +283,7 @@ impl AuditRuntimeView {
subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target_id,
state = "disabled",
"Changed audit target state"
"audit target state"
);
Ok(())
} else {
@@ -300,7 +300,7 @@ impl AuditRuntimeView {
subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target_id,
state = "removed",
"Changed audit target state"
"audit target state"
);
Ok(())
} else {
@@ -323,7 +323,7 @@ impl AuditRuntimeView {
subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target_id,
state = "upserted",
"Changed audit target state"
"audit target state"
);
Ok(())
}
@@ -349,7 +349,7 @@ impl AuditRuntimeFacade {
subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target.id(),
replay_key = %key,
"Delivered queued audit event"
"audit replay delivery"
);
observability::record_target_success();
}
@@ -361,7 +361,7 @@ impl AuditRuntimeFacade {
subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target.id(),
reason = "not_connected",
"Retrying queued audit event delivery"
"audit replay delivery"
);
}
rustfs_targets::TargetError::Timeout(_) => {
@@ -371,7 +371,7 @@ impl AuditRuntimeFacade {
subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target.id(),
reason = "timeout",
"Retrying queued audit event delivery"
"audit replay delivery"
);
}
_ => {}
@@ -383,7 +383,7 @@ impl AuditRuntimeFacade {
subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target.id(),
reason = %reason,
"Dropped queued audit payload"
"audit replay delivery"
);
observability::record_target_failure();
}
@@ -395,7 +395,7 @@ impl AuditRuntimeFacade {
target_id = %target.id(),
error = %error,
reason = "permanent_failure",
"Queued audit payload failed permanently"
"audit replay delivery"
);
target.record_final_failure();
observability::record_target_failure();
@@ -408,7 +408,7 @@ impl AuditRuntimeFacade {
target_id = %target.id(),
replay_key = %key,
reason = "retry_exhausted",
"Dropped queued audit payload after retry exhaustion"
"audit replay delivery"
);
target.record_final_failure();
observability::record_target_failure();
@@ -422,7 +422,7 @@ impl AuditRuntimeFacade {
replay_key = %key,
error = %error,
reason = "unreadable_entry",
"Skipped unreadable audit store entry"
"audit replay delivery"
);
}
}
@@ -436,7 +436,7 @@ impl AuditRuntimeFacade {
subsystem = LOG_SUBSYSTEM_PIPELINE,
target_id = %target_id,
replay_enabled = true,
"Audit replay stream started"
"audit replay stream"
);
} else {
debug!(
@@ -446,7 +446,7 @@ impl AuditRuntimeFacade {
target_id = %target_id,
replay_enabled = false,
reason = "no_store_configured",
"Audit replay stream skipped"
"audit replay stream"
);
}
}),
+3 -3
View File
@@ -198,7 +198,7 @@ impl AuditRegistry {
target_type = %target_type,
target_id = %target_id,
registry_key = %key,
"Created audit target registry key"
"audit target registry state"
);
key.to_string()
}
@@ -221,7 +221,7 @@ impl AuditRegistry {
target_type = %target_type,
target_id = %target_id,
state = "enabled",
"Audit target registry state changed"
"audit target registry state"
);
Ok(())
} else {
@@ -250,7 +250,7 @@ impl AuditRegistry {
target_type = %target_type,
target_id = %target_id,
state = "disabled",
"Audit target registry state changed"
"audit target registry state"
);
Ok(())
} else {
+8 -8
View File
@@ -120,7 +120,7 @@ impl AuditSystem {
subsystem = LOG_SUBSYSTEM_SYSTEM,
state = "targets_created",
target_count = targets.len(),
"Created audit targets"
"audit system state"
);
let activation = self.runtime_facade().activate_targets_with_replay(targets).await;
@@ -159,7 +159,7 @@ impl AuditSystem {
component = LOG_COMPONENT_AUDIT,
subsystem = LOG_SUBSYSTEM_SYSTEM,
state = "starting",
"Starting audit system"
"audit system state"
);
// Record system start
@@ -268,7 +268,7 @@ impl AuditSystem {
component = LOG_COMPONENT_AUDIT,
subsystem = LOG_SUBSYSTEM_SYSTEM,
state = "stopping",
"Stopping audit system"
"audit system state"
);
// Stop all stream tasks first
@@ -441,7 +441,7 @@ impl AuditSystem {
component = LOG_COMPONENT_AUDIT,
subsystem = LOG_SUBSYSTEM_SYSTEM,
state = "reloading",
"Reloading audit configuration"
"audit config reload"
);
observability::record_config_reload();
@@ -465,7 +465,7 @@ impl AuditSystem {
component = LOG_COMPONENT_AUDIT,
subsystem = LOG_SUBSYSTEM_SYSTEM,
state = "reloaded",
"Reloaded audit configuration"
"audit config reload"
);
Ok(())
}
@@ -513,7 +513,7 @@ fn info_audit_state(state: &str, reason: Option<&str>, target_count: Option<usiz
state,
reason = reason.unwrap_or_default(),
target_count = target_count.unwrap_or_default(),
"Changed audit system state"
"audit system state"
);
}
@@ -526,7 +526,7 @@ fn debug_audit_state(state: &str, reason: Option<&str>, error: Option<&str>, tar
reason = reason.unwrap_or_default(),
error = error.unwrap_or_default(),
target_count,
"Observed audit system state"
"audit system state"
);
}
@@ -537,7 +537,7 @@ fn warn_audit_state(state: &str, reason: Option<&str>) {
subsystem = LOG_SUBSYSTEM_SYSTEM,
state,
reason = reason.unwrap_or_default(),
"Audit system state transition skipped"
"audit system state"
);
}
@@ -77,7 +77,7 @@ use tokio::sync::mpsc::{Receiver, Sender};
use tokio::sync::{RwLock, mpsc};
use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn};
use tracing::{debug, error, warn};
use uuid::Uuid;
use xxhash_rust::xxh64;
@@ -762,13 +762,13 @@ impl TransitionState {
"Transition compensation backfill failed"
);
} else {
info!(
debug!(
event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %bucket,
state = "completed",
"Completed transition compensation backfill"
"Transition compensation completed"
);
}
@@ -958,7 +958,7 @@ impl TransitionState {
pub async fn init(api: Arc<ECStore>) {
let (configured, absolute_max, n) = resolve_transition_worker_count();
info!(
debug!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
@@ -968,7 +968,7 @@ impl TransitionState {
transition_queue_capacity = GLOBAL_TransitionState.transition_queue_capacity,
transition_queue_send_timeout_ms = GLOBAL_TransitionState.transition_queue_send_timeout.as_millis() as u64,
state = "configured",
"Lifecycle worker state resolved"
"Lifecycle worker configuration resolved"
);
//let mut transition_state = GLOBAL_TransitionState.write().await;
@@ -1158,7 +1158,7 @@ impl TransitionState {
GLOBAL_TransitionState.num_workers.store(current_workers, Ordering::SeqCst);
GLOBAL_TransitionState.record_scanner_transition_state();
info!(
debug!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
@@ -1169,7 +1169,7 @@ impl TransitionState {
current_transition_workers = current_workers,
pruned_finished_transition_workers = pruned_finished_workers,
state = "resized",
"Lifecycle worker state updated"
"Lifecycle worker pool resized"
);
}
}
@@ -2008,14 +2008,14 @@ pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn {
Arc::new(move || {
let tier = tier.clone();
Box::pin(async move {
info!(
debug!(
event = EVENT_LIFECYCLE_TIER_AUDIT,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
tier = %tier,
bytes = bytes,
state = "transition_completed",
"Lifecycle tier transition audit completed"
"Lifecycle tier transition recorded"
);
})
})
+87 -11
View File
@@ -29,7 +29,11 @@ use tokio::io::AsyncRead;
use tokio::spawn;
use tokio::time::timeout;
use tokio_util::sync::CancellationToken;
use tracing::{error, info, warn};
use tracing::{error, warn};
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_METACACHE: &str = "metacache";
const EVENT_METACACHE_LISTING: &str = "metacache_listing";
pub type AgreedFn = Box<dyn Fn(MetaCacheEntry) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
pub type PartialFn =
@@ -114,6 +118,9 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
return Err(DiskError::ErasureReadQuorum);
}
let log_bucket = opts.bucket.clone();
let log_path = opts.path.clone();
let mut jobs: Vec<tokio::task::JoinHandle<std::result::Result<(), DiskError>>> = Vec::new();
let mut readers = Vec::with_capacity(opts.disks.len());
let fds = opts.fallback_disks.iter().flatten().cloned().collect::<VecDeque<_>>();
@@ -180,7 +187,17 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
match disk.walk_dir(wakl_opts, &mut wr).await {
Ok(_res) => {}
Err(err) => {
info!("walk dir err {:?}", &err);
warn!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "walk_dir_failed",
error = ?err,
"Metacache walk_dir failed"
);
last_err = Some(err);
need_fallback = true;
}
@@ -205,7 +222,16 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
}
let Some(disk) = disk_op else {
warn!("list_path_raw: fallback disk is none");
warn!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "fallback_disk_missing",
"Metacache fallback disk missing"
);
let err = last_err.unwrap_or(DiskError::DiskNotFound);
record_producer_error(&producer_errs_clone, disk_idx, &err);
return Err(err);
@@ -234,7 +260,17 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
last_err = None;
}
Err(err) => {
error!("walk dir2 err {:?}", &err);
error!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "fallback_walk_dir_failed",
error = ?err,
"Metacache fallback walk_dir failed"
);
last_err = Some(err);
}
}
@@ -355,11 +391,15 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
)
.increment(1);
warn!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
drive = %endpoint,
bucket = %opts.bucket,
path = %opts.path,
timeout_ms = peek_timeout.as_millis(),
"list_path_raw reader peek timed out; excluding drive from current merge"
state = "peek_timed_out",
"Metacache reader peek timed out"
);
let (detached_rd, write_half) = tokio::io::duplex(1);
drop(write_half);
@@ -439,8 +479,14 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
});
error!(
"list_path_raw: has_err > 0 && has_err > opts.disks.len() - opts.min_disks break, err: {:?}",
&combined_err.join(", ")
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %opts.bucket,
path = %opts.path,
state = "quorum_failed",
error = %combined_err.join(", "),
"Metacache listing quorum failed"
);
return Err(DiskError::other(combined_err.join(", ")));
}
@@ -493,7 +539,16 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
});
if let Err(err) = revjob.await.map_err(std::io::Error::other)? {
error!("list_path_raw: revjob err {:?}", err);
error!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %log_bucket,
path = %log_path,
state = "merge_job_failed",
error = ?err,
"Metacache merge job failed"
);
cancel_rx.cancel();
for job in jobs {
job.abort();
@@ -521,9 +576,23 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
Ok(Ok(())) => {}
Ok(Err(err)) => {
if matches!(err, DiskError::FileNotFound | DiskError::VolumeNotFound) {
warn!("list_path_raw producer missing path {:?}", err);
warn!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
state = "producer_missing_path",
error = ?err,
"Metacache producer missing path"
);
} else {
error!("list_path_raw producer err {:?}", err);
error!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
state = "producer_failed",
error = ?err,
"Metacache producer failed"
);
}
job_errs.push(err);
}
@@ -531,7 +600,14 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
if err.is_cancelled() {
continue;
}
error!("list_path_raw join err {:?}", err);
error!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
state = "producer_join_failed",
error = ?err,
"Metacache producer join failed"
);
job_errs.push(err.into());
}
}
+28 -4
View File
@@ -472,7 +472,15 @@ fn resolve_local_disk_root(ep_path: &str) -> Result<PathBuf> {
impl LocalDisk {
pub async fn new(ep: &Endpoint, cleanup: bool) -> Result<Self> {
debug!("Creating local disk");
debug!(
event = EVENT_DISK_LOCAL_STARTUP_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
endpoint = %ep,
state = "create_started",
cleanup,
"Local disk creation started"
);
let endpoint_path = ep.get_file_path();
let root = resolve_local_disk_root(&endpoint_path).inspect_err(|err| {
log_startup_disk_error("resolve_local_disk_root", Path::new(&endpoint_path), err);
@@ -501,13 +509,21 @@ impl LocalDisk {
root = ?root,
state = "failed",
error = ?err,
"Disk local startup cleanup failed"
"Local disk startup cleanup failed"
);
}
// Use optimized path resolution instead of absolutize_virtually
let format_path = root.join(RUSTFS_META_BUCKET).join(super::FORMAT_CONFIG_FILE);
debug!("format_path: {:?}", format_path);
debug!(
event = EVENT_DISK_LOCAL_STARTUP_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
root = ?root,
format_path = ?format_path,
state = "format_path_resolved",
"Local disk format path resolved"
);
let (format_data, format_meta) = read_file_exists(&format_path).await.inspect_err(|err| {
log_startup_disk_error("read_format_json", &format_path, err);
})?;
@@ -639,7 +655,15 @@ impl LocalDisk {
let root = disk.root.clone();
tokio::spawn(Self::cleanup_deleted_objects_loop(root, exit_rx));
debug!("LocalDisk created: {:?}", disk);
debug!(
event = EVENT_DISK_LOCAL_STARTUP_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
endpoint = %disk.endpoint,
root = ?disk.root,
state = "created",
"Local disk created"
);
Ok(disk)
}
+323 -43
View File
@@ -71,6 +71,12 @@ use time::{Duration, OffsetDateTime};
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn};
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_POOLS: &str = "pools";
const EVENT_DECOMMISSION_STATE: &str = "decommission_state";
const EVENT_DECOMMISSION_BUCKET: &str = "decommission_bucket";
const EVENT_DECOMMISSION_ENTRY: &str = "decommission_entry";
pub const POOL_META_NAME: &str = "pool.bin";
pub const POOL_META_FORMAT: u16 = 1;
pub const POOL_META_VERSION: u16 = 1;
@@ -1280,7 +1286,15 @@ impl ECStore {
take_decommission_canceler(cancelers.as_mut_slice(), idx)
};
if !cancel_decommission_canceler(canceler) {
warn!("decommission_cancel: no active canceler found for pool {}", idx);
warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "cancel_skipped",
reason = "no_active_canceler",
"Decommission cancel skipped"
);
}
if should_reload_pool_meta && let Some(notification_sys) = get_global_notification_sys() {
@@ -1338,7 +1352,15 @@ impl ECStore {
let store = store.clone();
tokio::spawn(async move {
if let Err(err) = store.do_decommission_in_routine(canceler, idx).await {
error!("decommission: routine failed for idx {}: {err}", idx);
error!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "routine_failed",
error = %err,
"Decommission routine failed"
);
}
});
}
@@ -1350,7 +1372,14 @@ impl ECStore {
pub async fn decommission(&self, rx: CancellationToken, indices: Vec<usize>) -> Result<()> {
let indices = dedup_indices(&indices);
warn!("decommission: {:?}", indices);
info!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_indices = ?indices,
state = "requested",
"Decommission requested"
);
validate_start_decommission_request(&indices, self.single_pool())?;
ensure_decommission_not_rebalancing(self.is_rebalance_conflicting_with_decommission().await)?;
@@ -1363,8 +1392,13 @@ impl ECStore {
for idx in indices {
if let Err(cancel_err) = self.decommission_cancel(idx).await {
error!(
"decommission: failed to rollback decommission state for idx {} after spawn error: {:?}",
idx, cancel_err
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "rollback_failed",
error = ?cancel_err,
"Decommission rollback failed after spawn error"
);
if rollback_err.is_none() {
rollback_err = Some(Error::other(format!("decommission rollback failed for idx {idx}: {cancel_err}")));
@@ -1390,10 +1424,28 @@ impl ECStore {
lock_retention: Option<DefaultRetention>,
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
) -> Result<()> {
warn!("decommission_entry: {} {}", &bucket, &entry.name);
debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %entry.name,
state = "started",
"Decommission entry started"
);
wk.give().await;
if entry.is_dir() {
warn!("decommission_entry: skip dir {}", &entry.name);
debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %entry.name,
state = "skipped_directory",
"Decommission entry skipped directory"
);
return Ok(());
}
@@ -1426,7 +1478,16 @@ impl ECStore {
if should_skip_decommission_delete_marker(version, remaining_versions, replication_config.is_some()) {
//
decommissioned += 1;
info!("decommission_pool: DELETE marked object with no other non-current versions will be skipped");
debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %version.name,
state = "skipped_delete_marker",
"Decommission delete marker skipped"
);
continue;
}
@@ -1447,8 +1508,16 @@ impl ECStore {
{
if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) {
warn!(
"decommission_pool: ignore delete-marker copy for {}/{} version {:?}: {:?}",
&bucket, &version.name, &version_id, &err
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %version.name,
version_id = ?version_id,
state = "ignored_delete_marker_copy",
error = ?err,
"Decommission delete marker copy ignored"
);
ignore = true;
cleanup_ignored = true;
@@ -1463,7 +1532,16 @@ impl ECStore {
if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) {
decommissioned += 1;
}
info!("decommission_pool: ignore {}", &version.name);
debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %version.name,
state = "ignored",
"Decommission entry ignored"
);
continue;
}
@@ -1483,9 +1561,17 @@ impl ECStore {
decommissioned += 1;
}
info!(
"decommission_pool: DecomCopyDeleteMarker {} {} {:?} {:?}",
&bucket, &version.name, &version_id, error
debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %version.name,
version_id = ?version_id,
result = ?error,
state = "delete_marker_copied",
"Decommission delete marker copied"
);
continue;
}
@@ -1571,8 +1657,15 @@ impl ECStore {
}
warn!(
"decommission_pool: decommission_object done {}/{} {}",
&bucket_name, &object_name, &version.name
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket_name,
object = %object_name,
version = %version.name,
state = "object_migrated",
"Decommission object migrated"
);
failure = false;
@@ -1583,7 +1676,16 @@ impl ECStore {
if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) {
decommissioned += 1;
}
info!("decommission_pool: ignore {}", &version.name);
debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %version.name,
state = "ignored",
"Decommission entry ignored"
);
continue;
}
@@ -1624,12 +1726,17 @@ impl ECStore {
resolve_decommission_entry_cleanup_delete_result(cleanup_result, bucket.as_str(), entry.name.as_str())?
} else if decommissioned != fivs.versions.len() || expired > 0 {
warn!(
"decommission_pool: source object retained for {}/{} because only {}/{} versions were decommissioned and {} expired by lifecycle",
&bucket,
&entry.name,
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %entry.name,
decommissioned,
fivs.versions.len(),
expired
total_versions = fivs.versions.len(),
expired,
state = "source_retained",
"Decommission source object retained"
);
}
@@ -1667,7 +1774,16 @@ impl ECStore {
}
}
warn!("decommission_pool: decommission_entry done {} {}", &bucket, &entry.name);
debug!(
event = EVENT_DECOMMISSION_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bucket,
object = %entry.name,
state = "completed",
"Decommission entry completed"
);
Ok(())
}
@@ -1705,7 +1821,16 @@ impl ECStore {
for (set_idx, set) in pool.disk_set.iter().enumerate() {
wk.clone().take().await;
warn!("decommission_pool: decommission_pool {} {}", set_idx, &bi.name);
debug!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
set_index = set_idx,
bucket = %bi.name,
state = "listing_worker_started",
"Decommission listing worker started"
);
let decommission_entry: ListCallback = Arc::new({
let this = Arc::clone(self);
@@ -1753,23 +1878,69 @@ impl ECStore {
let worker = tokio::spawn(async move {
loop {
if rx_clone.is_cancelled() {
warn!("decommission_pool: cancel {}", set_id);
debug!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
set_index = set_id,
bucket = %bi.name,
state = "listing_worker_cancelled",
"Decommission listing worker cancelled"
);
break;
}
warn!("decommission_pool: list_objects_to_decommission {} {}", set_id, &bi.name);
debug!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
set_index = set_id,
bucket = %bi.name,
state = "listing_started",
"Decommission listing started"
);
match set
.list_objects_to_decommission(rx_clone.clone(), bi.clone(), decommission_entry.clone())
.await
{
Ok(_) => {
warn!("decommission_pool: list_objects_to_decommission {} done", set_id);
debug!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
set_index = set_id,
bucket = %bi.name,
state = "listing_completed",
"Decommission listing completed"
);
break;
}
Err(err) => {
error!("decommission_pool: list_objects_to_decommission {} err {:?}", set_id, &err);
error!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
set_index = set_id,
bucket = %bi.name,
state = "listing_failed",
error = ?err,
"Decommission listing failed"
);
if is_err_bucket_not_found(&err) {
warn!("decommission_pool: list_objects_to_decommission {} volume not found", set_id);
warn!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
set_index = set_id,
bucket = %bi.name,
state = "listing_bucket_missing",
"Decommission listing bucket missing"
);
break;
}
@@ -1783,7 +1954,15 @@ impl ECStore {
listing_workers.push((set_id, worker));
}
warn!("decommission_pool: decommission_pool wait {} {}", idx, &bi.name);
debug!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bi.name,
state = "waiting_for_workers",
"Decommission waiting for workers"
);
let mut listing_worker_error = None;
for (set_id, worker) in listing_workers {
@@ -1807,11 +1986,28 @@ impl ECStore {
}
if let Err(err) = decommission_cancel_signal_result(rx.is_cancelled()) {
warn!("decommission_pool: canceled after wait {} {}", idx, &bi.name);
warn!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bi.name,
state = "cancelled_after_wait",
error = %err,
"Decommission bucket cancelled after wait"
);
return Err(err);
}
warn!("decommission_pool: decommission_pool done {} {}", idx, &bi.name);
debug!(
event = EVENT_DECOMMISSION_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
bucket = %bi.name,
state = "completed",
"Decommission bucket completed"
);
Ok(())
}
@@ -1821,7 +2017,14 @@ impl ECStore {
defer!(|| async {
let mut cancelers = self.decommission_cancelers.write().await;
if take_decommission_canceler(cancelers.as_mut_slice(), idx).is_none() {
warn!("decommission: canceler already cleared for pool {}", idx);
warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "canceler_already_cleared",
"Decommission canceler already cleared"
);
}
});
@@ -1830,7 +2033,14 @@ impl ECStore {
let (final_state, canceled, cmd_line) = {
let pool_meta = self.pool_meta.read().await;
let Some(pool) = pool_meta.pools.get(idx) else {
error!("decommission: pool metadata missing for idx {}", idx);
error!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "pool_metadata_missing",
"Decommission pool metadata missing"
);
return Err(Error::other(format!(
"failed to resolve decommission final state: pool metadata missing for idx {idx}"
)));
@@ -1849,29 +2059,75 @@ impl ECStore {
};
if let Err(err) = result {
error!("decom err {:?}", &err);
error!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "background_failed",
error = ?err,
"Decommission background routine failed"
);
if is_err_operation_canceled(&err) || should_preserve_decommission_canceled_state(canceled, rx.is_cancelled()) {
warn!("decommission: canceled for pool {}, preserving canceled state", cmd_line);
warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
cmd_line = %cmd_line,
state = "cancelled_preserved",
"Decommission cancelled; preserving canceled state"
);
return Ok(());
}
resolve_decommission_terminal_mark_after_error_result(self.decommission_failed(idx).await, idx, &err)?;
warn!("decommission: decommission_failed {}", idx);
warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "marked_failed",
"Decommission marked failed"
);
return Ok(());
}
warn!("decommission: decommission_in_background complete {}", idx);
debug!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "background_complete",
"Decommission background routine completed"
);
if should_preserve_decommission_canceled_state(canceled, rx.is_cancelled()) {
warn!("decommission: canceled for pool {}, skipping terminal state overwrite", cmd_line);
warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
cmd_line = %cmd_line,
state = "terminal_state_preserved",
"Decommission terminal state preserved after cancellation"
);
return Ok(());
}
match final_state {
DecommissionFinalState::Complete => {
warn!("Decommissioning complete for pool {}, verifying for any pending objects", cmd_line);
debug!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
cmd_line = %cmd_line,
state = "verifying_completion",
"Decommission completion verification started"
);
if let Err(err) = self.check_after_decommission(idx).await {
resolve_decommission_terminal_mark_result(self.decommission_failed(idx).await, "failed", &cmd_line)?;
return Err(Error::other(format!(
@@ -1879,16 +2135,40 @@ impl ECStore {
)));
}
warn!("Decommissioning complete for pool {}, marking completed state", cmd_line);
info!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
cmd_line = %cmd_line,
state = "marking_completed",
"Decommission marking completed state"
);
resolve_decommission_terminal_mark_result(self.complete_decommission(idx).await, "completed", &cmd_line)?;
}
DecommissionFinalState::Failed => {
warn!("Decommissioning finished with failed items for pool {}, marking failed state", cmd_line);
warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
cmd_line = %cmd_line,
state = "marking_failed",
"Decommission marking failed state"
);
resolve_decommission_terminal_mark_result(self.decommission_failed(idx).await, "failed", &cmd_line)?;
}
}
warn!("Decommissioning complete for pool {}", cmd_line);
info!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
cmd_line = %cmd_line,
state = "completed",
"Decommission completed"
);
Ok(())
}
+176 -30
View File
@@ -858,7 +858,14 @@ impl ECStore {
"init_rebalance_meta",
)?;
info!("init_rebalance_meta: rebalance meta saved");
info!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "metadata_initialized",
bucket_count = bucktes.len(),
"Rebalance metadata initialized"
);
let id = meta.id.clone();
@@ -898,9 +905,16 @@ impl ECStore {
#[tracing::instrument(skip(self))]
pub async fn next_rebal_bucket(&self, pool_index: usize) -> Result<Option<String>> {
info!("next_rebal_bucket: pool_index: {}", pool_index);
let rebalance_meta = self.rebalance_meta.read().await;
info!("next_rebal_bucket: rebalance_meta: {:?}", rebalance_meta);
debug!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
has_meta = rebalance_meta.is_some(),
state = "next_bucket_lookup",
"Rebalance next bucket lookup"
);
resolve_next_rebalance_bucket(rebalance_meta.as_ref(), pool_index)
}
@@ -931,20 +945,38 @@ impl ECStore {
let rebalance_meta = self.rebalance_meta.read().await;
if let Some(meta) = rebalance_meta.as_ref() {
meta.pool_stats.iter().enumerate().for_each(|(i, v)| {
info!(
"is_rebalance_started: pool_index: {}, participating: {:?}, status: {:?}",
i, v.participating, v.info.status
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index = i,
participating = v.participating,
status = ?v.info.status,
state = "status_inspected",
"Rebalance status inspected"
);
});
let started = is_rebalance_conflicting_with_decommission(meta);
if started {
info!("is_rebalance_started: rebalance started");
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "running",
"Rebalance is running"
);
return true;
}
}
info!("is_rebalance_started: rebalance not started");
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "not_running",
"Rebalance is not running"
);
false
}
@@ -1108,15 +1140,23 @@ impl ECStore {
workers_started += 1;
tokio::spawn(async move {
if let Err(err) = store.rebalance_buckets(rx_clone, pool_idx).await {
error!("Rebalance failed for pool {}: {}", pool_idx, err);
error!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index = pool_idx,
state = "pool_failed",
error = %err,
"Rebalance pool failed"
);
} else {
info!(
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index = pool_idx,
state = "completed",
"Completed rebalance pool"
"Rebalance pool completed"
);
}
});
@@ -1139,7 +1179,8 @@ impl ECStore {
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "started",
"Started rebalance"
worker_count = workers_started,
"Rebalance started"
);
Ok(())
}
@@ -1240,7 +1281,7 @@ impl ECStore {
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "pool_started",
"Started rebalance worker"
"Rebalance worker started"
);
let mut final_result: Result<()> = Ok(());
let mut deferred_buckets = HashSet::new();
@@ -1267,7 +1308,15 @@ impl ECStore {
let next_bucket = match self.next_rebal_bucket(pool_index).await {
Ok(bucket) => bucket,
Err(err) => {
error!("next_rebal_bucket failed for pool {}: {:?}", pool_index, err);
error!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "next_bucket_failed",
error = ?err,
"Rebalance next bucket lookup failed"
);
final_result = Err(resolve_rebalance_terminal_error(
err.clone(),
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
@@ -1294,7 +1343,16 @@ impl ECStore {
) {
Ok(outcome) => outcome,
Err(err) => {
error!("Error rebalancing bucket {}: {:?}", bucket, err);
error!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
state = "bucket_failed",
error = ?err,
"Rebalance bucket failed"
);
final_result = Err(resolve_rebalance_terminal_error(
err.clone(),
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
@@ -1308,7 +1366,16 @@ impl ECStore {
let err = Error::other(format!(
"rebalance bucket {bucket} deferred repeatedly due to transient object failures: {last_error}"
));
error!("Error rebalancing bucket {}: {:?}", bucket, err);
error!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
state = "bucket_deferred_repeatedly",
error = ?err,
"Rebalance bucket failed after repeated deferral"
);
final_result = Err(resolve_rebalance_terminal_error(
err.clone(),
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
@@ -1327,7 +1394,16 @@ impl ECStore {
"Deferred rebalance bucket after transient object failures"
);
if let Err(err) = self.defer_rebalance_bucket(pool_index, bucket.clone(), last_error).await {
error!("defer_rebalance_bucket failed for pool {}: {:?}", pool_index, err);
error!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
state = "defer_failed",
error = ?err,
"Rebalance bucket defer failed"
);
final_result = Err(resolve_rebalance_terminal_error(
err.clone(),
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
@@ -1347,7 +1423,15 @@ impl ECStore {
"Completed rebalance bucket"
);
if let Err(err) = self.bucket_rebalance_done(pool_index, bucket).await {
error!("bucket_rebalance_done failed for pool {}: {:?}", pool_index, err);
error!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "bucket_done_mark_failed",
error = ?err,
"Rebalance bucket completion mark failed"
);
final_result = Err(resolve_rebalance_terminal_error(
err.clone(),
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
@@ -1374,7 +1458,7 @@ impl ECStore {
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "pool_done",
"Finished rebalance worker"
"Rebalance worker finished"
);
if final_result.is_ok()
@@ -1388,7 +1472,14 @@ impl ECStore {
{
final_result = Err(err);
}
info!("Pool {} rebalancing is done2", pool_index);
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "pool_result_returned",
"Rebalance worker result returned"
);
final_result
}
@@ -2591,7 +2682,15 @@ impl ECStore {
ensure_valid_rebalance_pool_index(self.pools.len(), pool_index)?;
// Placeholder for actual bucket rebalance logic
info!("Rebalancing bucket {} in pool {}", bucket, pool_index);
debug!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
state = "entry_scan_started",
"Rebalance bucket entry scan started"
);
// TODO: other config
// if bucket != RUSTFS_META_BUCKET{
@@ -2765,9 +2864,14 @@ impl ECStore {
let pool = clone_first_arc(&self.pools, "save_rebalance_stats: no pools available")?;
info!(
"save_rebalance_stats: save rebalance meta, pool_idx: {}, opt: {:?}, meta: {:?}",
pool_idx, opt, meta_to_save
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index = pool_idx,
save_opt = ?opt,
state = "metadata_save_requested",
"Rebalance metadata save requested"
);
let stage = format!("save_rebalance_stats for pool {pool_idx} opt {opt:?}");
resolve_rebalance_meta_save_result(
@@ -2838,12 +2942,27 @@ impl SetDisks {
bucket: String,
cb: ListCallback,
) -> Result<()> {
info!("list_objects_to_rebalance: start list_objects_to_rebalance");
debug!(
event = EVENT_REBALANCE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
bucket = %bucket,
state = "started",
"Rebalance listing started"
);
// Placeholder for actual object listing logic
let (disks, _) = self.get_online_disks_with_healing(false).await;
ensure_rebalance_listing_disks_available(!disks.is_empty(), &bucket)?;
info!("list_objects_to_rebalance: get online disks with healing");
debug!(
event = EVENT_REBALANCE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
bucket = %bucket,
disk_count = disks.len(),
state = "disks_resolved",
"Rebalance listing disks resolved"
);
let listing_quorum = self.set_drive_count.div_ceil(2);
let resolver = MetadataResolutionParams {
@@ -2863,7 +2982,14 @@ impl SetDisks {
min_disks: listing_quorum,
skip_walkdir_total_timeout: true,
agreed: Some(Box::new(move |entry: MetaCacheEntry| {
info!("list_objects_to_rebalance: agreed: {:?}", &entry.name);
debug!(
event = EVENT_REBALANCE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
entry = %entry.name,
state = "agreed_entry",
"Rebalance listing agreed entry"
);
Box::pin(cb1(entry))
})),
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
@@ -2873,11 +2999,24 @@ impl SetDisks {
match entries.resolve(resolver) {
Some(entry) => {
info!("list_objects_to_rebalance: list_objects_to_decommission get {}", &entry.name);
debug!(
event = EVENT_REBALANCE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
entry = %entry.name,
state = "resolved_partial_entry",
"Rebalance listing resolved partial entry"
);
Box::pin(async move { cb(entry).await })
}
None => {
info!("list_objects_to_rebalance: list_objects_to_decommission get none");
debug!(
event = EVENT_REBALANCE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "partial_entry_missing",
"Rebalance listing partial entry missing"
);
Box::pin(async {})
}
}
@@ -2887,7 +3026,14 @@ impl SetDisks {
)
.await?;
info!("list_objects_to_rebalance: list_objects_to_rebalance done");
debug!(
event = EVENT_REBALANCE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
bucket = %bucket,
state = "completed",
"Rebalance listing completed"
);
Ok(())
}
}
+363 -37
View File
@@ -65,7 +65,7 @@ use tokio::{
};
use tokio_util::sync::CancellationToken;
use tonic::{Request, service::interceptor::InterceptedService, transport::Channel};
use tracing::{debug, info, warn};
use tracing::{debug, warn};
use uuid::Uuid;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
@@ -76,6 +76,10 @@ enum FailureHealthAction {
const REMOTE_DISK_OPEN_WRITE_MAX_ATTEMPTS: usize = 2;
const REMOTE_DISK_OPEN_WRITE_RETRY_BACKOFF: Duration = Duration::from_millis(20);
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_REMOTE_DISK: &str = "remote_disk";
const EVENT_REMOTE_DISK_HEALTH: &str = "remote_disk_health";
const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc";
async fn copy_stream_with_buffer<R, W>(reader: &mut R, writer: &mut W, buffer_size: usize) -> io::Result<u64>
where
@@ -275,7 +279,16 @@ impl RemoteDisk {
if initial_probe_ok {
health.record_operation_success(&endpoint, "connectivity_probe_success");
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
warn!("Remote disk health check failed for {}: marking as faulty", addr);
warn!(
event = EVENT_REMOTE_DISK_HEALTH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %endpoint,
addr,
state = "initial_probe_failed",
result = "mark_faulty",
"Remote disk initial health probe failed"
);
// Start recovery monitoring
let health_clone = Arc::clone(&health);
@@ -291,7 +304,15 @@ impl RemoteDisk {
loop {
tokio::select! {
_ = cancel_token.cancelled() => {
debug!("Health monitoring cancelled for remote disk: {}", addr);
debug!(
event = EVENT_REMOTE_DISK_HEALTH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %endpoint,
addr,
state = "monitor_cancelled",
"Remote disk health monitor cancelled"
);
return;
}
_ = interval.tick() => {
@@ -320,7 +341,16 @@ impl RemoteDisk {
if Self::perform_connectivity_check(&addr).await.is_ok() {
health.record_operation_success(&endpoint, "connectivity_probe_success");
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
warn!("Remote disk health check failed for {}: marking as faulty", addr);
warn!(
event = EVENT_REMOTE_DISK_HEALTH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %endpoint,
addr,
state = "probe_failed",
result = "mark_faulty",
"Remote disk health probe failed"
);
// Start recovery monitoring
let health_clone = Arc::clone(&health);
@@ -354,9 +384,25 @@ impl RemoteDisk {
_ = interval.tick() => {
if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
info!("Remote disk recovery probe succeeded: {}", addr);
debug!(
event = EVENT_REMOTE_DISK_HEALTH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %endpoint,
addr,
state = "recovery_probe_succeeded",
"Remote disk recovery probe succeeded"
);
if became_online {
info!("Remote disk recovered: {}", addr);
debug!(
event = EVENT_REMOTE_DISK_HEALTH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %endpoint,
addr,
state = "recovered",
"Remote disk recovered"
);
return;
}
} else {
@@ -469,7 +515,16 @@ impl RemoteDisk {
{
// Check if disk is faulty
if self.health.is_faulty() {
warn!("remote disk {} health is faulty, returning error", self.to_string());
warn!(
event = EVENT_REMOTE_DISK_HEALTH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
addr = %self.addr,
op,
state = "faulty_short_circuit",
"Remote disk operation short-circuited by faulty state"
);
return Err(DiskError::FaultyDisk);
}
@@ -519,10 +574,14 @@ impl RemoteDisk {
self.mark_faulty_and_evict("operation_timeout").await;
}
warn!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
addr = %self.addr,
op,
timeout_ms = timeout_duration.as_millis(),
state = "timeout",
"Remote disk operation timed out"
);
Err(DiskError::Timeout)
@@ -547,11 +606,15 @@ impl RemoteDisk {
)
.increment(1);
warn!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
addr = %self.addr,
op,
timeout_ms = timeout_duration.as_millis(),
"Remote disk operation returned a network-like error"
state = "network_like_error",
"Remote disk operation returned network-like error"
);
if failure_health_action == FailureHealthAction::MarkFailure {
self.mark_faulty_and_evict("operation_network_error").await;
@@ -574,13 +637,26 @@ impl RemoteDisk {
.increment(1);
if transitioned_to_offline {
warn!(
"Remote disk marked faulty after timeout: endpoint={}, addr={}, reason={}",
self.endpoint, self.addr, reason
event = EVENT_REMOTE_DISK_HEALTH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
addr = %self.addr,
reason,
state = "marked_faulty",
"Remote disk marked faulty"
);
} else {
warn!(
"Remote disk marked suspect after timeout: endpoint={}, addr={}, reason={}, state={:?}",
self.endpoint, self.addr, reason, state
event = EVENT_REMOTE_DISK_HEALTH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
addr = %self.addr,
reason,
runtime_state = ?state,
state = "marked_suspect",
"Remote disk marked suspect"
);
}
counter!(
@@ -589,11 +665,15 @@ impl RemoteDisk {
"reason" => reason.to_string()
)
.increment(1);
info!(
debug!(
event = EVENT_REMOTE_DISK_HEALTH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
addr = %self.addr,
reason,
"Evicting cached remote disk connection after fault transition"
state = "evict_cached_connection",
"Remote disk cached connection evicted"
);
evict_failed_connection(&self.addr).await;
}
@@ -705,7 +785,16 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn make_volume(&self, volume: &str) -> Result<()> {
info!("make_volume");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
op = "make_volume",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -733,7 +822,16 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> {
info!("make_volumes");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume_count = volumes.len(),
op = "make_volumes",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -761,7 +859,15 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn list_volumes(&self) -> Result<Vec<VolumeInfo>> {
info!("list_volumes");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
op = "list_volumes",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -794,7 +900,16 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn stat_volume(&self, volume: &str) -> Result<VolumeInfo> {
info!("stat_volume");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
op = "stat_volume",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -824,7 +939,16 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn delete_volume(&self, volume: &str) -> Result<()> {
info!("delete_volume {}/{}", self.endpoint.to_string(), volume);
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
op = "delete_volume",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -909,7 +1033,17 @@ impl DiskAPI for RemoteDisk {
force_del_marker: bool,
opts: DeleteOptions,
) -> Result<()> {
info!("delete_version");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
op = "delete_version",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -946,7 +1080,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn delete_versions(&self, volume: &str, versions: Vec<FileInfoVersions>, opts: DeleteOptions) -> Vec<Option<Error>> {
info!("delete_versions");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
version_count = versions.len(),
op = "delete_versions",
state = "started",
"Remote disk RPC started"
);
if self.health.is_faulty() {
return vec![Some(DiskError::FaultyDisk); versions.len()];
@@ -1041,7 +1185,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> {
info!("delete_paths");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path_count = paths.len(),
op = "delete_paths",
state = "started",
"Remote disk RPC started"
);
let paths = paths.to_owned();
self.execute_with_timeout(
@@ -1071,7 +1225,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
info!("write_metadata {}/{}", volume, path);
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
op = "write_metadata",
state = "started",
"Remote disk RPC started"
);
let file_info = serde_json::to_string(&fi)?;
let file_info_bin = encode_msgpack(&fi)?;
@@ -1134,7 +1298,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> {
info!("update_metadata");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
op = "update_metadata",
state = "started",
"Remote disk RPC started"
);
let file_info = serde_json::to_string(&fi)?;
let opts_str = serde_json::to_string(&opts)?;
let file_info_bin = encode_msgpack(&fi)?;
@@ -1180,7 +1354,18 @@ impl DiskAPI for RemoteDisk {
version_id: &str,
opts: &ReadOptions,
) -> Result<FileInfo> {
info!("read_version");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
version_id,
op = "read_version",
state = "started",
"Remote disk RPC started"
);
let opts_str = serde_json::to_string(opts)?;
let opts_bin = encode_msgpack(opts)?;
@@ -1217,7 +1402,18 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(level = "debug", skip(self))]
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
info!("read_xl {}/{}/{}", self.endpoint.to_string(), volume, path);
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
read_data,
op = "read_xl",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -1257,7 +1453,19 @@ impl DiskAPI for RemoteDisk {
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp> {
info!("rename_data {}/{}/{}/{}", self.addr, self.endpoint.to_string(), dst_volume, dst_path);
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
src_volume,
src_path,
dst_volume,
dst_path,
op = "rename_data",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout_for_op(
"rename_data",
@@ -1325,7 +1533,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self, wr))]
async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
info!("walk_dir {}", self.endpoint.to_string());
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
bucket = %opts.bucket,
base_dir = %opts.base_dir,
op = "walk_dir",
state = "started",
"Remote disk RPC started"
);
let disk = self.disk_ref().await;
let body = serde_json::to_vec(&opts)?;
@@ -1443,7 +1661,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(level = "debug", skip(self))]
async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> {
info!("append_file {}/{}", volume, path);
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
op = "append_file",
state = "started",
"Remote disk RPC started"
);
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
@@ -1487,7 +1715,19 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(level = "debug", skip(self))]
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
info!("rename_file");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
src_volume,
src_path,
dst_volume,
dst_path,
op = "rename_file",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -1518,7 +1758,19 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Bytes) -> Result<()> {
info!("rename_part {}/{}", src_volume, src_path);
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
src_volume,
src_path,
dst_volume,
dst_path,
op = "rename_part",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -1550,7 +1802,19 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
info!("delete {}/{}/{}", self.endpoint.to_string(), volume, path);
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
recursive = opt.recursive,
immediate = opt.immediate,
op = "delete",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -1581,7 +1845,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
info!("verify_file");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
op = "verify_file",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -1614,6 +1888,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn read_parts(&self, bucket: &str, paths: &[String]) -> Result<Vec<ObjectPartInfo>> {
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
bucket,
path_count = paths.len(),
op = "read_parts",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
let mut client = self
@@ -1642,7 +1927,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
info!("check_parts");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
op = "check_parts",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -1675,7 +1970,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>> {
info!("read_multiple {}/{}/{}", self.endpoint.to_string(), req.bucket, req.prefix);
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
bucket = %req.bucket,
prefix = %req.prefix,
op = "read_multiple",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -1721,7 +2026,18 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn write_all(&self, volume: &str, path: &str, data: Bytes) -> Result<()> {
info!("write_all");
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
bytes = data.len(),
op = "write_all",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
@@ -1779,7 +2095,17 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(skip(self))]
async fn read_all(&self, volume: &str, path: &str) -> Result<Bytes> {
info!("read_all {}/{}", volume, path);
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
volume,
path,
op = "read_all",
state = "started",
"Remote disk RPC started"
);
self.execute_with_timeout(
|| async {
+110 -10
View File
@@ -130,6 +130,12 @@ use tracing::error;
use tracing::{debug, info, warn};
use uuid::Uuid;
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_SET_DISK: &str = "set_disk";
const EVENT_SET_DISK_MULTIPART: &str = "set_disk_multipart";
const EVENT_SET_DISK_WRITE: &str = "set_disk_write";
const EVENT_SET_DISK_HEAL: &str = "set_disk_heal";
use crate::rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _};
pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024;
@@ -948,7 +954,16 @@ impl ObjectIO for SetDisks {
)
.await
{
error!("get_object_with_fileinfo {bucket}/{object} err {:?}", e);
error!(
event = EVENT_SET_DISK_WRITE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
bucket,
object,
state = "read_pipeline_failed",
error = ?e,
"Set disk object read pipeline failed"
);
};
});
@@ -1065,7 +1080,15 @@ impl ObjectIO for SetDisks {
{
Ok(writer) => (Some(writer), None),
Err(err) => {
warn!("create_bitrot_writer disk {}, err {:?}, skipping operation", disk.to_string(), err);
warn!(
event = EVENT_SET_DISK_WRITE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
disk = ?disk,
state = "bitrot_writer_skipped",
error = ?err,
"Set disk bitrot writer skipped"
);
(None, Some(err))
}
}
@@ -1085,7 +1108,18 @@ impl ObjectIO for SetDisks {
let nil_count = errors.iter().filter(|&e| e.is_none()).count();
if nil_count < write_quorum {
error!("not enough disks to write: {:?}", errors);
error!(
event = EVENT_SET_DISK_WRITE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
bucket,
object,
write_quorum,
available_writers = nil_count,
state = "write_quorum_unavailable",
error = ?errors,
"Set disk write quorum unavailable"
);
if let Some(write_err) = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, write_quorum) {
return Err(to_object_err(write_err.into(), vec![bucket, object]));
}
@@ -1136,7 +1170,17 @@ impl ObjectIO for SetDisks {
// }
if (w_size as i64) < data.size() {
warn!("put_object write size < data.size(), w_size={}, data.size={}", w_size, data.size());
warn!(
event = EVENT_SET_DISK_WRITE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
bucket,
object,
written_size = w_size,
expected_size = data.size(),
state = "short_write",
"Set disk write produced fewer bytes than expected"
);
return Err(Error::other(format!(
"put_object write size < data.size(), w_size={}, data.size={}",
w_size,
@@ -2976,7 +3020,15 @@ impl MultipartOperations for SetDisks {
{
Ok(writer) => writer,
Err(err) => {
warn!("create_bitrot_writer disk {}, err {:?}, skipping operation", disk.to_string(), err);
warn!(
event = EVENT_SET_DISK_MULTIPART,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
disk = ?disk,
state = "bitrot_writer_skipped",
error = ?err,
"Set disk multipart bitrot writer skipped"
);
errors.push(Some(err));
writers.push(None);
continue;
@@ -3021,7 +3073,18 @@ impl MultipartOperations for SetDisks {
let _ = mem::replace(&mut data.stream, reader);
if (w_size as i64) < data.size() {
warn!("put_object_part write size < data.size(), w_size={}, data.size={}", w_size, data.size());
warn!(
event = EVENT_SET_DISK_MULTIPART,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
bucket,
object,
part_number = part_id,
written_size = w_size,
expected_size = data.size(),
state = "short_write",
"Set disk multipart write produced fewer bytes than expected"
);
return Err(Error::other(format!(
"put_object_part write size < data.size(), w_size={}, data.size={}",
w_size,
@@ -3657,7 +3720,17 @@ impl MultipartOperations for SetDisks {
);
return Err(Error::InvalidPart(p.part_num, "".to_owned(), p.etag.clone().unwrap_or_default()));
};
info!(target:"rustfs_ecstore::set_disk", part_number = p.part_num, part_size = ext_part.size, part_actual_size = ext_part.actual_size, "Completing multipart part");
debug!(
target:"rustfs_ecstore::set_disk",
event = EVENT_SET_DISK_MULTIPART,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
part_number = p.part_num,
part_size = ext_part.size,
part_actual_size = ext_part.actual_size,
state = "part_validated",
"Set disk multipart part validated"
);
// Normalize ETags by removing quotes before comparison (PR #592 compatibility)
let client_etag = p.etag.as_ref().map(|e| rustfs_utils::path::trim_etag(e));
@@ -3988,7 +4061,16 @@ impl HealOperations for SetDisks {
let disks = disks.clone();
let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, false, false, false).await?;
if DiskError::is_all_not_found(&errs) {
debug!(bucket, object, version_id, "heal_object skipped missing object");
debug!(
event = EVENT_SET_DISK_HEAL,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
bucket,
object,
version_id,
state = "missing_object_skipped",
"Set disk heal skipped missing object"
);
let err = if !version_id.is_empty() {
Error::FileVersionNotFound
} else {
@@ -4356,7 +4438,16 @@ async fn disks_with_all_parts(
verify_resp = v;
}
Err(err) => {
info!("verify_file failed: {err:?}, object_name={}, index: {index}", object_name);
debug!(
event = EVENT_SET_DISK_HEAL,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
object = %object_name,
disk_index = index,
state = "verify_failed",
error = ?err,
"Set disk verify_file failed"
);
verify_err = Some(err);
}
}
@@ -4366,7 +4457,16 @@ async fn disks_with_all_parts(
verify_resp = v;
}
Err(err) => {
info!("check_parts failed: {err:?}, object_name={}, index: {index}", object_name);
debug!(
event = EVENT_SET_DISK_HEAL,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
object = %object_name,
disk_index = index,
state = "check_parts_failed",
error = ?err,
"Set disk check_parts failed"
);
verify_err = Some(err);
}
}
+17 -17
View File
@@ -69,7 +69,7 @@ impl HealChannelProcessor {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_CHANNEL,
state = "started",
"Heal channel processor state updated"
"Heal channel started"
);
loop {
@@ -85,7 +85,7 @@ impl HealChannelProcessor {
subsystem = LOG_SUBSYSTEM_CHANNEL,
state = "process_failed",
error = %e,
"Heal channel request processing failed"
"Heal channel processing failed"
);
}
}
@@ -96,7 +96,7 @@ impl HealChannelProcessor {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_CHANNEL,
state = "receiver_closed",
"Heal channel processor state updated"
"Heal channel receiver closed"
);
break;
}
@@ -105,7 +105,7 @@ impl HealChannelProcessor {
response = self.response_receiver.recv() => {
if let Some(response) = response {
// Handle response if needed
info!(
debug!(
target: "rustfs::heal::channel",
event = EVENT_HEAL_CHANNEL_RESPONSE,
component = LOG_COMPONENT_HEAL,
@@ -113,7 +113,7 @@ impl HealChannelProcessor {
request_id = %response.request_id,
success = response.success,
state = "received_local",
"Heal channel response observed"
"Heal response observed"
);
}
}
@@ -126,7 +126,7 @@ impl HealChannelProcessor {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_CHANNEL,
state = "stopped",
"Heal channel processor state updated"
"Heal channel stopped"
);
Ok(())
}
@@ -154,7 +154,7 @@ impl HealChannelProcessor {
request: HealChannelRequest,
response_tx: oneshot::Sender<std::result::Result<HealAdmissionResult, String>>,
) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::channel",
event = EVENT_HEAL_CHANNEL_REQUEST,
component = LOG_COMPONENT_HEAL,
@@ -163,7 +163,7 @@ impl HealChannelProcessor {
bucket = %request.bucket,
object_prefix = %request.object_prefix.as_deref().unwrap_or(""),
state = "start_received",
"Heal channel start request received"
"Heal start received"
);
// Convert channel request to heal request
@@ -185,7 +185,7 @@ impl HealChannelProcessor {
// Submit to heal manager
match self.heal_manager.submit_heal_request(heal_request).await {
Ok(admission) => {
info!(
debug!(
target: "rustfs::heal::channel",
event = EVENT_HEAL_CHANNEL_REQUEST,
component = LOG_COMPONENT_HEAL,
@@ -193,7 +193,7 @@ impl HealChannelProcessor {
request_id = %request.id,
admission = admission.result_label(),
state = "admission_decided",
"Heal channel admission decision completed"
"Heal admission decided"
);
let _ = response_tx.send(Ok(admission));
@@ -225,7 +225,7 @@ impl HealChannelProcessor {
request_id = %request.id,
state = "submit_failed",
error = %error_text,
"Heal channel start request failed"
"Heal start submission failed"
);
let _ = response_tx.send(Err(error_text.clone()));
@@ -251,7 +251,7 @@ impl HealChannelProcessor {
client_token: String,
response_tx: oneshot::Sender<std::result::Result<HealChannelResponse, String>>,
) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::channel",
event = EVENT_HEAL_CHANNEL_REQUEST,
component = LOG_COMPONENT_HEAL,
@@ -259,7 +259,7 @@ impl HealChannelProcessor {
request_id = %client_token,
heal_path = %heal_path,
state = "query_received",
"Heal channel query request received"
"Heal query received"
);
let (summary, detail, items) = match self.heal_manager.get_task_report_for_path(&heal_path, &client_token).await {
@@ -332,7 +332,7 @@ impl HealChannelProcessor {
client_token: String,
response_tx: oneshot::Sender<std::result::Result<HealChannelResponse, String>>,
) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::channel",
event = EVENT_HEAL_CHANNEL_REQUEST,
component = LOG_COMPONENT_HEAL,
@@ -340,7 +340,7 @@ impl HealChannelProcessor {
request_id = %client_token,
heal_path = %heal_path,
state = "cancel_received",
"Heal channel cancel request received"
"Heal cancel received"
);
let request_id = if client_token.is_empty() {
@@ -451,7 +451,7 @@ impl HealChannelProcessor {
request_id = %response.request_id,
state = "enqueue_local_failed",
error = %e,
"Heal channel response enqueue failed"
"Heal response local enqueue failed"
);
}
// Always attempt to broadcast, even if local send failed
@@ -464,7 +464,7 @@ impl HealChannelProcessor {
subsystem = LOG_SUBSYSTEM_CHANNEL,
state = "broadcast_failed",
error = %e,
"Heal channel response broadcast failed"
"Heal response broadcast failed"
);
}
}
+27 -27
View File
@@ -27,7 +27,7 @@ use std::sync::{
atomic::{AtomicUsize, Ordering},
};
use tokio::sync::{RwLock, Semaphore};
use tracing::{error, info, warn};
use tracing::{debug, error, info, warn};
const LOG_COMPONENT_HEAL: &str = "heal";
const LOG_SUBSYSTEM_ERASURE_HEALER: &str = "erasure_healer";
@@ -99,7 +99,7 @@ impl ErasureSetHealer {
/// execute erasure set heal with resume
#[tracing::instrument(skip(self, buckets), fields(set_disk_id = %set_disk_id, bucket_count = buckets.len()))]
pub async fn heal_erasure_set(&self, buckets: &[String], set_disk_id: &str) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
@@ -162,7 +162,7 @@ impl ErasureSetHealer {
Ok(manager) => {
let state = manager.get_state().await;
if state.set_disk_id == set_disk_id && ResumeUtils::can_resume_task(&self.disk, &task_id).await {
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
@@ -170,7 +170,7 @@ impl ErasureSetHealer {
task_id,
set_disk_id,
state = "resume_found",
"Erasure set resume state selected"
"Erasure set resume selected"
);
return Ok(task_id);
}
@@ -193,7 +193,7 @@ impl ErasureSetHealer {
// create new task id
let task_id = format!("{}_{}", set_disk_id, ResumeUtils::generate_task_id());
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
@@ -215,7 +215,7 @@ impl ErasureSetHealer {
) -> Result<(ResumeManager, CheckpointManager)> {
// check if resume state exists
if ResumeManager::has_resume_state(&self.disk, task_id).await {
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
@@ -235,7 +235,7 @@ impl ErasureSetHealer {
Ok((resume_manager, checkpoint_manager))
} else {
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
@@ -243,7 +243,7 @@ impl ErasureSetHealer {
task_id,
set_disk_id,
state = "creating_new",
"Erasure set resume state creating"
"Erasure set resume created"
);
let resume_manager = ResumeManager::new(
@@ -273,7 +273,7 @@ impl ErasureSetHealer {
let state = resume_manager.get_state().await;
let checkpoint = checkpoint_manager.get_checkpoint().await;
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
@@ -282,7 +282,7 @@ impl ErasureSetHealer {
current_bucket_index = checkpoint.current_bucket_index,
current_object_index = checkpoint.current_object_index,
state = "resuming",
"Erasure set heal resumed from checkpoint"
"Erasure set resumed"
);
// 2. initialize progress
@@ -349,7 +349,7 @@ impl ErasureSetHealer {
match bucket_result {
Ok(_) => {
resume_manager.complete_bucket(bucket).await?;
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_BUCKET_STATE,
component = LOG_COMPONENT_HEAL,
@@ -357,7 +357,7 @@ impl ErasureSetHealer {
set_disk_id,
bucket,
state = "completed",
"Erasure set bucket heal completed"
"Erasure set bucket completed"
);
}
Err(e) => {
@@ -383,14 +383,14 @@ impl ErasureSetHealer {
// 5. mark task completed
resume_manager.mark_completed().await?;
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
set_disk_id,
state = "completed",
"Erasure set heal completed"
"Erasure set completed"
);
Ok(())
}
@@ -411,7 +411,7 @@ impl ErasureSetHealer {
resume_manager: &ResumeManager,
checkpoint_manager: &CheckpointManager,
) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_BUCKET_STATE,
component = LOG_COMPONENT_HEAL,
@@ -421,7 +421,7 @@ impl ErasureSetHealer {
bucket_index,
current_object_index = *current_object_index,
state = "started",
"Erasure set bucket heal started"
"Erasure set bucket started"
);
// 1. get bucket info
@@ -584,7 +584,7 @@ impl ErasureSetHealer {
Ok(true) => {
*successful_objects += 1;
checkpoint_manager.add_processed_object(object.clone()).await?;
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
component = LOG_COMPONENT_HEAL,
@@ -593,13 +593,13 @@ impl ErasureSetHealer {
bucket,
object = %object,
state = "healed",
"Erasure set object heal completed"
"Erasure set object healed"
);
}
Ok(false) => {
checkpoint_manager.add_processed_object(object.clone()).await?;
*successful_objects += 1;
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
component = LOG_COMPONENT_HEAL,
@@ -608,7 +608,7 @@ impl ErasureSetHealer {
bucket,
object = %object,
state = "missing_treated_as_ok",
"Erasure set object heal skipped because object disappeared"
"Erasure set missing object treated as ok"
);
}
Err(Error::TaskCancelled) => {
@@ -743,14 +743,14 @@ impl ErasureSetHealer {
bucket: &str,
progress: &Arc<RwLock<HealProgress>>,
) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_BUCKET_STATE,
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
bucket,
state = "started",
"Erasure set bucket heal started"
"Erasure set bucket started"
);
// 1. get bucket info
@@ -847,7 +847,7 @@ impl ErasureSetHealer {
p.set_current_object(Some(format!("completed bucket: {bucket}")));
}
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_BUCKET_STATE,
component = LOG_COMPONENT_HEAL,
@@ -857,7 +857,7 @@ impl ErasureSetHealer {
total_failed,
total_scanned,
state = "completed",
"Erasure set bucket heal completed"
"Erasure set bucket completed"
);
Ok(())
@@ -890,7 +890,7 @@ impl ErasureSetHealer {
match storage.heal_object(&bucket, &object, None, &heal_opts).await {
Ok((_result, None)) => {
info!(
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
component = LOG_COMPONENT_HEAL,
@@ -898,7 +898,7 @@ impl ErasureSetHealer {
bucket,
object = %object,
state = "healed",
"Erasure set object heal completed"
"Erasure set object healed"
);
Ok(())
}
@@ -956,7 +956,7 @@ impl ErasureSetHealer {
success_count,
total,
state = "summary",
"Erasure set heal summary recorded"
"Erasure set summary recorded"
);
if failure_count > 0 {
+56 -36
View File
@@ -558,7 +558,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER,
state = "already_running",
"Heal manager state unchanged"
"Heal manager already running"
);
return Ok(());
}
@@ -571,7 +571,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER,
state = "starting",
"Heal manager state updated"
"Heal manager starting"
);
// start scheduler
@@ -586,7 +586,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER,
state = "running",
"Heal manager state updated"
"Heal manager started"
);
Ok(())
}
@@ -599,7 +599,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER,
state = "stopping",
"Heal manager state updated"
"Heal manager stopping"
);
// cancel all tasks
@@ -617,7 +617,7 @@ impl HealManager {
state = "task_cancel_failed",
task_id = %task.id,
error = %e,
"Heal manager failed to cancel active task"
"Heal active task cancellation failed"
);
}
}
@@ -636,7 +636,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER,
state = "stopped",
"Heal manager state updated"
"Heal manager stopped"
);
Ok(())
}
@@ -659,7 +659,7 @@ impl HealManager {
match admission {
HealAdmissionResult::Merged => {
info!(
debug!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_QUEUE_ADMISSION,
component = LOG_COMPONENT_HEAL,
@@ -667,7 +667,7 @@ impl HealManager {
request_id = %request.id,
priority = ?request.priority,
result = "merged_duplicate",
"Heal queue admission decided"
"Heal queue request merged"
);
}
HealAdmissionResult::Dropped(reason) => {
@@ -680,7 +680,7 @@ impl HealManager {
priority = ?request.priority,
reason = reason.as_str(),
result = "dropped_duplicate",
"Heal queue admission decided"
"Heal queue request dropped"
);
}
HealAdmissionResult::Accepted | HealAdmissionResult::Full => {}
@@ -707,7 +707,7 @@ impl HealManager {
queue_len,
queue_capacity,
result = "accepted_by_displacement",
"Heal queue admission decided"
"Heal queue request accepted by displacement"
);
drop(queue);
if config.event_driven_scheduler_enable {
@@ -726,7 +726,7 @@ impl HealManager {
queue_len,
queue_capacity,
result = "full_no_displacement_candidate",
"Heal queue admission decided"
"Heal queue request rejected without displacement"
);
return Ok(HealAdmissionResult::Full);
}
@@ -745,7 +745,7 @@ impl HealManager {
queue_capacity,
reason = reason.as_str(),
result = "dropped_full",
"Heal queue admission decided"
"Heal queue request dropped"
);
}
HealAdmissionResult::Full => {
@@ -759,7 +759,7 @@ impl HealManager {
queue_len,
queue_capacity,
result = "rejected_full",
"Heal queue admission decided"
"Heal queue request rejected"
);
}
HealAdmissionResult::Accepted | HealAdmissionResult::Merged => {}
@@ -779,7 +779,7 @@ impl HealManager {
queue_capacity,
queue_usage_pct = (queue_len * 100) / queue_capacity,
result = "queue_pressure_high",
"Heal queue pressure increased"
"Heal queue pressure high"
);
}
@@ -793,7 +793,7 @@ impl HealManager {
// Log queue statistics periodically (when adding high/urgent priority items)
if matches!(priority, HealPriority::High | HealPriority::Urgent) {
let stats = queue.get_priority_stats();
info!(
debug!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_QUEUE_ADMISSION,
component = LOG_COMPONENT_HEAL,
@@ -812,7 +812,7 @@ impl HealManager {
drop(queue);
info!(
debug!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_QUEUE_ADMISSION,
component = LOG_COMPONENT_HEAL,
@@ -821,7 +821,7 @@ impl HealManager {
priority = ?priority,
queue_len = queue_len + 1,
result = "accepted",
"Heal queue admission decided"
"Heal queue request accepted"
);
if config.event_driven_scheduler_enable {
self.notify.notify_one();
@@ -1119,7 +1119,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER,
state = "shutdown",
"Heal scheduler state updated"
"Heal scheduler stopped"
);
break;
}
@@ -1158,13 +1158,17 @@ impl HealManager {
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
state = "started",
interval = ?duration,
"Heal auto disk scanner state updated"
"Heal auto disk scanner started"
);
tokio::spawn(async move {
let mut interval = interval(duration);
loop {
let mut candidate_count = 0usize;
let mut skipped_duplicate_count = 0usize;
let mut skipped_invalid_count = 0usize;
let mut enqueued_count = 0usize;
tokio::select! {
_ = cancel_token.cancelled() => {
info!(
@@ -1173,7 +1177,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
state = "shutdown",
"Heal auto disk scanner state updated"
"Heal auto disk scanner stopped"
);
break;
}
@@ -1185,14 +1189,15 @@ impl HealManager {
// detect unformatted disk via get_disk_id()
match disk.get_disk_id().await {
Err(DiskError::UnformattedDisk) => {
info!(
candidate_count += 1;
debug!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_AUTO_SCAN_DISK,
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
endpoint = %disk.endpoint(),
disk_state = "unformatted",
"Heal auto disk scanner detected candidate disk"
"Heal auto-scan candidate detected"
);
endpoints.push(disk.endpoint());
}
@@ -1205,7 +1210,7 @@ impl HealManager {
endpoint = %disk.endpoint(),
disk_state = "check_failed",
error = ?e,
"Heal auto disk scanner failed to inspect disk"
"Heal auto-scan disk inspection failed"
);
}
Ok(_) => {
@@ -1222,7 +1227,7 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
state = "idle",
"Heal auto disk scanner found no candidate disks"
"Heal auto disk scanner idle"
);
continue;
}
@@ -1238,7 +1243,7 @@ impl HealManager {
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
state = "bucket_list_failed",
error = %e,
"Heal auto disk scanner failed to list buckets"
"Heal auto-scan bucket listing failed"
);
continue;
}
@@ -1256,8 +1261,9 @@ impl HealManager {
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
endpoint = %ep,
result = "skipped_invalid_set_disk_id",
"Heal auto disk scanner skipped enqueue"
"Heal auto-scan enqueue skipped"
);
skipped_invalid_count += 1;
continue;
};
// skip if already queued or healing
@@ -1283,7 +1289,8 @@ impl HealManager {
}
if skip {
info!(
skipped_duplicate_count += 1;
debug!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
component = LOG_COMPONENT_HEAL,
@@ -1291,7 +1298,7 @@ impl HealManager {
endpoint = %ep,
set_disk_id,
result = "skipped_duplicate",
"Heal auto disk scanner skipped enqueue"
"Heal auto-scan duplicate skipped"
);
continue;
}
@@ -1312,7 +1319,8 @@ impl HealManager {
if config.event_driven_scheduler_enable {
notify.notify_one();
}
info!(
enqueued_count += 1;
debug!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
component = LOG_COMPONENT_HEAL,
@@ -1321,10 +1329,22 @@ impl HealManager {
set_disk_id,
bucket_count = buckets.len(),
result = "enqueued",
"Heal auto disk scanner enqueued task"
"Heal auto-scan task enqueued"
);
}
}
info!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_AUTO_SCAN_STATE,
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
state = "cycle_completed",
candidate_count,
enqueued_count,
skipped_duplicate_count,
skipped_invalid_count,
"Heal auto-scan cycle completed"
);
}
}
}
@@ -1405,7 +1425,7 @@ impl HealManager {
// start heal task
tokio::spawn(async move {
info!(
debug!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_SCHEDULER_STATE,
component = LOG_COMPONENT_HEAL,
@@ -1415,12 +1435,12 @@ impl HealManager {
heal_type = %task_type_label_for_spawn,
set = %task_set_label_for_spawn,
state = "task_started",
"Heal scheduler started task"
"Heal scheduler task started"
);
let result = task.execute().await;
match result {
Ok(_) => {
info!(
debug!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_SCHEDULER_STATE,
component = LOG_COMPONENT_HEAL,
@@ -1429,7 +1449,7 @@ impl HealManager {
heal_type = %task_type_label_for_spawn,
set = %task_set_label_for_spawn,
state = "task_completed",
"Heal scheduler finished task"
"Heal scheduler task completed"
);
}
Err(e) => {
@@ -1443,7 +1463,7 @@ impl HealManager {
set = %task_set_label_for_spawn,
state = "task_failed",
error = %e,
"Heal scheduler observed task failure"
"Heal scheduler task failed"
);
}
}
@@ -1500,7 +1520,7 @@ impl HealManager {
queue_len = remaining,
active_tasks = active_heals_guard.len(),
state = "backlog_high",
"Heal queue backlog summary recorded"
"Heal queue backlog high"
);
}
}
+5 -5
View File
@@ -20,7 +20,7 @@ use std::path::Path;
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tokio::sync::RwLock;
use tracing::{debug, info, warn};
use tracing::{debug, warn};
use uuid::Uuid;
const LOG_COMPONENT_HEAL: &str = "heal";
@@ -299,7 +299,7 @@ impl ResumeManager {
let _ = self.disk.delete(RUSTFS_META_BUCKET, path_str, Default::default()).await;
}
info!(
debug!(
target: "rustfs::heal::resume",
event = EVENT_HEAL_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
@@ -524,7 +524,7 @@ impl CheckpointManager {
let _ = self.disk.delete(RUSTFS_META_BUCKET, path_str, Default::default()).await;
}
info!(
debug!(
target: "rustfs::heal::resume",
event = EVENT_HEAL_CHECKPOINT_STATE,
component = LOG_COMPONENT_HEAL,
@@ -649,7 +649,7 @@ impl ResumeUtils {
let age_hours = (current_time - state.last_update) / 3600;
if age_hours > max_age_hours {
info!(
debug!(
target: "rustfs::heal::resume",
event = EVENT_HEAL_RESUME_STATE,
component = LOG_COMPONENT_HEAL,
@@ -657,7 +657,7 @@ impl ResumeUtils {
task_id,
age_hours,
state = "expired_cleanup_started",
"Heal resume state cleanup started"
"Heal resume cleanup started"
);
if let Err(e) = resume_manager.cleanup().await {
warn!(
+33 -33
View File
@@ -24,7 +24,7 @@ use rustfs_ecstore::{
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_storage_api::{BucketInfo, DiskSetSelector, StorageAdminApi};
use std::sync::Arc;
use tracing::{debug, error, info, warn};
use tracing::{debug, error, warn};
const LOG_COMPONENT_HEAL: &str = "heal";
const LOG_SUBSYSTEM_STORAGE: &str = "storage";
@@ -206,7 +206,7 @@ impl HealStorageAPI for ECStoreHealStorage {
bucket,
object,
result = "not_found",
"Heal storage request finished"
"Heal storage object metadata missing"
);
Ok(None)
} else {
@@ -333,7 +333,7 @@ impl HealStorageAPI for ECStoreHealStorage {
.await
{
Ok(_) => {
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_OBJECT_IO,
component = LOG_COMPONENT_HEAL,
@@ -342,7 +342,7 @@ impl HealStorageAPI for ECStoreHealStorage {
bucket,
object,
result = "ok",
"Heal storage request finished"
"Heal storage object write completed"
);
Ok(())
}
@@ -378,7 +378,7 @@ impl HealStorageAPI for ECStoreHealStorage {
match self.ecstore.delete_object(bucket, object, Default::default()).await {
Ok(_) => {
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_OBJECT_IO,
component = LOG_COMPONENT_HEAL,
@@ -387,7 +387,7 @@ impl HealStorageAPI for ECStoreHealStorage {
bucket,
object,
result = "ok",
"Heal storage request finished"
"Heal storage object delete completed"
);
Ok(())
}
@@ -447,7 +447,7 @@ impl HealStorageAPI for ECStoreHealStorage {
let mut stream = reader.stream;
match tokio::io::copy(&mut stream, &mut tokio::io::sink()).await {
Ok(_) => {
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_OBJECT_VERIFY,
component = LOG_COMPONENT_HEAL,
@@ -455,7 +455,7 @@ impl HealStorageAPI for ECStoreHealStorage {
bucket,
object,
state = "ok",
"Heal storage object verification finished"
"Heal storage object verified"
);
Ok(true)
}
@@ -544,7 +544,7 @@ impl HealStorageAPI for ECStoreHealStorage {
// After healing, try to read the object data
match self.get_object_data(bucket, object).await? {
Some(data) => {
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_REPAIR_OP,
component = LOG_COMPONENT_HEAL,
@@ -554,7 +554,7 @@ impl HealStorageAPI for ECStoreHealStorage {
object,
bytes = data.len(),
state = "ok",
"Heal storage repair finished"
"Heal storage EC decode rebuild completed"
);
Ok(data)
}
@@ -608,7 +608,7 @@ impl HealStorageAPI for ECStoreHealStorage {
// TODO: implement disk status check using ecstore
// For now, return Ok status
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_ADMIN_OP,
component = LOG_COMPONENT_HEAL,
@@ -617,7 +617,7 @@ impl HealStorageAPI for ECStoreHealStorage {
endpoint = ?endpoint,
result = "ok",
disk_status = "ok",
"Heal storage admin operation finished"
"Heal storage disk status resolved"
);
Ok(DiskStatus::Ok)
}
@@ -640,7 +640,7 @@ impl HealStorageAPI for ECStoreHealStorage {
if error.is_some() {
return Err(Error::other(format!("Format failed: {error:?}")));
}
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_ADMIN_OP,
component = LOG_COMPONENT_HEAL,
@@ -648,7 +648,7 @@ impl HealStorageAPI for ECStoreHealStorage {
operation = "format_disk",
endpoint = ?endpoint,
result = "ok",
"Heal storage admin operation finished"
"Heal storage disk format completed"
);
Ok(())
}
@@ -726,7 +726,7 @@ impl HealStorageAPI for ECStoreHealStorage {
match self.heal_bucket(bucket, &heal_opts).await {
Ok(_) => {
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_REPAIR_OP,
component = LOG_COMPONENT_HEAL,
@@ -734,7 +734,7 @@ impl HealStorageAPI for ECStoreHealStorage {
operation = "heal_bucket_metadata",
bucket,
result = "ok",
"Heal storage repair finished"
"Heal storage bucket metadata repaired"
);
Ok(())
}
@@ -816,7 +816,7 @@ impl HealStorageAPI for ECStoreHealStorage {
bucket,
object,
result = "not_found",
"Heal storage request finished"
"Heal storage object absence confirmed"
);
Ok(false)
} else if is_transient_object_exists_error(&e) {
@@ -923,7 +923,7 @@ impl HealStorageAPI for ECStoreHealStorage {
match self.ecstore.heal_object(bucket, object, version_id_str, opts).await {
Ok((result, ecstore_error)) => {
let error = ecstore_error.map(Error::other);
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_REPAIR_OP,
component = LOG_COMPONENT_HEAL,
@@ -935,7 +935,7 @@ impl HealStorageAPI for ECStoreHealStorage {
drives_after = result.after.drives.len(),
has_error = error.is_some(),
result = "ok",
"Heal storage repair finished"
"Heal storage object repair completed"
);
Ok((result, error))
}
@@ -974,7 +974,7 @@ impl HealStorageAPI for ECStoreHealStorage {
match self.ecstore.heal_bucket(bucket, opts).await {
Ok(result) => {
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_REPAIR_OP,
component = LOG_COMPONENT_HEAL,
@@ -983,7 +983,7 @@ impl HealStorageAPI for ECStoreHealStorage {
bucket,
drives_after = result.after.drives.len(),
result = "ok",
"Heal storage repair finished"
"Heal storage bucket repair completed"
);
Ok(result)
}
@@ -1019,7 +1019,7 @@ impl HealStorageAPI for ECStoreHealStorage {
match self.ecstore.heal_format(dry_run).await {
Ok((result, ecstore_error)) => {
let error = ecstore_error.map(Error::other);
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_REPAIR_OP,
component = LOG_COMPONENT_HEAL,
@@ -1028,7 +1028,7 @@ impl HealStorageAPI for ECStoreHealStorage {
drives_after = result.after.drives.len(),
has_error = error.is_some(),
result = "ok",
"Heal storage repair finished"
"Heal storage format repair completed"
);
Ok((result, error))
}
@@ -1103,7 +1103,7 @@ impl HealStorageAPI for ECStoreHealStorage {
}
}
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_ADMIN_OP,
component = LOG_COMPONENT_HEAL,
@@ -1113,7 +1113,7 @@ impl HealStorageAPI for ECStoreHealStorage {
prefix,
object_count = all_objects.len(),
result = "ok",
"Heal storage admin operation finished"
"Heal storage object listing completed"
);
Ok(all_objects)
}
@@ -1175,12 +1175,12 @@ impl HealStorageAPI for ECStoreHealStorage {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_STORAGE,
operation = "list_objects_for_heal_page",
bucket,
prefix,
object_count = page_count,
is_truncated = list_info.is_truncated,
state = "page_loaded",
"Heal storage admin operation finished"
bucket,
prefix,
object_count = page_count,
is_truncated = list_info.is_truncated,
state = "page_loaded",
"Heal storage object listing page loaded"
);
Ok((page_objects, list_info.next_continuation_token, list_info.is_truncated))
@@ -1210,7 +1210,7 @@ impl HealStorageAPI for ECStoreHealStorage {
// Find the first available disk
if let Some(disk_store) = disks.into_iter().flatten().next() {
info!(
debug!(
target: "rustfs::heal::storage",
event = EVENT_HEAL_STORAGE_ADMIN_OP,
component = LOG_COMPONENT_HEAL,
@@ -1219,7 +1219,7 @@ impl HealStorageAPI for ECStoreHealStorage {
set_disk_id,
result = "ok",
disk = ?disk_store,
"Heal storage admin operation finished"
"Heal storage resume disk resolved"
);
return Ok(disk_store);
}
+51 -51
View File
@@ -445,7 +445,7 @@ impl HealTask {
heal_type = self.heal_type.log_kind(),
state = "started",
queue_delay = ?queue_delay,
"Heal task state updated"
"Heal task started"
);
let result = match &self.heal_type {
@@ -484,7 +484,7 @@ impl HealTask {
task_id = %self.id,
heal_type = self.heal_type.log_kind(),
state = "completed",
"Heal task state updated"
"Heal task completed"
);
}
Err(Error::TaskCancelled) => {
@@ -498,7 +498,7 @@ impl HealTask {
task_id = %self.id,
heal_type = self.heal_type.log_kind(),
state = "cancelled",
"Heal task state updated"
"Heal task cancelled"
);
}
Err(Error::TaskTimeout) => {
@@ -512,7 +512,7 @@ impl HealTask {
task_id = %self.id,
heal_type = self.heal_type.log_kind(),
state = "timed_out",
"Heal task state updated"
"Heal task timed out"
);
}
Err(e) => {
@@ -527,7 +527,7 @@ impl HealTask {
heal_type = self.heal_type.log_kind(),
state = "failed",
error = %e,
"Heal task state updated"
"Heal task failed"
);
}
}
@@ -539,7 +539,7 @@ impl HealTask {
self.cancel_token.cancel();
let mut status = self.status.write().await;
*status = HealTaskStatus::Cancelled;
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_TASK_STATE,
component = LOG_COMPONENT_HEAL,
@@ -548,7 +548,7 @@ impl HealTask {
heal_type = self.heal_type.log_kind(),
state = "cancelled",
source = "manual",
"Heal task state updated"
"Heal task cancellation requested"
);
Ok(())
}
@@ -572,7 +572,7 @@ impl HealTask {
// specific heal implementation method
#[tracing::instrument(skip(self), fields(bucket = %bucket, object = %object, version_id = ?version_id))]
async fn heal_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -582,7 +582,7 @@ impl HealTask {
object,
version_id = ?version_id,
stage = "start",
"Heal object workflow started"
"Heal object started"
);
// update progress
@@ -625,7 +625,7 @@ impl HealTask {
"Heal target object is missing"
);
if self.options.recreate_missing {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -634,7 +634,7 @@ impl HealTask {
bucket,
object,
stage = "recreate_missing",
"Heal object recovery started"
"Heal object recreate requested"
);
return self.recreate_missing_object(bucket, object, version_id).await;
} else {
@@ -690,7 +690,7 @@ impl HealTask {
// Check if this is a "File not found" error during delete operations
let error_msg = format!("{e}");
if error_msg.contains("File not found") || error_msg.contains("not found") {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -699,7 +699,7 @@ impl HealTask {
bucket,
object,
result = "treated_as_deleted",
"Heal object finished after target disappeared during repair"
"Heal missing object treated as deleted"
);
{
let mut progress = self.progress.write().await;
@@ -723,7 +723,7 @@ impl HealTask {
// If heal failed and remove_corrupted is enabled, delete the corrupted object
if self.options.remove_corrupted {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_CLEANUP,
component = LOG_COMPONENT_HEAL,
@@ -737,7 +737,7 @@ impl HealTask {
);
if !self.options.dry_run {
self.await_with_control(self.storage.delete_object(bucket, object)).await?;
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_CLEANUP,
component = LOG_COMPONENT_HEAL,
@@ -747,7 +747,7 @@ impl HealTask {
object,
action = "delete_corrupted_object",
result = "deleted",
"Heal object cleanup completed"
"Heal corrupted object deleted"
);
}
}
@@ -775,7 +775,7 @@ impl HealTask {
"Heal object stage entered"
);
let object_size = result.object_size as u64;
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -786,7 +786,7 @@ impl HealTask {
object_size = object_size,
drives_healed = result.after.drives.len(),
result = "ok",
"Heal object completed"
"Heal object repaired"
);
{
@@ -806,7 +806,7 @@ impl HealTask {
// Check if this is a "File not found" error during delete operations
let error_msg = format!("{e}");
if error_msg.contains("File not found") || error_msg.contains("not found") {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -815,7 +815,7 @@ impl HealTask {
bucket,
object,
result = "treated_as_deleted",
"Heal object finished after target disappeared during repair"
"Heal missing object treated as deleted"
);
{
let mut progress = self.progress.write().await;
@@ -839,7 +839,7 @@ impl HealTask {
// If heal failed and remove_corrupted is enabled, delete the corrupted object
if self.options.remove_corrupted {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_CLEANUP,
component = LOG_COMPONENT_HEAL,
@@ -853,7 +853,7 @@ impl HealTask {
);
if !self.options.dry_run {
self.await_with_control(self.storage.delete_object(bucket, object)).await?;
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_CLEANUP,
component = LOG_COMPONENT_HEAL,
@@ -863,7 +863,7 @@ impl HealTask {
object,
action = "delete_corrupted_object",
result = "deleted",
"Heal object cleanup completed"
"Heal corrupted object deleted"
);
}
}
@@ -882,7 +882,7 @@ impl HealTask {
/// Recreate missing object (for EC decode scenarios)
async fn recreate_missing_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -892,7 +892,7 @@ impl HealTask {
object,
version_id = ?version_id,
stage = "recreate_missing",
"Heal object recovery started"
"Heal object recreate started"
);
// Use ecstore's heal_object with recreate option
@@ -932,7 +932,7 @@ impl HealTask {
}
let object_size = result.object_size as u64;
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -942,7 +942,7 @@ impl HealTask {
object,
object_size,
result = "recreated",
"Heal object recovery finished"
"Heal object recreated"
);
{
@@ -975,7 +975,7 @@ impl HealTask {
}
async fn heal_bucket(&self, bucket: &str) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_BUCKET_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -984,7 +984,7 @@ impl HealTask {
bucket,
stage = "start",
recursive = self.options.recursive,
"Heal bucket workflow started"
"Heal bucket started"
);
// update progress
@@ -1060,7 +1060,7 @@ impl HealTask {
match heal_result {
Ok(result) => {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_BUCKET_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -1269,7 +1269,7 @@ impl HealTask {
});
}
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_BUCKET_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -1287,7 +1287,7 @@ impl HealTask {
}
async fn heal_metadata(&self, bucket: &str, object: &str) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_METADATA_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -1296,7 +1296,7 @@ impl HealTask {
bucket,
object,
stage = "start",
"Heal metadata workflow started"
"Heal metadata started"
);
// update progress
@@ -1400,7 +1400,7 @@ impl HealTask {
});
}
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_METADATA_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -1410,7 +1410,7 @@ impl HealTask {
object,
drives_healed = result.after.drives.len(),
result = "ok",
"Heal metadata completed"
"Heal metadata repaired"
);
{
@@ -1447,7 +1447,7 @@ impl HealTask {
}
async fn heal_mrf(&self, meta_path: &str) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_MRF_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -1455,7 +1455,7 @@ impl HealTask {
task_id = %self.id,
meta_path,
stage = "start",
"Heal MRF workflow started"
"Heal MRF started"
);
// update progress
@@ -1530,7 +1530,7 @@ impl HealTask {
});
}
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_MRF_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -1541,7 +1541,7 @@ impl HealTask {
object = %object,
drives_healed = result.after.drives.len(),
result = "ok",
"Heal MRF completed"
"Heal MRF repaired"
);
{
@@ -1579,7 +1579,7 @@ impl HealTask {
}
async fn heal_ec_decode(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_EC_DECODE_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -1589,7 +1589,7 @@ impl HealTask {
object,
version_id = ?version_id,
stage = "start",
"Heal EC decode workflow started"
"Heal EC decode started"
);
// update progress
@@ -1694,7 +1694,7 @@ impl HealTask {
}
let object_size = result.object_size as u64;
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_EC_DECODE_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -1705,7 +1705,7 @@ impl HealTask {
object_size,
drives_healed = result.after.drives.len(),
result = "ok",
"Heal EC decode completed"
"Heal EC decode repaired"
);
{
@@ -1742,7 +1742,7 @@ impl HealTask {
}
async fn heal_erasure_set(&self, buckets: Vec<String>, set_disk_id: String) -> Result<()> {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_ERASURE_SET_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -1751,7 +1751,7 @@ impl HealTask {
set_disk_id,
bucket_count = buckets.len(),
stage = "start",
"Heal erasure set workflow started"
"Heal erasure set started"
);
// update progress
@@ -1762,7 +1762,7 @@ impl HealTask {
}
let buckets = if buckets.is_empty() {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_ERASURE_SET_STAGE,
component = LOG_COMPONENT_HEAL,
@@ -1770,7 +1770,7 @@ impl HealTask {
task_id = %self.id,
set_disk_id,
stage = "list_buckets",
"Heal erasure set resolved bucket list from storage"
"Heal erasure set bucket list resolved"
);
let bucket_infos = self.await_with_control(self.storage.list_buckets()).await?;
bucket_infos.into_iter().map(|info| info.name).collect()
@@ -1814,7 +1814,7 @@ impl HealTask {
});
}
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_ERASURE_SET_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -1823,7 +1823,7 @@ impl HealTask {
set_disk_id,
drives_healed = result.after.drives.len(),
result = "format_ok",
"Heal erasure set format repair completed"
"Heal erasure set format repaired"
);
}
Err(Error::TaskCancelled) => return Err(Error::TaskCancelled),
@@ -1970,7 +1970,7 @@ impl HealTask {
match result {
Ok(_) => {
info!(
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_ERASURE_SET_RESULT,
component = LOG_COMPONENT_HEAL,
@@ -1979,7 +1979,7 @@ impl HealTask {
set_disk_id,
bucket_count = buckets.len(),
result = "ok",
"Heal erasure set completed"
"Heal erasure set repaired"
);
Ok(())
}
+79 -11
View File
@@ -19,7 +19,13 @@ use rustfs_ecstore::store::ECStore;
use std::sync::{Arc, OnceLock};
use store::object::ObjectStore;
use sys::IamSys;
use tracing::{error, info, instrument, warn};
use tracing::{debug, error, info, instrument, warn};
const LOG_COMPONENT_IAM: &str = "iam";
const LOG_SUBSYSTEM_RUNTIME: &str = "runtime";
const LOG_SUBSYSTEM_OIDC: &str = "oidc";
const EVENT_IAM_STATE: &str = "iam_state";
const EVENT_OIDC_STATE: &str = "oidc_state";
pub mod cache;
pub mod error;
@@ -37,11 +43,23 @@ static OIDC_SYS: OnceLock<Arc<OidcSys>> = OnceLock::new();
#[instrument(skip(ecstore))]
pub async fn init_iam_sys(ecstore: Arc<ECStore>) -> Result<()> {
if IAM_SYS.get().is_some() {
info!("IAM system already initialized, skipping.");
info!(
event = EVENT_IAM_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "already_initialized",
"IAM runtime already initialized"
);
return Ok(());
}
info!("Starting IAM system initialization sequence...");
info!(
event = EVENT_IAM_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "starting",
"IAM runtime starting"
);
// 1. Create the persistent storage adapter
let storage_adapter = ObjectStore::new(ecstore);
@@ -55,11 +73,23 @@ pub async fn init_iam_sys(ecstore: Arc<ECStore>) -> Result<()> {
// 4. Securely set the global singleton
if IAM_SYS.set(iam_instance).is_err() {
error!("Critical: Race condition detected during IAM initialization!");
error!(
event = EVENT_IAM_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "singleton_set_failed",
"IAM runtime singleton set failed"
);
return Err(Error::IamSysAlreadyInitialized);
}
info!("IAM system initialization completed successfully.");
info!(
event = EVENT_IAM_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "ready",
"IAM runtime ready"
);
Ok(())
}
@@ -84,29 +114,67 @@ pub fn get_global_iam_sys() -> Option<Arc<IamSys<ObjectStore>>> {
/// Initialize the global OIDC system. Non-fatal if no OIDC providers are configured.
pub async fn init_oidc_sys() -> Result<()> {
if OIDC_SYS.get().is_some() {
info!("OIDC system already initialized, skipping.");
debug!(
event = EVENT_OIDC_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_OIDC,
state = "already_initialized",
"OIDC runtime already initialized"
);
return Ok(());
}
info!("Starting OIDC system initialization...");
debug!(
event = EVENT_OIDC_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_OIDC,
state = "starting",
"OIDC runtime starting"
);
let oidc_sys = match OidcSys::new().await {
Ok(sys) => {
if sys.has_providers() {
info!("OIDC system initialized with {} provider(s)", sys.list_providers().len());
debug!(
event = EVENT_OIDC_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_OIDC,
provider_count = sys.list_providers().len(),
state = "ready",
"OIDC runtime ready"
);
} else {
info!("No OIDC providers configured");
debug!(
event = EVENT_OIDC_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_OIDC,
state = "empty",
"OIDC runtime has no providers"
);
}
sys
}
Err(e) => {
warn!("OIDC initialization failed (non-fatal): {}", e);
warn!(
event = EVENT_OIDC_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_OIDC,
state = "init_failed_non_fatal",
error = %e,
"OIDC runtime initialization failed"
);
OidcSys::empty().map_err(Error::StringError)?
}
};
if OIDC_SYS.set(Arc::new(oidc_sys)).is_err() {
warn!("Race condition during OIDC initialization (non-fatal)");
warn!(
event = EVENT_OIDC_STATE,
component = LOG_COMPONENT_IAM,
subsystem = LOG_SUBSYSTEM_OIDC,
state = "singleton_set_race",
"OIDC runtime singleton set raced"
);
}
Ok(())
+14 -10
View File
@@ -54,7 +54,7 @@ use tokio::{
},
};
use tracing::warn;
use tracing::{error, info};
use tracing::{debug, error};
const IAM_FORMAT_FILE: &str = "format.json";
const IAM_FORMAT_VERSION_1: i32 = 1;
@@ -153,10 +153,14 @@ where
if let Err(e) = self.clone().load().await {
if attempt == MAX_RETRIES - 1 {
self.state.store(IamState::Error as u8, Ordering::SeqCst);
warn!("IAM failed to load initial data after {} attempts: {:?}", MAX_RETRIES, e);
warn!(
attempts = MAX_RETRIES,
error = ?e,
"IAM initial load failed"
);
load_error = Some(e);
} else {
warn!("IAM load failed, retrying... attempt {}", attempt + 1);
warn!(attempt = attempt + 1, max_attempts = MAX_RETRIES, "IAM load retry scheduled");
tokio::time::sleep(INITIAL_LOAD_RETRY_DELAY).await;
}
} else {
@@ -169,7 +173,7 @@ where
}
self.state.store(IamState::Ready as u8, Ordering::SeqCst);
info!("IAM System successfully initialized and marked as READY");
debug!(state = "ready", "IAM manager ready");
// Background ticker for synchronization
// Check if environment variable is set
@@ -185,20 +189,20 @@ where
loop {
select! {
_ = ticker.tick() => {
info!("iam load ticker");
debug!(source = "ticker", "IAM reload tick");
if let Err(err) =s.clone().load().await{
warn!("iam load err {:?}", err);
warn!(source = "ticker", error = ?err, "IAM reload failed");
}
},
i = receiver.recv() => {
info!("iam load receiver");
debug!(source = "receiver", "IAM reload signal received");
match i {
Some(t) => {
let last = s.last_timestamp.load(Ordering::Relaxed);
if last <= t {
info!("iam load receiver load");
debug!(source = "receiver", "IAM reload accepted");
if let Err(err) =s.clone().load().await{
warn!("iam load err {:?}", err);
warn!(source = "receiver", error = ?err, "IAM reload failed");
}
ticker.reset();
}
@@ -1294,7 +1298,7 @@ where
let cache = self.cache.snapshot();
let users = Arc::clone(&cache.users);
if let Some(x) = users.get(access_key) {
warn!("user already exists: {:?}", x);
warn!(error = ?x, "IAM user already exists");
if x.credentials.is_temp() {
return Err(Error::IAMActionNotAllowed);
}
+5 -5
View File
@@ -40,7 +40,7 @@ use std::pin::Pin;
use std::sync::{LazyLock, Mutex, MutexGuard, RwLock};
use std::time::{Duration as StdDuration, Instant};
use tokio::time::sleep;
use tracing::{error, info, warn};
use tracing::{debug, error, warn};
use url::Url;
const OIDC_JWKS_REFRESH_INTERVAL: StdDuration = StdDuration::from_secs(24 * 60 * 60);
@@ -76,7 +76,7 @@ fn lock_oidc_plugin_authn_metrics<'a, T>(mutex: &'a Mutex<T>, metric: &'static s
match mutex.lock() {
Ok(guard) => guard,
Err(err) => {
warn!("recovering poisoned OIDC plugin authn metrics lock: {}", metric);
warn!(metric, "Recovering poisoned OIDC authn metrics lock");
err.into_inner()
}
}
@@ -417,18 +417,18 @@ impl OidcSys {
for sourced_config in parsed_configs {
let config = sourced_config.config;
if !config.enabled {
info!("OIDC provider '{}' is disabled, skipping", config.id);
debug!(provider = %config.id, "OIDC provider disabled");
continue;
}
match Self::discover_provider(&config, &http_client).await {
Ok(state) => {
info!("OIDC provider '{}' discovered successfully", config.id);
debug!(provider = %config.id, "OIDC provider discovered");
provider_states.insert(config.id.clone(), state);
configs.insert(config.id.clone(), config);
}
Err(e) => {
error!("Failed to discover OIDC provider '{}': {}", config.id, e);
error!(provider = %config.id, error = %e, "OIDC provider discovery failed");
}
}
}
+31 -33
View File
@@ -41,7 +41,7 @@ use std::time::{Duration, Instant};
use std::{collections::HashMap, sync::Arc};
use tokio::sync::mpsc::{self, Sender};
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn};
use tracing::{debug, error, warn};
pub static IAM_CONFIG_PREFIX: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_CONFIG_PREFIX}/iam"));
pub static IAM_CONFIG_USERS_PREFIX: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_CONFIG_PREFIX}/iam/users/"));
@@ -307,11 +307,11 @@ impl ObjectStore {
}
Err(StorageError::PreconditionFailed) => {
Self::complete_lazy_rewrite(path.as_str(), false);
debug!("iam lazy rewrite skipped due to stale etag, path: {}", path);
debug!(path = %path, state = "stale_etag", "IAM lazy rewrite skipped");
}
Err(err) => {
Self::complete_lazy_rewrite(path.as_str(), false);
warn!("iam lazy rewrite failed, path: {}, err: {}", path, err);
warn!(path = %path, error = %err, state = "rewrite_failed", "IAM lazy rewrite failed");
}
}
});
@@ -377,7 +377,7 @@ impl ObjectStore {
bucket = Self::BUCKET_NAME,
prefix = %path,
error = %err,
"system path walk failed"
"IAM config walk failed"
);
let _ = sender_on_error
.send(StringOrErr {
@@ -643,7 +643,7 @@ impl Store for ObjectStore {
let outcome = match Self::decrypt_data_with_source(&data) {
Ok(v) => v,
Err(err) => {
warn!("config decrypt failed, keeping file: {}, path: {}", err, path_ref);
warn!(path = %path_ref, error = %err, "IAM config decrypt failed; keeping file");
// keep the config file when decrypt failed - do not delete
return Err(Error::ConfigNotFound);
}
@@ -692,7 +692,7 @@ impl Store for ObjectStore {
tokio::time::sleep(std::time::Duration::from_millis(wait_ms)).await;
}
Err(e) => {
error!("Final failure saving IAM config to {}: {:?}", path_ref, e);
error!(path = %path_ref, error = ?e, "IAM config save failed");
return Err(e.into());
}
}
@@ -717,7 +717,7 @@ impl Store for ObjectStore {
debug!("Saving IAM identity to path: {}", path);
self.save_iam_config(user_identity, path).await.map_err(|e| {
error!("ObjectStore save failure for {}: {:?}", name, e);
error!(name, error = ?e, "IAM identity save failed");
e
})
}
@@ -739,10 +739,10 @@ impl Store for ObjectStore {
.await
.map_err(|err| {
if is_err_config_not_found(&err) {
warn!("load_user_identity failed: no such user, name: {name}, user_type: {user_type:?}");
warn!(name, user_type = ?user_type, "IAM user identity missing");
Error::NoSuchUser(name.to_owned())
} else {
warn!("load_user_identity failed: {err:?}, name: {name}, user_type: {user_type:?}");
warn!(name, user_type = ?user_type, error = ?err, "IAM user identity load failed");
err
}
})?;
@@ -750,9 +750,7 @@ impl Store for ObjectStore {
if u.credentials.is_expired() {
let _ = self.delete_iam_config(get_user_identity_path(name, user_type)).await;
let _ = self.delete_iam_config(get_mapped_policy_path(name, user_type, false)).await;
warn!(
"load_user_identity failed: user is expired, delete the user and mapped policy, name: {name}, user_type: {user_type:?}"
);
warn!(name, user_type = ?user_type, "IAM user identity expired and was removed");
return Err(Error::NoSuchUser(name.to_owned()));
}
@@ -776,7 +774,7 @@ impl Store for ObjectStore {
let _ = self.delete_iam_config(get_user_identity_path(name, user_type)).await;
let _ = self.delete_iam_config(get_mapped_policy_path(name, user_type, false)).await;
}
warn!("extract_jwt_claims failed: {err:?}, name: {name}, user_type: {user_type:?}");
warn!(name, user_type = ?user_type, error = ?err, "IAM JWT claim extraction failed");
return Err(Error::NoSuchUser(name.to_owned()));
}
}
@@ -804,7 +802,7 @@ impl Store for ObjectStore {
while let Some(v) = rx.recv().await {
if let Some(err) = v.err {
warn!("list_iam_config_items {:?}", err);
warn!(error = ?err, "IAM config item listing failed");
let _ = ctx.cancel();
return Err(err);
@@ -866,7 +864,7 @@ impl Store for ObjectStore {
while let Some(v) = rx.recv().await {
if let Some(err) = v.err {
warn!("list_iam_config_items {:?}", err);
warn!(error = ?err, "IAM config item listing failed");
let _ = ctx.cancel();
return Err(err);
@@ -934,7 +932,7 @@ impl Store for ObjectStore {
while let Some(v) = rx.recv().await {
if let Some(err) = v.err {
warn!("list_iam_config_items {:?}", err);
warn!(error = ?err, "IAM config item listing failed");
let _ = ctx.cancel();
return Err(err);
@@ -1009,7 +1007,7 @@ impl Store for ObjectStore {
while let Some(v) = rx.recv().await {
if let Some(err) = v.err {
warn!("list_iam_config_items {:?}", err);
warn!(error = ?err, "IAM config item listing failed");
let _ = ctx.cancel();
return Err(err);
@@ -1044,7 +1042,7 @@ impl Store for ObjectStore {
let policy_name = rustfs_utils::path::dir(&policies_list[idx]);
info!("load policy: {}", policy_name);
debug!(policy = %policy_name, "IAM policy loaded");
policy_docs_cache.insert(policy_name, p);
}
@@ -1059,7 +1057,7 @@ impl Store for ObjectStore {
}
let policy_name = rustfs_utils::path::dir(&policies_list[idx]);
info!("load policy: {}", policy_name);
debug!(policy = %policy_name, "IAM policy loaded");
policy_docs_cache.insert(policy_name, p);
}
@@ -1083,7 +1081,7 @@ impl Store for ObjectStore {
}
let name = rustfs_utils::path::dir(&item_name_list[idx]);
info!("load reg user: {}", name);
debug!(user = %name, "IAM regular user loaded");
user_items_cache.insert(name, p);
}
break;
@@ -1097,7 +1095,7 @@ impl Store for ObjectStore {
}
let name = rustfs_utils::path::dir(&item_name_list[idx]);
info!("load reg user: {}", name);
debug!(user = %name, "IAM regular user loaded");
user_items_cache.insert(name, p);
}
@@ -1112,7 +1110,7 @@ impl Store for ObjectStore {
for item in item_name_list.iter() {
let name = rustfs_utils::path::dir(item);
info!("load group: {}", name);
debug!(group = %name, "IAM group loaded");
if let Err(err) = self.load_group(&name, &mut items_cache).await {
return Err(Error::other(format!("load group failed: {err}")));
};
@@ -1140,7 +1138,7 @@ impl Store for ObjectStore {
}
let name = item_name_list[idx].trim_end_matches(".json").to_owned();
info!("load user policy: {}", name);
debug!(user = %name, "IAM user policy loaded");
items_cache.insert(name, p);
}
break;
@@ -1156,7 +1154,7 @@ impl Store for ObjectStore {
}
let name = item_name_list[idx].trim_end_matches(".json").to_owned();
info!("load user policy: {}", name);
debug!(user = %name, "IAM user policy loaded");
items_cache.insert(name, p);
}
@@ -1174,7 +1172,7 @@ impl Store for ObjectStore {
for item in item_name_list.iter() {
let name = item.trim_end_matches(".json");
info!("load group policy: {}", name);
debug!(group = %name, "IAM group policy loaded");
if let Err(err) = self.load_mapped_policy(name, UserType::Reg, true, &mut items_cache).await
&& !is_err_no_such_policy(&err)
{
@@ -1193,7 +1191,7 @@ impl Store for ObjectStore {
for item in item_name_list.iter() {
let name = rustfs_utils::path::dir(item);
info!("load svc user: {}", name);
debug!(user = %name, "IAM service user loaded");
if let Err(err) = self.load_user(&name, UserType::Svc, &mut items_cache).await
&& !is_err_no_such_user(&err)
{
@@ -1204,7 +1202,7 @@ impl Store for ObjectStore {
for (_, v) in items_cache.iter() {
let parent = v.credentials.parent_user.clone();
if !user_items_cache.contains_key(&parent) {
info!("load sts user policy: {}", parent);
debug!(user = %parent, "IAM STS parent policy loaded");
if let Err(err) = self
.load_mapped_policy(&parent, UserType::Sts, false, &mut sts_policies_cache)
.await
@@ -1223,12 +1221,12 @@ impl Store for ObjectStore {
// sts users
if let Some(item_name_list) = listed_config_items.get(STS_LIST_KEY) {
for item in item_name_list.iter() {
info!("load sts user path: {}", item);
debug!(path = %item, "IAM STS user path discovered");
let name = rustfs_utils::path::dir(item);
info!("load sts user: {}", name);
debug!(user = %name, "IAM STS user loaded");
if let Err(err) = self.load_user(&name, UserType::Sts, &mut sts_items_cache).await {
info!("load sts user failed: {}", err);
debug!(user = %name, error = %err, "IAM STS user load failed");
};
}
}
@@ -1237,12 +1235,12 @@ impl Store for ObjectStore {
if let Some(item_name_list) = listed_config_items.get(POLICY_DB_STS_USERS_LIST_KEY) {
for item in item_name_list.iter() {
let name = item.trim_end_matches(".json");
info!("load sts user policy: {}", name);
debug!(user = %name, "IAM STS user policy loaded");
if let Err(err) = self
.load_mapped_policy(name, UserType::Sts, false, &mut sts_policies_cache)
.await
{
info!("load sts user policy failed: {}", err);
debug!(user = %name, error = %err, "IAM STS user policy load failed");
};
}
}
@@ -1264,7 +1262,7 @@ impl Store for ObjectStore {
cache.replace_sts_policies(sts_policies_cache);
cache.build_user_group_memberships();
} else {
warn!("skip IAM full reload cache commit because one or more IAM caches changed during reload");
warn!("IAM full reload cache commit skipped due to concurrent cache changes");
}
});
+12 -12
View File
@@ -34,7 +34,7 @@ use std::path::PathBuf;
use std::time::Duration;
use tokio::fs;
use tokio::sync::RwLock;
use tracing::{debug, info, warn};
use tracing::{debug, warn};
/// Local KMS client that stores keys in local files
pub struct LocalKmsClient {
@@ -74,7 +74,7 @@ impl LocalKmsClient {
// Create key directory if it doesn't exist
if !config.key_dir.exists() {
fs::create_dir_all(&config.key_dir).await?;
info!("Created KMS key directory: {:?}", config.key_dir);
debug!(path = ?config.key_dir, "KMS key directory created");
}
// Initialize master cipher if master key is provided
@@ -219,7 +219,7 @@ impl LocalKmsClient {
fs::rename(&temp_path, &key_path).await?;
info!("Saved master key {} to {:?}", master_key.key_id, key_path);
debug!(key_id = %master_key.key_id, path = ?key_path, "Local KMS master key saved");
Ok(())
}
@@ -278,7 +278,7 @@ impl KmsClient for LocalKmsClient {
let data_key = DataKeyInfo::new(envelope.key_id, 1, Some(plaintext_key), ciphertext, request.key_spec.clone());
info!("Generated data key for master key: {}", request.master_key_id);
debug!(key_id = %request.master_key_id, "Local KMS data key generated");
Ok(data_key)
}
@@ -334,7 +334,7 @@ impl KmsClient for LocalKmsClient {
.decrypt_with_master_key(&envelope.master_key_id, &envelope.encrypted_key, &envelope.nonce)
.await?;
info!("Successfully decrypted data");
debug!("Local KMS data decrypted");
Ok(plaintext)
}
@@ -367,7 +367,7 @@ impl KmsClient for LocalKmsClient {
let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key.clone());
info!("Created master key: {}", key_id);
debug!(key_id, "Local KMS master key created");
Ok(master_key)
}
@@ -453,7 +453,7 @@ impl KmsClient for LocalKmsClient {
let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key);
info!("Enabled key: {}", key_id);
debug!(key_id, "Local KMS key enabled");
Ok(())
}
@@ -470,7 +470,7 @@ impl KmsClient for LocalKmsClient {
let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key);
info!("Disabled key: {}", key_id);
debug!(key_id, "Local KMS key disabled");
Ok(())
}
@@ -492,7 +492,7 @@ impl KmsClient for LocalKmsClient {
let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key);
warn!("Scheduled key deletion: {}", key_id);
debug!(key_id, "Local KMS key deletion scheduled");
Ok(())
}
@@ -509,7 +509,7 @@ impl KmsClient for LocalKmsClient {
let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key);
info!("Canceled deletion for key: {}", key_id);
debug!(key_id, "Local KMS key deletion canceled");
Ok(())
}
@@ -528,7 +528,7 @@ impl KmsClient for LocalKmsClient {
let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key.clone());
info!("Rotated key: {}", key_id);
debug!(key_id, "Local KMS key rotated");
Ok(master_key)
}
@@ -721,7 +721,7 @@ impl KmsBackend for LocalKmsBackend {
let mut cache = self.client.key_cache.write().await;
cache.remove(key_id);
info!("Immediately deleted key: {}", key_id);
debug!(key_id, "Local KMS key deleted immediately");
// Return success response for immediate deletion
let key_metadata = KeyMetadata {
+13 -13
View File
@@ -98,7 +98,7 @@ impl VaultKmsClient {
let client =
VaultClient::new(settings).map_err(|e| KmsError::backend_error(format!("Failed to create Vault client: {e}")))?;
info!("Successfully connected to Vault at {}", config.address);
info!(address = %config.address, "Vault KMS backend connected");
Ok(Self {
client,
@@ -136,7 +136,7 @@ impl VaultKmsClient {
// If encrypted_key_material is empty, generate and store it (fix for old keys)
if key_data.encrypted_key_material.is_empty() {
warn!("Key {} has empty encrypted_key_material, generating and storing new key material", key_id);
warn!(key_id, "Vault KMS key material missing; regenerating");
let key_material = generate_key_material(&key_data.algorithm)?;
key_data.encrypted_key_material = self.encrypt_key_material(&key_material).await?;
// Store the updated key data back to Vault
@@ -147,7 +147,7 @@ impl VaultKmsClient {
let key_material = match self.decrypt_key_material(&key_data.encrypted_key_material).await {
Ok(km) => km,
Err(e) => {
warn!("Failed to decrypt key material for key {}: {}, generating new key material", key_id, e);
warn!(key_id, error = %e, "Vault KMS key material decrypt failed; regenerating");
let new_key_material = generate_key_material(&key_data.algorithm)?;
key_data.encrypted_key_material = self.encrypt_key_material(&new_key_material).await?;
// Store the updated key data back to Vault
@@ -210,7 +210,7 @@ impl VaultKmsClient {
// If encrypted_key_material is empty, generate it (this handles the case where
// an old key was created without proper key material)
if existing_key_data.encrypted_key_material.is_empty() {
warn!("Key {} has empty encrypted_key_material, generating new key material", key_id);
warn!(key_id, "Vault KMS key metadata missing encrypted key material");
let key_material = generate_key_material(&existing_key_data.algorithm)?;
existing_key_data.encrypted_key_material = self.encrypt_key_material(&key_material).await?;
}
@@ -316,7 +316,7 @@ impl KmsClient for VaultKmsClient {
let data_key = DataKeyInfo::new(envelope.key_id, 1, Some(plaintext_key), ciphertext, request.key_spec.clone());
info!("Generated data key for master key: {}", request.master_key_id);
debug!(key_id = %request.master_key_id, "Vault KMS data key generated");
Ok(data_key)
}
@@ -373,7 +373,7 @@ impl KmsClient for VaultKmsClient {
.decrypt_with_master_key(&envelope.master_key_id, &envelope.encrypted_key, &envelope.nonce)
.await?;
info!("Successfully decrypted data");
debug!("Vault KMS data decrypted");
Ok(plaintext)
}
@@ -418,7 +418,7 @@ impl KmsClient for VaultKmsClient {
created_by: None,
};
info!("Successfully created master key: {}", key_id);
debug!(key_id, "Vault KMS master key created");
Ok(master_key)
}
@@ -486,7 +486,7 @@ impl KmsClient for VaultKmsClient {
key_data.status = KeyStatus::Active;
self.store_key_data(key_id, &key_data).await?;
info!("Enabled key: {}", key_id);
debug!(key_id, "Vault KMS key enabled");
Ok(())
}
@@ -497,7 +497,7 @@ impl KmsClient for VaultKmsClient {
key_data.status = KeyStatus::Disabled;
self.store_key_data(key_id, &key_data).await?;
info!("Disabled key: {}", key_id);
debug!(key_id, "Vault KMS key disabled");
Ok(())
}
@@ -513,7 +513,7 @@ impl KmsClient for VaultKmsClient {
key_data.status = KeyStatus::PendingDeletion;
self.store_key_data(key_id, &key_data).await?;
info!("Scheduled key deletion: {}", key_id);
debug!(key_id, "Vault KMS key deletion scheduled");
Ok(())
}
@@ -524,7 +524,7 @@ impl KmsClient for VaultKmsClient {
key_data.status = KeyStatus::Active;
self.store_key_data(key_id, &key_data).await?;
info!("Canceled key deletion: {}", key_id);
debug!(key_id, "Vault KMS key deletion canceled");
Ok(())
}
@@ -553,7 +553,7 @@ impl KmsClient for VaultKmsClient {
created_by: None,
};
info!("Successfully rotated key: {}", key_id);
debug!(key_id, "Vault KMS key rotated");
Ok(master_key)
}
@@ -573,7 +573,7 @@ impl KmsClient for VaultKmsClient {
debug!("Vault health check passed - 404 error is expected when no keys exist yet");
Ok(())
} else {
warn!("Vault health check failed: {}", e);
warn!(error = %e, "Vault KMS health check failed");
Err(e)
}
}
+19 -7
View File
@@ -24,7 +24,7 @@ use rand::random;
use std::collections::HashMap;
use std::io::Cursor;
use tokio::io::{AsyncRead, AsyncReadExt};
use tracing::{debug, info};
use tracing::debug;
use zeroize::Zeroize;
/// Data key for object encryption
@@ -209,7 +209,7 @@ impl ObjectEncryptionService {
// Generate a unique random nonce for this data key
// This ensures each object/part gets a unique base nonce for streaming encryption
let nonce: [u8; 12] = random();
tracing::info!("Generated random nonce for data key: {:02x?}", nonce);
tracing::debug!("Generated random nonce for data key");
let data_key = DataKey {
plaintext_key: data_key_response
@@ -302,7 +302,7 @@ impl ObjectEncryptionService {
key_id: actual_key_id.to_string(),
};
if let Err(KmsError::KeyNotFound { .. }) = self.kms_manager.describe_key(describe_req).await {
info!("Auto-creating SSE-S3 key: {}", actual_key_id);
debug!(key_id = %actual_key_id, "Auto-creating SSE-S3 key");
let create_req = CreateKeyRequest {
key_name: Some(actual_key_id.to_string()),
key_usage: KeyUsage::EncryptDecrypt,
@@ -364,7 +364,13 @@ impl ObjectEncryptionService {
encrypted_data_key: data_key.ciphertext_blob,
};
info!("Successfully encrypted object {}/{} ({} bytes)", bucket, object_key, original_size);
debug!(
bucket,
object = object_key,
original_size,
algorithm = %algorithm.as_str(),
"Object encrypted"
);
Ok(EncryptionResult { ciphertext, metadata })
}
@@ -429,7 +435,13 @@ impl ObjectEncryptionService {
// Decrypt the data
let plaintext = cipher.decrypt(&ciphertext, &metadata.iv, tag, &aad)?;
info!("Successfully decrypted object {}/{} ({} bytes)", bucket, object_key, plaintext.len());
debug!(
bucket,
object = object_key,
plaintext_len = plaintext.len(),
algorithm = %metadata.algorithm,
"Object decrypted"
);
Ok(Box::new(Cursor::new(plaintext)))
}
@@ -507,7 +519,7 @@ impl ObjectEncryptionService {
encrypted_data_key: Vec::new(), // Empty for SSE-C
};
info!(
debug!(
"Successfully encrypted object {}/{} with SSE-C ({} bytes)",
bucket, object_key, original_size
);
@@ -568,7 +580,7 @@ impl ObjectEncryptionService {
// Decrypt the data
let plaintext = cipher.decrypt(&ciphertext, &metadata.iv, tag, &aad)?;
info!(
debug!(
"Successfully decrypted SSE-C object {}/{} ({} bytes)",
bucket,
object_key,
+61 -11
View File
@@ -25,7 +25,11 @@ use std::sync::{
atomic::{AtomicU64, Ordering},
};
use tokio::sync::{Mutex, RwLock};
use tracing::{error, info, warn};
use tracing::{debug, error, info, warn};
const LOG_COMPONENT_KMS: &str = "kms";
const LOG_SUBSYSTEM_SERVICE: &str = "service";
const EVENT_KMS_SERVICE_STATE: &str = "kms_service_state";
/// KMS service status
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
@@ -106,7 +110,13 @@ impl KmsServiceManager {
*status = KmsServiceStatus::Configured;
}
info!("KMS configuration updated successfully");
debug!(
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
state = "configured",
"KMS service configured"
);
Ok(())
}
@@ -132,7 +142,14 @@ impl KmsServiceManager {
}
};
info!("Starting KMS service with backend: {:?}", config.backend);
info!(
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
backend = ?config.backend,
state = "starting",
"KMS service starting"
);
match self.create_service_version(&config).await {
Ok(service_version) => {
@@ -146,7 +163,13 @@ impl KmsServiceManager {
*status = KmsServiceStatus::Running;
}
info!("KMS service started successfully");
debug!(
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
state = "running",
"KMS service running"
);
Ok(())
}
Err(e) => {
@@ -170,7 +193,13 @@ impl KmsServiceManager {
/// Internal stop implementation (called within lifecycle mutex)
async fn stop_internal(&self) -> Result<()> {
info!("Stopping KMS service");
debug!(
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
state = "stopping",
"KMS service stopping"
);
// Atomically clear current service version (lock-free, instant)
// Note: Existing Arc references will keep the service alive until operations complete
@@ -184,7 +213,13 @@ impl KmsServiceManager {
}
}
info!("KMS service stopped successfully (existing operations may continue)");
debug!(
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
state = "configured",
"KMS service stopped"
);
Ok(())
}
@@ -201,7 +236,13 @@ impl KmsServiceManager {
pub async fn reconfigure(&self, new_config: KmsConfig) -> Result<()> {
let _guard = self.lifecycle_mutex.lock().await;
info!("Reconfiguring KMS service (zero-downtime)");
debug!(
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
state = "reconfiguring",
"KMS service reconfiguring"
);
new_config.validate()?;
// Configure with new config
@@ -230,13 +271,22 @@ impl KmsServiceManager {
if let Some(old_ver) = old_version {
info!(
"KMS service reconfigured successfully: version {} -> {} (old service will be cleaned up when operations complete)",
old_ver, new_service_version.version
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
old_version = old_ver,
new_version = new_service_version.version,
state = "running",
"KMS service reconfigured"
);
} else {
info!(
"KMS service reconfigured successfully: version {} (service started)",
new_service_version.version
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
new_version = new_service_version.version,
state = "running",
"KMS service started from reconfigure"
);
}
Ok(())
+4 -4
View File
@@ -69,7 +69,7 @@ impl NotifyBucketConfigManager {
bucket = %bucket,
region = %cfg.region,
available_arn_count = arn_list.len(),
"Loaded available notify target ARNs for bucket config validation"
"notify bucket config validation"
);
if let Err(e) = cfg.validate(&cfg.region, &arn_list) {
@@ -81,7 +81,7 @@ impl NotifyBucketConfigManager {
region = %cfg.region,
error = %e,
result = "validation_failed",
"Bucket notification config validation failed"
"notify bucket config validation"
);
if !matches!(e, ParseConfigError::ArnNotFound(_)) {
return Err(NotificationError::BucketNotification(e.to_string()));
@@ -94,7 +94,7 @@ impl NotifyBucketConfigManager {
region = %cfg.region,
error = %e,
result = "missing_target_arn",
"Bucket notification config references missing target ARN; keeping compatibility and loading remaining rules"
"notify bucket config validation"
);
}
@@ -107,7 +107,7 @@ impl NotifyBucketConfigManager {
bucket = %bucket,
region = %cfg.region,
rule_count = cfg.get_rules_map().inner().len(),
"Loaded bucket notification config"
"notify bucket config state"
);
Ok(())
}
+15 -15
View File
@@ -91,7 +91,7 @@ impl NotifyConfigManager {
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
state = "initializing",
"Initializing notification system"
"notify runtime lifecycle"
);
let config = {
@@ -111,7 +111,7 @@ impl NotifyConfigManager {
subsystem = LOG_SUBSYSTEM_CONFIG,
state = "targets_created",
target_count = targets.len(),
"Created notification targets"
"notify runtime lifecycle"
);
if targets.is_empty() {
debug!(
@@ -121,7 +121,7 @@ impl NotifyConfigManager {
state = "idle",
reason = "no_targets_configured",
hint = %notify_configuration_hint(),
"Notification runtime has no configured targets"
"notify runtime lifecycle"
);
}
@@ -132,7 +132,7 @@ impl NotifyConfigManager {
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
state = "initialized",
"Initialized notification system"
"notify runtime lifecycle"
);
Ok(())
}
@@ -156,13 +156,13 @@ impl NotifyConfigManager {
if let Some(targets_of_type) = config.0.get_mut(&ttype) {
if targets_of_type.remove(&tname).is_some() {
info!(
event = EVENT_NOTIFY_CONFIG_UPDATE,
component = LOG_COMPONENT_NOTIFY,
event = EVENT_NOTIFY_CONFIG_UPDATE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
action = "remove_target",
target_id = %target_id,
result = "removed",
"Removed notification target from configuration"
"notify config update"
);
changed = true;
}
@@ -178,7 +178,7 @@ impl NotifyConfigManager {
action = "remove_target",
target_id = %target_id,
result = "not_found",
"Notification target not found in configuration"
"notify config update"
);
}
changed
@@ -246,7 +246,7 @@ impl NotifyConfigManager {
target_type = %target_type,
target_name = %target_name,
result = "not_found",
"Notification target configuration not found"
"notify config update"
);
}
debug!(
@@ -264,7 +264,7 @@ impl NotifyConfigManager {
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
state = "reloading",
"Reloading notification configuration"
"notify runtime lifecycle"
);
self.update_config(new_config.clone()).await;
@@ -281,7 +281,7 @@ impl NotifyConfigManager {
subsystem = LOG_SUBSYSTEM_CONFIG,
state = "targets_created",
target_count = targets.len(),
"Created notification targets from reloaded configuration"
"notify runtime lifecycle"
);
if targets.is_empty() {
debug!(
@@ -291,7 +291,7 @@ impl NotifyConfigManager {
state = "idle",
reason = "no_targets_configured",
hint = %notify_configuration_hint(),
"Notification runtime has no configured targets after reload"
"notify runtime lifecycle"
);
}
@@ -302,7 +302,7 @@ impl NotifyConfigManager {
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
state = "reloaded",
"Reloaded notification configuration"
"notify runtime lifecycle"
);
Ok(())
}
@@ -333,7 +333,7 @@ impl NotifyConfigManager {
subsystem = LOG_SUBSYSTEM_CONFIG,
action = "reload_if_changed",
result = "unchanged",
"Notification configuration unchanged; skipping reload"
"notify config update"
);
return Ok(());
}
@@ -348,7 +348,7 @@ impl NotifyConfigManager {
subsystem = LOG_SUBSYSTEM_CONFIG,
action = "reload_if_changed",
result = "updated",
"Notification configuration updated; reloading runtime"
"notify config update"
);
self.reload_config(new_config).await
}
+10 -1
View File
@@ -23,6 +23,9 @@ use std::sync::{Arc, OnceLock};
use tracing::error;
static NOTIFICATION_SYSTEM: OnceLock<Arc<NotificationSystem>> = OnceLock::new();
const LOG_COMPONENT_NOTIFY: &str = "notify";
const LOG_SUBSYSTEM_GLOBAL: &str = "global";
const EVENT_NOTIFY_GLOBAL_STATE: &str = "notify_global_state";
/// Initialize the global notification system with the given configuration.
/// This function should only be called once throughout the application life cycle.
@@ -102,7 +105,13 @@ pub mod notifier_global {
// If the notification system itself cannot be retrieved, it will be returned directly
Some(sys) => sys,
None => {
error!("Notification system is not initialized.");
error!(
event = EVENT_NOTIFY_GLOBAL_STATE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_GLOBAL,
state = "uninitialized",
"notify global state"
);
return;
}
};
+3 -3
View File
@@ -372,7 +372,7 @@ impl Drop for NotificationSystem {
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_INTEGRATION,
state = "dropping",
"Notification system instance is being dropped"
"notify system integration state"
);
let snapshot = self.snapshot_metrics();
@@ -394,7 +394,7 @@ impl Drop for NotificationSystem {
metric_name = name,
metric_value = value,
metric_kind = if is_gauge { "gauge" } else { "counter" },
"Notification shutdown metric snapshot"
"notify system integration state"
);
}
@@ -406,7 +406,7 @@ impl Drop for NotificationSystem {
subsystem = LOG_SUBSYSTEM_INTEGRATION,
status_key = %key,
status_value = %value,
"Notification system status snapshot"
"notify system integration state"
);
}
}
+6 -6
View File
@@ -96,7 +96,7 @@ impl EventNotifier {
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_DISPATCH,
state = "targets_cleared",
"Removed all notify targets"
"notify runtime lifecycle"
);
}
@@ -136,7 +136,7 @@ impl EventNotifier {
bucket = %bucket_name,
object = %object_key,
target_count = target_ids_len,
"Dispatching notify event"
"notify dispatch"
);
for target_id in target_ids {
// `get` now returns Option<Arc<dyn Target + Send + Sync>>
@@ -166,7 +166,7 @@ impl EventNotifier {
subsystem = LOG_SUBSYSTEM_DISPATCH,
target_id = %target_for_task.id(),
deferred = is_deferred,
"Prepared notify target dispatch"
"notify dispatch"
);
// Use cloned data in closures to avoid borrowing conflicts
// Create an EntityTarget from the event
@@ -217,7 +217,7 @@ impl EventNotifier {
subsystem = LOG_SUBSYSTEM_DISPATCH,
target_id = %target_name_for_task,
deferred = is_deferred,
"Completed notify target dispatch"
"notify dispatch"
);
}
});
@@ -256,7 +256,7 @@ impl EventNotifier {
subsystem = LOG_SUBSYSTEM_DISPATCH,
bucket = %bucket_name,
target_count = target_ids_len,
"Finished notify dispatch fan-out"
"notify dispatch"
);
}
@@ -284,7 +284,7 @@ impl EventNotifier {
subsystem = LOG_SUBSYSTEM_DISPATCH,
state = "targets_initialized",
target_count = target_list_guard.len(),
"Initialized notify runtime targets"
"notify runtime lifecycle"
);
Ok(())
}
+2 -2
View File
@@ -69,7 +69,7 @@ impl NotifyRuleEngine {
bucket = %bucket,
state = "updated",
event_count,
"Updated bucket notification rules"
"notify bucket rules state"
);
}
@@ -85,7 +85,7 @@ impl NotifyRuleEngine {
subsystem = LOG_SUBSYSTEM_RULE_ENGINE,
bucket = %bucket,
state = "removed",
"Removed bucket notification rules"
"notify bucket rules state"
);
}
}
+5 -5
View File
@@ -66,7 +66,7 @@ impl NotifyRuntimeFacade {
subsystem = LOG_SUBSYSTEM_RUNTIME,
target_id = %target_id,
state = "replay_started",
"Started notify replay worker"
"notify runtime lifecycle"
);
} else {
debug!(
@@ -76,7 +76,7 @@ impl NotifyRuntimeFacade {
target_id = %target_id,
state = "replay_skipped",
reason = "no_store_configured",
"Skipped notify replay worker startup"
"notify runtime lifecycle"
);
}
}),
@@ -122,7 +122,7 @@ impl NotifyRuntimeFacade {
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "stopping",
"Stopping notification runtime"
"notify runtime lifecycle"
);
let active_targets = self.replay_workers.read().await.len();
@@ -132,7 +132,7 @@ impl NotifyRuntimeFacade {
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "replay_stopping",
active_targets,
"Stopping notify replay workers"
"notify runtime lifecycle"
);
{
@@ -160,7 +160,7 @@ impl NotifyRuntimeFacade {
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "stopped",
"Stopped notification runtime"
"notify runtime lifecycle"
);
}
}
+2 -2
View File
@@ -203,7 +203,7 @@ impl Dial9SessionGuard {
component = LOG_COMPONENT_OBS,
subsystem = LOG_SUBSYSTEM_DIAL9,
state = "shutdown_requested",
"dial9 state changed"
"dial9 state"
);
// TelemetryGuard handles flushing automatically when dropped
}
@@ -219,7 +219,7 @@ impl Drop for Dial9SessionGuard {
component = LOG_COMPONENT_OBS,
subsystem = LOG_SUBSYSTEM_DIAL9,
state = "flushed",
"dial9 state changed"
"dial9 state"
);
}
}
+3 -3
View File
@@ -175,7 +175,7 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
output_format = "json",
logger_level,
is_production,
"local logging state changed"
"local logging state"
);
OtelGuard {
@@ -293,7 +293,7 @@ fn init_file_logging_internal(
stdout_mirror_enabled = stdout_guard.is_some(),
is_production,
logger_level,
"local logging state changed"
"local logging state"
);
Ok(OtelGuard {
@@ -482,7 +482,7 @@ pub fn spawn_cleanup_task(
zstd_level,
zstd_fallback_to_gzip,
zstd_workers,
"log cleaner state changed"
"log cleaner state"
);
tokio::spawn(async move {
+5 -5
View File
@@ -655,7 +655,7 @@ where
state = "requested",
username = %masked_username,
path = %path_str,
"ftps object delete state changed"
"FTPS delete requested"
);
let (bucket, key) = self
@@ -749,7 +749,7 @@ where
username = %masked_username,
path = %path_str,
bucket = %bucket,
"ftps directory state changed"
"FTPS directory created"
);
Ok(())
}
@@ -793,7 +793,7 @@ where
state = "removed",
path = %path_str,
bucket = %bucket,
"ftps directory state changed"
"FTPS directory removed"
);
Ok(())
}
@@ -807,7 +807,7 @@ where
subsystem = LOG_SUBSYSTEM_FTPS_DRIVER,
state = "already_removed",
bucket = %bucket,
"ftps directory state changed"
"FTPS directory already removed"
);
Ok(())
} else {
@@ -877,7 +877,7 @@ where
username = %MaskedAccessKey(&user.username),
from = %from_str,
to = %to_str,
"ftps rename state changed"
"FTPS rename unsupported"
);
Err(Error::new(
+24 -24
View File
@@ -107,7 +107,7 @@ where
bind_addr = %self.config.bind_addr,
tls_enabled = self.config.tls_enabled,
ftps_required = self.config.ftps_required,
"ftps server state changed"
"FTPS server starting"
);
let (reload_shutdown_tx, reload_shutdown_rx) = watch::channel(false);
@@ -121,14 +121,14 @@ where
// Configure passive ports for data connections
if let Some(passive_ports) = &self.config.passive_ports {
let range = self.config.parse_passive_ports()?;
info!(
debug!(
event = EVENT_FTPS_CONFIG_STATE,
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
state = "passive_ports_configured",
passive_ports = %passive_ports,
passive_port_range = ?range,
"ftps config state changed"
"FTPS passive ports configured"
);
server_builder = server_builder.passive_ports(range);
} else {
@@ -137,19 +137,19 @@ where
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
result = "system_assigned_passive_ports",
"ftps config state changed"
"FTPS passive ports defaulted"
);
}
// Configure external IP address for passive mode
if let Some(ref external_ip) = self.config.external_ip {
info!(
debug!(
event = EVENT_FTPS_CONFIG_STATE,
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
state = "external_ip_configured",
external_ip = %external_ip,
"ftps config state changed"
"FTPS external IP configured"
);
server_builder = server_builder.passive_host(external_ip.as_str());
}
@@ -157,13 +157,13 @@ where
// Configure both active and passive mode support
use libunftp::options::ActivePassiveMode;
server_builder = server_builder.active_passive_mode(ActivePassiveMode::ActiveAndPassive);
info!(
debug!(
event = EVENT_FTPS_CONFIG_STATE,
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
state = "active_passive_mode_enabled",
mode = "active_and_passive",
"ftps config state changed"
"FTPS active/passive mode configured"
);
// Configure FTPS / TLS
@@ -175,7 +175,7 @@ where
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
state = "enabled",
cert_dir = %cert_dir,
"ftps tls state changed"
"FTPS TLS enabled"
);
let resolver = ReloadableServerCertResolver::load_from_directory(cert_dir)
@@ -197,12 +197,12 @@ where
server_builder = server_builder.ftps_manual::<std::path::PathBuf>(Arc::new(server_config));
if self.config.ftps_required {
info!(
debug!(
event = EVENT_FTPS_TLS_STATE,
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
state = "required",
"ftps tls state changed"
"FTPS TLS required"
);
server_builder = server_builder.ftps_required(FtpsRequired::All, FtpsRequired::All);
}
@@ -218,7 +218,7 @@ where
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
state = "disabled",
mode = "plain_ftp",
"ftps tls state changed"
"FTPS TLS disabled"
);
}
@@ -248,13 +248,13 @@ where
let _ = reload_shutdown_tx.send(true);
match result {
Ok(Ok(())) => {
info!(
debug!(
event = EVENT_FTPS_SERVER_STATE,
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
state = "stopped",
result = "ok",
"ftps server state changed"
"FTPS server stopped"
);
Ok(())
}
@@ -288,7 +288,7 @@ where
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
state = "shutdown_requested",
"ftps server state changed"
"FTPS shutdown requested"
);
let _ = reload_shutdown_tx.send(true);
// libunftp listen() is not easily cancellable gracefully without dropping the future.
@@ -330,7 +330,7 @@ impl UserDetailProvider for FtpsUserDetailProvider {
result = "iam_unavailable",
phase = "user_detail",
error = %e,
"ftps auth state changed"
"FTPS user-detail IAM unavailable"
);
UserDetailError::ImplPropagated("Internal authentication service unavailable".to_string(), Some(Box::new(e)))
})?;
@@ -344,7 +344,7 @@ impl UserDetailProvider for FtpsUserDetailProvider {
phase = "user_detail",
username = %masked_username,
error = %e,
"ftps auth state changed"
"FTPS user-detail key check failed"
);
UserDetailError::ImplPropagated("Authentication verification failed".to_string(), Some(Box::new(e)))
})?;
@@ -357,7 +357,7 @@ impl UserDetailProvider for FtpsUserDetailProvider {
result = "identity_missing",
phase = "user_detail",
username = %masked_username,
"ftps auth state changed"
"FTPS user-detail identity missing"
);
UserDetailError::UserNotFound {
username: principal.username.clone(),
@@ -406,7 +406,7 @@ impl Authenticator for FtpsAuthenticator {
result = "iam_unavailable",
phase = "authenticate",
error = %e,
"ftps auth state changed"
"FTPS auth IAM unavailable"
);
AuthenticationError::ImplPropagated("Internal authentication service unavailable".to_string(), Some(Box::new(e)))
})?;
@@ -433,7 +433,7 @@ impl Authenticator for FtpsAuthenticator {
phase = "authenticate",
username = %masked_username,
error = %e,
"ftps auth state changed"
"FTPS auth key check failed"
);
AuthenticationError::ImplPropagated("Authentication verification failed".to_string(), Some(Box::new(e)))
})?;
@@ -446,7 +446,7 @@ impl Authenticator for FtpsAuthenticator {
result = "invalid_access_key",
phase = "authenticate",
username = %masked_username,
"ftps auth state changed"
"FTPS auth rejected access key"
);
return Err(AuthenticationError::BadUser);
}
@@ -459,7 +459,7 @@ impl Authenticator for FtpsAuthenticator {
result = "identity_missing",
phase = "authenticate",
username = %masked_username,
"ftps auth state changed"
"FTPS auth identity missing"
);
AuthenticationError::BadUser
})?;
@@ -472,7 +472,7 @@ impl Authenticator for FtpsAuthenticator {
result = "invalid_secret_key",
phase = "authenticate",
username = %masked_username,
"ftps auth state changed"
"FTPS auth rejected secret key"
);
return Err(AuthenticationError::BadPassword);
}
@@ -484,7 +484,7 @@ impl Authenticator for FtpsAuthenticator {
result = "authenticated",
phase = "authenticate",
username = %masked_username,
"ftps auth state changed"
"FTPS auth accepted"
);
Ok(Principal {
username: username.to_string(),
+2 -2
View File
@@ -188,7 +188,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
peer = %self.session_context.source_ip,
user = %MaskedAccessKey(&self.session_context.principal.user_identity.credentials.access_key),
result = "read_only_rejected",
"sftp driver state changed"
"SFTP write rejected by read-only mode"
);
return Err(SftpError::code(StatusCode::PermissionDenied));
}
@@ -250,7 +250,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
op = op,
timeout_secs = self.backend_op_timeout_secs,
result = "timeout",
"sftp backend state changed"
"SFTP backend operation timed out"
);
Err(SftpError::code(StatusCode::Failure))
}
+8 -8
View File
@@ -255,7 +255,7 @@ fn spawn_host_key_reload_loop(config: SftpConfig, holder: Arc<SshConfigHolder>,
state = "enabled",
host_key_dir = %config.host_key_dir.display(),
interval_secs,
"sftp host key reload state changed"
"SFTP host-key reload enabled"
);
tokio::spawn(async move {
@@ -265,13 +265,13 @@ fn spawn_host_key_reload_loop(config: SftpConfig, holder: Arc<SshConfigHolder>,
loop {
tokio::select! {
_ = shutdown_token.cancelled() => {
info!(
debug!(
event = EVENT_SFTP_HOST_KEY_RELOAD_STATE,
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_SFTP_SERVER,
state = "stopped",
host_key_dir = %config.host_key_dir.display(),
"sftp host key reload state changed"
"SFTP host-key reload stopped"
);
break;
}
@@ -280,14 +280,14 @@ fn spawn_host_key_reload_loop(config: SftpConfig, holder: Arc<SshConfigHolder>,
match holder.reload_from_config(&config).await {
Ok(Some(host_key_count)) => {
info!(
debug!(
event = EVENT_SFTP_HOST_KEY_RELOAD_STATE,
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_SFTP_SERVER,
result = "reloaded",
host_key_dir = %config.host_key_dir.display(),
host_key_count,
"sftp host key reload state changed"
"SFTP host keys reloaded"
);
}
Ok(None) => {
@@ -297,7 +297,7 @@ fn spawn_host_key_reload_loop(config: SftpConfig, holder: Arc<SshConfigHolder>,
subsystem = LOG_SUBSYSTEM_SFTP_SERVER,
result = "unchanged",
host_key_dir = %config.host_key_dir.display(),
"sftp host key reload state changed"
"SFTP host keys unchanged"
);
}
Err(err) => {
@@ -308,7 +308,7 @@ fn spawn_host_key_reload_loop(config: SftpConfig, holder: Arc<SshConfigHolder>,
result = "reload_failed",
host_key_dir = %config.host_key_dir.display(),
err = %err,
"sftp host key reload state changed"
"SFTP host-key reload failed"
);
}
}
@@ -398,7 +398,7 @@ where
state = "listening",
bind_addr = %self.config.bind_addr,
read_only = self.config.read_only,
"sftp server state changed"
"SFTP server listening"
);
let mut sessions: JoinSet<()> = JoinSet::new();
+15 -15
View File
@@ -75,7 +75,7 @@ pub(super) fn write_dispatch_byte_count(phase: &WritePhase, part_size: u64, satu
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_SFTP_WRITE,
result = "byte_count_overflow",
"sftp write state changed"
"SFTP write byte-count overflow"
);
Err(SftpError::code(StatusCode::Failure))
}
@@ -90,7 +90,7 @@ pub(super) fn write_dispatch_byte_count(phase: &WritePhase, part_size: u64, satu
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_SFTP_WRITE,
result = "handle_poisoned",
"sftp write state changed"
"SFTP write handle poisoned"
);
Err(SftpError::code(StatusCode::Failure))
}
@@ -115,7 +115,7 @@ pub(super) fn write_dispatch_append_bytes(phase: &mut WritePhase, data: &[u8]) -
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_SFTP_WRITE,
result = "append_on_failed_handle",
"sftp write state changed"
"SFTP write append rejected on failed handle"
);
Err(SftpError::code(StatusCode::Failure))
}
@@ -368,7 +368,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
key = %sanitise_control_bytes(key),
attempt = attempt,
state = "retrying_put_object",
"sftp write state changed",
"SFTP put_object retry scheduled",
);
}
@@ -414,7 +414,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
bucket = %sanitise_control_bytes(bucket),
key = %sanitise_control_bytes(key),
result = "retry_loop_fell_through",
"sftp write state changed",
"SFTP put_object retry loop fell through",
);
Err(SftpError::code(StatusCode::Failure))
}
@@ -459,7 +459,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
upload_id = %upload_id,
part_number = part_number,
result = "etag_missing",
"sftp multipart state changed"
"SFTP multipart part missing etag"
);
SftpError::code(StatusCode::Failure)
})?;
@@ -543,7 +543,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
bucket = %sanitise_control_bytes(bucket),
key = %sanitise_control_bytes(key),
result = "upload_id_missing",
"sftp multipart state changed"
"SFTP multipart upload missing upload_id"
);
SftpError::code(StatusCode::Failure)
})?;
@@ -708,7 +708,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_SFTP_WRITE,
result = "buffering_phase_lost",
"sftp multipart state changed"
"SFTP multipart buffering phase lost"
);
return Err(SftpError::code(StatusCode::Failure));
}
@@ -772,7 +772,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
key = %key,
limit = S3_MAX_MULTIPART_PARTS,
result = "parts_limit_exceeded",
"sftp multipart state changed",
"SFTP multipart parts limit exceeded",
);
let upload_id_for_fail = upload_id.clone();
let abort_authorized_for_fail = *abort_authorized;
@@ -791,7 +791,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_SFTP_WRITE,
result = "flush_without_streaming",
"sftp multipart state changed"
"SFTP multipart flush requested without streaming state"
);
return Err(SftpError::code(StatusCode::Failure));
}
@@ -817,7 +817,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_SFTP_WRITE,
result = "post_upload_phase_missing",
"sftp multipart state changed"
"SFTP multipart post-upload phase missing"
);
Err(SftpError::code(StatusCode::Failure))
}
@@ -887,7 +887,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
err = ?abort_err,
context = context,
result = "abort_failed",
"sftp abort state changed",
"SFTP multipart abort failed",
);
}
} else {
@@ -901,7 +901,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
access_key = %MaskedAccessKey(self.access_key()),
context = context,
result = "abort_skipped_unauthorized",
"sftp abort state changed",
"SFTP multipart abort skipped by policy",
);
}
}
@@ -959,7 +959,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
upload_id = %upload_id,
limit = S3_MAX_MULTIPART_PARTS,
result = "final_part_limit_exceeded",
"sftp multipart state changed",
"SFTP multipart final-part limit exceeded",
);
self.close_abort_or_skip(bucket, key, &upload_id, abort_authorized, "parts-limit breach")
.await;
@@ -1080,7 +1080,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
upload_id = %mp.upload_id,
part_number = part_number,
result = "etag_missing",
"sftp multipart copy state changed"
"SFTP multipart copy part missing etag"
);
SftpError::code(StatusCode::Failure)
})?;
+14 -14
View File
@@ -423,7 +423,7 @@ where
bucket = %bucket,
object = %key,
file_size,
"webdav object write state changed"
"WebDAV object flush completed"
);
// Buffer already cleared by std::mem::take above
Ok(())
@@ -438,7 +438,7 @@ where
object = %key,
file_size,
error = %e,
"webdav object write state changed"
"WebDAV object flush failed"
);
Err(FsError::GeneralFailure)
}
@@ -1219,7 +1219,7 @@ where
result = "not_found",
bucket = %bucket,
error = %e,
"webdav bucket metadata state changed"
"WebDAV bucket metadata listed"
);
Err(FsError::NotFound)
}
@@ -1279,7 +1279,7 @@ where
state = "created",
bucket = %bucket,
object = %dir_key,
"webdav directory state changed"
"WebDAV directory marker created"
);
return Ok(());
}
@@ -1292,7 +1292,7 @@ where
bucket = %bucket,
object = %dir_key,
error = %e,
"webdav directory state changed"
"WebDAV directory marker create failed"
);
return Err(FsError::GeneralFailure);
}
@@ -1320,7 +1320,7 @@ where
subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
state = "bucket_created",
bucket = %bucket,
"webdav directory state changed"
"WebDAV bucket created"
);
Ok(())
}
@@ -1332,7 +1332,7 @@ where
state = "bucket_create_failed",
bucket = %bucket,
error = %e,
"webdav directory state changed"
"WebDAV bucket create failed"
);
Err(FsError::GeneralFailure)
}
@@ -1465,7 +1465,7 @@ where
state = "deleted",
bucket = %bucket,
object = %key,
"webdav object delete state changed"
"WebDAV object deleted"
);
Ok(())
}
@@ -1478,7 +1478,7 @@ where
bucket = %bucket,
object = %key,
error = %e,
"webdav object delete state changed"
"WebDAV object delete failed"
);
Err(FsError::GeneralFailure)
}
@@ -1529,7 +1529,7 @@ where
dst_bucket = %dst_bucket,
dst_object = %dst_key,
error = %e,
"webdav rename state changed"
"WebDAV rename source delete failed"
);
FsError::GeneralFailure
})?;
@@ -1543,7 +1543,7 @@ where
src_object = %src_key,
dst_bucket = %dst_bucket,
dst_object = %dst_key,
"webdav rename state changed"
"WebDAV file renamed"
);
return Ok(());
}
@@ -1603,7 +1603,7 @@ where
dst_bucket = %dst_bucket,
dst_prefix = %dst_prefix,
error = %e,
"webdav rename state changed"
"WebDAV rename directory listing failed"
);
FsError::GeneralFailure
})?;
@@ -1652,7 +1652,7 @@ where
src_object = %src_key,
dst_bucket = %dst_bucket,
dst_object = %dst_key,
"webdav rename state changed"
"WebDAV rename source not found"
);
return Err(FsError::NotFound);
}
@@ -1666,7 +1666,7 @@ where
src_object = %src_key,
dst_bucket = %dst_bucket,
dst_object = %dst_key,
"webdav rename state changed"
"WebDAV directory renamed"
);
Ok(())
}
+11 -11
View File
@@ -86,7 +86,7 @@ where
bind_addr = %self.config.bind_addr,
tls_enabled = self.config.tls_enabled,
max_body_size = self.config.max_body_size,
"webdav server state changed"
"WebDAV server starting"
);
let listener = TcpListener::bind(self.config.bind_addr).await?;
@@ -96,7 +96,7 @@ where
subsystem = LOG_SUBSYSTEM_WEBDAV_SERVER,
state = "listening",
bind_addr = %self.config.bind_addr,
"webdav server state changed"
"WebDAV server listening"
);
let (reload_shutdown_tx, reload_shutdown_rx) = watch::channel(false);
@@ -109,7 +109,7 @@ where
subsystem = LOG_SUBSYSTEM_WEBDAV_SERVER,
state = "enabled",
cert_dir = %cert_dir,
"webdav tls state changed"
"WebDAV TLS enabled"
);
let resolver = ReloadableServerCertResolver::load_from_directory(cert_dir)
@@ -211,7 +211,7 @@ where
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_WEBDAV_SERVER,
state = "shutdown_requested",
"webdav server state changed"
"WebDAV shutdown requested"
);
let _ = reload_shutdown_tx.send(true);
break;
@@ -225,7 +225,7 @@ where
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_WEBDAV_SERVER,
state = "stopped",
"webdav server state changed"
"WebDAV server stopped"
);
Ok(())
}
@@ -376,7 +376,7 @@ where
result = "iam_unavailable",
source_ip = %source_ip,
error = %e,
"webdav auth state changed"
"WebDAV auth IAM unavailable"
);
WebDavInitError::Server("Internal authentication service unavailable".to_string())
})?;
@@ -403,7 +403,7 @@ where
source_ip = %source_ip,
access_key = %masked_access_key,
error = %e,
"webdav auth state changed"
"WebDAV auth key check failed"
);
WebDavInitError::Server("Authentication verification failed".to_string())
})?;
@@ -416,7 +416,7 @@ where
result = "invalid_access_key",
source_ip = %source_ip,
access_key = %masked_access_key,
"webdav auth state changed"
"WebDAV auth rejected access key"
);
return Err(WebDavInitError::Server("Invalid credentials".to_string()));
}
@@ -429,7 +429,7 @@ where
result = "identity_missing",
source_ip = %source_ip,
access_key = %masked_access_key,
"webdav auth state changed"
"WebDAV auth identity missing"
);
WebDavInitError::Server("User not found".to_string())
})?;
@@ -442,7 +442,7 @@ where
result = "invalid_secret_key",
source_ip = %source_ip,
access_key = %masked_access_key,
"webdav auth state changed"
"WebDAV auth rejected secret key"
);
return Err(WebDavInitError::Server("Invalid credentials".to_string()));
}
@@ -454,7 +454,7 @@ where
result = "authenticated",
source_ip = %source_ip,
access_key = %masked_access_key,
"webdav auth state changed"
"WebDAV auth accepted"
);
Ok(SessionContext::new(
+24 -24
View File
@@ -97,7 +97,7 @@ fn resolve_scanner_runtime_config() -> crate::runtime_config::ScannerRuntimeConf
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "resolve_failed",
error = %err,
"Scanner runtime config resolution failed; using last applied config"
"Scanner runtime config fallback applied"
);
current_scanner_runtime_config()
}
@@ -179,7 +179,7 @@ async fn persisted_usage_cache_is_cold_for_startup(storeapi: &Arc<ECStore>) -> b
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "startup_inspect_failed",
error = %err,
"Scanner startup cache inspection failed; keeping configured startup delay"
"Scanner startup cache inspection failed"
);
return false;
}
@@ -198,7 +198,7 @@ async fn persisted_usage_cache_is_cold_for_startup(storeapi: &Arc<ECStore>) -> b
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "startup_decode_failed",
error = %err,
"Scanner startup cache decode failed; skipping startup delay"
"Scanner startup cache decode failed"
);
true
}
@@ -222,7 +222,7 @@ async fn initial_scanner_startup_usage_state(storeapi: &Arc<ECStore>) -> (bool,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "startup_bucket_inspect_failed",
error = %err,
"Scanner startup bucket inspection failed; keeping configured startup delay"
"Scanner startup bucket inspection failed"
);
false
}
@@ -418,7 +418,7 @@ async fn configure_scanner_defaults(storeapi: &Arc<ECStore>) {
replication_active = features.replication,
feature_inspection_failed = features.inspection_failed,
state = "single_disk_defaults_applied",
"Scanner defaults updated for single-disk deployment"
"Scanner defaults applied"
);
} else {
set_scanner_default_speed(ScannerSpeed::Default);
@@ -562,7 +562,7 @@ pub async fn read_background_heal_info(storeapi: Arc<ECStore>) -> BackgroundHeal
path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "decode_failed",
error = %e,
"Scanner background heal state decode failed"
"Scanner background heal decode failed"
);
BackgroundHealInfo::default()
}),
@@ -577,7 +577,7 @@ pub async fn read_background_heal_info(storeapi: Arc<ECStore>) -> BackgroundHeal
path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "read_failed",
error = %e,
"Scanner background heal state read failed"
"Scanner background heal read failed"
);
}
BackgroundHealInfo::default()
@@ -605,7 +605,7 @@ pub async fn save_background_heal_info(storeapi: Arc<ECStore>, info: BackgroundH
path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "encode_failed",
error = %e,
"Scanner background heal state encode failed"
"Scanner background heal encode failed"
);
return;
}
@@ -621,7 +621,7 @@ pub async fn save_background_heal_info(storeapi: Arc<ECStore>, info: BackgroundH
path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "save_failed",
error = %e,
"Scanner background heal state save failed"
"Scanner background heal save failed"
);
}
}
@@ -650,7 +650,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "refresh_failed",
error = %err,
"Scanner runtime config refresh failed; using last applied config"
"Scanner runtime config refresh failed"
);
}
let configured_cycle_interval = scanner_cycle_interval();
@@ -685,7 +685,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
cycle = cycle_info.current,
scan_mode = ?scan_mode,
state = "started",
"Scanner cycle state updated"
"Scanner cycle started"
);
let _scan_mode_guard = ScannerScanModeGuard::new(scan_mode);
if let Some(new_heal_info) = background_heal_info_for_scan_start(
@@ -730,7 +730,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
max_objects = ?cycle_budget.max_objects(),
max_directories = ?cycle_budget.max_directories(),
state = "budget_reached",
"Scanner cycle stopped after reaching budget"
"Scanner cycle budget reached"
);
let budget_reason = cycle_budget.reason();
emit_scan_cycle_partial_with_source(
@@ -751,7 +751,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
state = "failed",
duration = ?now.elapsed(),
error = %e,
"Scanner cycle state updated"
"Scanner cycle failed"
);
emit_scan_cycle_complete(false, cycle_start.elapsed());
if let Some(new_heal_info) = background_heal_info_for_scan_complete(background_heal_info.clone(), scan_mode) {
@@ -772,7 +772,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
max_objects = ?cycle_budget.max_objects(),
max_directories = ?cycle_budget.max_directories(),
state = "budget_reached",
"Scanner cycle stopped after reaching budget"
"Scanner cycle budget reached"
);
global_metrics().finish_scan_cycle_work(cycle_work_start);
let budget_reason = cycle_budget.reason();
@@ -806,7 +806,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
state = "completed",
duration = ?now.elapsed(),
cycles_total = cycle_info.cycle_completed.len(),
"Scanner cycle state updated"
"Scanner cycle completed"
);
retain_recent_cycle_completions(&mut cycle_info.cycle_completed);
@@ -830,14 +830,14 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
"Scanner state persistence failed"
);
} else {
info!(
debug!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %&*DATA_USAGE_BLOOM_NAME_PATH,
state = "saved",
"Scanner state persisted"
"Scanner state saved"
);
}
}
@@ -854,7 +854,7 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) ->
subsystem = LOG_SUBSYSTEM_RUNTIME,
lock_name = "leader.lock",
state = "acquired",
"Scanner leader lock state updated"
"Scanner leader lock acquired"
);
guard
}
@@ -867,7 +867,7 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) ->
lock_name = "leader.lock",
state = "contended",
error = ?e,
"Scanner leader lock state updated"
"Scanner leader lock contended"
);
return Ok(());
}
@@ -881,7 +881,7 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) ->
lock_name = "leader.lock",
state = "create_failed",
error = %e,
"Scanner leader lock state updated"
"Scanner leader lock creation failed"
);
return Ok(());
}
@@ -977,7 +977,7 @@ pub async fn store_data_usage_in_backend(
&& let (Some(new_ts), Some(existing_ts)) = (data_usage_info.last_update, existing.last_update)
&& new_ts <= existing_ts
{
info!(
debug!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
@@ -986,7 +986,7 @@ pub async fn store_data_usage_in_backend(
incoming_last_update = ?new_ts,
existing_last_update = ?existing_ts,
state = "skip_stale_update",
"Scanner data usage persistence skipped stale update"
"Scanner stale data usage update skipped"
);
continue;
}
@@ -1003,7 +1003,7 @@ pub async fn store_data_usage_in_backend(
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "encode_failed",
error = %e,
"Scanner data usage persistence encode failed"
"Scanner data usage encode failed"
);
continue;
}
@@ -1040,7 +1040,7 @@ pub async fn store_data_usage_in_backend(
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "save_failed",
error = %e,
"Scanner data usage persistence failed"
"Scanner data usage save failed"
);
} else {
rustfs_ecstore::data_usage::replace_bucket_usage_memory_from_info(&data_usage_info).await;
+6 -6
View File
@@ -66,7 +66,7 @@ use time::OffsetDateTime;
use tokio::select;
use tokio::sync::mpsc;
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn};
use tracing::{debug, error, warn};
const LOG_COMPONENT_SCANNER: &str = "scanner";
const LOG_SUBSYSTEM_FOLDER: &str = "folder";
@@ -639,7 +639,7 @@ impl ScannerItem {
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
object_path = %self.object_path(),
state = "started",
"Scanner lifecycle action evaluation started"
"Scanner lifecycle evaluation started"
);
let versioning_config = match BucketVersioningSys::get(&self.bucket).await {
@@ -990,7 +990,7 @@ impl ScannerItem {
let age = now - mod_time;
age.whole_seconds().max(0)
});
info!(
debug!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_HEAL_ADMISSION,
component = LOG_COMPONENT_SCANNER,
@@ -1003,7 +1003,7 @@ impl ScannerItem {
original_scan_mode = %HealScanMode::Deep.as_str(),
effective_scan_mode = %scan_mode.as_str(),
state = "downgraded_to_normal",
"Scanner heal admission downgraded deep scan during cooldown"
"Scanner heal deep scan downgraded"
);
}
@@ -1688,14 +1688,14 @@ impl FolderScanner {
if found_objects && is_erasure().await {
// If we found an object in erasure mode, we skip subdirs (only datadirs)...
info!(
debug!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
folder = %folder.name,
state = "erasure_object_found",
"Scanner folder stopped descending after finding erasure object"
"Scanner folder descent stopped after erasure object"
);
break;
}
+14 -14
View File
@@ -900,7 +900,7 @@ impl ScannerIOCache for SetDisks {
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "scan_started",
"Scanner disk bucket state updated"
"Scanner disk bucket scan started"
);
let cache_name = path_join_buf(&[&bucket.name, DATA_USAGE_CACHE_NAME]);
@@ -910,13 +910,13 @@ impl ScannerIOCache for SetDisks {
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "cache_load_failed",
error = %e,
"Scanner disk bucket state updated"
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "cache_load_failed",
error = %e,
"Scanner disk bucket cache load failed"
);
}
@@ -942,7 +942,7 @@ impl ScannerIOCache for SetDisks {
bucket = %bucket.name,
cache_name = ?cache.info.name,
state = "cache_ready",
"Scanner disk bucket state updated"
"Scanner disk bucket cache ready"
);
let (updates_tx, mut updates_rx) = mpsc::channel::<DataUsageEntry>(1);
@@ -997,7 +997,7 @@ impl ScannerIOCache for SetDisks {
bucket = %bucket.name,
state = "cancelled",
error = %e,
"Scanner disk bucket state updated"
"Scanner disk bucket scan cancelled"
);
} else {
error!(
@@ -1008,7 +1008,7 @@ impl ScannerIOCache for SetDisks {
bucket = %bucket.name,
state = "scan_failed",
error = %e,
"Scanner disk bucket state updated"
"Scanner disk bucket scan failed"
);
}
@@ -1104,7 +1104,7 @@ impl ScannerIOCache for SetDisks {
bucket = %bucket.name,
cache_name = %cache.info.name,
state = "scan_completed",
"Scanner disk bucket state updated"
"Scanner disk bucket scan completed"
);
if let Err(e) = update_fut.await {
@@ -1132,7 +1132,7 @@ impl ScannerIOCache for SetDisks {
bucket = %bucket.name,
cache_name = %cache.info.name,
state = "send_root_entry",
"Scanner data usage stream progress updated"
"Scanner root entry publish started"
);
if let Err(e) = send_cache_root_entry_info(&bucket_result_tx_clone_clone, &cache).await {
@@ -1182,7 +1182,7 @@ impl ScannerIOCache for SetDisks {
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
state = "set_scan_completed",
"Scanner set-level disk scan completed"
"Scanner set scan completed"
);
Ok(())
+108 -16
View File
@@ -30,6 +30,10 @@ use std::{
use tracing::{debug, warn};
use uuid::Uuid;
const LOG_COMPONENT_TARGETS: &str = "targets";
const LOG_SUBSYSTEM_STORE: &str = "store";
const EVENT_TARGET_STORE_STATE: &str = "target_store_state";
fn resolve_queue_store_compression_from_env_value(value: Option<&str>) -> bool {
value
.and_then(|value| value.parse::<EnableState>().ok().map(|state| state.is_enabled()))
@@ -83,7 +87,14 @@ impl std::fmt::Display for Key {
/// Parses a string into a Key
pub fn parse_key(s: &str) -> Key {
debug!("Parsing key: {}", s);
debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "parse_key",
key = %s,
"target store state"
);
let mut name = s.to_string();
let mut extension = String::new();
@@ -111,8 +122,16 @@ pub fn parse_key(s: &str) -> Key {
}
debug!(
"Parsed key - name: {}, extension: {}, item_count: {}, compress: {}",
name, extension, item_count, compress
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "parse_key",
result = "parsed",
key_name = %name,
extension = %extension,
item_count,
compressed = compress,
"target store state"
);
Key {
@@ -276,7 +295,15 @@ impl<T: Serialize + DeserializeOwned + Send + Sync> QueueStore<T> {
.read()
.map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?;
let path = self.file_path(key);
debug!("Reading file for key: {},path: {}", key, path.display());
debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "read_file",
key = %key,
path = %path.display(),
"target store state"
);
let data = std::fs::read(&path).map_err(|e| {
if e.kind() == std::io::ErrorKind::NotFound {
StoreError::NotFound
@@ -342,7 +369,14 @@ impl<T: Serialize + DeserializeOwned + Send + Sync> QueueStore<T> {
std::fs::write(&path, data).map_err(StoreError::Io)?;
}
let modified = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_nanos() as i64;
debug!("Wrote event to store: {}", key);
debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "write_file",
key = %key,
"target store state"
);
Ok(modified)
}
@@ -409,7 +443,15 @@ where
}
}
debug!("Opened store at: {:?}", self.directory);
debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
state = "opened",
store_dir = ?self.directory,
entry_count = entries_map.len(),
"target store state"
);
Ok(())
}
@@ -471,7 +513,14 @@ where
}
fn get_multiple(&self, key: &Self::Key) -> Result<Vec<T>, Self::Error> {
debug!("Reading items from store for key: {}", key);
debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "read_batch",
key = %key,
"target store state"
);
let data = self.get_raw(key)?;
if data.is_empty() {
return Err(StoreError::Deserialization("Cannot deserialize empty data".to_string()));
@@ -500,10 +549,15 @@ where
if items.len() < key.item_count && !items.is_empty() {
// Partial read
warn!(
"Expected {} items for key {}, but only found {}. Possible data corruption or incorrect item_count.",
key.item_count,
key,
items.len()
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "read_batch",
key = %key,
expected_items = key.item_count,
actual_items = items.len(),
reason = "partial_batch_read",
"target store state"
);
// Depending on strictness, this could be an error.
} else if items.is_empty() {
@@ -538,7 +592,15 @@ where
Ok(()) => {}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
// File already gone — still clean up the entries map to avoid stale keys.
warn!("File not found for key {} during del, cleaning up entries map.", key);
warn!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "delete",
key = %key,
result = "file_missing",
"target store state"
);
}
Err(e) => return Err(StoreError::Io(e)),
}
@@ -550,9 +612,25 @@ where
.map_err(|_| StoreError::Internal("Failed to acquire write lock on entries".to_string()))?;
if entries.remove(&key.to_key_string()).is_none() {
debug!("Key {} not found in entries map during del, might have been already removed.", key);
debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "delete",
key = %key,
result = "entry_missing",
"target store state"
);
}
debug!("Deleted event from store: {}", key.to_string());
debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "delete",
key = %key,
result = "deleted",
"target store state"
);
Ok(())
}
@@ -580,7 +658,14 @@ where
let entries = match self.entries.read() {
Ok(entries) => entries,
Err(_) => {
debug!("Failed to acquire read lock on entries for listing");
debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "list",
result = "lock_unavailable",
"target store state"
);
return Vec::new();
}
};
@@ -597,7 +682,14 @@ where
match self.entries.read() {
Ok(entries) => entries.len(),
Err(_) => {
debug!("Failed to acquire read lock on entries for len");
debug!(
event = EVENT_TARGET_STORE_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_STORE,
action = "len",
result = "lock_unavailable",
"target store state"
);
0
}
}
+464 -56
View File
@@ -60,6 +60,10 @@ const DEFAULT_MQTT_TLS_PORT: u16 = 8883;
const DEFAULT_MQTT_WSS_PORT: u16 = 443;
const MAX_MQTT_PACKET_SIZE_BYTES: u32 = 100 * 1024 * 1024;
const DEFAULT_MQTT_WS_PATH_ALLOWLIST: &[&str] = &["/", "/mqtt"];
const LOG_COMPONENT_TARGETS: &str = "targets";
const LOG_SUBSYSTEM_MQTT: &str = "mqtt";
const EVENT_MQTT_TARGET_STATE: &str = "mqtt_target_state";
const EVENT_MQTT_DELIVERY_STATE: &str = "mqtt_delivery_state";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum MQTTTlsPolicy {
@@ -574,7 +578,14 @@ where
Some(MAX_MQTT_PACKET_SIZE_BYTES),
)?;
info!(target_id = %target_id, "MQTT target created");
info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "created",
"mqtt target state"
);
Ok(MQTTTarget::<E> {
id: target_id,
args,
@@ -593,7 +604,14 @@ where
#[instrument(skip(self), fields(target_id = %self.id))]
async fn init(&self) -> Result<(), TargetError> {
if self.connected.load(Ordering::SeqCst) {
debug!(target_id = %self.id, "Already connected.");
debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "already_connected",
"mqtt target state"
);
return Ok(());
}
@@ -607,7 +625,14 @@ where
let _ = bg_task_manager
.init_cell
.get_or_try_init(|| async {
debug!(target_id = %target_id_clone, "Initializing MQTT background task.");
debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id_clone,
state = "background_task_initializing",
"mqtt target state"
);
// Use the latest MqttOptions (may have been updated by TLS reload coordinator).
let mqtt_options: MqttOptions = (**pending_mqtt_options.load()).clone();
@@ -615,30 +640,67 @@ where
let (new_client, eventloop) = AsyncClient::builder(mqtt_options).capacity(10).build();
if let Err(e) = new_client.subscribe(&args_clone.topic, args_clone.qos).await {
error!(target_id = %target_id_clone, error = %e, "Failed to subscribe to MQTT topic during init");
error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id_clone,
state = "subscribe_failed",
error = %e,
"mqtt target state"
);
return Err(TargetError::Network(format!("MQTT subscribe failed: {e}")));
}
let mut rx_guard = bg_task_manager.initial_cancel_rx.lock().await;
let cancel_rx = rx_guard.take().ok_or_else(|| {
error!(target_id = %target_id_clone, "MQTT cancel receiver already taken for task.");
error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id_clone,
state = "cancel_receiver_unavailable",
"mqtt target state"
);
TargetError::Configuration("MQTT cancel receiver already taken for task".to_string())
})?;
drop(rx_guard);
*client_arc.lock().await = Some(new_client.clone());
info!(target_id = %target_id_clone, "Spawning MQTT event loop task.");
info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id_clone,
state = "event_loop_spawning",
"mqtt target state"
);
let task_handle =
tokio::spawn(run_mqtt_event_loop(eventloop, connected_arc.clone(), target_id_clone.clone(), cancel_rx));
Ok(task_handle)
})
.await
.map_err(|e: TargetError| {
error!(target_id = %self.id, error = %e, "Failed to initialize MQTT background task");
error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "background_task_init_failed",
error = %e,
"mqtt target state"
);
e
})?;
debug!(target_id = %self.id, "MQTT background task initialized successfully.");
debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "background_task_initialized",
"mqtt target state"
);
match tokio::time::timeout(DEFAULT_CONNECTION_TIMEOUT, async {
while !self.connected.load(Ordering::SeqCst) {
@@ -646,23 +708,51 @@ where
&& handle.is_finished()
&& !self.connected.load(Ordering::SeqCst)
{
error!(target_id = %self.id, "MQTT background task exited prematurely before connection was established.");
error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "background_task_exited_before_connect",
"mqtt target state"
);
return Err(TargetError::Network("MQTT background task exited prematurely".to_string()));
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
debug!(target_id = %self.id, "MQTT target connected successfully.");
debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "connected",
"mqtt target state"
);
Ok(())
})
.await
{
Ok(Ok(_)) => {
info!(target_id = %self.id, "MQTT target initialized and connected.");
info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "ready",
"mqtt target state"
);
Ok(())
}
Ok(Err(e)) => Err(e),
Err(_) => {
error!(target_id = %self.id, "Timeout waiting for MQTT connection after task spawn.");
error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "connect_timeout",
"mqtt target state"
);
Err(TargetError::Network("Timeout waiting for MQTT connection".to_string()))
}
}
@@ -680,12 +770,16 @@ where
.ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?;
debug!(
target = %self.id,
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
bucket = %meta.bucket_name,
object = %meta.object_name,
event = %meta.event_name,
payload_len = body.len(),
"Sending MQTT payload"
state = "publishing",
"mqtt delivery state"
);
client
@@ -693,7 +787,16 @@ where
.await
.map_err(|e| {
if e.to_string().contains("Connection") || e.to_string().contains("Timeout") {
warn!(target_id = %self.id, error = %e, "Publish failed due to connection issue, marking as not connected.");
warn!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "publish_failed",
reason = "connectivity_error",
error = %e,
"mqtt delivery state"
);
let err = TargetError::NotConnected;
mark_target_disconnected_on_connectivity_error(&self.connected, &err);
err
@@ -702,7 +805,15 @@ where
}
})?;
debug!(target_id = %self.id, topic = %self.args.topic, "Event published to MQTT topic");
debug!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
topic = %self.args.topic,
state = "published",
"mqtt delivery state"
);
self.delivery_counters.record_success();
Ok(())
}
@@ -781,14 +892,28 @@ async fn run_mqtt_event_loop(
target_id: TargetID,
mut cancel_rx: mpsc::Receiver<()>,
) {
info!(target_id = %target_id, "MQTT event loop task started.");
info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "event_loop_started",
"mqtt target state"
);
let mut initial_connection_established = false;
loop {
tokio::select! {
biased;
_ = cancel_rx.recv() => {
info!(target_id = %target_id, "MQTT event loop task received cancellation signal. Shutting down.");
info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "cancellation_received",
"mqtt target state"
);
break;
}
polled_event_result = async {
@@ -796,7 +921,14 @@ async fn run_mqtt_event_loop(
match tokio::time::timeout(EVENT_LOOP_POLL_TIMEOUT, eventloop.poll()).await {
Ok(result) => Some(result),
Err(_) => {
debug!(target_id = %target_id, "MQTT poll timed out (EVENT_LOOP_POLL_TIMEOUT) while not connected or status pending.");
debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "poll_timeout",
"mqtt target state"
);
connected_status.store(false, Ordering::SeqCst);
None
}
@@ -810,15 +942,38 @@ async fn run_mqtt_event_loop(
trace!(target_id = %target_id, event = ?notification, "Received MQTT event");
match notification {
rumqttc::Event::Incoming(Incoming::ConnAck(_conn_ack)) => {
info!(target_id = %target_id, "MQTT connected (ConnAck).");
info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "connack_received",
"mqtt target state"
);
connected_status.store(true, Ordering::SeqCst);
initial_connection_established = true;
}
rumqttc::Event::Incoming(Incoming::Publish(publish)) => {
debug!(target_id = %target_id, topic = ?publish.topic, payload_len = publish.payload.len(), "Received message on subscribed topic.");
debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "publish_received",
topic = ?publish.topic,
payload_len = publish.payload.len(),
"mqtt target state"
);
}
rumqttc::Event::Incoming(Incoming::Disconnect(_)) => {
info!(target_id = %target_id, "Received Disconnect packet from broker. MQTT connection lost.");
info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "broker_disconnected",
"mqtt target state"
);
connected_status.store(false, Ordering::SeqCst);
}
rumqttc::Event::Incoming(Incoming::PingResp(_)) => {
@@ -832,7 +987,14 @@ async fn run_mqtt_event_loop(
}
// Process other incoming packet types as needed (PubRec, PubRel, PubComp, UnsubAck)
rumqttc::Event::Outgoing(Outgoing::Disconnect) => {
info!(target_id = %target_id, "MQTT outgoing disconnect initiated by client.");
info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "client_disconnect_requested",
"mqtt target state"
);
connected_status.store(false, Ordering::SeqCst);
}
rumqttc::Event::Outgoing(Outgoing::PingReq) => {
@@ -848,7 +1010,15 @@ async fn run_mqtt_event_loop(
}
Some(Err(e)) => {
connected_status.store(false, Ordering::SeqCst);
error!(target_id = %target_id, error = %e, "Error from MQTT event loop poll");
error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "poll_failed",
error = %e,
"mqtt target state"
);
if matches!(e,
ConnectionError::Io(_) |
@@ -856,12 +1026,28 @@ async fn run_mqtt_event_loop(
ConnectionError::ConnectionRefused(_) |
ConnectionError::Tls(_)
) {
warn!(target_id = %target_id, error = %e, "MQTT connection error. Relying on rumqttc for reconnection if applicable.");
warn!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "reconnect_pending",
error = %e,
"mqtt target state"
);
}
// Here you can decide whether to break loops based on the error type.
// For example, for some unrecoverable errors.
if is_fatal_mqtt_error(&e) {
error!(target_id = %target_id, error = %e, "Fatal MQTT error, terminating event loop.");
error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "fatal_error",
error = %e,
"mqtt target state"
);
break;
}
// rumqttc's eventloop.poll() may return Err and terminate after some errors,
@@ -871,7 +1057,14 @@ async fn run_mqtt_event_loop(
tokio::time::sleep(Duration::from_secs(1)).await;
}
None => {
warn!(target_id = %target_id, "Timeout during initial poll or pending state, will retry.");
warn!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "poll_retry_scheduled",
"mqtt target state"
);
continue;
}
}
@@ -879,7 +1072,14 @@ async fn run_mqtt_event_loop(
}
}
connected_status.store(false, Ordering::SeqCst);
info!(target_id = %target_id, "MQTT event loop task finished.");
info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %target_id,
state = "event_loop_finished",
"mqtt target state"
);
}
/// Check whether the given MQTT connection error should be considered a fatal error,
@@ -938,26 +1138,61 @@ where
#[instrument(skip(self), fields(target_id = %self.id))]
async fn is_active(&self) -> Result<bool, TargetError> {
debug!(target_id = %self.id, "Checking if MQTT target is active.");
debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "activity_check",
"mqtt target state"
);
if self.client.lock().await.is_none() && !self.connected.load(Ordering::SeqCst) {
// Check if the background task is running and has not panicked
if let Some(handle) = self.bg_task_manager.init_cell.get()
&& handle.is_finished()
{
error!(target_id = %self.id, "MQTT background task has finished, possibly due to an error. Target is not active.");
error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "inactive_background_task_finished",
"mqtt target state"
);
return Err(TargetError::Network("MQTT background task terminated".to_string()));
}
debug!(target_id = %self.id, "MQTT client not yet initialized or task not running/connected.");
debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "inactive_client_unavailable",
"mqtt target state"
);
return Err(TargetError::Configuration(
"MQTT client not available or not initialized/connected".to_string(),
));
}
if self.connected.load(Ordering::SeqCst) {
debug!(target_id = %self.id, "MQTT target is active (connected flag is true).");
debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "active",
"mqtt target state"
);
Ok(true)
} else {
debug!(target_id = %self.id, "MQTT target is not connected (connected flag is false).");
debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "inactive_not_connected",
"mqtt target state"
);
Err(TargetError::NotConnected)
}
}
@@ -973,14 +1208,36 @@ where
};
if let Some(store) = &self.store {
debug!(target_id = %self.id, "Event saved to store start");
debug!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "store_enqueue_started",
"mqtt delivery state"
);
match persist_queued_payload_to_store(store.as_ref(), &queued) {
Ok(_) => {
debug!(target_id = %self.id, "Event saved to store for MQTT target successfully.");
debug!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "store_enqueued",
"mqtt delivery state"
);
Ok(())
}
Err(e) => {
error!(target_id = %self.id, error = %e, "Failed to save event to store");
error!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "store_enqueue_failed",
error = %e,
"mqtt delivery state"
);
self.delivery_counters.record_final_failure();
Err(e)
}
@@ -991,18 +1248,47 @@ where
}
if !self.connected.load(Ordering::SeqCst) {
warn!(target_id = %self.id, "Attempting to send directly but not connected; trying to init.");
warn!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "direct_send_requires_init",
"mqtt target state"
);
// Call the struct's init method, not the trait's default
match MQTTTarget::<E>::init(self).await {
Ok(_) => debug!(target_id = %self.id, "MQTT target initialized successfully."),
Ok(_) => debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "init_completed",
"mqtt target state"
),
Err(e) => {
error!(target_id = %self.id, error = %e, "Failed to initialize MQTT target.");
error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "init_failed",
error = %e,
"mqtt target state"
);
self.delivery_counters.record_final_failure();
return Err(TargetError::NotConnected);
}
}
if !self.connected.load(Ordering::SeqCst) {
error!(target_id = %self.id, "Cannot save (send directly) as target is not active after init attempt.");
error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "init_completed_not_connected",
"mqtt target state"
);
self.delivery_counters.record_final_failure();
return Err(TargetError::NotConnected);
}
@@ -1017,50 +1303,143 @@ where
#[instrument(skip(self, body, meta), fields(target_id = %self.id))]
async fn send_raw_from_store(&self, key: Key, body: Vec<u8>, meta: QueuedPayloadMeta) -> Result<(), TargetError> {
debug!(target_id = %self.id, ?key, "Attempting to send queued payload from store.");
debug!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
?key,
state = "store_replay_started",
"mqtt delivery state"
);
if !self.is_enabled() {
return Err(TargetError::Disabled);
}
if !self.connected.load(Ordering::SeqCst) {
warn!(target_id = %self.id, "Not connected; trying to init before sending from store.");
warn!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "store_replay_requires_init",
"mqtt target state"
);
match MQTTTarget::<E>::init(self).await {
Ok(_) => debug!(target_id = %self.id, "MQTT target initialized successfully."),
Ok(_) => debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "init_completed",
"mqtt target state"
),
Err(e) => {
error!(target_id = %self.id, error = %e, "Failed to initialize MQTT target.");
error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "init_failed",
error = %e,
"mqtt target state"
);
return Err(TargetError::NotConnected);
}
}
if !self.connected.load(Ordering::SeqCst) {
error!(target_id = %self.id, "Cannot send from store as target is not active after init attempt.");
error!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "init_completed_not_connected",
"mqtt target state"
);
return Err(TargetError::NotConnected);
}
}
debug!(target_id = %self.id, ?key, "Sending event from store.");
debug!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
?key,
state = "store_replay_publishing",
"mqtt delivery state"
);
if let Err(e) = self.send_body(body, &meta).await {
if matches!(e, TargetError::NotConnected) {
warn!(target_id = %self.id, "Failed to send event from store: Not connected. Event remains in store.");
warn!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
?key,
state = "store_replay_deferred",
reason = "not_connected",
"mqtt delivery state"
);
return Err(TargetError::NotConnected);
}
error!(target_id = %self.id, error = %e, "Failed to send event from store with an unexpected error.");
error!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
?key,
state = "store_replay_failed",
error = %e,
"mqtt delivery state"
);
return Err(e);
}
debug!(target_id = %self.id, ?key, "Event sent from store successfully.");
debug!(
event = EVENT_MQTT_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
?key,
state = "store_replay_published",
"mqtt delivery state"
);
Ok(())
}
async fn close(&self) -> Result<(), TargetError> {
info!(target_id = %self.id, "Attempting to close MQTT target.");
info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "closing",
"mqtt target state"
);
if let Err(e) = self.bg_task_manager.cancel_tx.send(()).await {
warn!(target_id = %self.id, error = %e, "Failed to send cancel signal to MQTT background task. It might have already exited.");
warn!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "cancel_signal_failed",
error = %e,
"mqtt target state"
);
}
// Wait for the task to finish if it was initialized
if let Some(_task_handle) = self.bg_task_manager.init_cell.get() {
debug!(target_id = %self.id, "Waiting for MQTT background task to complete...");
debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "waiting_for_event_loop",
"mqtt target state"
);
// It's tricky to await here if close is called from a sync context or Drop
// For async close, this is fine. Consider a timeout.
// let _ = tokio::time::timeout(Duration::from_secs(5), task_handle.await).await;
@@ -1069,9 +1448,24 @@ where
}
if let Some(client_instance) = self.client.lock().await.take() {
info!(target_id = %self.id, "Disconnecting MQTT client.");
info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "disconnecting_client",
"mqtt target state"
);
if let Err(e) = client_instance.disconnect().await {
warn!(target_id = %self.id, error = %e, "Error during MQTT client disconnect.");
warn!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "disconnect_failed",
error = %e,
"mqtt target state"
);
}
}
@@ -1083,7 +1477,14 @@ where
}
self.connected.store(false, Ordering::SeqCst);
info!(target_id = %self.id, "MQTT target close method finished.");
info!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "closed",
"mqtt target state"
);
Ok(())
}
@@ -1097,7 +1498,14 @@ where
async fn init(&self) -> Result<(), TargetError> {
if !self.is_enabled() {
debug!(target_id = %self.id, "Target is disabled, skipping init.");
debug!(
event = EVENT_MQTT_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_MQTT,
target_id = %self.id,
state = "disabled",
"mqtt target state"
);
return Ok(());
}
// Call the internal init logic
+120 -22
View File
@@ -45,6 +45,11 @@ use std::{
use tokio::sync::mpsc;
use tracing::{debug, error, info, instrument, warn};
const LOG_COMPONENT_TARGETS: &str = "targets";
const LOG_SUBSYSTEM_WEBHOOK: &str = "webhook";
const EVENT_WEBHOOK_TARGET_STATE: &str = "webhook_target_state";
const EVENT_WEBHOOK_DELIVERY_STATE: &str = "webhook_delivery_state";
/// Arguments for configuring a Webhook target
#[derive(Clone)]
pub struct WebhookArgs {
@@ -190,7 +195,14 @@ where
// Create a cancel channel
let (cancel_sender, _) = mpsc::channel(1);
info!(target_id = %target_id.id, "Webhook target created");
info!(
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %target_id.id,
state = "created",
"webhook target state"
);
Ok(WebhookTarget::<E> {
id: target_id,
args,
@@ -220,8 +232,13 @@ where
// DANGEROUS: For testing only, skip all certificate verification
client_builder = client_builder.danger_accept_invalid_certs(true);
warn!(
"Webhook target '{}' is configured to skip TLS verification. This is insecure and should not be used in production.",
args.endpoint
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
endpoint = %args.endpoint,
state = "tls_verification_skipped",
fallback = "danger_accept_invalid_certs",
"webhook target state"
);
} else if !args.client_ca.is_empty() {
// Use user-provided custom CA certificate
@@ -301,10 +318,14 @@ where
match tokio::time::timeout(Duration::from_secs(5), client.head(health_check_url.as_str()).send()).await {
Ok(Ok(resp)) => {
debug!(
target = %self.id,
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
status = %resp.status(),
health_check_url = %health_check_url,
"Webhook health check request succeeded"
state = "reachability_probe_succeeded",
"webhook target state"
);
Ok(true)
}
@@ -337,7 +358,15 @@ where
// behavior matches real delivery while avoiding path-specific false negatives.
match self.probe_reachability().await {
Ok(true) => {
debug!("Webhook target {} reachability probe succeeded via {:?}", self.id, self.health_check_url);
debug!(
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
health_check_url = ?self.health_check_url,
state = "reachable",
"webhook target state"
);
}
Ok(false) => {
return Err(TargetError::NotConnected);
@@ -348,7 +377,14 @@ where
}
self.initialized.store(true, Ordering::SeqCst);
info!("Webhook target {} initialized", self.id);
info!(
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
state = "initialized",
"webhook target state"
);
Ok(())
}
@@ -357,14 +393,17 @@ where
}
async fn send_body(&self, body: Vec<u8>, meta: &QueuedPayloadMeta) -> Result<(), TargetError> {
info!("Webhook sending queued payload to target: {}", self.id);
debug!(
target = %self.id,
event = EVENT_WEBHOOK_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
bucket = %meta.bucket_name,
object = %meta.object_name,
event = %meta.event_name,
payload_event = %meta.event_name,
payload_len = body.len(),
"Sending webhook payload"
state = "sending",
"webhook delivery state"
);
// When a TLS reload adapter is attached, it drives client rebuilds in
@@ -406,7 +445,15 @@ where
let status = resp.status();
if status.is_success() {
debug!("Event sent to webhook target: {}", self.id);
debug!(
event = EVENT_WEBHOOK_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
status = %status,
state = "sent",
"webhook delivery state"
);
self.delivery_counters.record_success();
Ok(())
} else if status == StatusCode::FORBIDDEN {
@@ -454,13 +501,28 @@ where
self.delivery_counters.record_final_failure();
return Err(e);
}
debug!("Event saved to store for target: {}", self.id);
debug!(
event = EVENT_WEBHOOK_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
state = "store_enqueued",
"webhook delivery state"
);
Ok(())
} else {
match self.init().await {
Ok(_) => (),
Err(e) => {
error!("Failed to initialize Webhook target {}: {}", self.id.id, e);
error!(
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id.id,
state = "init_failed",
error = %e,
"webhook target state"
);
self.delivery_counters.record_final_failure();
return Err(TargetError::NotConnected);
}
@@ -474,13 +536,27 @@ where
}
async fn send_raw_from_store(&self, key: Key, body: Vec<u8>, meta: QueuedPayloadMeta) -> Result<(), TargetError> {
debug!("Sending queued payload from store for target: {}, key: {}", self.id, key);
debug!(
event = EVENT_WEBHOOK_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
key = %key,
state = "store_replay_started",
"webhook delivery state"
);
match self.init().await {
Ok(_) => {
debug!("Event sent to store for target: {}", self.name());
}
Ok(_) => {}
Err(e) => {
error!("Failed to initialize Webhook target {}: {}", self.id.id, e);
error!(
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id.id,
state = "init_failed",
error = %e,
"webhook target state"
);
return Err(TargetError::NotConnected);
}
}
@@ -492,7 +568,15 @@ where
return Err(e);
}
debug!("Event sent from store and deleted for target: {}", self.id);
debug!(
event = EVENT_WEBHOOK_DELIVERY_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
key = %key,
state = "store_replay_sent",
"webhook delivery state"
);
Ok(())
}
@@ -500,7 +584,14 @@ where
// Send cancel signal to background tasks
let _ = self.cancel_sender.try_send(());
// Adapter cleanup is done by the coordinator; no local state to reset.
info!("Webhook target closed: {}", self.id);
info!(
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
state = "closed",
"webhook target state"
);
Ok(())
}
@@ -515,7 +606,14 @@ where
async fn init(&self) -> Result<(), TargetError> {
if !self.is_enabled() {
debug!("Webhook target {} is disabled, skipping initialization", self.id);
debug!(
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
target_id = %self.id,
state = "disabled",
"webhook target state"
);
return Ok(());
}
self.init_inner().await