mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-26 05:56:50 +00:00
refactor(logging): normalize admin telemetry and error messages (#3430)
This commit is contained in:
@@ -160,7 +160,7 @@ impl AuditMetrics {
|
||||
component = LOG_COMPONENT_AUDIT,
|
||||
subsystem = LOG_SUBSYSTEM_OBSERVABILITY,
|
||||
state = "config_reloaded",
|
||||
"Audit observability state updated"
|
||||
"audit observability state"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -173,7 +173,7 @@ impl AuditMetrics {
|
||||
component = LOG_COMPONENT_AUDIT,
|
||||
subsystem = LOG_SUBSYSTEM_OBSERVABILITY,
|
||||
state = "system_started",
|
||||
"Audit observability state updated"
|
||||
"audit observability state"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -249,7 +249,7 @@ impl AuditMetrics {
|
||||
component = LOG_COMPONENT_AUDIT,
|
||||
subsystem = LOG_SUBSYSTEM_OBSERVABILITY,
|
||||
state = "metrics_reset",
|
||||
"Audit observability state updated"
|
||||
"audit observability state"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -266,7 +266,7 @@ impl AuditRuntimeView {
|
||||
subsystem = LOG_SUBSYSTEM_PIPELINE,
|
||||
target_id = %target_id,
|
||||
state = "enabled",
|
||||
"Changed audit target state"
|
||||
"audit target state"
|
||||
);
|
||||
Ok(())
|
||||
} else {
|
||||
@@ -283,7 +283,7 @@ impl AuditRuntimeView {
|
||||
subsystem = LOG_SUBSYSTEM_PIPELINE,
|
||||
target_id = %target_id,
|
||||
state = "disabled",
|
||||
"Changed audit target state"
|
||||
"audit target state"
|
||||
);
|
||||
Ok(())
|
||||
} else {
|
||||
@@ -300,7 +300,7 @@ impl AuditRuntimeView {
|
||||
subsystem = LOG_SUBSYSTEM_PIPELINE,
|
||||
target_id = %target_id,
|
||||
state = "removed",
|
||||
"Changed audit target state"
|
||||
"audit target state"
|
||||
);
|
||||
Ok(())
|
||||
} else {
|
||||
@@ -323,7 +323,7 @@ impl AuditRuntimeView {
|
||||
subsystem = LOG_SUBSYSTEM_PIPELINE,
|
||||
target_id = %target_id,
|
||||
state = "upserted",
|
||||
"Changed audit target state"
|
||||
"audit target state"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -349,7 +349,7 @@ impl AuditRuntimeFacade {
|
||||
subsystem = LOG_SUBSYSTEM_PIPELINE,
|
||||
target_id = %target.id(),
|
||||
replay_key = %key,
|
||||
"Delivered queued audit event"
|
||||
"audit replay delivery"
|
||||
);
|
||||
observability::record_target_success();
|
||||
}
|
||||
@@ -361,7 +361,7 @@ impl AuditRuntimeFacade {
|
||||
subsystem = LOG_SUBSYSTEM_PIPELINE,
|
||||
target_id = %target.id(),
|
||||
reason = "not_connected",
|
||||
"Retrying queued audit event delivery"
|
||||
"audit replay delivery"
|
||||
);
|
||||
}
|
||||
rustfs_targets::TargetError::Timeout(_) => {
|
||||
@@ -371,7 +371,7 @@ impl AuditRuntimeFacade {
|
||||
subsystem = LOG_SUBSYSTEM_PIPELINE,
|
||||
target_id = %target.id(),
|
||||
reason = "timeout",
|
||||
"Retrying queued audit event delivery"
|
||||
"audit replay delivery"
|
||||
);
|
||||
}
|
||||
_ => {}
|
||||
@@ -383,7 +383,7 @@ impl AuditRuntimeFacade {
|
||||
subsystem = LOG_SUBSYSTEM_PIPELINE,
|
||||
target_id = %target.id(),
|
||||
reason = %reason,
|
||||
"Dropped queued audit payload"
|
||||
"audit replay delivery"
|
||||
);
|
||||
observability::record_target_failure();
|
||||
}
|
||||
@@ -395,7 +395,7 @@ impl AuditRuntimeFacade {
|
||||
target_id = %target.id(),
|
||||
error = %error,
|
||||
reason = "permanent_failure",
|
||||
"Queued audit payload failed permanently"
|
||||
"audit replay delivery"
|
||||
);
|
||||
target.record_final_failure();
|
||||
observability::record_target_failure();
|
||||
@@ -408,7 +408,7 @@ impl AuditRuntimeFacade {
|
||||
target_id = %target.id(),
|
||||
replay_key = %key,
|
||||
reason = "retry_exhausted",
|
||||
"Dropped queued audit payload after retry exhaustion"
|
||||
"audit replay delivery"
|
||||
);
|
||||
target.record_final_failure();
|
||||
observability::record_target_failure();
|
||||
@@ -422,7 +422,7 @@ impl AuditRuntimeFacade {
|
||||
replay_key = %key,
|
||||
error = %error,
|
||||
reason = "unreadable_entry",
|
||||
"Skipped unreadable audit store entry"
|
||||
"audit replay delivery"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -436,7 +436,7 @@ impl AuditRuntimeFacade {
|
||||
subsystem = LOG_SUBSYSTEM_PIPELINE,
|
||||
target_id = %target_id,
|
||||
replay_enabled = true,
|
||||
"Audit replay stream started"
|
||||
"audit replay stream"
|
||||
);
|
||||
} else {
|
||||
debug!(
|
||||
@@ -446,7 +446,7 @@ impl AuditRuntimeFacade {
|
||||
target_id = %target_id,
|
||||
replay_enabled = false,
|
||||
reason = "no_store_configured",
|
||||
"Audit replay stream skipped"
|
||||
"audit replay stream"
|
||||
);
|
||||
}
|
||||
}),
|
||||
|
||||
@@ -198,7 +198,7 @@ impl AuditRegistry {
|
||||
target_type = %target_type,
|
||||
target_id = %target_id,
|
||||
registry_key = %key,
|
||||
"Created audit target registry key"
|
||||
"audit target registry state"
|
||||
);
|
||||
key.to_string()
|
||||
}
|
||||
@@ -221,7 +221,7 @@ impl AuditRegistry {
|
||||
target_type = %target_type,
|
||||
target_id = %target_id,
|
||||
state = "enabled",
|
||||
"Audit target registry state changed"
|
||||
"audit target registry state"
|
||||
);
|
||||
Ok(())
|
||||
} else {
|
||||
@@ -250,7 +250,7 @@ impl AuditRegistry {
|
||||
target_type = %target_type,
|
||||
target_id = %target_id,
|
||||
state = "disabled",
|
||||
"Audit target registry state changed"
|
||||
"audit target registry state"
|
||||
);
|
||||
Ok(())
|
||||
} else {
|
||||
|
||||
@@ -120,7 +120,7 @@ impl AuditSystem {
|
||||
subsystem = LOG_SUBSYSTEM_SYSTEM,
|
||||
state = "targets_created",
|
||||
target_count = targets.len(),
|
||||
"Created audit targets"
|
||||
"audit system state"
|
||||
);
|
||||
|
||||
let activation = self.runtime_facade().activate_targets_with_replay(targets).await;
|
||||
@@ -159,7 +159,7 @@ impl AuditSystem {
|
||||
component = LOG_COMPONENT_AUDIT,
|
||||
subsystem = LOG_SUBSYSTEM_SYSTEM,
|
||||
state = "starting",
|
||||
"Starting audit system"
|
||||
"audit system state"
|
||||
);
|
||||
|
||||
// Record system start
|
||||
@@ -268,7 +268,7 @@ impl AuditSystem {
|
||||
component = LOG_COMPONENT_AUDIT,
|
||||
subsystem = LOG_SUBSYSTEM_SYSTEM,
|
||||
state = "stopping",
|
||||
"Stopping audit system"
|
||||
"audit system state"
|
||||
);
|
||||
|
||||
// Stop all stream tasks first
|
||||
@@ -441,7 +441,7 @@ impl AuditSystem {
|
||||
component = LOG_COMPONENT_AUDIT,
|
||||
subsystem = LOG_SUBSYSTEM_SYSTEM,
|
||||
state = "reloading",
|
||||
"Reloading audit configuration"
|
||||
"audit config reload"
|
||||
);
|
||||
|
||||
observability::record_config_reload();
|
||||
@@ -465,7 +465,7 @@ impl AuditSystem {
|
||||
component = LOG_COMPONENT_AUDIT,
|
||||
subsystem = LOG_SUBSYSTEM_SYSTEM,
|
||||
state = "reloaded",
|
||||
"Reloaded audit configuration"
|
||||
"audit config reload"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -513,7 +513,7 @@ fn info_audit_state(state: &str, reason: Option<&str>, target_count: Option<usiz
|
||||
state,
|
||||
reason = reason.unwrap_or_default(),
|
||||
target_count = target_count.unwrap_or_default(),
|
||||
"Changed audit system state"
|
||||
"audit system state"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -526,7 +526,7 @@ fn debug_audit_state(state: &str, reason: Option<&str>, error: Option<&str>, tar
|
||||
reason = reason.unwrap_or_default(),
|
||||
error = error.unwrap_or_default(),
|
||||
target_count,
|
||||
"Observed audit system state"
|
||||
"audit system state"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -537,7 +537,7 @@ fn warn_audit_state(state: &str, reason: Option<&str>) {
|
||||
subsystem = LOG_SUBSYSTEM_SYSTEM,
|
||||
state,
|
||||
reason = reason.unwrap_or_default(),
|
||||
"Audit system state transition skipped"
|
||||
"audit system state"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -77,7 +77,7 @@ use tokio::sync::mpsc::{Receiver, Sender};
|
||||
use tokio::sync::{RwLock, mpsc};
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, warn};
|
||||
use tracing::{debug, error, warn};
|
||||
use uuid::Uuid;
|
||||
use xxhash_rust::xxh64;
|
||||
|
||||
@@ -762,13 +762,13 @@ impl TransitionState {
|
||||
"Transition compensation backfill failed"
|
||||
);
|
||||
} else {
|
||||
info!(
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket = %bucket,
|
||||
state = "completed",
|
||||
"Completed transition compensation backfill"
|
||||
"Transition compensation completed"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -958,7 +958,7 @@ impl TransitionState {
|
||||
|
||||
pub async fn init(api: Arc<ECStore>) {
|
||||
let (configured, absolute_max, n) = resolve_transition_worker_count();
|
||||
info!(
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_WORKER_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
@@ -968,7 +968,7 @@ impl TransitionState {
|
||||
transition_queue_capacity = GLOBAL_TransitionState.transition_queue_capacity,
|
||||
transition_queue_send_timeout_ms = GLOBAL_TransitionState.transition_queue_send_timeout.as_millis() as u64,
|
||||
state = "configured",
|
||||
"Lifecycle worker state resolved"
|
||||
"Lifecycle worker configuration resolved"
|
||||
);
|
||||
|
||||
//let mut transition_state = GLOBAL_TransitionState.write().await;
|
||||
@@ -1158,7 +1158,7 @@ impl TransitionState {
|
||||
GLOBAL_TransitionState.num_workers.store(current_workers, Ordering::SeqCst);
|
||||
GLOBAL_TransitionState.record_scanner_transition_state();
|
||||
|
||||
info!(
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_WORKER_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
@@ -1169,7 +1169,7 @@ impl TransitionState {
|
||||
current_transition_workers = current_workers,
|
||||
pruned_finished_transition_workers = pruned_finished_workers,
|
||||
state = "resized",
|
||||
"Lifecycle worker state updated"
|
||||
"Lifecycle worker pool resized"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -2008,14 +2008,14 @@ pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn {
|
||||
Arc::new(move || {
|
||||
let tier = tier.clone();
|
||||
Box::pin(async move {
|
||||
info!(
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_TIER_AUDIT,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
tier = %tier,
|
||||
bytes = bytes,
|
||||
state = "transition_completed",
|
||||
"Lifecycle tier transition audit completed"
|
||||
"Lifecycle tier transition recorded"
|
||||
);
|
||||
})
|
||||
})
|
||||
|
||||
@@ -29,7 +29,11 @@ use tokio::io::AsyncRead;
|
||||
use tokio::spawn;
|
||||
use tokio::time::timeout;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{error, info, warn};
|
||||
use tracing::{error, warn};
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_METACACHE: &str = "metacache";
|
||||
const EVENT_METACACHE_LISTING: &str = "metacache_listing";
|
||||
|
||||
pub type AgreedFn = Box<dyn Fn(MetaCacheEntry) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
|
||||
pub type PartialFn =
|
||||
@@ -114,6 +118,9 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
|
||||
return Err(DiskError::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
let log_bucket = opts.bucket.clone();
|
||||
let log_path = opts.path.clone();
|
||||
|
||||
let mut jobs: Vec<tokio::task::JoinHandle<std::result::Result<(), DiskError>>> = Vec::new();
|
||||
let mut readers = Vec::with_capacity(opts.disks.len());
|
||||
let fds = opts.fallback_disks.iter().flatten().cloned().collect::<VecDeque<_>>();
|
||||
@@ -180,7 +187,17 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
|
||||
match disk.walk_dir(wakl_opts, &mut wr).await {
|
||||
Ok(_res) => {}
|
||||
Err(err) => {
|
||||
info!("walk dir err {:?}", &err);
|
||||
warn!(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
disk_index = disk_idx,
|
||||
state = "walk_dir_failed",
|
||||
error = ?err,
|
||||
"Metacache walk_dir failed"
|
||||
);
|
||||
last_err = Some(err);
|
||||
need_fallback = true;
|
||||
}
|
||||
@@ -205,7 +222,16 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
|
||||
}
|
||||
|
||||
let Some(disk) = disk_op else {
|
||||
warn!("list_path_raw: fallback disk is none");
|
||||
warn!(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
disk_index = disk_idx,
|
||||
state = "fallback_disk_missing",
|
||||
"Metacache fallback disk missing"
|
||||
);
|
||||
let err = last_err.unwrap_or(DiskError::DiskNotFound);
|
||||
record_producer_error(&producer_errs_clone, disk_idx, &err);
|
||||
return Err(err);
|
||||
@@ -234,7 +260,17 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
|
||||
last_err = None;
|
||||
}
|
||||
Err(err) => {
|
||||
error!("walk dir2 err {:?}", &err);
|
||||
error!(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
disk_index = disk_idx,
|
||||
state = "fallback_walk_dir_failed",
|
||||
error = ?err,
|
||||
"Metacache fallback walk_dir failed"
|
||||
);
|
||||
last_err = Some(err);
|
||||
}
|
||||
}
|
||||
@@ -355,11 +391,15 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
|
||||
)
|
||||
.increment(1);
|
||||
warn!(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
drive = %endpoint,
|
||||
bucket = %opts.bucket,
|
||||
path = %opts.path,
|
||||
timeout_ms = peek_timeout.as_millis(),
|
||||
"list_path_raw reader peek timed out; excluding drive from current merge"
|
||||
state = "peek_timed_out",
|
||||
"Metacache reader peek timed out"
|
||||
);
|
||||
let (detached_rd, write_half) = tokio::io::duplex(1);
|
||||
drop(write_half);
|
||||
@@ -439,8 +479,14 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
|
||||
});
|
||||
|
||||
error!(
|
||||
"list_path_raw: has_err > 0 && has_err > opts.disks.len() - opts.min_disks break, err: {:?}",
|
||||
&combined_err.join(", ")
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts.bucket,
|
||||
path = %opts.path,
|
||||
state = "quorum_failed",
|
||||
error = %combined_err.join(", "),
|
||||
"Metacache listing quorum failed"
|
||||
);
|
||||
return Err(DiskError::other(combined_err.join(", ")));
|
||||
}
|
||||
@@ -493,7 +539,16 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
|
||||
});
|
||||
|
||||
if let Err(err) = revjob.await.map_err(std::io::Error::other)? {
|
||||
error!("list_path_raw: revjob err {:?}", err);
|
||||
error!(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %log_bucket,
|
||||
path = %log_path,
|
||||
state = "merge_job_failed",
|
||||
error = ?err,
|
||||
"Metacache merge job failed"
|
||||
);
|
||||
cancel_rx.cancel();
|
||||
for job in jobs {
|
||||
job.abort();
|
||||
@@ -521,9 +576,23 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(err)) => {
|
||||
if matches!(err, DiskError::FileNotFound | DiskError::VolumeNotFound) {
|
||||
warn!("list_path_raw producer missing path {:?}", err);
|
||||
warn!(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
state = "producer_missing_path",
|
||||
error = ?err,
|
||||
"Metacache producer missing path"
|
||||
);
|
||||
} else {
|
||||
error!("list_path_raw producer err {:?}", err);
|
||||
error!(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
state = "producer_failed",
|
||||
error = ?err,
|
||||
"Metacache producer failed"
|
||||
);
|
||||
}
|
||||
job_errs.push(err);
|
||||
}
|
||||
@@ -531,7 +600,14 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
|
||||
if err.is_cancelled() {
|
||||
continue;
|
||||
}
|
||||
error!("list_path_raw join err {:?}", err);
|
||||
error!(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
state = "producer_join_failed",
|
||||
error = ?err,
|
||||
"Metacache producer join failed"
|
||||
);
|
||||
job_errs.push(err.into());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -472,7 +472,15 @@ fn resolve_local_disk_root(ep_path: &str) -> Result<PathBuf> {
|
||||
|
||||
impl LocalDisk {
|
||||
pub async fn new(ep: &Endpoint, cleanup: bool) -> Result<Self> {
|
||||
debug!("Creating local disk");
|
||||
debug!(
|
||||
event = EVENT_DISK_LOCAL_STARTUP_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
endpoint = %ep,
|
||||
state = "create_started",
|
||||
cleanup,
|
||||
"Local disk creation started"
|
||||
);
|
||||
let endpoint_path = ep.get_file_path();
|
||||
let root = resolve_local_disk_root(&endpoint_path).inspect_err(|err| {
|
||||
log_startup_disk_error("resolve_local_disk_root", Path::new(&endpoint_path), err);
|
||||
@@ -501,13 +509,21 @@ impl LocalDisk {
|
||||
root = ?root,
|
||||
state = "failed",
|
||||
error = ?err,
|
||||
"Disk local startup cleanup failed"
|
||||
"Local disk startup cleanup failed"
|
||||
);
|
||||
}
|
||||
|
||||
// Use optimized path resolution instead of absolutize_virtually
|
||||
let format_path = root.join(RUSTFS_META_BUCKET).join(super::FORMAT_CONFIG_FILE);
|
||||
debug!("format_path: {:?}", format_path);
|
||||
debug!(
|
||||
event = EVENT_DISK_LOCAL_STARTUP_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
root = ?root,
|
||||
format_path = ?format_path,
|
||||
state = "format_path_resolved",
|
||||
"Local disk format path resolved"
|
||||
);
|
||||
let (format_data, format_meta) = read_file_exists(&format_path).await.inspect_err(|err| {
|
||||
log_startup_disk_error("read_format_json", &format_path, err);
|
||||
})?;
|
||||
@@ -639,7 +655,15 @@ impl LocalDisk {
|
||||
|
||||
let root = disk.root.clone();
|
||||
tokio::spawn(Self::cleanup_deleted_objects_loop(root, exit_rx));
|
||||
debug!("LocalDisk created: {:?}", disk);
|
||||
debug!(
|
||||
event = EVENT_DISK_LOCAL_STARTUP_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
endpoint = %disk.endpoint,
|
||||
root = ?disk.root,
|
||||
state = "created",
|
||||
"Local disk created"
|
||||
);
|
||||
Ok(disk)
|
||||
}
|
||||
|
||||
|
||||
+323
-43
@@ -71,6 +71,12 @@ use time::{Duration, OffsetDateTime};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_POOLS: &str = "pools";
|
||||
const EVENT_DECOMMISSION_STATE: &str = "decommission_state";
|
||||
const EVENT_DECOMMISSION_BUCKET: &str = "decommission_bucket";
|
||||
const EVENT_DECOMMISSION_ENTRY: &str = "decommission_entry";
|
||||
|
||||
pub const POOL_META_NAME: &str = "pool.bin";
|
||||
pub const POOL_META_FORMAT: u16 = 1;
|
||||
pub const POOL_META_VERSION: u16 = 1;
|
||||
@@ -1280,7 +1286,15 @@ impl ECStore {
|
||||
take_decommission_canceler(cancelers.as_mut_slice(), idx)
|
||||
};
|
||||
if !cancel_decommission_canceler(canceler) {
|
||||
warn!("decommission_cancel: no active canceler found for pool {}", idx);
|
||||
warn!(
|
||||
event = EVENT_DECOMMISSION_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
state = "cancel_skipped",
|
||||
reason = "no_active_canceler",
|
||||
"Decommission cancel skipped"
|
||||
);
|
||||
}
|
||||
|
||||
if should_reload_pool_meta && let Some(notification_sys) = get_global_notification_sys() {
|
||||
@@ -1338,7 +1352,15 @@ impl ECStore {
|
||||
let store = store.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Err(err) = store.do_decommission_in_routine(canceler, idx).await {
|
||||
error!("decommission: routine failed for idx {}: {err}", idx);
|
||||
error!(
|
||||
event = EVENT_DECOMMISSION_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
state = "routine_failed",
|
||||
error = %err,
|
||||
"Decommission routine failed"
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -1350,7 +1372,14 @@ impl ECStore {
|
||||
pub async fn decommission(&self, rx: CancellationToken, indices: Vec<usize>) -> Result<()> {
|
||||
let indices = dedup_indices(&indices);
|
||||
|
||||
warn!("decommission: {:?}", indices);
|
||||
info!(
|
||||
event = EVENT_DECOMMISSION_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_indices = ?indices,
|
||||
state = "requested",
|
||||
"Decommission requested"
|
||||
);
|
||||
validate_start_decommission_request(&indices, self.single_pool())?;
|
||||
|
||||
ensure_decommission_not_rebalancing(self.is_rebalance_conflicting_with_decommission().await)?;
|
||||
@@ -1363,8 +1392,13 @@ impl ECStore {
|
||||
for idx in indices {
|
||||
if let Err(cancel_err) = self.decommission_cancel(idx).await {
|
||||
error!(
|
||||
"decommission: failed to rollback decommission state for idx {} after spawn error: {:?}",
|
||||
idx, cancel_err
|
||||
event = EVENT_DECOMMISSION_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
state = "rollback_failed",
|
||||
error = ?cancel_err,
|
||||
"Decommission rollback failed after spawn error"
|
||||
);
|
||||
if rollback_err.is_none() {
|
||||
rollback_err = Some(Error::other(format!("decommission rollback failed for idx {idx}: {cancel_err}")));
|
||||
@@ -1390,10 +1424,28 @@ impl ECStore {
|
||||
lock_retention: Option<DefaultRetention>,
|
||||
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
|
||||
) -> Result<()> {
|
||||
warn!("decommission_entry: {} {}", &bucket, &entry.name);
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
bucket = %bucket,
|
||||
object = %entry.name,
|
||||
state = "started",
|
||||
"Decommission entry started"
|
||||
);
|
||||
wk.give().await;
|
||||
if entry.is_dir() {
|
||||
warn!("decommission_entry: skip dir {}", &entry.name);
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
bucket = %bucket,
|
||||
object = %entry.name,
|
||||
state = "skipped_directory",
|
||||
"Decommission entry skipped directory"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -1426,7 +1478,16 @@ impl ECStore {
|
||||
if should_skip_decommission_delete_marker(version, remaining_versions, replication_config.is_some()) {
|
||||
//
|
||||
decommissioned += 1;
|
||||
info!("decommission_pool: DELETE marked object with no other non-current versions will be skipped");
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
bucket = %bucket,
|
||||
object = %version.name,
|
||||
state = "skipped_delete_marker",
|
||||
"Decommission delete marker skipped"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1447,8 +1508,16 @@ impl ECStore {
|
||||
{
|
||||
if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) {
|
||||
warn!(
|
||||
"decommission_pool: ignore delete-marker copy for {}/{} version {:?}: {:?}",
|
||||
&bucket, &version.name, &version_id, &err
|
||||
event = EVENT_DECOMMISSION_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
bucket = %bucket,
|
||||
object = %version.name,
|
||||
version_id = ?version_id,
|
||||
state = "ignored_delete_marker_copy",
|
||||
error = ?err,
|
||||
"Decommission delete marker copy ignored"
|
||||
);
|
||||
ignore = true;
|
||||
cleanup_ignored = true;
|
||||
@@ -1463,7 +1532,16 @@ impl ECStore {
|
||||
if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) {
|
||||
decommissioned += 1;
|
||||
}
|
||||
info!("decommission_pool: ignore {}", &version.name);
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
bucket = %bucket,
|
||||
object = %version.name,
|
||||
state = "ignored",
|
||||
"Decommission entry ignored"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1483,9 +1561,17 @@ impl ECStore {
|
||||
decommissioned += 1;
|
||||
}
|
||||
|
||||
info!(
|
||||
"decommission_pool: DecomCopyDeleteMarker {} {} {:?} {:?}",
|
||||
&bucket, &version.name, &version_id, error
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
bucket = %bucket,
|
||||
object = %version.name,
|
||||
version_id = ?version_id,
|
||||
result = ?error,
|
||||
state = "delete_marker_copied",
|
||||
"Decommission delete marker copied"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
@@ -1571,8 +1657,15 @@ impl ECStore {
|
||||
}
|
||||
|
||||
warn!(
|
||||
"decommission_pool: decommission_object done {}/{} {}",
|
||||
&bucket_name, &object_name, &version.name
|
||||
event = EVENT_DECOMMISSION_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
bucket = %bucket_name,
|
||||
object = %object_name,
|
||||
version = %version.name,
|
||||
state = "object_migrated",
|
||||
"Decommission object migrated"
|
||||
);
|
||||
|
||||
failure = false;
|
||||
@@ -1583,7 +1676,16 @@ impl ECStore {
|
||||
if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) {
|
||||
decommissioned += 1;
|
||||
}
|
||||
info!("decommission_pool: ignore {}", &version.name);
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
bucket = %bucket,
|
||||
object = %version.name,
|
||||
state = "ignored",
|
||||
"Decommission entry ignored"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1624,12 +1726,17 @@ impl ECStore {
|
||||
resolve_decommission_entry_cleanup_delete_result(cleanup_result, bucket.as_str(), entry.name.as_str())?
|
||||
} else if decommissioned != fivs.versions.len() || expired > 0 {
|
||||
warn!(
|
||||
"decommission_pool: source object retained for {}/{} because only {}/{} versions were decommissioned and {} expired by lifecycle",
|
||||
&bucket,
|
||||
&entry.name,
|
||||
event = EVENT_DECOMMISSION_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
bucket = %bucket,
|
||||
object = %entry.name,
|
||||
decommissioned,
|
||||
fivs.versions.len(),
|
||||
expired
|
||||
total_versions = fivs.versions.len(),
|
||||
expired,
|
||||
state = "source_retained",
|
||||
"Decommission source object retained"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1667,7 +1774,16 @@ impl ECStore {
|
||||
}
|
||||
}
|
||||
|
||||
warn!("decommission_pool: decommission_entry done {} {}", &bucket, &entry.name);
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
bucket = %bucket,
|
||||
object = %entry.name,
|
||||
state = "completed",
|
||||
"Decommission entry completed"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1705,7 +1821,16 @@ impl ECStore {
|
||||
for (set_idx, set) in pool.disk_set.iter().enumerate() {
|
||||
wk.clone().take().await;
|
||||
|
||||
warn!("decommission_pool: decommission_pool {} {}", set_idx, &bi.name);
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
set_index = set_idx,
|
||||
bucket = %bi.name,
|
||||
state = "listing_worker_started",
|
||||
"Decommission listing worker started"
|
||||
);
|
||||
|
||||
let decommission_entry: ListCallback = Arc::new({
|
||||
let this = Arc::clone(self);
|
||||
@@ -1753,23 +1878,69 @@ impl ECStore {
|
||||
let worker = tokio::spawn(async move {
|
||||
loop {
|
||||
if rx_clone.is_cancelled() {
|
||||
warn!("decommission_pool: cancel {}", set_id);
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
set_index = set_id,
|
||||
bucket = %bi.name,
|
||||
state = "listing_worker_cancelled",
|
||||
"Decommission listing worker cancelled"
|
||||
);
|
||||
break;
|
||||
}
|
||||
warn!("decommission_pool: list_objects_to_decommission {} {}", set_id, &bi.name);
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
set_index = set_id,
|
||||
bucket = %bi.name,
|
||||
state = "listing_started",
|
||||
"Decommission listing started"
|
||||
);
|
||||
|
||||
match set
|
||||
.list_objects_to_decommission(rx_clone.clone(), bi.clone(), decommission_entry.clone())
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
warn!("decommission_pool: list_objects_to_decommission {} done", set_id);
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
set_index = set_id,
|
||||
bucket = %bi.name,
|
||||
state = "listing_completed",
|
||||
"Decommission listing completed"
|
||||
);
|
||||
break;
|
||||
}
|
||||
Err(err) => {
|
||||
error!("decommission_pool: list_objects_to_decommission {} err {:?}", set_id, &err);
|
||||
error!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
set_index = set_id,
|
||||
bucket = %bi.name,
|
||||
state = "listing_failed",
|
||||
error = ?err,
|
||||
"Decommission listing failed"
|
||||
);
|
||||
if is_err_bucket_not_found(&err) {
|
||||
warn!("decommission_pool: list_objects_to_decommission {} volume not found", set_id);
|
||||
warn!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
set_index = set_id,
|
||||
bucket = %bi.name,
|
||||
state = "listing_bucket_missing",
|
||||
"Decommission listing bucket missing"
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -1783,7 +1954,15 @@ impl ECStore {
|
||||
listing_workers.push((set_id, worker));
|
||||
}
|
||||
|
||||
warn!("decommission_pool: decommission_pool wait {} {}", idx, &bi.name);
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
bucket = %bi.name,
|
||||
state = "waiting_for_workers",
|
||||
"Decommission waiting for workers"
|
||||
);
|
||||
|
||||
let mut listing_worker_error = None;
|
||||
for (set_id, worker) in listing_workers {
|
||||
@@ -1807,11 +1986,28 @@ impl ECStore {
|
||||
}
|
||||
|
||||
if let Err(err) = decommission_cancel_signal_result(rx.is_cancelled()) {
|
||||
warn!("decommission_pool: canceled after wait {} {}", idx, &bi.name);
|
||||
warn!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
bucket = %bi.name,
|
||||
state = "cancelled_after_wait",
|
||||
error = %err,
|
||||
"Decommission bucket cancelled after wait"
|
||||
);
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
warn!("decommission_pool: decommission_pool done {} {}", idx, &bi.name);
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
bucket = %bi.name,
|
||||
state = "completed",
|
||||
"Decommission bucket completed"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1821,7 +2017,14 @@ impl ECStore {
|
||||
defer!(|| async {
|
||||
let mut cancelers = self.decommission_cancelers.write().await;
|
||||
if take_decommission_canceler(cancelers.as_mut_slice(), idx).is_none() {
|
||||
warn!("decommission: canceler already cleared for pool {}", idx);
|
||||
warn!(
|
||||
event = EVENT_DECOMMISSION_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
state = "canceler_already_cleared",
|
||||
"Decommission canceler already cleared"
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1830,7 +2033,14 @@ impl ECStore {
|
||||
let (final_state, canceled, cmd_line) = {
|
||||
let pool_meta = self.pool_meta.read().await;
|
||||
let Some(pool) = pool_meta.pools.get(idx) else {
|
||||
error!("decommission: pool metadata missing for idx {}", idx);
|
||||
error!(
|
||||
event = EVENT_DECOMMISSION_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
state = "pool_metadata_missing",
|
||||
"Decommission pool metadata missing"
|
||||
);
|
||||
return Err(Error::other(format!(
|
||||
"failed to resolve decommission final state: pool metadata missing for idx {idx}"
|
||||
)));
|
||||
@@ -1849,29 +2059,75 @@ impl ECStore {
|
||||
};
|
||||
|
||||
if let Err(err) = result {
|
||||
error!("decom err {:?}", &err);
|
||||
error!(
|
||||
event = EVENT_DECOMMISSION_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
state = "background_failed",
|
||||
error = ?err,
|
||||
"Decommission background routine failed"
|
||||
);
|
||||
|
||||
if is_err_operation_canceled(&err) || should_preserve_decommission_canceled_state(canceled, rx.is_cancelled()) {
|
||||
warn!("decommission: canceled for pool {}, preserving canceled state", cmd_line);
|
||||
warn!(
|
||||
event = EVENT_DECOMMISSION_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
cmd_line = %cmd_line,
|
||||
state = "cancelled_preserved",
|
||||
"Decommission cancelled; preserving canceled state"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
resolve_decommission_terminal_mark_after_error_result(self.decommission_failed(idx).await, idx, &err)?;
|
||||
warn!("decommission: decommission_failed {}", idx);
|
||||
warn!(
|
||||
event = EVENT_DECOMMISSION_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
state = "marked_failed",
|
||||
"Decommission marked failed"
|
||||
);
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
warn!("decommission: decommission_in_background complete {}", idx);
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
state = "background_complete",
|
||||
"Decommission background routine completed"
|
||||
);
|
||||
|
||||
if should_preserve_decommission_canceled_state(canceled, rx.is_cancelled()) {
|
||||
warn!("decommission: canceled for pool {}, skipping terminal state overwrite", cmd_line);
|
||||
warn!(
|
||||
event = EVENT_DECOMMISSION_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
cmd_line = %cmd_line,
|
||||
state = "terminal_state_preserved",
|
||||
"Decommission terminal state preserved after cancellation"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
match final_state {
|
||||
DecommissionFinalState::Complete => {
|
||||
warn!("Decommissioning complete for pool {}, verifying for any pending objects", cmd_line);
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
cmd_line = %cmd_line,
|
||||
state = "verifying_completion",
|
||||
"Decommission completion verification started"
|
||||
);
|
||||
if let Err(err) = self.check_after_decommission(idx).await {
|
||||
resolve_decommission_terminal_mark_result(self.decommission_failed(idx).await, "failed", &cmd_line)?;
|
||||
return Err(Error::other(format!(
|
||||
@@ -1879,16 +2135,40 @@ impl ECStore {
|
||||
)));
|
||||
}
|
||||
|
||||
warn!("Decommissioning complete for pool {}, marking completed state", cmd_line);
|
||||
info!(
|
||||
event = EVENT_DECOMMISSION_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
cmd_line = %cmd_line,
|
||||
state = "marking_completed",
|
||||
"Decommission marking completed state"
|
||||
);
|
||||
resolve_decommission_terminal_mark_result(self.complete_decommission(idx).await, "completed", &cmd_line)?;
|
||||
}
|
||||
DecommissionFinalState::Failed => {
|
||||
warn!("Decommissioning finished with failed items for pool {}, marking failed state", cmd_line);
|
||||
warn!(
|
||||
event = EVENT_DECOMMISSION_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
cmd_line = %cmd_line,
|
||||
state = "marking_failed",
|
||||
"Decommission marking failed state"
|
||||
);
|
||||
resolve_decommission_terminal_mark_result(self.decommission_failed(idx).await, "failed", &cmd_line)?;
|
||||
}
|
||||
}
|
||||
|
||||
warn!("Decommissioning complete for pool {}", cmd_line);
|
||||
info!(
|
||||
event = EVENT_DECOMMISSION_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
cmd_line = %cmd_line,
|
||||
state = "completed",
|
||||
"Decommission completed"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
+176
-30
@@ -858,7 +858,14 @@ impl ECStore {
|
||||
"init_rebalance_meta",
|
||||
)?;
|
||||
|
||||
info!("init_rebalance_meta: rebalance meta saved");
|
||||
info!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
state = "metadata_initialized",
|
||||
bucket_count = bucktes.len(),
|
||||
"Rebalance metadata initialized"
|
||||
);
|
||||
|
||||
let id = meta.id.clone();
|
||||
|
||||
@@ -898,9 +905,16 @@ impl ECStore {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
pub async fn next_rebal_bucket(&self, pool_index: usize) -> Result<Option<String>> {
|
||||
info!("next_rebal_bucket: pool_index: {}", pool_index);
|
||||
let rebalance_meta = self.rebalance_meta.read().await;
|
||||
info!("next_rebal_bucket: rebalance_meta: {:?}", rebalance_meta);
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
has_meta = rebalance_meta.is_some(),
|
||||
state = "next_bucket_lookup",
|
||||
"Rebalance next bucket lookup"
|
||||
);
|
||||
resolve_next_rebalance_bucket(rebalance_meta.as_ref(), pool_index)
|
||||
}
|
||||
|
||||
@@ -931,20 +945,38 @@ impl ECStore {
|
||||
let rebalance_meta = self.rebalance_meta.read().await;
|
||||
if let Some(meta) = rebalance_meta.as_ref() {
|
||||
meta.pool_stats.iter().enumerate().for_each(|(i, v)| {
|
||||
info!(
|
||||
"is_rebalance_started: pool_index: {}, participating: {:?}, status: {:?}",
|
||||
i, v.participating, v.info.status
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index = i,
|
||||
participating = v.participating,
|
||||
status = ?v.info.status,
|
||||
state = "status_inspected",
|
||||
"Rebalance status inspected"
|
||||
);
|
||||
});
|
||||
|
||||
let started = is_rebalance_conflicting_with_decommission(meta);
|
||||
if started {
|
||||
info!("is_rebalance_started: rebalance started");
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
state = "running",
|
||||
"Rebalance is running"
|
||||
);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
info!("is_rebalance_started: rebalance not started");
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
state = "not_running",
|
||||
"Rebalance is not running"
|
||||
);
|
||||
false
|
||||
}
|
||||
|
||||
@@ -1108,15 +1140,23 @@ impl ECStore {
|
||||
workers_started += 1;
|
||||
tokio::spawn(async move {
|
||||
if let Err(err) = store.rebalance_buckets(rx_clone, pool_idx).await {
|
||||
error!("Rebalance failed for pool {}: {}", pool_idx, err);
|
||||
error!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index = pool_idx,
|
||||
state = "pool_failed",
|
||||
error = %err,
|
||||
"Rebalance pool failed"
|
||||
);
|
||||
} else {
|
||||
info!(
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index = pool_idx,
|
||||
state = "completed",
|
||||
"Completed rebalance pool"
|
||||
"Rebalance pool completed"
|
||||
);
|
||||
}
|
||||
});
|
||||
@@ -1139,7 +1179,8 @@ impl ECStore {
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
state = "started",
|
||||
"Started rebalance"
|
||||
worker_count = workers_started,
|
||||
"Rebalance started"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -1240,7 +1281,7 @@ impl ECStore {
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
state = "pool_started",
|
||||
"Started rebalance worker"
|
||||
"Rebalance worker started"
|
||||
);
|
||||
let mut final_result: Result<()> = Ok(());
|
||||
let mut deferred_buckets = HashSet::new();
|
||||
@@ -1267,7 +1308,15 @@ impl ECStore {
|
||||
let next_bucket = match self.next_rebal_bucket(pool_index).await {
|
||||
Ok(bucket) => bucket,
|
||||
Err(err) => {
|
||||
error!("next_rebal_bucket failed for pool {}: {:?}", pool_index, err);
|
||||
error!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
state = "next_bucket_failed",
|
||||
error = ?err,
|
||||
"Rebalance next bucket lookup failed"
|
||||
);
|
||||
final_result = Err(resolve_rebalance_terminal_error(
|
||||
err.clone(),
|
||||
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
|
||||
@@ -1294,7 +1343,16 @@ impl ECStore {
|
||||
) {
|
||||
Ok(outcome) => outcome,
|
||||
Err(err) => {
|
||||
error!("Error rebalancing bucket {}: {:?}", bucket, err);
|
||||
error!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
state = "bucket_failed",
|
||||
error = ?err,
|
||||
"Rebalance bucket failed"
|
||||
);
|
||||
final_result = Err(resolve_rebalance_terminal_error(
|
||||
err.clone(),
|
||||
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
|
||||
@@ -1308,7 +1366,16 @@ impl ECStore {
|
||||
let err = Error::other(format!(
|
||||
"rebalance bucket {bucket} deferred repeatedly due to transient object failures: {last_error}"
|
||||
));
|
||||
error!("Error rebalancing bucket {}: {:?}", bucket, err);
|
||||
error!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
state = "bucket_deferred_repeatedly",
|
||||
error = ?err,
|
||||
"Rebalance bucket failed after repeated deferral"
|
||||
);
|
||||
final_result = Err(resolve_rebalance_terminal_error(
|
||||
err.clone(),
|
||||
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
|
||||
@@ -1327,7 +1394,16 @@ impl ECStore {
|
||||
"Deferred rebalance bucket after transient object failures"
|
||||
);
|
||||
if let Err(err) = self.defer_rebalance_bucket(pool_index, bucket.clone(), last_error).await {
|
||||
error!("defer_rebalance_bucket failed for pool {}: {:?}", pool_index, err);
|
||||
error!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
state = "defer_failed",
|
||||
error = ?err,
|
||||
"Rebalance bucket defer failed"
|
||||
);
|
||||
final_result = Err(resolve_rebalance_terminal_error(
|
||||
err.clone(),
|
||||
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
|
||||
@@ -1347,7 +1423,15 @@ impl ECStore {
|
||||
"Completed rebalance bucket"
|
||||
);
|
||||
if let Err(err) = self.bucket_rebalance_done(pool_index, bucket).await {
|
||||
error!("bucket_rebalance_done failed for pool {}: {:?}", pool_index, err);
|
||||
error!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
state = "bucket_done_mark_failed",
|
||||
error = ?err,
|
||||
"Rebalance bucket completion mark failed"
|
||||
);
|
||||
final_result = Err(resolve_rebalance_terminal_error(
|
||||
err.clone(),
|
||||
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
|
||||
@@ -1374,7 +1458,7 @@ impl ECStore {
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
state = "pool_done",
|
||||
"Finished rebalance worker"
|
||||
"Rebalance worker finished"
|
||||
);
|
||||
|
||||
if final_result.is_ok()
|
||||
@@ -1388,7 +1472,14 @@ impl ECStore {
|
||||
{
|
||||
final_result = Err(err);
|
||||
}
|
||||
info!("Pool {} rebalancing is done2", pool_index);
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
state = "pool_result_returned",
|
||||
"Rebalance worker result returned"
|
||||
);
|
||||
final_result
|
||||
}
|
||||
|
||||
@@ -2591,7 +2682,15 @@ impl ECStore {
|
||||
ensure_valid_rebalance_pool_index(self.pools.len(), pool_index)?;
|
||||
|
||||
// Placeholder for actual bucket rebalance logic
|
||||
info!("Rebalancing bucket {} in pool {}", bucket, pool_index);
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
state = "entry_scan_started",
|
||||
"Rebalance bucket entry scan started"
|
||||
);
|
||||
|
||||
// TODO: other config
|
||||
// if bucket != RUSTFS_META_BUCKET{
|
||||
@@ -2765,9 +2864,14 @@ impl ECStore {
|
||||
|
||||
let pool = clone_first_arc(&self.pools, "save_rebalance_stats: no pools available")?;
|
||||
|
||||
info!(
|
||||
"save_rebalance_stats: save rebalance meta, pool_idx: {}, opt: {:?}, meta: {:?}",
|
||||
pool_idx, opt, meta_to_save
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index = pool_idx,
|
||||
save_opt = ?opt,
|
||||
state = "metadata_save_requested",
|
||||
"Rebalance metadata save requested"
|
||||
);
|
||||
let stage = format!("save_rebalance_stats for pool {pool_idx} opt {opt:?}");
|
||||
resolve_rebalance_meta_save_result(
|
||||
@@ -2838,12 +2942,27 @@ impl SetDisks {
|
||||
bucket: String,
|
||||
cb: ListCallback,
|
||||
) -> Result<()> {
|
||||
info!("list_objects_to_rebalance: start list_objects_to_rebalance");
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
bucket = %bucket,
|
||||
state = "started",
|
||||
"Rebalance listing started"
|
||||
);
|
||||
// Placeholder for actual object listing logic
|
||||
let (disks, _) = self.get_online_disks_with_healing(false).await;
|
||||
ensure_rebalance_listing_disks_available(!disks.is_empty(), &bucket)?;
|
||||
|
||||
info!("list_objects_to_rebalance: get online disks with healing");
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
bucket = %bucket,
|
||||
disk_count = disks.len(),
|
||||
state = "disks_resolved",
|
||||
"Rebalance listing disks resolved"
|
||||
);
|
||||
let listing_quorum = self.set_drive_count.div_ceil(2);
|
||||
|
||||
let resolver = MetadataResolutionParams {
|
||||
@@ -2863,7 +2982,14 @@ impl SetDisks {
|
||||
min_disks: listing_quorum,
|
||||
skip_walkdir_total_timeout: true,
|
||||
agreed: Some(Box::new(move |entry: MetaCacheEntry| {
|
||||
info!("list_objects_to_rebalance: agreed: {:?}", &entry.name);
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
entry = %entry.name,
|
||||
state = "agreed_entry",
|
||||
"Rebalance listing agreed entry"
|
||||
);
|
||||
Box::pin(cb1(entry))
|
||||
})),
|
||||
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
|
||||
@@ -2873,11 +2999,24 @@ impl SetDisks {
|
||||
|
||||
match entries.resolve(resolver) {
|
||||
Some(entry) => {
|
||||
info!("list_objects_to_rebalance: list_objects_to_decommission get {}", &entry.name);
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
entry = %entry.name,
|
||||
state = "resolved_partial_entry",
|
||||
"Rebalance listing resolved partial entry"
|
||||
);
|
||||
Box::pin(async move { cb(entry).await })
|
||||
}
|
||||
None => {
|
||||
info!("list_objects_to_rebalance: list_objects_to_decommission get none");
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
state = "partial_entry_missing",
|
||||
"Rebalance listing partial entry missing"
|
||||
);
|
||||
Box::pin(async {})
|
||||
}
|
||||
}
|
||||
@@ -2887,7 +3026,14 @@ impl SetDisks {
|
||||
)
|
||||
.await?;
|
||||
|
||||
info!("list_objects_to_rebalance: list_objects_to_rebalance done");
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
bucket = %bucket,
|
||||
state = "completed",
|
||||
"Rebalance listing completed"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,7 +65,7 @@ use tokio::{
|
||||
};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tonic::{Request, service::interceptor::InterceptedService, transport::Channel};
|
||||
use tracing::{debug, info, warn};
|
||||
use tracing::{debug, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
@@ -76,6 +76,10 @@ enum FailureHealthAction {
|
||||
|
||||
const REMOTE_DISK_OPEN_WRITE_MAX_ATTEMPTS: usize = 2;
|
||||
const REMOTE_DISK_OPEN_WRITE_RETRY_BACKOFF: Duration = Duration::from_millis(20);
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_REMOTE_DISK: &str = "remote_disk";
|
||||
const EVENT_REMOTE_DISK_HEALTH: &str = "remote_disk_health";
|
||||
const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc";
|
||||
|
||||
async fn copy_stream_with_buffer<R, W>(reader: &mut R, writer: &mut W, buffer_size: usize) -> io::Result<u64>
|
||||
where
|
||||
@@ -275,7 +279,16 @@ impl RemoteDisk {
|
||||
if initial_probe_ok {
|
||||
health.record_operation_success(&endpoint, "connectivity_probe_success");
|
||||
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
|
||||
warn!("Remote disk health check failed for {}: marking as faulty", addr);
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %endpoint,
|
||||
addr,
|
||||
state = "initial_probe_failed",
|
||||
result = "mark_faulty",
|
||||
"Remote disk initial health probe failed"
|
||||
);
|
||||
|
||||
// Start recovery monitoring
|
||||
let health_clone = Arc::clone(&health);
|
||||
@@ -291,7 +304,15 @@ impl RemoteDisk {
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = cancel_token.cancelled() => {
|
||||
debug!("Health monitoring cancelled for remote disk: {}", addr);
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %endpoint,
|
||||
addr,
|
||||
state = "monitor_cancelled",
|
||||
"Remote disk health monitor cancelled"
|
||||
);
|
||||
return;
|
||||
}
|
||||
_ = interval.tick() => {
|
||||
@@ -320,7 +341,16 @@ impl RemoteDisk {
|
||||
if Self::perform_connectivity_check(&addr).await.is_ok() {
|
||||
health.record_operation_success(&endpoint, "connectivity_probe_success");
|
||||
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
|
||||
warn!("Remote disk health check failed for {}: marking as faulty", addr);
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %endpoint,
|
||||
addr,
|
||||
state = "probe_failed",
|
||||
result = "mark_faulty",
|
||||
"Remote disk health probe failed"
|
||||
);
|
||||
|
||||
// Start recovery monitoring
|
||||
let health_clone = Arc::clone(&health);
|
||||
@@ -354,9 +384,25 @@ impl RemoteDisk {
|
||||
_ = interval.tick() => {
|
||||
if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
|
||||
let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
|
||||
info!("Remote disk recovery probe succeeded: {}", addr);
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %endpoint,
|
||||
addr,
|
||||
state = "recovery_probe_succeeded",
|
||||
"Remote disk recovery probe succeeded"
|
||||
);
|
||||
if became_online {
|
||||
info!("Remote disk recovered: {}", addr);
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %endpoint,
|
||||
addr,
|
||||
state = "recovered",
|
||||
"Remote disk recovered"
|
||||
);
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
@@ -469,7 +515,16 @@ impl RemoteDisk {
|
||||
{
|
||||
// Check if disk is faulty
|
||||
if self.health.is_faulty() {
|
||||
warn!("remote disk {} health is faulty, returning error", self.to_string());
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
addr = %self.addr,
|
||||
op,
|
||||
state = "faulty_short_circuit",
|
||||
"Remote disk operation short-circuited by faulty state"
|
||||
);
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
|
||||
@@ -519,10 +574,14 @@ impl RemoteDisk {
|
||||
self.mark_faulty_and_evict("operation_timeout").await;
|
||||
}
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = timeout_duration.as_millis(),
|
||||
state = "timeout",
|
||||
"Remote disk operation timed out"
|
||||
);
|
||||
Err(DiskError::Timeout)
|
||||
@@ -547,11 +606,15 @@ impl RemoteDisk {
|
||||
)
|
||||
.increment(1);
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = timeout_duration.as_millis(),
|
||||
"Remote disk operation returned a network-like error"
|
||||
state = "network_like_error",
|
||||
"Remote disk operation returned network-like error"
|
||||
);
|
||||
if failure_health_action == FailureHealthAction::MarkFailure {
|
||||
self.mark_faulty_and_evict("operation_network_error").await;
|
||||
@@ -574,13 +637,26 @@ impl RemoteDisk {
|
||||
.increment(1);
|
||||
if transitioned_to_offline {
|
||||
warn!(
|
||||
"Remote disk marked faulty after timeout: endpoint={}, addr={}, reason={}",
|
||||
self.endpoint, self.addr, reason
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
addr = %self.addr,
|
||||
reason,
|
||||
state = "marked_faulty",
|
||||
"Remote disk marked faulty"
|
||||
);
|
||||
} else {
|
||||
warn!(
|
||||
"Remote disk marked suspect after timeout: endpoint={}, addr={}, reason={}, state={:?}",
|
||||
self.endpoint, self.addr, reason, state
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
addr = %self.addr,
|
||||
reason,
|
||||
runtime_state = ?state,
|
||||
state = "marked_suspect",
|
||||
"Remote disk marked suspect"
|
||||
);
|
||||
}
|
||||
counter!(
|
||||
@@ -589,11 +665,15 @@ impl RemoteDisk {
|
||||
"reason" => reason.to_string()
|
||||
)
|
||||
.increment(1);
|
||||
info!(
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
addr = %self.addr,
|
||||
reason,
|
||||
"Evicting cached remote disk connection after fault transition"
|
||||
state = "evict_cached_connection",
|
||||
"Remote disk cached connection evicted"
|
||||
);
|
||||
evict_failed_connection(&self.addr).await;
|
||||
}
|
||||
@@ -705,7 +785,16 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn make_volume(&self, volume: &str) -> Result<()> {
|
||||
info!("make_volume");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
volume,
|
||||
op = "make_volume",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
@@ -733,7 +822,16 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> {
|
||||
info!("make_volumes");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
volume_count = volumes.len(),
|
||||
op = "make_volumes",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
@@ -761,7 +859,15 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn list_volumes(&self) -> Result<Vec<VolumeInfo>> {
|
||||
info!("list_volumes");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
op = "list_volumes",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
@@ -794,7 +900,16 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn stat_volume(&self, volume: &str) -> Result<VolumeInfo> {
|
||||
info!("stat_volume");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
volume,
|
||||
op = "stat_volume",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
@@ -824,7 +939,16 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn delete_volume(&self, volume: &str) -> Result<()> {
|
||||
info!("delete_volume {}/{}", self.endpoint.to_string(), volume);
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
volume,
|
||||
op = "delete_volume",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
@@ -909,7 +1033,17 @@ impl DiskAPI for RemoteDisk {
|
||||
force_del_marker: bool,
|
||||
opts: DeleteOptions,
|
||||
) -> Result<()> {
|
||||
info!("delete_version");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
volume,
|
||||
path,
|
||||
op = "delete_version",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
@@ -946,7 +1080,17 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn delete_versions(&self, volume: &str, versions: Vec<FileInfoVersions>, opts: DeleteOptions) -> Vec<Option<Error>> {
|
||||
info!("delete_versions");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
volume,
|
||||
version_count = versions.len(),
|
||||
op = "delete_versions",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
if self.health.is_faulty() {
|
||||
return vec![Some(DiskError::FaultyDisk); versions.len()];
|
||||
@@ -1041,7 +1185,17 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> {
|
||||
info!("delete_paths");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
volume,
|
||||
path_count = paths.len(),
|
||||
op = "delete_paths",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
let paths = paths.to_owned();
|
||||
|
||||
self.execute_with_timeout(
|
||||
@@ -1071,7 +1225,17 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
|
||||
info!("write_metadata {}/{}", volume, path);
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
volume,
|
||||
path,
|
||||
op = "write_metadata",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
let file_info = serde_json::to_string(&fi)?;
|
||||
let file_info_bin = encode_msgpack(&fi)?;
|
||||
|
||||
@@ -1134,7 +1298,17 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> {
|
||||
info!("update_metadata");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
volume,
|
||||
path,
|
||||
op = "update_metadata",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
let file_info = serde_json::to_string(&fi)?;
|
||||
let opts_str = serde_json::to_string(&opts)?;
|
||||
let file_info_bin = encode_msgpack(&fi)?;
|
||||
@@ -1180,7 +1354,18 @@ impl DiskAPI for RemoteDisk {
|
||||
version_id: &str,
|
||||
opts: &ReadOptions,
|
||||
) -> Result<FileInfo> {
|
||||
info!("read_version");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
volume,
|
||||
path,
|
||||
version_id,
|
||||
op = "read_version",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
let opts_str = serde_json::to_string(opts)?;
|
||||
let opts_bin = encode_msgpack(opts)?;
|
||||
|
||||
@@ -1217,7 +1402,18 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
|
||||
info!("read_xl {}/{}/{}", self.endpoint.to_string(), volume, path);
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
volume,
|
||||
path,
|
||||
read_data,
|
||||
op = "read_xl",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
@@ -1257,7 +1453,19 @@ impl DiskAPI for RemoteDisk {
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
) -> Result<RenameDataResp> {
|
||||
info!("rename_data {}/{}/{}/{}", self.addr, self.endpoint.to_string(), dst_volume, dst_path);
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
src_volume,
|
||||
src_path,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
op = "rename_data",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
self.execute_with_timeout_for_op(
|
||||
"rename_data",
|
||||
@@ -1325,7 +1533,17 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(skip(self, wr))]
|
||||
async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
|
||||
info!("walk_dir {}", self.endpoint.to_string());
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
bucket = %opts.bucket,
|
||||
base_dir = %opts.base_dir,
|
||||
op = "walk_dir",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
let disk = self.disk_ref().await;
|
||||
let body = serde_json::to_vec(&opts)?;
|
||||
@@ -1443,7 +1661,17 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> {
|
||||
info!("append_file {}/{}", volume, path);
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
volume,
|
||||
path,
|
||||
op = "append_file",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
@@ -1487,7 +1715,19 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
|
||||
info!("rename_file");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
src_volume,
|
||||
src_path,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
op = "rename_file",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
@@ -1518,7 +1758,19 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Bytes) -> Result<()> {
|
||||
info!("rename_part {}/{}", src_volume, src_path);
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
src_volume,
|
||||
src_path,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
op = "rename_part",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
@@ -1550,7 +1802,19 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
|
||||
info!("delete {}/{}/{}", self.endpoint.to_string(), volume, path);
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
volume,
|
||||
path,
|
||||
recursive = opt.recursive,
|
||||
immediate = opt.immediate,
|
||||
op = "delete",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
@@ -1581,7 +1845,17 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
|
||||
info!("verify_file");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
volume,
|
||||
path,
|
||||
op = "verify_file",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
@@ -1614,6 +1888,17 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn read_parts(&self, bucket: &str, paths: &[String]) -> Result<Vec<ObjectPartInfo>> {
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
bucket,
|
||||
path_count = paths.len(),
|
||||
op = "read_parts",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let mut client = self
|
||||
@@ -1642,7 +1927,17 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
|
||||
info!("check_parts");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
volume,
|
||||
path,
|
||||
op = "check_parts",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
@@ -1675,7 +1970,17 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>> {
|
||||
info!("read_multiple {}/{}/{}", self.endpoint.to_string(), req.bucket, req.prefix);
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
bucket = %req.bucket,
|
||||
prefix = %req.prefix,
|
||||
op = "read_multiple",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
@@ -1721,7 +2026,18 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn write_all(&self, volume: &str, path: &str, data: Bytes) -> Result<()> {
|
||||
info!("write_all");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
volume,
|
||||
path,
|
||||
bytes = data.len(),
|
||||
op = "write_all",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
@@ -1779,7 +2095,17 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn read_all(&self, volume: &str, path: &str) -> Result<Bytes> {
|
||||
info!("read_all {}/{}", volume, path);
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
volume,
|
||||
path,
|
||||
op = "read_all",
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
|
||||
+110
-10
@@ -130,6 +130,12 @@ use tracing::error;
|
||||
use tracing::{debug, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_SET_DISK: &str = "set_disk";
|
||||
const EVENT_SET_DISK_MULTIPART: &str = "set_disk_multipart";
|
||||
const EVENT_SET_DISK_WRITE: &str = "set_disk_write";
|
||||
const EVENT_SET_DISK_HEAL: &str = "set_disk_heal";
|
||||
|
||||
use crate::rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _};
|
||||
|
||||
pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024;
|
||||
@@ -948,7 +954,16 @@ impl ObjectIO for SetDisks {
|
||||
)
|
||||
.await
|
||||
{
|
||||
error!("get_object_with_fileinfo {bucket}/{object} err {:?}", e);
|
||||
error!(
|
||||
event = EVENT_SET_DISK_WRITE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
bucket,
|
||||
object,
|
||||
state = "read_pipeline_failed",
|
||||
error = ?e,
|
||||
"Set disk object read pipeline failed"
|
||||
);
|
||||
};
|
||||
});
|
||||
|
||||
@@ -1065,7 +1080,15 @@ impl ObjectIO for SetDisks {
|
||||
{
|
||||
Ok(writer) => (Some(writer), None),
|
||||
Err(err) => {
|
||||
warn!("create_bitrot_writer disk {}, err {:?}, skipping operation", disk.to_string(), err);
|
||||
warn!(
|
||||
event = EVENT_SET_DISK_WRITE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
disk = ?disk,
|
||||
state = "bitrot_writer_skipped",
|
||||
error = ?err,
|
||||
"Set disk bitrot writer skipped"
|
||||
);
|
||||
(None, Some(err))
|
||||
}
|
||||
}
|
||||
@@ -1085,7 +1108,18 @@ impl ObjectIO for SetDisks {
|
||||
|
||||
let nil_count = errors.iter().filter(|&e| e.is_none()).count();
|
||||
if nil_count < write_quorum {
|
||||
error!("not enough disks to write: {:?}", errors);
|
||||
error!(
|
||||
event = EVENT_SET_DISK_WRITE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
bucket,
|
||||
object,
|
||||
write_quorum,
|
||||
available_writers = nil_count,
|
||||
state = "write_quorum_unavailable",
|
||||
error = ?errors,
|
||||
"Set disk write quorum unavailable"
|
||||
);
|
||||
if let Some(write_err) = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, write_quorum) {
|
||||
return Err(to_object_err(write_err.into(), vec![bucket, object]));
|
||||
}
|
||||
@@ -1136,7 +1170,17 @@ impl ObjectIO for SetDisks {
|
||||
// }
|
||||
|
||||
if (w_size as i64) < data.size() {
|
||||
warn!("put_object write size < data.size(), w_size={}, data.size={}", w_size, data.size());
|
||||
warn!(
|
||||
event = EVENT_SET_DISK_WRITE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
bucket,
|
||||
object,
|
||||
written_size = w_size,
|
||||
expected_size = data.size(),
|
||||
state = "short_write",
|
||||
"Set disk write produced fewer bytes than expected"
|
||||
);
|
||||
return Err(Error::other(format!(
|
||||
"put_object write size < data.size(), w_size={}, data.size={}",
|
||||
w_size,
|
||||
@@ -2976,7 +3020,15 @@ impl MultipartOperations for SetDisks {
|
||||
{
|
||||
Ok(writer) => writer,
|
||||
Err(err) => {
|
||||
warn!("create_bitrot_writer disk {}, err {:?}, skipping operation", disk.to_string(), err);
|
||||
warn!(
|
||||
event = EVENT_SET_DISK_MULTIPART,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
disk = ?disk,
|
||||
state = "bitrot_writer_skipped",
|
||||
error = ?err,
|
||||
"Set disk multipart bitrot writer skipped"
|
||||
);
|
||||
errors.push(Some(err));
|
||||
writers.push(None);
|
||||
continue;
|
||||
@@ -3021,7 +3073,18 @@ impl MultipartOperations for SetDisks {
|
||||
let _ = mem::replace(&mut data.stream, reader);
|
||||
|
||||
if (w_size as i64) < data.size() {
|
||||
warn!("put_object_part write size < data.size(), w_size={}, data.size={}", w_size, data.size());
|
||||
warn!(
|
||||
event = EVENT_SET_DISK_MULTIPART,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
bucket,
|
||||
object,
|
||||
part_number = part_id,
|
||||
written_size = w_size,
|
||||
expected_size = data.size(),
|
||||
state = "short_write",
|
||||
"Set disk multipart write produced fewer bytes than expected"
|
||||
);
|
||||
return Err(Error::other(format!(
|
||||
"put_object_part write size < data.size(), w_size={}, data.size={}",
|
||||
w_size,
|
||||
@@ -3657,7 +3720,17 @@ impl MultipartOperations for SetDisks {
|
||||
);
|
||||
return Err(Error::InvalidPart(p.part_num, "".to_owned(), p.etag.clone().unwrap_or_default()));
|
||||
};
|
||||
info!(target:"rustfs_ecstore::set_disk", part_number = p.part_num, part_size = ext_part.size, part_actual_size = ext_part.actual_size, "Completing multipart part");
|
||||
debug!(
|
||||
target:"rustfs_ecstore::set_disk",
|
||||
event = EVENT_SET_DISK_MULTIPART,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
part_number = p.part_num,
|
||||
part_size = ext_part.size,
|
||||
part_actual_size = ext_part.actual_size,
|
||||
state = "part_validated",
|
||||
"Set disk multipart part validated"
|
||||
);
|
||||
|
||||
// Normalize ETags by removing quotes before comparison (PR #592 compatibility)
|
||||
let client_etag = p.etag.as_ref().map(|e| rustfs_utils::path::trim_etag(e));
|
||||
@@ -3988,7 +4061,16 @@ impl HealOperations for SetDisks {
|
||||
let disks = disks.clone();
|
||||
let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, false, false, false).await?;
|
||||
if DiskError::is_all_not_found(&errs) {
|
||||
debug!(bucket, object, version_id, "heal_object skipped missing object");
|
||||
debug!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
state = "missing_object_skipped",
|
||||
"Set disk heal skipped missing object"
|
||||
);
|
||||
let err = if !version_id.is_empty() {
|
||||
Error::FileVersionNotFound
|
||||
} else {
|
||||
@@ -4356,7 +4438,16 @@ async fn disks_with_all_parts(
|
||||
verify_resp = v;
|
||||
}
|
||||
Err(err) => {
|
||||
info!("verify_file failed: {err:?}, object_name={}, index: {index}", object_name);
|
||||
debug!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
object = %object_name,
|
||||
disk_index = index,
|
||||
state = "verify_failed",
|
||||
error = ?err,
|
||||
"Set disk verify_file failed"
|
||||
);
|
||||
verify_err = Some(err);
|
||||
}
|
||||
}
|
||||
@@ -4366,7 +4457,16 @@ async fn disks_with_all_parts(
|
||||
verify_resp = v;
|
||||
}
|
||||
Err(err) => {
|
||||
info!("check_parts failed: {err:?}, object_name={}, index: {index}", object_name);
|
||||
debug!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
object = %object_name,
|
||||
disk_index = index,
|
||||
state = "check_parts_failed",
|
||||
error = ?err,
|
||||
"Set disk check_parts failed"
|
||||
);
|
||||
verify_err = Some(err);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,7 +69,7 @@ impl HealChannelProcessor {
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_CHANNEL,
|
||||
state = "started",
|
||||
"Heal channel processor state updated"
|
||||
"Heal channel started"
|
||||
);
|
||||
|
||||
loop {
|
||||
@@ -85,7 +85,7 @@ impl HealChannelProcessor {
|
||||
subsystem = LOG_SUBSYSTEM_CHANNEL,
|
||||
state = "process_failed",
|
||||
error = %e,
|
||||
"Heal channel request processing failed"
|
||||
"Heal channel processing failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -96,7 +96,7 @@ impl HealChannelProcessor {
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_CHANNEL,
|
||||
state = "receiver_closed",
|
||||
"Heal channel processor state updated"
|
||||
"Heal channel receiver closed"
|
||||
);
|
||||
break;
|
||||
}
|
||||
@@ -105,7 +105,7 @@ impl HealChannelProcessor {
|
||||
response = self.response_receiver.recv() => {
|
||||
if let Some(response) = response {
|
||||
// Handle response if needed
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::channel",
|
||||
event = EVENT_HEAL_CHANNEL_RESPONSE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -113,7 +113,7 @@ impl HealChannelProcessor {
|
||||
request_id = %response.request_id,
|
||||
success = response.success,
|
||||
state = "received_local",
|
||||
"Heal channel response observed"
|
||||
"Heal response observed"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -126,7 +126,7 @@ impl HealChannelProcessor {
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_CHANNEL,
|
||||
state = "stopped",
|
||||
"Heal channel processor state updated"
|
||||
"Heal channel stopped"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -154,7 +154,7 @@ impl HealChannelProcessor {
|
||||
request: HealChannelRequest,
|
||||
response_tx: oneshot::Sender<std::result::Result<HealAdmissionResult, String>>,
|
||||
) -> Result<()> {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::channel",
|
||||
event = EVENT_HEAL_CHANNEL_REQUEST,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -163,7 +163,7 @@ impl HealChannelProcessor {
|
||||
bucket = %request.bucket,
|
||||
object_prefix = %request.object_prefix.as_deref().unwrap_or(""),
|
||||
state = "start_received",
|
||||
"Heal channel start request received"
|
||||
"Heal start received"
|
||||
);
|
||||
|
||||
// Convert channel request to heal request
|
||||
@@ -185,7 +185,7 @@ impl HealChannelProcessor {
|
||||
// Submit to heal manager
|
||||
match self.heal_manager.submit_heal_request(heal_request).await {
|
||||
Ok(admission) => {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::channel",
|
||||
event = EVENT_HEAL_CHANNEL_REQUEST,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -193,7 +193,7 @@ impl HealChannelProcessor {
|
||||
request_id = %request.id,
|
||||
admission = admission.result_label(),
|
||||
state = "admission_decided",
|
||||
"Heal channel admission decision completed"
|
||||
"Heal admission decided"
|
||||
);
|
||||
|
||||
let _ = response_tx.send(Ok(admission));
|
||||
@@ -225,7 +225,7 @@ impl HealChannelProcessor {
|
||||
request_id = %request.id,
|
||||
state = "submit_failed",
|
||||
error = %error_text,
|
||||
"Heal channel start request failed"
|
||||
"Heal start submission failed"
|
||||
);
|
||||
let _ = response_tx.send(Err(error_text.clone()));
|
||||
|
||||
@@ -251,7 +251,7 @@ impl HealChannelProcessor {
|
||||
client_token: String,
|
||||
response_tx: oneshot::Sender<std::result::Result<HealChannelResponse, String>>,
|
||||
) -> Result<()> {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::channel",
|
||||
event = EVENT_HEAL_CHANNEL_REQUEST,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -259,7 +259,7 @@ impl HealChannelProcessor {
|
||||
request_id = %client_token,
|
||||
heal_path = %heal_path,
|
||||
state = "query_received",
|
||||
"Heal channel query request received"
|
||||
"Heal query received"
|
||||
);
|
||||
|
||||
let (summary, detail, items) = match self.heal_manager.get_task_report_for_path(&heal_path, &client_token).await {
|
||||
@@ -332,7 +332,7 @@ impl HealChannelProcessor {
|
||||
client_token: String,
|
||||
response_tx: oneshot::Sender<std::result::Result<HealChannelResponse, String>>,
|
||||
) -> Result<()> {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::channel",
|
||||
event = EVENT_HEAL_CHANNEL_REQUEST,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -340,7 +340,7 @@ impl HealChannelProcessor {
|
||||
request_id = %client_token,
|
||||
heal_path = %heal_path,
|
||||
state = "cancel_received",
|
||||
"Heal channel cancel request received"
|
||||
"Heal cancel received"
|
||||
);
|
||||
|
||||
let request_id = if client_token.is_empty() {
|
||||
@@ -451,7 +451,7 @@ impl HealChannelProcessor {
|
||||
request_id = %response.request_id,
|
||||
state = "enqueue_local_failed",
|
||||
error = %e,
|
||||
"Heal channel response enqueue failed"
|
||||
"Heal response local enqueue failed"
|
||||
);
|
||||
}
|
||||
// Always attempt to broadcast, even if local send failed
|
||||
@@ -464,7 +464,7 @@ impl HealChannelProcessor {
|
||||
subsystem = LOG_SUBSYSTEM_CHANNEL,
|
||||
state = "broadcast_failed",
|
||||
error = %e,
|
||||
"Heal channel response broadcast failed"
|
||||
"Heal response broadcast failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ use std::sync::{
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
use tokio::sync::{RwLock, Semaphore};
|
||||
use tracing::{error, info, warn};
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
const LOG_COMPONENT_HEAL: &str = "heal";
|
||||
const LOG_SUBSYSTEM_ERASURE_HEALER: &str = "erasure_healer";
|
||||
@@ -99,7 +99,7 @@ impl ErasureSetHealer {
|
||||
/// execute erasure set heal with resume
|
||||
#[tracing::instrument(skip(self, buckets), fields(set_disk_id = %set_disk_id, bucket_count = buckets.len()))]
|
||||
pub async fn heal_erasure_set(&self, buckets: &[String], set_disk_id: &str) -> Result<()> {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_RESUME_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -162,7 +162,7 @@ impl ErasureSetHealer {
|
||||
Ok(manager) => {
|
||||
let state = manager.get_state().await;
|
||||
if state.set_disk_id == set_disk_id && ResumeUtils::can_resume_task(&self.disk, &task_id).await {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_RESUME_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -170,7 +170,7 @@ impl ErasureSetHealer {
|
||||
task_id,
|
||||
set_disk_id,
|
||||
state = "resume_found",
|
||||
"Erasure set resume state selected"
|
||||
"Erasure set resume selected"
|
||||
);
|
||||
return Ok(task_id);
|
||||
}
|
||||
@@ -193,7 +193,7 @@ impl ErasureSetHealer {
|
||||
|
||||
// create new task id
|
||||
let task_id = format!("{}_{}", set_disk_id, ResumeUtils::generate_task_id());
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_RESUME_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -215,7 +215,7 @@ impl ErasureSetHealer {
|
||||
) -> Result<(ResumeManager, CheckpointManager)> {
|
||||
// check if resume state exists
|
||||
if ResumeManager::has_resume_state(&self.disk, task_id).await {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_RESUME_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -235,7 +235,7 @@ impl ErasureSetHealer {
|
||||
|
||||
Ok((resume_manager, checkpoint_manager))
|
||||
} else {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_RESUME_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -243,7 +243,7 @@ impl ErasureSetHealer {
|
||||
task_id,
|
||||
set_disk_id,
|
||||
state = "creating_new",
|
||||
"Erasure set resume state creating"
|
||||
"Erasure set resume created"
|
||||
);
|
||||
|
||||
let resume_manager = ResumeManager::new(
|
||||
@@ -273,7 +273,7 @@ impl ErasureSetHealer {
|
||||
let state = resume_manager.get_state().await;
|
||||
let checkpoint = checkpoint_manager.get_checkpoint().await;
|
||||
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_RESUME_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -282,7 +282,7 @@ impl ErasureSetHealer {
|
||||
current_bucket_index = checkpoint.current_bucket_index,
|
||||
current_object_index = checkpoint.current_object_index,
|
||||
state = "resuming",
|
||||
"Erasure set heal resumed from checkpoint"
|
||||
"Erasure set resumed"
|
||||
);
|
||||
|
||||
// 2. initialize progress
|
||||
@@ -349,7 +349,7 @@ impl ErasureSetHealer {
|
||||
match bucket_result {
|
||||
Ok(_) => {
|
||||
resume_manager.complete_bucket(bucket).await?;
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_BUCKET_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -357,7 +357,7 @@ impl ErasureSetHealer {
|
||||
set_disk_id,
|
||||
bucket,
|
||||
state = "completed",
|
||||
"Erasure set bucket heal completed"
|
||||
"Erasure set bucket completed"
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
@@ -383,14 +383,14 @@ impl ErasureSetHealer {
|
||||
// 5. mark task completed
|
||||
resume_manager.mark_completed().await?;
|
||||
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_RESUME_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
|
||||
set_disk_id,
|
||||
state = "completed",
|
||||
"Erasure set heal completed"
|
||||
"Erasure set completed"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -411,7 +411,7 @@ impl ErasureSetHealer {
|
||||
resume_manager: &ResumeManager,
|
||||
checkpoint_manager: &CheckpointManager,
|
||||
) -> Result<()> {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_BUCKET_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -421,7 +421,7 @@ impl ErasureSetHealer {
|
||||
bucket_index,
|
||||
current_object_index = *current_object_index,
|
||||
state = "started",
|
||||
"Erasure set bucket heal started"
|
||||
"Erasure set bucket started"
|
||||
);
|
||||
|
||||
// 1. get bucket info
|
||||
@@ -584,7 +584,7 @@ impl ErasureSetHealer {
|
||||
Ok(true) => {
|
||||
*successful_objects += 1;
|
||||
checkpoint_manager.add_processed_object(object.clone()).await?;
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -593,13 +593,13 @@ impl ErasureSetHealer {
|
||||
bucket,
|
||||
object = %object,
|
||||
state = "healed",
|
||||
"Erasure set object heal completed"
|
||||
"Erasure set object healed"
|
||||
);
|
||||
}
|
||||
Ok(false) => {
|
||||
checkpoint_manager.add_processed_object(object.clone()).await?;
|
||||
*successful_objects += 1;
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -608,7 +608,7 @@ impl ErasureSetHealer {
|
||||
bucket,
|
||||
object = %object,
|
||||
state = "missing_treated_as_ok",
|
||||
"Erasure set object heal skipped because object disappeared"
|
||||
"Erasure set missing object treated as ok"
|
||||
);
|
||||
}
|
||||
Err(Error::TaskCancelled) => {
|
||||
@@ -743,14 +743,14 @@ impl ErasureSetHealer {
|
||||
bucket: &str,
|
||||
progress: &Arc<RwLock<HealProgress>>,
|
||||
) -> Result<()> {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_BUCKET_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
|
||||
bucket,
|
||||
state = "started",
|
||||
"Erasure set bucket heal started"
|
||||
"Erasure set bucket started"
|
||||
);
|
||||
|
||||
// 1. get bucket info
|
||||
@@ -847,7 +847,7 @@ impl ErasureSetHealer {
|
||||
p.set_current_object(Some(format!("completed bucket: {bucket}")));
|
||||
}
|
||||
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_BUCKET_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -857,7 +857,7 @@ impl ErasureSetHealer {
|
||||
total_failed,
|
||||
total_scanned,
|
||||
state = "completed",
|
||||
"Erasure set bucket heal completed"
|
||||
"Erasure set bucket completed"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
@@ -890,7 +890,7 @@ impl ErasureSetHealer {
|
||||
|
||||
match storage.heal_object(&bucket, &object, None, &heal_opts).await {
|
||||
Ok((_result, None)) => {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -898,7 +898,7 @@ impl ErasureSetHealer {
|
||||
bucket,
|
||||
object = %object,
|
||||
state = "healed",
|
||||
"Erasure set object heal completed"
|
||||
"Erasure set object healed"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -956,7 +956,7 @@ impl ErasureSetHealer {
|
||||
success_count,
|
||||
total,
|
||||
state = "summary",
|
||||
"Erasure set heal summary recorded"
|
||||
"Erasure set summary recorded"
|
||||
);
|
||||
|
||||
if failure_count > 0 {
|
||||
|
||||
@@ -558,7 +558,7 @@ impl HealManager {
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
state = "already_running",
|
||||
"Heal manager state unchanged"
|
||||
"Heal manager already running"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
@@ -571,7 +571,7 @@ impl HealManager {
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
state = "starting",
|
||||
"Heal manager state updated"
|
||||
"Heal manager starting"
|
||||
);
|
||||
|
||||
// start scheduler
|
||||
@@ -586,7 +586,7 @@ impl HealManager {
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
state = "running",
|
||||
"Heal manager state updated"
|
||||
"Heal manager started"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -599,7 +599,7 @@ impl HealManager {
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
state = "stopping",
|
||||
"Heal manager state updated"
|
||||
"Heal manager stopping"
|
||||
);
|
||||
|
||||
// cancel all tasks
|
||||
@@ -617,7 +617,7 @@ impl HealManager {
|
||||
state = "task_cancel_failed",
|
||||
task_id = %task.id,
|
||||
error = %e,
|
||||
"Heal manager failed to cancel active task"
|
||||
"Heal active task cancellation failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -636,7 +636,7 @@ impl HealManager {
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
state = "stopped",
|
||||
"Heal manager state updated"
|
||||
"Heal manager stopped"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -659,7 +659,7 @@ impl HealManager {
|
||||
|
||||
match admission {
|
||||
HealAdmissionResult::Merged => {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -667,7 +667,7 @@ impl HealManager {
|
||||
request_id = %request.id,
|
||||
priority = ?request.priority,
|
||||
result = "merged_duplicate",
|
||||
"Heal queue admission decided"
|
||||
"Heal queue request merged"
|
||||
);
|
||||
}
|
||||
HealAdmissionResult::Dropped(reason) => {
|
||||
@@ -680,7 +680,7 @@ impl HealManager {
|
||||
priority = ?request.priority,
|
||||
reason = reason.as_str(),
|
||||
result = "dropped_duplicate",
|
||||
"Heal queue admission decided"
|
||||
"Heal queue request dropped"
|
||||
);
|
||||
}
|
||||
HealAdmissionResult::Accepted | HealAdmissionResult::Full => {}
|
||||
@@ -707,7 +707,7 @@ impl HealManager {
|
||||
queue_len,
|
||||
queue_capacity,
|
||||
result = "accepted_by_displacement",
|
||||
"Heal queue admission decided"
|
||||
"Heal queue request accepted by displacement"
|
||||
);
|
||||
drop(queue);
|
||||
if config.event_driven_scheduler_enable {
|
||||
@@ -726,7 +726,7 @@ impl HealManager {
|
||||
queue_len,
|
||||
queue_capacity,
|
||||
result = "full_no_displacement_candidate",
|
||||
"Heal queue admission decided"
|
||||
"Heal queue request rejected without displacement"
|
||||
);
|
||||
return Ok(HealAdmissionResult::Full);
|
||||
}
|
||||
@@ -745,7 +745,7 @@ impl HealManager {
|
||||
queue_capacity,
|
||||
reason = reason.as_str(),
|
||||
result = "dropped_full",
|
||||
"Heal queue admission decided"
|
||||
"Heal queue request dropped"
|
||||
);
|
||||
}
|
||||
HealAdmissionResult::Full => {
|
||||
@@ -759,7 +759,7 @@ impl HealManager {
|
||||
queue_len,
|
||||
queue_capacity,
|
||||
result = "rejected_full",
|
||||
"Heal queue admission decided"
|
||||
"Heal queue request rejected"
|
||||
);
|
||||
}
|
||||
HealAdmissionResult::Accepted | HealAdmissionResult::Merged => {}
|
||||
@@ -779,7 +779,7 @@ impl HealManager {
|
||||
queue_capacity,
|
||||
queue_usage_pct = (queue_len * 100) / queue_capacity,
|
||||
result = "queue_pressure_high",
|
||||
"Heal queue pressure increased"
|
||||
"Heal queue pressure high"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -793,7 +793,7 @@ impl HealManager {
|
||||
// Log queue statistics periodically (when adding high/urgent priority items)
|
||||
if matches!(priority, HealPriority::High | HealPriority::Urgent) {
|
||||
let stats = queue.get_priority_stats();
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -812,7 +812,7 @@ impl HealManager {
|
||||
|
||||
drop(queue);
|
||||
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -821,7 +821,7 @@ impl HealManager {
|
||||
priority = ?priority,
|
||||
queue_len = queue_len + 1,
|
||||
result = "accepted",
|
||||
"Heal queue admission decided"
|
||||
"Heal queue request accepted"
|
||||
);
|
||||
if config.event_driven_scheduler_enable {
|
||||
self.notify.notify_one();
|
||||
@@ -1119,7 +1119,7 @@ impl HealManager {
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
state = "shutdown",
|
||||
"Heal scheduler state updated"
|
||||
"Heal scheduler stopped"
|
||||
);
|
||||
break;
|
||||
}
|
||||
@@ -1158,13 +1158,17 @@ impl HealManager {
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
state = "started",
|
||||
interval = ?duration,
|
||||
"Heal auto disk scanner state updated"
|
||||
"Heal auto disk scanner started"
|
||||
);
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut interval = interval(duration);
|
||||
|
||||
loop {
|
||||
let mut candidate_count = 0usize;
|
||||
let mut skipped_duplicate_count = 0usize;
|
||||
let mut skipped_invalid_count = 0usize;
|
||||
let mut enqueued_count = 0usize;
|
||||
tokio::select! {
|
||||
_ = cancel_token.cancelled() => {
|
||||
info!(
|
||||
@@ -1173,7 +1177,7 @@ impl HealManager {
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
state = "shutdown",
|
||||
"Heal auto disk scanner state updated"
|
||||
"Heal auto disk scanner stopped"
|
||||
);
|
||||
break;
|
||||
}
|
||||
@@ -1185,14 +1189,15 @@ impl HealManager {
|
||||
// detect unformatted disk via get_disk_id()
|
||||
match disk.get_disk_id().await {
|
||||
Err(DiskError::UnformattedDisk) => {
|
||||
info!(
|
||||
candidate_count += 1;
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_DISK,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint = %disk.endpoint(),
|
||||
disk_state = "unformatted",
|
||||
"Heal auto disk scanner detected candidate disk"
|
||||
"Heal auto-scan candidate detected"
|
||||
);
|
||||
endpoints.push(disk.endpoint());
|
||||
}
|
||||
@@ -1205,7 +1210,7 @@ impl HealManager {
|
||||
endpoint = %disk.endpoint(),
|
||||
disk_state = "check_failed",
|
||||
error = ?e,
|
||||
"Heal auto disk scanner failed to inspect disk"
|
||||
"Heal auto-scan disk inspection failed"
|
||||
);
|
||||
}
|
||||
Ok(_) => {
|
||||
@@ -1222,7 +1227,7 @@ impl HealManager {
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
state = "idle",
|
||||
"Heal auto disk scanner found no candidate disks"
|
||||
"Heal auto disk scanner idle"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
@@ -1238,7 +1243,7 @@ impl HealManager {
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
state = "bucket_list_failed",
|
||||
error = %e,
|
||||
"Heal auto disk scanner failed to list buckets"
|
||||
"Heal auto-scan bucket listing failed"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
@@ -1256,8 +1261,9 @@ impl HealManager {
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint = %ep,
|
||||
result = "skipped_invalid_set_disk_id",
|
||||
"Heal auto disk scanner skipped enqueue"
|
||||
"Heal auto-scan enqueue skipped"
|
||||
);
|
||||
skipped_invalid_count += 1;
|
||||
continue;
|
||||
};
|
||||
// skip if already queued or healing
|
||||
@@ -1283,7 +1289,8 @@ impl HealManager {
|
||||
}
|
||||
|
||||
if skip {
|
||||
info!(
|
||||
skipped_duplicate_count += 1;
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -1291,7 +1298,7 @@ impl HealManager {
|
||||
endpoint = %ep,
|
||||
set_disk_id,
|
||||
result = "skipped_duplicate",
|
||||
"Heal auto disk scanner skipped enqueue"
|
||||
"Heal auto-scan duplicate skipped"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
@@ -1312,7 +1319,8 @@ impl HealManager {
|
||||
if config.event_driven_scheduler_enable {
|
||||
notify.notify_one();
|
||||
}
|
||||
info!(
|
||||
enqueued_count += 1;
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -1321,10 +1329,22 @@ impl HealManager {
|
||||
set_disk_id,
|
||||
bucket_count = buckets.len(),
|
||||
result = "enqueued",
|
||||
"Heal auto disk scanner enqueued task"
|
||||
"Heal auto-scan task enqueued"
|
||||
);
|
||||
}
|
||||
}
|
||||
info!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
state = "cycle_completed",
|
||||
candidate_count,
|
||||
enqueued_count,
|
||||
skipped_duplicate_count,
|
||||
skipped_invalid_count,
|
||||
"Heal auto-scan cycle completed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1405,7 +1425,7 @@ impl HealManager {
|
||||
|
||||
// start heal task
|
||||
tokio::spawn(async move {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_SCHEDULER_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -1415,12 +1435,12 @@ impl HealManager {
|
||||
heal_type = %task_type_label_for_spawn,
|
||||
set = %task_set_label_for_spawn,
|
||||
state = "task_started",
|
||||
"Heal scheduler started task"
|
||||
"Heal scheduler task started"
|
||||
);
|
||||
let result = task.execute().await;
|
||||
match result {
|
||||
Ok(_) => {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_SCHEDULER_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -1429,7 +1449,7 @@ impl HealManager {
|
||||
heal_type = %task_type_label_for_spawn,
|
||||
set = %task_set_label_for_spawn,
|
||||
state = "task_completed",
|
||||
"Heal scheduler finished task"
|
||||
"Heal scheduler task completed"
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
@@ -1443,7 +1463,7 @@ impl HealManager {
|
||||
set = %task_set_label_for_spawn,
|
||||
state = "task_failed",
|
||||
error = %e,
|
||||
"Heal scheduler observed task failure"
|
||||
"Heal scheduler task failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1500,7 +1520,7 @@ impl HealManager {
|
||||
queue_len = remaining,
|
||||
active_tasks = active_heals_guard.len(),
|
||||
state = "backlog_high",
|
||||
"Heal queue backlog summary recorded"
|
||||
"Heal queue backlog high"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{debug, info, warn};
|
||||
use tracing::{debug, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
const LOG_COMPONENT_HEAL: &str = "heal";
|
||||
@@ -299,7 +299,7 @@ impl ResumeManager {
|
||||
let _ = self.disk.delete(RUSTFS_META_BUCKET, path_str, Default::default()).await;
|
||||
}
|
||||
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_RESUME_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -524,7 +524,7 @@ impl CheckpointManager {
|
||||
let _ = self.disk.delete(RUSTFS_META_BUCKET, path_str, Default::default()).await;
|
||||
}
|
||||
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_CHECKPOINT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -649,7 +649,7 @@ impl ResumeUtils {
|
||||
let age_hours = (current_time - state.last_update) / 3600;
|
||||
|
||||
if age_hours > max_age_hours {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_RESUME_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -657,7 +657,7 @@ impl ResumeUtils {
|
||||
task_id,
|
||||
age_hours,
|
||||
state = "expired_cleanup_started",
|
||||
"Heal resume state cleanup started"
|
||||
"Heal resume cleanup started"
|
||||
);
|
||||
if let Err(e) = resume_manager.cleanup().await {
|
||||
warn!(
|
||||
|
||||
@@ -24,7 +24,7 @@ use rustfs_ecstore::{
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use rustfs_storage_api::{BucketInfo, DiskSetSelector, StorageAdminApi};
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, error, info, warn};
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
const LOG_COMPONENT_HEAL: &str = "heal";
|
||||
const LOG_SUBSYSTEM_STORAGE: &str = "storage";
|
||||
@@ -206,7 +206,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
bucket,
|
||||
object,
|
||||
result = "not_found",
|
||||
"Heal storage request finished"
|
||||
"Heal storage object metadata missing"
|
||||
);
|
||||
Ok(None)
|
||||
} else {
|
||||
@@ -333,7 +333,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -342,7 +342,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
bucket,
|
||||
object,
|
||||
result = "ok",
|
||||
"Heal storage request finished"
|
||||
"Heal storage object write completed"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -378,7 +378,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
|
||||
match self.ecstore.delete_object(bucket, object, Default::default()).await {
|
||||
Ok(_) => {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -387,7 +387,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
bucket,
|
||||
object,
|
||||
result = "ok",
|
||||
"Heal storage request finished"
|
||||
"Heal storage object delete completed"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -447,7 +447,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
let mut stream = reader.stream;
|
||||
match tokio::io::copy(&mut stream, &mut tokio::io::sink()).await {
|
||||
Ok(_) => {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_VERIFY,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -455,7 +455,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
bucket,
|
||||
object,
|
||||
state = "ok",
|
||||
"Heal storage object verification finished"
|
||||
"Heal storage object verified"
|
||||
);
|
||||
Ok(true)
|
||||
}
|
||||
@@ -544,7 +544,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
// After healing, try to read the object data
|
||||
match self.get_object_data(bucket, object).await? {
|
||||
Some(data) => {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_REPAIR_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -554,7 +554,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
object,
|
||||
bytes = data.len(),
|
||||
state = "ok",
|
||||
"Heal storage repair finished"
|
||||
"Heal storage EC decode rebuild completed"
|
||||
);
|
||||
Ok(data)
|
||||
}
|
||||
@@ -608,7 +608,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
|
||||
// TODO: implement disk status check using ecstore
|
||||
// For now, return Ok status
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -617,7 +617,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
endpoint = ?endpoint,
|
||||
result = "ok",
|
||||
disk_status = "ok",
|
||||
"Heal storage admin operation finished"
|
||||
"Heal storage disk status resolved"
|
||||
);
|
||||
Ok(DiskStatus::Ok)
|
||||
}
|
||||
@@ -640,7 +640,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
if error.is_some() {
|
||||
return Err(Error::other(format!("Format failed: {error:?}")));
|
||||
}
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -648,7 +648,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
operation = "format_disk",
|
||||
endpoint = ?endpoint,
|
||||
result = "ok",
|
||||
"Heal storage admin operation finished"
|
||||
"Heal storage disk format completed"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -726,7 +726,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
|
||||
match self.heal_bucket(bucket, &heal_opts).await {
|
||||
Ok(_) => {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_REPAIR_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -734,7 +734,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
operation = "heal_bucket_metadata",
|
||||
bucket,
|
||||
result = "ok",
|
||||
"Heal storage repair finished"
|
||||
"Heal storage bucket metadata repaired"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -816,7 +816,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
bucket,
|
||||
object,
|
||||
result = "not_found",
|
||||
"Heal storage request finished"
|
||||
"Heal storage object absence confirmed"
|
||||
);
|
||||
Ok(false)
|
||||
} else if is_transient_object_exists_error(&e) {
|
||||
@@ -923,7 +923,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
match self.ecstore.heal_object(bucket, object, version_id_str, opts).await {
|
||||
Ok((result, ecstore_error)) => {
|
||||
let error = ecstore_error.map(Error::other);
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_REPAIR_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -935,7 +935,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
drives_after = result.after.drives.len(),
|
||||
has_error = error.is_some(),
|
||||
result = "ok",
|
||||
"Heal storage repair finished"
|
||||
"Heal storage object repair completed"
|
||||
);
|
||||
Ok((result, error))
|
||||
}
|
||||
@@ -974,7 +974,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
|
||||
match self.ecstore.heal_bucket(bucket, opts).await {
|
||||
Ok(result) => {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_REPAIR_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -983,7 +983,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
bucket,
|
||||
drives_after = result.after.drives.len(),
|
||||
result = "ok",
|
||||
"Heal storage repair finished"
|
||||
"Heal storage bucket repair completed"
|
||||
);
|
||||
Ok(result)
|
||||
}
|
||||
@@ -1019,7 +1019,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
match self.ecstore.heal_format(dry_run).await {
|
||||
Ok((result, ecstore_error)) => {
|
||||
let error = ecstore_error.map(Error::other);
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_REPAIR_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -1028,7 +1028,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
drives_after = result.after.drives.len(),
|
||||
has_error = error.is_some(),
|
||||
result = "ok",
|
||||
"Heal storage repair finished"
|
||||
"Heal storage format repair completed"
|
||||
);
|
||||
Ok((result, error))
|
||||
}
|
||||
@@ -1103,7 +1103,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
}
|
||||
}
|
||||
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -1113,7 +1113,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
prefix,
|
||||
object_count = all_objects.len(),
|
||||
result = "ok",
|
||||
"Heal storage admin operation finished"
|
||||
"Heal storage object listing completed"
|
||||
);
|
||||
Ok(all_objects)
|
||||
}
|
||||
@@ -1175,12 +1175,12 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "list_objects_for_heal_page",
|
||||
bucket,
|
||||
prefix,
|
||||
object_count = page_count,
|
||||
is_truncated = list_info.is_truncated,
|
||||
state = "page_loaded",
|
||||
"Heal storage admin operation finished"
|
||||
bucket,
|
||||
prefix,
|
||||
object_count = page_count,
|
||||
is_truncated = list_info.is_truncated,
|
||||
state = "page_loaded",
|
||||
"Heal storage object listing page loaded"
|
||||
);
|
||||
|
||||
Ok((page_objects, list_info.next_continuation_token, list_info.is_truncated))
|
||||
@@ -1210,7 +1210,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
|
||||
// Find the first available disk
|
||||
if let Some(disk_store) = disks.into_iter().flatten().next() {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -1219,7 +1219,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
set_disk_id,
|
||||
result = "ok",
|
||||
disk = ?disk_store,
|
||||
"Heal storage admin operation finished"
|
||||
"Heal storage resume disk resolved"
|
||||
);
|
||||
return Ok(disk_store);
|
||||
}
|
||||
|
||||
@@ -445,7 +445,7 @@ impl HealTask {
|
||||
heal_type = self.heal_type.log_kind(),
|
||||
state = "started",
|
||||
queue_delay = ?queue_delay,
|
||||
"Heal task state updated"
|
||||
"Heal task started"
|
||||
);
|
||||
|
||||
let result = match &self.heal_type {
|
||||
@@ -484,7 +484,7 @@ impl HealTask {
|
||||
task_id = %self.id,
|
||||
heal_type = self.heal_type.log_kind(),
|
||||
state = "completed",
|
||||
"Heal task state updated"
|
||||
"Heal task completed"
|
||||
);
|
||||
}
|
||||
Err(Error::TaskCancelled) => {
|
||||
@@ -498,7 +498,7 @@ impl HealTask {
|
||||
task_id = %self.id,
|
||||
heal_type = self.heal_type.log_kind(),
|
||||
state = "cancelled",
|
||||
"Heal task state updated"
|
||||
"Heal task cancelled"
|
||||
);
|
||||
}
|
||||
Err(Error::TaskTimeout) => {
|
||||
@@ -512,7 +512,7 @@ impl HealTask {
|
||||
task_id = %self.id,
|
||||
heal_type = self.heal_type.log_kind(),
|
||||
state = "timed_out",
|
||||
"Heal task state updated"
|
||||
"Heal task timed out"
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
@@ -527,7 +527,7 @@ impl HealTask {
|
||||
heal_type = self.heal_type.log_kind(),
|
||||
state = "failed",
|
||||
error = %e,
|
||||
"Heal task state updated"
|
||||
"Heal task failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -539,7 +539,7 @@ impl HealTask {
|
||||
self.cancel_token.cancel();
|
||||
let mut status = self.status.write().await;
|
||||
*status = HealTaskStatus::Cancelled;
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_TASK_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -548,7 +548,7 @@ impl HealTask {
|
||||
heal_type = self.heal_type.log_kind(),
|
||||
state = "cancelled",
|
||||
source = "manual",
|
||||
"Heal task state updated"
|
||||
"Heal task cancellation requested"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -572,7 +572,7 @@ impl HealTask {
|
||||
// specific heal implementation method
|
||||
#[tracing::instrument(skip(self), fields(bucket = %bucket, object = %object, version_id = ?version_id))]
|
||||
async fn heal_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -582,7 +582,7 @@ impl HealTask {
|
||||
object,
|
||||
version_id = ?version_id,
|
||||
stage = "start",
|
||||
"Heal object workflow started"
|
||||
"Heal object started"
|
||||
);
|
||||
|
||||
// update progress
|
||||
@@ -625,7 +625,7 @@ impl HealTask {
|
||||
"Heal target object is missing"
|
||||
);
|
||||
if self.options.recreate_missing {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -634,7 +634,7 @@ impl HealTask {
|
||||
bucket,
|
||||
object,
|
||||
stage = "recreate_missing",
|
||||
"Heal object recovery started"
|
||||
"Heal object recreate requested"
|
||||
);
|
||||
return self.recreate_missing_object(bucket, object, version_id).await;
|
||||
} else {
|
||||
@@ -690,7 +690,7 @@ impl HealTask {
|
||||
// Check if this is a "File not found" error during delete operations
|
||||
let error_msg = format!("{e}");
|
||||
if error_msg.contains("File not found") || error_msg.contains("not found") {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -699,7 +699,7 @@ impl HealTask {
|
||||
bucket,
|
||||
object,
|
||||
result = "treated_as_deleted",
|
||||
"Heal object finished after target disappeared during repair"
|
||||
"Heal missing object treated as deleted"
|
||||
);
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
@@ -723,7 +723,7 @@ impl HealTask {
|
||||
|
||||
// If heal failed and remove_corrupted is enabled, delete the corrupted object
|
||||
if self.options.remove_corrupted {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_CLEANUP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -737,7 +737,7 @@ impl HealTask {
|
||||
);
|
||||
if !self.options.dry_run {
|
||||
self.await_with_control(self.storage.delete_object(bucket, object)).await?;
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_CLEANUP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -747,7 +747,7 @@ impl HealTask {
|
||||
object,
|
||||
action = "delete_corrupted_object",
|
||||
result = "deleted",
|
||||
"Heal object cleanup completed"
|
||||
"Heal corrupted object deleted"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -775,7 +775,7 @@ impl HealTask {
|
||||
"Heal object stage entered"
|
||||
);
|
||||
let object_size = result.object_size as u64;
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -786,7 +786,7 @@ impl HealTask {
|
||||
object_size = object_size,
|
||||
drives_healed = result.after.drives.len(),
|
||||
result = "ok",
|
||||
"Heal object completed"
|
||||
"Heal object repaired"
|
||||
);
|
||||
|
||||
{
|
||||
@@ -806,7 +806,7 @@ impl HealTask {
|
||||
// Check if this is a "File not found" error during delete operations
|
||||
let error_msg = format!("{e}");
|
||||
if error_msg.contains("File not found") || error_msg.contains("not found") {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -815,7 +815,7 @@ impl HealTask {
|
||||
bucket,
|
||||
object,
|
||||
result = "treated_as_deleted",
|
||||
"Heal object finished after target disappeared during repair"
|
||||
"Heal missing object treated as deleted"
|
||||
);
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
@@ -839,7 +839,7 @@ impl HealTask {
|
||||
|
||||
// If heal failed and remove_corrupted is enabled, delete the corrupted object
|
||||
if self.options.remove_corrupted {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_CLEANUP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -853,7 +853,7 @@ impl HealTask {
|
||||
);
|
||||
if !self.options.dry_run {
|
||||
self.await_with_control(self.storage.delete_object(bucket, object)).await?;
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_CLEANUP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -863,7 +863,7 @@ impl HealTask {
|
||||
object,
|
||||
action = "delete_corrupted_object",
|
||||
result = "deleted",
|
||||
"Heal object cleanup completed"
|
||||
"Heal corrupted object deleted"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -882,7 +882,7 @@ impl HealTask {
|
||||
|
||||
/// Recreate missing object (for EC decode scenarios)
|
||||
async fn recreate_missing_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -892,7 +892,7 @@ impl HealTask {
|
||||
object,
|
||||
version_id = ?version_id,
|
||||
stage = "recreate_missing",
|
||||
"Heal object recovery started"
|
||||
"Heal object recreate started"
|
||||
);
|
||||
|
||||
// Use ecstore's heal_object with recreate option
|
||||
@@ -932,7 +932,7 @@ impl HealTask {
|
||||
}
|
||||
|
||||
let object_size = result.object_size as u64;
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -942,7 +942,7 @@ impl HealTask {
|
||||
object,
|
||||
object_size,
|
||||
result = "recreated",
|
||||
"Heal object recovery finished"
|
||||
"Heal object recreated"
|
||||
);
|
||||
|
||||
{
|
||||
@@ -975,7 +975,7 @@ impl HealTask {
|
||||
}
|
||||
|
||||
async fn heal_bucket(&self, bucket: &str) -> Result<()> {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -984,7 +984,7 @@ impl HealTask {
|
||||
bucket,
|
||||
stage = "start",
|
||||
recursive = self.options.recursive,
|
||||
"Heal bucket workflow started"
|
||||
"Heal bucket started"
|
||||
);
|
||||
|
||||
// update progress
|
||||
@@ -1060,7 +1060,7 @@ impl HealTask {
|
||||
|
||||
match heal_result {
|
||||
Ok(result) => {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -1269,7 +1269,7 @@ impl HealTask {
|
||||
});
|
||||
}
|
||||
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -1287,7 +1287,7 @@ impl HealTask {
|
||||
}
|
||||
|
||||
async fn heal_metadata(&self, bucket: &str, object: &str) -> Result<()> {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_METADATA_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -1296,7 +1296,7 @@ impl HealTask {
|
||||
bucket,
|
||||
object,
|
||||
stage = "start",
|
||||
"Heal metadata workflow started"
|
||||
"Heal metadata started"
|
||||
);
|
||||
|
||||
// update progress
|
||||
@@ -1400,7 +1400,7 @@ impl HealTask {
|
||||
});
|
||||
}
|
||||
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_METADATA_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -1410,7 +1410,7 @@ impl HealTask {
|
||||
object,
|
||||
drives_healed = result.after.drives.len(),
|
||||
result = "ok",
|
||||
"Heal metadata completed"
|
||||
"Heal metadata repaired"
|
||||
);
|
||||
|
||||
{
|
||||
@@ -1447,7 +1447,7 @@ impl HealTask {
|
||||
}
|
||||
|
||||
async fn heal_mrf(&self, meta_path: &str) -> Result<()> {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_MRF_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -1455,7 +1455,7 @@ impl HealTask {
|
||||
task_id = %self.id,
|
||||
meta_path,
|
||||
stage = "start",
|
||||
"Heal MRF workflow started"
|
||||
"Heal MRF started"
|
||||
);
|
||||
|
||||
// update progress
|
||||
@@ -1530,7 +1530,7 @@ impl HealTask {
|
||||
});
|
||||
}
|
||||
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_MRF_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -1541,7 +1541,7 @@ impl HealTask {
|
||||
object = %object,
|
||||
drives_healed = result.after.drives.len(),
|
||||
result = "ok",
|
||||
"Heal MRF completed"
|
||||
"Heal MRF repaired"
|
||||
);
|
||||
|
||||
{
|
||||
@@ -1579,7 +1579,7 @@ impl HealTask {
|
||||
}
|
||||
|
||||
async fn heal_ec_decode(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_EC_DECODE_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -1589,7 +1589,7 @@ impl HealTask {
|
||||
object,
|
||||
version_id = ?version_id,
|
||||
stage = "start",
|
||||
"Heal EC decode workflow started"
|
||||
"Heal EC decode started"
|
||||
);
|
||||
|
||||
// update progress
|
||||
@@ -1694,7 +1694,7 @@ impl HealTask {
|
||||
}
|
||||
|
||||
let object_size = result.object_size as u64;
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_EC_DECODE_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -1705,7 +1705,7 @@ impl HealTask {
|
||||
object_size,
|
||||
drives_healed = result.after.drives.len(),
|
||||
result = "ok",
|
||||
"Heal EC decode completed"
|
||||
"Heal EC decode repaired"
|
||||
);
|
||||
|
||||
{
|
||||
@@ -1742,7 +1742,7 @@ impl HealTask {
|
||||
}
|
||||
|
||||
async fn heal_erasure_set(&self, buckets: Vec<String>, set_disk_id: String) -> Result<()> {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -1751,7 +1751,7 @@ impl HealTask {
|
||||
set_disk_id,
|
||||
bucket_count = buckets.len(),
|
||||
stage = "start",
|
||||
"Heal erasure set workflow started"
|
||||
"Heal erasure set started"
|
||||
);
|
||||
|
||||
// update progress
|
||||
@@ -1762,7 +1762,7 @@ impl HealTask {
|
||||
}
|
||||
|
||||
let buckets = if buckets.is_empty() {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -1770,7 +1770,7 @@ impl HealTask {
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
stage = "list_buckets",
|
||||
"Heal erasure set resolved bucket list from storage"
|
||||
"Heal erasure set bucket list resolved"
|
||||
);
|
||||
let bucket_infos = self.await_with_control(self.storage.list_buckets()).await?;
|
||||
bucket_infos.into_iter().map(|info| info.name).collect()
|
||||
@@ -1814,7 +1814,7 @@ impl HealTask {
|
||||
});
|
||||
}
|
||||
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -1823,7 +1823,7 @@ impl HealTask {
|
||||
set_disk_id,
|
||||
drives_healed = result.after.drives.len(),
|
||||
result = "format_ok",
|
||||
"Heal erasure set format repair completed"
|
||||
"Heal erasure set format repaired"
|
||||
);
|
||||
}
|
||||
Err(Error::TaskCancelled) => return Err(Error::TaskCancelled),
|
||||
@@ -1970,7 +1970,7 @@ impl HealTask {
|
||||
|
||||
match result {
|
||||
Ok(_) => {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -1979,7 +1979,7 @@ impl HealTask {
|
||||
set_disk_id,
|
||||
bucket_count = buckets.len(),
|
||||
result = "ok",
|
||||
"Heal erasure set completed"
|
||||
"Heal erasure set repaired"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
+79
-11
@@ -19,7 +19,13 @@ use rustfs_ecstore::store::ECStore;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use store::object::ObjectStore;
|
||||
use sys::IamSys;
|
||||
use tracing::{error, info, instrument, warn};
|
||||
use tracing::{debug, error, info, instrument, warn};
|
||||
|
||||
const LOG_COMPONENT_IAM: &str = "iam";
|
||||
const LOG_SUBSYSTEM_RUNTIME: &str = "runtime";
|
||||
const LOG_SUBSYSTEM_OIDC: &str = "oidc";
|
||||
const EVENT_IAM_STATE: &str = "iam_state";
|
||||
const EVENT_OIDC_STATE: &str = "oidc_state";
|
||||
|
||||
pub mod cache;
|
||||
pub mod error;
|
||||
@@ -37,11 +43,23 @@ static OIDC_SYS: OnceLock<Arc<OidcSys>> = OnceLock::new();
|
||||
#[instrument(skip(ecstore))]
|
||||
pub async fn init_iam_sys(ecstore: Arc<ECStore>) -> Result<()> {
|
||||
if IAM_SYS.get().is_some() {
|
||||
info!("IAM system already initialized, skipping.");
|
||||
info!(
|
||||
event = EVENT_IAM_STATE,
|
||||
component = LOG_COMPONENT_IAM,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "already_initialized",
|
||||
"IAM runtime already initialized"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
info!("Starting IAM system initialization sequence...");
|
||||
info!(
|
||||
event = EVENT_IAM_STATE,
|
||||
component = LOG_COMPONENT_IAM,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "starting",
|
||||
"IAM runtime starting"
|
||||
);
|
||||
|
||||
// 1. Create the persistent storage adapter
|
||||
let storage_adapter = ObjectStore::new(ecstore);
|
||||
@@ -55,11 +73,23 @@ pub async fn init_iam_sys(ecstore: Arc<ECStore>) -> Result<()> {
|
||||
|
||||
// 4. Securely set the global singleton
|
||||
if IAM_SYS.set(iam_instance).is_err() {
|
||||
error!("Critical: Race condition detected during IAM initialization!");
|
||||
error!(
|
||||
event = EVENT_IAM_STATE,
|
||||
component = LOG_COMPONENT_IAM,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "singleton_set_failed",
|
||||
"IAM runtime singleton set failed"
|
||||
);
|
||||
return Err(Error::IamSysAlreadyInitialized);
|
||||
}
|
||||
|
||||
info!("IAM system initialization completed successfully.");
|
||||
info!(
|
||||
event = EVENT_IAM_STATE,
|
||||
component = LOG_COMPONENT_IAM,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "ready",
|
||||
"IAM runtime ready"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -84,29 +114,67 @@ pub fn get_global_iam_sys() -> Option<Arc<IamSys<ObjectStore>>> {
|
||||
/// Initialize the global OIDC system. Non-fatal if no OIDC providers are configured.
|
||||
pub async fn init_oidc_sys() -> Result<()> {
|
||||
if OIDC_SYS.get().is_some() {
|
||||
info!("OIDC system already initialized, skipping.");
|
||||
debug!(
|
||||
event = EVENT_OIDC_STATE,
|
||||
component = LOG_COMPONENT_IAM,
|
||||
subsystem = LOG_SUBSYSTEM_OIDC,
|
||||
state = "already_initialized",
|
||||
"OIDC runtime already initialized"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
info!("Starting OIDC system initialization...");
|
||||
debug!(
|
||||
event = EVENT_OIDC_STATE,
|
||||
component = LOG_COMPONENT_IAM,
|
||||
subsystem = LOG_SUBSYSTEM_OIDC,
|
||||
state = "starting",
|
||||
"OIDC runtime starting"
|
||||
);
|
||||
|
||||
let oidc_sys = match OidcSys::new().await {
|
||||
Ok(sys) => {
|
||||
if sys.has_providers() {
|
||||
info!("OIDC system initialized with {} provider(s)", sys.list_providers().len());
|
||||
debug!(
|
||||
event = EVENT_OIDC_STATE,
|
||||
component = LOG_COMPONENT_IAM,
|
||||
subsystem = LOG_SUBSYSTEM_OIDC,
|
||||
provider_count = sys.list_providers().len(),
|
||||
state = "ready",
|
||||
"OIDC runtime ready"
|
||||
);
|
||||
} else {
|
||||
info!("No OIDC providers configured");
|
||||
debug!(
|
||||
event = EVENT_OIDC_STATE,
|
||||
component = LOG_COMPONENT_IAM,
|
||||
subsystem = LOG_SUBSYSTEM_OIDC,
|
||||
state = "empty",
|
||||
"OIDC runtime has no providers"
|
||||
);
|
||||
}
|
||||
sys
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("OIDC initialization failed (non-fatal): {}", e);
|
||||
warn!(
|
||||
event = EVENT_OIDC_STATE,
|
||||
component = LOG_COMPONENT_IAM,
|
||||
subsystem = LOG_SUBSYSTEM_OIDC,
|
||||
state = "init_failed_non_fatal",
|
||||
error = %e,
|
||||
"OIDC runtime initialization failed"
|
||||
);
|
||||
OidcSys::empty().map_err(Error::StringError)?
|
||||
}
|
||||
};
|
||||
|
||||
if OIDC_SYS.set(Arc::new(oidc_sys)).is_err() {
|
||||
warn!("Race condition during OIDC initialization (non-fatal)");
|
||||
warn!(
|
||||
event = EVENT_OIDC_STATE,
|
||||
component = LOG_COMPONENT_IAM,
|
||||
subsystem = LOG_SUBSYSTEM_OIDC,
|
||||
state = "singleton_set_race",
|
||||
"OIDC runtime singleton set raced"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
+14
-10
@@ -54,7 +54,7 @@ use tokio::{
|
||||
},
|
||||
};
|
||||
use tracing::warn;
|
||||
use tracing::{error, info};
|
||||
use tracing::{debug, error};
|
||||
|
||||
const IAM_FORMAT_FILE: &str = "format.json";
|
||||
const IAM_FORMAT_VERSION_1: i32 = 1;
|
||||
@@ -153,10 +153,14 @@ where
|
||||
if let Err(e) = self.clone().load().await {
|
||||
if attempt == MAX_RETRIES - 1 {
|
||||
self.state.store(IamState::Error as u8, Ordering::SeqCst);
|
||||
warn!("IAM failed to load initial data after {} attempts: {:?}", MAX_RETRIES, e);
|
||||
warn!(
|
||||
attempts = MAX_RETRIES,
|
||||
error = ?e,
|
||||
"IAM initial load failed"
|
||||
);
|
||||
load_error = Some(e);
|
||||
} else {
|
||||
warn!("IAM load failed, retrying... attempt {}", attempt + 1);
|
||||
warn!(attempt = attempt + 1, max_attempts = MAX_RETRIES, "IAM load retry scheduled");
|
||||
tokio::time::sleep(INITIAL_LOAD_RETRY_DELAY).await;
|
||||
}
|
||||
} else {
|
||||
@@ -169,7 +173,7 @@ where
|
||||
}
|
||||
|
||||
self.state.store(IamState::Ready as u8, Ordering::SeqCst);
|
||||
info!("IAM System successfully initialized and marked as READY");
|
||||
debug!(state = "ready", "IAM manager ready");
|
||||
|
||||
// Background ticker for synchronization
|
||||
// Check if environment variable is set
|
||||
@@ -185,20 +189,20 @@ where
|
||||
loop {
|
||||
select! {
|
||||
_ = ticker.tick() => {
|
||||
info!("iam load ticker");
|
||||
debug!(source = "ticker", "IAM reload tick");
|
||||
if let Err(err) =s.clone().load().await{
|
||||
warn!("iam load err {:?}", err);
|
||||
warn!(source = "ticker", error = ?err, "IAM reload failed");
|
||||
}
|
||||
},
|
||||
i = receiver.recv() => {
|
||||
info!("iam load receiver");
|
||||
debug!(source = "receiver", "IAM reload signal received");
|
||||
match i {
|
||||
Some(t) => {
|
||||
let last = s.last_timestamp.load(Ordering::Relaxed);
|
||||
if last <= t {
|
||||
info!("iam load receiver load");
|
||||
debug!(source = "receiver", "IAM reload accepted");
|
||||
if let Err(err) =s.clone().load().await{
|
||||
warn!("iam load err {:?}", err);
|
||||
warn!(source = "receiver", error = ?err, "IAM reload failed");
|
||||
}
|
||||
ticker.reset();
|
||||
}
|
||||
@@ -1294,7 +1298,7 @@ where
|
||||
let cache = self.cache.snapshot();
|
||||
let users = Arc::clone(&cache.users);
|
||||
if let Some(x) = users.get(access_key) {
|
||||
warn!("user already exists: {:?}", x);
|
||||
warn!(error = ?x, "IAM user already exists");
|
||||
if x.credentials.is_temp() {
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
|
||||
@@ -40,7 +40,7 @@ use std::pin::Pin;
|
||||
use std::sync::{LazyLock, Mutex, MutexGuard, RwLock};
|
||||
use std::time::{Duration as StdDuration, Instant};
|
||||
use tokio::time::sleep;
|
||||
use tracing::{error, info, warn};
|
||||
use tracing::{debug, error, warn};
|
||||
use url::Url;
|
||||
|
||||
const OIDC_JWKS_REFRESH_INTERVAL: StdDuration = StdDuration::from_secs(24 * 60 * 60);
|
||||
@@ -76,7 +76,7 @@ fn lock_oidc_plugin_authn_metrics<'a, T>(mutex: &'a Mutex<T>, metric: &'static s
|
||||
match mutex.lock() {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => {
|
||||
warn!("recovering poisoned OIDC plugin authn metrics lock: {}", metric);
|
||||
warn!(metric, "Recovering poisoned OIDC authn metrics lock");
|
||||
err.into_inner()
|
||||
}
|
||||
}
|
||||
@@ -417,18 +417,18 @@ impl OidcSys {
|
||||
for sourced_config in parsed_configs {
|
||||
let config = sourced_config.config;
|
||||
if !config.enabled {
|
||||
info!("OIDC provider '{}' is disabled, skipping", config.id);
|
||||
debug!(provider = %config.id, "OIDC provider disabled");
|
||||
continue;
|
||||
}
|
||||
|
||||
match Self::discover_provider(&config, &http_client).await {
|
||||
Ok(state) => {
|
||||
info!("OIDC provider '{}' discovered successfully", config.id);
|
||||
debug!(provider = %config.id, "OIDC provider discovered");
|
||||
provider_states.insert(config.id.clone(), state);
|
||||
configs.insert(config.id.clone(), config);
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to discover OIDC provider '{}': {}", config.id, e);
|
||||
error!(provider = %config.id, error = %e, "OIDC provider discovery failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ use std::time::{Duration, Instant};
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
use tokio::sync::mpsc::{self, Sender};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, warn};
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
pub static IAM_CONFIG_PREFIX: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_CONFIG_PREFIX}/iam"));
|
||||
pub static IAM_CONFIG_USERS_PREFIX: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_CONFIG_PREFIX}/iam/users/"));
|
||||
@@ -307,11 +307,11 @@ impl ObjectStore {
|
||||
}
|
||||
Err(StorageError::PreconditionFailed) => {
|
||||
Self::complete_lazy_rewrite(path.as_str(), false);
|
||||
debug!("iam lazy rewrite skipped due to stale etag, path: {}", path);
|
||||
debug!(path = %path, state = "stale_etag", "IAM lazy rewrite skipped");
|
||||
}
|
||||
Err(err) => {
|
||||
Self::complete_lazy_rewrite(path.as_str(), false);
|
||||
warn!("iam lazy rewrite failed, path: {}, err: {}", path, err);
|
||||
warn!(path = %path, error = %err, state = "rewrite_failed", "IAM lazy rewrite failed");
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -377,7 +377,7 @@ impl ObjectStore {
|
||||
bucket = Self::BUCKET_NAME,
|
||||
prefix = %path,
|
||||
error = %err,
|
||||
"system path walk failed"
|
||||
"IAM config walk failed"
|
||||
);
|
||||
let _ = sender_on_error
|
||||
.send(StringOrErr {
|
||||
@@ -643,7 +643,7 @@ impl Store for ObjectStore {
|
||||
let outcome = match Self::decrypt_data_with_source(&data) {
|
||||
Ok(v) => v,
|
||||
Err(err) => {
|
||||
warn!("config decrypt failed, keeping file: {}, path: {}", err, path_ref);
|
||||
warn!(path = %path_ref, error = %err, "IAM config decrypt failed; keeping file");
|
||||
// keep the config file when decrypt failed - do not delete
|
||||
return Err(Error::ConfigNotFound);
|
||||
}
|
||||
@@ -692,7 +692,7 @@ impl Store for ObjectStore {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(wait_ms)).await;
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Final failure saving IAM config to {}: {:?}", path_ref, e);
|
||||
error!(path = %path_ref, error = ?e, "IAM config save failed");
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
@@ -717,7 +717,7 @@ impl Store for ObjectStore {
|
||||
debug!("Saving IAM identity to path: {}", path);
|
||||
|
||||
self.save_iam_config(user_identity, path).await.map_err(|e| {
|
||||
error!("ObjectStore save failure for {}: {:?}", name, e);
|
||||
error!(name, error = ?e, "IAM identity save failed");
|
||||
e
|
||||
})
|
||||
}
|
||||
@@ -739,10 +739,10 @@ impl Store for ObjectStore {
|
||||
.await
|
||||
.map_err(|err| {
|
||||
if is_err_config_not_found(&err) {
|
||||
warn!("load_user_identity failed: no such user, name: {name}, user_type: {user_type:?}");
|
||||
warn!(name, user_type = ?user_type, "IAM user identity missing");
|
||||
Error::NoSuchUser(name.to_owned())
|
||||
} else {
|
||||
warn!("load_user_identity failed: {err:?}, name: {name}, user_type: {user_type:?}");
|
||||
warn!(name, user_type = ?user_type, error = ?err, "IAM user identity load failed");
|
||||
err
|
||||
}
|
||||
})?;
|
||||
@@ -750,9 +750,7 @@ impl Store for ObjectStore {
|
||||
if u.credentials.is_expired() {
|
||||
let _ = self.delete_iam_config(get_user_identity_path(name, user_type)).await;
|
||||
let _ = self.delete_iam_config(get_mapped_policy_path(name, user_type, false)).await;
|
||||
warn!(
|
||||
"load_user_identity failed: user is expired, delete the user and mapped policy, name: {name}, user_type: {user_type:?}"
|
||||
);
|
||||
warn!(name, user_type = ?user_type, "IAM user identity expired and was removed");
|
||||
return Err(Error::NoSuchUser(name.to_owned()));
|
||||
}
|
||||
|
||||
@@ -776,7 +774,7 @@ impl Store for ObjectStore {
|
||||
let _ = self.delete_iam_config(get_user_identity_path(name, user_type)).await;
|
||||
let _ = self.delete_iam_config(get_mapped_policy_path(name, user_type, false)).await;
|
||||
}
|
||||
warn!("extract_jwt_claims failed: {err:?}, name: {name}, user_type: {user_type:?}");
|
||||
warn!(name, user_type = ?user_type, error = ?err, "IAM JWT claim extraction failed");
|
||||
return Err(Error::NoSuchUser(name.to_owned()));
|
||||
}
|
||||
}
|
||||
@@ -804,7 +802,7 @@ impl Store for ObjectStore {
|
||||
|
||||
while let Some(v) = rx.recv().await {
|
||||
if let Some(err) = v.err {
|
||||
warn!("list_iam_config_items {:?}", err);
|
||||
warn!(error = ?err, "IAM config item listing failed");
|
||||
let _ = ctx.cancel();
|
||||
|
||||
return Err(err);
|
||||
@@ -866,7 +864,7 @@ impl Store for ObjectStore {
|
||||
|
||||
while let Some(v) = rx.recv().await {
|
||||
if let Some(err) = v.err {
|
||||
warn!("list_iam_config_items {:?}", err);
|
||||
warn!(error = ?err, "IAM config item listing failed");
|
||||
let _ = ctx.cancel();
|
||||
|
||||
return Err(err);
|
||||
@@ -934,7 +932,7 @@ impl Store for ObjectStore {
|
||||
|
||||
while let Some(v) = rx.recv().await {
|
||||
if let Some(err) = v.err {
|
||||
warn!("list_iam_config_items {:?}", err);
|
||||
warn!(error = ?err, "IAM config item listing failed");
|
||||
let _ = ctx.cancel();
|
||||
|
||||
return Err(err);
|
||||
@@ -1009,7 +1007,7 @@ impl Store for ObjectStore {
|
||||
|
||||
while let Some(v) = rx.recv().await {
|
||||
if let Some(err) = v.err {
|
||||
warn!("list_iam_config_items {:?}", err);
|
||||
warn!(error = ?err, "IAM config item listing failed");
|
||||
let _ = ctx.cancel();
|
||||
|
||||
return Err(err);
|
||||
@@ -1044,7 +1042,7 @@ impl Store for ObjectStore {
|
||||
|
||||
let policy_name = rustfs_utils::path::dir(&policies_list[idx]);
|
||||
|
||||
info!("load policy: {}", policy_name);
|
||||
debug!(policy = %policy_name, "IAM policy loaded");
|
||||
|
||||
policy_docs_cache.insert(policy_name, p);
|
||||
}
|
||||
@@ -1059,7 +1057,7 @@ impl Store for ObjectStore {
|
||||
}
|
||||
|
||||
let policy_name = rustfs_utils::path::dir(&policies_list[idx]);
|
||||
info!("load policy: {}", policy_name);
|
||||
debug!(policy = %policy_name, "IAM policy loaded");
|
||||
policy_docs_cache.insert(policy_name, p);
|
||||
}
|
||||
|
||||
@@ -1083,7 +1081,7 @@ impl Store for ObjectStore {
|
||||
}
|
||||
|
||||
let name = rustfs_utils::path::dir(&item_name_list[idx]);
|
||||
info!("load reg user: {}", name);
|
||||
debug!(user = %name, "IAM regular user loaded");
|
||||
user_items_cache.insert(name, p);
|
||||
}
|
||||
break;
|
||||
@@ -1097,7 +1095,7 @@ impl Store for ObjectStore {
|
||||
}
|
||||
|
||||
let name = rustfs_utils::path::dir(&item_name_list[idx]);
|
||||
info!("load reg user: {}", name);
|
||||
debug!(user = %name, "IAM regular user loaded");
|
||||
user_items_cache.insert(name, p);
|
||||
}
|
||||
|
||||
@@ -1112,7 +1110,7 @@ impl Store for ObjectStore {
|
||||
|
||||
for item in item_name_list.iter() {
|
||||
let name = rustfs_utils::path::dir(item);
|
||||
info!("load group: {}", name);
|
||||
debug!(group = %name, "IAM group loaded");
|
||||
if let Err(err) = self.load_group(&name, &mut items_cache).await {
|
||||
return Err(Error::other(format!("load group failed: {err}")));
|
||||
};
|
||||
@@ -1140,7 +1138,7 @@ impl Store for ObjectStore {
|
||||
}
|
||||
|
||||
let name = item_name_list[idx].trim_end_matches(".json").to_owned();
|
||||
info!("load user policy: {}", name);
|
||||
debug!(user = %name, "IAM user policy loaded");
|
||||
items_cache.insert(name, p);
|
||||
}
|
||||
break;
|
||||
@@ -1156,7 +1154,7 @@ impl Store for ObjectStore {
|
||||
}
|
||||
|
||||
let name = item_name_list[idx].trim_end_matches(".json").to_owned();
|
||||
info!("load user policy: {}", name);
|
||||
debug!(user = %name, "IAM user policy loaded");
|
||||
items_cache.insert(name, p);
|
||||
}
|
||||
|
||||
@@ -1174,7 +1172,7 @@ impl Store for ObjectStore {
|
||||
for item in item_name_list.iter() {
|
||||
let name = item.trim_end_matches(".json");
|
||||
|
||||
info!("load group policy: {}", name);
|
||||
debug!(group = %name, "IAM group policy loaded");
|
||||
if let Err(err) = self.load_mapped_policy(name, UserType::Reg, true, &mut items_cache).await
|
||||
&& !is_err_no_such_policy(&err)
|
||||
{
|
||||
@@ -1193,7 +1191,7 @@ impl Store for ObjectStore {
|
||||
|
||||
for item in item_name_list.iter() {
|
||||
let name = rustfs_utils::path::dir(item);
|
||||
info!("load svc user: {}", name);
|
||||
debug!(user = %name, "IAM service user loaded");
|
||||
if let Err(err) = self.load_user(&name, UserType::Svc, &mut items_cache).await
|
||||
&& !is_err_no_such_user(&err)
|
||||
{
|
||||
@@ -1204,7 +1202,7 @@ impl Store for ObjectStore {
|
||||
for (_, v) in items_cache.iter() {
|
||||
let parent = v.credentials.parent_user.clone();
|
||||
if !user_items_cache.contains_key(&parent) {
|
||||
info!("load sts user policy: {}", parent);
|
||||
debug!(user = %parent, "IAM STS parent policy loaded");
|
||||
if let Err(err) = self
|
||||
.load_mapped_policy(&parent, UserType::Sts, false, &mut sts_policies_cache)
|
||||
.await
|
||||
@@ -1223,12 +1221,12 @@ impl Store for ObjectStore {
|
||||
// sts users
|
||||
if let Some(item_name_list) = listed_config_items.get(STS_LIST_KEY) {
|
||||
for item in item_name_list.iter() {
|
||||
info!("load sts user path: {}", item);
|
||||
debug!(path = %item, "IAM STS user path discovered");
|
||||
|
||||
let name = rustfs_utils::path::dir(item);
|
||||
info!("load sts user: {}", name);
|
||||
debug!(user = %name, "IAM STS user loaded");
|
||||
if let Err(err) = self.load_user(&name, UserType::Sts, &mut sts_items_cache).await {
|
||||
info!("load sts user failed: {}", err);
|
||||
debug!(user = %name, error = %err, "IAM STS user load failed");
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1237,12 +1235,12 @@ impl Store for ObjectStore {
|
||||
if let Some(item_name_list) = listed_config_items.get(POLICY_DB_STS_USERS_LIST_KEY) {
|
||||
for item in item_name_list.iter() {
|
||||
let name = item.trim_end_matches(".json");
|
||||
info!("load sts user policy: {}", name);
|
||||
debug!(user = %name, "IAM STS user policy loaded");
|
||||
if let Err(err) = self
|
||||
.load_mapped_policy(name, UserType::Sts, false, &mut sts_policies_cache)
|
||||
.await
|
||||
{
|
||||
info!("load sts user policy failed: {}", err);
|
||||
debug!(user = %name, error = %err, "IAM STS user policy load failed");
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1264,7 +1262,7 @@ impl Store for ObjectStore {
|
||||
cache.replace_sts_policies(sts_policies_cache);
|
||||
cache.build_user_group_memberships();
|
||||
} else {
|
||||
warn!("skip IAM full reload cache commit because one or more IAM caches changed during reload");
|
||||
warn!("IAM full reload cache commit skipped due to concurrent cache changes");
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
use tokio::fs;
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{debug, info, warn};
|
||||
use tracing::{debug, warn};
|
||||
|
||||
/// Local KMS client that stores keys in local files
|
||||
pub struct LocalKmsClient {
|
||||
@@ -74,7 +74,7 @@ impl LocalKmsClient {
|
||||
// Create key directory if it doesn't exist
|
||||
if !config.key_dir.exists() {
|
||||
fs::create_dir_all(&config.key_dir).await?;
|
||||
info!("Created KMS key directory: {:?}", config.key_dir);
|
||||
debug!(path = ?config.key_dir, "KMS key directory created");
|
||||
}
|
||||
|
||||
// Initialize master cipher if master key is provided
|
||||
@@ -219,7 +219,7 @@ impl LocalKmsClient {
|
||||
|
||||
fs::rename(&temp_path, &key_path).await?;
|
||||
|
||||
info!("Saved master key {} to {:?}", master_key.key_id, key_path);
|
||||
debug!(key_id = %master_key.key_id, path = ?key_path, "Local KMS master key saved");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -278,7 +278,7 @@ impl KmsClient for LocalKmsClient {
|
||||
|
||||
let data_key = DataKeyInfo::new(envelope.key_id, 1, Some(plaintext_key), ciphertext, request.key_spec.clone());
|
||||
|
||||
info!("Generated data key for master key: {}", request.master_key_id);
|
||||
debug!(key_id = %request.master_key_id, "Local KMS data key generated");
|
||||
Ok(data_key)
|
||||
}
|
||||
|
||||
@@ -334,7 +334,7 @@ impl KmsClient for LocalKmsClient {
|
||||
.decrypt_with_master_key(&envelope.master_key_id, &envelope.encrypted_key, &envelope.nonce)
|
||||
.await?;
|
||||
|
||||
info!("Successfully decrypted data");
|
||||
debug!("Local KMS data decrypted");
|
||||
Ok(plaintext)
|
||||
}
|
||||
|
||||
@@ -367,7 +367,7 @@ impl KmsClient for LocalKmsClient {
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key.clone());
|
||||
|
||||
info!("Created master key: {}", key_id);
|
||||
debug!(key_id, "Local KMS master key created");
|
||||
Ok(master_key)
|
||||
}
|
||||
|
||||
@@ -453,7 +453,7 @@ impl KmsClient for LocalKmsClient {
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key);
|
||||
|
||||
info!("Enabled key: {}", key_id);
|
||||
debug!(key_id, "Local KMS key enabled");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -470,7 +470,7 @@ impl KmsClient for LocalKmsClient {
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key);
|
||||
|
||||
info!("Disabled key: {}", key_id);
|
||||
debug!(key_id, "Local KMS key disabled");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -492,7 +492,7 @@ impl KmsClient for LocalKmsClient {
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key);
|
||||
|
||||
warn!("Scheduled key deletion: {}", key_id);
|
||||
debug!(key_id, "Local KMS key deletion scheduled");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -509,7 +509,7 @@ impl KmsClient for LocalKmsClient {
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key);
|
||||
|
||||
info!("Canceled deletion for key: {}", key_id);
|
||||
debug!(key_id, "Local KMS key deletion canceled");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -528,7 +528,7 @@ impl KmsClient for LocalKmsClient {
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key.clone());
|
||||
|
||||
info!("Rotated key: {}", key_id);
|
||||
debug!(key_id, "Local KMS key rotated");
|
||||
Ok(master_key)
|
||||
}
|
||||
|
||||
@@ -721,7 +721,7 @@ impl KmsBackend for LocalKmsBackend {
|
||||
let mut cache = self.client.key_cache.write().await;
|
||||
cache.remove(key_id);
|
||||
|
||||
info!("Immediately deleted key: {}", key_id);
|
||||
debug!(key_id, "Local KMS key deleted immediately");
|
||||
|
||||
// Return success response for immediate deletion
|
||||
let key_metadata = KeyMetadata {
|
||||
|
||||
@@ -98,7 +98,7 @@ impl VaultKmsClient {
|
||||
let client =
|
||||
VaultClient::new(settings).map_err(|e| KmsError::backend_error(format!("Failed to create Vault client: {e}")))?;
|
||||
|
||||
info!("Successfully connected to Vault at {}", config.address);
|
||||
info!(address = %config.address, "Vault KMS backend connected");
|
||||
|
||||
Ok(Self {
|
||||
client,
|
||||
@@ -136,7 +136,7 @@ impl VaultKmsClient {
|
||||
|
||||
// If encrypted_key_material is empty, generate and store it (fix for old keys)
|
||||
if key_data.encrypted_key_material.is_empty() {
|
||||
warn!("Key {} has empty encrypted_key_material, generating and storing new key material", key_id);
|
||||
warn!(key_id, "Vault KMS key material missing; regenerating");
|
||||
let key_material = generate_key_material(&key_data.algorithm)?;
|
||||
key_data.encrypted_key_material = self.encrypt_key_material(&key_material).await?;
|
||||
// Store the updated key data back to Vault
|
||||
@@ -147,7 +147,7 @@ impl VaultKmsClient {
|
||||
let key_material = match self.decrypt_key_material(&key_data.encrypted_key_material).await {
|
||||
Ok(km) => km,
|
||||
Err(e) => {
|
||||
warn!("Failed to decrypt key material for key {}: {}, generating new key material", key_id, e);
|
||||
warn!(key_id, error = %e, "Vault KMS key material decrypt failed; regenerating");
|
||||
let new_key_material = generate_key_material(&key_data.algorithm)?;
|
||||
key_data.encrypted_key_material = self.encrypt_key_material(&new_key_material).await?;
|
||||
// Store the updated key data back to Vault
|
||||
@@ -210,7 +210,7 @@ impl VaultKmsClient {
|
||||
// If encrypted_key_material is empty, generate it (this handles the case where
|
||||
// an old key was created without proper key material)
|
||||
if existing_key_data.encrypted_key_material.is_empty() {
|
||||
warn!("Key {} has empty encrypted_key_material, generating new key material", key_id);
|
||||
warn!(key_id, "Vault KMS key metadata missing encrypted key material");
|
||||
let key_material = generate_key_material(&existing_key_data.algorithm)?;
|
||||
existing_key_data.encrypted_key_material = self.encrypt_key_material(&key_material).await?;
|
||||
}
|
||||
@@ -316,7 +316,7 @@ impl KmsClient for VaultKmsClient {
|
||||
|
||||
let data_key = DataKeyInfo::new(envelope.key_id, 1, Some(plaintext_key), ciphertext, request.key_spec.clone());
|
||||
|
||||
info!("Generated data key for master key: {}", request.master_key_id);
|
||||
debug!(key_id = %request.master_key_id, "Vault KMS data key generated");
|
||||
Ok(data_key)
|
||||
}
|
||||
|
||||
@@ -373,7 +373,7 @@ impl KmsClient for VaultKmsClient {
|
||||
.decrypt_with_master_key(&envelope.master_key_id, &envelope.encrypted_key, &envelope.nonce)
|
||||
.await?;
|
||||
|
||||
info!("Successfully decrypted data");
|
||||
debug!("Vault KMS data decrypted");
|
||||
Ok(plaintext)
|
||||
}
|
||||
|
||||
@@ -418,7 +418,7 @@ impl KmsClient for VaultKmsClient {
|
||||
created_by: None,
|
||||
};
|
||||
|
||||
info!("Successfully created master key: {}", key_id);
|
||||
debug!(key_id, "Vault KMS master key created");
|
||||
Ok(master_key)
|
||||
}
|
||||
|
||||
@@ -486,7 +486,7 @@ impl KmsClient for VaultKmsClient {
|
||||
key_data.status = KeyStatus::Active;
|
||||
self.store_key_data(key_id, &key_data).await?;
|
||||
|
||||
info!("Enabled key: {}", key_id);
|
||||
debug!(key_id, "Vault KMS key enabled");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -497,7 +497,7 @@ impl KmsClient for VaultKmsClient {
|
||||
key_data.status = KeyStatus::Disabled;
|
||||
self.store_key_data(key_id, &key_data).await?;
|
||||
|
||||
info!("Disabled key: {}", key_id);
|
||||
debug!(key_id, "Vault KMS key disabled");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -513,7 +513,7 @@ impl KmsClient for VaultKmsClient {
|
||||
key_data.status = KeyStatus::PendingDeletion;
|
||||
self.store_key_data(key_id, &key_data).await?;
|
||||
|
||||
info!("Scheduled key deletion: {}", key_id);
|
||||
debug!(key_id, "Vault KMS key deletion scheduled");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -524,7 +524,7 @@ impl KmsClient for VaultKmsClient {
|
||||
key_data.status = KeyStatus::Active;
|
||||
self.store_key_data(key_id, &key_data).await?;
|
||||
|
||||
info!("Canceled key deletion: {}", key_id);
|
||||
debug!(key_id, "Vault KMS key deletion canceled");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -553,7 +553,7 @@ impl KmsClient for VaultKmsClient {
|
||||
created_by: None,
|
||||
};
|
||||
|
||||
info!("Successfully rotated key: {}", key_id);
|
||||
debug!(key_id, "Vault KMS key rotated");
|
||||
Ok(master_key)
|
||||
}
|
||||
|
||||
@@ -573,7 +573,7 @@ impl KmsClient for VaultKmsClient {
|
||||
debug!("Vault health check passed - 404 error is expected when no keys exist yet");
|
||||
Ok(())
|
||||
} else {
|
||||
warn!("Vault health check failed: {}", e);
|
||||
warn!(error = %e, "Vault KMS health check failed");
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ use rand::random;
|
||||
use std::collections::HashMap;
|
||||
use std::io::Cursor;
|
||||
use tokio::io::{AsyncRead, AsyncReadExt};
|
||||
use tracing::{debug, info};
|
||||
use tracing::debug;
|
||||
use zeroize::Zeroize;
|
||||
|
||||
/// Data key for object encryption
|
||||
@@ -209,7 +209,7 @@ impl ObjectEncryptionService {
|
||||
// Generate a unique random nonce for this data key
|
||||
// This ensures each object/part gets a unique base nonce for streaming encryption
|
||||
let nonce: [u8; 12] = random();
|
||||
tracing::info!("Generated random nonce for data key: {:02x?}", nonce);
|
||||
tracing::debug!("Generated random nonce for data key");
|
||||
|
||||
let data_key = DataKey {
|
||||
plaintext_key: data_key_response
|
||||
@@ -302,7 +302,7 @@ impl ObjectEncryptionService {
|
||||
key_id: actual_key_id.to_string(),
|
||||
};
|
||||
if let Err(KmsError::KeyNotFound { .. }) = self.kms_manager.describe_key(describe_req).await {
|
||||
info!("Auto-creating SSE-S3 key: {}", actual_key_id);
|
||||
debug!(key_id = %actual_key_id, "Auto-creating SSE-S3 key");
|
||||
let create_req = CreateKeyRequest {
|
||||
key_name: Some(actual_key_id.to_string()),
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
@@ -364,7 +364,13 @@ impl ObjectEncryptionService {
|
||||
encrypted_data_key: data_key.ciphertext_blob,
|
||||
};
|
||||
|
||||
info!("Successfully encrypted object {}/{} ({} bytes)", bucket, object_key, original_size);
|
||||
debug!(
|
||||
bucket,
|
||||
object = object_key,
|
||||
original_size,
|
||||
algorithm = %algorithm.as_str(),
|
||||
"Object encrypted"
|
||||
);
|
||||
|
||||
Ok(EncryptionResult { ciphertext, metadata })
|
||||
}
|
||||
@@ -429,7 +435,13 @@ impl ObjectEncryptionService {
|
||||
// Decrypt the data
|
||||
let plaintext = cipher.decrypt(&ciphertext, &metadata.iv, tag, &aad)?;
|
||||
|
||||
info!("Successfully decrypted object {}/{} ({} bytes)", bucket, object_key, plaintext.len());
|
||||
debug!(
|
||||
bucket,
|
||||
object = object_key,
|
||||
plaintext_len = plaintext.len(),
|
||||
algorithm = %metadata.algorithm,
|
||||
"Object decrypted"
|
||||
);
|
||||
|
||||
Ok(Box::new(Cursor::new(plaintext)))
|
||||
}
|
||||
@@ -507,7 +519,7 @@ impl ObjectEncryptionService {
|
||||
encrypted_data_key: Vec::new(), // Empty for SSE-C
|
||||
};
|
||||
|
||||
info!(
|
||||
debug!(
|
||||
"Successfully encrypted object {}/{} with SSE-C ({} bytes)",
|
||||
bucket, object_key, original_size
|
||||
);
|
||||
@@ -568,7 +580,7 @@ impl ObjectEncryptionService {
|
||||
// Decrypt the data
|
||||
let plaintext = cipher.decrypt(&ciphertext, &metadata.iv, tag, &aad)?;
|
||||
|
||||
info!(
|
||||
debug!(
|
||||
"Successfully decrypted SSE-C object {}/{} ({} bytes)",
|
||||
bucket,
|
||||
object_key,
|
||||
|
||||
@@ -25,7 +25,11 @@ use std::sync::{
|
||||
atomic::{AtomicU64, Ordering},
|
||||
};
|
||||
use tokio::sync::{Mutex, RwLock};
|
||||
use tracing::{error, info, warn};
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
const LOG_COMPONENT_KMS: &str = "kms";
|
||||
const LOG_SUBSYSTEM_SERVICE: &str = "service";
|
||||
const EVENT_KMS_SERVICE_STATE: &str = "kms_service_state";
|
||||
|
||||
/// KMS service status
|
||||
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
||||
@@ -106,7 +110,13 @@ impl KmsServiceManager {
|
||||
*status = KmsServiceStatus::Configured;
|
||||
}
|
||||
|
||||
info!("KMS configuration updated successfully");
|
||||
debug!(
|
||||
event = EVENT_KMS_SERVICE_STATE,
|
||||
component = LOG_COMPONENT_KMS,
|
||||
subsystem = LOG_SUBSYSTEM_SERVICE,
|
||||
state = "configured",
|
||||
"KMS service configured"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -132,7 +142,14 @@ impl KmsServiceManager {
|
||||
}
|
||||
};
|
||||
|
||||
info!("Starting KMS service with backend: {:?}", config.backend);
|
||||
info!(
|
||||
event = EVENT_KMS_SERVICE_STATE,
|
||||
component = LOG_COMPONENT_KMS,
|
||||
subsystem = LOG_SUBSYSTEM_SERVICE,
|
||||
backend = ?config.backend,
|
||||
state = "starting",
|
||||
"KMS service starting"
|
||||
);
|
||||
|
||||
match self.create_service_version(&config).await {
|
||||
Ok(service_version) => {
|
||||
@@ -146,7 +163,13 @@ impl KmsServiceManager {
|
||||
*status = KmsServiceStatus::Running;
|
||||
}
|
||||
|
||||
info!("KMS service started successfully");
|
||||
debug!(
|
||||
event = EVENT_KMS_SERVICE_STATE,
|
||||
component = LOG_COMPONENT_KMS,
|
||||
subsystem = LOG_SUBSYSTEM_SERVICE,
|
||||
state = "running",
|
||||
"KMS service running"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
@@ -170,7 +193,13 @@ impl KmsServiceManager {
|
||||
|
||||
/// Internal stop implementation (called within lifecycle mutex)
|
||||
async fn stop_internal(&self) -> Result<()> {
|
||||
info!("Stopping KMS service");
|
||||
debug!(
|
||||
event = EVENT_KMS_SERVICE_STATE,
|
||||
component = LOG_COMPONENT_KMS,
|
||||
subsystem = LOG_SUBSYSTEM_SERVICE,
|
||||
state = "stopping",
|
||||
"KMS service stopping"
|
||||
);
|
||||
|
||||
// Atomically clear current service version (lock-free, instant)
|
||||
// Note: Existing Arc references will keep the service alive until operations complete
|
||||
@@ -184,7 +213,13 @@ impl KmsServiceManager {
|
||||
}
|
||||
}
|
||||
|
||||
info!("KMS service stopped successfully (existing operations may continue)");
|
||||
debug!(
|
||||
event = EVENT_KMS_SERVICE_STATE,
|
||||
component = LOG_COMPONENT_KMS,
|
||||
subsystem = LOG_SUBSYSTEM_SERVICE,
|
||||
state = "configured",
|
||||
"KMS service stopped"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -201,7 +236,13 @@ impl KmsServiceManager {
|
||||
pub async fn reconfigure(&self, new_config: KmsConfig) -> Result<()> {
|
||||
let _guard = self.lifecycle_mutex.lock().await;
|
||||
|
||||
info!("Reconfiguring KMS service (zero-downtime)");
|
||||
debug!(
|
||||
event = EVENT_KMS_SERVICE_STATE,
|
||||
component = LOG_COMPONENT_KMS,
|
||||
subsystem = LOG_SUBSYSTEM_SERVICE,
|
||||
state = "reconfiguring",
|
||||
"KMS service reconfiguring"
|
||||
);
|
||||
new_config.validate()?;
|
||||
|
||||
// Configure with new config
|
||||
@@ -230,13 +271,22 @@ impl KmsServiceManager {
|
||||
|
||||
if let Some(old_ver) = old_version {
|
||||
info!(
|
||||
"KMS service reconfigured successfully: version {} -> {} (old service will be cleaned up when operations complete)",
|
||||
old_ver, new_service_version.version
|
||||
event = EVENT_KMS_SERVICE_STATE,
|
||||
component = LOG_COMPONENT_KMS,
|
||||
subsystem = LOG_SUBSYSTEM_SERVICE,
|
||||
old_version = old_ver,
|
||||
new_version = new_service_version.version,
|
||||
state = "running",
|
||||
"KMS service reconfigured"
|
||||
);
|
||||
} else {
|
||||
info!(
|
||||
"KMS service reconfigured successfully: version {} (service started)",
|
||||
new_service_version.version
|
||||
event = EVENT_KMS_SERVICE_STATE,
|
||||
component = LOG_COMPONENT_KMS,
|
||||
subsystem = LOG_SUBSYSTEM_SERVICE,
|
||||
new_version = new_service_version.version,
|
||||
state = "running",
|
||||
"KMS service started from reconfigure"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
|
||||
@@ -69,7 +69,7 @@ impl NotifyBucketConfigManager {
|
||||
bucket = %bucket,
|
||||
region = %cfg.region,
|
||||
available_arn_count = arn_list.len(),
|
||||
"Loaded available notify target ARNs for bucket config validation"
|
||||
"notify bucket config validation"
|
||||
);
|
||||
|
||||
if let Err(e) = cfg.validate(&cfg.region, &arn_list) {
|
||||
@@ -81,7 +81,7 @@ impl NotifyBucketConfigManager {
|
||||
region = %cfg.region,
|
||||
error = %e,
|
||||
result = "validation_failed",
|
||||
"Bucket notification config validation failed"
|
||||
"notify bucket config validation"
|
||||
);
|
||||
if !matches!(e, ParseConfigError::ArnNotFound(_)) {
|
||||
return Err(NotificationError::BucketNotification(e.to_string()));
|
||||
@@ -94,7 +94,7 @@ impl NotifyBucketConfigManager {
|
||||
region = %cfg.region,
|
||||
error = %e,
|
||||
result = "missing_target_arn",
|
||||
"Bucket notification config references missing target ARN; keeping compatibility and loading remaining rules"
|
||||
"notify bucket config validation"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -107,7 +107,7 @@ impl NotifyBucketConfigManager {
|
||||
bucket = %bucket,
|
||||
region = %cfg.region,
|
||||
rule_count = cfg.get_rules_map().inner().len(),
|
||||
"Loaded bucket notification config"
|
||||
"notify bucket config state"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -91,7 +91,7 @@ impl NotifyConfigManager {
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_CONFIG,
|
||||
state = "initializing",
|
||||
"Initializing notification system"
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
|
||||
let config = {
|
||||
@@ -111,7 +111,7 @@ impl NotifyConfigManager {
|
||||
subsystem = LOG_SUBSYSTEM_CONFIG,
|
||||
state = "targets_created",
|
||||
target_count = targets.len(),
|
||||
"Created notification targets"
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
if targets.is_empty() {
|
||||
debug!(
|
||||
@@ -121,7 +121,7 @@ impl NotifyConfigManager {
|
||||
state = "idle",
|
||||
reason = "no_targets_configured",
|
||||
hint = %notify_configuration_hint(),
|
||||
"Notification runtime has no configured targets"
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -132,7 +132,7 @@ impl NotifyConfigManager {
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_CONFIG,
|
||||
state = "initialized",
|
||||
"Initialized notification system"
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -156,13 +156,13 @@ impl NotifyConfigManager {
|
||||
if let Some(targets_of_type) = config.0.get_mut(&ttype) {
|
||||
if targets_of_type.remove(&tname).is_some() {
|
||||
info!(
|
||||
event = EVENT_NOTIFY_CONFIG_UPDATE,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
event = EVENT_NOTIFY_CONFIG_UPDATE,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_CONFIG,
|
||||
action = "remove_target",
|
||||
target_id = %target_id,
|
||||
result = "removed",
|
||||
"Removed notification target from configuration"
|
||||
"notify config update"
|
||||
);
|
||||
changed = true;
|
||||
}
|
||||
@@ -178,7 +178,7 @@ impl NotifyConfigManager {
|
||||
action = "remove_target",
|
||||
target_id = %target_id,
|
||||
result = "not_found",
|
||||
"Notification target not found in configuration"
|
||||
"notify config update"
|
||||
);
|
||||
}
|
||||
changed
|
||||
@@ -246,7 +246,7 @@ impl NotifyConfigManager {
|
||||
target_type = %target_type,
|
||||
target_name = %target_name,
|
||||
result = "not_found",
|
||||
"Notification target configuration not found"
|
||||
"notify config update"
|
||||
);
|
||||
}
|
||||
debug!(
|
||||
@@ -264,7 +264,7 @@ impl NotifyConfigManager {
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_CONFIG,
|
||||
state = "reloading",
|
||||
"Reloading notification configuration"
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
|
||||
self.update_config(new_config.clone()).await;
|
||||
@@ -281,7 +281,7 @@ impl NotifyConfigManager {
|
||||
subsystem = LOG_SUBSYSTEM_CONFIG,
|
||||
state = "targets_created",
|
||||
target_count = targets.len(),
|
||||
"Created notification targets from reloaded configuration"
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
if targets.is_empty() {
|
||||
debug!(
|
||||
@@ -291,7 +291,7 @@ impl NotifyConfigManager {
|
||||
state = "idle",
|
||||
reason = "no_targets_configured",
|
||||
hint = %notify_configuration_hint(),
|
||||
"Notification runtime has no configured targets after reload"
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -302,7 +302,7 @@ impl NotifyConfigManager {
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_CONFIG,
|
||||
state = "reloaded",
|
||||
"Reloaded notification configuration"
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -333,7 +333,7 @@ impl NotifyConfigManager {
|
||||
subsystem = LOG_SUBSYSTEM_CONFIG,
|
||||
action = "reload_if_changed",
|
||||
result = "unchanged",
|
||||
"Notification configuration unchanged; skipping reload"
|
||||
"notify config update"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
@@ -348,7 +348,7 @@ impl NotifyConfigManager {
|
||||
subsystem = LOG_SUBSYSTEM_CONFIG,
|
||||
action = "reload_if_changed",
|
||||
result = "updated",
|
||||
"Notification configuration updated; reloading runtime"
|
||||
"notify config update"
|
||||
);
|
||||
self.reload_config(new_config).await
|
||||
}
|
||||
|
||||
@@ -23,6 +23,9 @@ use std::sync::{Arc, OnceLock};
|
||||
use tracing::error;
|
||||
|
||||
static NOTIFICATION_SYSTEM: OnceLock<Arc<NotificationSystem>> = OnceLock::new();
|
||||
const LOG_COMPONENT_NOTIFY: &str = "notify";
|
||||
const LOG_SUBSYSTEM_GLOBAL: &str = "global";
|
||||
const EVENT_NOTIFY_GLOBAL_STATE: &str = "notify_global_state";
|
||||
|
||||
/// Initialize the global notification system with the given configuration.
|
||||
/// This function should only be called once throughout the application life cycle.
|
||||
@@ -102,7 +105,13 @@ pub mod notifier_global {
|
||||
// If the notification system itself cannot be retrieved, it will be returned directly
|
||||
Some(sys) => sys,
|
||||
None => {
|
||||
error!("Notification system is not initialized.");
|
||||
error!(
|
||||
event = EVENT_NOTIFY_GLOBAL_STATE,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_GLOBAL,
|
||||
state = "uninitialized",
|
||||
"notify global state"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -372,7 +372,7 @@ impl Drop for NotificationSystem {
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_INTEGRATION,
|
||||
state = "dropping",
|
||||
"Notification system instance is being dropped"
|
||||
"notify system integration state"
|
||||
);
|
||||
|
||||
let snapshot = self.snapshot_metrics();
|
||||
@@ -394,7 +394,7 @@ impl Drop for NotificationSystem {
|
||||
metric_name = name,
|
||||
metric_value = value,
|
||||
metric_kind = if is_gauge { "gauge" } else { "counter" },
|
||||
"Notification shutdown metric snapshot"
|
||||
"notify system integration state"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -406,7 +406,7 @@ impl Drop for NotificationSystem {
|
||||
subsystem = LOG_SUBSYSTEM_INTEGRATION,
|
||||
status_key = %key,
|
||||
status_value = %value,
|
||||
"Notification system status snapshot"
|
||||
"notify system integration state"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -96,7 +96,7 @@ impl EventNotifier {
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_DISPATCH,
|
||||
state = "targets_cleared",
|
||||
"Removed all notify targets"
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -136,7 +136,7 @@ impl EventNotifier {
|
||||
bucket = %bucket_name,
|
||||
object = %object_key,
|
||||
target_count = target_ids_len,
|
||||
"Dispatching notify event"
|
||||
"notify dispatch"
|
||||
);
|
||||
for target_id in target_ids {
|
||||
// `get` now returns Option<Arc<dyn Target + Send + Sync>>
|
||||
@@ -166,7 +166,7 @@ impl EventNotifier {
|
||||
subsystem = LOG_SUBSYSTEM_DISPATCH,
|
||||
target_id = %target_for_task.id(),
|
||||
deferred = is_deferred,
|
||||
"Prepared notify target dispatch"
|
||||
"notify dispatch"
|
||||
);
|
||||
// Use cloned data in closures to avoid borrowing conflicts
|
||||
// Create an EntityTarget from the event
|
||||
@@ -217,7 +217,7 @@ impl EventNotifier {
|
||||
subsystem = LOG_SUBSYSTEM_DISPATCH,
|
||||
target_id = %target_name_for_task,
|
||||
deferred = is_deferred,
|
||||
"Completed notify target dispatch"
|
||||
"notify dispatch"
|
||||
);
|
||||
}
|
||||
});
|
||||
@@ -256,7 +256,7 @@ impl EventNotifier {
|
||||
subsystem = LOG_SUBSYSTEM_DISPATCH,
|
||||
bucket = %bucket_name,
|
||||
target_count = target_ids_len,
|
||||
"Finished notify dispatch fan-out"
|
||||
"notify dispatch"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -284,7 +284,7 @@ impl EventNotifier {
|
||||
subsystem = LOG_SUBSYSTEM_DISPATCH,
|
||||
state = "targets_initialized",
|
||||
target_count = target_list_guard.len(),
|
||||
"Initialized notify runtime targets"
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -69,7 +69,7 @@ impl NotifyRuleEngine {
|
||||
bucket = %bucket,
|
||||
state = "updated",
|
||||
event_count,
|
||||
"Updated bucket notification rules"
|
||||
"notify bucket rules state"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -85,7 +85,7 @@ impl NotifyRuleEngine {
|
||||
subsystem = LOG_SUBSYSTEM_RULE_ENGINE,
|
||||
bucket = %bucket,
|
||||
state = "removed",
|
||||
"Removed bucket notification rules"
|
||||
"notify bucket rules state"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -66,7 +66,7 @@ impl NotifyRuntimeFacade {
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
target_id = %target_id,
|
||||
state = "replay_started",
|
||||
"Started notify replay worker"
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
} else {
|
||||
debug!(
|
||||
@@ -76,7 +76,7 @@ impl NotifyRuntimeFacade {
|
||||
target_id = %target_id,
|
||||
state = "replay_skipped",
|
||||
reason = "no_store_configured",
|
||||
"Skipped notify replay worker startup"
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
}
|
||||
}),
|
||||
@@ -122,7 +122,7 @@ impl NotifyRuntimeFacade {
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "stopping",
|
||||
"Stopping notification runtime"
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
|
||||
let active_targets = self.replay_workers.read().await.len();
|
||||
@@ -132,7 +132,7 @@ impl NotifyRuntimeFacade {
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "replay_stopping",
|
||||
active_targets,
|
||||
"Stopping notify replay workers"
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
|
||||
{
|
||||
@@ -160,7 +160,7 @@ impl NotifyRuntimeFacade {
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "stopped",
|
||||
"Stopped notification runtime"
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -203,7 +203,7 @@ impl Dial9SessionGuard {
|
||||
component = LOG_COMPONENT_OBS,
|
||||
subsystem = LOG_SUBSYSTEM_DIAL9,
|
||||
state = "shutdown_requested",
|
||||
"dial9 state changed"
|
||||
"dial9 state"
|
||||
);
|
||||
// TelemetryGuard handles flushing automatically when dropped
|
||||
}
|
||||
@@ -219,7 +219,7 @@ impl Drop for Dial9SessionGuard {
|
||||
component = LOG_COMPONENT_OBS,
|
||||
subsystem = LOG_SUBSYSTEM_DIAL9,
|
||||
state = "flushed",
|
||||
"dial9 state changed"
|
||||
"dial9 state"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -175,7 +175,7 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
|
||||
output_format = "json",
|
||||
logger_level,
|
||||
is_production,
|
||||
"local logging state changed"
|
||||
"local logging state"
|
||||
);
|
||||
|
||||
OtelGuard {
|
||||
@@ -293,7 +293,7 @@ fn init_file_logging_internal(
|
||||
stdout_mirror_enabled = stdout_guard.is_some(),
|
||||
is_production,
|
||||
logger_level,
|
||||
"local logging state changed"
|
||||
"local logging state"
|
||||
);
|
||||
|
||||
Ok(OtelGuard {
|
||||
@@ -482,7 +482,7 @@ pub fn spawn_cleanup_task(
|
||||
zstd_level,
|
||||
zstd_fallback_to_gzip,
|
||||
zstd_workers,
|
||||
"log cleaner state changed"
|
||||
"log cleaner state"
|
||||
);
|
||||
|
||||
tokio::spawn(async move {
|
||||
|
||||
@@ -655,7 +655,7 @@ where
|
||||
state = "requested",
|
||||
username = %masked_username,
|
||||
path = %path_str,
|
||||
"ftps object delete state changed"
|
||||
"FTPS delete requested"
|
||||
);
|
||||
|
||||
let (bucket, key) = self
|
||||
@@ -749,7 +749,7 @@ where
|
||||
username = %masked_username,
|
||||
path = %path_str,
|
||||
bucket = %bucket,
|
||||
"ftps directory state changed"
|
||||
"FTPS directory created"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -793,7 +793,7 @@ where
|
||||
state = "removed",
|
||||
path = %path_str,
|
||||
bucket = %bucket,
|
||||
"ftps directory state changed"
|
||||
"FTPS directory removed"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -807,7 +807,7 @@ where
|
||||
subsystem = LOG_SUBSYSTEM_FTPS_DRIVER,
|
||||
state = "already_removed",
|
||||
bucket = %bucket,
|
||||
"ftps directory state changed"
|
||||
"FTPS directory already removed"
|
||||
);
|
||||
Ok(())
|
||||
} else {
|
||||
@@ -877,7 +877,7 @@ where
|
||||
username = %MaskedAccessKey(&user.username),
|
||||
from = %from_str,
|
||||
to = %to_str,
|
||||
"ftps rename state changed"
|
||||
"FTPS rename unsupported"
|
||||
);
|
||||
|
||||
Err(Error::new(
|
||||
|
||||
@@ -107,7 +107,7 @@ where
|
||||
bind_addr = %self.config.bind_addr,
|
||||
tls_enabled = self.config.tls_enabled,
|
||||
ftps_required = self.config.ftps_required,
|
||||
"ftps server state changed"
|
||||
"FTPS server starting"
|
||||
);
|
||||
let (reload_shutdown_tx, reload_shutdown_rx) = watch::channel(false);
|
||||
|
||||
@@ -121,14 +121,14 @@ where
|
||||
// Configure passive ports for data connections
|
||||
if let Some(passive_ports) = &self.config.passive_ports {
|
||||
let range = self.config.parse_passive_ports()?;
|
||||
info!(
|
||||
debug!(
|
||||
event = EVENT_FTPS_CONFIG_STATE,
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
|
||||
state = "passive_ports_configured",
|
||||
passive_ports = %passive_ports,
|
||||
passive_port_range = ?range,
|
||||
"ftps config state changed"
|
||||
"FTPS passive ports configured"
|
||||
);
|
||||
server_builder = server_builder.passive_ports(range);
|
||||
} else {
|
||||
@@ -137,19 +137,19 @@ where
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
|
||||
result = "system_assigned_passive_ports",
|
||||
"ftps config state changed"
|
||||
"FTPS passive ports defaulted"
|
||||
);
|
||||
}
|
||||
|
||||
// Configure external IP address for passive mode
|
||||
if let Some(ref external_ip) = self.config.external_ip {
|
||||
info!(
|
||||
debug!(
|
||||
event = EVENT_FTPS_CONFIG_STATE,
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
|
||||
state = "external_ip_configured",
|
||||
external_ip = %external_ip,
|
||||
"ftps config state changed"
|
||||
"FTPS external IP configured"
|
||||
);
|
||||
server_builder = server_builder.passive_host(external_ip.as_str());
|
||||
}
|
||||
@@ -157,13 +157,13 @@ where
|
||||
// Configure both active and passive mode support
|
||||
use libunftp::options::ActivePassiveMode;
|
||||
server_builder = server_builder.active_passive_mode(ActivePassiveMode::ActiveAndPassive);
|
||||
info!(
|
||||
debug!(
|
||||
event = EVENT_FTPS_CONFIG_STATE,
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
|
||||
state = "active_passive_mode_enabled",
|
||||
mode = "active_and_passive",
|
||||
"ftps config state changed"
|
||||
"FTPS active/passive mode configured"
|
||||
);
|
||||
|
||||
// Configure FTPS / TLS
|
||||
@@ -175,7 +175,7 @@ where
|
||||
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
|
||||
state = "enabled",
|
||||
cert_dir = %cert_dir,
|
||||
"ftps tls state changed"
|
||||
"FTPS TLS enabled"
|
||||
);
|
||||
|
||||
let resolver = ReloadableServerCertResolver::load_from_directory(cert_dir)
|
||||
@@ -197,12 +197,12 @@ where
|
||||
server_builder = server_builder.ftps_manual::<std::path::PathBuf>(Arc::new(server_config));
|
||||
|
||||
if self.config.ftps_required {
|
||||
info!(
|
||||
debug!(
|
||||
event = EVENT_FTPS_TLS_STATE,
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
|
||||
state = "required",
|
||||
"ftps tls state changed"
|
||||
"FTPS TLS required"
|
||||
);
|
||||
server_builder = server_builder.ftps_required(FtpsRequired::All, FtpsRequired::All);
|
||||
}
|
||||
@@ -218,7 +218,7 @@ where
|
||||
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
|
||||
state = "disabled",
|
||||
mode = "plain_ftp",
|
||||
"ftps tls state changed"
|
||||
"FTPS TLS disabled"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -248,13 +248,13 @@ where
|
||||
let _ = reload_shutdown_tx.send(true);
|
||||
match result {
|
||||
Ok(Ok(())) => {
|
||||
info!(
|
||||
debug!(
|
||||
event = EVENT_FTPS_SERVER_STATE,
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
|
||||
state = "stopped",
|
||||
result = "ok",
|
||||
"ftps server state changed"
|
||||
"FTPS server stopped"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -288,7 +288,7 @@ where
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_FTPS_SERVER,
|
||||
state = "shutdown_requested",
|
||||
"ftps server state changed"
|
||||
"FTPS shutdown requested"
|
||||
);
|
||||
let _ = reload_shutdown_tx.send(true);
|
||||
// libunftp listen() is not easily cancellable gracefully without dropping the future.
|
||||
@@ -330,7 +330,7 @@ impl UserDetailProvider for FtpsUserDetailProvider {
|
||||
result = "iam_unavailable",
|
||||
phase = "user_detail",
|
||||
error = %e,
|
||||
"ftps auth state changed"
|
||||
"FTPS user-detail IAM unavailable"
|
||||
);
|
||||
UserDetailError::ImplPropagated("Internal authentication service unavailable".to_string(), Some(Box::new(e)))
|
||||
})?;
|
||||
@@ -344,7 +344,7 @@ impl UserDetailProvider for FtpsUserDetailProvider {
|
||||
phase = "user_detail",
|
||||
username = %masked_username,
|
||||
error = %e,
|
||||
"ftps auth state changed"
|
||||
"FTPS user-detail key check failed"
|
||||
);
|
||||
UserDetailError::ImplPropagated("Authentication verification failed".to_string(), Some(Box::new(e)))
|
||||
})?;
|
||||
@@ -357,7 +357,7 @@ impl UserDetailProvider for FtpsUserDetailProvider {
|
||||
result = "identity_missing",
|
||||
phase = "user_detail",
|
||||
username = %masked_username,
|
||||
"ftps auth state changed"
|
||||
"FTPS user-detail identity missing"
|
||||
);
|
||||
UserDetailError::UserNotFound {
|
||||
username: principal.username.clone(),
|
||||
@@ -406,7 +406,7 @@ impl Authenticator for FtpsAuthenticator {
|
||||
result = "iam_unavailable",
|
||||
phase = "authenticate",
|
||||
error = %e,
|
||||
"ftps auth state changed"
|
||||
"FTPS auth IAM unavailable"
|
||||
);
|
||||
AuthenticationError::ImplPropagated("Internal authentication service unavailable".to_string(), Some(Box::new(e)))
|
||||
})?;
|
||||
@@ -433,7 +433,7 @@ impl Authenticator for FtpsAuthenticator {
|
||||
phase = "authenticate",
|
||||
username = %masked_username,
|
||||
error = %e,
|
||||
"ftps auth state changed"
|
||||
"FTPS auth key check failed"
|
||||
);
|
||||
AuthenticationError::ImplPropagated("Authentication verification failed".to_string(), Some(Box::new(e)))
|
||||
})?;
|
||||
@@ -446,7 +446,7 @@ impl Authenticator for FtpsAuthenticator {
|
||||
result = "invalid_access_key",
|
||||
phase = "authenticate",
|
||||
username = %masked_username,
|
||||
"ftps auth state changed"
|
||||
"FTPS auth rejected access key"
|
||||
);
|
||||
return Err(AuthenticationError::BadUser);
|
||||
}
|
||||
@@ -459,7 +459,7 @@ impl Authenticator for FtpsAuthenticator {
|
||||
result = "identity_missing",
|
||||
phase = "authenticate",
|
||||
username = %masked_username,
|
||||
"ftps auth state changed"
|
||||
"FTPS auth identity missing"
|
||||
);
|
||||
AuthenticationError::BadUser
|
||||
})?;
|
||||
@@ -472,7 +472,7 @@ impl Authenticator for FtpsAuthenticator {
|
||||
result = "invalid_secret_key",
|
||||
phase = "authenticate",
|
||||
username = %masked_username,
|
||||
"ftps auth state changed"
|
||||
"FTPS auth rejected secret key"
|
||||
);
|
||||
return Err(AuthenticationError::BadPassword);
|
||||
}
|
||||
@@ -484,7 +484,7 @@ impl Authenticator for FtpsAuthenticator {
|
||||
result = "authenticated",
|
||||
phase = "authenticate",
|
||||
username = %masked_username,
|
||||
"ftps auth state changed"
|
||||
"FTPS auth accepted"
|
||||
);
|
||||
Ok(Principal {
|
||||
username: username.to_string(),
|
||||
|
||||
@@ -188,7 +188,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
peer = %self.session_context.source_ip,
|
||||
user = %MaskedAccessKey(&self.session_context.principal.user_identity.credentials.access_key),
|
||||
result = "read_only_rejected",
|
||||
"sftp driver state changed"
|
||||
"SFTP write rejected by read-only mode"
|
||||
);
|
||||
return Err(SftpError::code(StatusCode::PermissionDenied));
|
||||
}
|
||||
@@ -250,7 +250,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
op = op,
|
||||
timeout_secs = self.backend_op_timeout_secs,
|
||||
result = "timeout",
|
||||
"sftp backend state changed"
|
||||
"SFTP backend operation timed out"
|
||||
);
|
||||
Err(SftpError::code(StatusCode::Failure))
|
||||
}
|
||||
|
||||
@@ -255,7 +255,7 @@ fn spawn_host_key_reload_loop(config: SftpConfig, holder: Arc<SshConfigHolder>,
|
||||
state = "enabled",
|
||||
host_key_dir = %config.host_key_dir.display(),
|
||||
interval_secs,
|
||||
"sftp host key reload state changed"
|
||||
"SFTP host-key reload enabled"
|
||||
);
|
||||
|
||||
tokio::spawn(async move {
|
||||
@@ -265,13 +265,13 @@ fn spawn_host_key_reload_loop(config: SftpConfig, holder: Arc<SshConfigHolder>,
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = shutdown_token.cancelled() => {
|
||||
info!(
|
||||
debug!(
|
||||
event = EVENT_SFTP_HOST_KEY_RELOAD_STATE,
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_SFTP_SERVER,
|
||||
state = "stopped",
|
||||
host_key_dir = %config.host_key_dir.display(),
|
||||
"sftp host key reload state changed"
|
||||
"SFTP host-key reload stopped"
|
||||
);
|
||||
break;
|
||||
}
|
||||
@@ -280,14 +280,14 @@ fn spawn_host_key_reload_loop(config: SftpConfig, holder: Arc<SshConfigHolder>,
|
||||
|
||||
match holder.reload_from_config(&config).await {
|
||||
Ok(Some(host_key_count)) => {
|
||||
info!(
|
||||
debug!(
|
||||
event = EVENT_SFTP_HOST_KEY_RELOAD_STATE,
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_SFTP_SERVER,
|
||||
result = "reloaded",
|
||||
host_key_dir = %config.host_key_dir.display(),
|
||||
host_key_count,
|
||||
"sftp host key reload state changed"
|
||||
"SFTP host keys reloaded"
|
||||
);
|
||||
}
|
||||
Ok(None) => {
|
||||
@@ -297,7 +297,7 @@ fn spawn_host_key_reload_loop(config: SftpConfig, holder: Arc<SshConfigHolder>,
|
||||
subsystem = LOG_SUBSYSTEM_SFTP_SERVER,
|
||||
result = "unchanged",
|
||||
host_key_dir = %config.host_key_dir.display(),
|
||||
"sftp host key reload state changed"
|
||||
"SFTP host keys unchanged"
|
||||
);
|
||||
}
|
||||
Err(err) => {
|
||||
@@ -308,7 +308,7 @@ fn spawn_host_key_reload_loop(config: SftpConfig, holder: Arc<SshConfigHolder>,
|
||||
result = "reload_failed",
|
||||
host_key_dir = %config.host_key_dir.display(),
|
||||
err = %err,
|
||||
"sftp host key reload state changed"
|
||||
"SFTP host-key reload failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -398,7 +398,7 @@ where
|
||||
state = "listening",
|
||||
bind_addr = %self.config.bind_addr,
|
||||
read_only = self.config.read_only,
|
||||
"sftp server state changed"
|
||||
"SFTP server listening"
|
||||
);
|
||||
|
||||
let mut sessions: JoinSet<()> = JoinSet::new();
|
||||
|
||||
@@ -75,7 +75,7 @@ pub(super) fn write_dispatch_byte_count(phase: &WritePhase, part_size: u64, satu
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_SFTP_WRITE,
|
||||
result = "byte_count_overflow",
|
||||
"sftp write state changed"
|
||||
"SFTP write byte-count overflow"
|
||||
);
|
||||
Err(SftpError::code(StatusCode::Failure))
|
||||
}
|
||||
@@ -90,7 +90,7 @@ pub(super) fn write_dispatch_byte_count(phase: &WritePhase, part_size: u64, satu
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_SFTP_WRITE,
|
||||
result = "handle_poisoned",
|
||||
"sftp write state changed"
|
||||
"SFTP write handle poisoned"
|
||||
);
|
||||
Err(SftpError::code(StatusCode::Failure))
|
||||
}
|
||||
@@ -115,7 +115,7 @@ pub(super) fn write_dispatch_append_bytes(phase: &mut WritePhase, data: &[u8]) -
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_SFTP_WRITE,
|
||||
result = "append_on_failed_handle",
|
||||
"sftp write state changed"
|
||||
"SFTP write append rejected on failed handle"
|
||||
);
|
||||
Err(SftpError::code(StatusCode::Failure))
|
||||
}
|
||||
@@ -368,7 +368,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
key = %sanitise_control_bytes(key),
|
||||
attempt = attempt,
|
||||
state = "retrying_put_object",
|
||||
"sftp write state changed",
|
||||
"SFTP put_object retry scheduled",
|
||||
);
|
||||
}
|
||||
|
||||
@@ -414,7 +414,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
bucket = %sanitise_control_bytes(bucket),
|
||||
key = %sanitise_control_bytes(key),
|
||||
result = "retry_loop_fell_through",
|
||||
"sftp write state changed",
|
||||
"SFTP put_object retry loop fell through",
|
||||
);
|
||||
Err(SftpError::code(StatusCode::Failure))
|
||||
}
|
||||
@@ -459,7 +459,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
upload_id = %upload_id,
|
||||
part_number = part_number,
|
||||
result = "etag_missing",
|
||||
"sftp multipart state changed"
|
||||
"SFTP multipart part missing etag"
|
||||
);
|
||||
SftpError::code(StatusCode::Failure)
|
||||
})?;
|
||||
@@ -543,7 +543,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
bucket = %sanitise_control_bytes(bucket),
|
||||
key = %sanitise_control_bytes(key),
|
||||
result = "upload_id_missing",
|
||||
"sftp multipart state changed"
|
||||
"SFTP multipart upload missing upload_id"
|
||||
);
|
||||
SftpError::code(StatusCode::Failure)
|
||||
})?;
|
||||
@@ -708,7 +708,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_SFTP_WRITE,
|
||||
result = "buffering_phase_lost",
|
||||
"sftp multipart state changed"
|
||||
"SFTP multipart buffering phase lost"
|
||||
);
|
||||
return Err(SftpError::code(StatusCode::Failure));
|
||||
}
|
||||
@@ -772,7 +772,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
key = %key,
|
||||
limit = S3_MAX_MULTIPART_PARTS,
|
||||
result = "parts_limit_exceeded",
|
||||
"sftp multipart state changed",
|
||||
"SFTP multipart parts limit exceeded",
|
||||
);
|
||||
let upload_id_for_fail = upload_id.clone();
|
||||
let abort_authorized_for_fail = *abort_authorized;
|
||||
@@ -791,7 +791,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_SFTP_WRITE,
|
||||
result = "flush_without_streaming",
|
||||
"sftp multipart state changed"
|
||||
"SFTP multipart flush requested without streaming state"
|
||||
);
|
||||
return Err(SftpError::code(StatusCode::Failure));
|
||||
}
|
||||
@@ -817,7 +817,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_SFTP_WRITE,
|
||||
result = "post_upload_phase_missing",
|
||||
"sftp multipart state changed"
|
||||
"SFTP multipart post-upload phase missing"
|
||||
);
|
||||
Err(SftpError::code(StatusCode::Failure))
|
||||
}
|
||||
@@ -887,7 +887,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
err = ?abort_err,
|
||||
context = context,
|
||||
result = "abort_failed",
|
||||
"sftp abort state changed",
|
||||
"SFTP multipart abort failed",
|
||||
);
|
||||
}
|
||||
} else {
|
||||
@@ -901,7 +901,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
access_key = %MaskedAccessKey(self.access_key()),
|
||||
context = context,
|
||||
result = "abort_skipped_unauthorized",
|
||||
"sftp abort state changed",
|
||||
"SFTP multipart abort skipped by policy",
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -959,7 +959,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
upload_id = %upload_id,
|
||||
limit = S3_MAX_MULTIPART_PARTS,
|
||||
result = "final_part_limit_exceeded",
|
||||
"sftp multipart state changed",
|
||||
"SFTP multipart final-part limit exceeded",
|
||||
);
|
||||
self.close_abort_or_skip(bucket, key, &upload_id, abort_authorized, "parts-limit breach")
|
||||
.await;
|
||||
@@ -1080,7 +1080,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
upload_id = %mp.upload_id,
|
||||
part_number = part_number,
|
||||
result = "etag_missing",
|
||||
"sftp multipart copy state changed"
|
||||
"SFTP multipart copy part missing etag"
|
||||
);
|
||||
SftpError::code(StatusCode::Failure)
|
||||
})?;
|
||||
|
||||
@@ -423,7 +423,7 @@ where
|
||||
bucket = %bucket,
|
||||
object = %key,
|
||||
file_size,
|
||||
"webdav object write state changed"
|
||||
"WebDAV object flush completed"
|
||||
);
|
||||
// Buffer already cleared by std::mem::take above
|
||||
Ok(())
|
||||
@@ -438,7 +438,7 @@ where
|
||||
object = %key,
|
||||
file_size,
|
||||
error = %e,
|
||||
"webdav object write state changed"
|
||||
"WebDAV object flush failed"
|
||||
);
|
||||
Err(FsError::GeneralFailure)
|
||||
}
|
||||
@@ -1219,7 +1219,7 @@ where
|
||||
result = "not_found",
|
||||
bucket = %bucket,
|
||||
error = %e,
|
||||
"webdav bucket metadata state changed"
|
||||
"WebDAV bucket metadata listed"
|
||||
);
|
||||
Err(FsError::NotFound)
|
||||
}
|
||||
@@ -1279,7 +1279,7 @@ where
|
||||
state = "created",
|
||||
bucket = %bucket,
|
||||
object = %dir_key,
|
||||
"webdav directory state changed"
|
||||
"WebDAV directory marker created"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
@@ -1292,7 +1292,7 @@ where
|
||||
bucket = %bucket,
|
||||
object = %dir_key,
|
||||
error = %e,
|
||||
"webdav directory state changed"
|
||||
"WebDAV directory marker create failed"
|
||||
);
|
||||
return Err(FsError::GeneralFailure);
|
||||
}
|
||||
@@ -1320,7 +1320,7 @@ where
|
||||
subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
|
||||
state = "bucket_created",
|
||||
bucket = %bucket,
|
||||
"webdav directory state changed"
|
||||
"WebDAV bucket created"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -1332,7 +1332,7 @@ where
|
||||
state = "bucket_create_failed",
|
||||
bucket = %bucket,
|
||||
error = %e,
|
||||
"webdav directory state changed"
|
||||
"WebDAV bucket create failed"
|
||||
);
|
||||
Err(FsError::GeneralFailure)
|
||||
}
|
||||
@@ -1465,7 +1465,7 @@ where
|
||||
state = "deleted",
|
||||
bucket = %bucket,
|
||||
object = %key,
|
||||
"webdav object delete state changed"
|
||||
"WebDAV object deleted"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -1478,7 +1478,7 @@ where
|
||||
bucket = %bucket,
|
||||
object = %key,
|
||||
error = %e,
|
||||
"webdav object delete state changed"
|
||||
"WebDAV object delete failed"
|
||||
);
|
||||
Err(FsError::GeneralFailure)
|
||||
}
|
||||
@@ -1529,7 +1529,7 @@ where
|
||||
dst_bucket = %dst_bucket,
|
||||
dst_object = %dst_key,
|
||||
error = %e,
|
||||
"webdav rename state changed"
|
||||
"WebDAV rename source delete failed"
|
||||
);
|
||||
FsError::GeneralFailure
|
||||
})?;
|
||||
@@ -1543,7 +1543,7 @@ where
|
||||
src_object = %src_key,
|
||||
dst_bucket = %dst_bucket,
|
||||
dst_object = %dst_key,
|
||||
"webdav rename state changed"
|
||||
"WebDAV file renamed"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
@@ -1603,7 +1603,7 @@ where
|
||||
dst_bucket = %dst_bucket,
|
||||
dst_prefix = %dst_prefix,
|
||||
error = %e,
|
||||
"webdav rename state changed"
|
||||
"WebDAV rename directory listing failed"
|
||||
);
|
||||
FsError::GeneralFailure
|
||||
})?;
|
||||
@@ -1652,7 +1652,7 @@ where
|
||||
src_object = %src_key,
|
||||
dst_bucket = %dst_bucket,
|
||||
dst_object = %dst_key,
|
||||
"webdav rename state changed"
|
||||
"WebDAV rename source not found"
|
||||
);
|
||||
return Err(FsError::NotFound);
|
||||
}
|
||||
@@ -1666,7 +1666,7 @@ where
|
||||
src_object = %src_key,
|
||||
dst_bucket = %dst_bucket,
|
||||
dst_object = %dst_key,
|
||||
"webdav rename state changed"
|
||||
"WebDAV directory renamed"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -86,7 +86,7 @@ where
|
||||
bind_addr = %self.config.bind_addr,
|
||||
tls_enabled = self.config.tls_enabled,
|
||||
max_body_size = self.config.max_body_size,
|
||||
"webdav server state changed"
|
||||
"WebDAV server starting"
|
||||
);
|
||||
|
||||
let listener = TcpListener::bind(self.config.bind_addr).await?;
|
||||
@@ -96,7 +96,7 @@ where
|
||||
subsystem = LOG_SUBSYSTEM_WEBDAV_SERVER,
|
||||
state = "listening",
|
||||
bind_addr = %self.config.bind_addr,
|
||||
"webdav server state changed"
|
||||
"WebDAV server listening"
|
||||
);
|
||||
let (reload_shutdown_tx, reload_shutdown_rx) = watch::channel(false);
|
||||
|
||||
@@ -109,7 +109,7 @@ where
|
||||
subsystem = LOG_SUBSYSTEM_WEBDAV_SERVER,
|
||||
state = "enabled",
|
||||
cert_dir = %cert_dir,
|
||||
"webdav tls state changed"
|
||||
"WebDAV TLS enabled"
|
||||
);
|
||||
|
||||
let resolver = ReloadableServerCertResolver::load_from_directory(cert_dir)
|
||||
@@ -211,7 +211,7 @@ where
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBDAV_SERVER,
|
||||
state = "shutdown_requested",
|
||||
"webdav server state changed"
|
||||
"WebDAV shutdown requested"
|
||||
);
|
||||
let _ = reload_shutdown_tx.send(true);
|
||||
break;
|
||||
@@ -225,7 +225,7 @@ where
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBDAV_SERVER,
|
||||
state = "stopped",
|
||||
"webdav server state changed"
|
||||
"WebDAV server stopped"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -376,7 +376,7 @@ where
|
||||
result = "iam_unavailable",
|
||||
source_ip = %source_ip,
|
||||
error = %e,
|
||||
"webdav auth state changed"
|
||||
"WebDAV auth IAM unavailable"
|
||||
);
|
||||
WebDavInitError::Server("Internal authentication service unavailable".to_string())
|
||||
})?;
|
||||
@@ -403,7 +403,7 @@ where
|
||||
source_ip = %source_ip,
|
||||
access_key = %masked_access_key,
|
||||
error = %e,
|
||||
"webdav auth state changed"
|
||||
"WebDAV auth key check failed"
|
||||
);
|
||||
WebDavInitError::Server("Authentication verification failed".to_string())
|
||||
})?;
|
||||
@@ -416,7 +416,7 @@ where
|
||||
result = "invalid_access_key",
|
||||
source_ip = %source_ip,
|
||||
access_key = %masked_access_key,
|
||||
"webdav auth state changed"
|
||||
"WebDAV auth rejected access key"
|
||||
);
|
||||
return Err(WebDavInitError::Server("Invalid credentials".to_string()));
|
||||
}
|
||||
@@ -429,7 +429,7 @@ where
|
||||
result = "identity_missing",
|
||||
source_ip = %source_ip,
|
||||
access_key = %masked_access_key,
|
||||
"webdav auth state changed"
|
||||
"WebDAV auth identity missing"
|
||||
);
|
||||
WebDavInitError::Server("User not found".to_string())
|
||||
})?;
|
||||
@@ -442,7 +442,7 @@ where
|
||||
result = "invalid_secret_key",
|
||||
source_ip = %source_ip,
|
||||
access_key = %masked_access_key,
|
||||
"webdav auth state changed"
|
||||
"WebDAV auth rejected secret key"
|
||||
);
|
||||
return Err(WebDavInitError::Server("Invalid credentials".to_string()));
|
||||
}
|
||||
@@ -454,7 +454,7 @@ where
|
||||
result = "authenticated",
|
||||
source_ip = %source_ip,
|
||||
access_key = %masked_access_key,
|
||||
"webdav auth state changed"
|
||||
"WebDAV auth accepted"
|
||||
);
|
||||
|
||||
Ok(SessionContext::new(
|
||||
|
||||
@@ -97,7 +97,7 @@ fn resolve_scanner_runtime_config() -> crate::runtime_config::ScannerRuntimeConf
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "resolve_failed",
|
||||
error = %err,
|
||||
"Scanner runtime config resolution failed; using last applied config"
|
||||
"Scanner runtime config fallback applied"
|
||||
);
|
||||
current_scanner_runtime_config()
|
||||
}
|
||||
@@ -179,7 +179,7 @@ async fn persisted_usage_cache_is_cold_for_startup(storeapi: &Arc<ECStore>) -> b
|
||||
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
state = "startup_inspect_failed",
|
||||
error = %err,
|
||||
"Scanner startup cache inspection failed; keeping configured startup delay"
|
||||
"Scanner startup cache inspection failed"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
@@ -198,7 +198,7 @@ async fn persisted_usage_cache_is_cold_for_startup(storeapi: &Arc<ECStore>) -> b
|
||||
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
state = "startup_decode_failed",
|
||||
error = %err,
|
||||
"Scanner startup cache decode failed; skipping startup delay"
|
||||
"Scanner startup cache decode failed"
|
||||
);
|
||||
true
|
||||
}
|
||||
@@ -222,7 +222,7 @@ async fn initial_scanner_startup_usage_state(storeapi: &Arc<ECStore>) -> (bool,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "startup_bucket_inspect_failed",
|
||||
error = %err,
|
||||
"Scanner startup bucket inspection failed; keeping configured startup delay"
|
||||
"Scanner startup bucket inspection failed"
|
||||
);
|
||||
false
|
||||
}
|
||||
@@ -418,7 +418,7 @@ async fn configure_scanner_defaults(storeapi: &Arc<ECStore>) {
|
||||
replication_active = features.replication,
|
||||
feature_inspection_failed = features.inspection_failed,
|
||||
state = "single_disk_defaults_applied",
|
||||
"Scanner defaults updated for single-disk deployment"
|
||||
"Scanner defaults applied"
|
||||
);
|
||||
} else {
|
||||
set_scanner_default_speed(ScannerSpeed::Default);
|
||||
@@ -562,7 +562,7 @@ pub async fn read_background_heal_info(storeapi: Arc<ECStore>) -> BackgroundHeal
|
||||
path = %&*BACKGROUND_HEAL_INFO_PATH,
|
||||
state = "decode_failed",
|
||||
error = %e,
|
||||
"Scanner background heal state decode failed"
|
||||
"Scanner background heal decode failed"
|
||||
);
|
||||
BackgroundHealInfo::default()
|
||||
}),
|
||||
@@ -577,7 +577,7 @@ pub async fn read_background_heal_info(storeapi: Arc<ECStore>) -> BackgroundHeal
|
||||
path = %&*BACKGROUND_HEAL_INFO_PATH,
|
||||
state = "read_failed",
|
||||
error = %e,
|
||||
"Scanner background heal state read failed"
|
||||
"Scanner background heal read failed"
|
||||
);
|
||||
}
|
||||
BackgroundHealInfo::default()
|
||||
@@ -605,7 +605,7 @@ pub async fn save_background_heal_info(storeapi: Arc<ECStore>, info: BackgroundH
|
||||
path = %&*BACKGROUND_HEAL_INFO_PATH,
|
||||
state = "encode_failed",
|
||||
error = %e,
|
||||
"Scanner background heal state encode failed"
|
||||
"Scanner background heal encode failed"
|
||||
);
|
||||
return;
|
||||
}
|
||||
@@ -621,7 +621,7 @@ pub async fn save_background_heal_info(storeapi: Arc<ECStore>, info: BackgroundH
|
||||
path = %&*BACKGROUND_HEAL_INFO_PATH,
|
||||
state = "save_failed",
|
||||
error = %e,
|
||||
"Scanner background heal state save failed"
|
||||
"Scanner background heal save failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -650,7 +650,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "refresh_failed",
|
||||
error = %err,
|
||||
"Scanner runtime config refresh failed; using last applied config"
|
||||
"Scanner runtime config refresh failed"
|
||||
);
|
||||
}
|
||||
let configured_cycle_interval = scanner_cycle_interval();
|
||||
@@ -685,7 +685,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
|
||||
cycle = cycle_info.current,
|
||||
scan_mode = ?scan_mode,
|
||||
state = "started",
|
||||
"Scanner cycle state updated"
|
||||
"Scanner cycle started"
|
||||
);
|
||||
let _scan_mode_guard = ScannerScanModeGuard::new(scan_mode);
|
||||
if let Some(new_heal_info) = background_heal_info_for_scan_start(
|
||||
@@ -730,7 +730,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
|
||||
max_objects = ?cycle_budget.max_objects(),
|
||||
max_directories = ?cycle_budget.max_directories(),
|
||||
state = "budget_reached",
|
||||
"Scanner cycle stopped after reaching budget"
|
||||
"Scanner cycle budget reached"
|
||||
);
|
||||
let budget_reason = cycle_budget.reason();
|
||||
emit_scan_cycle_partial_with_source(
|
||||
@@ -751,7 +751,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
|
||||
state = "failed",
|
||||
duration = ?now.elapsed(),
|
||||
error = %e,
|
||||
"Scanner cycle state updated"
|
||||
"Scanner cycle failed"
|
||||
);
|
||||
emit_scan_cycle_complete(false, cycle_start.elapsed());
|
||||
if let Some(new_heal_info) = background_heal_info_for_scan_complete(background_heal_info.clone(), scan_mode) {
|
||||
@@ -772,7 +772,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
|
||||
max_objects = ?cycle_budget.max_objects(),
|
||||
max_directories = ?cycle_budget.max_directories(),
|
||||
state = "budget_reached",
|
||||
"Scanner cycle stopped after reaching budget"
|
||||
"Scanner cycle budget reached"
|
||||
);
|
||||
global_metrics().finish_scan_cycle_work(cycle_work_start);
|
||||
let budget_reason = cycle_budget.reason();
|
||||
@@ -806,7 +806,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
|
||||
state = "completed",
|
||||
duration = ?now.elapsed(),
|
||||
cycles_total = cycle_info.cycle_completed.len(),
|
||||
"Scanner cycle state updated"
|
||||
"Scanner cycle completed"
|
||||
);
|
||||
|
||||
retain_recent_cycle_completions(&mut cycle_info.cycle_completed);
|
||||
@@ -830,14 +830,14 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
|
||||
"Scanner state persistence failed"
|
||||
);
|
||||
} else {
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %&*DATA_USAGE_BLOOM_NAME_PATH,
|
||||
state = "saved",
|
||||
"Scanner state persisted"
|
||||
"Scanner state saved"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -854,7 +854,7 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) ->
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
lock_name = "leader.lock",
|
||||
state = "acquired",
|
||||
"Scanner leader lock state updated"
|
||||
"Scanner leader lock acquired"
|
||||
);
|
||||
guard
|
||||
}
|
||||
@@ -867,7 +867,7 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) ->
|
||||
lock_name = "leader.lock",
|
||||
state = "contended",
|
||||
error = ?e,
|
||||
"Scanner leader lock state updated"
|
||||
"Scanner leader lock contended"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
@@ -881,7 +881,7 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) ->
|
||||
lock_name = "leader.lock",
|
||||
state = "create_failed",
|
||||
error = %e,
|
||||
"Scanner leader lock state updated"
|
||||
"Scanner leader lock creation failed"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
@@ -977,7 +977,7 @@ pub async fn store_data_usage_in_backend(
|
||||
&& let (Some(new_ts), Some(existing_ts)) = (data_usage_info.last_update, existing.last_update)
|
||||
&& new_ts <= existing_ts
|
||||
{
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
@@ -986,7 +986,7 @@ pub async fn store_data_usage_in_backend(
|
||||
incoming_last_update = ?new_ts,
|
||||
existing_last_update = ?existing_ts,
|
||||
state = "skip_stale_update",
|
||||
"Scanner data usage persistence skipped stale update"
|
||||
"Scanner stale data usage update skipped"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
@@ -1003,7 +1003,7 @@ pub async fn store_data_usage_in_backend(
|
||||
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
state = "encode_failed",
|
||||
error = %e,
|
||||
"Scanner data usage persistence encode failed"
|
||||
"Scanner data usage encode failed"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
@@ -1040,7 +1040,7 @@ pub async fn store_data_usage_in_backend(
|
||||
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
state = "save_failed",
|
||||
error = %e,
|
||||
"Scanner data usage persistence failed"
|
||||
"Scanner data usage save failed"
|
||||
);
|
||||
} else {
|
||||
rustfs_ecstore::data_usage::replace_bucket_usage_memory_from_info(&data_usage_info).await;
|
||||
|
||||
@@ -66,7 +66,7 @@ use time::OffsetDateTime;
|
||||
use tokio::select;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, warn};
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
const LOG_COMPONENT_SCANNER: &str = "scanner";
|
||||
const LOG_SUBSYSTEM_FOLDER: &str = "folder";
|
||||
@@ -639,7 +639,7 @@ impl ScannerItem {
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
object_path = %self.object_path(),
|
||||
state = "started",
|
||||
"Scanner lifecycle action evaluation started"
|
||||
"Scanner lifecycle evaluation started"
|
||||
);
|
||||
|
||||
let versioning_config = match BucketVersioningSys::get(&self.bucket).await {
|
||||
@@ -990,7 +990,7 @@ impl ScannerItem {
|
||||
let age = now - mod_time;
|
||||
age.whole_seconds().max(0)
|
||||
});
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::scanner::folder",
|
||||
event = EVENT_SCANNER_HEAL_ADMISSION,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
@@ -1003,7 +1003,7 @@ impl ScannerItem {
|
||||
original_scan_mode = %HealScanMode::Deep.as_str(),
|
||||
effective_scan_mode = %scan_mode.as_str(),
|
||||
state = "downgraded_to_normal",
|
||||
"Scanner heal admission downgraded deep scan during cooldown"
|
||||
"Scanner heal deep scan downgraded"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1688,14 +1688,14 @@ impl FolderScanner {
|
||||
|
||||
if found_objects && is_erasure().await {
|
||||
// If we found an object in erasure mode, we skip subdirs (only datadirs)...
|
||||
info!(
|
||||
debug!(
|
||||
target: "rustfs::scanner::folder",
|
||||
event = EVENT_SCANNER_FOLDER_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_FOLDER,
|
||||
folder = %folder.name,
|
||||
state = "erasure_object_found",
|
||||
"Scanner folder stopped descending after finding erasure object"
|
||||
"Scanner folder descent stopped after erasure object"
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -900,7 +900,7 @@ impl ScannerIOCache for SetDisks {
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
bucket = %bucket.name,
|
||||
state = "scan_started",
|
||||
"Scanner disk bucket state updated"
|
||||
"Scanner disk bucket scan started"
|
||||
);
|
||||
|
||||
let cache_name = path_join_buf(&[&bucket.name, DATA_USAGE_CACHE_NAME]);
|
||||
@@ -910,13 +910,13 @@ impl ScannerIOCache for SetDisks {
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_DISK_BUCKET_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
bucket = %bucket.name,
|
||||
cache_name = %cache_name,
|
||||
state = "cache_load_failed",
|
||||
error = %e,
|
||||
"Scanner disk bucket state updated"
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
bucket = %bucket.name,
|
||||
cache_name = %cache_name,
|
||||
state = "cache_load_failed",
|
||||
error = %e,
|
||||
"Scanner disk bucket cache load failed"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -942,7 +942,7 @@ impl ScannerIOCache for SetDisks {
|
||||
bucket = %bucket.name,
|
||||
cache_name = ?cache.info.name,
|
||||
state = "cache_ready",
|
||||
"Scanner disk bucket state updated"
|
||||
"Scanner disk bucket cache ready"
|
||||
);
|
||||
|
||||
let (updates_tx, mut updates_rx) = mpsc::channel::<DataUsageEntry>(1);
|
||||
@@ -997,7 +997,7 @@ impl ScannerIOCache for SetDisks {
|
||||
bucket = %bucket.name,
|
||||
state = "cancelled",
|
||||
error = %e,
|
||||
"Scanner disk bucket state updated"
|
||||
"Scanner disk bucket scan cancelled"
|
||||
);
|
||||
} else {
|
||||
error!(
|
||||
@@ -1008,7 +1008,7 @@ impl ScannerIOCache for SetDisks {
|
||||
bucket = %bucket.name,
|
||||
state = "scan_failed",
|
||||
error = %e,
|
||||
"Scanner disk bucket state updated"
|
||||
"Scanner disk bucket scan failed"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1104,7 +1104,7 @@ impl ScannerIOCache for SetDisks {
|
||||
bucket = %bucket.name,
|
||||
cache_name = %cache.info.name,
|
||||
state = "scan_completed",
|
||||
"Scanner disk bucket state updated"
|
||||
"Scanner disk bucket scan completed"
|
||||
);
|
||||
|
||||
if let Err(e) = update_fut.await {
|
||||
@@ -1132,7 +1132,7 @@ impl ScannerIOCache for SetDisks {
|
||||
bucket = %bucket.name,
|
||||
cache_name = %cache.info.name,
|
||||
state = "send_root_entry",
|
||||
"Scanner data usage stream progress updated"
|
||||
"Scanner root entry publish started"
|
||||
);
|
||||
|
||||
if let Err(e) = send_cache_root_entry_info(&bucket_result_tx_clone_clone, &cache).await {
|
||||
@@ -1182,7 +1182,7 @@ impl ScannerIOCache for SetDisks {
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
state = "set_scan_completed",
|
||||
"Scanner set-level disk scan completed"
|
||||
"Scanner set scan completed"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
|
||||
+108
-16
@@ -30,6 +30,10 @@ use std::{
|
||||
use tracing::{debug, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
const LOG_COMPONENT_TARGETS: &str = "targets";
|
||||
const LOG_SUBSYSTEM_STORE: &str = "store";
|
||||
const EVENT_TARGET_STORE_STATE: &str = "target_store_state";
|
||||
|
||||
fn resolve_queue_store_compression_from_env_value(value: Option<&str>) -> bool {
|
||||
value
|
||||
.and_then(|value| value.parse::<EnableState>().ok().map(|state| state.is_enabled()))
|
||||
@@ -83,7 +87,14 @@ impl std::fmt::Display for Key {
|
||||
|
||||
/// Parses a string into a Key
|
||||
pub fn parse_key(s: &str) -> Key {
|
||||
debug!("Parsing key: {}", s);
|
||||
debug!(
|
||||
event = EVENT_TARGET_STORE_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_STORE,
|
||||
action = "parse_key",
|
||||
key = %s,
|
||||
"target store state"
|
||||
);
|
||||
|
||||
let mut name = s.to_string();
|
||||
let mut extension = String::new();
|
||||
@@ -111,8 +122,16 @@ pub fn parse_key(s: &str) -> Key {
|
||||
}
|
||||
|
||||
debug!(
|
||||
"Parsed key - name: {}, extension: {}, item_count: {}, compress: {}",
|
||||
name, extension, item_count, compress
|
||||
event = EVENT_TARGET_STORE_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_STORE,
|
||||
action = "parse_key",
|
||||
result = "parsed",
|
||||
key_name = %name,
|
||||
extension = %extension,
|
||||
item_count,
|
||||
compressed = compress,
|
||||
"target store state"
|
||||
);
|
||||
|
||||
Key {
|
||||
@@ -276,7 +295,15 @@ impl<T: Serialize + DeserializeOwned + Send + Sync> QueueStore<T> {
|
||||
.read()
|
||||
.map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?;
|
||||
let path = self.file_path(key);
|
||||
debug!("Reading file for key: {},path: {}", key, path.display());
|
||||
debug!(
|
||||
event = EVENT_TARGET_STORE_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_STORE,
|
||||
action = "read_file",
|
||||
key = %key,
|
||||
path = %path.display(),
|
||||
"target store state"
|
||||
);
|
||||
let data = std::fs::read(&path).map_err(|e| {
|
||||
if e.kind() == std::io::ErrorKind::NotFound {
|
||||
StoreError::NotFound
|
||||
@@ -342,7 +369,14 @@ impl<T: Serialize + DeserializeOwned + Send + Sync> QueueStore<T> {
|
||||
std::fs::write(&path, data).map_err(StoreError::Io)?;
|
||||
}
|
||||
let modified = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_nanos() as i64;
|
||||
debug!("Wrote event to store: {}", key);
|
||||
debug!(
|
||||
event = EVENT_TARGET_STORE_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_STORE,
|
||||
action = "write_file",
|
||||
key = %key,
|
||||
"target store state"
|
||||
);
|
||||
Ok(modified)
|
||||
}
|
||||
|
||||
@@ -409,7 +443,15 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
debug!("Opened store at: {:?}", self.directory);
|
||||
debug!(
|
||||
event = EVENT_TARGET_STORE_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_STORE,
|
||||
state = "opened",
|
||||
store_dir = ?self.directory,
|
||||
entry_count = entries_map.len(),
|
||||
"target store state"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -471,7 +513,14 @@ where
|
||||
}
|
||||
|
||||
fn get_multiple(&self, key: &Self::Key) -> Result<Vec<T>, Self::Error> {
|
||||
debug!("Reading items from store for key: {}", key);
|
||||
debug!(
|
||||
event = EVENT_TARGET_STORE_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_STORE,
|
||||
action = "read_batch",
|
||||
key = %key,
|
||||
"target store state"
|
||||
);
|
||||
let data = self.get_raw(key)?;
|
||||
if data.is_empty() {
|
||||
return Err(StoreError::Deserialization("Cannot deserialize empty data".to_string()));
|
||||
@@ -500,10 +549,15 @@ where
|
||||
if items.len() < key.item_count && !items.is_empty() {
|
||||
// Partial read
|
||||
warn!(
|
||||
"Expected {} items for key {}, but only found {}. Possible data corruption or incorrect item_count.",
|
||||
key.item_count,
|
||||
key,
|
||||
items.len()
|
||||
event = EVENT_TARGET_STORE_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_STORE,
|
||||
action = "read_batch",
|
||||
key = %key,
|
||||
expected_items = key.item_count,
|
||||
actual_items = items.len(),
|
||||
reason = "partial_batch_read",
|
||||
"target store state"
|
||||
);
|
||||
// Depending on strictness, this could be an error.
|
||||
} else if items.is_empty() {
|
||||
@@ -538,7 +592,15 @@ where
|
||||
Ok(()) => {}
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
|
||||
// File already gone — still clean up the entries map to avoid stale keys.
|
||||
warn!("File not found for key {} during del, cleaning up entries map.", key);
|
||||
warn!(
|
||||
event = EVENT_TARGET_STORE_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_STORE,
|
||||
action = "delete",
|
||||
key = %key,
|
||||
result = "file_missing",
|
||||
"target store state"
|
||||
);
|
||||
}
|
||||
Err(e) => return Err(StoreError::Io(e)),
|
||||
}
|
||||
@@ -550,9 +612,25 @@ where
|
||||
.map_err(|_| StoreError::Internal("Failed to acquire write lock on entries".to_string()))?;
|
||||
|
||||
if entries.remove(&key.to_key_string()).is_none() {
|
||||
debug!("Key {} not found in entries map during del, might have been already removed.", key);
|
||||
debug!(
|
||||
event = EVENT_TARGET_STORE_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_STORE,
|
||||
action = "delete",
|
||||
key = %key,
|
||||
result = "entry_missing",
|
||||
"target store state"
|
||||
);
|
||||
}
|
||||
debug!("Deleted event from store: {}", key.to_string());
|
||||
debug!(
|
||||
event = EVENT_TARGET_STORE_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_STORE,
|
||||
action = "delete",
|
||||
key = %key,
|
||||
result = "deleted",
|
||||
"target store state"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -580,7 +658,14 @@ where
|
||||
let entries = match self.entries.read() {
|
||||
Ok(entries) => entries,
|
||||
Err(_) => {
|
||||
debug!("Failed to acquire read lock on entries for listing");
|
||||
debug!(
|
||||
event = EVENT_TARGET_STORE_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_STORE,
|
||||
action = "list",
|
||||
result = "lock_unavailable",
|
||||
"target store state"
|
||||
);
|
||||
return Vec::new();
|
||||
}
|
||||
};
|
||||
@@ -597,7 +682,14 @@ where
|
||||
match self.entries.read() {
|
||||
Ok(entries) => entries.len(),
|
||||
Err(_) => {
|
||||
debug!("Failed to acquire read lock on entries for len");
|
||||
debug!(
|
||||
event = EVENT_TARGET_STORE_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_STORE,
|
||||
action = "len",
|
||||
result = "lock_unavailable",
|
||||
"target store state"
|
||||
);
|
||||
0
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60,6 +60,10 @@ const DEFAULT_MQTT_TLS_PORT: u16 = 8883;
|
||||
const DEFAULT_MQTT_WSS_PORT: u16 = 443;
|
||||
const MAX_MQTT_PACKET_SIZE_BYTES: u32 = 100 * 1024 * 1024;
|
||||
const DEFAULT_MQTT_WS_PATH_ALLOWLIST: &[&str] = &["/", "/mqtt"];
|
||||
const LOG_COMPONENT_TARGETS: &str = "targets";
|
||||
const LOG_SUBSYSTEM_MQTT: &str = "mqtt";
|
||||
const EVENT_MQTT_TARGET_STATE: &str = "mqtt_target_state";
|
||||
const EVENT_MQTT_DELIVERY_STATE: &str = "mqtt_delivery_state";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum MQTTTlsPolicy {
|
||||
@@ -574,7 +578,14 @@ where
|
||||
Some(MAX_MQTT_PACKET_SIZE_BYTES),
|
||||
)?;
|
||||
|
||||
info!(target_id = %target_id, "MQTT target created");
|
||||
info!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %target_id,
|
||||
state = "created",
|
||||
"mqtt target state"
|
||||
);
|
||||
Ok(MQTTTarget::<E> {
|
||||
id: target_id,
|
||||
args,
|
||||
@@ -593,7 +604,14 @@ where
|
||||
#[instrument(skip(self), fields(target_id = %self.id))]
|
||||
async fn init(&self) -> Result<(), TargetError> {
|
||||
if self.connected.load(Ordering::SeqCst) {
|
||||
debug!(target_id = %self.id, "Already connected.");
|
||||
debug!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "already_connected",
|
||||
"mqtt target state"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -607,7 +625,14 @@ where
|
||||
let _ = bg_task_manager
|
||||
.init_cell
|
||||
.get_or_try_init(|| async {
|
||||
debug!(target_id = %target_id_clone, "Initializing MQTT background task.");
|
||||
debug!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %target_id_clone,
|
||||
state = "background_task_initializing",
|
||||
"mqtt target state"
|
||||
);
|
||||
|
||||
// Use the latest MqttOptions (may have been updated by TLS reload coordinator).
|
||||
let mqtt_options: MqttOptions = (**pending_mqtt_options.load()).clone();
|
||||
@@ -615,30 +640,67 @@ where
|
||||
let (new_client, eventloop) = AsyncClient::builder(mqtt_options).capacity(10).build();
|
||||
|
||||
if let Err(e) = new_client.subscribe(&args_clone.topic, args_clone.qos).await {
|
||||
error!(target_id = %target_id_clone, error = %e, "Failed to subscribe to MQTT topic during init");
|
||||
error!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %target_id_clone,
|
||||
state = "subscribe_failed",
|
||||
error = %e,
|
||||
"mqtt target state"
|
||||
);
|
||||
return Err(TargetError::Network(format!("MQTT subscribe failed: {e}")));
|
||||
}
|
||||
|
||||
let mut rx_guard = bg_task_manager.initial_cancel_rx.lock().await;
|
||||
let cancel_rx = rx_guard.take().ok_or_else(|| {
|
||||
error!(target_id = %target_id_clone, "MQTT cancel receiver already taken for task.");
|
||||
error!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %target_id_clone,
|
||||
state = "cancel_receiver_unavailable",
|
||||
"mqtt target state"
|
||||
);
|
||||
TargetError::Configuration("MQTT cancel receiver already taken for task".to_string())
|
||||
})?;
|
||||
drop(rx_guard);
|
||||
|
||||
*client_arc.lock().await = Some(new_client.clone());
|
||||
|
||||
info!(target_id = %target_id_clone, "Spawning MQTT event loop task.");
|
||||
info!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %target_id_clone,
|
||||
state = "event_loop_spawning",
|
||||
"mqtt target state"
|
||||
);
|
||||
let task_handle =
|
||||
tokio::spawn(run_mqtt_event_loop(eventloop, connected_arc.clone(), target_id_clone.clone(), cancel_rx));
|
||||
Ok(task_handle)
|
||||
})
|
||||
.await
|
||||
.map_err(|e: TargetError| {
|
||||
error!(target_id = %self.id, error = %e, "Failed to initialize MQTT background task");
|
||||
error!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "background_task_init_failed",
|
||||
error = %e,
|
||||
"mqtt target state"
|
||||
);
|
||||
e
|
||||
})?;
|
||||
debug!(target_id = %self.id, "MQTT background task initialized successfully.");
|
||||
debug!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "background_task_initialized",
|
||||
"mqtt target state"
|
||||
);
|
||||
|
||||
match tokio::time::timeout(DEFAULT_CONNECTION_TIMEOUT, async {
|
||||
while !self.connected.load(Ordering::SeqCst) {
|
||||
@@ -646,23 +708,51 @@ where
|
||||
&& handle.is_finished()
|
||||
&& !self.connected.load(Ordering::SeqCst)
|
||||
{
|
||||
error!(target_id = %self.id, "MQTT background task exited prematurely before connection was established.");
|
||||
error!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "background_task_exited_before_connect",
|
||||
"mqtt target state"
|
||||
);
|
||||
return Err(TargetError::Network("MQTT background task exited prematurely".to_string()));
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
debug!(target_id = %self.id, "MQTT target connected successfully.");
|
||||
debug!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "connected",
|
||||
"mqtt target state"
|
||||
);
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(Ok(_)) => {
|
||||
info!(target_id = %self.id, "MQTT target initialized and connected.");
|
||||
info!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "ready",
|
||||
"mqtt target state"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Ok(Err(e)) => Err(e),
|
||||
Err(_) => {
|
||||
error!(target_id = %self.id, "Timeout waiting for MQTT connection after task spawn.");
|
||||
error!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "connect_timeout",
|
||||
"mqtt target state"
|
||||
);
|
||||
Err(TargetError::Network("Timeout waiting for MQTT connection".to_string()))
|
||||
}
|
||||
}
|
||||
@@ -680,12 +770,16 @@ where
|
||||
.ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?;
|
||||
|
||||
debug!(
|
||||
target = %self.id,
|
||||
event = EVENT_MQTT_DELIVERY_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
bucket = %meta.bucket_name,
|
||||
object = %meta.object_name,
|
||||
event = %meta.event_name,
|
||||
payload_len = body.len(),
|
||||
"Sending MQTT payload"
|
||||
state = "publishing",
|
||||
"mqtt delivery state"
|
||||
);
|
||||
|
||||
client
|
||||
@@ -693,7 +787,16 @@ where
|
||||
.await
|
||||
.map_err(|e| {
|
||||
if e.to_string().contains("Connection") || e.to_string().contains("Timeout") {
|
||||
warn!(target_id = %self.id, error = %e, "Publish failed due to connection issue, marking as not connected.");
|
||||
warn!(
|
||||
event = EVENT_MQTT_DELIVERY_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "publish_failed",
|
||||
reason = "connectivity_error",
|
||||
error = %e,
|
||||
"mqtt delivery state"
|
||||
);
|
||||
let err = TargetError::NotConnected;
|
||||
mark_target_disconnected_on_connectivity_error(&self.connected, &err);
|
||||
err
|
||||
@@ -702,7 +805,15 @@ where
|
||||
}
|
||||
})?;
|
||||
|
||||
debug!(target_id = %self.id, topic = %self.args.topic, "Event published to MQTT topic");
|
||||
debug!(
|
||||
event = EVENT_MQTT_DELIVERY_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
topic = %self.args.topic,
|
||||
state = "published",
|
||||
"mqtt delivery state"
|
||||
);
|
||||
self.delivery_counters.record_success();
|
||||
Ok(())
|
||||
}
|
||||
@@ -781,14 +892,28 @@ async fn run_mqtt_event_loop(
|
||||
target_id: TargetID,
|
||||
mut cancel_rx: mpsc::Receiver<()>,
|
||||
) {
|
||||
info!(target_id = %target_id, "MQTT event loop task started.");
|
||||
info!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %target_id,
|
||||
state = "event_loop_started",
|
||||
"mqtt target state"
|
||||
);
|
||||
let mut initial_connection_established = false;
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancel_rx.recv() => {
|
||||
info!(target_id = %target_id, "MQTT event loop task received cancellation signal. Shutting down.");
|
||||
info!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %target_id,
|
||||
state = "cancellation_received",
|
||||
"mqtt target state"
|
||||
);
|
||||
break;
|
||||
}
|
||||
polled_event_result = async {
|
||||
@@ -796,7 +921,14 @@ async fn run_mqtt_event_loop(
|
||||
match tokio::time::timeout(EVENT_LOOP_POLL_TIMEOUT, eventloop.poll()).await {
|
||||
Ok(result) => Some(result),
|
||||
Err(_) => {
|
||||
debug!(target_id = %target_id, "MQTT poll timed out (EVENT_LOOP_POLL_TIMEOUT) while not connected or status pending.");
|
||||
debug!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %target_id,
|
||||
state = "poll_timeout",
|
||||
"mqtt target state"
|
||||
);
|
||||
connected_status.store(false, Ordering::SeqCst);
|
||||
None
|
||||
}
|
||||
@@ -810,15 +942,38 @@ async fn run_mqtt_event_loop(
|
||||
trace!(target_id = %target_id, event = ?notification, "Received MQTT event");
|
||||
match notification {
|
||||
rumqttc::Event::Incoming(Incoming::ConnAck(_conn_ack)) => {
|
||||
info!(target_id = %target_id, "MQTT connected (ConnAck).");
|
||||
info!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %target_id,
|
||||
state = "connack_received",
|
||||
"mqtt target state"
|
||||
);
|
||||
connected_status.store(true, Ordering::SeqCst);
|
||||
initial_connection_established = true;
|
||||
}
|
||||
rumqttc::Event::Incoming(Incoming::Publish(publish)) => {
|
||||
debug!(target_id = %target_id, topic = ?publish.topic, payload_len = publish.payload.len(), "Received message on subscribed topic.");
|
||||
debug!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %target_id,
|
||||
state = "publish_received",
|
||||
topic = ?publish.topic,
|
||||
payload_len = publish.payload.len(),
|
||||
"mqtt target state"
|
||||
);
|
||||
}
|
||||
rumqttc::Event::Incoming(Incoming::Disconnect(_)) => {
|
||||
info!(target_id = %target_id, "Received Disconnect packet from broker. MQTT connection lost.");
|
||||
info!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %target_id,
|
||||
state = "broker_disconnected",
|
||||
"mqtt target state"
|
||||
);
|
||||
connected_status.store(false, Ordering::SeqCst);
|
||||
}
|
||||
rumqttc::Event::Incoming(Incoming::PingResp(_)) => {
|
||||
@@ -832,7 +987,14 @@ async fn run_mqtt_event_loop(
|
||||
}
|
||||
// Process other incoming packet types as needed (PubRec, PubRel, PubComp, UnsubAck)
|
||||
rumqttc::Event::Outgoing(Outgoing::Disconnect) => {
|
||||
info!(target_id = %target_id, "MQTT outgoing disconnect initiated by client.");
|
||||
info!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %target_id,
|
||||
state = "client_disconnect_requested",
|
||||
"mqtt target state"
|
||||
);
|
||||
connected_status.store(false, Ordering::SeqCst);
|
||||
}
|
||||
rumqttc::Event::Outgoing(Outgoing::PingReq) => {
|
||||
@@ -848,7 +1010,15 @@ async fn run_mqtt_event_loop(
|
||||
}
|
||||
Some(Err(e)) => {
|
||||
connected_status.store(false, Ordering::SeqCst);
|
||||
error!(target_id = %target_id, error = %e, "Error from MQTT event loop poll");
|
||||
error!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %target_id,
|
||||
state = "poll_failed",
|
||||
error = %e,
|
||||
"mqtt target state"
|
||||
);
|
||||
|
||||
if matches!(e,
|
||||
ConnectionError::Io(_) |
|
||||
@@ -856,12 +1026,28 @@ async fn run_mqtt_event_loop(
|
||||
ConnectionError::ConnectionRefused(_) |
|
||||
ConnectionError::Tls(_)
|
||||
) {
|
||||
warn!(target_id = %target_id, error = %e, "MQTT connection error. Relying on rumqttc for reconnection if applicable.");
|
||||
warn!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %target_id,
|
||||
state = "reconnect_pending",
|
||||
error = %e,
|
||||
"mqtt target state"
|
||||
);
|
||||
}
|
||||
// Here you can decide whether to break loops based on the error type.
|
||||
// For example, for some unrecoverable errors.
|
||||
if is_fatal_mqtt_error(&e) {
|
||||
error!(target_id = %target_id, error = %e, "Fatal MQTT error, terminating event loop.");
|
||||
error!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %target_id,
|
||||
state = "fatal_error",
|
||||
error = %e,
|
||||
"mqtt target state"
|
||||
);
|
||||
break;
|
||||
}
|
||||
// rumqttc's eventloop.poll() may return Err and terminate after some errors,
|
||||
@@ -871,7 +1057,14 @@ async fn run_mqtt_event_loop(
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
None => {
|
||||
warn!(target_id = %target_id, "Timeout during initial poll or pending state, will retry.");
|
||||
warn!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %target_id,
|
||||
state = "poll_retry_scheduled",
|
||||
"mqtt target state"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
@@ -879,7 +1072,14 @@ async fn run_mqtt_event_loop(
|
||||
}
|
||||
}
|
||||
connected_status.store(false, Ordering::SeqCst);
|
||||
info!(target_id = %target_id, "MQTT event loop task finished.");
|
||||
info!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %target_id,
|
||||
state = "event_loop_finished",
|
||||
"mqtt target state"
|
||||
);
|
||||
}
|
||||
|
||||
/// Check whether the given MQTT connection error should be considered a fatal error,
|
||||
@@ -938,26 +1138,61 @@ where
|
||||
|
||||
#[instrument(skip(self), fields(target_id = %self.id))]
|
||||
async fn is_active(&self) -> Result<bool, TargetError> {
|
||||
debug!(target_id = %self.id, "Checking if MQTT target is active.");
|
||||
debug!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "activity_check",
|
||||
"mqtt target state"
|
||||
);
|
||||
if self.client.lock().await.is_none() && !self.connected.load(Ordering::SeqCst) {
|
||||
// Check if the background task is running and has not panicked
|
||||
if let Some(handle) = self.bg_task_manager.init_cell.get()
|
||||
&& handle.is_finished()
|
||||
{
|
||||
error!(target_id = %self.id, "MQTT background task has finished, possibly due to an error. Target is not active.");
|
||||
error!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "inactive_background_task_finished",
|
||||
"mqtt target state"
|
||||
);
|
||||
return Err(TargetError::Network("MQTT background task terminated".to_string()));
|
||||
}
|
||||
debug!(target_id = %self.id, "MQTT client not yet initialized or task not running/connected.");
|
||||
debug!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "inactive_client_unavailable",
|
||||
"mqtt target state"
|
||||
);
|
||||
return Err(TargetError::Configuration(
|
||||
"MQTT client not available or not initialized/connected".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if self.connected.load(Ordering::SeqCst) {
|
||||
debug!(target_id = %self.id, "MQTT target is active (connected flag is true).");
|
||||
debug!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "active",
|
||||
"mqtt target state"
|
||||
);
|
||||
Ok(true)
|
||||
} else {
|
||||
debug!(target_id = %self.id, "MQTT target is not connected (connected flag is false).");
|
||||
debug!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "inactive_not_connected",
|
||||
"mqtt target state"
|
||||
);
|
||||
Err(TargetError::NotConnected)
|
||||
}
|
||||
}
|
||||
@@ -973,14 +1208,36 @@ where
|
||||
};
|
||||
|
||||
if let Some(store) = &self.store {
|
||||
debug!(target_id = %self.id, "Event saved to store start");
|
||||
debug!(
|
||||
event = EVENT_MQTT_DELIVERY_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "store_enqueue_started",
|
||||
"mqtt delivery state"
|
||||
);
|
||||
match persist_queued_payload_to_store(store.as_ref(), &queued) {
|
||||
Ok(_) => {
|
||||
debug!(target_id = %self.id, "Event saved to store for MQTT target successfully.");
|
||||
debug!(
|
||||
event = EVENT_MQTT_DELIVERY_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "store_enqueued",
|
||||
"mqtt delivery state"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
error!(target_id = %self.id, error = %e, "Failed to save event to store");
|
||||
error!(
|
||||
event = EVENT_MQTT_DELIVERY_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "store_enqueue_failed",
|
||||
error = %e,
|
||||
"mqtt delivery state"
|
||||
);
|
||||
self.delivery_counters.record_final_failure();
|
||||
Err(e)
|
||||
}
|
||||
@@ -991,18 +1248,47 @@ where
|
||||
}
|
||||
|
||||
if !self.connected.load(Ordering::SeqCst) {
|
||||
warn!(target_id = %self.id, "Attempting to send directly but not connected; trying to init.");
|
||||
warn!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "direct_send_requires_init",
|
||||
"mqtt target state"
|
||||
);
|
||||
// Call the struct's init method, not the trait's default
|
||||
match MQTTTarget::<E>::init(self).await {
|
||||
Ok(_) => debug!(target_id = %self.id, "MQTT target initialized successfully."),
|
||||
Ok(_) => debug!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "init_completed",
|
||||
"mqtt target state"
|
||||
),
|
||||
Err(e) => {
|
||||
error!(target_id = %self.id, error = %e, "Failed to initialize MQTT target.");
|
||||
error!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "init_failed",
|
||||
error = %e,
|
||||
"mqtt target state"
|
||||
);
|
||||
self.delivery_counters.record_final_failure();
|
||||
return Err(TargetError::NotConnected);
|
||||
}
|
||||
}
|
||||
if !self.connected.load(Ordering::SeqCst) {
|
||||
error!(target_id = %self.id, "Cannot save (send directly) as target is not active after init attempt.");
|
||||
error!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "init_completed_not_connected",
|
||||
"mqtt target state"
|
||||
);
|
||||
self.delivery_counters.record_final_failure();
|
||||
return Err(TargetError::NotConnected);
|
||||
}
|
||||
@@ -1017,50 +1303,143 @@ where
|
||||
|
||||
#[instrument(skip(self, body, meta), fields(target_id = %self.id))]
|
||||
async fn send_raw_from_store(&self, key: Key, body: Vec<u8>, meta: QueuedPayloadMeta) -> Result<(), TargetError> {
|
||||
debug!(target_id = %self.id, ?key, "Attempting to send queued payload from store.");
|
||||
debug!(
|
||||
event = EVENT_MQTT_DELIVERY_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
?key,
|
||||
state = "store_replay_started",
|
||||
"mqtt delivery state"
|
||||
);
|
||||
|
||||
if !self.is_enabled() {
|
||||
return Err(TargetError::Disabled);
|
||||
}
|
||||
|
||||
if !self.connected.load(Ordering::SeqCst) {
|
||||
warn!(target_id = %self.id, "Not connected; trying to init before sending from store.");
|
||||
warn!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "store_replay_requires_init",
|
||||
"mqtt target state"
|
||||
);
|
||||
match MQTTTarget::<E>::init(self).await {
|
||||
Ok(_) => debug!(target_id = %self.id, "MQTT target initialized successfully."),
|
||||
Ok(_) => debug!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "init_completed",
|
||||
"mqtt target state"
|
||||
),
|
||||
Err(e) => {
|
||||
error!(target_id = %self.id, error = %e, "Failed to initialize MQTT target.");
|
||||
error!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "init_failed",
|
||||
error = %e,
|
||||
"mqtt target state"
|
||||
);
|
||||
return Err(TargetError::NotConnected);
|
||||
}
|
||||
}
|
||||
if !self.connected.load(Ordering::SeqCst) {
|
||||
error!(target_id = %self.id, "Cannot send from store as target is not active after init attempt.");
|
||||
error!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "init_completed_not_connected",
|
||||
"mqtt target state"
|
||||
);
|
||||
return Err(TargetError::NotConnected);
|
||||
}
|
||||
}
|
||||
|
||||
debug!(target_id = %self.id, ?key, "Sending event from store.");
|
||||
debug!(
|
||||
event = EVENT_MQTT_DELIVERY_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
?key,
|
||||
state = "store_replay_publishing",
|
||||
"mqtt delivery state"
|
||||
);
|
||||
if let Err(e) = self.send_body(body, &meta).await {
|
||||
if matches!(e, TargetError::NotConnected) {
|
||||
warn!(target_id = %self.id, "Failed to send event from store: Not connected. Event remains in store.");
|
||||
warn!(
|
||||
event = EVENT_MQTT_DELIVERY_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
?key,
|
||||
state = "store_replay_deferred",
|
||||
reason = "not_connected",
|
||||
"mqtt delivery state"
|
||||
);
|
||||
return Err(TargetError::NotConnected);
|
||||
}
|
||||
error!(target_id = %self.id, error = %e, "Failed to send event from store with an unexpected error.");
|
||||
error!(
|
||||
event = EVENT_MQTT_DELIVERY_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
?key,
|
||||
state = "store_replay_failed",
|
||||
error = %e,
|
||||
"mqtt delivery state"
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
debug!(target_id = %self.id, ?key, "Event sent from store successfully.");
|
||||
debug!(
|
||||
event = EVENT_MQTT_DELIVERY_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
?key,
|
||||
state = "store_replay_published",
|
||||
"mqtt delivery state"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn close(&self) -> Result<(), TargetError> {
|
||||
info!(target_id = %self.id, "Attempting to close MQTT target.");
|
||||
info!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "closing",
|
||||
"mqtt target state"
|
||||
);
|
||||
|
||||
if let Err(e) = self.bg_task_manager.cancel_tx.send(()).await {
|
||||
warn!(target_id = %self.id, error = %e, "Failed to send cancel signal to MQTT background task. It might have already exited.");
|
||||
warn!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "cancel_signal_failed",
|
||||
error = %e,
|
||||
"mqtt target state"
|
||||
);
|
||||
}
|
||||
|
||||
// Wait for the task to finish if it was initialized
|
||||
if let Some(_task_handle) = self.bg_task_manager.init_cell.get() {
|
||||
debug!(target_id = %self.id, "Waiting for MQTT background task to complete...");
|
||||
debug!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "waiting_for_event_loop",
|
||||
"mqtt target state"
|
||||
);
|
||||
// It's tricky to await here if close is called from a sync context or Drop
|
||||
// For async close, this is fine. Consider a timeout.
|
||||
// let _ = tokio::time::timeout(Duration::from_secs(5), task_handle.await).await;
|
||||
@@ -1069,9 +1448,24 @@ where
|
||||
}
|
||||
|
||||
if let Some(client_instance) = self.client.lock().await.take() {
|
||||
info!(target_id = %self.id, "Disconnecting MQTT client.");
|
||||
info!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "disconnecting_client",
|
||||
"mqtt target state"
|
||||
);
|
||||
if let Err(e) = client_instance.disconnect().await {
|
||||
warn!(target_id = %self.id, error = %e, "Error during MQTT client disconnect.");
|
||||
warn!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "disconnect_failed",
|
||||
error = %e,
|
||||
"mqtt target state"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1083,7 +1477,14 @@ where
|
||||
}
|
||||
|
||||
self.connected.store(false, Ordering::SeqCst);
|
||||
info!(target_id = %self.id, "MQTT target close method finished.");
|
||||
info!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "closed",
|
||||
"mqtt target state"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1097,7 +1498,14 @@ where
|
||||
|
||||
async fn init(&self) -> Result<(), TargetError> {
|
||||
if !self.is_enabled() {
|
||||
debug!(target_id = %self.id, "Target is disabled, skipping init.");
|
||||
debug!(
|
||||
event = EVENT_MQTT_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_MQTT,
|
||||
target_id = %self.id,
|
||||
state = "disabled",
|
||||
"mqtt target state"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
// Call the internal init logic
|
||||
|
||||
@@ -45,6 +45,11 @@ use std::{
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{debug, error, info, instrument, warn};
|
||||
|
||||
const LOG_COMPONENT_TARGETS: &str = "targets";
|
||||
const LOG_SUBSYSTEM_WEBHOOK: &str = "webhook";
|
||||
const EVENT_WEBHOOK_TARGET_STATE: &str = "webhook_target_state";
|
||||
const EVENT_WEBHOOK_DELIVERY_STATE: &str = "webhook_delivery_state";
|
||||
|
||||
/// Arguments for configuring a Webhook target
|
||||
#[derive(Clone)]
|
||||
pub struct WebhookArgs {
|
||||
@@ -190,7 +195,14 @@ where
|
||||
|
||||
// Create a cancel channel
|
||||
let (cancel_sender, _) = mpsc::channel(1);
|
||||
info!(target_id = %target_id.id, "Webhook target created");
|
||||
info!(
|
||||
event = EVENT_WEBHOOK_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBHOOK,
|
||||
target_id = %target_id.id,
|
||||
state = "created",
|
||||
"webhook target state"
|
||||
);
|
||||
Ok(WebhookTarget::<E> {
|
||||
id: target_id,
|
||||
args,
|
||||
@@ -220,8 +232,13 @@ where
|
||||
// DANGEROUS: For testing only, skip all certificate verification
|
||||
client_builder = client_builder.danger_accept_invalid_certs(true);
|
||||
warn!(
|
||||
"Webhook target '{}' is configured to skip TLS verification. This is insecure and should not be used in production.",
|
||||
args.endpoint
|
||||
event = EVENT_WEBHOOK_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBHOOK,
|
||||
endpoint = %args.endpoint,
|
||||
state = "tls_verification_skipped",
|
||||
fallback = "danger_accept_invalid_certs",
|
||||
"webhook target state"
|
||||
);
|
||||
} else if !args.client_ca.is_empty() {
|
||||
// Use user-provided custom CA certificate
|
||||
@@ -301,10 +318,14 @@ where
|
||||
match tokio::time::timeout(Duration::from_secs(5), client.head(health_check_url.as_str()).send()).await {
|
||||
Ok(Ok(resp)) => {
|
||||
debug!(
|
||||
target = %self.id,
|
||||
event = EVENT_WEBHOOK_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBHOOK,
|
||||
target_id = %self.id,
|
||||
status = %resp.status(),
|
||||
health_check_url = %health_check_url,
|
||||
"Webhook health check request succeeded"
|
||||
state = "reachability_probe_succeeded",
|
||||
"webhook target state"
|
||||
);
|
||||
Ok(true)
|
||||
}
|
||||
@@ -337,7 +358,15 @@ where
|
||||
// behavior matches real delivery while avoiding path-specific false negatives.
|
||||
match self.probe_reachability().await {
|
||||
Ok(true) => {
|
||||
debug!("Webhook target {} reachability probe succeeded via {:?}", self.id, self.health_check_url);
|
||||
debug!(
|
||||
event = EVENT_WEBHOOK_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBHOOK,
|
||||
target_id = %self.id,
|
||||
health_check_url = ?self.health_check_url,
|
||||
state = "reachable",
|
||||
"webhook target state"
|
||||
);
|
||||
}
|
||||
Ok(false) => {
|
||||
return Err(TargetError::NotConnected);
|
||||
@@ -348,7 +377,14 @@ where
|
||||
}
|
||||
|
||||
self.initialized.store(true, Ordering::SeqCst);
|
||||
info!("Webhook target {} initialized", self.id);
|
||||
info!(
|
||||
event = EVENT_WEBHOOK_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBHOOK,
|
||||
target_id = %self.id,
|
||||
state = "initialized",
|
||||
"webhook target state"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -357,14 +393,17 @@ where
|
||||
}
|
||||
|
||||
async fn send_body(&self, body: Vec<u8>, meta: &QueuedPayloadMeta) -> Result<(), TargetError> {
|
||||
info!("Webhook sending queued payload to target: {}", self.id);
|
||||
debug!(
|
||||
target = %self.id,
|
||||
event = EVENT_WEBHOOK_DELIVERY_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBHOOK,
|
||||
target_id = %self.id,
|
||||
bucket = %meta.bucket_name,
|
||||
object = %meta.object_name,
|
||||
event = %meta.event_name,
|
||||
payload_event = %meta.event_name,
|
||||
payload_len = body.len(),
|
||||
"Sending webhook payload"
|
||||
state = "sending",
|
||||
"webhook delivery state"
|
||||
);
|
||||
|
||||
// When a TLS reload adapter is attached, it drives client rebuilds in
|
||||
@@ -406,7 +445,15 @@ where
|
||||
|
||||
let status = resp.status();
|
||||
if status.is_success() {
|
||||
debug!("Event sent to webhook target: {}", self.id);
|
||||
debug!(
|
||||
event = EVENT_WEBHOOK_DELIVERY_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBHOOK,
|
||||
target_id = %self.id,
|
||||
status = %status,
|
||||
state = "sent",
|
||||
"webhook delivery state"
|
||||
);
|
||||
self.delivery_counters.record_success();
|
||||
Ok(())
|
||||
} else if status == StatusCode::FORBIDDEN {
|
||||
@@ -454,13 +501,28 @@ where
|
||||
self.delivery_counters.record_final_failure();
|
||||
return Err(e);
|
||||
}
|
||||
debug!("Event saved to store for target: {}", self.id);
|
||||
debug!(
|
||||
event = EVENT_WEBHOOK_DELIVERY_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBHOOK,
|
||||
target_id = %self.id,
|
||||
state = "store_enqueued",
|
||||
"webhook delivery state"
|
||||
);
|
||||
Ok(())
|
||||
} else {
|
||||
match self.init().await {
|
||||
Ok(_) => (),
|
||||
Err(e) => {
|
||||
error!("Failed to initialize Webhook target {}: {}", self.id.id, e);
|
||||
error!(
|
||||
event = EVENT_WEBHOOK_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBHOOK,
|
||||
target_id = %self.id.id,
|
||||
state = "init_failed",
|
||||
error = %e,
|
||||
"webhook target state"
|
||||
);
|
||||
self.delivery_counters.record_final_failure();
|
||||
return Err(TargetError::NotConnected);
|
||||
}
|
||||
@@ -474,13 +536,27 @@ where
|
||||
}
|
||||
|
||||
async fn send_raw_from_store(&self, key: Key, body: Vec<u8>, meta: QueuedPayloadMeta) -> Result<(), TargetError> {
|
||||
debug!("Sending queued payload from store for target: {}, key: {}", self.id, key);
|
||||
debug!(
|
||||
event = EVENT_WEBHOOK_DELIVERY_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBHOOK,
|
||||
target_id = %self.id,
|
||||
key = %key,
|
||||
state = "store_replay_started",
|
||||
"webhook delivery state"
|
||||
);
|
||||
match self.init().await {
|
||||
Ok(_) => {
|
||||
debug!("Event sent to store for target: {}", self.name());
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
error!("Failed to initialize Webhook target {}: {}", self.id.id, e);
|
||||
error!(
|
||||
event = EVENT_WEBHOOK_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBHOOK,
|
||||
target_id = %self.id.id,
|
||||
state = "init_failed",
|
||||
error = %e,
|
||||
"webhook target state"
|
||||
);
|
||||
return Err(TargetError::NotConnected);
|
||||
}
|
||||
}
|
||||
@@ -492,7 +568,15 @@ where
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
debug!("Event sent from store and deleted for target: {}", self.id);
|
||||
debug!(
|
||||
event = EVENT_WEBHOOK_DELIVERY_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBHOOK,
|
||||
target_id = %self.id,
|
||||
key = %key,
|
||||
state = "store_replay_sent",
|
||||
"webhook delivery state"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -500,7 +584,14 @@ where
|
||||
// Send cancel signal to background tasks
|
||||
let _ = self.cancel_sender.try_send(());
|
||||
// Adapter cleanup is done by the coordinator; no local state to reset.
|
||||
info!("Webhook target closed: {}", self.id);
|
||||
info!(
|
||||
event = EVENT_WEBHOOK_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBHOOK,
|
||||
target_id = %self.id,
|
||||
state = "closed",
|
||||
"webhook target state"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -515,7 +606,14 @@ where
|
||||
|
||||
async fn init(&self) -> Result<(), TargetError> {
|
||||
if !self.is_enabled() {
|
||||
debug!("Webhook target {} is disabled, skipping initialization", self.id);
|
||||
debug!(
|
||||
event = EVENT_WEBHOOK_TARGET_STATE,
|
||||
component = LOG_COMPONENT_TARGETS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBHOOK,
|
||||
target_id = %self.id,
|
||||
state = "disabled",
|
||||
"webhook target state"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
self.init_inner().await
|
||||
|
||||
Reference in New Issue
Block a user