gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-31 09:18:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(logging): reduce runtime noise (#3363)

Browse Source
This commit is contained in:
houseme
2026-06-11 19:49:01 +08:00
committed by GitHub
parent a0b6636b61
commit 0a987d870b
19 changed files with 1974 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+220 -42
View File
@@ -81,6 +81,13 @@ use tracing::{debug, error, info, warn};
use uuid::Uuid;
use xxhash_rust::xxh64;
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
const EVENT_LIFECYCLE_WORKER_STATE: &str = "lifecycle_worker_state";
const EVENT_LIFECYCLE_TRANSITION_COMPENSATION: &str = "lifecycle_transition_compensation";
const EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP: &str = "lifecycle_stale_multipart_cleanup";
const EVENT_LIFECYCLE_SCAN_SKIPPED: &str = "lifecycle_scan_skipped";
pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
pub type TraceFn =
Arc<dyn Fn(String, HashMap<String, String>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
@@ -189,7 +196,15 @@ impl LifecycleSys {
Ok((lc, _)) => Some(lc),
Err(Error::ConfigNotFound) => None,
Err(err) => {
warn!(bucket, error = ?err, "failed to load lifecycle config");
debug!(
event = EVENT_LIFECYCLE_SCAN_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket,
error = ?err,
reason = "lifecycle_config_unavailable",
"Skipped lifecycle config lookup"
);
None
}
}
@@ -204,13 +219,16 @@ impl LifecycleSys {
let name = name.clone();
let version_id = version_id.clone();
Box::pin(async move {
info!(
debug!(
bucket = %bucket,
object = %name,
version_id = %version_id,
action = %_action,
"ILM lifecycle trace: {} on {}/{} (version: {})",
_action, bucket, name, version_id
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "trace",
"Lifecycle trace event"
);
})
})
@@ -482,7 +500,14 @@ impl ExpiryState {
loop {
select! {
_ = cancel_token.cancelled() => {
info!("lifecycle expiry worker received shutdown signal, exiting");
debug!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "stopped",
reason = "shutdown_signal",
"Lifecycle expiry worker stopped"
);
break;
}
v = rx.recv() => {
@@ -513,12 +538,16 @@ impl ExpiryState {
else if v.as_any().is::<Jentry>() {
let v = v.as_any().downcast_ref::<Jentry>().expect("Jentry downcast failed");
if let Err(err) = delete_object_from_remote_tier(&v.obj_name, &v.version_id, &v.tier_name).await {
warn!(
debug!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
object = %v.obj_name,
version_id = %v.version_id,
tier = %v.tier_name,
error = ?err,
"failed to delete transitioned object from remote tier"
reason = "remote_tier_delete_failed",
"Lifecycle worker skipped remote tier delete"
);
}
}
@@ -532,14 +561,18 @@ impl ExpiryState {
)
.await
{
warn!(
debug!(
bucket = %oi.bucket,
object = %oi.name,
remote_object = %oi.transitioned_object.name,
remote_version_id = %oi.transitioned_object.version_id,
tier = %oi.transitioned_object.tier,
error = ?err,
"failed to sweep transitioned free version from remote tier"
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
reason = "remote_tier_delete_failed",
"Lifecycle worker skipped remote tier delete"
);
continue;
}
@@ -562,14 +595,18 @@ impl ExpiryState {
}
Err(err) if is_err_version_not_found(&err) || is_err_object_not_found(&err) => continue,
Err(err) => {
warn!(
debug!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %oi.bucket,
object = %oi.name,
remote_object = %oi.transitioned_object.name,
remote_version_id = %oi.transitioned_object.version_id,
tier = %oi.transitioned_object.tier,
error = ?err,
"failed to delete transitioned free version after remote tier sweep"
reason = "local_free_version_delete_failed",
"Lifecycle worker failed local free-version cleanup"
);
break;
}
@@ -577,19 +614,29 @@ impl ExpiryState {
}
if !deleted_locally {
warn!(
debug!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %oi.bucket,
object = %oi.name,
remote_object = %oi.transitioned_object.name,
remote_version_id = %oi.transitioned_object.version_id,
tier = %oi.transitioned_object.tier,
"transitioned free version was not found during local cleanup"
reason = "local_free_version_missing",
"Lifecycle worker could not find transitioned free version locally"
);
}
}
else {
//info!("Invalid work type - {:?}", v);
warn!("lifecycle worker received unsupported operation type");
debug!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "unsupported_task",
"Lifecycle worker received unsupported operation type"
);
}
}
}
@@ -689,14 +736,37 @@ impl TransitionState {
scheduled.lock().unwrap().remove(&bucket);
Self::add_counter(&state.compensation_running_tasks, -1);
state.record_scanner_transition_state();
warn!(bucket = %bucket, "transition compensation skipped because object layer is unavailable");
debug!(
event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %bucket,
state = "skipped",
reason = "object_layer_unavailable",
"Skipped transition compensation"
);
return;
};
if let Err(err) = enqueue_transition_for_existing_objects(api, &bucket).await {
warn!(bucket = %bucket, error = ?err, "transition compensation backfill failed");
warn!(
event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %bucket,
state = "failed",
error = ?err,
"Transition compensation backfill failed"
);
} else {
info!(bucket = %bucket, "transition compensation backfill completed");
info!(
event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %bucket,
state = "completed",
"Completed transition compensation backfill"
);
}
scheduled.lock().unwrap().remove(&bucket);
@@ -744,43 +814,59 @@ impl TransitionState {
match failure {
ImmediateEnqueueFailure::ForcedTimeout => {
Self::inc_counter(&self.queue_send_timeout_tasks);
warn!(
debug!(
bucket = %oi.bucket,
object = %oi.name,
source = ?src,
compensation_scheduled = scheduled,
event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "queue_timeout_forced",
"transition enqueue forced into timeout path for test fault injection"
);
}
ImmediateEnqueueFailure::QueueClosed { timeout_ms } => match timeout_ms {
Some(timeout_ms) => {
warn!(
debug!(
bucket = %oi.bucket,
object = %oi.name,
source = ?src,
timeout_ms,
compensation_scheduled = scheduled,
event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "queue_closed",
"transition enqueue failed because the queue is closed"
);
}
None => {
warn!(
debug!(
bucket = %oi.bucket,
object = %oi.name,
source = ?src,
compensation_scheduled = scheduled,
event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "queue_closed",
"transition enqueue failed because the queue is closed"
);
}
},
ImmediateEnqueueFailure::QueueSendTimedOut { timeout_ms } => {
Self::inc_counter(&self.queue_send_timeout_tasks);
warn!(
debug!(
bucket = %oi.bucket,
object = %oi.name,
source = ?src,
timeout_ms,
compensation_scheduled = scheduled,
event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "queue_send_timed_out",
"transition enqueue timed out under backpressure"
);
}
@@ -850,10 +936,14 @@ impl TransitionState {
);
}
Err(async_channel::TrySendError::Closed(_)) => {
warn!(
debug!(
bucket = %oi.bucket,
object = %oi.name,
source = ?src,
event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "queue_closed",
"transition enqueue failed because the queue is closed"
);
}
@@ -1229,7 +1319,14 @@ async fn cleanup_empty_multipart_sha_dirs_on_local_disks(set: &Arc<SetDisks>) {
Ok(entries) => entries,
Err(err) => {
if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound {
warn!(error = ?err, "failed to list multipart root during empty sha cleanup");
debug!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
error = ?err,
reason = "multipart_root_list_failed",
"Skipped empty multipart sha cleanup"
);
}
continue;
}
@@ -1244,7 +1341,15 @@ async fn cleanup_empty_multipart_sha_dirs_on_local_disks(set: &Arc<SetDisks>) {
Ok(entries) => entries,
Err(err) => {
if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound {
warn!(sha_dir = %sha_dir, error = ?err, "failed to list multipart sha dir during empty sha cleanup");
debug!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
sha_dir = %sha_dir,
error = ?err,
reason = "multipart_sha_dir_list_failed",
"Skipped empty multipart sha cleanup"
);
}
continue;
}
@@ -1260,7 +1365,15 @@ async fn cleanup_empty_multipart_sha_dirs_on_local_disks(set: &Arc<SetDisks>) {
&& err != DiskError::FileNotFound
&& err != DiskError::VolumeNotFound
{
warn!(sha_dir = %sha_dir, error = ?err, "failed to remove empty multipart sha dir");
debug!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
sha_dir = %sha_dir,
error = ?err,
reason = "multipart_sha_dir_remove_failed",
"Failed to remove empty multipart sha dir"
);
}
}
}
@@ -1282,7 +1395,14 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
Ok(entries) => entries,
Err(err) => {
if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound {
warn!(error = ?err, "failed to list multipart root during stale cleanup");
debug!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
error = ?err,
reason = "multipart_root_list_failed",
"Skipped stale multipart cleanup"
);
}
continue;
}
@@ -1297,7 +1417,15 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
Ok(entries) => entries,
Err(err) => {
if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound {
warn!(sha_dir = %sha_dir, error = ?err, "failed to list multipart sha dir during stale cleanup");
debug!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
sha_dir = %sha_dir,
error = ?err,
reason = "multipart_sha_dir_list_failed",
"Skipped stale multipart cleanup"
);
}
continue;
}
@@ -1317,7 +1445,15 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
Ok(candidate) => candidate,
Err(err) => {
if err != DiskError::FileNotFound {
warn!(path = %candidate_path, error = ?err, "failed to read multipart metadata during stale cleanup");
debug!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
path = %candidate_path,
error = ?err,
reason = "multipart_metadata_read_failed",
"Multipart metadata unavailable during stale cleanup"
);
}
let initiated = initiated_from_upload_dir(&upload_dir, None);
StaleMultipartUploadCandidate {
@@ -1351,18 +1487,39 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
deleted += 1;
let upload_id = encode_stale_upload_id(&upload_dir);
if let Some(metadata) = candidate.metadata.as_ref() {
info!(
debug!(
bucket = metadata.get(RUSTFS_MULTIPART_BUCKET_KEY).cloned().unwrap_or_default(),
object = metadata.get(RUSTFS_MULTIPART_OBJECT_KEY).cloned().unwrap_or_default(),
upload_id = %upload_id,
due = ?due,
"removed stale multipart upload"
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "removed",
"Removed stale multipart upload"
);
} else {
info!(path = %candidate.path, upload_id = %upload_id, due = ?due, "removed stale multipart upload");
debug!(
path = %candidate.path,
upload_id = %upload_id,
due = ?due,
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "removed",
"Removed stale multipart upload"
);
}
}
Err(err) => warn!(path = %candidate.path, error = ?err, "failed to remove stale multipart upload"),
Err(err) => debug!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
path = %candidate.path,
error = ?err,
reason = "multipart_remove_failed",
"Failed to remove stale multipart upload"
),
}
}
@@ -1402,7 +1559,13 @@ pub fn init_background_stale_multipart_upload_cleanup(api: Arc<ECStore>) {
let deleted = cleanup_stale_multipart_uploads_once_at(api, OffsetDateTime::now_utc(), default_expiry).await;
if deleted > 0 {
info!(deleted, "completed stale multipart cleanup pass");
debug!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
deleted,
"Completed stale multipart cleanup pass"
);
}
}
});
@@ -2061,9 +2224,14 @@ pub async fn eval_action_from_lifecycle(
oi: &ObjectInfo,
) -> lifecycle::Event {
let event = lc.eval(&oi.to_lifecycle_opts()).await;
//if serverDebugLog {
info!("lifecycle: Secondary scan: {}", event.action);
//}
debug!(
event = EVENT_LIFECYCLE_SCAN_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
action = ?event.action,
state = "evaluated",
"Evaluated lifecycle action during secondary scan"
);
let lock_enabled = if let Some(lr) = lr { lr.mode.is_some() } else { false };
@@ -2079,15 +2247,25 @@ pub async fn eval_action_from_lifecycle(
if lock_enabled && check_object_lock_for_deletion(&oi.bucket, oi, false).await.is_some() {
//if serverDebugLog {
if oi.version_id.is_some() {
info!(
"lifecycle: {} v({}) is locked, not deleting",
oi.name,
oi.version_id.map(|v| v.to_string()).unwrap_or_default()
debug!(
event = EVENT_LIFECYCLE_SCAN_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
object = %oi.name,
version_id = %oi.version_id.map(|v| v.to_string()).unwrap_or_default(),
reason = "object_locked",
"Skipped lifecycle delete because object version is locked"
);
} else {
info!("lifecycle: {} is locked, not deleting", oi.name);
debug!(
event = EVENT_LIFECYCLE_SCAN_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
object = %oi.name,
reason = "object_locked",
"Skipped lifecycle delete because object is locked"
);
}
//}
return lifecycle::Event::default();
}
if let Some(rcfg) = rcfg
crates/ecstore/src/bucket/metadata.rs
+148 -17
View File
@@ -799,59 +799,139 @@ impl BucketMetadata {
fn parse_all_configs(&mut self) -> Result<()> {
if let Err(e) = self.parse_policy_config() {
tracing::warn!(bucket = %self.name, config = "policy", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "policy",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.notification_config_xml.is_empty()
&& let Err(e) = deserialize::<NotificationConfiguration>(&self.notification_config_xml)
.map(|c| self.notification_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "notification", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "notification",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.lifecycle_config_xml.is_empty()
&& let Err(e) =
deserialize::<BucketLifecycleConfiguration>(&self.lifecycle_config_xml).map(|c| self.lifecycle_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "lifecycle", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "lifecycle",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.object_lock_config_xml.is_empty()
&& let Err(e) =
deserialize::<ObjectLockConfiguration>(&self.object_lock_config_xml).map(|c| self.object_lock_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "object_lock", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "object_lock",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.versioning_config_xml.is_empty()
&& let Err(e) =
deserialize::<VersioningConfiguration>(&self.versioning_config_xml).map(|c| self.versioning_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "versioning", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "versioning",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.encryption_config_xml.is_empty()
&& let Err(e) =
deserialize::<ServerSideEncryptionConfiguration>(&self.encryption_config_xml).map(|c| self.sse_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "encryption", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "encryption",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.tagging_config_xml.is_empty()
&& let Err(e) = deserialize::<Tagging>(&self.tagging_config_xml).map(|c| self.tagging_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "tagging", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "tagging",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.quota_config_json.is_empty()
&& let Err(e) = serde_json::from_slice(&self.quota_config_json).map(|c| self.quota_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "quota", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "quota",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.replication_config_xml.is_empty()
&& let Err(e) =
deserialize::<ReplicationConfiguration>(&self.replication_config_xml).map(|c| self.replication_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "replication", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "replication",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.bucket_targets_config_json.is_empty() {
if let Err(e) = serde_json::from_slice::<BucketTargets>(&self.bucket_targets_config_json)
.map(|t| self.bucket_target_config = Some(t))
{
tracing::warn!(bucket = %self.name, config = "bucket_targets", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "bucket_targets",
error = %e,
"Failed to parse bucket metadata config"
);
self.bucket_target_config = Some(BucketTargets::default());
}
} else {
@@ -860,45 +940,96 @@ impl BucketMetadata {
if !self.cors_config_xml.is_empty()
&& let Err(e) = deserialize::<CORSConfiguration>(&self.cors_config_xml).map(|c| self.cors_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "cors", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "cors",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.logging_config_xml.is_empty()
&& let Err(e) = deserialize::<BucketLoggingStatus>(&self.logging_config_xml).map(|c| self.logging_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "logging", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "logging",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.website_config_xml.is_empty()
&& let Err(e) = deserialize::<WebsiteConfiguration>(&self.website_config_xml).map(|c| self.website_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "website", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "website",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.accelerate_config_xml.is_empty()
&& let Err(e) =
deserialize::<AccelerateConfiguration>(&self.accelerate_config_xml).map(|c| self.accelerate_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "accelerate", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "accelerate",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.request_payment_config_xml.is_empty()
&& let Err(e) = deserialize::<RequestPaymentConfiguration>(&self.request_payment_config_xml)
.map(|c| self.request_payment_config = Some(c))
{
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "request_payment",
error = %e,
"parse_all_configs: failed to parse"
"Failed to parse bucket metadata config"
);
}
if !self.public_access_block_config_xml.is_empty()
&& let Err(e) = deserialize::<PublicAccessBlockConfiguration>(&self.public_access_block_config_xml)
.map(|c| self.public_access_block_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "public_access_block", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "public_access_block",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.bucket_acl_config_json.is_empty()
&& let Err(e) = String::from_utf8(self.bucket_acl_config_json.clone()).map(|acl| self.bucket_acl_config = Some(acl))
{
tracing::warn!(bucket = %self.name, config = "bucket_acl", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "bucket_acl",
error = %e,
"Failed to parse bucket metadata config"
);
}
Ok(())
crates/ecstore/src/bucket/replication/replication_pool.rs
+100 -16
View File
@@ -56,7 +56,15 @@ use tokio::sync::mpsc::Sender;
use tokio::task::JoinHandle;
use tokio::time::Duration;
use tokio_util::sync::CancellationToken;
use tracing::{info, instrument, warn};
use tracing::{debug, info, instrument};
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_REPLICATION: &str = "replication";
const EVENT_REPLICATION_WORKER_RESIZE_SKIPPED: &str = "replication_worker_resize_skipped";
const EVENT_REPLICATION_WORKER_RESIZED: &str = "replication_worker_resized";
const EVENT_REPLICATION_BACKPRESSURE: &str = "replication_backpressure";
const EVENT_REPLICATION_RESYNC_LOAD_SKIPPED: &str = "replication_resync_load_skipped";
const EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED: &str = "replication_config_lookup_skipped";
// Worker limits
pub const WORKER_MAX_LIMIT: usize = 500;
@@ -358,18 +366,31 @@ impl<S: StorageAPI> ReplicationPool<S> {
let mut workers = self.workers.write().await;
if (check_old > 0 && workers.len() != check_old) || n == workers.len() || n < 1 {
warn!(
"resize_workers: skipping resize - check_old_mismatch={}, same_size={}, invalid_n={}",
check_old > 0 && workers.len() != check_old,
n == workers.len(),
n < 1
debug!(
event = EVENT_REPLICATION_WORKER_RESIZE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
check_old_mismatch = check_old > 0 && workers.len() != check_old,
same_size = n == workers.len(),
invalid_target_size = n < 1,
current_workers = workers.len(),
target_workers = n,
"Skipped replication worker resize"
);
return;
}
// Add workers if needed
if workers.len() < n {
info!("resize_workers: adding workers from {} to {}", workers.len(), n);
info!(
event = EVENT_REPLICATION_WORKER_RESIZED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
action = "increase",
from_workers = workers.len(),
to_workers = n,
"Resized replication workers"
);
}
while workers.len() < n {
@@ -416,7 +437,15 @@ impl<S: StorageAPI> ReplicationPool<S> {
// Remove workers if needed
if workers.len() > n {
warn!("resize_workers: removing workers from {} to {}", workers.len(), n);
info!(
event = EVENT_REPLICATION_WORKER_RESIZED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
action = "decrease",
from_workers = workers.len(),
to_workers = n,
"Resized replication workers"
);
}
while workers.len() > n {
@@ -607,11 +636,26 @@ impl<S: StorageAPI> ReplicationPool<S> {
match priority {
ReplicationPriority::Fast => {
// Log warning about unable to keep up
info!("Warning: Unable to keep up with incoming traffic");
debug!(
event = EVENT_REPLICATION_BACKPRESSURE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
queue_type = "object",
priority = "fast",
recommendation = "none",
"Replication queue is backpressured"
);
}
ReplicationPriority::Slow => {
info!("Warning: Unable to keep up with incoming traffic - recommend increasing replication priority to auto");
debug!(
event = EVENT_REPLICATION_BACKPRESSURE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
queue_type = "object",
priority = "slow",
recommendation = "set_priority_auto",
"Replication queue is backpressured"
);
}
ReplicationPriority::Auto => {
let max_w = std::cmp::min(max_workers, WORKER_MAX_LIMIT);
@@ -667,10 +711,26 @@ impl<S: StorageAPI> ReplicationPool<S> {
match priority {
ReplicationPriority::Fast => {
info!("Warning: Unable to keep up with incoming deletes");
debug!(
event = EVENT_REPLICATION_BACKPRESSURE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
queue_type = "delete",
priority = "fast",
recommendation = "none",
"Replication delete queue is backpressured"
);
}
ReplicationPriority::Slow => {
info!("Warning: Unable to keep up with incoming deletes - recommend increasing replication priority to auto");
debug!(
event = EVENT_REPLICATION_BACKPRESSURE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
queue_type = "delete",
priority = "slow",
recommendation = "set_priority_auto",
"Replication delete queue is backpressured"
);
}
ReplicationPriority::Auto => {
let max_w = std::cmp::min(max_workers, WORKER_MAX_LIMIT);
@@ -930,7 +990,15 @@ impl<S: StorageAPI> ReplicationPool<S> {
Ok(meta) => meta,
Err(err) => {
if !matches!(err, EcstoreError::VolumeNotFound) {
warn!("Error loading resync metadata for bucket {bucket}: {err:?}");
debug!(
event = EVENT_REPLICATION_RESYNC_LOAD_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
bucket,
error = ?err,
reason = "metadata_load_failed",
"Skipped replication resync metadata load"
);
}
continue;
}
@@ -1179,7 +1247,15 @@ pub async fn queue_replication_heal(bucket: &str, oi: ObjectInfo, retry_count: u
let rcfg = match metadata_sys::get_replication_config(bucket).await {
Ok((config, _)) => config,
Err(err) => {
warn!("Failed to get replication config for bucket {}: {}", bucket, err);
debug!(
event = EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
bucket,
error = %err,
reason = "config_lookup_failed",
"Skipped replication heal queue due to missing replication config"
);
return;
}
@@ -1188,7 +1264,15 @@ pub async fn queue_replication_heal(bucket: &str, oi: ObjectInfo, retry_count: u
let tgts = match BucketTargetSys::get().list_bucket_targets(bucket).await {
Ok(targets) => Some(targets),
Err(err) => {
warn!("Failed to list bucket targets for bucket {}: {}", bucket, err);
debug!(
event = EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
bucket,
error = %err,
reason = "target_list_failed",
"Skipped bucket target list during replication heal queue setup"
);
None
}
};
crates/ecstore/src/bucket/replication/replication_resyncer.rs
+262 -49
View File
@@ -85,9 +85,18 @@ use tokio::task::JoinSet;
use tokio::time::Duration as TokioDuration;
use tokio_util::io::ReaderStream;
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, instrument, warn};
use tracing::{debug, error, instrument, warn};
use uuid::Uuid;
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_REPLICATION_RESYNC: &str = "replication_resync";
const EVENT_RESYNC_STATUS_UPDATE_SKIPPED: &str = "replication_resync_status_update_skipped";
const EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED: &str = "replication_resync_config_lookup_skipped";
const EVENT_RESYNC_OBJECT_PROCESSED: &str = "replication_resync_object_processed";
const EVENT_RESYNC_RUNTIME_SKIPPED: &str = "replication_resync_runtime_skipped";
const EVENT_REPLICATION_DELETE_SKIPPED: &str = "replication_delete_skipped";
const EVENT_REPLICATION_FORCE_DELETE_SKIPPED: &str = "replication_force_delete_skipped";
pub(crate) const REPLICATION_DIR: &str = ".replication";
pub(crate) const RESYNC_FILE_NAME: &str = "resync.bin";
pub(crate) const RESYNC_META_FORMAT: u16 = 1;
@@ -508,12 +517,16 @@ impl ReplicationResyncer {
};
if !resync_state_accepts_update(state, &opts) {
warn!(
debug!(
event = EVENT_RESYNC_STATUS_UPDATE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %opts.bucket,
arn = %opts.arn,
incoming_resync_id = %opts.resync_id,
current_resync_id = %state.resync_id,
"ignoring stale resync status update"
reason = "stale_status_update",
"Skipped stale resync status update"
);
return Ok(());
}
@@ -565,12 +578,16 @@ impl ReplicationResyncer {
};
if !resync_state_accepts_update(state, &opts) {
warn!(
debug!(
event = EVENT_RESYNC_STATUS_UPDATE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %opts.bucket,
arn = %opts.arn,
incoming_resync_id = %opts.resync_id,
current_resync_id = %state.resync_id,
"ignoring stale resync stats update"
reason = "stale_stats_update",
"Skipped stale resync stats update"
);
return;
}
@@ -676,7 +693,15 @@ impl ReplicationResyncer {
let targets = match BucketTargetSys::get().list_bucket_targets(&opts.bucket).await {
Ok(targets) => targets,
Err(err) => {
warn!("Failed to list bucket targets: {}", err);
debug!(
event = EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %opts.bucket,
error = %err,
reason = "target_list_failed",
"Failed to list bucket targets during resync"
);
self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone())
.await;
return;
@@ -847,14 +872,17 @@ impl ReplicationResyncer {
(roi.size, None)
};
info!(
"resynced reset_id:{} object: {}/{}-{} size:{} err:{:?}",
reset_id,
bucket_name,
roi.name,
roi.version_id.unwrap_or_default(),
debug!(
event = EVENT_RESYNC_OBJECT_PROCESSED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
reset_id = %reset_id,
bucket = %bucket_name,
object = %roi.name,
version_id = %roi.version_id.unwrap_or_default(),
size,
err,
error = ?err,
"Processed resync object"
);
if cancel_token.is_cancelled() {
@@ -1548,7 +1576,14 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
let _rcfg = match get_replication_config(&bucket).await {
Ok(Some(config)) => config,
Ok(None) => {
warn!("No replication config found for bucket: {}", bucket);
debug!(
event = EVENT_REPLICATION_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
reason = "replication_config_missing",
"Skipping replication delete because replication config is missing"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -1567,7 +1602,15 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
return;
}
Err(err) => {
warn!("replication config for bucket: {} error: {}", bucket, err);
debug!(
event = EVENT_REPLICATION_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
error = %err,
reason = "replication_config_lookup_failed",
"Skipping replication delete because replication config lookup failed"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -1605,30 +1648,42 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
match source_marker_state {
Ok(info) if info.delete_marker && info.version_id == Some(delete_marker_version_id) => {}
Ok(_) => {
warn!(
debug!(
event = EVENT_REPLICATION_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket,
object = dobj.delete_object.object_name,
version_id = %delete_marker_version_id,
"skipping stale delete-marker replication because source version is no longer a delete marker"
reason = "source_not_delete_marker",
"Skipping stale delete-marker replication"
);
return;
}
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {
warn!(
debug!(
event = EVENT_REPLICATION_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket,
object = dobj.delete_object.object_name,
version_id = %delete_marker_version_id,
"skipping stale delete-marker replication because source version no longer exists"
reason = "source_version_missing",
"Skipping stale delete-marker replication"
);
return;
}
Err(err) => {
warn!(
debug!(
event = EVENT_REPLICATION_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket,
object = dobj.delete_object.object_name,
version_id = %delete_marker_version_id,
error = %err,
"failed to verify source delete-marker state before replication"
reason = "source_state_verification_failed",
"Failed to verify source delete-marker state before replication"
);
}
}
@@ -1645,9 +1700,15 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
) {
Ok(dsc) => dsc,
Err(err) => {
warn!(
"failed to parse replicate decision for bucket:{} arn:{} error:{}",
bucket, dobj.target_arn, err
debug!(
event = EVENT_REPLICATION_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %dobj.target_arn,
error = %err,
reason = "replicate_decision_parse_failed",
"Failed to parse replicate decision"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
@@ -1672,9 +1733,15 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
{
Ok(ns_lock) => ns_lock,
Err(e) => {
warn!(
"failed to get ns lock for bucket:{} object:{} error:{}",
bucket, dobj.delete_object.object_name, e
debug!(
event = EVENT_REPLICATION_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
object = %dobj.delete_object.object_name,
error = %e,
reason = "ns_lock_unavailable",
"Skipping replication delete"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
@@ -1697,9 +1764,15 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
let _lock_guard = match ns_lock.get_write_lock(get_lock_acquire_timeout()).await {
Ok(lock_guard) => lock_guard,
Err(e) => {
warn!(
"failed to get write lock for bucket:{} object:{} error:{}",
bucket, dobj.delete_object.object_name, e
debug!(
event = EVENT_REPLICATION_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
object = %dobj.delete_object.object_name,
error = %e,
reason = "write_lock_unavailable",
"Skipping replication delete"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
@@ -1741,7 +1814,15 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
// Get the remote target client
let Some(tgt_client) = BucketTargetSys::get().get_remote_target_client(&bucket, &tgt_entry.arn).await else {
warn!("failed to get target for bucket:{:?}, arn:{:?}", &bucket, &tgt_entry.arn);
debug!(
event = EVENT_REPLICATION_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_entry.arn,
reason = "target_client_missing",
"Skipping replication delete because target client is unavailable"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -1969,7 +2050,14 @@ async fn replicate_force_delete_to_targets<S: StorageAPI>(dobj: &DeletedObjectRe
let rcfg = match get_replication_config(bucket).await {
Ok(Some(config)) => config,
Ok(None) => {
warn!("replicate force-delete: no replication config for bucket:{}", bucket);
debug!(
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
reason = "replication_config_missing",
"Skipping replication force-delete because replication config is missing"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -1985,7 +2073,15 @@ async fn replicate_force_delete_to_targets<S: StorageAPI>(dobj: &DeletedObjectRe
return;
}
Err(err) => {
warn!("replicate force-delete: replication config error bucket:{} error:{}", bucket, err);
debug!(
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
error = %err,
reason = "replication_config_lookup_failed",
"Skipping replication force-delete because replication config lookup failed"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -2064,7 +2160,15 @@ async fn replicate_force_delete_to_targets<S: StorageAPI>(dobj: &DeletedObjectRe
for arn in tgt_arns {
let Some(tgt_client) = BucketTargetSys::get().get_remote_target_client(bucket, &arn).await else {
warn!("replicate force-delete: failed to get target client bucket:{} arn:{}", bucket, arn);
debug!(
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %arn,
reason = "target_client_missing",
"Skipping replication force-delete because target client is unavailable"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -2300,7 +2404,14 @@ pub async fn replicate_object<S: StorageAPI>(roi: ReplicateObjectInfo, storage:
let cfg = match get_replication_config(&bucket).await {
Ok(Some(config)) => config,
Ok(None) => {
warn!("No replication config found for bucket: {}", bucket);
debug!(
event = EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
reason = "replication_config_missing",
"Skipping replication object because replication config is missing"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -2338,7 +2449,15 @@ pub async fn replicate_object<S: StorageAPI>(roi: ReplicateObjectInfo, storage:
for arn in tgt_arns {
let Some(tgt_client) = BucketTargetSys::get().get_remote_target_client(&bucket, &arn).await else {
warn!("failed to get target for bucket:{:?}, arn:{:?}", &bucket, &arn);
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %arn,
reason = "target_client_missing",
"Skipping replication object target"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -2479,7 +2598,16 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
}
if BucketTargetSys::get().is_offline(&tgt_client.to_url()).await {
warn!("target is offline: {}", tgt_client.to_url());
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
reason = "target_offline",
endpoint = %tgt_client.to_url(),
"Skipping replication object target"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -2509,7 +2637,16 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
Ok(gr) => gr,
Err(e) => {
if !is_err_object_not_found(&e) || is_err_version_not_found(&e) {
warn!("failed to get object reader for bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e);
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
error = %e,
reason = "object_reader_unavailable",
"Skipping replication object target"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
@@ -2532,7 +2669,16 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
let size = match object_info.get_actual_size() {
Ok(size) => size,
Err(e) => {
warn!("failed to get actual size for bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e);
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
error = %e,
reason = "actual_size_unavailable",
"Skipping replication object target"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -2546,7 +2692,15 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
};
if tgt_client.bucket.is_empty() {
warn!("target bucket is empty: {}", tgt_client.bucket);
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
reason = "target_bucket_empty",
"Skipping replication object target"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -2702,7 +2856,16 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
};
if BucketTargetSys::get().is_offline(&tgt_client.to_url()).await {
warn!("target is offline: {}", tgt_client.to_url());
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
target = %tgt_client.to_url(),
reason = "target_offline",
"Skipped replication because target is offline"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -2732,7 +2895,16 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
Ok(gr) => gr,
Err(e) => {
if !is_err_object_not_found(&e) || is_err_version_not_found(&e) {
warn!("failed to get object reader for bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e);
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
error = %e,
reason = "object_reader_unavailable",
"Skipped replication because object reader is unavailable"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -2763,7 +2935,16 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
let size = match object_info.get_actual_size() {
Ok(size) => size,
Err(e) => {
warn!("failed to get actual size for bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e);
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
error = %e,
reason = "actual_size_unavailable",
"Skipped replication because actual object size is unavailable"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -2779,7 +2960,15 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
// TODO: SSE
if tgt_client.bucket.is_empty() {
warn!("target bucket is empty: {}", tgt_client.bucket);
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
reason = "target_bucket_empty",
"Skipped replication because target bucket is empty"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -2801,7 +2990,16 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
};
if let Err(err) = sopts.set(AMZ_TAGGING_DIRECTIVE, "ACCESS") {
warn!("failed to set replication tagging directive header: {err}");
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
error = %err,
reason = "tagging_directive_header_invalid",
"Skipped replication tagging directive header detail"
);
}
match head_object_with_proxy_stats(
@@ -2877,9 +3075,15 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
}
Err(e2) => {
rinfo.error = Some(e2.to_string());
warn!(
"replication head_object fallback failed bucket:{} arn:{} error:{}",
bucket, tgt_client.arn, e2
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
error = %e2,
reason = "head_object_fallback_failed",
"Failed replication head-object fallback"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
@@ -2897,7 +3101,16 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
replication_action = ReplicationAction::All;
} else {
rinfo.error = Some(e.to_string());
warn!("failed to head object for bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e);
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
error = %e,
reason = "head_object_failed",
"Skipped replication because head-object failed"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),