gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-24 05:06:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(logging): reduce runtime noise (#3363)

Browse Source
This commit is contained in:
houseme
2026-06-11 19:49:01 +08:00
committed by GitHub
parent a0b6636b61
commit 0a987d870b
19 changed files with 1974 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+220 -42
View File
@@ -81,6 +81,13 @@ use tracing::{debug, error, info, warn};
use uuid::Uuid;
use xxhash_rust::xxh64;
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
const EVENT_LIFECYCLE_WORKER_STATE: &str = "lifecycle_worker_state";
const EVENT_LIFECYCLE_TRANSITION_COMPENSATION: &str = "lifecycle_transition_compensation";
const EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP: &str = "lifecycle_stale_multipart_cleanup";
const EVENT_LIFECYCLE_SCAN_SKIPPED: &str = "lifecycle_scan_skipped";
pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
pub type TraceFn =
Arc<dyn Fn(String, HashMap<String, String>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
@@ -189,7 +196,15 @@ impl LifecycleSys {
Ok((lc, _)) => Some(lc),
Err(Error::ConfigNotFound) => None,
Err(err) => {
warn!(bucket, error = ?err, "failed to load lifecycle config");
debug!(
event = EVENT_LIFECYCLE_SCAN_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket,
error = ?err,
reason = "lifecycle_config_unavailable",
"Skipped lifecycle config lookup"
);
None
}
}
@@ -204,13 +219,16 @@ impl LifecycleSys {
let name = name.clone();
let version_id = version_id.clone();
Box::pin(async move {
info!(
debug!(
bucket = %bucket,
object = %name,
version_id = %version_id,
action = %_action,
"ILM lifecycle trace: {} on {}/{} (version: {})",
_action, bucket, name, version_id
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "trace",
"Lifecycle trace event"
);
})
})
@@ -482,7 +500,14 @@ impl ExpiryState {
loop {
select! {
_ = cancel_token.cancelled() => {
info!("lifecycle expiry worker received shutdown signal, exiting");
debug!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "stopped",
reason = "shutdown_signal",
"Lifecycle expiry worker stopped"
);
break;
}
v = rx.recv() => {
@@ -513,12 +538,16 @@ impl ExpiryState {
else if v.as_any().is::<Jentry>() {
let v = v.as_any().downcast_ref::<Jentry>().expect("Jentry downcast failed");
if let Err(err) = delete_object_from_remote_tier(&v.obj_name, &v.version_id, &v.tier_name).await {
warn!(
debug!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
object = %v.obj_name,
version_id = %v.version_id,
tier = %v.tier_name,
error = ?err,
"failed to delete transitioned object from remote tier"
reason = "remote_tier_delete_failed",
"Lifecycle worker skipped remote tier delete"
);
}
}
@@ -532,14 +561,18 @@ impl ExpiryState {
)
.await
{
warn!(
debug!(
bucket = %oi.bucket,
object = %oi.name,
remote_object = %oi.transitioned_object.name,
remote_version_id = %oi.transitioned_object.version_id,
tier = %oi.transitioned_object.tier,
error = ?err,
"failed to sweep transitioned free version from remote tier"
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
reason = "remote_tier_delete_failed",
"Lifecycle worker skipped remote tier delete"
);
continue;
}
@@ -562,14 +595,18 @@ impl ExpiryState {
}
Err(err) if is_err_version_not_found(&err) || is_err_object_not_found(&err) => continue,
Err(err) => {
warn!(
debug!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %oi.bucket,
object = %oi.name,
remote_object = %oi.transitioned_object.name,
remote_version_id = %oi.transitioned_object.version_id,
tier = %oi.transitioned_object.tier,
error = ?err,
"failed to delete transitioned free version after remote tier sweep"
reason = "local_free_version_delete_failed",
"Lifecycle worker failed local free-version cleanup"
);
break;
}
@@ -577,19 +614,29 @@ impl ExpiryState {
}
if !deleted_locally {
warn!(
debug!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %oi.bucket,
object = %oi.name,
remote_object = %oi.transitioned_object.name,
remote_version_id = %oi.transitioned_object.version_id,
tier = %oi.transitioned_object.tier,
"transitioned free version was not found during local cleanup"
reason = "local_free_version_missing",
"Lifecycle worker could not find transitioned free version locally"
);
}
}
else {
//info!("Invalid work type - {:?}", v);
warn!("lifecycle worker received unsupported operation type");
debug!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "unsupported_task",
"Lifecycle worker received unsupported operation type"
);
}
}
}
@@ -689,14 +736,37 @@ impl TransitionState {
scheduled.lock().unwrap().remove(&bucket);
Self::add_counter(&state.compensation_running_tasks, -1);
state.record_scanner_transition_state();
warn!(bucket = %bucket, "transition compensation skipped because object layer is unavailable");
debug!(
event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %bucket,
state = "skipped",
reason = "object_layer_unavailable",
"Skipped transition compensation"
);
return;
};
if let Err(err) = enqueue_transition_for_existing_objects(api, &bucket).await {
warn!(bucket = %bucket, error = ?err, "transition compensation backfill failed");
warn!(
event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %bucket,
state = "failed",
error = ?err,
"Transition compensation backfill failed"
);
} else {
info!(bucket = %bucket, "transition compensation backfill completed");
info!(
event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %bucket,
state = "completed",
"Completed transition compensation backfill"
);
}
scheduled.lock().unwrap().remove(&bucket);
@@ -744,43 +814,59 @@ impl TransitionState {
match failure {
ImmediateEnqueueFailure::ForcedTimeout => {
Self::inc_counter(&self.queue_send_timeout_tasks);
warn!(
debug!(
bucket = %oi.bucket,
object = %oi.name,
source = ?src,
compensation_scheduled = scheduled,
event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "queue_timeout_forced",
"transition enqueue forced into timeout path for test fault injection"
);
}
ImmediateEnqueueFailure::QueueClosed { timeout_ms } => match timeout_ms {
Some(timeout_ms) => {
warn!(
debug!(
bucket = %oi.bucket,
object = %oi.name,
source = ?src,
timeout_ms,
compensation_scheduled = scheduled,
event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "queue_closed",
"transition enqueue failed because the queue is closed"
);
}
None => {
warn!(
debug!(
bucket = %oi.bucket,
object = %oi.name,
source = ?src,
compensation_scheduled = scheduled,
event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "queue_closed",
"transition enqueue failed because the queue is closed"
);
}
},
ImmediateEnqueueFailure::QueueSendTimedOut { timeout_ms } => {
Self::inc_counter(&self.queue_send_timeout_tasks);
warn!(
debug!(
bucket = %oi.bucket,
object = %oi.name,
source = ?src,
timeout_ms,
compensation_scheduled = scheduled,
event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "queue_send_timed_out",
"transition enqueue timed out under backpressure"
);
}
@@ -850,10 +936,14 @@ impl TransitionState {
);
}
Err(async_channel::TrySendError::Closed(_)) => {
warn!(
debug!(
bucket = %oi.bucket,
object = %oi.name,
source = ?src,
event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "queue_closed",
"transition enqueue failed because the queue is closed"
);
}
@@ -1229,7 +1319,14 @@ async fn cleanup_empty_multipart_sha_dirs_on_local_disks(set: &Arc<SetDisks>) {
Ok(entries) => entries,
Err(err) => {
if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound {
warn!(error = ?err, "failed to list multipart root during empty sha cleanup");
debug!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
error = ?err,
reason = "multipart_root_list_failed",
"Skipped empty multipart sha cleanup"
);
}
continue;
}
@@ -1244,7 +1341,15 @@ async fn cleanup_empty_multipart_sha_dirs_on_local_disks(set: &Arc<SetDisks>) {
Ok(entries) => entries,
Err(err) => {
if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound {
warn!(sha_dir = %sha_dir, error = ?err, "failed to list multipart sha dir during empty sha cleanup");
debug!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
sha_dir = %sha_dir,
error = ?err,
reason = "multipart_sha_dir_list_failed",
"Skipped empty multipart sha cleanup"
);
}
continue;
}
@@ -1260,7 +1365,15 @@ async fn cleanup_empty_multipart_sha_dirs_on_local_disks(set: &Arc<SetDisks>) {
&& err != DiskError::FileNotFound
&& err != DiskError::VolumeNotFound
{
warn!(sha_dir = %sha_dir, error = ?err, "failed to remove empty multipart sha dir");
debug!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
sha_dir = %sha_dir,
error = ?err,
reason = "multipart_sha_dir_remove_failed",
"Failed to remove empty multipart sha dir"
);
}
}
}
@@ -1282,7 +1395,14 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
Ok(entries) => entries,
Err(err) => {
if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound {
warn!(error = ?err, "failed to list multipart root during stale cleanup");
debug!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
error = ?err,
reason = "multipart_root_list_failed",
"Skipped stale multipart cleanup"
);
}
continue;
}
@@ -1297,7 +1417,15 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
Ok(entries) => entries,
Err(err) => {
if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound {
warn!(sha_dir = %sha_dir, error = ?err, "failed to list multipart sha dir during stale cleanup");
debug!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
sha_dir = %sha_dir,
error = ?err,
reason = "multipart_sha_dir_list_failed",
"Skipped stale multipart cleanup"
);
}
continue;
}
@@ -1317,7 +1445,15 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
Ok(candidate) => candidate,
Err(err) => {
if err != DiskError::FileNotFound {
warn!(path = %candidate_path, error = ?err, "failed to read multipart metadata during stale cleanup");
debug!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
path = %candidate_path,
error = ?err,
reason = "multipart_metadata_read_failed",
"Multipart metadata unavailable during stale cleanup"
);
}
let initiated = initiated_from_upload_dir(&upload_dir, None);
StaleMultipartUploadCandidate {
@@ -1351,18 +1487,39 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
deleted += 1;
let upload_id = encode_stale_upload_id(&upload_dir);
if let Some(metadata) = candidate.metadata.as_ref() {
info!(
debug!(
bucket = metadata.get(RUSTFS_MULTIPART_BUCKET_KEY).cloned().unwrap_or_default(),
object = metadata.get(RUSTFS_MULTIPART_OBJECT_KEY).cloned().unwrap_or_default(),
upload_id = %upload_id,
due = ?due,
"removed stale multipart upload"
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "removed",
"Removed stale multipart upload"
);
} else {
info!(path = %candidate.path, upload_id = %upload_id, due = ?due, "removed stale multipart upload");
debug!(
path = %candidate.path,
upload_id = %upload_id,
due = ?due,
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "removed",
"Removed stale multipart upload"
);
}
}
Err(err) => warn!(path = %candidate.path, error = ?err, "failed to remove stale multipart upload"),
Err(err) => debug!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
path = %candidate.path,
error = ?err,
reason = "multipart_remove_failed",
"Failed to remove stale multipart upload"
),
}
}
@@ -1402,7 +1559,13 @@ pub fn init_background_stale_multipart_upload_cleanup(api: Arc<ECStore>) {
let deleted = cleanup_stale_multipart_uploads_once_at(api, OffsetDateTime::now_utc(), default_expiry).await;
if deleted > 0 {
info!(deleted, "completed stale multipart cleanup pass");
debug!(
event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
deleted,
"Completed stale multipart cleanup pass"
);
}
}
});
@@ -2061,9 +2224,14 @@ pub async fn eval_action_from_lifecycle(
oi: &ObjectInfo,
) -> lifecycle::Event {
let event = lc.eval(&oi.to_lifecycle_opts()).await;
//if serverDebugLog {
info!("lifecycle: Secondary scan: {}", event.action);
//}
debug!(
event = EVENT_LIFECYCLE_SCAN_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
action = ?event.action,
state = "evaluated",
"Evaluated lifecycle action during secondary scan"
);
let lock_enabled = if let Some(lr) = lr { lr.mode.is_some() } else { false };
@@ -2079,15 +2247,25 @@ pub async fn eval_action_from_lifecycle(
if lock_enabled && check_object_lock_for_deletion(&oi.bucket, oi, false).await.is_some() {
//if serverDebugLog {
if oi.version_id.is_some() {
info!(
"lifecycle: {} v({}) is locked, not deleting",
oi.name,
oi.version_id.map(|v| v.to_string()).unwrap_or_default()
debug!(
event = EVENT_LIFECYCLE_SCAN_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
object = %oi.name,
version_id = %oi.version_id.map(|v| v.to_string()).unwrap_or_default(),
reason = "object_locked",
"Skipped lifecycle delete because object version is locked"
);
} else {
info!("lifecycle: {} is locked, not deleting", oi.name);
debug!(
event = EVENT_LIFECYCLE_SCAN_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
object = %oi.name,
reason = "object_locked",
"Skipped lifecycle delete because object is locked"
);
}
//}
return lifecycle::Event::default();
}
if let Some(rcfg) = rcfg
crates/ecstore/src/bucket/metadata.rs
+148 -17
View File
@@ -799,59 +799,139 @@ impl BucketMetadata {
fn parse_all_configs(&mut self) -> Result<()> {
if let Err(e) = self.parse_policy_config() {
tracing::warn!(bucket = %self.name, config = "policy", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "policy",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.notification_config_xml.is_empty()
&& let Err(e) = deserialize::<NotificationConfiguration>(&self.notification_config_xml)
.map(|c| self.notification_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "notification", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "notification",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.lifecycle_config_xml.is_empty()
&& let Err(e) =
deserialize::<BucketLifecycleConfiguration>(&self.lifecycle_config_xml).map(|c| self.lifecycle_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "lifecycle", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "lifecycle",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.object_lock_config_xml.is_empty()
&& let Err(e) =
deserialize::<ObjectLockConfiguration>(&self.object_lock_config_xml).map(|c| self.object_lock_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "object_lock", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "object_lock",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.versioning_config_xml.is_empty()
&& let Err(e) =
deserialize::<VersioningConfiguration>(&self.versioning_config_xml).map(|c| self.versioning_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "versioning", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "versioning",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.encryption_config_xml.is_empty()
&& let Err(e) =
deserialize::<ServerSideEncryptionConfiguration>(&self.encryption_config_xml).map(|c| self.sse_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "encryption", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "encryption",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.tagging_config_xml.is_empty()
&& let Err(e) = deserialize::<Tagging>(&self.tagging_config_xml).map(|c| self.tagging_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "tagging", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "tagging",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.quota_config_json.is_empty()
&& let Err(e) = serde_json::from_slice(&self.quota_config_json).map(|c| self.quota_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "quota", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "quota",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.replication_config_xml.is_empty()
&& let Err(e) =
deserialize::<ReplicationConfiguration>(&self.replication_config_xml).map(|c| self.replication_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "replication", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "replication",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.bucket_targets_config_json.is_empty() {
if let Err(e) = serde_json::from_slice::<BucketTargets>(&self.bucket_targets_config_json)
.map(|t| self.bucket_target_config = Some(t))
{
tracing::warn!(bucket = %self.name, config = "bucket_targets", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "bucket_targets",
error = %e,
"Failed to parse bucket metadata config"
);
self.bucket_target_config = Some(BucketTargets::default());
}
} else {
@@ -860,45 +940,96 @@ impl BucketMetadata {
if !self.cors_config_xml.is_empty()
&& let Err(e) = deserialize::<CORSConfiguration>(&self.cors_config_xml).map(|c| self.cors_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "cors", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "cors",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.logging_config_xml.is_empty()
&& let Err(e) = deserialize::<BucketLoggingStatus>(&self.logging_config_xml).map(|c| self.logging_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "logging", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "logging",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.website_config_xml.is_empty()
&& let Err(e) = deserialize::<WebsiteConfiguration>(&self.website_config_xml).map(|c| self.website_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "website", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "website",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.accelerate_config_xml.is_empty()
&& let Err(e) =
deserialize::<AccelerateConfiguration>(&self.accelerate_config_xml).map(|c| self.accelerate_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "accelerate", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "accelerate",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.request_payment_config_xml.is_empty()
&& let Err(e) = deserialize::<RequestPaymentConfiguration>(&self.request_payment_config_xml)
.map(|c| self.request_payment_config = Some(c))
{
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "request_payment",
error = %e,
"parse_all_configs: failed to parse"
"Failed to parse bucket metadata config"
);
}
if !self.public_access_block_config_xml.is_empty()
&& let Err(e) = deserialize::<PublicAccessBlockConfiguration>(&self.public_access_block_config_xml)
.map(|c| self.public_access_block_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "public_access_block", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "public_access_block",
error = %e,
"Failed to parse bucket metadata config"
);
}
if !self.bucket_acl_config_json.is_empty()
&& let Err(e) = String::from_utf8(self.bucket_acl_config_json.clone()).map(|acl| self.bucket_acl_config = Some(acl))
{
tracing::warn!(bucket = %self.name, config = "bucket_acl", error = %e, "parse_all_configs: failed to parse");
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "bucket_acl",
error = %e,
"Failed to parse bucket metadata config"
);
}
Ok(())
crates/ecstore/src/bucket/replication/replication_pool.rs
+100 -16
View File
@@ -56,7 +56,15 @@ use tokio::sync::mpsc::Sender;
use tokio::task::JoinHandle;
use tokio::time::Duration;
use tokio_util::sync::CancellationToken;
use tracing::{info, instrument, warn};
use tracing::{debug, info, instrument};
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_REPLICATION: &str = "replication";
const EVENT_REPLICATION_WORKER_RESIZE_SKIPPED: &str = "replication_worker_resize_skipped";
const EVENT_REPLICATION_WORKER_RESIZED: &str = "replication_worker_resized";
const EVENT_REPLICATION_BACKPRESSURE: &str = "replication_backpressure";
const EVENT_REPLICATION_RESYNC_LOAD_SKIPPED: &str = "replication_resync_load_skipped";
const EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED: &str = "replication_config_lookup_skipped";
// Worker limits
pub const WORKER_MAX_LIMIT: usize = 500;
@@ -358,18 +366,31 @@ impl<S: StorageAPI> ReplicationPool<S> {
let mut workers = self.workers.write().await;
if (check_old > 0 && workers.len() != check_old) || n == workers.len() || n < 1 {
warn!(
"resize_workers: skipping resize - check_old_mismatch={}, same_size={}, invalid_n={}",
check_old > 0 && workers.len() != check_old,
n == workers.len(),
n < 1
debug!(
event = EVENT_REPLICATION_WORKER_RESIZE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
check_old_mismatch = check_old > 0 && workers.len() != check_old,
same_size = n == workers.len(),
invalid_target_size = n < 1,
current_workers = workers.len(),
target_workers = n,
"Skipped replication worker resize"
);
return;
}
// Add workers if needed
if workers.len() < n {
info!("resize_workers: adding workers from {} to {}", workers.len(), n);
info!(
event = EVENT_REPLICATION_WORKER_RESIZED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
action = "increase",
from_workers = workers.len(),
to_workers = n,
"Resized replication workers"
);
}
while workers.len() < n {
@@ -416,7 +437,15 @@ impl<S: StorageAPI> ReplicationPool<S> {
// Remove workers if needed
if workers.len() > n {
warn!("resize_workers: removing workers from {} to {}", workers.len(), n);
info!(
event = EVENT_REPLICATION_WORKER_RESIZED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
action = "decrease",
from_workers = workers.len(),
to_workers = n,
"Resized replication workers"
);
}
while workers.len() > n {
@@ -607,11 +636,26 @@ impl<S: StorageAPI> ReplicationPool<S> {
match priority {
ReplicationPriority::Fast => {
// Log warning about unable to keep up
info!("Warning: Unable to keep up with incoming traffic");
debug!(
event = EVENT_REPLICATION_BACKPRESSURE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
queue_type = "object",
priority = "fast",
recommendation = "none",
"Replication queue is backpressured"
);
}
ReplicationPriority::Slow => {
info!("Warning: Unable to keep up with incoming traffic - recommend increasing replication priority to auto");
debug!(
event = EVENT_REPLICATION_BACKPRESSURE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
queue_type = "object",
priority = "slow",
recommendation = "set_priority_auto",
"Replication queue is backpressured"
);
}
ReplicationPriority::Auto => {
let max_w = std::cmp::min(max_workers, WORKER_MAX_LIMIT);
@@ -667,10 +711,26 @@ impl<S: StorageAPI> ReplicationPool<S> {
match priority {
ReplicationPriority::Fast => {
info!("Warning: Unable to keep up with incoming deletes");
debug!(
event = EVENT_REPLICATION_BACKPRESSURE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
queue_type = "delete",
priority = "fast",
recommendation = "none",
"Replication delete queue is backpressured"
);
}
ReplicationPriority::Slow => {
info!("Warning: Unable to keep up with incoming deletes - recommend increasing replication priority to auto");
debug!(
event = EVENT_REPLICATION_BACKPRESSURE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
queue_type = "delete",
priority = "slow",
recommendation = "set_priority_auto",
"Replication delete queue is backpressured"
);
}
ReplicationPriority::Auto => {
let max_w = std::cmp::min(max_workers, WORKER_MAX_LIMIT);
@@ -930,7 +990,15 @@ impl<S: StorageAPI> ReplicationPool<S> {
Ok(meta) => meta,
Err(err) => {
if !matches!(err, EcstoreError::VolumeNotFound) {
warn!("Error loading resync metadata for bucket {bucket}: {err:?}");
debug!(
event = EVENT_REPLICATION_RESYNC_LOAD_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
bucket,
error = ?err,
reason = "metadata_load_failed",
"Skipped replication resync metadata load"
);
}
continue;
}
@@ -1179,7 +1247,15 @@ pub async fn queue_replication_heal(bucket: &str, oi: ObjectInfo, retry_count: u
let rcfg = match metadata_sys::get_replication_config(bucket).await {
Ok((config, _)) => config,
Err(err) => {
warn!("Failed to get replication config for bucket {}: {}", bucket, err);
debug!(
event = EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
bucket,
error = %err,
reason = "config_lookup_failed",
"Skipped replication heal queue due to missing replication config"
);
return;
}
@@ -1188,7 +1264,15 @@ pub async fn queue_replication_heal(bucket: &str, oi: ObjectInfo, retry_count: u
let tgts = match BucketTargetSys::get().list_bucket_targets(bucket).await {
Ok(targets) => Some(targets),
Err(err) => {
warn!("Failed to list bucket targets for bucket {}: {}", bucket, err);
debug!(
event = EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
bucket,
error = %err,
reason = "target_list_failed",
"Skipped bucket target list during replication heal queue setup"
);
None
}
};
crates/ecstore/src/bucket/replication/replication_resyncer.rs
+262 -49
View File
@@ -85,9 +85,18 @@ use tokio::task::JoinSet;
use tokio::time::Duration as TokioDuration;
use tokio_util::io::ReaderStream;
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, instrument, warn};
use tracing::{debug, error, instrument, warn};
use uuid::Uuid;
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_REPLICATION_RESYNC: &str = "replication_resync";
const EVENT_RESYNC_STATUS_UPDATE_SKIPPED: &str = "replication_resync_status_update_skipped";
const EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED: &str = "replication_resync_config_lookup_skipped";
const EVENT_RESYNC_OBJECT_PROCESSED: &str = "replication_resync_object_processed";
const EVENT_RESYNC_RUNTIME_SKIPPED: &str = "replication_resync_runtime_skipped";
const EVENT_REPLICATION_DELETE_SKIPPED: &str = "replication_delete_skipped";
const EVENT_REPLICATION_FORCE_DELETE_SKIPPED: &str = "replication_force_delete_skipped";
pub(crate) const REPLICATION_DIR: &str = ".replication";
pub(crate) const RESYNC_FILE_NAME: &str = "resync.bin";
pub(crate) const RESYNC_META_FORMAT: u16 = 1;
@@ -508,12 +517,16 @@ impl ReplicationResyncer {
};
if !resync_state_accepts_update(state, &opts) {
warn!(
debug!(
event = EVENT_RESYNC_STATUS_UPDATE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %opts.bucket,
arn = %opts.arn,
incoming_resync_id = %opts.resync_id,
current_resync_id = %state.resync_id,
"ignoring stale resync status update"
reason = "stale_status_update",
"Skipped stale resync status update"
);
return Ok(());
}
@@ -565,12 +578,16 @@ impl ReplicationResyncer {
};
if !resync_state_accepts_update(state, &opts) {
warn!(
debug!(
event = EVENT_RESYNC_STATUS_UPDATE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %opts.bucket,
arn = %opts.arn,
incoming_resync_id = %opts.resync_id,
current_resync_id = %state.resync_id,
"ignoring stale resync stats update"
reason = "stale_stats_update",
"Skipped stale resync stats update"
);
return;
}
@@ -676,7 +693,15 @@ impl ReplicationResyncer {
let targets = match BucketTargetSys::get().list_bucket_targets(&opts.bucket).await {
Ok(targets) => targets,
Err(err) => {
warn!("Failed to list bucket targets: {}", err);
debug!(
event = EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %opts.bucket,
error = %err,
reason = "target_list_failed",
"Failed to list bucket targets during resync"
);
self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone())
.await;
return;
@@ -847,14 +872,17 @@ impl ReplicationResyncer {
(roi.size, None)
};
info!(
"resynced reset_id:{} object: {}/{}-{} size:{} err:{:?}",
reset_id,
bucket_name,
roi.name,
roi.version_id.unwrap_or_default(),
debug!(
event = EVENT_RESYNC_OBJECT_PROCESSED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
reset_id = %reset_id,
bucket = %bucket_name,
object = %roi.name,
version_id = %roi.version_id.unwrap_or_default(),
size,
err,
error = ?err,
"Processed resync object"
);
if cancel_token.is_cancelled() {
@@ -1548,7 +1576,14 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
let _rcfg = match get_replication_config(&bucket).await {
Ok(Some(config)) => config,
Ok(None) => {
warn!("No replication config found for bucket: {}", bucket);
debug!(
event = EVENT_REPLICATION_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
reason = "replication_config_missing",
"Skipping replication delete because replication config is missing"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -1567,7 +1602,15 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
return;
}
Err(err) => {
warn!("replication config for bucket: {} error: {}", bucket, err);
debug!(
event = EVENT_REPLICATION_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
error = %err,
reason = "replication_config_lookup_failed",
"Skipping replication delete because replication config lookup failed"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -1605,30 +1648,42 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
match source_marker_state {
Ok(info) if info.delete_marker && info.version_id == Some(delete_marker_version_id) => {}
Ok(_) => {
warn!(
debug!(
event = EVENT_REPLICATION_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket,
object = dobj.delete_object.object_name,
version_id = %delete_marker_version_id,
"skipping stale delete-marker replication because source version is no longer a delete marker"
reason = "source_not_delete_marker",
"Skipping stale delete-marker replication"
);
return;
}
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {
warn!(
debug!(
event = EVENT_REPLICATION_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket,
object = dobj.delete_object.object_name,
version_id = %delete_marker_version_id,
"skipping stale delete-marker replication because source version no longer exists"
reason = "source_version_missing",
"Skipping stale delete-marker replication"
);
return;
}
Err(err) => {
warn!(
debug!(
event = EVENT_REPLICATION_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket,
object = dobj.delete_object.object_name,
version_id = %delete_marker_version_id,
error = %err,
"failed to verify source delete-marker state before replication"
reason = "source_state_verification_failed",
"Failed to verify source delete-marker state before replication"
);
}
}
@@ -1645,9 +1700,15 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
) {
Ok(dsc) => dsc,
Err(err) => {
warn!(
"failed to parse replicate decision for bucket:{} arn:{} error:{}",
bucket, dobj.target_arn, err
debug!(
event = EVENT_REPLICATION_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %dobj.target_arn,
error = %err,
reason = "replicate_decision_parse_failed",
"Failed to parse replicate decision"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
@@ -1672,9 +1733,15 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
{
Ok(ns_lock) => ns_lock,
Err(e) => {
warn!(
"failed to get ns lock for bucket:{} object:{} error:{}",
bucket, dobj.delete_object.object_name, e
debug!(
event = EVENT_REPLICATION_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
object = %dobj.delete_object.object_name,
error = %e,
reason = "ns_lock_unavailable",
"Skipping replication delete"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
@@ -1697,9 +1764,15 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
let _lock_guard = match ns_lock.get_write_lock(get_lock_acquire_timeout()).await {
Ok(lock_guard) => lock_guard,
Err(e) => {
warn!(
"failed to get write lock for bucket:{} object:{} error:{}",
bucket, dobj.delete_object.object_name, e
debug!(
event = EVENT_REPLICATION_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
object = %dobj.delete_object.object_name,
error = %e,
reason = "write_lock_unavailable",
"Skipping replication delete"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
@@ -1741,7 +1814,15 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
// Get the remote target client
let Some(tgt_client) = BucketTargetSys::get().get_remote_target_client(&bucket, &tgt_entry.arn).await else {
warn!("failed to get target for bucket:{:?}, arn:{:?}", &bucket, &tgt_entry.arn);
debug!(
event = EVENT_REPLICATION_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_entry.arn,
reason = "target_client_missing",
"Skipping replication delete because target client is unavailable"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -1969,7 +2050,14 @@ async fn replicate_force_delete_to_targets<S: StorageAPI>(dobj: &DeletedObjectRe
let rcfg = match get_replication_config(bucket).await {
Ok(Some(config)) => config,
Ok(None) => {
warn!("replicate force-delete: no replication config for bucket:{}", bucket);
debug!(
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
reason = "replication_config_missing",
"Skipping replication force-delete because replication config is missing"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -1985,7 +2073,15 @@ async fn replicate_force_delete_to_targets<S: StorageAPI>(dobj: &DeletedObjectRe
return;
}
Err(err) => {
warn!("replicate force-delete: replication config error bucket:{} error:{}", bucket, err);
debug!(
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
error = %err,
reason = "replication_config_lookup_failed",
"Skipping replication force-delete because replication config lookup failed"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -2064,7 +2160,15 @@ async fn replicate_force_delete_to_targets<S: StorageAPI>(dobj: &DeletedObjectRe
for arn in tgt_arns {
let Some(tgt_client) = BucketTargetSys::get().get_remote_target_client(bucket, &arn).await else {
warn!("replicate force-delete: failed to get target client bucket:{} arn:{}", bucket, arn);
debug!(
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %arn,
reason = "target_client_missing",
"Skipping replication force-delete because target client is unavailable"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -2300,7 +2404,14 @@ pub async fn replicate_object<S: StorageAPI>(roi: ReplicateObjectInfo, storage:
let cfg = match get_replication_config(&bucket).await {
Ok(Some(config)) => config,
Ok(None) => {
warn!("No replication config found for bucket: {}", bucket);
debug!(
event = EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
reason = "replication_config_missing",
"Skipping replication object because replication config is missing"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -2338,7 +2449,15 @@ pub async fn replicate_object<S: StorageAPI>(roi: ReplicateObjectInfo, storage:
for arn in tgt_arns {
let Some(tgt_client) = BucketTargetSys::get().get_remote_target_client(&bucket, &arn).await else {
warn!("failed to get target for bucket:{:?}, arn:{:?}", &bucket, &arn);
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %arn,
reason = "target_client_missing",
"Skipping replication object target"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -2479,7 +2598,16 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
}
if BucketTargetSys::get().is_offline(&tgt_client.to_url()).await {
warn!("target is offline: {}", tgt_client.to_url());
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
reason = "target_offline",
endpoint = %tgt_client.to_url(),
"Skipping replication object target"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -2509,7 +2637,16 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
Ok(gr) => gr,
Err(e) => {
if !is_err_object_not_found(&e) || is_err_version_not_found(&e) {
warn!("failed to get object reader for bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e);
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
error = %e,
reason = "object_reader_unavailable",
"Skipping replication object target"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
@@ -2532,7 +2669,16 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
let size = match object_info.get_actual_size() {
Ok(size) => size,
Err(e) => {
warn!("failed to get actual size for bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e);
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
error = %e,
reason = "actual_size_unavailable",
"Skipping replication object target"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -2546,7 +2692,15 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
};
if tgt_client.bucket.is_empty() {
warn!("target bucket is empty: {}", tgt_client.bucket);
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
reason = "target_bucket_empty",
"Skipping replication object target"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -2702,7 +2856,16 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
};
if BucketTargetSys::get().is_offline(&tgt_client.to_url()).await {
warn!("target is offline: {}", tgt_client.to_url());
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
target = %tgt_client.to_url(),
reason = "target_offline",
"Skipped replication because target is offline"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -2732,7 +2895,16 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
Ok(gr) => gr,
Err(e) => {
if !is_err_object_not_found(&e) || is_err_version_not_found(&e) {
warn!("failed to get object reader for bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e);
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
error = %e,
reason = "object_reader_unavailable",
"Skipped replication because object reader is unavailable"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -2763,7 +2935,16 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
let size = match object_info.get_actual_size() {
Ok(size) => size,
Err(e) => {
warn!("failed to get actual size for bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e);
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
error = %e,
reason = "actual_size_unavailable",
"Skipped replication because actual object size is unavailable"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -2779,7 +2960,15 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
// TODO: SSE
if tgt_client.bucket.is_empty() {
warn!("target bucket is empty: {}", tgt_client.bucket);
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
reason = "target_bucket_empty",
"Skipped replication because target bucket is empty"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
@@ -2801,7 +2990,16 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
};
if let Err(err) = sopts.set(AMZ_TAGGING_DIRECTIVE, "ACCESS") {
warn!("failed to set replication tagging directive header: {err}");
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
error = %err,
reason = "tagging_directive_header_invalid",
"Skipped replication tagging directive header detail"
);
}
match head_object_with_proxy_stats(
@@ -2877,9 +3075,15 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
}
Err(e2) => {
rinfo.error = Some(e2.to_string());
warn!(
"replication head_object fallback failed bucket:{} arn:{} error:{}",
bucket, tgt_client.arn, e2
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
error = %e2,
reason = "head_object_fallback_failed",
"Failed replication head-object fallback"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
@@ -2897,7 +3101,16 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
replication_action = ReplicationAction::All;
} else {
rinfo.error = Some(e.to_string());
warn!("failed to head object for bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e);
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
error = %e,
reason = "head_object_failed",
"Skipped replication because head-object failed"
);
send_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
crates/ecstore/src/client/object_handlers_common.rs
+31 -5
View File
@@ -13,7 +13,12 @@
// limitations under the License.
use std::sync::Arc;
use tracing::warn;
use tracing::debug;
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
const EVENT_LIFECYCLE_CLEANUP_SKIPPED: &str = "lifecycle_cleanup_skipped";
const EVENT_LIFECYCLE_CLEANUP_FAILED: &str = "lifecycle_cleanup_failed";
use crate::bucket::lifecycle::lifecycle;
use crate::bucket::replication::{DeletedObjectReplicationInfo, check_replicate_delete, schedule_replication_delete};
@@ -40,7 +45,15 @@ pub async fn delete_object_versions(api: &Arc<ECStore>, bucket: &str, to_del: &[
let version_suspended = match BucketVersioningSys::get(bucket).await {
Ok(vc) => vc.suspended(),
Err(err) => {
warn!(bucket, error = ?err, "failed to get versioning config during lifecycle noncurrent version cleanup");
debug!(
event = EVENT_LIFECYCLE_CLEANUP_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket,
error = ?err,
reason = "versioning_config_unavailable",
"Skipped lifecycle noncurrent version cleanup"
);
return;
}
};
@@ -70,12 +83,16 @@ pub async fn delete_object_versions(api: &Arc<ECStore>, bucket: &str, to_del: &[
.then(|| lifecycle_version_delete_replication_state(dsc.to_string(), dsc.pending_status()))
}
Err(err) => {
warn!(
debug!(
event = EVENT_LIFECYCLE_CLEANUP_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket,
object = %object.object_name,
version_id = ?version_id,
error = ?err,
"failed to get object info during lifecycle noncurrent version cleanup; skipping delete replication scheduling"
reason = "object_info_unavailable",
"Skipped lifecycle delete replication scheduling"
);
None
}
@@ -119,7 +136,16 @@ pub async fn delete_object_versions(api: &Arc<ECStore>, bucket: &str, to_del: &[
.and_then(|o| o.version_id)
.map(|v| v.to_string())
.unwrap_or_default();
warn!(bucket, object = obj_name, version_id = %vid, error = ?e, "failed to delete noncurrent version during lifecycle cleanup");
debug!(
event = EVENT_LIFECYCLE_CLEANUP_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket,
object = obj_name,
version_id = %vid,
error = ?e,
"Failed lifecycle noncurrent version cleanup"
);
}
}
if remaining.is_empty() {
crates/ecstore/src/rebalance.rs
+409 -65
View File
@@ -39,7 +39,14 @@ use std::sync::Arc;
use time::OffsetDateTime;
use tokio::time::{Duration, Instant};
use tokio_util::sync::CancellationToken;
use tracing::{error, info, warn};
use tracing::{debug, error, info, warn};
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_REBALANCE: &str = "rebalance";
const EVENT_REBALANCE_STATE: &str = "rebalance_state";
const EVENT_REBALANCE_BUCKET: &str = "rebalance_bucket";
const EVENT_REBALANCE_ENTRY: &str = "rebalance_entry";
const EVENT_REBALANCE_LISTING: &str = "rebalance_listing";
use uuid::Uuid;
const REBAL_META_FMT: u16 = 1; // Replace with actual format value
@@ -573,7 +580,13 @@ impl RebalanceMeta {
pub async fn load_with_opts<S: ObjectIO>(&mut self, store: Arc<S>, opts: ObjectOptions) -> Result<()> {
let (data, _) = read_config_with_metadata(store, REBAL_META_NAME, &opts).await?;
if data.is_empty() {
info!("rebalanceMeta load_with_opts: no data");
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "metadata_empty",
"Rebalance metadata is empty"
);
return Ok(());
}
if data.len() <= 4 {
@@ -595,7 +608,13 @@ impl RebalanceMeta {
self.last_refreshed_at = Some(OffsetDateTime::now_utc());
info!("rebalanceMeta load_with_opts: loaded meta done");
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "metadata_loaded",
"Loaded rebalance metadata"
);
Ok(())
}
@@ -605,7 +624,14 @@ impl RebalanceMeta {
pub async fn save_with_opts<S: ObjectIO>(&self, store: Arc<S>, opts: ObjectOptions) -> Result<()> {
if self.pool_stats.is_empty() {
info!("rebalanceMeta save_with_opts: no pool stats");
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "metadata_save_skipped",
reason = "no_pool_stats",
"Skipped rebalance metadata save"
);
return Ok(());
}
@@ -658,10 +684,15 @@ impl ECStore {
#[tracing::instrument(skip_all)]
pub async fn load_rebalance_meta(&self) -> Result<()> {
let mut meta = RebalanceMeta::new();
info!("rebalanceMeta: store load rebalance meta");
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "metadata_loading",
"Loading rebalance metadata"
);
let pool = clone_first_arc(&self.pools, "rebalanceMeta: no pools available")?;
if resolve_rebalance_meta_load_result(meta.load(pool).await)? {
info!("rebalanceMeta: rebalance meta loaded0");
{
let mut rebalance_meta = self.rebalance_meta.write().await;
@@ -670,12 +701,23 @@ impl ECStore {
drop(rebalance_meta);
}
info!("rebalanceMeta: rebalance meta loaded1");
resolve_load_rebalance_stats_update_result(self.update_rebalance_stats().await)?;
info!("rebalanceMeta: rebalance meta loaded2");
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "metadata_loaded",
"Loaded rebalance metadata"
);
} else {
info!("rebalanceMeta: not found, rebalance not started");
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "metadata_missing",
reason = "rebalance_not_started",
"Rebalance metadata not found"
);
}
Ok(())
@@ -690,13 +732,25 @@ impl ECStore {
clone_rebalance_pool_stats(rebalance_meta.as_ref())?
};
info!("update_rebalance_stats: pool_stats: {:?}", &pool_stats);
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_count = pool_stats.len(),
"Refreshing rebalance stats snapshot"
);
for i in 0..self.pools.len() {
if pool_stats.get(i).is_none() {
info!("update_rebalance_stats: pool {} not found", i);
let mut rebalance_meta = self.rebalance_meta.write().await;
info!("update_rebalance_stats: pool {} not found, add", i);
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index = i,
state = "pool_stat_missing",
"Adding missing rebalance pool stats entry"
);
if let Some(meta) = rebalance_meta.as_mut() {
meta.pool_stats.push(RebalanceStats::default());
}
@@ -706,7 +760,13 @@ impl ECStore {
}
if ok {
info!("update_rebalance_stats: save rebalance meta");
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "metadata_saving",
"Saving rebalance metadata after stats refresh"
);
let rebalance_meta = self.rebalance_meta.read().await;
if let Some(meta) = rebalance_meta.as_ref() {
@@ -732,7 +792,14 @@ impl ECStore {
#[tracing::instrument(skip(self))]
pub async fn init_rebalance_meta(&self, bucktes: Vec<String>) -> Result<String> {
info!("init_rebalance_meta: start rebalance");
info!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "initializing",
bucket_count = bucktes.len(),
"Initializing rebalance metadata"
);
let si = StorageAdminApi::storage_info(self).await;
let mut disk_stats = vec![DiskStat::default(); self.pools.len()];
@@ -924,7 +991,13 @@ impl ECStore {
#[tracing::instrument(skip_all)]
pub async fn start_rebalance(self: &Arc<Self>) -> Result<()> {
info!("start_rebalance: start rebalance");
info!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "starting",
"Starting rebalance"
);
let decommission_running = self.is_decommission_running().await;
// let rebalance_meta = self.rebalance_meta.read().await;
@@ -940,7 +1013,14 @@ impl ECStore {
return Err(Error::ConfigNotFound);
};
if should_skip_start_rebalance(meta.cancel.is_some(), is_rebalance_in_progress(meta)) {
info!("start_rebalance: already in progress, skip duplicate start");
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "start_skipped",
reason = "already_in_progress",
"Skipped duplicate rebalance start"
);
return Ok(());
}
if complete_rebalance_pools_at_goal(meta, OffsetDateTime::now_utc()) {
@@ -963,24 +1043,45 @@ impl ECStore {
let participants = if let Some(ref meta) = *self.rebalance_meta.read().await {
resolve_rebalance_participants(meta.pool_stats.as_slice(), self.pools.len())
} else {
info!("start_rebalance:2 rebalance_meta is None exit");
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "start_skipped",
reason = "metadata_missing",
"Skipped rebalance start because metadata is unavailable"
);
Vec::new()
};
for (idx, participating) in participants.iter().enumerate() {
if !*participating {
info!("start_rebalance: pool {} is not participating, skipping", idx);
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index = idx,
state = "pool_skipped",
reason = "not_participating",
"Skipped rebalance pool"
);
continue;
}
if !get_global_endpoints().as_ref().get(idx).is_some_and(|v| {
info!("start_rebalance: pool {} endpoints: {:?}", idx, v.endpoints);
v.endpoints.as_ref().first().is_some_and(|e| {
info!("start_rebalance: pool {} endpoint: {:?}, is_local: {}", idx, e, e.is_local);
e.is_local
})
}) {
info!("start_rebalance: pool {} is not local, skipping", idx);
if !get_global_endpoints()
.as_ref()
.get(idx)
.is_some_and(|v| v.endpoints.as_ref().first().is_some_and(|e| e.is_local))
{
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index = idx,
state = "pool_skipped",
reason = "not_local",
"Skipped non-local rebalance pool"
);
continue;
}
@@ -991,12 +1092,25 @@ impl ECStore {
if let Err(err) = store.rebalance_buckets(rx_clone, pool_idx).await {
error!("Rebalance failed for pool {}: {}", pool_idx, err);
} else {
info!("Rebalance completed for pool {}", pool_idx);
info!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index = pool_idx,
state = "completed",
"Completed rebalance pool"
);
}
});
}
info!("start_rebalance: rebalance started done");
info!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
state = "started",
"Started rebalance"
);
Ok(())
}
@@ -1029,9 +1143,13 @@ impl ECStore {
pool_stat.info.status,
&terminal_event,
) {
info!(
"rebalance_buckets: preserving stopped status for pool {}",
pool_index
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "stopped_preserved",
"Preserved stopped rebalance status"
);
} else {
apply_rebalance_terminal_event(
@@ -1058,11 +1176,27 @@ impl ECStore {
return Err(wrapped);
}
} else {
info!(msg);
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "metadata_saved",
message = %msg,
"Saved rebalance metadata"
);
}
if quit {
info!("{}: exiting save_task", msg);
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "save_task_exiting",
message = %msg,
"Exiting rebalance save task"
);
return Ok(());
}
@@ -1070,13 +1204,28 @@ impl ECStore {
}
});
info!("Pool {} rebalancing is started", pool_index);
info!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "pool_started",
"Started rebalance worker"
);
let mut final_result: Result<()> = Ok(());
let mut deferred_buckets = HashSet::new();
loop {
if rx.is_cancelled() {
info!("Pool {} rebalancing is stopped", pool_index);
info!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "pool_stopped",
reason = "cancelled",
"Stopped rebalance worker"
);
let err = Error::OperationCanceled;
final_result = Err(resolve_rebalance_terminal_error(
err.clone(),
@@ -1098,7 +1247,15 @@ impl ECStore {
};
if let Some(bucket) = next_bucket {
info!("Rebalancing bucket: start {}", bucket);
debug!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
state = "started",
"Starting rebalance bucket"
);
let outcome = match resolve_rebalance_bucket_result(
self.rebalance_bucket(rx.clone(), bucket.clone(), pool_index).await,
@@ -1129,7 +1286,16 @@ impl ECStore {
break;
}
warn!("Rebalance bucket deferred due to transient object failures: {bucket}: {last_error}");
warn!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
state = "deferred",
error = %last_error,
"Deferred rebalance bucket after transient object failures"
);
if let Err(err) = self.defer_rebalance_bucket(pool_index, bucket.clone(), last_error).await {
error!("defer_rebalance_bucket failed for pool {}: {:?}", pool_index, err);
final_result = Err(resolve_rebalance_terminal_error(
@@ -1141,7 +1307,15 @@ impl ECStore {
continue;
}
info!("Rebalance bucket: done {} ", bucket);
debug!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
state = "completed",
"Completed rebalance bucket"
);
if let Err(err) = self.bucket_rebalance_done(pool_index, bucket).await {
error!("bucket_rebalance_done failed for pool {}: {:?}", pool_index, err);
final_result = Err(resolve_rebalance_terminal_error(
@@ -1151,12 +1325,27 @@ impl ECStore {
break;
}
} else {
info!("Rebalance bucket: no bucket to rebalance");
debug!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "idle",
reason = "no_bucket_to_rebalance",
"No rebalance bucket available"
);
break;
}
}
info!("Pool {} rebalancing is done", pool_index);
info!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "pool_done",
"Finished rebalance worker"
);
if final_result.is_ok()
&& let Err(err) = send_rebalance_done_signal(&done_tx, Ok(()), pool_index).await
@@ -1181,7 +1370,14 @@ impl ECStore {
{
// Check if the pool's rebalance status is already completed
if pool_stat.info.status == RebalStatus::Completed {
info!("check_if_rebalance_done: pool {} is already completed", pool_index);
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "already_completed",
"Rebalance pool is already completed"
);
return true;
}
@@ -1202,7 +1398,15 @@ impl ECStore {
{
pool_stat.info.status = RebalStatus::Completed;
pool_stat.info.end_time = Some(OffsetDateTime::now_utc());
info!("check_if_rebalance_done: pool {} is completed, pfi: {}", pool_index, pfi);
info!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "completed",
percent_free = pfi,
"Marked rebalance pool completed"
);
return true;
}
}
@@ -1271,21 +1475,53 @@ fn resolve_next_rebalance_bucket(meta: Option<&RebalanceMeta>, pool_index: usize
};
if pool_stat.info.status == RebalStatus::Completed || !pool_stat.participating {
info!("next_rebal_bucket: pool_index: {} completed or not participating", pool_index);
debug!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "unavailable",
reason = "completed_or_not_participating",
"No rebalance bucket available"
);
return Ok(None);
}
if pool_stat.buckets.is_empty() {
info!("next_rebal_bucket: pool_index: {} buckets is empty", pool_index);
debug!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "unavailable",
reason = "bucket_queue_empty",
"No rebalance bucket available"
);
return Ok(None);
}
if let Some(bucket) = next_rebal_bucket_from_stat(pool_stat) {
info!("next_rebal_bucket: pool_index: {} bucket: {}", pool_index, bucket);
debug!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
state = "selected",
"Selected rebalance bucket"
);
return Ok(Some(bucket));
}
info!("next_rebal_bucket: pool_index: {} None", pool_index);
debug!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "unavailable",
reason = "selection_returned_none",
"No rebalance bucket available"
);
Ok(None)
}
@@ -1299,10 +1535,17 @@ fn mark_rebalance_bucket_done(meta: Option<&mut RebalanceMeta>, pool_index: usiz
return Err(invalid_rebalance_pool_index_error(pool_index, meta.pool_stats.len()));
};
info!("bucket_rebalance_done: buckets {:?}", &pool_stat.buckets);
if take_bucket_from_rebalance_queue(pool_stat, bucket) {
info!("bucket_rebalance_done: bucket {} rebalanced", bucket);
debug!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
state = "queue_removed",
remaining_bucket_count = pool_stat.buckets.len(),
"Removed bucket from rebalance queue"
);
if has_deferred_rebalance_error(pool_stat) {
pool_stat.info.last_error = None;
}
@@ -2036,7 +2279,16 @@ impl ECStore {
bucket_configs: Arc<RebalanceBucketConfigs>,
// wk: Arc<Workers>,
) -> Result<RebalanceEntryOutcome> {
info!("rebalance_entry: start rebalance_entry");
debug!(
event = EVENT_REBALANCE_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
bucket = %bucket,
object = %entry.name,
pool_index,
state = "started",
"Starting rebalance entry"
);
// defer!(|| async {
// warn!("rebalance_entry: defer give worker start");
@@ -2045,12 +2297,32 @@ impl ECStore {
// });
if entry.is_dir() {
info!("rebalance_entry: entry is dir, skipping");
debug!(
event = EVENT_REBALANCE_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
bucket = %bucket,
object = %entry.name,
pool_index,
state = "skipped",
reason = "directory_entry",
"Skipped rebalance entry"
);
return Ok(RebalanceEntryOutcome::Completed);
}
if self.check_if_rebalance_done(pool_index).await {
info!("rebalance_entry: rebalance done, skipping pool {}", pool_index);
debug!(
event = EVENT_REBALANCE_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
object = %entry.name,
state = "skipped",
reason = "pool_completed",
"Skipped rebalance entry"
);
return Ok(RebalanceEntryOutcome::Completed);
}
@@ -2077,16 +2349,33 @@ impl ECStore {
.await
{
expired += 1;
info!("rebalance_entry {} Entry {} expired by lifecycle, skipping", &bucket, version.name);
debug!(
event = EVENT_REBALANCE_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
object = %version.name,
state = "skipped",
reason = "expired_by_lifecycle",
"Skipped rebalance version"
);
continue;
}
let remaining_versions = fivs.versions.len() - expired;
if should_skip_rebalance_delete_marker(version, remaining_versions, bucket_configs.replication_config.is_some()) {
rebalanced += 1;
info!(
"rebalance_entry Entry {} is deleted and last version without replication, skipping",
version.name
debug!(
event = EVENT_REBALANCE_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
object = %version.name,
state = "skipped",
reason = "last_delete_marker_without_replication",
"Skipped rebalance version"
);
continue;
}
@@ -2112,7 +2401,17 @@ impl ECStore {
if should_count_rebalance_version_complete(&result) {
rebalanced += 1;
}
info!("rebalance_entry {} Entry {} is already deleted, skipping", &bucket, version.name);
debug!(
event = EVENT_REBALANCE_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
object = %version.name,
state = "skipped",
reason = "already_deleted",
"Skipped rebalance version"
);
continue;
}
@@ -2131,8 +2430,15 @@ impl ECStore {
if should_defer_rebalance_entry_failure(&err) {
let deferred_error = format!("{REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX} {err}");
warn!(
"rebalance_entry {} deferring transient migration failure for {}/{:?}: {}",
&bucket, &version.name, &version.version_id, err
event = EVENT_REBALANCE_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
object = %version.name,
state = "deferred",
error = %err,
"Deferred rebalance entry after transient migration failure"
);
if let Err(stats_err) = self.update_rebalance_last_error(pool_index, deferred_error.clone()).await {
error!(
@@ -2192,7 +2498,16 @@ impl ECStore {
entry.name.as_str(),
)
.map_err(|err| with_rebalance_entry_context("cleanup_source", bucket.as_str(), entry.name.as_str(), err))?;
info!("rebalance_entry {} Entry {} deleted successfully", &bucket, &entry.name);
debug!(
event = EVENT_REBALANCE_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
object = %entry.name,
state = "source_deleted",
"Deleted rebalance source entry"
);
}
Ok(RebalanceEntryOutcome::Completed)
@@ -2288,7 +2603,14 @@ impl ECStore {
let task = tokio::spawn(async move {
let _permit = permit;
info!("rebalance_entry: rebalance entry task start");
debug!(
event = EVENT_REBALANCE_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
set_index = set_idx,
state = "task_started",
"Started rebalance entry task"
);
let result = this.rebalance_entry(bucket, pool_index, entry, set, bucket_configs).await;
if let Err(err) = &result {
error!("rebalance_entry: rebalance entry failed: {err}");
@@ -2298,7 +2620,14 @@ impl ECStore {
callback_rx.cancel();
}
}
info!("rebalance_entry: rebalance entry task done");
debug!(
event = EVENT_REBALANCE_ENTRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
set_index = set_idx,
state = "task_completed",
"Completed rebalance entry task"
);
result
});
@@ -2327,7 +2656,14 @@ impl ECStore {
if let Err(err) = &result {
error!("Rebalance worker {} error: {}", set_idx, err);
} else {
info!("Rebalance worker {} done", set_idx);
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
set_index = set_idx,
state = "worker_completed",
"Completed rebalance worker"
);
}
result
});
@@ -2355,7 +2691,15 @@ impl ECStore {
return Ok(RebalanceBucketOutcome::Deferred { last_error });
}
info!("rebalance_bucket: rebalance_bucket done");
debug!(
event = EVENT_REBALANCE_BUCKET,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
bucket = %bucket,
state = "completed",
"Finished rebalance bucket"
);
Ok(RebalanceBucketOutcome::Completed)
}
crates/ecstore/src/store/init.rs
+62 -12
View File
@@ -18,6 +18,14 @@ use crate::global::{
GLOBAL_EventNotifier, GLOBAL_LOCAL_DISK_ID_MAP, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, GLOBAL_TierConfigMgr,
get_global_bucket_monitor, is_dist_erasure, is_first_cluster_node_local,
};
use tracing::{debug, error, info, warn};
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_STORE_INIT: &str = "store_init";
const EVENT_DECOMMISSION_RESUME_RETRY: &str = "decommission_resume_retry";
const EVENT_DECOMMISSION_RESUME_FAILED: &str = "decommission_resume_failed";
const EVENT_STORE_FORMAT_RETRY: &str = "store_format_retry";
const EVENT_ECSTORE_INIT_STATUS: &str = "ecstore_init_status";
fn pool_first_endpoint_is_local(pool: &crate::endpoints::PoolEndpoints) -> bool {
pool.endpoints.as_ref().first().is_some_and(|endpoint| endpoint.is_local)
@@ -71,19 +79,27 @@ async fn resume_local_decommission_after_init(store: Arc<ECStore>, rx: Cancellat
.await
{
error!(
"store init failed to resume decommission workers for pools {:?}: {}",
pool_indices, spawn_err
event = EVENT_DECOMMISSION_RESUME_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_STORE_INIT,
pool_indices = ?pool_indices,
error = %spawn_err,
reason = "spawn_workers_failed",
"Failed to resume decommission workers"
);
}
return;
}
Err(err) if should_retry_local_decommission_resume(&err, attempt) => {
warn!(
"store init decommission resume missing config for pools {:?}, retry {}/{}: {}",
pool_indices,
attempt + 1,
LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES + 1,
err
event = EVENT_DECOMMISSION_RESUME_RETRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_STORE_INIT,
pool_indices = ?pool_indices,
retry_count = attempt + 1,
retry_limit = LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES + 1,
error = %err,
"Retrying decommission resume after missing config"
);
tokio::select! {
_ = rx.cancelled() => return,
@@ -91,7 +107,15 @@ async fn resume_local_decommission_after_init(store: Arc<ECStore>, rx: Cancellat
}
}
Err(err) => {
error!("store init failed to resume decommission for pools {:?}: {}", pool_indices, err);
error!(
event = EVENT_DECOMMISSION_RESUME_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_STORE_INIT,
pool_indices = ?pool_indices,
error = %err,
reason = "resume_failed",
"Failed to resume decommission"
);
return;
}
}
@@ -113,7 +137,13 @@ impl ECStore {
let mut local_disks = Vec::new();
info!("ECStore new address: {}", address.to_string());
debug!(
event = EVENT_ECSTORE_INIT_STATUS,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_STORE_INIT,
address = %address,
"Initializing ECStore address"
);
let mut host = address.ip().to_string();
if host.is_empty() {
host = GLOBAL_RUSTFS_HOST.read().await.to_string()
@@ -122,7 +152,14 @@ impl ECStore {
if port.is_empty() {
port = GLOBAL_RUSTFS_PORT.read().await.to_string()
}
info!("ECStore new host: {}, port: {}", host, port);
debug!(
event = EVENT_ECSTORE_INIT_STATUS,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_STORE_INIT,
host = %host,
port = %port,
"Initializing ECStore host"
);
init_local_peer(&endpoint_pools, &host, &port).await;
// debug!("endpoint_pools: {:?}", endpoint_pools);
@@ -179,10 +216,23 @@ impl ECStore {
if interval < 16 {
interval *= 2;
}
info!("retrying get formats after {:?}", interval);
debug!(
event = EVENT_STORE_FORMAT_RETRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_STORE_INIT,
retry_count = times,
retry_delay_secs = interval,
"Retrying storage format load"
);
select! {
_ = tokio::signal::ctrl_c() => {
info!("got ctrl+c, exits");
info!(
event = EVENT_STORE_FORMAT_RETRY,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_STORE_INIT,
reason = "ctrl_c",
"Interrupted storage format retry loop"
);
exit(0);
}
_ = sleep(Duration::from_secs(interval)) => {
crates/ecstore/src/store/peer.rs
+29 -3
View File
@@ -14,6 +14,12 @@
use super::*;
use crate::global::GLOBAL_LOCAL_DISK_ID_MAP;
use tracing::{debug, error};
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_DISK_STARTUP: &str = "disk_startup";
const EVENT_LOCAL_DISK_ID_PREWARM_SKIPPED: &str = "local_disk_id_prewarm_skipped";
const EVENT_LOCK_CLIENT_INITIALIZATION_FAILED: &str = "lock_client_initialization_failed";
async fn remember_local_disk_id(disk: &DiskStore) -> Option<Uuid> {
let disk_id = disk.get_disk_id().await.ok().flatten()?;
@@ -94,7 +100,14 @@ pub async fn all_local_disk() -> Vec<DiskStore> {
pub async fn prewarm_local_disk_id_map() {
for disk in all_local_disk().await {
if let Err(err) = disk.get_disk_id().await {
warn!("prewarm_local_disk_id_map: failed to load disk id for {}: {}", disk.endpoint(), err);
debug!(
event = EVENT_LOCAL_DISK_ID_PREWARM_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK_STARTUP,
disk_endpoint = %disk.endpoint(),
error = %err,
"Skipped local disk id prewarm"
);
continue;
}
@@ -159,7 +172,14 @@ pub fn init_lock_clients(endpoint_pools: EndpointServerPools) {
if !first_local_client_set {
if let Err(e) = crate::global::set_global_lock_client(local_client.clone()) {
// If already set, ignore the error (another thread may have set it)
warn!("set_global_lock_client error: {:?}", e);
debug!(
event = EVENT_LOCK_CLIENT_INITIALIZATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK_STARTUP,
error = ?e,
reason = "global_lock_client_already_set",
"Skipped global lock client publication"
);
} else {
first_local_client_set = true;
}
@@ -173,7 +193,13 @@ pub fn init_lock_clients(endpoint_pools: EndpointServerPools) {
// Store the lock clients map globally
if crate::global::set_global_lock_clients(clients).is_err() {
error!("init_lock_clients: error setting lock clients");
error!(
event = EVENT_LOCK_CLIENT_INITIALIZATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK_STARTUP,
reason = "set_global_lock_clients_failed",
"Failed to initialize lock clients"
);
}
}
crates/ecstore/src/tier/tier.rs
+24 -4
View File
@@ -1230,10 +1230,20 @@ where
return;
}
}
Err(err) => warn!("legacy tier config is incompatible, skip local migration: {}", err),
Err(err) => debug!(
bucket = RUSTFS_META_BUCKET,
path = %legacy_path,
error = %err,
"Skipping incompatible legacy tier config migration"
),
},
Ok(None) => {}
Err(err) => warn!("read legacy local tier config failed: {}", err),
Err(err) => debug!(
bucket = RUSTFS_META_BUCKET,
path = %legacy_path,
error = %err,
"Skipping legacy tier config migration after read failure"
),
}
match read_tier_config_from_bucket(api.clone(), MIGRATING_META_BUCKET, &target_path, &opts).await {
@@ -1243,10 +1253,20 @@ where
info!("Migrated compatible tier config from migrating metadata bucket");
}
}
Err(err) => warn!("migrating tier config is incompatible, skip migration: {}", err),
Err(err) => debug!(
bucket = MIGRATING_META_BUCKET,
path = %target_path,
error = %err,
"Skipping incompatible migrating tier config"
),
},
Ok(None) => {}
Err(err) => warn!("read migrating tier config failed: {}", err),
Err(err) => debug!(
bucket = MIGRATING_META_BUCKET,
path = %target_path,
error = %err,
"Skipping migrating tier config after read failure"
),
}
}