gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-13 16:46:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: preserve request ids across async recovery logs (#3451)

Browse Source 
* feat(obs): promote request ids in structured logs

* refactor(tracing): propagate spans into request tasks

* test(ecstore): baseline recovery monitor log chains

* fix(replication): reduce startup resync log noise

* chore(docs): stop tracking local recovery baseline

* chore(obs): polish request id logging cleanup
This commit is contained in:
houseme
2026-06-14 23:16:04 +08:00
committed by GitHub
parent 9499391370
commit efaf07d323
21 changed files with 1194 additions and 360 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/bucket/replication/replication_pool.rs
+63 -29
View File
@@ -64,9 +64,14 @@ const EVENT_REPLICATION_WORKER_RESIZE_SKIPPED: &str = "replication_worker_resize
const EVENT_REPLICATION_WORKER_RESIZED: &str = "replication_worker_resized";
const EVENT_REPLICATION_BACKPRESSURE: &str = "replication_backpressure";
const EVENT_REPLICATION_RESYNC_LOAD_SKIPPED: &str = "replication_resync_load_skipped";
const EVENT_REPLICATION_RESYNC_RECOVERED: &str = "replication_resync_recovered";
const EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED: &str = "replication_config_lookup_skipped";
const EVENT_REPLICATION_MRF_QUEUE_OVERFLOW: &str = "replication_mrf_queue_overflow";
fn should_auto_resume_resync(status: ResyncStatusType) -> bool {
matches!(status, ResyncStatusType::ResyncPending | ResyncStatusType::ResyncStarted)
}
// Worker limits
pub const WORKER_MAX_LIMIT: usize = 500;
pub const WORKER_MIN_LIMIT: usize = 50;
@@ -1017,6 +1022,11 @@ impl<S: StorageAPI + NamespaceLocking> ReplicationPool<S> {
// Note: Leader lock implementation would be needed here
// let _lock_guard = global_leader_lock.get_lock().await?;
let mut recovered_statuses = Vec::new();
let mut restart_opts = Vec::new();
let mut recovered_bucket_count = 0usize;
let mut skipped_failed_target_count = 0usize;
for bucket in buckets {
let meta = match load_bucket_resync_metadata(bucket, self.storage.clone()).await {
Ok(meta) => meta,
@@ -1036,39 +1046,53 @@ impl<S: StorageAPI + NamespaceLocking> ReplicationPool<S> {
}
};
// Store metadata in resyncer
{
let mut status_map = self.resyncer.status_map.write().await;
status_map.insert(bucket.clone(), meta.clone());
if meta.targets_map.is_empty() {
continue;
}
// Process target statistics
let target_stats = meta.clone_tgt_stats();
for (arn, stats) in target_stats {
match stats.resync_status {
ResyncStatusType::ResyncFailed | ResyncStatusType::ResyncStarted | ResyncStatusType::ResyncPending => {
// Note: This would spawn a resync task in a real implementation
// For now, we just log the resync request
let ctx = CancellationToken::new();
let bucket_clone = bucket.clone();
let resync = self.resyncer.clone();
let storage = self.storage.clone();
let opts = ResyncOpts {
bucket: bucket_clone,
arn,
resync_id: stats.resync_id,
resync_before: stats.resync_before_date,
};
tokio::spawn(async move {
resync.register_cancel_token(&opts, ctx.clone()).await;
Box::pin(resync.clone().resync_bucket(ctx, storage, true, opts.clone())).await;
resync.clear_cancel_token(&opts).await;
});
}
_ => {}
recovered_bucket_count += 1;
for (arn, stats) in &meta.targets_map {
if should_auto_resume_resync(stats.resync_status) {
restart_opts.push(ResyncOpts {
bucket: bucket.clone(),
arn: arn.clone(),
resync_id: stats.resync_id.clone(),
resync_before: stats.resync_before_date,
});
} else if stats.resync_status == ResyncStatusType::ResyncFailed {
skipped_failed_target_count += 1;
}
}
recovered_statuses.push((bucket.clone(), meta));
}
if !recovered_statuses.is_empty() {
let mut status_map = self.resyncer.status_map.write().await;
status_map.extend(recovered_statuses);
}
if !restart_opts.is_empty() || skipped_failed_target_count > 0 {
info!(
event = EVENT_REPLICATION_RESYNC_RECOVERED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
recovered_buckets = recovered_bucket_count,
resumed_targets = restart_opts.len(),
skipped_failed_targets = skipped_failed_target_count,
"Recovered replication resync state from persisted metadata; failed targets require manual resync restart"
);
}
for opts in restart_opts {
let ctx = CancellationToken::new();
let resync = self.resyncer.clone();
let storage = self.storage.clone();
tokio::spawn(async move {
resync.register_cancel_token(&opts, ctx.clone()).await;
Box::pin(resync.clone().resync_bucket(ctx, storage, true, opts.clone())).await;
resync.clear_cancel_token(&opts).await;
});
}
Ok(())
@@ -1498,4 +1522,14 @@ mod tests {
admission.merge(ReplicationQueueAdmission::Missed);
assert_eq!(admission, ReplicationQueueAdmission::Missed);
}
#[test]
fn auto_resume_resync_only_for_inflight_states() {
assert!(should_auto_resume_resync(ResyncStatusType::ResyncPending));
assert!(should_auto_resume_resync(ResyncStatusType::ResyncStarted));
assert!(!should_auto_resume_resync(ResyncStatusType::NoResync));
assert!(!should_auto_resume_resync(ResyncStatusType::ResyncCanceled));
assert!(!should_auto_resume_resync(ResyncStatusType::ResyncCompleted));
assert!(!should_auto_resume_resync(ResyncStatusType::ResyncFailed));
}
}
crates/ecstore/src/bucket/replication/replication_resyncer.rs
+27 -13
View File
@@ -87,7 +87,7 @@ use tokio::task::JoinSet;
use tokio::time::Duration as TokioDuration;
use tokio_util::io::ReaderStream;
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, instrument, warn};
use tracing::{debug, error, instrument, trace, warn};
use uuid::Uuid;
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
@@ -920,18 +920,32 @@ impl ReplicationResyncer {
(roi.size, None)
};
debug!(
event = EVENT_RESYNC_OBJECT_PROCESSED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
reset_id = %reset_id,
bucket = %bucket_name,
object = %roi.name,
version_id = %roi.version_id.unwrap_or_default(),
size,
error = ?err,
"Processed resync object"
);
if err.is_some() {
debug!(
event = EVENT_RESYNC_OBJECT_PROCESSED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
reset_id = %reset_id,
bucket = %bucket_name,
object = %roi.name,
version_id = %roi.version_id.unwrap_or_default(),
size,
error = ?err,
"Processed resync object with verification error"
);
} else {
trace!(
event = EVENT_RESYNC_OBJECT_PROCESSED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
reset_id = %reset_id,
bucket = %bucket_name,
object = %roi.name,
version_id = %roi.version_id.unwrap_or_default(),
size,
"Processed resync object"
);
}
if cancel_token.is_cancelled() {
return;