gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-26 05:56:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(replication): make resync recovery resilient (#5883)

Browse Source 
Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
cxymds
2026-08-09 19:42:06 +08:00
committed by GitHub
parent ec7f5f7b7d
commit 1be636b914
7 changed files with 447 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/bucket/replication/replication_pool.rs
+76 -17
View File
@@ -710,8 +710,8 @@ pub struct ReplicationPool<S: ReplicationStorage> {
mrf_save_tx: Sender<MrfReplicateEntry>,
mrf_save_rx: Mutex<Option<Receiver<MrfReplicateEntry>>>,
// Control channels
mrf_worker_kill_tx: Sender<()>,
// MRF worker lifecycle
mrf_worker_cancellations: Mutex<Vec<CancellationToken>>,
mrf_stop_tx: Sender<()>,
// Worker size tracking
@@ -734,7 +734,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
// Create MRF channels
let (mrf_replica_tx, mrf_replica_rx) = mpsc::channel(100000);
let (mrf_save_tx, mrf_save_rx) = mpsc::channel(100000);
let (mrf_worker_kill_tx, _mrf_worker_kill_rx) = mpsc::channel(worker_counts.mrf_workers);
let (mrf_stop_tx, _mrf_stop_rx) = mpsc::channel(1);
let pool = Arc::new(Self {
@@ -752,7 +751,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
mrf_replica_rx: Arc::new(Mutex::new(mrf_replica_rx)),
mrf_save_tx,
mrf_save_rx: Mutex::new(Some(mrf_save_rx)),
mrf_worker_kill_tx,
mrf_worker_cancellations: Mutex::new(Vec::with_capacity(worker_counts.mrf_workers)),
mrf_stop_tx,
mrf_worker_size: AtomicI32::new(0),
task_handles: Mutex::new(Vec::new()),
@@ -896,12 +895,12 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
/// Resizes the failed workers pool
pub async fn resize_failed_workers(&self, n: i32) {
// Spawn workers up to n. Each worker shares the receiver via Arc<Mutex<...>>.
// The mutex is held only while calling recv() — released before processing — so
// all workers process entries concurrently (the dequeue step is serialised but
// the replication I/O is not).
while self.mrf_worker_size.load(Ordering::SeqCst) < n {
self.mrf_worker_size.fetch_add(1, Ordering::SeqCst);
let target = mrf_worker_size_to_count(n);
let mut cancellations = self.mrf_worker_cancellations.lock().await;
while cancellations.len() < target {
let cancellation = CancellationToken::new();
cancellations.push(cancellation.clone());
let active_counter = self.active_mrf_workers.clone();
let stats = self.stats.clone();
@@ -910,7 +909,18 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
let handle = tokio::spawn(async move {
loop {
let operation = { mrf_rx.lock().await.recv().await };
let operation = tokio::select! {
biased;
operation = async {
let mut receiver = mrf_rx.lock().await;
tokio::select! {
biased;
operation = receiver.recv() => operation,
_ = cancellation.cancelled() => None,
}
} => operation,
_ = cancellation.cancelled() => break,
};
let Some(operation) = operation else { break };
let _active = ActiveWorkerGuard::new(active_counter.clone());
@@ -920,11 +930,13 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
self.task_handles.lock().await.push(handle);
}
// Remove workers if needed
while self.mrf_worker_size.load(Ordering::SeqCst) > n {
self.mrf_worker_size.fetch_sub(1, Ordering::SeqCst);
let _ = self.mrf_worker_kill_tx.try_send(());
while cancellations.len() > target {
if let Some(cancellation) = cancellations.pop() {
cancellation.cancel();
}
}
self.mrf_worker_size.store(n.max(0), Ordering::SeqCst);
}
/// Resizes worker priority and counts
@@ -3350,7 +3362,6 @@ mod tests {
) -> Arc<ReplicationPool<LoadResyncNodeStore>> {
let (mrf_replica_tx, mrf_replica_rx) = mpsc::channel(1);
let (mrf_save_tx, mrf_save_rx) = mpsc::channel(mrf_save_capacity);
let (mrf_worker_kill_tx, _) = mpsc::channel(1);
let (mrf_stop_tx, _) = mpsc::channel(1);
Arc::new(ReplicationPool {
@@ -3368,7 +3379,7 @@ mod tests {
mrf_replica_rx: Arc::new(Mutex::new(mrf_replica_rx)),
mrf_save_tx,
mrf_save_rx: Mutex::new(Some(mrf_save_rx)),
mrf_worker_kill_tx,
mrf_worker_cancellations: Mutex::new(Vec::new()),
mrf_stop_tx,
mrf_worker_size: AtomicI32::new(0),
task_handles: Mutex::new(Vec::new()),
@@ -3971,6 +3982,54 @@ mod tests {
);
}
#[tokio::test]
async fn resize_failed_workers_cancels_idle_workers() {
let shared = empty_resync_shared_state();
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("mrf-resize", shared))).await;
pool.resize_failed_workers(4).await;
assert_eq!(pool.mrf_worker_cancellations.lock().await.len(), 4);
assert_eq!(pool.mrf_worker_size.load(Ordering::SeqCst), 4);
pool.resize_failed_workers(1).await;
tokio::time::timeout(Duration::from_secs(10), async {
loop {
let finished = pool
.task_handles
.lock()
.await
.iter()
.filter(|handle| handle.is_finished())
.count();
if finished == 3 {
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("canceled MRF workers should exit while the shared queue is idle");
assert_eq!(pool.mrf_worker_cancellations.lock().await.len(), 1);
assert_eq!(pool.mrf_worker_size.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn resize_failed_workers_is_idempotent_across_growth_and_shrink() {
let shared = empty_resync_shared_state();
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("mrf-resize-repeat", shared))).await;
for target in [2, 4, 1, 4, 4] {
pool.resize_failed_workers(target).await;
assert_eq!(
pool.mrf_worker_cancellations.lock().await.len(),
usize::try_from(target).expect("test worker count should fit usize")
);
assert_eq!(pool.mrf_worker_size.load(Ordering::SeqCst), target);
}
}
#[test]
fn replicate_object_info_from_object_info_preserves_ssec_checksum() {
let checksum = bytes::Bytes::from_static(b"ssec-checksum");
crates/ecstore/src/bucket/replication/replication_resyncer.rs
+108 -79
View File
@@ -69,16 +69,17 @@ use rustfs_s3_types::EventName;
use rustfs_utils::http::{
AMZ_TAGGING_DIRECTIVE, SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_STATUS, has_internal_suffix, insert_str,
};
use rustfs_utils::{DEFAULT_SIP_HASH_KEY, sip_hash};
use rustfs_utils::{DEFAULT_SIP_HASH_KEY, get_env_usize, sip_hash};
#[cfg(test)]
use s3s::dto::ReplicationConfiguration;
use std::collections::HashMap;
use std::fmt::Display;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use time::OffsetDateTime;
use time::format_description::well_known::Rfc3339;
use tokio::io::AsyncRead;
use tokio::sync::RwLock;
use tokio::sync::{OwnedSemaphorePermit, RwLock, Semaphore};
use tokio::task::{JoinHandle, JoinSet};
use tokio::time::Duration as TokioDuration;
use tokio_util::io::ReaderStream;
@@ -86,6 +87,9 @@ use tokio_util::sync::CancellationToken;
use tracing::{debug, error, instrument, trace, warn};
const BACKGROUND_WALKDIR_TIMEOUT: TokioDuration = TokioDuration::from_secs(60);
const ENV_REPL_RESYNC_MAX_JOBS: &str = "RUSTFS_REPL_RESYNC_MAX_JOBS";
const DEFAULT_REPL_RESYNC_MAX_JOBS: usize = 2;
const MAX_REPL_RESYNC_MAX_JOBS: usize = 32;
use uuid::Uuid;
const EVENT_RESYNC_STATUS_UPDATE_SKIPPED: &str = "replication_resync_status_update_skipped";
@@ -256,11 +260,20 @@ fn resync_status_duration(
type ResyncCancelKey = (String, String, String);
fn configured_resync_max_jobs() -> usize {
bounded_resync_max_jobs(get_env_usize(ENV_REPL_RESYNC_MAX_JOBS, DEFAULT_REPL_RESYNC_MAX_JOBS))
}
fn bounded_resync_max_jobs(value: usize) -> usize {
value.clamp(1, MAX_REPL_RESYNC_MAX_JOBS)
}
#[derive(Debug)]
pub struct ReplicationResyncer {
pub status_map: Arc<RwLock<HashMap<String, BucketReplicationResyncStatus>>>,
pub worker_size: usize,
pub(crate) cancel_tokens: Arc<RwLock<HashMap<ResyncCancelKey, CancellationToken>>>,
resync_admission: Arc<Semaphore>,
}
impl ReplicationResyncer {
@@ -269,6 +282,14 @@ impl ReplicationResyncer {
status_map: Arc::new(RwLock::new(HashMap::new())),
worker_size: RESYNC_WORKER_COUNT,
cancel_tokens: Arc::new(RwLock::new(HashMap::new())),
resync_admission: Arc::new(Semaphore::new(configured_resync_max_jobs())),
}
}
async fn acquire_resync_admission(&self, cancellation_token: &CancellationToken) -> Option<OwnedSemaphorePermit> {
tokio::select! {
permit = self.resync_admission.clone().acquire_owned() => permit.ok(),
_ = cancellation_token.cancelled() => None,
}
}
@@ -603,6 +624,10 @@ impl ReplicationResyncer {
}
};
let Some(_resync_admission_permit) = self.acquire_resync_admission(&cancellation_token).await else {
return;
};
let cfg = match get_replication_config(&opts.bucket).await {
Ok(cfg) => cfg,
Err(err) => {
@@ -715,55 +740,36 @@ impl ReplicationResyncer {
}
let (tx, mut rx) = tokio::sync::mpsc::channel(100);
if let Err(err) = storage
.clone()
.walk(
cancellation_token.clone(),
&opts.bucket,
"",
tx.clone(),
WalkOptions::default().with_walkdir_timeouts(BACKGROUND_WALKDIR_TIMEOUT),
)
.await
{
error!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %opts.bucket,
arn = %opts.arn,
reason = "walk_failed",
error = %err,
"Replication resync bucket walk failed"
);
self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone())
.await;
return;
}
drop(tx);
let status = {
self.status_map
.read()
let walk_failed = Arc::new(AtomicBool::new(false));
let walk_failed_task = walk_failed.clone();
let walk_storage = storage.clone();
let walk_cancellation = cancellation_token.clone();
let walk_bucket = opts.bucket.clone();
let walk_arn = opts.arn.clone();
let walk_task = tokio::spawn(async move {
if let Err(err) = walk_storage
.walk(
walk_cancellation,
&walk_bucket,
"",
tx,
WalkOptions::default().with_walkdir_timeouts(BACKGROUND_WALKDIR_TIMEOUT),
)
.await
.get(&opts.bucket)
.and_then(|status| status.targets_map.get(&opts.arn))
.cloned()
.unwrap_or_default()
};
// An empty checkpoint means no per-object progress was persisted before the
// interruption: resume from the beginning, otherwise `object.name != checkpoint`
// below would skip every object and mark the resync completed without work.
let mut last_checkpoint = if (status.resync_status == ResyncStatusType::ResyncStarted
|| status.resync_status == ResyncStatusType::ResyncFailed)
&& !status.object.is_empty()
{
Some(status.object)
} else {
None
};
{
walk_failed_task.store(true, Ordering::Relaxed);
error!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %walk_bucket,
arn = %walk_arn,
reason = "walk_failed",
error = %err,
"Replication resync bucket walk failed"
);
}
});
let mut worker_txs = Vec::new();
// mpsc, not broadcast: a lagging broadcast receiver returns Err(Lagged) which
@@ -773,7 +779,7 @@ impl ReplicationResyncer {
let opts_clone = opts.clone();
let self_clone = self.clone();
let mut futures = Vec::new();
let mut futures = vec![walk_task];
let results_fut = tokio::spawn(async move {
while let Some(st) = results_rx.recv().await {
@@ -962,6 +968,8 @@ impl ReplicationResyncer {
error = %err,
"Failed to receive resync object info"
);
cancellation_token.cancel();
drop(rx);
let worker_failed = finish_resync_workers(worker_txs, results_tx, futures, false).await;
if worker_failed {
error!(
@@ -980,6 +988,7 @@ impl ReplicationResyncer {
}
if cancellation_token.is_cancelled() {
drop(rx);
finish_resync_workers(worker_txs, results_tx, futures, true).await;
self.resync_bucket_mark_status(ResyncStatusType::ResyncCanceled, opts.clone(), storage.clone())
.await;
@@ -990,14 +999,6 @@ impl ReplicationResyncer {
continue;
};
if heal
&& let Some(checkpoint) = &last_checkpoint
&& &object.name != checkpoint
{
continue;
}
last_checkpoint = None;
let roi = match get_heal_replicate_object_info(&object, &rcfg).await {
Ok(roi) => roi,
Err(err) => {
@@ -1011,6 +1012,8 @@ impl ReplicationResyncer {
error = %err,
"Failed to classify object for replication resync"
);
cancellation_token.cancel();
drop(rx);
let worker_failed = finish_resync_workers(worker_txs, results_tx, futures, false).await;
if worker_failed {
error!(
@@ -1033,6 +1036,7 @@ impl ReplicationResyncer {
}
if cancellation_token.is_cancelled() {
drop(rx);
finish_resync_workers(worker_txs, results_tx, futures, true).await;
self.resync_bucket_mark_status(ResyncStatusType::ResyncCanceled, opts.clone(), storage.clone())
.await;
@@ -1052,6 +1056,8 @@ impl ReplicationResyncer {
error = %err,
"Failed to send resync object to worker"
);
cancellation_token.cancel();
drop(rx);
let worker_failed = finish_resync_workers(worker_txs, results_tx, futures, false).await;
if worker_failed {
error!(
@@ -1072,7 +1078,7 @@ impl ReplicationResyncer {
let worker_failed = finish_resync_workers(worker_txs, results_tx, futures, false).await;
let target_failed = self.target_has_resync_failures(&opts).await;
let status = if worker_failed || target_failed {
let status = if walk_failed.load(Ordering::Relaxed) || worker_failed || target_failed {
ResyncStatusType::ResyncFailed
} else {
ResyncStatusType::ResyncCompleted
@@ -2355,16 +2361,6 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
..Default::default()
};
if self.target_replication_status(&tgt_client.arn) == ReplicationStatusType::Completed
&& !self.existing_obj_resync.is_empty()
&& self.existing_obj_resync.must_resync_target(&tgt_client.arn)
{
rinfo.replication_status = ReplicationStatusType::Completed;
rinfo.replication_resynced = true;
return rinfo;
}
if ReplicationTargetStore::target_is_offline(&tgt_client).await {
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
@@ -2723,15 +2719,6 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
rinfo.prev_replication_status = object_info.target_replication_status(&tgt_client.arn);
if rinfo.prev_replication_status == ReplicationStatusType::Completed
&& !self.existing_obj_resync.is_empty()
&& self.existing_obj_resync.must_resync_target(&tgt_client.arn)
{
rinfo.replication_status = ReplicationStatusType::Completed;
rinfo.replication_resynced = true;
return rinfo;
}
let size = match object_info.get_actual_size() {
Ok(size) => size,
Err(e) => {
@@ -3251,6 +3238,48 @@ mod tests {
ReplicationTargetStore::register_test_target(target).await;
}
#[test]
fn resync_admission_configuration_is_bounded() {
assert_eq!(ENV_REPL_RESYNC_MAX_JOBS, "RUSTFS_REPL_RESYNC_MAX_JOBS");
assert_eq!(bounded_resync_max_jobs(0), 1);
assert_eq!(bounded_resync_max_jobs(DEFAULT_REPL_RESYNC_MAX_JOBS), 2);
assert_eq!(bounded_resync_max_jobs(1000), MAX_REPL_RESYNC_MAX_JOBS);
}
#[tokio::test]
async fn resync_admission_limits_jobs_and_wait_is_cancelable() {
let resyncer = ReplicationResyncer {
resync_admission: Arc::new(Semaphore::new(2)),
..ReplicationResyncer::new().await
};
let first = resyncer
.acquire_resync_admission(&CancellationToken::new())
.await
.expect("first resync should acquire admission");
let second = resyncer
.acquire_resync_admission(&CancellationToken::new())
.await
.expect("second resync should acquire admission");
let cancellation = CancellationToken::new();
let blocked = resyncer.acquire_resync_admission(&cancellation);
tokio::pin!(blocked);
assert!(
tokio::time::timeout(TokioDuration::from_millis(25), &mut blocked)
.await
.is_err()
);
cancellation.cancel();
assert!(
tokio::time::timeout(TokioDuration::from_secs(1), &mut blocked)
.await
.expect("canceled admission wait should finish")
.is_none()
);
drop((first, second));
}
#[test]
fn replication_target_offline_error_classifier_is_network_scoped() {
assert!(is_replication_target_offline_error(&"put_object dispatch failure: connector error"));