gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-27 15:37:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(replication): isolate queue backpressure decisions (#4202)

Browse Source
This commit is contained in:
Zhengchao An
2026-07-02 23:28:45 +08:00
committed by GitHub
parent a9894843d9
commit 29899f4731
4 changed files with 282 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/bucket/replication/replication_pool.rs
+84 -93
View File
@@ -34,11 +34,12 @@ use super::runtime_boundary as runtime_sources;
use super::{BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus};
use lazy_static::lazy_static;
use rustfs_replication::{
DeletedObjectReplicationInfo, LARGE_WORKER_COUNT, ReplicationHealQueueAction, ReplicationHealQueueResult,
ReplicationHealResyncDeletes, ReplicationOperation, ReplicationPoolOpts, ReplicationPriority, ReplicationQueueAdmission,
WORKER_MAX_LIMIT, initial_worker_counts, mrf_worker_size_to_count, next_large_worker_count, next_mrf_worker_count,
next_regular_worker_count, replication_heal_queue_action, resized_worker_counts, should_auto_resume_resync,
should_grow_large_workers, should_queue_large_object,
DeletedObjectReplicationInfo, LARGE_WORKER_COUNT, ReplicationBackpressureRecommendation, ReplicationBackpressureState,
ReplicationHealQueueAction, ReplicationHealQueueResult, ReplicationHealResyncDeletes, ReplicationOperation,
ReplicationPoolOpts, ReplicationPriority, ReplicationQueueAdmission, ReplicationWorkerQueue, WORKER_MAX_LIMIT,
initial_worker_counts, large_worker_backpressure_resize, mrf_worker_size_to_count, replication_backpressure_recommendation,
replication_heal_queue_action, resized_worker_counts, should_auto_resume_resync, should_queue_large_object,
worker_queue_for_replication_type,
};
use rustfs_utils::http::{SUFFIX_REPLICATION_TIMESTAMP, get_str};
use std::sync::Arc;
@@ -396,6 +397,73 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
workers.get(index).cloned()
}
async fn worker_queue_channel(
&self,
op_type: &ReplicationType,
bucket: &str,
object: &str,
size: i64,
) -> Option<Sender<ReplicationOperation>> {
match worker_queue_for_replication_type(op_type) {
ReplicationWorkerQueue::Mrf => Some(self.mrf_replica_tx.clone()),
ReplicationWorkerQueue::Regular => self.get_worker_ch(bucket, object, size).await,
}
}
async fn apply_queue_backpressure(&self, queue_type: &'static str, include_mrf_workers: bool, message: &'static str) {
let priority = self.priority.read().await.clone();
let max_workers = *self.max_workers.read().await;
let current_workers = self.workers.read().await.len();
let current_mrf_workers = self.mrf_worker_size.load(Ordering::SeqCst);
let recommendation = replication_backpressure_recommendation(
&priority,
ReplicationBackpressureState {
current_workers,
active_workers: self.active_workers(),
current_mrf_workers,
active_mrf_workers: self.active_mrf_workers(),
max_workers,
include_mrf_workers,
},
);
match recommendation {
ReplicationBackpressureRecommendation::KeepFast => {
debug!(
event = EVENT_REPLICATION_BACKPRESSURE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
queue_type,
priority = "fast",
recommendation = "none",
"{message}"
);
}
ReplicationBackpressureRecommendation::SetPriorityAuto => {
debug!(
event = EVENT_REPLICATION_BACKPRESSURE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
queue_type,
priority = "slow",
recommendation = "set_priority_auto",
"{message}"
);
}
ReplicationBackpressureRecommendation::Resize(resize) => {
if let Some(regular_workers) = resize.regular_workers {
self.resize_workers(regular_workers.new_count, regular_workers.existing_count)
.await;
}
if let Some(mrf_workers) = resize.mrf_workers {
self.resize_failed_workers(mrf_workers).await;
}
}
ReplicationBackpressureRecommendation::Noop => {}
}
}
/// Queues a replica task
pub async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission {
// If object is large, queue it to a static set of large workers
@@ -418,6 +486,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
// Try to add more workers if possible
let max_l_workers = *self.max_l_workers.read().await;
let existing = lrg_workers.len();
let resize = large_worker_backpressure_resize(existing, self.active_lrg_workers(), max_l_workers);
drop(lrg_workers);
// Queue to MRF if worker is busy.
@@ -425,10 +494,8 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
queue_mrf_save_admission(&self.mrf_save_tx, ri.to_mrf_entry(), &ri.bucket, &ri.name, "large_object")
.await;
if should_grow_large_workers(self.active_lrg_workers(), max_l_workers) {
let workers = next_large_worker_count(existing, max_l_workers);
self.resize_lrg_workers(workers, existing).await;
if let Some(resize) = resize {
self.resize_lrg_workers(resize.new_count, resize.existing_count).await;
}
return admission;
}
@@ -440,10 +507,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
// Handle regular sized objects
let ch = match ri.op_type {
ReplicationType::Heal | ReplicationType::ExistingObject => Some(self.mrf_replica_tx.clone()),
_ => self.get_worker_ch(&ri.bucket, &ri.name, ri.size).await,
};
let ch = self.worker_queue_channel(&ri.op_type, &ri.bucket, &ri.name, ri.size).await;
let Some(channel) = ch else {
return ReplicationQueueAdmission::Missed;
@@ -457,57 +521,17 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
let admission = queue_mrf_save_admission(&self.mrf_save_tx, ri.to_mrf_entry(), &ri.bucket, &ri.name, "object").await;
// Try to scale up workers based on priority
let priority = self.priority.read().await.clone();
let max_workers = *self.max_workers.read().await;
match priority {
ReplicationPriority::Fast => {
debug!(
event = EVENT_REPLICATION_BACKPRESSURE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
queue_type = "object",
priority = "fast",
recommendation = "none",
"Replication queue is backpressured"
);
}
ReplicationPriority::Slow => {
debug!(
event = EVENT_REPLICATION_BACKPRESSURE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
queue_type = "object",
priority = "slow",
recommendation = "set_priority_auto",
"Replication queue is backpressured"
);
}
ReplicationPriority::Auto => {
let workers = self.workers.read().await;
if let Some(new_count) = next_regular_worker_count(workers.len(), self.active_workers(), max_workers) {
let existing = workers.len();
drop(workers);
self.resize_workers(new_count, existing).await;
}
let current_mrf = self.mrf_worker_size.load(Ordering::SeqCst);
if let Some(new_mrf) = next_mrf_worker_count(current_mrf, self.active_mrf_workers(), max_workers) {
self.resize_failed_workers(new_mrf).await;
}
}
}
self.apply_queue_backpressure("object", true, "Replication queue is backpressured")
.await;
admission
}
/// Queues a replica delete task
pub async fn queue_replica_delete_task(&self, doi: DeletedObjectReplicationInfo) -> ReplicationQueueAdmission {
let ch = match doi.op_type {
ReplicationType::Heal | ReplicationType::ExistingObject => Some(self.mrf_replica_tx.clone()),
_ => self.get_worker_ch(&doi.bucket, &doi.delete_object.object_name, 0).await,
};
let ch = self
.worker_queue_channel(&doi.op_type, &doi.bucket, &doi.delete_object.object_name, 0)
.await;
let Some(channel) = ch else {
return ReplicationQueueAdmission::Missed;
@@ -526,41 +550,8 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
)
.await;
let priority = self.priority.read().await.clone();
let max_workers = *self.max_workers.read().await;
match priority {
ReplicationPriority::Fast => {
debug!(
event = EVENT_REPLICATION_BACKPRESSURE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
queue_type = "delete",
priority = "fast",
recommendation = "none",
"Replication delete queue is backpressured"
);
}
ReplicationPriority::Slow => {
debug!(
event = EVENT_REPLICATION_BACKPRESSURE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
queue_type = "delete",
priority = "slow",
recommendation = "set_priority_auto",
"Replication delete queue is backpressured"
);
}
ReplicationPriority::Auto => {
let workers = self.workers.read().await;
if let Some(new_count) = next_regular_worker_count(workers.len(), self.active_workers(), max_workers) {
let existing = workers.len();
drop(workers);
self.resize_workers(new_count, existing).await;
}
}
}
self.apply_queue_backpressure("delete", false, "Replication delete queue is backpressured")
.await;
admission
}