refactor(replication): isolate queue backpressure decisions (#4202)

2026-08-22 04:16:38 +00:00 · 2026-07-02 23:28:45 +08:00
parent a9894843d9
commit 29899f4731
4 changed files with 282 additions and 98 deletions
@@ -46,7 +46,8 @@ pub use operation::{
 };
 pub use queue::{
    ReplicationHealQueueAction, ReplicationHealQueueResult, ReplicationHealResyncDeletes, ReplicationOperation,
-    ReplicationPriority, ReplicationQueueAdmission, replication_heal_queue_action,
+    ReplicationPriority, ReplicationQueueAdmission, ReplicationWorkerQueue, mrf_save_admission, replication_heal_queue_action,
+    worker_queue_for_replication_type,
 };
 pub use resync::{
    BucketReplicationResyncStatus, Error, Result, ResyncOpts, ResyncStatusType, TargetReplicationResyncStatus,
@@ -56,8 +57,10 @@ pub use resync::{
 pub use rule::ReplicationRuleExt;
 pub use runtime::{
    LARGE_WORKER_COUNT, MIN_LARGE_OBJ_SIZE, MRF_WORKER_AUTO_DEFAULT, MRF_WORKER_MAX_LIMIT, MRF_WORKER_MIN_LIMIT,
-    ReplicationPoolOpts, ReplicationWorkerCounts, WORKER_AUTO_DEFAULT, WORKER_MAX_LIMIT, WORKER_MIN_LIMIT, initial_worker_counts,
-    mrf_worker_size_to_count, next_large_worker_count, next_mrf_worker_count, next_regular_worker_count, resized_worker_counts,
+    ReplicationBackpressureRecommendation, ReplicationBackpressureResize, ReplicationBackpressureState, ReplicationPoolOpts,
+    ReplicationWorkerCounts, ReplicationWorkerResize, WORKER_AUTO_DEFAULT, WORKER_MAX_LIMIT, WORKER_MIN_LIMIT,
+    initial_worker_counts, large_worker_backpressure_resize, mrf_worker_size_to_count, next_large_worker_count,
+    next_mrf_worker_count, next_regular_worker_count, replication_backpressure_recommendation, resized_worker_counts,
    should_grow_large_workers, should_queue_large_object, worker_counts_for_priority,
 };
 pub use rustfs_filemeta::{
@@ -40,6 +40,27 @@ impl ReplicationQueueAdmission {
    }
 }

+pub fn mrf_save_admission(saved: bool) -> ReplicationQueueAdmission {
+    if saved {
+        ReplicationQueueAdmission::Queued
+    } else {
+        ReplicationQueueAdmission::Missed
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ReplicationWorkerQueue {
+    Regular,
+    Mrf,
+}
+
+pub fn worker_queue_for_replication_type(op_type: &ReplicationType) -> ReplicationWorkerQueue {
+    match op_type {
+        ReplicationType::Heal | ReplicationType::ExistingObject => ReplicationWorkerQueue::Mrf,
+        _ => ReplicationWorkerQueue::Regular,
+    }
+}
+
 #[derive(Debug, Clone, Default)]
 pub struct ReplicationHealQueueResult {
    pub object_info: ReplicateObjectInfo,
@@ -247,11 +268,14 @@ mod tests {
    use rustfs_storage_api::DeletedObject;
    use uuid::Uuid;

-    use super::{ReplicationHealQueueAction, ReplicationOperation, ReplicationPriority, ReplicationQueueAdmission};
+    use super::{
+        ReplicationHealQueueAction, ReplicationOperation, ReplicationPriority, ReplicationQueueAdmission, ReplicationWorkerQueue,
+    };
    use crate::{
        DeletedObjectReplicationInfo, REPLICATE_EXISTING, REPLICATE_HEAL, REPLICATE_HEAL_DELETE, ReplicateDecision,
        ReplicateObjectInfo, ReplicateTargetDecision, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation,
-        ResyncTargetDecision, VersionPurgeStatusType, replication_heal_queue_action,
+        ResyncTargetDecision, VersionPurgeStatusType, mrf_save_admission, replication_heal_queue_action,
+        worker_queue_for_replication_type,
    };

    #[test]
@@ -268,6 +292,29 @@ mod tests {
        assert_eq!(admission, ReplicationQueueAdmission::Missed);
    }

+    #[test]
+    fn mrf_save_admission_maps_send_result() {
+        assert_eq!(mrf_save_admission(true), ReplicationQueueAdmission::Queued);
+        assert_eq!(mrf_save_admission(false), ReplicationQueueAdmission::Missed);
+    }
+
+    #[test]
+    fn worker_queue_routes_heal_and_existing_objects_to_mrf() {
+        assert_eq!(worker_queue_for_replication_type(&ReplicationType::Heal), ReplicationWorkerQueue::Mrf);
+        assert_eq!(
+            worker_queue_for_replication_type(&ReplicationType::ExistingObject),
+            ReplicationWorkerQueue::Mrf
+        );
+        assert_eq!(
+            worker_queue_for_replication_type(&ReplicationType::Object),
+            ReplicationWorkerQueue::Regular
+        );
+        assert_eq!(
+            worker_queue_for_replication_type(&ReplicationType::Delete),
+            ReplicationWorkerQueue::Regular
+        );
+    }
+
    #[test]
    fn heal_queue_action_routes_failed_objects_to_heal_queue() {
        let mut roi = replicate_object_info(ReplicationStatusType::Failed);
@@ -23,6 +23,36 @@ pub const MRF_WORKER_AUTO_DEFAULT: usize = 4;
 pub const LARGE_WORKER_COUNT: usize = 10;
 pub const MIN_LARGE_OBJ_SIZE: i64 = 128 * 1024 * 1024;

+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct ReplicationWorkerResize {
+    pub new_count: usize,
+    pub existing_count: usize,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct ReplicationBackpressureState {
+    pub current_workers: usize,
+    pub active_workers: i32,
+    pub current_mrf_workers: i32,
+    pub active_mrf_workers: i32,
+    pub max_workers: usize,
+    pub include_mrf_workers: bool,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct ReplicationBackpressureResize {
+    pub regular_workers: Option<ReplicationWorkerResize>,
+    pub mrf_workers: Option<i32>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ReplicationBackpressureRecommendation {
+    KeepFast,
+    SetPriorityAuto,
+    Resize(ReplicationBackpressureResize),
+    Noop,
+}
+
 #[derive(Debug, Clone)]
 pub struct ReplicationPoolOpts {
    pub priority: ReplicationPriority,
@@ -133,6 +163,49 @@ pub fn next_mrf_worker_count(current_mrf_workers: i32, active_mrf_workers: i32,
    }
 }

+pub fn large_worker_backpressure_resize(
+    existing_workers: usize,
+    active_lrg_workers: i32,
+    max_l_workers: usize,
+) -> Option<ReplicationWorkerResize> {
+    should_grow_large_workers(active_lrg_workers, max_l_workers).then_some(ReplicationWorkerResize {
+        new_count: next_large_worker_count(existing_workers, max_l_workers),
+        existing_count: existing_workers,
+    })
+}
+
+pub fn replication_backpressure_recommendation(
+    priority: &ReplicationPriority,
+    state: ReplicationBackpressureState,
+) -> ReplicationBackpressureRecommendation {
+    match priority {
+        ReplicationPriority::Fast => ReplicationBackpressureRecommendation::KeepFast,
+        ReplicationPriority::Slow => ReplicationBackpressureRecommendation::SetPriorityAuto,
+        ReplicationPriority::Auto => {
+            let regular_workers =
+                next_regular_worker_count(state.current_workers, state.active_workers, state.max_workers).map(|new_count| {
+                    ReplicationWorkerResize {
+                        new_count,
+                        existing_count: state.current_workers,
+                    }
+                });
+            let mrf_workers = state
+                .include_mrf_workers
+                .then(|| next_mrf_worker_count(state.current_mrf_workers, state.active_mrf_workers, state.max_workers))
+                .flatten();
+
+            if regular_workers.is_none() && mrf_workers.is_none() {
+                ReplicationBackpressureRecommendation::Noop
+            } else {
+                ReplicationBackpressureRecommendation::Resize(ReplicationBackpressureResize {
+                    regular_workers,
+                    mrf_workers,
+                })
+            }
+        }
+    }
+}
+
 fn auto_worker_count(current_workers: usize, auto_default: usize) -> usize {
    if current_workers < auto_default {
        current_workers.saturating_add(1).min(auto_default)
@@ -229,4 +302,74 @@ mod tests {
        assert!(should_grow_large_workers(1, LARGE_WORKER_COUNT));
        assert_eq!(next_large_worker_count(LARGE_WORKER_COUNT, LARGE_WORKER_COUNT), LARGE_WORKER_COUNT);
    }
+
+    #[test]
+    fn large_worker_resize_recommendation_respects_growth_limit() {
+        assert_eq!(
+            large_worker_backpressure_resize(3, 2, LARGE_WORKER_COUNT),
+            Some(ReplicationWorkerResize {
+                new_count: 4,
+                existing_count: 3
+            })
+        );
+        assert_eq!(
+            large_worker_backpressure_resize(LARGE_WORKER_COUNT, usize_to_i32_saturating(LARGE_WORKER_COUNT), LARGE_WORKER_COUNT),
+            None
+        );
+    }
+
+    #[test]
+    fn queue_backpressure_recommendation_preserves_priority_semantics() {
+        let state = ReplicationBackpressureState {
+            current_workers: 4,
+            active_workers: 4,
+            current_mrf_workers: 2,
+            active_mrf_workers: 2,
+            max_workers: WORKER_MAX_LIMIT,
+            include_mrf_workers: true,
+        };
+
+        assert_eq!(
+            replication_backpressure_recommendation(&ReplicationPriority::Fast, state),
+            ReplicationBackpressureRecommendation::KeepFast
+        );
+        assert_eq!(
+            replication_backpressure_recommendation(&ReplicationPriority::Slow, state),
+            ReplicationBackpressureRecommendation::SetPriorityAuto
+        );
+
+        assert_eq!(
+            replication_backpressure_recommendation(&ReplicationPriority::Auto, state),
+            ReplicationBackpressureRecommendation::Resize(ReplicationBackpressureResize {
+                regular_workers: Some(ReplicationWorkerResize {
+                    new_count: 5,
+                    existing_count: 4
+                }),
+                mrf_workers: Some(3),
+            })
+        );
+    }
+
+    #[test]
+    fn delete_queue_backpressure_skips_mrf_growth() {
+        let state = ReplicationBackpressureState {
+            current_workers: 4,
+            active_workers: 4,
+            current_mrf_workers: 2,
+            active_mrf_workers: 2,
+            max_workers: WORKER_MAX_LIMIT,
+            include_mrf_workers: false,
+        };
+
+        assert_eq!(
+            replication_backpressure_recommendation(&ReplicationPriority::Auto, state),
+            ReplicationBackpressureRecommendation::Resize(ReplicationBackpressureResize {
+                regular_workers: Some(ReplicationWorkerResize {
+                    new_count: 5,
+                    existing_count: 4
+                }),
+                mrf_workers: None,
+            })
+        );
+    }
 }