fix(ecstore): throttle data movement under read pressure (#3906)

rustfs/src/startup_background.rs

@@ -12,8 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::storage_api::startup::ECStore;
+use crate::storage_api::startup::{ECStore, set_workload_admission_snapshot_provider};
 use crate::workload_admission::RustFsWorkloadAdmissionSnapshotProvider;
+use rustfs_concurrency::WorkloadAdmissionSnapshotProvider;
 use rustfs_heal::{
     create_ahm_services_cancel_token, heal::storage::ECStoreHealStorage, init_heal_manager_with_workload_provider,
 };
@@ -45,9 +46,12 @@ pub(crate) async fn init_background_service_runtime(store: Arc<ECStore>) -> Resu
         "Background services configured"
     );
 
+    let workload_provider: Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync> =
+        Arc::new(RustFsWorkloadAdmissionSnapshotProvider);
+    let _ = set_workload_admission_snapshot_provider(workload_provider.clone());
+
     if enable_heal || enable_scanner {
         let heal_storage = Arc::new(ECStoreHealStorage::new(store));
-        let workload_provider = Arc::new(RustFsWorkloadAdmissionSnapshotProvider);
         init_heal_manager_with_workload_provider(heal_storage, None, Some(workload_provider)).await?;
     }
 

rustfs/src/storage/storage_api.rs

@@ -231,6 +231,7 @@ pub(crate) type ObjectLockBlockReason = ecstore_bucket::object_lock::objectlock_
 pub(crate) type ObjectStoreResolver = dyn Fn() -> Option<Arc<ECStore>> + Send + Sync + 'static;
 pub(crate) type PolicySys = ecstore_bucket::policy_sys::PolicySys;
 pub(crate) type PoolEndpoints = ecstore_layout::PoolEndpoints;
+pub(crate) type WorkloadAdmissionSnapshotProviderRef = rustfs_ecstore::WorkloadAdmissionSnapshotProviderRef;
 pub(crate) type QuotaError = ecstore_bucket::quota::QuotaError;
 pub(crate) type RawFileInfo = rustfs_filemeta::RawFileInfo;
 pub(crate) type ReadMultipleReq = ecstore_disk::ReadMultipleReq;
@@ -850,6 +851,12 @@ pub(crate) fn set_object_store_resolver(resolver: Arc<ObjectStoreResolver>) -> b
     ecstore_global::set_object_store_resolver(resolver)
 }
 
+pub(crate) fn set_workload_admission_snapshot_provider(
+    provider: WorkloadAdmissionSnapshotProviderRef,
+) -> std::result::Result<(), WorkloadAdmissionSnapshotProviderRef> {
+    rustfs_ecstore::set_workload_admission_snapshot_provider(provider)
+}
+
 pub(crate) fn get_global_notification_sys() -> Option<&'static NotificationSys> {
     ecstore_notification::get_global_notification_sys()
 }

rustfs/src/storage_api.rs

@@ -104,8 +104,8 @@ pub(crate) mod startup {
         init_bucket_metadata_sys, init_ecstore_config, init_global_config_sys, init_local_disks, init_lock_clients,
         new_global_notification_sys, prewarm_local_disk_id_map, process_lambda_configurations, process_queue_configurations,
         process_topic_configurations, set_global_endpoints, set_global_region, set_global_rustfs_port,
-        shutdown_background_services, try_migrate_bucket_metadata, try_migrate_iam_config, try_migrate_server_config,
-        update_erasure_type,
+        set_workload_admission_snapshot_provider, shutdown_background_services, try_migrate_bucket_metadata,
+        try_migrate_iam_config, try_migrate_server_config, update_erasure_type,
     };
 
     #[cfg(test)]