gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-31 17:28:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf(memory): add reclaim signals and cache controls (#2689)

Browse Source
This commit is contained in:
houseme
2026-04-27 00:42:35 +08:00
committed by GitHub
parent 438913b84a
commit 50d03ef021
23 changed files with 2253 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/heal/src/heal/manager.rs
+25 -1
View File
@@ -237,6 +237,14 @@ impl PriorityHealQueue {
}
}
fn publish_active_heal_count(active_heals: &HashMap<String, Arc<HealTask>>) {
crate::set_heal_active_tasks(active_heals.len());
}
fn publish_heal_queue_length(queue: &PriorityHealQueue) {
crate::set_heal_queue_length(queue.len());
}
/// Heal config
#[derive(Debug, Clone)]
pub struct HealConfig {
@@ -420,6 +428,8 @@ impl HealManager {
}
}
active_heals.clear();
publish_active_heal_count(&active_heals);
crate::set_heal_queue_length(0);
// update state
let mut state = self.state.write().await;
@@ -435,6 +445,7 @@ impl HealManager {
let mut queue = self.heal_queue.lock().await;
let queue_len = queue.len();
publish_heal_queue_length(&queue);
let queue_capacity = config.queue_size;
if queue.contains_key(&request) {
@@ -505,6 +516,7 @@ impl HealManager {
let push_outcome = queue.push(request);
debug_assert_eq!(push_outcome, QueuePushOutcome::Accepted);
publish_heal_queue_length(&queue);
// Log queue statistics periodically (when adding high/urgent priority items)
if matches!(priority, HealPriority::High | HealPriority::Urgent) {
@@ -544,7 +556,9 @@ impl HealManager {
/// Get task progress
pub async fn get_active_tasks_count(&self) -> usize {
self.active_heals.lock().await.len()
let active_heals = self.active_heals.lock().await;
publish_active_heal_count(&active_heals);
active_heals.len()
}
pub async fn get_task_progress(&self, task_id: &str) -> Result<HealProgress> {
@@ -564,6 +578,7 @@ impl HealManager {
if let Some(task) = active_heals.get(task_id) {
task.cancel().await?;
active_heals.remove(task_id);
publish_active_heal_count(&active_heals);
info!("Cancelled heal task: {}", task_id);
Ok(())
} else {
@@ -581,12 +596,14 @@ impl HealManager {
/// Get active task count
pub async fn get_active_task_count(&self) -> usize {
let active_heals = self.active_heals.lock().await;
publish_active_heal_count(&active_heals);
active_heals.len()
}
/// Get queue length
pub async fn get_queue_length(&self) -> usize {
let queue = self.heal_queue.lock().await;
publish_heal_queue_length(&queue);
queue.len()
}
@@ -731,6 +748,7 @@ impl HealManager {
);
let mut queue = heal_queue.lock().await;
if matches!(queue.push(req), QueuePushOutcome::Accepted) {
publish_heal_queue_length(&queue);
let config = config.read().await;
if config.event_driven_scheduler_enable {
notify.notify_one();
@@ -757,6 +775,7 @@ impl HealManager {
) {
let config = config.read().await;
let mut active_heals_guard = active_heals.lock().await;
publish_active_heal_count(&active_heals_guard);
// Check if new heal tasks can be started
let active_count = active_heals_guard.len();
@@ -769,6 +788,7 @@ impl HealManager {
let mut queue = heal_queue.lock().await;
let queue_len = queue.len();
publish_heal_queue_length(&queue);
if queue_len == 0 {
return;
@@ -804,6 +824,7 @@ impl HealManager {
let task = Arc::new(HealTask::from_request(request, storage.clone()));
let task_id = task.id.clone();
active_heals_guard.insert(task_id.clone(), task.clone());
publish_active_heal_count(&active_heals_guard);
update_task_running_metric_for_task(&active_heals_guard, task.as_ref());
let active_heals_clone = active_heals.clone();
let statistics_clone = statistics.clone();
@@ -828,6 +849,7 @@ impl HealManager {
}
let mut active_heals_guard = active_heals_clone.lock().await;
if let Some(completed_task) = active_heals_guard.remove(&task_id) {
publish_active_heal_count(&active_heals_guard);
update_task_running_metric_for_task(&active_heals_guard, completed_task.as_ref());
// update statistics
let mut stats = statistics_clone.write().await;
@@ -853,6 +875,8 @@ impl HealManager {
let mut stats = statistics.write().await;
stats.total_tasks += tasks_started as u64;
stats.update_running_tasks(active_heals_guard.len() as u64);
publish_active_heal_count(&active_heals_guard);
publish_heal_queue_length(&queue);
// Log queue status if items remain
if !queue.is_empty() {
crates/heal/src/lib.rs
+19
View File
@@ -17,6 +17,7 @@ pub mod heal;
pub use error::{Error, Result};
pub use heal::{HealManager, HealOptions, HealPriority, HealRequest, HealType, channel::HealChannelProcessor};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, OnceLock};
use tokio_util::sync::CancellationToken;
use tracing::{error, info};
@@ -55,6 +56,8 @@ static GLOBAL_HEAL_MANAGER: OnceLock<Arc<HealManager>> = OnceLock::new();
/// Global heal channel processor instance
static GLOBAL_HEAL_CHANNEL_PROCESSOR: OnceLock<Arc<tokio::sync::Mutex<HealChannelProcessor>>> = OnceLock::new();
static GLOBAL_HEAL_ACTIVE_TASKS: AtomicU64 = AtomicU64::new(0);
static GLOBAL_HEAL_QUEUE_LENGTH: AtomicU64 = AtomicU64::new(0);
/// Initialize and start heal manager with channel processor
pub async fn init_heal_manager(
@@ -107,3 +110,19 @@ pub fn get_heal_manager() -> Option<&'static Arc<HealManager>> {
pub fn get_heal_channel_processor() -> Option<&'static Arc<tokio::sync::Mutex<HealChannelProcessor>>> {
GLOBAL_HEAL_CHANNEL_PROCESSOR.get()
}
pub fn current_heal_active_tasks() -> u64 {
GLOBAL_HEAL_ACTIVE_TASKS.load(Ordering::Relaxed)
}
pub fn current_heal_queue_length() -> u64 {
GLOBAL_HEAL_QUEUE_LENGTH.load(Ordering::Relaxed)
}
pub(crate) fn set_heal_active_tasks(count: usize) {
GLOBAL_HEAL_ACTIVE_TASKS.store(count as u64, Ordering::Relaxed);
}
pub(crate) fn set_heal_queue_length(count: usize) {
GLOBAL_HEAL_QUEUE_LENGTH.store(count as u64, Ordering::Relaxed);
}