gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-24 13:16:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(storage): harden offline drive fail-fast paths (#2564)

Browse Source 
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>
This commit is contained in:
houseme
2026-04-16 17:21:45 +08:00
committed by GitHub
parent 579b124726
commit 28edfd6190
35 changed files with 4426 additions and 618 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/heal/src/error.rs
+8
View File
@@ -68,6 +68,9 @@ pub enum Error {
#[error("Invalid heal type: {heal_type}")]
InvalidHealType { heal_type: String },
#[error("Transient heal skip: {message}")]
TransientSkip { message: String },
#[error("Heal task cancelled")]
TaskCancelled,
@@ -92,6 +95,11 @@ impl Error {
{
Error::Other(error.into().to_string())
}
/// Create a transient skip error for retryable background heal checks.
pub fn transient_skip(message: impl Into<String>) -> Self {
Error::TransientSkip { message: message.into() }
}
}
impl From<Error> for std::io::Error {
crates/heal/src/heal/channel.rs
+136 -14
View File
@@ -19,11 +19,11 @@ use crate::heal::{
};
use crate::{Error, Result};
use rustfs_common::heal_channel::{
HealChannelCommand, HealChannelPriority, HealChannelReceiver, HealChannelRequest, HealChannelResponse, HealScanMode,
publish_heal_response,
HealAdmissionResult, HealChannelCommand, HealChannelPriority, HealChannelReceiver, HealChannelRequest, HealChannelResponse,
HealScanMode, publish_heal_response,
};
use std::sync::Arc;
use tokio::sync::mpsc;
use tokio::sync::{mpsc, oneshot};
use tracing::{debug, error, info};
/// Heal channel processor
@@ -82,14 +82,18 @@ impl HealChannelProcessor {
/// Process heal command
async fn process_command(&self, command: HealChannelCommand) -> Result<()> {
match command {
HealChannelCommand::Start(request) => self.process_start_request(request).await,
HealChannelCommand::Start { request, response_tx } => self.process_start_request(request, response_tx).await,
HealChannelCommand::Query { heal_path, client_token } => self.process_query_request(heal_path, client_token).await,
HealChannelCommand::Cancel { heal_path } => self.process_cancel_request(heal_path).await,
}
}
/// Process start request
async fn process_start_request(&self, request: HealChannelRequest) -> Result<()> {
async fn process_start_request(
&self,
request: HealChannelRequest,
response_tx: oneshot::Sender<std::result::Result<HealAdmissionResult, String>>,
) -> Result<()> {
info!(
"Processing heal start request: {} for bucket: {}/{}",
request.id,
@@ -98,31 +102,60 @@ impl HealChannelProcessor {
);
// Convert channel request to heal request
let heal_request = self.convert_to_heal_request(request.clone())?;
let heal_request = match self.convert_to_heal_request(request.clone()) {
Ok(heal_request) => heal_request,
Err(err) => {
let error_text = err.to_string();
let _ = response_tx.send(Err(error_text.clone()));
self.publish_response(HealChannelResponse {
request_id: request.id,
success: false,
data: None,
error: Some(error_text),
});
return Ok(());
}
};
// Submit to heal manager
match self.heal_manager.submit_heal_request(heal_request).await {
Ok(task_id) => {
info!("Successfully submitted heal request: {} as task: {}", request.id, task_id);
Ok(admission) => {
info!(
request_id = %request.id,
admission = admission.result_label(),
"Heal request admission decision completed"
);
let _ = response_tx.send(Ok(admission));
let (success, error) = match admission {
HealAdmissionResult::Accepted | HealAdmissionResult::Merged => (true, None),
HealAdmissionResult::Full => (false, Some("Heal request queue is full".to_string())),
HealAdmissionResult::Dropped(reason) => (false, Some(format!("Heal request dropped: {}", reason.as_str()))),
};
let response = HealChannelResponse {
request_id: request.id,
success: true,
data: Some(format!("Task ID: {task_id}").into_bytes()),
error: None,
success,
data: Some(
format!("admission={},reason={}", admission.result_label(), admission.reason_label()).into_bytes(),
),
error,
};
self.publish_response(response);
}
Err(e) => {
error!("Failed to submit heal request: {} - {}", request.id, e);
let error_text = e.to_string();
error!("Failed to submit heal request: {} - {}", request.id, error_text);
let _ = response_tx.send(Err(error_text.clone()));
// Send error response
let response = HealChannelResponse {
request_id: request.id,
success: false,
data: None,
error: Some(e.to_string()),
error: Some(error_text),
};
self.publish_response(response);
@@ -247,8 +280,9 @@ impl HealChannelProcessor {
#[cfg(test)]
mod tests {
use super::*;
use crate::heal::manager::HealConfig;
use crate::heal::storage::HealStorageAPI;
use rustfs_common::heal_channel::{HealChannelPriority, HealChannelRequest, HealScanMode};
use rustfs_common::heal_channel::{HealAdmissionResult, HealChannelPriority, HealChannelRequest, HealScanMode};
use std::sync::Arc;
// Mock storage for testing
@@ -569,4 +603,92 @@ mod tests {
let heal_request = processor.convert_to_heal_request(channel_request).unwrap();
assert!(matches!(heal_request.heal_type, HealType::Bucket { .. }));
}
#[tokio::test]
async fn test_process_start_request_returns_admission_result() {
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
let manager = Arc::new(HealManager::new(
storage,
Some(HealConfig {
queue_size: 1,
..HealConfig::default()
}),
));
let processor = HealChannelProcessor::new(manager);
let request = HealChannelRequest {
id: "admission-id".to_string(),
bucket: "bucket".to_string(),
object_prefix: Some("object".to_string()),
object_version_id: None,
disk: None,
priority: HealChannelPriority::Low,
scan_mode: Some(HealScanMode::Normal),
remove_corrupted: None,
recreate_missing: None,
update_parity: None,
recursive: None,
dry_run: None,
timeout_seconds: None,
pool_index: None,
set_index: None,
force_start: false,
};
let (tx, rx) = oneshot::channel();
processor
.process_start_request(request.clone(), tx)
.await
.expect("first admission should succeed");
assert_eq!(
rx.await
.expect("oneshot should resolve")
.expect("admission should be returned"),
HealAdmissionResult::Accepted
);
let (tx, rx) = oneshot::channel();
processor
.process_start_request(request, tx)
.await
.expect("duplicate admission should succeed");
assert_eq!(
rx.await
.expect("oneshot should resolve")
.expect("admission should be returned"),
HealAdmissionResult::Merged
);
}
#[tokio::test]
async fn test_process_start_request_returns_error_on_invalid_request() {
let heal_manager = create_test_heal_manager();
let processor = HealChannelProcessor::new(heal_manager);
let request = HealChannelRequest {
id: "invalid-id".to_string(),
bucket: "bucket".to_string(),
object_prefix: None,
object_version_id: None,
disk: Some("invalid".to_string()),
priority: HealChannelPriority::Normal,
scan_mode: None,
remove_corrupted: None,
recreate_missing: None,
update_parity: None,
recursive: None,
dry_run: None,
timeout_seconds: None,
pool_index: None,
set_index: None,
force_start: false,
};
let (tx, rx) = oneshot::channel();
processor
.process_start_request(request, tx)
.await
.expect("processor should surface invalid request through response channel");
assert!(rx.await.expect("oneshot should resolve").is_err());
}
}
crates/heal/src/heal/erasure_healer.rs
+196 -63
View File
@@ -18,11 +18,15 @@ use crate::heal::{
storage::HealStorageAPI,
};
use crate::{Error, Result};
use futures::future::join_all;
use futures::{StreamExt, future::join_all, stream::FuturesUnordered};
use metrics::gauge;
use rustfs_common::heal_channel::{HealOpts, HealScanMode};
use rustfs_ecstore::disk::DiskStore;
use std::sync::Arc;
use tokio::sync::RwLock;
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
use tokio::sync::{RwLock, Semaphore};
use tracing::{error, info, warn};
/// Erasure Set Healer
@@ -34,6 +38,29 @@ pub struct ErasureSetHealer {
}
impl ErasureSetHealer {
fn page_parallel_enabled() -> bool {
rustfs_utils::get_env_bool(
rustfs_config::ENV_HEAL_PAGE_PARALLEL_ENABLE,
rustfs_config::DEFAULT_HEAL_PAGE_PARALLEL_ENABLE,
)
}
fn heal_page_object_concurrency() -> usize {
rustfs_utils::get_env_usize(
rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY,
rustfs_config::DEFAULT_HEAL_PAGE_OBJECT_CONCURRENCY,
)
.max(1)
}
fn effective_heal_page_object_concurrency() -> usize {
if Self::page_parallel_enabled() {
Self::heal_page_object_concurrency()
} else {
1
}
}
pub fn new(
storage: Arc<dyn HealStorageAPI>,
progress: Arc<RwLock<HealProgress>>,
@@ -61,7 +88,7 @@ impl ErasureSetHealer {
// 3. execute heal with resume
let result = self
.execute_heal_with_resume(buckets, &resume_manager, &checkpoint_manager)
.execute_heal_with_resume(buckets, set_disk_id, &resume_manager, &checkpoint_manager)
.await;
// 4. cleanup resume state
@@ -144,6 +171,7 @@ impl ErasureSetHealer {
async fn execute_heal_with_resume(
&self,
buckets: &[String],
set_disk_id: &str,
resume_manager: &ResumeManager,
checkpoint_manager: &CheckpointManager,
) -> Result<()> {
@@ -182,6 +210,7 @@ impl ErasureSetHealer {
let bucket_result = self
.heal_bucket_with_resume(
bucket,
set_disk_id,
bucket_idx,
&mut current_object_index,
&mut processed_objects,
@@ -232,16 +261,17 @@ impl ErasureSetHealer {
/// heal single bucket with resume
#[allow(clippy::too_many_arguments)]
#[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, _skipped_objects, resume_manager, checkpoint_manager), fields(bucket = %bucket, bucket_index = bucket_index))]
#[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, skipped_objects, resume_manager, checkpoint_manager), fields(bucket = %bucket, bucket_index = bucket_index))]
async fn heal_bucket_with_resume(
&self,
bucket: &str,
set_disk_id: &str,
bucket_index: usize,
current_object_index: &mut usize,
processed_objects: &mut u64,
successful_objects: &mut u64,
failed_objects: &mut u64,
_skipped_objects: &mut u64,
skipped_objects: &mut u64,
resume_manager: &ResumeManager,
checkpoint_manager: &CheckpointManager,
) -> Result<()> {
@@ -259,6 +289,8 @@ impl ErasureSetHealer {
// 2. process objects with pagination to avoid loading all objects into memory
let mut continuation_token: Option<String> = None;
let mut global_obj_idx = 0usize;
let page_concurrency_limit = Self::effective_heal_page_object_concurrency();
let in_flight = Arc::new(AtomicUsize::new(0));
loop {
// Get one page of objects
@@ -266,69 +298,139 @@ impl ErasureSetHealer {
.storage
.list_objects_for_heal_page(bucket, "", continuation_token.as_deref())
.await?;
let checkpoint = checkpoint_manager.get_checkpoint().await;
let page_resume_index = *current_object_index;
let semaphore = Arc::new(Semaphore::new(page_concurrency_limit));
let mut page_tasks = FuturesUnordered::new();
// Process objects in this page
for object in objects {
// Skip objects before the checkpoint
if global_obj_idx < *current_object_index {
global_obj_idx += 1;
let object_idx = global_obj_idx;
global_obj_idx += 1;
if object_idx < *current_object_index {
continue;
}
// check if already processed
if checkpoint_manager.get_checkpoint().await.processed_objects.contains(&object) {
global_obj_idx += 1;
if checkpoint.processed_objects.contains(&object) || checkpoint.skipped_objects.contains(&object) {
continue;
}
// update current object
resume_manager
.set_current_item(Some(bucket.to_string()), Some(object.clone()))
.await?;
// Check if object still exists before attempting heal
let object_exists = match self.storage.object_exists(bucket, &object).await {
Ok(exists) => exists,
Err(e) => {
warn!("Failed to check existence of {}/{}: {}, marking as failed", bucket, object, e);
*failed_objects += 1;
checkpoint_manager.add_failed_object(object.clone()).await?;
global_obj_idx += 1;
*current_object_index = global_obj_idx;
continue;
}
};
let storage = self.storage.clone();
let bucket_name = bucket.to_string();
let object_name = object.clone();
let cancel_token = self.cancel_token.clone();
let in_flight = in_flight.clone();
let set_label = set_disk_id.to_string();
let permit = semaphore
.clone()
.acquire_owned()
.await
.map_err(|e| Error::other(format!("Failed to acquire page concurrency permit: {e}")))?;
if !object_exists {
info!(
target: "rustfs:heal:heal_bucket_with_resume" ,"Object {}/{} no longer exists, skipping heal (likely deleted intentionally)",
bucket, object
);
checkpoint_manager.add_processed_object(object.clone()).await?;
*successful_objects += 1; // Treat as successful - object is gone as intended
global_obj_idx += 1;
*current_object_index = global_obj_idx;
continue;
}
let current_in_flight = in_flight.fetch_add(1, Ordering::SeqCst) + 1;
gauge!(
"rustfs_heal_page_concurrency_current",
"set" => set_label.clone()
)
.set(current_in_flight as f64);
// heal object
let heal_opts = HealOpts {
scan_mode: HealScanMode::Normal,
remove: true,
recreate: true, // Keep recreate enabled for legitimate heal scenarios
..Default::default()
};
page_tasks.push(async move {
let _permit = permit;
let result = if cancel_token.is_cancelled() {
Err(Error::TaskCancelled)
} else {
let object_exists = match storage.object_exists(&bucket_name, &object_name).await {
Ok(exists) => exists,
Err(err @ Error::TransientSkip { .. }) => {
let current = in_flight.fetch_sub(1, Ordering::SeqCst) - 1;
gauge!(
"rustfs_heal_page_concurrency_current",
"set" => set_label.clone()
)
.set(current as f64);
return (object_name, Err(err));
}
Err(err) => {
let object_name_for_error = object_name.clone();
let current = in_flight.fetch_sub(1, Ordering::SeqCst) - 1;
gauge!(
"rustfs_heal_page_concurrency_current",
"set" => set_label.clone()
)
.set(current as f64);
return (
object_name,
Err(Error::other(format!(
"Failed to check existence of {}/{}: {}",
bucket_name, object_name_for_error, err
))),
);
}
};
match self.storage.heal_object(bucket, &object, None, &heal_opts).await {
Ok((_result, None)) => {
if !object_exists {
Ok(false)
} else {
let heal_opts = HealOpts {
scan_mode: HealScanMode::Normal,
remove: true,
recreate: true,
..Default::default()
};
match storage.heal_object(&bucket_name, &object_name, None, &heal_opts).await {
Ok((_result, None)) => Ok(true),
Ok((_, Some(err))) => Err(Error::other(err)),
Err(err) => Err(err),
}
}
};
let current = in_flight.fetch_sub(1, Ordering::SeqCst) - 1;
gauge!(
"rustfs_heal_page_concurrency_current",
"set" => set_label.clone()
)
.set(current as f64);
(object_name, result)
});
}
let mut completed_in_page = 0usize;
while let Some((object, result)) = page_tasks.next().await {
match result {
Ok(true) => {
*successful_objects += 1;
checkpoint_manager.add_processed_object(object.clone()).await?;
info!("Successfully healed object {}/{}", bucket, object);
}
Ok((_, Some(err))) => {
*failed_objects += 1;
checkpoint_manager.add_failed_object(object.clone()).await?;
warn!("Failed to heal object {}/{}: {}", bucket, object, err);
Ok(false) => {
checkpoint_manager.add_processed_object(object.clone()).await?;
*successful_objects += 1;
info!(
target: "rustfs:heal:heal_bucket_with_resume" ,"Object {}/{} no longer exists, skipping heal (likely deleted intentionally)",
bucket, object
);
}
Err(Error::TaskCancelled) => {
gauge!(
"rustfs_heal_page_concurrency_current",
"set" => set_disk_id.to_string()
)
.set(0.0);
return Err(Error::TaskCancelled);
}
Err(Error::TransientSkip { message }) => {
*skipped_objects += 1;
checkpoint_manager.add_skipped_object(object.clone()).await?;
warn!(
"Skipping heal for object {}/{} due to transient existence check error: {}",
bucket, object, message
);
}
Err(err) => {
*failed_objects += 1;
@@ -338,23 +440,23 @@ impl ErasureSetHealer {
}
*processed_objects += 1;
global_obj_idx += 1;
*current_object_index = global_obj_idx;
completed_in_page += 1;
// check cancel status
if self.cancel_token.is_cancelled() {
info!("Heal task cancelled during object processing");
return Err(Error::TaskCancelled);
}
// save checkpoint periodically
if global_obj_idx.is_multiple_of(100) {
checkpoint_manager
.update_position(bucket_index, *current_object_index)
.await?;
if completed_in_page.is_multiple_of(100) {
checkpoint_manager.update_position(bucket_index, page_resume_index).await?;
}
}
*current_object_index = global_obj_idx;
checkpoint_manager
.update_position(bucket_index, *current_object_index)
.await?;
gauge!(
"rustfs_heal_page_concurrency_current",
"set" => set_disk_id.to_string()
)
.set(0.0);
// Check if there are more pages
if !is_truncated {
break;
@@ -572,3 +674,34 @@ impl ErasureSetHealer {
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::ErasureSetHealer;
#[test]
fn heal_page_object_concurrency_uses_default_when_env_is_unset() {
temp_env::with_var_unset(rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, || {
assert_eq!(
ErasureSetHealer::heal_page_object_concurrency(),
rustfs_config::DEFAULT_HEAL_PAGE_OBJECT_CONCURRENCY
);
});
}
#[test]
fn heal_page_object_concurrency_respects_env_override() {
temp_env::with_var(rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("11"), || {
assert_eq!(ErasureSetHealer::heal_page_object_concurrency(), 11);
});
}
#[test]
fn effective_heal_page_object_concurrency_disables_parallelism_when_flag_is_off() {
temp_env::with_var(rustfs_config::ENV_HEAL_PAGE_PARALLEL_ENABLE, Some("false"), || {
temp_env::with_var(rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("11"), || {
assert_eq!(ErasureSetHealer::effective_heal_page_object_concurrency(), 1);
});
});
}
}
crates/heal/src/heal/manager.rs
+667 -71
View File
@@ -18,6 +18,8 @@ use crate::heal::{
task::{HealOptions, HealPriority, HealRequest, HealTask, HealTaskStatus, HealType},
};
use crate::{Error, Result};
use metrics::{counter, gauge};
use rustfs_common::heal_channel::{HealAdmissionDropReason, HealAdmissionResult};
use rustfs_ecstore::disk::DiskAPI;
use rustfs_ecstore::disk::error::DiskError;
use rustfs_ecstore::global::GLOBAL_LOCAL_DISK_MAP;
@@ -27,7 +29,7 @@ use std::{
time::{Duration, SystemTime},
};
use tokio::{
sync::{Mutex, RwLock},
sync::{Mutex, Notify, RwLock},
time::interval,
};
use tokio_util::sync::CancellationToken;
@@ -80,6 +82,12 @@ impl PartialOrd for PriorityQueueItem {
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum QueuePushOutcome {
Accepted,
Merged,
}
impl PriorityHealQueue {
fn new() -> Self {
Self {
@@ -93,16 +101,24 @@ impl PriorityHealQueue {
self.heap.len()
}
fn pop_next(&mut self) -> Option<HealRequest> {
self.heap.pop().map(|item| {
let key = Self::make_dedup_key(&item.request);
self.dedup_keys.remove(&key);
item.request
})
}
fn is_empty(&self) -> bool {
self.heap.is_empty()
}
fn push(&mut self, request: HealRequest) -> bool {
fn push(&mut self, request: HealRequest) -> QueuePushOutcome {
let key = Self::make_dedup_key(&request);
// Check for duplicates
if self.dedup_keys.contains(&key) {
return false; // Duplicate request, don't add
return QueuePushOutcome::Merged;
}
self.dedup_keys.insert(key);
@@ -112,7 +128,7 @@ impl PriorityHealQueue {
sequence: self.sequence,
request,
});
true
QueuePushOutcome::Accepted
}
/// Get statistics about queue contents by priority
@@ -124,6 +140,7 @@ impl PriorityHealQueue {
stats
}
#[cfg(test)]
fn pop(&mut self) -> Option<HealRequest> {
self.heap.pop().map(|item| {
let key = Self::make_dedup_key(&item.request);
@@ -132,6 +149,48 @@ impl PriorityHealQueue {
})
}
#[cfg(test)]
fn pop_runnable<F>(&mut self, can_run: F) -> Option<HealRequest>
where
F: Fn(&HealRequest) -> bool,
{
self.pop_runnable_with_skips(can_run, |_| None).0
}
fn pop_runnable_with_skips<F, G>(&mut self, can_run: F, skip_label: G) -> (Option<HealRequest>, Vec<String>)
where
F: Fn(&HealRequest) -> bool,
G: Fn(&HealRequest) -> Option<String>,
{
let mut deferred = Vec::new();
let mut selected = None;
let mut skipped = Vec::new();
while let Some(item) = self.heap.pop() {
if can_run(&item.request) {
selected = Some(item);
break;
}
if let Some(label) = skip_label(&item.request) {
skipped.push(label);
}
deferred.push(item);
}
for item in deferred {
self.heap.push(item);
}
(
selected.map(|item| {
let key = Self::make_dedup_key(&item.request);
self.dedup_keys.remove(&key);
item.request
}),
skipped,
)
}
/// Create a deduplication key from a heal request
fn make_dedup_key(request: &HealRequest) -> String {
match &request.heal_type {
@@ -187,10 +246,22 @@ pub struct HealConfig {
pub heal_interval: Duration,
/// Maximum concurrent heal tasks
pub max_concurrent_heals: usize,
/// Maximum concurrent heal tasks allowed for a single erasure set
pub max_concurrent_per_set: usize,
/// Task timeout
pub task_timeout: Duration,
/// Queue size
pub queue_size: usize,
/// Whether duplicate low-priority requests should merge into an existing queued request.
pub low_priority_merge_enable: bool,
/// Whether low-priority requests may be dropped when the queue is full.
pub low_priority_drop_when_full: bool,
/// Whether notify-driven scheduler wakeups are enabled.
pub event_driven_scheduler_enable: bool,
/// Whether per-set bulkhead scheduling is enabled.
pub set_bulkhead_enable: bool,
/// Whether erasure-set page parallelism is enabled.
pub page_parallel_enable: bool,
}
impl Default for HealConfig {
@@ -211,12 +282,42 @@ impl Default for HealConfig {
rustfs_config::ENV_HEAL_MAX_CONCURRENT_HEALS,
rustfs_config::DEFAULT_HEAL_MAX_CONCURRENT_HEALS,
);
let max_concurrent_per_set = rustfs_utils::get_env_usize(
rustfs_config::ENV_HEAL_MAX_CONCURRENT_PER_SET,
rustfs_config::DEFAULT_HEAL_MAX_CONCURRENT_PER_SET,
);
let low_priority_merge_enable = rustfs_utils::get_env_bool(
rustfs_config::ENV_HEAL_LOW_PRIORITY_MERGE_ENABLE,
rustfs_config::DEFAULT_HEAL_LOW_PRIORITY_MERGE_ENABLE,
);
let low_priority_drop_when_full = rustfs_utils::get_env_bool(
rustfs_config::ENV_HEAL_LOW_PRIORITY_DROP_WHEN_FULL,
rustfs_config::DEFAULT_HEAL_LOW_PRIORITY_DROP_WHEN_FULL,
);
let event_driven_scheduler_enable = rustfs_utils::get_env_bool(
rustfs_config::ENV_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE,
rustfs_config::DEFAULT_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE,
);
let set_bulkhead_enable = rustfs_utils::get_env_bool(
rustfs_config::ENV_HEAL_SET_BULKHEAD_ENABLE,
rustfs_config::DEFAULT_HEAL_SET_BULKHEAD_ENABLE,
);
let page_parallel_enable = rustfs_utils::get_env_bool(
rustfs_config::ENV_HEAL_PAGE_PARALLEL_ENABLE,
rustfs_config::DEFAULT_HEAL_PAGE_PARALLEL_ENABLE,
);
Self {
enable_auto_heal,
heal_interval, // 10 seconds
max_concurrent_heals, // max 4,
task_timeout, // 5 minutes
max_concurrent_per_set: std::cmp::min(max_concurrent_heals.max(1), max_concurrent_per_set.max(1)),
task_timeout, // 5 minutes
queue_size,
low_priority_merge_enable,
low_priority_drop_when_full,
event_driven_scheduler_enable,
set_bulkhead_enable,
page_parallel_enable,
}
}
}
@@ -254,9 +355,19 @@ pub struct HealManager {
cancel_token: CancellationToken,
/// Statistics
statistics: Arc<RwLock<HealStatistics>>,
/// Scheduler wake-up notifier for event-driven dispatch
notify: Arc<Notify>,
}
impl HealManager {
fn classify_full_admission(request: &HealRequest, config: &HealConfig) -> HealAdmissionResult {
if request.priority == HealPriority::Low && config.low_priority_drop_when_full {
HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)
} else {
HealAdmissionResult::Full
}
}
/// Create new HealManager
pub fn new(storage: Arc<dyn HealStorageAPI>, config: Option<HealConfig>) -> Self {
let config = config.unwrap_or_default();
@@ -268,6 +379,7 @@ impl HealManager {
storage,
cancel_token: CancellationToken::new(),
statistics: Arc::new(RwLock::new(HealStatistics::new())),
notify: Arc::new(Notify::new()),
}
}
@@ -318,17 +430,63 @@ impl HealManager {
}
/// Submit heal request
pub async fn submit_heal_request(&self, request: HealRequest) -> Result<String> {
pub async fn submit_heal_request(&self, request: HealRequest) -> Result<HealAdmissionResult> {
let config = self.config.read().await;
let mut queue = self.heal_queue.lock().await;
let queue_len = queue.len();
let queue_capacity = config.queue_size;
if queue.contains_key(&request) {
let admission = if request.priority == HealPriority::Low && !config.low_priority_merge_enable {
HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped)
} else {
HealAdmissionResult::Merged
};
match admission {
HealAdmissionResult::Merged => {
info!("Heal request already queued (duplicate merged): {}", request.id);
}
HealAdmissionResult::Dropped(reason) => {
warn!(
request_id = %request.id,
priority = ?request.priority,
reason = reason.as_str(),
"Dropping duplicate heal request due to admission policy"
);
}
HealAdmissionResult::Accepted | HealAdmissionResult::Full => {}
}
return Ok(admission);
}
if queue_len >= queue_capacity {
return Err(Error::ConfigurationError {
message: format!("Heal queue is full ({queue_len}/{queue_capacity})"),
});
let admission = Self::classify_full_admission(&request, &config);
match admission {
HealAdmissionResult::Dropped(reason) => {
warn!(
request_id = %request.id,
priority = ?request.priority,
queue_len,
queue_capacity,
reason = reason.as_str(),
"Dropping heal request because the queue is full"
);
}
HealAdmissionResult::Full => {
warn!(
request_id = %request.id,
priority = ?request.priority,
queue_len,
queue_capacity,
"Rejecting heal request because the queue is full"
);
}
HealAdmissionResult::Accepted | HealAdmissionResult::Merged => {}
}
return Ok(admission);
}
// Warn when queue is getting full (>80% capacity)
@@ -345,8 +503,8 @@ impl HealManager {
let request_id = request.id.clone();
let priority = request.priority;
// Try to push the request; if it's a duplicate, still return the request_id
let is_new = queue.push(request);
let push_outcome = queue.push(request);
debug_assert_eq!(push_outcome, QueuePushOutcome::Accepted);
// Log queue statistics periodically (when adding high/urgent priority items)
if matches!(priority, HealPriority::High | HealPriority::Urgent) {
@@ -364,13 +522,12 @@ impl HealManager {
drop(queue);
if is_new {
info!("Submitted heal request: {} with priority: {:?}", request_id, priority);
} else {
info!("Heal request already queued (duplicate): {}", request_id);
info!("Submitted heal request: {} with priority: {:?}", request_id, priority);
if config.event_driven_scheduler_enable {
self.notify.notify_one();
}
Ok(request_id)
Ok(HealAdmissionResult::Accepted)
}
/// Get task status
@@ -441,18 +598,23 @@ impl HealManager {
let cancel_token = self.cancel_token.clone();
let statistics = self.statistics.clone();
let storage = self.storage.clone();
let notify = self.notify.clone();
tokio::spawn(async move {
let mut interval = interval(config.read().await.heal_interval);
loop {
let event_driven_scheduler_enable = config.read().await.event_driven_scheduler_enable;
tokio::select! {
_ = cancel_token.cancelled() => {
info!("Heal scheduler received shutdown signal");
break;
}
_ = notify.notified(), if event_driven_scheduler_enable => {
Self::process_heal_queue(&heal_queue, &active_heals, &config, &statistics, &storage, &notify).await;
}
_ = interval.tick() => {
Self::process_heal_queue(&heal_queue, &active_heals, &config, &statistics, &storage).await;
Self::process_heal_queue(&heal_queue, &active_heals, &config, &statistics, &storage, &notify).await;
}
}
}
@@ -468,6 +630,7 @@ impl HealManager {
let active_heals = self.active_heals.clone();
let cancel_token = self.cancel_token.clone();
let storage = self.storage.clone();
let notify = self.notify.clone();
let mut duration = {
let config = config.read().await;
config.heal_interval
@@ -567,8 +730,13 @@ impl HealManager {
HealPriority::Normal,
);
let mut queue = heal_queue.lock().await;
queue.push(req);
info!("start_auto_disk_scanner: Enqueued auto erasure set heal for endpoint: {} (set_disk_id: {})", ep, set_disk_id);
if matches!(queue.push(req), QueuePushOutcome::Accepted) {
let config = config.read().await;
if config.event_driven_scheduler_enable {
notify.notify_one();
}
info!("start_auto_disk_scanner: Enqueued auto erasure set heal for endpoint: {} (set_disk_id: {})", ep, set_disk_id);
}
}
}
}
@@ -585,6 +753,7 @@ impl HealManager {
config: &Arc<RwLock<HealConfig>>,
statistics: &Arc<RwLock<HealStatistics>>,
storage: &Arc<dyn HealStorageAPI>,
notify: &Arc<Notify>,
) {
let config = config.read().await;
let mut active_heals_guard = active_heals.lock().await;
@@ -605,28 +774,49 @@ impl HealManager {
return;
}
// Process multiple tasks if:
// 1. We have available slots
// 2. Queue is not empty
// Prioritize urgent/high priority tasks by processing up to 2 tasks per cycle if available
let tasks_to_process = if queue_len > 0 {
std::cmp::min(available_slots, std::cmp::min(2, queue_len))
} else {
0
};
let mut running_per_set = running_erasure_set_counts(&active_heals_guard);
let mut tasks_started = 0usize;
for _ in 0..tasks_to_process {
if let Some(request) = queue.pop() {
for _ in 0..available_slots {
let selected_request = if config.set_bulkhead_enable {
let max_concurrent_per_set = config.max_concurrent_per_set;
let (selected_request, skipped_sets) = queue.pop_runnable_with_skips(
|request| can_schedule_request(request, &running_per_set, max_concurrent_per_set),
|request| heal_request_set_key(request).map(|_| heal_request_set_metric_label(request)),
);
for skipped_set in skipped_sets {
record_scheduler_skip(&skipped_set);
}
selected_request
} else {
queue.pop_next()
};
if let Some(request) = selected_request {
let task_priority = request.priority;
let task_type_label = heal_request_type_label(&request).to_string();
let task_set_label = heal_request_set_metric_label(&request);
if config.set_bulkhead_enable
&& let Some(set_key) = heal_request_set_key(&request)
{
*running_per_set.entry(set_key).or_insert(0) += 1;
}
let task = Arc::new(HealTask::from_request(request, storage.clone()));
let task_id = task.id.clone();
active_heals_guard.insert(task_id.clone(), task.clone());
update_task_running_metric_for_task(&active_heals_guard, task.as_ref());
let active_heals_clone = active_heals.clone();
let statistics_clone = statistics.clone();
let notify_clone = notify.clone();
let task_type_label_for_spawn = task_type_label.clone();
let task_set_label_for_spawn = task_set_label.clone();
// start heal task
tokio::spawn(async move {
info!("Starting heal task: {} with priority: {:?}", task_id, task_priority);
info!(
"Starting heal task: {} with priority: {:?}, type: {}, set: {}",
task_id, task_priority, task_type_label_for_spawn, task_set_label_for_spawn
);
let result = task.execute().await;
match result {
Ok(_) => {
@@ -638,6 +828,7 @@ impl HealManager {
}
let mut active_heals_guard = active_heals_clone.lock().await;
if let Some(completed_task) = active_heals_guard.remove(&task_id) {
update_task_running_metric_for_task(&active_heals_guard, completed_task.as_ref());
// update statistics
let mut stats = statistics_clone.write().await;
match completed_task.get_status().await {
@@ -650,7 +841,9 @@ impl HealManager {
}
stats.update_running_tasks(active_heals_guard.len() as u64);
}
notify_clone.notify_one();
});
tasks_started += 1;
} else {
break;
}
@@ -658,7 +851,8 @@ impl HealManager {
// Update statistics for all started tasks
let mut stats = statistics.write().await;
stats.total_tasks += tasks_to_process as u64;
stats.total_tasks += tasks_started as u64;
stats.update_running_tasks(active_heals_guard.len() as u64);
// Log queue status if items remain
if !queue.is_empty() {
@@ -681,10 +875,180 @@ impl std::fmt::Debug for HealManager {
}
}
fn heal_request_set_key(request: &HealRequest) -> Option<String> {
match &request.heal_type {
HealType::ErasureSet { set_disk_id, .. } => Some(set_disk_id.clone()),
_ => None,
}
}
fn heal_request_type_label(request: &HealRequest) -> &'static str {
match &request.heal_type {
HealType::Object { .. } => "object",
HealType::Bucket { .. } => "bucket",
HealType::ErasureSet { .. } => "erasure_set",
HealType::Metadata { .. } => "metadata",
HealType::MRF { .. } => "mrf",
HealType::ECDecode { .. } => "ec_decode",
}
}
fn heal_request_set_metric_label(request: &HealRequest) -> String {
heal_request_set_key(request).unwrap_or_else(|| match (request.options.pool_index, request.options.set_index) {
(Some(pool), Some(set)) => format!("pool_{pool}_set_{set}"),
_ => "global".to_string(),
})
}
fn record_scheduler_skip(set_label: &str) {
counter!(
"rustfs_heal_scheduler_skip_total",
"reason" => "set_limit".to_string(),
"set" => set_label.to_string()
)
.increment(1);
}
fn update_task_running_metric_for_task(active_heals: &HashMap<String, Arc<HealTask>>, task: &HealTask) {
let type_label = task.metric_type_label();
let set_label = task.metric_set_label();
let count = active_heals
.values()
.filter(|active_task| active_task.metric_type_label() == type_label && active_task.metric_set_label() == set_label)
.count();
gauge!(
"rustfs_heal_task_running",
"type" => type_label.to_string(),
"set" => set_label.clone()
)
.set(count as f64);
}
fn running_erasure_set_counts(active_heals: &HashMap<String, Arc<HealTask>>) -> HashMap<String, usize> {
let mut running = HashMap::new();
for task in active_heals.values() {
if let HealType::ErasureSet { set_disk_id, .. } = &task.heal_type {
*running.entry(set_disk_id.clone()).or_insert(0) += 1;
}
}
running
}
fn can_schedule_request(request: &HealRequest, running_per_set: &HashMap<String, usize>, max_concurrent_per_set: usize) -> bool {
match heal_request_set_key(request) {
Some(set_key) => running_per_set.get(&set_key).copied().unwrap_or(0) < max_concurrent_per_set,
None => true,
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::heal::storage::HealStorageAPI;
use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealType};
use rustfs_common::heal_channel::HealOpts;
use rustfs_ecstore::{
disk::{DiskStore, endpoint::Endpoint},
store_api::BucketInfo,
};
use rustfs_madmin::heal_commands::HealResultItem;
struct MockStorage;
#[async_trait::async_trait]
impl HealStorageAPI for MockStorage {
async fn get_object_meta(&self, _bucket: &str, _object: &str) -> Result<Option<rustfs_ecstore::store_api::ObjectInfo>> {
Ok(None)
}
async fn get_object_data(&self, _bucket: &str, _object: &str) -> Result<Option<Vec<u8>>> {
Ok(None)
}
async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> Result<()> {
Ok(())
}
async fn delete_object(&self, _bucket: &str, _object: &str) -> Result<()> {
Ok(())
}
async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> Result<bool> {
Ok(true)
}
async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> Result<Vec<u8>> {
Ok(Vec::new())
}
async fn get_disk_status(&self, _endpoint: &Endpoint) -> Result<crate::heal::storage::DiskStatus> {
Ok(crate::heal::storage::DiskStatus::Ok)
}
async fn format_disk(&self, _endpoint: &Endpoint) -> Result<()> {
Ok(())
}
async fn get_bucket_info(&self, _bucket: &str) -> Result<Option<BucketInfo>> {
Ok(None)
}
async fn heal_bucket_metadata(&self, _bucket: &str) -> Result<()> {
Ok(())
}
async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
Ok(Vec::new())
}
async fn object_exists(&self, _bucket: &str, _object: &str) -> Result<bool> {
Ok(false)
}
async fn get_object_size(&self, _bucket: &str, _object: &str) -> Result<Option<u64>> {
Ok(None)
}
async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> Result<Option<String>> {
Ok(None)
}
async fn heal_object(
&self,
_bucket: &str,
_object: &str,
_version_id: Option<&str>,
_opts: &HealOpts,
) -> Result<(HealResultItem, Option<Error>)> {
Ok((HealResultItem::default(), None))
}
async fn heal_bucket(&self, _bucket: &str, _opts: &HealOpts) -> Result<HealResultItem> {
Ok(HealResultItem::default())
}
async fn heal_format(&self, _dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
Ok((HealResultItem::default(), None))
}
async fn list_objects_for_heal(&self, _bucket: &str, _prefix: &str) -> Result<Vec<String>> {
Ok(Vec::new())
}
async fn list_objects_for_heal_page(
&self,
_bucket: &str,
_prefix: &str,
_continuation_token: Option<&str>,
) -> Result<(Vec<String>, Option<String>, bool)> {
Ok((Vec::new(), None, false))
}
async fn get_disk_for_resume(&self, _set_disk_id: &str) -> Result<DiskStore> {
Err(Error::other("not implemented in tests"))
}
}
#[test]
fn test_priority_queue_ordering() {
@@ -724,10 +1088,10 @@ mod tests {
);
// Add in random order: low, high, normal, urgent
assert!(queue.push(low_req));
assert!(queue.push(high_req));
assert!(queue.push(normal_req));
assert!(queue.push(urgent_req));
assert_eq!(queue.push(low_req), QueuePushOutcome::Accepted);
assert_eq!(queue.push(high_req), QueuePushOutcome::Accepted);
assert_eq!(queue.push(normal_req), QueuePushOutcome::Accepted);
assert_eq!(queue.push(urgent_req), QueuePushOutcome::Accepted);
assert_eq!(queue.len(), 4);
@@ -780,9 +1144,9 @@ mod tests {
let id2 = req2.id.clone();
let id3 = req3.id.clone();
assert!(queue.push(req1));
assert!(queue.push(req2));
assert!(queue.push(req3));
assert_eq!(queue.push(req1), QueuePushOutcome::Accepted);
assert_eq!(queue.push(req2), QueuePushOutcome::Accepted);
assert_eq!(queue.push(req3), QueuePushOutcome::Accepted);
// Should maintain FIFO order for same priority
let popped1 = queue.pop().unwrap();
@@ -820,11 +1184,11 @@ mod tests {
);
// First request should be added
assert!(queue.push(req1));
assert_eq!(queue.push(req1), QueuePushOutcome::Accepted);
assert_eq!(queue.len(), 1);
// Second request with same object should be rejected (duplicate)
assert!(!queue.push(req2));
assert_eq!(queue.push(req2), QueuePushOutcome::Merged);
assert_eq!(queue.len(), 1);
}
@@ -841,7 +1205,7 @@ mod tests {
HealPriority::Normal,
);
assert!(queue.push(req));
assert_eq!(queue.push(req), QueuePushOutcome::Accepted);
assert!(queue.contains_erasure_set("pool_0_set_1"));
assert!(!queue.contains_erasure_set("pool_0_set_2"));
}
@@ -929,7 +1293,8 @@ mod tests {
for (heal_type, priority) in requests {
let req = HealRequest::new(heal_type, HealOptions::default(), priority);
queue.push(req);
let outcome = queue.push(req);
assert_eq!(outcome, QueuePushOutcome::Accepted);
}
assert_eq!(queue.len(), 4);
@@ -954,32 +1319,41 @@ mod tests {
// Add requests with different priorities
for _ in 0..3 {
queue.push(HealRequest::new(
HealType::Bucket {
bucket: format!("bucket-low-{}", queue.len()),
},
HealOptions::default(),
HealPriority::Low,
));
assert_eq!(
queue.push(HealRequest::new(
HealType::Bucket {
bucket: format!("bucket-low-{}", queue.len()),
},
HealOptions::default(),
HealPriority::Low,
)),
QueuePushOutcome::Accepted
);
}
for _ in 0..2 {
queue.push(HealRequest::new(
HealType::Bucket {
bucket: format!("bucket-normal-{}", queue.len()),
},
HealOptions::default(),
HealPriority::Normal,
));
assert_eq!(
queue.push(HealRequest::new(
HealType::Bucket {
bucket: format!("bucket-normal-{}", queue.len()),
},
HealOptions::default(),
HealPriority::Normal,
)),
QueuePushOutcome::Accepted
);
}
queue.push(HealRequest::new(
HealType::Bucket {
bucket: "bucket-high".to_string(),
},
HealOptions::default(),
HealPriority::High,
));
assert_eq!(
queue.push(HealRequest::new(
HealType::Bucket {
bucket: "bucket-high".to_string(),
},
HealOptions::default(),
HealPriority::High,
)),
QueuePushOutcome::Accepted
);
let stats = queue.get_priority_stats();
@@ -995,13 +1369,16 @@ mod tests {
assert!(queue.is_empty());
queue.push(HealRequest::new(
HealType::Bucket {
bucket: "test".to_string(),
},
HealOptions::default(),
HealPriority::Normal,
));
assert_eq!(
queue.push(HealRequest::new(
HealType::Bucket {
bucket: "test".to_string(),
},
HealOptions::default(),
HealPriority::Normal,
)),
QueuePushOutcome::Accepted
);
assert!(!queue.is_empty());
@@ -1009,4 +1386,223 @@ mod tests {
assert!(queue.is_empty());
}
#[test]
fn test_priority_queue_pop_runnable_skips_blocked_erasure_set() {
let mut queue = PriorityHealQueue::new();
let blocked = HealRequest::new(
HealType::ErasureSet {
buckets: vec!["bucket-a".to_string()],
set_disk_id: "pool_0_set_1".to_string(),
},
HealOptions::default(),
HealPriority::Urgent,
);
let runnable = HealRequest::new(
HealType::ErasureSet {
buckets: vec!["bucket-b".to_string()],
set_disk_id: "pool_0_set_2".to_string(),
},
HealOptions::default(),
HealPriority::Normal,
);
assert_eq!(queue.push(blocked), QueuePushOutcome::Accepted);
assert_eq!(queue.push(runnable.clone()), QueuePushOutcome::Accepted);
let mut running = HashMap::new();
running.insert("pool_0_set_1".to_string(), 1);
let popped = queue
.pop_runnable(|request| can_schedule_request(request, &running, 1))
.expect("should find runnable request");
match popped.heal_type {
HealType::ErasureSet { set_disk_id, .. } => assert_eq!(set_disk_id, "pool_0_set_2"),
other => panic!("expected erasure set request, got {other:?}"),
}
}
#[test]
fn test_can_schedule_request_respects_per_set_limit() {
let request = HealRequest::new(
HealType::ErasureSet {
buckets: vec!["bucket".to_string()],
set_disk_id: "pool_0_set_1".to_string(),
},
HealOptions::default(),
HealPriority::Normal,
);
let mut running = HashMap::new();
running.insert("pool_0_set_1".to_string(), 1);
assert!(!can_schedule_request(&request, &running, 1));
assert!(can_schedule_request(&request, &running, 2));
}
#[tokio::test]
async fn test_submit_heal_request_returns_merged_for_duplicate() {
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
let manager = HealManager::new(storage, None);
let request = HealRequest::new(
HealType::Object {
bucket: "bucket".to_string(),
object: "object".to_string(),
version_id: None,
},
HealOptions::default(),
HealPriority::Low,
);
assert_eq!(
manager
.submit_heal_request(request.clone())
.await
.expect("first request should be accepted"),
HealAdmissionResult::Accepted
);
assert_eq!(
manager
.submit_heal_request(request)
.await
.expect("duplicate request should produce admission result"),
HealAdmissionResult::Merged
);
}
#[tokio::test]
async fn test_submit_heal_request_returns_merged_before_full_for_duplicate() {
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
let manager = HealManager::new(
storage,
Some(HealConfig {
queue_size: 1,
..HealConfig::default()
}),
);
let request = HealRequest::new(
HealType::Object {
bucket: "bucket".to_string(),
object: "object".to_string(),
version_id: None,
},
HealOptions::default(),
HealPriority::Low,
);
assert_eq!(
manager
.submit_heal_request(request.clone())
.await
.expect("first request should be accepted"),
HealAdmissionResult::Accepted
);
assert_eq!(
manager
.submit_heal_request(request)
.await
.expect("duplicate request should merge even when queue is full"),
HealAdmissionResult::Merged
);
}
#[tokio::test]
async fn test_submit_heal_request_returns_dropped_for_low_priority_when_full() {
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
let manager = HealManager::new(
storage,
Some(HealConfig {
queue_size: 1,
low_priority_drop_when_full: true,
..HealConfig::default()
}),
);
let accepted = HealRequest::new(
HealType::Bucket {
bucket: "bucket-a".to_string(),
},
HealOptions::default(),
HealPriority::Normal,
);
let dropped = HealRequest::new(
HealType::Bucket {
bucket: "bucket-b".to_string(),
},
HealOptions::default(),
HealPriority::Low,
);
assert_eq!(
manager
.submit_heal_request(accepted)
.await
.expect("first request should be accepted"),
HealAdmissionResult::Accepted
);
assert_eq!(
manager
.submit_heal_request(dropped)
.await
.expect("low priority request should be dropped with explicit admission result"),
HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)
);
}
#[test]
fn test_running_erasure_set_counts_groups_only_erasure_tasks() {
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
let erasure_task = Arc::new(HealTask::from_request(
HealRequest::new(
HealType::ErasureSet {
buckets: vec!["bucket".to_string()],
set_disk_id: "pool_0_set_1".to_string(),
},
HealOptions::default(),
HealPriority::Normal,
),
storage.clone(),
));
let object_task = Arc::new(HealTask::from_request(
HealRequest::new(
HealType::Object {
bucket: "bucket".to_string(),
object: "object".to_string(),
version_id: None,
},
HealOptions::default(),
HealPriority::Normal,
),
storage,
));
let mut active = HashMap::new();
active.insert(erasure_task.id.clone(), erasure_task);
active.insert(object_task.id.clone(), object_task);
let counts = running_erasure_set_counts(&active);
assert_eq!(counts.get("pool_0_set_1"), Some(&1));
assert_eq!(counts.len(), 1);
}
#[test]
fn test_heal_config_respects_feature_flags() {
temp_env::with_vars(
[
(rustfs_config::ENV_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE, Some("false")),
(rustfs_config::ENV_HEAL_SET_BULKHEAD_ENABLE, Some("false")),
(rustfs_config::ENV_HEAL_PAGE_PARALLEL_ENABLE, Some("false")),
],
|| {
let config = HealConfig::default();
assert!(!config.event_driven_scheduler_enable);
assert!(!config.set_bulkhead_enable);
assert!(!config.page_parallel_enable);
},
);
}
}
crates/heal/src/heal/storage.rs
+79 -4
View File
@@ -17,8 +17,11 @@ use async_trait::async_trait;
use rustfs_common::heal_channel::{HealOpts, HealScanMode};
use rustfs_ecstore::{
disk::{DiskStore, endpoint::Endpoint},
error::StorageError,
store::ECStore,
store_api::{BucketInfo, BucketOperations, HealOperations, ListOperations, ObjectIO, ObjectOperations, StorageAPI},
store_api::{
BucketInfo, BucketOperations, HealOperations, ListOperations, ObjectIO, ObjectOperations, ObjectOptions, StorageAPI,
},
};
use rustfs_madmin::heal_commands::HealResultItem;
use std::sync::Arc;
@@ -137,6 +140,37 @@ impl ECStoreHealStorage {
}
}
fn is_transient_object_exists_message(message: &str) -> bool {
let message = message.to_ascii_lowercase();
[
"failed to acquire read lock",
"lock acquisition failed",
"lock acquisition timeout",
"quorum not reached",
"deadline has elapsed",
"timed out",
"network error",
"transport error",
"connection refused",
]
.iter()
.any(|pattern| message.contains(pattern))
}
fn is_transient_object_exists_error(err: &StorageError) -> bool {
if err.is_quorum_error() {
return true;
}
match err {
StorageError::Lock(lock_err) => lock_err.is_retryable() || is_transient_object_exists_message(&lock_err.to_string()),
StorageError::Io(io_err) => is_transient_object_exists_message(&io_err.to_string()),
StorageError::SlowDown | StorageError::OperationCanceled => true,
_ => false,
}
}
#[async_trait]
impl HealStorageAPI for ECStoreHealStorage {
async fn get_object_meta(&self, bucket: &str, object: &str) -> Result<Option<rustfs_ecstore::store_api::ObjectInfo>> {
@@ -408,14 +442,24 @@ impl HealStorageAPI for ECStoreHealStorage {
async fn object_exists(&self, bucket: &str, object: &str) -> Result<bool> {
debug!("Checking object exists: {}/{}", bucket, object);
// Use get_object_info for efficient existence check without heavy heal operations
match self.ecstore.get_object_info(bucket, object, &Default::default()).await {
// Existence checks are best-effort for background heal scheduling, so avoid
// acquiring an extra namespace read lock here.
let opts = ObjectOptions {
no_lock: true,
..Default::default()
};
match self.ecstore.get_object_info(bucket, object, &opts).await {
Ok(_) => Ok(true), // Object exists
Err(e) => {
// Map ObjectNotFound to false, other errors must be propagated!
if matches!(e, rustfs_ecstore::error::StorageError::ObjectNotFound(_, _)) {
debug!("Object not found: {}/{}", bucket, object);
Ok(false)
} else if is_transient_object_exists_error(&e) {
warn!("Skipping object existence check for {}/{} due to transient error: {}", bucket, object, e);
Err(Error::transient_skip(format!(
"Skipped object existence check for {bucket}/{object}: {e}"
)))
} else {
error!("Error checking object existence {}/{}: {}", bucket, object, e);
Err(Error::other(e))
@@ -594,3 +638,34 @@ impl HealStorageAPI for ECStoreHealStorage {
})
}
}
#[cfg(test)]
mod tests {
use super::{is_transient_object_exists_error, is_transient_object_exists_message};
use rustfs_ecstore::error::StorageError;
#[test]
fn transient_object_exists_message_matches_lock_quorum_failures() {
assert!(is_transient_object_exists_message(
"Failed to acquire read lock: ns_loc: read lock acquisition failed on bucket/object: Quorum not reached: required 2, achieved 0"
));
assert!(is_transient_object_exists_message("deadline has elapsed"));
}
#[test]
fn transient_object_exists_error_matches_quorum_variants() {
assert!(is_transient_object_exists_error(&StorageError::ErasureReadQuorum));
assert!(is_transient_object_exists_error(&StorageError::InsufficientReadQuorum(
"bucket".to_string(),
"object".to_string(),
)));
}
#[test]
fn transient_object_exists_error_does_not_treat_not_found_as_transient() {
assert!(!is_transient_object_exists_error(&StorageError::ObjectNotFound(
"bucket".to_string(),
"object".to_string(),
)));
}
}
crates/heal/src/heal/task.rs
+79 -4
View File
@@ -14,6 +14,7 @@
use crate::heal::{ErasureSetHealer, progress::HealProgress, storage::HealStorageAPI};
use crate::{Error, Result};
use metrics::{counter, histogram};
use rustfs_common::heal_channel::{HealOpts, HealScanMode};
use serde::{Deserialize, Serialize};
use std::{
@@ -133,16 +134,20 @@ pub struct HealRequest {
pub priority: HealPriority,
/// Created time
pub created_at: SystemTime,
/// Queue admission time used for scheduler delay metrics
pub enqueued_at: SystemTime,
}
impl HealRequest {
pub fn new(heal_type: HealType, options: HealOptions, priority: HealPriority) -> Self {
let now = SystemTime::now();
Self {
id: Uuid::new_v4().to_string(),
heal_type,
options,
priority,
created_at: SystemTime::now(),
created_at: now,
enqueued_at: now,
}
}
@@ -193,6 +198,8 @@ pub struct HealTask {
pub progress: Arc<RwLock<HealProgress>>,
/// Created time
pub created_at: SystemTime,
/// Queue admission time
pub enqueued_at: SystemTime,
/// Started time
pub started_at: Arc<RwLock<Option<SystemTime>>>,
/// Completed time
@@ -214,6 +221,7 @@ impl HealTask {
status: Arc::new(RwLock::new(HealTaskStatus::Pending)),
progress: Arc::new(RwLock::new(HealProgress::new())),
created_at: request.created_at,
enqueued_at: request.enqueued_at,
started_at: Arc::new(RwLock::new(None)),
completed_at: Arc::new(RwLock::new(None)),
task_start_instant: Arc::new(RwLock::new(None)),
@@ -222,6 +230,27 @@ impl HealTask {
}
}
pub fn metric_type_label(&self) -> &'static str {
match &self.heal_type {
HealType::Object { .. } => "object",
HealType::Bucket { .. } => "bucket",
HealType::ErasureSet { .. } => "erasure_set",
HealType::Metadata { .. } => "metadata",
HealType::MRF { .. } => "mrf",
HealType::ECDecode { .. } => "ec_decode",
}
}
pub fn metric_set_label(&self) -> String {
match &self.heal_type {
HealType::ErasureSet { set_disk_id, .. } => set_disk_id.clone(),
_ => match (self.options.pool_index, self.options.set_index) {
(Some(pool), Some(set)) => format!("pool_{pool}_set_{set}"),
_ => "global".to_string(),
},
}
}
async fn remaining_timeout(&self) -> Result<Option<Duration>> {
if let Some(total) = self.options.timeout {
let start_instant = { *self.task_start_instant.read().await };
@@ -272,11 +301,26 @@ impl HealTask {
}
}
async fn skip_due_to_transient_object_exists(&self, bucket: &str, object: &str, err: &Error) -> Result<()> {
warn!(
"Skipping heal for {}/{} due to transient object existence check error: {}",
bucket, object, err
);
let mut progress = self.progress.write().await;
progress.set_current_object(Some(format!("skipped: {bucket}/{object}")));
progress.update_progress(0, 1, 0, 0);
Ok(())
}
#[tracing::instrument(skip(self), fields(task_id = %self.id, heal_type = ?self.heal_type))]
pub async fn execute(&self) -> Result<()> {
// update status and timestamps atomically to avoid race conditions
let now = SystemTime::now();
let start_instant = Instant::now();
let queue_delay = now.duration_since(self.enqueued_at).unwrap_or_default();
let type_label = self.metric_type_label().to_string();
let set_label = self.metric_set_label();
{
let mut status = self.status.write().await;
let mut started_at = self.started_at.write().await;
@@ -286,6 +330,19 @@ impl HealTask {
*task_start_instant = Some(start_instant);
}
histogram!(
"rustfs_heal_queue_delay_seconds",
"type" => type_label.clone(),
"set" => set_label.clone()
)
.record(queue_delay.as_secs_f64());
counter!(
"rustfs_heal_task_start_total",
"type" => type_label,
"set" => set_label
)
.increment(1);
info!("Task started");
let result = match &self.heal_type {
@@ -369,7 +426,13 @@ impl HealTask {
// Step 1: Check if object exists and get metadata
warn!("Step 1: Checking object existence and metadata");
self.check_control_flags().await?;
let object_exists = self.await_with_control(self.storage.object_exists(bucket, object)).await?;
let object_exists = match self.await_with_control(self.storage.object_exists(bucket, object)).await {
Ok(exists) => exists,
Err(err @ Error::TransientSkip { .. }) => {
return self.skip_due_to_transient_object_exists(bucket, object, &err).await;
}
Err(err) => return Err(err),
};
if !object_exists {
warn!("Object does not exist: {}/{}", bucket, object);
if self.options.recreate_missing {
@@ -631,7 +694,13 @@ impl HealTask {
// Step 1: Check if object exists
info!("Step 1: Checking object existence");
self.check_control_flags().await?;
let object_exists = self.await_with_control(self.storage.object_exists(bucket, object)).await?;
let object_exists = match self.await_with_control(self.storage.object_exists(bucket, object)).await {
Ok(exists) => exists,
Err(err @ Error::TransientSkip { .. }) => {
return self.skip_due_to_transient_object_exists(bucket, object, &err).await;
}
Err(err) => return Err(err),
};
if !object_exists {
warn!("Object does not exist: {}/{}", bucket, object);
return Err(Error::TaskExecutionFailed {
@@ -791,7 +860,13 @@ impl HealTask {
// Step 1: Check if object exists
info!("Step 1: Checking object existence");
self.check_control_flags().await?;
let object_exists = self.await_with_control(self.storage.object_exists(bucket, object)).await?;
let object_exists = match self.await_with_control(self.storage.object_exists(bucket, object)).await {
Ok(exists) => exists,
Err(err @ Error::TransientSkip { .. }) => {
return self.skip_due_to_transient_object_exists(bucket, object, &err).await;
}
Err(err) => return Err(err),
};
if !object_exists {
warn!("Object does not exist: {}/{}", bucket, object);
return Err(Error::TaskExecutionFailed {