gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-13 08:36:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(storage): harden offline drive fail-fast paths (#2564)

Browse Source 
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>
This commit is contained in:
houseme
2026-04-16 17:21:45 +08:00
committed by GitHub
parent 579b124726
commit 28edfd6190
35 changed files with 4426 additions and 618 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scanner/src/scanner_folder.rs
+357 -75
View File
@@ -15,7 +15,7 @@
use std::collections::HashSet;
use std::fs::FileType;
use std::io::ErrorKind;
use std::sync::Arc;
use std::sync::{Arc, Once};
use std::time::{Duration, SystemTime};
use crate::ReplTargetSizeSummary;
@@ -23,9 +23,12 @@ use crate::data_usage_define::{DataUsageCache, DataUsageEntry, DataUsageHash, Da
use crate::error::ScannerError;
use crate::scanner_io::ScannerIODisk as _;
use crate::sleeper::DynamicSleeper;
use rustfs_common::heal_channel::{HEAL_DELETE_DANGLING, HealChannelRequest, HealOpts, HealScanMode, send_heal_request};
use metrics::{counter, describe_counter};
use rustfs_common::heal_channel::{
HEAL_DELETE_DANGLING, HealAdmissionResult, HealChannelPriority, HealChannelRequest, HealScanMode,
send_heal_request_with_admission,
};
use rustfs_common::metrics::{IlmAction, Metric, Metrics, UpdateCurrentPathFn, current_path_updater};
use rustfs_ecstore::StorageAPI;
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_rule;
use rustfs_ecstore::bucket::lifecycle::evaluator::Evaluator;
@@ -65,6 +68,10 @@ const ENV_FAILED_OBJECT_TTL_SECS: &str = "RUSTFS_DATA_USAGE_FAILED_OBJECT_TTL_SE
const ENV_FAILED_OBJECTS_MAX: &str = "RUSTFS_DATA_USAGE_FAILED_OBJECTS_MAX";
const DEFAULT_FAILED_OBJECT_TTL_SECS: u32 = 86_400;
const DEFAULT_FAILED_OBJECTS_MAX: u32 = 10_000;
const METRIC_SCANNER_INLINE_HEAL_TOTAL: &str = "rustfs_scanner_inline_heal_total";
static SCANNER_INLINE_HEAL_WARN_ONCE: Once = Once::new();
static SCANNER_INLINE_HEAL_METRICS_ONCE: Once = Once::new();
pub fn data_usage_update_dir_cycles() -> u32 {
rustfs_utils::get_env_u32(ENV_DATA_USAGE_UPDATE_DIR_CYCLES, DATA_USAGE_UPDATE_DIR_CYCLES)
@@ -74,6 +81,40 @@ pub fn heal_object_select_prob() -> u32 {
rustfs_utils::get_env_u32(ENV_HEAL_OBJECT_SELECT_PROB, DEFAULT_HEAL_OBJECT_SELECT_PROB)
}
fn scanner_inline_heal_enabled() -> bool {
scanner_inline_heal_enabled_from_value(std::env::var(rustfs_config::ENV_SCANNER_INLINE_HEAL_ENABLE).ok().as_deref())
}
fn scanner_inline_heal_enabled_from_value(value: Option<&str>) -> bool {
match value {
Some(value) => matches!(value.trim().to_ascii_lowercase().as_str(), "1" | "true" | "on" | "yes"),
None => rustfs_config::DEFAULT_SCANNER_INLINE_HEAL_ENABLE,
}
}
fn ensure_scanner_inline_heal_metric_registered() {
SCANNER_INLINE_HEAL_METRICS_ONCE.call_once(|| {
describe_counter!(
METRIC_SCANNER_INLINE_HEAL_TOTAL,
"Total number of inline heal operations executed directly by scanner."
);
counter!(METRIC_SCANNER_INLINE_HEAL_TOTAL).increment(0);
});
}
fn warn_inline_heal_compat_requested() {
if !scanner_inline_heal_enabled() {
return;
}
SCANNER_INLINE_HEAL_WARN_ONCE.call_once(|| {
warn!(
env = rustfs_config::ENV_SCANNER_INLINE_HEAL_ENABLE,
"Inline scanner heal rollback is no longer supported; continuing to enqueue heal candidates asynchronously"
);
});
}
/// Cached folder information for scanning
#[derive(Clone, Debug)]
pub struct CachedFolder {
@@ -85,6 +126,154 @@ pub struct CachedFolder {
/// Type alias for get size function
pub type GetSizeFn = Box<dyn Fn(ScannerItem) -> Result<SizeSummary, StorageError> + Send + Sync>;
fn build_bucket_heal_request(bucket: String, priority: HealChannelPriority) -> HealChannelRequest {
HealChannelRequest {
bucket,
priority,
..Default::default()
}
}
fn build_object_heal_request(
bucket: String,
object: String,
version_id: Option<String>,
scan_mode: HealScanMode,
priority: HealChannelPriority,
) -> HealChannelRequest {
HealChannelRequest {
bucket,
object_prefix: Some(object),
object_version_id: version_id,
priority,
scan_mode: Some(scan_mode),
remove_corrupted: Some(HEAL_DELETE_DANGLING),
..Default::default()
}
}
fn heal_priority_label(priority: HealChannelPriority) -> &'static str {
match priority {
HealChannelPriority::Low => "low",
HealChannelPriority::Normal => "normal",
HealChannelPriority::High => "high",
HealChannelPriority::Critical => "critical",
}
}
fn describe_heal_admission(result: HealAdmissionResult) -> String {
match result {
HealAdmissionResult::Accepted | HealAdmissionResult::Merged => result.result_label().to_string(),
HealAdmissionResult::Full => "queue_full".to_string(),
HealAdmissionResult::Dropped(reason) => format!("dropped:{}", reason.as_str()),
}
}
fn record_high_priority_heal_escalation(
candidate_type: &'static str,
priority: HealChannelPriority,
result: HealAdmissionResult,
) {
counter!(
"rustfs_heal_candidate_priority_reject_total",
"type" => candidate_type.to_string(),
"priority" => heal_priority_label(priority).to_string(),
"result" => result.result_label().to_string(),
"reason" => result.reason_label().to_string()
)
.increment(1);
}
fn build_high_priority_heal_admission_error(
candidate_type: &'static str,
bucket: &str,
object: Option<&str>,
priority: HealChannelPriority,
result: HealAdmissionResult,
) -> ScannerError {
let object_text = object.map(|object| format!(", object='{object}'")).unwrap_or_default();
ScannerError::Other(format!(
"high-priority heal request was not admitted: type={candidate_type}, bucket='{bucket}'{object_text}, priority={}, admission={}",
heal_priority_label(priority),
describe_heal_admission(result)
))
}
fn record_heal_candidate_admission(candidate_type: &'static str, priority: HealChannelPriority, result: HealAdmissionResult) {
counter!(
"rustfs_heal_candidate_enqueue_total",
"type" => candidate_type.to_string(),
"priority" => heal_priority_label(priority).to_string(),
"result" => result.result_label().to_string()
)
.increment(1);
if matches!(result, HealAdmissionResult::Merged) {
counter!(
"rustfs_heal_candidate_merge_total",
"type" => candidate_type.to_string()
)
.increment(1);
}
if let HealAdmissionResult::Dropped(reason) = result {
counter!(
"rustfs_heal_candidate_drop_total",
"type" => candidate_type.to_string(),
"reason" => reason.as_str().to_string()
)
.increment(1);
}
}
async fn send_scanner_heal_request(
candidate_type: &'static str,
request: HealChannelRequest,
) -> Result<HealAdmissionResult, ScannerError> {
let priority = request.priority;
match send_heal_request_with_admission(request).await {
Ok(result) => {
record_heal_candidate_admission(candidate_type, priority, result);
Ok(result)
}
Err(err) => {
counter!(
"rustfs_heal_candidate_enqueue_total",
"type" => candidate_type.to_string(),
"priority" => heal_priority_label(priority).to_string(),
"result" => "channel_error".to_string()
)
.increment(1);
Err(ScannerError::Other(err))
}
}
}
async fn send_required_scanner_heal_request(
candidate_type: &'static str,
bucket: &str,
object: Option<&str>,
request: HealChannelRequest,
) -> Result<(), ScannerError> {
let priority = request.priority;
let result = send_scanner_heal_request(candidate_type, request).await?;
if result.is_admitted() {
return Ok(());
}
record_high_priority_heal_escalation(candidate_type, priority, result);
error!(
candidate_type,
bucket,
object = object.unwrap_or(""),
priority = heal_priority_label(priority),
admission = result.result_label(),
reason = result.reason_label(),
"High-priority heal request was not admitted; escalating to scanner failure"
);
Err(build_high_priority_heal_admission_error(candidate_type, bucket, object, priority, result))
}
/// Scanner item representing a file during scanning
#[derive(Clone, Debug)]
pub struct ScannerItem {
@@ -127,9 +316,8 @@ impl ScannerItem {
self.object_name = split.last().unwrap_or(&"").to_string();
}
pub async fn apply_actions<S: StorageAPI>(
pub async fn apply_actions(
&mut self,
store: Arc<S>,
object_infos: Vec<ObjectInfo>,
lock_retention: Option<Arc<ObjectLockConfiguration>>,
size_summary: &mut SizeSummary,
@@ -159,7 +347,7 @@ impl ScannerItem {
}
};
let size = self.heal_actions(store.clone(), oi, actual_size, size_summary).await;
let size = self.heal_actions(oi, actual_size, size_summary).await;
size_summary.actions_accounting(oi, size, actual_size);
@@ -246,7 +434,7 @@ impl ScannerItem {
}
IlmAction::NoneAction | IlmAction::ActionCount => {
size = self.heal_actions(store.clone(), oi, actual_size, size_summary).await;
size = self.heal_actions(oi, actual_size, size_summary).await;
}
}
@@ -262,22 +450,15 @@ impl ScannerItem {
self.alert_excessive_versions(remaining_versions, cumulative_size);
}
async fn heal_actions<S: StorageAPI>(
&mut self,
store: Arc<S>,
oi: &ObjectInfo,
actual_size: i64,
size_summary: &mut SizeSummary,
) -> i64 {
let mut size = actual_size;
async fn heal_actions(&mut self, oi: &ObjectInfo, actual_size: i64, size_summary: &mut SizeSummary) -> i64 {
if self.heal_enabled {
size = self.apply_heal(store, oi).await;
warn_inline_heal_compat_requested();
self.enqueue_heal(oi).await;
}
self.heal_replication(oi, size_summary).await;
size
actual_size
}
async fn heal_replication(&mut self, oi: &ObjectInfo, size_summary: &mut SizeSummary) {
@@ -332,10 +513,10 @@ impl ScannerItem {
}
}
async fn apply_heal<S: StorageAPI>(&mut self, store: Arc<S>, oi: &ObjectInfo) -> i64 {
async fn enqueue_heal(&mut self, oi: &ObjectInfo) {
let done_heal = Metrics::time(Metric::HealAbandonedObject);
debug!(
"apply_heal: bucket: {}, object_path: {}, version_id: {}",
"enqueue_heal: bucket: {}, object_path: {}, version_id: {}",
self.bucket,
self.object_path(),
oi.version_id.unwrap_or_default()
@@ -347,36 +528,32 @@ impl ScannerItem {
HealScanMode::Normal
};
let result = match store
.clone()
.heal_object(
self.bucket.as_str(),
self.object_path().as_str(),
let result = send_scanner_heal_request(
"object",
build_object_heal_request(
self.bucket.clone(),
self.object_path(),
oi.version_id
.map(|v| if v.is_nil() { "".to_string() } else { v.to_string() })
.unwrap_or_default()
.as_str(),
&HealOpts {
remove: HEAL_DELETE_DANGLING,
scan_mode,
..Default::default()
},
)
.await
{
Ok((result, err)) => {
if let Some(err) = err {
warn!("apply_heal: failed to heal object: {}", err);
}
result.object_size as i64
.and_then(|v| if v.is_nil() { None } else { Some(v.to_string()) }),
scan_mode,
HealChannelPriority::Low,
),
)
.await;
match result {
Ok(HealAdmissionResult::Accepted | HealAdmissionResult::Merged) => {}
Ok(result @ (HealAdmissionResult::Full | HealAdmissionResult::Dropped(_))) => {
warn!(
bucket = %self.bucket,
object = %self.object_path(),
admission = %describe_heal_admission(result),
"enqueue_heal: low-priority heal request was not admitted"
);
}
Err(e) => {
warn!("apply_heal: failed to heal object: {}", e);
0
}
};
Err(e) => warn!("enqueue_heal: failed to submit heal request: {}", e),
}
done_heal();
result
}
fn alert_excessive_versions(&self, _object_infos_length: usize, _cumulative_size: i64) {
@@ -964,12 +1141,13 @@ impl FolderScanner {
let (bucket, prefix) = path2_bucket_object(name.as_str());
if bucket != resolver.bucket {
send_heal_request(HealChannelRequest {
bucket: bucket.clone(),
..Default::default()
})
.await
.map_err(ScannerError::Other)?;
send_required_scanner_heal_request(
"bucket",
&bucket,
None,
build_bucket_heal_request(bucket.clone(), HealChannelPriority::High),
)
.await?;
}
resolver.bucket = bucket.clone();
@@ -1075,36 +1253,40 @@ impl FolderScanner {
Ok(fivs) => fivs,
Err(e) => {
error!("scan_folder: list_path_raw: failed to get file info versions: {}", e);
if let Err(e) = send_heal_request(HealChannelRequest {
bucket: bucket.clone(),
object_prefix: Some(entry.name.clone()),
..Default::default()
}).await {
error!("scan_folder: list_path_raw: failed to send heal request: {}", e);
continue;
}
send_required_scanner_heal_request(
"object",
&bucket,
Some(&entry.name),
build_object_heal_request(
bucket.clone(),
entry.name.clone(),
None,
self.scan_mode,
HealChannelPriority::High,
),
)
.await?;
found_objects = true;
continue;
}
};
for fiv in fivs.versions {
if let Err(e) = send_heal_request(HealChannelRequest {
bucket: bucket.clone(),
object_prefix: Some(entry.name.clone()),
object_version_id: fiv.version_id.map(|v| v.to_string()),
..Default::default()
}).await {
error!("scan_folder: list_path_raw: failed to send heal request: {}", e);
continue;
}
send_required_scanner_heal_request(
"object",
&bucket,
Some(&entry.name),
build_object_heal_request(
bucket.clone(),
entry.name.clone(),
fiv.version_id.and_then(|v| if v.is_nil() { None } else { Some(v.to_string()) }),
self.scan_mode,
HealChannelPriority::High,
),
)
.await?;
found_objects = true;
}
@@ -1233,6 +1415,8 @@ pub async fn scan_data_folder(
) -> Result<DataUsageCache, ScannerError> {
use crate::data_usage_define::DATA_USAGE_ROOT;
ensure_scanner_inline_heal_metric_registered();
// Check that we're not trying to scan the root
if cache.info.name.is_empty() || cache.info.name == DATA_USAGE_ROOT {
return Err(ScannerError::Other("internal error: root scan attempted".to_string()));
@@ -1518,6 +1702,104 @@ mod tests {
assert!(!scanner.new_cache.info.failed_objects.contains_key("expired"));
}
#[test]
fn test_scanner_inline_heal_enabled_defaults_to_false() {
assert!(!scanner_inline_heal_enabled_from_value(None));
}
#[test]
fn test_scanner_inline_heal_enabled_reads_env_override() {
assert!(scanner_inline_heal_enabled_from_value(Some("true")));
assert!(scanner_inline_heal_enabled_from_value(Some("YES")));
assert!(scanner_inline_heal_enabled_from_value(Some("1")));
assert!(!scanner_inline_heal_enabled_from_value(Some("false")));
}
#[test]
fn test_build_object_heal_request_omits_nil_version_id() {
let request = build_object_heal_request(
"bucket".to_string(),
"path/to/object".to_string(),
None,
HealScanMode::Deep,
HealChannelPriority::Low,
);
assert_eq!(request.bucket, "bucket");
assert_eq!(request.object_prefix.as_deref(), Some("path/to/object"));
assert!(request.object_version_id.is_none());
assert_eq!(request.scan_mode, Some(HealScanMode::Deep));
assert_eq!(request.priority, HealChannelPriority::Low);
assert_eq!(request.remove_corrupted, Some(HEAL_DELETE_DANGLING));
}
#[test]
fn test_heal_priority_label_matches_priority_names() {
assert_eq!(heal_priority_label(HealChannelPriority::Low), "low");
assert_eq!(heal_priority_label(HealChannelPriority::Normal), "normal");
assert_eq!(heal_priority_label(HealChannelPriority::High), "high");
assert_eq!(heal_priority_label(HealChannelPriority::Critical), "critical");
}
#[test]
fn test_describe_heal_admission_formats_unadmitted_results() {
assert_eq!(describe_heal_admission(HealAdmissionResult::Accepted), "accepted");
assert_eq!(describe_heal_admission(HealAdmissionResult::Merged), "merged");
assert_eq!(describe_heal_admission(HealAdmissionResult::Full), "queue_full");
assert_eq!(
describe_heal_admission(HealAdmissionResult::Dropped(
rustfs_common::heal_channel::HealAdmissionDropReason::QueueFull
)),
"dropped:queue_full"
);
}
#[test]
fn test_build_high_priority_heal_admission_error_contains_context() {
let err = build_high_priority_heal_admission_error(
"object",
"bucket-a",
Some("path/to/object"),
HealChannelPriority::High,
HealAdmissionResult::Full,
);
let err_text = err.to_string();
assert!(err_text.contains("type=object"));
assert!(err_text.contains("bucket='bucket-a'"));
assert!(err_text.contains("object='path/to/object'"));
assert!(err_text.contains("priority=high"));
assert!(err_text.contains("admission=queue_full"));
}
#[tokio::test]
async fn test_heal_actions_returns_actual_size_without_inline_heal() {
let temp_dir = std::env::temp_dir();
let file_type = std::fs::metadata(&temp_dir).unwrap().file_type();
let mut item = ScannerItem {
path: temp_dir.join("object").to_string_lossy().to_string(),
bucket: "bucket".to_string(),
prefix: "".to_string(),
object_name: "object".to_string(),
file_type,
lifecycle: None,
replication: None,
heal_enabled: true,
heal_bitrot: true,
debug: false,
};
let object_info = ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
..Default::default()
};
let mut size_summary = SizeSummary::default();
let size = item.heal_actions(&object_info, 123, &mut size_summary).await;
assert_eq!(size, 123);
}
#[tokio::test]
#[serial]
#[cfg(unix)]
crates/scanner/src/scanner_io.rs
+76 -13
View File
@@ -19,6 +19,7 @@ use crate::{
DataUsageInfo, SizeSummary, TierStats,
};
use futures::future::join_all;
use metrics::counter;
use rand::seq::SliceRandom as _;
use rustfs_common::heal_channel::HealScanMode;
use rustfs_common::metrics::{Metric, Metrics, emit_scan_bucket_drive_complete};
@@ -50,6 +51,24 @@ use tokio::time::Duration;
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, warn};
fn record_set_scan_failure(first_err: &mut Option<Error>, err: Error) {
if first_err.is_none() {
*first_err = Some(err);
}
}
fn finalize_nsscanner_result(results: &[DataUsageCache], first_err: Option<Error>) -> Result<()> {
if results.iter().any(|result| result.info.last_update.is_some()) {
return Ok(());
}
if let Some(err) = first_err {
return Err(err);
}
Ok(())
}
#[async_trait::async_trait]
pub trait ScannerIO: Send + Sync + Debug + 'static {
async fn nsscanner(
@@ -125,6 +144,8 @@ impl ScannerIO for ECStore {
let results_index_clone = results_index as usize;
// Clone the Arc to move it into the spawned task
let set_clone: Arc<SetDisks> = Arc::clone(set);
let pool_label = set.pool_index.to_string();
let set_label = set.set_index.to_string();
let child_token_clone = child_token.clone();
let want_cycle_clone = want_cycle;
@@ -150,9 +171,21 @@ impl ScannerIO for ECStore {
.nsscanner_cache(child_token_clone.clone(), all_buckets_clone, tx, want_cycle_clone, scan_mode_clone)
.await
{
error!("Failed to scan set: {e}");
let _ = first_err_mutex_clone.lock().await.insert(e);
child_token_clone.cancel();
counter!(
"rustfs_scanner_set_failure_total",
"pool" => pool_label.clone(),
"set" => set_label.clone(),
"stage" => "nsscanner_cache".to_string()
)
.increment(1);
error!(
pool = %pool_label,
set = %set_label,
error = %e,
"Failed to scan set; continuing scanner cycle"
);
let mut first_err = first_err_mutex_clone.lock().await;
record_set_scan_failure(&mut first_err, e);
}
});
wait_futs.push(scanner_fut);
@@ -162,6 +195,7 @@ impl ScannerIO for ECStore {
let (update_tx, mut update_rx) = tokio::sync::oneshot::channel::<()>();
let all_buckets_clone = all_buckets.iter().map(|b| b.name.clone()).collect::<Vec<String>>();
let results_mutex_for_updates = results_mutex.clone();
tokio::spawn(async move {
let mut last_update = SystemTime::UNIX_EPOCH;
let mut has_sent_once = false;
@@ -177,7 +211,7 @@ impl ScannerIO for ECStore {
break;
}
let results = results_mutex.lock().await;
let results = results_mutex_for_updates.lock().await;
let mut all_merged = DataUsageCache::default();
for result in results.iter() {
if result.info.last_update.is_none() {
@@ -196,7 +230,7 @@ impl ScannerIO for ECStore {
break;
}
_ = ticker.tick() => {
let results = results_mutex.lock().await;
let results = results_mutex_for_updates.lock().await;
let mut all_merged = DataUsageCache::default();
for result in results.iter() {
if result.info.last_update.is_none() {
@@ -223,7 +257,9 @@ impl ScannerIO for ECStore {
let _ = update_tx.send(());
Ok(())
let first_err = first_err_mutex.lock().await.take();
let results = results_mutex.lock().await.clone();
finalize_nsscanner_result(&results, first_err)
}
}
@@ -561,13 +597,7 @@ impl ScannerIODisk for Disk {
Err(_) => None,
};
let Some(ecstore) = new_object_layer_fn() else {
error!("ECStore not available");
return Err(StorageError::other("ECStore not available".to_string()));
};
item.apply_actions(ecstore, object_infos, lock_config, &mut size_summary)
.await;
item.apply_actions(object_infos, lock_config, &mut size_summary).await;
if !free_version_infos.is_empty() {
let mut expiry_state = GLOBAL_ExpiryState.write().await;
@@ -668,3 +698,36 @@ impl ScannerIODisk for Disk {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn record_set_scan_failure_preserves_first_error() {
let mut first = None;
record_set_scan_failure(&mut first, Error::other("first"));
record_set_scan_failure(&mut first, Error::other("second"));
let first = first.expect("first error should be recorded");
assert!(first.to_string().contains("first"));
}
#[test]
fn finalize_nsscanner_result_returns_ok_when_any_set_succeeds() {
let mut results = vec![DataUsageCache::default(), DataUsageCache::default()];
results[1].info.last_update = Some(SystemTime::now());
let result = finalize_nsscanner_result(&results, Some(Error::other("set failed")));
assert!(result.is_ok());
}
#[test]
fn finalize_nsscanner_result_returns_first_error_when_all_sets_fail() {
let results = vec![DataUsageCache::default(), DataUsageCache::default()];
let err = finalize_nsscanner_result(&results, Some(Error::other("set failed")))
.expect_err("all failed sets should bubble first error");
assert!(err.to_string().contains("set failed"));
}
}