mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-31 09:18:28 +00:00
@@ -144,16 +144,16 @@ fn new_dynamic_sleeper(factor: f64, max_wait: Duration, is_scanner: bool) -> Dyn
|
||||
}
|
||||
|
||||
/// Initialize and start the data scanner in the background
|
||||
///
|
||||
///
|
||||
/// This function starts a background task that continuously runs the data scanner
|
||||
/// with randomized intervals between cycles to avoid resource contention.
|
||||
///
|
||||
///
|
||||
/// # Features
|
||||
/// - Graceful shutdown support via cancellation token
|
||||
/// - Randomized sleep intervals to prevent synchronized scanning across nodes
|
||||
/// - Minimum sleep duration to avoid excessive CPU usage
|
||||
/// - Proper error handling and logging
|
||||
///
|
||||
///
|
||||
/// # Architecture
|
||||
/// 1. Initialize with random seed for sleep intervals
|
||||
/// 2. Run scanner cycles in a loop
|
||||
@@ -161,12 +161,12 @@ fn new_dynamic_sleeper(factor: f64, max_wait: Duration, is_scanner: bool) -> Dyn
|
||||
/// 4. Ensure minimum sleep duration to prevent CPU thrashing
|
||||
pub async fn init_data_scanner() {
|
||||
info!("Initializing data scanner background task");
|
||||
|
||||
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
// Run the data scanner
|
||||
run_data_scanner().await;
|
||||
|
||||
|
||||
// Calculate randomized sleep duration
|
||||
// Use random factor (0.0 to 1.0) multiplied by the scanner cycle duration
|
||||
let random_factor: f64 = {
|
||||
@@ -175,7 +175,7 @@ pub async fn init_data_scanner() {
|
||||
};
|
||||
let base_cycle_duration = SCANNER_CYCLE.load(Ordering::SeqCst) as f64;
|
||||
let sleep_duration_secs = random_factor * base_cycle_duration;
|
||||
|
||||
|
||||
// Ensure minimum sleep duration of 1 second to avoid high CPU usage
|
||||
let sleep_duration = if sleep_duration_secs < 1.0 {
|
||||
Duration::from_secs(1)
|
||||
@@ -183,8 +183,11 @@ pub async fn init_data_scanner() {
|
||||
Duration::from_secs_f64(sleep_duration_secs)
|
||||
};
|
||||
|
||||
info!(duration_secs = sleep_duration.as_secs(), "Data scanner sleeping before next cycle");
|
||||
|
||||
info!(
|
||||
duration_secs = sleep_duration.as_secs(),
|
||||
"Data scanner sleeping before next cycle"
|
||||
);
|
||||
|
||||
// Sleep with the calculated duration
|
||||
sleep(sleep_duration).await;
|
||||
}
|
||||
@@ -192,20 +195,20 @@ pub async fn init_data_scanner() {
|
||||
}
|
||||
|
||||
/// Run a single data scanner cycle
|
||||
///
|
||||
///
|
||||
/// This function performs one complete scan cycle, including:
|
||||
/// - Loading and updating cycle information
|
||||
/// - Determining scan mode based on healing configuration
|
||||
/// - Running the namespace scanner
|
||||
/// - Saving cycle completion state
|
||||
///
|
||||
///
|
||||
/// # Error Handling
|
||||
/// - Gracefully handles missing object layer
|
||||
/// - Continues operation even if individual steps fail
|
||||
/// - Logs errors appropriately without terminating the scanner
|
||||
async fn run_data_scanner() {
|
||||
info!("Starting data scanner cycle");
|
||||
|
||||
|
||||
// Get the object layer, return early if not available
|
||||
let Some(store) = new_object_layer_fn() else {
|
||||
error!("Object layer not initialized, skipping scanner cycle");
|
||||
@@ -232,20 +235,22 @@ async fn run_data_scanner() {
|
||||
|
||||
// Start metrics collection for this cycle
|
||||
let stop_fn = ScannerMetrics::log(ScannerMetric::ScanCycle);
|
||||
|
||||
|
||||
// Update cycle information
|
||||
cycle_info.current = cycle_info.next;
|
||||
cycle_info.started = Utc::now();
|
||||
|
||||
// Update global scanner metrics
|
||||
{
|
||||
globalScannerMetrics.write().await.set_cycle(Some(cycle_info.clone())).await;
|
||||
}
|
||||
globalScannerMetrics.set_cycle(Some(cycle_info.clone())).await;
|
||||
|
||||
// Read background healing information and determine scan mode
|
||||
let bg_heal_info = read_background_heal_info(store.clone()).await;
|
||||
let scan_mode =
|
||||
get_cycle_scan_mode(cycle_info.current, bg_heal_info.bitrot_start_cycle, bg_heal_info.bitrot_start_time).await;
|
||||
let scan_mode = get_cycle_scan_mode(
|
||||
cycle_info.current,
|
||||
bg_heal_info.bitrot_start_cycle,
|
||||
bg_heal_info.bitrot_start_time,
|
||||
)
|
||||
.await;
|
||||
|
||||
// Update healing info if scan mode changed
|
||||
if bg_heal_info.current_scan_mode != scan_mode {
|
||||
@@ -275,22 +280,26 @@ async fn run_data_scanner() {
|
||||
);
|
||||
|
||||
// Run the namespace scanner
|
||||
match store.clone().ns_scanner(tx, cycle_info.current as usize, scan_mode).await {
|
||||
match store
|
||||
.clone()
|
||||
.ns_scanner(tx, cycle_info.current as usize, scan_mode)
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
info!(cycle = cycle_info.current, "Namespace scanner completed successfully");
|
||||
|
||||
|
||||
// Update cycle completion information
|
||||
cycle_info.next += 1;
|
||||
cycle_info.current = 0;
|
||||
cycle_info.cycle_completed.push(Utc::now());
|
||||
|
||||
|
||||
// Maintain cycle completion history (keep only recent cycles)
|
||||
if cycle_info.cycle_completed.len() > DATA_USAGE_UPDATE_DIR_CYCLES as usize {
|
||||
let _ = cycle_info.cycle_completed.remove(0);
|
||||
}
|
||||
|
||||
// Update global metrics with completion info
|
||||
globalScannerMetrics.write().await.set_cycle(Some(cycle_info.clone())).await;
|
||||
globalScannerMetrics.set_cycle(Some(cycle_info.clone())).await;
|
||||
|
||||
// Persist updated cycle information
|
||||
// ignore error, continue.
|
||||
@@ -312,7 +321,7 @@ async fn run_data_scanner() {
|
||||
}
|
||||
|
||||
// Complete metrics collection for this cycle
|
||||
stop_fn(&scan_result).await;
|
||||
stop_fn(&scan_result);
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
@@ -560,7 +569,7 @@ impl ScannerItem {
|
||||
pub async fn apply_actions(&self, oi: &ObjectInfo, _size_s: &SizeSummary) -> (bool, usize) {
|
||||
let done = ScannerMetrics::time(ScannerMetric::Ilm);
|
||||
//todo: lifecycle
|
||||
done().await;
|
||||
done();
|
||||
|
||||
(false, oi.size)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user