feat(scanner): expose cycle observability controls (#3147)

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
Henry Guo
2026-06-01 05:55:46 +08:00
committed by GitHub
parent a14657f517
commit 76da2a48d0
9 changed files with 398 additions and 15 deletions
+20 -6
View File
@@ -110,6 +110,25 @@ pub enum HealScanMode {
Deep = 2,
}
impl HealScanMode {
pub const fn as_str(self) -> &'static str {
match self {
Self::Unknown => "unknown",
Self::Normal => "normal",
Self::Deep => "deep",
}
}
pub const fn from_u8(value: u8) -> Option<Self> {
match value {
0 => Some(Self::Unknown),
1 => Some(Self::Normal),
2 => Some(Self::Deep),
_ => None,
}
}
}
impl Serialize for HealScanMode {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
@@ -137,12 +156,7 @@ impl<'de> Deserialize<'de> for HealScanMode {
where
E: serde::de::Error,
{
match value {
0 => Ok(HealScanMode::Unknown),
1 => Ok(HealScanMode::Normal),
2 => Ok(HealScanMode::Deep),
_ => Err(E::custom(format!("invalid HealScanMode value: {value}"))),
}
HealScanMode::from_u8(value).ok_or_else(|| E::custom(format!("invalid HealScanMode value: {value}")))
}
fn visit_u64<E>(self, value: u64) -> Result<Self::Value, E>
+106 -2
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::heal_channel::HealScanMode;
use crate::last_minute::{AccElem, LastMinuteLatency};
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
@@ -22,7 +23,7 @@ use std::{
pin::Pin,
sync::{
Arc, Mutex, OnceLock,
atomic::{AtomicU64, Ordering},
atomic::{AtomicU8, AtomicU64, Ordering},
},
time::{Duration, SystemTime},
};
@@ -340,8 +341,19 @@ pub struct Metrics {
actions_latency: Vec<LockedLastMinuteLatency>,
current_paths: Arc<RwLock<HashMap<String, Arc<CurrentPathTracker>>>>,
cycle_info: Arc<RwLock<Option<CurrentCycle>>>,
current_scan_mode: AtomicU8,
last_scan_cycle_result: AtomicU8,
last_scan_cycle_duration_millis: AtomicU64,
failed_scan_cycles: AtomicU64,
}
const SCAN_CYCLE_RESULT_UNKNOWN: u8 = 0;
const SCAN_CYCLE_RESULT_SUCCESS: u8 = 1;
const SCAN_CYCLE_RESULT_ERROR: u8 = 2;
const SCAN_CYCLE_RESULT_UNKNOWN_LABEL: &str = "unknown";
const SCAN_CYCLE_RESULT_SUCCESS_LABEL: &str = "success";
const SCAN_CYCLE_RESULT_ERROR_LABEL: &str = "error";
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct CurrentCycle {
pub current: u64,
@@ -376,6 +388,11 @@ pub struct ScannerMetricsReport {
pub life_time_ilm: HashMap<String, u64>,
pub last_minute: ScannerLastMinute,
pub active_paths: Vec<String>,
pub current_scan_mode: String,
pub last_cycle_result: String,
pub last_cycle_result_code: u64,
pub last_cycle_duration_seconds: f64,
pub failed_cycles: u64,
}
impl CurrentCycle {
@@ -409,8 +426,25 @@ fn emit_otel_counter(metric: usize, count: u64) {
}
}
fn scan_cycle_result_label(result: u8) -> &'static str {
match result {
SCAN_CYCLE_RESULT_SUCCESS => SCAN_CYCLE_RESULT_SUCCESS_LABEL,
SCAN_CYCLE_RESULT_ERROR => SCAN_CYCLE_RESULT_ERROR_LABEL,
_ => SCAN_CYCLE_RESULT_UNKNOWN_LABEL,
}
}
fn duration_millis_saturated(duration: Duration) -> u64 {
duration.as_millis().min(u64::MAX as u128) as u64
}
pub fn emit_scan_cycle_complete(success: bool, duration: Duration) {
let result = if success { "success" } else { "error" };
let result = if success {
SCAN_CYCLE_RESULT_SUCCESS_LABEL
} else {
SCAN_CYCLE_RESULT_ERROR_LABEL
};
global_metrics().record_scan_cycle_complete(success, duration);
metrics::counter!(OTEL_SCANNER_CYCLES, "result" => result).increment(1);
if success {
metrics::gauge!(OTEL_SCANNER_CYCLE_DURATION_SECONDS).set(duration.as_secs_f64());
@@ -449,6 +483,10 @@ impl Metrics {
.collect(),
current_paths: Arc::new(RwLock::new(HashMap::new())),
cycle_info: Arc::new(RwLock::new(None)),
current_scan_mode: AtomicU8::new(HealScanMode::Unknown as u8),
last_scan_cycle_result: AtomicU8::new(SCAN_CYCLE_RESULT_UNKNOWN),
last_scan_cycle_duration_millis: AtomicU64::new(0),
failed_scan_cycles: AtomicU64::new(0),
}
}
@@ -583,6 +621,30 @@ impl Metrics {
self.cycle_info.read().await.clone()
}
pub fn set_current_scan_mode(&self, scan_mode: HealScanMode) {
self.current_scan_mode.store(scan_mode as u8, Ordering::Relaxed);
}
pub fn clear_current_scan_mode(&self) {
self.set_current_scan_mode(HealScanMode::Unknown);
}
pub fn current_scan_mode(&self) -> HealScanMode {
HealScanMode::from_u8(self.current_scan_mode.load(Ordering::Relaxed)).unwrap_or(HealScanMode::Unknown)
}
pub fn record_scan_cycle_complete(&self, success: bool, duration: Duration) {
let result = if success {
SCAN_CYCLE_RESULT_SUCCESS
} else {
self.failed_scan_cycles.fetch_add(1, Ordering::Relaxed);
SCAN_CYCLE_RESULT_ERROR
};
self.last_scan_cycle_result.store(result, Ordering::Relaxed);
self.last_scan_cycle_duration_millis
.store(duration_millis_saturated(duration), Ordering::Relaxed);
}
/// Snapshot of every path currently being scanned.
pub async fn get_current_paths(&self) -> Vec<String> {
let paths = self.current_paths.read().await;
@@ -618,6 +680,12 @@ impl Metrics {
m.collected_at = Utc::now();
m.active_paths = self.get_current_paths().await;
m.active_scan_paths = m.active_paths.len();
m.current_scan_mode = self.current_scan_mode().as_str().to_string();
let last_cycle_result = self.last_scan_cycle_result.load(Ordering::Relaxed);
m.last_cycle_result = scan_cycle_result_label(last_cycle_result).to_string();
m.last_cycle_result_code = last_cycle_result as u64;
m.last_cycle_duration_seconds = self.last_scan_cycle_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
m.failed_cycles = self.failed_scan_cycles.load(Ordering::Relaxed);
// Lifetime operation counts
for i in 0..Metric::Last as usize {
@@ -795,4 +863,40 @@ mod tests {
assert_eq!(report.current_started, cycle_started);
}
#[tokio::test]
async fn report_includes_current_scan_mode() {
let metrics = Metrics::new();
metrics.set_current_scan_mode(HealScanMode::Deep);
let report = metrics.report().await;
assert_eq!(report.current_scan_mode, HealScanMode::Deep.as_str());
}
#[tokio::test]
async fn report_includes_last_scan_cycle_result() {
let metrics = Metrics::new();
metrics.record_scan_cycle_complete(false, Duration::from_millis(1500));
let report = metrics.report().await;
assert_eq!(report.last_cycle_result, SCAN_CYCLE_RESULT_ERROR_LABEL);
assert_eq!(report.last_cycle_result_code, SCAN_CYCLE_RESULT_ERROR as u64);
assert_eq!(report.last_cycle_duration_seconds, 1.5);
assert_eq!(report.failed_cycles, 1);
}
#[tokio::test]
async fn report_tracks_successful_scan_cycle_without_failed_increment() {
let metrics = Metrics::new();
metrics.record_scan_cycle_complete(true, Duration::from_secs(2));
let report = metrics.report().await;
assert_eq!(report.last_cycle_result, SCAN_CYCLE_RESULT_SUCCESS_LABEL);
assert_eq!(report.last_cycle_result_code, SCAN_CYCLE_RESULT_SUCCESS as u64);
assert_eq!(report.last_cycle_duration_seconds, 2.0);
assert_eq!(report.failed_cycles, 0);
}
}
+8
View File
@@ -93,6 +93,11 @@ pub const ENV_SCANNER_MAX_CONCURRENT_SET_SCANS: &str = "RUSTFS_SCANNER_MAX_CONCU
/// - Example: `export RUSTFS_SCANNER_MAX_CONCURRENT_DISK_SCANS=1`
pub const ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS: &str = "RUSTFS_SCANNER_MAX_CONCURRENT_DISK_SCANS";
/// Environment variable that controls how often scanner object loops yield to the async runtime.
/// A value of `0` disables this extra object-count yield.
/// - Example: `export RUSTFS_SCANNER_YIELD_EVERY_N_OBJECTS=32`
pub const ENV_SCANNER_YIELD_EVERY_N_OBJECTS: &str = "RUSTFS_SCANNER_YIELD_EVERY_N_OBJECTS";
/// Default scanner idle mode.
pub const DEFAULT_SCANNER_IDLE_MODE: bool = true;
@@ -104,6 +109,9 @@ pub const DEFAULT_SCANNER_MAX_CONCURRENT_SET_SCANS: usize = 0;
/// `0` means no additional limit beyond available disks in the set.
pub const DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS: usize = 0;
/// Default object interval for cooperative scanner yields.
pub const DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS: u64 = 128;
/// Compatibility flag kept for Patch 3 rollback windows.
///
/// Inline scanner heal execution has been removed in favor of heal-candidate enqueue.
+69 -4
View File
@@ -21,8 +21,10 @@
use crate::metrics::report::PrometheusMetric;
use crate::metrics::schema::scanner::{
SCANNER_ACTIVE_PATHS_MD, SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_DIRECTORIES_SCANNED_MD,
SCANNER_LAST_ACTIVITY_SECONDS_MD, SCANNER_OBJECTS_SCANNED_MD, SCANNER_VERSIONS_SCANNED_MD,
SCANNER_ACTIVE_PATHS_MD, SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_COMPLETED_CYCLES_MD,
SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, SCANNER_CURRENT_CYCLE_MD, SCANNER_CURRENT_SCAN_MODE_MD, SCANNER_DIRECTORIES_SCANNED_MD,
SCANNER_FAILED_CYCLES_MD, SCANNER_LAST_ACTIVITY_SECONDS_MD, SCANNER_LAST_CYCLE_DURATION_SECONDS_MD,
SCANNER_LAST_CYCLE_RESULT_MD, SCANNER_OBJECTS_SCANNED_MD, SCANNER_VERSIONS_SCANNED_MD,
};
/// Scanner statistics.
@@ -42,6 +44,20 @@ pub struct ScannerStats {
pub last_activity_seconds: u64,
/// Number of scanner paths currently being processed
pub active_paths: u64,
/// Current scanner cycle number, or zero when idle
pub current_cycle: u64,
/// Number of scanner cycles completed since server start
pub completed_cycles: u64,
/// Seconds elapsed since the current scanner cycle started
pub current_cycle_age_seconds: u64,
/// Current scanner mode: 0 unknown or idle, 1 normal, 2 deep bitrot scan
pub current_scan_mode: u64,
/// Last scanner cycle result: 0 unknown, 1 success, 2 error
pub last_cycle_result: u64,
/// Duration in seconds of the last finished scanner cycle
pub last_cycle_duration_seconds: f64,
/// Number of scanner cycles that failed since server start
pub failed_cycles: u64,
}
/// Collects scanner metrics from the given stats.
@@ -57,6 +73,13 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
PrometheusMetric::from_descriptor(&SCANNER_VERSIONS_SCANNED_MD, stats.versions_scanned as f64),
PrometheusMetric::from_descriptor(&SCANNER_LAST_ACTIVITY_SECONDS_MD, stats.last_activity_seconds as f64),
PrometheusMetric::from_descriptor(&SCANNER_ACTIVE_PATHS_MD, stats.active_paths as f64),
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_MD, stats.current_cycle as f64),
PrometheusMetric::from_descriptor(&SCANNER_COMPLETED_CYCLES_MD, stats.completed_cycles as f64),
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, stats.current_cycle_age_seconds as f64),
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_SCAN_MODE_MD, stats.current_scan_mode as f64),
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_RESULT_MD, stats.last_cycle_result as f64),
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, stats.last_cycle_duration_seconds),
PrometheusMetric::from_descriptor(&SCANNER_FAILED_CYCLES_MD, stats.failed_cycles as f64),
]
}
@@ -75,12 +98,19 @@ mod tests {
versions_scanned: 1500000,
last_activity_seconds: 30,
active_paths: 4,
current_cycle: 12,
completed_cycles: 11,
current_cycle_age_seconds: 90,
current_scan_mode: 2,
last_cycle_result: 1,
last_cycle_duration_seconds: 42.5,
failed_cycles: 3,
};
let metrics = collect_scanner_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 7);
assert_eq!(metrics.len(), 14);
let objects = metrics.iter().find(|m| m.value == 1000000.0);
assert!(objects.is_some());
@@ -92,6 +122,41 @@ mod tests {
.iter()
.find(|m| m.name == SCANNER_ACTIVE_PATHS_MD.get_full_metric_name());
assert_eq!(active_paths.map(|m| m.value), Some(4.0));
let current_cycle = metrics
.iter()
.find(|m| m.name == SCANNER_CURRENT_CYCLE_MD.get_full_metric_name());
assert_eq!(current_cycle.map(|m| m.value), Some(12.0));
let completed_cycles = metrics
.iter()
.find(|m| m.name == SCANNER_COMPLETED_CYCLES_MD.get_full_metric_name());
assert_eq!(completed_cycles.map(|m| m.value), Some(11.0));
let current_cycle_age = metrics
.iter()
.find(|m| m.name == SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD.get_full_metric_name());
assert_eq!(current_cycle_age.map(|m| m.value), Some(90.0));
let current_scan_mode = metrics
.iter()
.find(|m| m.name == SCANNER_CURRENT_SCAN_MODE_MD.get_full_metric_name());
assert_eq!(current_scan_mode.map(|m| m.value), Some(2.0));
let last_cycle_result = metrics
.iter()
.find(|m| m.name == SCANNER_LAST_CYCLE_RESULT_MD.get_full_metric_name());
assert_eq!(last_cycle_result.map(|m| m.value), Some(1.0));
let last_cycle_duration = metrics
.iter()
.find(|m| m.name == SCANNER_LAST_CYCLE_DURATION_SECONDS_MD.get_full_metric_name());
assert_eq!(last_cycle_duration.map(|m| m.value), Some(42.5));
let failed_cycles = metrics
.iter()
.find(|m| m.name == SCANNER_FAILED_CYCLES_MD.get_full_metric_name());
assert_eq!(failed_cycles.map(|m| m.value), Some(3.0));
}
#[test]
@@ -99,7 +164,7 @@ mod tests {
let stats = ScannerStats::default();
let metrics = collect_scanner_metrics(&stats);
assert_eq!(metrics.len(), 7);
assert_eq!(metrics.len(), 14);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty());
@@ -279,6 +279,13 @@ pub enum MetricName {
ScannerVersionsScanned,
ScannerLastActivitySeconds,
ScannerActivePaths,
ScannerCurrentCycle,
ScannerCompletedCycles,
ScannerCurrentCycleAgeSeconds,
ScannerCurrentScanMode,
ScannerLastCycleResult,
ScannerLastCycleDurationSeconds,
ScannerFailedCycles,
// CPU system-related metrics
SysCPUAvgIdle,
@@ -620,6 +627,13 @@ impl MetricName {
Self::ScannerVersionsScanned => "versions_scanned".to_string(),
Self::ScannerLastActivitySeconds => "last_activity_seconds".to_string(),
Self::ScannerActivePaths => "active_paths".to_string(),
Self::ScannerCurrentCycle => "current_cycle".to_string(),
Self::ScannerCompletedCycles => "completed_cycles".to_string(),
Self::ScannerCurrentCycleAgeSeconds => "current_cycle_age_seconds".to_string(),
Self::ScannerCurrentScanMode => "current_scan_mode".to_string(),
Self::ScannerLastCycleResult => "last_cycle_result".to_string(),
Self::ScannerLastCycleDurationSeconds => "last_cycle_duration_seconds".to_string(),
Self::ScannerFailedCycles => "failed_cycles".to_string(),
// CPU system-related metrics
Self::SysCPUAvgIdle => "avg_idle".to_string(),
+63
View File
@@ -79,3 +79,66 @@ pub static SCANNER_ACTIVE_PATHS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|
subsystems::SCANNER,
)
});
pub static SCANNER_CURRENT_CYCLE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerCurrentCycle,
"Current scanner cycle number, or zero when no cycle is running.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_COMPLETED_CYCLES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerCompletedCycles,
"Total number of scanner cycles completed since server start.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerCurrentCycleAgeSeconds,
"Time elapsed (in seconds) since the current scanner cycle started, or zero when no cycle is running.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_CURRENT_SCAN_MODE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerCurrentScanMode,
"Current scanner mode: 0 unknown or idle, 1 normal, 2 deep bitrot scan.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_LAST_CYCLE_RESULT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerLastCycleResult,
"Last scanner cycle result: 0 unknown, 1 success, 2 error.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_LAST_CYCLE_DURATION_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerLastCycleDurationSeconds,
"Duration in seconds of the last finished scanner cycle.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_FAILED_CYCLES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::ScannerFailedCycles,
"Total number of scanner cycles that failed since server start.",
&[],
subsystems::SCANNER,
)
});
+65 -1
View File
@@ -27,6 +27,7 @@ use crate::metrics::collectors::{
ProcessStatusType, ReplicationStats, ResourceStats, ScannerStats,
};
use chrono::Utc;
use rustfs_common::heal_channel::HealScanMode;
use rustfs_common::metrics::global_metrics;
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{GLOBAL_ExpiryState, GLOBAL_TransitionState};
use rustfs_ecstore::bucket::metadata_sys::get_quota_config;
@@ -44,6 +45,26 @@ use std::time::Duration;
use sysinfo::{Networks, System};
use tracing::{instrument, warn};
fn current_scanner_cycle_age_seconds(
current_cycle: u64,
current_started: chrono::DateTime<Utc>,
now: chrono::DateTime<Utc>,
) -> u64 {
if current_cycle == 0 {
0
} else {
now.signed_duration_since(current_started).num_seconds().max(0) as u64
}
}
fn scanner_scan_mode_code(scan_mode: &str) -> u64 {
match scan_mode {
mode if mode == HealScanMode::Normal.as_str() => HealScanMode::Normal as u8 as u64,
mode if mode == HealScanMode::Deep.as_str() => HealScanMode::Deep as u8 as u64,
_ => HealScanMode::Unknown as u8 as u64,
}
}
const DRIVE_STATE_OK: &str = "ok";
const DRIVE_STATE_ONLINE: &str = "online";
const DRIVE_STATE_UNFORMATTED: &str = "unformatted";
@@ -869,13 +890,17 @@ pub async fn collect_ilm_metric_stats() -> Option<IlmStats> {
/// rustfs-obs scanner collector shape.
pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
let metrics = global_metrics().report().await;
let now = Utc::now();
let bucket_scans_finished = metrics.life_time_ops.get("scan_bucket_drive").copied().unwrap_or_default();
let completed_cycles = metrics.life_time_ops.get("scan_cycle").copied().unwrap_or_default();
let directories_scanned = metrics.life_time_ops.get("scan_folder").copied().unwrap_or_default();
let objects_scanned = metrics.life_time_ops.get("scan_object").copied().unwrap_or_default();
let versions_scanned = metrics.life_time_ilm.values().copied().sum();
let reference_time = metrics.cycles_completed_at.last().copied().unwrap_or(metrics.current_started);
let last_activity_seconds = Utc::now().signed_duration_since(reference_time).num_seconds().max(0) as u64;
let last_activity_seconds = now.signed_duration_since(reference_time).num_seconds().max(0) as u64;
let active_paths = metrics.active_scan_paths as u64;
let current_cycle_age_seconds = current_scanner_cycle_age_seconds(metrics.current_cycle, metrics.current_started, now);
let current_scan_mode = scanner_scan_mode_code(&metrics.current_scan_mode);
Some(ScannerStats {
bucket_scans_finished,
@@ -888,6 +913,13 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
versions_scanned,
last_activity_seconds,
active_paths,
current_cycle: metrics.current_cycle,
completed_cycles,
current_cycle_age_seconds,
current_scan_mode,
last_cycle_result: metrics.last_cycle_result_code,
last_cycle_duration_seconds: metrics.last_cycle_duration_seconds,
failed_cycles: metrics.failed_cycles,
})
}
@@ -957,4 +989,36 @@ mod tests {
assert_eq!(stats.write_health, 1);
assert_eq!(stats.health, 1);
}
#[test]
fn current_scanner_cycle_age_seconds_returns_zero_when_idle() {
let now = Utc::now();
assert_eq!(current_scanner_cycle_age_seconds(0, now - chrono::Duration::seconds(30), now), 0);
}
#[test]
fn current_scanner_cycle_age_seconds_clamps_future_start() {
let now = Utc::now();
assert_eq!(current_scanner_cycle_age_seconds(4, now + chrono::Duration::seconds(30), now), 0);
}
#[test]
fn current_scanner_cycle_age_seconds_reports_active_elapsed_time() {
let now = Utc::now();
assert_eq!(current_scanner_cycle_age_seconds(4, now - chrono::Duration::seconds(45), now), 45);
}
#[test]
fn scanner_scan_mode_code_maps_known_modes() {
assert_eq!(scanner_scan_mode_code(HealScanMode::Normal.as_str()), HealScanMode::Normal as u8 as u64);
assert_eq!(scanner_scan_mode_code(HealScanMode::Deep.as_str()), HealScanMode::Deep as u8 as u64);
}
#[test]
fn scanner_scan_mode_code_maps_unknown_mode() {
assert_eq!(scanner_scan_mode_code(""), HealScanMode::Unknown as u8 as u64);
}
}
+17
View File
@@ -438,6 +438,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
background_heal_info.bitrot_start_cycle,
background_heal_info.bitrot_start_time,
);
let _scan_mode_guard = ScannerScanModeGuard::new(scan_mode);
if let Some(new_heal_info) =
background_heal_info_for_scan_start(background_heal_info.clone(), cycle_info.current, scan_mode, Utc::now())
{
@@ -475,6 +476,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
cycle_info.next += 1;
cycle_info.current = 0;
cycle_info.cycle_completed.push(Utc::now());
global_metrics().clear_current_scan_mode();
info!(duration = ?now.elapsed(), cycles_total=cycle_info.cycle_completed.len(), "Success run data scanner cycle");
@@ -553,6 +555,21 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) ->
Ok(())
}
struct ScannerScanModeGuard;
impl ScannerScanModeGuard {
fn new(scan_mode: HealScanMode) -> Self {
global_metrics().set_current_scan_mode(scan_mode);
Self
}
}
impl Drop for ScannerScanModeGuard {
fn drop(&mut self) {
global_metrics().clear_current_scan_mode();
}
}
/// Store data usage info in backend. Will store all objects sent on the receiver until closed.
#[instrument(skip(ctx, storeapi))]
pub async fn store_data_usage_in_backend(
+36 -2
View File
@@ -57,7 +57,6 @@ use tracing::{debug, error, info, warn};
const DATA_USAGE_UPDATE_DIR_CYCLES: u32 = 16;
const DATA_SCANNER_COMPACT_LEAST_OBJECT: usize = 500;
const YIELD_EVERY_N_OBJECTS: u64 = 128;
const DATA_SCANNER_COMPACT_AT_CHILDREN: usize = 10000;
const DATA_SCANNER_COMPACT_AT_FOLDERS: usize = DATA_SCANNER_COMPACT_AT_CHILDREN / 4;
const DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS: usize = 250_000;
@@ -144,6 +143,17 @@ fn scanner_excess_folders_threshold() -> u64 {
)
}
fn scanner_yield_every_n_objects() -> u64 {
rustfs_utils::get_env_u64(
rustfs_config::ENV_SCANNER_YIELD_EVERY_N_OBJECTS,
rustfs_config::DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS,
)
}
fn should_yield_after_object(object_count: u64, yield_every: u64) -> bool {
yield_every > 0 && object_count.is_multiple_of(yield_every)
}
fn should_alert_excessive_versions(remaining_versions: usize, cumulative_size: i64) -> (bool, bool) {
let too_many_versions = remaining_versions as u64 >= scanner_excess_versions_threshold();
let too_large_versions = cumulative_size > 0 && cumulative_size as u64 >= scanner_excess_version_size_threshold();
@@ -916,6 +926,7 @@ impl FolderScanner {
let mut new_folders: Vec<CachedFolder> = Vec::new();
let mut found_objects = false;
let mut object_count: u64 = 0;
let yield_every_objects = scanner_yield_every_n_objects();
let dir_path = path_join_buf(&[&self.root, &folder.name]);
@@ -1098,7 +1109,7 @@ impl FolderScanner {
timer.sleep().await;
if object_count.is_multiple_of(YIELD_EVERY_N_OBJECTS) {
if should_yield_after_object(object_count, yield_every_objects) {
tokio::task::yield_now().await;
}
}
@@ -1816,6 +1827,29 @@ mod tests {
});
}
#[test]
#[serial]
fn test_scanner_yield_every_n_objects_uses_env() {
with_var(rustfs_config::ENV_SCANNER_YIELD_EVERY_N_OBJECTS, Some("32"), || {
assert_eq!(scanner_yield_every_n_objects(), 32);
});
}
#[test]
#[serial]
fn test_scanner_yield_every_n_objects_uses_default() {
with_var_unset(rustfs_config::ENV_SCANNER_YIELD_EVERY_N_OBJECTS, || {
assert_eq!(scanner_yield_every_n_objects(), rustfs_config::DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS);
});
}
#[test]
fn test_should_yield_after_object_respects_interval_and_disable() {
assert!(!should_yield_after_object(127, 128));
assert!(should_yield_after_object(128, 128));
assert!(!should_yield_after_object(128, 0));
}
#[tokio::test]
#[serial]
async fn test_record_failed_prunes_to_max_entries() {