gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-24 05:06:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Add RustFS Scanner Module and Multiple Bug Fixes (#1579)

Browse Source
This commit is contained in:
weisd
2026-01-22 13:39:38 +08:00
committed by GitHub
parent 6c5f8e591a
commit 6631407416
47 changed files with 6635 additions and 580 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scanner/src/data_usage_define.rs
+1601
View File
File diff suppressed because it is too large Load Diff
crates/scanner/src/error.rs
+36
View File
@@ -0,0 +1,36 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use thiserror::Error;
/// Scanner-related errors
#[derive(Error, Debug)]
#[non_exhaustive]
pub enum ScannerError {
/// Configuration error
#[error("Configuration error: {0}")]
Config(String),
/// I/O error
#[error("I/O error: {0}")]
Io(#[from] std::io::Error),
/// Serialization error
#[error("Serialization error: {0}")]
Serialization(#[from] serde_json::Error),
/// Generic error
#[error("Scanner error: {0}")]
Other(String),
}
crates/scanner/src/last_minute.rs
+886
View File
@@ -0,0 +1,886 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::time::{Duration, SystemTime, UNIX_EPOCH};
#[allow(dead_code)]
#[derive(Debug, Default)]
pub struct TimedAction {
count: u64,
acc_time: u64,
min_time: Option<u64>,
max_time: Option<u64>,
bytes: u64,
}
#[allow(dead_code)]
impl TimedAction {
// Avg returns the average time spent on the action.
pub fn avg(&self) -> Option<Duration> {
if self.count == 0 {
return None;
}
Some(Duration::from_nanos(self.acc_time / self.count))
}
// AvgBytes returns the average bytes processed.
pub fn avg_bytes(&self) -> u64 {
if self.count == 0 {
return 0;
}
self.bytes / self.count
}
// Merge other into t.
pub fn merge(&mut self, other: TimedAction) {
self.count += other.count;
self.acc_time += other.acc_time;
self.bytes += other.bytes;
if self.count == 0 {
self.min_time = other.min_time;
}
if let Some(other_min) = other.min_time {
self.min_time = self.min_time.map_or(Some(other_min), |min| Some(min.min(other_min)));
}
self.max_time = self
.max_time
.map_or(other.max_time, |max| Some(max.max(other.max_time.unwrap_or(0))));
}
}
#[allow(dead_code)]
#[derive(Debug)]
enum SizeCategory {
SizeLessThan1KiB = 0,
SizeLessThan1MiB,
SizeLessThan10MiB,
SizeLessThan100MiB,
SizeLessThan1GiB,
SizeGreaterThan1GiB,
// Add new entries here
SizeLastElemMarker,
}
impl std::fmt::Display for SizeCategory {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let s = match *self {
SizeCategory::SizeLessThan1KiB => "SizeLessThan1KiB",
SizeCategory::SizeLessThan1MiB => "SizeLessThan1MiB",
SizeCategory::SizeLessThan10MiB => "SizeLessThan10MiB",
SizeCategory::SizeLessThan100MiB => "SizeLessThan100MiB",
SizeCategory::SizeLessThan1GiB => "SizeLessThan1GiB",
SizeCategory::SizeGreaterThan1GiB => "SizeGreaterThan1GiB",
SizeCategory::SizeLastElemMarker => "SizeLastElemMarker",
};
write!(f, "{s}")
}
}
#[derive(Clone, Debug, Default, Copy)]
pub struct AccElem {
pub total: u64,
pub size: u64,
pub n: u64,
}
impl AccElem {
pub fn add(&mut self, dur: &Duration) {
let dur = dur.as_secs();
self.total = self.total.wrapping_add(dur);
self.n = self.n.wrapping_add(1);
}
pub fn merge(&mut self, b: &AccElem) {
self.n = self.n.wrapping_add(b.n);
self.total = self.total.wrapping_add(b.total);
self.size = self.size.wrapping_add(b.size);
}
pub fn avg(&self) -> Duration {
if self.n >= 1 && self.total > 0 {
return Duration::from_secs(self.total / self.n);
}
Duration::from_secs(0)
}
}
#[derive(Clone, Debug)]
pub struct LastMinuteLatency {
pub totals: Vec<AccElem>,
pub last_sec: u64,
}
impl Default for LastMinuteLatency {
fn default() -> Self {
Self {
totals: vec![AccElem::default(); 60],
last_sec: Default::default(),
}
}
}
impl LastMinuteLatency {
pub fn merge(&mut self, o: &LastMinuteLatency) -> LastMinuteLatency {
let mut merged = LastMinuteLatency::default();
let mut x = o.clone();
if self.last_sec > o.last_sec {
x.forward_to(self.last_sec);
merged.last_sec = self.last_sec;
} else {
self.forward_to(o.last_sec);
merged.last_sec = o.last_sec;
}
for i in 0..merged.totals.len() {
merged.totals[i] = AccElem {
total: self.totals[i].total + o.totals[i].total,
n: self.totals[i].n + o.totals[i].n,
size: self.totals[i].size + o.totals[i].size,
}
}
merged
}
pub fn add(&mut self, t: &Duration) {
let sec = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("Time went backwards")
.as_secs();
self.forward_to(sec);
let win_idx = sec % 60;
self.totals[win_idx as usize].add(t);
self.last_sec = sec;
}
pub fn add_all(&mut self, sec: u64, a: &AccElem) {
self.forward_to(sec);
let win_idx = sec % 60;
self.totals[win_idx as usize].merge(a);
self.last_sec = sec;
}
pub fn get_total(&mut self) -> AccElem {
let mut res = AccElem::default();
let sec = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("Time went backwards")
.as_secs();
self.forward_to(sec);
for elem in self.totals.iter() {
res.merge(elem);
}
res
}
pub fn forward_to(&mut self, t: u64) {
if self.last_sec >= t {
return;
}
if t - self.last_sec >= 60 {
self.totals = vec![AccElem::default(); 60];
self.last_sec = t;
return;
}
while self.last_sec != t {
let idx = (self.last_sec + 1) % 60;
self.totals[idx as usize] = AccElem::default();
self.last_sec += 1;
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::Duration;
#[test]
fn test_acc_elem_default() {
let elem = AccElem::default();
assert_eq!(elem.total, 0);
assert_eq!(elem.size, 0);
assert_eq!(elem.n, 0);
}
#[test]
fn test_acc_elem_add_single_duration() {
let mut elem = AccElem::default();
let duration = Duration::from_secs(5);
elem.add(&duration);
assert_eq!(elem.total, 5);
assert_eq!(elem.n, 1);
assert_eq!(elem.size, 0); // size is not modified by add
}
#[test]
fn test_acc_elem_add_multiple_durations() {
let mut elem = AccElem::default();
elem.add(&Duration::from_secs(3));
elem.add(&Duration::from_secs(7));
elem.add(&Duration::from_secs(2));
assert_eq!(elem.total, 12);
assert_eq!(elem.n, 3);
assert_eq!(elem.size, 0);
}
#[test]
fn test_acc_elem_add_zero_duration() {
let mut elem = AccElem::default();
let duration = Duration::from_secs(0);
elem.add(&duration);
assert_eq!(elem.total, 0);
assert_eq!(elem.n, 1);
}
#[test]
fn test_acc_elem_add_subsecond_duration() {
let mut elem = AccElem::default();
// Duration less than 1 second should be truncated to 0
let duration = Duration::from_millis(500);
elem.add(&duration);
assert_eq!(elem.total, 0); // as_secs() truncates subsecond values
assert_eq!(elem.n, 1);
}
#[test]
fn test_acc_elem_merge_empty_elements() {
let mut elem1 = AccElem::default();
let elem2 = AccElem::default();
elem1.merge(&elem2);
assert_eq!(elem1.total, 0);
assert_eq!(elem1.size, 0);
assert_eq!(elem1.n, 0);
}
#[test]
fn test_acc_elem_merge_with_data() {
let mut elem1 = AccElem {
total: 10,
size: 100,
n: 2,
};
let elem2 = AccElem {
total: 15,
size: 200,
n: 3,
};
elem1.merge(&elem2);
assert_eq!(elem1.total, 25);
assert_eq!(elem1.size, 300);
assert_eq!(elem1.n, 5);
}
#[test]
fn test_acc_elem_merge_one_empty() {
let mut elem1 = AccElem {
total: 10,
size: 100,
n: 2,
};
let elem2 = AccElem::default();
elem1.merge(&elem2);
assert_eq!(elem1.total, 10);
assert_eq!(elem1.size, 100);
assert_eq!(elem1.n, 2);
}
#[test]
fn test_acc_elem_avg_with_data() {
let elem = AccElem {
total: 15,
size: 0,
n: 3,
};
let avg = elem.avg();
assert_eq!(avg, Duration::from_secs(5)); // 15 / 3 = 5
}
#[test]
fn test_acc_elem_avg_zero_count() {
let elem = AccElem {
total: 10,
size: 0,
n: 0,
};
let avg = elem.avg();
assert_eq!(avg, Duration::from_secs(0));
}
#[test]
fn test_acc_elem_avg_zero_total() {
let elem = AccElem { total: 0, size: 0, n: 5 };
let avg = elem.avg();
assert_eq!(avg, Duration::from_secs(0));
}
#[test]
fn test_acc_elem_avg_rounding() {
let elem = AccElem {
total: 10,
size: 0,
n: 3,
};
let avg = elem.avg();
assert_eq!(avg, Duration::from_secs(3)); // 10 / 3 = 3 (integer division)
}
#[test]
fn test_last_minute_latency_default() {
let latency = LastMinuteLatency::default();
assert_eq!(latency.totals.len(), 60);
assert_eq!(latency.last_sec, 0);
// All elements should be default (empty)
for elem in &latency.totals {
assert_eq!(elem.total, 0);
assert_eq!(elem.size, 0);
assert_eq!(elem.n, 0);
}
}
#[test]
fn test_last_minute_latency_forward_to_same_time() {
let mut latency = LastMinuteLatency {
last_sec: 100,
..Default::default()
};
// Add some data to verify it's not cleared
latency.totals[0].total = 10;
latency.totals[0].n = 1;
latency.forward_to(100); // Same time
assert_eq!(latency.last_sec, 100);
assert_eq!(latency.totals[0].total, 10); // Data should remain
assert_eq!(latency.totals[0].n, 1);
}
#[test]
fn test_last_minute_latency_forward_to_past_time() {
let mut latency = LastMinuteLatency {
last_sec: 100,
..Default::default()
};
// Add some data to verify it's not cleared
latency.totals[0].total = 10;
latency.totals[0].n = 1;
latency.forward_to(50); // Past time
assert_eq!(latency.last_sec, 100); // Should not change
assert_eq!(latency.totals[0].total, 10); // Data should remain
assert_eq!(latency.totals[0].n, 1);
}
#[test]
fn test_last_minute_latency_forward_to_large_gap() {
let mut latency = LastMinuteLatency {
last_sec: 100,
..Default::default()
};
// Add some data to verify it's cleared
latency.totals[0].total = 10;
latency.totals[0].n = 1;
latency.forward_to(200); // Gap >= 60 seconds
assert_eq!(latency.last_sec, 200); // last_sec should be updated to target time
// All data should be cleared
for elem in &latency.totals {
assert_eq!(elem.total, 0);
assert_eq!(elem.size, 0);
assert_eq!(elem.n, 0);
}
}
#[test]
fn test_last_minute_latency_forward_to_small_gap() {
let mut latency = LastMinuteLatency {
last_sec: 100,
..Default::default()
};
// Add data at specific indices
latency.totals[41].total = 10; // (100 + 1) % 60 = 41
latency.totals[42].total = 20; // (100 + 2) % 60 = 42
latency.forward_to(102); // Forward by 2 seconds
assert_eq!(latency.last_sec, 102);
// The slots that were advanced should be cleared
assert_eq!(latency.totals[41].total, 0); // Cleared during forward
assert_eq!(latency.totals[42].total, 0); // Cleared during forward
}
#[test]
fn test_last_minute_latency_add_all() {
let mut latency = LastMinuteLatency::default();
let acc_elem = AccElem {
total: 15,
size: 100,
n: 3,
};
latency.add_all(1000, &acc_elem);
assert_eq!(latency.last_sec, 1000);
let idx = 1000 % 60; // Should be 40
assert_eq!(latency.totals[idx as usize].total, 15);
assert_eq!(latency.totals[idx as usize].size, 100);
assert_eq!(latency.totals[idx as usize].n, 3);
}
#[test]
fn test_last_minute_latency_add_all_multiple() {
let mut latency = LastMinuteLatency::default();
let acc_elem1 = AccElem {
total: 10,
size: 50,
n: 2,
};
let acc_elem2 = AccElem {
total: 20,
size: 100,
n: 4,
};
latency.add_all(1000, &acc_elem1);
latency.add_all(1000, &acc_elem2); // Same second
let idx = 1000 % 60;
assert_eq!(latency.totals[idx as usize].total, 30); // 10 + 20
assert_eq!(latency.totals[idx as usize].size, 150); // 50 + 100
assert_eq!(latency.totals[idx as usize].n, 6); // 2 + 4
}
#[test]
fn test_last_minute_latency_merge_same_time() {
let mut latency1 = LastMinuteLatency::default();
let mut latency2 = LastMinuteLatency::default();
latency1.last_sec = 1000;
latency2.last_sec = 1000;
// Add data to both
latency1.totals[0].total = 10;
latency1.totals[0].n = 2;
latency2.totals[0].total = 20;
latency2.totals[0].n = 3;
let merged = latency1.merge(&latency2);
assert_eq!(merged.last_sec, 1000);
assert_eq!(merged.totals[0].total, 30); // 10 + 20
assert_eq!(merged.totals[0].n, 5); // 2 + 3
}
#[test]
fn test_last_minute_latency_merge_different_times() {
let mut latency1 = LastMinuteLatency::default();
let mut latency2 = LastMinuteLatency::default();
latency1.last_sec = 1000;
latency2.last_sec = 1010; // 10 seconds later
// Add data to both
latency1.totals[0].total = 10;
latency2.totals[0].total = 20;
let merged = latency1.merge(&latency2);
assert_eq!(merged.last_sec, 1010); // Should use the later time
assert_eq!(merged.totals[0].total, 30);
}
#[test]
fn test_last_minute_latency_merge_empty() {
let mut latency1 = LastMinuteLatency::default();
let latency2 = LastMinuteLatency::default();
let merged = latency1.merge(&latency2);
assert_eq!(merged.last_sec, 0);
for elem in &merged.totals {
assert_eq!(elem.total, 0);
assert_eq!(elem.size, 0);
assert_eq!(elem.n, 0);
}
}
#[test]
fn test_last_minute_latency_window_wraparound() {
let mut latency = LastMinuteLatency::default();
// Test that indices wrap around correctly
for sec in 0..120 {
// Test for 2 minutes
let acc_elem = AccElem {
total: sec,
size: 0,
n: 1,
};
latency.add_all(sec, &acc_elem);
let expected_idx = sec % 60;
assert_eq!(latency.totals[expected_idx as usize].total, sec);
}
}
#[test]
fn test_last_minute_latency_time_progression() {
let mut latency = LastMinuteLatency::default();
// Add data at time 1000
latency.add_all(
1000,
&AccElem {
total: 10,
size: 0,
n: 1,
},
);
// Forward to time 1030 (30 seconds later)
latency.forward_to(1030);
// Original data should still be there
let idx_1000 = 1000 % 60;
assert_eq!(latency.totals[idx_1000 as usize].total, 10);
// Forward to time 1070 (70 seconds from original, > 60 seconds)
latency.forward_to(1070);
// All data should be cleared due to large gap
for elem in &latency.totals {
assert_eq!(elem.total, 0);
assert_eq!(elem.n, 0);
}
}
#[test]
fn test_last_minute_latency_realistic_scenario() {
let mut latency = LastMinuteLatency::default();
let base_time = 1000u64;
// Add data for exactly 60 seconds to fill the window
for i in 0..60 {
let current_time = base_time + i;
let duration_secs = i % 10 + 1; // Varying durations 1-10 seconds
let acc_elem = AccElem {
total: duration_secs,
size: 1024 * (i % 5 + 1), // Varying sizes
n: 1,
};
latency.add_all(current_time, &acc_elem);
}
// Count non-empty slots after filling the window
let mut non_empty_count = 0;
let mut total_n = 0;
let mut total_sum = 0;
for elem in &latency.totals {
if elem.n > 0 {
non_empty_count += 1;
total_n += elem.n;
total_sum += elem.total;
}
}
// We should have exactly 60 non-empty slots (one for each second in the window)
assert_eq!(non_empty_count, 60);
assert_eq!(total_n, 60); // 60 data points total
assert!(total_sum > 0);
// Test manual total calculation (get_total uses system time which interferes with test)
let mut manual_total = AccElem::default();
for elem in &latency.totals {
manual_total.merge(elem);
}
assert_eq!(manual_total.n, 60);
assert_eq!(manual_total.total, total_sum);
}
#[test]
fn test_acc_elem_clone_and_debug() {
let elem = AccElem {
total: 100,
size: 200,
n: 5,
};
let cloned = elem;
assert_eq!(elem.total, cloned.total);
assert_eq!(elem.size, cloned.size);
assert_eq!(elem.n, cloned.n);
// Test Debug trait
let debug_str = format!("{elem:?}");
assert!(debug_str.contains("100"));
assert!(debug_str.contains("200"));
assert!(debug_str.contains("5"));
}
#[test]
fn test_last_minute_latency_clone() {
let mut latency = LastMinuteLatency {
last_sec: 1000,
..Default::default()
};
latency.totals[0].total = 100;
latency.totals[0].n = 5;
let cloned = latency.clone();
assert_eq!(latency.last_sec, cloned.last_sec);
assert_eq!(latency.totals[0].total, cloned.totals[0].total);
assert_eq!(latency.totals[0].n, cloned.totals[0].n);
}
#[test]
fn test_edge_case_max_values() {
let mut elem = AccElem {
total: u64::MAX - 50,
size: u64::MAX - 50,
n: u64::MAX - 50,
};
let other = AccElem {
total: 100,
size: 100,
n: 100,
};
// This should not panic due to overflow, values will wrap around
elem.merge(&other);
// Values should wrap around due to overflow (wrapping_add behavior)
assert_eq!(elem.total, 49); // (u64::MAX - 50) + 100 wraps to 49
assert_eq!(elem.size, 49);
assert_eq!(elem.n, 49);
}
#[test]
fn test_forward_to_boundary_conditions() {
let mut latency = LastMinuteLatency {
last_sec: 59,
..Default::default()
};
// Add data at the last slot
latency.totals[59].total = 100;
latency.totals[59].n = 1;
// Forward exactly 60 seconds (boundary case)
latency.forward_to(119);
// All data should be cleared
for elem in &latency.totals {
assert_eq!(elem.total, 0);
assert_eq!(elem.n, 0);
}
}
#[test]
fn test_get_total_with_data() {
let mut latency = LastMinuteLatency::default();
// Set a recent timestamp to avoid forward_to clearing data
let current_time = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("Time went backwards")
.as_secs();
latency.last_sec = current_time;
// Add data to multiple slots
latency.totals[0] = AccElem {
total: 10,
size: 100,
n: 1,
};
latency.totals[1] = AccElem {
total: 20,
size: 200,
n: 2,
};
latency.totals[59] = AccElem {
total: 30,
size: 300,
n: 3,
};
let total = latency.get_total();
assert_eq!(total.total, 60);
assert_eq!(total.size, 600);
assert_eq!(total.n, 6);
}
#[test]
fn test_window_index_calculation() {
// Test that window index calculation works correctly
let _latency = LastMinuteLatency::default();
let acc_elem = AccElem { total: 1, size: 1, n: 1 };
// Test various timestamps
let test_cases = [(0, 0), (1, 1), (59, 59), (60, 0), (61, 1), (119, 59), (120, 0)];
for (timestamp, expected_idx) in test_cases {
let mut test_latency = LastMinuteLatency::default();
test_latency.add_all(timestamp, &acc_elem);
assert_eq!(
test_latency.totals[expected_idx].n, 1,
"Failed for timestamp {timestamp} (expected index {expected_idx})"
);
}
}
#[test]
fn test_concurrent_safety_simulation() {
// Simulate concurrent access patterns
let mut latency = LastMinuteLatency::default();
// Use current time to ensure data doesn't get cleared by get_total
let current_time = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("Time went backwards")
.as_secs();
// Simulate rapid additions within a 60-second window
for i in 0..1000 {
let acc_elem = AccElem {
total: (i % 10) + 1, // Ensure non-zero values
size: (i % 100) + 1,
n: 1,
};
// Keep all timestamps within the current minute window
latency.add_all(current_time - (i % 60), &acc_elem);
}
let total = latency.get_total();
assert!(total.n > 0, "Total count should be greater than 0");
assert!(total.total > 0, "Total time should be greater than 0");
}
#[test]
fn test_acc_elem_debug_format() {
let elem = AccElem {
total: 123,
size: 456,
n: 789,
};
let debug_str = format!("{elem:?}");
assert!(debug_str.contains("123"));
assert!(debug_str.contains("456"));
assert!(debug_str.contains("789"));
}
#[test]
fn test_large_values() {
let mut elem = AccElem::default();
// Test with large duration values
let large_duration = Duration::from_secs(u64::MAX / 2);
elem.add(&large_duration);
assert_eq!(elem.total, u64::MAX / 2);
assert_eq!(elem.n, 1);
// Test average calculation with large values
let avg = elem.avg();
assert_eq!(avg, Duration::from_secs(u64::MAX / 2));
}
#[test]
fn test_zero_duration_handling() {
let mut elem = AccElem::default();
let zero_duration = Duration::from_secs(0);
elem.add(&zero_duration);
assert_eq!(elem.total, 0);
assert_eq!(elem.n, 1);
assert_eq!(elem.avg(), Duration::from_secs(0));
}
}
const SIZE_LAST_ELEM_MARKER: usize = 10; // Assumed marker size is 10, modify according to actual situation
#[allow(dead_code)]
#[derive(Debug, Default)]
pub struct LastMinuteHistogram {
histogram: Vec<LastMinuteLatency>,
size: u32,
}
impl LastMinuteHistogram {
pub fn merge(&mut self, other: &LastMinuteHistogram) {
for i in 0..self.histogram.len() {
self.histogram[i].merge(&other.histogram[i]);
}
}
pub fn add(&mut self, size: i64, t: Duration) {
let index = size_to_tag(size);
self.histogram[index].add(&t);
}
pub fn get_avg_data(&mut self) -> [AccElem; SIZE_LAST_ELEM_MARKER] {
let mut res = [AccElem::default(); SIZE_LAST_ELEM_MARKER];
for (i, elem) in self.histogram.iter_mut().enumerate() {
res[i] = elem.get_total();
}
res
}
}
fn size_to_tag(size: i64) -> usize {
match size {
_ if size < 1024 => 0, // sizeLessThan1KiB
_ if size < 1024 * 1024 => 1, // sizeLessThan1MiB
_ if size < 10 * 1024 * 1024 => 2, // sizeLessThan10MiB
_ if size < 100 * 1024 * 1024 => 3, // sizeLessThan100MiB
_ if size < 1024 * 1024 * 1024 => 4, // sizeLessThan1GiB
_ => 5, // sizeGreaterThan1GiB
}
}
crates/scanner/src/lib.rs
+33
View File
@@ -0,0 +1,33 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![cfg_attr(docsrs, feature(doc_auto_cfg))]
#![warn(
// missing_docs,
rustdoc::missing_crate_level_docs,
unreachable_pub,
rust_2018_idioms
)]
pub mod data_usage_define;
pub mod error;
pub mod last_minute;
pub mod metrics;
pub mod scanner;
pub mod scanner_folder;
pub mod scanner_io;
pub use data_usage_define::*;
pub use error::ScannerError;
pub use scanner::init_data_scanner;
crates/scanner/src/metrics.rs
+576
View File
@@ -0,0 +1,576 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::last_minute::{AccElem, LastMinuteLatency};
use chrono::{DateTime, Utc};
use rustfs_madmin::metrics::ScannerMetrics as M_ScannerMetrics;
use serde::{Deserialize, Serialize};
use std::{
collections::HashMap,
fmt::Display,
pin::Pin,
sync::{
Arc, OnceLock,
atomic::{AtomicU64, Ordering},
},
time::{Duration, SystemTime},
};
use tokio::sync::{Mutex, RwLock};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum IlmAction {
NoneAction = 0,
DeleteAction,
DeleteVersionAction,
TransitionAction,
TransitionVersionAction,
DeleteRestoredAction,
DeleteRestoredVersionAction,
DeleteAllVersionsAction,
DelMarkerDeleteAllVersionsAction,
ActionCount,
}
impl IlmAction {
pub fn delete_restored(&self) -> bool {
*self == Self::DeleteRestoredAction || *self == Self::DeleteRestoredVersionAction
}
pub fn delete_versioned(&self) -> bool {
*self == Self::DeleteVersionAction || *self == Self::DeleteRestoredVersionAction
}
pub fn delete_all(&self) -> bool {
*self == Self::DeleteAllVersionsAction || *self == Self::DelMarkerDeleteAllVersionsAction
}
pub fn delete(&self) -> bool {
if self.delete_restored() {
return true;
}
*self == Self::DeleteVersionAction
|| *self == Self::DeleteAction
|| *self == Self::DeleteAllVersionsAction
|| *self == Self::DelMarkerDeleteAllVersionsAction
}
}
impl Display for IlmAction {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{self:?}")
}
}
pub static GLOBAL_METRICS: OnceLock<Arc<Metrics>> = OnceLock::new();
pub fn global_metrics() -> &'static Arc<Metrics> {
GLOBAL_METRICS.get_or_init(|| Arc::new(Metrics::new()))
}
#[derive(Clone, Debug, PartialEq, PartialOrd)]
pub enum Metric {
// START Realtime metrics, that only records
// last minute latencies and total operation count.
ReadMetadata = 0,
CheckMissing,
SaveUsage,
ApplyAll,
ApplyVersion,
TierObjSweep,
HealCheck,
Ilm,
CheckReplication,
Yield,
CleanAbandoned,
ApplyNonCurrent,
HealAbandonedVersion,
// START Trace metrics:
StartTrace,
ScanObject, // Scan object. All operations included.
HealAbandonedObject,
// END realtime metrics:
LastRealtime,
// Trace only metrics:
ScanFolder, // Scan a folder on disk, recursively.
ScanCycle, // Full cycle, cluster global.
ScanBucketDrive, // Single bucket on one drive.
CompactFolder, // Folder compacted.
// Must be last:
Last,
}
impl Metric {
/// Convert to string representation for metrics
pub fn as_str(self) -> &'static str {
match self {
Self::ReadMetadata => "read_metadata",
Self::CheckMissing => "check_missing",
Self::SaveUsage => "save_usage",
Self::ApplyAll => "apply_all",
Self::ApplyVersion => "apply_version",
Self::TierObjSweep => "tier_obj_sweep",
Self::HealCheck => "heal_check",
Self::Ilm => "ilm",
Self::CheckReplication => "check_replication",
Self::Yield => "yield",
Self::CleanAbandoned => "clean_abandoned",
Self::ApplyNonCurrent => "apply_non_current",
Self::HealAbandonedVersion => "heal_abandoned_version",
Self::StartTrace => "start_trace",
Self::ScanObject => "scan_object",
Self::HealAbandonedObject => "heal_abandoned_object",
Self::LastRealtime => "last_realtime",
Self::ScanFolder => "scan_folder",
Self::ScanCycle => "scan_cycle",
Self::ScanBucketDrive => "scan_bucket_drive",
Self::CompactFolder => "compact_folder",
Self::Last => "last",
}
}
/// Convert from index back to enum (safe version)
pub fn from_index(index: usize) -> Option<Self> {
if index >= Self::Last as usize {
return None;
}
// Safe conversion using match instead of unsafe transmute
match index {
0 => Some(Self::ReadMetadata),
1 => Some(Self::CheckMissing),
2 => Some(Self::SaveUsage),
3 => Some(Self::ApplyAll),
4 => Some(Self::ApplyVersion),
5 => Some(Self::TierObjSweep),
6 => Some(Self::HealCheck),
7 => Some(Self::Ilm),
8 => Some(Self::CheckReplication),
9 => Some(Self::Yield),
10 => Some(Self::CleanAbandoned),
11 => Some(Self::ApplyNonCurrent),
12 => Some(Self::HealAbandonedVersion),
13 => Some(Self::StartTrace),
14 => Some(Self::ScanObject),
15 => Some(Self::HealAbandonedObject),
16 => Some(Self::LastRealtime),
17 => Some(Self::ScanFolder),
18 => Some(Self::ScanCycle),
19 => Some(Self::ScanBucketDrive),
20 => Some(Self::CompactFolder),
21 => Some(Self::Last),
_ => None,
}
}
}
/// Thread-safe wrapper for LastMinuteLatency with atomic operations
#[derive(Default)]
pub struct LockedLastMinuteLatency {
latency: Arc<Mutex<LastMinuteLatency>>,
}
impl Clone for LockedLastMinuteLatency {
fn clone(&self) -> Self {
Self {
latency: Arc::clone(&self.latency),
}
}
}
impl LockedLastMinuteLatency {
pub fn new() -> Self {
Self {
latency: Arc::new(Mutex::new(LastMinuteLatency::default())),
}
}
/// Add a duration measurement
pub async fn add(&self, duration: Duration) {
self.add_size(duration, 0).await;
}
/// Add a duration measurement with size
pub async fn add_size(&self, duration: Duration, size: u64) {
let mut latency = self.latency.lock().await;
let now = SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
let elem = AccElem {
n: 1,
total: duration.as_secs(),
size,
};
latency.add_all(now, &elem);
}
/// Get total accumulated metrics for the last minute
pub async fn total(&self) -> AccElem {
let mut latency = self.latency.lock().await;
latency.get_total()
}
}
/// Current path tracker for monitoring active scan paths
struct CurrentPathTracker {
current_path: Arc<RwLock<String>>,
}
impl CurrentPathTracker {
fn new(initial_path: String) -> Self {
Self {
current_path: Arc::new(RwLock::new(initial_path)),
}
}
async fn update_path(&self, path: String) {
*self.current_path.write().await = path;
}
async fn get_path(&self) -> String {
self.current_path.read().await.clone()
}
}
/// Main scanner metrics structure
pub struct Metrics {
// All fields must be accessed atomically and aligned.
operations: Vec<AtomicU64>,
latency: Vec<LockedLastMinuteLatency>,
actions: Vec<AtomicU64>,
actions_latency: Vec<LockedLastMinuteLatency>,
// Current paths contains disk -> tracker mappings
current_paths: Arc<RwLock<HashMap<String, Arc<CurrentPathTracker>>>>,
// Cycle information
cycle_info: Arc<RwLock<Option<CurrentCycle>>>,
}
// This is a placeholder. We'll need to define this struct.
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct CurrentCycle {
pub current: u64,
pub next: u64,
pub cycle_completed: Vec<DateTime<Utc>>,
pub started: DateTime<Utc>,
}
impl CurrentCycle {
pub fn unmarshal(&mut self, buf: &[u8]) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
*self = rmp_serde::from_slice(buf)?;
Ok(())
}
pub fn marshal(&self) -> Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync>> {
Ok(rmp_serde::to_vec(self)?)
}
}
impl Metrics {
pub fn new() -> Self {
let operations = (0..Metric::Last as usize).map(|_| AtomicU64::new(0)).collect();
let latency = (0..Metric::LastRealtime as usize)
.map(|_| LockedLastMinuteLatency::new())
.collect();
Self {
operations,
latency,
actions: (0..IlmAction::ActionCount as usize).map(|_| AtomicU64::new(0)).collect(),
actions_latency: vec![LockedLastMinuteLatency::default(); IlmAction::ActionCount as usize],
current_paths: Arc::new(RwLock::new(HashMap::new())),
cycle_info: Arc::new(RwLock::new(None)),
}
}
/// Log scanner action with custom metadata - compatible with existing usage
pub fn log(metric: Metric) -> impl Fn(&HashMap<String, String>) {
let metric = metric as usize;
let start_time = SystemTime::now();
move |_custom: &HashMap<String, String>| {
let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
// Update operation count
global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed);
// Update latency for realtime metrics (spawn async task for this)
if (metric) < Metric::LastRealtime as usize {
let metric_index = metric;
tokio::spawn(async move {
global_metrics().latency[metric_index].add(duration).await;
});
}
// Log trace metrics
if metric as u8 > Metric::StartTrace as u8 {
//debug!(metric = metric.as_str(), duration_ms = duration.as_millis(), "Scanner trace metric");
}
}
}
/// Time scanner action with size - returns function that takes size
pub fn time_size(metric: Metric) -> impl Fn(u64) {
let metric = metric as usize;
let start_time = SystemTime::now();
move |size: u64| {
let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
// Update operation count
global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed);
// Update latency for realtime metrics with size (spawn async task)
if (metric) < Metric::LastRealtime as usize {
let metric_index = metric;
tokio::spawn(async move {
global_metrics().latency[metric_index].add_size(duration, size).await;
});
}
}
}
/// Time a scanner action - returns a closure to call when done
pub fn time(metric: Metric) -> impl Fn() {
let metric = metric as usize;
let start_time = SystemTime::now();
move || {
let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
// Update operation count
global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed);
// Update latency for realtime metrics (spawn async task)
if (metric) < Metric::LastRealtime as usize {
let metric_index = metric;
tokio::spawn(async move {
global_metrics().latency[metric_index].add(duration).await;
});
}
}
}
/// Time N scanner actions - returns function that takes count, then returns completion function
pub fn time_n(metric: Metric) -> Box<dyn Fn(usize) -> Box<dyn Fn() + Send + Sync> + Send + Sync> {
let metric = metric as usize;
let start_time = SystemTime::now();
Box::new(move |count: usize| {
Box::new(move || {
let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
// Update operation count
global_metrics().operations[metric].fetch_add(count as u64, Ordering::Relaxed);
// Update latency for realtime metrics (spawn async task)
if (metric) < Metric::LastRealtime as usize {
let metric_index = metric;
tokio::spawn(async move {
global_metrics().latency[metric_index].add(duration).await;
});
}
})
})
}
/// Time ILM action with versions - returns function that takes versions, then returns completion function
pub fn time_ilm(a: IlmAction) -> Box<dyn Fn(u64) -> Box<dyn Fn() + Send + Sync> + Send + Sync> {
let a_clone = a as usize;
if a_clone == IlmAction::NoneAction as usize || a_clone >= IlmAction::ActionCount as usize {
return Box::new(move |_: u64| Box::new(move || {}));
}
let start = SystemTime::now();
Box::new(move |versions: u64| {
Box::new(move || {
let duration = SystemTime::now().duration_since(start).unwrap_or(Duration::from_secs(0));
tokio::spawn(async move {
global_metrics().actions[a_clone].fetch_add(versions, Ordering::Relaxed);
global_metrics().actions_latency[a_clone].add(duration).await;
});
})
})
}
/// Increment time with specific duration
pub async fn inc_time(metric: Metric, duration: Duration) {
let metric = metric as usize;
// Update operation count
global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed);
// Update latency for realtime metrics
if (metric) < Metric::LastRealtime as usize {
global_metrics().latency[metric].add(duration).await;
}
}
/// Get lifetime operation count for a metric
pub fn lifetime(&self, metric: Metric) -> u64 {
let metric = metric as usize;
if (metric) >= Metric::Last as usize {
return 0;
}
self.operations[metric].load(Ordering::Relaxed)
}
/// Get last minute statistics for a metric
pub async fn last_minute(&self, metric: Metric) -> AccElem {
let metric = metric as usize;
if (metric) >= Metric::LastRealtime as usize {
return AccElem::default();
}
self.latency[metric].total().await
}
/// Set current cycle information
pub async fn set_cycle(&self, cycle: Option<CurrentCycle>) {
*self.cycle_info.write().await = cycle;
}
/// Get current cycle information
pub async fn get_cycle(&self) -> Option<CurrentCycle> {
self.cycle_info.read().await.clone()
}
/// Get current active paths
pub async fn get_current_paths(&self) -> Vec<String> {
let mut result = Vec::new();
let paths = self.current_paths.read().await;
for (disk, tracker) in paths.iter() {
let path = tracker.get_path().await;
result.push(format!("{disk}/{path}"));
}
result
}
/// Get number of active drives
pub async fn active_drives(&self) -> usize {
self.current_paths.read().await.len()
}
/// Generate metrics report
pub async fn report(&self) -> M_ScannerMetrics {
let mut metrics = M_ScannerMetrics::default();
// Set cycle information
if let Some(cycle) = self.get_cycle().await {
metrics.current_cycle = cycle.current;
metrics.cycles_completed_at = cycle.cycle_completed;
metrics.current_started = cycle.started;
}
metrics.collected_at = Utc::now();
metrics.active_paths = self.get_current_paths().await;
// Lifetime operations
for i in 0..Metric::Last as usize {
let count = self.operations[i].load(Ordering::Relaxed);
if count > 0
&& let Some(metric) = Metric::from_index(i)
{
metrics.life_time_ops.insert(metric.as_str().to_string(), count);
}
}
// Last minute statistics for realtime metrics
for i in 0..Metric::LastRealtime as usize {
let last_min = self.latency[i].total().await;
if last_min.n > 0
&& let Some(_metric) = Metric::from_index(i)
{
// Convert to madmin TimedAction format if needed
// This would require implementing the conversion
}
}
metrics
}
}
// Type aliases for compatibility with existing code
pub type UpdateCurrentPathFn = Arc<dyn Fn(&str) -> Pin<Box<dyn std::future::Future<Output = ()> + Send>> + Send + Sync>;
pub type CloseDiskFn = Arc<dyn Fn() -> Pin<Box<dyn std::future::Future<Output = ()> + Send>> + Send + Sync>;
/// Create a current path updater for tracking scan progress
pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn, CloseDiskFn) {
let tracker = Arc::new(CurrentPathTracker::new(initial.to_string()));
let disk_name = disk.to_string();
// Store the tracker in global metrics
let tracker_clone = Arc::clone(&tracker);
let disk_clone = disk_name.clone();
tokio::spawn(async move {
global_metrics().current_paths.write().await.insert(disk_clone, tracker_clone);
});
let update_fn = {
let tracker = Arc::clone(&tracker);
Arc::new(move |path: &str| -> Pin<Box<dyn std::future::Future<Output = ()> + Send>> {
let tracker = Arc::clone(&tracker);
let path = path.to_string();
Box::pin(async move {
tracker.update_path(path).await;
})
})
};
let done_fn = {
let disk_name = disk_name.clone();
Arc::new(move || -> Pin<Box<dyn std::future::Future<Output = ()> + Send>> {
let disk_name = disk_name.clone();
Box::pin(async move {
global_metrics().current_paths.write().await.remove(&disk_name);
})
})
};
(update_fn, done_fn)
}
impl Default for Metrics {
fn default() -> Self {
Self::new()
}
}
pub struct CloseDiskGuard(CloseDiskFn);
impl CloseDiskGuard {
pub fn new(close_disk: CloseDiskFn) -> Self {
Self(close_disk)
}
pub async fn close(&self) {
(self.0)().await;
}
}
impl Drop for CloseDiskGuard {
fn drop(&mut self) {
// Drop cannot be async, so we spawn the async cleanup task
// The task will run in the background and complete asynchronously
if let Ok(handle) = tokio::runtime::Handle::try_current() {
let close_fn = self.0.clone();
handle.spawn(async move {
close_fn().await;
});
} else {
// If we're not in a tokio runtime context, we can't spawn
// This is a best-effort cleanup, so we just skip it
}
}
}
crates/scanner/src/scanner.rs
+269
View File
@@ -0,0 +1,269 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use crate::data_usage_define::{BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_OBJ_NAME_PATH};
use crate::metrics::CurrentCycle;
use crate::metrics::global_metrics;
use crate::scanner_folder::data_usage_update_dir_cycles;
use crate::scanner_io::ScannerIO;
use crate::{DataUsageInfo, ScannerError};
use chrono::{DateTime, Utc};
use rustfs_common::heal_channel::HealScanMode;
use rustfs_config::{DEFAULT_DATA_SCANNER_START_DELAY_SECS, ENV_DATA_SCANNER_START_DELAY_SECS};
use rustfs_ecstore::StorageAPI as _;
use rustfs_ecstore::config::com::{read_config, save_config};
use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
use rustfs_ecstore::error::Error as EcstoreError;
use rustfs_ecstore::global::is_erasure_sd;
use rustfs_ecstore::store::ECStore;
use serde::{Deserialize, Serialize};
use tokio::sync::mpsc;
use tokio::time::Duration;
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn};
fn data_scanner_start_delay() -> Duration {
let secs = rustfs_utils::get_env_u64(ENV_DATA_SCANNER_START_DELAY_SECS, DEFAULT_DATA_SCANNER_START_DELAY_SECS);
Duration::from_secs(secs)
}
pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
let ctx_clone = ctx.clone();
let storeapi_clone = storeapi.clone();
tokio::spawn(async move {
let sleep_time = Duration::from_secs(rand::random::<u64>() % 5);
tokio::time::sleep(sleep_time).await;
loop {
if ctx_clone.is_cancelled() {
break;
}
if let Err(e) = run_data_scanner(ctx_clone.clone(), storeapi_clone.clone()).await {
error!("Failed to run data scanner: {e}");
}
tokio::time::sleep(data_scanner_start_delay()).await;
}
});
}
fn get_cycle_scan_mode(_current_cycle: u64, _bitrot_start_cycle: u64, _bitrot_start_time: Option<DateTime<Utc>>) -> HealScanMode {
// TODO: from config
HealScanMode::Normal
}
/// Background healing information
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct BackgroundHealInfo {
/// Bitrot scan start time
pub bitrot_start_time: Option<DateTime<Utc>>,
/// Bitrot scan start cycle
pub bitrot_start_cycle: u64,
/// Current scan mode
pub current_scan_mode: HealScanMode,
}
/// Read background healing information from storage
pub async fn read_background_heal_info(storeapi: Arc<ECStore>) -> BackgroundHealInfo {
// Skip for ErasureSD setup
if is_erasure_sd().await {
return BackgroundHealInfo::default();
}
// Get last healing information
match read_config(storeapi, &BACKGROUND_HEAL_INFO_PATH).await {
Ok(buf) => match serde_json::from_slice::<BackgroundHealInfo>(&buf) {
Ok(info) => info,
Err(e) => {
error!("Failed to unmarshal background heal info from {}: {}", &*BACKGROUND_HEAL_INFO_PATH, e);
BackgroundHealInfo::default()
}
},
Err(e) => {
// Only log if it's not a ConfigNotFound error
if e != EcstoreError::ConfigNotFound {
warn!("Failed to read background heal info from {}: {}", &*BACKGROUND_HEAL_INFO_PATH, e);
}
BackgroundHealInfo::default()
}
}
}
/// Save background healing information to storage
pub async fn save_background_heal_info(storeapi: Arc<ECStore>, info: BackgroundHealInfo) {
// Skip for ErasureSD setup
if is_erasure_sd().await {
return;
}
// Serialize to JSON
let data = match serde_json::to_vec(&info) {
Ok(data) => data,
Err(e) => {
error!("Failed to marshal background heal info: {}", e);
return;
}
};
// Save configuration
if let Err(e) = save_config(storeapi, &BACKGROUND_HEAL_INFO_PATH, data).await {
warn!("Failed to save background heal info to {}: {}", &*BACKGROUND_HEAL_INFO_PATH, e);
}
}
pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) -> Result<(), ScannerError> {
// TODO: leader lock
let _guard = match storeapi.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock").await {
Ok(guard) => guard,
Err(e) => {
error!("run_data_scanner: other node is running, failed to acquire leader lock: {e}");
return Ok(());
}
};
let mut cycle_info = CurrentCycle::default();
let buf = read_config(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH)
.await
.unwrap_or_default();
if buf.len() == 8 {
cycle_info.next = u64::from_le_bytes(buf.try_into().unwrap_or_default());
} else if buf.len() > 8 {
cycle_info.next = u64::from_le_bytes(buf[0..8].try_into().unwrap_or_default());
if let Err(e) = cycle_info.unmarshal(&buf[8..]) {
warn!("Failed to unmarshal cycle info: {e}");
}
}
let mut ticker = tokio::time::interval(data_scanner_start_delay());
loop {
tokio::select! {
_ = ctx.cancelled() => {
break;
}
_ = ticker.tick() => {
cycle_info.current = cycle_info.next;
cycle_info.started = Utc::now();
global_metrics().set_cycle(Some(cycle_info.clone())).await;
let background_heal_info = read_background_heal_info(storeapi.clone()).await;
let scan_mode = get_cycle_scan_mode(cycle_info.current, background_heal_info.bitrot_start_cycle, background_heal_info.bitrot_start_time);
if background_heal_info.current_scan_mode != scan_mode {
let mut new_heal_info = background_heal_info.clone();
new_heal_info.current_scan_mode = scan_mode;
if scan_mode == HealScanMode::Deep {
new_heal_info.bitrot_start_cycle = cycle_info.current;
new_heal_info.bitrot_start_time = Some(Utc::now());
}
save_background_heal_info(storeapi.clone(), new_heal_info).await;
}
let (sender, receiver) = tokio::sync::mpsc::channel::<DataUsageInfo>(1);
let storeapi_clone = storeapi.clone();
let ctx_clone = ctx.clone();
tokio::spawn(async move {
store_data_usage_in_backend(ctx_clone, storeapi_clone, receiver).await;
});
if let Err(e) = storeapi.clone().nsscanner(ctx.clone(), sender, cycle_info.current, scan_mode).await {
error!("Failed to scan namespace: {e}");
} else {
info!("Namespace scanned successfully");
cycle_info.next +=1;
cycle_info.current = 0;
cycle_info.cycle_completed.push(Utc::now());
if cycle_info.cycle_completed.len() >= data_usage_update_dir_cycles() as usize {
cycle_info.cycle_completed = cycle_info.cycle_completed.split_off(data_usage_update_dir_cycles() as usize);
}
global_metrics().set_cycle(Some(cycle_info.clone())).await;
let cycle_info_buf = cycle_info.marshal().unwrap_or_default();
let mut buf = Vec::with_capacity(cycle_info_buf.len() + 8);
buf.extend_from_slice(&cycle_info.next.to_le_bytes());
buf.extend_from_slice(&cycle_info_buf);
if let Err(e) = save_config(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, buf).await {
error!("Failed to save data usage bloom name to {}: {}", &*DATA_USAGE_BLOOM_NAME_PATH, e);
} else {
info!("Data usage bloom name saved successfully");
}
}
ticker.reset();
}
}
}
global_metrics().set_cycle(None).await;
Ok(())
}
/// Store data usage info in backend. Will store all objects sent on the receiver until closed.
pub async fn store_data_usage_in_backend(
ctx: CancellationToken,
storeapi: Arc<ECStore>,
mut receiver: mpsc::Receiver<DataUsageInfo>,
) {
let mut attempts = 1u32;
while let Some(data_usage_info) = receiver.recv().await {
if ctx.is_cancelled() {
break;
}
debug!("store_data_usage_in_backend: received data usage info: {:?}", &data_usage_info);
// Serialize to JSON
let data = match serde_json::to_vec(&data_usage_info) {
Ok(data) => data,
Err(e) => {
error!("Failed to marshal data usage info: {}", e);
continue;
}
};
// Save a backup every 10th update
if attempts > 10 {
let backup_path = format!("{:?}.bkp", &DATA_USAGE_OBJ_NAME_PATH);
if let Err(e) = save_config(storeapi.clone(), &backup_path, data.clone()).await {
warn!("Failed to save data usage backup to {}: {}", backup_path, e);
}
attempts = 1;
}
// Save main configuration
if let Err(e) = save_config(storeapi.clone(), &DATA_USAGE_OBJ_NAME_PATH, data).await {
error!("Failed to save data usage info to {:?}: {e}", &DATA_USAGE_OBJ_NAME_PATH);
}
attempts += 1;
}
}
crates/scanner/src/scanner_folder.rs
+1156
View File
File diff suppressed because it is too large Load Diff
crates/scanner/src/scanner_io.rs
+626
View File
@@ -0,0 +1,626 @@
use crate::scanner_folder::{ScannerItem, scan_data_folder};
use crate::{
DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, DataUsageCache, DataUsageCacheInfo, DataUsageEntry, DataUsageEntryInfo,
DataUsageInfo, SizeSummary, TierStats,
};
use futures::future::join_all;
use rand::seq::SliceRandom as _;
use rustfs_common::heal_channel::HealScanMode;
use rustfs_ecstore::bucket::bucket_target_sys::BucketTargetSys;
use rustfs_ecstore::bucket::lifecycle::lifecycle::Lifecycle;
use rustfs_ecstore::bucket::metadata_sys::{get_lifecycle_config, get_object_lock_config, get_replication_config};
use rustfs_ecstore::bucket::replication::{ReplicationConfig, ReplicationConfigurationExt};
use rustfs_ecstore::bucket::versioning::VersioningApi as _;
use rustfs_ecstore::bucket::versioning_sys::BucketVersioningSys;
use rustfs_ecstore::config::storageclass;
use rustfs_ecstore::disk::STORAGE_FORMAT_FILE;
use rustfs_ecstore::disk::{Disk, DiskAPI};
use rustfs_ecstore::error::{Error, StorageError};
use rustfs_ecstore::global::GLOBAL_TierConfigMgr;
use rustfs_ecstore::new_object_layer_fn;
use rustfs_ecstore::set_disk::SetDisks;
use rustfs_ecstore::store_api::{BucketInfo, BucketOptions, ObjectInfo};
use rustfs_ecstore::{StorageAPI, error::Result, store::ECStore};
use rustfs_filemeta::FileMeta;
use rustfs_utils::path::{SLASH_SEPARATOR_STR, path_join_buf};
use s3s::dto::{BucketLifecycleConfiguration, ReplicationConfiguration};
use std::collections::HashMap;
use std::time::SystemTime;
use std::{fmt::Debug, sync::Arc};
use time::OffsetDateTime;
use tokio::sync::{Mutex, mpsc};
use tokio::time::Duration;
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, warn};
#[async_trait::async_trait]
pub trait ScannerIO: Send + Sync + Debug + 'static {
async fn nsscanner(
&self,
ctx: CancellationToken,
updates: mpsc::Sender<DataUsageInfo>,
want_cycle: u64,
scan_mode: HealScanMode,
) -> Result<()>;
}
#[async_trait::async_trait]
pub trait ScannerIOCache: Send + Sync + Debug + 'static {
async fn nsscanner_cache(
self: Arc<Self>,
ctx: CancellationToken,
buckets: Vec<BucketInfo>,
updates: mpsc::Sender<DataUsageCache>,
want_cycle: u64,
scan_mode: HealScanMode,
) -> Result<()>;
}
#[async_trait::async_trait]
pub trait ScannerIODisk: Send + Sync + Debug + 'static {
async fn nsscanner_disk(
&self,
ctx: CancellationToken,
cache: DataUsageCache,
updates: Option<mpsc::Sender<DataUsageEntry>>,
scan_mode: HealScanMode,
) -> Result<DataUsageCache>;
async fn get_size(&self, item: ScannerItem) -> Result<SizeSummary>;
}
#[async_trait::async_trait]
impl ScannerIO for ECStore {
async fn nsscanner(
&self,
ctx: CancellationToken,
updates: mpsc::Sender<DataUsageInfo>,
want_cycle: u64,
scan_mode: HealScanMode,
) -> Result<()> {
let child_token = ctx.child_token();
let all_buckets = self.list_bucket(&BucketOptions::default()).await?;
if all_buckets.is_empty() {
if let Err(e) = updates.send(DataUsageInfo::default()).await {
error!("Failed to send data usage info: {}", e);
}
return Ok(());
}
let mut total_results = 0;
for pool in self.pools.iter() {
total_results += pool.disk_set.len();
}
let results = vec![DataUsageCache::default(); total_results];
let results_mutex: Arc<Mutex<Vec<DataUsageCache>>> = Arc::new(Mutex::new(results));
let first_err_mutex: Arc<Mutex<Option<Error>>> = Arc::new(Mutex::new(None));
let mut results_index: i32 = -1_i32;
let mut wait_futs = Vec::new();
for pool in self.pools.iter() {
for set in pool.disk_set.iter() {
results_index += 1;
let results_index_clone = results_index as usize;
// Clone the Arc to move it into the spawned task
let set_clone: Arc<SetDisks> = Arc::clone(set);
let child_token_clone = child_token.clone();
let want_cycle_clone = want_cycle;
let scan_mode_clone = scan_mode;
let results_mutex_clone = results_mutex.clone();
let first_err_mutex_clone = first_err_mutex.clone();
let (tx, mut rx) = tokio::sync::mpsc::channel::<DataUsageCache>(1);
// Spawn task to receive and store results
let receiver_fut = tokio::spawn(async move {
while let Some(result) = rx.recv().await {
let mut results = results_mutex_clone.lock().await;
results[results_index_clone] = result;
}
});
wait_futs.push(receiver_fut);
let all_buckets_clone = all_buckets.clone();
// Spawn task to run the scanner
let scanner_fut = tokio::spawn(async move {
if let Err(e) = set_clone
.nsscanner_cache(child_token_clone.clone(), all_buckets_clone, tx, want_cycle_clone, scan_mode_clone)
.await
{
error!("Failed to scan set: {e}");
let _ = first_err_mutex_clone.lock().await.insert(e);
child_token_clone.cancel();
}
});
wait_futs.push(scanner_fut);
}
}
let (update_tx, mut update_rx) = tokio::sync::oneshot::channel::<()>();
let all_buckets_clone = all_buckets.iter().map(|b| b.name.clone()).collect::<Vec<String>>();
tokio::spawn(async move {
let mut last_update = SystemTime::now();
let mut ticker = tokio::time::interval(Duration::from_secs(30));
loop {
tokio::select! {
_ = child_token.cancelled() => {
break;
}
res = &mut update_rx => {
if res.is_err() {
break;
}
let results = results_mutex.lock().await;
let mut all_merged = DataUsageCache::default();
for result in results.iter() {
if result.info.last_update.is_none() {
return;
}
all_merged.merge(result);
}
if all_merged.root().is_some() && all_merged.info.last_update.unwrap() > last_update
&& let Err(e) = updates
.send(all_merged.dui(&all_merged.info.name, &all_buckets_clone))
.await {
error!("Failed to send data usage info: {}", e);
}
break;
}
_ = ticker.tick() => {
let results = results_mutex.lock().await;
let mut all_merged = DataUsageCache::default();
for result in results.iter() {
if result.info.last_update.is_none() {
return;
}
all_merged.merge(result);
}
if all_merged.root().is_some() && all_merged.info.last_update.unwrap() > last_update {
if let Err(e) = updates
.send(all_merged.dui(&all_merged.info.name, &all_buckets_clone))
.await {
error!("Failed to send data usage info: {}", e);
}
last_update = all_merged.info.last_update.unwrap();
}
}
}
}
});
let _ = join_all(wait_futs).await;
let _ = update_tx.send(());
Ok(())
}
}
#[async_trait::async_trait]
impl ScannerIOCache for SetDisks {
async fn nsscanner_cache(
self: Arc<Self>,
ctx: CancellationToken,
buckets: Vec<BucketInfo>,
updates: mpsc::Sender<DataUsageCache>,
want_cycle: u64,
scan_mode: HealScanMode,
) -> Result<()> {
if buckets.is_empty() {
return Ok(());
}
let (disks, healing) = self.get_online_disks_with_healing(false).await;
if disks.is_empty() {
debug!("nsscanner_cache: no online disks available for set");
return Ok(());
}
let mut old_cache = DataUsageCache::default();
old_cache.load(self.clone(), DATA_USAGE_CACHE_NAME).await?;
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
next_cycle: old_cache.info.next_cycle,
..Default::default()
},
cache: HashMap::new(),
};
let (bucket_tx, bucket_rx) = mpsc::channel::<BucketInfo>(buckets.len());
let mut permutes = buckets.clone();
permutes.shuffle(&mut rand::rng());
for bucket in permutes.iter() {
if old_cache.find(&bucket.name).is_none()
&& let Err(e) = bucket_tx.send(bucket.clone()).await
{
error!("Failed to send bucket info: {}", e);
}
}
for bucket in permutes.iter() {
if let Some(c) = old_cache.find(&bucket.name) {
cache.replace(&bucket.name, DATA_USAGE_ROOT, c.clone());
if let Err(e) = bucket_tx.send(bucket.clone()).await {
error!("Failed to send bucket info: {}", e);
}
}
}
drop(bucket_tx);
let cache_mutex: Arc<Mutex<DataUsageCache>> = Arc::new(Mutex::new(cache));
let (bucket_result_tx, mut bucket_result_rx) = mpsc::channel::<DataUsageEntryInfo>(disks.len());
let cache_mutex_clone = cache_mutex.clone();
let store_clone = self.clone();
let ctx_clone = ctx.clone();
let send_update_fut = tokio::spawn(async move {
let mut ticker = tokio::time::interval(Duration::from_secs(3 + rand::random::<u64>() % 10));
let mut last_update = None;
loop {
tokio::select! {
_ = ctx_clone.cancelled() => {
break;
}
_ = ticker.tick() => {
let cache = cache_mutex_clone.lock().await;
if cache.info.last_update == last_update {
continue;
}
if let Err(e) = cache.save(store_clone.clone(), DATA_USAGE_CACHE_NAME).await {
error!("Failed to save data usage cache: {}", e);
}
if let Err(e) = updates.send(cache.clone()).await {
error!("Failed to send data usage cache: {}", e);
}
last_update = cache.info.last_update;
}
res = bucket_result_rx.recv() => {
if let Some(result) = res {
let mut cache = cache_mutex_clone.lock().await;
cache.replace(&result.name, &result.parent, result.entry);
cache.info.last_update = Some(SystemTime::now());
} else {
let mut cache = cache_mutex_clone.lock().await;
cache.info.next_cycle =want_cycle;
cache.info.last_update = Some(SystemTime::now());
if let Err(e) = cache.save(store_clone.clone(), DATA_USAGE_CACHE_NAME).await {
error!("Failed to save data usage cache: {}", e);
}
if let Err(e) = updates.send(cache.clone()).await {
error!("Failed to send data usage cache: {}", e);
}
return;
}
}
}
}
});
let mut futs = Vec::new();
let bucket_rx_mutex: Arc<Mutex<mpsc::Receiver<BucketInfo>>> = Arc::new(Mutex::new(bucket_rx));
let bucket_result_tx_clone: Arc<Mutex<mpsc::Sender<DataUsageEntryInfo>>> = Arc::new(Mutex::new(bucket_result_tx));
for disk in disks.into_iter() {
let bucket_rx_mutex_clone = bucket_rx_mutex.clone();
let ctx_clone = ctx.clone();
let store_clone_clone = self.clone();
let bucket_result_tx_clone_clone = bucket_result_tx_clone.clone();
let disk_clone = disk.clone();
futs.push(tokio::spawn(async move {
while let Some(bucket) = bucket_rx_mutex_clone.lock().await.recv().await {
if ctx_clone.is_cancelled() {
break;
}
debug!("nsscanner_disk: got bucket: {}", bucket.name);
let cache_name = path_join_buf(&[&bucket.name, DATA_USAGE_CACHE_NAME]);
let mut cache = DataUsageCache::default();
if let Err(e) = cache.load(store_clone_clone.clone(), &cache_name).await {
error!("Failed to load data usage cache: {}", e);
}
if cache.info.name.is_empty() {
cache.info.name = bucket.name.clone();
}
cache.info.skip_healing = healing;
cache.info.next_cycle = want_cycle;
if cache.info.name != bucket.name {
cache.info = DataUsageCacheInfo {
name: bucket.name.clone(),
next_cycle: want_cycle,
..Default::default()
};
}
debug!("nsscanner_disk: cache.info.name: {:?}", cache.info.name);
let (updates_tx, mut updates_rx) = mpsc::channel::<DataUsageEntry>(1);
let ctx_clone_clone = ctx_clone.clone();
let bucket_name_clone = bucket.name.clone();
let bucket_result_tx_clone_clone_clone = bucket_result_tx_clone_clone.clone();
let update_fut = tokio::spawn(async move {
while let Some(result) = updates_rx.recv().await {
if ctx_clone_clone.is_cancelled() {
break;
}
if let Err(e) = bucket_result_tx_clone_clone_clone
.lock()
.await
.send(DataUsageEntryInfo {
name: bucket_name_clone.clone(),
parent: DATA_USAGE_ROOT.to_string(),
entry: result,
})
.await
{
error!("Failed to send data usage entry info: {}", e);
}
}
});
let before = cache.info.last_update;
cache = match disk_clone
.nsscanner_disk(ctx_clone.clone(), cache.clone(), Some(updates_tx), scan_mode)
.await
{
Ok(cache) => cache,
Err(e) => {
error!("Failed to scan disk: {}", e);
if let (Some(last_update), Some(before_update)) = (cache.info.last_update, before)
&& last_update > before_update
&& let Err(e) = cache.save(store_clone_clone.clone(), cache_name.as_str()).await
{
error!("Failed to save data usage cache: {}", e);
}
if let Err(e) = update_fut.await {
error!("Failed to update data usage cache: {}", e);
}
continue;
}
};
debug!("nsscanner_disk: got cache: {}", cache.info.name);
if let Err(e) = update_fut.await {
error!("nsscanner_disk: Failed to update data usage cache: {}", e);
}
let root = if let Some(r) = cache.root() {
cache.flatten(&r)
} else {
DataUsageEntry::default()
};
if ctx_clone.is_cancelled() {
break;
}
debug!("nsscanner_disk: sending data usage entry info: {}", cache.info.name);
if let Err(e) = bucket_result_tx_clone_clone
.lock()
.await
.send(DataUsageEntryInfo {
name: cache.info.name.clone(),
parent: DATA_USAGE_ROOT.to_string(),
entry: root,
})
.await
{
error!("nsscanner_disk: Failed to send data usage entry info: {}", e);
}
if let Err(e) = cache.save(store_clone_clone.clone(), &cache_name).await {
error!("nsscanner_disk: Failed to save data usage cache: {}", e);
}
}
}));
}
let _ = join_all(futs).await;
drop(bucket_result_tx_clone);
send_update_fut.await?;
debug!("nsscanner_cache: done");
Ok(())
}
}
#[async_trait::async_trait]
impl ScannerIODisk for Disk {
async fn get_size(&self, mut item: ScannerItem) -> Result<SizeSummary> {
if !item.path.ends_with(&format!("{SLASH_SEPARATOR_STR}{STORAGE_FORMAT_FILE}")) {
return Err(StorageError::other("skip file".to_string()));
}
let data = match self.read_metadata(&item.bucket, &item.object_path()).await {
Ok(data) => data,
Err(e) => {
warn!(
"Failed to read metadata: {e}, bucket={}, object_path={}",
&item.bucket,
&item.object_path()
);
return Err(StorageError::other("skip file".to_string()));
}
};
item.transform_meta_dir();
let meta = FileMeta::load(&data)?;
let fivs = match meta.get_file_info_versions(item.bucket.as_str(), item.object_path().as_str(), false) {
Ok(versions) => versions,
Err(e) => {
error!("Failed to get file info versions: {}", e);
return Err(StorageError::other("skip file".to_string()));
}
};
let versioned = BucketVersioningSys::get(&item.bucket)
.await
.map(|v| v.versioned(&item.object_path()))
.unwrap_or(false);
let object_infos = fivs
.versions
.iter()
.map(|v| ObjectInfo::from_file_info(v, item.bucket.as_str(), item.object_path().as_str(), versioned))
.collect::<Vec<ObjectInfo>>();
let mut size_summary = SizeSummary::default();
let tiers = {
let tier_config_mgr = GLOBAL_TierConfigMgr.read().await;
tier_config_mgr.list_tiers()
};
for tier in tiers.iter() {
size_summary.tier_stats.insert(tier.name.clone(), TierStats::default());
}
if !size_summary.tier_stats.is_empty() {
size_summary
.tier_stats
.insert(storageclass::STANDARD.to_string(), TierStats::default());
size_summary
.tier_stats
.insert(storageclass::RRS.to_string(), TierStats::default());
}
let lock_config = match get_object_lock_config(&item.bucket).await {
Ok((cfg, _)) => Some(Arc::new(cfg)),
Err(_) => None,
};
let Some(ecstore) = new_object_layer_fn() else {
error!("ECStore not available");
return Err(StorageError::other("ECStore not available".to_string()));
};
item.apply_actions(ecstore, object_infos, lock_config, &mut size_summary)
.await;
// TODO: enqueueFreeVersion
Ok(size_summary)
}
async fn nsscanner_disk(
&self,
ctx: CancellationToken,
cache: DataUsageCache,
updates: Option<mpsc::Sender<DataUsageEntry>>,
scan_mode: HealScanMode,
) -> Result<DataUsageCache> {
let _guard = self.start_scan();
let mut cache = cache;
let (lifecycle_config, _) = get_lifecycle_config(&cache.info.name)
.await
.unwrap_or((BucketLifecycleConfiguration::default(), OffsetDateTime::now_utc()));
if lifecycle_config.has_active_rules("") {
cache.info.lifecycle = Some(Arc::new(lifecycle_config));
}
let (replication_config, _) = get_replication_config(&cache.info.name).await.unwrap_or((
ReplicationConfiguration {
role: "".to_string(),
rules: vec![],
},
OffsetDateTime::now_utc(),
));
if replication_config.has_active_rules("", true)
&& let Ok(targets) = BucketTargetSys::get().list_bucket_targets(&cache.info.name).await
{
cache.info.replication = Some(Arc::new(ReplicationConfig {
config: Some(replication_config),
remotes: Some(targets),
}));
}
// TODO: object lock
let Some(ecstore) = new_object_layer_fn() else {
error!("ECStore not available");
return Err(StorageError::other("ECStore not available".to_string()));
};
let disk_location = self.get_disk_location();
let (Some(pool_idx), Some(set_idx)) = (disk_location.pool_idx, disk_location.set_idx) else {
error!("Disk location not available");
return Err(StorageError::other("Disk location not available".to_string()));
};
let disks_result = ecstore.get_disks(pool_idx, set_idx).await?;
let Some(disk_idx) = disk_location.disk_idx else {
error!("Disk index not available");
return Err(StorageError::other("Disk index not available".to_string()));
};
let local_disk = if let Some(Some(local_disk)) = disks_result.get(disk_idx) {
local_disk.clone()
} else {
error!("Local disk not available");
return Err(StorageError::other("Local disk not available".to_string()));
};
let disks = disks_result.into_iter().flatten().collect::<Vec<Arc<Disk>>>();
// Create we_sleep function (always return false for now, can be enhanced later)
let we_sleep: Box<dyn Fn() -> bool + Send + Sync> = Box::new(|| false);
let result = scan_data_folder(ctx, disks, local_disk, cache, updates, scan_mode, we_sleep).await;
match result {
Ok(mut data_usage_info) => {
data_usage_info.info.last_update = Some(SystemTime::now());
Ok(data_usage_info)
}
Err(e) => Err(StorageError::other(format!("Failed to scan data folder: {e}"))),
}
}
}