gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-24 13:16:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

merge heal

Browse Source
This commit is contained in:
weisd
2024-11-20 17:40:11 +08:00
parent 706fce49b9 1d81a9e40f
commit 821bb6fe3f
64 changed files with 5332 additions and 771 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ecstore/src/heal/background_heal_ops.rs
+391 -34
View File
@@ -1,76 +1,424 @@
use std::sync::Arc;
use s3s::{S3Error, S3ErrorCode};
use std::{cmp::Ordering, env, path::PathBuf, sync::Arc, time::Duration};
use tokio::{
select,
sync::{
broadcast::Receiver as B_Receiver,
mpsc::{self, Receiver, Sender},
RwLock,
},
time::interval,
};
use crate::{error::Error, heal::heal_ops::NOP_HEAL, utils::path::SLASH_SEPARATOR};
use tracing::{error, info};
use uuid::Uuid;
use super::{
heal_commands::{HealOpts, HealResultItem},
heal_ops::HealSequence,
heal_ops::{new_bg_heal_sequence, HealSequence},
};
use crate::heal::error::ERR_RETRY_HEALING;
use crate::{
config::RUSTFS_CONFIG_PREFIX,
disk::{endpoint::Endpoint, error::DiskError, DiskAPI, DiskInfoOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
error::{Error, Result},
global::{GLOBAL_BackgroundHealRoutine, GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP},
heal::{
data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT},
data_usage_cache::DataUsageCache,
heal_commands::{init_healing_tracker, load_healing_tracker},
heal_ops::NOP_HEAL,
},
new_object_layer_fn,
store::get_disk_via_endpoint,
store_api::{BucketInfo, BucketOptions, StorageAPI},
utils::path::{path_join, SLASH_SEPARATOR},
};
#[derive(Clone, Debug)]
pub static DEFAULT_MONITOR_NEW_DISK_INTERVAL: Duration = Duration::from_secs(10);
pub async fn init_auto_heal() {
init_background_healing().await;
if let Ok(v) = env::var("_RUSTFS_AUTO_DRIVE_HEALING") {
if v == "on" {
GLOBAL_BackgroundHealState
.write()
.await
.push_heal_local_disks(&get_local_disks_to_heal().await)
.await;
tokio::spawn(async {
monitor_local_disks_and_heal().await;
});
}
}
}
async fn init_background_healing() {
let bg_seq = Arc::new(RwLock::new(new_bg_heal_sequence()));
for _ in 0..GLOBAL_BackgroundHealRoutine.read().await.workers {
let bg_seq_clone = bg_seq.clone();
tokio::spawn(async {
GLOBAL_BackgroundHealRoutine.write().await.add_worker(bg_seq_clone).await;
});
}
let _ = GLOBAL_BackgroundHealState
.write()
.await
.launch_new_heal_sequence(bg_seq)
.await;
}
async fn get_local_disks_to_heal() -> Vec<Endpoint> {
let mut disks_to_heal = Vec::new();
for (_, disk) in GLOBAL_LOCAL_DISK_MAP.read().await.iter() {
if let Some(disk) = disk {
if let Err(err) = disk.disk_info(&DiskInfoOptions::default()).await {
if let Some(DiskError::UnformattedDisk) = err.downcast_ref() {
disks_to_heal.push(disk.endpoint());
}
}
let h = disk.healing().await;
if let Some(h) = h {
if !h.finished {
disks_to_heal.push(disk.endpoint());
}
}
}
}
// todo
// if disks_to_heal.len() == GLOBAL_Endpoints.read().await.n {
// }
disks_to_heal
}
async fn monitor_local_disks_and_heal() {
let mut interval = interval(DEFAULT_MONITOR_NEW_DISK_INTERVAL);
loop {
interval.tick().await;
let heal_disks = GLOBAL_BackgroundHealState.read().await.get_heal_local_disk_endpoints().await;
if heal_disks.is_empty() {
interval.reset();
continue;
}
let layer = new_object_layer_fn();
let lock = layer.read().await;
let store = lock.as_ref().expect("errServerNotInitialized");
if let (_, Some(err)) = store.heal_format(false).await.expect("heal format failed") {
if let Some(DiskError::NoHealRequired) = err.downcast_ref() {
} else {
info!("heal format err: {}", err.to_string());
interval.reset();
continue;
}
}
for disk in heal_disks.into_ref().iter() {
let disk_clone = disk.clone();
tokio::spawn(async move {
GLOBAL_BackgroundHealState
.write()
.await
.set_disk_healing_status(disk_clone.clone(), true)
.await;
if heal_fresh_disk(&disk_clone).await.is_err() {
GLOBAL_BackgroundHealState
.write()
.await
.set_disk_healing_status(disk_clone.clone(), false)
.await;
return;
}
GLOBAL_BackgroundHealState
.write()
.await
.pop_heal_local_disks(&[disk_clone])
.await;
});
}
interval.reset();
}
}
async fn heal_fresh_disk(endpoint: &Endpoint) -> Result<()> {
let (pool_idx, set_idx) = (endpoint.pool_idx as usize, endpoint.disk_idx as usize);
let disk = match get_disk_via_endpoint(endpoint).await {
Some(disk) => disk,
None => {
return Err(Error::from_string(format!(
"Unexpected error disk must be initialized by now after formatting: {}",
endpoint
)))
}
};
if let Err(err) = disk.disk_info(&DiskInfoOptions::default()).await {
match err.downcast_ref() {
Some(DiskError::DriveIsRoot) => {
return Ok(());
}
Some(DiskError::UnformattedDisk) => {}
_ => {
return Err(err);
}
}
}
let mut tracker = match load_healing_tracker(&Some(disk.clone())).await {
Ok(tracker) => tracker,
Err(err) => {
match err.downcast_ref() {
Some(DiskError::FileNotFound) => {
return Ok(());
}
_ => {
info!(
"Unable to load healing tracker on '{}': {}, re-initializing..",
disk.to_string(),
err.to_string()
);
}
}
init_healing_tracker(disk.clone(), &Uuid::new_v4().to_string()).await?
}
};
info!(
"Healing drive '{}' - 'mc admin heal alias/ --verbose' to check the current status.",
endpoint.to_string()
);
let layer = new_object_layer_fn();
let lock = layer.read().await;
let store = match lock.as_ref() {
Some(s) => s,
None => return Err(Error::msg("errServerNotInitialized")),
};
let mut buckets = store.list_bucket(&BucketOptions::default()).await?;
buckets.push(BucketInfo {
name: path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(RUSTFS_CONFIG_PREFIX)])
.to_string_lossy()
.to_string(),
..Default::default()
});
buckets.push(BucketInfo {
name: path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(BUCKET_META_PREFIX)])
.to_string_lossy()
.to_string(),
..Default::default()
});
buckets.sort_by(|a, b| {
let a_has_prefix = a.name.starts_with(RUSTFS_META_BUCKET);
let b_has_prefix = b.name.starts_with(RUSTFS_META_BUCKET);
match (a_has_prefix, b_has_prefix) {
(true, false) => Ordering::Less,
(false, true) => Ordering::Greater,
_ => b.created.cmp(&a.created),
}
});
if let Ok(cache) = DataUsageCache::load(&store.pools[pool_idx].disk_set[set_idx], DATA_USAGE_CACHE_NAME).await {
let data_usage_info = cache.dui(DATA_USAGE_ROOT, &Vec::new());
tracker.objects_total_count = data_usage_info.objects_total_count;
tracker.objects_total_size = data_usage_info.objects_total_size;
};
tracker.set_queue_buckets(&buckets).await;
tracker.save().await?;
let tracker = Arc::new(RwLock::new(tracker));
let qb = tracker.read().await.queue_buckets.clone();
store.pools[pool_idx].disk_set[set_idx]
.clone()
.heal_erasure_set(&qb, tracker.clone())
.await?;
let mut tracker_w = tracker.write().await;
if tracker_w.items_failed > 0 && tracker_w.retry_attempts < 4 {
tracker_w.retry_attempts += 1;
tracker_w.reset_healing().await;
if let Err(err) = tracker_w.update().await {
info!("update tracker failed: {}", err.to_string());
}
return Err(Error::from_string(ERR_RETRY_HEALING));
}
if tracker_w.items_failed > 0 {
info!(
"Healing of drive '{}' is incomplete, retried {} times (healed: {}, skipped: {}, failed: {}).",
disk.to_string(),
tracker_w.retry_attempts,
tracker_w.items_healed,
tracker_w.item_skipped,
tracker_w.items_failed
);
} else if tracker_w.retry_attempts > 0 {
info!(
"Healing of drive '{}' is incomplete, retried {} times (healed: {}, skipped: {}).",
disk.to_string(),
tracker_w.retry_attempts,
tracker_w.items_healed,
tracker_w.item_skipped
);
} else {
info!(
"Healing of drive '{}' is finished (healed: {}, skipped: {}).",
disk.to_string(),
tracker_w.items_healed,
tracker_w.item_skipped
);
}
if tracker_w.heal_id.is_empty() {
if let Err(err) = tracker_w.delete().await {
error!("delete tracker failed: {}", err.to_string());
}
}
let layer = new_object_layer_fn();
let lock = layer.read().await;
let store = match lock.as_ref() {
Some(s) => s,
None => return Err(Error::from(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))),
};
let disks = store.get_disks(pool_idx, set_idx).await?;
for disk in disks.into_iter() {
if disk.is_none() {
continue;
}
let mut tracker = match load_healing_tracker(&disk).await {
Ok(tracker) => tracker,
Err(err) => {
match err.downcast_ref() {
Some(DiskError::FileNotFound) => {}
_ => {
info!("Unable to load healing tracker on '{:?}': {}, re-initializing..", disk, err.to_string());
}
}
continue;
}
};
if tracker.heal_id == tracker_w.heal_id {
tracker.finished = true;
tracker.update().await?;
}
}
Ok(())
}
#[derive(Debug)]
pub struct HealTask {
pub bucket: String,
pub object: String,
pub version_id: String,
pub opts: HealOpts,
pub resp_tx: Arc<Sender<HealResult>>,
pub resp_rx: Arc<Receiver<HealResult>>,
pub resp_tx: Option<Sender<HealResult>>,
pub resp_rx: Option<Receiver<HealResult>>,
}
impl HealTask {
pub fn new(bucket: &str, object: &str, version_id: &str, opts: &HealOpts) -> Self {
let (tx, rx) = mpsc::channel(10);
Self {
bucket: bucket.to_string(),
object: object.to_string(),
version_id: version_id.to_string(),
opts: opts.clone(),
resp_tx: tx.into(),
resp_rx: rx.into(),
opts: *opts,
resp_tx: None,
resp_rx: None,
}
}
}
pub struct HealResult {
pub result: HealResultItem,
err: Error,
pub err: Option<Error>,
}
pub struct HealRoutine {
tasks_tx: Sender<HealTask>,
pub tasks_tx: Sender<HealTask>,
tasks_rx: Receiver<HealTask>,
workers: usize,
}
impl HealRoutine {
pub async fn add_worker(&mut self, mut ctx: B_Receiver<bool>, bgseq: &HealSequence) {
pub fn new() -> Arc<RwLock<Self>> {
let mut workers = num_cpus::get() / 2;
if let Ok(env_heal_workers) = env::var("_RUSTFS_HEAL_WORKERS") {
if let Ok(num_healers) = env_heal_workers.parse::<usize>() {
workers = num_healers;
}
}
if workers == 0 {
workers = 4;
}
let (tx, rx) = mpsc::channel(100);
Arc::new(RwLock::new(Self {
tasks_tx: tx,
tasks_rx: rx,
workers,
}))
}
pub async fn add_worker(&mut self, bgseq: Arc<RwLock<HealSequence>>) {
loop {
select! {
task = self.tasks_rx.recv() => {
let mut res = HealResultItem::default();
let mut err: Error;
match task {
Some(task) => {
if task.bucket == NOP_HEAL {
err = Error::from_string("skip file");
} else if task.bucket == SLASH_SEPARATOR {
(res, err) = heal_disk_format(task.opts).await;
let mut d_res = HealResultItem::default();
let d_err: Option<Error>;
match self.tasks_rx.recv().await {
Some(task) => {
if task.bucket == NOP_HEAL {
d_err = Some(Error::from_string("skip file"));
} else if task.bucket == SLASH_SEPARATOR {
match heal_disk_format(task.opts).await {
Ok((res, err)) => {
d_res = res;
d_err = err;
}
},
None => return,
Err(err) => d_err = Some(err),
}
} else {
let layer = new_object_layer_fn();
let lock = layer.read().await;
let store = lock.as_ref().expect("Not init");
if task.object.is_empty() {
match store.heal_bucket(&task.bucket, &task.opts).await {
Ok(res) => {
d_res = res;
d_err = None;
}
Err(err) => d_err = Some(err),
}
} else {
match store
.heal_object(&task.bucket, &task.object, &task.version_id, &task.opts)
.await
{
Ok((res, err)) => {
d_res = res;
d_err = err;
}
Err(err) => d_err = Some(err),
}
}
}
if let Some(resp_tx) = task.resp_tx {
let _ = resp_tx
.send(HealResult {
result: d_res,
err: d_err,
})
.await;
} else {
// when respCh is not set caller is not waiting but we
// update the relevant metrics for them
if d_err.is_none() {
bgseq.write().await.count_healed(d_res.heal_item_type);
} else {
bgseq.write().await.count_failed(d_res.heal_item_type);
}
}
}
_ = ctx.recv() => {
return;
}
None => return,
}
}
}
@@ -80,6 +428,15 @@ impl HealRoutine {
// }
async fn heal_disk_format(opts: HealOpts) -> (HealResultItem, Error) {
todo!()
async fn heal_disk_format(opts: HealOpts) -> Result<(HealResultItem, Option<Error>)> {
let layer = new_object_layer_fn();
let lock = layer.read().await;
let store = lock.as_ref().expect("Not init");
let (res, err) = store.heal_format(opts.dry_run).await?;
// return any error, ignore error returned when disks have
// already healed.
if err.is_some() {
return Ok((HealResultItem::default(), err));
}
Ok((res, err))
}
ecstore/src/heal/data_scanner.rs
+1086
View File
File diff suppressed because it is too large Load Diff
ecstore/src/heal/data_scanner_metric.rs
+223
View File
@@ -0,0 +1,223 @@
use common::last_minute::{AccElem, LastMinuteLatency};
use lazy_static::lazy_static;
use std::future::Future;
use std::pin::Pin;
use std::sync::atomic::AtomicU64;
use std::sync::Once;
use std::time::{Duration, UNIX_EPOCH};
use std::{
collections::HashMap,
sync::{
atomic::{AtomicU32, Ordering},
Arc,
},
time::SystemTime,
};
use tokio::sync::RwLock;
use super::data_scanner::{CurrentScannerCycle, UpdateCurrentPathFn};
lazy_static! {
pub static ref globalScannerMetrics: Arc<RwLock<ScannerMetrics>> = Arc::new(RwLock::new(ScannerMetrics::new()));
}
#[derive(Clone, Debug, PartialEq, PartialOrd)]
pub enum ScannerMetric {
// START Realtime metrics, that only to records
// last minute latencies and total operation count.
ReadMetadata = 0,
CheckMissing,
SaveUsage,
ApplyAll,
ApplyVersion,
TierObjSweep,
HealCheck,
Ilm,
CheckReplication,
Yield,
CleanAbandoned,
ApplyNonCurrent,
HealAbandonedVersion,
// START Trace metrics:
StartTrace,
ScanObject, // Scan object. All operations included.
HealAbandonedObject,
// END realtime metrics:
LastRealtime,
// Trace only metrics:
ScanFolder, // Scan a folder on disk, recursively.
ScanCycle, // Full cycle, cluster global.
ScanBucketDrive, // Single bucket on one drive.
CompactFolder, // Folder compacted.
// Must be last:
Last,
}
static INIT: Once = Once::new();
#[derive(Default)]
pub struct LockedLastMinuteLatency {
cached_sec: AtomicU64,
cached: AccElem,
mu: RwLock<bool>,
latency: LastMinuteLatency,
}
impl Clone for LockedLastMinuteLatency {
fn clone(&self) -> Self {
Self {
cached_sec: AtomicU64::new(0),
cached: self.cached.clone(),
mu: RwLock::new(true),
latency: self.latency.clone(),
}
}
}
impl LockedLastMinuteLatency {
pub async fn add(&mut self, value: &Duration) {
self.add_size(value, 0).await;
}
pub async fn add_size(&mut self, value: &Duration, sz: u64) {
let t = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("Time went backwards")
.as_secs();
INIT.call_once(|| {
self.cached = AccElem::default();
self.cached_sec.store(t, Ordering::SeqCst);
});
let last_t = self.cached_sec.load(Ordering::SeqCst);
if last_t != t
&& self
.cached_sec
.compare_exchange(last_t, t, Ordering::SeqCst, Ordering::SeqCst)
.is_ok()
{
let old = self.cached.clone();
self.cached = AccElem::default();
let a = AccElem {
size: old.size,
total: old.total,
n: old.n,
};
let _ = self.mu.write().await;
self.latency.add_all(t - 1, &a);
}
self.cached.n += 1;
self.cached.total += value.as_secs();
self.cached.size += sz;
}
pub async fn total(&mut self) -> AccElem {
let _ = self.mu.read().await;
self.latency.get_total()
}
}
pub type LogFn = Arc<dyn Fn(&HashMap<String, String>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
pub type TimeSizeFn = Arc<dyn Fn(u64) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
pub struct ScannerMetrics {
operations: Vec<AtomicU32>,
latency: Vec<LockedLastMinuteLatency>,
cycle_info: RwLock<Option<CurrentScannerCycle>>,
current_paths: HashMap<String, String>,
}
impl Default for ScannerMetrics {
fn default() -> Self {
Self::new()
}
}
impl ScannerMetrics {
pub fn new() -> Self {
Self {
operations: (0..ScannerMetric::Last as usize).map(|_| AtomicU32::new(0)).collect(),
latency: vec![LockedLastMinuteLatency::default(); ScannerMetric::LastRealtime as usize],
cycle_info: RwLock::new(None),
current_paths: HashMap::new(),
}
}
pub async fn set_cycle(&mut self, c: Option<CurrentScannerCycle>) {
*self.cycle_info.write().await = c;
}
pub fn log(s: ScannerMetric) -> LogFn {
let start = SystemTime::now();
let s_clone = s as usize;
Arc::new(move |_custom: &HashMap<String, String>| {
Box::pin(async move {
let duration = SystemTime::now().duration_since(start).unwrap_or(Duration::from_secs(0));
let mut sm_w = globalScannerMetrics.write().await;
sm_w.operations[s_clone].fetch_add(1, Ordering::SeqCst);
if s_clone < ScannerMetric::LastRealtime as usize {
sm_w.latency[s_clone].add(&duration).await;
}
})
})
}
pub async fn time_size(s: ScannerMetric) -> TimeSizeFn {
let start = SystemTime::now();
let s_clone = s as usize;
Arc::new(move |sz: u64| {
Box::pin(async move {
let duration = SystemTime::now().duration_since(start).unwrap_or(Duration::from_secs(0));
let mut sm_w = globalScannerMetrics.write().await;
sm_w.operations[s_clone].fetch_add(1, Ordering::SeqCst);
if s_clone < ScannerMetric::LastRealtime as usize {
sm_w.latency[s_clone].add_size(&duration, sz).await;
}
})
})
}
pub fn time(s: ScannerMetric) -> TimeFn {
let start = SystemTime::now();
let s_clone = s as usize;
Arc::new(move || {
Box::pin(async move {
let duration = SystemTime::now().duration_since(start).unwrap_or(Duration::from_secs(0));
let mut sm_w = globalScannerMetrics.write().await;
sm_w.operations[s_clone].fetch_add(1, Ordering::SeqCst);
if s_clone < ScannerMetric::LastRealtime as usize {
sm_w.latency[s_clone].add(&duration).await;
}
})
})
}
}
pub type CloseDiskFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
pub fn current_path_updater(disk: &str, _initial: &str) -> (UpdateCurrentPathFn, CloseDiskFn) {
let disk_1 = disk.to_string();
let disk_2 = disk.to_string();
(
Arc::new(move |path: &str| {
let disk_inner = disk_1.clone();
let path = path.to_string();
Box::pin(async move {
globalScannerMetrics
.write()
.await
.current_paths
.insert(disk_inner, path.to_string());
})
}),
Arc::new(move || {
let disk_inner = disk_2.clone();
Box::pin(async move {
globalScannerMetrics.write().await.current_paths.remove(&disk_inner);
})
}),
)
}
ecstore/src/heal/data_usage.rs
+139
View File
@@ -0,0 +1,139 @@
use std::{collections::HashMap, time::SystemTime};
use lazy_static::lazy_static;
use serde::{Deserialize, Serialize};
use tokio::sync::mpsc::Receiver;
use tracing::info;
use crate::{
config::common::save_config,
disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
new_object_layer_fn,
utils::path::SLASH_SEPARATOR,
};
pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;
const DATA_USAGE_OBJ_NAME: &str = ".usage.json";
const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin";
pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
lazy_static! {
pub static ref DATA_USAGE_BUCKET: String = format!("{}{}{}", RUSTFS_META_BUCKET, SLASH_SEPARATOR, BUCKET_META_PREFIX);
pub static ref DATA_USAGE_OBJ_NAME_PATH: String = format!("{}{}{}", BUCKET_META_PREFIX, SLASH_SEPARATOR, DATA_USAGE_OBJ_NAME);
pub static ref DATA_USAGE_BLOOM_NAME_PATH: String =
format!("{}{}{}", BUCKET_META_PREFIX, SLASH_SEPARATOR, DATA_USAGE_BLOOM_NAME);
pub static ref BACKGROUND_HEAL_INFO_PATH: String =
format!("{}{}{}", BUCKET_META_PREFIX, SLASH_SEPARATOR, ".background-heal.json");
}
// BucketTargetUsageInfo - bucket target usage info provides
// - replicated size for all objects sent to this target
// - replica size for all objects received from this target
// - replication pending size for all objects pending replication to this target
// - replication failed size for all objects failed replication to this target
// - replica pending count
// - replica failed count
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct BucketTargetUsageInfo {
pub replication_pending_size: u64,
pub replication_failed_size: u64,
pub replicated_size: u64,
pub replica_size: u64,
pub replication_pending_count: u64,
pub replication_failed_count: u64,
pub replicated_count: u64,
}
// BucketUsageInfo - bucket usage info provides
// - total size of the bucket
// - total objects in a bucket
// - object size histogram per bucket
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct BucketUsageInfo {
pub size: u64,
// Following five fields suffixed with V1 are here for backward compatibility
// Total Size for objects that have not yet been replicated
pub replication_pending_size_v1: u64,
// Total size for objects that have witness one or more failures and will be retried
pub replication_failed_size_v1: u64,
// Total size for objects that have been replicated to destination
pub replicated_size_v1: u64,
// Total number of objects pending replication
pub replication_pending_count_v1: u64,
// Total number of objects that failed replication
pub replication_failed_count_v1: u64,
pub objects_count: u64,
pub object_size_histogram: HashMap<String, u64>,
pub object_versions_histogram: HashMap<String, u64>,
pub versions_count: u64,
pub delete_markers_count: u64,
pub replica_size: u64,
pub replica_count: u64,
pub replication_info: HashMap<String, BucketTargetUsageInfo>,
}
// DataUsageInfo represents data usage stats of the underlying Object API
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct DataUsageInfo {
pub total_capacity: u64,
pub total_used_capacity: u64,
pub total_free_capacity: u64,
// LastUpdate is the timestamp of when the data usage info was last updated.
// This does not indicate a full scan.
pub last_update: Option<SystemTime>,
// Objects total count across all buckets
pub objects_total_count: u64,
// Versions total count across all buckets
pub versions_total_count: u64,
// Delete markers total count across all buckets
pub delete_markers_total_count: u64,
// Objects total size across all buckets
pub objects_total_size: u64,
pub replication_info: HashMap<String, BucketTargetUsageInfo>,
// Total number of buckets in this cluster
pub buckets_count: u64,
// Buckets usage info provides following information across all buckets
// - total size of the bucket
// - total objects in a bucket
// - object size histogram per bucket
pub buckets_usage: HashMap<String, BucketUsageInfo>,
// Deprecated kept here for backward compatibility reasons.
pub bucket_sizes: HashMap<String, u64>,
// Todo: TierStats
// TierStats contains per-tier stats of all configured remote tiers
}
pub async fn store_data_usage_in_backend(mut rx: Receiver<DataUsageInfo>) {
let layer = new_object_layer_fn();
let lock = layer.read().await;
let store = match lock.as_ref() {
Some(s) => s,
None => {
info!("errServerNotInitialized");
return;
}
};
let mut attempts = 1;
loop {
match rx.recv().await {
Some(data_usage_info) => {
if let Ok(data) = serde_json::to_vec(&data_usage_info) {
if attempts > 10 {
let _ = save_config(store, &format!("{}{}", *DATA_USAGE_OBJ_NAME_PATH, ".bkp"), &data).await;
attempts += 1;
}
let _ = save_config(store, &DATA_USAGE_OBJ_NAME_PATH, &data).await;
attempts += 1;
} else {
continue;
}
}
None => {
return;
}
}
}
}
ecstore/src/heal/data_usage_cache.rs
+860
View File
@@ -0,0 +1,860 @@
use crate::config::common::save_config;
use crate::disk::error::DiskError;
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
use crate::error::{Error, Result};
use crate::new_object_layer_fn;
use crate::set_disk::SetDisks;
use crate::store_api::{BucketInfo, HTTPRangeSpec, ObjectIO, ObjectOptions};
use bytesize::ByteSize;
use http::HeaderMap;
use path_clean::PathClean;
use rand::Rng;
use rmp_serde::Serializer;
use s3s::dto::ReplicationConfiguration;
use s3s::{S3Error, S3ErrorCode};
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::hash::{DefaultHasher, Hash, Hasher};
use std::path::Path;
use std::time::{Duration, SystemTime};
use tokio::sync::mpsc::Sender;
use tokio::time::sleep;
use super::data_scanner::{SizeSummary, DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS};
use super::data_usage::{BucketTargetUsageInfo, BucketUsageInfo, DataUsageInfo, DATA_USAGE_ROOT};
// DATA_USAGE_BUCKET_LEN must be length of ObjectsHistogramIntervals
pub const DATA_USAGE_BUCKET_LEN: usize = 11;
pub const DATA_USAGE_VERSION_LEN: usize = 7;
pub type DataUsageHashMap = HashSet<String>;
struct ObjectHistogramInterval {
name: &'static str,
start: u64,
end: u64,
}
const OBJECTS_HISTOGRAM_INTERVALS: [ObjectHistogramInterval; DATA_USAGE_BUCKET_LEN] = [
ObjectHistogramInterval {
name: "LESS_THAN_1024_B",
start: 0,
end: ByteSize::kib(1).as_u64() - 1,
},
ObjectHistogramInterval {
name: "BETWEEN_1024_B_AND_64_KB",
start: ByteSize::kib(1).as_u64(),
end: ByteSize::kib(64).as_u64() - 1,
},
ObjectHistogramInterval {
name: "BETWEEN_64_KB_AND_256_KB",
start: ByteSize::kib(64).as_u64(),
end: ByteSize::kib(256).as_u64() - 1,
},
ObjectHistogramInterval {
name: "BETWEEN_256_KB_AND_512_KB",
start: ByteSize::kib(256).as_u64(),
end: ByteSize::kib(512).as_u64() - 1,
},
ObjectHistogramInterval {
name: "BETWEEN_512_KB_AND_1_MB",
start: ByteSize::kib(512).as_u64(),
end: ByteSize::mib(1).as_u64() - 1,
},
ObjectHistogramInterval {
name: "BETWEEN_1024B_AND_1_MB",
start: ByteSize::kib(1).as_u64(),
end: ByteSize::mib(1).as_u64() - 1,
},
ObjectHistogramInterval {
name: "BETWEEN_1_MB_AND_10_MB",
start: ByteSize::mib(1).as_u64(),
end: ByteSize::mib(10).as_u64() - 1,
},
ObjectHistogramInterval {
name: "BETWEEN_10_MB_AND_64_MB",
start: ByteSize::mib(10).as_u64(),
end: ByteSize::mib(64).as_u64() - 1,
},
ObjectHistogramInterval {
name: "BETWEEN_64_MB_AND_128_MB",
start: ByteSize::mib(64).as_u64(),
end: ByteSize::mib(128).as_u64() - 1,
},
ObjectHistogramInterval {
name: "BETWEEN_128_MB_AND_512_MB",
start: ByteSize::mib(128).as_u64(),
end: ByteSize::mib(512).as_u64() - 1,
},
ObjectHistogramInterval {
name: "GREATER_THAN_512_MB",
start: ByteSize::mib(512).as_u64(),
end: u64::MAX,
},
];
const OBJECTS_VERSION_COUNT_INTERVALS: [ObjectHistogramInterval; DATA_USAGE_VERSION_LEN] = [
ObjectHistogramInterval {
name: "UNVERSIONED",
start: 0,
end: 0,
},
ObjectHistogramInterval {
name: "SINGLE_VERSION",
start: 1,
end: 1,
},
ObjectHistogramInterval {
name: "BETWEEN_2_AND_10",
start: 2,
end: 9,
},
ObjectHistogramInterval {
name: "BETWEEN_10_AND_100",
start: 10,
end: 99,
},
ObjectHistogramInterval {
name: "BETWEEN_100_AND_1000",
start: 100,
end: 999,
},
ObjectHistogramInterval {
name: "BETWEEN_1000_AND_10000",
start: 1000,
end: 9999,
},
ObjectHistogramInterval {
name: "GREATER_THAN_10000",
start: 10000,
end: u64::MAX,
},
];
// sizeHistogram is a size histogram.
#[derive(Clone, Serialize, Deserialize)]
pub struct SizeHistogram(Vec<u64>);
impl Default for SizeHistogram {
fn default() -> Self {
Self(vec![0; DATA_USAGE_BUCKET_LEN])
}
}
impl SizeHistogram {
fn add(&mut self, size: u64) {
for (idx, interval) in OBJECTS_HISTOGRAM_INTERVALS.iter().enumerate() {
if size >= interval.start && size <= interval.end {
self.0[idx] += 1;
break;
}
}
}
pub fn to_map(&self) -> HashMap<String, u64> {
let mut res = HashMap::new();
let mut spl_count = 0;
for (count, oh) in self.0.iter().zip(OBJECTS_HISTOGRAM_INTERVALS.iter()) {
if ByteSize::kib(1).as_u64() == oh.start && oh.end == ByteSize::mib(1).as_u64() - 1 {
res.insert(oh.name.to_string(), spl_count);
} else if ByteSize::kib(1).as_u64() <= oh.start && oh.end < ByteSize::mib(1).as_u64() {
spl_count += count;
res.insert(oh.name.to_string(), *count);
} else {
res.insert(oh.name.to_string(), *count);
}
}
res
}
}
// versionsHistogram is a histogram of number of versions in an object.
#[derive(Clone, Serialize, Deserialize)]
pub struct VersionsHistogram(Vec<u64>);
impl Default for VersionsHistogram {
fn default() -> Self {
Self(vec![0; DATA_USAGE_VERSION_LEN])
}
}
impl VersionsHistogram {
fn add(&mut self, size: u64) {
for (idx, interval) in OBJECTS_VERSION_COUNT_INTERVALS.iter().enumerate() {
if size >= interval.start && size <= interval.end {
self.0[idx] += 1;
break;
}
}
}
pub fn to_map(&self) -> HashMap<String, u64> {
let mut res = HashMap::new();
for (count, ov) in self.0.iter().zip(OBJECTS_VERSION_COUNT_INTERVALS.iter()) {
res.insert(ov.name.to_string(), *count);
}
res
}
}
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct ReplicationStats {
pub pending_size: u64,
pub replicated_size: u64,
pub failed_size: u64,
pub failed_count: u64,
pub pending_count: u64,
pub missed_threshold_size: u64,
pub after_threshold_size: u64,
pub missed_threshold_count: u64,
pub after_threshold_count: u64,
pub replicated_count: u64,
}
impl ReplicationStats {
pub fn empty(&self) -> bool {
self.replicated_size == 0 && self.failed_size == 0 && self.failed_count == 0
}
}
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct ReplicationAllStats {
pub targets: HashMap<String, ReplicationStats>,
pub replica_size: u64,
pub replica_count: u64,
}
impl ReplicationAllStats {
pub fn empty(&self) -> bool {
if self.replica_size != 0 && self.replica_count != 0 {
return false;
}
for (_, v) in self.targets.iter() {
if !v.empty() {
return false;
}
}
true
}
}
#[derive(Clone, Default, Serialize, Deserialize)]
pub struct DataUsageEntry {
pub children: DataUsageHashMap,
// These fields do no include any children.
pub size: usize,
pub objects: usize,
pub versions: usize,
pub delete_markers: usize,
pub obj_sizes: SizeHistogram,
pub obj_versions: VersionsHistogram,
pub replication_stats: Option<ReplicationAllStats>,
// Todo: tier
// pub all_tier_stats: ,
pub compacted: bool,
}
impl DataUsageEntry {
pub fn add_child(&mut self, hash: &DataUsageHash) {
if self.children.contains(&hash.key()) {
return;
}
self.children.insert(hash.key());
}
pub fn add_sizes(&mut self, summary: &SizeSummary) {
self.size += summary.total_size;
self.versions += summary.versions;
self.delete_markers += summary.delete_markers;
self.obj_sizes.add(summary.total_size as u64);
self.obj_versions.add(summary.versions as u64);
let replication_stats = if self.replication_stats.is_none() {
self.replication_stats = Some(ReplicationAllStats::default());
self.replication_stats.as_mut().unwrap()
} else {
self.replication_stats.as_mut().unwrap()
};
replication_stats.replica_size += summary.replica_size as u64;
replication_stats.replica_count += summary.replica_count as u64;
for (arn, st) in &summary.repl_target_stats {
let tgt_stat = replication_stats
.targets
.entry(arn.to_string())
.or_insert(ReplicationStats::default());
tgt_stat.pending_size += st.pending_size as u64;
tgt_stat.failed_size += st.failed_size as u64;
tgt_stat.replicated_size += st.replicated_size as u64;
tgt_stat.replicated_count += st.replicated_count as u64;
tgt_stat.failed_count += st.failed_count as u64;
tgt_stat.pending_count += st.pending_count as u64;
}
// Todo:: tiers
}
pub fn merge(&mut self, other: &DataUsageEntry) {
self.objects += other.objects;
self.versions += other.versions;
self.delete_markers += other.delete_markers;
self.size += other.size;
if let Some(o_rep) = &other.replication_stats {
if self.replication_stats.is_none() {
self.replication_stats = Some(ReplicationAllStats::default());
}
let s_rep = self.replication_stats.as_mut().unwrap();
s_rep.targets.clear();
s_rep.replica_size += o_rep.replica_size;
s_rep.replica_count += o_rep.replica_count;
for (arn, stat) in o_rep.targets.iter() {
let st = s_rep.targets.entry(arn.clone()).or_default();
*st = ReplicationStats {
pending_size: stat.pending_size + st.pending_size,
failed_size: stat.failed_size + st.failed_size,
replicated_size: stat.replicated_size + st.replicated_size,
pending_count: stat.pending_count + st.pending_count,
failed_count: stat.failed_count + st.failed_count,
replicated_count: stat.replicated_count + st.replicated_count,
..Default::default()
};
}
}
for (i, v) in other.obj_sizes.0.iter().enumerate() {
self.obj_sizes.0[i] += v;
}
for (i, v) in other.obj_versions.0.iter().enumerate() {
self.obj_versions.0[i] += v;
}
// todo: tiers
}
}
#[derive(Clone)]
pub struct DataUsageEntryInfo {
pub name: String,
pub parent: String,
pub entry: DataUsageEntry,
}
#[derive(Clone, Default, Serialize, Deserialize)]
pub struct DataUsageCacheInfo {
pub name: String,
pub next_cycle: u32,
pub last_update: Option<SystemTime>,
pub skip_healing: bool,
// todo: life_cycle
// pub life_cycle:
#[serde(skip)]
pub updates: Option<Sender<DataUsageEntry>>,
#[serde(skip)]
pub replication: Option<ReplicationConfiguration>,
}
// impl Default for DataUsageCacheInfo {
// fn default() -> Self {
// Self {
// name: Default::default(),
// next_cycle: Default::default(),
// last_update: SystemTime::now(),
// skip_healing: Default::default(),
// updates: Default::default(),
// replication: Default::default(),
// }
// }
// }
#[derive(Clone, Default, Serialize, Deserialize)]
pub struct DataUsageCache {
pub info: DataUsageCacheInfo,
pub cache: HashMap<String, DataUsageEntry>,
}
impl DataUsageCache {
pub async fn load(store: &SetDisks, name: &str) -> Result<Self> {
let mut d = DataUsageCache::default();
let mut retries = 0;
while retries < 5 {
let path = Path::new(BUCKET_META_PREFIX).join(name);
match store
.get_object_reader(
RUSTFS_META_BUCKET,
path.to_str().unwrap(),
HTTPRangeSpec::nil(),
HeaderMap::new(),
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await
{
Ok(mut reader) => {
if let Ok(info) = Self::unmarshal(&reader.read_all().await?) {
d = info
}
break;
}
Err(err) => match err.downcast_ref::<DiskError>() {
Some(DiskError::FileNotFound) | Some(DiskError::VolumeNotFound) => {
match store
.get_object_reader(
RUSTFS_META_BUCKET,
name,
HTTPRangeSpec::nil(),
HeaderMap::new(),
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await
{
Ok(mut reader) => {
if let Ok(info) = Self::unmarshal(&reader.read_all().await?) {
d = info
}
break;
}
Err(_) => match err.downcast_ref::<DiskError>() {
Some(DiskError::FileNotFound) | Some(DiskError::VolumeNotFound) => {
break;
}
_ => {}
},
}
}
_ => {}
},
}
retries += 1;
let dur = {
let mut rng = rand::thread_rng();
rng.gen_range(0..1_000)
};
sleep(Duration::from_millis(dur)).await;
}
Ok(d)
}
pub async fn save(&self, name: &str) -> Result<()> {
let buf = self.marshal_msg()?;
let buf_clone = buf.clone();
let layer = new_object_layer_fn();
let lock = layer.read().await;
let store = match lock.as_ref() {
Some(s) => s,
None => return Err(Error::from(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))),
};
let store_clone = store.clone();
let name_clone = name.to_string();
tokio::spawn(async move {
let _ = save_config(&store_clone, &format!("{}{}", &name_clone, ".bkp"), &buf_clone).await;
});
save_config(store, name, &buf).await
}
pub fn replace(&mut self, path: &str, parent: &str, e: DataUsageEntry) {
let hash = hash_path(path);
self.cache.insert(hash.key(), e);
if !parent.is_empty() {
let phash = hash_path(parent);
let p = {
let p = self.cache.entry(phash.key()).or_default();
p.add_child(&hash);
p.clone()
};
self.cache.insert(phash.key(), p);
}
}
pub fn replace_hashed(&mut self, hash: &DataUsageHash, parent: &Option<DataUsageHash>, e: &DataUsageEntry) {
self.cache.insert(hash.key(), e.clone());
if let Some(parent) = parent {
self.cache.entry(parent.key()).or_default().add_child(hash);
}
}
pub fn find(&self, path: &str) -> Option<DataUsageEntry> {
self.cache.get(&hash_path(path).key()).cloned()
}
pub fn find_children_copy(&mut self, h: DataUsageHash) -> DataUsageHashMap {
self.cache.entry(h.string()).or_default().children.clone()
}
pub fn flatten(&self, root: &DataUsageEntry) -> DataUsageEntry {
let mut root = root.clone();
for id in root.children.clone().iter() {
if let Some(e) = self.cache.get(id) {
let mut e = e.clone();
if !e.children.is_empty() {
e = self.flatten(&e);
}
root.merge(&e);
}
}
root.children.clear();
root
}
pub fn copy_with_children(&mut self, src: &DataUsageCache, hash: &DataUsageHash, parent: &Option<DataUsageHash>) {
if let Some(e) = src.cache.get(&hash.string()) {
self.cache.insert(hash.key(), e.clone());
for ch in e.children.iter() {
if *ch == hash.key() {
return;
}
self.copy_with_children(src, &DataUsageHash(ch.to_string()), &Some(hash.clone()));
}
if let Some(parent) = parent {
let p = self.cache.entry(parent.key()).or_default();
p.add_child(hash);
}
}
}
pub fn delete_recursive(&mut self, hash: &DataUsageHash) {
let mut need_remove = Vec::new();
if let Some(v) = self.cache.get(&hash.string()) {
for child in v.children.iter() {
need_remove.push(child.clone());
}
}
self.cache.remove(&hash.string());
need_remove.iter().for_each(|child| {
self.delete_recursive(&DataUsageHash(child.to_string()));
});
}
pub fn size_recursive(&self, path: &str) -> Option<DataUsageEntry> {
match self.find(path) {
Some(root) => {
if root.children.is_empty() {
return None;
}
let mut flat = self.flatten(&root);
if flat.replication_stats.is_some() && flat.replication_stats.as_ref().unwrap().empty() {
flat.replication_stats = None;
}
Some(flat)
}
None => None,
}
}
pub fn search_parent(&self, hash: &DataUsageHash) -> Option<DataUsageHash> {
let want = hash.key();
if let Some(last_index) = want.rfind('/') {
if let Some(v) = self.find(&want[0..last_index]) {
if v.children.contains(&want) {
let found = hash_path(&want[0..last_index]);
return Some(found);
}
}
}
for (k, v) in self.cache.iter() {
if v.children.contains(&want) {
let found = DataUsageHash(k.clone());
return Some(found);
}
}
None
}
pub fn is_compacted(&self, hash: &DataUsageHash) -> bool {
match self.cache.get(&hash.key()) {
Some(due) => due.compacted,
None => false,
}
}
pub fn force_compact(&mut self, limit: usize) {
if self.cache.len() < limit {
return;
}
let top = hash_path(&self.info.name).key();
let top_e = match self.find(&top) {
Some(e) => e,
None => return,
};
if top_e.children.len() > DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS.try_into().unwrap() {
self.reduce_children_of(&hash_path(&self.info.name), limit, true);
}
if self.cache.len() <= limit {
return;
}
let mut found = HashSet::new();
found.insert(top);
mark(self, &top_e, &mut found);
self.cache.retain(|k, _| {
if !found.contains(k) {
return false;
}
true
});
}
pub fn reduce_children_of(&mut self, path: &DataUsageHash, limit: usize, compact_self: bool) {
let e = match self.cache.get(&path.key()) {
Some(e) => e,
None => return,
};
if e.compacted {
return;
}
if e.children.len() > limit && compact_self {
let mut flat = self.size_recursive(&path.key()).unwrap_or_default();
flat.compacted = true;
self.delete_recursive(path);
self.replace_hashed(path, &None, &flat);
return;
}
let total = self.total_children_rec(&path.key());
if total < limit {
return;
}
let mut leaves = Vec::new();
let mut remove = total - limit;
add(self, path, &mut leaves);
leaves.sort_by(|a, b| a.objects.cmp(&b.objects));
while remove > 0 && !leaves.is_empty() {
let e = leaves.first().unwrap();
let candidate = e.path.clone();
if candidate == *path && !compact_self {
break;
}
let removing = self.total_children_rec(&candidate.key());
let mut flat = match self.size_recursive(&candidate.key()) {
Some(flat) => flat,
None => {
leaves.remove(0);
continue;
}
};
flat.compacted = true;
self.delete_recursive(&candidate);
self.replace_hashed(&candidate, &None, &flat);
remove -= removing;
leaves.remove(0);
}
}
pub fn total_children_rec(&self, path: &str) -> usize {
let root = self.find(path);
if root.is_none() {
return 0;
}
let root = root.unwrap();
if root.children.is_empty() {
return 0;
}
let mut n = root.children.len();
for ch in root.children.iter() {
n += self.total_children_rec(ch);
}
n
}
pub fn merge(&mut self, o: &DataUsageCache) {
let mut existing_root = self.root();
let other_root = o.root();
if existing_root.is_none() && other_root.is_none() {
return;
}
if other_root.is_none() {
return;
}
if existing_root.is_none() {
*self = o.clone();
return;
}
if o.info.last_update.gt(&self.info.last_update) {
self.info.last_update = o.info.last_update;
}
existing_root.as_mut().unwrap().merge(other_root.as_ref().unwrap());
self.cache.insert(hash_path(&self.info.name).key(), existing_root.unwrap());
let e_hash = self.root_hash();
for key in other_root.as_ref().unwrap().children.iter() {
let entry = &o.cache[key];
let flat = o.flatten(entry);
let mut existing = self.cache[key].clone();
existing.merge(&flat);
self.replace_hashed(&DataUsageHash(key.clone()), &Some(e_hash.clone()), &existing);
}
}
pub fn root_hash(&self) -> DataUsageHash {
hash_path(&self.info.name)
}
pub fn root(&self) -> Option<DataUsageEntry> {
self.find(&self.info.name)
}
pub fn dui(&self, path: &str, buckets: &[BucketInfo]) -> DataUsageInfo {
let e = match self.find(path) {
Some(e) => e,
None => return DataUsageInfo::default(),
};
let flat = self.flatten(&e);
DataUsageInfo {
last_update: self.info.last_update,
objects_total_count: flat.objects as u64,
versions_total_count: flat.versions as u64,
delete_markers_total_count: flat.delete_markers as u64,
objects_total_size: flat.size as u64,
buckets_count: e.children.len() as u64,
buckets_usage: self.buckets_usage_info(buckets),
..Default::default()
}
}
pub fn buckets_usage_info(&self, buckets: &[BucketInfo]) -> HashMap<String, BucketUsageInfo> {
let mut dst = HashMap::new();
for bucket in buckets.iter() {
let e = match self.find(&bucket.name) {
Some(e) => e,
None => continue,
};
let flat = self.flatten(&e);
let mut bui = BucketUsageInfo {
size: flat.size as u64,
versions_count: flat.versions as u64,
objects_count: flat.objects as u64,
delete_markers_count: flat.delete_markers as u64,
object_size_histogram: flat.obj_sizes.to_map(),
object_versions_histogram: flat.obj_versions.to_map(),
..Default::default()
};
if let Some(rs) = &flat.replication_stats {
bui.replica_size = rs.replica_size;
bui.replica_count = rs.replica_count;
for (arn, stat) in rs.targets.iter() {
bui.replication_info.insert(
arn.clone(),
BucketTargetUsageInfo {
replication_pending_size: stat.pending_size,
replicated_size: stat.replicated_size,
replication_failed_size: stat.failed_size,
replication_pending_count: stat.pending_count,
replication_failed_count: stat.failed_count,
replicated_count: stat.replicated_count,
..Default::default()
},
);
}
}
dst.insert(bucket.name.clone(), bui);
}
dst
}
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
let mut buf = Vec::new();
self.serialize(&mut Serializer::new(&mut buf))?;
Ok(buf)
}
pub fn unmarshal(buf: &[u8]) -> Result<Self> {
let t: Self = rmp_serde::from_slice(buf)?;
Ok(t)
}
}
#[derive(Default, Clone)]
struct Inner {
objects: usize,
path: DataUsageHash,
}
fn add(data_usage_cache: &DataUsageCache, path: &DataUsageHash, leaves: &mut Vec<Inner>) {
let e = match data_usage_cache.cache.get(&path.key()) {
Some(e) => e,
None => return,
};
if !e.children.is_empty() {
return;
}
let sz = data_usage_cache.size_recursive(&path.key()).unwrap_or_default();
leaves.push(Inner {
objects: sz.objects,
path: path.clone(),
});
for ch in e.children.iter() {
add(data_usage_cache, &DataUsageHash(ch.clone()), leaves);
}
}
fn mark(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut HashSet<String>) {
for k in entry.children.iter() {
found.insert(k.to_string());
if let Some(ch) = duc.cache.get(k) {
mark(duc, ch, found);
}
}
}
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct DataUsageHash(pub String);
impl DataUsageHash {
pub fn string(&self) -> String {
self.0.clone()
}
pub fn key(&self) -> String {
self.0.clone()
}
pub fn mod_(&self, cycle: u32, cycles: u32) -> bool {
if cycles <= 1 {
return cycles == 1;
}
let hash = self.calculate_hash();
hash as u32 % cycles == cycle % cycles
}
pub fn mod_alt(&self, cycle: u32, cycles: u32) -> bool {
if cycles <= 1 {
return cycles == 1;
}
let hash = self.calculate_hash();
(hash >> 32) as u32 % cycles == cycle % cycles
}
fn calculate_hash(&self) -> u64 {
let mut hasher = DefaultHasher::new();
self.0.hash(&mut hasher);
hasher.finish()
}
}
pub fn hash_path(data: &str) -> DataUsageHash {
let mut data = data;
if data != DATA_USAGE_ROOT {
data = data.trim_matches('/');
}
DataUsageHash(Path::new(&data).clean().to_string_lossy().to_string())
}
ecstore/src/heal/error.rs
+5
View File
@@ -0,0 +1,5 @@
pub const ERR_IGNORE_FILE_CONTRIB: &str = "ignore this file's contribution toward data-usage";
pub const ERR_SKIP_FILE: &str = "skip this file";
pub const ERR_HEAL_STOP_SIGNALLED: &str = "heal stop signaled";
pub const ERR_HEAL_IDLE_TIMEOUT: &str = "healing results were not consumed for too long";
pub const ERR_RETRY_HEALING: &str = "some items failed to heal, we will retry healing this drive again";
ecstore/src/heal/heal_commands.rs
+71 -65
View File
@@ -1,14 +1,14 @@
use std::{
path::Path,
time::{SystemTime, UNIX_EPOCH},
};
use std::{path::Path, time::SystemTime};
use lazy_static::lazy_static;
use serde::{Deserialize, Serialize};
use time::OffsetDateTime;
use tokio::sync::RwLock;
use crate::{
disk::{DeleteOptions, DiskAPI, DiskStore, BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
error::{Error, Result},
global::GLOBAL_BackgroundHealState,
heal::heal_ops::HEALING_TRACKER_FILENAME,
new_object_layer_fn,
store_api::{BucketInfo, StorageAPI},
@@ -37,7 +37,11 @@ pub const DRIVE_STATE_ROOT_MOUNT: &str = "root-mount";
pub const DRIVE_STATE_UNKNOWN: &str = "unknown";
pub const DRIVE_STATE_UNFORMATTED: &str = "unformatted"; // only returned by disk
#[derive(Clone, Copy, Debug, Default)]
lazy_static! {
pub static ref TIME_SENTINEL: OffsetDateTime = OffsetDateTime::from_unix_timestamp(0).unwrap();
}
#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)]
pub struct HealOpts {
pub recursive: bool,
pub dry_run: bool,
@@ -50,7 +54,7 @@ pub struct HealOpts {
pub set: Option<usize>,
}
#[derive(Clone, Debug)]
#[derive(Clone, Debug, Default)]
pub struct HealDriveInfo {
pub uuid: String,
pub endpoint: String,
@@ -74,11 +78,21 @@ pub struct HealResultItem {
pub object_size: usize,
}
#[derive(Debug, Default, Serialize, Deserialize)]
#[derive(Debug, Serialize, Deserialize)]
pub struct HealStartSuccess {
pub client_token: String,
pub client_address: String,
pub start_time: u64,
pub start_time: SystemTime,
}
impl Default for HealStartSuccess {
fn default() -> Self {
Self {
client_token: Default::default(),
client_address: Default::default(),
start_time: SystemTime::now(),
}
}
}
pub type HealStopSuccess = HealStartSuccess;
@@ -91,8 +105,8 @@ pub struct HealingDisk {
pub disk_index: Option<usize>,
pub endpoint: String,
pub path: String,
pub started: u64,
pub last_update: u64,
pub started: Option<OffsetDateTime>,
pub last_update: Option<SystemTime>,
pub retry_attempts: u64,
pub objects_total_count: u64,
pub objects_total_size: u64,
@@ -121,8 +135,8 @@ pub struct HealingTracker {
pub disk_index: Option<usize>,
pub path: String,
pub endpoint: String,
pub started: u64,
pub last_update: u64,
pub started: Option<OffsetDateTime>,
pub last_update: Option<SystemTime>,
pub objects_total_count: u64,
pub objects_total_size: u64,
pub items_healed: u64,
@@ -150,9 +164,7 @@ pub struct HealingTracker {
impl HealingTracker {
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
serde_json::to_string(self)
.map(|s| s.as_bytes().to_vec())
.map_err(|err| Error::from_string(err.to_string()))
serde_json::to_vec(self).map_err(|err| Error::from_string(err.to_string()))
}
pub fn unmarshal_msg(data: &[u8]) -> Result<Self> {
@@ -177,7 +189,7 @@ impl HealingTracker {
self.object = String::new();
}
pub async fn get_last_update(&self) -> u64 {
pub async fn get_last_update(&self) -> Option<SystemTime> {
let _ = self.mu.read().await;
self.last_update
@@ -224,7 +236,7 @@ impl HealingTracker {
pub async fn update(&mut self) -> Result<()> {
if let Some(disk) = &self.disk {
if healing(&disk.path().to_string_lossy().to_string()).await?.is_none() {
if healing(disk.path().to_string_lossy().as_ref()).await?.is_none() {
return Err(Error::from_string(format!("healingTracker: drive {} is not marked as healing", self.id)));
}
let _ = self.mu.write().await;
@@ -252,14 +264,11 @@ impl HealingTracker {
(self.pool_index, self.set_index, self.disk_index) = store.get_pool_and_set(&self.id).await?;
}
self.last_update = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("Time went backwards")
.as_secs();
self.last_update = Some(SystemTime::now());
let htracker_bytes = self.marshal_msg()?;
// TODO: globalBackgroundHealState
GLOBAL_BackgroundHealState.write().await.update_heal_status(self).await;
if let Some(disk) = &self.disk {
let file_path = Path::new(BUCKET_META_PREFIX).join(HEALING_TRACKER_FILENAME);
@@ -270,7 +279,7 @@ impl HealingTracker {
Ok(())
}
async fn delete(&self) -> Result<()> {
pub async fn delete(&self) -> Result<()> {
if let Some(disk) = &self.disk {
let file_path = Path::new(BUCKET_META_PREFIX).join(HEALING_TRACKER_FILENAME);
return disk
@@ -289,7 +298,7 @@ impl HealingTracker {
Ok(())
}
async fn is_healed(&self, bucket: &str) -> bool {
pub async fn is_healed(&self, bucket: &str) -> bool {
let _ = self.mu.read().await;
for v in self.healed_buckets.iter() {
if v == bucket {
@@ -300,7 +309,7 @@ impl HealingTracker {
false
}
async fn resume(&mut self) {
pub async fn resume(&mut self) {
let _ = self.mu.write().await;
self.items_healed = self.resume_items_healed;
@@ -311,7 +320,7 @@ impl HealingTracker {
self.bytes_skipped = self.resume_bytes_skipped;
}
async fn bucket_done(&mut self, bucket: &str) {
pub async fn bucket_done(&mut self, bucket: &str) {
let _ = self.mu.write().await;
self.resume_items_healed = self.items_healed;
@@ -325,7 +334,7 @@ impl HealingTracker {
self.queue_buckets.retain(|x| x != bucket);
}
async fn set_queue_buckets(&mut self, buckets: &[BucketInfo]) {
pub async fn set_queue_buckets(&mut self, buckets: &[BucketInfo]) {
let _ = self.mu.write().await;
buckets.iter().for_each(|bucket| {
@@ -373,40 +382,40 @@ impl Clone for HealingTracker {
Self {
disk: self.disk.clone(),
id: self.id.clone(),
pool_index: self.pool_index.clone(),
set_index: self.set_index.clone(),
disk_index: self.disk_index.clone(),
pool_index: self.pool_index,
set_index: self.set_index,
disk_index: self.disk_index,
path: self.path.clone(),
endpoint: self.endpoint.clone(),
started: self.started.clone(),
last_update: self.last_update.clone(),
objects_total_count: self.objects_total_count.clone(),
objects_total_size: self.objects_total_size.clone(),
items_healed: self.items_healed.clone(),
items_failed: self.items_failed.clone(),
item_skipped: self.item_skipped.clone(),
bytes_done: self.bytes_done.clone(),
bytes_failed: self.bytes_failed.clone(),
bytes_skipped: self.bytes_skipped.clone(),
started: self.started,
last_update: self.last_update,
objects_total_count: self.objects_total_count,
objects_total_size: self.objects_total_size,
items_healed: self.items_healed,
items_failed: self.items_failed,
item_skipped: self.item_skipped,
bytes_done: self.bytes_done,
bytes_failed: self.bytes_failed,
bytes_skipped: self.bytes_skipped,
bucket: self.bucket.clone(),
object: self.object.clone(),
resume_items_healed: self.resume_items_healed.clone(),
resume_items_failed: self.resume_items_failed.clone(),
resume_items_skipped: self.resume_items_skipped.clone(),
resume_bytes_done: self.resume_bytes_done.clone(),
resume_bytes_failed: self.resume_bytes_failed.clone(),
resume_bytes_skipped: self.resume_bytes_skipped.clone(),
resume_items_healed: self.resume_items_healed,
resume_items_failed: self.resume_items_failed,
resume_items_skipped: self.resume_items_skipped,
resume_bytes_done: self.resume_bytes_done,
resume_bytes_failed: self.resume_bytes_failed,
resume_bytes_skipped: self.resume_bytes_skipped,
queue_buckets: self.queue_buckets.clone(),
healed_buckets: self.healed_buckets.clone(),
heal_id: self.heal_id.clone(),
retry_attempts: self.retry_attempts.clone(),
finished: self.finished.clone(),
retry_attempts: self.retry_attempts,
finished: self.finished,
mu: RwLock::new(false),
}
}
}
async fn load_healing_tracker(disk: &Option<DiskStore>) -> Result<HealingTracker> {
pub async fn load_healing_tracker(disk: &Option<DiskStore>) -> Result<HealingTracker> {
if let Some(disk) = disk {
let disk_id = disk.get_disk_id().await?;
if let Some(disk_id) = disk_id {
@@ -430,23 +439,20 @@ async fn load_healing_tracker(disk: &Option<DiskStore>) -> Result<HealingTracker
}
}
async fn init_healing_tracker(disk: DiskStore, heal_id: String) -> Result<HealingTracker> {
let mut healing_tracker = HealingTracker::default();
healing_tracker.id = disk.get_disk_id().await?.map_or("".to_string(), |id| id.to_string());
healing_tracker.heal_id = heal_id;
healing_tracker.path = disk.to_string();
healing_tracker.endpoint = disk.endpoint().to_string();
healing_tracker.started = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("Time went backwards")
.as_secs();
pub async fn init_healing_tracker(disk: DiskStore, heal_id: &str) -> Result<HealingTracker> {
let disk_location = disk.get_disk_location();
healing_tracker.pool_index = disk_location.pool_idx;
healing_tracker.set_index = disk_location.set_idx;
healing_tracker.disk_index = disk_location.disk_idx;
healing_tracker.disk = Some(disk);
Ok(healing_tracker)
Ok(HealingTracker {
id: disk.get_disk_id().await?.map_or("".to_string(), |id| id.to_string()),
heal_id: heal_id.to_string(),
path: disk.to_string(),
endpoint: disk.endpoint().to_string(),
started: Some(OffsetDateTime::now_utc()),
pool_index: disk_location.pool_idx,
set_index: disk_location.set_idx,
disk_index: disk_location.disk_idx,
disk: Some(disk),
..Default::default()
})
}
pub async fn healing(derive_path: &str) -> Result<Option<HealingTracker>> {
ecstore/src/heal/heal_ops.rs
+335 -117
View File
@@ -1,12 +1,34 @@
use super::{
background_heal_ops::HealTask,
data_scanner::HEAL_DELETE_DANGLING,
error::ERR_SKIP_FILE,
heal_commands::{
HealItemType, HealOpts, HealResultItem, HealScanMode, HealStopSuccess, HealingDisk, HealingTracker,
HEAL_ITEM_BUCKET_METADATA,
},
};
use crate::heal::heal_commands::{HEAL_ITEM_BUCKET, HEAL_ITEM_OBJECT};
use crate::store_api::StorageAPI;
use crate::{
config::common::CONFIG_PREFIX,
disk::RUSTFS_META_BUCKET,
global::GLOBAL_BackgroundHealRoutine,
heal::{
error::ERR_HEAL_STOP_SIGNALLED,
heal_commands::{HealDriveInfo, DRIVE_STATE_OK},
},
};
use crate::{
disk::{endpoint::Endpoint, MetaCacheEntry},
endpoints::Endpoints,
error::{Error, Result},
global::GLOBAL_IsDistErasure,
heal::heal_commands::HEAL_UNKNOWN_SCAN,
heal::heal_commands::{HealStartSuccess, HEAL_UNKNOWN_SCAN},
new_object_layer_fn,
utils::path::has_profix,
};
use lazy_static::lazy_static;
use s3s::{S3Error, S3ErrorCode};
use std::{
collections::HashMap,
future::Future,
@@ -24,18 +46,13 @@ use tokio::{
},
time::{interval, sleep},
};
use tracing::info;
use uuid::Uuid;
use super::{
background_heal_ops::HealTask,
heal_commands::{HealItemType, HealOpts, HealResultItem, HealScanMode, HealStopSuccess, HealingDisk, HealingTracker},
};
type HealStatusSummary = String;
type ItemsMap = HashMap<HealItemType, usize>;
pub type HealObjectFn = Arc<dyn Fn(&str, &str, &str, HealScanMode) -> Result<()> + Send + Sync>;
pub type HealEntryFn =
Box<dyn Fn(String, MetaCacheEntry, HealScanMode) -> Pin<Box<dyn Future<Output = Result<()>> + Send>> + Send>;
Arc<dyn Fn(String, MetaCacheEntry, HealScanMode) -> Pin<Box<dyn Future<Output = Result<()>> + Send>> + Send + Sync + 'static>;
pub const BG_HEALING_UUID: &str = "0000-0000-0000-0000";
pub const HEALING_TRACKER_FILENAME: &str = ".healing.bin";
@@ -45,6 +62,10 @@ const HEAL_RUNNING_STATUS: &str = "running";
const HEAL_STOPPED_STATUS: &str = "stopped";
const HEAL_FINISHED_STATUS: &str = "finished";
pub const RUESTFS_RESERVED_BUCKET: &str = "rustfs";
pub const RUESTFS_RESERVED_BUCKET_PATH: &str = "/rustfs";
pub const LOGIN_PATH_PREFIX: &str = "/login";
const MAX_UNCONSUMED_HEAL_RESULT_ITEMS: usize = 1000;
const HEAL_UNCONSUMED_TIMEOUT: std::time::Duration = Duration::from_secs(24 * 60 * 60);
pub const NOP_HEAL: &str = "";
@@ -60,12 +81,13 @@ pub struct HealSequenceStatus {
pub items: Vec<HealResultItem>,
}
#[derive(Debug, Default)]
pub struct HealSource {
pub bucket: String,
pub object: String,
pub version_id: String,
pub no_wait: bool,
opts: Option<HealOpts>,
pub opts: Option<HealOpts>,
}
#[derive(Clone, Debug)]
@@ -73,8 +95,8 @@ pub struct HealSequence {
pub bucket: String,
pub object: String,
pub report_progress: bool,
pub start_time: u64,
pub end_time: Arc<RwLock<u64>>,
pub start_time: SystemTime,
pub end_time: Arc<RwLock<SystemTime>>,
pub client_token: String,
pub client_address: String,
pub force_started: bool,
@@ -93,6 +115,30 @@ pub struct HealSequence {
rx: Arc<RwLock<Receiver<bool>>>,
}
pub fn new_bg_heal_sequence() -> HealSequence {
let hs = HealOpts {
remove: HEAL_DELETE_DANGLING,
..Default::default()
};
HealSequence {
start_time: SystemTime::now(),
client_token: BG_HEALING_UUID.to_string(),
bucket: RUESTFS_RESERVED_BUCKET.to_string(),
setting: hs,
current_status: Arc::new(RwLock::new(HealSequenceStatus {
summary: HEAL_NOT_STARTED_STATUS.to_string(),
heal_setting: hs,
..Default::default()
})),
report_progress: false,
scanned_items_map: HashMap::new(),
healed_items_map: HashMap::new(),
heal_failed_items_map: HashMap::new(),
..Default::default()
}
}
impl Default for HealSequence {
fn default() -> Self {
let (h_tx, h_rx) = mpsc::channel(1);
@@ -101,8 +147,8 @@ impl Default for HealSequence {
bucket: Default::default(),
object: Default::default(),
report_progress: Default::default(),
start_time: Default::default(),
end_time: Default::default(),
start_time: SystemTime::now(),
end_time: Arc::new(RwLock::new(SystemTime::now())),
client_token: Default::default(),
client_address: Default::default(),
force_started: Default::default(),
@@ -144,23 +190,23 @@ impl HealSequence {
}
impl HealSequence {
fn get_scanned_items_count(&self) -> usize {
fn _get_scanned_items_count(&self) -> usize {
self.scanned_items_map.values().sum()
}
fn get_scanned_items_map(&self) -> ItemsMap {
fn _get_scanned_items_map(&self) -> ItemsMap {
self.scanned_items_map.clone()
}
fn get_healed_items_map(&self) -> ItemsMap {
fn _get_healed_items_map(&self) -> ItemsMap {
self.healed_items_map.clone()
}
fn get_heal_failed_items_map(&self) -> ItemsMap {
fn _get_heal_failed_items_map(&self) -> ItemsMap {
self.heal_failed_items_map.clone()
}
fn count_failed(&mut self, heal_type: HealItemType) {
pub fn count_failed(&mut self, heal_type: HealItemType) {
*self.heal_failed_items_map.entry(heal_type).or_insert(0) += 1;
self.last_heal_activity = SystemTime::now()
.duration_since(UNIX_EPOCH)
@@ -168,7 +214,7 @@ impl HealSequence {
.as_secs();
}
fn count_scanned(&mut self, heal_type: HealItemType) {
pub fn count_scanned(&mut self, heal_type: HealItemType) {
*self.scanned_items_map.entry(heal_type).or_insert(0) += 1;
self.last_heal_activity = SystemTime::now()
.duration_since(UNIX_EPOCH)
@@ -176,7 +222,7 @@ impl HealSequence {
.as_secs();
}
fn count_healed(&mut self, heal_type: HealItemType) {
pub fn count_healed(&mut self, heal_type: HealItemType) {
*self.healed_items_map.entry(heal_type).or_insert(0) += 1;
self.last_heal_activity = SystemTime::now()
.duration_since(UNIX_EPOCH)
@@ -193,11 +239,11 @@ impl HealSequence {
}
async fn has_ended(&self) -> bool {
if self.client_token == BG_HEALING_UUID.to_string() {
if self.client_token == *BG_HEALING_UUID {
return false;
}
!(*(self.end_time.read().await) == self.start_time)
*(self.end_time.read().await) != self.start_time
}
async fn stop(&self) {
@@ -208,6 +254,7 @@ impl HealSequence {
async fn push_heal_result_item(&self, r: &HealResultItem) -> Result<()> {
let mut r = r.clone();
let mut interval_timer = interval(HEAL_UNCONSUMED_TIMEOUT);
#[allow(unused_assignments)]
let mut items_len = 0;
loop {
{
@@ -244,7 +291,7 @@ impl HealSequence {
Ok(())
}
async fn queue_heal_task(&mut self, source: HealSource, heal_type: HealItemType) -> Result<()> {
pub async fn queue_heal_task(&mut self, source: HealSource, heal_type: HealItemType) -> Result<()> {
let mut task = HealTask::new(&source.bucket, &source.object, &source.version_id, &self.setting);
if let Some(opts) = source.opts {
task.opts = opts;
@@ -252,31 +299,104 @@ impl HealSequence {
task.opts.scan_mode = HEAL_UNKNOWN_SCAN;
}
self.count_scanned(heal_type);
self.count_scanned(heal_type.clone());
if source.no_wait {}
todo!()
}
fn heal_disk_meta() -> Result<()> {
todo!()
}
fn heal_items(&self, buckets_only: bool) -> Result<()> {
if self.client_token == BG_HEALING_UUID.to_string() {
if source.no_wait {
let task_str = format!("{:?}", task);
if GLOBAL_BackgroundHealRoutine.read().await.tasks_tx.try_send(task).is_ok() {
info!("Task in the queue: {:?}", task_str);
}
return Ok(());
}
todo!()
let (resp_tx, mut resp_rx) = mpsc::channel(1);
task.resp_tx = Some(resp_tx);
let task_str = format!("{:?}", task);
if GLOBAL_BackgroundHealRoutine.read().await.tasks_tx.try_send(task).is_ok() {
info!("Task in the queue: {:?}", task_str);
}
let count_ok_drives = |drivers: &[HealDriveInfo]| {
let mut count = 0;
for drive in drivers.iter() {
if drive.state == DRIVE_STATE_OK {
count += 1;
}
}
count
};
match resp_rx.recv().await {
Some(mut res) => {
if res.err.is_none() {
self.count_healed(heal_type.clone());
} else {
self.count_failed(heal_type.clone());
}
if !self.report_progress {
if let Some(err) = res.err {
if err.to_string() == ERR_SKIP_FILE {
return Ok(());
}
return Err(err);
} else {
return Ok(());
}
}
res.result.heal_item_type = heal_type.clone();
if let Some(err) = res.err.as_ref() {
res.result.detail = err.to_string();
}
if res.result.parity_blocks > 0 && res.result.data_blocks > 0 && res.result.data_blocks > res.result.parity_blocks
{
let got = count_ok_drives(&res.result.after);
if got < res.result.parity_blocks {
res.result.detail = format!(
"quorum loss - expected {} minimum, got drive states in OK {}",
res.result.parity_blocks, got
);
}
}
self.push_heal_result_item(&res.result).await
}
None => Ok(()),
}
}
async fn traverse_and_heal(&self) {
async fn heal_disk_meta(h: Arc<RwLock<HealSequence>>) -> Result<()> {
HealSequence::heal_rustfs_sys_meta(h, CONFIG_PREFIX).await
}
async fn heal_items(h: Arc<RwLock<HealSequence>>, buckets_only: bool) -> Result<()> {
if h.read().await.client_token == *BG_HEALING_UUID {
return Ok(());
}
Self::heal_disk_meta(h.clone()).await?;
let bucket = h.read().await.bucket.clone();
Self::heal_bucket(h.clone(), &bucket, buckets_only).await
}
async fn traverse_and_heal(h: Arc<RwLock<HealSequence>>) {
let buckets_only = false;
let result = match Self::heal_items(h.clone(), buckets_only).await {
Ok(_) => None,
Err(err) => Some(err),
};
let _ = h.read().await.traverse_and_heal_done_tx.read().await.send(result).await;
}
fn heal_rustfs_sys_meta(&self, meta_prefix: String) -> Result<()> {
todo!()
async fn heal_rustfs_sys_meta(h: Arc<RwLock<HealSequence>>, meta_prefix: &str) -> Result<()> {
let layer = new_object_layer_fn();
let lock = layer.read().await;
let store = match lock.as_ref() {
Some(s) => s,
None => return Err(Error::from(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))),
};
let setting = h.read().await.setting;
store
.heal_objects(RUSTFS_META_BUCKET, meta_prefix, &setting, h.clone(), true)
.await
}
async fn is_done(&self) -> bool {
@@ -286,13 +406,101 @@ impl HealSequence {
}
false
}
pub async fn heal_bucket(hs: Arc<RwLock<HealSequence>>, bucket: &str, bucket_only: bool) -> Result<()> {
let (object, setting) = {
let mut hs_w = hs.write().await;
hs_w.queue_heal_task(
HealSource {
bucket: bucket.to_string(),
..Default::default()
},
HEAL_ITEM_BUCKET.to_string(),
)
.await?;
if bucket_only {
return Ok(());
}
if !hs_w.setting.recursive {
if !hs_w.object.is_empty() {
HealSequence::heal_object(hs.clone(), bucket, &hs_w.object, "", hs_w.setting.scan_mode).await?;
}
return Ok(());
}
(hs_w.object.clone(), hs_w.setting)
};
let layer = new_object_layer_fn();
let lock = layer.read().await;
let store = match lock.as_ref() {
Some(s) => s,
None => return Err(Error::from(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))),
};
store.heal_objects(bucket, &object, &setting, hs.clone(), false).await
}
pub async fn heal_object(
hs: Arc<RwLock<HealSequence>>,
bucket: &str,
object: &str,
version_id: &str,
_scan_mode: HealScanMode,
) -> Result<()> {
let mut hs_w = hs.write().await;
if hs_w.is_quitting().await {
return Err(Error::from_string(ERR_HEAL_STOP_SIGNALLED));
}
let setting = hs_w.setting;
hs_w.queue_heal_task(
HealSource {
bucket: bucket.to_string(),
object: object.to_string(),
version_id: version_id.to_string(),
opts: Some(setting),
..Default::default()
},
HEAL_ITEM_OBJECT.to_string(),
)
.await?;
Ok(())
}
pub async fn heal_meta_object(
hs: Arc<RwLock<HealSequence>>,
bucket: &str,
object: &str,
version_id: &str,
_scan_mode: HealScanMode,
) -> Result<()> {
let mut hs_w = hs.write().await;
if hs_w.is_quitting().await {
return Err(Error::from_string(ERR_HEAL_STOP_SIGNALLED));
}
hs_w.queue_heal_task(
HealSource {
bucket: bucket.to_string(),
object: object.to_string(),
version_id: version_id.to_string(),
..Default::default()
},
HEAL_ITEM_BUCKET_METADATA.to_string(),
)
.await?;
Ok(())
}
}
pub async fn heal_sequence_start(h: Arc<HealSequence>) {
pub async fn heal_sequence_start(h: Arc<RwLock<HealSequence>>) {
let r = h.read().await;
{
let mut current_status_w = h.current_status.write().await;
(*current_status_w).summary = HEAL_RUNNING_STATUS.to_string();
(*current_status_w).start_time = SystemTime::now()
let mut current_status_w = r.current_status.write().await;
current_status_w.summary = HEAL_RUNNING_STATUS.to_string();
current_status_w.start_time = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("Time went backwards")
.as_secs();
@@ -300,42 +508,35 @@ pub async fn heal_sequence_start(h: Arc<HealSequence>) {
let h_clone = h.clone();
spawn(async move {
h_clone.traverse_and_heal().await;
HealSequence::traverse_and_heal(h_clone).await;
});
let h_clone_1 = h.clone();
let mut x = h.traverse_and_heal_done_rx.write().await;
let mut x = r.traverse_and_heal_done_rx.write().await;
select! {
_ = h.is_done() => {
*(h.end_time.write().await) = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("Time went backwards")
.as_secs();
let mut current_status_w = h.current_status.write().await;
(*current_status_w).summary = HEAL_FINISHED_STATUS.to_string();
_ = r.is_done() => {
*(r.end_time.write().await) = SystemTime::now();
let mut current_status_w = r.current_status.write().await;
current_status_w.summary = HEAL_FINISHED_STATUS.to_string();
spawn(async move {
let mut rx_w = h_clone_1.traverse_and_heal_done_rx.write().await;
rx_w.recv().await;
spawn(async move {
let binding = h_clone_1.read().await;
let mut rx_w = binding.traverse_and_heal_done_rx.write().await;
rx_w.recv().await;
});
}
result = x.recv() => {
match result {
Some(err) => {
match err {
Some(err) => {
let mut current_status_w = h.current_status.write().await;
(current_status_w).summary = HEAL_STOPPED_STATUS.to_string();
(current_status_w).failure_detail = err.to_string();
},
None => {
let mut current_status_w = h.current_status.write().await;
(current_status_w).summary = HEAL_FINISHED_STATUS.to_string();
}
if let Some(err) = result {
match err {
Some(err) => {
let mut current_status_w = r.current_status.write().await;
(current_status_w).summary = HEAL_STOPPED_STATUS.to_string();
(current_status_w).failure_detail = err.to_string();
},
None => {
let mut current_status_w = r.current_status.write().await;
(current_status_w).summary = HEAL_FINISHED_STATUS.to_string();
}
},
None => {
}
}
@@ -347,22 +548,37 @@ pub async fn heal_sequence_start(h: Arc<HealSequence>) {
pub struct AllHealState {
mu: RwLock<bool>,
heal_seq_map: HashMap<String, HealSequence>,
heal_seq_map: HashMap<String, Arc<RwLock<HealSequence>>>,
heal_local_disks: HashMap<Endpoint, bool>,
heal_status: HashMap<String, HealingTracker>,
}
impl AllHealState {
pub fn new(cleanup: bool) -> Self {
let hstate = AllHealState::default();
pub fn new(cleanup: bool) -> Arc<RwLock<Self>> {
let hstate = Arc::new(RwLock::new(AllHealState::default()));
let (_, mut rx) = broadcast::channel(1);
if cleanup {
// spawn(f);
let hstate_clone = hstate.clone();
tokio::spawn(async move {
loop {
select! {
result = rx.recv() =>{
if let Ok(true) = result {
return;
}
}
_ = sleep(Duration::from_secs(5 * 60)) => {
hstate_clone.write().await.periodic_heal_seqs_clean().await;
}
}
}
});
}
hstate
}
async fn pop_heal_local_disks(&mut self, heal_local_disks: &[Endpoint]) {
pub async fn pop_heal_local_disks(&mut self, heal_local_disks: &[Endpoint]) {
let _ = self.mu.write().await;
self.heal_local_disks.retain(|k, _| {
@@ -382,14 +598,14 @@ impl AllHealState {
});
}
async fn update_heal_status(&mut self, tracker: &HealingTracker) {
pub async fn update_heal_status(&mut self, tracker: &HealingTracker) {
let _ = self.mu.write().await;
let _ = tracker.mu.read().await;
self.heal_status.insert(tracker.id.clone(), tracker.clone());
}
async fn get_local_healing_disks(&self) -> HashMap<String, HealingDisk> {
pub async fn get_local_healing_disks(&self) -> HashMap<String, HealingDisk> {
let _ = self.mu.read().await;
let mut dst = HashMap::new();
@@ -400,7 +616,7 @@ impl AllHealState {
dst
}
async fn get_heal_local_disk_endpoints(&self) -> Endpoints {
pub async fn get_heal_local_disk_endpoints(&self) -> Endpoints {
let _ = self.mu.read().await;
let mut endpoints = Vec::new();
@@ -413,13 +629,13 @@ impl AllHealState {
Endpoints::from(endpoints)
}
async fn set_disk_healing_status(&mut self, ep: Endpoint, healing: bool) {
pub async fn set_disk_healing_status(&mut self, ep: Endpoint, healing: bool) {
let _ = self.mu.write().await;
self.heal_local_disks.insert(ep, healing);
}
async fn push_heal_local_disks(&mut self, heal_local_disks: &[Endpoint]) {
pub async fn push_heal_local_disks(&mut self, heal_local_disks: &[Endpoint]) {
let _ = self.mu.write().await;
heal_local_disks.iter().for_each(|heal_local_disk| {
@@ -427,37 +643,28 @@ impl AllHealState {
});
}
async fn periodic_heal_seqs_clean(&mut self, mut rx: Receiver<bool>) {
loop {
select! {
result = rx.recv() =>{
if let Ok(true) = result {
return;
}
}
_ = sleep(Duration::from_secs(5 * 60)) => {
let _ = self.mu.write().await;
let now = SystemTime::now();
pub async fn periodic_heal_seqs_clean(&mut self) {
let _ = self.mu.write().await;
let now = SystemTime::now();
let mut keys_to_reomve = Vec::new();
for (k, v) in self.heal_seq_map.iter() {
if v.has_ended().await && (UNIX_EPOCH + Duration::from_secs(*(v.end_time.read().await)) + KEEP_HEAL_SEQ_STATE_DURATION) < now {
keys_to_reomve.push(k.clone())
}
}
for key in keys_to_reomve.iter() {
self.heal_seq_map.remove(key);
}
}
let mut keys_to_reomve = Vec::new();
for (k, v) in self.heal_seq_map.iter() {
let r = v.read().await;
if r.has_ended().await && now.duration_since(*(r.end_time.read().await)).unwrap() > KEEP_HEAL_SEQ_STATE_DURATION {
keys_to_reomve.push(k.clone())
}
}
for key in keys_to_reomve.iter() {
self.heal_seq_map.remove(key);
}
}
async fn get_heal_sequence_by_token(&self, token: &str) -> (Option<HealSequence>, bool) {
pub async fn get_heal_sequence_by_token(&self, token: &str) -> (Option<Arc<RwLock<HealSequence>>>, bool) {
let _ = self.mu.read().await;
for v in self.heal_seq_map.values() {
if v.client_token == token {
let r = v.read().await;
if r.client_token == token {
return (Some(v.clone()), true);
}
}
@@ -465,15 +672,16 @@ impl AllHealState {
(None, false)
}
async fn get_heal_sequence(&self, path: &str) -> Option<HealSequence> {
pub async fn get_heal_sequence(&self, path: &str) -> Option<Arc<RwLock<HealSequence>>> {
let _ = self.mu.read().await;
self.heal_seq_map.get(path).cloned()
}
async fn stop_heal_sequence(&mut self, path: &str) -> Result<Vec<u8>> {
pub async fn stop_heal_sequence(&mut self, path: &str) -> Result<Vec<u8>> {
let mut hsp = HealStopSuccess::default();
if let Some(he) = self.get_heal_sequence(path).await {
let he = he.read().await;
let client_token = he.client_token.clone();
if *GLOBAL_IsDistErasure.read().await {
// TODO: proxy
@@ -513,23 +721,22 @@ impl AllHealState {
// `keepHealSeqStateDuration`. This function also launches a
// background routine to clean up heal results after the
// aforementioned duration.
pub async fn launch_new_heal_sequence(&mut self, heal_sequence: &HealSequence) -> Result<Vec<u8>> {
let path = Path::new(&heal_sequence.bucket).join(heal_sequence.object.clone());
pub async fn launch_new_heal_sequence(&mut self, heal_sequence: Arc<RwLock<HealSequence>>) -> Result<Vec<u8>> {
let r = heal_sequence.read().await;
let path = Path::new(&r.bucket).join(r.object.clone());
let path_s = path.to_str().unwrap();
if heal_sequence.force_started {
if r.force_started {
self.stop_heal_sequence(path_s).await?;
} else {
if let Some(hs) = self.get_heal_sequence(path_s).await {
if !hs.has_ended().await {
return Err(Error::from_string(format!("Heal is already running on the given path (use force-start option to stop and start afresh). The heal was started by IP {} at {}, token is {}", heal_sequence.client_address, heal_sequence.start_time, heal_sequence.client_token)));
}
} else if let Some(hs) = self.get_heal_sequence(path_s).await {
if !hs.read().await.has_ended().await {
return Err(Error::from_string(format!("Heal is already running on the given path (use force-start option to stop and start afresh). The heal was started by IP {} at {:?}, token is {}", r.client_address, r.start_time, r.client_token)));
}
}
let _ = self.mu.write().await;
for (k, v) in self.heal_seq_map.iter() {
if !v.has_ended().await && (has_profix(k, path_s) || has_profix(path_s, k)) {
if !v.read().await.has_ended().await && (has_profix(k, path_s) || has_profix(path_s, k)) {
return Err(Error::from_string(format!(
"The provided heal sequence path overlaps with an existing heal path: {}",
k
@@ -539,14 +746,25 @@ impl AllHealState {
self.heal_seq_map.insert(path_s.to_string(), heal_sequence.clone());
let client_token = heal_sequence.client_token.clone();
let client_token = r.client_token.clone();
if *GLOBAL_IsDistErasure.read().await {
// TODO: proxy
}
if heal_sequence.client_token == BG_HEALING_UUID {
if r.client_token == BG_HEALING_UUID {
// For background heal do nothing, do not spawn an unnecessary goroutine.
} else {
let heal_sequence_clone = heal_sequence.clone();
tokio::spawn(async {
heal_sequence_start(heal_sequence_clone).await;
});
}
todo!()
let b = serde_json::to_vec(&HealStartSuccess {
client_token,
client_address: r.client_address.clone(),
start_time: r.start_time,
})?;
Ok(b)
}
}
ecstore/src/heal/mod.rs
+5
View File
@@ -1,3 +1,8 @@
pub mod background_heal_ops;
pub mod data_scanner;
pub mod data_scanner_metric;
pub mod data_usage;
pub mod data_usage_cache;
pub mod error;
pub mod heal_commands;
pub mod heal_ops;