gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-25 21:46:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

auto heal(1)

Browse Source 
Signed-off-by: root <root@DESKTOP-QLJNS6S.localdomain>
This commit is contained in:
root
2024-11-16 20:43:34 +08:00
parent 7eed1a6db3
commit 01e0e4b673
20 changed files with 988 additions and 1791 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ecstore/src/heal/background_heal_ops.rs
+92 -64
View File
@@ -1,17 +1,14 @@
use std::{env, sync::Arc};
use tokio::{
select,
sync::{
broadcast::Receiver as B_Receiver,
mpsc::{self, Receiver, Sender},
RwLock,
},
use tokio::sync::{
mpsc::{self, Receiver, Sender},
RwLock,
};
use crate::{
disk::error::DiskError,
endpoints::Endpoints,
error::{Error, Result},
global::{GLOBAL_BackgroundHealRoutine, GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP},
heal::heal_ops::NOP_HEAL,
new_object_layer_fn,
store_api::StorageAPI,
@@ -20,9 +17,38 @@ use crate::{
use super::{
heal_commands::{HealOpts, HealResultItem},
heal_ops::HealSequence,
heal_ops::{new_bg_heal_sequence, HealSequence},
};
pub async fn init_auto_heal() {
init_background_healing().await;
if let Ok(v) = env::var("_RUSTFS_AUTO_DRIVE_HEALING") {
if v == "on" {
// GLOBAL_BackgroundHealState.write().await.push_heal_local_disks(heal_local_disks).await;
}
}
}
async fn init_background_healing() {
let bg_seq = Arc::new(RwLock::new(new_bg_heal_sequence()));
for _ in 0..GLOBAL_BackgroundHealRoutine.read().await.workers {
let bg_seq_clone = bg_seq.clone();
tokio::spawn(async {
GLOBAL_BackgroundHealRoutine.write().await.add_worker(bg_seq_clone).await;
});
}
let _ = GLOBAL_BackgroundHealState
.write()
.await
.launch_new_heal_sequence(bg_seq)
.await;
}
async fn get_local_disks_to_heal() -> Endpoints {
for (_, disk) in GLOBAL_LOCAL_DISK_MAP.read().await.iter() {}
todo!()
}
#[derive(Clone, Debug)]
pub struct HealTask {
pub bucket: String,
@@ -49,7 +75,7 @@ impl HealTask {
pub struct HealResult {
pub result: HealResultItem,
err: Option<Error>,
_err: Option<Error>,
}
pub struct HealRoutine {
@@ -79,66 +105,68 @@ impl HealRoutine {
}))
}
pub async fn add_worker(&mut self, mut ctx: B_Receiver<bool>, bgseq: &mut HealSequence) {
pub async fn add_worker(&mut self, bgseq: Arc<RwLock<HealSequence>>) {
loop {
select! {
task = self.tasks_rx.recv() => {
let mut d_res = HealResultItem::default();
let d_err: Option<Error>;
match task {
Some(task) => {
if task.bucket == NOP_HEAL {
d_err = Some(Error::from_string("skip file"));
} else if task.bucket == SLASH_SEPARATOR {
match heal_disk_format(task.opts).await {
Ok((res, err)) => {
d_res = res;
d_err = err;
},
Err(err) => {d_err = Some(err)},
}
} else {
let layer = new_object_layer_fn();
let lock = layer.read().await;
let store = lock
.as_ref()
.expect("Not init");
if task.object.is_empty() {
match store.heal_object(&task.bucket, &task.object, &task.version_id, &task.opts).await {
Ok((res, err)) => {
d_res = res;
d_err = err;
},
Err(err) => {d_err = Some(err)},
}
} else {
match store.heal_object(&task.bucket, &task.object, &task.version_id, &task.opts).await {
Ok((res, err)) => {
d_res = res;
d_err = err;
},
Err(err) => {d_err = Some(err)},
}
}
let mut d_res = HealResultItem::default();
let d_err: Option<Error>;
match self.tasks_rx.recv().await {
Some(task) => {
if task.bucket == NOP_HEAL {
d_err = Some(Error::from_string("skip file"));
} else if task.bucket == SLASH_SEPARATOR {
match heal_disk_format(task.opts).await {
Ok((res, err)) => {
d_res = res;
d_err = err;
}
if let Some(resp_tx) = task.resp_tx {
let _ = resp_tx.send(HealResult{result: d_res, err: d_err}).await;
} else {
// when respCh is not set caller is not waiting but we
// update the relevant metrics for them
if d_err.is_none() {
bgseq.count_healed(d_res.heal_item_type);
} else {
bgseq.count_failed(d_res.heal_item_type);
Err(err) => d_err = Some(err),
}
} else {
let layer = new_object_layer_fn();
let lock = layer.read().await;
let store = lock.as_ref().expect("Not init");
if task.object.is_empty() {
match store
.heal_object(&task.bucket, &task.object, &task.version_id, &task.opts)
.await
{
Ok((res, err)) => {
d_res = res;
d_err = err;
}
Err(err) => d_err = Some(err),
}
},
None => return,
} else {
match store
.heal_object(&task.bucket, &task.object, &task.version_id, &task.opts)
.await
{
Ok((res, err)) => {
d_res = res;
d_err = err;
}
Err(err) => d_err = Some(err),
}
}
}
if let Some(resp_tx) = task.resp_tx {
let _ = resp_tx
.send(HealResult {
result: d_res,
_err: d_err,
})
.await;
} else {
// when respCh is not set caller is not waiting but we
// update the relevant metrics for them
if d_err.is_none() {
bgseq.write().await.count_healed(d_res.heal_item_type);
} else {
bgseq.write().await.count_failed(d_res.heal_item_type);
}
}
}
_ = ctx.recv() => {
return;
}
None => return,
}
}
}
ecstore/src/heal/data_scanner.rs
+21 -14
View File
@@ -34,7 +34,6 @@ use super::{
data_usage_cache::{DataUsageCache, DataUsageEntry, DataUsageHash},
heal_commands::{HealScanMode, HEAL_DEEP_SCAN, HEAL_NORMAL_SCAN},
};
use crate::heal::data_scanner_metric::current_path_updater;
use crate::heal::data_usage::DATA_USAGE_ROOT;
use crate::{
cache_value::metacache_set::{list_path_raw, ListPathRawOptions},
@@ -54,15 +53,16 @@ use crate::{
},
new_object_layer_fn,
peer::is_reserved_or_invalid_bucket,
store::{ECStore, ListPathOptions},
store::ECStore,
utils::path::{path_join, path_to_bucket_object, path_to_bucket_object_with_base_path, SLASH_SEPARATOR},
};
use crate::{disk::local::LocalDisk, heal::data_scanner_metric::current_path_updater};
use crate::{
disk::DiskAPI,
store_api::{FileInfo, ObjectInfo},
};
const DATA_SCANNER_SLEEP_PER_FOLDER: Duration = Duration::from_millis(1); // Time to wait between folders.
const _DATA_SCANNER_SLEEP_PER_FOLDER: Duration = Duration::from_millis(1); // Time to wait between folders.
const DATA_USAGE_UPDATE_DIR_CYCLES: u32 = 16; // Visit all folders every n cycles.
const DATA_SCANNER_COMPACT_LEAST_OBJECT: u64 = 500; // Compact when there are less than this many objects in a branch.
const DATA_SCANNER_COMPACT_AT_CHILDREN: u64 = 10000; // Compact when there are this many children in a branch.
@@ -70,12 +70,12 @@ const DATA_SCANNER_COMPACT_AT_FOLDERS: u64 = DATA_SCANNER_COMPACT_AT_CHILDREN /
pub const DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS: u64 = 250_000; // Compact when this many subfolders in a single folder (even top level).
const DATA_SCANNER_START_DELAY: Duration = Duration::from_secs(60); // Time to wait on startup and between cycles.
const HEAL_DELETE_DANGLING: bool = true;
pub const HEAL_DELETE_DANGLING: bool = true;
const HEAL_OBJECT_SELECT_PROB: u64 = 1024; // Overall probability of a file being scanned; one in n.
// static SCANNER_SLEEPER: () = new_dynamic_sleeper(2, Duration::from_secs(1), true); // Keep defaults same as config defaults
static SCANNER_CYCLE: AtomicU64 = AtomicU64::new(DATA_SCANNER_START_DELAY.as_secs());
static SCANNER_IDLE_MODE: AtomicU32 = AtomicU32::new(0); // default is throttled when idle
static _SCANNER_IDLE_MODE: AtomicU32 = AtomicU32::new(0); // default is throttled when idle
static SCANNER_EXCESS_OBJECT_VERSIONS: AtomicU64 = AtomicU64::new(100);
static SCANNER_EXCESS_OBJECT_VERSIONS_TOTAL_SIZE: AtomicU64 = AtomicU64::new(1024 * 1024 * 1024 * 1024); // 1 TB
static SCANNER_EXCESS_FOLDERS: AtomicU64 = AtomicU64::new(50_000);
@@ -167,16 +167,18 @@ async fn run_data_scanner() {
cycle_info.current = 0;
cycle_info.cycle_completed.push(SystemTime::now());
if cycle_info.cycle_completed.len() > DATA_USAGE_UPDATE_DIR_CYCLES as usize {
cycle_info.cycle_completed = cycle_info.cycle_completed[cycle_info.cycle_completed.len() - DATA_USAGE_UPDATE_DIR_CYCLES as usize..].to_vec();
cycle_info.cycle_completed = cycle_info.cycle_completed
[cycle_info.cycle_completed.len() - DATA_USAGE_UPDATE_DIR_CYCLES as usize..]
.to_vec();
}
globalScannerMetrics.write().await.set_cycle(Some(cycle_info.clone())).await;
let mut tmp = Vec::new();
tmp.write_u64::<LittleEndian>(cycle_info.next).unwrap();
let _ = save_config(store, &DATA_USAGE_BLOOM_NAME_PATH, &tmp).await;
},
}
Err(err) => {
res.insert("error".to_string(), err.to_string());
},
}
}
stop_fn(&res).await;
sleep(Duration::from_secs(SCANNER_CYCLE.load(std::sync::atomic::Ordering::SeqCst))).await;
@@ -265,7 +267,7 @@ impl Default for CurrentScannerCycle {
}
impl CurrentScannerCycle {
pub fn marshal_msg(&self, buf: &[u8]) -> Result<Vec<u8>> {
pub fn marshal_msg(&self, next_buf: &[u8]) -> Result<Vec<u8>> {
let len: u32 = 4;
let mut wr = Vec::new();
@@ -291,7 +293,7 @@ impl CurrentScannerCycle {
.serialize(&mut Serializer::new(&mut buf))
.expect("Serialization failed");
rmp::encode::write_bin(&mut wr, &buf)?;
let mut result = buf.to_vec();
let mut result = next_buf.to_vec();
result.extend(wr.iter());
Ok(result)
}
@@ -355,7 +357,7 @@ fn timestamp_to_system_time(timestamp: u64) -> SystemTime {
}
#[derive(Clone, Debug, Default)]
struct Heal {
pub struct Heal {
enabled: bool,
bitrot: bool,
}
@@ -403,7 +405,12 @@ impl ScannerItem {
cumulative_size += obj_info.size;
}
if cumulative_size >= SCANNER_EXCESS_OBJECT_VERSIONS_TOTAL_SIZE.load(Ordering::SeqCst).try_into().unwrap() {
if cumulative_size
>= SCANNER_EXCESS_OBJECT_VERSIONS_TOTAL_SIZE
.load(Ordering::SeqCst)
.try_into()
.unwrap()
{
//todo
}
@@ -421,7 +428,7 @@ impl ScannerItem {
Ok(object_infos)
}
pub async fn apply_actions(&self, oi: &ObjectInfo, size_s: &SizeSummary) -> (bool, usize) {
pub async fn apply_actions(&self, _oi: &ObjectInfo, _size_s: &SizeSummary) -> (bool, usize) {
let done = ScannerMetrics::time(ScannerMetric::Ilm);
//todo: lifecycle
done().await;
@@ -1014,7 +1021,7 @@ pub fn has_active_rules(config: &ReplicationConfiguration, prefix: &str, recursi
false
}
pub type LocalDrive = Arc<dyn DiskAPI>;
pub type LocalDrive = Arc<LocalDisk>;
pub async fn scan_data_folder(
disks: &[Option<DiskStore>],
drive: LocalDrive,
ecstore/src/heal/data_scanner_metric.rs
+11 -11
View File
@@ -79,8 +79,8 @@ impl Clone for LockedLastMinuteLatency {
}
impl LockedLastMinuteLatency {
pub fn add(&mut self, value: &Duration) {
self.add_size(value, 0);
pub async fn add(&mut self, value: &Duration) {
self.add_size(value, 0).await;
}
pub async fn add_size(&mut self, value: &Duration, sz: u64) {
@@ -105,16 +105,16 @@ impl LockedLastMinuteLatency {
a.size = old.size;
a.total = old.total;
a.n = old.n;
self.mu.write().await;
let _ = self.mu.write().await;
self.latency.add_all(t - 1, &a);
}
self.cached.n += 1;
self.cached.total += value.as_secs();
self.cached.size != sz;
self.cached.size += sz;
}
pub async fn total(&mut self) -> AccElem {
self.mu.read().await;
let _ = self.mu.read().await;
self.latency.get_total()
}
}
@@ -147,19 +147,19 @@ impl ScannerMetrics {
pub fn log(s: ScannerMetric) -> LogFn {
let start = SystemTime::now();
let s_clone = s as usize;
Arc::new(move |custom: &HashMap<String, String>| {
Arc::new(move |_custom: &HashMap<String, String>| {
Box::pin(async move {
let duration = SystemTime::now().duration_since(start).unwrap_or(Duration::from_secs(0));
let mut sm_w = globalScannerMetrics.write().await;
sm_w.operations[s_clone].fetch_add(1, Ordering::SeqCst);
if s_clone < ScannerMetric::LastRealtime as usize {
sm_w.latency[s_clone].add(&duration);
sm_w.latency[s_clone].add(&duration).await;
}
})
})
}
pub fn time_size(s: ScannerMetric) -> TimeSizeFn {
pub async fn time_size(s: ScannerMetric) -> TimeSizeFn {
let start = SystemTime::now();
let s_clone = s as usize;
Arc::new(move |sz: u64| {
@@ -168,7 +168,7 @@ impl ScannerMetrics {
let mut sm_w = globalScannerMetrics.write().await;
sm_w.operations[s_clone].fetch_add(1, Ordering::SeqCst);
if s_clone < ScannerMetric::LastRealtime as usize {
sm_w.latency[s_clone].add_size(&duration, sz);
sm_w.latency[s_clone].add_size(&duration, sz).await;
}
})
})
@@ -183,7 +183,7 @@ impl ScannerMetrics {
let mut sm_w = globalScannerMetrics.write().await;
sm_w.operations[s_clone].fetch_add(1, Ordering::SeqCst);
if s_clone < ScannerMetric::LastRealtime as usize {
sm_w.latency[s_clone].add(&duration);
sm_w.latency[s_clone].add(&duration).await;
}
})
})
@@ -191,7 +191,7 @@ impl ScannerMetrics {
}
pub type CloseDiskFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn, CloseDiskFn) {
pub fn current_path_updater(disk: &str, _initial: &str) -> (UpdateCurrentPathFn, CloseDiskFn) {
let disk_1 = disk.to_string();
let disk_2 = disk.to_string();
(
ecstore/src/heal/data_usage_cache.rs
+12 -11
View File
@@ -5,7 +5,6 @@ use crate::error::{Error, Result};
use crate::new_object_layer_fn;
use crate::set_disk::SetDisks;
use crate::store_api::{BucketInfo, HTTPRangeSpec, ObjectIO, ObjectOptions};
use bytes::Bytes;
use bytesize::ByteSize;
use http::HeaderMap;
use path_clean::PathClean;
@@ -17,7 +16,6 @@ use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::hash::{DefaultHasher, Hash, Hasher};
use std::path::Path;
use std::sync::Arc;
use std::time::{Duration, SystemTime};
use std::u64;
use tokio::sync::mpsc::Sender;
@@ -761,15 +759,18 @@ impl DataUsageCache {
bui.replica_count = rs.replica_count;
for (arn, stat) in rs.targets.iter() {
bui.replication_info.insert(arn.clone(), BucketTargetUsageInfo {
replication_pending_size: stat.pending_size,
replicated_size: stat.replicated_size,
replication_failed_size: stat.failed_size,
replication_pending_count: stat.pending_count,
replication_failed_count: stat.failed_count,
replicated_count: stat.replicated_count,
..Default::default()
});
bui.replication_info.insert(
arn.clone(),
BucketTargetUsageInfo {
replication_pending_size: stat.pending_size,
replicated_size: stat.replicated_size,
replication_failed_size: stat.failed_size,
replication_pending_count: stat.pending_count,
replication_failed_count: stat.failed_count,
replicated_count: stat.replicated_count,
..Default::default()
},
);
}
}
dst.insert(bucket.name.clone(), bui);
ecstore/src/heal/heal_commands.rs
+12 -2
View File
@@ -75,11 +75,21 @@ pub struct HealResultItem {
pub object_size: usize,
}
#[derive(Debug, Default, Serialize, Deserialize)]
#[derive(Debug, Serialize, Deserialize)]
pub struct HealStartSuccess {
pub client_token: String,
pub client_address: String,
pub start_time: u64,
pub start_time: SystemTime,
}
impl Default for HealStartSuccess {
fn default() -> Self {
Self {
client_token: Default::default(),
client_address: Default::default(),
start_time: SystemTime::now(),
}
}
}
pub type HealStopSuccess = HealStartSuccess;
ecstore/src/heal/heal_ops.rs
+69 -32
View File
@@ -3,7 +3,7 @@ use crate::{
endpoints::Endpoints,
error::{Error, Result},
global::GLOBAL_IsDistErasure,
heal::heal_commands::HEAL_UNKNOWN_SCAN,
heal::heal_commands::{HealStartSuccess, HEAL_UNKNOWN_SCAN},
utils::path::has_profix,
};
use lazy_static::lazy_static;
@@ -28,6 +28,7 @@ use uuid::Uuid;
use super::{
background_heal_ops::HealTask,
data_scanner::HEAL_DELETE_DANGLING,
heal_commands::{HealItemType, HealOpts, HealResultItem, HealScanMode, HealStopSuccess, HealingDisk, HealingTracker},
};
@@ -45,6 +46,10 @@ const HEAL_RUNNING_STATUS: &str = "running";
const HEAL_STOPPED_STATUS: &str = "stopped";
const HEAL_FINISHED_STATUS: &str = "finished";
pub const RUESTFS_RESERVED_BUCKET: &str = "rustfs";
pub const RUESTFS_RESERVED_BUCKET_PATH: &str = "/rustfs";
pub const LOGIN_PATH_PREFIX: &str = "/login";
const MAX_UNCONSUMED_HEAL_RESULT_ITEMS: usize = 1000;
const HEAL_UNCONSUMED_TIMEOUT: std::time::Duration = Duration::from_secs(24 * 60 * 60);
pub const NOP_HEAL: &str = "";
@@ -74,8 +79,8 @@ pub struct HealSequence {
pub bucket: String,
pub object: String,
pub report_progress: bool,
pub start_time: u64,
pub end_time: Arc<RwLock<u64>>,
pub start_time: SystemTime,
pub end_time: Arc<RwLock<SystemTime>>,
pub client_token: String,
pub client_address: String,
pub force_started: bool,
@@ -94,6 +99,30 @@ pub struct HealSequence {
rx: Arc<RwLock<Receiver<bool>>>,
}
pub fn new_bg_heal_sequence() -> HealSequence {
let hs = HealOpts {
remove: HEAL_DELETE_DANGLING,
..Default::default()
};
HealSequence {
start_time: SystemTime::now(),
client_token: BG_HEALING_UUID.to_string(),
bucket: RUESTFS_RESERVED_BUCKET.to_string(),
setting: hs,
current_status: Arc::new(RwLock::new(HealSequenceStatus {
summary: HEAL_NOT_STARTED_STATUS.to_string(),
heal_setting: hs,
..Default::default()
})),
report_progress: false,
scanned_items_map: HashMap::new(),
healed_items_map: HashMap::new(),
heal_failed_items_map: HashMap::new(),
..Default::default()
}
}
impl Default for HealSequence {
fn default() -> Self {
let (h_tx, h_rx) = mpsc::channel(1);
@@ -102,8 +131,8 @@ impl Default for HealSequence {
bucket: Default::default(),
object: Default::default(),
report_progress: Default::default(),
start_time: Default::default(),
end_time: Default::default(),
start_time: SystemTime::now(),
end_time: Arc::new(RwLock::new(SystemTime::now())),
client_token: Default::default(),
client_address: Default::default(),
force_started: Default::default(),
@@ -289,9 +318,10 @@ impl HealSequence {
}
}
pub async fn heal_sequence_start(h: Arc<HealSequence>) {
pub async fn heal_sequence_start(h: Arc<RwLock<HealSequence>>) {
let r = h.read().await;
{
let mut current_status_w = h.current_status.write().await;
let mut current_status_w = r.current_status.write().await;
(*current_status_w).summary = HEAL_RUNNING_STATUS.to_string();
(*current_status_w).start_time = SystemTime::now()
.duration_since(UNIX_EPOCH)
@@ -301,22 +331,20 @@ pub async fn heal_sequence_start(h: Arc<HealSequence>) {
let h_clone = h.clone();
spawn(async move {
h_clone.traverse_and_heal().await;
h_clone.read().await.traverse_and_heal().await;
});
let h_clone_1 = h.clone();
let mut x = h.traverse_and_heal_done_rx.write().await;
let mut x = r.traverse_and_heal_done_rx.write().await;
select! {
_ = h.is_done() => {
*(h.end_time.write().await) = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("Time went backwards")
.as_secs();
let mut current_status_w = h.current_status.write().await;
_ = r.is_done() => {
*(r.end_time.write().await) = SystemTime::now();
let mut current_status_w = r.current_status.write().await;
(*current_status_w).summary = HEAL_FINISHED_STATUS.to_string();
spawn(async move {
let mut rx_w = h_clone_1.traverse_and_heal_done_rx.write().await;
let binding = h_clone_1.read().await;
let mut rx_w = binding.traverse_and_heal_done_rx.write().await;
rx_w.recv().await;
});
}
@@ -325,12 +353,12 @@ pub async fn heal_sequence_start(h: Arc<HealSequence>) {
Some(err) => {
match err {
Some(err) => {
let mut current_status_w = h.current_status.write().await;
let mut current_status_w = r.current_status.write().await;
(current_status_w).summary = HEAL_STOPPED_STATUS.to_string();
(current_status_w).failure_detail = err.to_string();
},
None => {
let mut current_status_w = h.current_status.write().await;
let mut current_status_w = r.current_status.write().await;
(current_status_w).summary = HEAL_FINISHED_STATUS.to_string();
}
}
@@ -429,13 +457,13 @@ impl AllHealState {
Endpoints::from(endpoints)
}
async fn set_disk_healing_status(&mut self, ep: Endpoint, healing: bool) {
pub async fn set_disk_healing_status(&mut self, ep: Endpoint, healing: bool) {
let _ = self.mu.write().await;
self.heal_local_disks.insert(ep, healing);
}
async fn push_heal_local_disks(&mut self, heal_local_disks: &[Endpoint]) {
pub async fn push_heal_local_disks(&mut self, heal_local_disks: &[Endpoint]) {
let _ = self.mu.write().await;
heal_local_disks.iter().for_each(|heal_local_disk| {
@@ -450,9 +478,7 @@ impl AllHealState {
let mut keys_to_reomve = Vec::new();
for (k, v) in self.heal_seq_map.iter() {
let r = v.read().await;
if r.has_ended().await
&& (UNIX_EPOCH + Duration::from_secs(*(r.end_time.read().await)) + KEEP_HEAL_SEQ_STATE_DURATION) < now
{
if r.has_ended().await && now.duration_since(*(r.end_time.read().await)).unwrap() > KEEP_HEAL_SEQ_STATE_DURATION {
keys_to_reomve.push(k.clone())
}
}
@@ -523,15 +549,16 @@ impl AllHealState {
// `keepHealSeqStateDuration`. This function also launches a
// background routine to clean up heal results after the
// aforementioned duration.
pub async fn launch_new_heal_sequence(&mut self, heal_sequence: &HealSequence) -> Result<Vec<u8>> {
let path = Path::new(&heal_sequence.bucket).join(heal_sequence.object.clone());
pub async fn launch_new_heal_sequence(&mut self, heal_sequence: Arc<RwLock<HealSequence>>) -> Result<Vec<u8>> {
let r = heal_sequence.read().await;
let path = Path::new(&r.bucket).join(r.object.clone());
let path_s = path.to_str().unwrap();
if heal_sequence.force_started {
if r.force_started {
self.stop_heal_sequence(path_s).await?;
} else {
if let Some(hs) = self.get_heal_sequence(path_s).await {
if !hs.read().await.has_ended().await {
return Err(Error::from_string(format!("Heal is already running on the given path (use force-start option to stop and start afresh). The heal was started by IP {} at {}, token is {}", heal_sequence.client_address, heal_sequence.start_time, heal_sequence.client_token)));
return Err(Error::from_string(format!("Heal is already running on the given path (use force-start option to stop and start afresh). The heal was started by IP {} at {:?}, token is {}", r.client_address, r.start_time, r.client_token)));
}
}
}
@@ -547,17 +574,27 @@ impl AllHealState {
}
}
self.heal_seq_map
.insert(path_s.to_string(), Arc::new(RwLock::new(heal_sequence.clone())));
self.heal_seq_map.insert(path_s.to_string(), heal_sequence.clone());
let client_token = heal_sequence.client_token.clone();
let client_token = r.client_token.clone();
if *GLOBAL_IsDistErasure.read().await {
// TODO: proxy
}
if heal_sequence.client_token == BG_HEALING_UUID {
if r.client_token == BG_HEALING_UUID {
// For background heal do nothing, do not spawn an unnecessary goroutine.
} else {
let heal_sequence_clone = heal_sequence.clone();
tokio::spawn(async {
heal_sequence_start(heal_sequence_clone).await;
});
}
todo!()
let b = serde_json::to_vec(&HealStartSuccess {
client_token,
client_address: r.client_address.clone(),
start_time: r.start_time,
})?;
Ok(b)
}
}