gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-12 08:06:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

auto heal(1)

Browse Source 
Signed-off-by: root <root@DESKTOP-QLJNS6S.localdomain>
This commit is contained in:
root
2024-11-16 20:43:34 +08:00
parent 7eed1a6db3
commit 01e0e4b673
20 changed files with 988 additions and 1791 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ecstore/src/disk/local.rs
+8 -8
View File
@@ -20,8 +20,8 @@ use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE};
use crate::error::{Error, Result};
use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold};
use crate::heal::data_scanner::{has_active_rules, scan_data_folder, ScannerItem, SizeSummary};
use crate::heal::data_scanner_metric::{globalScannerMetrics, ScannerMetric, ScannerMetrics};
use crate::heal::data_usage_cache::{self, DataUsageCache, DataUsageEntry};
use crate::heal::data_scanner_metric::{ScannerMetric, ScannerMetrics};
use crate::heal::data_usage_cache::{DataUsageCache, DataUsageEntry};
use crate::heal::error::{ERR_IGNORE_FILE_CONTRIB, ERR_SKIP_FILE};
use crate::heal::heal_commands::HealScanMode;
use crate::new_object_layer_fn;
@@ -40,7 +40,6 @@ use crate::{
};
use common::defer;
use path_absolutize::Absolutize;
use s3s::dto::{ReplicationConfiguration, ReplicationRuleStatus};
use std::collections::{HashMap, HashSet};
use std::fmt::Debug;
use std::io::Cursor;
@@ -1880,7 +1879,7 @@ impl DiskAPI for LocalDisk {
}
async fn ns_scanner(
self: Arc<Self>,
&self,
cache: &DataUsageCache,
updates: Sender<DataUsageEntry>,
scan_mode: HealScanMode,
@@ -1908,7 +1907,7 @@ impl DiskAPI for LocalDisk {
};
let loc = self.get_disk_location();
let disks = store.get_disks(loc.pool_idx.unwrap(), loc.disk_idx.unwrap()).await?;
let disk = self.clone();
let disk = Arc::new(LocalDisk::new(&self.endpoint(), false).await?);
let disk_clone = disk.clone();
let mut cache = cache.clone();
cache.info.updates = Some(updates.clone());
@@ -1925,7 +1924,7 @@ impl DiskAPI for LocalDisk {
}
let stop_fn = ScannerMetrics::log(ScannerMetric::ScanObject);
let mut res = HashMap::new();
let done_sz = ScannerMetrics::time_size(ScannerMetric::ReadMetadata);
let done_sz = ScannerMetrics::time_size(ScannerMetric::ReadMetadata).await;
let buf = match disk.read_metadata(item.path.clone()).await {
Ok(buf) => buf,
Err(err) => {
@@ -1965,7 +1964,7 @@ impl DiskAPI for LocalDisk {
let mut obj_deleted = false;
for info in obj_infos.iter() {
let done = ScannerMetrics::time(ScannerMetric::ApplyVersion);
let mut sz = 0;
let sz: usize;
(obj_deleted, sz) = item.apply_actions(&info, &size_s).await;
done().await;
@@ -1994,7 +1993,8 @@ impl DiskAPI for LocalDisk {
}
for frer_version in fivs.free_versions.iter() {
let _obj_info = frer_version.to_object_info(&item.bucket, &item.object_path().to_string_lossy(), versioned);
let _obj_info =
frer_version.to_object_info(&item.bucket, &item.object_path().to_string_lossy(), versioned);
let done = ScannerMetrics::time(ScannerMetric::TierObjSweep);
done().await;
}
ecstore/src/disk/mod.rs
+8 -8
View File
@@ -1,7 +1,7 @@
pub mod endpoint;
pub mod error;
pub mod format;
mod local;
pub mod local;
pub mod os;
pub mod remote;
@@ -16,7 +16,7 @@ const STORAGE_FORMAT_FILE: &str = "xl.meta";
use crate::{
erasure::Writer,
error::{Error, Result},
file_meta::{merge_file_meta_versions, FileMeta, FileMetaShallowVersion, FileMetaVersion},
file_meta::{merge_file_meta_versions, FileMeta, FileMetaShallowVersion},
heal::{
data_usage_cache::{DataUsageCache, DataUsageEntry},
heal_commands::HealScanMode,
@@ -344,14 +344,14 @@ impl DiskAPI for Disk {
}
async fn ns_scanner(
self: Arc<Self>,
&self,
cache: &DataUsageCache,
updates: Sender<DataUsageEntry>,
scan_mode: HealScanMode,
) -> Result<DataUsageCache> {
match &*self {
Disk::Local(local_disk) => Arc::new(local_disk).ns_scanner(cache, updates, scan_mode).await,
Disk::Remote(remote_disk) => Arc::new(remote_disk).ns_scanner(cache, updates, scan_mode).await,
Disk::Local(local_disk) => local_disk.ns_scanner(cache, updates, scan_mode).await,
Disk::Remote(remote_disk) => remote_disk.ns_scanner(cache, updates, scan_mode).await,
}
}
}
@@ -453,7 +453,7 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
async fn read_all(&self, volume: &str, path: &str) -> Result<Vec<u8>>;
async fn disk_info(&self, opts: &DiskInfoOptions) -> Result<DiskInfo>;
async fn ns_scanner(
self: Arc<Self>,
&self,
cache: &DataUsageCache,
updates: Sender<DataUsageEntry>,
scan_mode: HealScanMode,
@@ -1303,10 +1303,10 @@ impl Reader for RemoteFileReader {
Err(Error::from_string(error_info))
}
}
async fn seek(&mut self, offset: usize) -> Result<()> {
async fn seek(&mut self, _offset: usize) -> Result<()> {
unimplemented!()
}
async fn read_exact(&mut self, buf: &mut [u8]) -> Result<usize> {
async fn read_exact(&mut self, _buf: &mut [u8]) -> Result<usize> {
unimplemented!()
}
}
ecstore/src/disk/remote.rs
+39 -7
View File
@@ -1,16 +1,17 @@
use std::{path::PathBuf, sync::Arc};
use std::path::PathBuf;
use futures::lock::Mutex;
use protos::{
node_service_time_out_client,
proto_gen::node_service::{
CheckPartsRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest,
DiskInfoRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest,
ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequst, StatVolumeRequest,
UpdateMetadataRequest, VerifyFileRequest, WalkDirRequest, WriteAllRequest, WriteMetadataRequest,
DiskInfoRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, NsScannerRequest,
ReadAllRequest, ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequst,
StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WalkDirRequest, WriteAllRequest, WriteMetadataRequest,
},
};
use tokio::sync::mpsc::Sender;
use tokio::sync::mpsc::{self, Sender};
use tokio_stream::{wrappers::ReceiverStream, StreamExt};
use tonic::Request;
use tracing::info;
use uuid::Uuid;
@@ -736,11 +737,42 @@ impl DiskAPI for RemoteDisk {
}
async fn ns_scanner(
self: Arc<Self>,
&self,
cache: &DataUsageCache,
updates: Sender<DataUsageEntry>,
scan_mode: HealScanMode,
) -> Result<DataUsageCache> {
todo!()
info!("ns_scanner");
let cache = serde_json::to_string(cache)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::from_string(format!("can not get client, err: {}", err)))?;
let (tx, rx) = mpsc::channel(10);
let in_stream = ReceiverStream::new(rx);
let mut response = client.ns_scanner(in_stream).await?.into_inner();
let request = NsScannerRequest {
disk: self.root.to_string_lossy().to_string(),
cache,
scan_mode: scan_mode as u64,
};
tx.send(request).await?;
loop {
match response.next().await {
Some(Ok(resp)) => {
if !resp.update.is_empty() {
let data_usage_cache = serde_json::from_str::<DataUsageEntry>(&resp.update)?;
let _ = updates.send(data_usage_cache).await;
} else if !resp.data_usage_cache.is_empty() {
let data_usage_cache = serde_json::from_str::<DataUsageCache>(&resp.data_usage_cache)?;
return Ok(data_usage_cache);
} else {
return Err(Error::from_string("scan was interrupted"));
}
}
_ => return Err(Error::from_string("scan was interrupted")),
}
}
}
}