gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-07 22:03:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

merge heal

Browse Source
This commit is contained in:
weisd
2024-11-20 17:40:11 +08:00
parent 706fce49b9 1d81a9e40f
commit 821bb6fe3f
64 changed files with 5332 additions and 771 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/grpc.rs
+125 -8
View File
@@ -6,6 +6,7 @@ use ecstore::{
UpdateMetadataOpts, WalkDirOptions,
},
erasure::Writer,
heal::{data_usage_cache::DataUsageCache, heal_commands::HealOpts},
peer::{LocalPeerS3Client, PeerS3Client},
store::{all_local_disk_path, find_local_disk},
store_api::{BucketOptions, DeleteBucketOptions, FileInfo, MakeBucketOptions},
@@ -20,14 +21,14 @@ use protos::{
DeleteBucketResponse, DeletePathsRequest, DeletePathsResponse, DeleteRequest, DeleteResponse, DeleteVersionRequest,
DeleteVersionResponse, DeleteVersionsRequest, DeleteVersionsResponse, DeleteVolumeRequest, DeleteVolumeResponse,
DiskInfoRequest, DiskInfoResponse, GenerallyLockRequest, GenerallyLockResponse, GetBucketInfoRequest,
GetBucketInfoResponse, ListBucketRequest, ListBucketResponse, ListDirRequest, ListDirResponse, ListVolumesRequest,
ListVolumesResponse, MakeBucketRequest, MakeBucketResponse, MakeVolumeRequest, MakeVolumeResponse, MakeVolumesRequest,
MakeVolumesResponse, PingRequest, PingResponse, ReadAllRequest, ReadAllResponse, ReadAtRequest, ReadAtResponse,
ReadMultipleRequest, ReadMultipleResponse, ReadVersionRequest, ReadVersionResponse, ReadXlRequest, ReadXlResponse,
RenameDataRequest, RenameDataResponse, RenameFileRequst, RenameFileResponse, RenamePartRequst, RenamePartResponse,
StatVolumeRequest, StatVolumeResponse, UpdateMetadataRequest, UpdateMetadataResponse, VerifyFileRequest,
VerifyFileResponse, WalkDirRequest, WalkDirResponse, WriteAllRequest, WriteAllResponse, WriteMetadataRequest,
WriteMetadataResponse, WriteRequest, WriteResponse,
GetBucketInfoResponse, HealBucketRequest, HealBucketResponse, ListBucketRequest, ListBucketResponse, ListDirRequest,
ListDirResponse, ListVolumesRequest, ListVolumesResponse, MakeBucketRequest, MakeBucketResponse, MakeVolumeRequest,
MakeVolumeResponse, MakeVolumesRequest, MakeVolumesResponse, NsScannerRequest, NsScannerResponse, PingRequest,
PingResponse, ReadAllRequest, ReadAllResponse, ReadAtRequest, ReadAtResponse, ReadMultipleRequest, ReadMultipleResponse,
ReadVersionRequest, ReadVersionResponse, ReadXlRequest, ReadXlResponse, RenameDataRequest, RenameDataResponse,
RenameFileRequst, RenameFileResponse, RenamePartRequst, RenamePartResponse, StatVolumeRequest, StatVolumeResponse,
UpdateMetadataRequest, UpdateMetadataResponse, VerifyFileRequest, VerifyFileResponse, WalkDirRequest, WalkDirResponse,
WriteAllRequest, WriteAllResponse, WriteMetadataRequest, WriteMetadataResponse, WriteRequest, WriteResponse,
},
};
use tokio::sync::mpsc;
@@ -109,6 +110,32 @@ impl Node for NodeService {
}))
}
async fn heal_bucket(&self, request: Request<HealBucketRequest>) -> Result<Response<HealBucketResponse>, Status> {
debug!("heal bucket");
let request = request.into_inner();
let options = match serde_json::from_str::<HealOpts>(&request.options) {
Ok(options) => options,
Err(err) => {
return Ok(tonic::Response::new(HealBucketResponse {
success: false,
error_info: Some(format!("decode HealOpts failed: {}", err)),
}))
}
};
match self.local_peer.heal_bucket(&request.bucket, &options).await {
Ok(_) => Ok(tonic::Response::new(HealBucketResponse {
success: true,
error_info: None,
})),
Err(err) => Ok(tonic::Response::new(HealBucketResponse {
success: false,
error_info: Some(format!("heal bucket failed: {}", err)),
})),
}
}
async fn list_bucket(&self, request: Request<ListBucketRequest>) -> Result<Response<ListBucketResponse>, Status> {
debug!("list bucket");
@@ -1270,6 +1297,96 @@ impl Node for NodeService {
}
}
type NsScannerStream = ResponseStream<NsScannerResponse>;
async fn ns_scanner(&self, request: Request<Streaming<NsScannerRequest>>) -> Result<Response<Self::NsScannerStream>, Status> {
info!("ns_scanner");
let mut in_stream = request.into_inner();
let (tx, rx) = mpsc::channel(10);
tokio::spawn(async move {
match in_stream.next().await {
Some(Ok(request)) => {
if let Some(disk) = find_local_disk(&request.disk).await {
let cache = match serde_json::from_str::<DataUsageCache>(&request.cache) {
Ok(cache) => cache,
Err(_) => {
tx.send(Ok(NsScannerResponse {
success: false,
update: "".to_string(),
data_usage_cache: "".to_string(),
error_info: Some("can not decode DataUsageCache".to_string()),
}))
.await
.expect("working rx");
return;
}
};
let (updates_tx, mut updates_rx) = mpsc::channel(100);
let tx_clone = tx.clone();
let task = tokio::spawn(async move {
loop {
match updates_rx.recv().await {
Some(update) => {
let update = serde_json::to_string(&update).expect("encode failed");
tx_clone
.send(Ok(NsScannerResponse {
success: true,
update,
data_usage_cache: "".to_string(),
error_info: Some("can not decode DataUsageCache".to_string()),
}))
.await
.expect("working rx");
}
None => return,
}
}
});
let data_usage_cache = disk.ns_scanner(&cache, updates_tx, request.scan_mode as usize).await;
let _ = task.await;
match data_usage_cache {
Ok(data_usage_cache) => {
let data_usage_cache = serde_json::to_string(&data_usage_cache).expect("encode failed");
tx.send(Ok(NsScannerResponse {
success: true,
update: "".to_string(),
data_usage_cache,
error_info: Some("can not decode DataUsageCache".to_string()),
}))
.await
.expect("working rx");
}
Err(_) => {
tx.send(Ok(NsScannerResponse {
success: false,
update: "".to_string(),
data_usage_cache: "".to_string(),
error_info: Some("scanner failed".to_string()),
}))
.await
.expect("working rx");
}
}
} else {
tx.send(Ok(NsScannerResponse {
success: false,
update: "".to_string(),
data_usage_cache: "".to_string(),
error_info: Some("can not find disk".to_string()),
}))
.await
.expect("working rx");
}
}
_ => todo!(),
}
});
let out_stream = ReceiverStream::new(rx);
Ok(tonic::Response::new(Box::pin(out_stream)))
}
async fn lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
let request = request.into_inner();
match &serde_json::from_str::<LockArgs>(&request.args) {
rustfs/src/main.rs
+3
View File
@@ -7,6 +7,7 @@ use clap::Parser;
use common::error::{Error, Result};
use ecstore::{
endpoints::EndpointServerPools,
heal::data_scanner::init_data_scanner,
set_global_endpoints,
store::{init_local_disks, ECStore},
update_erasure_type,
@@ -184,6 +185,8 @@ async fn run(opt: config::Opt) -> Result<()> {
.map_err(|err| Error::from_string(err.to_string()))?;
store.init().await.map_err(|err| Error::from_string(err.to_string()))?;
// init scanner
init_data_scanner().await;
tokio::select! {
_ = tokio::signal::ctrl_c() => {
rustfs/src/storage/acess.rs
+2 -2
View File
@@ -10,7 +10,7 @@ use s3s::auth::Credentials;
use s3s::{dto::*, s3_error, S3Request, S3Result};
use uuid::Uuid;
#[derive(Debug, Default, Clone)]
#[derive(Default, Clone)]
struct ReqInfo {
pub card: Option<Credentials>,
pub action: Option<Action>,
@@ -51,7 +51,7 @@ impl S3Access for FS {
let req_info = ReqInfo {
card: cx.credentials().cloned(),
action: action,
action,
..Default::default()
};
rustfs/src/storage/ecfs.rs
+10 -17
View File
@@ -341,15 +341,14 @@ impl S3 for FS {
let content_type = {
if let Some(content_type) = info.content_type {
let ct = match ContentType::from_str(&content_type) {
match ContentType::from_str(&content_type) {
Ok(res) => Some(res),
Err(err) => {
error!("parse content-type err {} {:?}", &content_type, err);
//
None
}
};
ct
}
} else {
None
}
@@ -411,15 +410,14 @@ impl S3 for FS {
let content_type = {
if let Some(content_type) = info.content_type {
let ct = match ContentType::from_str(&content_type) {
match ContentType::from_str(&content_type) {
Ok(res) => Some(res),
Err(err) => {
error!("parse content-type err {} {:?}", &content_type, err);
//
None
}
};
ct
}
} else {
None
}
@@ -1536,9 +1534,7 @@ impl S3 for FS {
}
}
let mut grants = Vec::new();
grants.push(Grant {
let grants = vec![Grant {
grantee: Some(Grantee {
type_: Type::from_static(Type::CANONICAL_USER),
display_name: None,
@@ -1547,12 +1543,11 @@ impl S3 for FS {
uri: None,
}),
permission: Some(Permission::from_static(Permission::FULL_CONTROL)),
});
}];
Ok(S3Response::new(GetBucketAclOutput {
grants: Some(grants),
owner: Some(RUSTFS_OWNER.to_owned()),
..Default::default()
}))
}
@@ -1589,7 +1584,7 @@ impl S3 for FS {
v.grants.is_some_and(|gs| {
//
!gs.is_empty()
&& gs.get(0).is_some_and(|g| {
&& gs.first().is_some_and(|g| {
g.to_owned()
.permission
.is_some_and(|p| p.as_str() == Permission::FULL_CONTROL)
@@ -1617,9 +1612,7 @@ impl S3 for FS {
return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{}", e)));
}
let mut grants = Vec::new();
grants.push(Grant {
let grants = vec![Grant {
grantee: Some(Grantee {
type_: Type::from_static(Type::CANONICAL_USER),
display_name: None,
@@ -1628,7 +1621,7 @@ impl S3 for FS {
uri: None,
}),
permission: Some(Permission::from_static(Permission::FULL_CONTROL)),
});
}];
Ok(S3Response::new(GetObjectAclOutput {
grants: Some(grants),
@@ -1665,7 +1658,7 @@ impl S3 for FS {
v.grants.is_some_and(|gs| {
//
!gs.is_empty()
&& gs.get(0).is_some_and(|g| {
&& gs.first().is_some_and(|g| {
g.to_owned()
.permission
.is_some_and(|p| p.as_str() == Permission::FULL_CONTROL)