From e4f5ca7ea85058249639eeb095ebdb2a8a90a098 Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Tue, 15 Oct 2024 10:29:05 +0800 Subject: [PATCH 01/12] tmp --- Cargo.lock | 7 + .../generated/flatbuffers_generated/models.rs | 207 +- .../src/generated/proto_gen/node_service.rs | 1825 +++++++++++++---- common/protos/src/node.proto | 12 + ecstore/Cargo.toml | 3 +- ecstore/src/bucket/metadata.rs | 11 + ecstore/src/cache_value/cache.rs | 88 + ecstore/src/cache_value/mod.rs | 1 + ecstore/src/disk/local.rs | 15 +- ecstore/src/disk/mod.rs | 62 +- ecstore/src/disk/remote.rs | 36 +- ecstore/src/heal/heal_commands.rs | 39 + ecstore/src/heal/heal_ops.rs | 178 ++ ecstore/src/heal/mod.rs | 2 + ecstore/src/lib.rs | 2 + ecstore/src/store.rs | 136 ++ ecstore/src/store_api.rs | 13 +- rustfs/src/grpc.rs | 41 + 18 files changed, 2204 insertions(+), 474 deletions(-) create mode 100644 ecstore/src/cache_value/cache.rs create mode 100644 ecstore/src/cache_value/mod.rs create mode 100644 ecstore/src/heal/heal_commands.rs create mode 100644 ecstore/src/heal/heal_ops.rs create mode 100644 ecstore/src/heal/mod.rs diff --git a/Cargo.lock b/Cargo.lock index ac19a510d..c908b52d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -478,6 +478,7 @@ dependencies = [ "common", "crc32fast", "futures", + "glob", "hex-simd", "http", "lazy_static", @@ -713,6 +714,12 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "gloo-timers" version = "0.3.0" diff --git a/common/protos/src/generated/flatbuffers_generated/models.rs b/common/protos/src/generated/flatbuffers_generated/models.rs index e4949fdcf..aa1f6ae2f 100644 --- a/common/protos/src/generated/flatbuffers_generated/models.rs +++ b/common/protos/src/generated/flatbuffers_generated/models.rs @@ -1,9 +1,10 @@ // automatically generated by the FlatBuffers compiler, do not modify + // @generated -use core::cmp::Ordering; use core::mem; +use core::cmp::Ordering; extern crate flatbuffers; use self::flatbuffers::{EndianScalar, Follow}; @@ -11,114 +12,112 @@ use self::flatbuffers::{EndianScalar, Follow}; #[allow(unused_imports, dead_code)] pub mod models { - use core::cmp::Ordering; - use core::mem; + use core::mem; + use core::cmp::Ordering; - extern crate flatbuffers; - use self::flatbuffers::{EndianScalar, Follow}; + extern crate flatbuffers; + use self::flatbuffers::{EndianScalar, Follow}; - pub enum PingBodyOffset {} - #[derive(Copy, Clone, PartialEq)] +pub enum PingBodyOffset {} +#[derive(Copy, Clone, PartialEq)] - pub struct PingBody<'a> { - pub _tab: flatbuffers::Table<'a>, +pub struct PingBody<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for PingBody<'a> { + type Inner = PingBody<'a>; + #[inline] + unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { _tab: flatbuffers::Table::new(buf, loc) } + } +} + +impl<'a> PingBody<'a> { + pub const VT_PAYLOAD: flatbuffers::VOffsetT = 4; + + pub const fn get_fully_qualified_name() -> &'static str { + "models.PingBody" + } + + #[inline] + pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + PingBody { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, + args: &'args PingBodyArgs<'args> + ) -> flatbuffers::WIPOffset> { + let mut builder = PingBodyBuilder::new(_fbb); + if let Some(x) = args.payload { builder.add_payload(x); } + builder.finish() + } + + + #[inline] + pub fn payload(&self) -> Option> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { self._tab.get::>>(PingBody::VT_PAYLOAD, None)} + } +} + +impl flatbuffers::Verifiable for PingBody<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, pos: usize + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>>("payload", Self::VT_PAYLOAD, false)? + .finish(); + Ok(()) + } +} +pub struct PingBodyArgs<'a> { + pub payload: Option>>, +} +impl<'a> Default for PingBodyArgs<'a> { + #[inline] + fn default() -> Self { + PingBodyArgs { + payload: None, } + } +} - impl<'a> flatbuffers::Follow<'a> for PingBody<'a> { - type Inner = PingBody<'a>; - #[inline] - unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { - Self { - _tab: flatbuffers::Table::new(buf, loc), - } - } +pub struct PingBodyBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> PingBodyBuilder<'a, 'b, A> { + #[inline] + pub fn add_payload(&mut self, payload: flatbuffers::WIPOffset>) { + self.fbb_.push_slot_always::>(PingBody::VT_PAYLOAD, payload); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> PingBodyBuilder<'a, 'b, A> { + let start = _fbb.start_table(); + PingBodyBuilder { + fbb_: _fbb, + start_: start, } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} - impl<'a> PingBody<'a> { - pub const VT_PAYLOAD: flatbuffers::VOffsetT = 4; +impl core::fmt::Debug for PingBody<'_> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let mut ds = f.debug_struct("PingBody"); + ds.field("payload", &self.payload()); + ds.finish() + } +} +} // pub mod models - pub const fn get_fully_qualified_name() -> &'static str { - "models.PingBody" - } - - #[inline] - pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self { - PingBody { _tab: table } - } - #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, - args: &'args PingBodyArgs<'args>, - ) -> flatbuffers::WIPOffset> { - let mut builder = PingBodyBuilder::new(_fbb); - if let Some(x) = args.payload { - builder.add_payload(x); - } - builder.finish() - } - - #[inline] - pub fn payload(&self) -> Option> { - // Safety: - // Created from valid Table for this object - // which contains a valid value in this slot - unsafe { - self._tab - .get::>>(PingBody::VT_PAYLOAD, None) - } - } - } - - impl flatbuffers::Verifiable for PingBody<'_> { - #[inline] - fn run_verifier(v: &mut flatbuffers::Verifier, pos: usize) -> Result<(), flatbuffers::InvalidFlatbuffer> { - use self::flatbuffers::Verifiable; - v.visit_table(pos)? - .visit_field::>>("payload", Self::VT_PAYLOAD, false)? - .finish(); - Ok(()) - } - } - pub struct PingBodyArgs<'a> { - pub payload: Option>>, - } - impl<'a> Default for PingBodyArgs<'a> { - #[inline] - fn default() -> Self { - PingBodyArgs { payload: None } - } - } - - pub struct PingBodyBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, - start_: flatbuffers::WIPOffset, - } - impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> PingBodyBuilder<'a, 'b, A> { - #[inline] - pub fn add_payload(&mut self, payload: flatbuffers::WIPOffset>) { - self.fbb_ - .push_slot_always::>(PingBody::VT_PAYLOAD, payload); - } - #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> PingBodyBuilder<'a, 'b, A> { - let start = _fbb.start_table(); - PingBodyBuilder { - fbb_: _fbb, - start_: start, - } - } - #[inline] - pub fn finish(self) -> flatbuffers::WIPOffset> { - let o = self.fbb_.end_table(self.start_); - flatbuffers::WIPOffset::new(o.value()) - } - } - - impl core::fmt::Debug for PingBody<'_> { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - let mut ds = f.debug_struct("PingBody"); - ds.field("payload", &self.payload()); - ds.finish() - } - } -} // pub mod models diff --git a/common/protos/src/generated/proto_gen/node_service.rs b/common/protos/src/generated/proto_gen/node_service.rs index c31199d6f..c2f957181 100644 --- a/common/protos/src/generated/proto_gen/node_service.rs +++ b/common/protos/src/generated/proto_gen/node_service.rs @@ -507,6 +507,22 @@ pub struct DeleteVolumeResponse { #[prost(string, optional, tag = "2")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, } +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DiskInfoRequest { + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub opts: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DiskInfoResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, tag = "2")] + pub disk_info: ::prost::alloc::string::String, + #[prost(string, optional, tag = "3")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} /// lock api have same argument type #[derive(Clone, PartialEq, ::prost::Message)] pub struct GenerallyLockRequest { @@ -522,9 +538,15 @@ pub struct GenerallyLockResponse { } /// Generated client implementations. pub mod node_service_client { - #![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)] - use tonic::codegen::http::Uri; + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; + use tonic::codegen::http::Uri; #[derive(Debug, Clone)] pub struct NodeServiceClient { inner: tonic::client::Grpc, @@ -555,16 +577,22 @@ pub mod node_service_client { let inner = tonic::client::Grpc::with_origin(inner, origin); Self { inner } } - pub fn with_interceptor(inner: T, interceptor: F) -> NodeServiceClient> + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> NodeServiceClient> where F: tonic::service::Interceptor, T::ResponseBody: Default, T: tonic::codegen::Service< http::Request, - Response = http::Response<>::ResponseBody>, + Response = http::Response< + >::ResponseBody, + >, >, - >>::Error: - Into + std::marker::Send + std::marker::Sync, + , + >>::Error: Into + std::marker::Send + std::marker::Sync, { NodeServiceClient::new(InterceptedService::new(inner, interceptor)) } @@ -607,9 +635,15 @@ pub mod node_service_client { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/Ping"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/Ping", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "Ping")); @@ -618,13 +652,22 @@ pub mod node_service_client { pub async fn list_bucket( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ListBucket"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/ListBucket", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ListBucket")); @@ -633,13 +676,22 @@ pub mod node_service_client { pub async fn make_bucket( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/MakeBucket"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/MakeBucket", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "MakeBucket")); @@ -648,13 +700,22 @@ pub mod node_service_client { pub async fn get_bucket_info( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/GetBucketInfo"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/GetBucketInfo", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "GetBucketInfo")); @@ -663,13 +724,22 @@ pub mod node_service_client { pub async fn delete_bucket( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/DeleteBucket"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/DeleteBucket", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "DeleteBucket")); @@ -678,13 +748,22 @@ pub mod node_service_client { pub async fn read_all( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReadAll"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/ReadAll", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ReadAll")); @@ -693,13 +772,22 @@ pub mod node_service_client { pub async fn write_all( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/WriteAll"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/WriteAll", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "WriteAll")); @@ -712,9 +800,15 @@ pub mod node_service_client { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/Delete"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/Delete", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "Delete")); @@ -723,13 +817,22 @@ pub mod node_service_client { pub async fn rename_part( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RenamePart"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/RenamePart", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "RenamePart")); @@ -738,13 +841,22 @@ pub mod node_service_client { pub async fn rename_file( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RenameFile"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/RenameFile", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "RenameFile")); @@ -757,9 +869,15 @@ pub mod node_service_client { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/Write"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/Write", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "Write")); @@ -768,13 +886,22 @@ pub mod node_service_client { pub async fn write_stream( &mut self, request: impl tonic::IntoStreamingRequest, - ) -> std::result::Result>, tonic::Status> { + ) -> std::result::Result< + tonic::Response>, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/WriteStream"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/WriteStream", + ); let mut req = request.into_streaming_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "WriteStream")); @@ -784,13 +911,22 @@ pub mod node_service_client { pub async fn read_at( &mut self, request: impl tonic::IntoStreamingRequest, - ) -> std::result::Result>, tonic::Status> { + ) -> std::result::Result< + tonic::Response>, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReadAt"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/ReadAt", + ); let mut req = request.into_streaming_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ReadAt")); @@ -799,13 +935,22 @@ pub mod node_service_client { pub async fn list_dir( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ListDir"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/ListDir", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ListDir")); @@ -814,13 +959,22 @@ pub mod node_service_client { pub async fn walk_dir( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/WalkDir"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/WalkDir", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "WalkDir")); @@ -829,13 +983,22 @@ pub mod node_service_client { pub async fn rename_data( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RenameData"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/RenameData", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "RenameData")); @@ -844,13 +1007,22 @@ pub mod node_service_client { pub async fn make_volumes( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/MakeVolumes"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/MakeVolumes", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "MakeVolumes")); @@ -859,13 +1031,22 @@ pub mod node_service_client { pub async fn make_volume( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/MakeVolume"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/MakeVolume", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "MakeVolume")); @@ -874,13 +1055,22 @@ pub mod node_service_client { pub async fn list_volumes( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ListVolumes"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/ListVolumes", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ListVolumes")); @@ -889,13 +1079,22 @@ pub mod node_service_client { pub async fn stat_volume( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/StatVolume"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/StatVolume", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "StatVolume")); @@ -904,13 +1103,22 @@ pub mod node_service_client { pub async fn delete_paths( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/DeletePaths"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/DeletePaths", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "DeletePaths")); @@ -919,13 +1127,22 @@ pub mod node_service_client { pub async fn update_metadata( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/UpdateMetadata"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/UpdateMetadata", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "UpdateMetadata")); @@ -934,13 +1151,22 @@ pub mod node_service_client { pub async fn write_metadata( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/WriteMetadata"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/WriteMetadata", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "WriteMetadata")); @@ -949,13 +1175,22 @@ pub mod node_service_client { pub async fn read_version( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReadVersion"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/ReadVersion", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ReadVersion")); @@ -968,9 +1203,15 @@ pub mod node_service_client { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReadXL"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/ReadXL", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ReadXL")); @@ -979,13 +1220,22 @@ pub mod node_service_client { pub async fn delete_version( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/DeleteVersion"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/DeleteVersion", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "DeleteVersion")); @@ -994,13 +1244,22 @@ pub mod node_service_client { pub async fn delete_versions( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/DeleteVersions"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/DeleteVersions", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "DeleteVersions")); @@ -1009,13 +1268,22 @@ pub mod node_service_client { pub async fn read_multiple( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReadMultiple"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/ReadMultiple", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ReadMultiple")); @@ -1024,28 +1292,70 @@ pub mod node_service_client { pub async fn delete_volume( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/DeleteVolume"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/DeleteVolume", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "DeleteVolume")); self.inner.unary(req, path, codec).await } - pub async fn lock( + pub async fn disk_info( &mut self, - request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/Lock"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/DiskInfo", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "DiskInfo")); + self.inner.unary(req, path, codec).await + } + pub async fn lock( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/Lock", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "Lock")); @@ -1054,13 +1364,22 @@ pub mod node_service_client { pub async fn un_lock( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/UnLock"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/UnLock", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "UnLock")); @@ -1069,13 +1388,22 @@ pub mod node_service_client { pub async fn r_lock( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RLock"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/RLock", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "RLock")); @@ -1084,13 +1412,22 @@ pub mod node_service_client { pub async fn r_un_lock( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RUnLock"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/RUnLock", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "RUnLock")); @@ -1099,13 +1436,22 @@ pub mod node_service_client { pub async fn force_un_lock( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ForceUnLock"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/ForceUnLock", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ForceUnLock")); @@ -1114,13 +1460,22 @@ pub mod node_service_client { pub async fn refresh( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/Refresh"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/Refresh", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "Refresh")); @@ -1130,7 +1485,13 @@ pub mod node_service_client { } /// Generated server implementations. pub mod node_service_server { - #![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)] + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; /// Generated trait containing gRPC methods that should be implemented for use with NodeServiceServer. #[async_trait] @@ -1143,19 +1504,31 @@ pub mod node_service_server { async fn list_bucket( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn make_bucket( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn get_bucket_info( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn delete_bucket( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn read_all( &self, request: tonic::Request, @@ -1163,7 +1536,10 @@ pub mod node_service_server { async fn write_all( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn delete( &self, request: tonic::Request, @@ -1171,25 +1547,38 @@ pub mod node_service_server { async fn rename_part( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn rename_file( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn write( &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; /// Server streaming response type for the WriteStream method. - type WriteStreamStream: tonic::codegen::tokio_stream::Stream> + type WriteStreamStream: tonic::codegen::tokio_stream::Stream< + Item = std::result::Result, + > + std::marker::Send + 'static; async fn write_stream( &self, request: tonic::Request>, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; /// Server streaming response type for the ReadAt method. - type ReadAtStream: tonic::codegen::tokio_stream::Stream> + type ReadAtStream: tonic::codegen::tokio_stream::Stream< + Item = std::result::Result, + > + std::marker::Send + 'static; /// rpc Append(AppendRequest) returns (AppendResponse) {}; @@ -1208,39 +1597,66 @@ pub mod node_service_server { async fn rename_data( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn make_volumes( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn make_volume( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn list_volumes( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn stat_volume( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn delete_paths( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn update_metadata( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn write_metadata( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn read_version( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn read_xl( &self, request: tonic::Request, @@ -1248,43 +1664,80 @@ pub mod node_service_server { async fn delete_version( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn delete_versions( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn read_multiple( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn delete_volume( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + async fn disk_info( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn lock( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn un_lock( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn r_lock( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn r_un_lock( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn force_un_lock( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn refresh( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; } #[derive(Debug)] pub struct NodeServiceServer { @@ -1307,7 +1760,10 @@ pub mod node_service_server { max_encoding_message_size: None, } } - pub fn with_interceptor(inner: T, interceptor: F) -> InterceptedService + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> InterceptedService where F: tonic::service::Interceptor, { @@ -1351,7 +1807,10 @@ pub mod node_service_server { type Response = http::Response; type Error = std::convert::Infallible; type Future = BoxFuture; - fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { Poll::Ready(Ok(())) } fn call(&mut self, req: http::Request) -> Self::Future { @@ -1359,12 +1818,21 @@ pub mod node_service_server { "/node_service.NodeService/Ping" => { #[allow(non_camel_case_types)] struct PingSvc(pub Arc); - impl tonic::server::UnaryService for PingSvc { + impl tonic::server::UnaryService + for PingSvc { type Response = super::PingResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::ping(&inner, request).await }; + let fut = async move { + ::ping(&inner, request).await + }; Box::pin(fut) } } @@ -1377,8 +1845,14 @@ pub mod node_service_server { let method = PingSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1387,12 +1861,23 @@ pub mod node_service_server { "/node_service.NodeService/ListBucket" => { #[allow(non_camel_case_types)] struct ListBucketSvc(pub Arc); - impl tonic::server::UnaryService for ListBucketSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for ListBucketSvc { type Response = super::ListBucketResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::list_bucket(&inner, request).await }; + let fut = async move { + ::list_bucket(&inner, request).await + }; Box::pin(fut) } } @@ -1405,8 +1890,14 @@ pub mod node_service_server { let method = ListBucketSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1415,12 +1906,23 @@ pub mod node_service_server { "/node_service.NodeService/MakeBucket" => { #[allow(non_camel_case_types)] struct MakeBucketSvc(pub Arc); - impl tonic::server::UnaryService for MakeBucketSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for MakeBucketSvc { type Response = super::MakeBucketResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::make_bucket(&inner, request).await }; + let fut = async move { + ::make_bucket(&inner, request).await + }; Box::pin(fut) } } @@ -1433,8 +1935,14 @@ pub mod node_service_server { let method = MakeBucketSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1443,12 +1951,23 @@ pub mod node_service_server { "/node_service.NodeService/GetBucketInfo" => { #[allow(non_camel_case_types)] struct GetBucketInfoSvc(pub Arc); - impl tonic::server::UnaryService for GetBucketInfoSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for GetBucketInfoSvc { type Response = super::GetBucketInfoResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::get_bucket_info(&inner, request).await }; + let fut = async move { + ::get_bucket_info(&inner, request).await + }; Box::pin(fut) } } @@ -1461,8 +1980,14 @@ pub mod node_service_server { let method = GetBucketInfoSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1471,12 +1996,23 @@ pub mod node_service_server { "/node_service.NodeService/DeleteBucket" => { #[allow(non_camel_case_types)] struct DeleteBucketSvc(pub Arc); - impl tonic::server::UnaryService for DeleteBucketSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for DeleteBucketSvc { type Response = super::DeleteBucketResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::delete_bucket(&inner, request).await }; + let fut = async move { + ::delete_bucket(&inner, request).await + }; Box::pin(fut) } } @@ -1489,8 +2025,14 @@ pub mod node_service_server { let method = DeleteBucketSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1499,12 +2041,23 @@ pub mod node_service_server { "/node_service.NodeService/ReadAll" => { #[allow(non_camel_case_types)] struct ReadAllSvc(pub Arc); - impl tonic::server::UnaryService for ReadAllSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for ReadAllSvc { type Response = super::ReadAllResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::read_all(&inner, request).await }; + let fut = async move { + ::read_all(&inner, request).await + }; Box::pin(fut) } } @@ -1517,8 +2070,14 @@ pub mod node_service_server { let method = ReadAllSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1527,12 +2086,23 @@ pub mod node_service_server { "/node_service.NodeService/WriteAll" => { #[allow(non_camel_case_types)] struct WriteAllSvc(pub Arc); - impl tonic::server::UnaryService for WriteAllSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for WriteAllSvc { type Response = super::WriteAllResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::write_all(&inner, request).await }; + let fut = async move { + ::write_all(&inner, request).await + }; Box::pin(fut) } } @@ -1545,8 +2115,14 @@ pub mod node_service_server { let method = WriteAllSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1555,12 +2131,23 @@ pub mod node_service_server { "/node_service.NodeService/Delete" => { #[allow(non_camel_case_types)] struct DeleteSvc(pub Arc); - impl tonic::server::UnaryService for DeleteSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for DeleteSvc { type Response = super::DeleteResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::delete(&inner, request).await }; + let fut = async move { + ::delete(&inner, request).await + }; Box::pin(fut) } } @@ -1573,8 +2160,14 @@ pub mod node_service_server { let method = DeleteSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1583,12 +2176,23 @@ pub mod node_service_server { "/node_service.NodeService/RenamePart" => { #[allow(non_camel_case_types)] struct RenamePartSvc(pub Arc); - impl tonic::server::UnaryService for RenamePartSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for RenamePartSvc { type Response = super::RenamePartResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::rename_part(&inner, request).await }; + let fut = async move { + ::rename_part(&inner, request).await + }; Box::pin(fut) } } @@ -1601,8 +2205,14 @@ pub mod node_service_server { let method = RenamePartSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1611,12 +2221,23 @@ pub mod node_service_server { "/node_service.NodeService/RenameFile" => { #[allow(non_camel_case_types)] struct RenameFileSvc(pub Arc); - impl tonic::server::UnaryService for RenameFileSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for RenameFileSvc { type Response = super::RenameFileResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::rename_file(&inner, request).await }; + let fut = async move { + ::rename_file(&inner, request).await + }; Box::pin(fut) } } @@ -1629,8 +2250,14 @@ pub mod node_service_server { let method = RenameFileSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1639,12 +2266,21 @@ pub mod node_service_server { "/node_service.NodeService/Write" => { #[allow(non_camel_case_types)] struct WriteSvc(pub Arc); - impl tonic::server::UnaryService for WriteSvc { + impl tonic::server::UnaryService + for WriteSvc { type Response = super::WriteResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::write(&inner, request).await }; + let fut = async move { + ::write(&inner, request).await + }; Box::pin(fut) } } @@ -1657,8 +2293,14 @@ pub mod node_service_server { let method = WriteSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1667,13 +2309,26 @@ pub mod node_service_server { "/node_service.NodeService/WriteStream" => { #[allow(non_camel_case_types)] struct WriteStreamSvc(pub Arc); - impl tonic::server::StreamingService for WriteStreamSvc { + impl< + T: NodeService, + > tonic::server::StreamingService + for WriteStreamSvc { type Response = super::WriteResponse; type ResponseStream = T::WriteStreamStream; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request>) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request< + tonic::Streaming, + >, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::write_stream(&inner, request).await }; + let fut = async move { + ::write_stream(&inner, request).await + }; Box::pin(fut) } } @@ -1686,8 +2341,14 @@ pub mod node_service_server { let method = WriteStreamSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.streaming(method, req).await; Ok(res) }; @@ -1696,13 +2357,26 @@ pub mod node_service_server { "/node_service.NodeService/ReadAt" => { #[allow(non_camel_case_types)] struct ReadAtSvc(pub Arc); - impl tonic::server::StreamingService for ReadAtSvc { + impl< + T: NodeService, + > tonic::server::StreamingService + for ReadAtSvc { type Response = super::ReadAtResponse; type ResponseStream = T::ReadAtStream; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request>) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request< + tonic::Streaming, + >, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::read_at(&inner, request).await }; + let fut = async move { + ::read_at(&inner, request).await + }; Box::pin(fut) } } @@ -1715,8 +2389,14 @@ pub mod node_service_server { let method = ReadAtSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.streaming(method, req).await; Ok(res) }; @@ -1725,12 +2405,23 @@ pub mod node_service_server { "/node_service.NodeService/ListDir" => { #[allow(non_camel_case_types)] struct ListDirSvc(pub Arc); - impl tonic::server::UnaryService for ListDirSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for ListDirSvc { type Response = super::ListDirResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::list_dir(&inner, request).await }; + let fut = async move { + ::list_dir(&inner, request).await + }; Box::pin(fut) } } @@ -1743,8 +2434,14 @@ pub mod node_service_server { let method = ListDirSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1753,12 +2450,23 @@ pub mod node_service_server { "/node_service.NodeService/WalkDir" => { #[allow(non_camel_case_types)] struct WalkDirSvc(pub Arc); - impl tonic::server::UnaryService for WalkDirSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for WalkDirSvc { type Response = super::WalkDirResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::walk_dir(&inner, request).await }; + let fut = async move { + ::walk_dir(&inner, request).await + }; Box::pin(fut) } } @@ -1771,8 +2479,14 @@ pub mod node_service_server { let method = WalkDirSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1781,12 +2495,23 @@ pub mod node_service_server { "/node_service.NodeService/RenameData" => { #[allow(non_camel_case_types)] struct RenameDataSvc(pub Arc); - impl tonic::server::UnaryService for RenameDataSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for RenameDataSvc { type Response = super::RenameDataResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::rename_data(&inner, request).await }; + let fut = async move { + ::rename_data(&inner, request).await + }; Box::pin(fut) } } @@ -1799,8 +2524,14 @@ pub mod node_service_server { let method = RenameDataSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1809,12 +2540,23 @@ pub mod node_service_server { "/node_service.NodeService/MakeVolumes" => { #[allow(non_camel_case_types)] struct MakeVolumesSvc(pub Arc); - impl tonic::server::UnaryService for MakeVolumesSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for MakeVolumesSvc { type Response = super::MakeVolumesResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::make_volumes(&inner, request).await }; + let fut = async move { + ::make_volumes(&inner, request).await + }; Box::pin(fut) } } @@ -1827,8 +2569,14 @@ pub mod node_service_server { let method = MakeVolumesSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1837,12 +2585,23 @@ pub mod node_service_server { "/node_service.NodeService/MakeVolume" => { #[allow(non_camel_case_types)] struct MakeVolumeSvc(pub Arc); - impl tonic::server::UnaryService for MakeVolumeSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for MakeVolumeSvc { type Response = super::MakeVolumeResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::make_volume(&inner, request).await }; + let fut = async move { + ::make_volume(&inner, request).await + }; Box::pin(fut) } } @@ -1855,8 +2614,14 @@ pub mod node_service_server { let method = MakeVolumeSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1865,12 +2630,23 @@ pub mod node_service_server { "/node_service.NodeService/ListVolumes" => { #[allow(non_camel_case_types)] struct ListVolumesSvc(pub Arc); - impl tonic::server::UnaryService for ListVolumesSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for ListVolumesSvc { type Response = super::ListVolumesResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::list_volumes(&inner, request).await }; + let fut = async move { + ::list_volumes(&inner, request).await + }; Box::pin(fut) } } @@ -1883,8 +2659,14 @@ pub mod node_service_server { let method = ListVolumesSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1893,12 +2675,23 @@ pub mod node_service_server { "/node_service.NodeService/StatVolume" => { #[allow(non_camel_case_types)] struct StatVolumeSvc(pub Arc); - impl tonic::server::UnaryService for StatVolumeSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for StatVolumeSvc { type Response = super::StatVolumeResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::stat_volume(&inner, request).await }; + let fut = async move { + ::stat_volume(&inner, request).await + }; Box::pin(fut) } } @@ -1911,8 +2704,14 @@ pub mod node_service_server { let method = StatVolumeSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1921,12 +2720,23 @@ pub mod node_service_server { "/node_service.NodeService/DeletePaths" => { #[allow(non_camel_case_types)] struct DeletePathsSvc(pub Arc); - impl tonic::server::UnaryService for DeletePathsSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for DeletePathsSvc { type Response = super::DeletePathsResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::delete_paths(&inner, request).await }; + let fut = async move { + ::delete_paths(&inner, request).await + }; Box::pin(fut) } } @@ -1939,8 +2749,14 @@ pub mod node_service_server { let method = DeletePathsSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1949,12 +2765,23 @@ pub mod node_service_server { "/node_service.NodeService/UpdateMetadata" => { #[allow(non_camel_case_types)] struct UpdateMetadataSvc(pub Arc); - impl tonic::server::UnaryService for UpdateMetadataSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for UpdateMetadataSvc { type Response = super::UpdateMetadataResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::update_metadata(&inner, request).await }; + let fut = async move { + ::update_metadata(&inner, request).await + }; Box::pin(fut) } } @@ -1967,8 +2794,14 @@ pub mod node_service_server { let method = UpdateMetadataSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1977,12 +2810,23 @@ pub mod node_service_server { "/node_service.NodeService/WriteMetadata" => { #[allow(non_camel_case_types)] struct WriteMetadataSvc(pub Arc); - impl tonic::server::UnaryService for WriteMetadataSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for WriteMetadataSvc { type Response = super::WriteMetadataResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::write_metadata(&inner, request).await }; + let fut = async move { + ::write_metadata(&inner, request).await + }; Box::pin(fut) } } @@ -1995,8 +2839,14 @@ pub mod node_service_server { let method = WriteMetadataSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -2005,12 +2855,23 @@ pub mod node_service_server { "/node_service.NodeService/ReadVersion" => { #[allow(non_camel_case_types)] struct ReadVersionSvc(pub Arc); - impl tonic::server::UnaryService for ReadVersionSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for ReadVersionSvc { type Response = super::ReadVersionResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::read_version(&inner, request).await }; + let fut = async move { + ::read_version(&inner, request).await + }; Box::pin(fut) } } @@ -2023,8 +2884,14 @@ pub mod node_service_server { let method = ReadVersionSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -2033,12 +2900,23 @@ pub mod node_service_server { "/node_service.NodeService/ReadXL" => { #[allow(non_camel_case_types)] struct ReadXLSvc(pub Arc); - impl tonic::server::UnaryService for ReadXLSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for ReadXLSvc { type Response = super::ReadXlResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::read_xl(&inner, request).await }; + let fut = async move { + ::read_xl(&inner, request).await + }; Box::pin(fut) } } @@ -2051,8 +2929,14 @@ pub mod node_service_server { let method = ReadXLSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -2061,12 +2945,23 @@ pub mod node_service_server { "/node_service.NodeService/DeleteVersion" => { #[allow(non_camel_case_types)] struct DeleteVersionSvc(pub Arc); - impl tonic::server::UnaryService for DeleteVersionSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for DeleteVersionSvc { type Response = super::DeleteVersionResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::delete_version(&inner, request).await }; + let fut = async move { + ::delete_version(&inner, request).await + }; Box::pin(fut) } } @@ -2079,8 +2974,14 @@ pub mod node_service_server { let method = DeleteVersionSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -2089,12 +2990,23 @@ pub mod node_service_server { "/node_service.NodeService/DeleteVersions" => { #[allow(non_camel_case_types)] struct DeleteVersionsSvc(pub Arc); - impl tonic::server::UnaryService for DeleteVersionsSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for DeleteVersionsSvc { type Response = super::DeleteVersionsResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::delete_versions(&inner, request).await }; + let fut = async move { + ::delete_versions(&inner, request).await + }; Box::pin(fut) } } @@ -2107,8 +3019,14 @@ pub mod node_service_server { let method = DeleteVersionsSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -2117,12 +3035,23 @@ pub mod node_service_server { "/node_service.NodeService/ReadMultiple" => { #[allow(non_camel_case_types)] struct ReadMultipleSvc(pub Arc); - impl tonic::server::UnaryService for ReadMultipleSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for ReadMultipleSvc { type Response = super::ReadMultipleResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::read_multiple(&inner, request).await }; + let fut = async move { + ::read_multiple(&inner, request).await + }; Box::pin(fut) } } @@ -2135,8 +3064,14 @@ pub mod node_service_server { let method = ReadMultipleSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -2145,12 +3080,23 @@ pub mod node_service_server { "/node_service.NodeService/DeleteVolume" => { #[allow(non_camel_case_types)] struct DeleteVolumeSvc(pub Arc); - impl tonic::server::UnaryService for DeleteVolumeSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for DeleteVolumeSvc { type Response = super::DeleteVolumeResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::delete_volume(&inner, request).await }; + let fut = async move { + ::delete_volume(&inner, request).await + }; Box::pin(fut) } } @@ -2163,8 +3109,59 @@ pub mod node_service_server { let method = DeleteVolumeSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/DiskInfo" => { + #[allow(non_camel_case_types)] + struct DiskInfoSvc(pub Arc); + impl< + T: NodeService, + > tonic::server::UnaryService + for DiskInfoSvc { + type Response = super::DiskInfoResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::disk_info(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = DiskInfoSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -2173,12 +3170,23 @@ pub mod node_service_server { "/node_service.NodeService/Lock" => { #[allow(non_camel_case_types)] struct LockSvc(pub Arc); - impl tonic::server::UnaryService for LockSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for LockSvc { type Response = super::GenerallyLockResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::lock(&inner, request).await }; + let fut = async move { + ::lock(&inner, request).await + }; Box::pin(fut) } } @@ -2191,8 +3199,14 @@ pub mod node_service_server { let method = LockSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -2201,12 +3215,23 @@ pub mod node_service_server { "/node_service.NodeService/UnLock" => { #[allow(non_camel_case_types)] struct UnLockSvc(pub Arc); - impl tonic::server::UnaryService for UnLockSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for UnLockSvc { type Response = super::GenerallyLockResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::un_lock(&inner, request).await }; + let fut = async move { + ::un_lock(&inner, request).await + }; Box::pin(fut) } } @@ -2219,8 +3244,14 @@ pub mod node_service_server { let method = UnLockSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -2229,12 +3260,23 @@ pub mod node_service_server { "/node_service.NodeService/RLock" => { #[allow(non_camel_case_types)] struct RLockSvc(pub Arc); - impl tonic::server::UnaryService for RLockSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for RLockSvc { type Response = super::GenerallyLockResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::r_lock(&inner, request).await }; + let fut = async move { + ::r_lock(&inner, request).await + }; Box::pin(fut) } } @@ -2247,8 +3289,14 @@ pub mod node_service_server { let method = RLockSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -2257,12 +3305,23 @@ pub mod node_service_server { "/node_service.NodeService/RUnLock" => { #[allow(non_camel_case_types)] struct RUnLockSvc(pub Arc); - impl tonic::server::UnaryService for RUnLockSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for RUnLockSvc { type Response = super::GenerallyLockResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::r_un_lock(&inner, request).await }; + let fut = async move { + ::r_un_lock(&inner, request).await + }; Box::pin(fut) } } @@ -2275,8 +3334,14 @@ pub mod node_service_server { let method = RUnLockSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -2285,12 +3350,23 @@ pub mod node_service_server { "/node_service.NodeService/ForceUnLock" => { #[allow(non_camel_case_types)] struct ForceUnLockSvc(pub Arc); - impl tonic::server::UnaryService for ForceUnLockSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for ForceUnLockSvc { type Response = super::GenerallyLockResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::force_un_lock(&inner, request).await }; + let fut = async move { + ::force_un_lock(&inner, request).await + }; Box::pin(fut) } } @@ -2303,8 +3379,14 @@ pub mod node_service_server { let method = ForceUnLockSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -2313,12 +3395,23 @@ pub mod node_service_server { "/node_service.NodeService/Refresh" => { #[allow(non_camel_case_types)] struct RefreshSvc(pub Arc); - impl tonic::server::UnaryService for RefreshSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for RefreshSvc { type Response = super::GenerallyLockResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::refresh(&inner, request).await }; + let fut = async move { + ::refresh(&inner, request).await + }; Box::pin(fut) } } @@ -2331,20 +3424,36 @@ pub mod node_service_server { let method = RefreshSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; Box::pin(fut) } - _ => Box::pin(async move { - let mut response = http::Response::new(empty_body()); - let headers = response.headers_mut(); - headers.insert(tonic::Status::GRPC_STATUS, (tonic::Code::Unimplemented as i32).into()); - headers.insert(http::header::CONTENT_TYPE, tonic::metadata::GRPC_CONTENT_TYPE); - Ok(response) - }), + _ => { + Box::pin(async move { + let mut response = http::Response::new(empty_body()); + let headers = response.headers_mut(); + headers + .insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers + .insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) + }) + } } } } diff --git a/common/protos/src/node.proto b/common/protos/src/node.proto index 4e84eedba..51ac6cbc0 100644 --- a/common/protos/src/node.proto +++ b/common/protos/src/node.proto @@ -345,6 +345,17 @@ message DeleteVolumeResponse { optional string error_info = 2; } +message DiskInfoRequest { + string disk = 1; + string opts = 2; +} + +message DiskInfoResponse { + bool success = 1; + string disk_info = 2; + optional string error_info = 3; +} + // lock api have same argument type message GenerallyLockRequest { string args = 1; @@ -392,6 +403,7 @@ service NodeService { rpc DeleteVersions(DeleteVersionsRequest) returns (DeleteVersionsResponse) {}; rpc ReadMultiple(ReadMultipleRequest) returns (ReadMultipleResponse) {}; rpc DeleteVolume(DeleteVolumeRequest) returns (DeleteVolumeResponse) {}; + rpc DiskInfo(DiskInfoRequest) returns (DiskInfoResponse) {}; /* -------------------------------lock service-------------------------- */ diff --git a/ecstore/Cargo.toml b/ecstore/Cargo.toml index af02e32eb..9a6a3f6c7 100644 --- a/ecstore/Cargo.toml +++ b/ecstore/Cargo.toml @@ -12,9 +12,10 @@ rust-version.workspace = true async-trait.workspace = true backon.workspace = true bytes.workspace = true +common.workspace = true +glob = "0.3.1" thiserror.workspace = true futures.workspace = true -common.workspace = true tracing.workspace = true serde.workspace = true time.workspace = true diff --git a/ecstore/src/bucket/metadata.rs b/ecstore/src/bucket/metadata.rs index f2297e29c..63ca1c584 100644 --- a/ecstore/src/bucket/metadata.rs +++ b/ecstore/src/bucket/metadata.rs @@ -19,6 +19,7 @@ use crate::config::common::{read_config, save_config}; use crate::error::{Error, Result}; use crate::disk::BUCKET_META_PREFIX; +use crate::file_meta::FileMetaShallowVersion; use crate::store::ECStore; pub const BUCKET_METADATA_FILE: &str = ".metadata.bin"; @@ -397,6 +398,16 @@ where s.serialize_bytes(&buf) } +#[derive(Debug, Default)] +pub struct MetadataResolutionParams { + pub dir_quorum: usize, + pub obj_quorum: usize, + pub requested_versions: usize, + pub bucket: String, + pub strict: bool, + pub candidates: Vec>, +} + #[cfg(test)] mod test { diff --git a/ecstore/src/cache_value/cache.rs b/ecstore/src/cache_value/cache.rs new file mode 100644 index 000000000..231263b61 --- /dev/null +++ b/ecstore/src/cache_value/cache.rs @@ -0,0 +1,88 @@ +use std::{ + ptr, + sync::{atomic::{AtomicPtr, AtomicU64, Ordering}, Arc, Mutex}, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use tokio::spawn; + +use crate::error::{Error, Result}; + +type UpdateFn = Box Result>; + +#[derive(Clone, Debug, Default)] +pub struct Opts { + return_last_good: bool, + no_wait: bool, +} + +pub struct Cache { + update_fn: UpdateFn, + ttl: Duration, + opts: Opts, + val: AtomicPtr, + last_update_ms: AtomicU64, + updating: Arc>, +} + +impl Cache { + pub fn new(update_fn: UpdateFn, ttl: Duration, opts: Opts) -> Self { + let val = AtomicPtr::new(ptr::null_mut()); + Self { + update_fn, + ttl, + opts, + val, + last_update_ms: AtomicU64::new(0), + updating: Arc::new(Mutex::new(false)), + } + } + + pub fn get(&self) -> Result { + let v_ptr = self.val.load(Ordering::SeqCst); + let v = if v_ptr.is_null() { None } else { Some(unsafe { *v_ptr }) }; + + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs(); + if v.is_some() && now - self.last_update_ms.load(Ordering::SeqCst) < self.ttl.as_secs() { + return Ok(v.unwrap()); + } + + if self.opts.no_wait && v.is_some() && now - self.last_update_ms.load(Ordering::SeqCst) < self.ttl.as_secs() * 2 { + if self.updating.try_lock().is_ok() { + let this = self.clone(); + spawn(async { + let _ = this.update().await; + }); + self.update(); + } + + return Ok(v.unwrap()); + } + + match self.updating.lock() { + Ok(_) => { + if let Ok(duration) = SystemTime::now().duration_since(UNIX_EPOCH + Duration::from_secs(self.last_update_ms.load(Ordering::SeqCst))) { + if duration < self.ttl { + return Ok(v.unwrap()); + } + } + + match self.update() { + + } + }, + Err(err) => { + return Err(Error::from_string(err.to_string())); + } + } + + todo!() + } + + async fn update(&self) -> Result<()> { + todo!() + } +} diff --git a/ecstore/src/cache_value/mod.rs b/ecstore/src/cache_value/mod.rs new file mode 100644 index 000000000..a5c08fdb0 --- /dev/null +++ b/ecstore/src/cache_value/mod.rs @@ -0,0 +1 @@ +pub mod cache; diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 6a28a8951..00adf0b5c 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -1,8 +1,9 @@ use super::error::{is_sys_err_io, is_sys_err_not_empty, is_sys_err_too_many_files, os_is_not_exist, os_is_permission}; use super::{endpoint::Endpoint, error::DiskError, format::FormatV3}; use super::{ - os, DeleteOptions, DiskAPI, DiskLocation, FileInfoVersions, FileReader, FileWriter, MetaCacheEntry, ReadMultipleReq, - ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, + os, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions, FileReader, FileWriter, + MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, + WalkDirOptions, }; use crate::disk::error::{ convert_access_error, is_sys_err_handle_invalid, is_sys_err_invalid_arg, is_sys_err_is_dir, is_sys_err_not_dir, @@ -560,6 +561,10 @@ impl LocalDisk { Ok(f) } + + fn get_metrics(&self) -> DiskMetrics { + DiskMetrics::default() + } } fn is_root_path(path: impl AsRef) -> bool { @@ -1599,6 +1604,12 @@ impl DiskAPI for LocalDisk { Ok(()) } + + async fn disk_info(&self, opts: &DiskInfoOptions) -> Result { + let mut info = DiskInfo::default(); + + Ok(info) + } } #[cfg(test)] diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs index f420168b9..9ef049e3d 100644 --- a/ecstore/src/disk/mod.rs +++ b/ecstore/src/disk/mod.rs @@ -26,7 +26,7 @@ use protos::proto_gen::node_service::{ node_service_client::NodeServiceClient, ReadAtRequest, ReadAtResponse, WriteRequest, WriteResponse, }; use serde::{Deserialize, Serialize}; -use std::{fmt::Debug, io::SeekFrom, path::PathBuf, sync::Arc}; +use std::{collections::HashMap, fmt::Debug, io::SeekFrom, path::PathBuf, sync::Arc, usize}; use time::OffsetDateTime; use tokio::{ fs::File, @@ -133,6 +133,7 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { // CleanAbandonedData async fn write_all(&self, volume: &str, path: &str, data: Vec) -> Result<()>; async fn read_all(&self, volume: &str, path: &str) -> Result>; + async fn disk_info(&self, opts: &DiskInfoOptions) -> Result; } #[derive(Debug, Serialize, Deserialize)] @@ -152,6 +153,45 @@ impl DiskLocation { } } +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct DiskInfoOptions { + pub disk_id: String, + pub metrics: bool, + pub noop: bool, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct DiskInfo { + pub total: u64, + pub free: u64, + pub used: u64, + pub used_inodes: u64, + pub free_inodes: u64, + pub major: u32, + pub minor: u32, + pub nr_requests: u64, + pub fs_type: String, + pub root_disk: bool, + pub healing: bool, + pub scanning: bool, + pub endpoint: String, + pub mount_path: String, + pub id: String, + pub rotational: bool, + pub metrics: DiskMetrics, + pub error: String, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct DiskMetrics { + api_calls: HashMap, + total_waiting: u32, + total_errors_availability: u64, + total_errors_timeout: u64, + total_writes: u64, + total_deletes: u64, +} + #[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct FileInfoVersions { // Name of the volume. @@ -263,6 +303,26 @@ impl MetaCacheEntry { return Ok(Some(fi)); } + + pub fn file_info_versions(&self, bucket: &str) -> Result { + if self.is_dir() { + return Ok(FileInfoVersions { + volume: bucket.to_string(), + name: self.name.clone(), + versions: vec![FileInfo { + volume: bucket.to_string(), + name: self.name.clone(), + ..Default::default() + }], + ..Default::default() + }); + } + + let mut fm = FileMeta::new(); + fm.unmarshal_msg(&self.metadata)?; + + Ok(fm.into_file_info_versions(bucket, self.name.as_str(), false)?) + } } #[derive(Debug, Default)] diff --git a/ecstore/src/disk/remote.rs b/ecstore/src/disk/remote.rs index 43fbd4086..4b82c43cd 100644 --- a/ecstore/src/disk/remote.rs +++ b/ecstore/src/disk/remote.rs @@ -4,10 +4,10 @@ use futures::lock::Mutex; use protos::{ node_service_time_out_client, proto_gen::node_service::{ - DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, ListDirRequest, - ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest, ReadMultipleRequest, ReadVersionRequest, - ReadXlRequest, RenameDataRequest, RenameFileRequst, StatVolumeRequest, UpdateMetadataRequest, WalkDirRequest, - WriteAllRequest, WriteMetadataRequest, + DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, + ListDirRequest, ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest, ReadMultipleRequest, + ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequst, StatVolumeRequest, UpdateMetadataRequest, + WalkDirRequest, WriteAllRequest, WriteMetadataRequest, }, }; use tonic::Request; @@ -15,9 +15,9 @@ use tracing::info; use uuid::Uuid; use super::{ - endpoint::Endpoint, DeleteOptions, DiskAPI, DiskLocation, DiskOption, FileInfoVersions, FileReader, FileWriter, - MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RemoteFileReader, RemoteFileWriter, RenameDataResp, - UpdateMetadataOpts, VolumeInfo, WalkDirOptions, + endpoint::Endpoint, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, FileInfoVersions, + FileReader, FileWriter, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RemoteFileReader, RemoteFileWriter, + RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, }; use crate::{ disk::error::DiskError, @@ -658,4 +658,26 @@ impl DiskAPI for RemoteDisk { Ok(()) } + + async fn disk_info(&self, opts: &DiskInfoOptions) -> Result { + info!("delete_volume"); + let opts = serde_json::to_string(&opts)?; + let mut client = node_service_time_out_client(&self.addr) + .await + .map_err(|err| Error::from_string(format!("can not get client, err: {}", err)))?; + let request = Request::new(DiskInfoRequest { + disk: self.root.to_string_lossy().to_string(), + opts, + }); + + let response = client.disk_info(request).await?.into_inner(); + + if !response.success { + return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); + } + + let disk_info = serde_json::from_str::(&response.disk_info)?; + + Ok(disk_info) + } } diff --git a/ecstore/src/heal/heal_commands.rs b/ecstore/src/heal/heal_commands.rs new file mode 100644 index 000000000..e6cf29b6b --- /dev/null +++ b/ecstore/src/heal/heal_commands.rs @@ -0,0 +1,39 @@ +pub type HealScanMode = usize; +pub type HealItemType = String; + +#[derive(Clone, Copy, Debug, Default)] +pub struct HealOpts { + pub recursive: bool, + pub dry_run: bool, + pub remove: bool, + pub recreate: bool, + pub scan_mode: HealScanMode, + pub update_parity: bool, + pub no_lock: bool, + pub pool: Option, + pub set: Option, +} + +#[derive(Debug)] +struct HealDriveInfo { + uuid: String, + endpoint: String, + state: String, +} + +#[derive(Debug)] +pub struct HealResultItem { + pub result_index: usize, + pub heal_item_type: HealItemType, + pub bucket: String, + pub object: String, + pub version_id: String, + pub detail: String, + pub parity_blocks: usize, + pub data_blocks: usize, + pub disk_count: usize, + pub set_count: usize, + pub before: Vec, + pub after: Vec, + pub object_size: usize, +} diff --git a/ecstore/src/heal/heal_ops.rs b/ecstore/src/heal/heal_ops.rs new file mode 100644 index 000000000..05877e634 --- /dev/null +++ b/ecstore/src/heal/heal_ops.rs @@ -0,0 +1,178 @@ +use crate::{ + disk::MetaCacheEntry, + error::{Error, Result}, +}; +use lazy_static::lazy_static; +use std::{collections::HashMap, time::Instant}; +use uuid::Uuid; + +use super::heal_commands::{HealItemType, HealOpts, HealResultItem, HealScanMode}; + +type HealStatusSummary = String; +type ItemsMap = HashMap; +pub type HealObjectFn = Box Result<()> + Send>; +pub type HealEntryFn = Box Result<()> + Send>; + +lazy_static! { + static ref HEAL_NOT_STARTED_STATUS: HealStatusSummary = String::from("not started"); + static ref bgHealingUUID: String = String::from("0000-0000-0000-0000"); +} + +lazy_static! {} + +#[derive(Debug)] +pub struct HealSequenceStatus { + summary: HealStatusSummary, + failure_detail: String, + start_time: Instant, + heal_setting: HealOpts, + items: Vec, +} + +impl Default for HealSequenceStatus { + fn default() -> Self { + Self { + summary: Default::default(), + failure_detail: Default::default(), + start_time: Instant::now(), + heal_setting: Default::default(), + items: Default::default(), + } + } +} + +#[derive(Debug)] +pub struct HealSequence { + pub bucket: String, + pub object: String, + pub report_progress: bool, + pub start_time: Instant, + pub end_time: Instant, + pub client_token: String, + pub client_address: String, + pub force_started: bool, + pub setting: HealOpts, + pub current_status: HealSequenceStatus, + pub last_sent_result_index: usize, + pub scanned_items_map: ItemsMap, + pub healed_items_map: ItemsMap, + pub heal_failed_items_map: ItemsMap, + pub last_heal_activity: Instant, +} + +impl HealSequence { + pub fn new(bucket: &str, obj_profix: &str, client_addr: &str, hs: HealOpts, force_start: bool) -> Self { + let client_token = Uuid::new_v4().to_string(); + + Self { + bucket: bucket.to_string(), + object: obj_profix.to_string(), + report_progress: true, + client_token, + client_address: client_addr.to_string(), + force_started: force_start, + setting: hs, + current_status: HealSequenceStatus { + summary: HEAL_NOT_STARTED_STATUS.to_string(), + heal_setting: hs, + ..Default::default() + }, + ..Default::default() + } + } +} + +impl HealSequence { + fn get_scanned_items_count(&self) -> usize { + self.scanned_items_map.values().sum() + } + + fn get_scanned_items_map(&self) -> ItemsMap { + self.scanned_items_map.clone() + } + + fn get_healed_items_map(&self) -> ItemsMap { + self.healed_items_map.clone() + } + + fn get_heal_failed_items_map(&self) -> ItemsMap { + self.heal_failed_items_map.clone() + } + + fn count_failed(&mut self, heal_type: HealItemType) { + *self.heal_failed_items_map.entry(heal_type).or_insert(0) += 1; + self.last_heal_activity = Instant::now(); + } + + fn count_scanned(&mut self, heal_type: HealItemType) { + *self.scanned_items_map.entry(heal_type).or_insert(0) += 1; + self.last_heal_activity = Instant::now() + } + + fn count_healed(&mut self, heal_type: HealItemType) { + *self.healed_items_map.entry(heal_type).or_insert(0) += 1; + self.last_heal_activity = Instant::now() + } + + fn is_quitting(&self) -> bool { + todo!() + } + + fn has_ended(&self) -> bool { + if self.client_token == bgHealingUUID.to_string() { + return false; + } + + !(self.end_time == self.start_time) + } + + fn stop(&self) { + todo!() + } + + fn push_heal_result_item(&self, r: HealResultItem) -> Result<()> { + todo!() + } + + fn heal_disk_meta() -> Result<()> { + todo!() + } + + fn heal_items(&self, buckets_only: bool) -> Result<()> { + if self.client_token == bgHealingUUID.to_string() { + return Ok(()); + } + + todo!() + } + + fn traverse_and_heal(&self) { + let buckets_only = false; + } + + fn heal_rustfs_sys_meta(&self, meta_prefix: String) -> Result<()> { + todo!() + } +} + +impl Default for HealSequence { + fn default() -> Self { + Self { + bucket: Default::default(), + object: Default::default(), + report_progress: Default::default(), + start_time: Instant::now(), + end_time: Instant::now(), + client_token: Default::default(), + client_address: Default::default(), + force_started: Default::default(), + setting: Default::default(), + current_status: Default::default(), + last_sent_result_index: Default::default(), + scanned_items_map: Default::default(), + healed_items_map: Default::default(), + heal_failed_items_map: Default::default(), + last_heal_activity: Instant::now(), + } + } +} diff --git a/ecstore/src/heal/mod.rs b/ecstore/src/heal/mod.rs new file mode 100644 index 000000000..b43ec4b06 --- /dev/null +++ b/ecstore/src/heal/mod.rs @@ -0,0 +1,2 @@ +pub mod heal_commands; +pub mod heal_ops; diff --git a/ecstore/src/lib.rs b/ecstore/src/lib.rs index 6d9e13a2a..5d1346b25 100644 --- a/ecstore/src/lib.rs +++ b/ecstore/src/lib.rs @@ -1,3 +1,4 @@ +pub mod cache_value; mod chunk_stream; mod config; pub mod disk; @@ -7,6 +8,7 @@ pub mod erasure; pub mod error; mod file_meta; mod global; +mod heal; pub mod peer; mod quorum; pub mod set_disk; diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index 1f12dd94e..b767a0412 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -3,7 +3,10 @@ use crate::bucket::metadata; use crate::bucket::metadata_sys::set_bucket_metadata; use crate::disk::endpoint::EndpointType; +use crate::disk::MetaCacheEntry; use crate::global::{is_dist_erasure, set_object_layer, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES}; +use crate::heal::heal_commands::{HealOpts, HealResultItem, HealScanMode}; +use crate::heal::heal_ops::HealObjectFn; use crate::store_api::ObjectIO; use crate::{ bucket::metadata::BucketMetadata, @@ -23,6 +26,7 @@ use crate::{ use backon::{ExponentialBuilder, Retryable}; use common::globals::{GLOBAL_Local_Node_Name, GLOBAL_Rustfs_Host, GLOBAL_Rustfs_Port}; use futures::future::join_all; +use glob::Pattern; use http::HeaderMap; use s3s::dto::{ObjectLockConfiguration, ObjectLockEnabled}; use std::{ @@ -776,6 +780,138 @@ impl StorageAPI for ECStore { } unimplemented!() } + + async fn heal_format(&self, dry_run: bool) -> Result { + unimplemented!() + } + async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result { + unimplemented!() + } + async fn heal_object(&self, bucket: &str, object: &str, version_id: &str, opts: &HealOpts) -> Result { + let object = utils::path::encode_dir_object(object); + let mut errs = HashMap::new(); + let mut results = HashMap::new(); + for (idx, pool) in self.pools.iter().enumerate() { + //TODO: IsSuspended + match pool.heal_object(bucket, &object, version_id, opts).await { + Ok(mut result) => { + result.object = utils::path::decode_dir_object(&result.object); + results.insert(idx, result); + } + Err(err) => { + errs.insert(idx, err); + } + } + } + + // Return the first nil error + for i in 0..self.pools.len() { + if errs.get(&i).is_none() { + return Ok(results.remove(&i).unwrap()); + } + } + + // No pool returned a nil error, return the first non 'not found' error + for (k, err) in errs.iter() { + match err.downcast_ref::() { + Some(DiskError::FileNotFound) | Some(DiskError::FileVersionNotFound) => {} + _ => return Ok(results.remove(&k).unwrap()), + } + } + + // At this stage, all errors are 'not found' + if !version_id.is_empty() { + return Err(Error::new(DiskError::FileVersionNotFound)); + } + + Err(Error::new(DiskError::FileNotFound)) + } + async fn heal_objects(&self, bucket: &str, prefix: &str, opts: &HealOpts, func: HealObjectFn) -> Result<()> { + let heal_entry = |bucket: String, entry: MetaCacheEntry, scan_mode: HealScanMode| async move { + if entry.is_dir() { + return Ok(()); + } + + // We might land at .metacache, .trash, .multipart + // no need to heal them skip, only when bucket + // is '.rustfs.sys' + if bucket == RUSTFS_META_BUCKET { + if Pattern::new("buckets/*/.metacache/*") + .map(|p| p.matches(&entry.name)) + .unwrap_or(false) + || Pattern::new("tmp/*").map(|p| p.matches(&entry.name)).unwrap_or(false) + || Pattern::new("multipart/*").map(|p| p.matches(&entry.name)).unwrap_or(false) + || Pattern::new("tmp-old/*").map(|p| p.matches(&entry.name)).unwrap_or(false) + { + return Ok(()); + } + } + + match entry.file_info_versions(&bucket) { + Ok(fivs) => { + if opts.remove && !opts.dry_run { + if let Err(err) = self.check_abandoned_parts(&bucket, &entry.name, opts).await { + return Err(Error::from_string(format!( + "unable to check object {}/{} for abandoned data: {}", + bucket, entry.name, err + ))); + } + } + + for version in fivs.versions.iter() { + let version_id = version.version_id.map_or("".to_string(), |version_id| version_id.to_string()); + if let Err(err) = func(&bucket, &entry.name, &version_id, scan_mode) { + match err.downcast_ref::() { + Some(DiskError::FileNotFound) | Some(DiskError::FileVersionNotFound) => {} + _ => return Err(err), + } + } + } + } + Err(_) => { + return func(&bucket, &entry.name, "", scan_mode); + } + } + Ok(()) + }; + + for (idx, pool) in self.pools.iter().enumerate() { + if opts.pool.is_some() && opts.pool.unwrap() != idx { + continue; + } + //TODO: IsSuspended + + for (idx, set) in pool.disk_set.iter().enumerate() { + if opts.set.is_some() && opts.set.unwrap() != idx { + continue; + } + + set.list + } + } + todo!() + } + + async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> { + let object = utils::path::encode_dir_object(object); + if self.single_pool() { + return self.pools[0].check_abandoned_parts(bucket, &object, opts).await; + } + + let mut errs = Vec::new(); + for pool in self.pools.iter() { + //TODO: IsSuspended + if let Err(err) = pool.check_abandoned_parts(bucket, &object, opts).await { + errs.push(err); + } + } + + if !errs.is_empty() { + return Err(errs[0]); + } + + Ok(()) + } } async fn init_local_peer(endpoint_pools: &EndpointServerPools, host: &String, port: &String) { diff --git a/ecstore/src/store_api.rs b/ecstore/src/store_api.rs index 948d31e49..514a7742f 100644 --- a/ecstore/src/store_api.rs +++ b/ecstore/src/store_api.rs @@ -1,6 +1,12 @@ use std::collections::HashMap; -use crate::error::{Error, Result}; +use crate::{ + error::{Error, Result}, + heal::{ + heal_commands::{HealOpts, HealResultItem}, + heal_ops::HealObjectFn, + }, +}; use futures::StreamExt; use http::HeaderMap; use rmp_serde::Serializer; @@ -601,4 +607,9 @@ pub trait StorageAPI: ObjectIO { uploaded_parts: Vec, opts: &ObjectOptions, ) -> Result; + async fn heal_format(&self, dry_run: bool) -> Result; + async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result; + async fn heal_object(&self, bucket: &str, object: &str, version_id: &str, opts: &HealOpts) -> Result; + async fn heal_objects(&self, bucket: &str, prefix: &str, opts: &HealOpts, func: HealObjectFn) -> Result<()>; + async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()>; } diff --git a/rustfs/src/grpc.rs b/rustfs/src/grpc.rs index 0780a96ed..2be43631d 100644 --- a/rustfs/src/grpc.rs +++ b/rustfs/src/grpc.rs @@ -1125,6 +1125,47 @@ impl Node for NodeService { } } + async fn disk_info(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + let opts = match serde_json::from_str::(&request.opts) { + Ok(opts) => opts, + Err(_) => { + return Ok(tonic::Response::new(ReadMultipleResponse { + success: false, + read_multiple_resps: Vec::new(), + error_info: Some("can not decode ReadMultipleReq".to_string()), + })); + } + }; + match disk.disk_info(&opts).await { + Ok(disk_info) => match serde_json::to_string(&disk_info) { + Ok(disk_info) => Ok(tonic::Response::new(DiskInfoResponse { + success: true, + disk_info, + error_info: None, + })), + Err(err) => Ok(tonic::Response::new(DiskInfoResponse { + success: false, + disk_info: "".to_string(), + error_info: Some(err.to_string()), + })), + }, + Err(err) => Ok(tonic::Response::new(DiskInfoResponse { + success: false, + disk_info: "".to_string(), + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(DiskInfoResponse { + success: false, + disk_info: "".to_string(), + error_info: Some("can not find disk".to_string()), + })) + } + } + async fn lock(&self, request: Request) -> Result, Status> { let request = request.into_inner(); match &serde_json::from_str::(&request.args) { From 3f2772c74c9855d01eb95baf34d97524b8a403e9 Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Tue, 15 Oct 2024 18:28:55 +0800 Subject: [PATCH 02/12] healing tracker Signed-off-by: junxiang Mu <1948535941@qq.com> --- Cargo.lock | 19 +++ ecstore/Cargo.toml | 1 + ecstore/src/cache/cache.rs | 119 +++++++++++++++ ecstore/src/cache/mod.rs | 1 + ecstore/src/cache_value/cache.rs | 81 ++++++---- ecstore/src/disk/local.rs | 74 +++++++++- ecstore/src/disk/mod.rs | 24 ++- ecstore/src/disk/os.rs | 10 +- ecstore/src/disk/remote.rs | 4 +- ecstore/src/global.rs | 3 +- ecstore/src/heal/heal_commands.rs | 237 ++++++++++++++++++++++++++++++ ecstore/src/heal/heal_ops.rs | 7 +- ecstore/src/store.rs | 14 ++ ecstore/src/store_api.rs | 1 + ecstore/src/utils/mod.rs | 1 + ecstore/src/utils/stat_linux.rs | 84 +++++++++++ 16 files changed, 636 insertions(+), 44 deletions(-) create mode 100644 ecstore/src/cache/cache.rs create mode 100644 ecstore/src/cache/mod.rs create mode 100644 ecstore/src/utils/stat_linux.rs diff --git a/Cargo.lock b/Cargo.lock index c908b52d8..aba59a7fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -327,6 +327,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chrono" version = "0.4.38" @@ -484,6 +490,7 @@ dependencies = [ "lazy_static", "lock", "netif", + "nix", "num_cpus", "openssl", "path-absolutize", @@ -1105,6 +1112,18 @@ dependencies = [ "winapi", ] +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.6.0", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "7.1.3" diff --git a/ecstore/Cargo.toml b/ecstore/Cargo.toml index 9a6a3f6c7..89eef4670 100644 --- a/ecstore/Cargo.toml +++ b/ecstore/Cargo.toml @@ -31,6 +31,7 @@ lazy_static.workspace = true lock.workspace = true regex = "1.11.0" netif = "0.1.6" +nix = { version = "0.29.0", features = ["fs"] } path-absolutize = "3.1.1" protos.workspace = true rmp-serde = "1.3.0" diff --git a/ecstore/src/cache/cache.rs b/ecstore/src/cache/cache.rs new file mode 100644 index 000000000..b5e0059b0 --- /dev/null +++ b/ecstore/src/cache/cache.rs @@ -0,0 +1,119 @@ +use std::{ + fmt::Debug, + ptr, + sync::{ + atomic::{AtomicPtr, AtomicU64, Ordering}, + Arc, + }, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use serde::{Deserialize, Serialize}; +use tokio::{spawn, sync::Mutex}; + +use crate::error::Result; + +type UpdateFn = Arc Result + Send + Sync>; + +#[derive(Clone, Debug, Default)] +pub struct Opts { + return_last_good: bool, + no_wait: bool, +} + +#[derive(Deserialize, Serialize)] +pub struct Cache { + update_fn: UpdateFn, + ttl: Duration, + opts: Opts, + val: AtomicPtr, + last_update_ms: AtomicU64, + updating: Arc>, +} + +impl Cache { + pub fn new(update_fn: UpdateFn, ttl: Duration, opts: Opts) -> Self { + let val = AtomicPtr::new(ptr::null_mut()); + Self { + update_fn, + ttl, + opts, + val, + last_update_ms: AtomicU64::new(0), + updating: Arc::new(Mutex::new(false)), + } + } + + pub async fn get(self: Arc) -> Result { + let v_ptr = self.val.load(Ordering::SeqCst); + let v = if v_ptr.is_null() { + None + } else { + Some(unsafe { (*v_ptr).clone() }) + }; + + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs(); + if v.is_some() && now - self.last_update_ms.load(Ordering::SeqCst) < self.ttl.as_secs() { + return Ok(v.unwrap()); + } + + if self.opts.no_wait && v.is_some() && now - self.last_update_ms.load(Ordering::SeqCst) < self.ttl.as_secs() * 2 { + if self.updating.try_lock().is_ok() { + let this = Arc::clone(&self); + spawn(async move { + let _ = this.update().await; + }); + } + + return Ok(v.unwrap()); + } + + let _ = self.updating.lock().await; + + if let Ok(duration) = + SystemTime::now().duration_since(UNIX_EPOCH + Duration::from_secs(self.last_update_ms.load(Ordering::SeqCst))) + { + if duration < self.ttl { + return Ok(v.unwrap()); + } + } + + match self.update().await { + Ok(_) => { + let v_ptr = self.val.load(Ordering::SeqCst); + let v = if v_ptr.is_null() { + None + } else { + Some(unsafe { (*v_ptr).clone() }) + }; + Ok(v.unwrap()) + } + Err(err) => Err(err), + } + } + + async fn update(&self) -> Result<()> { + match (self.update_fn)() { + Ok(val) => { + self.val.store(Box::into_raw(Box::new(val)), Ordering::SeqCst); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs(); + self.last_update_ms.store(now, Ordering::SeqCst); + Ok(()) + } + Err(err) => { + let v_ptr = self.val.load(Ordering::SeqCst); + if self.opts.return_last_good && !v_ptr.is_null() { + return Ok(()); + } + + return Err(err); + } + } + } +} diff --git a/ecstore/src/cache/mod.rs b/ecstore/src/cache/mod.rs new file mode 100644 index 000000000..a5c08fdb0 --- /dev/null +++ b/ecstore/src/cache/mod.rs @@ -0,0 +1 @@ +pub mod cache; diff --git a/ecstore/src/cache_value/cache.rs b/ecstore/src/cache_value/cache.rs index 231263b61..77dd104b1 100644 --- a/ecstore/src/cache_value/cache.rs +++ b/ecstore/src/cache_value/cache.rs @@ -1,14 +1,18 @@ use std::{ + fmt::Debug, ptr, - sync::{atomic::{AtomicPtr, AtomicU64, Ordering}, Arc, Mutex}, + sync::{ + atomic::{AtomicPtr, AtomicU64, Ordering}, + Arc, + }, time::{Duration, SystemTime, UNIX_EPOCH}, }; -use tokio::spawn; +use tokio::{spawn, sync::Mutex}; -use crate::error::{Error, Result}; +use crate::error::Result; -type UpdateFn = Box Result>; +type UpdateFn = Arc Result + Send + Sync>; #[derive(Clone, Debug, Default)] pub struct Opts { @@ -16,7 +20,7 @@ pub struct Opts { no_wait: bool, } -pub struct Cache { +pub struct Cache { update_fn: UpdateFn, ttl: Duration, opts: Opts, @@ -25,7 +29,7 @@ pub struct Cache { updating: Arc>, } -impl Cache { +impl Cache { pub fn new(update_fn: UpdateFn, ttl: Duration, opts: Opts) -> Self { let val = AtomicPtr::new(ptr::null_mut()); Self { @@ -38,9 +42,13 @@ impl Cache { } } - pub fn get(&self) -> Result { + pub async fn get(self: Arc) -> Result { let v_ptr = self.val.load(Ordering::SeqCst); - let v = if v_ptr.is_null() { None } else { Some(unsafe { *v_ptr }) }; + let v = if v_ptr.is_null() { + None + } else { + Some(unsafe { (*v_ptr).clone() }) + }; let now = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -52,37 +60,58 @@ impl Cache { if self.opts.no_wait && v.is_some() && now - self.last_update_ms.load(Ordering::SeqCst) < self.ttl.as_secs() * 2 { if self.updating.try_lock().is_ok() { - let this = self.clone(); - spawn(async { + let this = Arc::clone(&self); + spawn(async move { let _ = this.update().await; }); - self.update(); } return Ok(v.unwrap()); } - match self.updating.lock() { - Ok(_) => { - if let Ok(duration) = SystemTime::now().duration_since(UNIX_EPOCH + Duration::from_secs(self.last_update_ms.load(Ordering::SeqCst))) { - if duration < self.ttl { - return Ok(v.unwrap()); - } - } + let _ = self.updating.lock().await; - match self.update() { - - } - }, - Err(err) => { - return Err(Error::from_string(err.to_string())); + if let Ok(duration) = + SystemTime::now().duration_since(UNIX_EPOCH + Duration::from_secs(self.last_update_ms.load(Ordering::SeqCst))) + { + if duration < self.ttl { + return Ok(v.unwrap()); } } - todo!() + match self.update().await { + Ok(_) => { + let v_ptr = self.val.load(Ordering::SeqCst); + let v = if v_ptr.is_null() { + None + } else { + Some(unsafe { (*v_ptr).clone() }) + }; + Ok(v.unwrap()) + } + Err(err) => Err(err), + } } async fn update(&self) -> Result<()> { - todo!() + match (self.update_fn)() { + Ok(val) => { + self.val.store(Box::into_raw(Box::new(val)), Ordering::SeqCst); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs(); + self.last_update_ms.store(now, Ordering::SeqCst); + Ok(()) + } + Err(err) => { + let v_ptr = self.val.load(Ordering::SeqCst); + if self.opts.return_last_good && !v_ptr.is_null() { + return Ok(()); + } + + return Err(err); + } + } } } diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 00adf0b5c..9fda27fb9 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -1,10 +1,12 @@ use super::error::{is_sys_err_io, is_sys_err_not_empty, is_sys_err_too_many_files, os_is_not_exist, os_is_permission}; +use super::os::is_root_disk; use super::{endpoint::Endpoint, error::DiskError, format::FormatV3}; use super::{ os, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions, FileReader, FileWriter, - MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, - WalkDirOptions, + Info, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, + WalkDirOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET, }; +use crate::cache_value::cache::Cache; use crate::disk::error::{ convert_access_error, is_sys_err_handle_invalid, is_sys_err_invalid_arg, is_sys_err_is_dir, is_sys_err_not_dir, map_err_not_exists, os_err_to_file_err, @@ -12,14 +14,19 @@ use crate::disk::error::{ use crate::disk::os::check_path_length; use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE}; use crate::error::{Error, Result}; -use crate::utils::fs::{lstat, O_APPEND, O_CREATE, O_RDONLY, O_WRONLY}; +use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold}; +use crate::heal::heal_commands::HealingTracker; +use crate::heal::heal_ops::HEALING_TRACKER_FILENAME; +use crate::utils::fs::{lstat, read_file, O_APPEND, O_CREATE, O_RDONLY, O_WRONLY}; use crate::utils::path::{clean, has_suffix, SLASH_SEPARATOR}; +use crate::utils::stat_linux::get_info; use crate::{ file_meta::FileMeta, store_api::{FileInfo, RawFileInfo}, utils, }; use path_absolutize::Absolutize; +use std::fmt::Debug; use std::{ fs::Metadata, path::{Path, PathBuf}, @@ -49,18 +56,29 @@ impl FormatInfo { } } -#[derive(Debug)] pub struct LocalDisk { pub root: PathBuf, pub format_path: PathBuf, pub format_info: RwLock, pub endpoint: Endpoint, + disk_info_cache: Cache, // pub id: Mutex>, // pub format_data: Mutex>, // pub format_file_info: Mutex>, // pub format_last_check: Mutex>, } +impl Debug for LocalDisk { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LocalDisk") + .field("root", &self.root) + .field("format_path", &self.format_path) + .field("format_info", &self.format_info) + .field("endpoint", &self.endpoint) + .finish() + } +} + impl LocalDisk { pub async fn new(ep: &Endpoint, cleanup: bool) -> Result { let root = fs::canonicalize(ep.url.path()).await?; @@ -101,6 +119,33 @@ impl LocalDisk { last_check: format_last_check, }; + let derive_path = root.to_string_lossy().to_string(); + let disk_id = id.map_or("".to_string(), |id| id.to_string()); + let update_fn = || async move { + if let Ok((info, root)) = get_disk_info(&derive_path).await { + let mut disk_info = DiskInfo { + total: info.total, + free: info.free, + used: info.used, + used_inodes: info.files - info.ffree, + free_inodes: info.ffree, + major: info.major, + minor: info.minor, + fs_type: info.fstype, + root_disk: root, + id: disk_id, + ..Default::default() + }; + if root { + return Err(Error::new(DiskError::DriveIsRoot)); + } + + // disk_info.healing = + } else { + return Ok(DiskInfo::default()); + } + }; + // TODO: DIRECT suport // TODD: DiskInfo let disk = Self { @@ -1612,6 +1657,27 @@ impl DiskAPI for LocalDisk { } } +async fn get_disk_info(drive_path: &str) -> Result<(Info, bool)> { + check_path_length(drive_path)?; + + let disk_info = get_info(drive_path, false)?; + let root_drive = if !*GLOBAL_IsErasureSD.read().await { + let root_disk_threshold = *GLOBAL_RootDiskThreshold.read().await; + if root_disk_threshold > 0 { + disk_info.total <= root_disk_threshold + } else { + match is_root_disk(drive_path, SLASH_SEPARATOR) { + Ok(result) => result, + Err(_) => false, + } + } + } else { + false + }; + + Ok((disk_info, root_drive)) +} + #[cfg(test)] mod test { diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs index 9ef049e3d..9ab82343c 100644 --- a/ecstore/src/disk/mod.rs +++ b/ecstore/src/disk/mod.rs @@ -14,10 +14,7 @@ pub const FORMAT_CONFIG_FILE: &str = "format.json"; const STORAGE_FORMAT_FILE: &str = "xl.meta"; use crate::{ - erasure::{ReadAt, Write}, - error::{Error, Result}, - file_meta::FileMeta, - store_api::{FileInfo, RawFileInfo}, + erasure::{ReadAt, Write}, error::{Error, Result}, file_meta::FileMeta, heal::heal_commands::HealingTracker, store_api::{FileInfo, RawFileInfo} }; use endpoint::Endpoint; @@ -167,8 +164,8 @@ pub struct DiskInfo { pub used: u64, pub used_inodes: u64, pub free_inodes: u64, - pub major: u32, - pub minor: u32, + pub major: u64, + pub minor: u64, pub nr_requests: u64, pub fs_type: String, pub root_disk: bool, @@ -192,6 +189,21 @@ pub struct DiskMetrics { total_deletes: u64, } +#[derive(Clone, Debug, Default)] +pub struct Info { + pub total: u64, + pub free: u64, + pub used: u64, + pub files: u64, + pub ffree: u64, + pub fstype: String, + pub major: u64, + pub minor: u64, + pub name: String, + pub rotational: bool, + pub nrrequests: u64, +} + #[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct FileInfoVersions { // Name of the volume. diff --git a/ecstore/src/disk/os.rs b/ecstore/src/disk/os.rs index 367a36ccd..75f43166e 100644 --- a/ecstore/src/disk/os.rs +++ b/ecstore/src/disk/os.rs @@ -8,7 +8,7 @@ use tokio::fs; use crate::{ disk::error::{is_sys_err_not_dir, is_sys_err_path_not_found, os_is_not_exist}, error::{Error, Result}, - utils, + utils::{self, stat_linux::same_disk}, }; use super::error::{os_err_to_file_err, os_is_exist, DiskError}; @@ -51,6 +51,14 @@ pub fn check_path_length(path_name: &str) -> Result<()> { Ok(()) } +pub fn is_root_disk(disk_path: &str, root_disk: &str) -> Result { + if cfg!(target_os = "windows") { + return Ok(false); + } + + same_disk(disk_path, root_disk) +} + pub async fn make_dir_all(path: impl AsRef, base_dir: impl AsRef) -> Result<()> { check_path_length(path.as_ref().to_string_lossy().to_string().as_str())?; diff --git a/ecstore/src/disk/remote.rs b/ecstore/src/disk/remote.rs index 4b82c43cd..d66d9b947 100644 --- a/ecstore/src/disk/remote.rs +++ b/ecstore/src/disk/remote.rs @@ -20,9 +20,7 @@ use super::{ RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, }; use crate::{ - disk::error::DiskError, - error::{Error, Result}, - store_api::{FileInfo, RawFileInfo}, + disk::error::DiskError, error::{Error, Result}, heal::heal_commands::HealingTracker, store_api::{FileInfo, RawFileInfo} }; use protos::proto_gen::node_service::RenamePartRequst; diff --git a/ecstore/src/global.rs b/ecstore/src/global.rs index a5735544e..30c94bdb7 100644 --- a/ecstore/src/global.rs +++ b/ecstore/src/global.rs @@ -13,10 +13,11 @@ lazy_static! { pub static ref GLOBAL_LOCAL_DISK: Arc>>> = Arc::new(RwLock::new(Vec::new())); static ref GLOBAL_IsErasure: RwLock = RwLock::new(false); static ref GLOBAL_IsDistErasure: RwLock = RwLock::new(false); - static ref GLOBAL_IsErasureSD: RwLock = RwLock::new(false); + pub static ref GLOBAL_IsErasureSD: RwLock = RwLock::new(false); pub static ref GLOBAL_LOCAL_DISK_MAP: Arc>>> = Arc::new(RwLock::new(HashMap::new())); pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc> = Arc::new(RwLock::new(Vec::new())); pub static ref GLOBAL_Endpoints: RwLock = RwLock::new(EndpointServerPools(Vec::new())); + pub static ref GLOBAL_RootDiskThreshold: RwLock = RwLock::new(0); } pub async fn set_global_endpoints(eps: Vec) { diff --git a/ecstore/src/heal/heal_commands.rs b/ecstore/src/heal/heal_commands.rs index e6cf29b6b..827543690 100644 --- a/ecstore/src/heal/heal_commands.rs +++ b/ecstore/src/heal/heal_commands.rs @@ -1,3 +1,13 @@ +use std::{path::Path, time::{SystemTime, UNIX_EPOCH}}; + +use serde::{Deserialize, Serialize}; +use tokio::sync::RwLock; + +use crate::{ + disk::{DiskStore, BUCKET_META_PREFIX, RUSTFS_META_BUCKET}, + error::{Error, Result}, heal::heal_ops::HEALING_TRACKER_FILENAME, new_object_layer_fn, store_api::StorageAPI, utils::fs::read_file, +}; + pub type HealScanMode = usize; pub type HealItemType = String; @@ -37,3 +47,230 @@ pub struct HealResultItem { pub after: Vec, pub object_size: usize, } + +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct HealingTracker { + #[serde(skip_serializing, skip_deserializing)] + disk: Option, + + id: String, + pool_index: Option, + set_index: Option, + disk_index: Option, + path: String, + endpoint: String, + started: u64, + last_update: u64, + + objects_total_count: u64, + objects_total_size: u64, + + items_healed: u64, + items_failed: u64, + + bytes_done: u64, + bytes_failed: u64, + + bucket: String, + object: String, + + resume_items_healed: u64, + resume_items_failed: u64, + resume_items_skipped: u64, + resume_bytes_done: u64, + resume_bytes_failed: u64, + resume_bytes_skipped: u64, + + queue_buckets: Vec, + + healed_buckets: Vec, + + heal_id: String, + + item_skipped: u64, + bytes_skipped: u64, + + retry_attempts: u64, + + finished: bool, + + #[serde(skip_serializing, skip_deserializing)] + mu: RwLock, +} + +impl HealingTracker { + pub fn marshal_msg(&self) -> Result> { + serde_json::to_string(self) + .map(|s| s.as_bytes().to_vec()) + .map_err(|err| Error::from_string(err.to_string())) + } + + pub fn unmarshal_msg(data: &[u8]) -> Result { + serde_json::from_slice::(data) + .map_err(|err| Error::from_string(err.to_string())) + } + + pub async fn reset_healing(&mut self) { + self.mu.write().await; + self.items_healed = 0; + self.items_failed = 0; + self.bytes_done = 0; + self.bytes_failed = 0; + self.resume_items_healed = 0; + self.resume_items_failed = 0; + self.resume_bytes_done = 0; + self.resume_bytes_failed = 0; + self.item_skipped = 0; + self.bytes_skipped = 0; + + self.healed_buckets = Vec::new(); + self.bucket = String::new(); + self.object = String::new(); + } + + pub async fn get_last_update(&self) -> u64 { + self.mu.read().await; + + self.last_update + } + + pub async fn get_bucket(&self) -> String { + self.mu.read().await; + + self.bucket.clone() + } + + pub async fn set_bucket(&mut self, bucket: &str) { + self.mu.write().await; + + self.bucket = bucket.to_string(); + } + + pub async fn get_object(&self) -> String { + self.mu.read().await; + + self.object.clone() + } + + pub async fn set_object(&mut self, object: &str) { + self.mu.write().await; + + self.object = object.to_string(); + } + + pub async fn update_progress(&mut self, success: bool, skipped: bool, by: u64) { + self.mu.write().await; + + if success { + self.items_healed += 1; + self.bytes_done += by; + } else if skipped { + self.item_skipped += 1; + self.bytes_skipped += by; + } else { + self.items_failed += 1; + self.bytes_failed += by; + } + } + + pub async fn update(&mut self) -> Result<()> { + if let Some(disk) = &self.disk { + if healing(&disk.path().to_string_lossy().to_string()).await?.is_none() { + return Err(Error::from_string(format!("healingTracker: drive {} is not marked as healing", self.id))); + } + self.mu.write().await; + if self.id.is_empty() || self.pool_index.is_none() || self.set_index.is_none() || self.disk_index.is_none() { + self.id = disk.get_disk_id().await?.map_or("".to_string(), |id| id.to_string()); + let disk_location = disk.get_disk_location(); + self.pool_index = disk_location.pool_idx; + self.set_index = disk_location.set_idx; + self.disk_index = disk_location.disk_idx; + } + } + + self.save().await + } + + pub async fn save(&mut self) -> Result<()> { + self.mu.write().await; + if self.pool_index.is_none() || self.set_index.is_none() || self.disk_index.is_none() { + let layer = new_object_layer_fn(); + let lock = layer.read().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(Error::from_string("Not init".to_string())), + }; + (self.pool_index, self.set_index, self.disk_index) = store.get_pool_and_set(&self.id).await?; + } + + self.last_update = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs(); + + let htracker_bytes = self.marshal_msg()?; + + if let Some(disk) = &self.disk { + let file_path = Path::new(BUCKET_META_PREFIX).join(HEALING_TRACKER_FILENAME); + return disk.write_all(RUSTFS_META_BUCKET, file_path.to_str().unwrap(), htracker_bytes).await; + } + Ok(()) + } +} + +async fn load_healing_tracker(disk: &Option) -> Result { + if let Some(disk) = disk { + let disk_id = disk.get_disk_id().await?; + if let Some(disk_id) = disk_id { + let disk_id = disk_id.to_string(); + let file_path = Path::new(BUCKET_META_PREFIX).join(HEALING_TRACKER_FILENAME); + let data = disk.read_all(RUSTFS_META_BUCKET, file_path.to_str().unwrap()).await?; + let mut healing_tracker = HealingTracker::unmarshal_msg(&data)?; + if healing_tracker.id != disk_id && !healing_tracker.id.is_empty() { + return Err(Error::from_string(format!("loadHealingTracker: drive id mismatch expected {}, got {}", healing_tracker.id, disk_id))); + } + healing_tracker.id = disk_id; + return Ok(healing_tracker); + } else { + return Err(Error::from_string("loadHealingTracker: disk not have id")); + } + + } else { + return Err(Error::from_string("loadHealingTracker: nil drive given")); + } +} + +async fn init_healing_tracker(disk: DiskStore, heal_id: String) -> Result { + let mut healing_tracker = HealingTracker::default(); + healing_tracker.id = disk.get_disk_id().await?.map_or("".to_string(), |id| id.to_string()); + healing_tracker.heal_id = heal_id; + healing_tracker.path = disk.to_string(); + healing_tracker.endpoint = disk.endpoint().to_string(); + healing_tracker.started = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs(); + let disk_location = disk.get_disk_location(); + healing_tracker.pool_index = disk_location.pool_idx; + healing_tracker.set_index = disk_location.set_idx; + healing_tracker.disk_index = disk_location.disk_idx; + healing_tracker.disk = Some(disk); + + Ok(healing_tracker) +} + +pub async fn healing(derive_path: &str) -> Result> { + let healing_file = Path::new(derive_path) + .join(RUSTFS_META_BUCKET) + .join(BUCKET_META_PREFIX) + .join(HEALING_TRACKER_FILENAME); + + let b = read_file(healing_file).await?; + if b.is_empty() { + return Ok(None); + } + + let healing_tracker = HealingTracker::unmarshal_msg(&b)?; + + Ok(Some(healing_tracker)) +} diff --git a/ecstore/src/heal/heal_ops.rs b/ecstore/src/heal/heal_ops.rs index 05877e634..67a2d5b4a 100644 --- a/ecstore/src/heal/heal_ops.rs +++ b/ecstore/src/heal/heal_ops.rs @@ -15,8 +15,9 @@ pub type HealEntryFn = Box Resul lazy_static! { static ref HEAL_NOT_STARTED_STATUS: HealStatusSummary = String::from("not started"); - static ref bgHealingUUID: String = String::from("0000-0000-0000-0000"); } +pub const BG_HEALING_UUID: &str = "0000-0000-0000-0000"; +pub const HEALING_TRACKER_FILENAME: &str = ".healing.bin"; lazy_static! {} @@ -119,7 +120,7 @@ impl HealSequence { } fn has_ended(&self) -> bool { - if self.client_token == bgHealingUUID.to_string() { + if self.client_token == BG_HEALING_UUID.to_string() { return false; } @@ -139,7 +140,7 @@ impl HealSequence { } fn heal_items(&self, buckets_only: bool) -> Result<()> { - if self.client_token == bgHealingUUID.to_string() { + if self.client_token == BG_HEALING_UUID.to_string() { return Ok(()); } diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index b767a0412..dd320e8b0 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -892,6 +892,20 @@ impl StorageAPI for ECStore { todo!() } + async fn get_pool_and_set(&self, id: &str) -> Result<(Option, Option, Option)> { + for (pool_idx, pool) in self.pools.iter().enumerate(){ + for (set_idx, set) in pool.format.erasure.sets.iter().enumerate() { + for (disk_idx, disk_id) in set.iter().enumerate() { + if disk_id.to_string() == id { + return Ok((Some(pool_idx), Some(set_idx), Some(disk_idx))); + } + } + } + } + + Err(Error::new(DiskError::DiskNotFound)) + } + async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> { let object = utils::path::encode_dir_object(object); if self.single_pool() { diff --git a/ecstore/src/store_api.rs b/ecstore/src/store_api.rs index 514a7742f..1df9741d6 100644 --- a/ecstore/src/store_api.rs +++ b/ecstore/src/store_api.rs @@ -611,5 +611,6 @@ pub trait StorageAPI: ObjectIO { async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result; async fn heal_object(&self, bucket: &str, object: &str, version_id: &str, opts: &HealOpts) -> Result; async fn heal_objects(&self, bucket: &str, prefix: &str, opts: &HealOpts, func: HealObjectFn) -> Result<()>; + async fn get_pool_and_set(&self, id: &str) -> Result<(Option, Option, Option)>; async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()>; } diff --git a/ecstore/src/utils/mod.rs b/ecstore/src/utils/mod.rs index b5ca93bd3..bf3b6e4fd 100644 --- a/ecstore/src/utils/mod.rs +++ b/ecstore/src/utils/mod.rs @@ -4,4 +4,5 @@ pub mod fs; pub mod hash; pub mod net; pub mod path; +pub mod stat_linux; pub mod wildcard; diff --git a/ecstore/src/utils/stat_linux.rs b/ecstore/src/utils/stat_linux.rs new file mode 100644 index 000000000..254c71e6b --- /dev/null +++ b/ecstore/src/utils/stat_linux.rs @@ -0,0 +1,84 @@ +use nix::sys::{ + stat::{major, minor, stat}, + statfs::{statfs, FsType}, +}; + +use crate::{ + disk::Info, + error::{Error, Result}, +}; + +use lazy_static::lazy_static; +use std::collections::HashMap; + +lazy_static! { + static ref FS_TYPE_TO_STRING_MAP: HashMap<&'static str, &'static str> = { + let mut m = HashMap::new(); + m.insert("1021994", "TMPFS"); + m.insert("137d", "EXT"); + m.insert("4244", "HFS"); + m.insert("4d44", "MSDOS"); + m.insert("52654973", "REISERFS"); + m.insert("5346544e", "NTFS"); + m.insert("58465342", "XFS"); + m.insert("61756673", "AUFS"); + m.insert("6969", "NFS"); + m.insert("ef51", "EXT2OLD"); + m.insert("ef53", "EXT4"); + m.insert("f15f", "ecryptfs"); + m.insert("794c7630", "overlayfs"); + m.insert("2fc12fc1", "zfs"); + m.insert("ff534d42", "cifs"); + m.insert("53464846", "wslfs"); + m + }; +} + +fn get_fs_type(ftype: FsType) -> String { + let binding = format!("{:?}", ftype); + let fs_type_hex = binding.as_str(); + match FS_TYPE_TO_STRING_MAP.get(fs_type_hex) { + Some(fs_type_string) => fs_type_string.to_string(), + None => "UNKNOWN".to_string(), + } +} + +pub fn get_info(path: &str, first_time: bool) -> Result { + let statfs = statfs(path)?; + let reserved_blocks = statfs.blocks_free() - statfs.blocks_available(); + let mut info = Info { + total: statfs.block_size() as u64 * (statfs.blocks() - reserved_blocks), + free: statfs.blocks() as u64 * statfs.blocks_available(), + files: statfs.files(), + ffree: statfs.files_free(), + fstype: get_fs_type(statfs.filesystem_type()), + ..Default::default() + }; + + let stat = stat(path)?; + let dev_id = stat.st_dev as u64; + info.major = major(dev_id); + info.minor = minor(dev_id); + + if info.free > info.total { + return Err(Error::from_string(format!( + "detected free space {} > total drive space {}, fs corruption at {}. please run 'fsck'", + info.free, info.total, path + ))); + } + + info.used = info.total - info.free; + + if first_time { + // todo + } + + Ok(info) +} + +pub fn same_disk(disk1: &str, disk2: &str) -> Result { + let stat1 = stat(disk1)?; + let stat2 = stat(disk2)?; + + Ok(stat1.st_dev == stat2.st_dev) +} From 76fb6775d1279178888af94e3934d2cb39c4c401 Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Mon, 21 Oct 2024 16:38:50 +0800 Subject: [PATCH 03/12] heal object Signed-off-by: junxiang Mu <1948535941@qq.com> --- ecstore/src/bucket/metadata.rs | 11 - ecstore/src/cache/cache.rs | 119 ------ ecstore/src/cache/mod.rs | 1 - ecstore/src/cache_value/cache.rs | 2 +- ecstore/src/cache_value/metacache_set.rs | 230 +++++++++++ ecstore/src/cache_value/mod.rs | 1 + ecstore/src/disk/endpoint.rs | 2 +- ecstore/src/disk/local.rs | 127 ++++-- ecstore/src/disk/mod.rs | 197 ++++++++- ecstore/src/disk/remote.rs | 4 +- ecstore/src/global.rs | 4 +- ecstore/src/heal/background_heal_ops.rs | 89 ++++ ecstore/src/heal/heal_commands.rs | 297 +++++++++++--- ecstore/src/heal/heal_ops.rs | 494 ++++++++++++++++++++--- ecstore/src/heal/mod.rs | 1 + ecstore/src/sets.rs | 28 +- ecstore/src/store.rs | 128 +++--- ecstore/src/utils/path.rs | 12 + rustfs/src/grpc.rs | 20 +- 19 files changed, 1392 insertions(+), 375 deletions(-) delete mode 100644 ecstore/src/cache/cache.rs delete mode 100644 ecstore/src/cache/mod.rs create mode 100644 ecstore/src/cache_value/metacache_set.rs create mode 100644 ecstore/src/heal/background_heal_ops.rs diff --git a/ecstore/src/bucket/metadata.rs b/ecstore/src/bucket/metadata.rs index 63ca1c584..f2297e29c 100644 --- a/ecstore/src/bucket/metadata.rs +++ b/ecstore/src/bucket/metadata.rs @@ -19,7 +19,6 @@ use crate::config::common::{read_config, save_config}; use crate::error::{Error, Result}; use crate::disk::BUCKET_META_PREFIX; -use crate::file_meta::FileMetaShallowVersion; use crate::store::ECStore; pub const BUCKET_METADATA_FILE: &str = ".metadata.bin"; @@ -398,16 +397,6 @@ where s.serialize_bytes(&buf) } -#[derive(Debug, Default)] -pub struct MetadataResolutionParams { - pub dir_quorum: usize, - pub obj_quorum: usize, - pub requested_versions: usize, - pub bucket: String, - pub strict: bool, - pub candidates: Vec>, -} - #[cfg(test)] mod test { diff --git a/ecstore/src/cache/cache.rs b/ecstore/src/cache/cache.rs deleted file mode 100644 index b5e0059b0..000000000 --- a/ecstore/src/cache/cache.rs +++ /dev/null @@ -1,119 +0,0 @@ -use std::{ - fmt::Debug, - ptr, - sync::{ - atomic::{AtomicPtr, AtomicU64, Ordering}, - Arc, - }, - time::{Duration, SystemTime, UNIX_EPOCH}, -}; - -use serde::{Deserialize, Serialize}; -use tokio::{spawn, sync::Mutex}; - -use crate::error::Result; - -type UpdateFn = Arc Result + Send + Sync>; - -#[derive(Clone, Debug, Default)] -pub struct Opts { - return_last_good: bool, - no_wait: bool, -} - -#[derive(Deserialize, Serialize)] -pub struct Cache { - update_fn: UpdateFn, - ttl: Duration, - opts: Opts, - val: AtomicPtr, - last_update_ms: AtomicU64, - updating: Arc>, -} - -impl Cache { - pub fn new(update_fn: UpdateFn, ttl: Duration, opts: Opts) -> Self { - let val = AtomicPtr::new(ptr::null_mut()); - Self { - update_fn, - ttl, - opts, - val, - last_update_ms: AtomicU64::new(0), - updating: Arc::new(Mutex::new(false)), - } - } - - pub async fn get(self: Arc) -> Result { - let v_ptr = self.val.load(Ordering::SeqCst); - let v = if v_ptr.is_null() { - None - } else { - Some(unsafe { (*v_ptr).clone() }) - }; - - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("Time went backwards") - .as_secs(); - if v.is_some() && now - self.last_update_ms.load(Ordering::SeqCst) < self.ttl.as_secs() { - return Ok(v.unwrap()); - } - - if self.opts.no_wait && v.is_some() && now - self.last_update_ms.load(Ordering::SeqCst) < self.ttl.as_secs() * 2 { - if self.updating.try_lock().is_ok() { - let this = Arc::clone(&self); - spawn(async move { - let _ = this.update().await; - }); - } - - return Ok(v.unwrap()); - } - - let _ = self.updating.lock().await; - - if let Ok(duration) = - SystemTime::now().duration_since(UNIX_EPOCH + Duration::from_secs(self.last_update_ms.load(Ordering::SeqCst))) - { - if duration < self.ttl { - return Ok(v.unwrap()); - } - } - - match self.update().await { - Ok(_) => { - let v_ptr = self.val.load(Ordering::SeqCst); - let v = if v_ptr.is_null() { - None - } else { - Some(unsafe { (*v_ptr).clone() }) - }; - Ok(v.unwrap()) - } - Err(err) => Err(err), - } - } - - async fn update(&self) -> Result<()> { - match (self.update_fn)() { - Ok(val) => { - self.val.store(Box::into_raw(Box::new(val)), Ordering::SeqCst); - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("Time went backwards") - .as_secs(); - self.last_update_ms.store(now, Ordering::SeqCst); - Ok(()) - } - Err(err) => { - let v_ptr = self.val.load(Ordering::SeqCst); - if self.opts.return_last_good && !v_ptr.is_null() { - return Ok(()); - } - - return Err(err); - } - } - } -} diff --git a/ecstore/src/cache/mod.rs b/ecstore/src/cache/mod.rs deleted file mode 100644 index a5c08fdb0..000000000 --- a/ecstore/src/cache/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod cache; diff --git a/ecstore/src/cache_value/cache.rs b/ecstore/src/cache_value/cache.rs index 77dd104b1..fa4367197 100644 --- a/ecstore/src/cache_value/cache.rs +++ b/ecstore/src/cache_value/cache.rs @@ -12,7 +12,7 @@ use tokio::{spawn, sync::Mutex}; use crate::error::Result; -type UpdateFn = Arc Result + Send + Sync>; +type UpdateFn = Box Result + Send + Sync>; #[derive(Clone, Debug, Default)] pub struct Opts { diff --git a/ecstore/src/cache_value/metacache_set.rs b/ecstore/src/cache_value/metacache_set.rs new file mode 100644 index 000000000..845f5630a --- /dev/null +++ b/ecstore/src/cache_value/metacache_set.rs @@ -0,0 +1,230 @@ +use std::sync::Arc; + +use tokio::{ + spawn, + sync::{ + broadcast::Receiver as B_Receiver, + mpsc::{self}, + RwLock, + }, +}; + +use crate::{ + disk::{DiskStore, MetaCacheEntries, MetaCacheEntry, WalkDirOptions}, + error::{Error, Result}, +}; + +#[derive(Default)] +pub struct ListPathRawOptions { + pub disks: Vec>, + pub fallback_disks: Vec>, + pub bucket: String, + pub path: String, + pub recursice: bool, + pub filter_prefix: String, + pub forward_to: String, + pub min_disks: usize, + pub report_not_found: bool, + pub per_disk_limit: i32, + pub agreed: Option>, + pub partial: Option]) + Send + Sync>>, + pub finished: Option]) + Send + Sync>>, +} + +impl Clone for ListPathRawOptions { + fn clone(&self) -> Self { + Self { + disks: self.disks.clone(), + fallback_disks: self.fallback_disks.clone(), + bucket: self.bucket.clone(), + path: self.path.clone(), + recursice: self.recursice.clone(), + filter_prefix: self.filter_prefix.clone(), + forward_to: self.forward_to.clone(), + min_disks: self.min_disks.clone(), + report_not_found: self.report_not_found.clone(), + per_disk_limit: self.per_disk_limit.clone(), + ..Default::default() + } + } +} + +pub async fn list_path_raw(mut rx: B_Receiver, opts: ListPathRawOptions) -> Result<()> { + if opts.disks.is_empty() { + return Err(Error::from_string("list_path_raw: 0 drives provided")); + } + + let mut readers = Vec::with_capacity(opts.disks.len()); + let fds = Arc::new(RwLock::new(opts.fallback_disks.clone())); + for disk in opts.disks.iter() { + let disk = disk.clone(); + let opts_clone = opts.clone(); + let fds_clone = fds.clone(); + let (m_tx, m_rx) = mpsc::channel::(100); + readers.push(m_rx); + spawn(async move { + let mut need_fallback = false; + if disk.is_none() { + need_fallback = true; + } else { + match disk + .as_ref() + .unwrap() + .walk_dir(WalkDirOptions { + bucket: opts_clone.bucket.clone(), + base_dir: opts_clone.path.clone(), + recursive: opts_clone.recursice.clone(), + report_notfound: opts_clone.report_not_found, + filter_prefix: opts_clone.filter_prefix.clone(), + forward_to: opts_clone.forward_to.clone(), + limit: opts_clone.per_disk_limit, + ..Default::default() + }) + .await + { + Ok(r) => { + for v in r.iter() { + let _ = m_tx.send(v.to_owned()).await; + } + } + Err(_) => need_fallback = true, + } + } + + while need_fallback { + let f_disk = loop { + let mut fds_w = fds_clone.write().await; + if fds_w.is_empty() { + break None; + } + let fd = fds_w.remove(0); + if fd.is_some() && fd.as_ref().unwrap().is_online().await { + break fd; + } + }; + if f_disk.is_none() { + break; + } + match disk + .as_ref() + .unwrap() + .walk_dir(WalkDirOptions { + bucket: opts_clone.bucket.clone(), + base_dir: opts_clone.path.clone(), + recursive: opts_clone.recursice, + report_notfound: opts_clone.report_not_found, + filter_prefix: opts_clone.filter_prefix.clone(), + forward_to: opts_clone.forward_to.clone(), + limit: opts_clone.per_disk_limit, + ..Default::default() + }) + .await + { + Ok(r) => { + for v in r.iter() { + let _ = m_tx.send(v.to_owned()).await; + } + need_fallback = false; + } + Err(_) => break, + } + } + }); + } + + let errs: Vec> = Vec::with_capacity(readers.len()); + loop { + let mut current = MetaCacheEntry::default(); + let (mut at_eof, mut has_err, mut agree) = (0, 0, 0); + if rx.try_recv().is_ok() { + return Err(Error::from_string("canceled")); + } + let mut top_entries: Vec = Vec::with_capacity(readers.len()); + // top_entries.clear(); + + for (i, r) in readers.iter_mut().enumerate() { + if errs[i].is_none() { + has_err += 1; + continue; + } + let entry = match r.recv().await { + Some(entry) => entry, + None => { + at_eof += 1; + continue; + }, + }; + // If no current, add it. + if current.name.is_empty() { + top_entries.insert(i, entry.clone()); + current = entry; + agree += 1; + continue; + } + // If exact match, we agree. + if let Ok((_, true)) = current.matches(&entry, true) { + top_entries.insert(i, entry); + agree += 1; + continue; + } + // If only the name matches we didn't agree, but add it for resolution. + if entry.name == current.name { + top_entries.insert(i, entry); + continue; + } + // We got different entries + if entry.name > current.name { + continue; + } + // We got a new, better current. + // Clear existing entries. + top_entries.clear(); + agree += 1; + top_entries.insert(i, entry.clone()); + current = entry; + } + + if has_err > 0 && has_err > opts.disks.len() - opts.min_disks { + if let Some(finished_fn) = opts.finished.clone() { + finished_fn(&errs); + } + let mut combined_err = Vec::new(); + errs.iter().zip(opts.disks.iter()).for_each(|(err, disk)| { + match (err, disk) { + (Some(err), Some(disk)) => { + combined_err.push(format!("drive {} returned: {}", disk.to_string(), err)); + }, + (Some(err), None) => { + combined_err.push(err.to_string()); + }, + _ => {}, + } + }); + + return Err(Error::from_string(combined_err.join(", "))); + } + + // Break if all at EOF or error. + if at_eof + has_err == readers.len() { + if has_err > 0 { + if let Some(finished_fn) = opts.finished.clone() { + finished_fn(&errs); + } + break; + } + } + + if agree == readers.len() { + if let Some(agreed_fn) = opts.agreed.clone() { + agreed_fn(current); + } + continue; + } + + if let Some(partial_fn) = opts.partial.clone() { + partial_fn(MetaCacheEntries(top_entries), &errs); + } + } + + Ok(()) +} diff --git a/ecstore/src/cache_value/mod.rs b/ecstore/src/cache_value/mod.rs index a5c08fdb0..0a4c430d1 100644 --- a/ecstore/src/cache_value/mod.rs +++ b/ecstore/src/cache_value/mod.rs @@ -1 +1,2 @@ pub mod cache; +pub mod metacache_set; diff --git a/ecstore/src/disk/endpoint.rs b/ecstore/src/disk/endpoint.rs index 78ae38a26..6a6ba743f 100644 --- a/ecstore/src/disk/endpoint.rs +++ b/ecstore/src/disk/endpoint.rs @@ -16,7 +16,7 @@ pub enum EndpointType { } /// any type of endpoint. -#[derive(Debug, PartialEq, Eq, Clone)] +#[derive(Debug, PartialEq, Eq, Clone, Hash)] pub struct Endpoint { pub url: url::Url, pub is_local: bool, diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 9fda27fb9..6772cc72f 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -4,9 +4,9 @@ use super::{endpoint::Endpoint, error::DiskError, format::FormatV3}; use super::{ os, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions, FileReader, FileWriter, Info, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, - WalkDirOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET, + WalkDirOptions, }; -use crate::cache_value::cache::Cache; +use crate::cache_value::cache::{Cache, Opts}; use crate::disk::error::{ convert_access_error, is_sys_err_handle_invalid, is_sys_err_invalid_arg, is_sys_err_is_dir, is_sys_err_not_dir, map_err_not_exists, os_err_to_file_err, @@ -15,9 +15,7 @@ use crate::disk::os::check_path_length; use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE}; use crate::error::{Error, Result}; use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold}; -use crate::heal::heal_commands::HealingTracker; -use crate::heal::heal_ops::HEALING_TRACKER_FILENAME; -use crate::utils::fs::{lstat, read_file, O_APPEND, O_CREATE, O_RDONLY, O_WRONLY}; +use crate::utils::fs::{lstat, O_APPEND, O_CREATE, O_RDONLY, O_WRONLY}; use crate::utils::path::{clean, has_suffix, SLASH_SEPARATOR}; use crate::utils::stat_linux::get_info; use crate::{ @@ -26,7 +24,11 @@ use crate::{ utils, }; use path_absolutize::Absolutize; +use tokio::runtime::Runtime; use std::fmt::Debug; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::Arc; +use std::time::Duration; use std::{ fs::Metadata, path::{Path, PathBuf}, @@ -61,7 +63,13 @@ pub struct LocalDisk { pub format_path: PathBuf, pub format_info: RwLock, pub endpoint: Endpoint, - disk_info_cache: Cache, + pub disk_info_cache: Arc>, + pub scanning: AtomicU32, + pub rotational: bool, + pub fstype: String, + pub major: u64, + pub minor: u64, + pub nrrequests: u64, // pub id: Mutex>, // pub format_data: Mutex>, // pub format_file_info: Mutex>, @@ -118,46 +126,77 @@ impl LocalDisk { file_info: format_meta, last_check: format_last_check, }; - - let derive_path = root.to_string_lossy().to_string(); - let disk_id = id.map_or("".to_string(), |id| id.to_string()); - let update_fn = || async move { - if let Ok((info, root)) = get_disk_info(&derive_path).await { - let mut disk_info = DiskInfo { - total: info.total, - free: info.free, - used: info.used, - used_inodes: info.files - info.ffree, - free_inodes: info.ffree, - major: info.major, - minor: info.minor, - fs_type: info.fstype, - root_disk: root, - id: disk_id, - ..Default::default() - }; - if root { - return Err(Error::new(DiskError::DriveIsRoot)); + let root_clone = root.clone(); + let disk_id = Arc::new(id.map_or("".to_string(), |id| id.to_string())); + let update_fn = move || { + let rt = Runtime::new().unwrap(); + rt.block_on(async { + match get_disk_info(root.clone()).await { + Ok((info, root)) => { + let disk_info = DiskInfo { + total: info.total, + free: info.free, + used: info.used, + used_inodes: info.files - info.ffree, + free_inodes: info.ffree, + major: info.major, + minor: info.minor, + fs_type: info.fstype, + root_disk: root, + id: disk_id.to_string(), + ..Default::default() + }; + if root { + return Err(Error::new(DiskError::DriveIsRoot)); + } + + // disk_info.healing = + Ok(disk_info) + }, + Err(err) => { + Err(err) + } } - - // disk_info.healing = - } else { - return Ok(DiskInfo::default()); - } + }) }; - + + let cache = Cache::new(Box::new(update_fn), Duration::from_secs(1), Opts::default()); + // TODO: DIRECT suport // TODD: DiskInfo - let disk = Self { - root, + let mut disk = Self { + root: root_clone.clone(), endpoint: ep.clone(), format_path, format_info: RwLock::new(format_info), + disk_info_cache: Arc::new(cache), + scanning: AtomicU32::new(0), + rotational: Default::default(), + fstype: Default::default(), + minor: Default::default(), + major: Default::default(), + nrrequests: Default::default(), // // format_legacy, // format_file_info: Mutex::new(format_meta), // format_data: Mutex::new(format_data), // format_last_check: Mutex::new(format_last_check), }; + let (info, root) = get_disk_info(root_clone).await?; + disk.major = info.major; + disk.minor = info.minor; + disk.fstype = info.fstype; + + if root { + return Err(Error::new(DiskError::DriveIsRoot)); + } + + if info.nrrequests > 0 { + disk.nrrequests = info.nrrequests; + } + + if info.rotational { + disk.rotational = true; + } disk.make_meta_volumes().await?; @@ -607,6 +646,7 @@ impl LocalDisk { Ok(f) } + #[allow(dead_code)] fn get_metrics(&self) -> DiskMetrics { DiskMetrics::default() } @@ -1650,23 +1690,30 @@ impl DiskAPI for LocalDisk { Ok(()) } - async fn disk_info(&self, opts: &DiskInfoOptions) -> Result { - let mut info = DiskInfo::default(); + async fn disk_info(&self, _: &DiskInfoOptions) -> Result { + let mut info = Cache::get(self.disk_info_cache.clone()).await?; + // TODO: nr_requests, rotational + info.nr_requests = self.nrrequests; + info.rotational = self.rotational; + info.mount_path = self.path().to_str().unwrap().to_string(); + info.endpoint = self.endpoint.to_string(); + info.scanning = self.scanning.load(Ordering::SeqCst) == 1; Ok(info) } } -async fn get_disk_info(drive_path: &str) -> Result<(Info, bool)> { - check_path_length(drive_path)?; +async fn get_disk_info(drive_path: PathBuf) -> Result<(Info, bool)> { + let drive_path = drive_path.to_string_lossy().to_string(); + check_path_length(&drive_path)?; - let disk_info = get_info(drive_path, false)?; + let disk_info = get_info(&drive_path, false)?; let root_drive = if !*GLOBAL_IsErasureSD.read().await { let root_disk_threshold = *GLOBAL_RootDiskThreshold.read().await; if root_disk_threshold > 0 { disk_info.total <= root_disk_threshold } else { - match is_root_disk(drive_path, SLASH_SEPARATOR) { + match is_root_disk(&drive_path, SLASH_SEPARATOR) { Ok(result) => result, Err(_) => false, } diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs index 9ab82343c..0fa68ff2f 100644 --- a/ecstore/src/disk/mod.rs +++ b/ecstore/src/disk/mod.rs @@ -14,7 +14,10 @@ pub const FORMAT_CONFIG_FILE: &str = "format.json"; const STORAGE_FORMAT_FILE: &str = "xl.meta"; use crate::{ - erasure::{ReadAt, Write}, error::{Error, Result}, file_meta::FileMeta, heal::heal_commands::HealingTracker, store_api::{FileInfo, RawFileInfo} + erasure::{ReadAt, Write}, + error::{Error, Result}, + file_meta::{merge_file_meta_versions, FileMeta, FileMetaShallowVersion}, + store_api::{FileInfo, RawFileInfo}, }; use endpoint::Endpoint; @@ -23,7 +26,7 @@ use protos::proto_gen::node_service::{ node_service_client::NodeServiceClient, ReadAtRequest, ReadAtResponse, WriteRequest, WriteResponse, }; use serde::{Deserialize, Serialize}; -use std::{collections::HashMap, fmt::Debug, io::SeekFrom, path::PathBuf, sync::Arc, usize}; +use std::{cmp::Ordering, collections::HashMap, fmt::Debug, io::SeekFrom, path::PathBuf, sync::Arc, usize}; use time::OffsetDateTime; use tokio::{ fs::File, @@ -247,7 +250,17 @@ pub struct WalkDirOptions { pub disk_id: String, } -#[derive(Debug, Default, Serialize, Deserialize)] +#[derive(Clone, Debug, Default)] +pub struct MetadataResolutionParams { + pub dir_quorum: usize, + pub obj_quorum: usize, + pub requested_versions: usize, + pub bucket: String, + pub strict: bool, + pub candidates: Vec>, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct MetaCacheEntry { // name is the full name of the object including prefixes pub name: String, @@ -335,6 +348,184 @@ impl MetaCacheEntry { Ok(fm.into_file_info_versions(bucket, self.name.as_str(), false)?) } + + pub fn matches(&self, other: &MetaCacheEntry, strict: bool) -> Result<(Option, bool)> { + let mut prefer = None; + if self.name != other.name { + if self.name < other.name { + return Ok((Some(self.clone()), false)); + } + return Ok((Some(other.clone()), false)); + } + + if other.is_dir() || self.is_dir() { + if self.is_dir() { + return Ok((Some(self.clone()), other.is_dir())); + } + + return Ok((Some(other.clone()), other.is_dir() == self.is_dir())); + } + let self_vers = match &self.cached { + Some(file_meta) => file_meta.clone(), + None => FileMeta::load(&self.metadata)?, + }; + let other_vers = match &other.cached { + Some(file_meta) => file_meta.clone(), + None => FileMeta::load(&other.metadata)?, + }; + + if self_vers.versions.len() != other_vers.versions.len() { + match self_vers.lastest_mod_time().cmp(&other_vers.lastest_mod_time()) { + Ordering::Greater => { + return Ok((Some(self.clone()), false)); + } + Ordering::Less => { + return Ok((Some(self.clone()), false)); + } + _ => {} + } + + if self_vers.versions.len() > other_vers.versions.len() { + return Ok((Some(self.clone()), false)); + } + return Ok((Some(self.clone()), false)); + } + + for (s_version, o_version) in self_vers.versions.iter().zip(other_vers.versions.iter()) { + if s_version.header != o_version.header { + if s_version.header.has_ec() != o_version.header.has_ec() { + // One version has EC and the other doesn't - may have been written later. + // Compare without considering EC. + let (mut a, mut b) = (s_version.header.clone(), o_version.header.clone()); + (a.ec_n, a.ec_m, b.ec_n, b.ec_m) = (0, 0, 0, 0); + if a == b { + continue; + } + } + + if !strict && s_version.header.matches_not_strict(&o_version.header) { + if prefer.is_none() { + if s_version.header.sorts_before(&o_version.header) { + prefer = Some(self.clone()); + } else { + prefer = Some(other.clone()); + } + } + + continue; + } + + if prefer.is_some() { + return Ok((prefer, false)); + } + + if s_version.header.sorts_before(&o_version.header) { + return Ok((Some(self.clone()), false)); + } + + return Ok((Some(other.clone()), false)); + } + } + + if prefer.is_none() { + prefer = Some(self.clone()); + } + + Ok((prefer, true)) + } +} + +pub struct MetaCacheEntries(pub Vec); + +impl MetaCacheEntries { + pub fn resolve(&self, mut params: MetadataResolutionParams) -> Result> { + if self.0.is_empty() { + return Ok(None); + } + + let mut dir_exists = 0; + let mut selected = None; + + params.candidates.clear(); + let mut objs_agree = 0; + let mut objs_valid = 0; + + for entry in self.0.iter() { + if entry.name.is_empty() { + continue; + } + if entry.is_dir() { + dir_exists += 1; + selected = Some(entry.clone()); + continue; + } + + objs_valid += 1; + + match &entry.cached { + Some(file_meta) => { + params.candidates.push(file_meta.versions.clone()); + } + None => { + params.candidates.push(FileMeta::load(&entry.metadata)?.versions); + } + } + + if selected.is_none() { + selected = Some(entry.clone()); + objs_agree = 1; + continue; + } + + if let (Some(prefer), true) = entry.matches(selected.as_ref().unwrap(), params.strict)? { + selected = Some(prefer); + objs_agree += 1; + continue; + } + } + + // Return dir entries, if enough... + if selected.is_some() && selected.as_ref().unwrap().is_dir() && dir_exists >= params.dir_quorum { + return Ok(selected); + } + // If we would never be able to reach read quorum. + if objs_valid < params.obj_quorum { + return Ok(None); + } + // If all objects agree. + if selected.is_some() && objs_agree == objs_valid { + return Ok(selected); + } + // If cached is nil we shall skip the entry. + if selected.is_none() || (selected.is_some() && selected.as_ref().unwrap().cached.is_none()) { + return Ok(None); + } + // Merge if we have disagreement. + // Create a new merged result. + selected = Some(MetaCacheEntry { + name: selected.as_ref().unwrap().name.clone(), + cached: Some(FileMeta { + meta_ver: selected.as_ref().unwrap().cached.as_ref().unwrap().meta_ver.clone(), + ..Default::default() + }), + _reusable: true, + ..Default::default() + }); + + selected.as_mut().unwrap().cached.as_mut().unwrap().versions = + merge_file_meta_versions(params.obj_quorum, params.strict, params.requested_versions, ¶ms.candidates); + if selected.as_ref().unwrap().cached.as_ref().unwrap().versions.is_empty() { + return Ok(None); + } + + selected.as_mut().unwrap().metadata = selected.as_ref().unwrap().cached.as_ref().unwrap().marshal_msg()?; + + Ok(selected) + } + + pub fn first_found(&self) -> (Option, usize) { + (self.0.iter().find(|x| !x.name.is_empty()).cloned(), self.0.len()) + } } #[derive(Debug, Default)] diff --git a/ecstore/src/disk/remote.rs b/ecstore/src/disk/remote.rs index d66d9b947..4b82c43cd 100644 --- a/ecstore/src/disk/remote.rs +++ b/ecstore/src/disk/remote.rs @@ -20,7 +20,9 @@ use super::{ RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, }; use crate::{ - disk::error::DiskError, error::{Error, Result}, heal::heal_commands::HealingTracker, store_api::{FileInfo, RawFileInfo} + disk::error::DiskError, + error::{Error, Result}, + store_api::{FileInfo, RawFileInfo}, }; use protos::proto_gen::node_service::RenamePartRequst; diff --git a/ecstore/src/global.rs b/ecstore/src/global.rs index 30c94bdb7..c263e0f55 100644 --- a/ecstore/src/global.rs +++ b/ecstore/src/global.rs @@ -11,8 +11,8 @@ use crate::{ lazy_static! { pub static ref GLOBAL_OBJECT_API: Arc>> = Arc::new(RwLock::new(None)); pub static ref GLOBAL_LOCAL_DISK: Arc>>> = Arc::new(RwLock::new(Vec::new())); - static ref GLOBAL_IsErasure: RwLock = RwLock::new(false); - static ref GLOBAL_IsDistErasure: RwLock = RwLock::new(false); + pub static ref GLOBAL_IsErasure: RwLock = RwLock::new(false); + pub static ref GLOBAL_IsDistErasure: RwLock = RwLock::new(false); pub static ref GLOBAL_IsErasureSD: RwLock = RwLock::new(false); pub static ref GLOBAL_LOCAL_DISK_MAP: Arc>>> = Arc::new(RwLock::new(HashMap::new())); pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc> = Arc::new(RwLock::new(Vec::new())); diff --git a/ecstore/src/heal/background_heal_ops.rs b/ecstore/src/heal/background_heal_ops.rs new file mode 100644 index 000000000..136d56836 --- /dev/null +++ b/ecstore/src/heal/background_heal_ops.rs @@ -0,0 +1,89 @@ +use std::sync::Arc; + +use tokio::{ + select, + sync::{ + broadcast::Receiver as B_Receiver, + mpsc::{self, Receiver, Sender}, + }, +}; + +use crate::{ + error::Error, + heal::heal_ops::NOP_HEAL, + utils::path::SLASH_SEPARATOR, +}; + +use super::{ + heal_commands::{HealOpts, HealResultItem}, + heal_ops::HealSequence, +}; + +#[derive(Clone, Debug)] +pub struct HealTask { + pub bucket: String, + pub object: String, + pub version_id: String, + pub opts: HealOpts, + pub resp_tx: Arc>, + pub resp_rx: Arc>, +} + +impl HealTask { + pub fn new(bucket: &str, object: &str, version_id: &str, opts: &HealOpts) -> Self { + let (tx, rx) = mpsc::channel(10); + Self { + bucket: bucket.to_string(), + object: object.to_string(), + version_id: version_id.to_string(), + opts: opts.clone(), + resp_tx: tx.into(), + resp_rx: rx.into(), + } + } +} + +pub struct HealResult { + pub result: HealResultItem, + err: Error, +} + +pub struct HealRoutine { + tasks_tx: Sender, + tasks_rx: Receiver, + workers: usize, +} + +impl HealRoutine { + pub async fn add_worker(&mut self, mut ctx: B_Receiver, bgseq: &HealSequence) { + loop { + select! { + task = self.tasks_rx.recv() => { + let mut res = HealResultItem::default(); + let mut err: Error; + match task { + Some(task) => { + if task.bucket == NOP_HEAL { + err = Error::from_string("skip file"); + } else if task.bucket == SLASH_SEPARATOR { + (res, err) = heal_disk_format(task.opts).await; + } + }, + None => return, + } + } + _ = ctx.recv() => { + return; + } + } + } + } +} + +// pub fn active_listeners() -> Result { + +// } + +async fn heal_disk_format(opts: HealOpts) -> (HealResultItem, Error) { + todo!() +} diff --git a/ecstore/src/heal/heal_commands.rs b/ecstore/src/heal/heal_commands.rs index 827543690..d6fff5eb1 100644 --- a/ecstore/src/heal/heal_commands.rs +++ b/ecstore/src/heal/heal_commands.rs @@ -1,16 +1,27 @@ -use std::{path::Path, time::{SystemTime, UNIX_EPOCH}}; +use std::{ + path::Path, + time::{SystemTime, UNIX_EPOCH}, +}; use serde::{Deserialize, Serialize}; use tokio::sync::RwLock; use crate::{ - disk::{DiskStore, BUCKET_META_PREFIX, RUSTFS_META_BUCKET}, - error::{Error, Result}, heal::heal_ops::HEALING_TRACKER_FILENAME, new_object_layer_fn, store_api::StorageAPI, utils::fs::read_file, + disk::{DeleteOptions, DiskStore, BUCKET_META_PREFIX, RUSTFS_META_BUCKET}, + error::{Error, Result}, + heal::heal_ops::HEALING_TRACKER_FILENAME, + new_object_layer_fn, + store_api::{BucketInfo, StorageAPI}, + utils::fs::read_file, }; pub type HealScanMode = usize; pub type HealItemType = String; +pub const HEAL_UNKNOWN_SCAN: HealScanMode = 0; +pub const HEAL_NORMAL_SCAN: HealScanMode = 1; +pub const HEAL_DEEP_SCAN: HealScanMode = 2; + #[derive(Clone, Copy, Debug, Default)] pub struct HealOpts { pub recursive: bool, @@ -24,14 +35,14 @@ pub struct HealOpts { pub set: Option, } -#[derive(Debug)] +#[derive(Clone, Debug)] struct HealDriveInfo { uuid: String, endpoint: String, state: String, } -#[derive(Debug)] +#[derive(Clone, Debug, Default)] pub struct HealResultItem { pub result_index: usize, pub heal_item_type: HealItemType, @@ -48,54 +59,78 @@ pub struct HealResultItem { pub object_size: usize, } +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct HealStartSuccess { + pub client_token: String, + pub client_address: String, + pub start_time: u64, +} + +pub type HealStopSuccess = HealStartSuccess; + +pub struct HealingDisk { + pub id: String, + pub heal_id: String, + pub pool_index: Option, + pub set_index: Option, + pub disk_index: Option, + pub endpoint: String, + pub path: String, + pub started: u64, + pub last_update: u64, + pub retry_attempts: u64, + pub objects_total_count: u64, + pub objects_total_size: u64, + pub items_healed: u64, + pub items_failed: u64, + pub item_skipped: u64, + pub bytes_done: u64, + pub bytes_failed: u64, + pub bytes_skipped: u64, + pub objects_healed: u64, + pub objects_failed: u64, + pub bucket: String, + pub object: String, + pub queue_buckets: Vec, + pub healed_buckets: Vec, + pub finished: bool, +} + #[derive(Debug, Default, Deserialize, Serialize)] pub struct HealingTracker { #[serde(skip_serializing, skip_deserializing)] - disk: Option, - - id: String, - pool_index: Option, - set_index: Option, - disk_index: Option, - path: String, - endpoint: String, - started: u64, - last_update: u64, - - objects_total_count: u64, - objects_total_size: u64, - - items_healed: u64, - items_failed: u64, - - bytes_done: u64, - bytes_failed: u64, - - bucket: String, - object: String, - - resume_items_healed: u64, - resume_items_failed: u64, - resume_items_skipped: u64, - resume_bytes_done: u64, - resume_bytes_failed: u64, - resume_bytes_skipped: u64, - - queue_buckets: Vec, - - healed_buckets: Vec, - - heal_id: String, - - item_skipped: u64, - bytes_skipped: u64, - - retry_attempts: u64, - - finished: bool, - + pub disk: Option, + pub id: String, + pub pool_index: Option, + pub set_index: Option, + pub disk_index: Option, + pub path: String, + pub endpoint: String, + pub started: u64, + pub last_update: u64, + pub objects_total_count: u64, + pub objects_total_size: u64, + pub items_healed: u64, + pub items_failed: u64, + pub item_skipped: u64, + pub bytes_done: u64, + pub bytes_failed: u64, + pub bytes_skipped: u64, + pub bucket: String, + pub object: String, + pub resume_items_healed: u64, + pub resume_items_failed: u64, + pub resume_items_skipped: u64, + pub resume_bytes_done: u64, + pub resume_bytes_failed: u64, + pub resume_bytes_skipped: u64, + pub queue_buckets: Vec, + pub healed_buckets: Vec, + pub heal_id: String, + pub retry_attempts: u64, + pub finished: bool, #[serde(skip_serializing, skip_deserializing)] - mu: RwLock, + pub mu: RwLock, } impl HealingTracker { @@ -106,8 +141,7 @@ impl HealingTracker { } pub fn unmarshal_msg(data: &[u8]) -> Result { - serde_json::from_slice::(data) - .map_err(|err| Error::from_string(err.to_string())) + serde_json::from_slice::(data).map_err(|err| Error::from_string(err.to_string())) } pub async fn reset_healing(&mut self) { @@ -204,18 +238,157 @@ impl HealingTracker { } self.last_update = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("Time went backwards") - .as_secs(); - + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs(); + let htracker_bytes = self.marshal_msg()?; + // TODO: globalBackgroundHealState + if let Some(disk) = &self.disk { let file_path = Path::new(BUCKET_META_PREFIX).join(HEALING_TRACKER_FILENAME); - return disk.write_all(RUSTFS_META_BUCKET, file_path.to_str().unwrap(), htracker_bytes).await; + return disk + .write_all(RUSTFS_META_BUCKET, file_path.to_str().unwrap(), htracker_bytes) + .await; } Ok(()) } + + async fn delete(&self) -> Result<()> { + if let Some(disk) = &self.disk { + let file_path = Path::new(BUCKET_META_PREFIX).join(HEALING_TRACKER_FILENAME); + return disk + .delete( + RUSTFS_META_BUCKET, + file_path.to_str().unwrap(), + DeleteOptions { + recursive: false, + immediate: false, + ..Default::default() + }, + ) + .await; + } + + Ok(()) + } + + async fn is_healed(&self, bucket: &str) -> bool { + self.mu.read().await; + for v in self.healed_buckets.iter() { + if v == bucket { + return true; + } + } + + false + } + + async fn resume(&mut self) { + self.mu.write().await; + + self.items_healed = self.resume_items_healed; + self.items_failed = self.resume_items_failed; + self.item_skipped = self.resume_items_skipped; + self.bytes_done = self.resume_bytes_done; + self.bytes_failed = self.resume_bytes_failed; + self.bytes_skipped = self.resume_bytes_skipped; + } + + async fn bucket_done(&mut self, bucket: &str) { + self.mu.write().await; + + self.resume_items_healed = self.items_healed; + self.resume_items_failed = self.items_failed; + self.resume_items_skipped = self.item_skipped; + self.resume_bytes_done = self.bytes_done; + self.resume_bytes_failed = self.bytes_failed; + self.resume_bytes_skipped = self.bytes_skipped; + self.healed_buckets.push(bucket.to_string()); + + self.queue_buckets.retain(|x| x != bucket); + } + + async fn set_queue_buckets(&mut self, buckets: &[BucketInfo]) { + self.mu.write().await; + + buckets.iter().for_each(|bucket| { + if !self.healed_buckets.contains(&bucket.name) { + self.queue_buckets.push(bucket.name.clone()); + } + }); + } + + pub async fn to_healing_disk(&self) -> HealingDisk { + self.mu.read().await; + + HealingDisk { + id: self.id.clone(), + heal_id: self.heal_id.clone(), + pool_index: self.pool_index, + set_index: self.set_index, + disk_index: self.disk_index, + endpoint: self.endpoint.clone(), + path: self.path.clone(), + started: self.started, + last_update: self.last_update, + retry_attempts: self.retry_attempts, + objects_total_count: self.objects_total_count, + objects_total_size: self.objects_total_size, + items_healed: self.items_healed, + items_failed: self.items_failed, + item_skipped: self.item_skipped, + bytes_done: self.bytes_done, + bytes_failed: self.bytes_failed, + bytes_skipped: self.bytes_skipped, + objects_healed: self.items_healed, + objects_failed: self.items_failed, + bucket: self.bucket.clone(), + object: self.object.clone(), + queue_buckets: self.queue_buckets.clone(), + healed_buckets: self.healed_buckets.clone(), + finished: self.finished, + } + } +} + +impl Clone for HealingTracker { + fn clone(&self) -> Self { + Self { + disk: self.disk.clone(), + id: self.id.clone(), + pool_index: self.pool_index.clone(), + set_index: self.set_index.clone(), + disk_index: self.disk_index.clone(), + path: self.path.clone(), + endpoint: self.endpoint.clone(), + started: self.started.clone(), + last_update: self.last_update.clone(), + objects_total_count: self.objects_total_count.clone(), + objects_total_size: self.objects_total_size.clone(), + items_healed: self.items_healed.clone(), + items_failed: self.items_failed.clone(), + item_skipped: self.item_skipped.clone(), + bytes_done: self.bytes_done.clone(), + bytes_failed: self.bytes_failed.clone(), + bytes_skipped: self.bytes_skipped.clone(), + bucket: self.bucket.clone(), + object: self.object.clone(), + resume_items_healed: self.resume_items_healed.clone(), + resume_items_failed: self.resume_items_failed.clone(), + resume_items_skipped: self.resume_items_skipped.clone(), + resume_bytes_done: self.resume_bytes_done.clone(), + resume_bytes_failed: self.resume_bytes_failed.clone(), + resume_bytes_skipped: self.resume_bytes_skipped.clone(), + queue_buckets: self.queue_buckets.clone(), + healed_buckets: self.healed_buckets.clone(), + heal_id: self.heal_id.clone(), + retry_attempts: self.retry_attempts.clone(), + finished: self.finished.clone(), + mu: RwLock::new(false), + } + } } async fn load_healing_tracker(disk: &Option) -> Result { @@ -227,14 +400,16 @@ async fn load_healing_tracker(disk: &Option) -> Result Result; -pub type HealObjectFn = Box Result<()> + Send>; -pub type HealEntryFn = Box Result<()> + Send>; +pub type HealObjectFn = Arc Result<()> + Send + Sync>; +pub type HealEntryFn = + Box Pin> + Send>> + Send>; -lazy_static! { - static ref HEAL_NOT_STARTED_STATUS: HealStatusSummary = String::from("not started"); -} pub const BG_HEALING_UUID: &str = "0000-0000-0000-0000"; pub const HEALING_TRACKER_FILENAME: &str = ".healing.bin"; +const KEEP_HEAL_SEQ_STATE_DURATION: std::time::Duration = Duration::from_secs(10 * 60); +const HEAL_NOT_STARTED_STATUS: &str = "not started"; +const HEAL_RUNNING_STATUS: &str = "running"; +const HEAL_STOPPED_STATUS: &str = "stopped"; +const HEAL_FINISHED_STATUS: &str = "finished"; + +const MAX_UNCONSUMED_HEAL_RESULT_ITEMS: usize = 1000; +const HEAL_UNCONSUMED_TIMEOUT: std::time::Duration = Duration::from_secs(24 * 60 * 60); +pub const NOP_HEAL: &str = ""; lazy_static! {} -#[derive(Debug)] +#[derive(Clone, Debug, Default)] pub struct HealSequenceStatus { - summary: HealStatusSummary, - failure_detail: String, - start_time: Instant, - heal_setting: HealOpts, - items: Vec, + pub summary: HealStatusSummary, + pub failure_detail: String, + pub start_time: u64, + pub heal_setting: HealOpts, + pub items: Vec, } -impl Default for HealSequenceStatus { - fn default() -> Self { - Self { - summary: Default::default(), - failure_detail: Default::default(), - start_time: Instant::now(), - heal_setting: Default::default(), - items: Default::default(), - } - } +pub struct HealSource { + pub bucket: String, + pub object: String, + pub version_id: String, + pub no_wait: bool, + opts: Option, } -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct HealSequence { pub bucket: String, pub object: String, pub report_progress: bool, - pub start_time: Instant, - pub end_time: Instant, + pub start_time: u64, + pub end_time: Arc>, pub client_token: String, pub client_address: String, pub force_started: bool, pub setting: HealOpts, - pub current_status: HealSequenceStatus, + pub current_status: Arc>, pub last_sent_result_index: usize, pub scanned_items_map: ItemsMap, pub healed_items_map: ItemsMap, pub heal_failed_items_map: ItemsMap, - pub last_heal_activity: Instant, + pub last_heal_activity: u64, + + traverse_and_heal_done_tx: Arc>>>, + traverse_and_heal_done_rx: Arc>>>, + + tx: Arc>>, + rx: Arc>>, +} + +impl Default for HealSequence { + fn default() -> Self { + let (h_tx, h_rx) = mpsc::channel(1); + let (tx, rx) = broadcast::channel(1); + Self { + bucket: Default::default(), + object: Default::default(), + report_progress: Default::default(), + start_time: Default::default(), + end_time: Default::default(), + client_token: Default::default(), + client_address: Default::default(), + force_started: Default::default(), + setting: Default::default(), + current_status: Default::default(), + last_sent_result_index: Default::default(), + scanned_items_map: Default::default(), + healed_items_map: Default::default(), + heal_failed_items_map: Default::default(), + last_heal_activity: Default::default(), + traverse_and_heal_done_tx: Arc::new(RwLock::new(h_tx)), + traverse_and_heal_done_rx: Arc::new(RwLock::new(h_rx)), + tx: Arc::new(RwLock::new(tx)), + rx: Arc::new(RwLock::new(rx)), + } + } } impl HealSequence { @@ -73,11 +133,11 @@ impl HealSequence { client_address: client_addr.to_string(), force_started: force_start, setting: hs, - current_status: HealSequenceStatus { + current_status: Arc::new(RwLock::new(HealSequenceStatus { summary: HEAL_NOT_STARTED_STATUS.to_string(), heal_setting: hs, ..Default::default() - }, + })), ..Default::default() } } @@ -102,36 +162,100 @@ impl HealSequence { fn count_failed(&mut self, heal_type: HealItemType) { *self.heal_failed_items_map.entry(heal_type).or_insert(0) += 1; - self.last_heal_activity = Instant::now(); + self.last_heal_activity = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs(); } fn count_scanned(&mut self, heal_type: HealItemType) { *self.scanned_items_map.entry(heal_type).or_insert(0) += 1; - self.last_heal_activity = Instant::now() + self.last_heal_activity = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs(); } fn count_healed(&mut self, heal_type: HealItemType) { *self.healed_items_map.entry(heal_type).or_insert(0) += 1; - self.last_heal_activity = Instant::now() + self.last_heal_activity = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs(); } - fn is_quitting(&self) -> bool { - todo!() + async fn is_quitting(&self) -> bool { + let mut w = self.rx.write().await; + if w.try_recv().is_ok() { + return true; + } + false } - fn has_ended(&self) -> bool { + async fn has_ended(&self) -> bool { if self.client_token == BG_HEALING_UUID.to_string() { return false; } - !(self.end_time == self.start_time) + !(*(self.end_time.read().await) == self.start_time) } - fn stop(&self) { - todo!() + async fn stop(&self) { + let w = self.tx.write().await; + w.send(true); } - fn push_heal_result_item(&self, r: HealResultItem) -> Result<()> { + async fn push_heal_result_item(&self, r: &HealResultItem) -> Result<()> { + let mut r = r.clone(); + let mut interval_timer = interval(HEAL_UNCONSUMED_TIMEOUT); + let mut items_len = 0; + loop { + { + let current_status_r = self.current_status.read().await; + items_len = current_status_r.items.len(); + } + + if items_len == MAX_UNCONSUMED_HEAL_RESULT_ITEMS { + select! { + _ = sleep(Duration::from_secs(1)) => { + + } + _ = self.is_done() => { + return Err(Error::from_string("stopped")); + } + _ = interval_timer.tick() => { + return Err(Error::from_string("timeout")); + } + } + } else { + break; + } + } + + let mut current_status_w = self.current_status.write().await; + if items_len > 0 { + r.result_index = 1 + current_status_w.items[items_len - 1].result_index; + } else { + r.result_index = 1 + self.last_sent_result_index; + } + + current_status_w.items.push(r); + + Ok(()) + } + + async fn queue_heal_task(&mut self, source: HealSource, heal_type: HealItemType) -> Result<()> { + let mut task = HealTask::new(&source.bucket, &source.object, &source.version_id, &self.setting); + if let Some(opts) = source.opts { + task.opts = opts; + } else { + task.opts.scan_mode = HEAL_UNKNOWN_SCAN; + } + + self.count_scanned(heal_type); + + if source.no_wait {} + todo!() } @@ -147,33 +271,283 @@ impl HealSequence { todo!() } - fn traverse_and_heal(&self) { + async fn traverse_and_heal(&self) { let buckets_only = false; } fn heal_rustfs_sys_meta(&self, meta_prefix: String) -> Result<()> { todo!() } + + async fn is_done(&self) -> bool { + let mut rx_w = self.rx.write().await; + if let Ok(true) = rx_w.recv().await { + return true; + } + false + } } -impl Default for HealSequence { - fn default() -> Self { - Self { - bucket: Default::default(), - object: Default::default(), - report_progress: Default::default(), - start_time: Instant::now(), - end_time: Instant::now(), - client_token: Default::default(), - client_address: Default::default(), - force_started: Default::default(), - setting: Default::default(), - current_status: Default::default(), - last_sent_result_index: Default::default(), - scanned_items_map: Default::default(), - healed_items_map: Default::default(), - heal_failed_items_map: Default::default(), - last_heal_activity: Instant::now(), +pub async fn heal_sequence_start(h: Arc) { + { + let mut current_status_w = h.current_status.write().await; + (*current_status_w).summary = HEAL_RUNNING_STATUS.to_string(); + (*current_status_w).start_time = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs(); + } + + let h_clone = h.clone(); + spawn(async move { + h_clone.traverse_and_heal().await; + }); + + let h_clone_1 = h.clone(); + let mut x = h.traverse_and_heal_done_rx.write().await; + select! { + _ = h.is_done() => { + *(h.end_time.write().await) = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs(); + let mut current_status_w = h.current_status.write().await; + (*current_status_w).summary = HEAL_FINISHED_STATUS.to_string(); + + spawn(async move { + let mut rx_w = h_clone_1.traverse_and_heal_done_rx.write().await; + rx_w.recv().await; + }); + } + result = x.recv() => { + match result { + Some(err) => { + match err { + Some(err) => { + let mut current_status_w = h.current_status.write().await; + (*current_status_w).summary = HEAL_STOPPED_STATUS.to_string(); + (*current_status_w).failure_detail = err.to_string(); + }, + None => { + let mut current_status_w = h.current_status.write().await; + (*current_status_w).summary = HEAL_FINISHED_STATUS.to_string(); + } + } + }, + None => { + return; + } + } + } } } + +#[derive(Debug, Default)] +pub struct AllHealState { + mu: RwLock, + + heal_seq_map: HashMap, + heal_local_disks: HashMap, + heal_status: HashMap, +} + +impl AllHealState { + pub fn new(cleanup: bool) -> Self { + let hstate = AllHealState::default(); + if cleanup { + // spawn(f); + } + + hstate + } + + async fn pop_heal_local_disks(&mut self, heal_local_disks: &[Endpoint]) { + self.mu.write().await; + + self.heal_local_disks.retain(|k, _| { + if heal_local_disks.contains(k) { + return false; + } + true + }); + + let heal_local_disks = heal_local_disks.iter().map(|s| s.to_string()).collect::>(); + self.heal_status.retain(|_, v| { + if heal_local_disks.contains(&v.endpoint) { + return false; + } + + true + }); + } + + async fn update_heal_status(&mut self, tracker: &HealingTracker) { + self.mu.write().await; + tracker.mu.read().await; + + self.heal_status.insert(tracker.id.clone(), tracker.clone()); + } + + async fn get_local_healing_disks(&self) -> HashMap { + self.mu.read().await; + + let mut dst = HashMap::new(); + for v in self.heal_status.values() { + dst.insert(v.endpoint.clone(), v.to_healing_disk().await); + } + + dst + } + + async fn get_heal_local_disk_endpoints(&self) -> Endpoints { + self.mu.read().await; + + let mut endpoints = Vec::new(); + self.heal_local_disks.iter().for_each(|(k, v)| { + if !v { + endpoints.push(k.clone()); + } + }); + + Endpoints::from(endpoints) + } + + async fn set_disk_healing_status(&mut self, ep: Endpoint, healing: bool) { + self.mu.write().await; + + self.heal_local_disks.insert(ep, healing); + } + + async fn push_heal_local_disks(&mut self, heal_local_disks: &[Endpoint]) { + self.mu.write().await; + + heal_local_disks.iter().for_each(|heal_local_disk| { + self.heal_local_disks.insert(heal_local_disk.clone(), false); + }); + } + + async fn periodic_heal_seqs_clean(&mut self, mut rx: Receiver) { + loop { + select! { + result = rx.recv() =>{ + if let Ok(true) = result { + return; + } + } + _ = sleep(Duration::from_secs(5 * 60)) => { + self.mu.write().await; + let now = SystemTime::now(); + + let mut keys_to_reomve = Vec::new(); + for (k, v) in self.heal_seq_map.iter() { + if v.has_ended().await && (UNIX_EPOCH + Duration::from_secs(*(v.end_time.read().await)) + KEEP_HEAL_SEQ_STATE_DURATION) < now { + keys_to_reomve.push(k.clone()) + } + } + for key in keys_to_reomve.iter() { + self.heal_seq_map.remove(key); + } + } + } + } + } + + async fn get_heal_sequence_by_token(&self, token: &str) -> (Option, bool) { + self.mu.read().await; + + for v in self.heal_seq_map.values() { + if v.client_token == token { + return (Some(v.clone()), true); + } + } + + return (None, false); + } + + async fn get_heal_sequence(&self, path: &str) -> Option { + self.mu.read().await; + + self.heal_seq_map.get(path).cloned() + } + + async fn stop_heal_sequence(&mut self, path: &str) -> Result> { + let mut hsp = HealStopSuccess::default(); + if let Some(he) = self.get_heal_sequence(path).await { + let client_token = he.client_token.clone(); + if *GLOBAL_IsDistErasure.read().await { + // TODO: proxy + } + + hsp.client_token = client_token; + hsp.client_address = he.client_address.clone(); + hsp.start_time = he.start_time; + + he.stop(); + + loop { + if he.has_ended().await { + break; + } + + sleep(Duration::from_secs(1)).await; + } + + self.mu.write().await; + self.heal_seq_map.remove(path); + } else { + hsp.client_token = "unknown".to_string(); + } + + let b = serde_json::to_string(&hsp)?; + Ok(b.as_bytes().to_vec()) + } + + // LaunchNewHealSequence - launches a background routine that performs + // healing according to the healSequence argument. For each heal + // sequence, state is stored in the `globalAllHealState`, which is a + // map of the heal path to `healSequence` which holds state about the + // heal sequence. + // + // Heal results are persisted in server memory for + // `keepHealSeqStateDuration`. This function also launches a + // background routine to clean up heal results after the + // aforementioned duration. + pub async fn launch_new_heal_sequence(&mut self, heal_sequence: &HealSequence) -> Result> { + let path = Path::new(&heal_sequence.bucket).join(heal_sequence.object.clone()); + let path_s = path.to_str().unwrap(); + if heal_sequence.force_started { + self.stop_heal_sequence(path_s).await?; + } else { + if let Some(hs) = self.get_heal_sequence(path_s).await { + if !hs.has_ended().await { + return Err(Error::from_string(format!("Heal is already running on the given path (use force-start option to stop and start afresh). The heal was started by IP {} at {}, token is {}", heal_sequence.client_address, heal_sequence.start_time, heal_sequence.client_token))); + } + } + } + + self.mu.write().await; + + for (k, v) in self.heal_seq_map.iter() { + if !v.has_ended().await && (has_profix(&k, path_s) || has_profix(path_s, &k)) { + return Err(Error::from_string(format!( + "The provided heal sequence path overlaps with an existing heal path: {}", + k + ))); + } + } + + self.heal_seq_map.insert(path_s.to_string(), heal_sequence.clone()); + + let client_token = heal_sequence.client_token.clone(); + if *GLOBAL_IsDistErasure.read().await { + // TODO: proxy + } + + if heal_sequence.client_token == BG_HEALING_UUID { + // For background heal do nothing, do not spawn an unnecessary goroutine. + } else { + } + todo!() + } +} diff --git a/ecstore/src/heal/mod.rs b/ecstore/src/heal/mod.rs index b43ec4b06..38db00f2c 100644 --- a/ecstore/src/heal/mod.rs +++ b/ecstore/src/heal/mod.rs @@ -1,2 +1,3 @@ +pub mod background_heal_ops; pub mod heal_commands; pub mod heal_ops; diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index 9a5444aa7..8267a7061 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -12,17 +12,11 @@ use crate::{ disk::{ format::{DistributionAlgoVersion, FormatV3}, DiskStore, - }, - endpoints::PoolEndpoints, - error::{Error, Result}, - global::{is_dist_erasure, GLOBAL_LOCAL_DISK_SET_DRIVES}, - set_disk::SetDisks, - store_api::{ + }, endpoints::PoolEndpoints, error::{Error, Result}, global::{is_dist_erasure, GLOBAL_LOCAL_DISK_SET_DRIVES}, heal::{heal_commands::{HealOpts, HealResultItem}, heal_ops::HealObjectFn}, set_disk::SetDisks, store_api::{ BucketInfo, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, HTTPRangeSpec, ListObjectsV2Info, MakeBucketOptions, MultipartUploadResult, ObjectIO, ObjectInfo, ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI, - }, - utils::hash, + }, utils::hash }; use tokio::time::Duration; @@ -440,4 +434,22 @@ impl StorageAPI for Sets { async fn delete_bucket(&self, _bucket: &str, _opts: &DeleteBucketOptions) -> Result<()> { unimplemented!() } + async fn heal_format(&self, dry_run: bool) -> Result { + unimplemented!() + } + async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result { + unimplemented!() + } + async fn heal_object(&self, bucket: &str, object: &str, version_id: &str, opts: &HealOpts) -> Result { + unimplemented!() + } + async fn heal_objects(&self, bucket: &str, prefix: &str, opts: &HealOpts, func: HealObjectFn) -> Result<()> { + unimplemented!() + } + async fn get_pool_and_set(&self, id: &str) -> Result<(Option, Option, Option)> { + unimplemented!() + } + async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> { + unimplemented!() + } } diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index dd320e8b0..d47a65a19 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -7,6 +7,7 @@ use crate::disk::MetaCacheEntry; use crate::global::{is_dist_erasure, set_object_layer, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES}; use crate::heal::heal_commands::{HealOpts, HealResultItem, HealScanMode}; use crate::heal::heal_ops::HealObjectFn; +use crate::new_object_layer_fn; use crate::store_api::ObjectIO; use crate::{ bucket::metadata::BucketMetadata, @@ -827,54 +828,7 @@ impl StorageAPI for ECStore { Err(Error::new(DiskError::FileNotFound)) } async fn heal_objects(&self, bucket: &str, prefix: &str, opts: &HealOpts, func: HealObjectFn) -> Result<()> { - let heal_entry = |bucket: String, entry: MetaCacheEntry, scan_mode: HealScanMode| async move { - if entry.is_dir() { - return Ok(()); - } - - // We might land at .metacache, .trash, .multipart - // no need to heal them skip, only when bucket - // is '.rustfs.sys' - if bucket == RUSTFS_META_BUCKET { - if Pattern::new("buckets/*/.metacache/*") - .map(|p| p.matches(&entry.name)) - .unwrap_or(false) - || Pattern::new("tmp/*").map(|p| p.matches(&entry.name)).unwrap_or(false) - || Pattern::new("multipart/*").map(|p| p.matches(&entry.name)).unwrap_or(false) - || Pattern::new("tmp-old/*").map(|p| p.matches(&entry.name)).unwrap_or(false) - { - return Ok(()); - } - } - - match entry.file_info_versions(&bucket) { - Ok(fivs) => { - if opts.remove && !opts.dry_run { - if let Err(err) = self.check_abandoned_parts(&bucket, &entry.name, opts).await { - return Err(Error::from_string(format!( - "unable to check object {}/{} for abandoned data: {}", - bucket, entry.name, err - ))); - } - } - - for version in fivs.versions.iter() { - let version_id = version.version_id.map_or("".to_string(), |version_id| version_id.to_string()); - if let Err(err) = func(&bucket, &entry.name, &version_id, scan_mode) { - match err.downcast_ref::() { - Some(DiskError::FileNotFound) | Some(DiskError::FileVersionNotFound) => {} - _ => return Err(err), - } - } - } - } - Err(_) => { - return func(&bucket, &entry.name, "", scan_mode); - } - } - Ok(()) - }; - + let mut first_err = None; for (idx, pool) in self.pools.iter().enumerate() { if opts.pool.is_some() && opts.pool.unwrap() != idx { continue; @@ -886,14 +840,23 @@ impl StorageAPI for ECStore { continue; } - set.list + if let Err(err) = set.list_and_heal(bucket, prefix, opts, func.clone()).await { + if first_err.is_none() { + first_err = Some(err) + } + } } } - todo!() + + if first_err.is_some() { + return Err(first_err.unwrap()); + } + + Ok(()) } async fn get_pool_and_set(&self, id: &str) -> Result<(Option, Option, Option)> { - for (pool_idx, pool) in self.pools.iter().enumerate(){ + for (pool_idx, pool) in self.pools.iter().enumerate() { for (set_idx, set) in pool.format.erasure.sets.iter().enumerate() { for (disk_idx, disk_id) in set.iter().enumerate() { if disk_id.to_string() == id { @@ -921,7 +884,7 @@ impl StorageAPI for ECStore { } if !errs.is_empty() { - return Err(errs[0]); + return Err(errs[0].clone()); } Ok(()) @@ -950,3 +913,64 @@ async fn init_local_peer(endpoint_pools: &EndpointServerPools, host: &String, po *GLOBAL_Local_Node_Name.write().await = peer_set[0].clone(); } + +pub async fn heal_entry( + bucket: String, + entry: MetaCacheEntry, + scan_mode: HealScanMode, + opts: HealOpts, + func: HealObjectFn, +) -> Result<()> { + if entry.is_dir() { + return Ok(()); + } + + // We might land at .metacache, .trash, .multipart + // no need to heal them skip, only when bucket + // is '.rustfs.sys' + if bucket == RUSTFS_META_BUCKET { + if Pattern::new("buckets/*/.metacache/*") + .map(|p| p.matches(&entry.name)) + .unwrap_or(false) + || Pattern::new("tmp/*").map(|p| p.matches(&entry.name)).unwrap_or(false) + || Pattern::new("multipart/*").map(|p| p.matches(&entry.name)).unwrap_or(false) + || Pattern::new("tmp-old/*").map(|p| p.matches(&entry.name)).unwrap_or(false) + { + return Ok(()); + } + } + + let layer = new_object_layer_fn(); + let lock = layer.read().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(Error::msg("errServerNotInitialized")), + }; + + match entry.file_info_versions(&bucket) { + Ok(fivs) => { + if opts.remove && !opts.dry_run { + if let Err(err) = store.check_abandoned_parts(&bucket, &entry.name, &opts).await { + return Err(Error::from_string(format!( + "unable to check object {}/{} for abandoned data: {}", + bucket, entry.name, err + ))); + } + } + + for version in fivs.versions.iter() { + let version_id = version.version_id.map_or("".to_string(), |version_id| version_id.to_string()); + if let Err(err) = func(&bucket, &entry.name, &version_id, scan_mode) { + match err.downcast_ref::() { + Some(DiskError::FileNotFound) | Some(DiskError::FileVersionNotFound) => {} + _ => return Err(err), + } + } + } + } + Err(_) => { + return func(&bucket, &entry.name, "", scan_mode); + } + } + Ok(()) +} diff --git a/ecstore/src/utils/path.rs b/ecstore/src/utils/path.rs index 9b714d629..bf33313c0 100644 --- a/ecstore/src/utils/path.rs +++ b/ecstore/src/utils/path.rs @@ -38,6 +38,18 @@ pub fn retain_slash(s: &str) -> String { } } +pub fn strings_has_prefix_fold(s: &str, prefix: &str) -> bool { + s.len() >= prefix.len() && (s[..prefix.len()] == *prefix || s[..prefix.len()].eq_ignore_ascii_case(prefix)) +} + +pub fn has_profix(s: &str, prefix: &str) -> bool { + if cfg!(target_os = "windows") { + return strings_has_prefix_fold(s, prefix); + } + + s.starts_with(prefix) +} + pub struct LazyBuf { s: String, buf: Option>, diff --git a/rustfs/src/grpc.rs b/rustfs/src/grpc.rs index 2be43631d..9df7f9788 100644 --- a/rustfs/src/grpc.rs +++ b/rustfs/src/grpc.rs @@ -1,7 +1,7 @@ use std::{error::Error, io::ErrorKind, pin::Pin}; use ecstore::{ - disk::{DeleteOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadOptions, UpdateMetadataOpts, WalkDirOptions}, + disk::{DeleteOptions, DiskInfoOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadOptions, UpdateMetadataOpts, WalkDirOptions}, erasure::{ReadAt, Write}, peer::{LocalPeerS3Client, PeerS3Client}, store::{all_local_disk_path, find_local_disk}, @@ -13,17 +13,7 @@ use lock::{lock_args::LockArgs, Locker, GLOBAL_LOCAL_SERVER}; use protos::{ models::{PingBody, PingBodyBuilder}, proto_gen::node_service::{ - node_service_server::NodeService as Node, DeleteBucketRequest, DeleteBucketResponse, DeletePathsRequest, - DeletePathsResponse, DeleteRequest, DeleteResponse, DeleteVersionRequest, DeleteVersionResponse, DeleteVersionsRequest, - DeleteVersionsResponse, DeleteVolumeRequest, DeleteVolumeResponse, GenerallyLockRequest, GenerallyLockResponse, - GetBucketInfoRequest, GetBucketInfoResponse, ListBucketRequest, ListBucketResponse, ListDirRequest, ListDirResponse, - ListVolumesRequest, ListVolumesResponse, MakeBucketRequest, MakeBucketResponse, MakeVolumeRequest, MakeVolumeResponse, - MakeVolumesRequest, MakeVolumesResponse, PingRequest, PingResponse, ReadAllRequest, ReadAllResponse, ReadAtRequest, - ReadAtResponse, ReadMultipleRequest, ReadMultipleResponse, ReadVersionRequest, ReadVersionResponse, ReadXlRequest, - ReadXlResponse, RenameDataRequest, RenameDataResponse, RenameFileRequst, RenameFileResponse, RenamePartRequst, - RenamePartResponse, StatVolumeRequest, StatVolumeResponse, UpdateMetadataRequest, UpdateMetadataResponse, WalkDirRequest, - WalkDirResponse, WriteAllRequest, WriteAllResponse, WriteMetadataRequest, WriteMetadataResponse, WriteRequest, - WriteResponse, + node_service_server::NodeService as Node, DeleteBucketRequest, DeleteBucketResponse, DeletePathsRequest, DeletePathsResponse, DeleteRequest, DeleteResponse, DeleteVersionRequest, DeleteVersionResponse, DeleteVersionsRequest, DeleteVersionsResponse, DeleteVolumeRequest, DeleteVolumeResponse, DiskInfoRequest, DiskInfoResponse, GenerallyLockRequest, GenerallyLockResponse, GetBucketInfoRequest, GetBucketInfoResponse, ListBucketRequest, ListBucketResponse, ListDirRequest, ListDirResponse, ListVolumesRequest, ListVolumesResponse, MakeBucketRequest, MakeBucketResponse, MakeVolumeRequest, MakeVolumeResponse, MakeVolumesRequest, MakeVolumesResponse, PingRequest, PingResponse, ReadAllRequest, ReadAllResponse, ReadAtRequest, ReadAtResponse, ReadMultipleRequest, ReadMultipleResponse, ReadVersionRequest, ReadVersionResponse, ReadXlRequest, ReadXlResponse, RenameDataRequest, RenameDataResponse, RenameFileRequst, RenameFileResponse, RenamePartRequst, RenamePartResponse, StatVolumeRequest, StatVolumeResponse, UpdateMetadataRequest, UpdateMetadataResponse, WalkDirRequest, WalkDirResponse, WriteAllRequest, WriteAllResponse, WriteMetadataRequest, WriteMetadataResponse, WriteRequest, WriteResponse }, }; use tokio::sync::mpsc; @@ -1131,10 +1121,10 @@ impl Node for NodeService { let opts = match serde_json::from_str::(&request.opts) { Ok(opts) => opts, Err(_) => { - return Ok(tonic::Response::new(ReadMultipleResponse { + return Ok(tonic::Response::new(DiskInfoResponse { success: false, - read_multiple_resps: Vec::new(), - error_info: Some("can not decode ReadMultipleReq".to_string()), + disk_info: "".to_string(), + error_info: Some("can not decode DiskInfoOptions".to_string()), })); } }; From 9ff243a7a9f4cbd0ecbe6c7140950803d4efcc2d Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Tue, 22 Oct 2024 09:33:11 +0800 Subject: [PATCH 04/12] fmt Signed-off-by: junxiang Mu <1948535941@qq.com> --- ecstore/src/heal/heal_commands.rs | 5 +++++ ecstore/src/sets.rs | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ecstore/src/heal/heal_commands.rs b/ecstore/src/heal/heal_commands.rs index d6fff5eb1..8725e66ea 100644 --- a/ecstore/src/heal/heal_commands.rs +++ b/ecstore/src/heal/heal_commands.rs @@ -22,6 +22,11 @@ pub const HEAL_UNKNOWN_SCAN: HealScanMode = 0; pub const HEAL_NORMAL_SCAN: HealScanMode = 1; pub const HEAL_DEEP_SCAN: HealScanMode = 2; +pub const HEAL_ITEM_METADATA: &str = "metadata"; +pub const HEAL_ITEM_BUCKET: &str = "bucket"; +pub const HEAL_ITEM_BUCKET_METADATA: &str = "bucket-metadata"; +pub const HEAL_ITEM_OBJECT: &str = "object"; + #[derive(Clone, Copy, Debug, Default)] pub struct HealOpts { pub recursive: bool, diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index 8267a7061..fdaf7c724 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -441,7 +441,7 @@ impl StorageAPI for Sets { unimplemented!() } async fn heal_object(&self, bucket: &str, object: &str, version_id: &str, opts: &HealOpts) -> Result { - unimplemented!() + self.get_disks_by_key(object).heal_object(bucket, object, version_id, opts).await } async fn heal_objects(&self, bucket: &str, prefix: &str, opts: &HealOpts, func: HealObjectFn) -> Result<()> { unimplemented!() From 49593dd77df5ab8b1cbaf54cec126594eb71c159 Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Tue, 22 Oct 2024 14:31:28 +0800 Subject: [PATCH 05/12] Cross platform get_info Signed-off-by: junxiang Mu <1948535941@qq.com> --- Cargo.lock | 1 + ecstore/Cargo.toml | 5 +- ecstore/src/disk/error.rs | 17 +++ ecstore/src/disk/local.rs | 2 +- ecstore/src/disk/os.rs | 2 +- ecstore/src/heal/heal_commands.rs | 18 ++- ecstore/src/utils/mod.rs | 2 +- .../src/utils/{stat_linux.rs => os/linux.rs} | 0 ecstore/src/utils/os/mod.rs | 14 ++ ecstore/src/utils/os/unix.rs | 77 ++++++++++ ecstore/src/utils/os/windows.rs | 136 ++++++++++++++++++ rustfs/src/grpc.rs | 17 ++- 12 files changed, 281 insertions(+), 10 deletions(-) rename ecstore/src/utils/{stat_linux.rs => os/linux.rs} (100%) create mode 100644 ecstore/src/utils/os/mod.rs create mode 100644 ecstore/src/utils/os/unix.rs create mode 100644 ecstore/src/utils/os/windows.rs diff --git a/Cargo.lock b/Cargo.lock index aba59a7fe..4a670527a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -518,6 +518,7 @@ dependencies = [ "transform-stream", "url", "uuid", + "winapi", "xxhash-rust", ] diff --git a/ecstore/Cargo.toml b/ecstore/Cargo.toml index 89eef4670..b66875f03 100644 --- a/ecstore/Cargo.toml +++ b/ecstore/Cargo.toml @@ -31,7 +31,6 @@ lazy_static.workspace = true lock.workspace = true regex = "1.11.0" netif = "0.1.6" -nix = { version = "0.29.0", features = ["fs"] } path-absolutize = "3.1.1" protos.workspace = true rmp-serde = "1.3.0" @@ -53,7 +52,11 @@ num_cpus = "1.16" s3s-policy.workspace = true [target.'cfg(not(windows))'.dependencies] +nix = { version = "0.29.0", features = ["fs"] } openssl = "0.10.66" +[target.'cfg(windows)'.dependencies] +winapi = "0.3.9" + [dev-dependencies] tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } diff --git a/ecstore/src/disk/error.rs b/ecstore/src/disk/error.rs index 2a5414302..da8c7063e 100644 --- a/ecstore/src/disk/error.rs +++ b/ecstore/src/disk/error.rs @@ -401,3 +401,20 @@ pub fn convert_access_error(e: io::Error, per_err: DiskError) -> Error { Error::new(e) } + +pub fn is_all_not_found(errs: &[Option]) -> bool { + for err in errs.iter() { + if let Some(err) = err { + if let Some(err) = err.downcast_ref::() { + match err { + DiskError::FileNotFound | DiskError::VolumeNotFound | &DiskError::FileVersionNotFound =>{ + continue; + }, + _ => return false, + } + } + } + } + + !errs.is_empty() +} diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 6772cc72f..272b87276 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -17,7 +17,7 @@ use crate::error::{Error, Result}; use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold}; use crate::utils::fs::{lstat, O_APPEND, O_CREATE, O_RDONLY, O_WRONLY}; use crate::utils::path::{clean, has_suffix, SLASH_SEPARATOR}; -use crate::utils::stat_linux::get_info; +use crate::utils::os::get_info; use crate::{ file_meta::FileMeta, store_api::{FileInfo, RawFileInfo}, diff --git a/ecstore/src/disk/os.rs b/ecstore/src/disk/os.rs index 75f43166e..033cbb9e9 100644 --- a/ecstore/src/disk/os.rs +++ b/ecstore/src/disk/os.rs @@ -8,7 +8,7 @@ use tokio::fs; use crate::{ disk::error::{is_sys_err_not_dir, is_sys_err_path_not_found, os_is_not_exist}, error::{Error, Result}, - utils::{self, stat_linux::same_disk}, + utils::{self, os::same_disk}, }; use super::error::{os_err_to_file_err, os_is_exist, DiskError}; diff --git a/ecstore/src/heal/heal_commands.rs b/ecstore/src/heal/heal_commands.rs index 8725e66ea..1c1724ecf 100644 --- a/ecstore/src/heal/heal_commands.rs +++ b/ecstore/src/heal/heal_commands.rs @@ -27,6 +27,16 @@ pub const HEAL_ITEM_BUCKET: &str = "bucket"; pub const HEAL_ITEM_BUCKET_METADATA: &str = "bucket-metadata"; pub const HEAL_ITEM_OBJECT: &str = "object"; +pub const DRIVE_STATE_OK: &str = "ok"; +pub const DRIVE_STATE_OFFLINE: &str = "offline"; +pub const DRIVE_STATE_CORRUPT: &str = "corrupt"; +pub const DRIVE_STATE_MISSING: &str = "missing"; +pub const DRIVE_STATE_PERMISSION: &str = "permission-denied"; +pub const DRIVE_STATE_FAULTY: &str = "faulty"; +pub const DRIVE_STATE_ROOT_MOUNT: &str = "root-mount"; +pub const DRIVE_STATE_UNKNOWN: &str = "unknown"; +pub const DRIVE_STATE_UNFORMATTED: &str = "unformatted"; // only returned by disk + #[derive(Clone, Copy, Debug, Default)] pub struct HealOpts { pub recursive: bool, @@ -41,10 +51,10 @@ pub struct HealOpts { } #[derive(Clone, Debug)] -struct HealDriveInfo { - uuid: String, - endpoint: String, - state: String, +pub struct HealDriveInfo { + pub uuid: String, + pub endpoint: String, + pub state: String, } #[derive(Clone, Debug, Default)] diff --git a/ecstore/src/utils/mod.rs b/ecstore/src/utils/mod.rs index bf3b6e4fd..b12bf8048 100644 --- a/ecstore/src/utils/mod.rs +++ b/ecstore/src/utils/mod.rs @@ -4,5 +4,5 @@ pub mod fs; pub mod hash; pub mod net; pub mod path; -pub mod stat_linux; +pub mod os; pub mod wildcard; diff --git a/ecstore/src/utils/stat_linux.rs b/ecstore/src/utils/os/linux.rs similarity index 100% rename from ecstore/src/utils/stat_linux.rs rename to ecstore/src/utils/os/linux.rs diff --git a/ecstore/src/utils/os/mod.rs b/ecstore/src/utils/os/mod.rs new file mode 100644 index 000000000..7524d2ecb --- /dev/null +++ b/ecstore/src/utils/os/mod.rs @@ -0,0 +1,14 @@ +#[cfg(target_os = "linux")] +mod linux; +#[cfg(all(unix, not(target_os = "linux")))] +mod unix; +#[cfg(target_os = "windows")] +mod windows; + +#[cfg(target_os = "linux")] +pub use linux::get_info; +pub use linux::same_disk; +#[cfg(all(unix, not(target_os = "linux")))] +pub use unix::get_info; +#[cfg(target_os = "windows")] +pub use windows::get_info; \ No newline at end of file diff --git a/ecstore/src/utils/os/unix.rs b/ecstore/src/utils/os/unix.rs new file mode 100644 index 000000000..b60d8290f --- /dev/null +++ b/ecstore/src/utils/os/unix.rs @@ -0,0 +1,77 @@ +use crate::disk::Info; +use nix::sys::{statfs::statfs, stat::stat}; +use std::io::{Error, ErrorKind}; +use std::path::Path; + +/// returns total and free bytes available in a directory, e.g. `/`. +pub fn get_info(p: impl AsRef) -> std::io::Result { + let stat = statfs(p.as_ref())?; + + let bsize = stat.block_size() as u64; + let bfree = stat.blocks_free() as u64; + let bavail = stat.blocks_available() as u64; + let blocks = stat.blocks() as u64; + + let reserved = match bfree.checked_sub(bavail) { + Some(reserved) => reserved, + None => { + return Err(Error::new( + ErrorKind::Other, + format!( + "detected f_bavail space ({}) > f_bfree space ({}), fs corruption at ({}). please run 'fsck'", + bavail, + bfree, + p.as_ref().display() + ), + )) + } + }; + + let total = match blocks.checked_sub(reserved) { + Some(total) => total * bsize, + None => { + return Err(Error::new( + ErrorKind::Other, + format!( + "detected reserved space ({}) > blocks space ({}), fs corruption at ({}). please run 'fsck'", + reserved, + blocks, + p.as_ref().display() + ), + )) + } + }; + + let free = bavail * bsize; + let used = match total.checked_sub(free) { + Some(used) => used, + None => { + return Err(Error::new( + ErrorKind::Other, + format!( + "detected free space ({}) > total drive space ({}), fs corruption at ({}). please run 'fsck'", + free, + total, + p.as_ref().display() + ), + )) + } + }; + + Ok(Info { + total, + free, + used, + files: stat.files(), + ffree: stat.files_free(), + fstype: stat.filesystem_type_name().to_string(), + ..Default::default() + }) +} + +pub fn same_disk(disk1: &str, disk2: &str) -> Result { + let stat1 = stat(disk1)?; + let stat2 = stat(disk2)?; + + Ok(stat1.st_dev == stat2.st_dev) +} \ No newline at end of file diff --git a/ecstore/src/utils/os/windows.rs b/ecstore/src/utils/os/windows.rs new file mode 100644 index 000000000..65a9d0022 --- /dev/null +++ b/ecstore/src/utils/os/windows.rs @@ -0,0 +1,136 @@ +use crate::disk::Info; +use std::io::{Error, ErrorKind, Result}; +use std::mem; +use std::os::windows::ffi::OsStrExt; +use std::path::Path; +use winapi::shared::minwindef::{DWORD, MAX_PATH}; +use winapi::shared::ntdef::ULARGE_INTEGER; +use winapi::um::fileapi::{GetDiskFreeSpaceExW, GetDiskFreeSpaceW, GetVolumeInformationW, GetVolumePathNameW}; +use winapi::um::winnt::{LPCWSTR, WCHAR}; + +/// returns total and free bytes available in a directory, e.g. `C:\`. +pub fn get_info(p: impl AsRef) -> Result { + let path_wide: Vec = p + .as_ref() + .canonicalize()? + .into_os_string() + .encode_wide() + .chain(std::iter::once(0)) // Null-terminate the string + .collect(); + + let mut lp_free_bytes_available: ULARGE_INTEGER = unsafe { mem::zeroed() }; + let mut lp_total_number_of_bytes: ULARGE_INTEGER = unsafe { mem::zeroed() }; + let mut lp_total_number_of_free_bytes: ULARGE_INTEGER = unsafe { mem::zeroed() }; + + let success = unsafe { + GetDiskFreeSpaceExW( + path_wide.as_ptr(), + &mut lp_free_bytes_available, + &mut lp_total_number_of_bytes, + &mut lp_total_number_of_free_bytes, + ) + }; + if success == 0 { + return Err(Error::last_os_error()); + } + + let total = unsafe { *lp_total_number_of_bytes.QuadPart() }; + let free = unsafe { *lp_total_number_of_free_bytes.QuadPart() }; + + if free > total { + return Err(Error::new( + ErrorKind::Other, + format!( + "detected free space ({}) > total drive space ({}), fs corruption at ({}). please run 'fsck'", + free, + total, + p.as_ref().display() + ), + )); + } + + let mut lp_sectors_per_cluster: DWORD = 0; + let mut lp_bytes_per_sector: DWORD = 0; + let mut lp_number_of_free_clusters: DWORD = 0; + let mut lp_total_number_of_clusters: DWORD = 0; + + let success = unsafe { + GetDiskFreeSpaceW( + path_wide.as_ptr(), + &mut lp_sectors_per_cluster, + &mut lp_bytes_per_sector, + &mut lp_number_of_free_clusters, + &mut lp_total_number_of_clusters, + ) + }; + if success == 0 { + return Err(Error::last_os_error()); + } + + Ok(Info { + total, + free, + used: total - free, + files: lp_total_number_of_clusters as u64, + ffree: lp_number_of_free_clusters as u64, + fstype: get_fs_type(&path_wide)?, + ..Default::default() + }) +} + +/// returns leading volume name. +fn get_volume_name(v: &[WCHAR]) -> Result { + let volume_name_size: DWORD = MAX_PATH as _; + let mut lp_volume_name_buffer: [WCHAR; MAX_PATH] = [0; MAX_PATH]; + + let success = unsafe { GetVolumePathNameW(v.as_ptr(), lp_volume_name_buffer.as_mut_ptr(), volume_name_size) }; + + if success == 0 { + return Err(Error::last_os_error()); + } + + Ok(lp_volume_name_buffer.as_ptr()) +} + +fn utf16_to_string(v: &[WCHAR]) -> String { + let len = v.iter().position(|&x| x == 0).unwrap_or(v.len()); + String::from_utf16_lossy(&v[..len]) +} + +/// returns the filesystem type of the underlying mounted filesystem +fn get_fs_type(p: &[WCHAR]) -> Result { + let path = get_volume_name(p)?; + + let volume_name_size: DWORD = MAX_PATH as _; + let n_file_system_name_size: DWORD = MAX_PATH as _; + + let mut lp_volume_serial_number: DWORD = 0; + let mut lp_maximum_component_length: DWORD = 0; + let mut lp_file_system_flags: DWORD = 0; + + let mut lp_volume_name_buffer: [WCHAR; MAX_PATH] = [0; MAX_PATH]; + let mut lp_file_system_name_buffer: [WCHAR; MAX_PATH] = [0; MAX_PATH]; + + let success = unsafe { + GetVolumeInformationW( + path, + lp_volume_name_buffer.as_mut_ptr(), + volume_name_size, + &mut lp_volume_serial_number, + &mut lp_maximum_component_length, + &mut lp_file_system_flags, + lp_file_system_name_buffer.as_mut_ptr(), + n_file_system_name_size, + ) + }; + + if success == 0 { + return Err(Error::last_os_error()); + } + + Ok(utf16_to_string(&lp_file_system_name_buffer)) +} + +pub fn same_disk(disk1: &str, disk2: &str) -> Result { + Ok(false) +} \ No newline at end of file diff --git a/rustfs/src/grpc.rs b/rustfs/src/grpc.rs index 9df7f9788..e809a885f 100644 --- a/rustfs/src/grpc.rs +++ b/rustfs/src/grpc.rs @@ -1,7 +1,10 @@ use std::{error::Error, io::ErrorKind, pin::Pin}; use ecstore::{ - disk::{DeleteOptions, DiskInfoOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadOptions, UpdateMetadataOpts, WalkDirOptions}, + disk::{ + DeleteOptions, DiskInfoOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadOptions, UpdateMetadataOpts, + WalkDirOptions, + }, erasure::{ReadAt, Write}, peer::{LocalPeerS3Client, PeerS3Client}, store::{all_local_disk_path, find_local_disk}, @@ -13,7 +16,17 @@ use lock::{lock_args::LockArgs, Locker, GLOBAL_LOCAL_SERVER}; use protos::{ models::{PingBody, PingBodyBuilder}, proto_gen::node_service::{ - node_service_server::NodeService as Node, DeleteBucketRequest, DeleteBucketResponse, DeletePathsRequest, DeletePathsResponse, DeleteRequest, DeleteResponse, DeleteVersionRequest, DeleteVersionResponse, DeleteVersionsRequest, DeleteVersionsResponse, DeleteVolumeRequest, DeleteVolumeResponse, DiskInfoRequest, DiskInfoResponse, GenerallyLockRequest, GenerallyLockResponse, GetBucketInfoRequest, GetBucketInfoResponse, ListBucketRequest, ListBucketResponse, ListDirRequest, ListDirResponse, ListVolumesRequest, ListVolumesResponse, MakeBucketRequest, MakeBucketResponse, MakeVolumeRequest, MakeVolumeResponse, MakeVolumesRequest, MakeVolumesResponse, PingRequest, PingResponse, ReadAllRequest, ReadAllResponse, ReadAtRequest, ReadAtResponse, ReadMultipleRequest, ReadMultipleResponse, ReadVersionRequest, ReadVersionResponse, ReadXlRequest, ReadXlResponse, RenameDataRequest, RenameDataResponse, RenameFileRequst, RenameFileResponse, RenamePartRequst, RenamePartResponse, StatVolumeRequest, StatVolumeResponse, UpdateMetadataRequest, UpdateMetadataResponse, WalkDirRequest, WalkDirResponse, WriteAllRequest, WriteAllResponse, WriteMetadataRequest, WriteMetadataResponse, WriteRequest, WriteResponse + node_service_server::NodeService as Node, DeleteBucketRequest, DeleteBucketResponse, DeletePathsRequest, + DeletePathsResponse, DeleteRequest, DeleteResponse, DeleteVersionRequest, DeleteVersionResponse, DeleteVersionsRequest, + DeleteVersionsResponse, DeleteVolumeRequest, DeleteVolumeResponse, DiskInfoRequest, DiskInfoResponse, + GenerallyLockRequest, GenerallyLockResponse, GetBucketInfoRequest, GetBucketInfoResponse, ListBucketRequest, + ListBucketResponse, ListDirRequest, ListDirResponse, ListVolumesRequest, ListVolumesResponse, MakeBucketRequest, + MakeBucketResponse, MakeVolumeRequest, MakeVolumeResponse, MakeVolumesRequest, MakeVolumesResponse, PingRequest, + PingResponse, ReadAllRequest, ReadAllResponse, ReadAtRequest, ReadAtResponse, ReadMultipleRequest, ReadMultipleResponse, + ReadVersionRequest, ReadVersionResponse, ReadXlRequest, ReadXlResponse, RenameDataRequest, RenameDataResponse, + RenameFileRequst, RenameFileResponse, RenamePartRequst, RenamePartResponse, StatVolumeRequest, StatVolumeResponse, + UpdateMetadataRequest, UpdateMetadataResponse, WalkDirRequest, WalkDirResponse, WriteAllRequest, WriteAllResponse, + WriteMetadataRequest, WriteMetadataResponse, WriteRequest, WriteResponse, }, }; use tokio::sync::mpsc; From edb1e692f136d284af8772b153f1c82a39b7d569 Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Thu, 24 Oct 2024 10:16:52 +0800 Subject: [PATCH 06/12] bitrot(1) Signed-off-by: junxiang Mu <1948535941@qq.com> --- Cargo.lock | 17 ++++ ecstore/Cargo.toml | 2 + ecstore/src/bitrot.rs | 167 +++++++++++++++++++++++++++++++++++++ ecstore/src/disk/local.rs | 61 +++++++++++--- ecstore/src/disk/mod.rs | 5 ++ ecstore/src/disk/remote.rs | 9 +- ecstore/src/lib.rs | 1 + ecstore/src/store_api.rs | 15 +++- ecstore/src/utils/path.rs | 12 +++ 9 files changed, 274 insertions(+), 15 deletions(-) create mode 100644 ecstore/src/bitrot.rs diff --git a/Cargo.lock b/Cargo.lock index 4a670527a..a977e6def 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -276,6 +276,15 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -479,6 +488,7 @@ dependencies = [ "async-trait", "backon", "base64-simd", + "blake2", "byteorder", "bytes", "common", @@ -486,6 +496,7 @@ dependencies = [ "futures", "glob", "hex-simd", + "highway", "http", "lazy_static", "lock", @@ -796,6 +807,12 @@ dependencies = [ "vsimd", ] +[[package]] +name = "highway" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c706f1711006204c2ba8fb1a7bd55f689bbf7feca9ff40325206b5e140cff6df" + [[package]] name = "hmac" version = "0.12.1" diff --git a/ecstore/Cargo.toml b/ecstore/Cargo.toml index b66875f03..523ef2994 100644 --- a/ecstore/Cargo.toml +++ b/ecstore/Cargo.toml @@ -11,6 +11,7 @@ rust-version.workspace = true [dependencies] async-trait.workspace = true backon.workspace = true +blake2 = "0.10.6" bytes.workspace = true common.workspace = true glob = "0.3.1" @@ -23,6 +24,7 @@ serde_json.workspace = true tracing-error.workspace = true s3s.workspace = true http.workspace = true +highway = "1.2.0" url.workspace = true uuid = { version = "1.8.0", features = ["v4", "fast-rng", "serde"] } reed-solomon-erasure = { version = "6.0.0", features = ["simd-accel"] } diff --git a/ecstore/src/bitrot.rs b/ecstore/src/bitrot.rs new file mode 100644 index 000000000..822eb877d --- /dev/null +++ b/ecstore/src/bitrot.rs @@ -0,0 +1,167 @@ +use std::collections::HashMap; + +use blake2::{Blake2b, Blake2b512}; +use highway::{HighwayHash, HighwayHasher, Key}; +use lazy_static::lazy_static; +use sha2::{Digest, Sha256}; + +use crate::{ + disk::DiskStore, + erasure::{ReadAt, Write}, + error::Result, + store_api::BitrotAlgorithm, +}; + +lazy_static! { + static ref BITROT_ALGORITHMS: HashMap = { + let mut m = HashMap::new(); + m.insert(BitrotAlgorithm::SHA256, "sha256"); + m.insert(BitrotAlgorithm::BLAKE2b512, "blake2b"); + m.insert(BitrotAlgorithm::HighwayHash256, "highwayhash256"); + m.insert(BitrotAlgorithm::HighwayHash256S, "highwayhash256S"); + m + }; +} + +// const MAGIC_HIGHWAY_HASH256_KEY: &[u8] = &[ +// 0x4b, 0xe7, 0x34, 0xfa, 0x8e, 0x23, 0x8a, 0xcd, 0x26, 0x3e, 0x83, 0xe6, 0xbb, 0x96, 0x85, 0x52, 0x04, 0x0f, 0x93, 0x5d, 0xa3, +// 0x9f, 0x44, 0x14, 0x97, 0xe0, 0x9d, 0x13, 0x22, 0xde, 0x36, 0xa0, +// ]; +const MAGIC_HIGHWAY_HASH256_KEY: &[u64; 4] = &[3, 4, 2, 1]; + +pub enum Hasher { + SHA256(Sha256), + HighwayHash256(HighwayHasher), + BLAKE2b512(Blake2b512), +} + +impl Hasher { + pub fn update(&mut self, data: impl AsRef<[u8]>) { + match self { + Hasher::SHA256(core_wrapper) => { + core_wrapper.update(data); + } + Hasher::HighwayHash256(highway_hasher) => { + highway_hasher.append(data.as_ref()); + } + Hasher::BLAKE2b512(core_wrapper) => { + core_wrapper.update(data); + } + } + } + + pub fn finalize(self) -> Vec { + match self { + Hasher::SHA256(core_wrapper) => core_wrapper.finalize().to_vec(), + Hasher::HighwayHash256(highway_hasher) => highway_hasher + .finalize256() + .iter() + .flat_map(|&n| n.to_le_bytes()) // 使用小端字节序转换 + .collect(), + Hasher::BLAKE2b512(core_wrapper) => core_wrapper.finalize().to_vec(), + } + } +} + +impl BitrotAlgorithm { + pub fn new(&self) -> Hasher { + match self { + BitrotAlgorithm::SHA256 => Hasher::SHA256(Sha256::new()), + BitrotAlgorithm::HighwayHash256 | BitrotAlgorithm::HighwayHash256S => { + let key = Key(*MAGIC_HIGHWAY_HASH256_KEY); + Hasher::HighwayHash256(HighwayHasher::new(key)) + } + BitrotAlgorithm::BLAKE2b512 => Hasher::BLAKE2b512(Blake2b512::new()), + } + } + + pub fn available(&self) -> bool { + BITROT_ALGORITHMS.get(self).is_some() + } + + pub fn string(&self) -> String { + BITROT_ALGORITHMS.get(self).map_or("".to_string(), |s| s.to_string()) + } +} + +pub struct BitrotVerifier { + algorithm: BitrotAlgorithm, + sum: Vec, +} + +impl BitrotVerifier { + pub fn new(algorithm: BitrotAlgorithm, checksum: &[u8]) -> BitrotVerifier { + BitrotVerifier { + algorithm, + sum: checksum.to_vec(), + } + } +} + +pub fn bitrot_algorithm_from_string(s: &str) -> BitrotAlgorithm { + for (k, v) in BITROT_ALGORITHMS.iter() { + if *v == s { + return k.clone(); + } + } + + BitrotAlgorithm::HighwayHash256S +} + +type BitrotWriter = Box; + +pub fn new_bitrot_writer( + disk: DiskStore, + orig_volume: &str, + volume: &str, + file_path: &str, + length: usize, + algo: BitrotAlgorithm, + shard_size: usize, +) -> BitrotWriter { + todo!() +} + +pub struct WholeBitrotWriter { + disk: DiskStore, + volume: String, + file_path: String, + shard_size: usize, + hash: Hasher, +} + +impl WholeBitrotWriter { + pub fn new(disk: DiskStore, volume: &str, file_path: &str, algo: BitrotAlgorithm, shard_size: usize) -> Self { + WholeBitrotWriter { + disk, + volume: volume.to_string(), + file_path: file_path.to_string(), + shard_size, + hash: algo.new(), + } + } +} + +#[async_trait::async_trait] +impl Write for WholeBitrotWriter { + async fn write(&mut self, buf: &[u8]) -> Result<()> { + let mut file = self.disk.append_file(&self.volume, &self.file_path).await?; + let _ = file.write(buf).await?; + self.hash.update(buf); + + Ok(()) + } +} + +pub struct WholeBitrotReader { + disk: DiskStore, + volume: String, + file_path: String, +} + +#[async_trait::async_trait] +impl ReadAt for WholeBitrotReader { + async fn read_at(&mut self, offset: usize, length: usize) -> Result<(Vec, usize)> { + todo!() + } +} diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 272b87276..ae77cef42 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -2,9 +2,9 @@ use super::error::{is_sys_err_io, is_sys_err_not_empty, is_sys_err_too_many_file use super::os::is_root_disk; use super::{endpoint::Endpoint, error::DiskError, format::FormatV3}; use super::{ - os, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions, FileReader, FileWriter, - Info, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, - WalkDirOptions, + os, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions, + FileReader, FileWriter, Info, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, + UpdateMetadataOpts, VolumeInfo, WalkDirOptions, }; use crate::cache_value::cache::{Cache, Opts}; use crate::disk::error::{ @@ -15,16 +15,16 @@ use crate::disk::os::check_path_length; use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE}; use crate::error::{Error, Result}; use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold}; +use crate::store_api::BitrotAlgorithm; use crate::utils::fs::{lstat, O_APPEND, O_CREATE, O_RDONLY, O_WRONLY}; -use crate::utils::path::{clean, has_suffix, SLASH_SEPARATOR}; use crate::utils::os::get_info; +use crate::utils::path::{clean, has_suffix, SLASH_SEPARATOR}; use crate::{ file_meta::FileMeta, store_api::{FileInfo, RawFileInfo}, utils, }; use path_absolutize::Absolutize; -use tokio::runtime::Runtime; use std::fmt::Debug; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::Arc; @@ -36,6 +36,7 @@ use std::{ use time::OffsetDateTime; use tokio::fs::{self, File}; use tokio::io::{AsyncReadExt, AsyncWriteExt, ErrorKind}; +use tokio::runtime::Runtime; use tokio::sync::RwLock; use tracing::{error, warn}; use uuid::Uuid; @@ -149,19 +150,17 @@ impl LocalDisk { if root { return Err(Error::new(DiskError::DriveIsRoot)); } - + // disk_info.healing = Ok(disk_info) - }, - Err(err) => { - Err(err) } + Err(err) => Err(err), } }) }; - + let cache = Cache::new(Box::new(update_fn), Duration::from_secs(1), Opts::default()); - + // TODO: DIRECT suport // TODD: DiskInfo let mut disk = Self { @@ -650,6 +649,22 @@ impl LocalDisk { fn get_metrics(&self) -> DiskMetrics { DiskMetrics::default() } + + async fn bitrot_verify( + &self, + part_path: &PathBuf, + part_size: u64, + algo: BitrotAlgorithm, + sum: &[u8], + shard_size: u64, + ) -> Result<()> { + let file = utils::fs::open_file(part_path, O_CREATE | O_WRONLY) + .await + .map_err(os_err_to_file_err)?; + + + todo!() + } } fn is_root_path(path: impl AsRef) -> bool { @@ -846,6 +861,30 @@ impl DiskAPI for LocalDisk { Ok(()) } + async fn verify_file(&self, volume: &str, path: &str, fi: FileInfo) -> Result { + let volume_dir = self.get_bucket_path(volume)?; + if !skip_access_checks(volume) { + if let Err(e) = utils::fs::access(&volume_dir).await { + return Err(convert_access_error(e, DiskError::VolumeAccessDenied)); + } + } + + let mut resp = CheckPartsResp { + results: Vec::with_capacity(fi.parts.len()), + }; + + let erasure = fi.erasure; + fi.parts.iter().enumerate().for_each(|(i, part)| { + let checksum_info = erasure.get_checksum_info(part.number); + let part_path = Path::new(&volume_dir) + .join(path) + .join(fi.data_dir.map_or("".to_string(), |dir| dir.to_string())) + .join(format!("part.{}", part.number)); + + self.bi + }); + } + async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec) -> Result<()> { let src_volume_dir = self.get_bucket_path(src_volume)?; let dst_volume_dir = self.get_bucket_path(dst_volume)?; diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs index 0fa68ff2f..89a30746b 100644 --- a/ecstore/src/disk/mod.rs +++ b/ecstore/src/disk/mod.rs @@ -127,6 +127,7 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { // CheckParts async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>; // VerifyFile + async fn verify_file(&self, volume: &str, path: &str, fi: FileInfo) -> Result; // StatInfoFile // ReadParts async fn read_multiple(&self, req: ReadMultipleReq) -> Result>; @@ -136,6 +137,10 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { async fn disk_info(&self, opts: &DiskInfoOptions) -> Result; } +pub struct CheckPartsResp { + pub results: Vec, +} + #[derive(Debug, Serialize, Deserialize)] pub struct UpdateMetadataOpts { pub no_persistence: bool, diff --git a/ecstore/src/disk/remote.rs b/ecstore/src/disk/remote.rs index 4b82c43cd..8d120a065 100644 --- a/ecstore/src/disk/remote.rs +++ b/ecstore/src/disk/remote.rs @@ -15,9 +15,7 @@ use tracing::info; use uuid::Uuid; use super::{ - endpoint::Endpoint, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, FileInfoVersions, - FileReader, FileWriter, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RemoteFileReader, RemoteFileWriter, - RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, + endpoint::Endpoint, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, FileInfoVersions, FileReader, FileWriter, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RemoteFileReader, RemoteFileWriter, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions }; use crate::{ disk::error::DiskError, @@ -163,6 +161,11 @@ impl DiskAPI for RemoteDisk { Ok(()) } + + async fn verify_file(&self, volume: &str, path: &str, fi: FileInfo) -> Result { + unimplemented!() + } + async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec) -> Result<()> { info!("rename_part"); let mut client = node_service_time_out_client(&self.addr) diff --git a/ecstore/src/lib.rs b/ecstore/src/lib.rs index 5d1346b25..4533f1266 100644 --- a/ecstore/src/lib.rs +++ b/ecstore/src/lib.rs @@ -1,3 +1,4 @@ +pub mod bitrot; pub mod cache_value; mod chunk_stream; mod config; diff --git a/ecstore/src/store_api.rs b/ecstore/src/store_api.rs index 1df9741d6..51005a071 100644 --- a/ecstore/src/store_api.rs +++ b/ecstore/src/store_api.rs @@ -12,6 +12,7 @@ use http::HeaderMap; use rmp_serde::Serializer; use s3s::dto::StreamingBlob; use serde::{Deserialize, Serialize}; +use sha2::Sha256; use time::OffsetDateTime; use uuid::Uuid; @@ -240,6 +241,18 @@ pub struct ErasureInfo { pub checksums: Vec, } +impl ErasureInfo { + pub fn get_checksum_info(&self, part_number: usize) -> ChecksumInfo { + for sum in &self.checksums { + if sum.part_number == part_number { + return sum.clone(); + } + } + + ChecksumInfo {algorithm: BitrotAlgorithm::HighwayHash256S, ..Default::default()} + } +} + #[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone)] // ChecksumInfo - carries checksums of individual scattered parts per disk. pub struct ChecksumInfo { @@ -248,7 +261,7 @@ pub struct ChecksumInfo { pub hash: Vec, } -#[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone)] +#[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone, Eq, Hash)] // BitrotAlgorithm specifies a algorithm used for bitrot protection. pub enum BitrotAlgorithm { // SHA256 represents the SHA-256 hash function diff --git a/ecstore/src/utils/path.rs b/ecstore/src/utils/path.rs index bf33313c0..fc35031fc 100644 --- a/ecstore/src/utils/path.rs +++ b/ecstore/src/utils/path.rs @@ -1,3 +1,5 @@ +use std::path::PathBuf; + const GLOBAL_DIR_SUFFIX: &str = "__XLDIR__"; pub const SLASH_SEPARATOR: &str = "/"; @@ -50,6 +52,16 @@ pub fn has_profix(s: &str, prefix: &str) -> bool { s.starts_with(prefix) } +pub fn path_join(elem: &[PathBuf]) -> PathBuf { + let mut joined_path = PathBuf::new(); + + for path in elem { + joined_path.push(path); + } + + joined_path +} + pub struct LazyBuf { s: String, buf: Option>, From 57c6a08b29fbf9710832991beda9020a412b082d Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Tue, 29 Oct 2024 16:52:21 +0800 Subject: [PATCH 07/12] bitrot(2) Signed-off-by: junxiang Mu <1948535941@qq.com> --- ecstore/src/bitrot.rs | 157 ++++++++++++++++++++++++++++++++++++-- ecstore/src/disk/local.rs | 3 +- 2 files changed, 151 insertions(+), 9 deletions(-) diff --git a/ecstore/src/bitrot.rs b/ecstore/src/bitrot.rs index 822eb877d..f658fd095 100644 --- a/ecstore/src/bitrot.rs +++ b/ecstore/src/bitrot.rs @@ -1,14 +1,15 @@ -use std::collections::HashMap; +use std::{any::Any, collections::HashMap, io::Cursor}; -use blake2::{Blake2b, Blake2b512}; +use blake2::Blake2b512; +use hex_simd::{decode_to_vec, encode_to_string}; use highway::{HighwayHash, HighwayHasher, Key}; use lazy_static::lazy_static; -use sha2::{Digest, Sha256}; +use sha2::{digest::core_api::BlockSizeUser, Digest, Sha256}; use crate::{ - disk::DiskStore, + disk::{error::DiskError, DiskStore}, erasure::{ReadAt, Write}, - error::Result, + error::{Error, Result}, store_api::BitrotAlgorithm, }; @@ -61,6 +62,33 @@ impl Hasher { Hasher::BLAKE2b512(core_wrapper) => core_wrapper.finalize().to_vec(), } } + + pub fn size(&self) -> usize { + match self { + Hasher::SHA256(_) => Sha256::output_size(), + Hasher::HighwayHash256(_) => 32, + Hasher::BLAKE2b512(_) => Blake2b512::output_size(), + } + } + + pub fn block_size(&self) -> usize { + match self { + Hasher::SHA256(_) => Sha256::block_size(), + Hasher::HighwayHash256(_) => 64, + Hasher::BLAKE2b512(_) => Blake2b512::block_size(), + } + } + + pub fn reset(&mut self) { + match self { + Hasher::SHA256(core_wrapper) => core_wrapper.reset(), + Hasher::HighwayHash256(highway_hasher) => { + let key = Key(*MAGIC_HIGHWAY_HASH256_KEY); + *highway_hasher = HighwayHasher::new(key); + } , + Hasher::BLAKE2b512(core_wrapper) =>core_wrapper.reset(), + } + } } impl BitrotAlgorithm { @@ -112,13 +140,54 @@ type BitrotWriter = Box; pub fn new_bitrot_writer( disk: DiskStore, - orig_volume: &str, + _orig_volume: &str, volume: &str, file_path: &str, - length: usize, + _length: usize, algo: BitrotAlgorithm, shard_size: usize, ) -> BitrotWriter { + Box::new(WholeBitrotWriter::new(disk, volume, file_path, algo, shard_size)) +} + +type BitrotReader = Box; + +pub fn new_bitrot_reader( + disk: DiskStore, + _data: &[u8], + bucket: &str, + file_path: &str, + till_offset: usize, + algo: BitrotAlgorithm, + sum: &[u8], + _shard_size: usize, +) -> BitrotReader { + Box::new(WholeBitrotReader::new(disk, bucket, file_path, algo, till_offset, sum)) +} + +pub fn bitrot_shard_file_size(size: i64, _shard_size: i64, algo: BitrotAlgorithm) -> i64 { + if algo != BitrotAlgorithm::HighwayHash256S { + return size; + } + todo!() + // ceil_frac(size, shard_size) * algo.new().size() + size +} + +pub fn bitrot_verify( + r: Cursor>, + _want_size: usize, + _part_size: usize, + algo: BitrotAlgorithm, + want: Vec, + _shard_size: usize, +) -> Result<()> { + if algo != BitrotAlgorithm::HighwayHash256S { + let mut h = algo.new(); + h.update(r.into_inner()); + if h.finalize() != want { + return Err(Error::new(DiskError::FileCorrupt)); + } + } todo!() } @@ -131,6 +200,10 @@ pub struct WholeBitrotWriter { } impl WholeBitrotWriter { + fn as_any(&self) -> &dyn Any { + self + } + pub fn new(disk: DiskStore, volume: &str, file_path: &str, algo: BitrotAlgorithm, shard_size: usize) -> Self { WholeBitrotWriter { disk, @@ -157,11 +230,79 @@ pub struct WholeBitrotReader { disk: DiskStore, volume: String, file_path: String, + verifier: BitrotVerifier, + till_offset: usize, + buf: Option>, +} + +impl WholeBitrotReader { + pub fn new(disk: DiskStore, volume: &str, file_path: &str, algo: BitrotAlgorithm, till_offset: usize, sum: &[u8]) -> Self { + Self { + disk, + volume: volume.to_string(), + file_path: file_path.to_string(), + verifier: BitrotVerifier::new(algo, sum), + till_offset, + buf: None, + } + } } #[async_trait::async_trait] impl ReadAt for WholeBitrotReader { async fn read_at(&mut self, offset: usize, length: usize) -> Result<(Vec, usize)> { - todo!() + if self.buf.is_none() { + let buf_len = self.till_offset - offset; + let mut file = self.disk.read_file(&self.volume, &self.file_path).await?; + let (buf, _) = file.read_at(offset, buf_len).await?; + self.buf = Some(buf); + } + + if let Some(buf) = &mut self.buf { + if buf.len() < length { + return Err(Error::new(DiskError::LessData)); + } + + return Ok((buf.split_off(length), length)); + } + + Err(Error::new(DiskError::LessData)) } } + +#[test] +fn bitrot_self_test() -> Result<()> { + let mut checksums = HashMap::new(); + checksums.insert(BitrotAlgorithm::SHA256, "a7677ff19e0182e4d52e3a3db727804abc82a5818749336369552e54b838b004"); + checksums.insert(BitrotAlgorithm::BLAKE2b512, "e519b7d84b1c3c917985f544773a35cf265dcab10948be3550320d156bab612124a5ae2ae5a8c73c0eea360f68b0e28136f26e858756dbfe7375a7389f26c669"); + checksums.insert(BitrotAlgorithm::HighwayHash256, "c81c2386a1f565e805513d630d4e50ff26d11269b21c221cf50fc6c29d6ff75b"); + checksums.insert(BitrotAlgorithm::HighwayHash256S, "c81c2386a1f565e805513d630d4e50ff26d11269b21c221cf50fc6c29d6ff75b"); + + let iter = [BitrotAlgorithm::SHA256, BitrotAlgorithm::BLAKE2b512, BitrotAlgorithm:: HighwayHash256]; + + for algo in iter.iter() { + if !algo.available() || *algo != BitrotAlgorithm::HighwayHash256 { + continue; + } + let checksum = decode_to_vec(checksums.get(algo).unwrap()).unwrap(); + + let mut h = algo.new(); + let mut msg = Vec::with_capacity(h.size() * h.block_size()); + let mut sum = Vec::with_capacity(h.size()); + + for i in (0..h.size()*h.block_size()).step_by(h.size()) { + h.update(&msg); + sum = h.finalize(); + msg.extend(sum.clone()); + h = algo.new(); + } + + if checksum != sum { + println!("failed: {:?}, expect: {:?}, actual: {:?}", algo, checksum, sum); + return Err(Error::new(DiskError::FileCorrupt)); + } + println!("success: {:?}", algo); + } + + Ok(()) +} diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index ae77cef42..d37346aee 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -881,8 +881,9 @@ impl DiskAPI for LocalDisk { .join(fi.data_dir.map_or("".to_string(), |dir| dir.to_string())) .join(format!("part.{}", part.number)); - self.bi + // self.bi }); + todo!() } async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec) -> Result<()> { From e6cd184cd60b04a691d9d70befebe9fba3997593 Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Wed, 30 Oct 2024 11:26:49 +0800 Subject: [PATCH 08/12] bitrot test(1) Signed-off-by: junxiang Mu <1948535941@qq.com> --- Cargo.lock | 1 + Cargo.toml | 1 + ecstore/Cargo.toml | 1 + ecstore/src/bitrot.rs | 157 +++++++++++++++++++++++--------- ecstore/src/disk/mod.rs | 14 ++- ecstore/src/erasure.rs | 2 + ecstore/src/utils/os/linux.rs | 162 ++++++++++++++++++++-------------- 7 files changed, 230 insertions(+), 108 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a977e6def..2c884b2b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -517,6 +517,7 @@ dependencies = [ "serde_json", "sha2", "siphasher", + "tempfile", "thiserror", "time", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 2a50af2f9..c81b87305 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,6 +53,7 @@ s3s = { git = "https://github.com/Nugine/s3s.git", rev = "0caf79822ae1f2e7a7fad3 s3s-policy = { git = "https://github.com/Nugine/s3s.git", rev = "0caf79822ae1f2e7a7fad3c4a093c94dd2e2c33f" } serde = { version = "1.0.210", features = ["derive"] } serde_json = "1.0.128" +tempfile = "3.13.0" thiserror = "1.0.64" time = { version = "0.3.36", features = [ "std", diff --git a/ecstore/Cargo.toml b/ecstore/Cargo.toml index 523ef2994..a7ac29104 100644 --- a/ecstore/Cargo.toml +++ b/ecstore/Cargo.toml @@ -43,6 +43,7 @@ base64-simd = "0.8.0" sha2 = { version = "0.10.8", features = ["asm"] } hex-simd = "0.8.0" path-clean = "1.0.1" +tempfile.workspace = true tokio = { workspace = true, features = ["io-util", "sync"] } tokio-stream = "0.1.15" tonic.workspace = true diff --git a/ecstore/src/bitrot.rs b/ecstore/src/bitrot.rs index f658fd095..dd6ab8c01 100644 --- a/ecstore/src/bitrot.rs +++ b/ecstore/src/bitrot.rs @@ -1,7 +1,6 @@ use std::{any::Any, collections::HashMap, io::Cursor}; use blake2::Blake2b512; -use hex_simd::{decode_to_vec, encode_to_string}; use highway::{HighwayHash, HighwayHasher, Key}; use lazy_static::lazy_static; use sha2::{digest::core_api::BlockSizeUser, Digest, Sha256}; @@ -30,6 +29,7 @@ lazy_static! { // ]; const MAGIC_HIGHWAY_HASH256_KEY: &[u64; 4] = &[3, 4, 2, 1]; +#[derive(Clone)] pub enum Hasher { SHA256(Sha256), HighwayHash256(HighwayHasher), @@ -85,8 +85,8 @@ impl Hasher { Hasher::HighwayHash256(highway_hasher) => { let key = Key(*MAGIC_HIGHWAY_HASH256_KEY); *highway_hasher = HighwayHasher::new(key); - } , - Hasher::BLAKE2b512(core_wrapper) =>core_wrapper.reset(), + } + Hasher::BLAKE2b512(core_wrapper) => core_wrapper.reset(), } } } @@ -165,6 +165,14 @@ pub fn new_bitrot_reader( Box::new(WholeBitrotReader::new(disk, bucket, file_path, algo, till_offset, sum)) } +pub fn bitrot_writer_sum(w: &BitrotWriter) -> Vec { + if let Some(w) = w.as_any().downcast_ref::() { + return w.hash.clone().finalize(); + } + + Vec::new() +} + pub fn bitrot_shard_file_size(size: i64, _shard_size: i64, algo: BitrotAlgorithm) -> i64 { if algo != BitrotAlgorithm::HighwayHash256S { return size; @@ -195,21 +203,17 @@ pub struct WholeBitrotWriter { disk: DiskStore, volume: String, file_path: String, - shard_size: usize, - hash: Hasher, + _shard_size: usize, + pub hash: Hasher, } impl WholeBitrotWriter { - fn as_any(&self) -> &dyn Any { - self - } - pub fn new(disk: DiskStore, volume: &str, file_path: &str, algo: BitrotAlgorithm, shard_size: usize) -> Self { WholeBitrotWriter { disk, volume: volume.to_string(), file_path: file_path.to_string(), - shard_size, + _shard_size: shard_size, hash: algo.new(), } } @@ -217,6 +221,10 @@ impl WholeBitrotWriter { #[async_trait::async_trait] impl Write for WholeBitrotWriter { + fn as_any(&self) -> &dyn Any { + self + } + async fn write(&mut self, buf: &[u8]) -> Result<()> { let mut file = self.disk.append_file(&self.volume, &self.file_path).await?; let _ = file.write(buf).await?; @@ -230,7 +238,7 @@ pub struct WholeBitrotReader { disk: DiskStore, volume: String, file_path: String, - verifier: BitrotVerifier, + _verifier: BitrotVerifier, till_offset: usize, buf: Option>, } @@ -241,7 +249,7 @@ impl WholeBitrotReader { disk, volume: volume.to_string(), file_path: file_path.to_string(), - verifier: BitrotVerifier::new(algo, sum), + _verifier: BitrotVerifier::new(algo, sum), till_offset, buf: None, } @@ -270,39 +278,108 @@ impl ReadAt for WholeBitrotReader { } } -#[test] -fn bitrot_self_test() -> Result<()> { - let mut checksums = HashMap::new(); - checksums.insert(BitrotAlgorithm::SHA256, "a7677ff19e0182e4d52e3a3db727804abc82a5818749336369552e54b838b004"); - checksums.insert(BitrotAlgorithm::BLAKE2b512, "e519b7d84b1c3c917985f544773a35cf265dcab10948be3550320d156bab612124a5ae2ae5a8c73c0eea360f68b0e28136f26e858756dbfe7375a7389f26c669"); - checksums.insert(BitrotAlgorithm::HighwayHash256, "c81c2386a1f565e805513d630d4e50ff26d11269b21c221cf50fc6c29d6ff75b"); - checksums.insert(BitrotAlgorithm::HighwayHash256S, "c81c2386a1f565e805513d630d4e50ff26d11269b21c221cf50fc6c29d6ff75b"); +#[cfg(test)] +mod test { + use std::{collections::HashMap, fs}; - let iter = [BitrotAlgorithm::SHA256, BitrotAlgorithm::BLAKE2b512, BitrotAlgorithm:: HighwayHash256]; + use hex_simd::decode_to_vec; + use tempfile::TempDir; - for algo in iter.iter() { - if !algo.available() || *algo != BitrotAlgorithm::HighwayHash256 { - continue; - } - let checksum = decode_to_vec(checksums.get(algo).unwrap()).unwrap(); - - let mut h = algo.new(); - let mut msg = Vec::with_capacity(h.size() * h.block_size()); - let mut sum = Vec::with_capacity(h.size()); + use crate::{ + bitrot::{new_bitrot_writer, BITROT_ALGORITHMS}, + disk::{endpoint::Endpoint, error::DiskError, new_disk, DiskOption}, + error::{Error, Result}, + store_api::BitrotAlgorithm, + }; - for i in (0..h.size()*h.block_size()).step_by(h.size()) { - h.update(&msg); - sum = h.finalize(); - msg.extend(sum.clone()); - h = algo.new(); + use super::{bitrot_writer_sum, new_bitrot_reader}; + + #[test] + fn bitrot_self_test() -> Result<()> { + let mut checksums = HashMap::new(); + checksums.insert( + BitrotAlgorithm::SHA256, + "a7677ff19e0182e4d52e3a3db727804abc82a5818749336369552e54b838b004", + ); + checksums.insert(BitrotAlgorithm::BLAKE2b512, "e519b7d84b1c3c917985f544773a35cf265dcab10948be3550320d156bab612124a5ae2ae5a8c73c0eea360f68b0e28136f26e858756dbfe7375a7389f26c669"); + checksums.insert( + BitrotAlgorithm::HighwayHash256, + "c81c2386a1f565e805513d630d4e50ff26d11269b21c221cf50fc6c29d6ff75b", + ); + checksums.insert( + BitrotAlgorithm::HighwayHash256S, + "c81c2386a1f565e805513d630d4e50ff26d11269b21c221cf50fc6c29d6ff75b", + ); + + let iter = [ + BitrotAlgorithm::SHA256, + BitrotAlgorithm::BLAKE2b512, + BitrotAlgorithm::HighwayHash256, + ]; + + for algo in iter.iter() { + if !algo.available() || *algo != BitrotAlgorithm::HighwayHash256 { + continue; + } + let checksum = decode_to_vec(checksums.get(algo).unwrap()).unwrap(); + + let mut h = algo.new(); + let mut msg = Vec::with_capacity(h.size() * h.block_size()); + let mut sum = Vec::with_capacity(h.size()); + + for _ in (0..h.size() * h.block_size()).step_by(h.size()) { + h.update(&msg); + sum = h.finalize(); + msg.extend(sum.clone()); + h = algo.new(); + } + + if checksum != sum { + println!("failed: {:?}, expect: {:?}, actual: {:?}", algo, checksum, sum); + return Err(Error::new(DiskError::FileCorrupt)); + } + println!("success: {:?}", algo); } - if checksum != sum { - println!("failed: {:?}, expect: {:?}, actual: {:?}", algo, checksum, sum); - return Err(Error::new(DiskError::FileCorrupt)); - } - println!("success: {:?}", algo); + Ok(()) } - Ok(()) + #[tokio::test] + async fn test_all_bitrot_algorithms() -> Result<()> { + for algo in BITROT_ALGORITHMS.keys() { + if *algo == BitrotAlgorithm::HighwayHash256S { + continue; + } + test_bitrot_reader_writer_algo(algo.clone()).await?; + } + + Ok(()) + } + + async fn test_bitrot_reader_writer_algo(algo: BitrotAlgorithm) -> Result<()> { + let temp_dir = TempDir::new().unwrap().path().to_string_lossy().to_string(); + fs::create_dir_all(&temp_dir)?; + let volume = "testvol"; + let file_path = "testfile"; + + let ep = Endpoint::try_from(temp_dir.as_str())?; + let opt = DiskOption::default(); + let disk = new_disk(&ep, &opt).await?; + let _ = disk.make_volume(volume).await?; + let mut writer = new_bitrot_writer(disk.clone(), "", volume, file_path, 35, algo.clone(), 10); + + let _ = writer.write(b"aaaaaaaaaa").await?; + let _ = writer.write(b"aaaaaaaaaa").await?; + let _ = writer.write(b"aaaaaaaaaa").await?; + let _ = writer.write(b"aaaaa").await?; + + let mut reader = new_bitrot_reader(disk, b"", volume, file_path, 35, algo, &bitrot_writer_sum(&writer), 10); + let read_len = 10; + (_, _) = reader.read_at(0, read_len).await?; + (_, _) = reader.read_at(0, read_len).await?; + (_, _) = reader.read_at(0, read_len).await?; + (_, _) = reader.read_at(0, read_len / 2).await?; + + Ok(()) + } } diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs index 89a30746b..6408e8756 100644 --- a/ecstore/src/disk/mod.rs +++ b/ecstore/src/disk/mod.rs @@ -26,7 +26,7 @@ use protos::proto_gen::node_service::{ node_service_client::NodeServiceClient, ReadAtRequest, ReadAtResponse, WriteRequest, WriteResponse, }; use serde::{Deserialize, Serialize}; -use std::{cmp::Ordering, collections::HashMap, fmt::Debug, io::SeekFrom, path::PathBuf, sync::Arc, usize}; +use std::{any::Any, cmp::Ordering, collections::HashMap, fmt::Debug, io::SeekFrom, path::PathBuf, sync::Arc, usize}; use time::OffsetDateTime; use tokio::{ fs::File, @@ -646,6 +646,10 @@ pub enum FileWriter { #[async_trait::async_trait] impl Write for FileWriter { + fn as_any(&self) -> &dyn Any { + self + } + async fn write(&mut self, buf: &[u8]) -> Result<()> { match self { Self::Local(local_file_writer) => local_file_writer.write(buf).await, @@ -666,6 +670,10 @@ impl LocalFileWriter { #[async_trait::async_trait] impl Write for LocalFileWriter { + fn as_any(&self) -> &dyn Any { + self + } + async fn write(&mut self, buf: &[u8]) -> Result<()> { let _ = self.inner.write(buf).await?; self.inner.flush().await?; @@ -714,6 +722,10 @@ impl RemoteFileWriter { #[async_trait::async_trait] impl Write for RemoteFileWriter { + fn as_any(&self) -> &dyn Any { + self + } + async fn write(&mut self, buf: &[u8]) -> Result<()> { let request = WriteRequest { disk: self.root.to_string_lossy().to_string(), diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs index aae5ea128..4aac5c0ff 100644 --- a/ecstore/src/erasure.rs +++ b/ecstore/src/erasure.rs @@ -4,6 +4,7 @@ use bytes::Bytes; use futures::future::join_all; use futures::{Stream, StreamExt}; use reed_solomon_erasure::galois_8::ReedSolomon; +use std::any::Any; use std::fmt::Debug; use tokio::io::AsyncWriteExt; use tokio::io::DuplexStream; @@ -325,6 +326,7 @@ impl Erasure { #[async_trait::async_trait] pub trait Write { + fn as_any(&self) -> &dyn Any; async fn write(&mut self, buf: &[u8]) -> Result<()>; } diff --git a/ecstore/src/utils/os/linux.rs b/ecstore/src/utils/os/linux.rs index 254c71e6b..59b3951ca 100644 --- a/ecstore/src/utils/os/linux.rs +++ b/ecstore/src/utils/os/linux.rs @@ -1,79 +1,107 @@ -use nix::sys::{ - stat::{major, minor, stat}, - statfs::{statfs, FsType}, -}; +use nix::sys::stat::{self, stat}; +use nix::sys::statfs::{self, statfs, FsType}; +use std::io::{Error, ErrorKind}; +use std::path::Path; -use crate::{ - disk::Info, - error::{Error, Result}, -}; +use crate::{disk::Info, error::Result}; -use lazy_static::lazy_static; -use std::collections::HashMap; +/// returns total and free bytes available in a directory, e.g. `/`. +pub fn get_info(p: impl AsRef, _first_time: bool) -> std::io::Result { + let stat_fs = statfs(p.as_ref())?; -lazy_static! { - static ref FS_TYPE_TO_STRING_MAP: HashMap<&'static str, &'static str> = { - let mut m = HashMap::new(); - m.insert("1021994", "TMPFS"); - m.insert("137d", "EXT"); - m.insert("4244", "HFS"); - m.insert("4d44", "MSDOS"); - m.insert("52654973", "REISERFS"); - m.insert("5346544e", "NTFS"); - m.insert("58465342", "XFS"); - m.insert("61756673", "AUFS"); - m.insert("6969", "NFS"); - m.insert("ef51", "EXT2OLD"); - m.insert("ef53", "EXT4"); - m.insert("f15f", "ecryptfs"); - m.insert("794c7630", "overlayfs"); - m.insert("2fc12fc1", "zfs"); - m.insert("ff534d42", "cifs"); - m.insert("53464846", "wslfs"); - m + let bsize = stat_fs.block_size() as u64; + let bfree = stat_fs.blocks_free() as u64; + let bavail = stat_fs.blocks_available() as u64; + let blocks = stat_fs.blocks() as u64; + + let reserved = match bfree.checked_sub(bavail) { + Some(reserved) => reserved, + None => { + return Err(Error::new( + ErrorKind::Other, + format!( + "detected f_bavail space ({}) > f_bfree space ({}), fs corruption at ({}). please run 'fsck'", + bavail, + bfree, + p.as_ref().display() + ), + )) + } }; -} -fn get_fs_type(ftype: FsType) -> String { - let binding = format!("{:?}", ftype); - let fs_type_hex = binding.as_str(); - match FS_TYPE_TO_STRING_MAP.get(fs_type_hex) { - Some(fs_type_string) => fs_type_string.to_string(), - None => "UNKNOWN".to_string(), - } -} + let total = match blocks.checked_sub(reserved) { + Some(total) => total * bsize, + None => { + return Err(Error::new( + ErrorKind::Other, + format!( + "detected reserved space ({}) > blocks space ({}), fs corruption at ({}). please run 'fsck'", + reserved, + blocks, + p.as_ref().display() + ), + )) + } + }; + + let free = bavail * bsize; + let used = match total.checked_sub(free) { + Some(used) => used, + None => { + return Err(Error::new( + ErrorKind::Other, + format!( + "detected free space ({}) > total drive space ({}), fs corruption at ({}). please run 'fsck'", + free, + total, + p.as_ref().display() + ), + )) + } + }; + + let st = stat(p.as_ref())?; + + Ok(Info { + total, + free, + used, + files: stat_fs.files(), + ffree: stat_fs.files_free(), + fstype: get_fs_type(stat_fs.filesystem_type()).to_string(), + + major: stat::major(st.st_dev), + minor: stat::minor(st.st_dev), -pub fn get_info(path: &str, first_time: bool) -> Result { - let statfs = statfs(path)?; - let reserved_blocks = statfs.blocks_free() - statfs.blocks_available(); - let mut info = Info { - total: statfs.block_size() as u64 * (statfs.blocks() - reserved_blocks), - free: statfs.blocks() as u64 * statfs.blocks_available(), - files: statfs.files(), - ffree: statfs.files_free(), - fstype: get_fs_type(statfs.filesystem_type()), ..Default::default() - }; + }) +} - let stat = stat(path)?; - let dev_id = stat.st_dev as u64; - info.major = major(dev_id); - info.minor = minor(dev_id); +/// returns the filesystem type of the underlying mounted filesystem +/// +/// TODO The following mapping could not find the corresponding constant in `nix`: +/// +/// "137d" => "EXT", +/// "4244" => "HFS", +/// "5346544e" => "NTFS", +/// "61756673" => "AUFS", +/// "ef51" => "EXT2OLD", +/// "2fc12fc1" => "zfs", +/// "ff534d42" => "cifs", +/// "53464846" => "wslfs", +fn get_fs_type(fs_type: FsType) -> &'static str { + match fs_type { + statfs::TMPFS_MAGIC => "TMPFS", + statfs::MSDOS_SUPER_MAGIC => "MSDOS", + statfs::XFS_SUPER_MAGIC => "XFS", + statfs::NFS_SUPER_MAGIC => "NFS", + statfs::EXT4_SUPER_MAGIC => "EXT4", + statfs::ECRYPTFS_SUPER_MAGIC => "ecryptfs", + statfs::OVERLAYFS_SUPER_MAGIC => "overlayfs", + statfs::REISERFS_SUPER_MAGIC => "REISERFS", - if info.free > info.total { - return Err(Error::from_string(format!( - "detected free space {} > total drive space {}, fs corruption at {}. please run 'fsck'", - info.free, info.total, path - ))); + _ => "UNKNOWN", } - - info.used = info.total - info.free; - - if first_time { - // todo - } - - Ok(info) } pub fn same_disk(disk1: &str, disk2: &str) -> Result { @@ -81,4 +109,4 @@ pub fn same_disk(disk1: &str, disk2: &str) -> Result { let stat2 = stat(disk2)?; Ok(stat1.st_dev == stat2.st_dev) -} +} \ No newline at end of file From c0e6751893562d5334f8c32953cde429703c4cdc Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Wed, 30 Oct 2024 17:30:15 +0800 Subject: [PATCH 09/12] bitrot streaming Signed-off-by: junxiang Mu <1948535941@qq.com> --- Cargo.lock | 65 ++++++++++++ ecstore/Cargo.toml | 1 + ecstore/src/bitrot.rs | 199 +++++++++++++++++++++++++++++++---- ecstore/src/disk/local.rs | 12 +-- ecstore/src/erasure.rs | 3 + ecstore/src/heal/heal_ops.rs | 6 +- 6 files changed, 254 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2c884b2b6..5b1c17ecc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -502,6 +502,7 @@ dependencies = [ "lock", "netif", "nix", + "num", "num_cpus", "openssl", "path-absolutize", @@ -1172,12 +1173,76 @@ dependencies = [ "simdutf8", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" diff --git a/ecstore/Cargo.toml b/ecstore/Cargo.toml index a7ac29104..7dbd0afff 100644 --- a/ecstore/Cargo.toml +++ b/ecstore/Cargo.toml @@ -51,6 +51,7 @@ tower.workspace = true rmp = "0.8.14" byteorder = "1.5.0" xxhash-rust = { version = "0.8.12", features = ["xxh64"] } +num = "0.4.3" num_cpus = "1.16" s3s-policy.workspace = true diff --git a/ecstore/src/bitrot.rs b/ecstore/src/bitrot.rs index dd6ab8c01..ee8ddba63 100644 --- a/ecstore/src/bitrot.rs +++ b/ecstore/src/bitrot.rs @@ -4,6 +4,10 @@ use blake2::Blake2b512; use highway::{HighwayHash, HighwayHasher, Key}; use lazy_static::lazy_static; use sha2::{digest::core_api::BlockSizeUser, Digest, Sha256}; +use tokio::{ + spawn, + sync::mpsc::{self, Sender}, task::JoinHandle, +}; use crate::{ disk::{error::DiskError, DiskStore}, @@ -113,15 +117,15 @@ impl BitrotAlgorithm { } pub struct BitrotVerifier { - algorithm: BitrotAlgorithm, - sum: Vec, + _algorithm: BitrotAlgorithm, + _sum: Vec, } impl BitrotVerifier { pub fn new(algorithm: BitrotAlgorithm, checksum: &[u8]) -> BitrotVerifier { BitrotVerifier { - algorithm, - sum: checksum.to_vec(), + _algorithm: algorithm, + _sum: checksum.to_vec(), } } } @@ -136,32 +140,40 @@ pub fn bitrot_algorithm_from_string(s: &str) -> BitrotAlgorithm { BitrotAlgorithm::HighwayHash256S } -type BitrotWriter = Box; +type BitrotWriter = Box; -pub fn new_bitrot_writer( +pub async fn new_bitrot_writer( disk: DiskStore, - _orig_volume: &str, + orig_volume: &str, volume: &str, file_path: &str, - _length: usize, + length: usize, algo: BitrotAlgorithm, shard_size: usize, -) -> BitrotWriter { - Box::new(WholeBitrotWriter::new(disk, volume, file_path, algo, shard_size)) +) -> Result { + if algo == BitrotAlgorithm::HighwayHash256S { + return Ok(Box::new( + StreamingBitrotWriter::new(disk, orig_volume, volume, file_path, length, algo, shard_size).await?, + )); + } + Ok(Box::new(WholeBitrotWriter::new(disk, volume, file_path, algo, shard_size))) } type BitrotReader = Box; pub fn new_bitrot_reader( disk: DiskStore, - _data: &[u8], + data: &[u8], bucket: &str, file_path: &str, till_offset: usize, algo: BitrotAlgorithm, sum: &[u8], - _shard_size: usize, + shard_size: usize, ) -> BitrotReader { + if algo == BitrotAlgorithm::HighwayHash256S { + return Box::new(StreamingBitrotReader::new(disk, data, bucket, file_path, algo, till_offset, shard_size)); + } Box::new(WholeBitrotReader::new(disk, bucket, file_path, algo, till_offset, sum)) } @@ -173,12 +185,11 @@ pub fn bitrot_writer_sum(w: &BitrotWriter) -> Vec { Vec::new() } -pub fn bitrot_shard_file_size(size: i64, _shard_size: i64, algo: BitrotAlgorithm) -> i64 { +pub fn bitrot_shard_file_size(size: usize, shard_size: usize, algo: BitrotAlgorithm) -> usize { if algo != BitrotAlgorithm::HighwayHash256S { return size; } - todo!() - // ceil_frac(size, shard_size) * algo.new().size() + size + size.div_ceil(shard_size) * algo.new().size() + size } pub fn bitrot_verify( @@ -278,6 +289,148 @@ impl ReadAt for WholeBitrotReader { } } +struct StreamingBitrotWriter { + hasher: Hasher, + tx: Sender>>, + task: JoinHandle<()>, +} + +impl StreamingBitrotWriter { + pub async fn new( + disk: DiskStore, + orig_volume: &str, + volume: &str, + file_path: &str, + length: usize, + algo: BitrotAlgorithm, + shard_size: usize, + ) -> Result { + let hasher = algo.new(); + let (tx, mut rx) = mpsc::channel::>>(10); + + let total_file_size = length.div_ceil(shard_size) * hasher.size() + length; + let mut writer = disk.create_file(orig_volume, volume, file_path, total_file_size).await?; + + let task = spawn(async move { + loop { + if let Some(Some(buf)) = rx.recv().await { + let _ = writer.write(&buf).await.unwrap(); + continue; + } + + break; + } + }); + + Ok(StreamingBitrotWriter { hasher, tx, task }) + } +} + +#[async_trait::async_trait] +impl Write for StreamingBitrotWriter { + fn as_any(&self) -> &dyn Any { + self + } + + async fn write(&mut self, buf: &[u8]) -> Result<()> { + if buf.is_empty() { + return Ok(()); + } + self.hasher.reset(); + self.hasher.update(&buf); + let hash_bytes = self.hasher.clone().finalize(); + println!("hash_bytes len: {}, buf len: {}", hash_bytes.len(), buf.len()); + let _ = self.tx.send(Some(hash_bytes)).await?; + let _ = self.tx.send(Some(buf.to_vec())).await?; + + Ok(()) + } + + async fn close(self: Box) -> Result<()> { + let _ = self.tx.send(None).await?; + let _ = self.task.await; + Ok(()) + } +} + +struct StreamingBitrotReader { + disk: DiskStore, + _data: Vec, + volume: String, + file_path: String, + till_offset: usize, + curr_offset: usize, + hasher: Hasher, + shard_size: usize, + buf: Vec, + hash_bytes: Vec, +} + +impl StreamingBitrotReader { + pub fn new( + disk: DiskStore, + data: &[u8], + volume: &str, + file_path: &str, + algo: BitrotAlgorithm, + till_offset: usize, + shard_size: usize, + ) -> Self { + let hasher = algo.new(); + Self { + disk, + _data: data.to_vec(), + volume: volume.to_string(), + file_path: file_path.to_string(), + till_offset: till_offset.div_ceil(shard_size) * hasher.size() + till_offset, + curr_offset: 0, + hash_bytes: Vec::with_capacity(hasher.size()), + hasher, + shard_size, + buf: Vec::new(), + } + } +} + +#[async_trait::async_trait] +impl ReadAt for StreamingBitrotReader { + async fn read_at(&mut self, offset: usize, length: usize) -> Result<(Vec, usize)> { + println!("in read_at"); + if offset % self.shard_size != 0 { + return Err(Error::new(DiskError::Unexpected)); + } + if self.buf.is_empty() { + self.curr_offset = offset; + let stream_offset = (offset / self.shard_size) * self.hasher.size() + offset; + let buf_len = self.till_offset - stream_offset; + let mut file = self.disk.read_file(&self.volume, &self.file_path).await?; + println!("stream_offset: {}, buf_len: {}", stream_offset, buf_len); + let (buf, _) = file.read_at(stream_offset, buf_len).await?; + println!("buf: {:?}, len: {}", buf, buf.len()); + self.buf = buf; + } + if offset != self.curr_offset { + return Err(Error::new(DiskError::Unexpected)); + } + + self.hash_bytes = self.buf.drain(0..self.hash_bytes.capacity()).collect(); + let buf = self.buf.drain(0..length).collect::>(); + println!("self.buf: {:?}", self.buf); + self.hasher.reset(); + self.hasher.update(&buf); + let actual = self.hasher.clone().finalize(); + if actual != self.hash_bytes { + println!("except: {:?}, actual: {:?}", self.hash_bytes, actual); + return Err(Error::new(DiskError::FileCorrupt)); + } + + let readed_len = buf.len(); + self.curr_offset += readed_len; + + Ok((buf, readed_len)) + } +} + #[cfg(test)] mod test { use std::{collections::HashMap, fs}; @@ -347,9 +500,6 @@ mod test { #[tokio::test] async fn test_all_bitrot_algorithms() -> Result<()> { for algo in BITROT_ALGORITHMS.keys() { - if *algo == BitrotAlgorithm::HighwayHash256S { - continue; - } test_bitrot_reader_writer_algo(algo.clone()).await?; } @@ -366,19 +516,22 @@ mod test { let opt = DiskOption::default(); let disk = new_disk(&ep, &opt).await?; let _ = disk.make_volume(volume).await?; - let mut writer = new_bitrot_writer(disk.clone(), "", volume, file_path, 35, algo.clone(), 10); + let mut writer = new_bitrot_writer(disk.clone(), "", volume, file_path, 35, algo.clone(), 10).await?; let _ = writer.write(b"aaaaaaaaaa").await?; let _ = writer.write(b"aaaaaaaaaa").await?; let _ = writer.write(b"aaaaaaaaaa").await?; let _ = writer.write(b"aaaaa").await?; - let mut reader = new_bitrot_reader(disk, b"", volume, file_path, 35, algo, &bitrot_writer_sum(&writer), 10); + let sum = bitrot_writer_sum(&writer); + let _ = writer.close().await?; + + let mut reader = new_bitrot_reader(disk, b"", volume, file_path, 35, algo, &sum, 10); let read_len = 10; (_, _) = reader.read_at(0, read_len).await?; - (_, _) = reader.read_at(0, read_len).await?; - (_, _) = reader.read_at(0, read_len).await?; - (_, _) = reader.read_at(0, read_len / 2).await?; + (_, _) = reader.read_at(10, read_len).await?; + (_, _) = reader.read_at(20, read_len).await?; + (_, _) = reader.read_at(30, read_len / 2).await?; Ok(()) } diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index d37346aee..31bb69a9c 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -147,9 +147,9 @@ impl LocalDisk { id: disk_id.to_string(), ..Default::default() }; - if root { - return Err(Error::new(DiskError::DriveIsRoot)); - } + // if root { + // return Err(Error::new(DiskError::DriveIsRoot)); + // } // disk_info.healing = Ok(disk_info) @@ -185,9 +185,9 @@ impl LocalDisk { disk.minor = info.minor; disk.fstype = info.fstype; - if root { - return Err(Error::new(DiskError::DriveIsRoot)); - } + // if root { + // return Err(Error::new(DiskError::DriveIsRoot)); + // } if info.nrrequests > 0 { disk.nrrequests = info.nrrequests; diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs index 4aac5c0ff..5eecdd696 100644 --- a/ecstore/src/erasure.rs +++ b/ecstore/src/erasure.rs @@ -328,6 +328,9 @@ impl Erasure { pub trait Write { fn as_any(&self) -> &dyn Any; async fn write(&mut self, buf: &[u8]) -> Result<()>; + async fn close(self: Box) -> Result<()> { + Ok(()) + } } #[async_trait::async_trait] diff --git a/ecstore/src/heal/heal_ops.rs b/ecstore/src/heal/heal_ops.rs index 87ae312f2..d0ef9e555 100644 --- a/ecstore/src/heal/heal_ops.rs +++ b/ecstore/src/heal/heal_ops.rs @@ -483,7 +483,7 @@ impl AllHealState { hsp.client_address = he.client_address.clone(); hsp.start_time = he.start_time; - he.stop(); + he.stop().await; loop { if he.has_ended().await { @@ -493,7 +493,7 @@ impl AllHealState { sleep(Duration::from_secs(1)).await; } - self.mu.write().await; + let _ = self.mu.write().await; self.heal_seq_map.remove(path); } else { hsp.client_token = "unknown".to_string(); @@ -526,7 +526,7 @@ impl AllHealState { } } - self.mu.write().await; + let _ = self.mu.write().await; for (k, v) in self.heal_seq_map.iter() { if !v.has_ended().await && (has_profix(&k, path_s) || has_profix(path_s, &k)) { From 640cf1caf312928cb0fc470e4bc6b012744d2c6b Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Thu, 31 Oct 2024 11:18:00 +0800 Subject: [PATCH 10/12] support bitrot Signed-off-by: junxiang Mu <1948535941@qq.com> --- ecstore/src/bitrot.rs | 60 ++++++++++++++++++++----------- ecstore/src/erasure.rs | 34 ++++++++++++------ ecstore/src/heal/heal_commands.rs | 28 +++++++-------- ecstore/src/heal/heal_ops.rs | 24 ++++++------- ecstore/src/store_api.rs | 5 +-- 5 files changed, 93 insertions(+), 58 deletions(-) diff --git a/ecstore/src/bitrot.rs b/ecstore/src/bitrot.rs index ee8ddba63..57744c6db 100644 --- a/ecstore/src/bitrot.rs +++ b/ecstore/src/bitrot.rs @@ -6,7 +6,8 @@ use lazy_static::lazy_static; use sha2::{digest::core_api::BlockSizeUser, Digest, Sha256}; use tokio::{ spawn, - sync::mpsc::{self, Sender}, task::JoinHandle, + sync::mpsc::{self, Sender}, + task::JoinHandle, }; use crate::{ @@ -33,7 +34,7 @@ lazy_static! { // ]; const MAGIC_HIGHWAY_HASH256_KEY: &[u64; 4] = &[3, 4, 2, 1]; -#[derive(Clone)] +#[derive(Clone, Debug)] pub enum Hasher { SHA256(Sha256), HighwayHash256(HighwayHasher), @@ -116,6 +117,7 @@ impl BitrotAlgorithm { } } +#[derive(Debug)] pub struct BitrotVerifier { _algorithm: BitrotAlgorithm, _sum: Vec, @@ -140,7 +142,7 @@ pub fn bitrot_algorithm_from_string(s: &str) -> BitrotAlgorithm { BitrotAlgorithm::HighwayHash256S } -type BitrotWriter = Box; +pub type BitrotWriter = Box; pub async fn new_bitrot_writer( disk: DiskStore, @@ -159,7 +161,7 @@ pub async fn new_bitrot_writer( Ok(Box::new(WholeBitrotWriter::new(disk, volume, file_path, algo, shard_size))) } -type BitrotReader = Box; +pub type BitrotReader = Box; pub fn new_bitrot_reader( disk: DiskStore, @@ -177,6 +179,16 @@ pub fn new_bitrot_reader( Box::new(WholeBitrotReader::new(disk, bucket, file_path, algo, till_offset, sum)) } +pub async fn close_bitrot_writers(writers: &mut [Option]) -> Result<()> { + for w in writers.into_iter() { + if let Some(w) = w { + let _ = w.close().await?; + } + } + + Ok(()) +} + pub fn bitrot_writer_sum(w: &BitrotWriter) -> Vec { if let Some(w) = w.as_any().downcast_ref::() { return w.hash.clone().finalize(); @@ -245,6 +257,7 @@ impl Write for WholeBitrotWriter { } } +#[derive(Debug)] pub struct WholeBitrotReader { disk: DiskStore, volume: String, @@ -292,7 +305,7 @@ impl ReadAt for WholeBitrotReader { struct StreamingBitrotWriter { hasher: Hasher, tx: Sender>>, - task: JoinHandle<()>, + task: Option>, } impl StreamingBitrotWriter { @@ -322,7 +335,11 @@ impl StreamingBitrotWriter { } }); - Ok(StreamingBitrotWriter { hasher, tx, task }) + Ok(StreamingBitrotWriter { + hasher, + tx, + task: Some(task), + }) } } @@ -339,20 +356,22 @@ impl Write for StreamingBitrotWriter { self.hasher.reset(); self.hasher.update(&buf); let hash_bytes = self.hasher.clone().finalize(); - println!("hash_bytes len: {}, buf len: {}", hash_bytes.len(), buf.len()); let _ = self.tx.send(Some(hash_bytes)).await?; let _ = self.tx.send(Some(buf.to_vec())).await?; Ok(()) } - async fn close(self: Box) -> Result<()> { + async fn close(&mut self) -> Result<()> { let _ = self.tx.send(None).await?; - let _ = self.task.await; + if let Some(task) = self.task.take() { + let _ = task.await; // 等待任务完成 + } Ok(()) } } +#[derive(Debug)] struct StreamingBitrotReader { disk: DiskStore, _data: Vec, @@ -395,7 +414,6 @@ impl StreamingBitrotReader { #[async_trait::async_trait] impl ReadAt for StreamingBitrotReader { async fn read_at(&mut self, offset: usize, length: usize) -> Result<(Vec, usize)> { - println!("in read_at"); if offset % self.shard_size != 0 { return Err(Error::new(DiskError::Unexpected)); } @@ -404,9 +422,7 @@ impl ReadAt for StreamingBitrotReader { let stream_offset = (offset / self.shard_size) * self.hasher.size() + offset; let buf_len = self.till_offset - stream_offset; let mut file = self.disk.read_file(&self.volume, &self.file_path).await?; - println!("stream_offset: {}, buf_len: {}", stream_offset, buf_len); let (buf, _) = file.read_at(stream_offset, buf_len).await?; - println!("buf: {:?}, len: {}", buf, buf.len()); self.buf = buf; } if offset != self.curr_offset { @@ -415,12 +431,10 @@ impl ReadAt for StreamingBitrotReader { self.hash_bytes = self.buf.drain(0..self.hash_bytes.capacity()).collect(); let buf = self.buf.drain(0..length).collect::>(); - println!("self.buf: {:?}", self.buf); self.hasher.reset(); self.hasher.update(&buf); let actual = self.hasher.clone().finalize(); if actual != self.hash_bytes { - println!("except: {:?}, actual: {:?}", self.hash_bytes, actual); return Err(Error::new(DiskError::FileCorrupt)); } @@ -488,10 +502,8 @@ mod test { } if checksum != sum { - println!("failed: {:?}, expect: {:?}, actual: {:?}", algo, checksum, sum); return Err(Error::new(DiskError::FileCorrupt)); } - println!("success: {:?}", algo); } Ok(()) @@ -500,6 +512,9 @@ mod test { #[tokio::test] async fn test_all_bitrot_algorithms() -> Result<()> { for algo in BITROT_ALGORITHMS.keys() { + if *algo != BitrotAlgorithm::HighwayHash256S { + continue; + } test_bitrot_reader_writer_algo(algo.clone()).await?; } @@ -528,10 +543,15 @@ mod test { let mut reader = new_bitrot_reader(disk, b"", volume, file_path, 35, algo, &sum, 10); let read_len = 10; - (_, _) = reader.read_at(0, read_len).await?; - (_, _) = reader.read_at(10, read_len).await?; - (_, _) = reader.read_at(20, read_len).await?; - (_, _) = reader.read_at(30, read_len / 2).await?; + let mut result: Vec; + (result, _) = reader.read_at(0, read_len).await?; + assert_eq!(result, b"aaaaaaaaaa"); + (result, _) = reader.read_at(10, read_len).await?; + assert_eq!(result, b"aaaaaaaaaa"); + (result, _) = reader.read_at(20, read_len).await?; + assert_eq!(result, b"aaaaaaaaaa"); + (result, _) = reader.read_at(30, read_len / 2).await?; + assert_eq!(result, b"aaaaa"); Ok(()) } diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs index 5eecdd696..9ef335a1a 100644 --- a/ecstore/src/erasure.rs +++ b/ecstore/src/erasure.rs @@ -1,3 +1,4 @@ +use crate::bitrot::{close_bitrot_writers, BitrotReader, BitrotWriter}; use crate::error::{Error, Result, StdError}; use crate::quorum::{object_op_ignored_errs, reduce_write_quorum_errs}; use bytes::Bytes; @@ -15,13 +16,12 @@ use uuid::Uuid; use crate::chunk_stream::ChunkedStream; use crate::disk::error::DiskError; -use crate::disk::{FileReader, FileWriter}; pub struct Erasure { data_shards: usize, parity_shards: usize, encoder: Option, - block_size: usize, + pub block_size: usize, _id: Uuid, } @@ -48,7 +48,7 @@ impl Erasure { pub async fn encode( &self, body: S, - writers: &mut [Option], + writers: &mut [Option], // block_size: usize, total_size: usize, write_quorum: usize, @@ -111,6 +111,8 @@ impl Erasure { // debug!(" encode_data done shard block num {}", idx); + let _ = close_bitrot_writers(writers).await?; + Ok(total) // loop { @@ -124,7 +126,7 @@ impl Erasure { pub async fn decode( &self, writer: &mut DuplexStream, - readers: Vec>, + readers: Vec>, offset: usize, length: usize, total_length: usize, @@ -302,7 +304,7 @@ impl Erasure { } // 算出每个分片大小 - fn shard_size(&self, data_size: usize) -> usize { + pub fn shard_size(&self, data_size: usize) -> usize { (data_size + self.data_shards - 1) / self.data_shards } // returns final erasure size from original size. @@ -322,25 +324,37 @@ impl Erasure { // } // num_shards * self.shard_size(self.block_size) } + + pub fn shard_file_offset(&self, start_offset: usize, length: usize, total_length: usize) -> usize { + let shard_size = self.shard_size(self.block_size); + let shard_file_size = self.shard_file_size(total_length); + let end_shard = (start_offset + length) / self.block_size; + let mut till_offset = end_shard * shard_size + shard_size; + if till_offset > shard_file_size { + till_offset = shard_file_size; + } + + till_offset + } } #[async_trait::async_trait] pub trait Write { fn as_any(&self) -> &dyn Any; async fn write(&mut self, buf: &[u8]) -> Result<()>; - async fn close(self: Box) -> Result<()> { + async fn close(&mut self) -> Result<()> { Ok(()) } } #[async_trait::async_trait] -pub trait ReadAt { +pub trait ReadAt: Debug { async fn read_at(&mut self, offset: usize, length: usize) -> Result<(Vec, usize)>; } #[derive(Debug)] pub struct ShardReader { - readers: Vec>, // 磁盘 + readers: Vec>, // 磁盘 data_block_count: usize, // 总的分片数量 parity_block_count: usize, shard_size: usize, // 每个分片的块大小 一次读取一块 @@ -349,7 +363,7 @@ pub struct ShardReader { } impl ShardReader { - pub fn new(readers: Vec>, ec: &Erasure, offset: usize, total_length: usize) -> Self { + pub fn new(readers: Vec>, ec: &Erasure, offset: usize, total_length: usize) -> Self { Self { readers, data_block_count: ec.data_shards, @@ -386,7 +400,7 @@ impl ShardReader { continue; } - let disk: &mut FileReader = disk.as_mut().unwrap(); + let disk: &mut BitrotReader = disk.as_mut().unwrap(); futures.push(disk.read_at(self.offset, read_length)); } diff --git a/ecstore/src/heal/heal_commands.rs b/ecstore/src/heal/heal_commands.rs index 1c1724ecf..29d9bd550 100644 --- a/ecstore/src/heal/heal_commands.rs +++ b/ecstore/src/heal/heal_commands.rs @@ -160,7 +160,7 @@ impl HealingTracker { } pub async fn reset_healing(&mut self) { - self.mu.write().await; + let _ = self.mu.write().await; self.items_healed = 0; self.items_failed = 0; self.bytes_done = 0; @@ -178,37 +178,37 @@ impl HealingTracker { } pub async fn get_last_update(&self) -> u64 { - self.mu.read().await; + let _ = self.mu.read().await; self.last_update } pub async fn get_bucket(&self) -> String { - self.mu.read().await; + let _ = self.mu.read().await; self.bucket.clone() } pub async fn set_bucket(&mut self, bucket: &str) { - self.mu.write().await; + let _ = self.mu.write().await; self.bucket = bucket.to_string(); } pub async fn get_object(&self) -> String { - self.mu.read().await; + let _ = self.mu.read().await; self.object.clone() } pub async fn set_object(&mut self, object: &str) { - self.mu.write().await; + let _ = self.mu.write().await; self.object = object.to_string(); } pub async fn update_progress(&mut self, success: bool, skipped: bool, by: u64) { - self.mu.write().await; + let _ = self.mu.write().await; if success { self.items_healed += 1; @@ -227,7 +227,7 @@ impl HealingTracker { if healing(&disk.path().to_string_lossy().to_string()).await?.is_none() { return Err(Error::from_string(format!("healingTracker: drive {} is not marked as healing", self.id))); } - self.mu.write().await; + let _ = self.mu.write().await; if self.id.is_empty() || self.pool_index.is_none() || self.set_index.is_none() || self.disk_index.is_none() { self.id = disk.get_disk_id().await?.map_or("".to_string(), |id| id.to_string()); let disk_location = disk.get_disk_location(); @@ -241,7 +241,7 @@ impl HealingTracker { } pub async fn save(&mut self) -> Result<()> { - self.mu.write().await; + let _ = self.mu.write().await; if self.pool_index.is_none() || self.set_index.is_none() || self.disk_index.is_none() { let layer = new_object_layer_fn(); let lock = layer.read().await; @@ -290,7 +290,7 @@ impl HealingTracker { } async fn is_healed(&self, bucket: &str) -> bool { - self.mu.read().await; + let _ = self.mu.read().await; for v in self.healed_buckets.iter() { if v == bucket { return true; @@ -301,7 +301,7 @@ impl HealingTracker { } async fn resume(&mut self) { - self.mu.write().await; + let _ = self.mu.write().await; self.items_healed = self.resume_items_healed; self.items_failed = self.resume_items_failed; @@ -312,7 +312,7 @@ impl HealingTracker { } async fn bucket_done(&mut self, bucket: &str) { - self.mu.write().await; + let _ = self.mu.write().await; self.resume_items_healed = self.items_healed; self.resume_items_failed = self.items_failed; @@ -326,7 +326,7 @@ impl HealingTracker { } async fn set_queue_buckets(&mut self, buckets: &[BucketInfo]) { - self.mu.write().await; + let _ = self.mu.write().await; buckets.iter().for_each(|bucket| { if !self.healed_buckets.contains(&bucket.name) { @@ -336,7 +336,7 @@ impl HealingTracker { } pub async fn to_healing_disk(&self) -> HealingDisk { - self.mu.read().await; + let _ = self.mu.read().await; HealingDisk { id: self.id.clone(), diff --git a/ecstore/src/heal/heal_ops.rs b/ecstore/src/heal/heal_ops.rs index d0ef9e555..c75a15e4b 100644 --- a/ecstore/src/heal/heal_ops.rs +++ b/ecstore/src/heal/heal_ops.rs @@ -13,7 +13,7 @@ use std::{ path::Path, pin::Pin, sync::Arc, - time::{Duration, Instant, SystemTime, UNIX_EPOCH}, + time::{Duration, SystemTime, UNIX_EPOCH}, }; use tokio::{ select, spawn, @@ -202,7 +202,7 @@ impl HealSequence { async fn stop(&self) { let w = self.tx.write().await; - w.send(true); + let _ = w.send(true); } async fn push_heal_result_item(&self, r: &HealResultItem) -> Result<()> { @@ -363,7 +363,7 @@ impl AllHealState { } async fn pop_heal_local_disks(&mut self, heal_local_disks: &[Endpoint]) { - self.mu.write().await; + let _ = self.mu.write().await; self.heal_local_disks.retain(|k, _| { if heal_local_disks.contains(k) { @@ -383,14 +383,14 @@ impl AllHealState { } async fn update_heal_status(&mut self, tracker: &HealingTracker) { - self.mu.write().await; - tracker.mu.read().await; + let _ = self.mu.write().await; + let _ = tracker.mu.read().await; self.heal_status.insert(tracker.id.clone(), tracker.clone()); } async fn get_local_healing_disks(&self) -> HashMap { - self.mu.read().await; + let _ = self.mu.read().await; let mut dst = HashMap::new(); for v in self.heal_status.values() { @@ -401,7 +401,7 @@ impl AllHealState { } async fn get_heal_local_disk_endpoints(&self) -> Endpoints { - self.mu.read().await; + let _ = self.mu.read().await; let mut endpoints = Vec::new(); self.heal_local_disks.iter().for_each(|(k, v)| { @@ -414,13 +414,13 @@ impl AllHealState { } async fn set_disk_healing_status(&mut self, ep: Endpoint, healing: bool) { - self.mu.write().await; + let _ = self.mu.write().await; self.heal_local_disks.insert(ep, healing); } async fn push_heal_local_disks(&mut self, heal_local_disks: &[Endpoint]) { - self.mu.write().await; + let _ = self.mu.write().await; heal_local_disks.iter().for_each(|heal_local_disk| { self.heal_local_disks.insert(heal_local_disk.clone(), false); @@ -436,7 +436,7 @@ impl AllHealState { } } _ = sleep(Duration::from_secs(5 * 60)) => { - self.mu.write().await; + let _ = self.mu.write().await; let now = SystemTime::now(); let mut keys_to_reomve = Vec::new(); @@ -454,7 +454,7 @@ impl AllHealState { } async fn get_heal_sequence_by_token(&self, token: &str) -> (Option, bool) { - self.mu.read().await; + let _ = self.mu.read().await; for v in self.heal_seq_map.values() { if v.client_token == token { @@ -466,7 +466,7 @@ impl AllHealState { } async fn get_heal_sequence(&self, path: &str) -> Option { - self.mu.read().await; + let _ = self.mu.read().await; self.heal_seq_map.get(path).cloned() } diff --git a/ecstore/src/store_api.rs b/ecstore/src/store_api.rs index 51005a071..44910a4dc 100644 --- a/ecstore/src/store_api.rs +++ b/ecstore/src/store_api.rs @@ -12,7 +12,6 @@ use http::HeaderMap; use rmp_serde::Serializer; use s3s::dto::StreamingBlob; use serde::{Deserialize, Serialize}; -use sha2::Sha256; use time::OffsetDateTime; use uuid::Uuid; @@ -249,7 +248,7 @@ impl ErasureInfo { } } - ChecksumInfo {algorithm: BitrotAlgorithm::HighwayHash256S, ..Default::default()} + ChecksumInfo {algorithm: DEFAULT_BITROT_ALGO, ..Default::default()} } } @@ -261,6 +260,8 @@ pub struct ChecksumInfo { pub hash: Vec, } +pub const DEFAULT_BITROT_ALGO: BitrotAlgorithm = BitrotAlgorithm::HighwayHash256S; + #[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone, Eq, Hash)] // BitrotAlgorithm specifies a algorithm used for bitrot protection. pub enum BitrotAlgorithm { From a8d36fba8f9bf9687eac925079ee06d74b8db4f6 Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Fri, 1 Nov 2024 17:38:50 +0800 Subject: [PATCH 11/12] heal object(1) Signed-off-by: junxiang Mu <1948535941@qq.com> --- .../src/generated/proto_gen/node_service.rs | 194 ++++++++++++++++++ common/protos/src/node.proto | 28 +++ ecstore/src/bitrot.rs | 56 +++-- ecstore/src/disk/error.rs | 8 + ecstore/src/disk/local.rs | 108 ++++++++-- ecstore/src/disk/mod.rs | 6 +- ecstore/src/disk/remote.rs | 53 ++++- ecstore/src/store_api.rs | 68 +++++- rustfs/src/grpc.rs | 104 +++++++++- 9 files changed, 574 insertions(+), 51 deletions(-) diff --git a/common/protos/src/generated/proto_gen/node_service.rs b/common/protos/src/generated/proto_gen/node_service.rs index c2f957181..9d13aa58b 100644 --- a/common/protos/src/generated/proto_gen/node_service.rs +++ b/common/protos/src/generated/proto_gen/node_service.rs @@ -128,6 +128,48 @@ pub struct DeleteResponse { pub error_info: ::core::option::Option<::prost::alloc::string::String>, } #[derive(Clone, PartialEq, ::prost::Message)] +pub struct VerifyFileRequest { + /// indicate which one in the disks + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub volume: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub path: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub file_info: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct VerifyFileResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, tag = "2")] + pub check_parts_resp: ::prost::alloc::string::String, + #[prost(string, optional, tag = "3")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CheckPartsRequest { + /// indicate which one in the disks + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub volume: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub path: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub file_info: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CheckPartsResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, tag = "2")] + pub check_parts_resp: ::prost::alloc::string::String, + #[prost(string, optional, tag = "3")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] pub struct RenamePartRequst { #[prost(string, tag = "1")] pub disk: ::prost::alloc::string::String, @@ -814,6 +856,54 @@ pub mod node_service_client { .insert(GrpcMethod::new("node_service.NodeService", "Delete")); self.inner.unary(req, path, codec).await } + pub async fn verify_file( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/VerifyFile", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "VerifyFile")); + self.inner.unary(req, path, codec).await + } + pub async fn check_parts( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/CheckParts", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "CheckParts")); + self.inner.unary(req, path, codec).await + } pub async fn rename_part( &mut self, request: impl tonic::IntoRequest, @@ -1544,6 +1634,20 @@ pub mod node_service_server { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + async fn verify_file( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + async fn check_parts( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn rename_part( &self, request: tonic::Request, @@ -2173,6 +2277,96 @@ pub mod node_service_server { }; Box::pin(fut) } + "/node_service.NodeService/VerifyFile" => { + #[allow(non_camel_case_types)] + struct VerifyFileSvc(pub Arc); + impl< + T: NodeService, + > tonic::server::UnaryService + for VerifyFileSvc { + type Response = super::VerifyFileResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::verify_file(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = VerifyFileSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/CheckParts" => { + #[allow(non_camel_case_types)] + struct CheckPartsSvc(pub Arc); + impl< + T: NodeService, + > tonic::server::UnaryService + for CheckPartsSvc { + type Response = super::CheckPartsResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::check_parts(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = CheckPartsSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } "/node_service.NodeService/RenamePart" => { #[allow(non_camel_case_types)] struct RenamePartSvc(pub Arc); diff --git a/common/protos/src/node.proto b/common/protos/src/node.proto index 51ac6cbc0..f789a302f 100644 --- a/common/protos/src/node.proto +++ b/common/protos/src/node.proto @@ -88,6 +88,32 @@ message DeleteResponse { optional string error_info = 2; } +message VerifyFileRequest { + string disk = 1; // indicate which one in the disks + string volume = 2; + string path = 3; + string file_info = 4; +} + +message VerifyFileResponse { + bool success = 1; + string check_parts_resp = 2; + optional string error_info = 3; +} + +message CheckPartsRequest { + string disk = 1; // indicate which one in the disks + string volume = 2; + string path = 3; + string file_info = 4; +} + +message CheckPartsResponse { + bool success = 1; + string check_parts_resp = 2; + optional string error_info = 3; +} + message RenamePartRequst { string disk = 1; string src_volume = 2; @@ -381,6 +407,8 @@ service NodeService { rpc ReadAll(ReadAllRequest) returns (ReadAllResponse) {}; rpc WriteAll(WriteAllRequest) returns (WriteAllResponse) {}; rpc Delete(DeleteRequest) returns (DeleteResponse) {}; + rpc VerifyFile(VerifyFileRequest) returns (VerifyFileResponse) {}; + rpc CheckParts(CheckPartsRequest) returns (CheckPartsResponse) {}; rpc RenamePart(RenamePartRequst) returns (RenamePartResponse) {}; rpc RenameFile(RenameFileRequst) returns (RenameFileResponse) {}; rpc Write(WriteRequest) returns (WriteResponse) {}; diff --git a/ecstore/src/bitrot.rs b/ecstore/src/bitrot.rs index 57744c6db..7d24aefa3 100644 --- a/ecstore/src/bitrot.rs +++ b/ecstore/src/bitrot.rs @@ -1,4 +1,8 @@ -use std::{any::Any, collections::HashMap, io::Cursor}; +use std::{ + any::Any, + collections::HashMap, + io::{Cursor, Read}, +}; use blake2::Blake2b512; use highway::{HighwayHash, HighwayHasher, Key}; @@ -205,21 +209,49 @@ pub fn bitrot_shard_file_size(size: usize, shard_size: usize, algo: BitrotAlgori } pub fn bitrot_verify( - r: Cursor>, - _want_size: usize, - _part_size: usize, + r: &mut Cursor>, + want_size: usize, + part_size: usize, algo: BitrotAlgorithm, want: Vec, - _shard_size: usize, + mut shard_size: usize, ) -> Result<()> { if algo != BitrotAlgorithm::HighwayHash256S { let mut h = algo.new(); - h.update(r.into_inner()); + h.update(r.get_ref()); if h.finalize() != want { return Err(Error::new(DiskError::FileCorrupt)); } + + return Ok(()); } - todo!() + let mut h = algo.new(); + let mut hash_buf = vec![0; h.size()]; + let mut left = want_size; + + if left != bitrot_shard_file_size(part_size, shard_size, algo) { + return Err(Error::new(DiskError::FileCorrupt)); + } + + while left > 0 { + h.reset(); + let n = r.read(&mut hash_buf)?; + left -= n; + + if left < shard_size { + shard_size = left; + } + + let mut buf = vec![0; shard_size]; + let read = r.read(&mut buf)?; + h.update(buf); + left -= read; + if h.clone().finalize() != hash_buf[0..n] { + return Err(Error::new(DiskError::FileCorrupt)); + } + } + + Ok(()) } pub struct WholeBitrotWriter { @@ -295,7 +327,7 @@ impl ReadAt for WholeBitrotReader { return Err(Error::new(DiskError::LessData)); } - return Ok((buf.split_off(length), length)); + return Ok((buf.drain(0..length).collect::>(), length)); } Err(Error::new(DiskError::LessData)) @@ -453,10 +485,7 @@ mod test { use tempfile::TempDir; use crate::{ - bitrot::{new_bitrot_writer, BITROT_ALGORITHMS}, - disk::{endpoint::Endpoint, error::DiskError, new_disk, DiskOption}, - error::{Error, Result}, - store_api::BitrotAlgorithm, + bitrot::{new_bitrot_writer, BITROT_ALGORITHMS}, disk::{endpoint::Endpoint, error::DiskError, new_disk, DiskOption}, error::{Error, Result}, store_api::BitrotAlgorithm }; use super::{bitrot_writer_sum, new_bitrot_reader}; @@ -512,9 +541,6 @@ mod test { #[tokio::test] async fn test_all_bitrot_algorithms() -> Result<()> { for algo in BITROT_ALGORITHMS.keys() { - if *algo != BitrotAlgorithm::HighwayHash256S { - continue; - } test_bitrot_reader_writer_algo(algo.clone()).await?; } diff --git a/ecstore/src/disk/error.rs b/ecstore/src/disk/error.rs index da8c7063e..8fbfbf474 100644 --- a/ecstore/src/disk/error.rs +++ b/ecstore/src/disk/error.rs @@ -100,6 +100,12 @@ pub enum DiskError { #[error("more data was sent than what was advertised")] MoreData, + + #[error("outdated XL meta")] + OutdatedXLMeta, + + #[error("part missing or corrupt")] + PartMissingOrCorrupt, } impl DiskError { @@ -202,6 +208,8 @@ pub fn clone_disk_err(e: &DiskError) -> Error { DiskError::CrossDeviceLink => Error::new(DiskError::CrossDeviceLink), DiskError::LessData => Error::new(DiskError::LessData), DiskError::MoreData => Error::new(DiskError::MoreData), + DiskError::OutdatedXLMeta => Error::new(DiskError::OutdatedXLMeta), + DiskError::PartMissingOrCorrupt => Error::new(DiskError::PartMissingOrCorrupt), } } diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 31bb69a9c..1450d0b46 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -6,6 +6,7 @@ use super::{ FileReader, FileWriter, Info, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, }; +use crate::bitrot::bitrot_verify; use crate::cache_value::cache::{Cache, Opts}; use crate::disk::error::{ convert_access_error, is_sys_err_handle_invalid, is_sys_err_invalid_arg, is_sys_err_is_dir, is_sys_err_not_dir, @@ -15,8 +16,9 @@ use crate::disk::os::check_path_length; use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE}; use crate::error::{Error, Result}; use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold}; +use crate::set_disk::{conv_part_err_to_int, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, CHECK_PART_VOLUME_NOT_FOUND}; use crate::store_api::BitrotAlgorithm; -use crate::utils::fs::{lstat, O_APPEND, O_CREATE, O_RDONLY, O_WRONLY}; +use crate::utils::fs::{access, lstat, O_APPEND, O_CREATE, O_RDONLY, O_WRONLY}; use crate::utils::os::get_info; use crate::utils::path::{clean, has_suffix, SLASH_SEPARATOR}; use crate::{ @@ -26,6 +28,8 @@ use crate::{ }; use path_absolutize::Absolutize; use std::fmt::Debug; +use std::io::Cursor; +use std::os::unix::fs::MetadataExt; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::Arc; use std::time::Duration; @@ -38,7 +42,7 @@ use tokio::fs::{self, File}; use tokio::io::{AsyncReadExt, AsyncWriteExt, ErrorKind}; use tokio::runtime::Runtime; use tokio::sync::RwLock; -use tracing::{error, warn}; +use tracing::{error, info, warn}; use uuid::Uuid; #[derive(Debug)] @@ -180,7 +184,7 @@ impl LocalDisk { // format_data: Mutex::new(format_data), // format_last_check: Mutex::new(format_last_check), }; - let (info, root) = get_disk_info(root_clone).await?; + let (info, _root) = get_disk_info(root_clone).await?; disk.major = info.major; disk.minor = info.minor; disk.fstype = info.fstype; @@ -653,17 +657,18 @@ impl LocalDisk { async fn bitrot_verify( &self, part_path: &PathBuf, - part_size: u64, + part_size: usize, algo: BitrotAlgorithm, sum: &[u8], - shard_size: u64, + shard_size: usize, ) -> Result<()> { - let file = utils::fs::open_file(part_path, O_CREATE | O_WRONLY) + let mut file = utils::fs::open_file(part_path, O_CREATE | O_WRONLY) .await .map_err(os_err_to_file_err)?; - - todo!() + let mut data = Vec::new(); + let n = file.read_to_end(&mut data).await?; + bitrot_verify(&mut Cursor::new(data), n, part_size, algo, sum.to_vec(), shard_size) } } @@ -861,7 +866,7 @@ impl DiskAPI for LocalDisk { Ok(()) } - async fn verify_file(&self, volume: &str, path: &str, fi: FileInfo) -> Result { + async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result { let volume_dir = self.get_bucket_path(volume)?; if !skip_access_checks(volume) { if let Err(e) = utils::fs::access(&volume_dir).await { @@ -873,17 +878,92 @@ impl DiskAPI for LocalDisk { results: Vec::with_capacity(fi.parts.len()), }; - let erasure = fi.erasure; - fi.parts.iter().enumerate().for_each(|(i, part)| { + let erasure = &fi.erasure; + for (i, part) in fi.parts.iter().enumerate() { let checksum_info = erasure.get_checksum_info(part.number); let part_path = Path::new(&volume_dir) .join(path) .join(fi.data_dir.map_or("".to_string(), |dir| dir.to_string())) .join(format!("part.{}", part.number)); + let err = match self + .bitrot_verify( + &part_path, + erasure.shard_file_size(part.size), + checksum_info.algorithm, + &checksum_info.hash, + erasure.shard_size(erasure.block_size), + ) + .await + { + Ok(_) => None, + Err(err) => Some(err), + }; + resp.results[i] = conv_part_err_to_int(&err); + if resp.results[i] == CHECK_PART_UNKNOWN { + if let Some(err) = err { + match err.downcast_ref::() { + Some(DiskError::FileAccessDenied) => {} + _ => { + info!("part unknown, disk: {}, path: {:?}", self.to_string(), part_path); + } + } + } + } + } - // self.bi - }); - todo!() + Ok(resp) + } + + async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result { + let volume_dir = self.get_bucket_path(&volume)?; + check_path_length(volume_dir.join(path).to_string_lossy().as_ref())?; + let mut resp = CheckPartsResp { + results: vec![0; fi.parts.len()], + }; + + for (i, part) in fi.parts.iter().enumerate() { + let file_path = Path::new(&volume_dir) + .join(path) + .join(fi.data_dir.map_or("".to_string(), |dir| dir.to_string())) + .join(format!("part.{}", part.number)); + + match lstat(file_path).await { + Ok(st) => { + if st.is_dir() { + resp.results[i] = CHECK_PART_FILE_NOT_FOUND; + continue; + } + if (st.size() as usize) < fi.erasure.shard_file_size(part.size) { + resp.results[i] = CHECK_PART_FILE_CORRUPT; + continue; + } + + resp.results[i] = CHECK_PART_SUCCESS; + }, + Err(err) => { + match os_err_to_file_err(err).downcast_ref() { + Some(DiskError::FileNotFound) => { + if !skip_access_checks(volume) { + if let Err(err) = access(&volume_dir).await { + match err.kind() { + ErrorKind::NotFound => { + resp.results[i] = CHECK_PART_VOLUME_NOT_FOUND; + continue; + }, + _ => {}, + } + } + } + resp.results[i] = CHECK_PART_FILE_NOT_FOUND; + }, + _ => {}, + } + continue; + } + } + } + + Ok(resp) } async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec) -> Result<()> { diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs index 6408e8756..7ce541ee6 100644 --- a/ecstore/src/disk/mod.rs +++ b/ecstore/src/disk/mod.rs @@ -124,10 +124,11 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { // ReadFileStream async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()>; async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec) -> Result<()>; - // CheckParts async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>; // VerifyFile - async fn verify_file(&self, volume: &str, path: &str, fi: FileInfo) -> Result; + async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result; + // CheckParts + async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result; // StatInfoFile // ReadParts async fn read_multiple(&self, req: ReadMultipleReq) -> Result>; @@ -137,6 +138,7 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { async fn disk_info(&self, opts: &DiskInfoOptions) -> Result; } +#[derive(Debug, Default, Serialize, Deserialize)] pub struct CheckPartsResp { pub results: Vec, } diff --git a/ecstore/src/disk/remote.rs b/ecstore/src/disk/remote.rs index 8d120a065..496aa575e 100644 --- a/ecstore/src/disk/remote.rs +++ b/ecstore/src/disk/remote.rs @@ -4,10 +4,7 @@ use futures::lock::Mutex; use protos::{ node_service_time_out_client, proto_gen::node_service::{ - DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, - ListDirRequest, ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest, ReadMultipleRequest, - ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequst, StatVolumeRequest, UpdateMetadataRequest, - WalkDirRequest, WriteAllRequest, WriteMetadataRequest, + CheckPartsRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest, ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequst, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WalkDirRequest, WriteAllRequest, WriteMetadataRequest }, }; use tonic::Request; @@ -162,8 +159,52 @@ impl DiskAPI for RemoteDisk { Ok(()) } - async fn verify_file(&self, volume: &str, path: &str, fi: FileInfo) -> Result { - unimplemented!() + async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result { + info!("verify_file"); + let file_info = serde_json::to_string(&fi)?; + let mut client = node_service_time_out_client(&self.addr) + .await + .map_err(|err| Error::from_string(format!("can not get client, err: {}", err)))?; + let request = Request::new(VerifyFileRequest { + disk: self.root.to_string_lossy().to_string(), + volume: volume.to_string(), + path: path.to_string(), + file_info, + }); + + let response = client.verify_file(request).await?.into_inner(); + + if !response.success { + return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); + } + + let check_parts_resp = serde_json::from_str::(&response.check_parts_resp)?; + + Ok(check_parts_resp) + } + + async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result { + info!("check_parts"); + let file_info = serde_json::to_string(&fi)?; + let mut client = node_service_time_out_client(&self.addr) + .await + .map_err(|err| Error::from_string(format!("can not get client, err: {}", err)))?; + let request = Request::new(CheckPartsRequest { + disk: self.root.to_string_lossy().to_string(), + volume: volume.to_string(), + path: path.to_string(), + file_info, + }); + + let response = client.check_parts(request).await?.into_inner(); + + if !response.success { + return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); + } + + let check_parts_resp = serde_json::from_str::(&response.check_parts_resp)?; + + Ok(check_parts_resp) } async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec) -> Result<()> { diff --git a/ecstore/src/store_api.rs b/ecstore/src/store_api.rs index 44910a4dc..ab53e802e 100644 --- a/ecstore/src/store_api.rs +++ b/ecstore/src/store_api.rs @@ -1,11 +1,10 @@ use std::collections::HashMap; use crate::{ - error::{Error, Result}, - heal::{ + disk::error::DiskError, error::{Error, Result}, heal::{ heal_commands::{HealOpts, HealResultItem}, heal_ops::HealObjectFn, - }, + } }; use futures::StreamExt; use http::HeaderMap; @@ -17,6 +16,8 @@ use uuid::Uuid; pub const ERASURE_ALGORITHM: &str = "rs-vandermonde"; pub const BLOCK_SIZE_V2: usize = 1048576; // 1M +pub const RESERVED_METADATA_PREFIX: &str = "X-Rustfs-Internal-"; +pub const RESERVED_METADATA_PREFIX_LOWER: &str = "X-Rustfs-Internal-"; // #[derive(Debug, Clone)] #[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)] @@ -172,6 +173,7 @@ impl FileInfo { parts: self.parts.clone(), is_latest: self.is_latest, tags: self.tags.clone(), + ..Default::default() } } // to_part_offset 取offset 所在的part index, 返回part index, offset @@ -250,6 +252,29 @@ impl ErasureInfo { ChecksumInfo {algorithm: DEFAULT_BITROT_ALGO, ..Default::default()} } + + // 算出每个分片大小 + pub fn shard_size(&self, data_size: usize) -> usize { + (data_size + self.data_blocks - 1) / self.data_blocks + } + + // returns final erasure size from original size. + pub fn shard_file_size(&self, total_size: usize) -> usize { + if total_size == 0 { + return 0; + } + + let num_shards = total_size / self.block_size; + let last_block_size = total_size % self.block_size; + let last_shard_size = (last_block_size + self.data_blocks - 1) / self.data_blocks; + num_shards * self.shard_size(self.block_size) + last_shard_size + + // // 因为写入的时候ec需要补全，所以最后一个长度应该也是一样的 + // if last_block_size != 0 { + // num_shards += 1 + // } + // num_shards * self.shard_size(self.block_size) + } } #[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone)] @@ -481,7 +506,10 @@ pub struct ObjectInfo { pub name: String, pub mod_time: Option, pub size: usize, + // Actual size is the real size of the object uploaded by client. + pub actual_size: Option, pub is_dir: bool, + pub user_defined: HashMap, pub parity_blocks: usize, pub data_blocks: usize, pub version_id: Option, @@ -491,6 +519,40 @@ pub struct ObjectInfo { pub tags: Option>, } +impl ObjectInfo { + pub fn is_compressed(&self) -> bool { + self.user_defined.contains_key(&format!("{}compression", RESERVED_METADATA_PREFIX)) + } + + pub fn get_actual_size(&self) -> Result { + if let Some(actual_size) = self.actual_size { + return Ok(actual_size); + } + + if self.is_compressed() { + if let Some(size_str) = self.user_defined.get(&format!("{}actual-size", RESERVED_METADATA_PREFIX)) { + if !size_str.is_empty() { + // Todo: deal with error + let size = size_str.parse::()?; + return Ok(size); + } + } + let mut actual_size = 0; + self.parts.iter().for_each(|part| { + actual_size += part.actual_size; + }); + if actual_size == 0 && actual_size != self.size { + return Err(Error::from_string("invalid decompressed size")); + } + return Ok(actual_size); + } + + // TODO: IsEncrypted + + Ok(self.size) + } +} + #[derive(Debug, Default)] pub struct ListObjectsInfo { // Indicates whether the returned list objects response is truncated. A diff --git a/rustfs/src/grpc.rs b/rustfs/src/grpc.rs index e809a885f..e5caf61c4 100644 --- a/rustfs/src/grpc.rs +++ b/rustfs/src/grpc.rs @@ -16,17 +16,7 @@ use lock::{lock_args::LockArgs, Locker, GLOBAL_LOCAL_SERVER}; use protos::{ models::{PingBody, PingBodyBuilder}, proto_gen::node_service::{ - node_service_server::NodeService as Node, DeleteBucketRequest, DeleteBucketResponse, DeletePathsRequest, - DeletePathsResponse, DeleteRequest, DeleteResponse, DeleteVersionRequest, DeleteVersionResponse, DeleteVersionsRequest, - DeleteVersionsResponse, DeleteVolumeRequest, DeleteVolumeResponse, DiskInfoRequest, DiskInfoResponse, - GenerallyLockRequest, GenerallyLockResponse, GetBucketInfoRequest, GetBucketInfoResponse, ListBucketRequest, - ListBucketResponse, ListDirRequest, ListDirResponse, ListVolumesRequest, ListVolumesResponse, MakeBucketRequest, - MakeBucketResponse, MakeVolumeRequest, MakeVolumeResponse, MakeVolumesRequest, MakeVolumesResponse, PingRequest, - PingResponse, ReadAllRequest, ReadAllResponse, ReadAtRequest, ReadAtResponse, ReadMultipleRequest, ReadMultipleResponse, - ReadVersionRequest, ReadVersionResponse, ReadXlRequest, ReadXlResponse, RenameDataRequest, RenameDataResponse, - RenameFileRequst, RenameFileResponse, RenamePartRequst, RenamePartResponse, StatVolumeRequest, StatVolumeResponse, - UpdateMetadataRequest, UpdateMetadataResponse, WalkDirRequest, WalkDirResponse, WriteAllRequest, WriteAllResponse, - WriteMetadataRequest, WriteMetadataResponse, WriteRequest, WriteResponse, + node_service_server::NodeService as Node, CheckPartsRequest, CheckPartsResponse, DeleteBucketRequest, DeleteBucketResponse, DeletePathsRequest, DeletePathsResponse, DeleteRequest, DeleteResponse, DeleteVersionRequest, DeleteVersionResponse, DeleteVersionsRequest, DeleteVersionsResponse, DeleteVolumeRequest, DeleteVolumeResponse, DiskInfoRequest, DiskInfoResponse, GenerallyLockRequest, GenerallyLockResponse, GetBucketInfoRequest, GetBucketInfoResponse, ListBucketRequest, ListBucketResponse, ListDirRequest, ListDirResponse, ListVolumesRequest, ListVolumesResponse, MakeBucketRequest, MakeBucketResponse, MakeVolumeRequest, MakeVolumeResponse, MakeVolumesRequest, MakeVolumesResponse, PingRequest, PingResponse, ReadAllRequest, ReadAllResponse, ReadAtRequest, ReadAtResponse, ReadMultipleRequest, ReadMultipleResponse, ReadVersionRequest, ReadVersionResponse, ReadXlRequest, ReadXlResponse, RenameDataRequest, RenameDataResponse, RenameFileRequst, RenameFileResponse, RenamePartRequst, RenamePartResponse, StatVolumeRequest, StatVolumeResponse, UpdateMetadataRequest, UpdateMetadataResponse, VerifyFileRequest, VerifyFileResponse, WalkDirRequest, WalkDirResponse, WriteAllRequest, WriteAllResponse, WriteMetadataRequest, WriteMetadataResponse, WriteRequest, WriteResponse }, }; use tokio::sync::mpsc; @@ -306,6 +296,98 @@ impl Node for NodeService { } } + async fn verify_file(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + let file_info = match serde_json::from_str::(&request.file_info) { + Ok(file_info) => file_info, + Err(_) => { + return Ok(tonic::Response::new(VerifyFileResponse { + success: false, + check_parts_resp: "".to_string(), + error_info: Some("can not decode FileInfo".to_string()), + })); + } + }; + match disk.verify_file(&request.volume, &request.path, &file_info).await { + Ok(check_parts_resp) => { + let check_parts_resp = match serde_json::to_string(&check_parts_resp) { + Ok(check_parts_resp) => check_parts_resp, + Err(_) => { + return Ok(tonic::Response::new(VerifyFileResponse { + success: false, + check_parts_resp: String::new(), + error_info: Some("can not encode RenameDataResp".to_string()), + })); + } + }; + Ok(tonic::Response::new(VerifyFileResponse { + success: true, + check_parts_resp, + error_info: None, + })) + }, + Err(err) => Ok(tonic::Response::new(VerifyFileResponse { + success: false, + check_parts_resp: "".to_string(), + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(VerifyFileResponse { + success: false, + check_parts_resp: "".to_string(), + error_info: Some("can not find disk".to_string()), + })) + } + } + + async fn check_parts(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + let file_info = match serde_json::from_str::(&request.file_info) { + Ok(file_info) => file_info, + Err(_) => { + return Ok(tonic::Response::new(CheckPartsResponse { + success: false, + check_parts_resp: "".to_string(), + error_info: Some("can not decode FileInfo".to_string()), + })); + } + }; + match disk.verify_file(&request.volume, &request.path, &file_info).await { + Ok(check_parts_resp) => { + let check_parts_resp = match serde_json::to_string(&check_parts_resp) { + Ok(check_parts_resp) => check_parts_resp, + Err(_) => { + return Ok(tonic::Response::new(CheckPartsResponse { + success: false, + check_parts_resp: String::new(), + error_info: Some("can not encode RenameDataResp".to_string()), + })); + } + }; + Ok(tonic::Response::new(CheckPartsResponse { + success: true, + check_parts_resp, + error_info: None, + })) + }, + Err(err) => Ok(tonic::Response::new(CheckPartsResponse { + success: false, + check_parts_resp: "".to_string(), + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(CheckPartsResponse { + success: false, + check_parts_resp: "".to_string(), + error_info: Some("can not find disk".to_string()), + })) + } + } + async fn rename_part(&self, request: Request) -> Result, Status> { let request = request.into_inner(); if let Some(disk) = self.find_disk(&request.disk).await { From 1e28b9efa9c001e3a06f27d53b753132603abc88 Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Fri, 1 Nov 2024 17:43:23 +0800 Subject: [PATCH 12/12] fix import Signed-off-by: junxiang Mu <1948535941@qq.com>