diff --git a/Cargo.lock b/Cargo.lock index ca01c5e8b..7d74c4d58 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -427,7 +427,9 @@ checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" name = "common" version = "0.0.1" dependencies = [ + "async-trait", "lazy_static", + "scopeguard", "tokio", "tonic", "tracing-error", @@ -655,6 +657,7 @@ dependencies = [ "url", "uuid", "winapi", + "workers", "xxhash-rust", ] @@ -3207,6 +3210,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "workers" +version = "0.0.1" +dependencies = [ + "common", + "tokio", +] + [[package]] name = "write16" version = "1.0.0" diff --git a/Cargo.toml b/Cargo.toml index 77f3121ba..a930c0634 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ members = [ "common/protos", "api/admin", "reader", - "router", + "router", "common/workers", ] [workspace.package] @@ -23,6 +23,7 @@ version = "0.0.1" async-trait = "0.1.83" backon = "1.2.0" bytes = "1.8.0" +bytesize = "1.3.0" clap = { version = "4.5.20", features = ["derive"] } ecstore = { path = "./ecstore" } flatbuffers = "24.3.25" @@ -84,4 +85,4 @@ uuid = { version = "1.11.0", features = [ log = "0.4.22" axum = "0.7.7" md-5 = "0.10.6" -bytesize = "1.3.0" +workers = { path = "./common/workers" } diff --git a/common/common/Cargo.toml b/common/common/Cargo.toml index 0675b06b3..9031424e8 100644 --- a/common/common/Cargo.toml +++ b/common/common/Cargo.toml @@ -4,7 +4,9 @@ version.workspace = true edition.workspace = true [dependencies] +async-trait.workspace = true lazy_static.workspace = true +scopeguard = "1.2.0" tokio.workspace = true tonic.workspace = true -tracing-error.workspace = true \ No newline at end of file +tracing-error.workspace = true diff --git a/common/common/src/last_minute.rs b/common/common/src/last_minute.rs new file mode 100644 index 000000000..c7609d763 --- /dev/null +++ b/common/common/src/last_minute.rs @@ -0,0 +1,112 @@ +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +#[derive(Clone, Debug, Default)] +pub struct AccElem { + pub total: u64, + pub size: u64, + pub n: u64, +} + +impl AccElem { + pub fn add(&mut self, dur: &Duration) { + let dur = dur.as_secs(); + self.total += dur; + self.n += 1; + } + + pub fn merge(&mut self, b: &AccElem) { + self.n += b.n; + self.total += b.total; + self.size += b.size; + } + + pub fn avg(&self) -> Duration { + if self.n >= 1 && self.total > 0 { + return Duration::from_secs(self.total / self.n); + } + Duration::from_secs(0) + } +} + +#[derive(Clone)] +pub struct LastMinuteLatency { + pub totals: Vec, + pub last_sec: u64, +} + +impl Default for LastMinuteLatency { + fn default() -> Self { + Self { + totals: vec![AccElem::default(); 60], + last_sec: Default::default(), + } + } +} + +impl LastMinuteLatency { + pub fn merge(&mut self, o: &mut LastMinuteLatency) -> LastMinuteLatency { + let mut merged = LastMinuteLatency::default(); + if self.last_sec > o.last_sec { + o.forward_to(self.last_sec); + merged.last_sec = self.last_sec; + } else { + self.forward_to(o.last_sec); + merged.last_sec = o.last_sec; + } + + for i in 0..merged.totals.len() { + merged.totals[i] = AccElem { + total: self.totals[i].total + o.totals[i].total, + n: self.totals[i].n + o.totals[i].n, + size: self.totals[i].size + o.totals[i].size, + } + } + merged + } + + pub fn add(&mut self, t: &Duration) { + let sec = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs(); + self.forward_to(sec); + let win_idx = sec % 60; + self.totals[win_idx as usize].add(t); + self.last_sec = sec; + } + + pub fn add_all(&mut self, sec: u64, a: &AccElem) { + self.forward_to(sec); + let win_idx = sec % 60; + self.totals[win_idx as usize].merge(a); + self.last_sec = sec; + } + + pub fn get_total(&mut self) -> AccElem { + let mut res = AccElem::default(); + let sec = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs(); + self.forward_to(sec); + for elem in self.totals.iter() { + res.merge(elem); + } + res + } + + pub fn forward_to(&mut self, t: u64) { + if self.last_sec >= t { + return; + } + if t - self.last_sec >= 60 { + self.totals = vec![AccElem::default(); 60]; + return; + } + while self.last_sec != t { + let idx = (self.last_sec + 1) % 60; + self.totals[idx as usize] = AccElem::default(); + self.last_sec += 1; + } + } +} diff --git a/common/common/src/lib.rs b/common/common/src/lib.rs index b7a0605d3..a8998fee0 100644 --- a/common/common/src/lib.rs +++ b/common/common/src/lib.rs @@ -1,2 +1,23 @@ pub mod error; pub mod globals; +pub mod last_minute; + +/// Defers evaluation of a block of code until the end of the scope. +#[macro_export] +macro_rules! defer { + ($($body:tt)*) => { + let _guard = { + pub struct Guard(Option); + + impl Drop for Guard { + fn drop(&mut self) { + (self.0).take().map(|f| f()); + } + } + + Guard(Some(|| { + let _ = { $($body)* }; + })) + }; + }; +} diff --git a/common/lock/src/lrwmutex.rs b/common/lock/src/lrwmutex.rs index 1ff4ebb98..fa65f1a7c 100644 --- a/common/lock/src/lrwmutex.rs +++ b/common/lock/src/lrwmutex.rs @@ -135,14 +135,14 @@ mod test { let id = "foo"; let source = "dandan"; let timeout = Duration::from_secs(5); - assert_eq!(true, l_rw_lock.get_lock(id, source, &timeout).await); + assert!(l_rw_lock.get_lock(id, source, &timeout).await); l_rw_lock.un_lock().await; l_rw_lock.lock().await; - assert_eq!(false, l_rw_lock.get_r_lock(id, source, &timeout).await); + assert!(!(l_rw_lock.get_r_lock(id, source, &timeout).await)); l_rw_lock.un_lock().await; - assert_eq!(true, l_rw_lock.get_r_lock(id, source, &timeout).await); + assert!(l_rw_lock.get_r_lock(id, source, &timeout).await); Ok(()) } @@ -156,7 +156,7 @@ mod test { let one_fn = async { let one = Arc::clone(&l_rw_lock); let timeout = Duration::from_secs(1); - assert_eq!(true, one.get_lock(id, source, &timeout).await); + assert!(one.get_lock(id, source, &timeout).await); sleep(Duration::from_secs(5)).await; l_rw_lock.un_lock().await; }; @@ -164,9 +164,9 @@ mod test { let two_fn = async { let two = Arc::clone(&l_rw_lock); let timeout = Duration::from_secs(2); - assert_eq!(false, two.get_r_lock(id, source, &timeout).await); + assert!(!(two.get_r_lock(id, source, &timeout).await)); sleep(Duration::from_secs(5)).await; - assert_eq!(true, two.get_r_lock(id, source, &timeout).await); + assert!(two.get_r_lock(id, source, &timeout).await); two.un_r_lock().await; }; diff --git a/common/lock/src/namespace_lock.rs b/common/lock/src/namespace_lock.rs index 268458e73..2392ae0aa 100644 --- a/common/lock/src/namespace_lock.rs +++ b/common/lock/src/namespace_lock.rs @@ -287,7 +287,7 @@ mod test { }) .await?; - assert_eq!(result, true); + assert!(result); Ok(()) } } diff --git a/common/protos/src/generated/proto_gen/node_service.rs b/common/protos/src/generated/proto_gen/node_service.rs index 2a4fcf8ea..1ec3f589b 100644 --- a/common/protos/src/generated/proto_gen/node_service.rs +++ b/common/protos/src/generated/proto_gen/node_service.rs @@ -15,6 +15,20 @@ pub struct PingResponse { pub body: ::prost::alloc::vec::Vec, } #[derive(Clone, PartialEq, ::prost::Message)] +pub struct HealBucketRequest { + #[prost(string, tag = "1")] + pub bucket: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub options: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct HealBucketResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, optional, tag = "2")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ListBucketRequest { #[prost(string, tag = "1")] pub options: ::prost::alloc::string::String, @@ -565,6 +579,26 @@ pub struct DiskInfoResponse { #[prost(string, optional, tag = "3")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, } +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct NsScannerRequest { + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub cache: ::prost::alloc::string::String, + #[prost(uint64, tag = "3")] + pub scan_mode: u64, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct NsScannerResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, tag = "2")] + pub update: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub data_usage_cache: ::prost::alloc::string::String, + #[prost(string, optional, tag = "4")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} /// lock api have same argument type #[derive(Clone, PartialEq, ::prost::Message)] pub struct GenerallyLockRequest { @@ -673,6 +707,21 @@ pub mod node_service_client { .insert(GrpcMethod::new("node_service.NodeService", "Ping")); self.inner.unary(req, path, codec).await } + pub async fn heal_bucket( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/HealBucket"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "HealBucket")); + self.inner.unary(req, path, codec).await + } pub async fn list_bucket( &mut self, request: impl tonic::IntoRequest, @@ -1139,6 +1188,21 @@ pub mod node_service_client { .insert(GrpcMethod::new("node_service.NodeService", "DiskInfo")); self.inner.unary(req, path, codec).await } + pub async fn ns_scanner( + &mut self, + request: impl tonic::IntoStreamingRequest, + ) -> std::result::Result>, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/NsScanner"); + let mut req = request.into_streaming_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "NsScanner")); + self.inner.streaming(req, path, codec).await + } pub async fn lock( &mut self, request: impl tonic::IntoRequest, @@ -1243,6 +1307,10 @@ pub mod node_service_server { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + async fn heal_bucket( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; async fn list_bucket( &self, request: tonic::Request, @@ -1376,6 +1444,14 @@ pub mod node_service_server { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + /// Server streaming response type for the NsScanner method. + type NsScannerStream: tonic::codegen::tokio_stream::Stream> + + std::marker::Send + + 'static; + async fn ns_scanner( + &self, + request: tonic::Request>, + ) -> std::result::Result, tonic::Status>; async fn lock( &self, request: tonic::Request, @@ -1499,6 +1575,34 @@ pub mod node_service_server { }; Box::pin(fut) } + "/node_service.NodeService/HealBucket" => { + #[allow(non_camel_case_types)] + struct HealBucketSvc(pub Arc); + impl tonic::server::UnaryService for HealBucketSvc { + type Response = super::HealBucketResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::heal_bucket(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = HealBucketSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } "/node_service.NodeService/ListBucket" => { #[allow(non_camel_case_types)] struct ListBucketSvc(pub Arc); @@ -2369,6 +2473,35 @@ pub mod node_service_server { }; Box::pin(fut) } + "/node_service.NodeService/NsScanner" => { + #[allow(non_camel_case_types)] + struct NsScannerSvc(pub Arc); + impl tonic::server::StreamingService for NsScannerSvc { + type Response = super::NsScannerResponse; + type ResponseStream = T::NsScannerStream; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request>) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::ns_scanner(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = NsScannerSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.streaming(method, req).await; + Ok(res) + }; + Box::pin(fut) + } "/node_service.NodeService/Lock" => { #[allow(non_camel_case_types)] struct LockSvc(pub Arc); diff --git a/common/protos/src/node.proto b/common/protos/src/node.proto index f789a302f..f80fb10f3 100644 --- a/common/protos/src/node.proto +++ b/common/protos/src/node.proto @@ -12,6 +12,16 @@ message PingResponse { bytes body = 2; } +message HealBucketRequest { + string bucket = 1; + string options = 2; +} + +message HealBucketResponse { + bool success = 1; + optional string error_info = 2; +} + message ListBucketRequest { string options = 1; } @@ -382,6 +392,19 @@ message DiskInfoResponse { optional string error_info = 3; } +message NsScannerRequest { + string disk = 1; + string cache = 2; + uint64 scan_mode = 3; +} + +message NsScannerResponse { + bool success = 1; + string update = 2; + string data_usage_cache = 3; + optional string error_info = 4; +} + // lock api have same argument type message GenerallyLockRequest { string args = 1; @@ -397,6 +420,7 @@ message GenerallyLockResponse { service NodeService { /* -------------------------------meta service-------------------------- */ rpc Ping(PingRequest) returns (PingResponse) {}; + rpc HealBucket(HealBucketRequest) returns (HealBucketResponse) {}; rpc ListBucket(ListBucketRequest) returns (ListBucketResponse) {}; rpc MakeBucket(MakeBucketRequest) returns (MakeBucketResponse) {}; rpc GetBucketInfo(GetBucketInfoRequest) returns (GetBucketInfoResponse) {}; @@ -432,6 +456,7 @@ service NodeService { rpc ReadMultiple(ReadMultipleRequest) returns (ReadMultipleResponse) {}; rpc DeleteVolume(DeleteVolumeRequest) returns (DeleteVolumeResponse) {}; rpc DiskInfo(DiskInfoRequest) returns (DiskInfoResponse) {}; + rpc NsScanner(stream NsScannerRequest) returns (stream NsScannerResponse) {}; /* -------------------------------lock service-------------------------- */ diff --git a/common/workers/Cargo.toml b/common/workers/Cargo.toml new file mode 100644 index 000000000..dfb70d140 --- /dev/null +++ b/common/workers/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "workers" +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[dependencies] +common.workspace = true +tokio.workspace = true \ No newline at end of file diff --git a/common/workers/src/lib.rs b/common/workers/src/lib.rs new file mode 100644 index 000000000..3719b10aa --- /dev/null +++ b/common/workers/src/lib.rs @@ -0,0 +1 @@ +pub mod workers; diff --git a/common/workers/src/workers.rs b/common/workers/src/workers.rs new file mode 100644 index 000000000..5c03447f8 --- /dev/null +++ b/common/workers/src/workers.rs @@ -0,0 +1,87 @@ +use std::sync::Arc; +use tokio::sync::{Mutex, Notify}; + +pub struct Workers { + available: Mutex, // 可用的工作槽 + notify: Notify, // 用于通知等待的任务 + limit: usize, // 最大并发工作数 +} + +impl Workers { + // 创建 Workers 对象，允许最多 n 个作业并发执行 + pub fn new(n: usize) -> Result, &'static str> { + if n == 0 { + return Err("n must be > 0"); + } + + Ok(Arc::new(Workers { + available: Mutex::new(n), + notify: Notify::new(), + limit: n, + })) + } + + // 让一个作业获得执行的机会 + pub async fn take(&self) { + let mut available = self.available.lock().await; + while *available == 0 { + // 等待直到有可用槽 + self.notify.notified().await; + available = self.available.lock().await; + } + *available -= 1; // 减少可用槽 + } + + // 让一个作业释放其机会 + pub async fn give(&self) { + let mut available = self.available.lock().await; + *available += 1; // 增加可用槽 + self.notify.notify_one(); // 通知一个等待的任务 + } + + // 等待所有并发作业完成 + pub async fn wait(&self) { + loop { + { + let available = self.available.lock().await; + if *available == self.limit { + break; + } + } + // 等待直到所有槽都被释放 + self.notify.notified().await; + } + } + + pub async fn available(&self) -> usize { + *self.available.lock().await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + use tokio::time::sleep; + + #[tokio::test] + async fn test_workers() { + let workers = Arc::new(Workers::new(5).unwrap()); + + for _ in 0..4 { + let workers = workers.clone(); + tokio::spawn(async move { + workers.take().await; + sleep(Duration::from_secs(3)).await; + workers.give().await; + }); + } + + // Sleep: wait for spawn task started + sleep(Duration::from_secs(1)).await; + workers.wait().await; + if workers.available().await != workers.limit { + assert!(false); + } + } +} diff --git a/e2e_test/src/reliant/lock.rs b/e2e_test/src/reliant/lock.rs index fd5db5174..ea84b2787 100644 --- a/e2e_test/src/reliant/lock.rs +++ b/e2e_test/src/reliant/lock.rs @@ -33,13 +33,13 @@ async fn test_lock_unlock_rpc() -> Result<(), Box> { let response = client.lock(request).await?.into_inner(); println!("request ended"); if let Some(error_info) = response.error_info { - assert!(false, "can not get lock: {}", error_info); + panic!("can not get lock: {}", error_info); } let request = Request::new(GenerallyLockRequest { args }); let response = client.un_lock(request).await?.into_inner(); if let Some(error_info) = response.error_info { - assert!(false, "can not get un_lock: {}", error_info); + panic!("can not get un_lock: {}", error_info); } Ok(()) @@ -58,17 +58,16 @@ async fn test_lock_unlock_ns_lock() -> Result<(), Box> { vec![locker], ) .await; - assert_eq!( - ns.0.write() - .await - .get_lock(&Options { - timeout: Duration::from_secs(5), - retry_interval: Duration::from_secs(1), - }) - .await - .unwrap(), - true - ); + assert!(ns + .0 + .write() + .await + .get_lock(&Options { + timeout: Duration::from_secs(5), + retry_interval: Duration::from_secs(1), + }) + .await + .unwrap()); ns.0.write().await.un_lock().await.unwrap(); Ok(()) diff --git a/ecstore/Cargo.toml b/ecstore/Cargo.toml index 79370b32f..79023158b 100644 --- a/ecstore/Cargo.toml +++ b/ecstore/Cargo.toml @@ -60,6 +60,7 @@ s3s-policy.workspace = true rand.workspace = true pin-project-lite.workspace = true md-5.workspace = true +workers.workspace = true [target.'cfg(not(windows))'.dependencies] diff --git a/ecstore/src/bitrot.rs b/ecstore/src/bitrot.rs index 98c7a6896..198c6a226 100644 --- a/ecstore/src/bitrot.rs +++ b/ecstore/src/bitrot.rs @@ -103,7 +103,7 @@ impl Hasher { } impl BitrotAlgorithm { - pub fn new(&self) -> Hasher { + pub fn new_hasher(&self) -> Hasher { match self { BitrotAlgorithm::SHA256 => Hasher::SHA256(Sha256::new()), BitrotAlgorithm::HighwayHash256 | BitrotAlgorithm::HighwayHash256S => { @@ -186,10 +186,8 @@ pub fn new_bitrot_reader( } pub async fn close_bitrot_writers(writers: &mut [Option]) -> Result<()> { - for w in writers.into_iter() { - if let Some(w) = w { - let _ = w.close().await?; - } + for w in writers.iter_mut().flatten() { + w.close().await?; } Ok(()) @@ -207,7 +205,7 @@ pub fn bitrot_shard_file_size(size: usize, shard_size: usize, algo: BitrotAlgori if algo != BitrotAlgorithm::HighwayHash256S { return size; } - size.div_ceil(shard_size) * algo.new().size() + size + size.div_ceil(shard_size) * algo.new_hasher().size() + size } pub fn bitrot_verify( @@ -219,7 +217,7 @@ pub fn bitrot_verify( mut shard_size: usize, ) -> Result<()> { if algo != BitrotAlgorithm::HighwayHash256S { - let mut h = algo.new(); + let mut h = algo.new_hasher(); h.update(r.get_ref()); if h.finalize() != want { return Err(Error::new(DiskError::FileCorrupt)); @@ -227,7 +225,7 @@ pub fn bitrot_verify( return Ok(()); } - let mut h = algo.new(); + let mut h = algo.new_hasher(); let mut hash_buf = vec![0; h.size()]; let mut left = want_size; @@ -271,7 +269,7 @@ impl WholeBitrotWriter { volume: volume.to_string(), file_path: file_path.to_string(), _shard_size: shard_size, - hash: algo.new(), + hash: algo.new_hasher(), } } } @@ -353,7 +351,7 @@ impl StreamingBitrotWriter { algo: BitrotAlgorithm, shard_size: usize, ) -> Result { - let hasher = algo.new(); + let hasher = algo.new_hasher(); let (tx, mut rx) = mpsc::channel::>>(10); let total_file_size = length.div_ceil(shard_size) * hasher.size() + length; @@ -362,7 +360,7 @@ impl StreamingBitrotWriter { let task = spawn(async move { loop { if let Some(Some(buf)) = rx.recv().await { - let _ = writer.write(&buf).await.unwrap(); + writer.write(&buf).await.unwrap(); continue; } @@ -389,7 +387,7 @@ impl Writer for StreamingBitrotWriter { return Ok(()); } self.hasher.reset(); - self.hasher.update(&buf); + self.hasher.update(buf); let hash_bytes = self.hasher.clone().finalize(); let _ = self.tx.send(Some(hash_bytes)).await?; let _ = self.tx.send(Some(buf.to_vec())).await?; @@ -430,7 +428,7 @@ impl StreamingBitrotReader { till_offset: usize, shard_size: usize, ) -> Self { - let hasher = algo.new(); + let hasher = algo.new_hasher(); Self { disk, _data: data.to_vec(), @@ -489,7 +487,7 @@ pub struct BitrotFileWriter { impl BitrotFileWriter { pub fn new(inner: FileWriter, algo: BitrotAlgorithm, _shard_size: usize) -> Self { - let hasher = algo.new(); + let hasher = algo.new_hasher(); Self { inner, hasher, @@ -513,7 +511,7 @@ impl Writer for BitrotFileWriter { return Ok(()); } self.hasher.reset(); - self.hasher.update(&buf); + self.hasher.update(buf); let hash_bytes = self.hasher.clone().finalize(); let _ = self.inner.write(&hash_bytes).await?; let _ = self.inner.write(buf).await?; @@ -544,7 +542,7 @@ struct BitrotFileReader { impl BitrotFileReader { pub fn new(inner: FileReader, algo: BitrotAlgorithm, _till_offset: usize, shard_size: usize) -> Self { - let hasher = algo.new(); + let hasher = algo.new_hasher(); Self { inner, // till_offset: ceil(till_offset, shard_size) * hasher.size() + till_offset, @@ -648,7 +646,7 @@ mod test { } let checksum = decode_to_vec(checksums.get(algo).unwrap()).unwrap(); - let mut h = algo.new(); + let mut h = algo.new_hasher(); let mut msg = Vec::with_capacity(h.size() * h.block_size()); let mut sum = Vec::with_capacity(h.size()); @@ -656,7 +654,7 @@ mod test { h.update(&msg); sum = h.finalize(); msg.extend(sum.clone()); - h = algo.new(); + h = algo.new_hasher(); } if checksum != sum { diff --git a/ecstore/src/bucket/metadata.rs b/ecstore/src/bucket/metadata.rs index ac0134d86..a296a973f 100644 --- a/ecstore/src/bucket/metadata.rs +++ b/ecstore/src/bucket/metadata.rs @@ -9,7 +9,6 @@ use s3s::dto::{ BucketLifecycleConfiguration, NotificationConfiguration, ObjectLockConfiguration, ReplicationConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration, }; -use s3s::xml; use serde::Serializer; use serde::{Deserialize, Serialize}; use std::collections::HashMap; diff --git a/ecstore/src/bucket/metadata_sys.rs b/ecstore/src/bucket/metadata_sys.rs index 764a57b50..03ece0d3d 100644 --- a/ecstore/src/bucket/metadata_sys.rs +++ b/ecstore/src/bucket/metadata_sys.rs @@ -28,7 +28,7 @@ use super::target::BucketTargets; use lazy_static::lazy_static; lazy_static! { - static ref GLOBAL_BucketMetadataSys: Arc> = Arc::new(RwLock::new(BucketMetadataSys::new())); + pub static ref GLOBAL_BucketMetadataSys: Arc> = Arc::new(RwLock::new(BucketMetadataSys::new())); } pub async fn init_bucket_metadata_sys(api: ECStore, buckets: Vec) { diff --git a/ecstore/src/bucket/policy/bucket_policy.rs b/ecstore/src/bucket/policy/bucket_policy.rs index 1ed991515..2d4d183a5 100644 --- a/ecstore/src/bucket/policy/bucket_policy.rs +++ b/ecstore/src/bucket/policy/bucket_policy.rs @@ -2,7 +2,6 @@ use crate::error::{Error, Result}; // use rmp_serde::Serializer as rmpSerializer; use serde::{Deserialize, Serialize}; use std::collections::HashMap; -use std::collections::HashSet; use super::{ action::{Action, ActionSet, IAMActionConditionKeyMap}, diff --git a/ecstore/src/bucket/utils.rs b/ecstore/src/bucket/utils.rs index f7dc8dfe2..613f1b3ba 100644 --- a/ecstore/src/bucket/utils.rs +++ b/ecstore/src/bucket/utils.rs @@ -39,10 +39,8 @@ pub fn check_bucket_name_common(bucket_name: &str, strict: bool) -> Result<(), E if !VALID_BUCKET_NAME_STRICT.is_match(bucket_name_trimmed) { return Err(Error::msg("Bucket name contains invalid characters")); } - } else { - if !VALID_BUCKET_NAME.is_match(bucket_name_trimmed) { - return Err(Error::msg("Bucket name contains invalid characters")); - } + } else if !VALID_BUCKET_NAME.is_match(bucket_name_trimmed) { + return Err(Error::msg("Bucket name contains invalid characters")); } Ok(()) } diff --git a/ecstore/src/bucket/versioning/mod.rs b/ecstore/src/bucket/versioning/mod.rs index 5efe2f80d..6ad2e4014 100644 --- a/ecstore/src/bucket/versioning/mod.rs +++ b/ecstore/src/bucket/versioning/mod.rs @@ -40,10 +40,8 @@ impl VersioningApi for VersioningConfiguration { } if let Some(status) = self.status.as_ref() { - if status.as_str() == BucketVersioningStatus::ENABLED { - if prefix.is_empty() { - return false; - } + if status.as_str() == BucketVersioningStatus::ENABLED && prefix.is_empty() { + return false; // TODO: ExcludeFolders } diff --git a/ecstore/src/cache_value/cache.rs b/ecstore/src/cache_value/cache.rs index fa4367197..15f116dec 100644 --- a/ecstore/src/cache_value/cache.rs +++ b/ecstore/src/cache_value/cache.rs @@ -1,5 +1,7 @@ use std::{ fmt::Debug, + future::Future, + pin::Pin, ptr, sync::{ atomic::{AtomicPtr, AtomicU64, Ordering}, @@ -12,7 +14,7 @@ use tokio::{spawn, sync::Mutex}; use crate::error::Result; -type UpdateFn = Box Result + Send + Sync>; +pub type UpdateFn = Box Pin> + Send>> + Send + Sync + 'static>; #[derive(Clone, Debug, Default)] pub struct Opts { @@ -54,8 +56,10 @@ impl Cache { .duration_since(UNIX_EPOCH) .expect("Time went backwards") .as_secs(); - if v.is_some() && now - self.last_update_ms.load(Ordering::SeqCst) < self.ttl.as_secs() { - return Ok(v.unwrap()); + if now - self.last_update_ms.load(Ordering::SeqCst) < self.ttl.as_secs() { + if let Some(v) = v { + return Ok(v); + } } if self.opts.no_wait && v.is_some() && now - self.last_update_ms.load(Ordering::SeqCst) < self.ttl.as_secs() * 2 { @@ -94,7 +98,7 @@ impl Cache { } async fn update(&self) -> Result<()> { - match (self.update_fn)() { + match (self.update_fn)().await { Ok(val) => { self.val.store(Box::into_raw(Box::new(val)), Ordering::SeqCst); let now = SystemTime::now() @@ -110,7 +114,7 @@ impl Cache { return Ok(()); } - return Err(err); + Err(err) } } } diff --git a/ecstore/src/cache_value/metacache_set.rs b/ecstore/src/cache_value/metacache_set.rs index 42b81a20f..928849cf6 100644 --- a/ecstore/src/cache_value/metacache_set.rs +++ b/ecstore/src/cache_value/metacache_set.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::{future::Future, pin::Pin, sync::Arc}; use tokio::{ spawn, @@ -14,6 +14,10 @@ use crate::{ error::{Error, Result}, }; +type AgreedFn = Box Pin + Send>> + Send + 'static>; +type PartialFn = Box]) -> Pin + Send>> + Send + 'static>; +type FinishedFn = Box]) -> Pin + Send>> + Send + 'static>; + #[derive(Default)] pub struct ListPathRawOptions { pub disks: Vec>, @@ -26,9 +30,12 @@ pub struct ListPathRawOptions { pub min_disks: usize, pub report_not_found: bool, pub per_disk_limit: i32, - pub agreed: Option>, - pub partial: Option]) + Send + Sync>>, - pub finished: Option]) + Send + Sync>>, + pub agreed: Option, + pub partial: Option, + pub finished: Option, + // pub agreed: Option>, + // pub partial: Option]) + Send + Sync>>, + // pub finished: Option]) + Send + Sync>>, } impl Clone for ListPathRawOptions { @@ -38,12 +45,12 @@ impl Clone for ListPathRawOptions { fallback_disks: self.fallback_disks.clone(), bucket: self.bucket.clone(), path: self.path.clone(), - recursice: self.recursice.clone(), + recursice: self.recursice, filter_prefix: self.filter_prefix.clone(), forward_to: self.forward_to.clone(), - min_disks: self.min_disks.clone(), - report_not_found: self.report_not_found.clone(), - per_disk_limit: self.per_disk_limit.clone(), + min_disks: self.min_disks, + report_not_found: self.report_not_found, + per_disk_limit: self.per_disk_limit, ..Default::default() } } @@ -73,7 +80,7 @@ pub async fn list_path_raw(mut rx: B_Receiver, opts: ListPathRawOptions) - .walk_dir(WalkDirOptions { bucket: opts_clone.bucket.clone(), base_dir: opts_clone.path.clone(), - recursive: opts_clone.recursice.clone(), + recursive: opts_clone.recursice, report_notfound: opts_clone.report_not_found, filter_prefix: opts_clone.filter_prefix.clone(), forward_to: opts_clone.forward_to.clone(), @@ -185,8 +192,8 @@ pub async fn list_path_raw(mut rx: B_Receiver, opts: ListPathRawOptions) - } if has_err > 0 && has_err > opts.disks.len() - opts.min_disks { - if let Some(finished_fn) = opts.finished.clone() { - finished_fn(&errs); + if let Some(finished_fn) = opts.finished.as_ref() { + finished_fn(&errs).await; } let mut combined_err = Vec::new(); errs.iter().zip(opts.disks.iter()).for_each(|(err, disk)| match (err, disk) { @@ -203,24 +210,22 @@ pub async fn list_path_raw(mut rx: B_Receiver, opts: ListPathRawOptions) - } // Break if all at EOF or error. - if at_eof + has_err == readers.len() { - if has_err > 0 { - if let Some(finished_fn) = opts.finished.clone() { - finished_fn(&errs); - } - break; + if at_eof + has_err == readers.len() && has_err > 0 { + if let Some(finished_fn) = opts.finished.as_ref() { + finished_fn(&errs).await; } + break; } if agree == readers.len() { - if let Some(agreed_fn) = opts.agreed.clone() { - agreed_fn(current); + if let Some(agreed_fn) = opts.agreed.as_ref() { + agreed_fn(current).await; } continue; } - if let Some(partial_fn) = opts.partial.clone() { - partial_fn(MetaCacheEntries(top_entries), &errs); + if let Some(partial_fn) = opts.partial.as_ref() { + partial_fn(MetaCacheEntries(top_entries), &errs).await; } } diff --git a/ecstore/src/chunk_stream.rs b/ecstore/src/chunk_stream.rs index 80ad8eb22..886ee165c 100644 --- a/ecstore/src/chunk_stream.rs +++ b/ecstore/src/chunk_stream.rs @@ -1,256 +1,256 @@ -use crate::error::StdError; -use bytes::Bytes; -use futures::pin_mut; -use futures::stream::{Stream, StreamExt}; -use std::future::Future; -use std::pin::Pin; -use std::task::{Context, Poll}; -use transform_stream::AsyncTryStream; +// use crate::error::StdError; +// use bytes::Bytes; +// use futures::pin_mut; +// use futures::stream::{Stream, StreamExt}; +// use std::future::Future; +// use std::pin::Pin; +// use std::task::{Context, Poll}; +// use transform_stream::AsyncTryStream; -pub type SyncBoxFuture<'a, T> = Pin + Send + Sync + 'a>>; +// pub type SyncBoxFuture<'a, T> = Pin + Send + Sync + 'a>>; -pub struct ChunkedStream<'a> { - /// inner - inner: AsyncTryStream>>, +// pub struct ChunkedStream<'a> { +// /// inner +// inner: AsyncTryStream>>, - remaining_length: usize, -} +// remaining_length: usize, +// } -impl<'a> ChunkedStream<'a> { - pub fn new(body: S, content_length: usize, chunk_size: usize, need_padding: bool) -> Self - where - S: Stream> + Send + Sync + 'a, - { - let inner = AsyncTryStream::<_, _, SyncBoxFuture<'a, Result<(), StdError>>>::new(|mut y| { - #[allow(clippy::shadow_same)] // necessary for `pin_mut!` - Box::pin(async move { - pin_mut!(body); - // 上一次没用完的数据 - let mut prev_bytes = Bytes::new(); - let mut readed_size = 0; +// impl<'a> ChunkedStream<'a> { +// pub fn new(body: S, content_length: usize, chunk_size: usize, need_padding: bool) -> Self +// where +// S: Stream> + Send + Sync + 'a, +// { +// let inner = AsyncTryStream::<_, _, SyncBoxFuture<'a, Result<(), StdError>>>::new(|mut y| { +// #[allow(clippy::shadow_same)] // necessary for `pin_mut!` +// Box::pin(async move { +// pin_mut!(body); +// // 上一次没用完的数据 +// let mut prev_bytes = Bytes::new(); +// let mut readed_size = 0; - loop { - let data: Vec = { - // 读固定大小的数据 - match Self::read_data(body.as_mut(), prev_bytes, chunk_size).await { - None => break, - Some(Err(e)) => return Err(e), - Some(Ok((data, remaining_bytes))) => { - // debug!( - // "content_length:{},readed_size:{}, read_data data:{}, remaining_bytes: {} ", - // content_length, - // readed_size, - // data.len(), - // remaining_bytes.len() - // ); +// loop { +// let data: Vec = { +// // 读固定大小的数据 +// match Self::read_data(body.as_mut(), prev_bytes, chunk_size).await { +// None => break, +// Some(Err(e)) => return Err(e), +// Some(Ok((data, remaining_bytes))) => { +// // debug!( +// // "content_length:{},readed_size:{}, read_data data:{}, remaining_bytes: {} ", +// // content_length, +// // readed_size, +// // data.len(), +// // remaining_bytes.len() +// // ); - prev_bytes = remaining_bytes; - data - } - } - }; +// prev_bytes = remaining_bytes; +// data +// } +// } +// }; - for bytes in data { - readed_size += bytes.len(); - // debug!("readed_size {}, content_length {}", readed_size, content_length,); - y.yield_ok(bytes).await; - } +// for bytes in data { +// readed_size += bytes.len(); +// // debug!("readed_size {}, content_length {}", readed_size, content_length,); +// y.yield_ok(bytes).await; +// } - if readed_size + prev_bytes.len() >= content_length { - // debug!( - // "读完了 readed_size:{} + prev_bytes.len({}) == content_length {}", - // readed_size, - // prev_bytes.len(), - // content_length, - // ); +// if readed_size + prev_bytes.len() >= content_length { +// // debug!( +// // "读完了 readed_size:{} + prev_bytes.len({}) == content_length {}", +// // readed_size, +// // prev_bytes.len(), +// // content_length, +// // ); - // 填充0？ - if !need_padding { - y.yield_ok(prev_bytes).await; - break; - } +// // 填充0？ +// if !need_padding { +// y.yield_ok(prev_bytes).await; +// break; +// } - let mut bytes = vec![0u8; chunk_size]; - let (left, _) = bytes.split_at_mut(prev_bytes.len()); - left.copy_from_slice(&prev_bytes); +// let mut bytes = vec![0u8; chunk_size]; +// let (left, _) = bytes.split_at_mut(prev_bytes.len()); +// left.copy_from_slice(&prev_bytes); - y.yield_ok(Bytes::from(bytes)).await; +// y.yield_ok(Bytes::from(bytes)).await; - break; - } - } +// break; +// } +// } - // debug!("chunked stream exit"); +// // debug!("chunked stream exit"); - Ok(()) - }) - }); - Self { - inner, - remaining_length: content_length, - } - } - /// read data and return remaining bytes - async fn read_data( - mut body: Pin<&mut S>, - prev_bytes: Bytes, - data_size: usize, - ) -> Option, Bytes), StdError>> - where - S: Stream> + Send, - { - let mut bytes_buffer = Vec::new(); +// Ok(()) +// }) +// }); +// Self { +// inner, +// remaining_length: content_length, +// } +// } +// /// read data and return remaining bytes +// async fn read_data( +// mut body: Pin<&mut S>, +// prev_bytes: Bytes, +// data_size: usize, +// ) -> Option, Bytes), StdError>> +// where +// S: Stream> + Send, +// { +// let mut bytes_buffer = Vec::new(); - // 只执行一次 - let mut push_data_bytes = |mut bytes: Bytes| { - // debug!("read from body {} split per {}, prev_bytes: {}", bytes.len(), data_size, prev_bytes.len()); +// // 只执行一次 +// let mut push_data_bytes = |mut bytes: Bytes| { +// // debug!("read from body {} split per {}, prev_bytes: {}", bytes.len(), data_size, prev_bytes.len()); - if bytes.is_empty() { - return None; - } +// if bytes.is_empty() { +// return None; +// } - if data_size == 0 { - return Some(bytes); - } +// if data_size == 0 { +// return Some(bytes); +// } - // 合并上一次数据 - if !prev_bytes.is_empty() { - let need_size = data_size.wrapping_sub(prev_bytes.len()); - // debug!( - // " 上一次有剩余{},从这一次中取{},共：{}", - // prev_bytes.len(), - // need_size, - // prev_bytes.len() + need_size - // ); - if bytes.len() >= need_size { - let data = bytes.split_to(need_size); - let mut combined = Vec::new(); - combined.extend_from_slice(&prev_bytes); - combined.extend_from_slice(&data); +// // 合并上一次数据 +// if !prev_bytes.is_empty() { +// let need_size = data_size.wrapping_sub(prev_bytes.len()); +// // debug!( +// // " 上一次有剩余{},从这一次中取{},共：{}", +// // prev_bytes.len(), +// // need_size, +// // prev_bytes.len() + need_size +// // ); +// if bytes.len() >= need_size { +// let data = bytes.split_to(need_size); +// let mut combined = Vec::new(); +// combined.extend_from_slice(&prev_bytes); +// combined.extend_from_slice(&data); - // debug!( - // "取到的长度大于所需，取出需要的长度：{},与上一次合并得到：{}，bytes剩余：{}", - // need_size, - // combined.len(), - // bytes.len(), - // ); +// // debug!( +// // "取到的长度大于所需，取出需要的长度：{},与上一次合并得到：{}，bytes剩余：{}", +// // need_size, +// // combined.len(), +// // bytes.len(), +// // ); - bytes_buffer.push(Bytes::from(combined)); - } else { - let mut combined = Vec::new(); - combined.extend_from_slice(&prev_bytes); - combined.extend_from_slice(&bytes); +// bytes_buffer.push(Bytes::from(combined)); +// } else { +// let mut combined = Vec::new(); +// combined.extend_from_slice(&prev_bytes); +// combined.extend_from_slice(&bytes); - // debug!( - // "取到的长度小于所需，取出需要的长度：{},与上一次合并得到：{}，bytes剩余：{}，直接返回", - // need_size, - // combined.len(), - // bytes.len(), - // ); +// // debug!( +// // "取到的长度小于所需，取出需要的长度：{},与上一次合并得到：{}，bytes剩余：{}，直接返回", +// // need_size, +// // combined.len(), +// // bytes.len(), +// // ); - return Some(Bytes::from(combined)); - } - } +// return Some(Bytes::from(combined)); +// } +// } - // 取到的数据比需要的块大，从bytes中截取需要的块大小 - if data_size <= bytes.len() { - let n = bytes.len() / data_size; +// // 取到的数据比需要的块大，从bytes中截取需要的块大小 +// if data_size <= bytes.len() { +// let n = bytes.len() / data_size; - for _ in 0..n { - let data = bytes.split_to(data_size); +// for _ in 0..n { +// let data = bytes.split_to(data_size); - // println!("bytes_buffer.push: {}， 剩余：{}", data.len(), bytes.len()); - bytes_buffer.push(data); - } +// // println!("bytes_buffer.push: {}， 剩余：{}", data.len(), bytes.len()); +// bytes_buffer.push(data); +// } - Some(bytes) - } else { - // 不够 - Some(bytes) - } - }; +// Some(bytes) +// } else { +// // 不够 +// Some(bytes) +// } +// }; - // 剩余数据 - let remaining_bytes = 'outer: { - // // 如果上一次数据足够，跳出 - // if let Some(remaining_bytes) = push_data_bytes(prev_bytes) { - // println!("从剩下的取"); - // break 'outer remaining_bytes; - // } +// // 剩余数据 +// let remaining_bytes = 'outer: { +// // // 如果上一次数据足够，跳出 +// // if let Some(remaining_bytes) = push_data_bytes(prev_bytes) { +// // println!("从剩下的取"); +// // break 'outer remaining_bytes; +// // } - loop { - match body.next().await? { - Err(e) => return Some(Err(e)), - Ok(bytes) => { - if let Some(remaining_bytes) = push_data_bytes(bytes) { - break 'outer remaining_bytes; - } - } - } - } - }; +// loop { +// match body.next().await? { +// Err(e) => return Some(Err(e)), +// Ok(bytes) => { +// if let Some(remaining_bytes) = push_data_bytes(bytes) { +// break 'outer remaining_bytes; +// } +// } +// } +// } +// }; - Some(Ok((bytes_buffer, remaining_bytes))) - } +// Some(Ok((bytes_buffer, remaining_bytes))) +// } - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll>> { - let ans = Pin::new(&mut self.inner).poll_next(cx); - if let Poll::Ready(Some(Ok(ref bytes))) = ans { - self.remaining_length = self.remaining_length.saturating_sub(bytes.len()); - } - ans - } +// fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll>> { +// let ans = Pin::new(&mut self.inner).poll_next(cx); +// if let Poll::Ready(Some(Ok(ref bytes))) = ans { +// self.remaining_length = self.remaining_length.saturating_sub(bytes.len()); +// } +// ans +// } - // pub fn exact_remaining_length(&self) -> usize { - // self.remaining_length - // } -} +// // pub fn exact_remaining_length(&self) -> usize { +// // self.remaining_length +// // } +// } -impl Stream for ChunkedStream<'_> { - type Item = Result; +// impl Stream for ChunkedStream<'_> { +// type Item = Result; - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.poll(cx) - } +// fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { +// self.poll(cx) +// } - fn size_hint(&self) -> (usize, Option) { - (0, None) - } -} +// fn size_hint(&self) -> (usize, Option) { +// (0, None) +// } +// } -#[cfg(test)] -mod test { +// #[cfg(test)] +// mod test { - use super::*; +// use super::*; - #[tokio::test] - async fn test_chunked_stream() { - let chunk_size = 4; +// #[tokio::test] +// async fn test_chunked_stream() { +// let chunk_size = 4; - let data1 = vec![1u8; 7777]; // 65536 - let data2 = vec![1u8; 7777]; // 65536 +// let data1 = vec![1u8; 7777]; // 65536 +// let data2 = vec![1u8; 7777]; // 65536 - let content_length = data1.len() + data2.len(); +// let content_length = data1.len() + data2.len(); - let chunk1 = Bytes::from(data1); - let chunk2 = Bytes::from(data2); +// let chunk1 = Bytes::from(data1); +// let chunk2 = Bytes::from(data2); - let chunk_results: Vec> = vec![Ok(chunk1), Ok(chunk2)]; +// let chunk_results: Vec> = vec![Ok(chunk1), Ok(chunk2)]; - let stream = futures::stream::iter(chunk_results); +// let stream = futures::stream::iter(chunk_results); - let mut chunked_stream = ChunkedStream::new(stream, content_length, chunk_size, true); +// let mut chunked_stream = ChunkedStream::new(stream, content_length, chunk_size, true); - loop { - let ans1 = chunked_stream.next().await; - if ans1.is_none() { - break; - } +// loop { +// let ans1 = chunked_stream.next().await; +// if ans1.is_none() { +// break; +// } - let bytes = ans1.unwrap().unwrap(); - assert!(bytes.len() == chunk_size) - } +// let bytes = ans1.unwrap().unwrap(); +// assert!(bytes.len() == chunk_size) +// } - // assert_eq!(ans1.unwrap(), chunk1_data.as_slice()); - } -} +// // assert_eq!(ans1.unwrap(), chunk1_data.as_slice()); +// } +// } diff --git a/ecstore/src/config/common.rs b/ecstore/src/config/common.rs index e5b9e3d70..d5d44b44d 100644 --- a/ecstore/src/config/common.rs +++ b/ecstore/src/config/common.rs @@ -15,7 +15,7 @@ use s3s::dto::StreamingBlob; use s3s::Body; use tracing::error; -const CONFIG_PREFIX: &str = "config"; +pub const CONFIG_PREFIX: &str = "config"; const CONFIG_FILE: &str = "config.json"; pub const STORAGE_CLASS_SUB_SYS: &str = "storage_class"; @@ -161,32 +161,29 @@ async fn apply_dynamic_config(cfg: &mut Config, api: &ECStore) -> Result<()> { Ok(()) } -async fn apply_dynamic_config_for_sub_sys(cfg: &mut Config, api: &ECStore, subsys: &String) -> Result<()> { +async fn apply_dynamic_config_for_sub_sys(cfg: &mut Config, api: &ECStore, subsys: &str) -> Result<()> { let set_drive_counts = api.set_drive_counts(); - match subsys.as_str() { - STORAGE_CLASS_SUB_SYS => { - let kvs = match cfg.get_value(STORAGE_CLASS_SUB_SYS, DEFAULT_KV_KEY) { - Some(res) => res, - None => KVS::new(), - }; + if subsys == STORAGE_CLASS_SUB_SYS { + let kvs = match cfg.get_value(STORAGE_CLASS_SUB_SYS, DEFAULT_KV_KEY) { + Some(res) => res, + None => KVS::new(), + }; - for (i, count) in set_drive_counts.iter().enumerate() { - match storageclass::lookup_config(&kvs, *count) { - Ok(res) => { - if i == 0 && GLOBAL_StorageClass.get().is_none() { - if let Err(r) = GLOBAL_StorageClass.set(res) { - error!("GLOBAL_StorageClass.set failed {:?}", r); - } + for (i, count) in set_drive_counts.iter().enumerate() { + match storageclass::lookup_config(&kvs, *count) { + Ok(res) => { + if i == 0 && GLOBAL_StorageClass.get().is_none() { + if let Err(r) = GLOBAL_StorageClass.set(res) { + error!("GLOBAL_StorageClass.set failed {:?}", r); } } - Err(err) => { - error!("init storageclass err:{:?}", &err); - break; - } + } + Err(err) => { + error!("init storageclass err:{:?}", &err); + break; } } } - _ => {} } Ok(()) diff --git a/ecstore/src/config/heal.rs b/ecstore/src/config/heal.rs new file mode 100644 index 000000000..913997164 --- /dev/null +++ b/ecstore/src/config/heal.rs @@ -0,0 +1,65 @@ +use std::time::Duration; + +use crate::{ + error::{Error, Result}, + utils::bool_flag::parse_bool, +}; + +#[derive(Debug, Default)] +pub struct Config { + pub bitrot: String, + pub sleep: Duration, + pub io_count: usize, + pub drive_workers: usize, + pub cache: Duration, +} + +impl Config { + pub fn bitrot_scan_cycle(&self) -> Duration { + self.cache + } + + pub fn get_workers(&self) -> usize { + self.drive_workers + } + + pub fn update(&mut self, nopts: &Config) { + self.bitrot = nopts.bitrot.clone(); + self.io_count = nopts.io_count; + self.sleep = nopts.sleep; + self.drive_workers = nopts.drive_workers; + } +} + +const RUSTFS_BITROT_CYCLE_IN_MONTHS: u64 = 1; + +fn parse_bitrot_config(s: &str) -> Result { + match parse_bool(s) { + Ok(enabled) => { + if enabled { + Ok(Duration::from_secs_f64(0.0)) + } else { + Ok(Duration::from_secs_f64(-1.0)) + } + } + Err(_) => { + if !s.ends_with("m") { + return Err(Error::from_string("unknown format")); + } + + match s.trim_end_matches('m').parse::() { + Ok(months) => { + if months < RUSTFS_BITROT_CYCLE_IN_MONTHS { + return Err(Error::from_string(format!( + "minimum bitrot cycle is {} month(s)", + RUSTFS_BITROT_CYCLE_IN_MONTHS + ))); + } + + Ok(Duration::from_secs(months * 30 * 24 * 60)) + } + Err(err) => Err(err.into()), + } + } + } +} diff --git a/ecstore/src/config/mod.rs b/ecstore/src/config/mod.rs index 7da3f8469..a169d6988 100644 --- a/ecstore/src/config/mod.rs +++ b/ecstore/src/config/mod.rs @@ -1,5 +1,7 @@ pub mod common; pub mod error; +#[allow(dead_code)] +pub mod heal; pub mod storageclass; use crate::error::Result; @@ -17,8 +19,16 @@ lazy_static! { pub static ref GLOBAL_ConfigSys: ConfigSys = ConfigSys::new(); } +pub static RUSTFS_CONFIG_PREFIX: &str = "config"; + pub struct ConfigSys {} +impl Default for ConfigSys { + fn default() -> Self { + Self::new() + } +} + impl ConfigSys { pub fn new() -> Self { Self {} @@ -44,6 +54,12 @@ pub struct KV { #[derive(Debug, Deserialize, Serialize, Clone)] pub struct KVS(Vec); +impl Default for KVS { + fn default() -> Self { + Self::new() + } +} + impl KVS { pub fn new() -> Self { KVS(Vec::new()) @@ -69,6 +85,12 @@ impl KVS { #[derive(Debug, Clone)] pub struct Config(HashMap>); +impl Default for Config { + fn default() -> Self { + Self::new() + } +} + impl Config { pub fn new() -> Self { let mut cfg = Config(HashMap::new()); diff --git a/ecstore/src/config/storageclass.rs b/ecstore/src/config/storageclass.rs index 6e8917250..46bc0737b 100644 --- a/ecstore/src/config/storageclass.rs +++ b/ecstore/src/config/storageclass.rs @@ -228,7 +228,7 @@ pub fn parse_storage_class(env: &str) -> Result { // only two elements allowed in the string - "scheme" and "number of parity drives" if s.len() != 2 { - return Err(Error::msg(&format!( + return Err(Error::msg(format!( "Invalid storage class format: {}. Expected 'Scheme:Number of parity drives'.", env ))); @@ -236,13 +236,13 @@ pub fn parse_storage_class(env: &str) -> Result { // only allowed scheme is "EC" if s[0] != SCHEME_PREFIX { - return Err(Error::msg(&format!("Unsupported scheme {}. Supported scheme is EC.", s[0]))); + return Err(Error::msg(format!("Unsupported scheme {}. Supported scheme is EC.", s[0]))); } // Number of parity drives should be integer let parity_drives: usize = match s[1].parse() { Ok(num) => num, - Err(_) => return Err(Error::msg(&format!("Failed to parse parity value: {}.", s[1]))), + Err(_) => return Err(Error::msg(format!("Failed to parse parity value: {}.", s[1]))), }; Ok(StorageClass { parity: parity_drives }) diff --git a/ecstore/src/disk/endpoint.rs b/ecstore/src/disk/endpoint.rs index 254a1213b..8c90d35dc 100644 --- a/ecstore/src/disk/endpoint.rs +++ b/ecstore/src/disk/endpoint.rs @@ -2,6 +2,7 @@ use crate::error::{Error, Result}; use crate::utils::net; use path_absolutize::Absolutize; use path_clean::PathClean; +use std::fs; use std::{fmt::Display, path::Path}; use url::{ParseError, Url}; @@ -29,7 +30,13 @@ pub struct Endpoint { impl Display for Endpoint { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { if self.url.scheme() == "file" { - write!(f, "{}", self.url.path()) + write!( + f, + "{}", + fs::canonicalize(self.url.path()) + .map_err(|_| std::fmt::Error)? + .to_string_lossy() + ) } else { write!(f, "{}", self.url) } diff --git a/ecstore/src/disk/error.rs b/ecstore/src/disk/error.rs index fd20a91dc..0bce48455 100644 --- a/ecstore/src/disk/error.rs +++ b/ecstore/src/disk/error.rs @@ -106,6 +106,9 @@ pub enum DiskError { #[error("part missing or corrupt")] PartMissingOrCorrupt, + + #[error("No healing is required")] + NoHealRequired, } impl DiskError { @@ -210,6 +213,7 @@ pub fn clone_disk_err(e: &DiskError) -> Error { DiskError::MoreData => Error::new(DiskError::MoreData), DiskError::OutdatedXLMeta => Error::new(DiskError::OutdatedXLMeta), DiskError::PartMissingOrCorrupt => Error::new(DiskError::PartMissingOrCorrupt), + DiskError::NoHealRequired => Error::new(DiskError::NoHealRequired), } } @@ -261,14 +265,7 @@ pub fn os_err_to_file_err(e: io::Error) -> Error { } pub fn is_err_file_not_found(err: &Error) -> bool { - if let Some(e) = err.downcast_ref::() { - match e { - DiskError::FileNotFound => true, - _ => false, - } - } else { - false - } + matches!(err.downcast_ref::(), Some(DiskError::FileNotFound)) } pub fn is_sys_err_no_space(e: &io::Error) -> bool { @@ -422,18 +419,32 @@ pub fn convert_access_error(e: io::Error, per_err: DiskError) -> Error { } pub fn is_all_not_found(errs: &[Option]) -> bool { - for err in errs.iter() { - if let Some(err) = err { - if let Some(err) = err.downcast_ref::() { - match err { - DiskError::FileNotFound | DiskError::VolumeNotFound | &DiskError::FileVersionNotFound => { - continue; - } - _ => return false, + for err in errs.iter().flatten() { + if let Some(err) = err.downcast_ref::() { + match err { + DiskError::FileNotFound | DiskError::VolumeNotFound | &DiskError::FileVersionNotFound => { + continue; } + _ => return false, } } } !errs.is_empty() } + +pub fn is_all_buckets_not_found(errs: &[Option]) -> bool { + if errs.is_empty() { + return false; + } + let mut not_found_count = 0; + for err in errs.iter().flatten() { + match err.downcast_ref() { + Some(DiskError::VolumeNotFound) | Some(DiskError::DiskNotFound) => { + not_found_count += 1; + } + _ => {} + } + } + errs.len() == not_found_count +} diff --git a/ecstore/src/disk/format.rs b/ecstore/src/disk/format.rs index f2a9775cc..602ea629c 100644 --- a/ecstore/src/disk/format.rs +++ b/ecstore/src/disk/format.rs @@ -1,4 +1,4 @@ -use super::error::DiskError; +use super::{error::DiskError, DiskInfo}; use crate::error::{Error, Result}; use serde::{Deserialize, Serialize}; use serde_json::Error as JsonError; @@ -31,7 +31,7 @@ pub enum FormatBackend { /// /// The V3 format to support "large bucket" support where a bucket /// can span multiple erasure sets. -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] pub struct FormatErasureV3 { /// Version of 'xl' format. pub version: FormatErasureVersion, @@ -88,7 +88,7 @@ pub enum DistributionAlgoVersion { /// /// Ideally we will never have a situation where we will have to change the /// fields of this struct and deal with related migration. -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] pub struct FormatV3 { /// Version of the format config. pub version: FormatMetaVersion, @@ -103,8 +103,8 @@ pub struct FormatV3 { pub erasure: FormatErasureV3, // /// DiskInfo is an extended type which returns current // /// disk usage per path. - // #[serde(skip)] - // pub disk_info: Option, + #[serde(skip)] + pub disk_info: Option, } impl TryFrom<&[u8]> for FormatV3 { @@ -146,7 +146,7 @@ impl FormatV3 { format, id: Uuid::new_v4(), erasure, - // disk_info: None, + disk_info: None, } } diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 622a345db..c168e4231 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -6,10 +6,11 @@ use super::{endpoint::Endpoint, error::DiskError, format::FormatV3}; use super::{ os, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions, FileReader, FileWriter, Info, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, - UpdateMetadataOpts, VolumeInfo, WalkDirOptions, + UpdateMetadataOpts, VolumeInfo, WalkDirOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET, }; use crate::bitrot::bitrot_verify; -use crate::cache_value::cache::{Cache, Opts}; +use crate::bucket::metadata_sys::GLOBAL_BucketMetadataSys; +use crate::cache_value::cache::{Cache, Opts, UpdateFn}; use crate::disk::error::{ convert_access_error, is_sys_err_handle_invalid, is_sys_err_invalid_arg, is_sys_err_is_dir, is_sys_err_not_dir, map_err_not_exists, os_err_to_file_err, @@ -18,27 +19,35 @@ use crate::disk::os::{check_path_length, is_empty_dir}; use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE}; use crate::error::{Error, Result}; use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold}; +use crate::heal::data_scanner::{has_active_rules, scan_data_folder, ScannerItem, SizeSummary}; +use crate::heal::data_scanner_metric::{ScannerMetric, ScannerMetrics}; +use crate::heal::data_usage_cache::{DataUsageCache, DataUsageEntry}; +use crate::heal::error::{ERR_IGNORE_FILE_CONTRIB, ERR_SKIP_FILE}; +use crate::heal::heal_commands::{HealScanMode, HealingTracker}; +use crate::heal::heal_ops::HEALING_TRACKER_FILENAME; +use crate::new_object_layer_fn; use crate::set_disk::{ conv_part_err_to_int, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, CHECK_PART_VOLUME_NOT_FOUND, }; -use crate::store_api::BitrotAlgorithm; +use crate::store_api::{BitrotAlgorithm, StorageAPI}; use crate::utils::fs::{access, lstat, O_APPEND, O_CREATE, O_RDONLY, O_WRONLY}; use crate::utils::os::get_info; -use crate::utils::path::{clean, has_suffix, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR}; +use crate::utils::path::{clean, has_suffix, path_join, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR}; use crate::{ file_meta::FileMeta, store_api::{FileInfo, RawFileInfo}, utils, }; +use common::defer; use path_absolutize::Absolutize; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::fmt::Debug; use std::io::Cursor; use std::os::unix::fs::MetadataExt; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, SystemTime}; use std::{ fs::Metadata, path::{Path, PathBuf}, @@ -46,7 +55,7 @@ use std::{ use time::OffsetDateTime; use tokio::fs::{self, File}; use tokio::io::{AsyncReadExt, AsyncWriteExt, ErrorKind}; -use tokio::runtime::Runtime; +use tokio::sync::mpsc::Sender; use tokio::sync::RwLock; use tracing::{error, info, warn}; use uuid::Uuid; @@ -138,10 +147,10 @@ impl LocalDisk { last_check: format_last_check, }; let root_clone = root.clone(); - let disk_id = Arc::new(id.map_or("".to_string(), |id| id.to_string())); - let update_fn = move || { - let rt = Runtime::new().unwrap(); - rt.block_on(async { + let update_fn: UpdateFn = Box::new(move || { + let disk_id = id.map_or("".to_string(), |id| id.to_string()); + let root = root_clone.clone(); + Box::pin(async move { match get_disk_info(root.clone()).await { Ok((info, root)) => { let disk_info = DiskInfo { @@ -167,14 +176,14 @@ impl LocalDisk { Err(err) => Err(err), } }) - }; + }); - let cache = Cache::new(Box::new(update_fn), Duration::from_secs(1), Opts::default()); + let cache = Cache::new(update_fn, Duration::from_secs(1), Opts::default()); // TODO: DIRECT suport // TODD: DiskInfo let mut disk = Self { - root: root_clone.clone(), + root: root.clone(), endpoint: ep.clone(), format_path, format_info: RwLock::new(format_info), @@ -190,7 +199,7 @@ impl LocalDisk { // format_data: Mutex::new(format_data), // format_last_check: Mutex::new(format_last_check), }; - let (info, _root) = get_disk_info(root_clone).await?; + let (info, _root) = get_disk_info(root).await?; disk.major = info.major; disk.minor = info.minor; disk.fstype = info.fstype; @@ -932,7 +941,7 @@ impl DiskAPI for LocalDisk { } async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result { - let volume_dir = self.get_bucket_path(&volume)?; + let volume_dir = self.get_bucket_path(volume)?; check_path_length(volume_dir.join(path).to_string_lossy().as_ref())?; let mut resp = CheckPartsResp { results: vec![0; fi.parts.len()], @@ -958,22 +967,16 @@ impl DiskAPI for LocalDisk { resp.results[i] = CHECK_PART_SUCCESS; } Err(err) => { - match os_err_to_file_err(err).downcast_ref() { - Some(DiskError::FileNotFound) => { - if !skip_access_checks(volume) { - if let Err(err) = access(&volume_dir).await { - match err.kind() { - ErrorKind::NotFound => { - resp.results[i] = CHECK_PART_VOLUME_NOT_FOUND; - continue; - } - _ => {} - } + if let Some(DiskError::FileNotFound) = os_err_to_file_err(err).downcast_ref() { + if !skip_access_checks(volume) { + if let Err(err) = access(&volume_dir).await { + if err.kind() == ErrorKind::NotFound { + resp.results[i] = CHECK_PART_VOLUME_NOT_FOUND; + continue; } } - resp.results[i] = CHECK_PART_FILE_NOT_FOUND; } - _ => {} + resp.results[i] = CHECK_PART_FILE_NOT_FOUND; } continue; } @@ -1881,6 +1884,162 @@ impl DiskAPI for LocalDisk { Ok(info) } + + async fn ns_scanner( + &self, + cache: &DataUsageCache, + updates: Sender, + scan_mode: HealScanMode, + ) -> Result { + self.scanning.fetch_add(1, Ordering::SeqCst); + defer!(|| { self.scanning.fetch_sub(1, Ordering::SeqCst) }); + + // Check if the current bucket has replication configuration + if let Ok((rcfg, _)) = GLOBAL_BucketMetadataSys + .read() + .await + .get_replication_config(&cache.info.name) + .await + { + if has_active_rules(&rcfg, "", true) { + // TODO: globalBucketTargetSys + } + } + + let layer = new_object_layer_fn(); + let lock = layer.read().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(Error::msg("errServerNotInitialized")), + }; + let loc = self.get_disk_location(); + let disks = store.get_disks(loc.pool_idx.unwrap(), loc.disk_idx.unwrap()).await?; + let disk = Arc::new(LocalDisk::new(&self.endpoint(), false).await?); + let disk_clone = disk.clone(); + let mut cache = cache.clone(); + cache.info.updates = Some(updates.clone()); + let mut data_usage_info = scan_data_folder( + &disks, + disk, + &cache, + Box::new(move |item: &ScannerItem| { + let mut item = item.clone(); + let disk = disk_clone.clone(); + Box::pin(async move { + if item.path.ends_with(&format!("{}{}", SLASH_SEPARATOR, STORAGE_FORMAT_FILE)) { + return Err(Error::from_string(ERR_SKIP_FILE)); + } + let stop_fn = ScannerMetrics::log(ScannerMetric::ScanObject); + let mut res = HashMap::new(); + let done_sz = ScannerMetrics::time_size(ScannerMetric::ReadMetadata).await; + let buf = match disk.read_metadata(item.path.clone()).await { + Ok(buf) => buf, + Err(err) => { + res.insert("err".to_string(), err.to_string()); + stop_fn(&res).await; + return Err(Error::from_string(ERR_SKIP_FILE)); + } + }; + done_sz(buf.len() as u64).await; + res.insert("metasize".to_string(), buf.len().to_string()); + item.transform_meda_dir(); + let meta_cache = MetaCacheEntry { + name: item.object_path().to_string_lossy().to_string(), + metadata: buf, + ..Default::default() + }; + let fivs = match meta_cache.file_info_versions(&item.bucket) { + Ok(fivs) => fivs, + Err(err) => { + res.insert("err".to_string(), err.to_string()); + stop_fn(&res).await; + return Err(Error::from_string(ERR_SKIP_FILE)); + } + }; + let mut size_s = SizeSummary::default(); + let done = ScannerMetrics::time(ScannerMetric::ApplyAll); + let obj_infos = match item.apply_versions_actions(&fivs.versions).await { + Ok(obj_infos) => obj_infos, + Err(err) => { + res.insert("err".to_string(), err.to_string()); + stop_fn(&res).await; + return Err(Error::from_string(ERR_SKIP_FILE)); + } + }; + + let versioned = false; + let mut obj_deleted = false; + for info in obj_infos.iter() { + let done = ScannerMetrics::time(ScannerMetric::ApplyVersion); + let sz: usize; + (obj_deleted, sz) = item.apply_actions(info, &size_s).await; + done().await; + + if obj_deleted { + break; + } + + let actual_sz = match info.get_actual_size() { + Ok(size) => size, + Err(_) => continue, + }; + + if info.delete_marker { + size_s.delete_markers += 1; + } + + if info.version_id.is_some() && sz == actual_sz { + size_s.versions += 1; + } + + size_s.total_size += sz; + + if info.delete_marker { + continue; + } + } + + for frer_version in fivs.free_versions.iter() { + let _obj_info = + frer_version.to_object_info(&item.bucket, &item.object_path().to_string_lossy(), versioned); + let done = ScannerMetrics::time(ScannerMetric::TierObjSweep); + done().await; + } + + // todo: global trace + if obj_deleted { + return Err(Error::from_string(ERR_IGNORE_FILE_CONTRIB)); + } + done().await; + Ok(size_s) + }) + }), + scan_mode, + ) + .await?; + data_usage_info.info.last_update = Some(SystemTime::now()); + Ok(data_usage_info) + } + + async fn healing(&self) -> Option { + let healing_file = path_join(&[ + self.path(), + PathBuf::from(RUSTFS_META_BUCKET), + PathBuf::from(BUCKET_META_PREFIX), + PathBuf::from(HEALING_TRACKER_FILENAME), + ]); + let b = match fs::read(healing_file).await { + Ok(b) => b, + Err(_) => return None, + }; + if b.is_empty() { + return None; + } + match HealingTracker::unmarshal_msg(&b) { + Ok(h) => Some(h), + Err(_) => Some(HealingTracker::default()), + } + } } async fn get_disk_info(drive_path: PathBuf) -> Result<(Info, bool)> { @@ -1893,10 +2052,7 @@ async fn get_disk_info(drive_path: PathBuf) -> Result<(Info, bool)> { if root_disk_threshold > 0 { disk_info.total <= root_disk_threshold } else { - match is_root_disk(&drive_path, SLASH_SEPARATOR) { - Ok(result) => result, - Err(_) => false, - } + is_root_disk(&drive_path, SLASH_SEPARATOR).unwrap_or_default() } } else { false diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs index e57543058..ec393a1a2 100644 --- a/ecstore/src/disk/mod.rs +++ b/ecstore/src/disk/mod.rs @@ -1,7 +1,7 @@ pub mod endpoint; pub mod error; pub mod format; -mod local; +pub mod local; pub mod os; pub mod remote; @@ -14,9 +14,13 @@ pub const FORMAT_CONFIG_FILE: &str = "format.json"; const STORAGE_FORMAT_FILE: &str = "xl.meta"; use crate::{ - erasure::{ReadAt, Writer}, + erasure::Writer, error::{Error, Result}, file_meta::{merge_file_meta_versions, FileMeta, FileMetaShallowVersion}, + heal::{ + data_usage_cache::{DataUsageCache, DataUsageEntry}, + heal_commands::{HealScanMode, HealingTracker}, + }, store_api::{FileInfo, RawFileInfo}, }; use endpoint::Endpoint; @@ -35,7 +39,6 @@ use std::{ io::{Cursor, SeekFrom}, path::PathBuf, sync::Arc, - usize, }; use time::OffsetDateTime; use tokio::{ @@ -53,8 +56,8 @@ pub type DiskStore = Arc; #[derive(Debug)] pub enum Disk { - Local(LocalDisk), - Remote(RemoteDisk), + Local(Box), + Remote(Box), } #[async_trait::async_trait] @@ -338,15 +341,34 @@ impl DiskAPI for Disk { Disk::Remote(remote_disk) => remote_disk.disk_info(opts).await, } } + + async fn ns_scanner( + &self, + cache: &DataUsageCache, + updates: Sender, + scan_mode: HealScanMode, + ) -> Result { + match self { + Disk::Local(local_disk) => local_disk.ns_scanner(cache, updates, scan_mode).await, + Disk::Remote(remote_disk) => remote_disk.ns_scanner(cache, updates, scan_mode).await, + } + } + + async fn healing(&self) -> Option { + match self { + Disk::Local(local_disk) => local_disk.healing().await, + Disk::Remote(remote_disk) => remote_disk.healing().await, + } + } } pub async fn new_disk(ep: &endpoint::Endpoint, opt: &DiskOption) -> Result { if ep.is_local { let s = local::LocalDisk::new(ep, opt.cleanup).await?; - Ok(Arc::new(Disk::Local(s))) + Ok(Arc::new(Disk::Local(Box::new(s)))) } else { let remote_disk = remote::RemoteDisk::new(ep, opt).await?; - Ok(Arc::new(Disk::Remote(remote_disk))) + Ok(Arc::new(Disk::Remote(Box::new(remote_disk)))) } } @@ -436,6 +458,13 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { async fn write_all(&self, volume: &str, path: &str, data: Vec) -> Result<()>; async fn read_all(&self, volume: &str, path: &str) -> Result>; async fn disk_info(&self, opts: &DiskInfoOptions) -> Result; + async fn ns_scanner( + &self, + cache: &DataUsageCache, + updates: Sender, + scan_mode: HealScanMode, + ) -> Result; + async fn healing(&self) -> Option; } #[derive(Debug, Default, Serialize, Deserialize)] @@ -467,7 +496,7 @@ pub struct DiskInfoOptions { pub noop: bool, } -#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] pub struct DiskInfo { pub total: u64, pub free: u64, @@ -489,7 +518,7 @@ pub struct DiskInfo { pub error: String, } -#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] pub struct DiskMetrics { api_calls: HashMap, total_waiting: u32, @@ -653,7 +682,7 @@ impl MetaCacheEntry { let mut fm = FileMeta::new(); fm.unmarshal_msg(&self.metadata)?; - Ok(fm.into_file_info_versions(bucket, self.name.as_str(), false)?) + fm.into_file_info_versions(bucket, self.name.as_str(), false) } pub fn matches(&self, other: &MetaCacheEntry, strict: bool) -> Result<(Option, bool)> { @@ -812,7 +841,7 @@ impl MetaCacheEntries { selected = Some(MetaCacheEntry { name: selected.as_ref().unwrap().name.clone(), cached: Some(FileMeta { - meta_ver: selected.as_ref().unwrap().cached.as_ref().unwrap().meta_ver.clone(), + meta_ver: selected.as_ref().unwrap().cached.as_ref().unwrap().meta_ver, ..Default::default() }), _reusable: true, @@ -835,7 +864,7 @@ impl MetaCacheEntries { } } -#[derive(Debug, Default)] +#[derive(Clone, Debug, Default)] pub struct DiskOption { pub cleanup: bool, pub health_check: bool, @@ -1281,10 +1310,10 @@ impl Reader for RemoteFileReader { Err(Error::from_string(error_info)) } } - async fn seek(&mut self, offset: usize) -> Result<()> { + async fn seek(&mut self, _offset: usize) -> Result<()> { unimplemented!() } - async fn read_exact(&mut self, buf: &mut [u8]) -> Result { + async fn read_exact(&mut self, _buf: &mut [u8]) -> Result { unimplemented!() } } diff --git a/ecstore/src/disk/os.rs b/ecstore/src/disk/os.rs index 8375d608a..322d5f258 100644 --- a/ecstore/src/disk/os.rs +++ b/ecstore/src/disk/os.rs @@ -56,7 +56,7 @@ pub fn is_root_disk(disk_path: &str, root_disk: &str) -> Result { return Ok(false); } - Ok(same_disk(disk_path, root_disk)?) + same_disk(disk_path, root_disk) } pub async fn make_dir_all(path: impl AsRef, base_dir: impl AsRef) -> Result<()> { diff --git a/ecstore/src/disk/remote.rs b/ecstore/src/disk/remote.rs index 80fdbb5b7..ed69af25b 100644 --- a/ecstore/src/disk/remote.rs +++ b/ecstore/src/disk/remote.rs @@ -5,11 +5,13 @@ use protos::{ node_service_time_out_client, proto_gen::node_service::{ CheckPartsRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, - DiskInfoRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest, - ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequst, StatVolumeRequest, - UpdateMetadataRequest, VerifyFileRequest, WalkDirRequest, WriteAllRequest, WriteMetadataRequest, + DiskInfoRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, NsScannerRequest, + ReadAllRequest, ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequst, + StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WalkDirRequest, WriteAllRequest, WriteMetadataRequest, }, }; +use tokio::sync::mpsc::{self, Sender}; +use tokio_stream::{wrappers::ReceiverStream, StreamExt}; use tonic::Request; use tracing::info; use uuid::Uuid; @@ -22,6 +24,10 @@ use super::{ use crate::{ disk::error::DiskError, error::{Error, Result}, + heal::{ + data_usage_cache::{DataUsageCache, DataUsageEntry}, + heal_commands::{HealScanMode, HealingTracker}, + }, store_api::{FileInfo, RawFileInfo}, }; use protos::proto_gen::node_service::RenamePartRequst; @@ -747,4 +753,48 @@ impl DiskAPI for RemoteDisk { Ok(disk_info) } + + async fn ns_scanner( + &self, + cache: &DataUsageCache, + updates: Sender, + scan_mode: HealScanMode, + ) -> Result { + info!("ns_scanner"); + let cache = serde_json::to_string(cache)?; + let mut client = node_service_time_out_client(&self.addr) + .await + .map_err(|err| Error::from_string(format!("can not get client, err: {}", err)))?; + + let (tx, rx) = mpsc::channel(10); + let in_stream = ReceiverStream::new(rx); + let mut response = client.ns_scanner(in_stream).await?.into_inner(); + let request = NsScannerRequest { + disk: self.root.to_string_lossy().to_string(), + cache, + scan_mode: scan_mode as u64, + }; + tx.send(request).await?; + + loop { + match response.next().await { + Some(Ok(resp)) => { + if !resp.update.is_empty() { + let data_usage_cache = serde_json::from_str::(&resp.update)?; + let _ = updates.send(data_usage_cache).await; + } else if !resp.data_usage_cache.is_empty() { + let data_usage_cache = serde_json::from_str::(&resp.data_usage_cache)?; + return Ok(data_usage_cache); + } else { + return Err(Error::from_string("scan was interrupted")); + } + } + _ => return Err(Error::from_string("scan was interrupted")), + } + } + } + + async fn healing(&self) -> Option { + None + } } diff --git a/ecstore/src/endpoints.rs b/ecstore/src/endpoints.rs index 844fe936c..eeca16143 100644 --- a/ecstore/src/endpoints.rs +++ b/ecstore/src/endpoints.rs @@ -109,6 +109,24 @@ impl Endpoints { pub fn into_inner(self) -> Vec { self.0 } + + pub fn into_ref(&self) -> &Vec { + &self.0 + } + + // GetString - returns endpoint string of i-th endpoint (0-based), + // and empty string for invalid indexes. + pub fn get_string(&self, i: usize) -> String { + if i >= self.0.len() { + return "".to_string(); + } + + self.0[i].to_string() + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } } #[derive(Debug)] diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs index f0983aac6..ee1687833 100644 --- a/ecstore/src/erasure.rs +++ b/ecstore/src/erasure.rs @@ -19,6 +19,7 @@ use uuid::Uuid; // use crate::chunk_stream::ChunkedStream; use crate::disk::error::DiskError; +#[derive(Default)] pub struct Erasure { data_shards: usize, parity_shards: usize, @@ -417,6 +418,54 @@ impl Erasure { till_offset } + + pub async fn heal( + &self, + writers: &mut [Option], + readers: Vec>, + total_length: usize, + _prefer: &[bool], + ) -> Result<()> { + if writers.len() != self.parity_shards + self.data_shards { + return Err(Error::from_string("invalid argument")); + } + let mut reader = ShardReader::new(readers, self, 0, total_length); + + let start_block = 0; + let mut end_block = total_length / self.block_size; + if total_length % self.block_size != 0 { + end_block += 1; + } + + let mut errs = Vec::new(); + for _ in start_block..=end_block { + let mut bufs = reader.read().await?; + + if self.parity_shards > 0 { + self.encoder.as_ref().unwrap().reconstruct(&mut bufs)?; + } + + let shards = bufs.into_iter().flatten().collect::>(); + if shards.len() != self.parity_shards + self.data_shards { + return Err(Error::from_string("can not reconstruct data")); + } + + for (i, w) in writers.iter_mut().enumerate() { + if w.is_none() { + continue; + } + match w.as_mut().unwrap().write(shards[i].as_ref()).await { + Ok(_) => {} + Err(e) => errs.push(e), + } + } + } + if !errs.is_empty() { + return Err(errs[0].clone()); + } + + Ok(()) + } } #[async_trait::async_trait] diff --git a/ecstore/src/global.rs b/ecstore/src/global.rs index 16ac35be9..e2c46907a 100644 --- a/ecstore/src/global.rs +++ b/ecstore/src/global.rs @@ -6,6 +6,7 @@ use uuid::Uuid; use crate::{ disk::DiskStore, endpoints::{EndpointServerPools, PoolEndpoints, SetupType}, + heal::{background_heal_ops::HealRoutine, heal_ops::AllHealState}, store::ECStore, }; @@ -24,6 +25,8 @@ lazy_static! { pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc> = Arc::new(RwLock::new(Vec::new())); pub static ref GLOBAL_Endpoints: RwLock = RwLock::new(EndpointServerPools(Vec::new())); pub static ref GLOBAL_RootDiskThreshold: RwLock = RwLock::new(0); + pub static ref GLOBAL_BackgroundHealRoutine: Arc> = HealRoutine::new(); + pub static ref GLOBAL_BackgroundHealState: Arc> = AllHealState::new(false); static ref globalDeploymentIDPtr: RwLock = RwLock::new(Uuid::nil()); } @@ -34,7 +37,7 @@ pub async fn set_global_deployment_id(id: Uuid) { pub async fn get_global_deployment_id() -> Uuid { let id_ptr = globalDeploymentIDPtr.read().await; - id_ptr.clone() + *id_ptr } pub async fn set_global_endpoints(eps: Vec) { diff --git a/ecstore/src/heal/background_heal_ops.rs b/ecstore/src/heal/background_heal_ops.rs index 30f9e0563..c60287b5a 100644 --- a/ecstore/src/heal/background_heal_ops.rs +++ b/ecstore/src/heal/background_heal_ops.rs @@ -1,76 +1,424 @@ -use std::sync::Arc; - +use s3s::{S3Error, S3ErrorCode}; +use std::{cmp::Ordering, env, path::PathBuf, sync::Arc, time::Duration}; use tokio::{ - select, sync::{ - broadcast::Receiver as B_Receiver, mpsc::{self, Receiver, Sender}, + RwLock, }, + time::interval, }; - -use crate::{error::Error, heal::heal_ops::NOP_HEAL, utils::path::SLASH_SEPARATOR}; +use tracing::{error, info}; +use uuid::Uuid; use super::{ heal_commands::{HealOpts, HealResultItem}, - heal_ops::HealSequence, + heal_ops::{new_bg_heal_sequence, HealSequence}, +}; +use crate::heal::error::ERR_RETRY_HEALING; +use crate::{ + config::RUSTFS_CONFIG_PREFIX, + disk::{endpoint::Endpoint, error::DiskError, DiskAPI, DiskInfoOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET}, + error::{Error, Result}, + global::{GLOBAL_BackgroundHealRoutine, GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP}, + heal::{ + data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT}, + data_usage_cache::DataUsageCache, + heal_commands::{init_healing_tracker, load_healing_tracker}, + heal_ops::NOP_HEAL, + }, + new_object_layer_fn, + store::get_disk_via_endpoint, + store_api::{BucketInfo, BucketOptions, StorageAPI}, + utils::path::{path_join, SLASH_SEPARATOR}, }; -#[derive(Clone, Debug)] +pub static DEFAULT_MONITOR_NEW_DISK_INTERVAL: Duration = Duration::from_secs(10); + +pub async fn init_auto_heal() { + init_background_healing().await; + if let Ok(v) = env::var("_RUSTFS_AUTO_DRIVE_HEALING") { + if v == "on" { + GLOBAL_BackgroundHealState + .write() + .await + .push_heal_local_disks(&get_local_disks_to_heal().await) + .await; + tokio::spawn(async { + monitor_local_disks_and_heal().await; + }); + } + } +} + +async fn init_background_healing() { + let bg_seq = Arc::new(RwLock::new(new_bg_heal_sequence())); + for _ in 0..GLOBAL_BackgroundHealRoutine.read().await.workers { + let bg_seq_clone = bg_seq.clone(); + tokio::spawn(async { + GLOBAL_BackgroundHealRoutine.write().await.add_worker(bg_seq_clone).await; + }); + } + let _ = GLOBAL_BackgroundHealState + .write() + .await + .launch_new_heal_sequence(bg_seq) + .await; +} + +async fn get_local_disks_to_heal() -> Vec { + let mut disks_to_heal = Vec::new(); + for (_, disk) in GLOBAL_LOCAL_DISK_MAP.read().await.iter() { + if let Some(disk) = disk { + if let Err(err) = disk.disk_info(&DiskInfoOptions::default()).await { + if let Some(DiskError::UnformattedDisk) = err.downcast_ref() { + disks_to_heal.push(disk.endpoint()); + } + } + let h = disk.healing().await; + if let Some(h) = h { + if !h.finished { + disks_to_heal.push(disk.endpoint()); + } + } + } + } + + // todo + // if disks_to_heal.len() == GLOBAL_Endpoints.read().await.n { + + // } + disks_to_heal +} + +async fn monitor_local_disks_and_heal() { + let mut interval = interval(DEFAULT_MONITOR_NEW_DISK_INTERVAL); + + loop { + interval.tick().await; + let heal_disks = GLOBAL_BackgroundHealState.read().await.get_heal_local_disk_endpoints().await; + if heal_disks.is_empty() { + interval.reset(); + continue; + } + let layer = new_object_layer_fn(); + let lock = layer.read().await; + let store = lock.as_ref().expect("errServerNotInitialized"); + if let (_, Some(err)) = store.heal_format(false).await.expect("heal format failed") { + if let Some(DiskError::NoHealRequired) = err.downcast_ref() { + } else { + info!("heal format err: {}", err.to_string()); + interval.reset(); + continue; + } + } + + for disk in heal_disks.into_ref().iter() { + let disk_clone = disk.clone(); + tokio::spawn(async move { + GLOBAL_BackgroundHealState + .write() + .await + .set_disk_healing_status(disk_clone.clone(), true) + .await; + if heal_fresh_disk(&disk_clone).await.is_err() { + GLOBAL_BackgroundHealState + .write() + .await + .set_disk_healing_status(disk_clone.clone(), false) + .await; + return; + } + GLOBAL_BackgroundHealState + .write() + .await + .pop_heal_local_disks(&[disk_clone]) + .await; + }); + } + interval.reset(); + } +} + +async fn heal_fresh_disk(endpoint: &Endpoint) -> Result<()> { + let (pool_idx, set_idx) = (endpoint.pool_idx as usize, endpoint.disk_idx as usize); + let disk = match get_disk_via_endpoint(endpoint).await { + Some(disk) => disk, + None => { + return Err(Error::from_string(format!( + "Unexpected error disk must be initialized by now after formatting: {}", + endpoint + ))) + } + }; + + if let Err(err) = disk.disk_info(&DiskInfoOptions::default()).await { + match err.downcast_ref() { + Some(DiskError::DriveIsRoot) => { + return Ok(()); + } + Some(DiskError::UnformattedDisk) => {} + _ => { + return Err(err); + } + } + } + + let mut tracker = match load_healing_tracker(&Some(disk.clone())).await { + Ok(tracker) => tracker, + Err(err) => { + match err.downcast_ref() { + Some(DiskError::FileNotFound) => { + return Ok(()); + } + _ => { + info!( + "Unable to load healing tracker on '{}': {}, re-initializing..", + disk.to_string(), + err.to_string() + ); + } + } + init_healing_tracker(disk.clone(), &Uuid::new_v4().to_string()).await? + } + }; + + info!( + "Healing drive '{}' - 'mc admin heal alias/ --verbose' to check the current status.", + endpoint.to_string() + ); + + let layer = new_object_layer_fn(); + let lock = layer.read().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(Error::msg("errServerNotInitialized")), + }; + let mut buckets = store.list_bucket(&BucketOptions::default()).await?; + buckets.push(BucketInfo { + name: path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(RUSTFS_CONFIG_PREFIX)]) + .to_string_lossy() + .to_string(), + ..Default::default() + }); + buckets.push(BucketInfo { + name: path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(BUCKET_META_PREFIX)]) + .to_string_lossy() + .to_string(), + ..Default::default() + }); + + buckets.sort_by(|a, b| { + let a_has_prefix = a.name.starts_with(RUSTFS_META_BUCKET); + let b_has_prefix = b.name.starts_with(RUSTFS_META_BUCKET); + + match (a_has_prefix, b_has_prefix) { + (true, false) => Ordering::Less, + (false, true) => Ordering::Greater, + _ => b.created.cmp(&a.created), + } + }); + + if let Ok(cache) = DataUsageCache::load(&store.pools[pool_idx].disk_set[set_idx], DATA_USAGE_CACHE_NAME).await { + let data_usage_info = cache.dui(DATA_USAGE_ROOT, &Vec::new()); + tracker.objects_total_count = data_usage_info.objects_total_count; + tracker.objects_total_size = data_usage_info.objects_total_size; + }; + + tracker.set_queue_buckets(&buckets).await; + tracker.save().await?; + + let tracker = Arc::new(RwLock::new(tracker)); + let qb = tracker.read().await.queue_buckets.clone(); + store.pools[pool_idx].disk_set[set_idx] + .clone() + .heal_erasure_set(&qb, tracker.clone()) + .await?; + let mut tracker_w = tracker.write().await; + if tracker_w.items_failed > 0 && tracker_w.retry_attempts < 4 { + tracker_w.retry_attempts += 1; + tracker_w.reset_healing().await; + if let Err(err) = tracker_w.update().await { + info!("update tracker failed: {}", err.to_string()); + } + return Err(Error::from_string(ERR_RETRY_HEALING)); + } + + if tracker_w.items_failed > 0 { + info!( + "Healing of drive '{}' is incomplete, retried {} times (healed: {}, skipped: {}, failed: {}).", + disk.to_string(), + tracker_w.retry_attempts, + tracker_w.items_healed, + tracker_w.item_skipped, + tracker_w.items_failed + ); + } else if tracker_w.retry_attempts > 0 { + info!( + "Healing of drive '{}' is incomplete, retried {} times (healed: {}, skipped: {}).", + disk.to_string(), + tracker_w.retry_attempts, + tracker_w.items_healed, + tracker_w.item_skipped + ); + } else { + info!( + "Healing of drive '{}' is finished (healed: {}, skipped: {}).", + disk.to_string(), + tracker_w.items_healed, + tracker_w.item_skipped + ); + } + + if tracker_w.heal_id.is_empty() { + if let Err(err) = tracker_w.delete().await { + error!("delete tracker failed: {}", err.to_string()); + } + } + let layer = new_object_layer_fn(); + let lock = layer.read().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(Error::from(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))), + }; + let disks = store.get_disks(pool_idx, set_idx).await?; + for disk in disks.into_iter() { + if disk.is_none() { + continue; + } + let mut tracker = match load_healing_tracker(&disk).await { + Ok(tracker) => tracker, + Err(err) => { + match err.downcast_ref() { + Some(DiskError::FileNotFound) => {} + _ => { + info!("Unable to load healing tracker on '{:?}': {}, re-initializing..", disk, err.to_string()); + } + } + continue; + } + }; + if tracker.heal_id == tracker_w.heal_id { + tracker.finished = true; + tracker.update().await?; + } + } + Ok(()) +} + +#[derive(Debug)] pub struct HealTask { pub bucket: String, pub object: String, pub version_id: String, pub opts: HealOpts, - pub resp_tx: Arc>, - pub resp_rx: Arc>, + pub resp_tx: Option>, + pub resp_rx: Option>, } impl HealTask { pub fn new(bucket: &str, object: &str, version_id: &str, opts: &HealOpts) -> Self { - let (tx, rx) = mpsc::channel(10); Self { bucket: bucket.to_string(), object: object.to_string(), version_id: version_id.to_string(), - opts: opts.clone(), - resp_tx: tx.into(), - resp_rx: rx.into(), + opts: *opts, + resp_tx: None, + resp_rx: None, } } } pub struct HealResult { pub result: HealResultItem, - err: Error, + pub err: Option, } pub struct HealRoutine { - tasks_tx: Sender, + pub tasks_tx: Sender, tasks_rx: Receiver, workers: usize, } impl HealRoutine { - pub async fn add_worker(&mut self, mut ctx: B_Receiver, bgseq: &HealSequence) { + pub fn new() -> Arc> { + let mut workers = num_cpus::get() / 2; + if let Ok(env_heal_workers) = env::var("_RUSTFS_HEAL_WORKERS") { + if let Ok(num_healers) = env_heal_workers.parse::() { + workers = num_healers; + } + } + + if workers == 0 { + workers = 4; + } + + let (tx, rx) = mpsc::channel(100); + Arc::new(RwLock::new(Self { + tasks_tx: tx, + tasks_rx: rx, + workers, + })) + } + + pub async fn add_worker(&mut self, bgseq: Arc>) { loop { - select! { - task = self.tasks_rx.recv() => { - let mut res = HealResultItem::default(); - let mut err: Error; - match task { - Some(task) => { - if task.bucket == NOP_HEAL { - err = Error::from_string("skip file"); - } else if task.bucket == SLASH_SEPARATOR { - (res, err) = heal_disk_format(task.opts).await; + let mut d_res = HealResultItem::default(); + let d_err: Option; + match self.tasks_rx.recv().await { + Some(task) => { + if task.bucket == NOP_HEAL { + d_err = Some(Error::from_string("skip file")); + } else if task.bucket == SLASH_SEPARATOR { + match heal_disk_format(task.opts).await { + Ok((res, err)) => { + d_res = res; + d_err = err; } - }, - None => return, + Err(err) => d_err = Some(err), + } + } else { + let layer = new_object_layer_fn(); + let lock = layer.read().await; + let store = lock.as_ref().expect("Not init"); + if task.object.is_empty() { + match store.heal_bucket(&task.bucket, &task.opts).await { + Ok(res) => { + d_res = res; + d_err = None; + } + Err(err) => d_err = Some(err), + } + } else { + match store + .heal_object(&task.bucket, &task.object, &task.version_id, &task.opts) + .await + { + Ok((res, err)) => { + d_res = res; + d_err = err; + } + Err(err) => d_err = Some(err), + } + } + } + if let Some(resp_tx) = task.resp_tx { + let _ = resp_tx + .send(HealResult { + result: d_res, + err: d_err, + }) + .await; + } else { + // when respCh is not set caller is not waiting but we + // update the relevant metrics for them + if d_err.is_none() { + bgseq.write().await.count_healed(d_res.heal_item_type); + } else { + bgseq.write().await.count_failed(d_res.heal_item_type); + } } } - _ = ctx.recv() => { - return; - } + None => return, } } } @@ -80,6 +428,15 @@ impl HealRoutine { // } -async fn heal_disk_format(opts: HealOpts) -> (HealResultItem, Error) { - todo!() +async fn heal_disk_format(opts: HealOpts) -> Result<(HealResultItem, Option)> { + let layer = new_object_layer_fn(); + let lock = layer.read().await; + let store = lock.as_ref().expect("Not init"); + let (res, err) = store.heal_format(opts.dry_run).await?; + // return any error, ignore error returned when disks have + // already healed. + if err.is_some() { + return Ok((HealResultItem::default(), err)); + } + Ok((res, err)) } diff --git a/ecstore/src/heal/data_scanner.rs b/ecstore/src/heal/data_scanner.rs new file mode 100644 index 000000000..c6b1072e5 --- /dev/null +++ b/ecstore/src/heal/data_scanner.rs @@ -0,0 +1,1086 @@ +use std::{ + collections::{HashMap, HashSet}, + fs, + future::Future, + io::{Cursor, Read}, + path::{Path, PathBuf}, + pin::Pin, + sync::{ + atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, + Arc, + }, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; +use lazy_static::lazy_static; +use rand::Rng; +use rmp_serde::{Deserializer, Serializer}; +use s3s::dto::{ReplicationConfiguration, ReplicationRuleStatus}; +use serde::{Deserialize, Serialize}; +use tokio::{ + sync::{ + broadcast, + mpsc::{self, Sender}, + RwLock, + }, + time::sleep, +}; +use tracing::{error, info}; + +use super::{ + data_scanner_metric::{globalScannerMetrics, ScannerMetric, ScannerMetrics}, + data_usage::{store_data_usage_in_backend, DATA_USAGE_BLOOM_NAME_PATH}, + data_usage_cache::{DataUsageCache, DataUsageEntry, DataUsageHash}, + heal_commands::{HealScanMode, HEAL_DEEP_SCAN, HEAL_NORMAL_SCAN}, +}; +use crate::heal::data_usage::DATA_USAGE_ROOT; +use crate::{ + cache_value::metacache_set::{list_path_raw, ListPathRawOptions}, + config::{ + common::{read_config, save_config}, + heal::Config, + }, + disk::{error::DiskError, DiskInfoOptions, DiskStore, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}, + error::{Error, Result}, + global::{GLOBAL_BackgroundHealState, GLOBAL_IsErasure, GLOBAL_IsErasureSD}, + heal::{ + data_usage::BACKGROUND_HEAL_INFO_PATH, + data_usage_cache::{hash_path, DataUsageHashMap}, + error::ERR_IGNORE_FILE_CONTRIB, + heal_commands::{HEAL_ITEM_BUCKET, HEAL_ITEM_OBJECT}, + heal_ops::{HealSource, BG_HEALING_UUID}, + }, + new_object_layer_fn, + peer::is_reserved_or_invalid_bucket, + store::ECStore, + utils::path::{path_join, path_to_bucket_object, path_to_bucket_object_with_base_path, SLASH_SEPARATOR}, +}; +use crate::{disk::local::LocalDisk, heal::data_scanner_metric::current_path_updater}; +use crate::{ + disk::DiskAPI, + store_api::{FileInfo, ObjectInfo}, +}; + +const _DATA_SCANNER_SLEEP_PER_FOLDER: Duration = Duration::from_millis(1); // Time to wait between folders. +const DATA_USAGE_UPDATE_DIR_CYCLES: u32 = 16; // Visit all folders every n cycles. +const DATA_SCANNER_COMPACT_LEAST_OBJECT: u64 = 500; // Compact when there are less than this many objects in a branch. +const DATA_SCANNER_COMPACT_AT_CHILDREN: u64 = 10000; // Compact when there are this many children in a branch. +const DATA_SCANNER_COMPACT_AT_FOLDERS: u64 = DATA_SCANNER_COMPACT_AT_CHILDREN / 4; // Compact when this many subfolders in a single folder. +pub const DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS: u64 = 250_000; // Compact when this many subfolders in a single folder (even top level). +const DATA_SCANNER_START_DELAY: Duration = Duration::from_secs(60); // Time to wait on startup and between cycles. + +pub const HEAL_DELETE_DANGLING: bool = true; +const HEAL_OBJECT_SELECT_PROB: u64 = 1024; // Overall probability of a file being scanned; one in n. + +// static SCANNER_SLEEPER: () = new_dynamic_sleeper(2, Duration::from_secs(1), true); // Keep defaults same as config defaults +static SCANNER_CYCLE: AtomicU64 = AtomicU64::new(DATA_SCANNER_START_DELAY.as_secs()); +static _SCANNER_IDLE_MODE: AtomicU32 = AtomicU32::new(0); // default is throttled when idle +static SCANNER_EXCESS_OBJECT_VERSIONS: AtomicU64 = AtomicU64::new(100); +static SCANNER_EXCESS_OBJECT_VERSIONS_TOTAL_SIZE: AtomicU64 = AtomicU64::new(1024 * 1024 * 1024 * 1024); // 1 TB +static SCANNER_EXCESS_FOLDERS: AtomicU64 = AtomicU64::new(50_000); + +lazy_static! { + pub static ref globalHealConfig: Arc> = Arc::new(RwLock::new(Config::default())); +} + +pub async fn init_data_scanner() { + let mut r = rand::thread_rng(); + let random = r.gen_range(0.0..1.0); + tokio::spawn(async move { + loop { + run_data_scanner().await; + let duration = Duration::from_secs_f64(random * (SCANNER_CYCLE.load(std::sync::atomic::Ordering::SeqCst) as f64)); + let sleep_duration = if duration < Duration::new(1, 0) { + Duration::new(1, 0) + } else { + duration + }; + sleep(sleep_duration).await; + } + }); +} + +async fn run_data_scanner() { + let mut cycle_info = CurrentScannerCycle::default(); + let layer = new_object_layer_fn(); + let lock = layer.read().await; + let store = match lock.as_ref() { + Some(s) => s, + None => { + info!("errServerNotInitialized"); + return; + } + }; + let mut buf = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH) + .await + .map_or(Vec::new(), |buf| buf); + match buf.len().cmp(&8) { + std::cmp::Ordering::Less => {} + std::cmp::Ordering::Equal => { + cycle_info.next = match Cursor::new(buf).read_u64::() { + Ok(buf) => buf, + Err(_) => { + error!("can not decode DATA_USAGE_BLOOM_NAME_PATH"); + return; + } + }; + } + std::cmp::Ordering::Greater => { + cycle_info.next = match Cursor::new(buf[..8].to_vec()).read_u64::() { + Ok(buf) => buf, + Err(_) => { + error!("can not decode DATA_USAGE_BLOOM_NAME_PATH"); + return; + } + }; + let _ = cycle_info.unmarshal_msg(&buf.split_off(8)); + } + } + + loop { + let stop_fn = ScannerMetrics::log(ScannerMetric::ScanCycle); + cycle_info.current = cycle_info.next; + cycle_info.started = SystemTime::now(); + { + globalScannerMetrics.write().await.set_cycle(Some(cycle_info.clone())).await; + } + + let bg_heal_info = read_background_heal_info(store).await; + let scan_mode = + get_cycle_scan_mode(cycle_info.current, bg_heal_info.bitrot_start_cycle, bg_heal_info.bitrot_start_time).await; + if bg_heal_info.current_scan_mode != scan_mode { + let mut new_heal_info = bg_heal_info; + new_heal_info.current_scan_mode = scan_mode; + if scan_mode == HEAL_DEEP_SCAN { + new_heal_info.bitrot_start_time = SystemTime::now(); + new_heal_info.bitrot_start_cycle = cycle_info.current; + } + save_background_heal_info(store, &new_heal_info).await; + } + // Wait before starting next cycle and wait on startup. + let (tx, rx) = mpsc::channel(100); + tokio::spawn(async { + store_data_usage_in_backend(rx).await; + }); + let mut res = HashMap::new(); + res.insert("cycle".to_string(), cycle_info.current.to_string()); + match store.ns_scanner(tx, cycle_info.current as usize, scan_mode).await { + Ok(_) => { + cycle_info.next += 1; + cycle_info.current = 0; + cycle_info.cycle_completed.push(SystemTime::now()); + if cycle_info.cycle_completed.len() > DATA_USAGE_UPDATE_DIR_CYCLES as usize { + cycle_info.cycle_completed = cycle_info.cycle_completed + [cycle_info.cycle_completed.len() - DATA_USAGE_UPDATE_DIR_CYCLES as usize..] + .to_vec(); + } + globalScannerMetrics.write().await.set_cycle(Some(cycle_info.clone())).await; + let mut tmp = Vec::new(); + tmp.write_u64::(cycle_info.next).unwrap(); + let _ = save_config(store, &DATA_USAGE_BLOOM_NAME_PATH, &tmp).await; + } + Err(err) => { + res.insert("error".to_string(), err.to_string()); + } + } + stop_fn(&res).await; + sleep(Duration::from_secs(SCANNER_CYCLE.load(std::sync::atomic::Ordering::SeqCst))).await; + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct BackgroundHealInfo { + bitrot_start_time: SystemTime, + bitrot_start_cycle: u64, + current_scan_mode: HealScanMode, +} + +impl Default for BackgroundHealInfo { + fn default() -> Self { + Self { + bitrot_start_time: SystemTime::now(), + bitrot_start_cycle: Default::default(), + current_scan_mode: Default::default(), + } + } +} + +async fn read_background_heal_info(store: &ECStore) -> BackgroundHealInfo { + if *GLOBAL_IsErasureSD.read().await { + return BackgroundHealInfo::default(); + } + + let buf = read_config(store, &BACKGROUND_HEAL_INFO_PATH) + .await + .map_or(Vec::new(), |buf| buf); + if buf.is_empty() { + return BackgroundHealInfo::default(); + } + serde_json::from_slice::(&buf).map_or(BackgroundHealInfo::default(), |b| b) +} + +async fn save_background_heal_info(store: &ECStore, info: &BackgroundHealInfo) { + if *GLOBAL_IsErasureSD.read().await { + return; + } + let b = match serde_json::to_vec(info) { + Ok(info) => info, + Err(_) => return, + }; + let _ = save_config(store, &BACKGROUND_HEAL_INFO_PATH, &b).await; +} + +async fn get_cycle_scan_mode(current_cycle: u64, bitrot_start_cycle: u64, bitrot_start_time: SystemTime) -> HealScanMode { + let bitrot_cycle = globalHealConfig.read().await.bitrot_scan_cycle(); + let v = bitrot_cycle.as_secs_f64(); + if v == -1.0 { + return HEAL_NORMAL_SCAN; + } else if v == 0.0 { + return HEAL_DEEP_SCAN; + } + + if current_cycle - bitrot_start_cycle < HEAL_OBJECT_SELECT_PROB { + return HEAL_DEEP_SCAN; + } + + if bitrot_start_time.duration_since(SystemTime::now()).unwrap() > bitrot_cycle { + return HEAL_DEEP_SCAN; + } + + HEAL_NORMAL_SCAN +} + +#[derive(Clone, Debug)] +pub struct CurrentScannerCycle { + pub current: u64, + pub next: u64, + pub started: SystemTime, + pub cycle_completed: Vec, +} + +impl Default for CurrentScannerCycle { + fn default() -> Self { + Self { + current: Default::default(), + next: Default::default(), + started: SystemTime::now(), + cycle_completed: Default::default(), + } + } +} + +impl CurrentScannerCycle { + pub fn marshal_msg(&self, next_buf: &[u8]) -> Result> { + let len: u32 = 4; + let mut wr = Vec::new(); + + // 字段数量 + rmp::encode::write_map_len(&mut wr, len)?; + + // write "current" + rmp::encode::write_str(&mut wr, "current")?; + rmp::encode::write_uint(&mut wr, self.current)?; + + // write "next" + rmp::encode::write_str(&mut wr, "next")?; + rmp::encode::write_uint(&mut wr, self.next)?; + + // write "started" + rmp::encode::write_str(&mut wr, "started")?; + rmp::encode::write_uint(&mut wr, system_time_to_timestamp(&self.started))?; + + // write "cycle_completed" + rmp::encode::write_str(&mut wr, "cycle_completed")?; + let mut buf = Vec::new(); + self.cycle_completed + .serialize(&mut Serializer::new(&mut buf)) + .expect("Serialization failed"); + rmp::encode::write_bin(&mut wr, &buf)?; + let mut result = next_buf.to_vec(); + result.extend(wr.iter()); + Ok(result) + } + + #[tracing::instrument] + pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result { + let mut cur = Cursor::new(buf); + + let mut fields_len = rmp::decode::read_map_len(&mut cur)?; + + while fields_len > 0 { + fields_len -= 1; + + let str_len = rmp::decode::read_str_len(&mut cur)?; + + // ！！！ Vec::with_capacity(str_len) 失败，vec!正常 + let mut field_buff = vec![0u8; str_len as usize]; + + cur.read_exact(&mut field_buff)?; + + let field = String::from_utf8(field_buff)?; + + match field.as_str() { + "current" => { + let u: u64 = rmp::decode::read_int(&mut cur)?; + self.current = u; + } + + // "next" => { + // let u: u64 = rmp::decode::read_int(&mut cur)?; + // self.next = u; + // } + "started" => { + let u: u64 = rmp::decode::read_int(&mut cur)?; + let started = timestamp_to_system_time(u); + self.started = started; + } + "cycleCompleted" => { + let mut buf = Vec::new(); + let _ = cur.read_to_end(&mut buf)?; + let u: Vec = + Deserialize::deserialize(&mut Deserializer::new(&buf[..])).expect("Deserialization failed"); + self.cycle_completed = u; + } + name => return Err(Error::msg(format!("not suport field name {}", name))), + } + } + + Ok(cur.position()) + } +} + +// 将 SystemTime 转换为时间戳 +fn system_time_to_timestamp(time: &SystemTime) -> u64 { + time.duration_since(UNIX_EPOCH).expect("Time went backwards").as_secs() +} + +// 将时间戳转换为 SystemTime +fn timestamp_to_system_time(timestamp: u64) -> SystemTime { + UNIX_EPOCH + std::time::Duration::new(timestamp, 0) +} + +#[derive(Clone, Debug, Default)] +pub struct Heal { + enabled: bool, + bitrot: bool, +} + +#[derive(Clone)] +pub struct ScannerItem { + pub path: String, + pub bucket: String, + pub prefix: String, + pub object_name: String, + pub replication: Option, + // todo: lifecycle + // typ: fs::Permissions, + pub heal: Heal, + pub debug: bool, +} + +impl ScannerItem { + pub fn transform_meda_dir(&mut self) { + let split = self.prefix.split(SLASH_SEPARATOR).map(PathBuf::from).collect::>(); + if split.len() > 1 { + self.prefix = path_join(&split[0..split.len() - 1]).to_string_lossy().to_string(); + } else { + self.prefix = "".to_string(); + } + self.object_name = split.last().map_or("".to_string(), |v| v.to_string_lossy().to_string()); + } + + pub fn object_path(&self) -> PathBuf { + path_join(&[PathBuf::from(self.prefix.clone()), PathBuf::from(self.object_name.clone())]) + } + + pub async fn apply_versions_actions(&self, fivs: &[FileInfo]) -> Result> { + let obj_infos = self.apply_newer_noncurrent_version_limit(fivs).await?; + if obj_infos.len() >= SCANNER_EXCESS_OBJECT_VERSIONS.load(Ordering::SeqCst).try_into().unwrap() { + // todo + } + + let mut cumulative_size = 0; + for obj_info in obj_infos.iter() { + cumulative_size += obj_info.size; + } + + if cumulative_size + >= SCANNER_EXCESS_OBJECT_VERSIONS_TOTAL_SIZE + .load(Ordering::SeqCst) + .try_into() + .unwrap() + { + //todo + } + + Ok(obj_infos) + } + + pub async fn apply_newer_noncurrent_version_limit(&self, fivs: &[FileInfo]) -> Result> { + let done = ScannerMetrics::time(ScannerMetric::ApplyNonCurrent); + let mut object_infos = Vec::new(); + for info in fivs.iter() { + object_infos.push(info.to_object_info(&self.bucket, &self.object_path().to_string_lossy(), false)); + } + done().await; + + Ok(object_infos) + } + + pub async fn apply_actions(&self, _oi: &ObjectInfo, _size_s: &SizeSummary) -> (bool, usize) { + let done = ScannerMetrics::time(ScannerMetric::Ilm); + //todo: lifecycle + done().await; + + (false, 0) + } +} + +#[derive(Debug, Default)] +pub struct SizeSummary { + pub total_size: usize, + pub versions: usize, + pub delete_markers: usize, + pub replicated_size: usize, + pub replicated_count: usize, + pub pending_size: usize, + pub failed_size: usize, + pub replica_size: usize, + pub replica_count: usize, + pub pending_count: usize, + pub failed_count: usize, + pub repl_target_stats: HashMap, + // Todo: tires +} + +#[derive(Debug, Default)] +pub struct ReplTargetSizeSummary { + pub replicated_size: usize, + pub replicated_count: usize, + pub pending_size: usize, + pub failed_size: usize, + pub pending_count: usize, + pub failed_count: usize, +} + +#[derive(Debug, Clone)] +struct CachedFolder { + name: String, + parent: DataUsageHash, + object_heal_prob_div: u32, +} + +pub type GetSizeFn = + Box Pin> + Send>> + Send + Sync + 'static>; +pub type UpdateCurrentPathFn = Arc Pin + Send>> + Send + Sync + 'static>; + +struct FolderScanner { + root: String, + get_size: GetSizeFn, + old_cache: DataUsageCache, + new_cache: DataUsageCache, + update_cache: DataUsageCache, + data_usage_scanner_debug: bool, + heal_object_select: u32, + scan_mode: HealScanMode, + disks: Vec>, + disks_quorum: usize, + updates: Sender, + last_update: SystemTime, + update_current_path: UpdateCurrentPathFn, + skip_heal: AtomicBool, + drive: LocalDrive, +} + +impl FolderScanner { + async fn should_heal(&self) -> bool { + if self.skip_heal.load(Ordering::SeqCst) { + return false; + } + if self.heal_object_select == 0 { + return false; + } + if let Ok(info) = self.drive.disk_info(&DiskInfoOptions::default()).await { + if info.healing { + self.skip_heal.store(true, Ordering::SeqCst); + return false; + } + } + true + } + + async fn scan_folder(&mut self, folder: &CachedFolder, into: &mut DataUsageEntry) -> Result<()> { + let this_hash = hash_path(&folder.name); + let was_compacted = into.compacted; + + // do not really loop + let mut times = 1; + while times > 0 { + let mut abandoned_children: DataUsageHashMap = if !into.compacted { + self.old_cache.find_children_copy(this_hash.clone()) + } else { + HashSet::new() + }; + + let (_, prefix) = path_to_bucket_object_with_base_path(&self.root, &folder.name); + // Todo: lifeCycle + let replication_cfg = if self.old_cache.info.replication.is_some() + && has_active_rules(self.old_cache.info.replication.as_ref().unwrap(), &prefix, true) + { + self.old_cache.info.replication.clone() + } else { + None + }; + + let mut existing_folders = Vec::new(); + let mut new_folders = Vec::new(); + let mut found_objects: bool = false; + + let path = Path::new(&self.root).join(&folder.name); + if path.is_dir() { + for entry in fs::read_dir(path)? { + let entry = entry?; + let sub_path = entry.path(); + let ent_name = Path::new(&folder.name).join(&sub_path); + let (bucket, prefix) = path_to_bucket_object_with_base_path(&self.root, ent_name.to_str().unwrap()); + if bucket.is_empty() { + continue; + } + if is_reserved_or_invalid_bucket(&bucket, false) { + continue; + } + + if !sub_path.is_dir() { + let h = hash_path(ent_name.to_str().unwrap()); + if h == this_hash { + continue; + } + let this = CachedFolder { + name: ent_name.to_string_lossy().to_string(), + parent: this_hash.clone(), + object_heal_prob_div: folder.object_heal_prob_div, + }; + abandoned_children.remove(&h.key()); + if self.old_cache.cache.contains_key(&h.key()) { + existing_folders.push(this); + self.update_cache + .copy_with_children(&self.old_cache, &h, &Some(this_hash.clone())); + } else { + new_folders.push(this); + } + continue; + } + + let mut item = ScannerItem { + path: Path::new(&self.root).join(&ent_name).to_string_lossy().to_string(), + bucket, + prefix: Path::new(&prefix) + .parent() + .unwrap_or(Path::new("")) + .to_string_lossy() + .to_string(), + object_name: ent_name + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(), + debug: self.data_usage_scanner_debug, + replication: replication_cfg.clone(), + heal: Heal::default(), + }; + + item.heal.enabled = this_hash.mod_alt( + self.old_cache.info.next_cycle / folder.object_heal_prob_div, + self.heal_object_select / folder.object_heal_prob_div, + ) && self.should_heal().await; + item.heal.bitrot = self.scan_mode == HEAL_DEEP_SCAN; + + let (sz, err) = match (self.get_size)(&item).await { + Ok(sz) => (sz, None), + Err(err) => { + if err.to_string() != ERR_IGNORE_FILE_CONTRIB { + continue; + } + (SizeSummary::default(), Some(err)) + } + }; + // successfully read means we have a valid object. + found_objects = true; + // Remove filename i.e is the meta file to construct object name + item.transform_meda_dir(); + // Object already accounted for, remove from heal map, + // simply because getSize() function already heals the + // object. + abandoned_children.remove( + &path_join(&[PathBuf::from(item.bucket.clone()), item.object_path()]) + .to_string_lossy() + .to_string(), + ); + + if err.is_none() || err.unwrap().to_string() != ERR_IGNORE_FILE_CONTRIB { + into.add_sizes(&sz); + into.objects += 1; + } + } + } + if found_objects && *GLOBAL_IsErasure.read().await { + // If we found an object in erasure mode, we skip subdirs (only datadirs)... + break; + } + + let should_compact = self.new_cache.info.name != folder.name + && existing_folders.len() + new_folders.len() >= DATA_SCANNER_COMPACT_AT_FOLDERS.try_into().unwrap() + || existing_folders.len() + new_folders.len() >= DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS.try_into().unwrap(); + + let total_folders = existing_folders.len() + new_folders.len(); + if total_folders + > SCANNER_EXCESS_FOLDERS + .load(std::sync::atomic::Ordering::SeqCst) + .try_into() + .unwrap() + { + let _prefix_name = format!("{}/", folder.name.trim_end_matches('/')); + // todo: notification + } + + if !into.compacted && should_compact { + into.compacted = true; + new_folders.extend(existing_folders.clone()); + existing_folders.clear(); + } + + // Transfer existing + if !into.compacted { + for folder in existing_folders.iter() { + let h = hash_path(&folder.name); + self.update_cache + .copy_with_children(&self.old_cache, &h, &Some(folder.parent.clone())); + } + } + + // Scan new... + for folder in new_folders.iter() { + let h = hash_path(&folder.name); + if !into.compacted { + let mut found_any = false; + let mut parent = this_hash.clone(); + while parent != hash_path(&self.update_cache.info.name) { + let e = self.update_cache.find(&parent.key()); + if e.is_none() || e.as_ref().unwrap().compacted { + found_any = true; + break; + } + match self.update_cache.search_parent(&parent) { + Some(next) => { + parent = next; + } + None => { + found_any = true; + break; + } + } + } + if !found_any { + self.update_cache + .replace_hashed(&h, &Some(this_hash.clone()), &DataUsageEntry::default()); + } + } + (self.update_current_path)(&folder.name).await; + scan(folder, into, self).await; + // Add new folders if this is new and we don't have existing. + if !into.compacted { + if let Some(parent) = self.update_cache.find(&this_hash.key()) { + if !parent.compacted { + self.update_cache.delete_recursive(&h); + self.update_cache + .copy_with_children(&self.new_cache, &h, &Some(this_hash.clone())); + } + } + } + } + + // Scan existing... + for folder in existing_folders.iter() { + let h = hash_path(&folder.name); + if !into.compacted + && self.old_cache.is_compacted(&h) + && !h.mod_(self.old_cache.info.next_cycle, DATA_USAGE_UPDATE_DIR_CYCLES) + { + self.new_cache + .copy_with_children(&self.old_cache, &h, &Some(folder.parent.clone())); + into.add_child(&h); + continue; + } + (self.update_current_path)(&folder.name).await; + scan(folder, into, self).await; + } + + // Scan for healing + if abandoned_children.is_empty() || !self.should_heal().await { + break; + } + + if self.disks.is_empty() || self.disks_quorum == 0 { + break; + } + + let (bg_seq, found) = GLOBAL_BackgroundHealState + .read() + .await + .get_heal_sequence_by_token(BG_HEALING_UUID) + .await; + if !found { + break; + } + let bg_seq = bg_seq.unwrap(); + + let mut resolver = MetadataResolutionParams { + dir_quorum: self.disks_quorum, + obj_quorum: self.disks_quorum, + bucket: "".to_string(), + strict: false, + ..Default::default() + }; + + for k in abandoned_children.iter() { + if !self.should_heal().await { + break; + } + + let (bucket, prefix) = path_to_bucket_object(k); + (self.update_current_path)(k).await; + + if bucket != resolver.bucket { + bg_seq + .clone() + .write() + .await + .queue_heal_task( + HealSource { + bucket: bucket.clone(), + ..Default::default() + }, + HEAL_ITEM_BUCKET.to_owned(), + ) + .await?; + } + + resolver.bucket = bucket.clone(); + let found_objs = Arc::new(RwLock::new(false)); + let found_objs_clone = found_objs.clone(); + let (tx, rx) = broadcast::channel(1); + // let tx_partial = tx.clone(); + let tx_finished = tx.clone(); + let update_current_path_agreed = self.update_current_path.clone(); + let update_current_path_partial = self.update_current_path.clone(); + let resolver_clone = resolver.clone(); + let bg_seq_clone = bg_seq.clone(); + let lopts = ListPathRawOptions { + disks: self.disks.clone(), + bucket: bucket.clone(), + path: prefix.clone(), + recursice: true, + report_not_found: true, + min_disks: self.disks_quorum, + agreed: Some(Box::new(move |entry: MetaCacheEntry| { + Box::pin({ + let update_current_path_agreed = update_current_path_agreed.clone(); + async move { + update_current_path_agreed(&entry.name).await; + } + }) + })), + partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option]| { + Box::pin({ + let update_current_path_partial = update_current_path_partial.clone(); + // let tx_partial = tx_partial.clone(); + let resolver_partial = resolver_clone.clone(); + let bucket_partial = bucket.clone(); + let found_objs_clone = found_objs_clone.clone(); + let bg_seq_partial = bg_seq_clone.clone(); + async move { + // Todo + // if !fs.should_heal().await { + // let _ = tx_partial.send(true); + // return; + // } + let entry = match entries.resolve(resolver_partial) { + Ok(Some(entry)) => entry, + _ => match entries.first_found() { + (Some(entry), _) => entry, + _ => return, + }, + }; + + update_current_path_partial(&entry.name).await; + let mut custom = HashMap::new(); + if entry.is_dir() { + return; + } + + // We got an entry which we should be able to heal. + let fiv = match entry.file_info_versions(&bucket_partial) { + Ok(fiv) => fiv, + Err(_) => { + if let Err(err) = bg_seq_partial + .write() + .await + .queue_heal_task( + HealSource { + bucket: bucket_partial.clone(), + object: entry.name.clone(), + version_id: "".to_string(), + ..Default::default() + }, + HEAL_ITEM_OBJECT.to_string(), + ) + .await + { + match err.downcast_ref() { + Some(DiskError::FileNotFound) | Some(DiskError::FileVersionNotFound) => {} + _ => { + info!("{}", err.to_string()); + } + } + } else { + let mut w = found_objs_clone.write().await; + *w = true; + } + return; + } + }; + + custom.insert("versions", fiv.versions.len().to_string()); + let (mut success_versions, mut fail_versions) = (0, 0); + for ver in fiv.versions.iter() { + match bg_seq_partial + .write() + .await + .queue_heal_task( + HealSource { + bucket: bucket_partial.clone(), + object: fiv.name.clone(), + version_id: ver.version_id.map_or("".to_string(), |ver_id| ver_id.to_string()), + ..Default::default() + }, + HEAL_ITEM_OBJECT.to_string(), + ) + .await + { + Ok(_) => { + success_versions += 1; + + let mut w = found_objs_clone.write().await; + *w = true; + } + Err(_) => { + fail_versions += 1; + } + } + } + custom.insert("success_versions", success_versions.to_string()); + custom.insert("failed_versions", fail_versions.to_string()); + } + }) + })), + finished: Some(Box::new(move |_: &[Option]| { + Box::pin({ + let tx_finished = tx_finished.clone(); + async move { + let _ = tx_finished.send(true); + } + }) + })), + ..Default::default() + }; + let _ = list_path_raw(rx, lopts).await; + + if *found_objs.read().await { + let this = CachedFolder { + name: k.clone(), + parent: this_hash.clone(), + object_heal_prob_div: 1, + }; + scan(&this, into, self).await; + } + } + times -= 1; + } + if !was_compacted { + self.new_cache.replace_hashed(&this_hash, &Some(folder.parent.clone()), into); + } + + if !into.compacted && self.new_cache.info.name != folder.name { + let mut flat = self.new_cache.size_recursive(&this_hash.key()).unwrap_or_default(); + flat.compacted = true; + let compact = if flat.objects < DATA_SCANNER_COMPACT_LEAST_OBJECT.try_into().unwrap() { + true + } else { + // Compact if we only have objects as children... + let mut compact = true; + for k in into.children.iter() { + if let Some(v) = self.new_cache.cache.get(k) { + if !v.children.is_empty() || v.objects > 1 { + compact = false; + break; + } + } + } + compact + }; + if compact { + self.new_cache.delete_recursive(&this_hash); + self.new_cache.replace_hashed(&this_hash, &Some(folder.parent.clone()), &flat); + let mut total: HashMap = HashMap::new(); + total.insert("objects".to_string(), flat.objects.to_string()); + total.insert("size".to_string(), flat.size.to_string()); + if flat.versions > 0 { + total.insert("versions".to_string(), flat.versions.to_string()); + } + } + } + // Compact if too many children... + if !into.compacted { + self.new_cache.reduce_children_of( + &this_hash, + DATA_SCANNER_COMPACT_AT_CHILDREN.try_into().unwrap(), + self.new_cache.info.name != folder.name, + ); + } + if self.update_cache.cache.contains_key(&this_hash.key()) && !was_compacted { + // Replace if existed before. + if let Some(flat) = self.new_cache.size_recursive(&this_hash.key()) { + self.update_cache.delete_recursive(&this_hash); + self.update_cache + .replace_hashed(&this_hash, &Some(folder.parent.clone()), &flat); + } + } + Ok(()) + } + + async fn send_update(&mut self) { + if SystemTime::now().duration_since(self.last_update).unwrap() < Duration::from_secs(60) { + return; + } + if let Some(flat) = self.update_cache.size_recursive(&self.new_cache.info.name) { + let _ = self.updates.send(flat).await; + self.last_update = SystemTime::now(); + } + } +} + +async fn scan(folder: &CachedFolder, into: &mut DataUsageEntry, folder_scanner: &mut FolderScanner) { + let mut dst = if !into.compacted { + DataUsageEntry::default() + } else { + into.clone() + }; + + if Box::pin(folder_scanner.scan_folder(folder, &mut dst)).await.is_err() { + return; + } + if !into.compacted { + let h = DataUsageHash(folder.name.clone()); + into.add_child(&h); + folder_scanner.update_cache.delete_recursive(&h); + folder_scanner + .update_cache + .copy_with_children(&folder_scanner.new_cache, &h, &Some(folder.parent.clone())); + folder_scanner.send_update().await; + } +} + +pub fn has_active_rules(config: &ReplicationConfiguration, prefix: &str, recursive: bool) -> bool { + if config.rules.is_empty() { + return false; + } + + for rule in config.rules.iter() { + if rule + .status + .eq(&ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED)) + { + continue; + } + if !prefix.is_empty() { + if let Some(filter) = &rule.filter { + if let Some(r_prefix) = &filter.prefix { + if !r_prefix.is_empty() { + // incoming prefix must be in rule prefix + if !recursive && !prefix.starts_with(r_prefix) { + continue; + } + // If recursive, we can skip this rule if it doesn't match the tested prefix or level below prefix + // does not match + if recursive && !r_prefix.starts_with(prefix) && !prefix.starts_with(r_prefix) { + continue; + } + } + } + } + } + return true; + } + false +} + +pub type LocalDrive = Arc; +pub async fn scan_data_folder( + disks: &[Option], + drive: LocalDrive, + cache: &DataUsageCache, + get_size_fn: GetSizeFn, + heal_scan_mode: HealScanMode, +) -> Result { + if cache.info.name.is_empty() || cache.info.name == DATA_USAGE_ROOT { + return Err(Error::from_string("internal error: root scan attempted")); + } + + let base_path = drive.to_string(); + let (update_path, close_disk) = current_path_updater(&base_path, &cache.info.name); + let skip_heal = if *GLOBAL_IsErasure.read().await || cache.info.skip_healing { + AtomicBool::new(true) + } else { + AtomicBool::new(false) + }; + let mut s = FolderScanner { + root: base_path, + get_size: get_size_fn, + old_cache: cache.clone(), + new_cache: DataUsageCache::default(), + update_cache: DataUsageCache::default(), + data_usage_scanner_debug: false, + heal_object_select: 0, + scan_mode: heal_scan_mode, + updates: cache.info.updates.clone().unwrap(), + last_update: SystemTime::now(), + update_current_path: update_path, + disks: disks.to_vec(), + disks_quorum: disks.len() / 2, + skip_heal, + drive: drive.clone(), + }; + + if *GLOBAL_IsErasure.read().await || !cache.info.skip_healing { + s.heal_object_select = HEAL_OBJECT_SELECT_PROB as u32; + } + + let mut root = DataUsageEntry::default(); + let folder = CachedFolder { + name: cache.info.name.clone(), + object_heal_prob_div: 1, + parent: DataUsageHash("".to_string()), + }; + + if s.scan_folder(&folder, &mut root).await.is_err() { + close_disk().await; + } + s.new_cache + .force_compact(DATA_SCANNER_COMPACT_AT_CHILDREN.try_into().unwrap()); + s.new_cache.info.last_update = Some(SystemTime::now()); + s.new_cache.info.next_cycle = cache.info.next_cycle; + close_disk().await; + Ok(s.new_cache) +} + +// pub fn eval_action_from_lifecycle(lc: &BucketLifecycleConfiguration, lr: &ObjectLockConfiguration, rcfg: &ReplicationConfiguration， obj: &ObjectInfo) diff --git a/ecstore/src/heal/data_scanner_metric.rs b/ecstore/src/heal/data_scanner_metric.rs new file mode 100644 index 000000000..d45b372df --- /dev/null +++ b/ecstore/src/heal/data_scanner_metric.rs @@ -0,0 +1,223 @@ +use common::last_minute::{AccElem, LastMinuteLatency}; +use lazy_static::lazy_static; +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::AtomicU64; +use std::sync::Once; +use std::time::{Duration, UNIX_EPOCH}; +use std::{ + collections::HashMap, + sync::{ + atomic::{AtomicU32, Ordering}, + Arc, + }, + time::SystemTime, +}; +use tokio::sync::RwLock; + +use super::data_scanner::{CurrentScannerCycle, UpdateCurrentPathFn}; + +lazy_static! { + pub static ref globalScannerMetrics: Arc> = Arc::new(RwLock::new(ScannerMetrics::new())); +} + +#[derive(Clone, Debug, PartialEq, PartialOrd)] +pub enum ScannerMetric { + // START Realtime metrics, that only to records + // last minute latencies and total operation count. + ReadMetadata = 0, + CheckMissing, + SaveUsage, + ApplyAll, + ApplyVersion, + TierObjSweep, + HealCheck, + Ilm, + CheckReplication, + Yield, + CleanAbandoned, + ApplyNonCurrent, + HealAbandonedVersion, + + // START Trace metrics: + StartTrace, + ScanObject, // Scan object. All operations included. + HealAbandonedObject, + + // END realtime metrics: + LastRealtime, + + // Trace only metrics: + ScanFolder, // Scan a folder on disk, recursively. + ScanCycle, // Full cycle, cluster global. + ScanBucketDrive, // Single bucket on one drive. + CompactFolder, // Folder compacted. + + // Must be last: + Last, +} + +static INIT: Once = Once::new(); + +#[derive(Default)] +pub struct LockedLastMinuteLatency { + cached_sec: AtomicU64, + cached: AccElem, + mu: RwLock, + latency: LastMinuteLatency, +} + +impl Clone for LockedLastMinuteLatency { + fn clone(&self) -> Self { + Self { + cached_sec: AtomicU64::new(0), + cached: self.cached.clone(), + mu: RwLock::new(true), + latency: self.latency.clone(), + } + } +} + +impl LockedLastMinuteLatency { + pub async fn add(&mut self, value: &Duration) { + self.add_size(value, 0).await; + } + + pub async fn add_size(&mut self, value: &Duration, sz: u64) { + let t = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs(); + INIT.call_once(|| { + self.cached = AccElem::default(); + self.cached_sec.store(t, Ordering::SeqCst); + }); + let last_t = self.cached_sec.load(Ordering::SeqCst); + if last_t != t + && self + .cached_sec + .compare_exchange(last_t, t, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + let old = self.cached.clone(); + self.cached = AccElem::default(); + let a = AccElem { + size: old.size, + total: old.total, + n: old.n, + }; + let _ = self.mu.write().await; + self.latency.add_all(t - 1, &a); + } + self.cached.n += 1; + self.cached.total += value.as_secs(); + self.cached.size += sz; + } + + pub async fn total(&mut self) -> AccElem { + let _ = self.mu.read().await; + self.latency.get_total() + } +} + +pub type LogFn = Arc) -> Pin + Send>> + Send + Sync + 'static>; +pub type TimeSizeFn = Arc Pin + Send>> + Send + Sync + 'static>; +pub type TimeFn = Arc Pin + Send>> + Send + Sync + 'static>; + +pub struct ScannerMetrics { + operations: Vec, + latency: Vec, + cycle_info: RwLock>, + current_paths: HashMap, +} + +impl Default for ScannerMetrics { + fn default() -> Self { + Self::new() + } +} + +impl ScannerMetrics { + pub fn new() -> Self { + Self { + operations: (0..ScannerMetric::Last as usize).map(|_| AtomicU32::new(0)).collect(), + latency: vec![LockedLastMinuteLatency::default(); ScannerMetric::LastRealtime as usize], + cycle_info: RwLock::new(None), + current_paths: HashMap::new(), + } + } + + pub async fn set_cycle(&mut self, c: Option) { + *self.cycle_info.write().await = c; + } + + pub fn log(s: ScannerMetric) -> LogFn { + let start = SystemTime::now(); + let s_clone = s as usize; + Arc::new(move |_custom: &HashMap| { + Box::pin(async move { + let duration = SystemTime::now().duration_since(start).unwrap_or(Duration::from_secs(0)); + let mut sm_w = globalScannerMetrics.write().await; + sm_w.operations[s_clone].fetch_add(1, Ordering::SeqCst); + if s_clone < ScannerMetric::LastRealtime as usize { + sm_w.latency[s_clone].add(&duration).await; + } + }) + }) + } + + pub async fn time_size(s: ScannerMetric) -> TimeSizeFn { + let start = SystemTime::now(); + let s_clone = s as usize; + Arc::new(move |sz: u64| { + Box::pin(async move { + let duration = SystemTime::now().duration_since(start).unwrap_or(Duration::from_secs(0)); + let mut sm_w = globalScannerMetrics.write().await; + sm_w.operations[s_clone].fetch_add(1, Ordering::SeqCst); + if s_clone < ScannerMetric::LastRealtime as usize { + sm_w.latency[s_clone].add_size(&duration, sz).await; + } + }) + }) + } + + pub fn time(s: ScannerMetric) -> TimeFn { + let start = SystemTime::now(); + let s_clone = s as usize; + Arc::new(move || { + Box::pin(async move { + let duration = SystemTime::now().duration_since(start).unwrap_or(Duration::from_secs(0)); + let mut sm_w = globalScannerMetrics.write().await; + sm_w.operations[s_clone].fetch_add(1, Ordering::SeqCst); + if s_clone < ScannerMetric::LastRealtime as usize { + sm_w.latency[s_clone].add(&duration).await; + } + }) + }) + } +} + +pub type CloseDiskFn = Arc Pin + Send>> + Send + Sync + 'static>; +pub fn current_path_updater(disk: &str, _initial: &str) -> (UpdateCurrentPathFn, CloseDiskFn) { + let disk_1 = disk.to_string(); + let disk_2 = disk.to_string(); + ( + Arc::new(move |path: &str| { + let disk_inner = disk_1.clone(); + let path = path.to_string(); + Box::pin(async move { + globalScannerMetrics + .write() + .await + .current_paths + .insert(disk_inner, path.to_string()); + }) + }), + Arc::new(move || { + let disk_inner = disk_2.clone(); + Box::pin(async move { + globalScannerMetrics.write().await.current_paths.remove(&disk_inner); + }) + }), + ) +} diff --git a/ecstore/src/heal/data_usage.rs b/ecstore/src/heal/data_usage.rs new file mode 100644 index 000000000..067bcc641 --- /dev/null +++ b/ecstore/src/heal/data_usage.rs @@ -0,0 +1,139 @@ +use std::{collections::HashMap, time::SystemTime}; + +use lazy_static::lazy_static; +use serde::{Deserialize, Serialize}; +use tokio::sync::mpsc::Receiver; +use tracing::info; + +use crate::{ + config::common::save_config, + disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET}, + new_object_layer_fn, + utils::path::SLASH_SEPARATOR, +}; + +pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR; +const DATA_USAGE_OBJ_NAME: &str = ".usage.json"; +const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin"; +pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin"; +lazy_static! { + pub static ref DATA_USAGE_BUCKET: String = format!("{}{}{}", RUSTFS_META_BUCKET, SLASH_SEPARATOR, BUCKET_META_PREFIX); + pub static ref DATA_USAGE_OBJ_NAME_PATH: String = format!("{}{}{}", BUCKET_META_PREFIX, SLASH_SEPARATOR, DATA_USAGE_OBJ_NAME); + pub static ref DATA_USAGE_BLOOM_NAME_PATH: String = + format!("{}{}{}", BUCKET_META_PREFIX, SLASH_SEPARATOR, DATA_USAGE_BLOOM_NAME); + pub static ref BACKGROUND_HEAL_INFO_PATH: String = + format!("{}{}{}", BUCKET_META_PREFIX, SLASH_SEPARATOR, ".background-heal.json"); +} + +// BucketTargetUsageInfo - bucket target usage info provides +// - replicated size for all objects sent to this target +// - replica size for all objects received from this target +// - replication pending size for all objects pending replication to this target +// - replication failed size for all objects failed replication to this target +// - replica pending count +// - replica failed count +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct BucketTargetUsageInfo { + pub replication_pending_size: u64, + pub replication_failed_size: u64, + pub replicated_size: u64, + pub replica_size: u64, + pub replication_pending_count: u64, + pub replication_failed_count: u64, + pub replicated_count: u64, +} + +// BucketUsageInfo - bucket usage info provides +// - total size of the bucket +// - total objects in a bucket +// - object size histogram per bucket +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct BucketUsageInfo { + pub size: u64, + // Following five fields suffixed with V1 are here for backward compatibility + // Total Size for objects that have not yet been replicated + pub replication_pending_size_v1: u64, + // Total size for objects that have witness one or more failures and will be retried + pub replication_failed_size_v1: u64, + // Total size for objects that have been replicated to destination + pub replicated_size_v1: u64, + // Total number of objects pending replication + pub replication_pending_count_v1: u64, + // Total number of objects that failed replication + pub replication_failed_count_v1: u64, + + pub objects_count: u64, + pub object_size_histogram: HashMap, + pub object_versions_histogram: HashMap, + pub versions_count: u64, + pub delete_markers_count: u64, + pub replica_size: u64, + pub replica_count: u64, + pub replication_info: HashMap, +} + +// DataUsageInfo represents data usage stats of the underlying Object API +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct DataUsageInfo { + pub total_capacity: u64, + pub total_used_capacity: u64, + pub total_free_capacity: u64, + + // LastUpdate is the timestamp of when the data usage info was last updated. + // This does not indicate a full scan. + pub last_update: Option, + + // Objects total count across all buckets + pub objects_total_count: u64, + // Versions total count across all buckets + pub versions_total_count: u64, + // Delete markers total count across all buckets + pub delete_markers_total_count: u64, + // Objects total size across all buckets + pub objects_total_size: u64, + pub replication_info: HashMap, + + // Total number of buckets in this cluster + pub buckets_count: u64, + // Buckets usage info provides following information across all buckets + // - total size of the bucket + // - total objects in a bucket + // - object size histogram per bucket + pub buckets_usage: HashMap, + // Deprecated kept here for backward compatibility reasons. + pub bucket_sizes: HashMap, + // Todo: TierStats + // TierStats contains per-tier stats of all configured remote tiers +} + +pub async fn store_data_usage_in_backend(mut rx: Receiver) { + let layer = new_object_layer_fn(); + let lock = layer.read().await; + let store = match lock.as_ref() { + Some(s) => s, + None => { + info!("errServerNotInitialized"); + return; + } + }; + let mut attempts = 1; + loop { + match rx.recv().await { + Some(data_usage_info) => { + if let Ok(data) = serde_json::to_vec(&data_usage_info) { + if attempts > 10 { + let _ = save_config(store, &format!("{}{}", *DATA_USAGE_OBJ_NAME_PATH, ".bkp"), &data).await; + attempts += 1; + } + let _ = save_config(store, &DATA_USAGE_OBJ_NAME_PATH, &data).await; + attempts += 1; + } else { + continue; + } + } + None => { + return; + } + } + } +} diff --git a/ecstore/src/heal/data_usage_cache.rs b/ecstore/src/heal/data_usage_cache.rs new file mode 100644 index 000000000..371de38cf --- /dev/null +++ b/ecstore/src/heal/data_usage_cache.rs @@ -0,0 +1,860 @@ +use crate::config::common::save_config; +use crate::disk::error::DiskError; +use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET}; +use crate::error::{Error, Result}; +use crate::new_object_layer_fn; +use crate::set_disk::SetDisks; +use crate::store_api::{BucketInfo, HTTPRangeSpec, ObjectIO, ObjectOptions}; +use bytesize::ByteSize; +use http::HeaderMap; +use path_clean::PathClean; +use rand::Rng; +use rmp_serde::Serializer; +use s3s::dto::ReplicationConfiguration; +use s3s::{S3Error, S3ErrorCode}; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; +use std::hash::{DefaultHasher, Hash, Hasher}; +use std::path::Path; +use std::time::{Duration, SystemTime}; +use tokio::sync::mpsc::Sender; +use tokio::time::sleep; + +use super::data_scanner::{SizeSummary, DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS}; +use super::data_usage::{BucketTargetUsageInfo, BucketUsageInfo, DataUsageInfo, DATA_USAGE_ROOT}; + +// DATA_USAGE_BUCKET_LEN must be length of ObjectsHistogramIntervals +pub const DATA_USAGE_BUCKET_LEN: usize = 11; +pub const DATA_USAGE_VERSION_LEN: usize = 7; + +pub type DataUsageHashMap = HashSet; + +struct ObjectHistogramInterval { + name: &'static str, + start: u64, + end: u64, +} + +const OBJECTS_HISTOGRAM_INTERVALS: [ObjectHistogramInterval; DATA_USAGE_BUCKET_LEN] = [ + ObjectHistogramInterval { + name: "LESS_THAN_1024_B", + start: 0, + end: ByteSize::kib(1).as_u64() - 1, + }, + ObjectHistogramInterval { + name: "BETWEEN_1024_B_AND_64_KB", + start: ByteSize::kib(1).as_u64(), + end: ByteSize::kib(64).as_u64() - 1, + }, + ObjectHistogramInterval { + name: "BETWEEN_64_KB_AND_256_KB", + start: ByteSize::kib(64).as_u64(), + end: ByteSize::kib(256).as_u64() - 1, + }, + ObjectHistogramInterval { + name: "BETWEEN_256_KB_AND_512_KB", + start: ByteSize::kib(256).as_u64(), + end: ByteSize::kib(512).as_u64() - 1, + }, + ObjectHistogramInterval { + name: "BETWEEN_512_KB_AND_1_MB", + start: ByteSize::kib(512).as_u64(), + end: ByteSize::mib(1).as_u64() - 1, + }, + ObjectHistogramInterval { + name: "BETWEEN_1024B_AND_1_MB", + start: ByteSize::kib(1).as_u64(), + end: ByteSize::mib(1).as_u64() - 1, + }, + ObjectHistogramInterval { + name: "BETWEEN_1_MB_AND_10_MB", + start: ByteSize::mib(1).as_u64(), + end: ByteSize::mib(10).as_u64() - 1, + }, + ObjectHistogramInterval { + name: "BETWEEN_10_MB_AND_64_MB", + start: ByteSize::mib(10).as_u64(), + end: ByteSize::mib(64).as_u64() - 1, + }, + ObjectHistogramInterval { + name: "BETWEEN_64_MB_AND_128_MB", + start: ByteSize::mib(64).as_u64(), + end: ByteSize::mib(128).as_u64() - 1, + }, + ObjectHistogramInterval { + name: "BETWEEN_128_MB_AND_512_MB", + start: ByteSize::mib(128).as_u64(), + end: ByteSize::mib(512).as_u64() - 1, + }, + ObjectHistogramInterval { + name: "GREATER_THAN_512_MB", + start: ByteSize::mib(512).as_u64(), + end: u64::MAX, + }, +]; + +const OBJECTS_VERSION_COUNT_INTERVALS: [ObjectHistogramInterval; DATA_USAGE_VERSION_LEN] = [ + ObjectHistogramInterval { + name: "UNVERSIONED", + start: 0, + end: 0, + }, + ObjectHistogramInterval { + name: "SINGLE_VERSION", + start: 1, + end: 1, + }, + ObjectHistogramInterval { + name: "BETWEEN_2_AND_10", + start: 2, + end: 9, + }, + ObjectHistogramInterval { + name: "BETWEEN_10_AND_100", + start: 10, + end: 99, + }, + ObjectHistogramInterval { + name: "BETWEEN_100_AND_1000", + start: 100, + end: 999, + }, + ObjectHistogramInterval { + name: "BETWEEN_1000_AND_10000", + start: 1000, + end: 9999, + }, + ObjectHistogramInterval { + name: "GREATER_THAN_10000", + start: 10000, + end: u64::MAX, + }, +]; + +// sizeHistogram is a size histogram. +#[derive(Clone, Serialize, Deserialize)] +pub struct SizeHistogram(Vec); + +impl Default for SizeHistogram { + fn default() -> Self { + Self(vec![0; DATA_USAGE_BUCKET_LEN]) + } +} + +impl SizeHistogram { + fn add(&mut self, size: u64) { + for (idx, interval) in OBJECTS_HISTOGRAM_INTERVALS.iter().enumerate() { + if size >= interval.start && size <= interval.end { + self.0[idx] += 1; + break; + } + } + } + + pub fn to_map(&self) -> HashMap { + let mut res = HashMap::new(); + let mut spl_count = 0; + for (count, oh) in self.0.iter().zip(OBJECTS_HISTOGRAM_INTERVALS.iter()) { + if ByteSize::kib(1).as_u64() == oh.start && oh.end == ByteSize::mib(1).as_u64() - 1 { + res.insert(oh.name.to_string(), spl_count); + } else if ByteSize::kib(1).as_u64() <= oh.start && oh.end < ByteSize::mib(1).as_u64() { + spl_count += count; + res.insert(oh.name.to_string(), *count); + } else { + res.insert(oh.name.to_string(), *count); + } + } + res + } +} + +// versionsHistogram is a histogram of number of versions in an object. +#[derive(Clone, Serialize, Deserialize)] +pub struct VersionsHistogram(Vec); + +impl Default for VersionsHistogram { + fn default() -> Self { + Self(vec![0; DATA_USAGE_VERSION_LEN]) + } +} + +impl VersionsHistogram { + fn add(&mut self, size: u64) { + for (idx, interval) in OBJECTS_VERSION_COUNT_INTERVALS.iter().enumerate() { + if size >= interval.start && size <= interval.end { + self.0[idx] += 1; + break; + } + } + } + + pub fn to_map(&self) -> HashMap { + let mut res = HashMap::new(); + for (count, ov) in self.0.iter().zip(OBJECTS_VERSION_COUNT_INTERVALS.iter()) { + res.insert(ov.name.to_string(), *count); + } + res + } +} + +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct ReplicationStats { + pub pending_size: u64, + pub replicated_size: u64, + pub failed_size: u64, + pub failed_count: u64, + pub pending_count: u64, + pub missed_threshold_size: u64, + pub after_threshold_size: u64, + pub missed_threshold_count: u64, + pub after_threshold_count: u64, + pub replicated_count: u64, +} + +impl ReplicationStats { + pub fn empty(&self) -> bool { + self.replicated_size == 0 && self.failed_size == 0 && self.failed_count == 0 + } +} + +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct ReplicationAllStats { + pub targets: HashMap, + pub replica_size: u64, + pub replica_count: u64, +} + +impl ReplicationAllStats { + pub fn empty(&self) -> bool { + if self.replica_size != 0 && self.replica_count != 0 { + return false; + } + for (_, v) in self.targets.iter() { + if !v.empty() { + return false; + } + } + + true + } +} + +#[derive(Clone, Default, Serialize, Deserialize)] +pub struct DataUsageEntry { + pub children: DataUsageHashMap, + // These fields do no include any children. + pub size: usize, + pub objects: usize, + pub versions: usize, + pub delete_markers: usize, + pub obj_sizes: SizeHistogram, + pub obj_versions: VersionsHistogram, + pub replication_stats: Option, + // Todo: tier + // pub all_tier_stats: , + pub compacted: bool, +} + +impl DataUsageEntry { + pub fn add_child(&mut self, hash: &DataUsageHash) { + if self.children.contains(&hash.key()) { + return; + } + + self.children.insert(hash.key()); + } + + pub fn add_sizes(&mut self, summary: &SizeSummary) { + self.size += summary.total_size; + self.versions += summary.versions; + self.delete_markers += summary.delete_markers; + self.obj_sizes.add(summary.total_size as u64); + self.obj_versions.add(summary.versions as u64); + + let replication_stats = if self.replication_stats.is_none() { + self.replication_stats = Some(ReplicationAllStats::default()); + self.replication_stats.as_mut().unwrap() + } else { + self.replication_stats.as_mut().unwrap() + }; + replication_stats.replica_size += summary.replica_size as u64; + replication_stats.replica_count += summary.replica_count as u64; + + for (arn, st) in &summary.repl_target_stats { + let tgt_stat = replication_stats + .targets + .entry(arn.to_string()) + .or_insert(ReplicationStats::default()); + tgt_stat.pending_size += st.pending_size as u64; + tgt_stat.failed_size += st.failed_size as u64; + tgt_stat.replicated_size += st.replicated_size as u64; + tgt_stat.replicated_count += st.replicated_count as u64; + tgt_stat.failed_count += st.failed_count as u64; + tgt_stat.pending_count += st.pending_count as u64; + } + // Todo:: tiers + } + + pub fn merge(&mut self, other: &DataUsageEntry) { + self.objects += other.objects; + self.versions += other.versions; + self.delete_markers += other.delete_markers; + self.size += other.size; + if let Some(o_rep) = &other.replication_stats { + if self.replication_stats.is_none() { + self.replication_stats = Some(ReplicationAllStats::default()); + } + let s_rep = self.replication_stats.as_mut().unwrap(); + s_rep.targets.clear(); + s_rep.replica_size += o_rep.replica_size; + s_rep.replica_count += o_rep.replica_count; + for (arn, stat) in o_rep.targets.iter() { + let st = s_rep.targets.entry(arn.clone()).or_default(); + *st = ReplicationStats { + pending_size: stat.pending_size + st.pending_size, + failed_size: stat.failed_size + st.failed_size, + replicated_size: stat.replicated_size + st.replicated_size, + pending_count: stat.pending_count + st.pending_count, + failed_count: stat.failed_count + st.failed_count, + replicated_count: stat.replicated_count + st.replicated_count, + ..Default::default() + }; + } + } + + for (i, v) in other.obj_sizes.0.iter().enumerate() { + self.obj_sizes.0[i] += v; + } + + for (i, v) in other.obj_versions.0.iter().enumerate() { + self.obj_versions.0[i] += v; + } + + // todo: tiers + } +} + +#[derive(Clone)] +pub struct DataUsageEntryInfo { + pub name: String, + pub parent: String, + pub entry: DataUsageEntry, +} + +#[derive(Clone, Default, Serialize, Deserialize)] +pub struct DataUsageCacheInfo { + pub name: String, + pub next_cycle: u32, + pub last_update: Option, + pub skip_healing: bool, + // todo: life_cycle + // pub life_cycle: + #[serde(skip)] + pub updates: Option>, + #[serde(skip)] + pub replication: Option, +} + +// impl Default for DataUsageCacheInfo { +// fn default() -> Self { +// Self { +// name: Default::default(), +// next_cycle: Default::default(), +// last_update: SystemTime::now(), +// skip_healing: Default::default(), +// updates: Default::default(), +// replication: Default::default(), +// } +// } +// } + +#[derive(Clone, Default, Serialize, Deserialize)] +pub struct DataUsageCache { + pub info: DataUsageCacheInfo, + pub cache: HashMap, +} + +impl DataUsageCache { + pub async fn load(store: &SetDisks, name: &str) -> Result { + let mut d = DataUsageCache::default(); + let mut retries = 0; + while retries < 5 { + let path = Path::new(BUCKET_META_PREFIX).join(name); + match store + .get_object_reader( + RUSTFS_META_BUCKET, + path.to_str().unwrap(), + HTTPRangeSpec::nil(), + HeaderMap::new(), + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + { + Ok(mut reader) => { + if let Ok(info) = Self::unmarshal(&reader.read_all().await?) { + d = info + } + break; + } + Err(err) => match err.downcast_ref::() { + Some(DiskError::FileNotFound) | Some(DiskError::VolumeNotFound) => { + match store + .get_object_reader( + RUSTFS_META_BUCKET, + name, + HTTPRangeSpec::nil(), + HeaderMap::new(), + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + { + Ok(mut reader) => { + if let Ok(info) = Self::unmarshal(&reader.read_all().await?) { + d = info + } + break; + } + Err(_) => match err.downcast_ref::() { + Some(DiskError::FileNotFound) | Some(DiskError::VolumeNotFound) => { + break; + } + _ => {} + }, + } + } + _ => {} + }, + } + retries += 1; + let dur = { + let mut rng = rand::thread_rng(); + rng.gen_range(0..1_000) + }; + sleep(Duration::from_millis(dur)).await; + } + Ok(d) + } + + pub async fn save(&self, name: &str) -> Result<()> { + let buf = self.marshal_msg()?; + let buf_clone = buf.clone(); + let layer = new_object_layer_fn(); + let lock = layer.read().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(Error::from(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))), + }; + let store_clone = store.clone(); + let name_clone = name.to_string(); + tokio::spawn(async move { + let _ = save_config(&store_clone, &format!("{}{}", &name_clone, ".bkp"), &buf_clone).await; + }); + save_config(store, name, &buf).await + } + + pub fn replace(&mut self, path: &str, parent: &str, e: DataUsageEntry) { + let hash = hash_path(path); + self.cache.insert(hash.key(), e); + if !parent.is_empty() { + let phash = hash_path(parent); + let p = { + let p = self.cache.entry(phash.key()).or_default(); + p.add_child(&hash); + p.clone() + }; + self.cache.insert(phash.key(), p); + } + } + + pub fn replace_hashed(&mut self, hash: &DataUsageHash, parent: &Option, e: &DataUsageEntry) { + self.cache.insert(hash.key(), e.clone()); + if let Some(parent) = parent { + self.cache.entry(parent.key()).or_default().add_child(hash); + } + } + + pub fn find(&self, path: &str) -> Option { + self.cache.get(&hash_path(path).key()).cloned() + } + + pub fn find_children_copy(&mut self, h: DataUsageHash) -> DataUsageHashMap { + self.cache.entry(h.string()).or_default().children.clone() + } + + pub fn flatten(&self, root: &DataUsageEntry) -> DataUsageEntry { + let mut root = root.clone(); + for id in root.children.clone().iter() { + if let Some(e) = self.cache.get(id) { + let mut e = e.clone(); + if !e.children.is_empty() { + e = self.flatten(&e); + } + root.merge(&e); + } + } + root.children.clear(); + root + } + + pub fn copy_with_children(&mut self, src: &DataUsageCache, hash: &DataUsageHash, parent: &Option) { + if let Some(e) = src.cache.get(&hash.string()) { + self.cache.insert(hash.key(), e.clone()); + for ch in e.children.iter() { + if *ch == hash.key() { + return; + } + self.copy_with_children(src, &DataUsageHash(ch.to_string()), &Some(hash.clone())); + } + if let Some(parent) = parent { + let p = self.cache.entry(parent.key()).or_default(); + p.add_child(hash); + } + } + } + + pub fn delete_recursive(&mut self, hash: &DataUsageHash) { + let mut need_remove = Vec::new(); + if let Some(v) = self.cache.get(&hash.string()) { + for child in v.children.iter() { + need_remove.push(child.clone()); + } + } + self.cache.remove(&hash.string()); + need_remove.iter().for_each(|child| { + self.delete_recursive(&DataUsageHash(child.to_string())); + }); + } + + pub fn size_recursive(&self, path: &str) -> Option { + match self.find(path) { + Some(root) => { + if root.children.is_empty() { + return None; + } + let mut flat = self.flatten(&root); + if flat.replication_stats.is_some() && flat.replication_stats.as_ref().unwrap().empty() { + flat.replication_stats = None; + } + Some(flat) + } + None => None, + } + } + + pub fn search_parent(&self, hash: &DataUsageHash) -> Option { + let want = hash.key(); + if let Some(last_index) = want.rfind('/') { + if let Some(v) = self.find(&want[0..last_index]) { + if v.children.contains(&want) { + let found = hash_path(&want[0..last_index]); + return Some(found); + } + } + } + + for (k, v) in self.cache.iter() { + if v.children.contains(&want) { + let found = DataUsageHash(k.clone()); + return Some(found); + } + } + None + } + + pub fn is_compacted(&self, hash: &DataUsageHash) -> bool { + match self.cache.get(&hash.key()) { + Some(due) => due.compacted, + None => false, + } + } + + pub fn force_compact(&mut self, limit: usize) { + if self.cache.len() < limit { + return; + } + let top = hash_path(&self.info.name).key(); + let top_e = match self.find(&top) { + Some(e) => e, + None => return, + }; + if top_e.children.len() > DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS.try_into().unwrap() { + self.reduce_children_of(&hash_path(&self.info.name), limit, true); + } + if self.cache.len() <= limit { + return; + } + + let mut found = HashSet::new(); + found.insert(top); + mark(self, &top_e, &mut found); + self.cache.retain(|k, _| { + if !found.contains(k) { + return false; + } + true + }); + } + + pub fn reduce_children_of(&mut self, path: &DataUsageHash, limit: usize, compact_self: bool) { + let e = match self.cache.get(&path.key()) { + Some(e) => e, + None => return, + }; + + if e.compacted { + return; + } + + if e.children.len() > limit && compact_self { + let mut flat = self.size_recursive(&path.key()).unwrap_or_default(); + flat.compacted = true; + self.delete_recursive(path); + self.replace_hashed(path, &None, &flat); + return; + } + let total = self.total_children_rec(&path.key()); + if total < limit { + return; + } + + let mut leaves = Vec::new(); + let mut remove = total - limit; + add(self, path, &mut leaves); + leaves.sort_by(|a, b| a.objects.cmp(&b.objects)); + + while remove > 0 && !leaves.is_empty() { + let e = leaves.first().unwrap(); + let candidate = e.path.clone(); + if candidate == *path && !compact_self { + break; + } + let removing = self.total_children_rec(&candidate.key()); + let mut flat = match self.size_recursive(&candidate.key()) { + Some(flat) => flat, + None => { + leaves.remove(0); + continue; + } + }; + + flat.compacted = true; + self.delete_recursive(&candidate); + self.replace_hashed(&candidate, &None, &flat); + + remove -= removing; + leaves.remove(0); + } + } + + pub fn total_children_rec(&self, path: &str) -> usize { + let root = self.find(path); + + if root.is_none() { + return 0; + } + let root = root.unwrap(); + if root.children.is_empty() { + return 0; + } + + let mut n = root.children.len(); + for ch in root.children.iter() { + n += self.total_children_rec(ch); + } + n + } + + pub fn merge(&mut self, o: &DataUsageCache) { + let mut existing_root = self.root(); + let other_root = o.root(); + if existing_root.is_none() && other_root.is_none() { + return; + } + if other_root.is_none() { + return; + } + if existing_root.is_none() { + *self = o.clone(); + return; + } + if o.info.last_update.gt(&self.info.last_update) { + self.info.last_update = o.info.last_update; + } + + existing_root.as_mut().unwrap().merge(other_root.as_ref().unwrap()); + self.cache.insert(hash_path(&self.info.name).key(), existing_root.unwrap()); + let e_hash = self.root_hash(); + for key in other_root.as_ref().unwrap().children.iter() { + let entry = &o.cache[key]; + let flat = o.flatten(entry); + let mut existing = self.cache[key].clone(); + existing.merge(&flat); + self.replace_hashed(&DataUsageHash(key.clone()), &Some(e_hash.clone()), &existing); + } + } + + pub fn root_hash(&self) -> DataUsageHash { + hash_path(&self.info.name) + } + + pub fn root(&self) -> Option { + self.find(&self.info.name) + } + + pub fn dui(&self, path: &str, buckets: &[BucketInfo]) -> DataUsageInfo { + let e = match self.find(path) { + Some(e) => e, + None => return DataUsageInfo::default(), + }; + let flat = self.flatten(&e); + DataUsageInfo { + last_update: self.info.last_update, + objects_total_count: flat.objects as u64, + versions_total_count: flat.versions as u64, + delete_markers_total_count: flat.delete_markers as u64, + objects_total_size: flat.size as u64, + buckets_count: e.children.len() as u64, + buckets_usage: self.buckets_usage_info(buckets), + ..Default::default() + } + } + + pub fn buckets_usage_info(&self, buckets: &[BucketInfo]) -> HashMap { + let mut dst = HashMap::new(); + for bucket in buckets.iter() { + let e = match self.find(&bucket.name) { + Some(e) => e, + None => continue, + }; + let flat = self.flatten(&e); + let mut bui = BucketUsageInfo { + size: flat.size as u64, + versions_count: flat.versions as u64, + objects_count: flat.objects as u64, + delete_markers_count: flat.delete_markers as u64, + object_size_histogram: flat.obj_sizes.to_map(), + object_versions_histogram: flat.obj_versions.to_map(), + ..Default::default() + }; + if let Some(rs) = &flat.replication_stats { + bui.replica_size = rs.replica_size; + bui.replica_count = rs.replica_count; + + for (arn, stat) in rs.targets.iter() { + bui.replication_info.insert( + arn.clone(), + BucketTargetUsageInfo { + replication_pending_size: stat.pending_size, + replicated_size: stat.replicated_size, + replication_failed_size: stat.failed_size, + replication_pending_count: stat.pending_count, + replication_failed_count: stat.failed_count, + replicated_count: stat.replicated_count, + ..Default::default() + }, + ); + } + } + dst.insert(bucket.name.clone(), bui); + } + dst + } + + pub fn marshal_msg(&self) -> Result> { + let mut buf = Vec::new(); + + self.serialize(&mut Serializer::new(&mut buf))?; + + Ok(buf) + } + + pub fn unmarshal(buf: &[u8]) -> Result { + let t: Self = rmp_serde::from_slice(buf)?; + Ok(t) + } +} + +#[derive(Default, Clone)] +struct Inner { + objects: usize, + path: DataUsageHash, +} + +fn add(data_usage_cache: &DataUsageCache, path: &DataUsageHash, leaves: &mut Vec) { + let e = match data_usage_cache.cache.get(&path.key()) { + Some(e) => e, + None => return, + }; + if !e.children.is_empty() { + return; + } + + let sz = data_usage_cache.size_recursive(&path.key()).unwrap_or_default(); + leaves.push(Inner { + objects: sz.objects, + path: path.clone(), + }); + for ch in e.children.iter() { + add(data_usage_cache, &DataUsageHash(ch.clone()), leaves); + } +} + +fn mark(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut HashSet) { + for k in entry.children.iter() { + found.insert(k.to_string()); + if let Some(ch) = duc.cache.get(k) { + mark(duc, ch, found); + } + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct DataUsageHash(pub String); + +impl DataUsageHash { + pub fn string(&self) -> String { + self.0.clone() + } + + pub fn key(&self) -> String { + self.0.clone() + } + + pub fn mod_(&self, cycle: u32, cycles: u32) -> bool { + if cycles <= 1 { + return cycles == 1; + } + + let hash = self.calculate_hash(); + hash as u32 % cycles == cycle % cycles + } + + pub fn mod_alt(&self, cycle: u32, cycles: u32) -> bool { + if cycles <= 1 { + return cycles == 1; + } + + let hash = self.calculate_hash(); + (hash >> 32) as u32 % cycles == cycle % cycles + } + + fn calculate_hash(&self) -> u64 { + let mut hasher = DefaultHasher::new(); + self.0.hash(&mut hasher); + hasher.finish() + } +} + +pub fn hash_path(data: &str) -> DataUsageHash { + let mut data = data; + if data != DATA_USAGE_ROOT { + data = data.trim_matches('/'); + } + DataUsageHash(Path::new(&data).clean().to_string_lossy().to_string()) +} diff --git a/ecstore/src/heal/error.rs b/ecstore/src/heal/error.rs new file mode 100644 index 000000000..eb092ec90 --- /dev/null +++ b/ecstore/src/heal/error.rs @@ -0,0 +1,5 @@ +pub const ERR_IGNORE_FILE_CONTRIB: &str = "ignore this file's contribution toward data-usage"; +pub const ERR_SKIP_FILE: &str = "skip this file"; +pub const ERR_HEAL_STOP_SIGNALLED: &str = "heal stop signaled"; +pub const ERR_HEAL_IDLE_TIMEOUT: &str = "healing results were not consumed for too long"; +pub const ERR_RETRY_HEALING: &str = "some items failed to heal, we will retry healing this drive again"; diff --git a/ecstore/src/heal/heal_commands.rs b/ecstore/src/heal/heal_commands.rs index 39da448e2..6950cef1e 100644 --- a/ecstore/src/heal/heal_commands.rs +++ b/ecstore/src/heal/heal_commands.rs @@ -1,14 +1,14 @@ -use std::{ - path::Path, - time::{SystemTime, UNIX_EPOCH}, -}; +use std::{path::Path, time::SystemTime}; +use lazy_static::lazy_static; use serde::{Deserialize, Serialize}; +use time::OffsetDateTime; use tokio::sync::RwLock; use crate::{ disk::{DeleteOptions, DiskAPI, DiskStore, BUCKET_META_PREFIX, RUSTFS_META_BUCKET}, error::{Error, Result}, + global::GLOBAL_BackgroundHealState, heal::heal_ops::HEALING_TRACKER_FILENAME, new_object_layer_fn, store_api::{BucketInfo, StorageAPI}, @@ -37,7 +37,11 @@ pub const DRIVE_STATE_ROOT_MOUNT: &str = "root-mount"; pub const DRIVE_STATE_UNKNOWN: &str = "unknown"; pub const DRIVE_STATE_UNFORMATTED: &str = "unformatted"; // only returned by disk -#[derive(Clone, Copy, Debug, Default)] +lazy_static! { + pub static ref TIME_SENTINEL: OffsetDateTime = OffsetDateTime::from_unix_timestamp(0).unwrap(); +} + +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)] pub struct HealOpts { pub recursive: bool, pub dry_run: bool, @@ -50,7 +54,7 @@ pub struct HealOpts { pub set: Option, } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] pub struct HealDriveInfo { pub uuid: String, pub endpoint: String, @@ -74,11 +78,21 @@ pub struct HealResultItem { pub object_size: usize, } -#[derive(Debug, Default, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize)] pub struct HealStartSuccess { pub client_token: String, pub client_address: String, - pub start_time: u64, + pub start_time: SystemTime, +} + +impl Default for HealStartSuccess { + fn default() -> Self { + Self { + client_token: Default::default(), + client_address: Default::default(), + start_time: SystemTime::now(), + } + } } pub type HealStopSuccess = HealStartSuccess; @@ -91,8 +105,8 @@ pub struct HealingDisk { pub disk_index: Option, pub endpoint: String, pub path: String, - pub started: u64, - pub last_update: u64, + pub started: Option, + pub last_update: Option, pub retry_attempts: u64, pub objects_total_count: u64, pub objects_total_size: u64, @@ -121,8 +135,8 @@ pub struct HealingTracker { pub disk_index: Option, pub path: String, pub endpoint: String, - pub started: u64, - pub last_update: u64, + pub started: Option, + pub last_update: Option, pub objects_total_count: u64, pub objects_total_size: u64, pub items_healed: u64, @@ -150,9 +164,7 @@ pub struct HealingTracker { impl HealingTracker { pub fn marshal_msg(&self) -> Result> { - serde_json::to_string(self) - .map(|s| s.as_bytes().to_vec()) - .map_err(|err| Error::from_string(err.to_string())) + serde_json::to_vec(self).map_err(|err| Error::from_string(err.to_string())) } pub fn unmarshal_msg(data: &[u8]) -> Result { @@ -177,7 +189,7 @@ impl HealingTracker { self.object = String::new(); } - pub async fn get_last_update(&self) -> u64 { + pub async fn get_last_update(&self) -> Option { let _ = self.mu.read().await; self.last_update @@ -224,7 +236,7 @@ impl HealingTracker { pub async fn update(&mut self) -> Result<()> { if let Some(disk) = &self.disk { - if healing(&disk.path().to_string_lossy().to_string()).await?.is_none() { + if healing(disk.path().to_string_lossy().as_ref()).await?.is_none() { return Err(Error::from_string(format!("healingTracker: drive {} is not marked as healing", self.id))); } let _ = self.mu.write().await; @@ -252,14 +264,11 @@ impl HealingTracker { (self.pool_index, self.set_index, self.disk_index) = store.get_pool_and_set(&self.id).await?; } - self.last_update = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("Time went backwards") - .as_secs(); + self.last_update = Some(SystemTime::now()); let htracker_bytes = self.marshal_msg()?; - // TODO: globalBackgroundHealState + GLOBAL_BackgroundHealState.write().await.update_heal_status(self).await; if let Some(disk) = &self.disk { let file_path = Path::new(BUCKET_META_PREFIX).join(HEALING_TRACKER_FILENAME); @@ -270,7 +279,7 @@ impl HealingTracker { Ok(()) } - async fn delete(&self) -> Result<()> { + pub async fn delete(&self) -> Result<()> { if let Some(disk) = &self.disk { let file_path = Path::new(BUCKET_META_PREFIX).join(HEALING_TRACKER_FILENAME); return disk @@ -289,7 +298,7 @@ impl HealingTracker { Ok(()) } - async fn is_healed(&self, bucket: &str) -> bool { + pub async fn is_healed(&self, bucket: &str) -> bool { let _ = self.mu.read().await; for v in self.healed_buckets.iter() { if v == bucket { @@ -300,7 +309,7 @@ impl HealingTracker { false } - async fn resume(&mut self) { + pub async fn resume(&mut self) { let _ = self.mu.write().await; self.items_healed = self.resume_items_healed; @@ -311,7 +320,7 @@ impl HealingTracker { self.bytes_skipped = self.resume_bytes_skipped; } - async fn bucket_done(&mut self, bucket: &str) { + pub async fn bucket_done(&mut self, bucket: &str) { let _ = self.mu.write().await; self.resume_items_healed = self.items_healed; @@ -325,7 +334,7 @@ impl HealingTracker { self.queue_buckets.retain(|x| x != bucket); } - async fn set_queue_buckets(&mut self, buckets: &[BucketInfo]) { + pub async fn set_queue_buckets(&mut self, buckets: &[BucketInfo]) { let _ = self.mu.write().await; buckets.iter().for_each(|bucket| { @@ -373,40 +382,40 @@ impl Clone for HealingTracker { Self { disk: self.disk.clone(), id: self.id.clone(), - pool_index: self.pool_index.clone(), - set_index: self.set_index.clone(), - disk_index: self.disk_index.clone(), + pool_index: self.pool_index, + set_index: self.set_index, + disk_index: self.disk_index, path: self.path.clone(), endpoint: self.endpoint.clone(), - started: self.started.clone(), - last_update: self.last_update.clone(), - objects_total_count: self.objects_total_count.clone(), - objects_total_size: self.objects_total_size.clone(), - items_healed: self.items_healed.clone(), - items_failed: self.items_failed.clone(), - item_skipped: self.item_skipped.clone(), - bytes_done: self.bytes_done.clone(), - bytes_failed: self.bytes_failed.clone(), - bytes_skipped: self.bytes_skipped.clone(), + started: self.started, + last_update: self.last_update, + objects_total_count: self.objects_total_count, + objects_total_size: self.objects_total_size, + items_healed: self.items_healed, + items_failed: self.items_failed, + item_skipped: self.item_skipped, + bytes_done: self.bytes_done, + bytes_failed: self.bytes_failed, + bytes_skipped: self.bytes_skipped, bucket: self.bucket.clone(), object: self.object.clone(), - resume_items_healed: self.resume_items_healed.clone(), - resume_items_failed: self.resume_items_failed.clone(), - resume_items_skipped: self.resume_items_skipped.clone(), - resume_bytes_done: self.resume_bytes_done.clone(), - resume_bytes_failed: self.resume_bytes_failed.clone(), - resume_bytes_skipped: self.resume_bytes_skipped.clone(), + resume_items_healed: self.resume_items_healed, + resume_items_failed: self.resume_items_failed, + resume_items_skipped: self.resume_items_skipped, + resume_bytes_done: self.resume_bytes_done, + resume_bytes_failed: self.resume_bytes_failed, + resume_bytes_skipped: self.resume_bytes_skipped, queue_buckets: self.queue_buckets.clone(), healed_buckets: self.healed_buckets.clone(), heal_id: self.heal_id.clone(), - retry_attempts: self.retry_attempts.clone(), - finished: self.finished.clone(), + retry_attempts: self.retry_attempts, + finished: self.finished, mu: RwLock::new(false), } } } -async fn load_healing_tracker(disk: &Option) -> Result { +pub async fn load_healing_tracker(disk: &Option) -> Result { if let Some(disk) = disk { let disk_id = disk.get_disk_id().await?; if let Some(disk_id) = disk_id { @@ -430,23 +439,20 @@ async fn load_healing_tracker(disk: &Option) -> Result Result { - let mut healing_tracker = HealingTracker::default(); - healing_tracker.id = disk.get_disk_id().await?.map_or("".to_string(), |id| id.to_string()); - healing_tracker.heal_id = heal_id; - healing_tracker.path = disk.to_string(); - healing_tracker.endpoint = disk.endpoint().to_string(); - healing_tracker.started = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("Time went backwards") - .as_secs(); +pub async fn init_healing_tracker(disk: DiskStore, heal_id: &str) -> Result { let disk_location = disk.get_disk_location(); - healing_tracker.pool_index = disk_location.pool_idx; - healing_tracker.set_index = disk_location.set_idx; - healing_tracker.disk_index = disk_location.disk_idx; - healing_tracker.disk = Some(disk); - - Ok(healing_tracker) + Ok(HealingTracker { + id: disk.get_disk_id().await?.map_or("".to_string(), |id| id.to_string()), + heal_id: heal_id.to_string(), + path: disk.to_string(), + endpoint: disk.endpoint().to_string(), + started: Some(OffsetDateTime::now_utc()), + pool_index: disk_location.pool_idx, + set_index: disk_location.set_idx, + disk_index: disk_location.disk_idx, + disk: Some(disk), + ..Default::default() + }) } pub async fn healing(derive_path: &str) -> Result> { diff --git a/ecstore/src/heal/heal_ops.rs b/ecstore/src/heal/heal_ops.rs index 9ef9ee37a..c869e4ebb 100644 --- a/ecstore/src/heal/heal_ops.rs +++ b/ecstore/src/heal/heal_ops.rs @@ -1,12 +1,34 @@ +use super::{ + background_heal_ops::HealTask, + data_scanner::HEAL_DELETE_DANGLING, + error::ERR_SKIP_FILE, + heal_commands::{ + HealItemType, HealOpts, HealResultItem, HealScanMode, HealStopSuccess, HealingDisk, HealingTracker, + HEAL_ITEM_BUCKET_METADATA, + }, +}; +use crate::heal::heal_commands::{HEAL_ITEM_BUCKET, HEAL_ITEM_OBJECT}; +use crate::store_api::StorageAPI; +use crate::{ + config::common::CONFIG_PREFIX, + disk::RUSTFS_META_BUCKET, + global::GLOBAL_BackgroundHealRoutine, + heal::{ + error::ERR_HEAL_STOP_SIGNALLED, + heal_commands::{HealDriveInfo, DRIVE_STATE_OK}, + }, +}; use crate::{ disk::{endpoint::Endpoint, MetaCacheEntry}, endpoints::Endpoints, error::{Error, Result}, global::GLOBAL_IsDistErasure, - heal::heal_commands::HEAL_UNKNOWN_SCAN, + heal::heal_commands::{HealStartSuccess, HEAL_UNKNOWN_SCAN}, + new_object_layer_fn, utils::path::has_profix, }; use lazy_static::lazy_static; +use s3s::{S3Error, S3ErrorCode}; use std::{ collections::HashMap, future::Future, @@ -24,18 +46,13 @@ use tokio::{ }, time::{interval, sleep}, }; +use tracing::info; use uuid::Uuid; -use super::{ - background_heal_ops::HealTask, - heal_commands::{HealItemType, HealOpts, HealResultItem, HealScanMode, HealStopSuccess, HealingDisk, HealingTracker}, -}; - type HealStatusSummary = String; type ItemsMap = HashMap; -pub type HealObjectFn = Arc Result<()> + Send + Sync>; pub type HealEntryFn = - Box Pin> + Send>> + Send>; + Arc Pin> + Send>> + Send + Sync + 'static>; pub const BG_HEALING_UUID: &str = "0000-0000-0000-0000"; pub const HEALING_TRACKER_FILENAME: &str = ".healing.bin"; @@ -45,6 +62,10 @@ const HEAL_RUNNING_STATUS: &str = "running"; const HEAL_STOPPED_STATUS: &str = "stopped"; const HEAL_FINISHED_STATUS: &str = "finished"; +pub const RUESTFS_RESERVED_BUCKET: &str = "rustfs"; +pub const RUESTFS_RESERVED_BUCKET_PATH: &str = "/rustfs"; +pub const LOGIN_PATH_PREFIX: &str = "/login"; + const MAX_UNCONSUMED_HEAL_RESULT_ITEMS: usize = 1000; const HEAL_UNCONSUMED_TIMEOUT: std::time::Duration = Duration::from_secs(24 * 60 * 60); pub const NOP_HEAL: &str = ""; @@ -60,12 +81,13 @@ pub struct HealSequenceStatus { pub items: Vec, } +#[derive(Debug, Default)] pub struct HealSource { pub bucket: String, pub object: String, pub version_id: String, pub no_wait: bool, - opts: Option, + pub opts: Option, } #[derive(Clone, Debug)] @@ -73,8 +95,8 @@ pub struct HealSequence { pub bucket: String, pub object: String, pub report_progress: bool, - pub start_time: u64, - pub end_time: Arc>, + pub start_time: SystemTime, + pub end_time: Arc>, pub client_token: String, pub client_address: String, pub force_started: bool, @@ -93,6 +115,30 @@ pub struct HealSequence { rx: Arc>>, } +pub fn new_bg_heal_sequence() -> HealSequence { + let hs = HealOpts { + remove: HEAL_DELETE_DANGLING, + ..Default::default() + }; + + HealSequence { + start_time: SystemTime::now(), + client_token: BG_HEALING_UUID.to_string(), + bucket: RUESTFS_RESERVED_BUCKET.to_string(), + setting: hs, + current_status: Arc::new(RwLock::new(HealSequenceStatus { + summary: HEAL_NOT_STARTED_STATUS.to_string(), + heal_setting: hs, + ..Default::default() + })), + report_progress: false, + scanned_items_map: HashMap::new(), + healed_items_map: HashMap::new(), + heal_failed_items_map: HashMap::new(), + ..Default::default() + } +} + impl Default for HealSequence { fn default() -> Self { let (h_tx, h_rx) = mpsc::channel(1); @@ -101,8 +147,8 @@ impl Default for HealSequence { bucket: Default::default(), object: Default::default(), report_progress: Default::default(), - start_time: Default::default(), - end_time: Default::default(), + start_time: SystemTime::now(), + end_time: Arc::new(RwLock::new(SystemTime::now())), client_token: Default::default(), client_address: Default::default(), force_started: Default::default(), @@ -144,23 +190,23 @@ impl HealSequence { } impl HealSequence { - fn get_scanned_items_count(&self) -> usize { + fn _get_scanned_items_count(&self) -> usize { self.scanned_items_map.values().sum() } - fn get_scanned_items_map(&self) -> ItemsMap { + fn _get_scanned_items_map(&self) -> ItemsMap { self.scanned_items_map.clone() } - fn get_healed_items_map(&self) -> ItemsMap { + fn _get_healed_items_map(&self) -> ItemsMap { self.healed_items_map.clone() } - fn get_heal_failed_items_map(&self) -> ItemsMap { + fn _get_heal_failed_items_map(&self) -> ItemsMap { self.heal_failed_items_map.clone() } - fn count_failed(&mut self, heal_type: HealItemType) { + pub fn count_failed(&mut self, heal_type: HealItemType) { *self.heal_failed_items_map.entry(heal_type).or_insert(0) += 1; self.last_heal_activity = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -168,7 +214,7 @@ impl HealSequence { .as_secs(); } - fn count_scanned(&mut self, heal_type: HealItemType) { + pub fn count_scanned(&mut self, heal_type: HealItemType) { *self.scanned_items_map.entry(heal_type).or_insert(0) += 1; self.last_heal_activity = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -176,7 +222,7 @@ impl HealSequence { .as_secs(); } - fn count_healed(&mut self, heal_type: HealItemType) { + pub fn count_healed(&mut self, heal_type: HealItemType) { *self.healed_items_map.entry(heal_type).or_insert(0) += 1; self.last_heal_activity = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -193,11 +239,11 @@ impl HealSequence { } async fn has_ended(&self) -> bool { - if self.client_token == BG_HEALING_UUID.to_string() { + if self.client_token == *BG_HEALING_UUID { return false; } - !(*(self.end_time.read().await) == self.start_time) + *(self.end_time.read().await) != self.start_time } async fn stop(&self) { @@ -208,6 +254,7 @@ impl HealSequence { async fn push_heal_result_item(&self, r: &HealResultItem) -> Result<()> { let mut r = r.clone(); let mut interval_timer = interval(HEAL_UNCONSUMED_TIMEOUT); + #[allow(unused_assignments)] let mut items_len = 0; loop { { @@ -244,7 +291,7 @@ impl HealSequence { Ok(()) } - async fn queue_heal_task(&mut self, source: HealSource, heal_type: HealItemType) -> Result<()> { + pub async fn queue_heal_task(&mut self, source: HealSource, heal_type: HealItemType) -> Result<()> { let mut task = HealTask::new(&source.bucket, &source.object, &source.version_id, &self.setting); if let Some(opts) = source.opts { task.opts = opts; @@ -252,31 +299,104 @@ impl HealSequence { task.opts.scan_mode = HEAL_UNKNOWN_SCAN; } - self.count_scanned(heal_type); + self.count_scanned(heal_type.clone()); - if source.no_wait {} - - todo!() - } - - fn heal_disk_meta() -> Result<()> { - todo!() - } - - fn heal_items(&self, buckets_only: bool) -> Result<()> { - if self.client_token == BG_HEALING_UUID.to_string() { + if source.no_wait { + let task_str = format!("{:?}", task); + if GLOBAL_BackgroundHealRoutine.read().await.tasks_tx.try_send(task).is_ok() { + info!("Task in the queue: {:?}", task_str); + } return Ok(()); } - todo!() + let (resp_tx, mut resp_rx) = mpsc::channel(1); + task.resp_tx = Some(resp_tx); + + let task_str = format!("{:?}", task); + if GLOBAL_BackgroundHealRoutine.read().await.tasks_tx.try_send(task).is_ok() { + info!("Task in the queue: {:?}", task_str); + } + let count_ok_drives = |drivers: &[HealDriveInfo]| { + let mut count = 0; + for drive in drivers.iter() { + if drive.state == DRIVE_STATE_OK { + count += 1; + } + } + count + }; + + match resp_rx.recv().await { + Some(mut res) => { + if res.err.is_none() { + self.count_healed(heal_type.clone()); + } else { + self.count_failed(heal_type.clone()); + } + if !self.report_progress { + if let Some(err) = res.err { + if err.to_string() == ERR_SKIP_FILE { + return Ok(()); + } + return Err(err); + } else { + return Ok(()); + } + } + res.result.heal_item_type = heal_type.clone(); + if let Some(err) = res.err.as_ref() { + res.result.detail = err.to_string(); + } + if res.result.parity_blocks > 0 && res.result.data_blocks > 0 && res.result.data_blocks > res.result.parity_blocks + { + let got = count_ok_drives(&res.result.after); + if got < res.result.parity_blocks { + res.result.detail = format!( + "quorum loss - expected {} minimum, got drive states in OK {}", + res.result.parity_blocks, got + ); + } + } + self.push_heal_result_item(&res.result).await + } + None => Ok(()), + } } - async fn traverse_and_heal(&self) { + async fn heal_disk_meta(h: Arc>) -> Result<()> { + HealSequence::heal_rustfs_sys_meta(h, CONFIG_PREFIX).await + } + + async fn heal_items(h: Arc>, buckets_only: bool) -> Result<()> { + if h.read().await.client_token == *BG_HEALING_UUID { + return Ok(()); + } + + Self::heal_disk_meta(h.clone()).await?; + let bucket = h.read().await.bucket.clone(); + Self::heal_bucket(h.clone(), &bucket, buckets_only).await + } + + async fn traverse_and_heal(h: Arc>) { let buckets_only = false; + let result = match Self::heal_items(h.clone(), buckets_only).await { + Ok(_) => None, + Err(err) => Some(err), + }; + let _ = h.read().await.traverse_and_heal_done_tx.read().await.send(result).await; } - fn heal_rustfs_sys_meta(&self, meta_prefix: String) -> Result<()> { - todo!() + async fn heal_rustfs_sys_meta(h: Arc>, meta_prefix: &str) -> Result<()> { + let layer = new_object_layer_fn(); + let lock = layer.read().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(Error::from(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))), + }; + let setting = h.read().await.setting; + store + .heal_objects(RUSTFS_META_BUCKET, meta_prefix, &setting, h.clone(), true) + .await } async fn is_done(&self) -> bool { @@ -286,13 +406,101 @@ impl HealSequence { } false } + + pub async fn heal_bucket(hs: Arc>, bucket: &str, bucket_only: bool) -> Result<()> { + let (object, setting) = { + let mut hs_w = hs.write().await; + hs_w.queue_heal_task( + HealSource { + bucket: bucket.to_string(), + ..Default::default() + }, + HEAL_ITEM_BUCKET.to_string(), + ) + .await?; + + if bucket_only { + return Ok(()); + } + + if !hs_w.setting.recursive { + if !hs_w.object.is_empty() { + HealSequence::heal_object(hs.clone(), bucket, &hs_w.object, "", hs_w.setting.scan_mode).await?; + } + return Ok(()); + } + (hs_w.object.clone(), hs_w.setting) + }; + let layer = new_object_layer_fn(); + let lock = layer.read().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(Error::from(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))), + }; + store.heal_objects(bucket, &object, &setting, hs.clone(), false).await + } + + pub async fn heal_object( + hs: Arc>, + bucket: &str, + object: &str, + version_id: &str, + _scan_mode: HealScanMode, + ) -> Result<()> { + let mut hs_w = hs.write().await; + if hs_w.is_quitting().await { + return Err(Error::from_string(ERR_HEAL_STOP_SIGNALLED)); + } + + let setting = hs_w.setting; + hs_w.queue_heal_task( + HealSource { + bucket: bucket.to_string(), + object: object.to_string(), + version_id: version_id.to_string(), + opts: Some(setting), + ..Default::default() + }, + HEAL_ITEM_OBJECT.to_string(), + ) + .await?; + + Ok(()) + } + + pub async fn heal_meta_object( + hs: Arc>, + bucket: &str, + object: &str, + version_id: &str, + _scan_mode: HealScanMode, + ) -> Result<()> { + let mut hs_w = hs.write().await; + if hs_w.is_quitting().await { + return Err(Error::from_string(ERR_HEAL_STOP_SIGNALLED)); + } + + hs_w.queue_heal_task( + HealSource { + bucket: bucket.to_string(), + object: object.to_string(), + version_id: version_id.to_string(), + ..Default::default() + }, + HEAL_ITEM_BUCKET_METADATA.to_string(), + ) + .await?; + + Ok(()) + } } -pub async fn heal_sequence_start(h: Arc) { +pub async fn heal_sequence_start(h: Arc>) { + let r = h.read().await; { - let mut current_status_w = h.current_status.write().await; - (*current_status_w).summary = HEAL_RUNNING_STATUS.to_string(); - (*current_status_w).start_time = SystemTime::now() + let mut current_status_w = r.current_status.write().await; + current_status_w.summary = HEAL_RUNNING_STATUS.to_string(); + current_status_w.start_time = SystemTime::now() .duration_since(UNIX_EPOCH) .expect("Time went backwards") .as_secs(); @@ -300,42 +508,35 @@ pub async fn heal_sequence_start(h: Arc) { let h_clone = h.clone(); spawn(async move { - h_clone.traverse_and_heal().await; + HealSequence::traverse_and_heal(h_clone).await; }); let h_clone_1 = h.clone(); - let mut x = h.traverse_and_heal_done_rx.write().await; + let mut x = r.traverse_and_heal_done_rx.write().await; select! { - _ = h.is_done() => { - *(h.end_time.write().await) = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("Time went backwards") - .as_secs(); - let mut current_status_w = h.current_status.write().await; - (*current_status_w).summary = HEAL_FINISHED_STATUS.to_string(); + _ = r.is_done() => { + *(r.end_time.write().await) = SystemTime::now(); + let mut current_status_w = r.current_status.write().await; + current_status_w.summary = HEAL_FINISHED_STATUS.to_string(); - spawn(async move { - let mut rx_w = h_clone_1.traverse_and_heal_done_rx.write().await; - rx_w.recv().await; + spawn(async move { + let binding = h_clone_1.read().await; + let mut rx_w = binding.traverse_and_heal_done_rx.write().await; + rx_w.recv().await; }); } result = x.recv() => { - match result { - Some(err) => { - match err { - Some(err) => { - let mut current_status_w = h.current_status.write().await; - (current_status_w).summary = HEAL_STOPPED_STATUS.to_string(); - (current_status_w).failure_detail = err.to_string(); - }, - None => { - let mut current_status_w = h.current_status.write().await; - (current_status_w).summary = HEAL_FINISHED_STATUS.to_string(); - } + if let Some(err) = result { + match err { + Some(err) => { + let mut current_status_w = r.current_status.write().await; + (current_status_w).summary = HEAL_STOPPED_STATUS.to_string(); + (current_status_w).failure_detail = err.to_string(); + }, + None => { + let mut current_status_w = r.current_status.write().await; + (current_status_w).summary = HEAL_FINISHED_STATUS.to_string(); } - }, - None => { - } } @@ -347,22 +548,37 @@ pub async fn heal_sequence_start(h: Arc) { pub struct AllHealState { mu: RwLock, - heal_seq_map: HashMap, + heal_seq_map: HashMap>>, heal_local_disks: HashMap, heal_status: HashMap, } impl AllHealState { - pub fn new(cleanup: bool) -> Self { - let hstate = AllHealState::default(); + pub fn new(cleanup: bool) -> Arc> { + let hstate = Arc::new(RwLock::new(AllHealState::default())); + let (_, mut rx) = broadcast::channel(1); if cleanup { - // spawn(f); + let hstate_clone = hstate.clone(); + tokio::spawn(async move { + loop { + select! { + result = rx.recv() =>{ + if let Ok(true) = result { + return; + } + } + _ = sleep(Duration::from_secs(5 * 60)) => { + hstate_clone.write().await.periodic_heal_seqs_clean().await; + } + } + } + }); } hstate } - async fn pop_heal_local_disks(&mut self, heal_local_disks: &[Endpoint]) { + pub async fn pop_heal_local_disks(&mut self, heal_local_disks: &[Endpoint]) { let _ = self.mu.write().await; self.heal_local_disks.retain(|k, _| { @@ -382,14 +598,14 @@ impl AllHealState { }); } - async fn update_heal_status(&mut self, tracker: &HealingTracker) { + pub async fn update_heal_status(&mut self, tracker: &HealingTracker) { let _ = self.mu.write().await; let _ = tracker.mu.read().await; self.heal_status.insert(tracker.id.clone(), tracker.clone()); } - async fn get_local_healing_disks(&self) -> HashMap { + pub async fn get_local_healing_disks(&self) -> HashMap { let _ = self.mu.read().await; let mut dst = HashMap::new(); @@ -400,7 +616,7 @@ impl AllHealState { dst } - async fn get_heal_local_disk_endpoints(&self) -> Endpoints { + pub async fn get_heal_local_disk_endpoints(&self) -> Endpoints { let _ = self.mu.read().await; let mut endpoints = Vec::new(); @@ -413,13 +629,13 @@ impl AllHealState { Endpoints::from(endpoints) } - async fn set_disk_healing_status(&mut self, ep: Endpoint, healing: bool) { + pub async fn set_disk_healing_status(&mut self, ep: Endpoint, healing: bool) { let _ = self.mu.write().await; self.heal_local_disks.insert(ep, healing); } - async fn push_heal_local_disks(&mut self, heal_local_disks: &[Endpoint]) { + pub async fn push_heal_local_disks(&mut self, heal_local_disks: &[Endpoint]) { let _ = self.mu.write().await; heal_local_disks.iter().for_each(|heal_local_disk| { @@ -427,37 +643,28 @@ impl AllHealState { }); } - async fn periodic_heal_seqs_clean(&mut self, mut rx: Receiver) { - loop { - select! { - result = rx.recv() =>{ - if let Ok(true) = result { - return; - } - } - _ = sleep(Duration::from_secs(5 * 60)) => { - let _ = self.mu.write().await; - let now = SystemTime::now(); + pub async fn periodic_heal_seqs_clean(&mut self) { + let _ = self.mu.write().await; + let now = SystemTime::now(); - let mut keys_to_reomve = Vec::new(); - for (k, v) in self.heal_seq_map.iter() { - if v.has_ended().await && (UNIX_EPOCH + Duration::from_secs(*(v.end_time.read().await)) + KEEP_HEAL_SEQ_STATE_DURATION) < now { - keys_to_reomve.push(k.clone()) - } - } - for key in keys_to_reomve.iter() { - self.heal_seq_map.remove(key); - } - } + let mut keys_to_reomve = Vec::new(); + for (k, v) in self.heal_seq_map.iter() { + let r = v.read().await; + if r.has_ended().await && now.duration_since(*(r.end_time.read().await)).unwrap() > KEEP_HEAL_SEQ_STATE_DURATION { + keys_to_reomve.push(k.clone()) } } + for key in keys_to_reomve.iter() { + self.heal_seq_map.remove(key); + } } - async fn get_heal_sequence_by_token(&self, token: &str) -> (Option, bool) { + pub async fn get_heal_sequence_by_token(&self, token: &str) -> (Option>>, bool) { let _ = self.mu.read().await; for v in self.heal_seq_map.values() { - if v.client_token == token { + let r = v.read().await; + if r.client_token == token { return (Some(v.clone()), true); } } @@ -465,15 +672,16 @@ impl AllHealState { (None, false) } - async fn get_heal_sequence(&self, path: &str) -> Option { + pub async fn get_heal_sequence(&self, path: &str) -> Option>> { let _ = self.mu.read().await; self.heal_seq_map.get(path).cloned() } - async fn stop_heal_sequence(&mut self, path: &str) -> Result> { + pub async fn stop_heal_sequence(&mut self, path: &str) -> Result> { let mut hsp = HealStopSuccess::default(); if let Some(he) = self.get_heal_sequence(path).await { + let he = he.read().await; let client_token = he.client_token.clone(); if *GLOBAL_IsDistErasure.read().await { // TODO: proxy @@ -513,23 +721,22 @@ impl AllHealState { // `keepHealSeqStateDuration`. This function also launches a // background routine to clean up heal results after the // aforementioned duration. - pub async fn launch_new_heal_sequence(&mut self, heal_sequence: &HealSequence) -> Result> { - let path = Path::new(&heal_sequence.bucket).join(heal_sequence.object.clone()); + pub async fn launch_new_heal_sequence(&mut self, heal_sequence: Arc>) -> Result> { + let r = heal_sequence.read().await; + let path = Path::new(&r.bucket).join(r.object.clone()); let path_s = path.to_str().unwrap(); - if heal_sequence.force_started { + if r.force_started { self.stop_heal_sequence(path_s).await?; - } else { - if let Some(hs) = self.get_heal_sequence(path_s).await { - if !hs.has_ended().await { - return Err(Error::from_string(format!("Heal is already running on the given path (use force-start option to stop and start afresh). The heal was started by IP {} at {}, token is {}", heal_sequence.client_address, heal_sequence.start_time, heal_sequence.client_token))); - } + } else if let Some(hs) = self.get_heal_sequence(path_s).await { + if !hs.read().await.has_ended().await { + return Err(Error::from_string(format!("Heal is already running on the given path (use force-start option to stop and start afresh). The heal was started by IP {} at {:?}, token is {}", r.client_address, r.start_time, r.client_token))); } } let _ = self.mu.write().await; for (k, v) in self.heal_seq_map.iter() { - if !v.has_ended().await && (has_profix(k, path_s) || has_profix(path_s, k)) { + if !v.read().await.has_ended().await && (has_profix(k, path_s) || has_profix(path_s, k)) { return Err(Error::from_string(format!( "The provided heal sequence path overlaps with an existing heal path: {}", k @@ -539,14 +746,25 @@ impl AllHealState { self.heal_seq_map.insert(path_s.to_string(), heal_sequence.clone()); - let client_token = heal_sequence.client_token.clone(); + let client_token = r.client_token.clone(); if *GLOBAL_IsDistErasure.read().await { // TODO: proxy } - if heal_sequence.client_token == BG_HEALING_UUID { + if r.client_token == BG_HEALING_UUID { // For background heal do nothing, do not spawn an unnecessary goroutine. + } else { + let heal_sequence_clone = heal_sequence.clone(); + tokio::spawn(async { + heal_sequence_start(heal_sequence_clone).await; + }); } - todo!() + + let b = serde_json::to_vec(&HealStartSuccess { + client_token, + client_address: r.client_address.clone(), + start_time: r.start_time, + })?; + Ok(b) } } diff --git a/ecstore/src/heal/mod.rs b/ecstore/src/heal/mod.rs index 38db00f2c..f4a8e486c 100644 --- a/ecstore/src/heal/mod.rs +++ b/ecstore/src/heal/mod.rs @@ -1,3 +1,8 @@ pub mod background_heal_ops; +pub mod data_scanner; +pub mod data_scanner_metric; +pub mod data_usage; +pub mod data_usage_cache; +pub mod error; pub mod heal_commands; pub mod heal_ops; diff --git a/ecstore/src/lib.rs b/ecstore/src/lib.rs index 2c914c8c5..f2f4eb225 100644 --- a/ecstore/src/lib.rs +++ b/ecstore/src/lib.rs @@ -9,7 +9,7 @@ pub mod erasure; pub mod error; mod file_meta; mod global; -mod heal; +pub mod heal; pub mod peer; mod quorum; pub mod set_disk; diff --git a/ecstore/src/options.rs b/ecstore/src/options.rs index bf787c40a..2da19e2c6 100644 --- a/ecstore/src/options.rs +++ b/ecstore/src/options.rs @@ -77,7 +77,7 @@ fn get_default_opts( pub fn extract_metadata(headers: &HeaderMap) -> HashMap { let mut metadata = HashMap::new(); - extract_metadata_from_mime(&headers, &mut metadata); + extract_metadata_from_mime(headers, &mut metadata); metadata } diff --git a/ecstore/src/peer.rs b/ecstore/src/peer.rs index a56fdb426..369b47dfb 100644 --- a/ecstore/src/peer.rs +++ b/ecstore/src/peer.rs @@ -1,14 +1,26 @@ use async_trait::async_trait; use futures::future::join_all; use protos::node_service_time_out_client; -use protos::proto_gen::node_service::{DeleteBucketRequest, GetBucketInfoRequest, ListBucketRequest, MakeBucketRequest}; +use protos::proto_gen::node_service::{ + DeleteBucketRequest, GetBucketInfoRequest, HealBucketRequest, ListBucketRequest, MakeBucketRequest, +}; use regex::Regex; use std::{collections::HashMap, fmt::Debug, sync::Arc}; +use tokio::sync::RwLock; use tonic::Request; use tracing::warn; -use crate::disk::DiskAPI; +use crate::disk::error::{is_all_buckets_not_found, is_all_not_found}; +use crate::disk::{DiskAPI, DiskStore}; +use crate::global::GLOBAL_LOCAL_DISK_MAP; +use crate::heal::heal_commands::{ + HealDriveInfo, HealOpts, HealResultItem, DRIVE_STATE_CORRUPT, DRIVE_STATE_MISSING, DRIVE_STATE_OFFLINE, DRIVE_STATE_OK, + HEAL_ITEM_BUCKET, +}; +use crate::heal::heal_ops::RUESTFS_RESERVED_BUCKET; +use crate::quorum::{bucket_op_ignored_errs, reduce_write_quorum_errs}; use crate::store::all_local_disk; +use crate::utils::wildcard::is_rustfs_meta_bucket_name; use crate::{ disk::{self, error::DiskError, VolumeInfo}, endpoints::{EndpointServerPools, Node}, @@ -20,6 +32,7 @@ type Client = Arc>; #[async_trait] pub trait PeerS3Client: Debug + Sync + Send + 'static { + async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result; async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()>; async fn list_bucket(&self, opts: &BucketOptions) -> Result>; async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()>; @@ -61,6 +74,79 @@ impl S3PeerSys { } impl S3PeerSys { + pub async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result { + let mut opts = *opts; + let mut futures = Vec::with_capacity(self.clients.len()); + for client in self.clients.iter() { + // client_clon + futures.push(async move { + match client.get_bucket_info(bucket, &BucketOptions::default()).await { + Ok(_) => None, + Err(err) => Some(err), + } + }); + } + let errs = join_all(futures).await; + + let mut pool_errs = Vec::new(); + for pool_idx in 0..self.pools_count { + let mut per_pool_errs = Vec::new(); + for (i, client) in self.clients.iter().enumerate() { + if let Some(v) = client.get_pools() { + if v.contains(&pool_idx) { + per_pool_errs.push(errs[i].clone()); + } + } + } + let qu = per_pool_errs.len() / 2; + pool_errs.push(reduce_write_quorum_errs(&per_pool_errs, &bucket_op_ignored_errs(), qu)); + } + + if !opts.recreate { + opts.remove = is_all_not_found(&pool_errs); + opts.recursive = !opts.remove; + } + + let mut futures = Vec::new(); + let heal_bucket_results = Arc::new(RwLock::new(vec![HealResultItem::default(); self.clients.len()])); + for (idx, client) in self.clients.iter().enumerate() { + let opts_clone = opts; + let heal_bucket_results_clone = heal_bucket_results.clone(); + futures.push(async move { + match client.heal_bucket(bucket, &opts_clone).await { + Ok(res) => { + heal_bucket_results_clone.write().await[idx] = res; + None + } + Err(err) => Some(err), + } + }); + } + let errs = join_all(futures).await; + + for pool_idx in 0..self.pools_count { + let mut per_pool_errs = Vec::new(); + for (i, client) in self.clients.iter().enumerate() { + if let Some(v) = client.get_pools() { + if v.contains(&pool_idx) { + per_pool_errs.push(errs[i].clone()); + } + } + } + let qu = per_pool_errs.len() / 2; + if let Some(pool_err) = reduce_write_quorum_errs(&per_pool_errs, &bucket_op_ignored_errs(), qu) { + return Err(pool_err); + } + } + + for (i, err) in errs.iter().enumerate() { + if err.is_none() { + return Ok(heal_bucket_results.read().await[i].clone()); + } + } + Err(DiskError::VolumeNotFound.into()) + } + pub async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> { let mut futures = Vec::with_capacity(self.clients.len()); for cli in self.clients.iter() { @@ -236,6 +322,11 @@ impl PeerS3Client for LocalPeerS3Client { fn get_pools(&self) -> Option> { self.pools.clone() } + + async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result { + heal_bucket_local(bucket, opts).await + } + async fn list_bucket(&self, _opts: &BucketOptions) -> Result> { let local_disks = all_local_disk().await; @@ -420,6 +511,29 @@ impl PeerS3Client for RemotePeerS3Client { fn get_pools(&self) -> Option> { self.pools.clone() } + + async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result { + let options: String = serde_json::to_string(opts)?; + let mut client = node_service_time_out_client(&self.addr) + .await + .map_err(|err| Error::from_string(format!("can not get client, err: {}", err)))?; + let request = Request::new(HealBucketRequest { + bucket: bucket.to_string(), + options, + }); + let response = client.heal_bucket(request).await?.into_inner(); + if !response.success { + return Err(Error::from_string(response.error_info.unwrap_or_default())); + } + + Ok(HealResultItem { + heal_item_type: HEAL_ITEM_BUCKET.to_string(), + bucket: bucket.to_string(), + set_count: 0, + ..Default::default() + }) + } + async fn list_bucket(&self, opts: &BucketOptions) -> Result> { let options = serde_json::to_string(opts)?; let mut client = node_service_time_out_client(&self.addr) @@ -528,7 +642,7 @@ fn is_reserved_bucket(bucket_name: &str) -> bool { } // 检查桶名是否为保留名或无效名 -fn is_reserved_or_invalid_bucket(bucket_entry: &str, strict: bool) -> bool { +pub fn is_reserved_or_invalid_bucket(bucket_entry: &str, strict: bool) -> bool { if bucket_entry.is_empty() { return true; } @@ -538,3 +652,134 @@ fn is_reserved_or_invalid_bucket(bucket_entry: &str, strict: bool) -> bool { result || is_meta_bucket(bucket_entry) || is_reserved_bucket(bucket_entry) } + +pub async fn heal_bucket_local(bucket: &str, opts: &HealOpts) -> Result { + let disks = clone_drives().await; + let before_state = Arc::new(RwLock::new(vec![String::new(); disks.len()])); + let after_state = Arc::new(RwLock::new(vec![String::new(); disks.len()])); + + let mut futures = Vec::new(); + for (index, disk) in disks.iter().enumerate() { + let disk = disk.clone(); + let bucket = bucket.to_string(); + let bs_clone = before_state.clone(); + let as_clone = after_state.clone(); + futures.push(async move { + let disk = match disk { + Some(disk) => disk, + None => { + bs_clone.write().await[index] = DRIVE_STATE_OFFLINE.to_string(); + as_clone.write().await[index] = DRIVE_STATE_OFFLINE.to_string(); + return Some(Error::new(DiskError::DiskNotFound)); + } + }; + bs_clone.write().await[index] = DRIVE_STATE_OK.to_string(); + as_clone.write().await[index] = DRIVE_STATE_OK.to_string(); + + if bucket == RUESTFS_RESERVED_BUCKET { + return None; + } + + match disk.stat_volume(&bucket).await { + Ok(_) => None, + Err(err) => match err.downcast_ref() { + Some(DiskError::DiskNotFound) => { + bs_clone.write().await[index] = DRIVE_STATE_OFFLINE.to_string(); + as_clone.write().await[index] = DRIVE_STATE_OFFLINE.to_string(); + Some(err) + } + Some(DiskError::VolumeNotFound) => { + bs_clone.write().await[index] = DRIVE_STATE_MISSING.to_string(); + as_clone.write().await[index] = DRIVE_STATE_MISSING.to_string(); + Some(err) + } + _ => { + bs_clone.write().await[index] = DRIVE_STATE_CORRUPT.to_string(); + as_clone.write().await[index] = DRIVE_STATE_CORRUPT.to_string(); + Some(err) + } + }, + } + }); + } + let errs = join_all(futures).await; + let mut res = HealResultItem { + heal_item_type: HEAL_ITEM_BUCKET.to_string(), + bucket: bucket.to_string(), + disk_count: disks.len(), + set_count: 0, + ..Default::default() + }; + + if opts.dry_run { + return Ok(res); + } + + for (disk, state) in disks.iter().zip(before_state.read().await.iter()) { + res.before.push(HealDriveInfo { + uuid: "".to_string(), + endpoint: disk.clone().map(|s| s.to_string()).unwrap_or_default(), + state: state.to_string(), + }); + } + + if !is_rustfs_meta_bucket_name(bucket) && !is_all_buckets_not_found(&errs) && opts.remove { + let mut futures = Vec::new(); + for disk in disks.iter() { + let disk = disk.clone(); + let bucket = bucket.to_string(); + futures.push(async move { + match disk { + Some(disk) => { + let _ = disk.delete_volume(&bucket).await; + None + } + None => Some(Error::new(DiskError::DiskNotFound)), + } + }); + } + + let _ = join_all(futures).await; + } + + if !opts.remove { + let mut futures = Vec::new(); + for (idx, disk) in disks.iter().enumerate() { + let disk = disk.clone(); + let bucket = bucket.to_string(); + let bs_clone = before_state.clone(); + let as_clone = after_state.clone(); + let errs_clone = errs.clone(); + futures.push(async move { + if bs_clone.read().await[idx] == DRIVE_STATE_MISSING { + match disk.as_ref().unwrap().make_volume(&bucket).await { + Ok(_) => { + as_clone.write().await[idx] = DRIVE_STATE_OK.to_string(); + return None; + } + Err(err) => { + return Some(err); + } + } + } + errs_clone[idx].clone() + }); + } + + let _ = join_all(futures).await; + } + + for (disk, state) in disks.iter().zip(after_state.read().await.iter()) { + res.before.push(HealDriveInfo { + uuid: "".to_string(), + endpoint: disk.clone().map(|s| s.to_string()).unwrap_or_default(), + state: state.to_string(), + }); + } + + Ok(res) +} + +async fn clone_drives() -> Vec> { + GLOBAL_LOCAL_DISK_MAP.read().await.values().cloned().collect::>() +} diff --git a/ecstore/src/pools.rs b/ecstore/src/pools.rs index e8d7f49ae..d3ffec997 100644 --- a/ecstore/src/pools.rs +++ b/ecstore/src/pools.rs @@ -58,18 +58,19 @@ pub struct PoolDecommissionInfo { pub bytes_failed: usize, } -struct PoolSpaceInfo { +#[derive(Debug)] +pub struct PoolSpaceInfo { pub free: usize, pub total: usize, pub used: usize, } impl ECStore { - pub fn status(&self, idx: usize) -> Result { + pub fn status(&self, _idx: usize) -> Result { unimplemented!() } - async fn get_decommission_pool_space_info(&self, idx: usize) -> Result { + async fn _get_decommission_pool_space_info(&self, idx: usize) -> Result { if let Some(sets) = self.pools.get(idx) { let mut info = sets.storage_info().await; info.backend = self.backend_info().await; @@ -81,8 +82,8 @@ impl ECStore { } } -fn get_total_usable_capacity(disks: &Vec, info: &StorageInfo) -> usize { - for disk in disks.iter() { +fn _get_total_usable_capacity(disks: &Vec, _info: &StorageInfo) -> usize { + for _disk in disks.iter() { // if disk.pool_index < 0 || info.backend.standard_scdata.len() <= disk.pool_index { // continue; // } diff --git a/ecstore/src/quorum.rs b/ecstore/src/quorum.rs index aeb652c78..84d36df58 100644 --- a/ecstore/src/quorum.rs +++ b/ecstore/src/quorum.rs @@ -40,6 +40,16 @@ pub fn object_op_ignored_errs() -> Vec> { base } +// bucket_op_ignored_errs +pub fn bucket_op_ignored_errs() -> Vec> { + let mut base = base_ignored_errs(); + + let ext: Vec> = vec![Box::new(DiskError::DiskAccessDenied), Box::new(DiskError::UnformattedDisk)]; + + base.extend(ext); + base +} + // 用于检查错误是否被忽略的函数 fn is_err_ignored(err: &Error, ignored_errs: &[Box]) -> bool { ignored_errs.iter().any(|ignored_err| ignored_err.is(err)) diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index 203e3a3ad..5f494c630 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -10,15 +10,16 @@ use uuid::Uuid; use crate::{ disk::{ + error::DiskError, format::{DistributionAlgoVersion, FormatV3}, - DiskAPI, DiskStore, + new_disk, DiskAPI, DiskInfo, DiskOption, DiskStore, }, - endpoints::PoolEndpoints, + endpoints::{Endpoints, PoolEndpoints}, error::{Error, Result}, global::{is_dist_erasure, GLOBAL_LOCAL_DISK_SET_DRIVES}, - heal::{ - heal_commands::{HealOpts, HealResultItem}, - heal_ops::HealObjectFn, + heal::heal_commands::{ + HealDriveInfo, HealOpts, HealResultItem, DRIVE_STATE_CORRUPT, DRIVE_STATE_MISSING, DRIVE_STATE_OFFLINE, DRIVE_STATE_OK, + HEAL_ITEM_METADATA, }, set_disk::SetDisks, store_api::{ @@ -26,9 +27,11 @@ use crate::{ ListMultipartsInfo, ListObjectsV2Info, MakeBucketOptions, MultipartUploadResult, ObjectIO, ObjectInfo, ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI, StorageInfo, }, + store_init::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file}, utils::hash, }; +use crate::heal::heal_ops::HealSequence; use tokio::time::Duration; use tokio_util::sync::CancellationToken; use tracing::info; @@ -177,7 +180,7 @@ impl Sets { self.connect_disks().await; // TODO: config interval - let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(15 * 3)); + let mut interval = tokio::time::interval(Duration::from_secs(15 * 3)); let cloned_token = self.ctx.clone(); loop { tokio::select! { @@ -523,7 +526,7 @@ impl StorageAPI for Sets { .complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts) .await } - async fn get_disks(&self, pool_idx: usize, set_idx: usize) -> Result>> { + async fn get_disks(&self, _pool_idx: usize, _set_idx: usize) -> Result>> { unimplemented!() } @@ -534,24 +537,226 @@ impl StorageAPI for Sets { async fn delete_bucket(&self, _bucket: &str, _opts: &DeleteBucketOptions) -> Result<()> { unimplemented!() } - async fn heal_format(&self, dry_run: bool) -> Result { + async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option)> { + let (disks, _) = init_storage_disks_with_errors( + &self.endpoints.endpoints, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await; + let (formats, errs) = load_format_erasure_all(&disks, true).await; + if let Err(err) = check_format_erasure_values(&formats, self.set_drive_count) { + return Ok((HealResultItem::default(), Some(err))); + } + let ref_format = match get_format_erasure_in_quorum(&formats) { + Ok(format) => format, + Err(err) => return Ok((HealResultItem::default(), Some(err))), + }; + let mut res = HealResultItem { + heal_item_type: HEAL_ITEM_METADATA.to_string(), + detail: "disk-format".to_string(), + disk_count: self.set_count * self.set_drive_count, + set_count: self.set_count, + ..Default::default() + }; + let before_derives = formats_to_drives_info(&self.endpoints.endpoints, &formats, &errs); + res.before = vec![HealDriveInfo::default(); before_derives.len()]; + res.after = vec![HealDriveInfo::default(); before_derives.len()]; + + for v in before_derives.iter() { + res.before.push(v.clone()); + res.after.push(v.clone()); + } + if DiskError::UnformattedDisk.count_errs(&errs) == 0 { + return Ok((res, Some(Error::new(DiskError::NoHealRequired)))); + } + + if !self.format.eq(&ref_format) { + return Ok((res, Some(Error::new(DiskError::CorruptedFormat)))); + } + + let format_op_id = Uuid::new_v4().to_string(); + let (new_format_sets, _) = new_heal_format_sets(&ref_format, self.set_count, self.set_drive_count, &formats, &errs); + if !dry_run { + let mut tmp_new_formats = vec![None; self.set_count * self.set_drive_count]; + for (i, set) in new_format_sets.iter().enumerate() { + for (j, fm) in set.iter().enumerate() { + if let Some(fm) = fm { + res.after[i * self.set_drive_count + j].uuid = fm.erasure.this.to_string(); + res.after[i * self.set_drive_count + j].state = DRIVE_STATE_OK.to_string(); + tmp_new_formats[i * self.set_drive_count + j] = Some(fm.clone()); + } + } + } + // Save new formats `format.json` on unformatted disks. + for (fm, disk) in tmp_new_formats.iter_mut().zip(disks.iter()) { + if fm.is_some() && disk.is_some() && save_format_file(disk, fm, &format_op_id).await.is_err() { + let _ = disk.as_ref().unwrap().close().await; + *fm = None; + } + } + + for (index, fm) in tmp_new_formats.iter().enumerate() { + if let Some(fm) = fm { + let (m, n) = match ref_format.find_disk_index_by_disk_id(fm.erasure.this) { + Ok((m, n)) => (m, n), + Err(_) => continue, + }; + if let Some(set) = self.disk_set.get(m) { + if let Some(Some(disk)) = set.disks.read().await.get(n) { + let _ = disk.close().await; + } + } + + if let Some(Some(disk)) = disks.get(index) { + self.disk_set[m].renew_disk(&disk.endpoint()).await; + } + } + } + } + Ok((res, None)) + } + async fn heal_bucket(&self, _bucket: &str, _opts: &HealOpts) -> Result { unimplemented!() } - async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result { - unimplemented!() - } - async fn heal_object(&self, bucket: &str, object: &str, version_id: &str, opts: &HealOpts) -> Result { + async fn heal_object( + &self, + bucket: &str, + object: &str, + version_id: &str, + opts: &HealOpts, + ) -> Result<(HealResultItem, Option)> { self.get_disks_by_key(object) .heal_object(bucket, object, version_id, opts) .await } - async fn heal_objects(&self, bucket: &str, prefix: &str, opts: &HealOpts, func: HealObjectFn) -> Result<()> { + async fn heal_objects( + &self, + _bucket: &str, + _prefix: &str, + _opts: &HealOpts, + _hs: Arc>, + _is_meta: bool, + ) -> Result<()> { unimplemented!() } - async fn get_pool_and_set(&self, id: &str) -> Result<(Option, Option, Option)> { + async fn get_pool_and_set(&self, _id: &str) -> Result<(Option, Option, Option)> { unimplemented!() } - async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> { + async fn check_abandoned_parts(&self, _bucket: &str, _object: &str, _opts: &HealOpts) -> Result<()> { unimplemented!() } } + +async fn _close_storage_disks(disks: &[Option]) { + let mut futures = Vec::with_capacity(disks.len()); + for disk in disks.iter().flatten() { + let disk = disk.clone(); + futures.push(tokio::spawn(async move { + let _ = disk.close().await; + })); + } + let _ = join_all(futures).await; +} + +async fn init_storage_disks_with_errors( + endpoints: &Endpoints, + opts: &DiskOption, +) -> (Vec>, Vec>) { + // Bootstrap disks. + let disks = Arc::new(RwLock::new(vec![None; endpoints.as_ref().len()])); + let errs = Arc::new(RwLock::new(vec![None; endpoints.as_ref().len()])); + let mut futures = Vec::with_capacity(endpoints.as_ref().len()); + for (index, endpoint) in endpoints.as_ref().iter().enumerate() { + let ep = endpoint.clone(); + let opt = opts.clone(); + let disks_clone = disks.clone(); + let errs_clone = errs.clone(); + futures.push(tokio::spawn(async move { + match new_disk(&ep, &opt).await { + Ok(disk) => { + disks_clone.write().await[index] = Some(disk); + errs_clone.write().await[index] = None; + } + Err(err) => { + disks_clone.write().await[index] = None; + errs_clone.write().await[index] = Some(err); + } + } + })); + } + let _ = join_all(futures).await; + let disks = disks.read().await.clone(); + let errs = errs.read().await.clone(); + (disks, errs) +} + +fn formats_to_drives_info(endpoints: &Endpoints, formats: &[Option], errs: &[Option]) -> Vec { + let mut before_drives = Vec::with_capacity(endpoints.as_ref().len()); + for (index, format) in formats.iter().enumerate() { + let drive = endpoints.get_string(index); + let state = if format.is_some() { + DRIVE_STATE_OK + } else { + if let Some(Some(err)) = errs.get(index) { + match err.downcast_ref::() { + Some(DiskError::UnformattedDisk) => DRIVE_STATE_MISSING, + Some(DiskError::DiskNotFound) => DRIVE_STATE_OFFLINE, + _ => DRIVE_STATE_CORRUPT, + }; + } + DRIVE_STATE_CORRUPT + }; + + let uuid = if let Some(format) = format { + format.erasure.this.to_string() + } else { + "".to_string() + }; + before_drives.push(HealDriveInfo { + uuid, + endpoint: drive, + state: state.to_string(), + }); + } + before_drives +} + +fn new_heal_format_sets( + ref_format: &FormatV3, + set_count: usize, + set_drive_count: usize, + formats: &[Option], + errs: &[Option], +) -> (Vec>>, Vec>) { + let mut new_formats = vec![vec![None; set_drive_count]; set_count]; + let mut current_disks_info = vec![vec![DiskInfo::default(); set_drive_count]; set_count]; + for (i, set) in ref_format.erasure.sets.iter().enumerate() { + for j in 0..set.len() { + if let Some(Some(err)) = errs.get(i * set_drive_count + j) { + if let Some(DiskError::UnformattedDisk) = err.downcast_ref::() { + let mut fm = FormatV3::new(set_count, set_drive_count); + fm.id = ref_format.id; + fm.format = ref_format.format.clone(); + fm.version = ref_format.version.clone(); + fm.erasure.this = ref_format.erasure.sets[i][j]; + fm.erasure.sets = ref_format.erasure.sets.clone(); + fm.erasure.version = ref_format.erasure.version.clone(); + fm.erasure.distribution_algo = ref_format.erasure.distribution_algo.clone(); + new_formats[i][j] = Some(fm); + } + } + if let (Some(format), None) = (&formats[i * set_drive_count + j], &errs[i * set_drive_count + j]) { + if let Some(info) = &format.disk_info { + if !info.endpoint.is_empty() { + current_disks_info[i][j] = info.clone(); + } + } + } + } + } + + (new_formats, current_disks_info) +} diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index 1057c707d..fe5d517fa 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -1,18 +1,19 @@ #![allow(clippy::map_entry)] -use crate::bucket::metadata; use crate::bucket::metadata_sys::{self, init_bucket_metadata_sys, set_bucket_metadata}; use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname}; use crate::config::GLOBAL_StorageClass; use crate::config::{self, storageclass, GLOBAL_ConfigSys}; -use crate::disk::endpoint::EndpointType; +use crate::disk::endpoint::{Endpoint, EndpointType}; use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions, MetaCacheEntry}; use crate::global::{ is_dist_erasure, is_erasure_sd, set_global_deployment_id, set_object_layer, DISK_ASSUME_UNKNOWN_SIZE, DISK_FILL_FRACTION, DISK_MIN_INODES, DISK_RESERVE_FRACTION, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, }; -use crate::heal::heal_commands::{HealOpts, HealResultItem, HealScanMode}; -use crate::heal::heal_ops::HealObjectFn; +use crate::heal::data_usage::{DataUsageInfo, DATA_USAGE_ROOT}; +use crate::heal::data_usage_cache::{DataUsageCache, DataUsageCacheInfo}; +use crate::heal::heal_commands::{HealOpts, HealResultItem, HealScanMode, HEAL_ITEM_METADATA}; +use crate::heal::heal_ops::{HealEntryFn, HealSequence}; use crate::new_object_layer_fn; use crate::pools::PoolMeta; use crate::store_api::{BackendByte, BackendDisks, BackendInfo, ListMultipartsInfo, ObjectIO, StorageInfo}; @@ -45,18 +46,21 @@ use http::HeaderMap; use lazy_static::lazy_static; use rand::Rng; use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration}; +use s3s::{S3Error, S3ErrorCode}; use std::cmp::Ordering; use std::slice::Iter; +use std::time::SystemTime; use std::{ collections::{HashMap, HashSet}, sync::Arc, time::Duration, }; use time::OffsetDateTime; -use tokio::fs; -use tokio::sync::{RwLock, Semaphore}; - -use tracing::{debug, info, warn}; +use tokio::sync::mpsc::Sender; +use tokio::sync::{broadcast, mpsc, RwLock, Semaphore}; +use tokio::time::interval; +use tokio::{fs, select}; +use tracing::{debug, info}; use uuid::Uuid; const MAX_UPLOADS_LIST: usize = 10000; @@ -494,6 +498,103 @@ impl ECStore { internal_get_pool_info_existing_with_opts(&self.pools, bucket, object, opts).await } + pub async fn ns_scanner( + &self, + updates: Sender, + want_cycle: usize, + heal_scan_mode: HealScanMode, + ) -> Result<()> { + let all_buckets = self.list_bucket(&BucketOptions::default()).await?; + if all_buckets.is_empty() { + let _ = updates.send(DataUsageInfo::default()).await; + return Ok(()); + } + + let mut total_results = 0; + let mut result_index = 0; + self.pools.iter().for_each(|pool| { + total_results += pool.disk_set.len(); + }); + let results = Arc::new(RwLock::new(vec![DataUsageCache::default(); total_results])); + let (cancel, _) = broadcast::channel(100); + let first_err = Arc::new(RwLock::new(None)); + let mut futures = Vec::new(); + for pool in self.pools.iter() { + for set in pool.disk_set.iter() { + let index = result_index; + let results_clone = results.clone(); + let first_err_clone = first_err.clone(); + let cancel_clone = cancel.clone(); + let all_buckets_clone = all_buckets.clone(); + futures.push(async move { + let (tx, mut rx) = mpsc::channel(100); + let task = tokio::spawn(async move { + loop { + match rx.recv().await { + Some(info) => { + results_clone.write().await[index] = info; + } + None => { + return; + } + } + } + }); + if let Err(err) = set + .ns_scanner(&all_buckets_clone, want_cycle as u32, tx, heal_scan_mode) + .await + { + let mut f_w = first_err_clone.write().await; + if f_w.is_none() { + *f_w = Some(err); + } + let _ = cancel_clone.send(true); + return; + } + let _ = task.await; + }); + result_index += 1; + } + } + let (update_closer_tx, mut update_close_rx) = mpsc::channel(10); + let mut ctx_clone = cancel.subscribe(); + let all_buckets_clone = all_buckets.clone(); + let task = tokio::spawn(async move { + let mut last_update: Option = None; + let mut interval = interval(Duration::from_secs(30)); + let all_merged = Arc::new(RwLock::new(DataUsageCache::default())); + loop { + select! { + _ = ctx_clone.recv() => { + return; + } + _ = update_close_rx.recv() => { + update_scan(all_merged.clone(), results.clone(), &mut last_update, all_buckets_clone.clone(), updates.clone()).await; + return; + } + _ = interval.tick() => { + update_scan(all_merged.clone(), results.clone(), &mut last_update, all_buckets_clone.clone(), updates.clone()).await; + } + } + } + }); + let _ = join_all(futures).await; + let mut ctx_closer = cancel.subscribe(); + select! { + _ = update_closer_tx.send(true) => { + + } + _ = ctx_closer.recv() => { + + } + } + let _ = task.await; + if let Some(err) = first_err.read().await.as_ref() { + return Err(err.clone()); + } + Ok(()) + } + async fn get_latest_object_info_with_idx( &self, bucket: &str, @@ -515,8 +616,7 @@ impl ECStore { let mut idx_res = Vec::with_capacity(self.pools.len()); - let mut idx = 0; - for result in results { + for (idx, result) in results.into_iter().enumerate() { match result { Ok(res) => { idx_res.push(IndexRes { @@ -533,8 +633,6 @@ impl ECStore { }); } } - - idx += 1; } // TODO: test order @@ -589,6 +687,33 @@ impl ECStore { } } +async fn update_scan( + all_merged: Arc>, + results: Arc>>, + last_update: &mut Option, + all_buckets: Vec, + updates: Sender, +) { + let mut w = all_merged.write().await; + *w = DataUsageCache { + info: DataUsageCacheInfo { + name: DATA_USAGE_ROOT.to_string(), + ..Default::default() + }, + ..Default::default() + }; + for info in results.read().await.iter() { + if info.info.last_update.is_none() { + return; + } + w.merge(info); + } + if w.info.last_update > *last_update && w.root().is_none() { + let _ = updates.send(w.dui(&w.info.name, &all_buckets)).await; + *last_update = w.info.last_update; + } +} + pub async fn find_local_disk(disk_path: &String) -> Option { let disk_path = match fs::canonicalize(disk_path).await { Ok(disk_path) => disk_path, @@ -606,6 +731,14 @@ pub async fn find_local_disk(disk_path: &String) -> Option { None } +pub async fn get_disk_via_endpoint(endpoint: &Endpoint) -> Option { + let global_set_drives = GLOBAL_LOCAL_DISK_SET_DRIVES.read().await; + if global_set_drives.is_empty() { + return GLOBAL_LOCAL_DISK_MAP.read().await[&endpoint.to_string()].clone(); + } + global_set_drives[endpoint.pool_idx as usize][endpoint.set_idx as usize][endpoint.disk_idx as usize].clone() +} + pub async fn all_local_disk_path() -> Vec { let disk_map = GLOBAL_LOCAL_DISK_MAP.read().await; disk_map.keys().cloned().collect() @@ -1306,7 +1439,7 @@ impl StorageAPI for ECStore { Ok(ListMultipartsInfo { key_marker: key_marker.to_owned(), upload_id_marker: upload_id_marker.to_owned(), - max_uploads: max_uploads, + max_uploads, uploads, prefix: prefix.to_owned(), delimiter: delimiter.to_owned(), @@ -1439,53 +1572,187 @@ impl StorageAPI for ECStore { } counts } - async fn heal_format(&self, dry_run: bool) -> Result { - unimplemented!() + async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option)> { + let mut r = HealResultItem { + heal_item_type: HEAL_ITEM_METADATA.to_string(), + detail: "disk-format".to_string(), + ..Default::default() + }; + + let mut count_no_heal = 0; + for pool in self.pools.iter() { + let (mut result, err) = pool.heal_format(dry_run).await?; + if let Some(err) = err { + match err.downcast_ref::() { + Some(DiskError::NoHealRequired) => { + count_no_heal += 1; + } + _ => { + continue; + } + } + } + r.disk_count += result.disk_count; + r.set_count += result.set_count; + r.before.append(&mut result.before); + r.after.append(&mut result.after); + } + if count_no_heal == self.pools.len() { + return Ok((r, Some(Error::new(DiskError::NoHealRequired)))); + } + Ok((r, None)) } async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result { - unimplemented!() + self.peer_sys.heal_bucket(bucket, opts).await } - async fn heal_object(&self, bucket: &str, object: &str, version_id: &str, opts: &HealOpts) -> Result { + async fn heal_object( + &self, + bucket: &str, + object: &str, + version_id: &str, + opts: &HealOpts, + ) -> Result<(HealResultItem, Option)> { let object = utils::path::encode_dir_object(object); - let mut errs = HashMap::new(); - let mut results = HashMap::new(); + let errs = Arc::new(RwLock::new(vec![None; self.pools.len()])); + let results = Arc::new(RwLock::new(vec![HealResultItem::default(); self.pools.len()])); + let mut futures = Vec::with_capacity(self.pools.len()); for (idx, pool) in self.pools.iter().enumerate() { //TODO: IsSuspended - match pool.heal_object(bucket, &object, version_id, opts).await { - Ok(mut result) => { - result.object = utils::path::decode_dir_object(&result.object); - results.insert(idx, result); + let object = object.clone(); + let results = results.clone(); + let errs = errs.clone(); + futures.push(async move { + match pool.heal_object(bucket, &object, version_id, opts).await { + Ok((mut result, err)) => { + result.object = utils::path::decode_dir_object(&result.object); + results.write().await.insert(idx, result); + errs.write().await.insert(idx, err); + } + Err(err) => { + errs.write().await.insert(idx, Some(err)); + } } - Err(err) => { - errs.insert(idx, err); - } - } + }); } + let _ = join_all(futures).await; // Return the first nil error - for i in 0..self.pools.len() { - if !errs.contains_key(&i) { - return Ok(results.remove(&i).unwrap()); + for (index, err) in errs.read().await.iter().enumerate() { + if err.is_none() { + return Ok((results.write().await.remove(index), None)); } } // No pool returned a nil error, return the first non 'not found' error - for (k, err) in errs.iter() { - match err.downcast_ref::() { - Some(DiskError::FileNotFound) | Some(DiskError::FileVersionNotFound) => {} - _ => return Ok(results.remove(k).unwrap()), + for (index, err) in errs.read().await.iter().enumerate() { + match err { + Some(err) => match err.downcast_ref::() { + Some(DiskError::FileNotFound) | Some(DiskError::FileVersionNotFound) => {} + _ => return Ok((results.write().await.remove(index), Some(err.clone()))), + }, + None => { + return Ok((results.write().await.remove(index), None)); + } } } // At this stage, all errors are 'not found' if !version_id.is_empty() { - return Err(Error::new(DiskError::FileVersionNotFound)); + return Ok((HealResultItem::default(), Some(Error::new(DiskError::FileVersionNotFound)))); } - Err(Error::new(DiskError::FileNotFound)) + Ok((HealResultItem::default(), Some(Error::new(DiskError::FileNotFound)))) } - async fn heal_objects(&self, bucket: &str, prefix: &str, opts: &HealOpts, func: HealObjectFn) -> Result<()> { + async fn heal_objects( + &self, + bucket: &str, + prefix: &str, + opts: &HealOpts, + hs: Arc>, + is_meta: bool, + ) -> Result<()> { + let opts_clone = *opts; + let heal_entry: HealEntryFn = Arc::new(move |bucket: String, entry: MetaCacheEntry, scan_mode: HealScanMode| { + let opts_clone = opts_clone; + let hs_clone = hs.clone(); + Box::pin(async move { + if entry.is_dir() { + return Ok(()); + } + + if bucket == RUSTFS_META_BUCKET + && Pattern::new("buckets/*/.metacache/*") + .map(|p| p.matches(&entry.name)) + .unwrap_or(false) + || Pattern::new("tmp/*").map(|p| p.matches(&entry.name)).unwrap_or(false) + || Pattern::new("multipart/*").map(|p| p.matches(&entry.name)).unwrap_or(false) + || Pattern::new("tmp-old/*").map(|p| p.matches(&entry.name)).unwrap_or(false) + { + return Ok(()); + } + let fivs = match entry.file_info_versions(&bucket) { + Ok(fivs) => fivs, + Err(_) => { + if is_meta { + return HealSequence::heal_meta_object(hs_clone.clone(), &bucket, &entry.name, "", scan_mode).await; + } else { + return HealSequence::heal_object(hs_clone.clone(), &bucket, &entry.name, "", scan_mode).await; + } + } + }; + + if opts_clone.remove && !opts_clone.dry_run { + let layer = new_object_layer_fn(); + let lock = layer.read().await; + let store = match lock.as_ref() { + Some(s) => s, + None => { + return Err(Error::from(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))) + } + }; + if let Err(err) = store.check_abandoned_parts(&bucket, &entry.name, &opts_clone).await { + info!("unable to check object {}/{} for abandoned data: {}", bucket, entry.name, err.to_string()); + } + } + for version in fivs.versions.iter() { + if is_meta { + if let Err(err) = HealSequence::heal_meta_object( + hs_clone.clone(), + &bucket, + &version.name, + &version.version_id.map(|v| v.to_string()).unwrap_or("".to_string()), + scan_mode, + ) + .await + { + match err.downcast_ref() { + Some(DiskError::FileNotFound) | Some(DiskError::FileVersionNotFound) => {} + _ => { + return Err(err); + } + } + } + } else if let Err(err) = HealSequence::heal_object( + hs_clone.clone(), + &bucket, + &version.name, + &version.version_id.map(|v| v.to_string()).unwrap_or("".to_string()), + scan_mode, + ) + .await + { + match err.downcast_ref() { + Some(DiskError::FileNotFound) | Some(DiskError::FileVersionNotFound) => {} + _ => { + return Err(err); + } + } + } + } + Ok(()) + }) + }); let mut first_err = None; for (idx, pool) in self.pools.iter().enumerate() { if opts.pool.is_some() && opts.pool.unwrap() != idx { @@ -1498,7 +1765,7 @@ impl StorageAPI for ECStore { continue; } - if let Err(err) = set.list_and_heal(bucket, prefix, opts, func.clone()).await { + if let Err(err) = set.list_and_heal(bucket, prefix, opts, heal_entry.clone()).await { if first_err.is_none() { first_err = Some(err) } @@ -1736,67 +2003,6 @@ fn check_put_object_args(bucket: &str, object: &str) -> Result<()> { Ok(()) } -pub async fn heal_entry( - bucket: String, - entry: MetaCacheEntry, - scan_mode: HealScanMode, - opts: HealOpts, - func: HealObjectFn, -) -> Result<()> { - if entry.is_dir() { - return Ok(()); - } - - // We might land at .metacache, .trash, .multipart - // no need to heal them skip, only when bucket - // is '.rustfs.sys' - if bucket == RUSTFS_META_BUCKET { - if Pattern::new("buckets/*/.metacache/*") - .map(|p| p.matches(&entry.name)) - .unwrap_or(false) - || Pattern::new("tmp/*").map(|p| p.matches(&entry.name)).unwrap_or(false) - || Pattern::new("multipart/*").map(|p| p.matches(&entry.name)).unwrap_or(false) - || Pattern::new("tmp-old/*").map(|p| p.matches(&entry.name)).unwrap_or(false) - { - return Ok(()); - } - } - - let layer = new_object_layer_fn(); - let lock = layer.read().await; - let store = match lock.as_ref() { - Some(s) => s, - None => return Err(Error::msg("errServerNotInitialized")), - }; - - match entry.file_info_versions(&bucket) { - Ok(fivs) => { - if opts.remove && !opts.dry_run { - if let Err(err) = store.check_abandoned_parts(&bucket, &entry.name, &opts).await { - return Err(Error::from_string(format!( - "unable to check object {}/{} for abandoned data: {}", - bucket, entry.name, err - ))); - } - } - - for version in fivs.versions.iter() { - let version_id = version.version_id.map_or("".to_string(), |version_id| version_id.to_string()); - if let Err(err) = func(&bucket, &entry.name, &version_id, scan_mode) { - match err.downcast_ref::() { - Some(DiskError::FileNotFound) | Some(DiskError::FileVersionNotFound) => {} - _ => return Err(err), - } - } - } - } - Err(_) => { - return func(&bucket, &entry.name, "", scan_mode); - } - } - Ok(()) -} - async fn get_disk_infos(disks: &[Option]) -> Vec> { let opts = &DiskInfoOptions::default(); let mut res = vec![None; disks.len()]; diff --git a/ecstore/src/store_api.rs b/ecstore/src/store_api.rs index 16cf36b38..2ba047b58 100644 --- a/ecstore/src/store_api.rs +++ b/ecstore/src/store_api.rs @@ -1,12 +1,8 @@ -use std::collections::HashMap; - +use crate::heal::heal_ops::HealSequence; use crate::{ disk::DiskStore, error::{Error, Result}, - heal::{ - heal_commands::{HealOpts, HealResultItem}, - heal_ops::HealObjectFn, - }, + heal::heal_commands::{HealOpts, HealResultItem}, utils::path::decode_dir_object, xhttp, }; @@ -15,13 +11,18 @@ use http::HeaderMap; use rmp_serde::Serializer; use s3s::dto::StreamingBlob; use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::sync::Arc; use time::OffsetDateTime; +use tokio::sync::RwLock; use uuid::Uuid; pub const ERASURE_ALGORITHM: &str = "rs-vandermonde"; pub const BLOCK_SIZE_V2: usize = 1048576; // 1M pub const RESERVED_METADATA_PREFIX: &str = "X-Rustfs-Internal-"; pub const RESERVED_METADATA_PREFIX_LOWER: &str = "X-Rustfs-Internal-"; +pub const RUSTFS_HEALING: &str = "X-Rustfs-Internal-healing"; +pub const RUSTFS_DATA_MOVE: &str = "X-Rustfs-Internal-data-mov"; // #[derive(Debug, Clone)] #[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)] @@ -237,6 +238,16 @@ impl FileInfo { Err(Error::msg("part not found")) } + pub fn set_healing(&mut self) { + if self.metadata.is_none() { + self.metadata = Some(HashMap::new()); + } + + if let Some(metadata) = self.metadata.as_mut() { + metadata.insert(RUSTFS_HEALING.to_string(), "true".to_string()); + } + } + pub fn set_inline_data(&mut self) { if let Some(meta) = self.metadata.as_mut() { meta.insert("x-rustfs-inline-data".to_owned(), "true".to_owned()); @@ -978,10 +989,23 @@ pub trait StorageAPI: ObjectIO { async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result; async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; - async fn heal_format(&self, dry_run: bool) -> Result; + async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option)>; async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result; - async fn heal_object(&self, bucket: &str, object: &str, version_id: &str, opts: &HealOpts) -> Result; - async fn heal_objects(&self, bucket: &str, prefix: &str, opts: &HealOpts, func: HealObjectFn) -> Result<()>; + async fn heal_object( + &self, + bucket: &str, + object: &str, + version_id: &str, + opts: &HealOpts, + ) -> Result<(HealResultItem, Option)>; + async fn heal_objects( + &self, + bucket: &str, + prefix: &str, + opts: &HealOpts, + hs: Arc>, + is_meta: bool, + ) -> Result<()>; async fn get_pool_and_set(&self, id: &str) -> Result<(Option, Option, Option)>; async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()>; } diff --git a/ecstore/src/store_init.rs b/ecstore/src/store_init.rs index 6d8f7dd53..583c5546a 100644 --- a/ecstore/src/store_init.rs +++ b/ecstore/src/store_init.rs @@ -4,10 +4,11 @@ use crate::{ disk::{ error::DiskError, format::{FormatErasureVersion, FormatMetaVersion, FormatV3}, - new_disk, DiskOption, DiskStore, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET, + new_disk, DiskInfoOptions, DiskOption, DiskStore, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET, }, endpoints::Endpoints, error::{Error, Result}, + heal::heal_commands::init_healing_tracker, }; use futures::future::join_all; use std::{ @@ -114,7 +115,7 @@ fn init_format_erasure( fms } -fn get_format_erasure_in_quorum(formats: &[Option]) -> Result { +pub fn get_format_erasure_in_quorum(formats: &[Option]) -> Result { let mut countmap = HashMap::new(); for f in formats.iter() { @@ -148,7 +149,7 @@ fn get_format_erasure_in_quorum(formats: &[Option]) -> Result], // disks: &Vec>, set_drive_count: usize, @@ -184,7 +185,7 @@ fn check_format_erasure_value(format: &FormatV3) -> Result<()> { } // load_format_erasure_all 读取所有foramt.json -async fn load_format_erasure_all(disks: &[Option], heal: bool) -> (Vec>, Vec>) { +pub async fn load_format_erasure_all(disks: &[Option], heal: bool) -> (Vec>, Vec>) { let mut futures = Vec::with_capacity(disks.len()); let mut datas = Vec::with_capacity(disks.len()); let mut errors = Vec::with_capacity(disks.len()); @@ -220,7 +221,7 @@ async fn load_format_erasure_all(disks: &[Option], heal: bool) -> (Ve (datas, errors) } -pub async fn load_format_erasure(disk: &DiskStore, _heal: bool) -> Result { +pub async fn load_format_erasure(disk: &DiskStore, heal: bool) -> Result { let data = disk .read_all(RUSTFS_META_BUCKET, FORMAT_CONFIG_FILE) .await @@ -231,9 +232,17 @@ pub async fn load_format_erasure(disk: &DiskStore, _heal: bool) -> Result e, })?; - let fm = FormatV3::try_from(data.as_slice())?; + let mut fm = FormatV3::try_from(data.as_slice())?; - // TODO: heal + if heal { + let info = disk + .disk_info(&DiskInfoOptions { + noop: heal, + ..Default::default() + }) + .await?; + fm.disk_info = Some(info); + } Ok(fm) } @@ -242,7 +251,7 @@ async fn save_format_file_all(disks: &[Option], formats: &[Option], formats: &[Option, format: &Option) -> Result<()> { +pub async fn save_format_file(disk: &Option, format: &Option, heal_id: &str) -> Result<()> { if disk.is_none() { return Err(Error::new(DiskError::DiskNotFound)); } @@ -281,6 +290,10 @@ async fn save_format_file(disk: &Option, format: &Option) - .await?; disk.set_disk_id(Some(format.erasure.this)).await?; + if !heal_id.is_empty() { + let mut ht = init_healing_tracker(disk.clone(), heal_id).await?; + return ht.save().await; + } Ok(()) } diff --git a/ecstore/src/utils/bool_flag.rs b/ecstore/src/utils/bool_flag.rs new file mode 100644 index 000000000..45f4a4b11 --- /dev/null +++ b/ecstore/src/utils/bool_flag.rs @@ -0,0 +1,9 @@ +use crate::error::{Error, Result}; + +pub fn parse_bool(str: &str) -> Result { + match str { + "1" | "t" | "T" | "true" | "TRUE" | "True" | "on" | "ON" | "On" | "enabled" => Ok(true), + "0" | "f" | "F" | "false" | "FALSE" | "False" | "off" | "OFF" | "Off" | "disabled" => Ok(false), + _ => Err(Error::from_string(format!("ParseBool: parsing {}", str))), + } +} diff --git a/ecstore/src/utils/ellipses.rs b/ecstore/src/utils/ellipses.rs index 9badb80ce..07c33c1cf 100644 --- a/ecstore/src/utils/ellipses.rs +++ b/ecstore/src/utils/ellipses.rs @@ -40,6 +40,10 @@ impl Pattern { pub fn len(&self) -> usize { self.seq.len() } + + pub fn is_empty(&self) -> bool { + self.seq.is_empty() + } } /// contains a list of patterns provided in the input. diff --git a/ecstore/src/utils/mod.rs b/ecstore/src/utils/mod.rs index daff92a6c..65aac72b2 100644 --- a/ecstore/src/utils/mod.rs +++ b/ecstore/src/utils/mod.rs @@ -1,3 +1,4 @@ +pub mod bool_flag; pub mod crypto; pub mod ellipses; pub mod fs; diff --git a/ecstore/src/utils/path.rs b/ecstore/src/utils/path.rs index b505bd72f..654333ec2 100644 --- a/ecstore/src/utils/path.rs +++ b/ecstore/src/utils/path.rs @@ -69,6 +69,19 @@ pub fn path_join(elem: &[PathBuf]) -> PathBuf { joined_path } +pub fn path_to_bucket_object_with_base_path(bash_path: &str, path: &str) -> (String, String) { + let path = path.trim_start_matches(bash_path).trim_start_matches(SLASH_SEPARATOR); + if let Some(m) = path.find(SLASH_SEPARATOR) { + return (path[..m].to_string(), path[m + SLASH_SEPARATOR.len()..].to_string()); + } + + (path.to_string(), "".to_string()) +} + +pub fn path_to_bucket_object(s: &str) -> (String, String) { + path_to_bucket_object_with_base_path("", s) +} + pub fn base_dir_from_prefix(prefix: &str) -> String { let mut base_dir = dir(prefix).to_owned(); if base_dir == "." || base_dir == "./" || base_dir == "/" { diff --git a/ecstore/src/utils/wildcard.rs b/ecstore/src/utils/wildcard.rs index e46e118de..8e178d84f 100644 --- a/ecstore/src/utils/wildcard.rs +++ b/ecstore/src/utils/wildcard.rs @@ -1,3 +1,5 @@ +use crate::disk::RUSTFS_META_BUCKET; + pub fn match_simple(pattern: &str, name: &str) -> bool { if pattern.is_empty() { return name == pattern; @@ -65,3 +67,7 @@ pub fn match_as_pattern_prefix(pattern: &str, text: &str) -> bool { } text.len() <= pattern.len() } + +pub fn is_rustfs_meta_bucket_name(bucket: &str) -> bool { + bucket.starts_with(RUSTFS_META_BUCKET) +} diff --git a/rustfs/src/grpc.rs b/rustfs/src/grpc.rs index c17f67e67..0567f66f9 100644 --- a/rustfs/src/grpc.rs +++ b/rustfs/src/grpc.rs @@ -6,6 +6,7 @@ use ecstore::{ UpdateMetadataOpts, WalkDirOptions, }, erasure::Writer, + heal::{data_usage_cache::DataUsageCache, heal_commands::HealOpts}, peer::{LocalPeerS3Client, PeerS3Client}, store::{all_local_disk_path, find_local_disk}, store_api::{BucketOptions, DeleteBucketOptions, FileInfo, MakeBucketOptions}, @@ -20,14 +21,14 @@ use protos::{ DeleteBucketResponse, DeletePathsRequest, DeletePathsResponse, DeleteRequest, DeleteResponse, DeleteVersionRequest, DeleteVersionResponse, DeleteVersionsRequest, DeleteVersionsResponse, DeleteVolumeRequest, DeleteVolumeResponse, DiskInfoRequest, DiskInfoResponse, GenerallyLockRequest, GenerallyLockResponse, GetBucketInfoRequest, - GetBucketInfoResponse, ListBucketRequest, ListBucketResponse, ListDirRequest, ListDirResponse, ListVolumesRequest, - ListVolumesResponse, MakeBucketRequest, MakeBucketResponse, MakeVolumeRequest, MakeVolumeResponse, MakeVolumesRequest, - MakeVolumesResponse, PingRequest, PingResponse, ReadAllRequest, ReadAllResponse, ReadAtRequest, ReadAtResponse, - ReadMultipleRequest, ReadMultipleResponse, ReadVersionRequest, ReadVersionResponse, ReadXlRequest, ReadXlResponse, - RenameDataRequest, RenameDataResponse, RenameFileRequst, RenameFileResponse, RenamePartRequst, RenamePartResponse, - StatVolumeRequest, StatVolumeResponse, UpdateMetadataRequest, UpdateMetadataResponse, VerifyFileRequest, - VerifyFileResponse, WalkDirRequest, WalkDirResponse, WriteAllRequest, WriteAllResponse, WriteMetadataRequest, - WriteMetadataResponse, WriteRequest, WriteResponse, + GetBucketInfoResponse, HealBucketRequest, HealBucketResponse, ListBucketRequest, ListBucketResponse, ListDirRequest, + ListDirResponse, ListVolumesRequest, ListVolumesResponse, MakeBucketRequest, MakeBucketResponse, MakeVolumeRequest, + MakeVolumeResponse, MakeVolumesRequest, MakeVolumesResponse, NsScannerRequest, NsScannerResponse, PingRequest, + PingResponse, ReadAllRequest, ReadAllResponse, ReadAtRequest, ReadAtResponse, ReadMultipleRequest, ReadMultipleResponse, + ReadVersionRequest, ReadVersionResponse, ReadXlRequest, ReadXlResponse, RenameDataRequest, RenameDataResponse, + RenameFileRequst, RenameFileResponse, RenamePartRequst, RenamePartResponse, StatVolumeRequest, StatVolumeResponse, + UpdateMetadataRequest, UpdateMetadataResponse, VerifyFileRequest, VerifyFileResponse, WalkDirRequest, WalkDirResponse, + WriteAllRequest, WriteAllResponse, WriteMetadataRequest, WriteMetadataResponse, WriteRequest, WriteResponse, }, }; use tokio::sync::mpsc; @@ -109,6 +110,32 @@ impl Node for NodeService { })) } + async fn heal_bucket(&self, request: Request) -> Result, Status> { + debug!("heal bucket"); + let request = request.into_inner(); + let options = match serde_json::from_str::(&request.options) { + Ok(options) => options, + Err(err) => { + return Ok(tonic::Response::new(HealBucketResponse { + success: false, + error_info: Some(format!("decode HealOpts failed: {}", err)), + })) + } + }; + + match self.local_peer.heal_bucket(&request.bucket, &options).await { + Ok(_) => Ok(tonic::Response::new(HealBucketResponse { + success: true, + error_info: None, + })), + + Err(err) => Ok(tonic::Response::new(HealBucketResponse { + success: false, + error_info: Some(format!("heal bucket failed: {}", err)), + })), + } + } + async fn list_bucket(&self, request: Request) -> Result, Status> { debug!("list bucket"); @@ -1270,6 +1297,96 @@ impl Node for NodeService { } } + type NsScannerStream = ResponseStream; + async fn ns_scanner(&self, request: Request>) -> Result, Status> { + info!("ns_scanner"); + + let mut in_stream = request.into_inner(); + let (tx, rx) = mpsc::channel(10); + + tokio::spawn(async move { + match in_stream.next().await { + Some(Ok(request)) => { + if let Some(disk) = find_local_disk(&request.disk).await { + let cache = match serde_json::from_str::(&request.cache) { + Ok(cache) => cache, + Err(_) => { + tx.send(Ok(NsScannerResponse { + success: false, + update: "".to_string(), + data_usage_cache: "".to_string(), + error_info: Some("can not decode DataUsageCache".to_string()), + })) + .await + .expect("working rx"); + return; + } + }; + let (updates_tx, mut updates_rx) = mpsc::channel(100); + let tx_clone = tx.clone(); + let task = tokio::spawn(async move { + loop { + match updates_rx.recv().await { + Some(update) => { + let update = serde_json::to_string(&update).expect("encode failed"); + tx_clone + .send(Ok(NsScannerResponse { + success: true, + update, + data_usage_cache: "".to_string(), + error_info: Some("can not decode DataUsageCache".to_string()), + })) + .await + .expect("working rx"); + } + None => return, + } + } + }); + let data_usage_cache = disk.ns_scanner(&cache, updates_tx, request.scan_mode as usize).await; + let _ = task.await; + match data_usage_cache { + Ok(data_usage_cache) => { + let data_usage_cache = serde_json::to_string(&data_usage_cache).expect("encode failed"); + tx.send(Ok(NsScannerResponse { + success: true, + update: "".to_string(), + data_usage_cache, + error_info: Some("can not decode DataUsageCache".to_string()), + })) + .await + .expect("working rx"); + } + Err(_) => { + tx.send(Ok(NsScannerResponse { + success: false, + update: "".to_string(), + data_usage_cache: "".to_string(), + error_info: Some("scanner failed".to_string()), + })) + .await + .expect("working rx"); + } + } + } else { + tx.send(Ok(NsScannerResponse { + success: false, + update: "".to_string(), + data_usage_cache: "".to_string(), + error_info: Some("can not find disk".to_string()), + })) + .await + .expect("working rx"); + } + } + _ => todo!(), + } + }); + + let out_stream = ReceiverStream::new(rx); + Ok(tonic::Response::new(Box::pin(out_stream))) + } + async fn lock(&self, request: Request) -> Result, Status> { let request = request.into_inner(); match &serde_json::from_str::(&request.args) { diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index 46daded39..361982c34 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -7,6 +7,7 @@ use clap::Parser; use common::error::{Error, Result}; use ecstore::{ endpoints::EndpointServerPools, + heal::data_scanner::init_data_scanner, set_global_endpoints, store::{init_local_disks, ECStore}, update_erasure_type, @@ -184,6 +185,8 @@ async fn run(opt: config::Opt) -> Result<()> { .map_err(|err| Error::from_string(err.to_string()))?; store.init().await.map_err(|err| Error::from_string(err.to_string()))?; + // init scanner + init_data_scanner().await; tokio::select! { _ = tokio::signal::ctrl_c() => { diff --git a/rustfs/src/storage/acess.rs b/rustfs/src/storage/acess.rs index b437a3594..0d8552cab 100644 --- a/rustfs/src/storage/acess.rs +++ b/rustfs/src/storage/acess.rs @@ -10,7 +10,7 @@ use s3s::auth::Credentials; use s3s::{dto::*, s3_error, S3Request, S3Result}; use uuid::Uuid; -#[derive(Debug, Default, Clone)] +#[derive(Default, Clone)] struct ReqInfo { pub card: Option, pub action: Option, @@ -51,7 +51,7 @@ impl S3Access for FS { let req_info = ReqInfo { card: cx.credentials().cloned(), - action: action, + action, ..Default::default() }; diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 4b1f10902..ebcdb0cff 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -341,15 +341,14 @@ impl S3 for FS { let content_type = { if let Some(content_type) = info.content_type { - let ct = match ContentType::from_str(&content_type) { + match ContentType::from_str(&content_type) { Ok(res) => Some(res), Err(err) => { error!("parse content-type err {} {:?}", &content_type, err); // None } - }; - ct + } } else { None } @@ -411,15 +410,14 @@ impl S3 for FS { let content_type = { if let Some(content_type) = info.content_type { - let ct = match ContentType::from_str(&content_type) { + match ContentType::from_str(&content_type) { Ok(res) => Some(res), Err(err) => { error!("parse content-type err {} {:?}", &content_type, err); // None } - }; - ct + } } else { None } @@ -1536,9 +1534,7 @@ impl S3 for FS { } } - let mut grants = Vec::new(); - - grants.push(Grant { + let grants = vec![Grant { grantee: Some(Grantee { type_: Type::from_static(Type::CANONICAL_USER), display_name: None, @@ -1547,12 +1543,11 @@ impl S3 for FS { uri: None, }), permission: Some(Permission::from_static(Permission::FULL_CONTROL)), - }); + }]; Ok(S3Response::new(GetBucketAclOutput { grants: Some(grants), owner: Some(RUSTFS_OWNER.to_owned()), - ..Default::default() })) } @@ -1589,7 +1584,7 @@ impl S3 for FS { v.grants.is_some_and(|gs| { // !gs.is_empty() - && gs.get(0).is_some_and(|g| { + && gs.first().is_some_and(|g| { g.to_owned() .permission .is_some_and(|p| p.as_str() == Permission::FULL_CONTROL) @@ -1617,9 +1612,7 @@ impl S3 for FS { return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{}", e))); } - let mut grants = Vec::new(); - - grants.push(Grant { + let grants = vec![Grant { grantee: Some(Grantee { type_: Type::from_static(Type::CANONICAL_USER), display_name: None, @@ -1628,7 +1621,7 @@ impl S3 for FS { uri: None, }), permission: Some(Permission::from_static(Permission::FULL_CONTROL)), - }); + }]; Ok(S3Response::new(GetObjectAclOutput { grants: Some(grants), @@ -1665,7 +1658,7 @@ impl S3 for FS { v.grants.is_some_and(|gs| { // !gs.is_empty() - && gs.get(0).is_some_and(|g| { + && gs.first().is_some_and(|g| { g.to_owned() .permission .is_some_and(|p| p.as_str() == Permission::FULL_CONTROL)