From 750194e4dc2e45c4d7f1eb585fd1490cad9aa7dc Mon Sep 17 00:00:00 2001 From: weisd Date: Mon, 28 Apr 2025 13:16:11 +0800 Subject: [PATCH 01/38] test --- Cargo.toml | 2 +- ecstore/src/cache_value/metacache_set.rs | 15 ++- ecstore/src/disk/mod.rs | 147 +++++++++++++++-------- ecstore/src/file_meta.rs | 13 +- ecstore/src/heal/data_scanner.rs | 2 +- ecstore/src/pools.rs | 9 +- ecstore/src/rebalance.rs | 8 +- ecstore/src/set_disk.rs | 4 +- ecstore/src/store_list_objects.rs | 4 +- scripts/run.sh | 2 +- 10 files changed, 130 insertions(+), 76 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 59c48cc14..684dfb53b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -190,7 +190,7 @@ inherits = "dev" [profile.release] opt-level = 3 -lto = "fat" +lto = "thin" codegen-units = 1 panic = "abort" # Optional, remove the panic expansion code strip = true # strip symbol information to reduce binary size diff --git a/ecstore/src/cache_value/metacache_set.rs b/ecstore/src/cache_value/metacache_set.rs index e716a3051..c3e75d093 100644 --- a/ecstore/src/cache_value/metacache_set.rs +++ b/ecstore/src/cache_value/metacache_set.rs @@ -7,7 +7,7 @@ use common::error::{Error, Result}; use futures::future::join_all; use std::{future::Future, pin::Pin, sync::Arc}; use tokio::{spawn, sync::broadcast::Receiver as B_Receiver}; -use tracing::{error, info}; +use tracing::{error, info, warn}; pub type AgreedFn = Box Pin + Send>> + Send + 'static>; pub type PartialFn = Box]) -> Pin + Send>> + Send + 'static>; @@ -205,7 +205,7 @@ pub async fn list_path_raw(mut rx: B_Receiver, opts: ListPathRawOptions) - continue; } // If exact match, we agree. - if let Ok((_, true)) = current.matches(&entry, true) { + if let (_, true) = current.matches(Some(&entry), true) { top_entries[i] = Some(entry); agree += 1; @@ -224,7 +224,12 @@ pub async fn list_path_raw(mut rx: B_Receiver, opts: ListPathRawOptions) - // We got a new, better current. // Clear existing entries. top_entries = vec![None; top_entries.len()]; - agree += 1; + + for item in top_entries.iter_mut().take(i) { + *item = None; + } + + agree = 1; top_entries[i] = Some(entry.clone()); current = entry; } @@ -265,6 +270,7 @@ pub async fn list_path_raw(mut rx: B_Receiver, opts: ListPathRawOptions) - } } + warn!("list_path_raw: all at eof or error"); break; } @@ -272,6 +278,8 @@ pub async fn list_path_raw(mut rx: B_Receiver, opts: ListPathRawOptions) - for r in readers.iter_mut() { let _ = r.skip(1).await; } + + warn!("list_path_raw: agree == readers.len() {} ", ¤t.name); if let Some(agreed_fn) = opts.agreed.as_ref() { agreed_fn(current).await; } @@ -285,6 +293,7 @@ pub async fn list_path_raw(mut rx: B_Receiver, opts: ListPathRawOptions) - } } + warn!("list_path_raw: {} entries", top_entries.len()); if let Some(partial_fn) = opts.partial.as_ref() { partial_fn(MetaCacheEntries(top_entries), &errs).await; } diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs index ddefd1bed..cda289c2b 100644 --- a/ecstore/src/disk/mod.rs +++ b/ecstore/src/disk/mod.rs @@ -786,46 +786,62 @@ impl MetaCacheEntry { fm.into_file_info_versions(bucket, self.name.as_str(), false) } - pub fn matches(&self, other: &MetaCacheEntry, strict: bool) -> Result<(Option, bool)> { + pub fn matches(&self, other: Option<&MetaCacheEntry>, strict: bool) -> (Option, bool) { + if other.is_none() { + return (None, false); + } + + let other = other.unwrap(); + let mut prefer = None; if self.name != other.name { if self.name < other.name { - return Ok((Some(self.clone()), false)); + return (Some(self.clone()), false); } - return Ok((Some(other.clone()), false)); + return (Some(other.clone()), false); } if other.is_dir() || self.is_dir() { if self.is_dir() { - return Ok((Some(self.clone()), other.is_dir())); + return (Some(self.clone()), other.is_dir() == self.is_dir()); } - return Ok((Some(other.clone()), other.is_dir() == self.is_dir())); + return (Some(other.clone()), other.is_dir() == self.is_dir()); } let self_vers = match &self.cached { Some(file_meta) => file_meta.clone(), - None => FileMeta::load(&self.metadata)?, + None => match FileMeta::load(&self.metadata) { + Ok(meta) => meta, + Err(_) => { + return (None, false); + } + }, }; let other_vers = match &other.cached { Some(file_meta) => file_meta.clone(), - None => FileMeta::load(&other.metadata)?, + None => match FileMeta::load(&other.metadata) { + Ok(meta) => meta, + Err(_) => { + return (None, false); + } + }, }; if self_vers.versions.len() != other_vers.versions.len() { match self_vers.lastest_mod_time().cmp(&other_vers.lastest_mod_time()) { Ordering::Greater => { - return Ok((Some(self.clone()), false)); + return (Some(self.clone()), false); } Ordering::Less => { - return Ok((Some(self.clone()), false)); + return (Some(other.clone()), false); } _ => {} } if self_vers.versions.len() > other_vers.versions.len() { - return Ok((Some(self.clone()), false)); + return (Some(self.clone()), false); } - return Ok((Some(self.clone()), false)); + return (Some(other.clone()), false); } for (s_version, o_version) in self_vers.versions.iter().zip(other_vers.versions.iter()) { @@ -853,14 +869,14 @@ impl MetaCacheEntry { } if prefer.is_some() { - return Ok((prefer, false)); + return (prefer, false); } if s_version.header.sorts_before(&o_version.header) { - return Ok((Some(self.clone()), false)); + return (Some(self.clone()), false); } - return Ok((Some(other.clone()), false)); + return (Some(other.clone()), false); } } @@ -868,7 +884,7 @@ impl MetaCacheEntry { prefer = Some(self.clone()); } - Ok((prefer, true)) + (prefer, true) } pub fn xl_meta(&mut self) -> Result { @@ -900,9 +916,10 @@ impl MetaCacheEntries { pub fn as_ref(&self) -> &[Option] { &self.0 } - pub fn resolve(&self, mut params: MetadataResolutionParams) -> Result> { + pub fn resolve(&self, mut params: MetadataResolutionParams) -> Option { if self.0.is_empty() { - return Ok(None); + warn!("decommission_pool: entries resolve empty"); + return None; } let mut dir_exists = 0; @@ -913,76 +930,104 @@ impl MetaCacheEntries { let mut objs_valid = 0; for entry in self.0.iter().flatten() { + let mut entry = entry.clone(); + + warn!("decommission_pool: entries resolve entry {:?}", entry.name); if entry.name.is_empty() { continue; } if entry.is_dir() { dir_exists += 1; selected = Some(entry.clone()); + warn!("decommission_pool: entries resolve entry dir {:?}", entry.name); continue; } + let xl = match entry.xl_meta() { + Ok(xl) => xl, + Err(e) => { + warn!("decommission_pool: entries resolve entry xl_meta {:?}", e); + continue; + } + }; + objs_valid += 1; - match &entry.cached { - Some(file_meta) => { - params.candidates.push(file_meta.versions.clone()); - } - None => { - params.candidates.push(FileMeta::load(&entry.metadata)?.versions); - } - } + params.candidates.push(xl.versions.clone()); if selected.is_none() { selected = Some(entry.clone()); objs_agree = 1; + warn!("decommission_pool: entries resolve entry selected {:?}", entry.name); continue; } - if let (Some(prefer), true) = entry.matches(selected.as_ref().unwrap(), params.strict)? { - selected = Some(prefer); + if let (prefer, true) = entry.matches(selected.as_ref(), params.strict) { + selected = prefer; objs_agree += 1; + warn!("decommission_pool: entries resolve entry prefer {:?}", entry.name); continue; } } - // Return dir entries, if enough... - if selected.is_some() && selected.as_ref().unwrap().is_dir() && dir_exists >= params.dir_quorum { - return Ok(selected); + let Some(selected) = selected else { + warn!("decommission_pool: entries resolve entry no selected"); + return None; + }; + + if selected.is_dir() && dir_exists >= params.dir_quorum { + warn!("decommission_pool: entries resolve entry dir selected {:?}", selected.name); + return Some(selected); } + // If we would never be able to reach read quorum. if objs_valid < params.obj_quorum { - return Ok(None); + warn!( + "decommission_pool: entries resolve entry not enough objects {} < {}", + objs_valid, params.obj_quorum + ); + return None; } - // If all objects agree. - if selected.is_some() && objs_agree == objs_valid { - return Ok(selected); + + if objs_agree == objs_valid { + warn!("decommission_pool: entries resolve entry all agree {} == {}", objs_agree, objs_valid); + return Some(selected); } - // If cached is nil we shall skip the entry. - if selected.is_none() || (selected.is_some() && selected.as_ref().unwrap().cached.is_none()) { - return Ok(None); + + let Some(cached) = selected.cached else { + warn!("decommission_pool: entries resolve entry no cached"); + return None; + }; + + let versions = merge_file_meta_versions(params.obj_quorum, params.strict, params.requested_versions, ¶ms.candidates); + if versions.is_empty() { + warn!("decommission_pool: entries resolve entry no versions"); + return None; } + + let metadata = match cached.marshal_msg() { + Ok(meta) => meta, + Err(e) => { + warn!("decommission_pool: entries resolve entry marshal_msg {:?}", e); + return None; + } + }; + // Merge if we have disagreement. // Create a new merged result. - selected = Some(MetaCacheEntry { - name: selected.as_ref().unwrap().name.clone(), + let new_selected = MetaCacheEntry { + name: selected.name.clone(), cached: Some(FileMeta { - meta_ver: selected.as_ref().unwrap().cached.as_ref().unwrap().meta_ver, + meta_ver: cached.meta_ver, + versions, ..Default::default() }), reusable: true, - ..Default::default() - }); + metadata, + }; - selected.as_mut().unwrap().cached.as_mut().unwrap().versions = - merge_file_meta_versions(params.obj_quorum, params.strict, params.requested_versions, ¶ms.candidates); - if selected.as_ref().unwrap().cached.as_ref().unwrap().versions.is_empty() { - return Ok(None); - } - - selected.as_mut().unwrap().metadata = selected.as_ref().unwrap().cached.as_ref().unwrap().marshal_msg()?; - - Ok(selected) + warn!("decommission_pool: entries resolve entry selected {:?}", new_selected.name); + Some(new_selected) } pub fn first_found(&self) -> (Option, usize) { diff --git a/ecstore/src/file_meta.rs b/ecstore/src/file_meta.rs index e4da3882a..1234c07f4 100644 --- a/ecstore/src/file_meta.rs +++ b/ecstore/src/file_meta.rs @@ -916,11 +916,20 @@ impl FileMetaVersionHeader { } pub fn matches_not_strict(&self, o: &FileMetaVersionHeader) -> bool { + let mut ok = self.version_id == o.version_id && self.version_type == o.version_type && self.matches_ec(o); if self.version_id.is_none() { - return self.version_id == o.version_id && self.version_type == o.version_type && self.mod_time == o.mod_time; + ok = ok && self.mod_time == o.mod_time; } - self.version_id == o.version_id && self.version_type == o.version_type + ok + } + + pub fn matches_ec(&self, o: &FileMetaVersionHeader) -> bool { + if self.has_ec() && o.has_ec() { + return self.ec_n == o.ec_n && self.ec_m == o.ec_m; + } + + true } pub fn free_version(&self) -> bool { diff --git a/ecstore/src/heal/data_scanner.rs b/ecstore/src/heal/data_scanner.rs index b1b110a54..c806267f2 100644 --- a/ecstore/src/heal/data_scanner.rs +++ b/ecstore/src/heal/data_scanner.rs @@ -867,7 +867,7 @@ impl FolderScanner { // return; // } let entry = match entries.resolve(resolver_partial) { - Ok(Some(entry)) => entry, + Some(entry) => entry, _ => match entries.first_found() { (Some(entry), _) => entry, _ => return, diff --git a/ecstore/src/pools.rs b/ecstore/src/pools.rs index f6f996941..c151b0627 100644 --- a/ecstore/src/pools.rs +++ b/ecstore/src/pools.rs @@ -1330,22 +1330,17 @@ impl SetDisks { partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option]| { let resolver = resolver.clone(); let cb_func = cb_func.clone(); - match entries.resolve(resolver) { - Ok(Some(entry)) => { + Some(entry) => { warn!("decommission_pool: list_objects_to_decommission get {}", &entry.name); Box::pin(async move { cb_func(entry).await; }) } - Ok(None) => { + None => { warn!("decommission_pool: list_objects_to_decommission get none"); Box::pin(async {}) } - Err(err) => { - error!("decommission_pool: list_objects_to_decommission get err {:?}", &err); - Box::pin(async {}) - } } })), ..Default::default() diff --git a/ecstore/src/rebalance.rs b/ecstore/src/rebalance.rs index 62c5d696f..d6ec57113 100644 --- a/ecstore/src/rebalance.rs +++ b/ecstore/src/rebalance.rs @@ -1075,18 +1075,14 @@ impl SetDisks { let cb = cb.clone(); match entries.resolve(resolver) { - Ok(Some(entry)) => { + Some(entry) => { warn!("rebalance: list_objects_to_decommission get {}", &entry.name); Box::pin(async move { cb(entry).await }) } - Ok(None) => { + None => { warn!("rebalance: list_objects_to_decommission get none"); Box::pin(async {}) } - Err(err) => { - error!("rebalance: list_objects_to_decommission get err {:?}", &err); - Box::pin(async {}) - } } })), ..Default::default() diff --git a/ecstore/src/set_disk.rs b/ecstore/src/set_disk.rs index bd55ab39f..da3046a7b 100644 --- a/ecstore/src/set_disk.rs +++ b/ecstore/src/set_disk.rs @@ -2052,7 +2052,7 @@ impl SetDisks { let bucket_partial = bucket_partial.clone(); async move { let entry = match entries.resolve(resolver_partial) { - Ok(Some(entry)) => entry, + Some(entry) => entry, _ => match entries.first_found() { (Some(entry), _) => entry, _ => return, @@ -3505,7 +3505,7 @@ impl SetDisks { let heal_entry = heal_entry.clone(); let resolver = resolver.clone(); async move { - let entry = if let Ok(Some(entry)) = entries.resolve(resolver) { + let entry = if let Some(entry) = entries.resolve(resolver) { entry } else if let (Some(entry), _) = entries.first_found() { entry diff --git a/ecstore/src/store_list_objects.rs b/ecstore/src/store_list_objects.rs index 8b9118de3..445c56f0c 100644 --- a/ecstore/src/store_list_objects.rs +++ b/ecstore/src/store_list_objects.rs @@ -778,7 +778,7 @@ impl ECStore { let value = tx2.clone(); let resolver = resolver.clone(); async move { - if let Ok(Some(entry)) = entries.resolve(resolver) { + if let Some(entry) = entries.resolve(resolver) { if let Err(err) = value.send(entry).await { error!("list_path send fail {:?}", err); } @@ -1296,7 +1296,7 @@ impl SetDisks { let value = tx2.clone(); let resolver = resolver.clone(); async move { - if let Ok(Some(entry)) = entries.resolve(resolver) { + if let Some(entry) = entries.resolve(resolver) { if let Err(err) = value.send(entry).await { error!("list_path send fail {:?}", err); } diff --git a/scripts/run.sh b/scripts/run.sh index 690ac8511..352568ecc 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -33,7 +33,7 @@ export RUSTFS_CONSOLE_ENABLE=true export RUSTFS_CONSOLE_ADDRESS=":9002" # export RUSTFS_SERVER_DOMAINS="localhost:9000" # HTTPS 证书目录 - export RUSTFS_TLS_PATH="./deploy/certs" +# export RUSTFS_TLS_PATH="./deploy/certs" # 具体路径修改为配置文件真实路径，obs.example.toml 仅供参考 其中`RUSTFS_OBS_CONFIG` 和下面变量二选一 export RUSTFS_OBS_CONFIG="./deploy/config/obs.example.toml" From 26d69cdc7f31b5ca8ba63e9f388a27d2572b0f70 Mon Sep 17 00:00:00 2001 From: weisd Date: Mon, 28 Apr 2025 16:58:37 +0800 Subject: [PATCH 02/38] test --- ecstore/src/cache_value/metacache_set.rs | 10 +---- ecstore/src/pools.rs | 34 +++++++++++----- ecstore/src/rebalance.rs | 14 +++---- ecstore/src/store.rs | 49 +++++++++++++++--------- 4 files changed, 61 insertions(+), 46 deletions(-) diff --git a/ecstore/src/cache_value/metacache_set.rs b/ecstore/src/cache_value/metacache_set.rs index c3e75d093..1e32b4391 100644 --- a/ecstore/src/cache_value/metacache_set.rs +++ b/ecstore/src/cache_value/metacache_set.rs @@ -7,7 +7,7 @@ use common::error::{Error, Result}; use futures::future::join_all; use std::{future::Future, pin::Pin, sync::Arc}; use tokio::{spawn, sync::broadcast::Receiver as B_Receiver}; -use tracing::{error, info, warn}; +use tracing::error; pub type AgreedFn = Box Pin + Send>> + Send + 'static>; pub type PartialFn = Box]) -> Pin + Send>> + Send + 'static>; @@ -54,7 +54,6 @@ impl Clone for ListPathRawOptions { pub async fn list_path_raw(mut rx: B_Receiver, opts: ListPathRawOptions) -> Result<()> { // println!("list_path_raw {},{}", &opts.bucket, &opts.path); if opts.disks.is_empty() { - info!("list_path_raw 0 drives provided"); return Err(Error::from_string("list_path_raw: 0 drives provided")); } @@ -214,16 +213,12 @@ pub async fn list_path_raw(mut rx: B_Receiver, opts: ListPathRawOptions) - // If only the name matches we didn't agree, but add it for resolution. if entry.name == current.name { top_entries[i] = Some(entry); - continue; } // We got different entries if entry.name > current.name { continue; } - // We got a new, better current. - // Clear existing entries. - top_entries = vec![None; top_entries.len()]; for item in top_entries.iter_mut().take(i) { *item = None; @@ -270,7 +265,6 @@ pub async fn list_path_raw(mut rx: B_Receiver, opts: ListPathRawOptions) - } } - warn!("list_path_raw: all at eof or error"); break; } @@ -279,7 +273,6 @@ pub async fn list_path_raw(mut rx: B_Receiver, opts: ListPathRawOptions) - let _ = r.skip(1).await; } - warn!("list_path_raw: agree == readers.len() {} ", ¤t.name); if let Some(agreed_fn) = opts.agreed.as_ref() { agreed_fn(current).await; } @@ -293,7 +286,6 @@ pub async fn list_path_raw(mut rx: B_Receiver, opts: ListPathRawOptions) - } } - warn!("list_path_raw: {} entries", top_entries.len()); if let Some(partial_fn) = opts.partial.as_ref() { partial_fn(MetaCacheEntries(top_entries), &errs).await; } diff --git a/ecstore/src/pools.rs b/ecstore/src/pools.rs index c151b0627..8a0761006 100644 --- a/ecstore/src/pools.rs +++ b/ecstore/src/pools.rs @@ -31,6 +31,7 @@ use std::io::{Cursor, Write}; use std::path::PathBuf; use std::sync::Arc; use time::{Duration, OffsetDateTime}; +use tokio::io::AsyncReadExt; use tokio::sync::broadcast::Receiver as B_Receiver; use tracing::{error, info, warn}; @@ -910,7 +911,11 @@ impl ECStore { let wk = wk.clone(); let set = set.clone(); let rcfg = rcfg.clone(); - Box::pin(async move { this.decommission_entry(idx, entry, bucket, set, wk, rcfg).await }) + + Box::pin(async move { + wk.take().await; + this.decommission_entry(idx, entry, bucket, set, wk, rcfg).await + }) } }); @@ -918,6 +923,7 @@ impl ECStore { let mut rx = rx.resubscribe(); let bi = bi.clone(); let set_id = set_idx; + let wk_clone = wk.clone(); tokio::spawn(async move { loop { if rx.try_recv().is_ok() { @@ -945,6 +951,8 @@ impl ECStore { } } } + + wk_clone.give().await; }); } @@ -1166,7 +1174,7 @@ impl ECStore { #[tracing::instrument(skip(self, rd))] async fn decommission_object(self: Arc, pool_idx: usize, bucket: String, rd: GetObjectReader) -> Result<()> { - warn!("decommission_object: {} {}", &bucket, &rd.object_info.name); + warn!("decommission_object: start {} {}", &bucket, &rd.object_info.name); let object_info = rd.object_info.clone(); // TODO: check : use size or actual_size ? @@ -1194,8 +1202,6 @@ impl ECStore { } }; - // TODO: defer abort_multipart_upload - defer!(|| async { if let Err(err) = self .abort_multipart_upload(&bucket, &object_info.name, &res.upload_id, &ObjectOptions::default()) @@ -1210,9 +1216,13 @@ impl ECStore { let mut reader = rd.stream; for (i, part) in object_info.parts.iter().enumerate() { - // 每次从reader中读取一个part上传 + let mut chunk = vec![0u8; part.size]; - let mut data = PutObjReader::new(reader, part.size); + reader.read_exact(&mut chunk).await?; + + // 每次从reader中读取一个part上传 + let rd = Box::new(Cursor::new(chunk)); + let mut data = PutObjReader::new(rd, part.size); let pi = match self .put_object_part( @@ -1230,18 +1240,17 @@ impl ECStore { { Ok(pi) => pi, Err(err) => { - error!("decommission_object: put_object_part err {:?}", &err); + error!("decommission_object: put_object_part {} err {:?}", i, &err); return Err(err); } }; + warn!("decommission_object: put_object_part {} done {} {}", i, &bucket, &object_info.name); + parts[i] = CompletePart { part_num: pi.part_num, e_tag: pi.etag, }; - - // 把reader所有权拿回来? - reader = data.stream; } if let Err(err) = self @@ -1262,6 +1271,7 @@ impl ECStore { return Err(err); } + warn!("decommission_object: complete_multipart_upload done {} {}", &bucket, &object_info.name); return Ok(()); } @@ -1289,6 +1299,7 @@ impl ECStore { return Err(err); } + warn!("decommission_object: put_object done {} {}", &bucket, &object_info.name); Ok(()) } } @@ -1319,6 +1330,8 @@ impl SetDisks { ..Default::default() }; + let cb1 = cb_func.clone(); + list_path_raw( rx, ListPathRawOptions { @@ -1327,6 +1340,7 @@ impl SetDisks { path: bucket_info.prefix.clone(), recursice: true, min_disks: listing_quorum, + agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry)))), partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option]| { let resolver = resolver.clone(); let cb_func = cb_func.clone(); diff --git a/ecstore/src/rebalance.rs b/ecstore/src/rebalance.rs index d6ec57113..97255caf8 100644 --- a/ecstore/src/rebalance.rs +++ b/ecstore/src/rebalance.rs @@ -370,7 +370,7 @@ impl ECStore { let rebalance_meta = self.rebalance_meta.read().await; if let Some(meta) = rebalance_meta.as_ref() { if let Some(pool_stat) = meta.pool_stats.get(pool_index) { - if pool_stat.info.status != RebalStatus::Completed || !pool_stat.participating { + if pool_stat.info.status == RebalStatus::Completed || !pool_stat.participating { return Ok(None); } @@ -581,17 +581,17 @@ impl ECStore { } }); - tracing::warn!("Pool {} rebalancing is started", pool_index + 1); + warn!("Pool {} rebalancing is started", pool_index + 1); while let Some(bucket) = self.next_rebal_bucket(pool_index).await? { - tracing::info!("Rebalancing bucket: {}", bucket); + warn!("Rebalancing bucket: {}", bucket); if let Err(err) = self.rebalance_bucket(rx.resubscribe(), bucket.clone(), pool_index).await { if err.to_string().contains("not initialized") { warn!("rebalance_bucket: rebalance not initialized, continue"); continue; } - tracing::error!("Error rebalancing bucket {}: {:?}", bucket, err); + error!("Error rebalancing bucket {}: {:?}", bucket, err); done_tx.send(Err(err)).await.ok(); break; } @@ -599,7 +599,7 @@ impl ECStore { self.bucket_rebalance_done(pool_index, bucket).await?; } - tracing::warn!("Pool {} rebalancing is done", pool_index + 1); + warn!("Pool {} rebalancing is done", pool_index + 1); done_tx.send(Ok(())).await.ok(); save_task.await.ok(); @@ -838,8 +838,6 @@ impl ECStore { } }; - // TODO: defer abort_multipart_upload - defer!(|| async { if let Err(err) = self .abort_multipart_upload(&bucket, &object_info.name, &res.upload_id, &ObjectOptions::default()) @@ -1068,7 +1066,7 @@ impl SetDisks { bucket: bucket.clone(), recursice: true, min_disks: listing_quorum, - agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry.clone())))), + agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry)))), partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option]| { // let cb = cb.clone(); let resolver = resolver.clone(); diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index 5a2075775..821eb5305 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -524,7 +524,9 @@ impl ECStore { // TODO: 并发 for (idx, pool) in self.pools.iter().enumerate() { - // TODO: IsSuspended + if self.is_suspended(idx).await || self.is_pool_rebalancing(idx).await { + continue; + } n_sets[idx] = pool.set_count; @@ -714,16 +716,14 @@ impl ECStore { let mut def_pool = PoolObjInfo::default(); let mut has_def_pool = false; - let pool_meta = self.pool_meta.read().await; for pinfo in ress.iter() { - if opts.skip_decommissioned && pool_meta.is_suspended(pinfo.index) { + if opts.skip_decommissioned && self.is_suspended(pinfo.index).await { continue; } - // TODO:SkipRebalancing - // if opts.SkipRebalancing && z.IsPoolRebalancing(pinfo.Index) { - // continue - // } + if opts.skip_rebalancing && self.is_pool_rebalancing(pinfo.index).await { + continue; + } if pinfo.err.is_none() { return Ok((pinfo.clone(), self.pools_with_object(&ress, opts).await)); @@ -756,15 +756,15 @@ impl ECStore { async fn pools_with_object(&self, pools: &[PoolObjInfo], opts: &ObjectOptions) -> Vec { let mut errs = Vec::new(); - let pool_meta = self.pool_meta.read().await; + for pool in pools.iter() { - if opts.skip_decommissioned && pool_meta.is_suspended(pool.index) { + if opts.skip_decommissioned && self.is_suspended(pool.index).await { + continue; + } + + if opts.skip_rebalancing && self.is_pool_rebalancing(pool.index).await { continue; } - // TODO:SkipRebalancing - // if opts.SkipRebalancing && z.IsPoolRebalancing(pinfo.Index) { - // continue - // } if let Some(err) = &pool.err { if is_err_read_quorum(err) { @@ -1865,7 +1865,9 @@ impl StorageAPI for ECStore { } for pool in self.pools.iter() { - // TODO: IsSuspended + if self.is_suspended(pool.pool_idx).await { + continue; + } let err = match pool.put_object_part(bucket, object, upload_id, part_id, data, opts).await { Ok(res) => return Ok(res), Err(err) => { @@ -1915,6 +1917,9 @@ impl StorageAPI for ECStore { let mut uploads = Vec::new(); for pool in self.pools.iter() { + if self.is_suspended(pool.pool_idx).await { + continue; + } let res = pool .list_multipart_uploads( bucket, @@ -1948,7 +1953,9 @@ impl StorageAPI for ECStore { } for (idx, pool) in self.pools.iter().enumerate() { - // // TODO: IsSuspended + if self.is_suspended(idx).await || self.is_pool_rebalancing(idx).await { + continue; + } let res = pool .list_multipart_uploads(bucket, object, None, None, None, MAX_UPLOADS_LIST) .await?; @@ -1983,8 +1990,8 @@ impl StorageAPI for ECStore { return self.pools[0].get_multipart_info(bucket, object, upload_id, opts).await; } - for (idx, pool) in self.pools.iter().enumerate() { - if self.is_suspended(idx).await { + for pool in self.pools.iter() { + if self.is_suspended(pool.pool_idx).await { continue; } @@ -2017,7 +2024,9 @@ impl StorageAPI for ECStore { } for pool in self.pools.iter() { - // TODO: IsSuspended + if self.is_suspended(pool.pool_idx).await { + continue; + } let err = match pool.abort_multipart_upload(bucket, object, upload_id, opts).await { Ok(_) => return Ok(()), @@ -2060,7 +2069,9 @@ impl StorageAPI for ECStore { } for pool in self.pools.iter() { - // TODO: IsSuspended + if self.is_suspended(pool.pool_idx).await { + continue; + } let err = match pool .complete_multipart_upload(bucket, object, upload_id, uploaded_parts.clone(), opts) From 738a81e50e992419588f3d473cde8b63119fb8fe Mon Sep 17 00:00:00 2001 From: weisd Date: Mon, 28 Apr 2025 17:35:27 +0800 Subject: [PATCH 03/38] test --- ecstore/src/rebalance.rs | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/ecstore/src/rebalance.rs b/ecstore/src/rebalance.rs index 97255caf8..007cf7a4b 100644 --- a/ecstore/src/rebalance.rs +++ b/ecstore/src/rebalance.rs @@ -18,6 +18,7 @@ use common::defer; use common::error::{Error, Result}; use http::HeaderMap; use serde::{Deserialize, Serialize}; +use tokio::io::AsyncReadExt; use tokio::sync::broadcast::{self, Receiver as B_Receiver}; use tokio::time::{Duration, Instant}; use tracing::{error, info, warn}; @@ -389,10 +390,15 @@ impl ECStore { let mut rebalance_meta = self.rebalance_meta.write().await; if let Some(meta) = rebalance_meta.as_mut() { if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) { - if let Some(idx) = pool_stat.buckets.iter().position(|b| *b == bucket) { + warn!("bucket_rebalance_done: buckets {:?}", &pool_stat.buckets); + if let Some(idx) = pool_stat.buckets.iter().position(|b| b.as_str() == bucket.as_str()) { + warn!("bucket_rebalance_done: bucket {} rebalanced", &bucket); pool_stat.buckets.remove(idx); pool_stat.rebalanced_buckets.push(bucket); + return Ok(()); + } else { + warn!("bucket_rebalance_done: bucket {} not found", bucket); } } } @@ -584,7 +590,7 @@ impl ECStore { warn!("Pool {} rebalancing is started", pool_index + 1); while let Some(bucket) = self.next_rebal_bucket(pool_index).await? { - warn!("Rebalancing bucket: {}", bucket); + warn!("Rebalancing bucket: start {}", bucket); if let Err(err) = self.rebalance_bucket(rx.resubscribe(), bucket.clone(), pool_index).await { if err.to_string().contains("not initialized") { @@ -596,6 +602,7 @@ impl ECStore { break; } + warn!("Rebalance bucket: done {} ", bucket); self.bucket_rebalance_done(pool_index, bucket).await?; } @@ -632,7 +639,6 @@ impl ECStore { false } - #[allow(unused_assignments)] #[tracing::instrument(skip(self, wk, set))] async fn rebalance_entry( &self, @@ -854,7 +860,13 @@ impl ECStore { for (i, part) in object_info.parts.iter().enumerate() { // 每次从reader中读取一个part上传 - let mut data = PutObjReader::new(reader, part.size); + let mut chunk = vec![0u8; part.size]; + + reader.read_exact(&mut chunk).await?; + + // 每次从reader中读取一个part上传 + let rd = Box::new(Cursor::new(chunk)); + let mut data = PutObjReader::new(rd, part.size); let pi = match self .put_object_part( @@ -881,9 +893,6 @@ impl ECStore { part_num: pi.part_num, e_tag: pi.etag, }; - - // 把reader所有权拿回来? - reader = data.stream; } if let Err(err) = self @@ -937,7 +946,7 @@ impl ECStore { #[tracing::instrument(skip(self, rx))] async fn rebalance_bucket(self: &Arc, rx: B_Receiver, bucket: String, pool_index: usize) -> Result<()> { // Placeholder for actual bucket rebalance logic - tracing::info!("Rebalancing bucket {} in pool {}", bucket, pool_index); + warn!("Rebalancing bucket {} in pool {}", bucket, pool_index); // TODO: other config // if bucket != RUSTFS_META_BUCKET{ @@ -975,14 +984,12 @@ impl ECStore { let bucket = bucket.clone(); let wk = wk.clone(); tokio::spawn(async move { - defer!(|| async { - wk.clone().give().await; - }); if let Err(err) = set.list_objects_to_rebalance(rx, bucket, rebalance_entry).await { error!("Rebalance worker {} error: {}", set_idx, err); } else { info!("Rebalance worker {} done", set_idx); } + wk.clone().give().await; }); } From c1590a054c2d2c3630014c442704de82fb021f27 Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Mon, 28 Apr 2025 10:00:28 +0000 Subject: [PATCH 04/38] tmp Signed-off-by: junxiang Mu <1948535941@qq.com> --- ecstore/src/bitrot.rs | 1 + ecstore/src/erasure.rs | 126 +++++++++++++++++++++------------------- ecstore/src/io.rs | 54 +++++++++++++---- ecstore/src/set_disk.rs | 4 +- 4 files changed, 113 insertions(+), 72 deletions(-) diff --git a/ecstore/src/bitrot.rs b/ecstore/src/bitrot.rs index b64c84b3b..05e55a430 100644 --- a/ecstore/src/bitrot.rs +++ b/ecstore/src/bitrot.rs @@ -534,6 +534,7 @@ impl Writer for BitrotFileWriter { self } + #[tracing::instrument(level = "info", skip_all)] async fn write(&mut self, buf: Bytes) -> Result<()> { if buf.is_empty() { return Ok(()); diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs index 590889e78..d5a08af4b 100644 --- a/ecstore/src/erasure.rs +++ b/ecstore/src/erasure.rs @@ -6,8 +6,10 @@ use common::error::{Error, Result}; use futures::future::join_all; use reed_solomon_erasure::galois_8::ReedSolomon; use smallvec::SmallVec; +use tokio::sync::mpsc; use std::any::Any; use std::io::ErrorKind; +use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tracing::warn; @@ -52,8 +54,8 @@ impl Erasure { #[tracing::instrument(level = "debug", skip(self, reader, writers))] pub async fn encode( - &mut self, - reader: &mut S, + self: Arc, + mut reader: S, writers: &mut [Option], // block_size: usize, total_size: usize, @@ -67,40 +69,47 @@ impl Erasure { // body.map(|f| f.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))), // ); - let mut total: usize = 0; - let mut blocks = >::new(); - - loop { - if total_size > 0 { - let new_len = { - let remain = total_size - total; - if remain > self.block_size { - self.block_size - } else { - remain - } - }; - - if new_len == 0 && total > 0 { - break; - } - - self.buf.resize(new_len, 0u8); - match reader.read_exact(&mut self.buf).await { - Ok(res) => res, - Err(e) => { - if let ErrorKind::UnexpectedEof = e.kind() { - break; + let (tx, mut rx) = mpsc::channel(3); + let self_clone = self.clone(); + let task = tokio::spawn(async move { + let mut total: usize = 0; + let mut buf = Vec::new(); + loop { + let mut blocks = >::new(); + if total_size > 0 { + let new_len = { + let remain = total_size - total; + if remain > self_clone.block_size { + self_clone.block_size } else { - return Err(Error::new(e)); + remain } + }; + + if new_len == 0 && total > 0 { + break; } - }; - total += self.buf.len(); + + buf.resize(new_len, 0u8); + match reader.read_exact(&mut buf).await { + Ok(res) => res, + Err(e) => { + if let ErrorKind::UnexpectedEof = e.kind() { + break; + } else { + return Err(Error::new(e)); + } + } + }; + total += buf.len(); + } + self_clone.clone().encode_data(&buf, &mut blocks)?; + let _ = tx.send(blocks).await; } - - self.encode_data(&self.buf, &mut blocks)?; - + Ok(total) + }); + + while let Some(blocks) = rx.recv().await { let write_futures = writers.iter_mut().enumerate().map(|(i, w_op)| { let i_inner = i; let blocks_inner = blocks.clone(); @@ -130,8 +139,7 @@ impl Erasure { break; } } - - Ok(total) + task.await? // // let stream = ChunkedStream::new(body, self.block_size); // let stream = ChunkedStream::new(body, total_size, self.block_size, false); @@ -356,8 +364,8 @@ impl Erasure { self.data_shards + self.parity_shards } - #[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))] - pub fn encode_data(&self, data: &[u8], shards: &mut SmallVec<[Bytes; 16]>) -> Result<()> { + #[tracing::instrument(level = "info", skip_all, fields(data_len=data.len()))] + pub fn encode_data(self: Arc, data: &[u8], shards: &mut SmallVec<[Bytes; 16]>) -> Result<()> { let (shard_size, total_size) = self.need_size(data.len()); // 生成一个新的 所需的所有分片数据长度 @@ -618,34 +626,34 @@ impl ShardReader { #[cfg(test)] mod test { - use super::*; + // use super::*; - #[test] - fn test_erasure() { - let data_shards = 3; - let parity_shards = 2; - let data: &[u8] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]; - let ec = Erasure::new(data_shards, parity_shards, 1); - let mut shards = SmallVec::new(); - ec.encode_data(data, &mut shards).unwrap(); - println!("shards:{:?}", shards); + // #[test] + // fn test_erasure() { + // let data_shards = 3; + // let parity_shards = 2; + // let data: &[u8] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]; + // let ec = Erasure::new(data_shards, parity_shards, 1); + // let mut shards = SmallVec::new(); + // Arc::new(ec).encode_data(data, &mut shards).unwrap(); + // println!("shards:{:?}", shards); - let mut s: Vec<_> = shards - .iter() - .map(|d| if d.is_empty() { None } else { Some(d.to_vec()) }) - .collect(); + // let mut s: Vec<_> = shards + // .iter() + // .map(|d| if d.is_empty() { None } else { Some(d.to_vec()) }) + // .collect(); - // let mut s = shards_to_option_shards(&shards); + // // let mut s = shards_to_option_shards(&shards); - // s[0] = None; - s[4] = None; - s[3] = None; + // // s[0] = None; + // s[4] = None; + // s[3] = None; - println!("sss:{:?}", &s); + // println!("sss:{:?}", &s); - ec.decode_data(&mut s).unwrap(); - // ec.encoder.reconstruct(&mut s).unwrap(); + // ec.decode_data(&mut s).unwrap(); + // // ec.encoder.reconstruct(&mut s).unwrap(); - println!("sss:{:?}", &s); - } + // println!("sss:{:?}", &s); + // } } diff --git a/ecstore/src/io.rs b/ecstore/src/io.rs index f2affe8cb..6bdbd629b 100644 --- a/ecstore/src/io.rs +++ b/ecstore/src/io.rs @@ -2,7 +2,10 @@ use bytes::Bytes; use futures::TryStreamExt; use md5::Digest; use md5::Md5; +use pin_project_lite::pin_project; +use std::io; use std::pin::Pin; +use std::task::ready; use std::task::Context; use std::task::Poll; use tokio::io::AsyncRead; @@ -125,12 +128,19 @@ impl AsyncRead for HttpFileReader { } } -pub struct EtagReader { - inner: R, - bytes_tx: mpsc::Sender, - md5_rx: oneshot::Receiver, +pub trait { + } +pin_project! { + pub struct EtagReader { + inner: R, + bytes_tx: mpsc::Sender, + md5_rx: oneshot::Receiver, + } +} + + impl EtagReader { pub fn new(inner: R) -> Self { let (bytes_tx, mut bytes_rx) = mpsc::channel::(8); @@ -157,21 +167,43 @@ impl EtagReader { } impl AsyncRead for EtagReader { - #[tracing::instrument(level = "debug", skip_all)] - fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { - let poll = Pin::new(&mut self.inner).poll_read(cx, buf); - if let Poll::Ready(Ok(())) = &poll { - if buf.remaining() == 0 { + #[tracing::instrument(level = "info", skip_all)] + fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + let me = self.project(); + + loop { + let rem = buf.remaining(); + if rem != 0 { + ready!(Pin::new(&mut *me.inner).poll_read(cx, buf))?; + if buf.remaining() == rem { + return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "early eof")).into(); + } + } else { let bytes = buf.filled(); let bytes = Bytes::copy_from_slice(bytes); - let tx = self.bytes_tx.clone(); + let tx = me.bytes_tx.clone(); tokio::spawn(async move { if let Err(e) = tx.send(bytes).await { warn!("EtagReader send error: {:?}", e); } }); + return Poll::Ready(Ok(())); } } - poll + + // let poll = Pin::new(&mut self.inner).poll_read(cx, buf); + // if let Poll::Ready(Ok(())) = &poll { + // if buf.remaining() == 0 { + // let bytes = buf.filled(); + // let bytes = Bytes::copy_from_slice(bytes); + // let tx = self.bytes_tx.clone(); + // tokio::spawn(async move { + // if let Err(e) = tx.send(bytes).await { + // warn!("EtagReader send error: {:?}", e); + // } + // }); + // } + // } + // poll } } diff --git a/ecstore/src/set_disk.rs b/ecstore/src/set_disk.rs index bd55ab39f..3c2b033c8 100644 --- a/ecstore/src/set_disk.rs +++ b/ecstore/src/set_disk.rs @@ -3802,8 +3802,8 @@ impl ObjectIO for SetDisks { // TODO: etag from header - let w_size = erasure - .encode(&mut etag_stream, &mut writers, data.content_length, write_quorum) + let w_size = Arc::new(erasure) + .encode(etag_stream, &mut writers, data.content_length, write_quorum) .await?; // TODO: 出错，删除临时目录 if let Err(err) = close_bitrot_writers(&mut writers).await { From 66f5bf1bbcfbc6ab65ee3ceb9d122a3e37641f79 Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Mon, 28 Apr 2025 21:55:31 +0800 Subject: [PATCH 05/38] tmp1 Signed-off-by: junxiang Mu <1948535941@qq.com> --- ecstore/src/erasure.rs | 21 ++++++++++++--------- ecstore/src/io.rs | 12 ++++++++---- ecstore/src/set_disk.rs | 17 +++++++---------- 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs index d5a08af4b..56f5959c5 100644 --- a/ecstore/src/erasure.rs +++ b/ecstore/src/erasure.rs @@ -1,17 +1,18 @@ use crate::bitrot::{BitrotReader, BitrotWriter}; use crate::error::clone_err; +use crate::io::Etag; use crate::quorum::{object_op_ignored_errs, reduce_write_quorum_errs}; use bytes::{Bytes, BytesMut}; use common::error::{Error, Result}; use futures::future::join_all; use reed_solomon_erasure::galois_8::ReedSolomon; use smallvec::SmallVec; -use tokio::sync::mpsc; use std::any::Any; use std::io::ErrorKind; use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::sync::mpsc; use tracing::warn; use tracing::{error, info}; // use tracing::debug; @@ -28,7 +29,7 @@ pub struct Erasure { encoder: Option, pub block_size: usize, _id: Uuid, - buf: Vec, + _buf: Vec, } impl Erasure { @@ -48,7 +49,7 @@ impl Erasure { block_size, encoder, _id: Uuid::new_v4(), - buf: vec![0u8; block_size], + _buf: vec![0u8; block_size], } } @@ -60,9 +61,9 @@ impl Erasure { // block_size: usize, total_size: usize, write_quorum: usize, - ) -> Result + ) -> Result<(usize, String)> where - S: AsyncRead + Unpin + Send + 'static, + S: AsyncRead + Etag + Unpin + Send + 'static, { // pin_mut!(body); // let mut reader = tokio_util::io::StreamReader::new( @@ -85,11 +86,11 @@ impl Erasure { remain } }; - + if new_len == 0 && total > 0 { break; } - + buf.resize(new_len, 0u8); match reader.read_exact(&mut buf).await { Ok(res) => res, @@ -106,9 +107,11 @@ impl Erasure { self_clone.clone().encode_data(&buf, &mut blocks)?; let _ = tx.send(blocks).await; } - Ok(total) + // let etag = reader.etag().await; + let etag = String::new(); + Ok((total, etag)) }); - + while let Some(blocks) = rx.recv().await { let write_futures = writers.iter_mut().enumerate().map(|(i, w_op)| { let i_inner = i; diff --git a/ecstore/src/io.rs b/ecstore/src/io.rs index 6bdbd629b..056260fdf 100644 --- a/ecstore/src/io.rs +++ b/ecstore/src/io.rs @@ -1,3 +1,4 @@ +use async_trait::async_trait; use bytes::Bytes; use futures::TryStreamExt; use md5::Digest; @@ -128,8 +129,9 @@ impl AsyncRead for HttpFileReader { } } -pub trait { - +#[async_trait] +pub trait Etag { + async fn etag(self) -> String; } pin_project! { @@ -140,7 +142,6 @@ pin_project! { } } - impl EtagReader { pub fn new(inner: R) -> Self { let (bytes_tx, mut bytes_rx) = mpsc::channel::(8); @@ -158,8 +159,11 @@ impl EtagReader { EtagReader { inner, bytes_tx, md5_rx } } +} - pub async fn etag(self) -> String { +#[async_trait] +impl Etag for EtagReader { + async fn etag(self) -> String { drop(self.inner); drop(self.bytes_tx); self.md5_rx.await.unwrap() diff --git a/ecstore/src/set_disk.rs b/ecstore/src/set_disk.rs index 3c2b033c8..12491b105 100644 --- a/ecstore/src/set_disk.rs +++ b/ecstore/src/set_disk.rs @@ -3759,7 +3759,7 @@ impl ObjectIO for SetDisks { let tmp_object = format!("{}/{}/part.1", tmp_dir, fi.data_dir.unwrap()); - let mut erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size); + let erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size); let is_inline_buffer = { if let Some(sc) = GLOBAL_StorageClass.get() { @@ -3798,11 +3798,11 @@ impl ObjectIO for SetDisks { } let stream = replace(&mut data.stream, Box::new(empty())); - let mut etag_stream = EtagReader::new(stream); + let etag_stream = EtagReader::new(stream); // TODO: etag from header - let w_size = Arc::new(erasure) + let (w_size, etag) = Arc::new(erasure) .encode(etag_stream, &mut writers, data.content_length, write_quorum) .await?; // TODO: 出错，删除临时目录 @@ -3810,7 +3810,6 @@ impl ObjectIO for SetDisks { error!("close_bitrot_writers err {:?}", err); } - let etag = etag_stream.etag().await; //TODO: userDefined user_defined.insert("etag".to_owned(), etag.clone()); @@ -4408,21 +4407,19 @@ impl StorageAPI for SetDisks { } } - let mut erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size); + let erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size); let stream = replace(&mut data.stream, Box::new(empty())); - let mut etag_stream = EtagReader::new(stream); + let etag_stream = EtagReader::new(stream); - let w_size = erasure - .encode(&mut etag_stream, &mut writers, data.content_length, write_quorum) + let (w_size, mut etag) = Arc::new(erasure) + .encode(etag_stream, &mut writers, data.content_length, write_quorum) .await?; if let Err(err) = close_bitrot_writers(&mut writers).await { error!("close_bitrot_writers err {:?}", err); } - let mut etag = etag_stream.etag().await; - if let Some(ref tag) = opts.preserve_etag { etag = tag.clone(); } From 5a363c55237015a01b698d253c426a076ac4fca1 Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 28 Apr 2025 23:00:54 +0800 Subject: [PATCH 06/38] Comment certificate directory parameters --- scripts/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run.sh b/scripts/run.sh index 690ac8511..59703af95 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -33,7 +33,7 @@ export RUSTFS_CONSOLE_ENABLE=true export RUSTFS_CONSOLE_ADDRESS=":9002" # export RUSTFS_SERVER_DOMAINS="localhost:9000" # HTTPS 证书目录 - export RUSTFS_TLS_PATH="./deploy/certs" +# export RUSTFS_TLS_PATH="./deploy/certs" # 具体路径修改为配置文件真实路径，obs.example.toml 仅供参考 其中`RUSTFS_OBS_CONFIG` 和下面变量二选一 export RUSTFS_OBS_CONFIG="./deploy/config/obs.example.toml" From cdcf5d091759e1535e7ea97fe580194945ca9a47 Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Mon, 28 Apr 2025 06:25:20 +0000 Subject: [PATCH 07/38] fix readme Signed-off-by: junxiang Mu <1948535941@qq.com> --- README.md | 2 +- ecstore/src/erasure.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 511f6736e..812f13a3c 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ observability data formats (e.g. Jaeger, Prometheus, etc.) sending to one or mor 2. Run the following command: ```bash -docker-compose -f docker-compose.yml up -d +docker compose -f docker-compose.yml up -d ``` 3. Access the Grafana dashboard by navigating to `http://localhost:3000` in your browser. The default username and diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs index 021ae7bed..590889e78 100644 --- a/ecstore/src/erasure.rs +++ b/ecstore/src/erasure.rs @@ -470,7 +470,7 @@ impl Erasure { self.encoder.as_ref().unwrap().reconstruct(&mut bufs)?; } - let shards = bufs.into_iter().flatten().collect::>(); + let shards = bufs.into_iter().flatten().map(Bytes::from).collect::>(); if shards.len() != self.parity_shards + self.data_shards { return Err(Error::from_string("can not reconstruct data")); } @@ -479,7 +479,7 @@ impl Erasure { if w.is_none() { continue; } - match w.as_mut().unwrap().write(shards[i].clone().into()).await { + match w.as_mut().unwrap().write(shards[i].clone()).await { Ok(_) => {} Err(e) => { info!("write failed, err: {:?}", e); From 6aecd72acc6219ae791d9a069853b88c075768c6 Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 28 Apr 2025 14:37:28 +0800 Subject: [PATCH 08/38] improve readme.md --- .docker/observability/README.md | 16 ++++++++++++---- deploy/README.md | 18 +++++++++++++----- deploy/build/rustfs-zh.service | 4 ++++ deploy/build/rustfs.service | 4 ++++ deploy/certs/README.md | 16 +++++++++++++--- .../config/{.example.env => .example.obs.env} | 0 6 files changed, 46 insertions(+), 12 deletions(-) rename deploy/config/{.example.env => .example.obs.env} (100%) diff --git a/.docker/observability/README.md b/.docker/observability/README.md index a84f7592d..3d40319b1 100644 --- a/.docker/observability/README.md +++ b/.docker/observability/README.md @@ -6,7 +6,7 @@ This directory contains the observability stack for the application. The stack i - Grafana 11.6.0 - Loki 3.4.2 - Jaeger 2.4.0 -- Otel Collector 0.120.0 #0.121.0 remove loki +- Otel Collector 0.120.0 # 0.121.0 remove loki ## Prometheus @@ -47,8 +47,16 @@ observability data formats (e.g. Jaeger, Prometheus, etc.) sending to one or mor To deploy the observability stack, run the following command: +- docker latest version + ```bash -docker-compose -f docker-compose.yml -f docker-compose.override.yml up -d +docker compose -f docker-compose.yml -f docker-compose.override.yml up -d +``` + +- docker compose v2.0.0 or before + +```bash +docke-compose -f docker-compose.yml -f docker-compose.override.yml up -d ``` To access the Grafana dashboard, navigate to `http://localhost:3000` in your browser. The default username and password @@ -63,7 +71,7 @@ To access the Prometheus dashboard, navigate to `http://localhost:9090` in your To stop the observability stack, run the following command: ```bash -docker-compose -f docker-compose.yml -f docker-compose.override.yml down +docker compose -f docker-compose.yml -f docker-compose.override.yml down ``` ## How to remove data @@ -71,7 +79,7 @@ docker-compose -f docker-compose.yml -f docker-compose.override.yml down To remove the data generated by the observability stack, run the following command: ```bash -docker-compose -f docker-compose.yml -f docker-compose.override.yml down -v +docker compose -f docker-compose.yml -f docker-compose.override.yml down -v ``` ## How to configure diff --git a/deploy/README.md b/deploy/README.md index 4d3d4476a..2efdd85ec 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -23,13 +23,21 @@ managing and monitoring the system. | |--rustfs.service // systemd service file | |--rustfs-zh.service.md // systemd service file in Chinese |--certs -| |--README.md // certs readme -| |--rustfs_tls_cert.pem // API cert.pem -| |--rustfs_tls_key.pem // API key.pem -| |--rustfs_console_tls_cert.pem // console cert.pem -| |--rustfs_console_tls_key.pem // console key.pem +| ├── rustfs_cert.pem // Default｜fallback certificate +| ├── rustfs_key.pem // Default｜fallback private key +| ├── example.com/ // certificate directory of specific domain names +| │ ├── rustfs_cert.pem +| │ └── rustfs_key.pem +| ├── api.example.com/ +| │ ├── rustfs_cert.pem +| │ └── rustfs_key.pem +| └── cdn.example.com/ +| ├── rustfs_cert.pem +| └── rustfs_key.pem |--config | |--obs.example.yaml // example config | |--rustfs.env // env config | |--rustfs-zh.env // env config in Chinese +| |--.example.obs.env // example env config +| |--event.example.toml // event config ``` \ No newline at end of file diff --git a/deploy/build/rustfs-zh.service b/deploy/build/rustfs-zh.service index a1cd0df05..17351e675 100644 --- a/deploy/build/rustfs-zh.service +++ b/deploy/build/rustfs-zh.service @@ -51,6 +51,10 @@ ExecStart=/usr/local/bin/rustfs \ EnvironmentFile=-/etc/default/rustfs ExecStart=/usr/local/bin/rustfs $RUSTFS_VOLUMES $RUSTFS_OPTS +# standard output and error log configuration +StandardOutput=append:/data/deploy/rust/logs/rustfs.log +StandardError=append:/data/deploy/rust/logs/rustfs-err.log + # resource constraints LimitNOFILE=1048576 # 设置文件描述符上限为 1048576，支持高并发连接。 diff --git a/deploy/build/rustfs.service b/deploy/build/rustfs.service index df6e4067b..9c72e4276 100644 --- a/deploy/build/rustfs.service +++ b/deploy/build/rustfs.service @@ -31,6 +31,10 @@ ExecStart=/usr/local/bin/rustfs \ EnvironmentFile=-/etc/default/rustfs ExecStart=/usr/local/bin/rustfs $RUSTFS_VOLUMES $RUSTFS_OPTS +# service log configuration +StandardOutput=append:/data/deploy/rust/logs/rustfs.log +StandardError=append:/data/deploy/rust/logs/rustfs-err.log + # resource constraints LimitNOFILE=1048576 LimitNPROC=32768 diff --git a/deploy/certs/README.md b/deploy/certs/README.md index 84b733e79..e36d188b4 100644 --- a/deploy/certs/README.md +++ b/deploy/certs/README.md @@ -32,7 +32,17 @@ openssl req -x509 -newkey rsa:2048 -keyout key.pem -out cert.pem -days 365 -node ### TLS File ```text - rustfs_public.crt #api cert.pem - - rustfs_private.key #api key.pem +cd deploy/certs/ +ls -la + ├── rustfs_cert.pem // Default｜fallback certificate + ├── rustfs_key.pem // Default｜fallback private key + ├── example.com/ // certificate directory of specific domain names + │ ├── rustfs_cert.pem + │ └── rustfs_key.pem + ├── api.example.com/ + │ ├── rustfs_cert.pem + │ └── rustfs_key.pem + └── cdn.example.com/ + ├── rustfs_cert.pem + └── rustfs_key.pem ``` \ No newline at end of file diff --git a/deploy/config/.example.env b/deploy/config/.example.obs.env similarity index 100% rename from deploy/config/.example.env rename to deploy/config/.example.obs.env From 79d58a98f413498ec01bb132b32f4dc3f1a5c157 Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 28 Apr 2025 14:51:51 +0800 Subject: [PATCH 09/38] improve code for readme.md add chinese readme.md --- .docker/observability/README_ZH.md | 42 +++++++++ README.md | 141 ++++++++++++++++------------- README_ZH.md | 136 ++++++++++++++++++++++++++++ 3 files changed, 255 insertions(+), 64 deletions(-) create mode 100644 .docker/observability/README_ZH.md create mode 100644 README_ZH.md diff --git a/.docker/observability/README_ZH.md b/.docker/observability/README_ZH.md new file mode 100644 index 000000000..7ba5342bf --- /dev/null +++ b/.docker/observability/README_ZH.md @@ -0,0 +1,42 @@ +## 部署可观测性系统 + +OpenTelemetry Collector 提供了一个厂商中立的遥测数据处理方案，用于接收、处理和导出遥测数据。它消除了为支持多种开源可观测性数据格式（如 +Jaeger、Prometheus 等）而需要运行和维护多个代理/收集器的必要性。 + +### 快速部署 + +1. 进入 `.docker/observability` 目录 +2. 执行以下命令启动服务： + +```bash +docker compose up -d -f docker-compose.yml +``` + +### 访问监控面板 + +服务启动后，可通过以下地址访问各个监控面板： + +- Grafana: `http://localhost:3000` (默认账号/密码：`admin`/`admin`) +- Jaeger: `http://localhost:16686` +- Prometheus: `http://localhost:9090` + +## 配置可观测性 + +### 创建配置文件 + +1. 进入 `deploy/config` 目录 +2. 复制示例配置：`cp obs.toml.example obs.toml` +3. 编辑 `obs.toml` 配置文件，修改以下关键参数： + +| 配置项 | 说明 | 示例值 | +|-----------------|----------------------------|-----------------------| +| endpoint | OpenTelemetry Collector 地址 | http://localhost:4317 | +| service_name | 服务名称 | rustfs | +| service_version | 服务版本 | 1.0.0 | +| environment | 运行环境 | production | +| meter_interval | 指标导出间隔 (秒) | 30 | +| sample_ratio | 采样率 | 1.0 | +| use_stdout | 是否输出到控制台 | true/false | +| logger_level | 日志级别 | info | + +``` \ No newline at end of file diff --git a/README.md b/README.md index 812f13a3c..79d9a342c 100644 --- a/README.md +++ b/README.md @@ -1,54 +1,68 @@ -# How to compile RustFS +# RustFS -| Must package | Version | download link | -|--------------|---------|----------------------------------------------------------------------------------------------------------------------------------| -| Rust | 1.8.5 | https://www.rust-lang.org/tools/install | -| protoc | 30.2 | [protoc-30.2-linux-x86_64.zip](https://github.com/protocolbuffers/protobuf/releases/download/v30.2/protoc-30.2-linux-x86_64.zip) | -| flatc | 24.0+ | [Linux.flatc.binary.g++-13.zip](https://github.com/google/flatbuffers/releases/download/v25.2.10/Linux.flatc.binary.g++-13.zip) | +## English Documentation |[中文文档](README_ZH.md) -Download Links: +### Prerequisites -https://github.com/google/flatbuffers/releases/download/v25.2.10/Linux.flatc.binary.g++-13.zip +| Package | Version | Download Link | +|---------|---------|----------------------------------------------------------------------------------------------------------------------------------| +| Rust | 1.8.5+ | [rust-lang.org/tools/install](https://www.rust-lang.org/tools/install) | +| protoc | 30.2+ | [protoc-30.2-linux-x86_64.zip](https://github.com/protocolbuffers/protobuf/releases/download/v30.2/protoc-30.2-linux-x86_64.zip) | +| flatc | 24.0+ | [Linux.flatc.binary.g++-13.zip](https://github.com/google/flatbuffers/releases/download/v25.2.10/Linux.flatc.binary.g++-13.zip) | -https://github.com/protocolbuffers/protobuf/releases/download/v30.2/protoc-30.2-linux-x86_64.zip +### Building RustFS -generate protobuf code: +#### Generate Protobuf Code -```cargo run --bin gproto``` +```bash +cargo run --bin gproto +``` -Or use Docker: +#### Using Docker for Prerequisites -```yml +```yaml - uses: arduino/setup-protoc@v3 with: - version: "30.2" + version: "30.2" - uses: Nugine/setup-flatc@v1 with: - version: "25.2.10" + version: "25.2.10" ``` -# How to add Console web +#### Adding Console Web UI -1. `wget https://dl.rustfs.com/artifacts/console/rustfs-console-latest.zip` +1. Download the latest console UI: + ```bash + wget https://dl.rustfs.com/artifacts/console/rustfs-console-latest.zip + ``` +2. Create the static directory: + ```bash + mkdir -p ./rustfs/static + ``` +3. Extract and compile RustFS: + ```bash + unzip rustfs-console-latest.zip -d ./rustfs/static + cargo build + ``` -2. mkdir in this repos folder `./rustfs/static` +### Running RustFS -3. Compile RustFS +#### Configuration -# Star RustFS +Set the required environment variables: -Add Env Information: - -``` +```bash +# Basic config export RUSTFS_VOLUMES="./target/volume/test" export RUSTFS_ADDRESS="0.0.0.0:9000" export RUSTFS_CONSOLE_ENABLE=true export RUSTFS_CONSOLE_ADDRESS="0.0.0.0:9001" -# 具体路径修改为配置文件真实路径，obs.example.toml 仅供参考 其中`RUSTFS_OBS_CONFIG` 和下面变量二选一 -export RUSTFS_OBS_CONFIG="./deploy/config/obs.example.toml" -# 如下变量需要必须参数都有值才可以，以及会覆盖配置文件`obs.example.toml`中的值 +# Observability config (option 1: config file) +export RUSTFS_OBS_CONFIG="./deploy/config/obs.toml" + +# Observability config (option 2: environment variables) export RUSTFS__OBSERVABILITY__ENDPOINT=http://localhost:4317 export RUSTFS__OBSERVABILITY__USE_STDOUT=true export RUSTFS__OBSERVABILITY__SAMPLE_RATIO=2.0 @@ -57,6 +71,9 @@ export RUSTFS__OBSERVABILITY__SERVICE_NAME=rustfs export RUSTFS__OBSERVABILITY__SERVICE_VERSION=0.1.0 export RUSTFS__OBSERVABILITY__ENVIRONMENT=develop export RUSTFS__OBSERVABILITY__LOGGER_LEVEL=info +export RUSTFS__OBSERVABILITY__LOCAL_LOGGING_ENABLED=true + +# Logging sinks export RUSTFS__SINKS__FILE__ENABLED=true export RUSTFS__SINKS__FILE__PATH="./deploy/logs/rustfs.log" export RUSTFS__SINKS__WEBHOOK__ENABLED=false @@ -68,54 +85,50 @@ export RUSTFS__SINKS__KAFKA__TOPIC="" export RUSTFS__LOGGER__QUEUE_CAPACITY=10 ``` -You need replace your real data folder: +#### Start the service -``` +```bash ./rustfs /data/rustfs ``` -## How to deploy the observability stack +### Observability Stack -The OpenTelemetry Collector offers a vendor-agnostic implementation on how to receive, process, and export telemetry -data. It removes the need to run, operate, and maintain multiple agents/collectors in order to support open-source -observability data formats (e.g. Jaeger, Prometheus, etc.) sending to one or more open-source or commercial back-ends. +#### Deployment -1. Enter the `.docker/observability` directory, -2. Run the following command: +1. Navigate to the observability directory: + ```bash + cd .docker/observability + ``` -```bash -docker compose -f docker-compose.yml up -d -``` +2. Start the observability stack: + ```bash + docker compose up -d -f docker-compose.yml + ``` -3. Access the Grafana dashboard by navigating to `http://localhost:3000` in your browser. The default username and - password are `admin` and `admin`, respectively. +#### Access Monitoring Dashboards -4. Access the Jaeger dashboard by navigating to `http://localhost:16686` in your browser. +- Grafana: `http://localhost:3000` (credentials: `admin`/`admin`) +- Jaeger: `http://localhost:16686` +- Prometheus: `http://localhost:9090` -5. Access the Prometheus dashboard by navigating to `http://localhost:9090` in your browser. +#### Configuring Observability -## Create a new Observability configuration file - -#### 1. Enter the `deploy/config` directory, - -#### 2. Copy `obs.toml.example` to `obs.toml` - -#### 3. Modify the `obs.toml` configuration file - -##### 3.1. Modify the `endpoint` value to the address of the OpenTelemetry Collector - -##### 3.2. Modify the `service_name` value to the name of the service - -##### 3.3. Modify the `service_version` value to the version of the service - -##### 3.4. Modify the `environment` value to the environment of the service - -##### 3.5. Modify the `meter_interval` value to export interval - -##### 3.6. Modify the `sample_ratio` value to the sample ratio - -##### 3.7. Modify the `use_stdout` value to export to stdout - -##### 3.8. Modify the `logger_level` value to the logger level +1. Copy the example configuration: + ```bash + cd deploy/config + cp obs.toml.example obs.toml + ``` +2. Edit `obs.toml` with the following parameters: +| Parameter | Description | Example | +|----------------------|-----------------------------------|-----------------------| +| endpoint | OpenTelemetry Collector address | http://localhost:4317 | +| service_name | Service name | rustfs | +| service_version | Service version | 1.0.0 | +| environment | Runtime environment | production | +| meter_interval | Metrics export interval (seconds) | 30 | +| sample_ratio | Sampling ratio | 1.0 | +| use_stdout | Output to console | true/false | +| logger_level | Log level | info | +| local_logging_enable | stdout | true/false | diff --git a/README_ZH.md b/README_ZH.md new file mode 100644 index 000000000..c2016fb7c --- /dev/null +++ b/README_ZH.md @@ -0,0 +1,136 @@ +# RustFS + +## [English Documentation](README.md) ｜中文文档 + +### 前置要求 + +| 软件包 | 版本 | 下载链接 | +|--------|--------|----------------------------------------------------------------------------------------------------------------------------------| +| Rust | 1.8.5+ | [rust-lang.org/tools/install](https://www.rust-lang.org/tools/install) | +| protoc | 30.2+ | [protoc-30.2-linux-x86_64.zip](https://github.com/protocolbuffers/protobuf/releases/download/v30.2/protoc-30.2-linux-x86_64.zip) | +| flatc | 24.0+ | [Linux.flatc.binary.g++-13.zip](https://github.com/google/flatbuffers/releases/download/v25.2.10/Linux.flatc.binary.g++-13.zip) | + +### 构建 RustFS + +#### 生成 Protobuf 代码 + +```bash +cargo run --bin gproto +``` + +#### 使用 Docker 安装依赖 + +```yaml +- uses: arduino/setup-protoc@v3 + with: + version: "30.2" + +- uses: Nugine/setup-flatc@v1 + with: + version: "25.2.10" +``` + +#### 添加控制台 Web UI + +1. 下载最新的控制台 UI： + ```bash + wget https://dl.rustfs.com/artifacts/console/rustfs-console-latest.zip + ``` +2. 创建静态资源目录： + ```bash + mkdir -p ./rustfs/static + ``` +3. 解压并编译 RustFS： + ```bash + unzip rustfs-console-latest.zip -d ./rustfs/static + cargo build + ``` + +### 运行 RustFS + +#### 配置 + +设置必要的环境变量： + +```bash +# 基础配置 +export RUSTFS_VOLUMES="./target/volume/test" +export RUSTFS_ADDRESS="0.0.0.0:9000" +export RUSTFS_CONSOLE_ENABLE=true +export RUSTFS_CONSOLE_ADDRESS="0.0.0.0:9001" + +# 可观测性配置（方式一：配置文件） +export RUSTFS_OBS_CONFIG="./deploy/config/obs.toml" + +# 可观测性配置（方式二：环境变量） +export RUSTFS__OBSERVABILITY__ENDPOINT=http://localhost:4317 +export RUSTFS__OBSERVABILITY__USE_STDOUT=true +export RUSTFS__OBSERVABILITY__SAMPLE_RATIO=2.0 +export RUSTFS__OBSERVABILITY__METER_INTERVAL=30 +export RUSTFS__OBSERVABILITY__SERVICE_NAME=rustfs +export RUSTFS__OBSERVABILITY__SERVICE_VERSION=0.1.0 +export RUSTFS__OBSERVABILITY__ENVIRONMENT=develop +export RUSTFS__OBSERVABILITY__LOGGER_LEVEL=info +export RUSTFS__OBSERVABILITY__LOCAL_LOGGING_ENABLED=true + +# 日志接收器 +export RUSTFS__SINKS__FILE__ENABLED=true +export RUSTFS__SINKS__FILE__PATH="./deploy/logs/rustfs.log" +export RUSTFS__SINKS__WEBHOOK__ENABLED=false +export RUSTFS__SINKS__WEBHOOK__ENDPOINT="" +export RUSTFS__SINKS__WEBHOOK__AUTH_TOKEN="" +export RUSTFS__SINKS__KAFKA__ENABLED=false +export RUSTFS__SINKS__KAFKA__BOOTSTRAP_SERVERS="" +export RUSTFS__SINKS__KAFKA__TOPIC="" +export RUSTFS__LOGGER__QUEUE_CAPACITY=10 +``` + +#### 启动服务 + +```bash +./rustfs /data/rustfs +``` + +### 可观测性系统 + +#### 部署 + +1. 进入可观测性目录： + ```bash + cd .docker/observability + ``` + +2. 启动可观测性系统： + ```bash + docker compose up -d -f docker-compose.yml + ``` + +#### 访问监控面板 + +- Grafana: `http://localhost:3000` (默认账号/密码：`admin`/`admin`) +- Jaeger: `http://localhost:16686` +- Prometheus: `http://localhost:9090` + +#### 配置可观测性 + +1. 复制示例配置： + ```bash + cd deploy/config + cp obs.toml.example obs.toml + ``` + +2. 编辑 `obs.toml` 配置文件，参数如下： + +| 配置项 | 说明 | 示例值 | +|----------------------|----------------------------|-----------------------| +| endpoint | OpenTelemetry Collector 地址 | http://localhost:4317 | +| service_name | 服务名称 | rustfs | +| service_version | 服务版本 | 1.0.0 | +| environment | 运行环境 | production | +| meter_interval | 指标导出间隔 (秒) | 30 | +| sample_ratio | 采样率 | 1.0 | +| use_stdout | 是否输出到控制台 | true/false | +| logger_level | 日志级别 | info | +| local_logging_enable | 控制台是否答应日志 | true/false | + +``` \ No newline at end of file From e346d202282454a854bed91adb89d4453576c949 Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Tue, 29 Apr 2025 02:53:42 +0000 Subject: [PATCH 10/38] tmp1 Signed-off-by: junxiang Mu <1948535941@qq.com> --- crates/event-notifier/examples/webhook.rs | 8 +++---- crates/event-notifier/src/global.rs | 4 ++-- crates/obs/src/telemetry.rs | 2 +- ecstore/src/erasure.rs | 29 ++++++++--------------- 4 files changed, 17 insertions(+), 26 deletions(-) diff --git a/crates/event-notifier/examples/webhook.rs b/crates/event-notifier/examples/webhook.rs index c754f8223..4cdf02c66 100644 --- a/crates/event-notifier/examples/webhook.rs +++ b/crates/event-notifier/examples/webhook.rs @@ -37,7 +37,7 @@ async fn receive_webhook(Json(payload): Json) -> StatusCode { println!("current time:{:04}-{:02}-{:02} {:02}:{:02}:{:02}", year, month, day, hour, minute, second); println!( "received a webhook request time:{} content:\n {}", - seconds.to_string(), + seconds, serde_json::to_string_pretty(&payload).unwrap() ); StatusCode::OK @@ -66,10 +66,10 @@ fn convert_seconds_to_date(seconds: u64) -> (u32, u32, u32, u32, u32, u32) { // calculate month let days_in_month = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; - for m in 0..12 { - if total_seconds >= days_in_month[m] * seconds_per_day { + for m in &days_in_month { + if total_seconds >= m * seconds_per_day { month += 1; - total_seconds -= days_in_month[m] * seconds_per_day; + total_seconds -= m * seconds_per_day; } else { break; } diff --git a/crates/event-notifier/src/global.rs b/crates/event-notifier/src/global.rs index 25fadeeb4..19693532d 100644 --- a/crates/event-notifier/src/global.rs +++ b/crates/event-notifier/src/global.rs @@ -189,7 +189,7 @@ mod tests { let config = NotifierConfig::default(); let _ = initialize(config.clone()).await; // first initialization let result = initialize(config).await; // second initialization - assert!(!result.is_ok(), "Initialization should succeed"); + assert!(result.is_err(), "Initialization should succeed"); assert!(result.is_err(), "Re-initialization should fail"); } @@ -211,7 +211,7 @@ mod tests { ..Default::default() }; let result = initialize(config).await; - assert!(!result.is_err(), "Initialization with invalid config should fail"); + assert!(result.is_ok(), "Initialization with invalid config should fail"); assert!(is_initialized(), "System should not be marked as initialized after failure"); assert!(is_ready(), "System should not be marked as ready after failure"); } diff --git a/crates/obs/src/telemetry.rs b/crates/obs/src/telemetry.rs index 95bedfc4a..c73d35c0d 100644 --- a/crates/obs/src/telemetry.rs +++ b/crates/obs/src/telemetry.rs @@ -218,7 +218,7 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard { let tracer = tracer_provider.tracer(Cow::Borrowed(service_name).to_string()); // Configure registry to avoid repeated calls to filter methods - let _registry = tracing_subscriber::registry() + tracing_subscriber::registry() .with(filter) .with(ErrorLayer::default()) .with(if config.local_logging_enabled.unwrap_or(false) { diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs index 56f5959c5..247cadae3 100644 --- a/ecstore/src/erasure.rs +++ b/ecstore/src/erasure.rs @@ -53,7 +53,7 @@ impl Erasure { } } - #[tracing::instrument(level = "debug", skip(self, reader, writers))] + #[tracing::instrument(level = "info", skip(self, reader, writers))] pub async fn encode( self: Arc, mut reader: S, @@ -65,23 +65,16 @@ impl Erasure { where S: AsyncRead + Etag + Unpin + Send + 'static, { - // pin_mut!(body); - // let mut reader = tokio_util::io::StreamReader::new( - // body.map(|f| f.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))), - // ); - - let (tx, mut rx) = mpsc::channel(3); - let self_clone = self.clone(); + let (tx, mut rx) = mpsc::channel(5); let task = tokio::spawn(async move { - let mut total: usize = 0; let mut buf = Vec::new(); + let mut total: usize = 0; loop { - let mut blocks = >::new(); if total_size > 0 { let new_len = { let remain = total_size - total; - if remain > self_clone.block_size { - self_clone.block_size + if remain > self.block_size { + self.block_size } else { remain } @@ -104,11 +97,10 @@ impl Erasure { }; total += buf.len(); } - self_clone.clone().encode_data(&buf, &mut blocks)?; + let blocks = Arc::new(Box::pin(self.clone().encode_data(&buf)?)); let _ = tx.send(blocks).await; } - // let etag = reader.etag().await; - let etag = String::new(); + let etag = reader.etag().await; Ok((total, etag)) }); @@ -368,7 +360,7 @@ impl Erasure { } #[tracing::instrument(level = "info", skip_all, fields(data_len=data.len()))] - pub fn encode_data(self: Arc, data: &[u8], shards: &mut SmallVec<[Bytes; 16]>) -> Result<()> { + pub fn encode_data(self: Arc, data: &[u8]) -> Result> { let (shard_size, total_size) = self.need_size(data.len()); // 生成一个新的 所需的所有分片数据长度 @@ -390,14 +382,13 @@ impl Erasure { // 零拷贝分片，所有 shard 引用 data_buffer let mut data_buffer = data_buffer.freeze(); - shards.clear(); - shards.reserve(self.total_shard_count()); + let mut shards = Vec::with_capacity(self.total_shard_count()); for _ in 0..self.total_shard_count() { let shard = data_buffer.split_to(shard_size); shards.push(shard); } - Ok(()) + Ok(shards) } pub fn decode_data(&self, shards: &mut [Option>]) -> Result<()> { From 1dde7015deead33fe931179a261661885ccc1552 Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Tue, 29 Apr 2025 03:41:45 +0000 Subject: [PATCH 11/38] tmp2 Signed-off-by: junxiang Mu <1948535941@qq.com> --- ecstore/src/erasure.rs | 47 +++++++++++++++++++++--------------------- ecstore/src/io.rs | 15 -------------- 2 files changed, 23 insertions(+), 39 deletions(-) diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs index 247cadae3..942f461bd 100644 --- a/ecstore/src/erasure.rs +++ b/ecstore/src/erasure.rs @@ -619,35 +619,34 @@ impl ShardReader { #[cfg(test)] mod test { + use super::*; - // use super::*; + #[test] + fn test_erasure() { + let data_shards = 3; + let parity_shards = 2; + let data: &[u8] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]; + let ec = Erasure::new(data_shards, parity_shards, 1); + let shards = Arc::new(ec).encode_data(data).unwrap(); + println!("shards:{:?}", shards); - // #[test] - // fn test_erasure() { - // let data_shards = 3; - // let parity_shards = 2; - // let data: &[u8] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]; - // let ec = Erasure::new(data_shards, parity_shards, 1); - // let mut shards = SmallVec::new(); - // Arc::new(ec).encode_data(data, &mut shards).unwrap(); - // println!("shards:{:?}", shards); + let mut s: Vec<_> = shards + .iter() + .map(|d| if d.is_empty() { None } else { Some(d.to_vec()) }) + .collect(); - // let mut s: Vec<_> = shards - // .iter() - // .map(|d| if d.is_empty() { None } else { Some(d.to_vec()) }) - // .collect(); + // let mut s = shards_to_option_shards(&shards); - // // let mut s = shards_to_option_shards(&shards); + // s[0] = None; + s[4] = None; + s[3] = None; - // // s[0] = None; - // s[4] = None; - // s[3] = None; + println!("sss:{:?}", &s); - // println!("sss:{:?}", &s); + let ec = Erasure::new(data_shards, parity_shards, 1); + ec.decode_data(&mut s).unwrap(); + // ec.encoder.reconstruct(&mut s).unwrap(); - // ec.decode_data(&mut s).unwrap(); - // // ec.encoder.reconstruct(&mut s).unwrap(); - - // println!("sss:{:?}", &s); - // } + println!("sss:{:?}", &s); + } } diff --git a/ecstore/src/io.rs b/ecstore/src/io.rs index 056260fdf..2bd02c117 100644 --- a/ecstore/src/io.rs +++ b/ecstore/src/io.rs @@ -194,20 +194,5 @@ impl AsyncRead for EtagReader { return Poll::Ready(Ok(())); } } - - // let poll = Pin::new(&mut self.inner).poll_read(cx, buf); - // if let Poll::Ready(Ok(())) = &poll { - // if buf.remaining() == 0 { - // let bytes = buf.filled(); - // let bytes = Bytes::copy_from_slice(bytes); - // let tx = self.bytes_tx.clone(); - // tokio::spawn(async move { - // if let Err(e) = tx.send(bytes).await { - // warn!("EtagReader send error: {:?}", e); - // } - // }); - // } - // } - // poll } } From e5cd061e186317a811c3a5d347d3b1eab389c47f Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 29 Apr 2025 11:55:36 +0800 Subject: [PATCH 12/38] improve code for request and telemetry --- crates/obs/src/telemetry.rs | 15 ++++++--------- rustfs/src/main.rs | 35 +++++++++++++++++++---------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/crates/obs/src/telemetry.rs b/crates/obs/src/telemetry.rs index c73d35c0d..64c9ea46a 100644 --- a/crates/obs/src/telemetry.rs +++ b/crates/obs/src/telemetry.rs @@ -210,7 +210,8 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard { .with_thread_names(true) .with_thread_ids(true) .with_file(true) - .with_line_number(true); + .with_line_number(true) + .with_filter(build_env_filter(logger_level, None)); let filter = build_env_filter(logger_level, None); let otel_filter = build_env_filter(logger_level, None); @@ -231,16 +232,13 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard { .with(MetricsLayer::new(meter_provider.clone())) .init(); info!("Telemetry logging enabled: {:?}", config.local_logging_enabled); - // if config.local_logging_enabled.unwrap_or(false) { - // registry.with(fmt_layer).init(); - // } else { - // registry.init(); - // } if !endpoint.is_empty() { info!( - "OpenTelemetry telemetry initialized with OTLP endpoint: {}, logger_level: {}", - endpoint, logger_level + "OpenTelemetry telemetry initialized with OTLP endpoint: {}, logger_level: {},RUST_LOG env: {}", + endpoint, + logger_level, + std::env::var("RUST_LOG").unwrap_or_else(|_| "未设置".to_string()) ); } } @@ -255,7 +253,6 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard { fn build_env_filter(logger_level: &str, default_level: Option<&str>) -> EnvFilter { let level = default_level.unwrap_or(logger_level); let mut filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(level)); - if !matches!(logger_level, "trace" | "debug") { let directives: SmallVec<[&str; 5]> = smallvec::smallvec!["hyper", "tonic", "h2", "reqwest", "tower"]; for directive in directives { diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index 7871fba58..65fdb419e 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -62,8 +62,8 @@ use tokio_rustls::TlsAcceptor; use tonic::{metadata::MetadataValue, Request, Status}; use tower_http::cors::CorsLayer; use tower_http::trace::TraceLayer; -use tracing::Span; -use tracing::{debug, error, info, info_span, warn}; +use tracing::{debug, error, info, warn}; +use tracing::{instrument, Span}; #[cfg(all(target_os = "linux", target_env = "gnu"))] #[global_allocator] @@ -77,7 +77,7 @@ fn check_auth(req: Request<()>) -> Result, Status> { _ => Err(Status::unauthenticated("No valid auth token")), } } - +#[instrument] fn print_server_info() { let cfg = CONSOLE_CONFIG.get().unwrap(); let current_year = chrono::Utc::now().year(); @@ -95,8 +95,7 @@ async fn main() -> Result<()> { // Parse the obtained parameters let opt = config::Opt::parse(); - // config::init_config(opt.clone()); - + // Initialize the configuration init_license(opt.license.clone()); // Load the configuration file @@ -108,8 +107,13 @@ async fn main() -> Result<()> { // Store in global storage set_global_guard(guard)?; + // Run parameters + run(opt).await +} + +#[instrument] +async fn init_event_notifier(notifier_config: Option) { // Initialize event notifier - let notifier_config = opt.clone().event_config; if notifier_config.is_some() { info!("event_config is not empty"); tokio::spawn(async move { @@ -124,18 +128,16 @@ async fn main() -> Result<()> { } else { info!("event_config is empty"); } - - // Run parameters - run(opt).await } -// #[tokio::main] +#[instrument(skip(opt))] async fn run(opt: config::Opt) -> Result<()> { - let span = info_span!("trace-main-run"); - let _enter = span.enter(); - debug!("opt: {:?}", &opt); + // Initialize event notifier + let notifier_config = opt.event_config; + init_event_notifier(notifier_config).await; + let server_addr = net::parse_and_resolve_address(opt.address.as_str())?; let server_port = server_addr.port(); let server_address = server_addr.to_string(); @@ -175,7 +177,7 @@ async fn run(opt: config::Opt) -> Result<()> { // Detailed endpoint information (showing all API endpoints) let api_endpoints = format!("http://{}:{}", local_ip, server_port); let localhost_endpoint = format!("http://127.0.0.1:{}", server_port); - info!("API: {} {}", api_endpoints, localhost_endpoint); + info!(" API: {} {}", api_endpoints, localhost_endpoint); info!(" RootUser: {}", opt.access_key.clone()); info!(" RootPass: {}", opt.secret_key.clone()); if DEFAULT_ACCESS_KEY.eq(&opt.access_key) && DEFAULT_SECRET_KEY.eq(&opt.secret_key) { @@ -339,7 +341,7 @@ async fn run(opt: config::Opt) -> Result<()> { .layer( TraceLayer::new_for_http() .make_span_with(|request: &HttpRequest<_>| { - let span = tracing::debug_span!("http-request", + let span = tracing::info_span!("http-request", status_code = tracing::field::Empty, method = %request.method(), uri = %request.uri(), @@ -368,13 +370,14 @@ async fn run(opt: config::Opt) -> Result<()> { debug!("http response generated in {:?}", latency) }) .on_body_chunk(|chunk: &Bytes, latency: Duration, _span: &Span| { - info!(histogram.request.body.len = chunk.len(), "histogram request body lenght",); + info!(histogram.request.body.len = chunk.len(), "histogram request body length",); debug!("http body sending {} bytes in {:?}", chunk.len(), latency) }) .on_eos(|_trailers: Option<&HeaderMap>, stream_duration: Duration, _span: &Span| { debug!("http stream closed after {:?}", stream_duration) }) .on_failure(|_error, latency: Duration, _span: &Span| { + info!(counter.rustfs_api_requests_failure_total = 1_u64, "handle request api failure total"); debug!("http request failure error: {:?} in {:?}", _error, latency) }), ) From 7e1135df8f75e83c0b3070981b7621126ba97ebd Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Tue, 29 Apr 2025 10:53:03 +0000 Subject: [PATCH 13/38] tmp3 Signed-off-by: junxiang Mu <1948535941@qq.com> --- Cargo.lock | 1 + ecstore/src/disk/local.rs | 10 ++++---- ecstore/src/quorum.rs | 1 + ecstore/src/set_disk.rs | 50 ++++++++++++++++----------------------- ecstore/src/utils/fs.rs | 18 ++++++++++++++ rustfs/Cargo.toml | 1 + rustfs/src/main.rs | 12 +++++++++- 7 files changed, 58 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 81d6b4c3e..1c62f6659 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7269,6 +7269,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "shadow-rs", + "socket2", "tikv-jemallocator", "time", "tokio", diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 45abdbf5f..45bea2d96 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -38,7 +38,9 @@ use crate::set_disk::{ CHECK_PART_VOLUME_NOT_FOUND, }; use crate::store_api::{BitrotAlgorithm, StorageAPI}; -use crate::utils::fs::{access, lstat, remove, remove_all, rename, O_APPEND, O_CREATE, O_RDONLY, O_WRONLY}; +use crate::utils::fs::{ + access, lstat, remove, remove_all, remove_all_std, remove_std, rename, O_APPEND, O_CREATE, O_RDONLY, O_WRONLY, +}; use crate::utils::os::get_info; use crate::utils::path::{ self, clean, decode_dir_object, encode_dir_object, has_suffix, path_join, path_join_buf, GLOBAL_DIR_SUFFIX, @@ -315,9 +317,9 @@ impl LocalDisk { #[allow(unused_variables)] pub async fn move_to_trash(&self, delete_path: &PathBuf, recursive: bool, immediate_purge: bool) -> Result<()> { if recursive { - remove_all(delete_path).await?; + remove_all_std(delete_path)?; } else { - remove(delete_path).await?; + remove_std(delete_path)?; } return Ok(()); @@ -365,7 +367,7 @@ impl LocalDisk { Ok(()) } - // #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self))] pub async fn delete_file( &self, base_path: &PathBuf, diff --git a/ecstore/src/quorum.rs b/ecstore/src/quorum.rs index d89fa523b..d38177ca5 100644 --- a/ecstore/src/quorum.rs +++ b/ecstore/src/quorum.rs @@ -147,6 +147,7 @@ pub fn reduce_read_quorum_errs( // 根据写quorum验证错误数量 // 返回最大错误数量的下标，或QuorumError +#[tracing::instrument(level = "info", skip_all)] pub fn reduce_write_quorum_errs( errs: &[Option], ignored_errs: &[Box], diff --git a/ecstore/src/set_disk.rs b/ecstore/src/set_disk.rs index 12491b105..abf7422ac 100644 --- a/ecstore/src/set_disk.rs +++ b/ecstore/src/set_disk.rs @@ -407,7 +407,7 @@ impl SetDisks { } #[allow(dead_code)] - #[tracing::instrument(level = "debug", skip(self, disks))] + #[tracing::instrument(level = "info", skip(self, disks))] async fn commit_rename_data_dir( &self, disks: &[Option], @@ -417,40 +417,30 @@ impl SetDisks { write_quorum: usize, ) -> Result<()> { let file_path = format!("{}/{}", object, data_dir); - - let mut futures = Vec::with_capacity(disks.len()); - let mut errs = Vec::with_capacity(disks.len()); - - for disk in disks.iter() { + let futures = disks.iter().map(|disk| { let file_path = file_path.clone(); - futures.push(async move { + async move { if let Some(disk) = disk { - disk.delete( - bucket, - &file_path, - DeleteOptions { - recursive: true, - ..Default::default() - }, - ) - .await + match disk + .delete( + bucket, + &file_path, + DeleteOptions { + recursive: true, + ..Default::default() + }, + ) + .await + { + Ok(_) => None, + Err(e) => Some(e), + } } else { - Err(Error::new(DiskError::DiskNotFound)) - } - }); - } - - let results = join_all(futures).await; - for result in results { - match result { - Ok(_) => { - errs.push(None); - } - Err(e) => { - errs.push(Some(e)); + Some(Error::new(DiskError::DiskNotFound)) } } - } + }); + let errs: Vec> = join_all(futures).await; if let Some(err) = reduce_write_quorum_errs(&errs, object_op_ignored_errs().as_ref(), write_quorum) { return Err(err); diff --git a/ecstore/src/utils/fs.rs b/ecstore/src/utils/fs.rs index 463989c0f..9c0c591f1 100644 --- a/ecstore/src/utils/fs.rs +++ b/ecstore/src/utils/fs.rs @@ -132,6 +132,24 @@ pub async fn remove_all(path: impl AsRef) -> io::Result<()> { } } +pub fn remove_std(path: impl AsRef) -> io::Result<()> { + let meta = std::fs::metadata(path.as_ref())?; + if meta.is_dir() { + std::fs::remove_dir(path.as_ref()) + } else { + std::fs::remove_file(path.as_ref()) + } +} + +pub fn remove_all_std(path: impl AsRef) -> io::Result<()> { + let meta = std::fs::metadata(path.as_ref())?; + if meta.is_dir() { + std::fs::remove_dir_all(path.as_ref()) + } else { + std::fs::remove_file(path.as_ref()) + } +} + pub async fn mkdir(path: impl AsRef) -> io::Result<()> { fs::create_dir(path.as_ref()).await } diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 3019c060e..460e55bef 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -62,6 +62,7 @@ serde.workspace = true serde_json.workspace = true serde_urlencoded = { workspace = true } shadow-rs = { workspace = true, features = ["build", "metadata"] } +socket2 = "0.5.9" tracing.workspace = true time = { workspace = true, features = ["parsing", "formatting", "serde"] } tokio-util.workspace = true diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index 7871fba58..c5eb684a1 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -53,6 +53,7 @@ use rustfs_obs::{init_obs, init_process_observer, load_config, set_global_guard} use rustls::ServerConfig; use s3s::{host::MultiDomain, service::S3ServiceBuilder}; use service::hybrid; +use socket2::SockRef; use std::net::SocketAddr; use std::sync::Arc; use std::time::Duration; @@ -65,6 +66,8 @@ use tower_http::trace::TraceLayer; use tracing::Span; use tracing::{debug, error, info, info_span, warn}; +const MI_B: usize = 1024 * 1024; + #[cfg(all(target_os = "linux", target_env = "gnu"))] #[global_allocator] static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; @@ -425,9 +428,16 @@ async fn run(opt: config::Opt) -> Result<()> { } }; - if let Err(err) = socket.set_nodelay(true) { + let socket_ref = SockRef::from(&socket); + if let Err(err) = socket_ref.set_nodelay(true) { warn!(?err, "Failed to set TCP_NODELAY"); } + if let Err(err) = socket_ref.set_recv_buffer_size(4 * MI_B) { + warn!(?err, "Failed to set set_recv_buffer_size"); + } + if let Err(err) = socket_ref.set_send_buffer_size(4 * MI_B) { + warn!(?err, "Failed to set set_send_buffer_size"); + } if has_tls_certs { debug!("TLS certificates found, starting with SIGINT"); From 8f917e4a196592e6789c4f14a8ee09a87bdd8aa3 Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Tue, 29 Apr 2025 10:55:02 +0000 Subject: [PATCH 14/38] tmp4 Signed-off-by: junxiang Mu <1948535941@qq.com> --- rustfs/src/main.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index c5eb684a1..8355a3244 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -66,7 +66,7 @@ use tower_http::trace::TraceLayer; use tracing::Span; use tracing::{debug, error, info, info_span, warn}; -const MI_B: usize = 1024 * 1024; +// const MI_B: usize = 1024 * 1024; #[cfg(all(target_os = "linux", target_env = "gnu"))] #[global_allocator] @@ -432,12 +432,12 @@ async fn run(opt: config::Opt) -> Result<()> { if let Err(err) = socket_ref.set_nodelay(true) { warn!(?err, "Failed to set TCP_NODELAY"); } - if let Err(err) = socket_ref.set_recv_buffer_size(4 * MI_B) { - warn!(?err, "Failed to set set_recv_buffer_size"); - } - if let Err(err) = socket_ref.set_send_buffer_size(4 * MI_B) { - warn!(?err, "Failed to set set_send_buffer_size"); - } + // if let Err(err) = socket_ref.set_recv_buffer_size(4 * MI_B) { + // warn!(?err, "Failed to set set_recv_buffer_size"); + // } + // if let Err(err) = socket_ref.set_send_buffer_size(4 * MI_B) { + // warn!(?err, "Failed to set set_send_buffer_size"); + // } if has_tls_certs { debug!("TLS certificates found, starting with SIGINT"); From b411a38813b4c2754bde565d9204bdce91dcc728 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 29 Apr 2025 19:04:31 +0800 Subject: [PATCH 15/38] upgrade docker image version and fix docker comman --- .docker/observability/README_ZH.md | 4 +- .docker/observability/config/obs-multi.toml | 34 ++++ .docker/observability/config/obs.toml | 2 +- .docker/observability/docker-compose.yml | 10 +- README.md | 2 +- README_ZH.md | 2 +- docker-compose-obs.yaml | 34 ++-- rustfs/src/console.rs | 13 +- rustfs/src/license.rs | 4 + rustfs/src/server/service_state.rs | 10 +- rustfs/src/utils/certs.rs | 180 +++++++++++++++++++ rustfs/src/utils/mod.rs | 188 +------------------- 12 files changed, 264 insertions(+), 219 deletions(-) create mode 100644 .docker/observability/config/obs-multi.toml create mode 100644 rustfs/src/utils/certs.rs diff --git a/.docker/observability/README_ZH.md b/.docker/observability/README_ZH.md index 7ba5342bf..45474a2e8 100644 --- a/.docker/observability/README_ZH.md +++ b/.docker/observability/README_ZH.md @@ -9,7 +9,7 @@ Jaeger、Prometheus 等）而需要运行和维护多个代理/收集器的必 2. 执行以下命令启动服务： ```bash -docker compose up -d -f docker-compose.yml +docker compose -f docker-compose.yml up -d ``` ### 访问监控面板 @@ -34,7 +34,7 @@ docker compose up -d -f docker-compose.yml | service_name | 服务名称 | rustfs | | service_version | 服务版本 | 1.0.0 | | environment | 运行环境 | production | -| meter_interval | 指标导出间隔 (秒) | 30 | +| meter_interval | 指标导出间隔 (秒) | 30 | | sample_ratio | 采样率 | 1.0 | | use_stdout | 是否输出到控制台 | true/false | | logger_level | 日志级别 | info | diff --git a/.docker/observability/config/obs-multi.toml b/.docker/observability/config/obs-multi.toml new file mode 100644 index 000000000..e4ea037b1 --- /dev/null +++ b/.docker/observability/config/obs-multi.toml @@ -0,0 +1,34 @@ +[observability] +endpoint = "http://otel-collector:4317" # Default is "http://localhost:4317" if not specified +use_stdout = false # Output with stdout, true output, false no output +sample_ratio = 2.0 +meter_interval = 30 +service_name = "rustfs" +service_version = "0.1.0" +environments = "production" +logger_level = "debug" +local_logging_enabled = true + +[sinks] +[sinks.kafka] # Kafka sink is disabled by default +enabled = false +bootstrap_servers = "localhost:9092" +topic = "logs" +batch_size = 100 # Default is 100 if not specified +batch_timeout_ms = 1000 # Default is 1000ms if not specified + +[sinks.webhook] +enabled = false +endpoint = "http://localhost:8080/webhook" +auth_token = "" +batch_size = 100 # Default is 3 if not specified +batch_timeout_ms = 1000 # Default is 100ms if not specified + +[sinks.file] +enabled = true +path = "/root/data/logs/app.log" +batch_size = 10 +batch_timeout_ms = 1000 # Default is 8192 bytes if not specified + +[logger] +queue_capacity = 10 \ No newline at end of file diff --git a/.docker/observability/config/obs.toml b/.docker/observability/config/obs.toml index e4ea037b1..f77c25d84 100644 --- a/.docker/observability/config/obs.toml +++ b/.docker/observability/config/obs.toml @@ -1,5 +1,5 @@ [observability] -endpoint = "http://otel-collector:4317" # Default is "http://localhost:4317" if not specified +endpoint = "http://localhost:4317" # Default is "http://localhost:4317" if not specified use_stdout = false # Output with stdout, true output, false no output sample_ratio = 2.0 meter_interval = 30 diff --git a/.docker/observability/docker-compose.yml b/.docker/observability/docker-compose.yml index f6714bb26..55e4f84c8 100644 --- a/.docker/observability/docker-compose.yml +++ b/.docker/observability/docker-compose.yml @@ -1,6 +1,6 @@ services: otel-collector: - image: otel/opentelemetry-collector-contrib:0.120.0 + image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.124.0 environment: - TZ=Asia/Shanghai volumes: @@ -16,7 +16,7 @@ services: networks: - otel-network jaeger: - image: jaegertracing/jaeger:2.4.0 + image: jaegertracing/jaeger:2.5.0 environment: - TZ=Asia/Shanghai ports: @@ -26,7 +26,7 @@ services: networks: - otel-network prometheus: - image: prom/prometheus:v3.2.1 + image: prom/prometheus:v3.3.0 environment: - TZ=Asia/Shanghai volumes: @@ -36,7 +36,7 @@ services: networks: - otel-network loki: - image: grafana/loki:3.4.2 + image: grafana/loki:3.5.0 environment: - TZ=Asia/Shanghai volumes: @@ -47,7 +47,7 @@ services: networks: - otel-network grafana: - image: grafana/grafana:11.6.0 + image: grafana/grafana:11.6.1 ports: - "3000:3000" # Web UI environment: diff --git a/README.md b/README.md index 79d9a342c..e4818d4dd 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ export RUSTFS__LOGGER__QUEUE_CAPACITY=10 2. Start the observability stack: ```bash - docker compose up -d -f docker-compose.yml + docker compose -f docker-compose.yml up -d ``` #### Access Monitoring Dashboards diff --git a/README_ZH.md b/README_ZH.md index c2016fb7c..10f05bbef 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -102,7 +102,7 @@ export RUSTFS__LOGGER__QUEUE_CAPACITY=10 2. 启动可观测性系统： ```bash - docker compose up -d -f docker-compose.yml + docker compose -f docker-compose.yml up -d ``` #### 访问监控面板 diff --git a/docker-compose-obs.yaml b/docker-compose-obs.yaml index 505f287b5..f6d85b449 100644 --- a/docker-compose-obs.yaml +++ b/docker-compose-obs.yaml @@ -1,6 +1,6 @@ services: otel-collector: - image: otel/opentelemetry-collector-contrib:0.120.0 + image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.124.0 environment: - TZ=Asia/Shanghai volumes: @@ -16,7 +16,7 @@ services: networks: - rustfs-network jaeger: - image: jaegertracing/jaeger:2.4.0 + image: jaegertracing/jaeger:2.5.0 environment: - TZ=Asia/Shanghai ports: @@ -26,7 +26,7 @@ services: networks: - rustfs-network prometheus: - image: prom/prometheus:v3.2.1 + image: prom/prometheus:v3.3.0 environment: - TZ=Asia/Shanghai volumes: @@ -36,7 +36,7 @@ services: networks: - rustfs-network loki: - image: grafana/loki:3.4.2 + image: grafana/loki:3.5.0 environment: - TZ=Asia/Shanghai volumes: @@ -47,7 +47,7 @@ services: networks: - rustfs-network grafana: - image: grafana/grafana:11.6.0 + image: grafana/grafana:11.6.1 ports: - "3000:3000" # Web UI environment: @@ -63,10 +63,10 @@ services: container_name: node1 environment: - RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4} - - RUSTFS_ADDRESS=0.0.0.0:9000 + - RUSTFS_ADDRESS=:9000 - RUSTFS_CONSOLE_ENABLE=true - - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9002 - - RUSTFS_OBS_CONFIG=/etc/observability/config/obs.toml + - RUSTFS_CONSOLE_ADDRESS=:9002 + - RUSTFS_OBS_CONFIG=/etc/observability/config/obs-multi.toml platform: linux/amd64 ports: - "9001:9000" # 映射宿主机的 9001 端口到容器的 9000 端口 @@ -84,10 +84,10 @@ services: container_name: node2 environment: - RUSTFS_VOLUMES=/root/data/target/volume/test{1...4} - - RUSTFS_ADDRESS=0.0.0.0:9000 + - RUSTFS_ADDRESS=:9000 - RUSTFS_CONSOLE_ENABLE=true - - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9002 - - RUSTFS_OBS_CONFIG=/etc/observability/config/obs.toml + - RUSTFS_CONSOLE_ADDRESS=:9002 + - RUSTFS_OBS_CONFIG=/etc/observability/config/obs-multi.toml platform: linux/amd64 ports: - "9002:9000" # 映射宿主机的 9002 端口到容器的 9000 端口 @@ -105,10 +105,10 @@ services: container_name: node3 environment: - RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4} - - RUSTFS_ADDRESS=0.0.0.0:9000 + - RUSTFS_ADDRESS=:9000 - RUSTFS_CONSOLE_ENABLE=true - - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9002 - - RUSTFS_OBS_CONFIG=/etc/observability/config/obs.toml + - RUSTFS_CONSOLE_ADDRESS=:9002 + - RUSTFS_OBS_CONFIG=/etc/observability/config/obs-multi.toml platform: linux/amd64 ports: - "9003:9000" # 映射宿主机的 9003 端口到容器的 9000 端口 @@ -126,10 +126,10 @@ services: container_name: node4 environment: - RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4} - - RUSTFS_ADDRESS=0.0.0.0:9000 + - RUSTFS_ADDRESS=:9000 - RUSTFS_CONSOLE_ENABLE=true - - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9002 - - RUSTFS_OBS_CONFIG=/etc/observability/config/obs.toml + - RUSTFS_CONSOLE_ADDRESS=:9002 + - RUSTFS_OBS_CONFIG=/etc/observability/config/obs-multi.toml platform: linux/amd64 ports: - "9004:9000" # 映射宿主机的 9004 端口到容器的 9000 端口 diff --git a/rustfs/src/console.rs b/rustfs/src/console.rs index c3edbaaa1..0c86d4f3b 100644 --- a/rustfs/src/console.rs +++ b/rustfs/src/console.rs @@ -33,7 +33,8 @@ const RUSTFS_ADMIN_PREFIX: &str = "/rustfs/admin/v3"; #[folder = "$CARGO_MANIFEST_DIR/static"] struct StaticFiles; -async fn static_handler(uri: axum::http::Uri) -> impl IntoResponse { +/// Static file handler +async fn static_handler(uri: Uri) -> impl IntoResponse { let mut path = uri.path().trim_start_matches('/'); if path.is_empty() { path = "index.html" @@ -193,7 +194,7 @@ fn _is_private_ip(ip: std::net::IpAddr) -> bool { async fn config_handler(uri: Uri, Host(host): Host) -> impl IntoResponse { let scheme = uri.scheme().map(|s| s.as_str()).unwrap_or("http"); - // 从 uri 中获取 host，如果没有则使用 Host extractor 的值 + // Get the host from the uri and use the value of the host extractor if it doesn't have one let host = uri.host().unwrap_or(host.as_str()); let host = if host.contains(':') { @@ -203,7 +204,7 @@ async fn config_handler(uri: Uri, Host(host): Host) -> impl IntoResponse { host }; - // 将当前配置复制一份 + // Make a copy of the current configuration let mut cfg = CONSOLE_CONFIG.get().unwrap().clone(); let url = format!("{}://{}:{}", scheme, host, cfg.port); @@ -224,9 +225,9 @@ pub async fn start_static_file_server( secret_key: &str, tls_path: Option, ) { - // 配置 CORS + // Configure CORS let cors = CorsLayer::new() - .allow_origin(Any) // 生产环境建议指定具体域名 + .allow_origin(Any) // In the production environment, we recommend that you specify a specific domain name .allow_methods([http::Method::GET, http::Method::POST]) .allow_headers([header::CONTENT_TYPE]); // Create a route @@ -298,7 +299,7 @@ async fn start_server(server_addr: SocketAddr, tls_path: Option, app: Ro } #[allow(dead_code)] -/// HTTP 到 HTTPS 的 308 重定向 +/// 308 redirect for HTTP to HTTPS fn redirect_to_https(https_port: u16) -> Router { Router::new().route( "/*path", diff --git a/rustfs/src/license.rs b/rustfs/src/license.rs index 25d09c82c..2206c40e2 100644 --- a/rustfs/src/license.rs +++ b/rustfs/src/license.rs @@ -10,6 +10,7 @@ lazy_static::lazy_static! { static ref LICENSE: OnceLock = OnceLock::new(); } +/// Initialize the license pub fn init_license(license: Option) { if license.is_none() { error!("License is None"); @@ -23,10 +24,13 @@ pub fn init_license(license: Option) { }); } +/// Get the license pub fn get_license() -> Option { LICENSE.get().cloned() } +/// Check the license +/// This function checks if the license is valid. #[allow(unreachable_code)] pub fn license_check() -> Result<()> { return Ok(()); diff --git a/rustfs/src/server/service_state.rs b/rustfs/src/server/service_state.rs index ad137f66d..5390fb1e4 100644 --- a/rustfs/src/server/service_state.rs +++ b/rustfs/src/server/service_state.rs @@ -129,7 +129,7 @@ impl Default for ServiceStateManager { } } -// 使用示例 +// Example of use #[cfg(test)] mod tests { use super::*; @@ -138,18 +138,18 @@ mod tests { fn test_service_state_manager() { let manager = ServiceStateManager::new(); - // 初始状态应该是 Starting + // The initial state should be Starting assert_eq!(manager.current_state(), ServiceState::Starting); - // 更新状态到 Ready + // Update the status to Ready manager.update(ServiceState::Ready); assert_eq!(manager.current_state(), ServiceState::Ready); - // 更新状态到 Stopping + // Update the status to Stopping manager.update(ServiceState::Stopping); assert_eq!(manager.current_state(), ServiceState::Stopping); - // 更新状态到 Stopped + // Update the status to Stopped manager.update(ServiceState::Stopped); assert_eq!(manager.current_state(), ServiceState::Stopped); } diff --git a/rustfs/src/utils/certs.rs b/rustfs/src/utils/certs.rs new file mode 100644 index 000000000..115d43451 --- /dev/null +++ b/rustfs/src/utils/certs.rs @@ -0,0 +1,180 @@ +use crate::config::{RUSTFS_TLS_CERT, RUSTFS_TLS_KEY}; +use rustls::server::{ClientHello, ResolvesServerCert, ResolvesServerCertUsingSni}; +use rustls::sign::CertifiedKey; +use rustls_pemfile::{certs, private_key}; +use rustls_pki_types::{CertificateDer, PrivateKeyDer}; +use std::collections::HashMap; +use std::io::Error; +use std::path::Path; +use std::sync::Arc; +use std::{fs, io}; +use tracing::{debug, warn}; + +/// Load public certificate from file. +/// This function loads a public certificate from the specified file. +pub(crate) fn load_certs(filename: &str) -> io::Result>> { + // Open certificate file. + let cert_file = fs::File::open(filename).map_err(|e| error(format!("failed to open {}: {}", filename, e)))?; + let mut reader = io::BufReader::new(cert_file); + + // Load and return certificate. + let certs = certs(&mut reader) + .collect::, _>>() + .map_err(|_| error(format!("certificate file {} format error", filename)))?; + if certs.is_empty() { + return Err(error(format!("No valid certificate was found in the certificate file {}", filename))); + } + Ok(certs) +} + +/// Load private key from file. +/// This function loads a private key from the specified file. +pub(crate) fn load_private_key(filename: &str) -> io::Result> { + // Open keyfile. + let keyfile = fs::File::open(filename).map_err(|e| error(format!("failed to open {}: {}", filename, e)))?; + let mut reader = io::BufReader::new(keyfile); + + // Load and return a single private key. + private_key(&mut reader)?.ok_or_else(|| error(format!("no private key found in {}", filename))) +} + +/// error function +pub(crate) fn error(err: String) -> Error { + Error::new(io::ErrorKind::Other, err) +} + +/// Load all certificates and private keys in the directory +/// This function loads all certificate and private key pairs from the specified directory. +/// It looks for files named `rustfs_cert.pem` and `rustfs_key.pem` in each subdirectory. +/// The root directory can also contain a default certificate/private key pair. +pub(crate) fn load_all_certs_from_directory( + dir_path: &str, +) -> io::Result>, PrivateKeyDer<'static>)>> { + let mut cert_key_pairs = HashMap::new(); + let dir = Path::new(dir_path); + + if !dir.exists() || !dir.is_dir() { + return Err(error(format!( + "The certificate directory does not exist or is not a directory: {}", + dir_path + ))); + } + + // 1. First check whether there is a certificate/private key pair in the root directory + let root_cert_path = dir.join(RUSTFS_TLS_CERT); + let root_key_path = dir.join(RUSTFS_TLS_KEY); + + if root_cert_path.exists() && root_key_path.exists() { + debug!("find the root directory certificate: {:?}", root_cert_path); + let root_cert_str = root_cert_path + .to_str() + .ok_or_else(|| error(format!("Invalid UTF-8 in root certificate path: {:?}", root_cert_path)))?; + let root_key_str = root_key_path + .to_str() + .ok_or_else(|| error(format!("Invalid UTF-8 in root key path: {:?}", root_key_path)))?; + match load_cert_key_pair(root_cert_str, root_key_str) { + Ok((certs, key)) => { + // The root directory certificate is used as the default certificate and is stored using special keys. + cert_key_pairs.insert("default".to_string(), (certs, key)); + } + Err(e) => { + warn!("unable to load root directory certificate: {}", e); + } + } + } + + // 2.iterate through all folders in the directory + for entry in fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + + if path.is_dir() { + let domain_name = path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| error(format!("invalid domain name directory:{:?}", path)))?; + + // find certificate and private key files + let cert_path = path.join(RUSTFS_TLS_CERT); // e.g., rustfs_cert.pem + let key_path = path.join(RUSTFS_TLS_KEY); // e.g., rustfs_key.pem + + if cert_path.exists() && key_path.exists() { + debug!("find the domain name certificate: {} in {:?}", domain_name, cert_path); + match load_cert_key_pair(cert_path.to_str().unwrap(), key_path.to_str().unwrap()) { + Ok((certs, key)) => { + cert_key_pairs.insert(domain_name.to_string(), (certs, key)); + } + Err(e) => { + warn!("unable to load the certificate for {} domain name: {}", domain_name, e); + } + } + } + } + } + + if cert_key_pairs.is_empty() { + return Err(error(format!("No valid certificate/private key pair found in directory {}", dir_path))); + } + + Ok(cert_key_pairs) +} + +/// loading a single certificate private key pair +/// This function loads a certificate and private key from the specified paths. +/// It returns a tuple containing the certificate and private key. +fn load_cert_key_pair(cert_path: &str, key_path: &str) -> io::Result<(Vec>, PrivateKeyDer<'static>)> { + let certs = load_certs(cert_path)?; + let key = load_private_key(key_path)?; + Ok((certs, key)) +} + +/// Create a multi-cert resolver +/// This function loads all certificates and private keys from the specified directory. +/// It uses the first certificate/private key pair found in the root directory as the default certificate. +/// The rest of the certificates/private keys are used for SNI resolution. +/// +pub fn create_multi_cert_resolver( + cert_key_pairs: HashMap>, PrivateKeyDer<'static>)>, +) -> io::Result { + #[derive(Debug)] + struct MultiCertResolver { + cert_resolver: ResolvesServerCertUsingSni, + default_cert: Option>, + } + impl ResolvesServerCert for MultiCertResolver { + fn resolve(&self, client_hello: ClientHello) -> Option> { + // try matching certificates with sni + if let Some(cert) = self.cert_resolver.resolve(client_hello) { + return Some(cert); + } + + // If there is no matching SNI certificate, use the default certificate + self.default_cert.clone() + } + } + + let mut resolver = ResolvesServerCertUsingSni::new(); + let mut default_cert = None; + + for (domain, (certs, key)) in cert_key_pairs { + // create a signature + let signing_key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key) + .map_err(|_| error(format!("unsupported private key types:{}", domain)))?; + + // create a CertifiedKey + let certified_key = CertifiedKey::new(certs, signing_key); + if domain == "default" { + default_cert = Some(Arc::new(certified_key.clone())); + } else { + // add certificate to resolver + resolver + .add(&domain, certified_key) + .map_err(|e| error(format!("failed to add a domain name certificate:{},err: {:?}", domain, e)))?; + } + } + + Ok(MultiCertResolver { + cert_resolver: resolver, + default_cert, + }) +} diff --git a/rustfs/src/utils/mod.rs b/rustfs/src/utils/mod.rs index fad2718c8..9f391fee6 100644 --- a/rustfs/src/utils/mod.rs +++ b/rustfs/src/utils/mod.rs @@ -1,16 +1,11 @@ -use crate::config::{RUSTFS_TLS_CERT, RUSTFS_TLS_KEY}; -use rustls::server::{ClientHello, ResolvesServerCert, ResolvesServerCertUsingSni}; -use rustls::sign::CertifiedKey; -use rustls_pemfile::{certs, private_key}; -use rustls_pki_types::{CertificateDer, PrivateKeyDer}; -use std::collections::HashMap; -use std::fmt::Debug; -use std::io::Error; +mod certs; use std::net::IpAddr; -use std::path::Path; -use std::sync::Arc; -use std::{fs, io}; -use tracing::{debug, warn}; + +pub(crate) use certs::create_multi_cert_resolver; +pub(crate) use certs::error; +pub(crate) use certs::load_all_certs_from_directory; +pub(crate) use certs::load_certs; +pub(crate) use certs::load_private_key; /// Get the local IP address. /// This function retrieves the local IP address of the machine. @@ -21,172 +16,3 @@ pub(crate) fn get_local_ip() -> Option { Ok(IpAddr::V6(_)) => todo!(), } } - -/// Load public certificate from file. -/// This function loads a public certificate from the specified file. -pub(crate) fn load_certs(filename: &str) -> io::Result>> { - // Open certificate file. - let cert_file = fs::File::open(filename).map_err(|e| error(format!("failed to open {}: {}", filename, e)))?; - let mut reader = io::BufReader::new(cert_file); - - // Load and return certificate. - let certs = certs(&mut reader) - .collect::, _>>() - .map_err(|_| error(format!("certificate file {} format error", filename)))?; - if certs.is_empty() { - return Err(error(format!("No valid certificate was found in the certificate file {}", filename))); - } - Ok(certs) -} - -/// Load private key from file. -/// This function loads a private key from the specified file. -pub(crate) fn load_private_key(filename: &str) -> io::Result> { - // Open keyfile. - let keyfile = fs::File::open(filename).map_err(|e| error(format!("failed to open {}: {}", filename, e)))?; - let mut reader = io::BufReader::new(keyfile); - - // Load and return a single private key. - private_key(&mut reader)?.ok_or_else(|| error(format!("no private key found in {}", filename))) -} - -/// error function -pub(crate) fn error(err: String) -> Error { - Error::new(io::ErrorKind::Other, err) -} - -/// Load all certificates and private keys in the directory -/// This function loads all certificate and private key pairs from the specified directory. -/// It looks for files named `rustfs_cert.pem` and `rustfs_key.pem` in each subdirectory. -/// The root directory can also contain a default certificate/private key pair. -pub(crate) fn load_all_certs_from_directory( - dir_path: &str, -) -> io::Result>, PrivateKeyDer<'static>)>> { - let mut cert_key_pairs = HashMap::new(); - let dir = Path::new(dir_path); - - if !dir.exists() || !dir.is_dir() { - return Err(error(format!( - "The certificate directory does not exist or is not a directory: {}", - dir_path - ))); - } - - // 1. First check whether there is a certificate/private key pair in the root directory - let root_cert_path = dir.join(RUSTFS_TLS_CERT); - let root_key_path = dir.join(RUSTFS_TLS_KEY); - - if root_cert_path.exists() && root_key_path.exists() { - debug!("find the root directory certificate: {:?}", root_cert_path); - let root_cert_str = root_cert_path - .to_str() - .ok_or_else(|| error(format!("Invalid UTF-8 in root certificate path: {:?}", root_cert_path)))?; - let root_key_str = root_key_path - .to_str() - .ok_or_else(|| error(format!("Invalid UTF-8 in root key path: {:?}", root_key_path)))?; - match load_cert_key_pair(root_cert_str, root_key_str) { - Ok((certs, key)) => { - // The root directory certificate is used as the default certificate and is stored using special keys. - cert_key_pairs.insert("default".to_string(), (certs, key)); - } - Err(e) => { - warn!("unable to load root directory certificate: {}", e); - } - } - } - - // 2.iterate through all folders in the directory - for entry in fs::read_dir(dir)? { - let entry = entry?; - let path = entry.path(); - - if path.is_dir() { - let domain_name = path - .file_name() - .and_then(|name| name.to_str()) - .ok_or_else(|| error(format!("invalid domain name directory:{:?}", path)))?; - - // find certificate and private key files - let cert_path = path.join(RUSTFS_TLS_CERT); // e.g., rustfs_cert.pem - let key_path = path.join(RUSTFS_TLS_KEY); // e.g., rustfs_key.pem - - if cert_path.exists() && key_path.exists() { - debug!("find the domain name certificate: {} in {:?}", domain_name, cert_path); - match load_cert_key_pair(cert_path.to_str().unwrap(), key_path.to_str().unwrap()) { - Ok((certs, key)) => { - cert_key_pairs.insert(domain_name.to_string(), (certs, key)); - } - Err(e) => { - warn!("unable to load the certificate for {} domain name: {}", domain_name, e); - } - } - } - } - } - - if cert_key_pairs.is_empty() { - return Err(error(format!("No valid certificate/private key pair found in directory {}", dir_path))); - } - - Ok(cert_key_pairs) -} - -/// loading a single certificate private key pair -/// This function loads a certificate and private key from the specified paths. -/// It returns a tuple containing the certificate and private key. -fn load_cert_key_pair(cert_path: &str, key_path: &str) -> io::Result<(Vec>, PrivateKeyDer<'static>)> { - let certs = load_certs(cert_path)?; - let key = load_private_key(key_path)?; - Ok((certs, key)) -} - -/// Create a multi-cert resolver -/// This function loads all certificates and private keys from the specified directory. -/// It uses the first certificate/private key pair found in the root directory as the default certificate. -/// The rest of the certificates/private keys are used for SNI resolution. -/// -pub fn create_multi_cert_resolver( - cert_key_pairs: HashMap>, PrivateKeyDer<'static>)>, -) -> io::Result { - #[derive(Debug)] - struct MultiCertResolver { - cert_resolver: ResolvesServerCertUsingSni, - default_cert: Option>, - } - impl ResolvesServerCert for MultiCertResolver { - fn resolve(&self, client_hello: ClientHello) -> Option> { - // try matching certificates with sni - if let Some(cert) = self.cert_resolver.resolve(client_hello) { - return Some(cert); - } - - // If there is no matching SNI certificate, use the default certificate - self.default_cert.clone() - } - } - - let mut resolver = ResolvesServerCertUsingSni::new(); - let mut default_cert = None; - - for (domain, (certs, key)) in cert_key_pairs { - // create a signature - let signing_key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key) - .map_err(|_| error(format!("unsupported private key types:{}", domain)))?; - - // create a CertifiedKey - let certified_key = CertifiedKey::new(certs, signing_key); - if domain == "default" { - default_cert = Some(Arc::new(certified_key.clone())); - } else { - // add certificate to resolver - resolver - .add(&domain, certified_key) - .map_err(|e| error(format!("failed to add a domain name certificate:{},err: {:?}", domain, e)))?; - } - } - - Ok(MultiCertResolver { - cert_resolver: resolver, - default_cert, - }) -} From 05fad0aca0ce9bc072ce85118028870b796f3ae1 Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Tue, 29 Apr 2025 11:08:48 +0000 Subject: [PATCH 16/38] tmp5 Signed-off-by: junxiang Mu <1948535941@qq.com> --- rustfs/src/main.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index 8355a3244..c5eb684a1 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -66,7 +66,7 @@ use tower_http::trace::TraceLayer; use tracing::Span; use tracing::{debug, error, info, info_span, warn}; -// const MI_B: usize = 1024 * 1024; +const MI_B: usize = 1024 * 1024; #[cfg(all(target_os = "linux", target_env = "gnu"))] #[global_allocator] @@ -432,12 +432,12 @@ async fn run(opt: config::Opt) -> Result<()> { if let Err(err) = socket_ref.set_nodelay(true) { warn!(?err, "Failed to set TCP_NODELAY"); } - // if let Err(err) = socket_ref.set_recv_buffer_size(4 * MI_B) { - // warn!(?err, "Failed to set set_recv_buffer_size"); - // } - // if let Err(err) = socket_ref.set_send_buffer_size(4 * MI_B) { - // warn!(?err, "Failed to set set_send_buffer_size"); - // } + if let Err(err) = socket_ref.set_recv_buffer_size(4 * MI_B) { + warn!(?err, "Failed to set set_recv_buffer_size"); + } + if let Err(err) = socket_ref.set_send_buffer_size(4 * MI_B) { + warn!(?err, "Failed to set set_send_buffer_size"); + } if has_tls_certs { debug!("TLS certificates found, starting with SIGINT"); From 01d5383ce3a0c856add07adb8e868f8a1705656b Mon Sep 17 00:00:00 2001 From: houseme Date: Wed, 30 Apr 2025 00:31:55 +0800 Subject: [PATCH 17/38] Feature/bucket event notification (#365) * add tracing instrument * fix rebalance/decom * modify Telemetry filter order * feat: improve address binding and port handling mechanism (#366) * feat: improve address binding and port handling mechanism 1. Add support for ":port" format to enable dual-stack binding (IPv4/IPv6) 2. Implement automatic port allocation when port 0 is specified 3. Optimize server startup process with unified address resolution 4. Enhance error handling and logging for address resolution 5. Improve graceful shutdown with signal listening 6. Clean up commented code in console.rs Files: - ecstore/src/utils/net.rs - rustfs/src/console.rs - rustfs/src/main.rs Branch: feature/server-and-console-port * improve code for console * improve code * improve code for console and net.rs * Update rustfs/src/main.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update rustfs/src/utils/mod.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * upgrade config file * modify * fix readme Signed-off-by: junxiang Mu <1948535941@qq.com> * improve readme.md * improve code for readme.md add chinese readme.md * Implement Storage Service Event Notification System Added event notification capability to the storage module, enabling the storage service to publish object operation events. Key changes include: 1. Created `event_notifier` module providing core functionality: - `create_metadata()` - Creates event metadata objects with default configuration ID - `send_event()` - Asynchronously sends event notifications with error handling 2. Integrated the `rustfs_event_notifier` library: - Supports object creation, deletion, and access events - Provides event metadata building and management - Includes proper error propagation These changes enable the system to trigger notifications when storage operations occur, facilitating auditing, monitoring, and integration with other systems. * fix --------- Signed-off-by: junxiang Mu <1948535941@qq.com> Co-authored-by: weisd Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: junxiang Mu <1948535941@qq.com> --- crates/event-notifier/src/bus.rs | 2 + crates/event-notifier/src/event.rs | 181 +++++++++++++++++++++++++- crates/event-notifier/src/global.rs | 3 + crates/event-notifier/src/notifier.rs | 10 +- crates/event-notifier/src/store.rs | 2 + crates/obs/src/telemetry.rs | 1 - rustfs/src/storage/access.rs | 1 + rustfs/src/storage/ecfs.rs | 6 + rustfs/src/storage/event_notifier.rs | 17 +++ rustfs/src/storage/mod.rs | 1 + rustfs/src/storage/options.rs | 9 ++ 11 files changed, 230 insertions(+), 3 deletions(-) create mode 100644 rustfs/src/storage/event_notifier.rs diff --git a/crates/event-notifier/src/bus.rs b/crates/event-notifier/src/bus.rs index bd4b81c5d..5cabfc22e 100644 --- a/crates/event-notifier/src/bus.rs +++ b/crates/event-notifier/src/bus.rs @@ -7,11 +7,13 @@ use std::time::{SystemTime, UNIX_EPOCH}; use tokio::sync::mpsc; use tokio::time::Duration; use tokio_util::sync::CancellationToken; +use tracing::instrument; /// Handles incoming events from the producer. /// /// This function is responsible for receiving events from the producer and sending them to the appropriate adapters. /// It also handles the shutdown process and saves any pending logs to the event store. +#[instrument(skip_all)] pub async fn event_bus( mut rx: mpsc::Receiver, adapters: Vec>, diff --git a/crates/event-notifier/src/event.rs b/crates/event-notifier/src/event.rs index ce32aecca..55e4c0bd1 100644 --- a/crates/event-notifier/src/event.rs +++ b/crates/event-notifier/src/event.rs @@ -15,6 +15,18 @@ pub struct Identity { pub principal_id: String, } +impl Identity { + /// Create a new Identity instance + pub fn new(principal_id: String) -> Self { + Self { principal_id } + } + + /// Set the principal ID + pub fn set_principal_id(&mut self, principal_id: String) { + self.principal_id = principal_id; + } +} + /// A struct representing the bucket information #[derive(Serialize, Deserialize, Clone, Debug)] pub struct Bucket { @@ -24,6 +36,32 @@ pub struct Bucket { pub arn: String, } +impl Bucket { + /// Create a new Bucket instance + pub fn new(name: String, owner_identity: Identity, arn: String) -> Self { + Self { + name, + owner_identity, + arn, + } + } + + /// Set the name of the bucket + pub fn set_name(&mut self, name: String) { + self.name = name; + } + + /// Set the ARN of the bucket + pub fn set_arn(&mut self, arn: String) { + self.arn = arn; + } + + /// Set the owner identity of the bucket + pub fn set_owner_identity(&mut self, owner_identity: Identity) { + self.owner_identity = owner_identity; + } +} + /// A struct representing the object information #[derive(Serialize, Deserialize, Clone, Debug)] pub struct Object { @@ -41,6 +79,64 @@ pub struct Object { pub sequencer: String, } +impl Object { + /// Create a new Object instance + pub fn new( + key: String, + size: Option, + etag: Option, + content_type: Option, + user_metadata: Option>, + version_id: Option, + sequencer: String, + ) -> Self { + Self { + key, + size, + etag, + content_type, + user_metadata, + version_id, + sequencer, + } + } + + /// Set the key + pub fn set_key(&mut self, key: String) { + self.key = key; + } + + /// Set the size + pub fn set_size(&mut self, size: Option) { + self.size = size; + } + + /// Set the etag + pub fn set_etag(&mut self, etag: Option) { + self.etag = etag; + } + + /// Set the content type + pub fn set_content_type(&mut self, content_type: Option) { + self.content_type = content_type; + } + + /// Set the user metadata + pub fn set_user_metadata(&mut self, user_metadata: Option>) { + self.user_metadata = user_metadata; + } + + /// Set the version ID + pub fn set_version_id(&mut self, version_id: Option) { + self.version_id = version_id; + } + + /// Set the sequencer + pub fn set_sequencer(&mut self, sequencer: String) { + self.sequencer = sequencer; + } +} + /// A struct representing the metadata of the event #[derive(Serialize, Deserialize, Clone, Debug)] pub struct Metadata { @@ -52,6 +148,57 @@ pub struct Metadata { pub object: Object, } +impl Default for Metadata { + fn default() -> Self { + Self::new() + } +} +impl Metadata { + /// Create a new Metadata instance + pub fn create(schema_version: String, configuration_id: String, bucket: Bucket, object: Object) -> Self { + Self { + schema_version, + configuration_id, + bucket, + object, + } + } + + /// Create a new Metadata instance with default values + pub fn new() -> Self { + Self { + schema_version: "1.0".to_string(), + configuration_id: "default".to_string(), + bucket: Bucket::new( + "default".to_string(), + Identity::new("default".to_string()), + "arn:aws:s3:::default".to_string(), + ), + object: Object::new("default".to_string(), None, None, None, None, None, "default".to_string()), + } + } + + /// Set the schema version + pub fn set_schema_version(&mut self, schema_version: String) { + self.schema_version = schema_version; + } + + /// Set the configuration ID + pub fn set_configuration_id(&mut self, configuration_id: String) { + self.configuration_id = configuration_id; + } + + /// Set the bucket + pub fn set_bucket(&mut self, bucket: Bucket) { + self.bucket = bucket; + } + + /// Set the object + pub fn set_object(&mut self, object: Object) { + self.object = object; + } +} + /// A struct representing the source of the event #[derive(Serialize, Deserialize, Clone, Debug)] pub struct Source { @@ -61,6 +208,28 @@ pub struct Source { pub user_agent: String, } +impl Source { + /// Create a new Source instance + pub fn new(host: String, port: String, user_agent: String) -> Self { + Self { host, port, user_agent } + } + + /// Set the host + pub fn set_host(&mut self, host: String) { + self.host = host; + } + + /// Set the port + pub fn set_port(&mut self, port: String) { + self.port = port; + } + + /// Set the user agent + pub fn set_user_agent(&mut self, user_agent: String) { + self.user_agent = user_agent; + } +} + /// Builder for creating an Event. /// /// This struct is used to build an Event object with various parameters. @@ -301,7 +470,17 @@ pub struct Log { pub records: Vec, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, SerializeDisplay, DeserializeFromStr, Display, EnumString)] +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + SerializeDisplay, + DeserializeFromStr, + Display, + EnumString +)] #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] pub enum Name { ObjectAccessedGet, diff --git a/crates/event-notifier/src/global.rs b/crates/event-notifier/src/global.rs index 19693532d..0ffcb8b00 100644 --- a/crates/event-notifier/src/global.rs +++ b/crates/event-notifier/src/global.rs @@ -1,6 +1,7 @@ use crate::{create_adapters, Error, Event, NotifierConfig, NotifierSystem}; use std::sync::{atomic, Arc}; use tokio::sync::{Mutex, OnceCell}; +use tracing::instrument; static GLOBAL_SYSTEM: OnceCell>> = OnceCell::const_new(); static INITIALIZED: atomic::AtomicBool = atomic::AtomicBool::new(false); @@ -113,6 +114,7 @@ pub fn is_ready() -> bool { /// - The system is not initialized. /// - The system is not ready. /// - Sending the event fails. +#[instrument(fields(event))] pub async fn send_event(event: Event) -> Result<(), Error> { if !READY.load(atomic::Ordering::SeqCst) { return Err(Error::custom("Notification system not ready, please wait for initialization to complete")); @@ -124,6 +126,7 @@ pub async fn send_event(event: Event) -> Result<(), Error> { } /// Shuts down the notification system. +#[instrument] pub async fn shutdown() -> Result<(), Error> { if let Some(system) = GLOBAL_SYSTEM.get() { tracing::info!("Shutting down notification system start"); diff --git a/crates/event-notifier/src/notifier.rs b/crates/event-notifier/src/notifier.rs index 0b17ddddd..5ab17d371 100644 --- a/crates/event-notifier/src/notifier.rs +++ b/crates/event-notifier/src/notifier.rs @@ -2,6 +2,7 @@ use crate::{event_bus, ChannelAdapter, Error, Event, EventStore, NotifierConfig} use std::sync::Arc; use tokio::sync::mpsc; use tokio_util::sync::CancellationToken; +use tracing::instrument; /// The `NotificationSystem` struct represents the notification system. /// It manages the event bus and the adapters. @@ -18,6 +19,7 @@ pub struct NotifierSystem { impl NotifierSystem { /// Creates a new `NotificationSystem` instance. + #[instrument(skip(config))] pub async fn new(config: NotifierConfig) -> Result { let (tx, rx) = mpsc::channel::(config.channel_capacity); let store = Arc::new(EventStore::new(&config.store_path).await?); @@ -44,6 +46,7 @@ impl NotifierSystem { /// Starts the notification system. /// It initializes the event bus and the producer. + #[instrument(skip_all)] pub async fn start(&mut self, adapters: Vec>) -> Result<(), Error> { if self.shutdown.is_cancelled() { let error = Error::custom("System is shutting down"); @@ -67,6 +70,7 @@ impl NotifierSystem { /// Sends an event to the notification system. /// This method is used to send events to the event bus. + #[instrument(skip(self))] pub async fn send_event(&self, event: Event) -> Result<(), Error> { self.log(tracing::Level::DEBUG, "send_event", &format!("Sending event: {:?}", event)); if self.shutdown.is_cancelled() { @@ -85,6 +89,7 @@ impl NotifierSystem { /// Shuts down the notification system. /// This method is used to cancel the event bus and producer tasks. + #[instrument(skip(self))] pub async fn shutdown(&mut self) -> Result<(), Error> { tracing::info!("Shutting down the notification system"); self.shutdown.cancel(); @@ -112,10 +117,13 @@ impl NotifierSystem { self.shutdown.is_cancelled() } - fn handle_error(&self, context: &str, error: &Error) { + #[instrument(skip(self))] + pub fn handle_error(&self, context: &str, error: &Error) { self.log(tracing::Level::ERROR, context, &format!("{:?}", error)); // TODO Can be extended to record to files or send to monitoring systems } + + #[instrument(skip(self))] fn log(&self, level: tracing::Level, context: &str, message: &str) { match level { tracing::Level::ERROR => tracing::error!("[{}] {}", context, message), diff --git a/crates/event-notifier/src/store.rs b/crates/event-notifier/src/store.rs index eca26674a..249116152 100644 --- a/crates/event-notifier/src/store.rs +++ b/crates/event-notifier/src/store.rs @@ -5,6 +5,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use tokio::fs::{create_dir_all, File, OpenOptions}; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader, BufWriter}; use tokio::sync::RwLock; +use tracing::instrument; /// `EventStore` is a struct that manages the storage of event logs. pub struct EventStore { @@ -21,6 +22,7 @@ impl EventStore { }) } + #[instrument(skip(self))] pub async fn save_logs(&self, logs: &[Log]) -> Result<(), Error> { let _guard = self.lock.write().await; let file_path = format!( diff --git a/crates/obs/src/telemetry.rs b/crates/obs/src/telemetry.rs index 64c9ea46a..5a08321d3 100644 --- a/crates/obs/src/telemetry.rs +++ b/crates/obs/src/telemetry.rs @@ -231,7 +231,6 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard { .with(otel_layer) .with(MetricsLayer::new(meter_provider.clone())) .init(); - info!("Telemetry logging enabled: {:?}", config.local_logging_enabled); if !endpoint.is_empty() { info!( diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index e2cad0c07..3589923f8 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -20,6 +20,7 @@ pub(crate) struct ReqInfo { pub version_id: Option, } +/// Authorizes the request based on the action and credentials. pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3Result<()> { let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 252cd316f..e065fe7a1 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -141,6 +141,7 @@ impl S3 for FS { Ok(S3Response::new(output)) } + /// Copy an object from one location to another #[tracing::instrument(level = "debug", skip(self, req))] async fn copy_object(&self, req: S3Request) -> S3Result> { let CopyObjectInput { @@ -227,6 +228,7 @@ impl S3 for FS { Ok(S3Response::new(output)) } + /// Delete a bucket #[tracing::instrument(level = "debug", skip(self, req))] async fn delete_bucket(&self, req: S3Request) -> S3Result> { let input = req.input; @@ -249,6 +251,7 @@ impl S3 for FS { Ok(S3Response::new(DeleteBucketOutput {})) } + /// Delete an object #[tracing::instrument(level = "debug", skip(self, req))] async fn delete_object(&self, req: S3Request) -> S3Result> { let DeleteObjectInput { @@ -308,6 +311,7 @@ impl S3 for FS { Ok(S3Response::new(output)) } + /// Delete multiple objects #[tracing::instrument(level = "debug", skip(self, req))] async fn delete_objects(&self, req: S3Request) -> S3Result> { // info!("delete_objects args {:?}", req.input); @@ -367,6 +371,7 @@ impl S3 for FS { Ok(S3Response::new(output)) } + /// Get bucket location #[tracing::instrument(level = "debug", skip(self, req))] async fn get_bucket_location(&self, req: S3Request) -> S3Result> { // mc get 1 @@ -385,6 +390,7 @@ impl S3 for FS { Ok(S3Response::new(output)) } + /// Get bucket notification #[tracing::instrument( level = "debug", skip(self, req), diff --git a/rustfs/src/storage/event_notifier.rs b/rustfs/src/storage/event_notifier.rs new file mode 100644 index 000000000..292b45e31 --- /dev/null +++ b/rustfs/src/storage/event_notifier.rs @@ -0,0 +1,17 @@ +use rustfs_event_notifier::{Event, Metadata}; + +/// Create a new metadata object +#[allow(dead_code)] +pub(crate) fn create_metadata() -> Metadata { + // Create a new metadata object + let mut metadata = Metadata::new(); + metadata.set_configuration_id("test-config".to_string()); + // Return the created metadata object + metadata +} + +/// Create a new event object +#[allow(dead_code)] +pub(crate) async fn send_event(event: Event) -> Result<(), Box> { + rustfs_event_notifier::send_event(event).await.map_err(|e| e.into()) +} diff --git a/rustfs/src/storage/mod.rs b/rustfs/src/storage/mod.rs index d7d9d87ea..2f8ec8b88 100644 --- a/rustfs/src/storage/mod.rs +++ b/rustfs/src/storage/mod.rs @@ -1,4 +1,5 @@ pub mod access; pub mod ecfs; pub mod error; +mod event_notifier; pub mod options; diff --git a/rustfs/src/storage/options.rs b/rustfs/src/storage/options.rs index 18a13974a..ae0df5391 100644 --- a/rustfs/src/storage/options.rs +++ b/rustfs/src/storage/options.rs @@ -8,6 +8,7 @@ use lazy_static::lazy_static; use std::collections::HashMap; use uuid::Uuid; +/// Creates options for deleting an object in a bucket. pub async fn del_opts( bucket: &str, object: &str, @@ -56,6 +57,7 @@ pub async fn del_opts( Ok(opts) } +/// Creates options for getting an object from a bucket. pub async fn get_opts( bucket: &str, object: &str, @@ -105,6 +107,7 @@ pub async fn get_opts( Ok(opts) } +/// Creates options for putting an object in a bucket. pub async fn put_opts( bucket: &str, object: &str, @@ -151,6 +154,7 @@ pub async fn put_opts( Ok(opts) } +/// Creates options for copying an object in a bucket. pub async fn copy_dst_opts( bucket: &str, object: &str, @@ -172,6 +176,7 @@ pub fn put_opts_from_headers( get_default_opts(headers, metadata, false) } +/// Creates default options for getting an object from a bucket. pub fn get_default_opts( _headers: &HeaderMap, metadata: Option>, @@ -183,6 +188,7 @@ pub fn get_default_opts( }) } +/// Extracts metadata from headers and returns it as a HashMap. pub fn extract_metadata(headers: &HeaderMap) -> HashMap { let mut metadata = HashMap::new(); @@ -191,6 +197,7 @@ pub fn extract_metadata(headers: &HeaderMap) -> HashMap, metadata: &mut HashMap) { for (k, v) in headers.iter() { if let Some(key) = k.as_str().strip_prefix("x-amz-meta-") { @@ -219,7 +226,9 @@ pub fn extract_metadata_from_mime(headers: &HeaderMap, metadata: &m metadata.insert("content-type".to_owned(), "binary/octet-stream".to_owned()); } } + lazy_static! { + /// List of supported headers. static ref SUPPORTED_HEADERS: Vec<&'static str> = vec![ "content-type", "cache-control", From fc47ca9dd29e8fe2d332be3dbea1260f70ef4843 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 6 May 2025 08:54:35 +0800 Subject: [PATCH 18/38] upgrade version (#380) --- Cargo.lock | 90 ++++++++++++++++++++++++++-------------------- Cargo.toml | 28 +++++++++------ ecstore/Cargo.toml | 26 +++++++------- 3 files changed, 81 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1c62f6659..50200628c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -711,9 +711,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de45108900e1f9b9242f7f2e254aa3e2c029c921c258fe9e6b4217eeebd54288" +checksum = "021e862c184ae977658b36c4500f7feac3221ca5da43e3f25bd04ab6c79a29b5" dependencies = [ "axum-core", "bytes", @@ -799,7 +799,7 @@ dependencies = [ "hyper", "hyper-util", "pin-project-lite", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pemfile", "rustls-pki-types", "tokio", @@ -1204,9 +1204,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.40" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1430,7 +1430,7 @@ dependencies = [ "lazy_static", "scopeguard", "tokio", - "tonic 0.13.0", + "tonic 0.13.1", "tracing-error", ] @@ -3061,7 +3061,7 @@ dependencies = [ "serde", "serde_json", "tokio", - "tonic 0.13.0", + "tonic 0.13.1", "tower 0.5.2", "url", ] @@ -3091,7 +3091,7 @@ dependencies = [ "madmin", "md-5", "netif", - "nix", + "nix 0.30.1", "num", "num_cpus", "path-absolutize", @@ -3119,7 +3119,7 @@ dependencies = [ "tokio", "tokio-stream", "tokio-util", - "tonic 0.13.0", + "tonic 0.13.1", "tower 0.5.2", "tracing", "tracing-error", @@ -4129,7 +4129,7 @@ dependencies = [ "http", "hyper", "hyper-util", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pki-types", "tokio", "tokio-rustls 0.26.2", @@ -4780,14 +4780,14 @@ dependencies = [ [[package]] name = "libsystemd" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b85fe9dc49de659d05829fdf72b5770c0a5952d1055c34a39f6d4e932bce175d" +checksum = "19c97a761fc86953c5b885422b22c891dbf5bcb9dcc99d0110d6ce4c052759f0" dependencies = [ "hmac 0.12.1", "libc", "log", - "nix", + "nix 0.29.0", "nom 8.0.0", "once_cell", "serde", @@ -4847,13 +4847,13 @@ checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" [[package]] name = "local-ip-address" -version = "0.6.3" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3669cf5561f8d27e8fc84cc15e58350e70f557d4d65f70e3154e54cd2f8e1782" +checksum = "656b3b27f8893f7bbf9485148ff9a65f019e3f33bd5cdc87c83cab16b3fd9ec8" dependencies = [ "libc", "neli", - "thiserror 1.0.69", + "thiserror 2.0.12", "windows-sys 0.59.0", ] @@ -4870,7 +4870,7 @@ dependencies = [ "serde", "serde_json", "tokio", - "tonic 0.13.0", + "tonic 0.13.1", "tracing", "tracing-error", "url", @@ -5225,6 +5225,18 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags 2.9.0", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nodrop" version = "0.1.14" @@ -6587,7 +6599,7 @@ dependencies = [ "prost-build", "protobuf", "tokio", - "tonic 0.13.0", + "tonic 0.13.1", "tonic-build", "tower 0.5.2", ] @@ -6641,7 +6653,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.1", - "rustls 0.23.26", + "rustls 0.23.27", "socket2", "thiserror 2.0.12", "tokio", @@ -6660,7 +6672,7 @@ dependencies = [ "rand 0.9.1", "ring", "rustc-hash 2.1.1", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pki-types", "slab", "thiserror 2.0.12", @@ -6986,7 +6998,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pemfile", "rustls-pki-types", "serde", @@ -7143,9 +7155,9 @@ dependencies = [ [[package]] name = "rust-embed" -version = "8.7.0" +version = "8.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5fbc0ee50fcb99af7cebb442e5df7b5b45e9460ffa3f8f549cd26b862bec49d" +checksum = "60e425e204264b144d4c929d126d0de524b40a961686414bab5040f7465c71be" dependencies = [ "rust-embed-impl", "rust-embed-utils", @@ -7261,7 +7273,7 @@ dependencies = [ "rust-embed", "rustfs-event-notifier", "rustfs-obs", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pemfile", "rustls-pki-types", "s3s", @@ -7276,7 +7288,7 @@ dependencies = [ "tokio-rustls 0.26.2", "tokio-stream", "tokio-util", - "tonic 0.13.0", + "tonic 0.13.1", "tonic-build", "tower 0.5.2", "tower-http", @@ -7401,16 +7413,16 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.26" +version = "0.23.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df51b5869f3a441595eac5e8ff14d486ff285f7b8c0df8770e49c3b56351f0f0" +checksum = "730944ca083c1c233a75c09f199e973ca499344a2b7ba9e755c457e86fb4a321" dependencies = [ "aws-lc-rs", "log", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.1", + "rustls-webpki 0.103.2", "subtle", "zeroize", ] @@ -7459,9 +7471,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.1" +version = "0.103.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fef8b8769aaccf73098557a87cd1816b4f9c7c16811c9c77142aa695c16f2c03" +checksum = "7149975849f1abb3832b246010ef62ccc80d3a76169517ada7188252b9cfb437" dependencies = [ "aws-lc-rs", "ring", @@ -8642,7 +8654,7 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls 0.23.26", + "rustls 0.23.27", "tokio", ] @@ -8659,9 +8671,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" dependencies = [ "bytes", "futures-core", @@ -8756,9 +8768,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85839f0b32fd242bb3209262371d07feda6d780d16ee9d2bc88581b89da1549b" +checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" dependencies = [ "async-trait", "axum", @@ -8786,9 +8798,9 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85f0383fadd15609306383a90e85eaed44169f931a5d2be1b42c76ceff1825e" +checksum = "eac6f67be712d12f0b41328db3137e0d0757645d8904b4cb7d51cd9c2279e847" dependencies = [ "prettyplease", "proc-macro2", @@ -10253,7 +10265,7 @@ dependencies = [ "futures-sink", "futures-util", "hex", - "nix", + "nix 0.29.0", "ordered-stream", "rand 0.8.5", "serde", @@ -10284,7 +10296,7 @@ dependencies = [ "futures-core", "futures-lite", "hex", - "nix", + "nix 0.29.0", "ordered-stream", "serde", "serde_repr", diff --git a/Cargo.toml b/Cargo.toml index 59c48cc14..ea4fb6fd0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,13 +52,14 @@ atoi = "2.0.0" async-recursion = "1.1.1" async-trait = "0.1.88" atomic_enum = "0.3.0" -axum = "0.8.3" +axum = "0.8.4" axum-extra = "0.10.1" axum-server = { version = "0.7.2", features = ["tls-rustls"] } backon = "1.5.0" +blake2 = "0.10.6" bytes = "1.10.1" bytesize = "2.0.1" -chrono = { version = "0.4.40", features = ["serde"] } +chrono = { version = "0.4.41", features = ["serde"] } clap = { version = "4.5.37", features = ["derive", "env"] } config = "0.15.11" datafusion = "46.0.1" @@ -69,7 +70,9 @@ flatbuffers = "25.2.10" futures = "0.3.31" futures-core = "0.3.31" futures-util = "0.3.31" +glob = "0.3.2" hex = "0.4.3" +highway = { version = "1.3.0" } hyper = "1.6.0" hyper-util = { version = "0.1.11", features = [ "tokio", @@ -86,13 +89,15 @@ keyring = { version = "3.6.2", features = [ "sync-secret-service", ] } lazy_static = "1.5.0" -libsystemd = { version = "0.7.1" } -local-ip-address = "0.6.3" +libsystemd = { version = "0.7.2" } +local-ip-address = "0.6.5" matchit = "0.8.4" md-5 = "0.10.6" mime = "0.3.17" mime_guess = "2.0.5" netif = "0.1.6" +nix = { version = "0.30.1", features = ["fs"] } +num_cpus = { version = "1.16.0" } nvml-wrapper = "0.10.0" object_store = "0.11.2" opentelemetry = { version = "0.29.1" } @@ -113,6 +118,8 @@ prost-types = "0.13.5" protobuf = "3.7" rand = "0.8.5" rdkafka = { version = "0.37.0", features = ["tokio"] } +reed-solomon-erasure = { version = "6.0.0", features = ["simd-accel"] } +regex = { version = "1.11.1" } reqwest = { version = "0.12.15", default-features = false, features = [ "rustls-tls", "charset", @@ -129,8 +136,8 @@ rfd = { version = "0.15.3", default-features = false, features = [ rmp = "0.8.14" rmp-serde = "1.3.0" rumqttc = { version = "0.24" } -rust-embed = "8.7.0" -rustls = { version = "0.23.26" } +rust-embed = { version = "8.7.1" } +rustls = { version = "0.23.27" } rustls-pki-types = "1.11.0" rustls-pemfile = "2.2.0" s3s = { git = "https://github.com/Nugine/s3s.git", rev = "4733cdfb27b2713e832967232cbff413bb768c10" } @@ -156,11 +163,11 @@ time = { version = "0.3.41", features = [ "serde", ] } tokio = { version = "1.44.2", features = ["fs", "rt-multi-thread"] } -tonic = { version = "0.13.0", features = ["gzip"] } -tonic-build = "0.13.0" +tonic = { version = "0.13.1", features = ["gzip"] } +tonic-build = { version = "0.13.1" } tokio-rustls = { version = "0.26.2", default-features = false } -tokio-stream = "0.1.17" -tokio-util = { version = "0.7.14", features = ["io", "compat"] } +tokio-stream = { version = "0.1.17" } +tokio-util = { version = "0.7.15", features = ["io", "compat"] } tower = { version = "0.5.2", features = ["timeout"] } tower-http = { version = "0.6.2", features = ["cors"] } tracing = "0.1.41" @@ -176,6 +183,7 @@ uuid = { version = "1.16.0", features = [ "fast-rng", "macro-diagnostics", ] } +winapi = { version = "0.3.9" } [profile.wasm-dev] diff --git a/ecstore/Cargo.toml b/ecstore/Cargo.toml index 721c902fe..8ba12ac1d 100644 --- a/ecstore/Cargo.toml +++ b/ecstore/Cargo.toml @@ -13,12 +13,12 @@ workspace = true [dependencies] async-trait.workspace = true backon.workspace = true -blake2 = "0.10.6" +blake2 = { workspace = true } bytes.workspace = true common.workspace = true policy.workspace = true chrono.workspace = true -glob = "0.3.2" +glob = { workspace = true } thiserror.workspace = true flatbuffers.workspace = true futures.workspace = true @@ -30,16 +30,16 @@ serde_json.workspace = true tracing-error.workspace = true s3s.workspace = true http.workspace = true -highway = "1.3.0" +highway = { workspace = true } url.workspace = true uuid = { workspace = true, features = ["v4", "fast-rng", "serde"] } -reed-solomon-erasure = { version = "6.0.0", features = ["simd-accel"] } +reed-solomon-erasure = { workspace = true } transform-stream = "0.3.1" lazy_static.workspace = true lock.workspace = true -regex = "1.11.1" -netif = "0.1.6" -nix = { version = "0.29.0", features = ["fs"] } +regex = { workspace = true } +netif = { workspace = true } +nix = { workspace = true } path-absolutize = "3.1.1" protos.workspace = true rmp.workspace = true @@ -53,13 +53,13 @@ hex-simd = "0.8.0" path-clean = "1.0.1" tempfile.workspace = true tokio = { workspace = true, features = ["io-util", "sync", "signal"] } -tokio-stream = "0.1.17" +tokio-stream = { workspace = true } tonic.workspace = true tower.workspace = true byteorder = "1.5.0" xxhash-rust = { version = "0.8.15", features = ["xxh64"] } num = "0.4.3" -num_cpus = "1.16" +num_cpus = { workspace = true } s3s-policy.workspace = true rand.workspace = true pin-project-lite.workspace = true @@ -68,16 +68,14 @@ madmin.workspace = true workers.workspace = true reqwest = { workspace = true } urlencoding = "2.1.3" -smallvec = "1.15.0" +smallvec = { workspace = true } shadow-rs.workspace = true [target.'cfg(not(windows))'.dependencies] - -nix = { version = "0.29.0", features = ["fs"] } +nix = { workspace = true } [target.'cfg(windows)'.dependencies] - -winapi = "0.3.9" +winapi = { workspace = true } [dev-dependencies] From 0ac1095c70f62715bbfd81ac647400cb8c6dcdd3 Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Tue, 6 May 2025 11:00:18 +0800 Subject: [PATCH 19/38] support spec char as delimiter Signed-off-by: junxiang Mu <1948535941@qq.com> --- .cargo/config.toml | 7 -- Cargo.lock | 2 + common/common/src/lib.rs | 3 + ecstore/Cargo.toml | 2 +- rustfs/src/storage/ecfs.rs | 5 +- s3select/api/Cargo.toml | 2 + s3select/api/src/object_store.rs | 127 ++++++++++++++++++++--- s3select/api/src/query/mod.rs | 4 +- s3select/api/src/query/session.rs | 2 +- s3select/query/src/dispatcher/manager.rs | 10 +- s3select/query/src/instance.rs | 14 +-- 11 files changed, 139 insertions(+), 39 deletions(-) delete mode 100644 .cargo/config.toml diff --git a/.cargo/config.toml b/.cargo/config.toml deleted file mode 100644 index a1c92ecf3..000000000 --- a/.cargo/config.toml +++ /dev/null @@ -1,7 +0,0 @@ -[target.x86_64-unknown-linux-gnu] -rustflags = [ - "-C", "link-arg=-fuse-ld=bfd" -] - -[target.x86_64-unknown-linux-musl] -linker = "x86_64-linux-musl-gcc" \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 50200628c..e4c51bc8c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -185,12 +185,14 @@ dependencies = [ "async-trait", "bytes", "chrono", + "common", "datafusion", "ecstore", "futures", "futures-core", "http", "object_store", + "pin-project-lite", "s3s", "snafu", "tokio", diff --git a/common/common/src/lib.rs b/common/common/src/lib.rs index a8998fee0..90250c5eb 100644 --- a/common/common/src/lib.rs +++ b/common/common/src/lib.rs @@ -2,6 +2,9 @@ pub mod error; pub mod globals; pub mod last_minute; +// is ',' +pub static DEFAULT_DELIMITER: u8 = 44; + /// Defers evaluation of a block of code until the end of the scope. #[macro_export] macro_rules! defer { diff --git a/ecstore/Cargo.toml b/ecstore/Cargo.toml index 8ba12ac1d..007150d40 100644 --- a/ecstore/Cargo.toml +++ b/ecstore/Cargo.toml @@ -82,4 +82,4 @@ winapi = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } [build-dependencies] -shadow-rs.workspace = true +shadow-rs = { workspace = true, features = ["build", "metadata"] } diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index e065fe7a1..fd7b4c0c9 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -62,6 +62,7 @@ use s3s::S3; use s3s::{S3Request, S3Response}; use std::fmt::Debug; use std::str::FromStr; +use std::sync::Arc; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use tokio_util::io::ReaderStream; @@ -1896,14 +1897,14 @@ impl S3 for FS { ) -> S3Result> { info!("handle select_object_content"); - let input = req.input; + let input = Arc::new(req.input); info!("{:?}", input); let db = make_rustfsms(input.clone(), false).await.map_err(|e| { error!("make db failed, {}", e.to_string()); s3_error!(InternalError, "{}", e.to_string()) })?; - let query = Query::new(Context { input: input.clone() }, input.request.expression); + let query = Query::new(Context { input: input.clone() }, input.request.expression.clone()); let result = db .execute(&query) .await diff --git a/s3select/api/Cargo.toml b/s3select/api/Cargo.toml index 27e260c41..125849cdc 100644 --- a/s3select/api/Cargo.toml +++ b/s3select/api/Cargo.toml @@ -7,12 +7,14 @@ edition.workspace = true async-trait.workspace = true bytes.workspace = true chrono.workspace = true +common.workspace = true datafusion = { workspace = true } ecstore.workspace = true futures = { workspace = true } futures-core = { workspace = true } http.workspace = true object_store = { workspace = true } +pin-project-lite.workspace = true s3s.workspace = true snafu = { workspace = true, features = ["backtrace"] } tokio.workspace = true diff --git a/s3select/api/src/object_store.rs b/s3select/api/src/object_store.rs index 7772936ce..52132bc1a 100644 --- a/s3select/api/src/object_store.rs +++ b/s3select/api/src/object_store.rs @@ -1,6 +1,7 @@ use async_trait::async_trait; use bytes::Bytes; use chrono::Utc; +use common::DEFAULT_DELIMITER; use ecstore::io::READ_BUFFER_SIZE; use ecstore::new_object_layer_fn; use ecstore::store::ECStore; @@ -24,29 +25,54 @@ use object_store::PutOptions; use object_store::PutPayload; use object_store::PutResult; use object_store::{Error as o_Error, Result}; +use pin_project_lite::pin_project; use s3s::dto::SelectObjectContentInput; use s3s::s3_error; use s3s::S3Result; use std::ops::Range; +use std::pin::Pin; use std::sync::Arc; +use std::task::ready; +use std::task::Poll; +use tokio::io::AsyncRead; use tokio_util::io::ReaderStream; use tracing::info; use transform_stream::AsyncTryStream; #[derive(Debug)] pub struct EcObjectStore { - input: SelectObjectContentInput, + input: Arc, + need_convert: bool, + delimiter: String, store: Arc, } - impl EcObjectStore { - pub fn new(input: SelectObjectContentInput) -> S3Result { + pub fn new(input: Arc) -> S3Result { let Some(store) = new_object_layer_fn() else { return Err(s3_error!(InternalError, "ec store not inited")); }; - Ok(Self { input, store }) + let (need_convert, delimiter) = if let Some(csv) = input.request.input_serialization.csv.as_ref() { + if let Some(delimiter) = csv.field_delimiter.as_ref() { + if delimiter.len() > 1 { + (true, delimiter.to_owned()) + } else { + (false, String::new()) + } + } else { + (false, String::new()) + } + } else { + (false, String::new()) + }; + + Ok(Self { + input, + need_convert, + delimiter, + store, + }) } } @@ -79,16 +105,6 @@ impl ObjectStore for EcObjectStore { source: "can not get object info".into(), })?; - // let stream = stream::unfold(reader.stream, |mut blob| async move { - // match blob.next().await { - // Some(Ok(chunk)) => { - // let bytes = chunk; - // Some((Ok(bytes), blob)) - // } - // _ => None, - // } - // }) - // .boxed(); let meta = ObjectMeta { location: location.clone(), last_modified: Utc::now(), @@ -98,10 +114,21 @@ impl ObjectStore for EcObjectStore { }; let attributes = Attributes::default(); - Ok(GetResult { - payload: object_store::GetResultPayload::Stream( + let payload = if self.need_convert { + object_store::GetResultPayload::Stream( + bytes_stream( + ReaderStream::with_capacity(ConvertStream::new(reader.stream, self.delimiter.clone()), READ_BUFFER_SIZE), + reader.object_info.size, + ) + .boxed(), + ) + } else { + object_store::GetResultPayload::Stream( bytes_stream(ReaderStream::with_capacity(reader.stream, READ_BUFFER_SIZE), reader.object_info.size).boxed(), - ), + ) + }; + Ok(GetResult { + payload, meta, range: 0..reader.object_info.size, attributes, @@ -154,6 +181,54 @@ impl ObjectStore for EcObjectStore { } } +pin_project! { + struct ConvertStream { + inner: R, + delimiter: Vec, + } +} + +impl ConvertStream { + fn new(inner: R, delimiter: String) -> Self { + ConvertStream { + inner, + delimiter: delimiter.as_bytes().to_vec(), + } + } +} + +impl AsyncRead for ConvertStream { + #[tracing::instrument(level = "debug", skip_all)] + fn poll_read( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + let me = self.project(); + ready!(Pin::new(&mut *me.inner).poll_read(cx, buf))?; + let bytes = buf.filled(); + let replaced = replace_symbol(me.delimiter, bytes); + buf.clear(); + buf.put_slice(&replaced); + Poll::Ready(Ok(())) + } +} + +fn replace_symbol(delimiter: &[u8], slice: &[u8]) -> Vec { + let mut result = Vec::with_capacity(slice.len()); + let mut i = 0; + while i < slice.len() { + if slice[i..].starts_with(delimiter) { + result.push(DEFAULT_DELIMITER); + i += delimiter.len(); + } else { + result.push(slice[i]); + i += 1; + } + } + result +} + pub fn bytes_stream(stream: S, content_length: usize) -> impl Stream> + Send + 'static where S: Stream> + Send + 'static, @@ -175,3 +250,21 @@ where Ok(()) }) } + +#[cfg(test)] +mod test { + use super::replace_symbol; + + #[test] + fn test_replace() { + let ss = String::from("dandan&&is&&best"); + let slice = ss.as_bytes(); + let delimiter = b"&&"; + println!("len: {}", "╦".len()); + let result = replace_symbol(delimiter, slice); + match String::from_utf8(result) { + Ok(s) => println!("slice: {}", s), + Err(e) => eprintln!("Error converting to string: {}", e), + } + } +} diff --git a/s3select/api/src/query/mod.rs b/s3select/api/src/query/mod.rs index 6ddd2dc86..93736be2b 100644 --- a/s3select/api/src/query/mod.rs +++ b/s3select/api/src/query/mod.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use s3s::dto::SelectObjectContentInput; pub mod analyzer; @@ -16,7 +18,7 @@ pub mod session; #[derive(Clone)] pub struct Context { // maybe we need transfer some info? - pub input: SelectObjectContentInput, + pub input: Arc, } #[derive(Clone)] diff --git a/s3select/api/src/query/session.rs b/s3select/api/src/query/session.rs index 286ee9f8e..581cdf39b 100644 --- a/s3select/api/src/query/session.rs +++ b/s3select/api/src/query/session.rs @@ -66,7 +66,7 @@ impl SessionCtxFactory { 9,Ivy,24,Marketing,4800 10,Jack,38,Finance,7500"; let data_bytes = data.as_bytes(); - // let data = r#""year"╦"gender"╦"ethnicity"╦"firstname"╦"count"╦"rank" + // let data = r#""year"╦"gender"╦"ethnicity"╦"firstname"╦"count"╦"rank" // "2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"SOPHIA"╦"119"╦"1" // "2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"CHLOE"╦"106"╦"2" // "2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"EMILY"╦"93"╦"3" diff --git a/s3select/query/src/dispatcher/manager.rs b/s3select/query/src/dispatcher/manager.rs index 80543ed69..ee5386e20 100644 --- a/s3select/query/src/dispatcher/manager.rs +++ b/s3select/query/src/dispatcher/manager.rs @@ -49,7 +49,7 @@ lazy_static! { #[derive(Clone)] pub struct SimpleQueryDispatcher { - input: SelectObjectContentInput, + input: Arc, // client for default tenant _default_table_provider: TableHandleProviderRef, session_factory: Arc, @@ -164,7 +164,9 @@ impl SimpleQueryDispatcher { .map(|e| e.as_bytes().first().copied().unwrap_or_default()), ); if let Some(delimiter) = csv.field_delimiter.as_ref() { - file_format = file_format.with_delimiter(delimiter.as_bytes().first().copied().unwrap_or_default()); + if delimiter.len() == 1 { + file_format = file_format.with_delimiter(delimiter.as_bytes()[0]); + } } // TODO waiting for processing @junxiang Mu // if csv.file_header_info.is_some() {} @@ -272,7 +274,7 @@ impl Stream for TrackedRecordBatchStream { #[derive(Default, Clone)] pub struct SimpleQueryDispatcherBuilder { - input: Option, + input: Option>, default_table_provider: Option, session_factory: Option>, parser: Option>, @@ -283,7 +285,7 @@ pub struct SimpleQueryDispatcherBuilder { } impl SimpleQueryDispatcherBuilder { - pub fn with_input(mut self, input: SelectObjectContentInput) -> Self { + pub fn with_input(mut self, input: Arc) -> Self { self.input = Some(input); self } diff --git a/s3select/query/src/instance.rs b/s3select/query/src/instance.rs index 44952a4ac..344920631 100644 --- a/s3select/query/src/instance.rs +++ b/s3select/query/src/instance.rs @@ -63,7 +63,7 @@ where } } -pub async fn make_rustfsms(input: SelectObjectContentInput, is_test: bool) -> QueryResult { +pub async fn make_rustfsms(input: Arc, is_test: bool) -> QueryResult { // init Function Manager, we can define some UDF if need let func_manager = SimpleFunctionMetadataManager::default(); // TODO session config need load global system config @@ -95,6 +95,8 @@ pub async fn make_rustfsms(input: SelectObjectContentInput, is_test: bool) -> Qu #[cfg(test)] mod tests { + use std::sync::Arc; + use api::{ query::{Context, Query}, server::dbms::DatabaseManagerSystem, @@ -111,7 +113,7 @@ mod tests { #[ignore] async fn test_simple_sql() { let sql = "select * from S3Object"; - let input = SelectObjectContentInput { + let input = Arc::new(SelectObjectContentInput { bucket: "dandan".to_string(), expected_bucket_owner: None, key: "test.csv".to_string(), @@ -135,7 +137,7 @@ mod tests { request_progress: None, scan_range: None, }, - }; + }); let db = make_rustfsms(input.clone(), true).await.unwrap(); let query = Query::new(Context { input }, sql.to_string()); @@ -167,8 +169,8 @@ mod tests { #[tokio::test] #[ignore] async fn test_func_sql() { - let sql = "SELECT s._1 FROM S3Object s"; - let input = SelectObjectContentInput { + let sql = "SELECT * FROM S3Object s"; + let input = Arc::new(SelectObjectContentInput { bucket: "dandan".to_string(), expected_bucket_owner: None, key: "test.csv".to_string(), @@ -194,7 +196,7 @@ mod tests { request_progress: None, scan_range: None, }, - }; + }); let db = make_rustfsms(input.clone(), true).await.unwrap(); let query = Query::new(Context { input }, sql.to_string()); From 29ddf4dbc802a038b155efc86501aa2ba1f54d51 Mon Sep 17 00:00:00 2001 From: houseme Date: Wed, 7 May 2025 17:23:22 +0800 Subject: [PATCH 20/38] refactor: standardize constant management and fix typos (#387) * init rustfs config * init rustfs-utils crate * improve code for rustfs-config crate * add * improve code for comment * init rustfs config * improve code for rustfs-config crate * add * improve code for comment * Unified management of configurations and constants * fix: modify rustfs-config crate name * add default fn * improve code for rustfs config * refactor: standardize constant management and fix typos - Create centralized constants module for global static constants - Replace runtime format! expressions with compile-time constants - Fix DEFAULT_PORT reference issues in configuration arguments - Use const-str crate for compile-time string concatenation - Update tokio dependency from 1.42.2 to 1.45.0 - Ensure consistent naming convention for configuration constants * fix * Update common/workers/src/workers.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- Cargo.lock | 32 +++++++-- Cargo.toml | 26 ++++--- common/workers/src/workers.rs | 20 +++--- crates/config/Cargo.toml | 17 +++++ crates/config/src/config.rs | 23 ++++++ crates/config/src/constants/app.rs | 72 +++++++++++++++++++ crates/config/src/constants/mod.rs | 1 + crates/config/src/event/config.rs | 23 ++++++ crates/config/src/event/event.rs | 17 +++++ crates/config/src/event/mod.rs | 2 + crates/config/src/lib.rs | 9 +++ crates/config/src/observability/config.rs | 28 ++++++++ crates/config/src/observability/file_sink.rs | 25 +++++++ crates/config/src/observability/kafka_sink.rs | 23 ++++++ crates/config/src/observability/logger.rs | 21 ++++++ crates/config/src/observability/mod.rs | 8 +++ .../config/src/observability/observability.rs | 22 ++++++ crates/config/src/observability/otel.rs | 27 +++++++ crates/config/src/observability/sink.rs | 28 ++++++++ .../config/src/observability/webhook_sink.rs | 25 +++++++ crates/event-notifier/src/config.rs | 2 +- crates/event-notifier/src/event.rs | 32 +++------ crates/utils/Cargo.toml | 18 +++++ .../src/utils => crates/utils/src}/certs.rs | 40 ++++++----- crates/utils/src/ip.rs | 43 +++++++++++ crates/utils/src/lib.rs | 11 +++ crates/utils/src/net.rs | 0 ecstore/Cargo.toml | 1 + ecstore/src/global.rs | 12 ++-- ecstore/src/heal/heal_ops.rs | 4 +- ecstore/src/utils/path.rs | 2 +- rustfs/Cargo.toml | 9 ++- rustfs/README.md | 66 +++++++++-------- rustfs/src/config/mod.rs | 44 ++---------- rustfs/src/console.rs | 10 +-- rustfs/src/event.rs | 21 ++++++ rustfs/src/main.rs | 60 ++++++---------- rustfs/src/utils/mod.rs | 18 ----- 38 files changed, 637 insertions(+), 205 deletions(-) create mode 100644 crates/config/Cargo.toml create mode 100644 crates/config/src/config.rs create mode 100644 crates/config/src/constants/app.rs create mode 100644 crates/config/src/constants/mod.rs create mode 100644 crates/config/src/event/config.rs create mode 100644 crates/config/src/event/event.rs create mode 100644 crates/config/src/event/mod.rs create mode 100644 crates/config/src/lib.rs create mode 100644 crates/config/src/observability/config.rs create mode 100644 crates/config/src/observability/file_sink.rs create mode 100644 crates/config/src/observability/kafka_sink.rs create mode 100644 crates/config/src/observability/logger.rs create mode 100644 crates/config/src/observability/mod.rs create mode 100644 crates/config/src/observability/observability.rs create mode 100644 crates/config/src/observability/otel.rs create mode 100644 crates/config/src/observability/sink.rs create mode 100644 crates/config/src/observability/webhook_sink.rs create mode 100644 crates/utils/Cargo.toml rename {rustfs/src/utils => crates/utils/src}/certs.rs (78%) create mode 100644 crates/utils/src/ip.rs create mode 100644 crates/utils/src/lib.rs create mode 100644 crates/utils/src/net.rs create mode 100644 rustfs/src/event.rs delete mode 100644 rustfs/src/utils/mod.rs diff --git a/Cargo.lock b/Cargo.lock index e4c51bc8c..f3407172d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3107,6 +3107,7 @@ dependencies = [ "reqwest", "rmp", "rmp-serde", + "rustfs-config", "s3s", "s3s-policy", "serde", @@ -7258,7 +7259,6 @@ dependencies = [ "iam", "lazy_static", "libsystemd", - "local-ip-address", "lock", "madmin", "matchit", @@ -7273,11 +7273,11 @@ dependencies = [ "query", "rmp-serde", "rust-embed", + "rustfs-config", "rustfs-event-notifier", "rustfs-obs", + "rustfs-utils", "rustls 0.23.27", - "rustls-pemfile", - "rustls-pki-types", "s3s", "serde", "serde_json", @@ -7299,6 +7299,16 @@ dependencies = [ "uuid", ] +[[package]] +name = "rustfs-config" +version = "0.0.1" +dependencies = [ + "config", + "const-str", + "serde", + "serde_json", +] + [[package]] name = "rustfs-event-notifier" version = "0.0.1" @@ -7373,6 +7383,18 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "rustfs-utils" +version = "0.0.1" +dependencies = [ + "local-ip-address", + "rustfs-config", + "rustls 0.23.27", + "rustls-pemfile", + "rustls-pki-types", + "tracing", +] + [[package]] name = "rustix" version = "0.38.44" @@ -8611,9 +8633,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.44.2" +version = "1.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48" +checksum = "2513ca694ef9ede0fb23fe71a4ee4107cb102b9dc1930f6d0fd77aae068ae165" dependencies = [ "backtrace", "bytes", diff --git a/Cargo.toml b/Cargo.toml index ea4fb6fd0..645533ae1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,21 +1,23 @@ [workspace] members = [ - "madmin", # Management dashboard and admin API interface - "rustfs", # Core file system implementation - "ecstore", # Erasure coding storage implementation - "e2e_test", # End-to-end test suite + "appauth", # Application authentication and authorization + "cli/rustfs-gui", # Graphical user interface client "common/common", # Shared utilities and data structures "common/lock", # Distributed locking implementation "common/protos", # Protocol buffer definitions "common/workers", # Worker thread pools and task scheduling - "iam", # Identity and Access Management - "crypto", # Cryptography and security features - "cli/rustfs-gui", # Graphical user interface client - "crates/obs", # Observability utilities + "crates/config", # Configuration management "crates/event-notifier", # Event notification system + "crates/obs", # Observability utilities + "crates/utils", # Utility functions and helpers + "crypto", # Cryptography and security features + "ecstore", # Erasure coding storage implementation + "e2e_test", # End-to-end test suite + "iam", # Identity and Access Management + "madmin", # Management dashboard and admin API interface + "rustfs", # Core file system implementation "s3select/api", # S3 Select API interface "s3select/query", # S3 Select query engine - "appauth", # Application authentication and authorization ] resolver = "2" @@ -45,8 +47,10 @@ policy = { path = "./policy", version = "0.0.1" } protos = { path = "./common/protos", version = "0.0.1" } query = { path = "./s3select/query", version = "0.0.1" } rustfs = { path = "./rustfs", version = "0.0.1" } +rustfs-config = { path = "./crates/config", version = "0.0.1" } rustfs-obs = { path = "crates/obs", version = "0.0.1" } rustfs-event-notifier = { path = "crates/event-notifier", version = "0.0.1" } +rustfs-utils = { path = "crates/utils", version = "0.0.1" } workers = { path = "./common/workers", version = "0.0.1" } atoi = "2.0.0" async-recursion = "1.1.1" @@ -62,6 +66,7 @@ bytesize = "2.0.1" chrono = { version = "0.4.41", features = ["serde"] } clap = { version = "4.5.37", features = ["derive", "env"] } config = "0.15.11" +const-str = { version = "0.6.2", features = ["std", "proc"] } datafusion = "46.0.1" derive_builder = "0.20.2" dioxus = { version = "0.6.3", features = ["router"] } @@ -150,6 +155,7 @@ serde_with = "3.12.0" sha2 = "0.10.8" smallvec = { version = "1.15.0", features = ["serde"] } snafu = "0.8.5" +socket2 = "0.5.9" strum = { version = "0.27.1", features = ["derive"] } sysinfo = "0.34.2" tempfile = "3.19.1" @@ -162,7 +168,7 @@ time = { version = "0.3.41", features = [ "macros", "serde", ] } -tokio = { version = "1.44.2", features = ["fs", "rt-multi-thread"] } +tokio = { version = "1.45.0", features = ["fs", "rt-multi-thread"] } tonic = { version = "0.13.1", features = ["gzip"] } tonic-build = { version = "0.13.1" } tokio-rustls = { version = "0.26.2", default-features = false } diff --git a/common/workers/src/workers.rs b/common/workers/src/workers.rs index bd12d9285..f6a6a94ef 100644 --- a/common/workers/src/workers.rs +++ b/common/workers/src/workers.rs @@ -3,13 +3,13 @@ use tokio::sync::{Mutex, Notify}; use tracing::info; pub struct Workers { - available: Mutex, // 可用的工作槽 - notify: Notify, // 用于通知等待的任务 - limit: usize, // 最大并发工作数 + available: Mutex, // Available working slots + notify: Notify, // Used to notify waiting tasks + limit: usize, // Maximum number of concurrent jobs } impl Workers { - // 创建 Workers 对象，允许最多 n 个作业并发执行 + // Create a Workers object that allows up to n jobs to execute concurrently pub fn new(n: usize) -> Result, &'static str> { if n == 0 { return Err("n must be > 0"); @@ -22,7 +22,7 @@ impl Workers { })) } - // 让一个作业获得执行的机会 + // Give a job a chance to be executed pub async fn take(&self) { loop { let mut available = self.available.lock().await; @@ -37,15 +37,15 @@ impl Workers { } } - // 让一个作业释放其机会 + // Release a job's slot pub async fn give(&self) { let mut available = self.available.lock().await; info!("worker give, {}", *available); - *available += 1; // 增加可用槽 - self.notify.notify_one(); // 通知一个等待的任务 + *available += 1; // Increase available slots + self.notify.notify_one(); // Notify a waiting task } - // 等待所有并发作业完成 + // Wait for all concurrent jobs to complete pub async fn wait(&self) { loop { { @@ -54,7 +54,7 @@ impl Workers { break; } } - // 等待直到所有槽都被释放 + // Wait until all slots are freed self.notify.notified().await; } info!("worker wait end"); diff --git a/crates/config/Cargo.toml b/crates/config/Cargo.toml new file mode 100644 index 000000000..1a81e30d5 --- /dev/null +++ b/crates/config/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "rustfs-config" +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[dependencies] +config = { workspace = true } +const-str = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } + + +[lints] +workspace = true diff --git a/crates/config/src/config.rs b/crates/config/src/config.rs new file mode 100644 index 000000000..bebcbfd00 --- /dev/null +++ b/crates/config/src/config.rs @@ -0,0 +1,23 @@ +use crate::event::config::EventConfig; +use crate::ObservabilityConfig; + +/// RustFs configuration +pub struct RustFsConfig { + pub observability: ObservabilityConfig, + pub event: EventConfig, +} + +impl RustFsConfig { + pub fn new() -> Self { + Self { + observability: ObservabilityConfig::new(), + event: EventConfig::new(), + } + } +} + +impl Default for RustFsConfig { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/config/src/constants/app.rs b/crates/config/src/constants/app.rs new file mode 100644 index 000000000..294c7677f --- /dev/null +++ b/crates/config/src/constants/app.rs @@ -0,0 +1,72 @@ +use const_str::concat; + +/// Application name +/// Default value: RustFs +/// Environment variable: RUSTFS_APP_NAME +pub const APP_NAME: &str = "RustFs"; +/// Application version +/// Default value: 1.0.0 +/// Environment variable: RUSTFS_VERSION +pub const VERSION: &str = "0.0.1"; + +/// Default configuration logger level +/// Default value: info +/// Environment variable: RUSTFS_LOG_LEVEL +pub const DEFAULT_LOG_LEVEL: &str = "info"; + +/// maximum number of connections +/// This is the maximum number of connections that the server will accept. +/// This is used to limit the number of connections to the server. +pub const MAX_CONNECTIONS: usize = 100; +/// timeout for connections +/// This is the timeout for connections to the server. +/// This is used to limit the time that a connection can be open. +pub const DEFAULT_TIMEOUT_MS: u64 = 3000; + +/// Default Access Key +/// Default value: rustfsadmin +/// Environment variable: RUSTFS_ACCESS_KEY +/// Command line argument: --access-key +/// Example: RUSTFS_ACCESS_KEY=rustfsadmin +/// Example: --access-key rustfsadmin +pub const DEFAULT_ACCESS_KEY: &str = "rustfsadmin"; +/// Default Secret Key +/// Default value: rustfsadmin +/// Environment variable: RUSTFS_SECRET_KEY +/// Command line argument: --secret-key +/// Example: RUSTFS_SECRET_KEY=rustfsadmin +/// Example: --secret-key rustfsadmin +pub const DEFAULT_SECRET_KEY: &str = "rustfsadmin"; +/// Default configuration file for observability +/// Default value: config/obs.toml +/// Environment variable: RUSTFS_OBS_CONFIG +/// Command line argument: --obs-config +/// Example: RUSTFS_OBS_CONFIG=config/obs.toml +/// Example: --obs-config config/obs.toml +/// Example: --obs-config /etc/rustfs/obs.toml +pub const DEFAULT_OBS_CONFIG: &str = "config/obs.toml"; + +/// Default TLS key for rustfs +/// This is the default key for TLS. +pub const RUSTFS_TLS_KEY: &str = "rustfs_key.pem"; + +/// Default TLS cert for rustfs +/// This is the default cert for TLS. +pub const RUSTFS_TLS_CERT: &str = "rustfs_cert.pem"; + +/// Default port for rustfs +/// This is the default port for rustfs. +/// This is used to bind the server to a specific port. +pub const DEFAULT_PORT: u16 = 9000; + +/// Default address for rustfs +/// This is the default address for rustfs. +pub const DEFAULT_ADDRESS: &str = concat!(":", DEFAULT_PORT); + +/// Default port for rustfs console +/// This is the default port for rustfs console. +pub const DEFAULT_CONSOLE_PORT: u16 = 9002; + +/// Default address for rustfs console +/// This is the default address for rustfs console. +pub const DEFAULT_CONSOLE_ADDRESS: &str = concat!(":", DEFAULT_CONSOLE_PORT); diff --git a/crates/config/src/constants/mod.rs b/crates/config/src/constants/mod.rs new file mode 100644 index 000000000..04023c887 --- /dev/null +++ b/crates/config/src/constants/mod.rs @@ -0,0 +1 @@ +pub(crate) mod app; diff --git a/crates/config/src/event/config.rs b/crates/config/src/event/config.rs new file mode 100644 index 000000000..a8a430688 --- /dev/null +++ b/crates/config/src/event/config.rs @@ -0,0 +1,23 @@ +/// Event configuration module +pub struct EventConfig { + pub event_type: String, + pub event_source: String, + pub event_destination: String, +} + +impl EventConfig { + /// Creates a new instance of `EventConfig` with default values. + pub fn new() -> Self { + Self { + event_type: "default".to_string(), + event_source: "default".to_string(), + event_destination: "default".to_string(), + } + } +} + +impl Default for EventConfig { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/config/src/event/event.rs b/crates/config/src/event/event.rs new file mode 100644 index 000000000..70a103690 --- /dev/null +++ b/crates/config/src/event/event.rs @@ -0,0 +1,17 @@ +/// Event configuration module +pub struct EventConfig { + pub event_type: String, + pub event_source: String, + pub event_destination: String, +} + +impl EventConfig { + /// Creates a new instance of `EventConfig` with default values. + pub fn new() -> Self { + Self { + event_type: "default".to_string(), + event_source: "default".to_string(), + event_destination: "default".to_string(), + } + } +} diff --git a/crates/config/src/event/mod.rs b/crates/config/src/event/mod.rs new file mode 100644 index 000000000..602809247 --- /dev/null +++ b/crates/config/src/event/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod config; +pub(crate) mod event; diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs new file mode 100644 index 000000000..fd7b8bec2 --- /dev/null +++ b/crates/config/src/lib.rs @@ -0,0 +1,9 @@ +use crate::observability::config::ObservabilityConfig; + +mod config; +mod constants; +mod event; +mod observability; + +pub use config::RustFsConfig; +pub use constants::app::*; diff --git a/crates/config/src/observability/config.rs b/crates/config/src/observability/config.rs new file mode 100644 index 000000000..361f9a6c5 --- /dev/null +++ b/crates/config/src/observability/config.rs @@ -0,0 +1,28 @@ +use crate::observability::logger::LoggerConfig; +use crate::observability::otel::OtelConfig; +use crate::observability::sink::SinkConfig; +use serde::Deserialize; + +/// Observability configuration +#[derive(Debug, Deserialize, Clone)] +pub struct ObservabilityConfig { + pub otel: OtelConfig, + pub sinks: SinkConfig, + pub logger: Option, +} + +impl ObservabilityConfig { + pub fn new() -> Self { + Self { + otel: OtelConfig::new(), + sinks: SinkConfig::new(), + logger: Some(LoggerConfig::new()), + } + } +} + +impl Default for ObservabilityConfig { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/config/src/observability/file_sink.rs b/crates/config/src/observability/file_sink.rs new file mode 100644 index 000000000..d475376ef --- /dev/null +++ b/crates/config/src/observability/file_sink.rs @@ -0,0 +1,25 @@ +use serde::Deserialize; + +/// File sink configuration +#[derive(Debug, Deserialize, Clone)] +pub struct FileSinkConfig { + pub path: String, + pub max_size: u64, + pub max_backups: u64, +} + +impl FileSinkConfig { + pub fn new() -> Self { + Self { + path: "".to_string(), + max_size: 0, + max_backups: 0, + } + } +} + +impl Default for FileSinkConfig { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/config/src/observability/kafka_sink.rs b/crates/config/src/observability/kafka_sink.rs new file mode 100644 index 000000000..f40a979b9 --- /dev/null +++ b/crates/config/src/observability/kafka_sink.rs @@ -0,0 +1,23 @@ +use serde::Deserialize; + +/// Kafka sink configuration +#[derive(Debug, Deserialize, Clone)] +pub struct KafkaSinkConfig { + pub brokers: Vec, + pub topic: String, +} + +impl KafkaSinkConfig { + pub fn new() -> Self { + Self { + brokers: vec!["localhost:9092".to_string()], + topic: "rustfs".to_string(), + } + } +} + +impl Default for KafkaSinkConfig { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/config/src/observability/logger.rs b/crates/config/src/observability/logger.rs new file mode 100644 index 000000000..f6c70682c --- /dev/null +++ b/crates/config/src/observability/logger.rs @@ -0,0 +1,21 @@ +use serde::Deserialize; + +/// Logger configuration +#[derive(Debug, Deserialize, Clone)] +pub struct LoggerConfig { + pub queue_capacity: Option, +} + +impl LoggerConfig { + pub fn new() -> Self { + Self { + queue_capacity: Some(10000), + } + } +} + +impl Default for LoggerConfig { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/config/src/observability/mod.rs b/crates/config/src/observability/mod.rs new file mode 100644 index 000000000..65d9933b0 --- /dev/null +++ b/crates/config/src/observability/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod config; +pub(crate) mod file_sink; +pub(crate) mod kafka_sink; +pub(crate) mod logger; +pub(crate) mod observability; +pub(crate) mod otel; +pub(crate) mod sink; +pub(crate) mod webhook_sink; diff --git a/crates/config/src/observability/observability.rs b/crates/config/src/observability/observability.rs new file mode 100644 index 000000000..17b4e0704 --- /dev/null +++ b/crates/config/src/observability/observability.rs @@ -0,0 +1,22 @@ +use crate::observability::logger::LoggerConfig; +use crate::observability::otel::OtelConfig; +use crate::observability::sink::SinkConfig; +use serde::Deserialize; + +/// Observability configuration +#[derive(Debug, Deserialize, Clone)] +pub struct ObservabilityConfig { + pub otel: OtelConfig, + pub sinks: SinkConfig, + pub logger: Option, +} + +impl ObservabilityConfig { + pub fn new() -> Self { + Self { + otel: OtelConfig::new(), + sinks: SinkConfig::new(), + logger: Some(LoggerConfig::new()), + } + } +} diff --git a/crates/config/src/observability/otel.rs b/crates/config/src/observability/otel.rs new file mode 100644 index 000000000..4ac6618bd --- /dev/null +++ b/crates/config/src/observability/otel.rs @@ -0,0 +1,27 @@ +use serde::Deserialize; + +/// OpenTelemetry configuration +#[derive(Debug, Deserialize, Clone)] +pub struct OtelConfig { + pub endpoint: String, + pub service_name: String, + pub service_version: String, + pub resource_attributes: Vec, +} + +impl OtelConfig { + pub fn new() -> Self { + Self { + endpoint: "http://localhost:4317".to_string(), + service_name: "rustfs".to_string(), + service_version: "0.1.0".to_string(), + resource_attributes: vec![], + } + } +} + +impl Default for OtelConfig { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/config/src/observability/sink.rs b/crates/config/src/observability/sink.rs new file mode 100644 index 000000000..dcb37fa3b --- /dev/null +++ b/crates/config/src/observability/sink.rs @@ -0,0 +1,28 @@ +use crate::observability::file_sink::FileSinkConfig; +use crate::observability::kafka_sink::KafkaSinkConfig; +use crate::observability::webhook_sink::WebhookSinkConfig; +use serde::Deserialize; + +/// Sink configuration +#[derive(Debug, Deserialize, Clone)] +pub struct SinkConfig { + pub kafka: Option, + pub webhook: Option, + pub file: Option, +} + +impl SinkConfig { + pub fn new() -> Self { + Self { + kafka: None, + webhook: None, + file: Some(FileSinkConfig::new()), + } + } +} + +impl Default for SinkConfig { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/config/src/observability/webhook_sink.rs b/crates/config/src/observability/webhook_sink.rs new file mode 100644 index 000000000..494292039 --- /dev/null +++ b/crates/config/src/observability/webhook_sink.rs @@ -0,0 +1,25 @@ +use serde::Deserialize; + +/// Webhook sink configuration +#[derive(Debug, Deserialize, Clone)] +pub struct WebhookSinkConfig { + pub url: String, + pub method: String, + pub headers: Vec<(String, String)>, +} + +impl WebhookSinkConfig { + pub fn new() -> Self { + Self { + url: "http://localhost:8080/webhook".to_string(), + method: "POST".to_string(), + headers: vec![], + } + } +} + +impl Default for WebhookSinkConfig { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/event-notifier/src/config.rs b/crates/event-notifier/src/config.rs index ab46d8e8f..10429c34b 100644 --- a/crates/event-notifier/src/config.rs +++ b/crates/event-notifier/src/config.rs @@ -162,7 +162,7 @@ impl NotifierConfig { } } -const DEFAULT_CONFIG_FILE: &str = "obs"; +const DEFAULT_CONFIG_FILE: &str = "event"; /// Provide temporary directories as default storage paths fn default_store_path() -> String { diff --git a/crates/event-notifier/src/event.rs b/crates/event-notifier/src/event.rs index 55e4c0bd1..1bf100d75 100644 --- a/crates/event-notifier/src/event.rs +++ b/crates/event-notifier/src/event.rs @@ -154,16 +154,6 @@ impl Default for Metadata { } } impl Metadata { - /// Create a new Metadata instance - pub fn create(schema_version: String, configuration_id: String, bucket: Bucket, object: Object) -> Self { - Self { - schema_version, - configuration_id, - bucket, - object, - } - } - /// Create a new Metadata instance with default values pub fn new() -> Self { Self { @@ -178,6 +168,16 @@ impl Metadata { } } + /// Create a new Metadata instance + pub fn create(schema_version: String, configuration_id: String, bucket: Bucket, object: Object) -> Self { + Self { + schema_version, + configuration_id, + bucket, + object, + } + } + /// Set the schema version pub fn set_schema_version(&mut self, schema_version: String) { self.schema_version = schema_version; @@ -470,17 +470,7 @@ pub struct Log { pub records: Vec, } -#[derive( - Debug, - Clone, - Copy, - PartialEq, - Eq, - SerializeDisplay, - DeserializeFromStr, - Display, - EnumString -)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, SerializeDisplay, DeserializeFromStr, Display, EnumString)] #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] pub enum Name { ObjectAccessedGet, diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml new file mode 100644 index 000000000..13ee21e40 --- /dev/null +++ b/crates/utils/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "rustfs-utils" +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[dependencies] +local-ip-address = { workspace = true } +rustfs-config = { workspace = true } +rustls = { workspace = true } +rustls-pemfile = { workspace = true } +rustls-pki-types = { workspace = true } +tracing = { workspace = true } + +[lints] +workspace = true diff --git a/rustfs/src/utils/certs.rs b/crates/utils/src/certs.rs similarity index 78% rename from rustfs/src/utils/certs.rs rename to crates/utils/src/certs.rs index 115d43451..568fc6b69 100644 --- a/rustfs/src/utils/certs.rs +++ b/crates/utils/src/certs.rs @@ -1,4 +1,4 @@ -use crate::config::{RUSTFS_TLS_CERT, RUSTFS_TLS_KEY}; +use rustfs_config::{RUSTFS_TLS_CERT, RUSTFS_TLS_KEY}; use rustls::server::{ClientHello, ResolvesServerCert, ResolvesServerCertUsingSni}; use rustls::sign::CertifiedKey; use rustls_pemfile::{certs, private_key}; @@ -12,34 +12,37 @@ use tracing::{debug, warn}; /// Load public certificate from file. /// This function loads a public certificate from the specified file. -pub(crate) fn load_certs(filename: &str) -> io::Result>> { +pub fn load_certs(filename: &str) -> io::Result>> { // Open certificate file. - let cert_file = fs::File::open(filename).map_err(|e| error(format!("failed to open {}: {}", filename, e)))?; + let cert_file = fs::File::open(filename).map_err(|e| certs_error(format!("failed to open {}: {}", filename, e)))?; let mut reader = io::BufReader::new(cert_file); // Load and return certificate. let certs = certs(&mut reader) .collect::, _>>() - .map_err(|_| error(format!("certificate file {} format error", filename)))?; + .map_err(|_| certs_error(format!("certificate file {} format error", filename)))?; if certs.is_empty() { - return Err(error(format!("No valid certificate was found in the certificate file {}", filename))); + return Err(certs_error(format!( + "No valid certificate was found in the certificate file {}", + filename + ))); } Ok(certs) } /// Load private key from file. /// This function loads a private key from the specified file. -pub(crate) fn load_private_key(filename: &str) -> io::Result> { +pub fn load_private_key(filename: &str) -> io::Result> { // Open keyfile. - let keyfile = fs::File::open(filename).map_err(|e| error(format!("failed to open {}: {}", filename, e)))?; + let keyfile = fs::File::open(filename).map_err(|e| certs_error(format!("failed to open {}: {}", filename, e)))?; let mut reader = io::BufReader::new(keyfile); // Load and return a single private key. - private_key(&mut reader)?.ok_or_else(|| error(format!("no private key found in {}", filename))) + private_key(&mut reader)?.ok_or_else(|| certs_error(format!("no private key found in {}", filename))) } /// error function -pub(crate) fn error(err: String) -> Error { +pub fn certs_error(err: String) -> Error { Error::new(io::ErrorKind::Other, err) } @@ -47,14 +50,14 @@ pub(crate) fn error(err: String) -> Error { /// This function loads all certificate and private key pairs from the specified directory. /// It looks for files named `rustfs_cert.pem` and `rustfs_key.pem` in each subdirectory. /// The root directory can also contain a default certificate/private key pair. -pub(crate) fn load_all_certs_from_directory( +pub fn load_all_certs_from_directory( dir_path: &str, ) -> io::Result>, PrivateKeyDer<'static>)>> { let mut cert_key_pairs = HashMap::new(); let dir = Path::new(dir_path); if !dir.exists() || !dir.is_dir() { - return Err(error(format!( + return Err(certs_error(format!( "The certificate directory does not exist or is not a directory: {}", dir_path ))); @@ -68,10 +71,10 @@ pub(crate) fn load_all_certs_from_directory( debug!("find the root directory certificate: {:?}", root_cert_path); let root_cert_str = root_cert_path .to_str() - .ok_or_else(|| error(format!("Invalid UTF-8 in root certificate path: {:?}", root_cert_path)))?; + .ok_or_else(|| certs_error(format!("Invalid UTF-8 in root certificate path: {:?}", root_cert_path)))?; let root_key_str = root_key_path .to_str() - .ok_or_else(|| error(format!("Invalid UTF-8 in root key path: {:?}", root_key_path)))?; + .ok_or_else(|| certs_error(format!("Invalid UTF-8 in root key path: {:?}", root_key_path)))?; match load_cert_key_pair(root_cert_str, root_key_str) { Ok((certs, key)) => { // The root directory certificate is used as the default certificate and is stored using special keys. @@ -92,7 +95,7 @@ pub(crate) fn load_all_certs_from_directory( let domain_name = path .file_name() .and_then(|name| name.to_str()) - .ok_or_else(|| error(format!("invalid domain name directory:{:?}", path)))?; + .ok_or_else(|| certs_error(format!("invalid domain name directory:{:?}", path)))?; // find certificate and private key files let cert_path = path.join(RUSTFS_TLS_CERT); // e.g., rustfs_cert.pem @@ -113,7 +116,10 @@ pub(crate) fn load_all_certs_from_directory( } if cert_key_pairs.is_empty() { - return Err(error(format!("No valid certificate/private key pair found in directory {}", dir_path))); + return Err(certs_error(format!( + "No valid certificate/private key pair found in directory {}", + dir_path + ))); } Ok(cert_key_pairs) @@ -159,7 +165,7 @@ pub fn create_multi_cert_resolver( for (domain, (certs, key)) in cert_key_pairs { // create a signature let signing_key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key) - .map_err(|_| error(format!("unsupported private key types:{}", domain)))?; + .map_err(|_| certs_error(format!("unsupported private key types:{}", domain)))?; // create a CertifiedKey let certified_key = CertifiedKey::new(certs, signing_key); @@ -169,7 +175,7 @@ pub fn create_multi_cert_resolver( // add certificate to resolver resolver .add(&domain, certified_key) - .map_err(|e| error(format!("failed to add a domain name certificate:{},err: {:?}", domain, e)))?; + .map_err(|e| certs_error(format!("failed to add a domain name certificate:{},err: {:?}", domain, e)))?; } } diff --git a/crates/utils/src/ip.rs b/crates/utils/src/ip.rs new file mode 100644 index 000000000..3b63b12fc --- /dev/null +++ b/crates/utils/src/ip.rs @@ -0,0 +1,43 @@ +use std::net::{IpAddr, Ipv4Addr}; + +/// Get the IP address of the machine +/// +/// Priority is given to trying to get the IPv4 address, and if it fails, try to get the IPv6 address. +/// If both fail to retrieve, None is returned. +/// +/// # Returns +/// +/// * `Some(IpAddr)` - Native IP address (IPv4 or IPv6) +/// * `None` - Unable to obtain any native IP address +pub fn get_local_ip() -> Option { + local_ip_address::local_ip() + .ok() + .or_else(|| local_ip_address::local_ipv6().ok()) +} + +/// Get the IP address of the machine as a string +/// +/// If the IP address cannot be obtained, returns "127.0.0.1" as the default value. +/// +/// # Returns +/// +/// * `String` - Native IP address (IPv4 or IPv6) as a string, or the default value +pub fn get_local_ip_with_default() -> String { + get_local_ip() + .unwrap_or_else(|| IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))) // Provide a safe default value + .to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_get_local_ip() { + match get_local_ip() { + Some(ip) => println!("the ip address of this machine:{}", ip), + None => println!("Unable to obtain the IP address of the machine"), + } + assert!(get_local_ip().is_some()); + } +} diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs new file mode 100644 index 000000000..fbb5936b6 --- /dev/null +++ b/crates/utils/src/lib.rs @@ -0,0 +1,11 @@ +mod certs; +mod ip; +mod net; + +pub use certs::certs_error; +pub use certs::create_multi_cert_resolver; +pub use certs::load_all_certs_from_directory; +pub use certs::load_certs; +pub use certs::load_private_key; +pub use ip::get_local_ip; +pub use ip::get_local_ip_with_default; diff --git a/crates/utils/src/net.rs b/crates/utils/src/net.rs new file mode 100644 index 000000000..e69de29bb diff --git a/ecstore/Cargo.toml b/ecstore/Cargo.toml index 007150d40..0f9431fce 100644 --- a/ecstore/Cargo.toml +++ b/ecstore/Cargo.toml @@ -11,6 +11,7 @@ rust-version.workspace = true workspace = true [dependencies] +rustfs-config = { workspace = true } async-trait.workspace = true backon.workspace = true blake2 = { workspace = true } diff --git a/ecstore/src/global.rs b/ecstore/src/global.rs index 162aa9d3e..c60c6c594 100644 --- a/ecstore/src/global.rs +++ b/ecstore/src/global.rs @@ -20,8 +20,6 @@ pub const DISK_MIN_INODES: u64 = 1000; pub const DISK_FILL_FRACTION: f64 = 0.99; pub const DISK_RESERVE_FRACTION: f64 = 0.15; -pub const DEFAULT_PORT: u16 = 9000; - lazy_static! { static ref GLOBAL_RUSTFS_PORT: OnceLock = OnceLock::new(); pub static ref GLOBAL_OBJECT_API: OnceLock> = OnceLock::new(); @@ -41,31 +39,37 @@ lazy_static! { pub static ref GLOBAL_BOOT_TIME: OnceCell = OnceCell::new(); } +/// Get the global rustfs port pub fn global_rustfs_port() -> u16 { if let Some(p) = GLOBAL_RUSTFS_PORT.get() { *p } else { - DEFAULT_PORT + rustfs_config::DEFAULT_PORT } } +/// Set the global rustfs port pub fn set_global_rustfs_port(value: u16) { GLOBAL_RUSTFS_PORT.set(value).expect("set_global_rustfs_port fail"); } +/// Get the global rustfs port pub fn set_global_deployment_id(id: Uuid) { globalDeploymentIDPtr.set(id).unwrap(); } + +/// Get the global deployment id pub fn get_global_deployment_id() -> Option { globalDeploymentIDPtr.get().map(|v| v.to_string()) } - +/// Get the global deployment id pub fn set_global_endpoints(eps: Vec) { GLOBAL_Endpoints .set(EndpointServerPools::from(eps)) .expect("GLOBAL_Endpoints set failed") } +/// Get the global endpoints pub fn get_global_endpoints() -> EndpointServerPools { if let Some(eps) = GLOBAL_Endpoints.get() { eps.clone() diff --git a/ecstore/src/heal/heal_ops.rs b/ecstore/src/heal/heal_ops.rs index 6f4614578..f2e065fc7 100644 --- a/ecstore/src/heal/heal_ops.rs +++ b/ecstore/src/heal/heal_ops.rs @@ -17,7 +17,7 @@ use crate::{ global::GLOBAL_IsDistErasure, heal::heal_commands::{HealStartSuccess, HEAL_UNKNOWN_SCAN}, new_object_layer_fn, - utils::path::has_profix, + utils::path::has_prefix, }; use crate::{ heal::heal_commands::{HEAL_ITEM_BUCKET, HEAL_ITEM_OBJECT}, @@ -786,7 +786,7 @@ impl AllHealState { let _ = self.mu.write().await; for (k, v) in self.heal_seq_map.read().await.iter() { - if (has_profix(k, path_s) || has_profix(path_s, k)) && !v.has_ended().await { + if (has_prefix(k, path_s) || has_prefix(path_s, k)) && !v.has_ended().await { return Err(Error::from_string(format!( "The provided heal sequence path overlaps with an existing heal path: {}", k diff --git a/ecstore/src/utils/path.rs b/ecstore/src/utils/path.rs index 0e38eac2f..0c63b960a 100644 --- a/ecstore/src/utils/path.rs +++ b/ecstore/src/utils/path.rs @@ -52,7 +52,7 @@ pub fn strings_has_prefix_fold(s: &str, prefix: &str) -> bool { s.len() >= prefix.len() && (s[..prefix.len()] == *prefix || s[..prefix.len()].eq_ignore_ascii_case(prefix)) } -pub fn has_profix(s: &str, prefix: &str) -> bool { +pub fn has_prefix(s: &str, prefix: &str) -> bool { if cfg!(target_os = "windows") { return strings_has_prefix_fold(s, prefix); } diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 460e55bef..7f7ec6902 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -30,7 +30,7 @@ clap.workspace = true crypto = { workspace = true } datafusion = { workspace = true } common.workspace = true -const-str = { version = "0.6.1", features = ["std", "proc"] } +const-str = { workspace = true } ecstore.workspace = true policy.workspace = true flatbuffers.workspace = true @@ -42,7 +42,6 @@ http.workspace = true http-body.workspace = true iam = { workspace = true } lock.workspace = true -local-ip-address = { workspace = true } matchit = { workspace = true } mime.workspace = true mime_guess = { workspace = true } @@ -51,18 +50,18 @@ pin-project-lite.workspace = true protos.workspace = true query = { workspace = true } rmp-serde.workspace = true +rustfs-config = { workspace = true } rustfs-event-notifier = { workspace = true } rustfs-obs = { workspace = true } +rustfs-utils = { workspace = true } rustls.workspace = true -rustls-pemfile.workspace = true -rustls-pki-types.workspace = true rust-embed = { workspace = true, features = ["interpolate-folder-path"] } s3s.workspace = true serde.workspace = true serde_json.workspace = true serde_urlencoded = { workspace = true } shadow-rs = { workspace = true, features = ["build", "metadata"] } -socket2 = "0.5.9" +socket2 = { workspace = true } tracing.workspace = true time = { workspace = true, features = ["parsing", "formatting", "serde"] } tokio-util.workspace = true diff --git a/rustfs/README.md b/rustfs/README.md index 86b2ab767..147e7ed7e 100644 --- a/rustfs/README.md +++ b/rustfs/README.md @@ -1,30 +1,36 @@ -rustfs/ -├── Cargo.toml -├── src/ -│ ├── main.rs # 主入口 -│ ├── admin/ -│ │ └── mod.rs # 管理接口 -│ ├── auth/ -│ │ └── mod.rs # 认证模块 -│ ├── config/ -│ │ ├── mod.rs # 配置模块 -│ │ └── options.rs # 命令行参数 -│ ├── console/ -│ │ ├── mod.rs # 控制台模块 -│ │ └── server.rs # 控制台服务器 -│ ├── grpc/ -│ │ └── mod.rs # gRPC 服务 -│ ├── license/ -│ │ └── mod.rs # 许可证管理 -│ ├── logging/ -│ │ └── mod.rs # 日志管理 -│ ├── server/ -│ │ ├── mod.rs # 服务器实现 -│ │ ├── connection.rs # 连接处理 -│ │ ├── service.rs # 服务实现 -│ │ └── state.rs # 状态管理 -│ ├── storage/ -│ │ ├── mod.rs # 存储模块 -│ │ └── fs.rs # 文件系统实现 -│ └── utils/ -│ └── mod.rs # 工具函数 \ No newline at end of file +# RustFS + +RustFS is a simple file system written in Rust. It is designed to be a learning project for those who want to understand +how file systems work and how to implement them in Rust. + +## Features + +- Simple file system structure +- Basic file operations (create, read, write, delete) +- Directory support +- File metadata (size, creation time, etc.) +- Basic error handling +- Unit tests for core functionality +- Documentation for public API +- Example usage +- License information +- Contributing guidelines +- Changelog +- Code of conduct +- Acknowledgements +- Contact information +- Links to additional resources + +## Getting Started + +To get started with RustFS, clone the repository and build the project: + +```bash +git clone git@github.com:rustfs/s3-rustfs.git +cd rustfs +cargo build +``` + +## Usage + +To use RustFS, you can create a new file system instance and perform basic file operations. Here is an example: diff --git a/rustfs/src/config/mod.rs b/rustfs/src/config/mod.rs index 7a7bdc4c4..a93b6ddaf 100644 --- a/rustfs/src/config/mod.rs +++ b/rustfs/src/config/mod.rs @@ -1,40 +1,8 @@ use clap::Parser; use const_str::concat; -use ecstore::global::DEFAULT_PORT; use std::string::ToString; shadow_rs::shadow!(build); -/// Default Access Key -/// Default value: rustfsadmin -/// Environment variable: RUSTFS_ACCESS_KEY -/// Command line argument: --access-key -/// Example: RUSTFS_ACCESS_KEY=rustfsadmin -/// Example: --access-key rustfsadmin -pub const DEFAULT_ACCESS_KEY: &str = "rustfsadmin"; -/// Default Secret Key -/// Default value: rustfsadmin -/// Environment variable: RUSTFS_SECRET_KEY -/// Command line argument: --secret-key -/// Example: RUSTFS_SECRET_KEY=rustfsadmin -/// Example: --secret-key rustfsadmin -pub const DEFAULT_SECRET_KEY: &str = "rustfsadmin"; -/// Default configuration file for observability -/// Default value: config/obs.toml -/// Environment variable: RUSTFS_OBS_CONFIG -/// Command line argument: --obs-config -/// Example: RUSTFS_OBS_CONFIG=config/obs.toml -/// Example: --obs-config config/obs.toml -/// Example: --obs-config /etc/rustfs/obs.toml -pub const DEFAULT_OBS_CONFIG: &str = "config/obs.toml"; - -/// Default TLS key for rustfs -/// This is the default key for TLS. -pub(crate) const RUSTFS_TLS_KEY: &str = "rustfs_key.pem"; - -/// Default TLS cert for rustfs -/// This is the default cert for TLS. -pub(crate) const RUSTFS_TLS_CERT: &str = "rustfs_cert.pem"; - #[allow(clippy::const_is_empty)] const SHORT_VERSION: &str = { if !build::TAG.is_empty() { @@ -67,7 +35,7 @@ pub struct Opt { pub volumes: Vec, /// bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname - #[arg(long, default_value_t = format!("0.0.0.0:{}", DEFAULT_PORT), env = "RUSTFS_ADDRESS")] + #[arg(long, default_value_t = rustfs_config::DEFAULT_ADDRESS.to_string(), env = "RUSTFS_ADDRESS")] pub address: String, /// Domain name used for virtual-hosted-style requests. @@ -75,17 +43,19 @@ pub struct Opt { pub server_domains: Vec, /// Access key used for authentication. - #[arg(long, default_value_t = DEFAULT_ACCESS_KEY.to_string(), env = "RUSTFS_ACCESS_KEY")] + #[arg(long, default_value_t = rustfs_config::DEFAULT_ACCESS_KEY.to_string(), env = "RUSTFS_ACCESS_KEY")] pub access_key: String, /// Secret key used for authentication. - #[arg(long, default_value_t = DEFAULT_SECRET_KEY.to_string(), env = "RUSTFS_SECRET_KEY")] + #[arg(long, default_value_t = rustfs_config::DEFAULT_SECRET_KEY.to_string(), env = "RUSTFS_SECRET_KEY")] pub secret_key: String, + /// Enable console server #[arg(long, default_value_t = false, env = "RUSTFS_CONSOLE_ENABLE")] pub console_enable: bool, - #[arg(long, default_value_t = format!("127.0.0.1:{}", 9002), env = "RUSTFS_CONSOLE_ADDRESS")] + /// Console server bind address + #[arg(long, default_value_t = rustfs_config::DEFAULT_CONSOLE_ADDRESS.to_string(), env = "RUSTFS_CONSOLE_ADDRESS")] pub console_address: String, /// rustfs endpoint for console @@ -94,7 +64,7 @@ pub struct Opt { /// Observability configuration file /// Default value: config/obs.toml - #[arg(long, default_value_t = DEFAULT_OBS_CONFIG.to_string(), env = "RUSTFS_OBS_CONFIG")] + #[arg(long, default_value_t = rustfs_config::DEFAULT_OBS_CONFIG.to_string(), env = "RUSTFS_OBS_CONFIG")] pub obs_config: String, /// tls path for rustfs api and console. diff --git a/rustfs/src/console.rs b/rustfs/src/console.rs index 0c86d4f3b..0523991d6 100644 --- a/rustfs/src/console.rs +++ b/rustfs/src/console.rs @@ -1,4 +1,3 @@ -use crate::config::{RUSTFS_TLS_CERT, RUSTFS_TLS_KEY}; use crate::license::get_license; use axum::{ body::Body, @@ -8,6 +7,7 @@ use axum::{ Router, }; use axum_extra::extract::Host; +use rustfs_config::{RUSTFS_TLS_CERT, RUSTFS_TLS_KEY}; use std::io; use axum::response::Redirect; @@ -17,7 +17,7 @@ use mime_guess::from_path; use rust_embed::RustEmbed; use serde::Serialize; use shadow_rs::shadow; -use std::net::{Ipv4Addr, SocketAddr}; +use std::net::{IpAddr, SocketAddr}; use std::sync::OnceLock; use std::time::Duration; use tokio::signal; @@ -73,7 +73,7 @@ pub(crate) struct Config { } impl Config { - fn new(local_ip: Ipv4Addr, port: u16, version: &str, date: &str) -> Self { + fn new(local_ip: IpAddr, port: u16, version: &str, date: &str) -> Self { Config { port, api: Api { @@ -144,7 +144,7 @@ struct License { pub(crate) static CONSOLE_CONFIG: OnceLock = OnceLock::new(); #[allow(clippy::const_is_empty)] -pub(crate) fn init_console_cfg(local_ip: Ipv4Addr, port: u16) { +pub(crate) fn init_console_cfg(local_ip: IpAddr, port: u16) { CONSOLE_CONFIG.get_or_init(|| { let ver = { if !build::TAG.is_empty() { @@ -220,7 +220,7 @@ async fn config_handler(uri: Uri, Host(host): Host) -> impl IntoResponse { pub async fn start_static_file_server( addrs: &str, - local_ip: Ipv4Addr, + local_ip: IpAddr, access_key: &str, secret_key: &str, tls_path: Option, diff --git a/rustfs/src/event.rs b/rustfs/src/event.rs new file mode 100644 index 000000000..99e2a75c3 --- /dev/null +++ b/rustfs/src/event.rs @@ -0,0 +1,21 @@ +use rustfs_event_notifier::NotifierConfig; +use tracing::{error, info, instrument}; + +#[instrument] +pub(crate) async fn init_event_notifier(notifier_config: Option) { + // Initialize event notifier + if notifier_config.is_some() { + info!("event_config is not empty"); + tokio::spawn(async move { + let config = NotifierConfig::event_load_config(notifier_config); + let result = rustfs_event_notifier::initialize(config).await; + if let Err(e) = result { + error!("Failed to initialize event notifier: {}", e); + } else { + info!("Event notifier initialized successfully"); + } + }); + } else { + info!("event_config is empty"); + } +} diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index ee47cfa4e..bf94a6f6b 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -2,18 +2,18 @@ mod admin; mod auth; mod config; mod console; +mod event; mod grpc; pub mod license; mod logging; mod server; mod service; mod storage; -mod utils; + use crate::auth::IAMAuth; use crate::console::{init_console_cfg, CONSOLE_CONFIG}; // Ensure the correct path for parse_license is imported use crate::server::{wait_for_shutdown, ServiceState, ServiceStateManager, ShutdownSignal, SHUTDOWN_TIMEOUT}; -use crate::utils::error; use bytes::Bytes; use chrono::Datelike; use clap::Parser; @@ -21,7 +21,6 @@ use common::{ error::{Error, Result}, globals::set_global_addr, }; -use config::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, RUSTFS_TLS_CERT, RUSTFS_TLS_KEY}; use ecstore::bucket::metadata_sys::init_bucket_metadata_sys; use ecstore::config as ecconfig; use ecstore::config::GLOBAL_ConfigSys; @@ -48,7 +47,7 @@ use hyper_util::{ use iam::init_iam_sys; use license::init_license; use protos::proto_gen::node_service::node_service_server::NodeServiceServer; -use rustfs_event_notifier::NotifierConfig; +use rustfs_config::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, RUSTFS_TLS_CERT, RUSTFS_TLS_KEY}; use rustfs_obs::{init_obs, init_process_observer, load_config, set_global_guard}; use rustls::ServerConfig; use s3s::{host::MultiDomain, service::S3ServiceBuilder}; @@ -85,7 +84,7 @@ fn print_server_info() { let cfg = CONSOLE_CONFIG.get().unwrap(); let current_year = chrono::Utc::now().year(); - // 使用自定义宏打印服务器信息 + // Use custom macros to print server information info!("RustFS Object Storage Server"); info!("Copyright: 2024-{} RustFS, Inc", current_year); info!("License: {}", cfg.license()); @@ -114,32 +113,12 @@ async fn main() -> Result<()> { run(opt).await } -#[instrument] -async fn init_event_notifier(notifier_config: Option) { - // Initialize event notifier - if notifier_config.is_some() { - info!("event_config is not empty"); - tokio::spawn(async move { - let config = NotifierConfig::event_load_config(notifier_config); - let result = rustfs_event_notifier::initialize(config).await; - if let Err(e) = result { - error!("Failed to initialize event notifier: {}", e); - } else { - info!("Event notifier initialized successfully"); - } - }); - } else { - info!("event_config is empty"); - } -} - #[instrument(skip(opt))] async fn run(opt: config::Opt) -> Result<()> { debug!("opt: {:?}", &opt); // Initialize event notifier - let notifier_config = opt.event_config; - init_event_notifier(notifier_config).await; + event::init_event_notifier(opt.event_config).await; let server_addr = net::parse_and_resolve_address(opt.address.as_str())?; let server_port = server_addr.port(); @@ -147,16 +126,17 @@ async fn run(opt: config::Opt) -> Result<()> { debug!("server_address {}", &server_address); - //设置 AK 和 SK + // Set up AK and SK iam::init_global_action_cred(Some(opt.access_key.clone()), Some(opt.secret_key.clone()))?; set_global_rustfs_port(server_port); - //监听地址，端口从参数中获取 + // The listening address and port are obtained from the parameters let listener = TcpListener::bind(server_address.clone()).await?; - //获取监听地址 + // Obtain the listener address let local_addr: SocketAddr = listener.local_addr()?; - let local_ip = utils::get_local_ip().ok_or(local_addr.ip()).unwrap(); + // let local_ip = utils::get_local_ip().ok_or(local_addr.ip()).unwrap(); + let local_ip = rustfs_utils::get_local_ip().ok_or(local_addr.ip()).unwrap(); // 用于 rpc let (endpoint_pools, setup_type) = EndpointServerPools::from_volumes(server_address.clone().as_str(), opt.volumes.clone()) @@ -203,13 +183,13 @@ async fn run(opt: config::Opt) -> Result<()> { set_global_endpoints(endpoint_pools.as_ref().clone()); update_erasure_type(setup_type).await; - // 初始化本地磁盘 + // Initialize the local disk init_local_disks(endpoint_pools.clone()) .await .map_err(|err| Error::from_string(err.to_string()))?; // Setup S3 service - // 本项目使用 s3s 库来实现 s3 服务 + // This project uses the S3S library to implement S3 services let s3_service = { let store = storage::ecfs::FS::new(); // let mut b = S3ServiceBuilder::new(storage::ecfs::FS::new(server_address.clone(), endpoint_pools).await?); @@ -217,7 +197,7 @@ async fn run(opt: config::Opt) -> Result<()> { let access_key = opt.access_key.clone(); let secret_key = opt.secret_key.clone(); - //显示 info 信息 + // Displays info information debug!("authentication is enabled {}, {}", &access_key, &secret_key); b.set_auth(IAMAuth::new(access_key, secret_key)); @@ -268,7 +248,7 @@ async fn run(opt: config::Opt) -> Result<()> { debug!("Found TLS directory, checking for certificates"); // 1. Try to load all certificates directly (including root and subdirectories) - match utils::load_all_certs_from_directory(&tls_path) { + match rustfs_utils::load_all_certs_from_directory(&tls_path) { Ok(cert_key_pairs) if !cert_key_pairs.is_empty() => { debug!("Found {} certificates, starting with HTTPS", cert_key_pairs.len()); let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); @@ -276,7 +256,7 @@ async fn run(opt: config::Opt) -> Result<()> { // create a multi certificate configuration let mut server_config = ServerConfig::builder() .with_no_client_auth() - .with_cert_resolver(Arc::new(utils::create_multi_cert_resolver(cert_key_pairs)?)); + .with_cert_resolver(Arc::new(rustfs_utils::create_multi_cert_resolver(cert_key_pairs)?)); server_config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"http/1.0".to_vec()]; Some(TlsAcceptor::from(Arc::new(server_config))) @@ -291,12 +271,14 @@ async fn run(opt: config::Opt) -> Result<()> { if has_single_cert { debug!("Found legacy single TLS certificate, starting with HTTPS"); let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); - let certs = utils::load_certs(cert_path.as_str()).map_err(|e| error(e.to_string()))?; - let key = utils::load_private_key(key_path.as_str()).map_err(|e| error(e.to_string()))?; + let certs = + rustfs_utils::load_certs(cert_path.as_str()).map_err(|e| rustfs_utils::certs_error(e.to_string()))?; + let key = rustfs_utils::load_private_key(key_path.as_str()) + .map_err(|e| rustfs_utils::certs_error(e.to_string()))?; let mut server_config = ServerConfig::builder() .with_no_client_auth() .with_single_cert(certs, key) - .map_err(|e| error(e.to_string()))?; + .map_err(|e| rustfs_utils::certs_error(e.to_string()))?; server_config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"http/1.0".to_vec()]; Some(TlsAcceptor::from(Arc::new(server_config))) } else { @@ -446,7 +428,7 @@ async fn run(opt: config::Opt) -> Result<()> { debug!("TLS certificates found, starting with SIGINT"); let tls_socket = match tls_acceptor .as_ref() - .ok_or_else(|| error("TLS not configured".to_string())) + .ok_or_else(|| rustfs_utils::certs_error("TLS not configured".to_string())) .unwrap() .accept(socket) .await diff --git a/rustfs/src/utils/mod.rs b/rustfs/src/utils/mod.rs deleted file mode 100644 index 9f391fee6..000000000 --- a/rustfs/src/utils/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -mod certs; -use std::net::IpAddr; - -pub(crate) use certs::create_multi_cert_resolver; -pub(crate) use certs::error; -pub(crate) use certs::load_all_certs_from_directory; -pub(crate) use certs::load_certs; -pub(crate) use certs::load_private_key; - -/// Get the local IP address. -/// This function retrieves the local IP address of the machine. -pub(crate) fn get_local_ip() -> Option { - match local_ip_address::local_ip() { - Ok(IpAddr::V4(ip)) => Some(ip), - Err(_) => None, - Ok(IpAddr::V6(_)) => todo!(), - } -} From 76fdefeca432bad92e3f81b9fa8c018d34f166bf Mon Sep 17 00:00:00 2001 From: weisd Date: Thu, 8 May 2025 15:44:28 +0800 Subject: [PATCH 21/38] feat: auto-extract support --- Cargo.lock | 60 ++++++++++++++++++ Cargo.toml | 45 +++++++------ crates/zip/Cargo.toml | 28 +++++++++ crates/zip/src/lib.rs | 124 ++++++++++++++++++++++++++++++++++++ ecstore/src/disk/os.rs | 3 +- ecstore/src/set_disk.rs | 9 +-- rustfs/Cargo.toml | 9 ++- rustfs/src/storage/ecfs.rs | 126 +++++++++++++++++++++++++++++++++++-- 8 files changed, 369 insertions(+), 35 deletions(-) create mode 100644 crates/zip/Cargo.toml create mode 100644 crates/zip/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index f3407172d..cce1d3bcb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3270,6 +3270,18 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "filetime" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys 0.59.0", +] + [[package]] name = "fixedbitset" version = "0.5.7" @@ -4779,6 +4791,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.9.0", "libc", + "redox_syscall 0.5.11", ] [[package]] @@ -6894,6 +6907,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.11" @@ -7289,6 +7311,7 @@ dependencies = [ "tokio", "tokio-rustls 0.26.2", "tokio-stream", + "tokio-tar", "tokio-util", "tonic 0.13.1", "tonic-build", @@ -7297,6 +7320,7 @@ dependencies = [ "tracing", "transform-stream", "uuid", + "zip", ] [[package]] @@ -8693,6 +8717,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-tar" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5714c010ca3e5c27114c1cdeb9d14641ace49874aa5626d7149e47aedace75" +dependencies = [ + "filetime", + "futures-core", + "libc", + "redox_syscall 0.3.5", + "tokio", + "tokio-stream", + "xattr", +] + [[package]] name = "tokio-util" version = "0.7.15" @@ -10213,6 +10252,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "xattr" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d65cbf2f12c15564212d48f4e3dfb87923d25d611f2aed18f4cb23f0413d89e" +dependencies = [ + "libc", + "rustix 1.0.5", +] + [[package]] name = "xdg-home" version = "1.3.0" @@ -10476,6 +10525,17 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "zip" +version = "0.0.1" +dependencies = [ + "async-compression", + "tokio", + "tokio-stream", + "tokio-tar", + "xz2", +] + [[package]] name = "zstd" version = "0.13.3" diff --git a/Cargo.toml b/Cargo.toml index 645533ae1..1d349f805 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,23 +1,24 @@ [workspace] members = [ - "appauth", # Application authentication and authorization - "cli/rustfs-gui", # Graphical user interface client - "common/common", # Shared utilities and data structures - "common/lock", # Distributed locking implementation - "common/protos", # Protocol buffer definitions - "common/workers", # Worker thread pools and task scheduling - "crates/config", # Configuration management + "appauth", # Application authentication and authorization + "cli/rustfs-gui", # Graphical user interface client + "common/common", # Shared utilities and data structures + "common/lock", # Distributed locking implementation + "common/protos", # Protocol buffer definitions + "common/workers", # Worker thread pools and task scheduling + "crates/config", # Configuration management "crates/event-notifier", # Event notification system - "crates/obs", # Observability utilities - "crates/utils", # Utility functions and helpers - "crypto", # Cryptography and security features - "ecstore", # Erasure coding storage implementation - "e2e_test", # End-to-end test suite - "iam", # Identity and Access Management - "madmin", # Management dashboard and admin API interface - "rustfs", # Core file system implementation - "s3select/api", # S3 Select API interface - "s3select/query", # S3 Select query engine + "crates/obs", # Observability utilities + "crates/utils", # Utility functions and helpers + "crypto", # Cryptography and security features + "ecstore", # Erasure coding storage implementation + "e2e_test", # End-to-end test suite + "iam", # Identity and Access Management + "madmin", # Management dashboard and admin API interface + "rustfs", # Core file system implementation + "s3select/api", # S3 Select API interface + "s3select/query", # S3 Select query engine + "crates/zip", ] resolver = "2" @@ -47,11 +48,13 @@ policy = { path = "./policy", version = "0.0.1" } protos = { path = "./common/protos", version = "0.0.1" } query = { path = "./s3select/query", version = "0.0.1" } rustfs = { path = "./rustfs", version = "0.0.1" } +zip = { path = "./crates/zip", version = "0.0.1" } rustfs-config = { path = "./crates/config", version = "0.0.1" } rustfs-obs = { path = "crates/obs", version = "0.0.1" } rustfs-event-notifier = { path = "crates/event-notifier", version = "0.0.1" } rustfs-utils = { path = "crates/utils", version = "0.0.1" } workers = { path = "./common/workers", version = "0.0.1" } +tokio-tar = "0.3.1" atoi = "2.0.0" async-recursion = "1.1.1" async-trait = "0.1.88" @@ -113,7 +116,9 @@ opentelemetry-appender-tracing = { version = "0.29.1", features = [ opentelemetry_sdk = { version = "0.29.0" } opentelemetry-stdout = { version = "0.29.0" } opentelemetry-otlp = { version = "0.29.0" } -opentelemetry-semantic-conventions = { version = "0.29.0", features = ["semconv_experimental"] } +opentelemetry-semantic-conventions = { version = "0.29.0", features = [ + "semconv_experimental", +] } parking_lot = "0.12.3" pin-project-lite = "0.2.16" # pin-utils = "0.1.0" @@ -206,8 +211,8 @@ inherits = "dev" opt-level = 3 lto = "fat" codegen-units = 1 -panic = "abort" # Optional, remove the panic expansion code -strip = true # strip symbol information to reduce binary size +panic = "abort" # Optional, remove the panic expansion code +strip = true # strip symbol information to reduce binary size [profile.production] inherits = "release" diff --git a/crates/zip/Cargo.toml b/crates/zip/Cargo.toml new file mode 100644 index 000000000..a73f083b0 --- /dev/null +++ b/crates/zip/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "zip" +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + + +[dependencies] +async-compression = { version = "0.4.0", features = [ + "tokio", + "bzip2", + "gzip", + "zlib", + "zstd", + "xz", +] } +# async_zip = { version = "0.0.17", features = ["tokio"] } +# rc-zip-tokio = "4.2.6" +tokio = { version = "1.45.0", features = ["full"] } +tokio-stream = "0.1.17" +tokio-tar = { workspace = true } +xz2 = { version = "0.1", optional = true, features = ["static"] } + + +[lints] +workspace = true diff --git a/crates/zip/src/lib.rs b/crates/zip/src/lib.rs new file mode 100644 index 000000000..0da854c55 --- /dev/null +++ b/crates/zip/src/lib.rs @@ -0,0 +1,124 @@ +use async_compression::tokio::bufread::{BzDecoder, GzipDecoder, XzDecoder, ZlibDecoder, ZstdDecoder}; +use tokio::io::{self, AsyncRead, BufReader}; +use tokio_stream::StreamExt; +use tokio_tar::Archive; + +#[derive(Debug, PartialEq)] +pub enum CompressionFormat { + Gzip, //.gz + Bzip2, //.bz2 + // Lz4, //.lz4 + Zip, + Xz, //.xz + Zlib, //.z + Zstd, //.zst + Unknown, +} + +impl CompressionFormat { + pub fn from_extension(ext: &str) -> Self { + match ext { + "gz" => CompressionFormat::Gzip, + "bz2" => CompressionFormat::Bzip2, + // "lz4" => CompressionFormat::Lz4, + "zip" => CompressionFormat::Zip, + "xz" => CompressionFormat::Xz, + "zlib" => CompressionFormat::Zlib, + "zst" => CompressionFormat::Zstd, + _ => CompressionFormat::Unknown, + } + } + + pub fn get_decoder(&self, input: R) -> io::Result> + where + R: AsyncRead + Send + Unpin + 'static, + { + let reader = BufReader::new(input); + + let decoder: Box = match self { + CompressionFormat::Gzip => Box::new(GzipDecoder::new(reader)), + CompressionFormat::Bzip2 => Box::new(BzDecoder::new(reader)), + // CompressionFormat::Lz4 => Box::new(Lz4Decoder::new(reader)), + CompressionFormat::Zlib => Box::new(ZlibDecoder::new(reader)), + CompressionFormat::Xz => Box::new(XzDecoder::new(reader)), + CompressionFormat::Zstd => Box::new(ZstdDecoder::new(reader)), + _ => return Err(io::Error::new(io::ErrorKind::InvalidInput, "Unsupported file format")), + }; + + Ok(decoder) + } +} + +pub async fn decompress(input: R, format: CompressionFormat, mut callback: F) -> io::Result<()> +where + R: AsyncRead + Send + Unpin + 'static, + F: AsyncFnMut(tokio_tar::Entry>>) -> std::io::Result<()> + Send + 'static, +{ + // 打开输入文件 + // println!("format {:?}", format); + + let decoder = format.get_decoder(input)?; + + // let reader: BufReader = BufReader::new(input); + + // // 根据文件扩展名选择解压器 + // let decoder: Box = match format { + // CompressionFormat::Gzip => Box::new(GzipDecoder::new(reader)), + // CompressionFormat::Bzip2 => Box::new(BzDecoder::new(reader)), + // // CompressionFormat::Lz4 => Box::new(Lz4Decoder::new(reader)), + // CompressionFormat::Zlib => Box::new(ZlibDecoder::new(reader)), + // CompressionFormat::Xz => Box::new(XzDecoder::new(reader)), + // CompressionFormat::Zstd => Box::new(ZstdDecoder::new(reader)), + // // CompressionFormat::Zip => Box::new(DeflateDecoder::new(reader)), + // _ => { + // return Err(io::Error::new(io::ErrorKind::InvalidInput, "Unsupported file format")); + // } + // }; + + let mut ar = Archive::new(decoder); + let mut entries = ar.entries().unwrap(); + while let Some(entry) = entries.next().await { + let f = match entry { + Ok(f) => f, + Err(e) => { + println!("Error reading entry: {}", e); + return Err(e); + } + }; + // println!("{}", f.path().unwrap().display()); + callback(f).await?; + } + + Ok(()) +} + +// #[tokio::test] +// async fn test_decompress() -> io::Result<()> { +// use std::path::Path; +// use tokio::fs::File; + +// let input_path = "/Users/weisd/Downloads/wsd.tar.gz"; // 替换为你的压缩文件路径 + +// let f = File::open(input_path).await?; + +// let Some(ext) = Path::new(input_path).extension().and_then(|s| s.to_str()) else { +// return Err(io::Error::new(io::ErrorKind::InvalidInput, "Unsupported file format")); +// }; + +// match decompress( +// f, +// CompressionFormat::from_extension(ext), +// |entry: tokio_tar::Entry>>| async move { +// let path = entry.path().unwrap(); +// println!("Extracted: {}", path.display()); +// Ok(()) +// }, +// ) +// .await +// { +// Ok(_) => println!("解压成功！"), +// Err(e) => println!("解压失败: {}", e), +// } + +// Ok(()) +// } diff --git a/ecstore/src/disk/os.rs b/ecstore/src/disk/os.rs index ae88611a8..579bd5386 100644 --- a/ecstore/src/disk/os.rs +++ b/ecstore/src/disk/os.rs @@ -9,7 +9,6 @@ use crate::{ }; use common::error::{Error, Result}; use tokio::fs; -use tracing::info; use super::error::{os_err_to_file_err, os_is_exist, DiskError}; @@ -137,7 +136,7 @@ pub async fn reliable_rename( ) -> io::Result<()> { if let Some(parent) = dst_file_path.as_ref().parent() { if !file_exists(parent).await { - info!("reliable_rename reliable_mkdir_all parent: {:?}", parent); + // info!("reliable_rename reliable_mkdir_all parent: {:?}", parent); reliable_mkdir_all(parent, base_dir.as_ref()).await?; } } diff --git a/ecstore/src/set_disk.rs b/ecstore/src/set_disk.rs index abf7422ac..ca2220b30 100644 --- a/ecstore/src/set_disk.rs +++ b/ecstore/src/set_disk.rs @@ -421,7 +421,7 @@ impl SetDisks { let file_path = file_path.clone(); async move { if let Some(disk) = disk { - match disk + (disk .delete( bucket, &file_path, @@ -430,11 +430,8 @@ impl SetDisks { ..Default::default() }, ) - .await - { - Ok(_) => None, - Err(e) => Some(e), - } + .await) + .err() } else { Some(Error::new(DiskError::DiskNotFound)) } diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 7f7ec6902..3023a9cab 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -15,6 +15,8 @@ path = "src/main.rs" workspace = true [dependencies] +zip = { workspace = true } +tokio-tar = { workspace = true } madmin = { workspace = true } api = { workspace = true } appauth = { workspace = true } @@ -77,7 +79,12 @@ tokio-stream.workspace = true tonic = { workspace = true } tower.workspace = true transform-stream.workspace = true -tower-http = { workspace = true, features = ["trace", "compression-deflate", "compression-gzip", "cors"] } +tower-http = { workspace = true, features = [ + "trace", + "compression-deflate", + "compression-gzip", + "cors", +] } uuid = { workspace = true } [target.'cfg(target_os = "linux")'.dependencies] diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index fd7b4c0c9..418ecf077 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -4,6 +4,10 @@ use super::options::extract_metadata; use super::options::put_opts; use crate::auth::get_condition_values; use crate::storage::access::ReqInfo; +use crate::storage::error::to_s3_error; +use crate::storage::options::copy_dst_opts; +use crate::storage::options::copy_src_opts; +use crate::storage::options::{extract_metadata_from_mime, get_opts}; use api::query::Context; use api::query::Query; use api::server::dbms::DatabaseManagerSystem; @@ -61,10 +65,12 @@ use s3s::S3Result; use s3s::S3; use s3s::{S3Request, S3Response}; use std::fmt::Debug; +use std::path::Path; use std::str::FromStr; use std::sync::Arc; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; +use tokio_tar::Archive; use tokio_util::io::ReaderStream; use tokio_util::io::StreamReader; use tracing::debug; @@ -73,11 +79,7 @@ use tracing::info; use tracing::warn; use transform_stream::AsyncTryStream; use uuid::Uuid; - -use crate::storage::error::to_s3_error; -use crate::storage::options::copy_dst_opts; -use crate::storage::options::copy_src_opts; -use crate::storage::options::{extract_metadata_from_mime, get_opts}; +use zip::CompressionFormat; macro_rules! try_ { ($result:expr) => { @@ -107,6 +109,108 @@ impl FS { // let store: ECStore = ECStore::new(address, endpoint_pools).await?; Self {} } + + async fn put_object_extract(&self, req: S3Request) -> S3Result> { + let PutObjectInput { body, bucket, key, .. } = req.input; + + let Some(body) = body else { return Err(s3_error!(IncompleteBody)) }; + + let body = StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string())))); + + // let etag_stream = EtagReader::new(body); + + let Some(ext) = Path::new(&key).extension().and_then(|s| s.to_str()) else { + return Err(s3_error!(InvalidArgument, "key extension not found")); + }; + + let ext = ext.to_owned(); + + // TODO: spport zip + let decoder = CompressionFormat::from_extension(&ext).get_decoder(body).map_err(|e| { + error!("get_decoder err {:?}", e); + s3_error!(InvalidArgument, "get_decoder err") + })?; + + let mut ar = Archive::new(decoder); + let mut entries = ar.entries().map_err(|e| { + error!("get entries err {:?}", e); + s3_error!(InvalidArgument, "get entries err") + })?; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + let prefix = req + .headers + .get("X-Amz-Meta-RustFs-Snowball-Prefix") + .map(|v| v.to_str().unwrap_or_default()) + .unwrap_or_default(); + + while let Some(entry) = entries.next().await { + let f = match entry { + Ok(f) => f, + Err(e) => { + println!("Error reading entry: {}", e); + return Err(s3_error!(InvalidArgument, "Error reading entry {:?}", e)); + } + }; + + if f.header().entry_type().is_dir() { + continue; + } + + if let Ok(fpath) = f.path() { + let mut fpath = fpath.to_string_lossy().to_string(); + + if !prefix.is_empty() { + fpath = format!("{}/{}", prefix, fpath); + } + + let size = f.header().size().unwrap_or_default() as usize; + + println!("Extracted: {}, size {}", fpath, size); + + let mut reader = PutObjReader::new(Box::new(f), size); + + let _obj_info = store + .put_object(&bucket, &fpath, &mut reader, &ObjectOptions::default()) + .await + .map_err(to_s3_error)?; + + // let e_tag = obj_info.etag; + + // // store.put_object(bucket, object, data, opts); + + // let output = PutObjectOutput { + // e_tag, + // ..Default::default() + // }; + } + } + + // match decompress( + // body, + // CompressionFormat::from_extension(&ext), + // |entry: tokio_tar::Entry>>| async move { + // let path = entry.path().unwrap(); + // println!("Extracted: {}", path.display()); + // Ok(()) + // }, + // ) + // .await + // { + // Ok(_) => println!("解压成功！"), + // Err(e) => println!("解压失败: {}", e), + // } + + // TODO: etag + let output = PutObjectOutput { + // e_tag: Some(etag_stream.etag().await), + ..Default::default() + }; + Ok(S3Response::new(output)) + } } #[async_trait::async_trait] impl S3 for FS { @@ -409,6 +513,8 @@ impl S3 for FS { .. } = req.input; + // TODO: getObjectInArchiveFileHandler object = xxx.zip/xxx/xxx.xxx + // let range = HTTPRangeSpec::nil(); let h = HeaderMap::new(); @@ -804,8 +910,16 @@ impl S3 for FS { Ok(S3Response::new(output)) } - #[tracing::instrument(level = "debug", skip(self, req))] + // #[tracing::instrument(level = "debug", skip(self, req))] async fn put_object(&self, req: S3Request) -> S3Result> { + if req + .headers + .get("X-Amz-Meta-Snowball-Auto-Extract") + .is_some_and(|v| v.to_str().unwrap_or_default() == "true") + { + return self.put_object_extract(req).await; + } + let input = req.input; if let Some(ref storage_class) = input.storage_class { From 7a94363b389e8cdfebef00a2c54b703250e5e20c Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Thu, 8 May 2025 18:37:28 +0800 Subject: [PATCH 22/38] improve speed Signed-off-by: junxiang Mu <1948535941@qq.com> --- ecstore/src/disk/local.rs | 15 +++++---- ecstore/src/disk/os.rs | 9 ++--- ecstore/src/erasure.rs | 71 --------------------------------------- ecstore/src/file_meta.rs | 6 +++- ecstore/src/set_disk.rs | 57 +++++++++++++++++++++---------- ecstore/src/store.rs | 1 + ecstore/src/utils/fs.rs | 12 +++++++ 7 files changed, 70 insertions(+), 101 deletions(-) diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 45bea2d96..bd10ccf69 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -261,7 +261,7 @@ impl LocalDisk { #[tracing::instrument(level = "debug", skip(self))] async fn check_format_json(&self) -> Result { - let md = fs::metadata(&self.format_path).await.map_err(|e| match e.kind() { + let md = std::fs::metadata(&self.format_path).map_err(|e| match e.kind() { ErrorKind::NotFound => DiskError::DiskNotFound, ErrorKind::PermissionDenied => DiskError::FileAccessDenied, _ => { @@ -367,7 +367,7 @@ impl LocalDisk { Ok(()) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(level = "debug", skip(self))] pub async fn delete_file( &self, base_path: &PathBuf, @@ -690,6 +690,7 @@ impl LocalDisk { } // write_all_private with check_path_length + #[tracing::instrument(level = "debug", skip_all)] pub async fn write_all_private( &self, volume: &str, @@ -1215,7 +1216,7 @@ impl DiskAPI for LocalDisk { Ok(data) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(level = "debug", skip_all)] async fn write_all(&self, volume: &str, path: &str, data: Vec) -> Result<()> { self.write_all_public(volume, path, data).await } @@ -1723,7 +1724,7 @@ impl DiskAPI for LocalDisk { Ok(()) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(level = "debug", skip(self))] async fn rename_data( &self, src_volume: &str, @@ -1734,7 +1735,7 @@ impl DiskAPI for LocalDisk { ) -> Result { let src_volume_dir = self.get_bucket_path(src_volume)?; if !skip_access_checks(src_volume) { - if let Err(e) = utils::fs::access(&src_volume_dir).await { + if let Err(e) = utils::fs::access_std(&src_volume_dir) { info!("access checks failed, src_volume_dir: {:?}, err: {}", src_volume_dir, e.to_string()); return Err(convert_access_error(e, DiskError::VolumeAccessDenied)); } @@ -1742,7 +1743,7 @@ impl DiskAPI for LocalDisk { let dst_volume_dir = self.get_bucket_path(dst_volume)?; if !skip_access_checks(dst_volume) { - if let Err(e) = utils::fs::access(&dst_volume_dir).await { + if let Err(e) = utils::fs::access_std(&dst_volume_dir) { info!("access checks failed, dst_volume_dir: {:?}, err: {}", dst_volume_dir, e.to_string()); return Err(convert_access_error(e, DiskError::VolumeAccessDenied)); } @@ -1915,7 +1916,7 @@ impl DiskAPI for LocalDisk { if let Some(src_file_path_parent) = src_file_path.parent() { if src_volume != super::RUSTFS_META_MULTIPART_BUCKET { - let _ = utils::fs::remove(src_file_path_parent).await; + let _ = utils::fs::remove_std(src_file_path_parent); } else { let _ = self .delete_file(&dst_volume_dir, &src_file_path_parent.to_path_buf(), true, false) diff --git a/ecstore/src/disk/os.rs b/ecstore/src/disk/os.rs index ae88611a8..a6dfb15b2 100644 --- a/ecstore/src/disk/os.rs +++ b/ecstore/src/disk/os.rs @@ -108,6 +108,7 @@ pub async fn read_dir(path: impl AsRef, count: i32) -> Result> Ok(volumes) } +#[tracing::instrument(level = "debug", skip_all)] pub async fn rename_all( src_file_path: impl AsRef, dst_file_path: impl AsRef, @@ -136,7 +137,7 @@ pub async fn reliable_rename( base_dir: impl AsRef, ) -> io::Result<()> { if let Some(parent) = dst_file_path.as_ref().parent() { - if !file_exists(parent).await { + if !file_exists(parent) { info!("reliable_rename reliable_mkdir_all parent: {:?}", parent); reliable_mkdir_all(parent, base_dir.as_ref()).await?; } @@ -144,7 +145,7 @@ pub async fn reliable_rename( let mut i = 0; loop { - if let Err(e) = utils::fs::rename(src_file_path.as_ref(), dst_file_path.as_ref()).await { + if let Err(e) = utils::fs::rename_std(src_file_path.as_ref(), dst_file_path.as_ref()) { if os_is_not_exist(&e) && i == 0 { i += 1; continue; @@ -221,6 +222,6 @@ pub async fn os_mkdir_all(dir_path: impl AsRef, base_dir: impl AsRef Ok(()) } -pub async fn file_exists(path: impl AsRef) -> bool { - fs::metadata(path.as_ref()).await.map(|_| true).unwrap_or(false) +pub fn file_exists(path: impl AsRef) -> bool { + std::fs::metadata(path.as_ref()).map(|_| true).unwrap_or(false) } diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs index 942f461bd..ac0eee897 100644 --- a/ecstore/src/erasure.rs +++ b/ecstore/src/erasure.rs @@ -135,77 +135,6 @@ impl Erasure { } } task.await? - - // // let stream = ChunkedStream::new(body, self.block_size); - // let stream = ChunkedStream::new(body, total_size, self.block_size, false); - // let mut total: usize = 0; - // // let mut idx = 0; - // pin_mut!(stream); - - // // warn!("encode start..."); - - // loop { - // match stream.next().await { - // Some(result) => match result { - // Ok(data) => { - // total += data.len(); - - // // EOF - // if data.is_empty() { - // break; - // } - - // // idx += 1; - // // warn!("encode {} get data {:?}", data.len(), data.to_vec()); - - // let blocks = self.encode_data(data.as_ref())?; - - // // warn!( - // // "encode shard size: {}/{} from block_size {}, total_size {} ", - // // blocks[0].len(), - // // blocks.len(), - // // data.len(), - // // total_size - // // ); - - // let mut errs = Vec::new(); - - // for (i, w_op) in writers.iter_mut().enumerate() { - // if let Some(w) = w_op { - // match w.write(blocks[i].as_ref()).await { - // Ok(_) => errs.push(None), - // Err(e) => errs.push(Some(e)), - // } - // } else { - // errs.push(Some(Error::new(DiskError::DiskNotFound))); - // } - // } - - // let none_count = errs.iter().filter(|&x| x.is_none()).count(); - // if none_count >= write_quorum { - // continue; - // } - - // if let Some(err) = reduce_write_quorum_errs(&errs, object_op_ignored_errs().as_ref(), write_quorum) { - // warn!("Erasure encode errs {:?}", &errs); - // return Err(err); - // } - // } - // Err(e) => { - // warn!("poll result err {:?}", &e); - // return Err(Error::msg(e.to_string())); - // } - // }, - // None => { - // // warn!("poll empty result"); - // break; - // } - // } - // } - - // let _ = close_bitrot_writers(writers).await?; - - // Ok(total) } pub async fn decode( diff --git a/ecstore/src/file_meta.rs b/ecstore/src/file_meta.rs index e4da3882a..e374e456c 100644 --- a/ecstore/src/file_meta.rs +++ b/ecstore/src/file_meta.rs @@ -58,10 +58,12 @@ impl FileMeta { } // isXL2V1Format + #[tracing::instrument(level = "debug", skip_all)] pub fn is_xl2_v1_format(buf: &[u8]) -> bool { !matches!(Self::check_xl2_v1(buf), Err(_e)) } + #[tracing::instrument(level = "debug", skip_all)] pub fn load(buf: &[u8]) -> Result { let mut xl = FileMeta::default(); xl.unmarshal_msg(buf)?; @@ -245,7 +247,7 @@ impl FileMeta { } } - #[tracing::instrument] + #[tracing::instrument(level = "debug", skip_all)] pub fn marshal_msg(&self) -> Result> { let mut wr = Vec::new(); @@ -363,6 +365,7 @@ impl FileMeta { } // shard_data_dir_count 查询 vid下data_dir的数量 + #[tracing::instrument(level = "debug", skip_all)] pub fn shard_data_dir_count(&self, vid: &Option, data_dir: &Option) -> usize { self.versions .iter() @@ -434,6 +437,7 @@ impl FileMeta { } // 添加版本 + #[tracing::instrument(level = "debug", skip_all)] pub fn add_version(&mut self, fi: FileInfo) -> Result<()> { let vid = fi.version_id; diff --git a/ecstore/src/set_disk.rs b/ecstore/src/set_disk.rs index abf7422ac..6e7387834 100644 --- a/ecstore/src/set_disk.rs +++ b/ecstore/src/set_disk.rs @@ -284,10 +284,20 @@ impl SetDisks { // let mut ress = Vec::with_capacity(disks.len()); let mut errs = Vec::with_capacity(disks.len()); - for (i, disk) in disks.iter().enumerate() { - let mut file_info = file_infos[i].clone(); + let src_bucket = Arc::new(src_bucket.to_string()); + let src_object = Arc::new(src_object.to_string()); + let dst_bucket = Arc::new(dst_bucket.to_string()); + let dst_object = Arc::new(dst_object.to_string()); - futures.push(async move { + for (i, (disk, file_info)) in disks.iter().zip(file_infos.iter()).enumerate() { + let mut file_info = file_info.clone(); + let disk = disk.clone(); + let src_bucket = src_bucket.clone(); + let src_object = src_object.clone(); + let dst_object = dst_object.clone(); + let dst_bucket = dst_bucket.clone(); + + futures.push(tokio::spawn(async move { if file_info.erasure.index == 0 { file_info.erasure.index = i + 1; } @@ -297,12 +307,12 @@ impl SetDisks { } if let Some(disk) = disk { - disk.rename_data(src_bucket, src_object, file_info, dst_bucket, dst_object) + disk.rename_data(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object) .await } else { Err(Error::new(DiskError::DiskNotFound)) } - }) + })); } let mut disk_versions = vec![None; disks.len()]; @@ -311,15 +321,13 @@ impl SetDisks { let results = join_all(futures).await; for (idx, result) in results.iter().enumerate() { - match result { + match result.as_ref().map_err(|_| Error::new(DiskError::Unexpected))? { Ok(res) => { data_dirs[idx] = res.old_data_dir; disk_versions[idx].clone_from(&res.sign); - // ress.push(Some(res)); errs.push(None); } Err(e) => { - // ress.push(None); errs.push(Some(clone_err(e))); } } @@ -336,11 +344,14 @@ impl SetDisks { if let Some(disk) = disks[i].as_ref() { let fi = file_infos[i].clone(); let old_data_dir = data_dirs[i]; - futures.push(async move { + let disk = disk.clone(); + let src_bucket = src_bucket.clone(); + let src_object = src_object.clone(); + futures.push(tokio::spawn(async move { let _ = disk .delete_version( - src_bucket, - src_object, + &src_bucket, + &src_object, fi, false, DeleteOptions { @@ -354,7 +365,7 @@ impl SetDisks { debug!("rename_data delete_version err {:?}", e); e }); - }); + })); } } @@ -407,7 +418,7 @@ impl SetDisks { } #[allow(dead_code)] - #[tracing::instrument(level = "info", skip(self, disks))] + #[tracing::instrument(level = "debug", skip(self, disks))] async fn commit_rename_data_dir( &self, disks: &[Option], @@ -416,14 +427,17 @@ impl SetDisks { data_dir: &str, write_quorum: usize, ) -> Result<()> { - let file_path = format!("{}/{}", object, data_dir); + let file_path = Arc::new(format!("{}/{}", object, data_dir)); + let bucket = Arc::new(bucket.to_string()); let futures = disks.iter().map(|disk| { let file_path = file_path.clone(); - async move { + let bucket = bucket.clone(); + let disk = disk.clone(); + tokio::spawn(async move { if let Some(disk) = disk { match disk .delete( - bucket, + &bucket, &file_path, DeleteOptions { recursive: true, @@ -438,9 +452,16 @@ impl SetDisks { } else { Some(Error::new(DiskError::DiskNotFound)) } - } + }) }); - let errs: Vec> = join_all(futures).await; + let errs: Vec> = join_all(futures) + .await + .into_iter() + .map(|e| match e { + Ok(e) => e, + Err(_) => Some(Error::new(DiskError::Unexpected)), + }) + .collect(); if let Some(err) = reduce_write_quorum_errs(&errs, object_op_ignored_errs().as_ref(), write_quorum) { return Err(err); diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index 5a2075775..d480e0ac2 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -2549,6 +2549,7 @@ fn check_abort_multipart_args(bucket: &str, object: &str, upload_id: &str) -> Re check_multipart_object_args(bucket, object, upload_id) } +#[tracing::instrument(level = "debug")] fn check_put_object_args(bucket: &str, object: &str) -> Result<()> { if !is_meta_bucketname(bucket) && check_valid_bucket_name_strict(bucket).is_err() { return Err(Error::new(StorageError::BucketNameInvalid(bucket.to_string()))); diff --git a/ecstore/src/utils/fs.rs b/ecstore/src/utils/fs.rs index 9c0c591f1..f60e28f08 100644 --- a/ecstore/src/utils/fs.rs +++ b/ecstore/src/utils/fs.rs @@ -106,6 +106,11 @@ pub async fn access(path: impl AsRef) -> io::Result<()> { Ok(()) } +pub fn access_std(path: impl AsRef) -> io::Result<()> { + std::fs::metadata(path)?; + Ok(()) +} + pub async fn lstat(path: impl AsRef) -> io::Result { fs::metadata(path).await } @@ -114,6 +119,7 @@ pub async fn make_dir_all(path: impl AsRef) -> io::Result<()> { fs::create_dir_all(path.as_ref()).await } +#[tracing::instrument(level = "debug", skip_all)] pub async fn remove(path: impl AsRef) -> io::Result<()> { let meta = fs::metadata(path.as_ref()).await?; if meta.is_dir() { @@ -132,6 +138,7 @@ pub async fn remove_all(path: impl AsRef) -> io::Result<()> { } } +#[tracing::instrument(level = "debug", skip_all)] pub fn remove_std(path: impl AsRef) -> io::Result<()> { let meta = std::fs::metadata(path.as_ref())?; if meta.is_dir() { @@ -158,6 +165,11 @@ pub async fn rename(from: impl AsRef, to: impl AsRef) -> io::Result< fs::rename(from, to).await } +pub fn rename_std(from: impl AsRef, to: impl AsRef) -> io::Result<()> { + std::fs::rename(from, to) +} + +#[tracing::instrument(level = "debug", skip_all)] pub async fn read_file(path: impl AsRef) -> io::Result> { fs::read(path.as_ref()).await } From 5cb040f86362e58404a4ee56662b8e6d2d5b1cb1 Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Thu, 8 May 2025 19:34:58 +0800 Subject: [PATCH 23/38] fix zero size object bug Signed-off-by: junxiang Mu <1948535941@qq.com> --- crates/utils/src/net.rs | 1 + ecstore/src/erasure.rs | 9 ++++----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/utils/src/net.rs b/crates/utils/src/net.rs index e69de29bb..8b1378917 100644 --- a/crates/utils/src/net.rs +++ b/crates/utils/src/net.rs @@ -0,0 +1 @@ + diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs index ac0eee897..702d393ee 100644 --- a/ecstore/src/erasure.rs +++ b/ecstore/src/erasure.rs @@ -67,7 +67,7 @@ impl Erasure { { let (tx, mut rx) = mpsc::channel(5); let task = tokio::spawn(async move { - let mut buf = Vec::new(); + let mut buf = vec![0u8; self.block_size]; let mut total: usize = 0; loop { if total_size > 0 { @@ -99,6 +99,9 @@ impl Erasure { } let blocks = Arc::new(Box::pin(self.clone().encode_data(&buf)?)); let _ = tx.send(blocks).await; + if total_size == 0 { + break; + } } let etag = reader.etag().await; Ok((total, etag)) @@ -129,10 +132,6 @@ impl Erasure { warn!("Erasure encode errs {:?}", &errs); return Err(err); } - - if total_size == 0 { - break; - } } task.await? } From fd03ba54f3dc0d7664b57c896e8e949cd0fd16c4 Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Fri, 9 May 2025 17:13:25 +0800 Subject: [PATCH 24/38] improve multi put speed Signed-off-by: junxiang Mu <1948535941@qq.com> --- ecstore/src/disk/local.rs | 10 +-- ecstore/src/set_disk.rs | 127 +++++++++++++++++++++++--------------- ecstore/src/utils/fs.rs | 4 ++ 3 files changed, 87 insertions(+), 54 deletions(-) diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index bd10ccf69..a814baca3 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -39,7 +39,7 @@ use crate::set_disk::{ }; use crate::store_api::{BitrotAlgorithm, StorageAPI}; use crate::utils::fs::{ - access, lstat, remove, remove_all, remove_all_std, remove_std, rename, O_APPEND, O_CREATE, O_RDONLY, O_WRONLY, + access, lstat, lstat_std, remove, remove_all, remove_all_std, remove_std, rename, O_APPEND, O_CREATE, O_RDONLY, O_WRONLY, }; use crate::utils::os::get_info; use crate::utils::path::{ @@ -1337,10 +1337,10 @@ impl DiskAPI for LocalDisk { let src_volume_dir = self.get_bucket_path(src_volume)?; let dst_volume_dir = self.get_bucket_path(dst_volume)?; if !skip_access_checks(src_volume) { - utils::fs::access(&src_volume_dir).await.map_err(map_err_not_exists)? + utils::fs::access_std(&src_volume_dir).map_err(map_err_not_exists)? } if !skip_access_checks(dst_volume) { - utils::fs::access(&dst_volume_dir).await.map_err(map_err_not_exists)? + utils::fs::access_std(&dst_volume_dir).map_err(map_err_not_exists)? } let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR); @@ -1363,7 +1363,7 @@ impl DiskAPI for LocalDisk { check_path_length(dst_file_path.to_string_lossy().as_ref())?; if src_is_dir { - let meta_op = match lstat(&src_file_path).await { + let meta_op = match lstat_std(&src_file_path) { Ok(meta) => Some(meta), Err(e) => { if is_sys_err_io(&e) { @@ -1384,7 +1384,7 @@ impl DiskAPI for LocalDisk { } } - if let Err(e) = utils::fs::remove(&dst_file_path).await { + if let Err(e) = utils::fs::remove_std(&dst_file_path) { if is_sys_err_not_empty(&e) || is_sys_err_not_dir(&e) { warn!("rename_part remove dst failed {:?} err {:?}", &dst_file_path, e); return Err(Error::new(DiskError::FileAccessDenied)); diff --git a/ecstore/src/set_disk.rs b/ecstore/src/set_disk.rs index 6e7387834..38dbd9cf5 100644 --- a/ecstore/src/set_disk.rs +++ b/ecstore/src/set_disk.rs @@ -513,24 +513,33 @@ impl SetDisks { meta: Vec, write_quorum: usize, ) -> Result>> { - let mut futures = Vec::with_capacity(disks.len()); + let src_bucket = Arc::new(src_bucket.to_string()); + let src_object = Arc::new(src_object.to_string()); + let dst_bucket = Arc::new(dst_bucket.to_string()); + let dst_object = Arc::new(dst_object.to_string()); let mut errs = Vec::with_capacity(disks.len()); - for disk in disks.iter() { + let futures = disks.iter().map(|disk| { + let disk = disk.clone(); let meta = meta.clone(); - futures.push(async move { + let src_bucket = src_bucket.clone(); + let src_object = src_object.clone(); + let dst_bucket = dst_bucket.clone(); + let dst_object = dst_object.clone(); + tokio::spawn(async move { if let Some(disk) = disk { - disk.rename_part(src_bucket, src_object, dst_bucket, dst_object, meta).await + disk.rename_part(&src_bucket, &src_object, &dst_bucket, &dst_object, meta) + .await } else { Err(Error::new(DiskError::DiskNotFound)) } }) - } + }); let results = join_all(futures).await; for result in results { - match result { + match result? { Ok(_) => { errs.push(None); } @@ -542,7 +551,7 @@ impl SetDisks { if let Some(err) = reduce_write_quorum_errs(&errs, object_op_ignored_errs().as_ref(), write_quorum) { warn!("rename_part errs {:?}", &errs); - Self::cleanup_multipart_path(disks, &[dst_object.to_owned(), format!("{}.meta", dst_object)]).await; + Self::cleanup_multipart_path(disks, &[dst_object.to_string(), format!("{}.meta", dst_object)]).await; return Err(err); } @@ -944,7 +953,7 @@ impl SetDisks { let disks = disks.clone(); let (parts_metadata, errs) = - Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_id_path, "", false, false).await; + Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_id_path, "", false, false).await?; let map_err_notfound = |err: Error| { if is_err_object_not_found(&err) { @@ -1114,39 +1123,47 @@ impl SetDisks { version_id: &str, read_data: bool, healing: bool, - ) -> (Vec, Vec>) { - let mut futures = Vec::with_capacity(disks.len()); + ) -> Result<(Vec, Vec>)> { let mut ress = Vec::with_capacity(disks.len()); let mut errors = Vec::with_capacity(disks.len()); - - for disk in disks.iter() { - let opts = ReadOptions { - read_data, - healing, - ..Default::default() - }; - futures.push(async move { + let opts = Arc::new(ReadOptions { + read_data, + healing, + ..Default::default() + }); + let org_bucket = Arc::new(org_bucket.to_string()); + let bucket = Arc::new(bucket.to_string()); + let object = Arc::new(object.to_string()); + let version_id = Arc::new(version_id.to_string()); + let futures = disks.iter().map(|disk| { + let disk = disk.clone(); + let opts = opts.clone(); + let org_bucket = org_bucket.clone(); + let bucket = bucket.clone(); + let object = object.clone(); + let version_id = version_id.clone(); + tokio::spawn(async move { if let Some(disk) = disk { if version_id.is_empty() { - match disk.read_xl(bucket, object, read_data).await { + match disk.read_xl(&bucket, &object, read_data).await { Ok(info) => { - let fi = file_info_from_raw(info, bucket, object, read_data).await?; + let fi = file_info_from_raw(info, &bucket, &object, read_data).await?; Ok(fi) } Err(err) => Err(err), } } else { - disk.read_version(org_bucket, bucket, object, version_id, &opts).await + disk.read_version(&org_bucket, &bucket, &object, &version_id, &opts).await } } else { Err(Error::new(DiskError::DiskNotFound)) } }) - } + }); let results = join_all(futures).await; for result in results { - match result { + match result? { Ok(res) => { ress.push(res); errors.push(None); @@ -1157,7 +1174,7 @@ impl SetDisks { } } } - (ress, errors) + Ok((ress, errors)) } async fn read_all_xl( @@ -1770,7 +1787,7 @@ impl SetDisks { let vid = opts.version_id.clone().unwrap_or_default(); // TODO: 优化并发 可用数量中断 - let (parts_metadata, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, vid.as_str(), read_data, false).await; + let (parts_metadata, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, vid.as_str(), read_data, false).await?; // warn!("get_object_fileinfo parts_metadata {:?}", &parts_metadata); // warn!("get_object_fileinfo {}/{} errs {:?}", bucket, object, &errs); @@ -2177,7 +2194,7 @@ impl SetDisks { let disks = { self.disks.read().await.clone() }; - let (mut parts_metadata, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, true, true).await; + let (mut parts_metadata, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, true, true).await?; if is_all_not_found(&errs) { warn!( "heal_object failed, all obj part not found, bucket: {}, obj: {}, version_id: {}", @@ -3958,7 +3975,7 @@ impl StorageAPI for SetDisks { let (mut metas, errs) = { if let Some(vid) = &src_opts.version_id { - Self::read_all_fileinfo(&disks, "", src_bucket, src_object, vid, true, false).await + Self::read_all_fileinfo(&disks, "", src_bucket, src_object, vid, true, false).await? } else { Self::read_all_xl(&disks, src_bucket, src_object, true, false).await } @@ -4252,7 +4269,7 @@ impl StorageAPI for SetDisks { false, false, ) - .await + .await? } else { Self::read_all_xl(&disks, bucket, object, false, false).await } @@ -4392,30 +4409,42 @@ impl StorageAPI for SetDisks { let part_suffix = format!("part.{}", part_id); let tmp_part = format!("{}x{}", Uuid::new_v4(), OffsetDateTime::now_utc().unix_timestamp()); - let tmp_part_path = format!("{}/{}", tmp_part, part_suffix); + let tmp_part_path = Arc::new(format!("{}/{}", tmp_part, part_suffix)); let mut writers = Vec::with_capacity(disks.len()); let erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size); + let shared_size = erasure.shard_size(erasure.block_size); - for disk in disks.iter() { - if let Some(disk) = disk { - // let writer = disk.append_file(RUSTFS_META_TMP_BUCKET, &tmp_part_path).await?; - // let filewriter = disk - // .create_file("", RUSTFS_META_TMP_BUCKET, &tmp_part_path, data.content_length) - // .await?; - let writer = new_bitrot_filewriter( - disk.clone(), - RUSTFS_META_TMP_BUCKET, - &tmp_part_path, - false, - DEFAULT_BITROT_ALGO, - erasure.shard_size(erasure.block_size), - ) - .await?; - writers.push(Some(writer)); - } else { - writers.push(None); - } + let futures = disks.iter().map(|disk| { + let disk = disk.clone(); + let tmp_part_path = tmp_part_path.clone(); + tokio::spawn(async move { + if let Some(disk) = disk { + // let writer = disk.append_file(RUSTFS_META_TMP_BUCKET, &tmp_part_path).await?; + // let filewriter = disk + // .create_file("", RUSTFS_META_TMP_BUCKET, &tmp_part_path, data.content_length) + // .await?; + match new_bitrot_filewriter( + disk.clone(), + RUSTFS_META_TMP_BUCKET, + &tmp_part_path, + false, + DEFAULT_BITROT_ALGO, + shared_size, + ) + .await + { + Ok(writer) => Ok(Some(writer)), + Err(e) => Err(e), + } + } else { + Ok(None) + } + }) + }); + for x in join_all(futures).await { + let x = x??; + writers.push(x); } let erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size); @@ -5044,7 +5073,7 @@ impl StorageAPI for SetDisks { let disks = self.disks.read().await; let disks = disks.clone(); - let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, false, false).await; + let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, false, false).await?; if is_all_not_found(&errs) { warn!( "heal_object failed, all obj part not found, bucket: {}, obj: {}, version_id: {}", diff --git a/ecstore/src/utils/fs.rs b/ecstore/src/utils/fs.rs index f60e28f08..d8110ca63 100644 --- a/ecstore/src/utils/fs.rs +++ b/ecstore/src/utils/fs.rs @@ -115,6 +115,10 @@ pub async fn lstat(path: impl AsRef) -> io::Result { fs::metadata(path).await } +pub fn lstat_std(path: impl AsRef) -> io::Result { + std::fs::metadata(path) +} + pub async fn make_dir_all(path: impl AsRef) -> io::Result<()> { fs::create_dir_all(path.as_ref()).await } From db100a6db99157064a1da15b19eaafdb5852313a Mon Sep 17 00:00:00 2001 From: weisd Date: Fri, 9 May 2025 22:57:09 +0800 Subject: [PATCH 25/38] rm log --- ecstore/src/disk/os.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecstore/src/disk/os.rs b/ecstore/src/disk/os.rs index df539da40..41056a45a 100644 --- a/ecstore/src/disk/os.rs +++ b/ecstore/src/disk/os.rs @@ -137,7 +137,7 @@ pub async fn reliable_rename( ) -> io::Result<()> { if let Some(parent) = dst_file_path.as_ref().parent() { if !file_exists(parent) { - info!("reliable_rename reliable_mkdir_all parent: {:?}", parent); + // info!("reliable_rename reliable_mkdir_all parent: {:?}", parent); reliable_mkdir_all(parent, base_dir.as_ref()).await?; } } From 2a9a60197be46a9e02fd79c98e417d0019a51640 Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 10 May 2025 00:15:05 +0800 Subject: [PATCH 26/38] # Add aarch64-apple-darwin Build Target Support Added ARM64 macOS (Apple Silicon) build target support to the CI/CD pipeline by: 1. Including `aarch64-apple-darwin` as a new build variant in the build matrix 2. Adding proper exclusion rules to ensure the target only runs on macOS runners 3. Ensuring compatibility with the existing build scripts and packaging process This change enables native builds for Apple Silicon Macs, improving performance for users with M1/M2/M3/M4 processors while maintaining the same artifact organization and deployment process. --- .github/actions/setup/action.yml | 8 ++++++++ .github/workflows/build.yml | 14 +++++++++++--- scripts/build.py | 2 +- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index f74dc6fbe..73861f253 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -6,17 +6,25 @@ inputs: rust-version: required: true default: "stable" + description: "Rust version to use" cache-shared-key: required: true default: "" + description: "Cache key for shared cache" cache-save-if: required: true default: true + description: "Cache save condition" + run-os: + required: true + default: "ubuntu-latest" + description: "Running system" runs: using: "composite" steps: - name: Install system dependencies + if: inputs.run-os == 'ubuntu-latest' shell: bash run: | sudo apt update diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 01e49f6c5..6e75fe72f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,21 +11,29 @@ on: jobs: build-rustfs: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: matrix: + os: [ ubuntu-latest, macos-latest ] variant: - # - { profile: dev, target: x86_64-unknown-linux-musl, glibc: "default" } - { profile: release, target: x86_64-unknown-linux-musl, glibc: "default" } - { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" } - # - { profile: release, target: x86_64-unknown-linux-gnu, glibc: "2.31" } + - { profile: release, target: aarch64-apple-darwin, glibc: "default" } + exclude: + - os: macos-latest + variant: { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" } + - os: ubuntu-latest + variant: { profile: release, target: aarch64-apple-darwin, glibc: "default" } + - os: macos-latest + variant: { profile: release, target: x86_64-unknown-linux-musl, glibc: "default" } steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup with: cache-shared-key: rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.${{ matrix.variant.glibc }} + run-os: ${{ matrix.os }} - name: Download and Extract Static Assets run: | diff --git a/scripts/build.py b/scripts/build.py index 106a25fed..f1beb0789 100755 --- a/scripts/build.py +++ b/scripts/build.py @@ -34,7 +34,7 @@ def main(args: CliArgs): use_zigbuild = True use_old_glibc = True - if args.target and args.target == "x86_64-unknown-linux-musl": + if args.target and args.target != "x86_64-unknown-linux-gnu": shell("rustup target add " + args.target) cmd = ["cargo", "build"] From b1358d47116d9d22eedab0b3b03d49f194a5af9d Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 10 May 2025 00:24:14 +0800 Subject: [PATCH 27/38] # Expand ARM64 Linux Support in Build Pipeline Added support for both ARM64 Linux variants to the CI/CD build pipeline: 1. Enabled the previously commented `aarch64-unknown-linux-gnu` target build 2. Re-enabled the `aarch64-unknown-linux-musl` target build 3. Updated the build matrix to ensure proper runner selection: - Ubuntu runners build all Linux targets - macOS runners build only Apple Silicon targets 4. Maintained compatibility with the existing build scripts and packaging process This expansion gives users more options for deploying on ARM64 Linux platforms, supporting both glibc and musl libc environments for maximum compatibility and performance. --- .github/workflows/build.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6e75fe72f..b05bf7915 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,16 +17,22 @@ jobs: matrix: os: [ ubuntu-latest, macos-latest ] variant: - - { profile: release, target: x86_64-unknown-linux-musl, glibc: "default" } + #- { profile: release, target: x86_64-unknown-linux-musl, glibc: "default" } - { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" } - { profile: release, target: aarch64-apple-darwin, glibc: "default" } + - { profile: release, target: aarch64-unknown-linux-gnu, glibc: "default" } + #- { profile: release, target: aarch64-unknown-linux-musl, glibc: "default" } exclude: + - os: ubuntu-latest + variant: { profile: release, target: x86_64-apple-darwin, glibc: "default" } - os: macos-latest variant: { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" } - - os: ubuntu-latest - variant: { profile: release, target: aarch64-apple-darwin, glibc: "default" } - os: macos-latest variant: { profile: release, target: x86_64-unknown-linux-musl, glibc: "default" } + - os: macos-latest + variant: { profile: release, target: aarch64-unknown-linux-musl, glibc: "default" } + - os: macos-latest + variant: { profile: release, target: aarch64-unknown-linux-gnu, glibc: "default" } steps: - uses: actions/checkout@v4 From f64458018b6426c3fa4f4613d79a6d141444321b Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 10 May 2025 00:27:54 +0800 Subject: [PATCH 28/38] fix --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b05bf7915..8afe88681 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,7 +20,7 @@ jobs: #- { profile: release, target: x86_64-unknown-linux-musl, glibc: "default" } - { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" } - { profile: release, target: aarch64-apple-darwin, glibc: "default" } - - { profile: release, target: aarch64-unknown-linux-gnu, glibc: "default" } + #- { profile: release, target: aarch64-unknown-linux-gnu, glibc: "default" } #- { profile: release, target: aarch64-unknown-linux-musl, glibc: "default" } exclude: - os: ubuntu-latest From a7115ba699379597efc3376a2c3489f6a0763652 Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 10 May 2025 00:30:01 +0800 Subject: [PATCH 29/38] test --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8afe88681..6b22ece1f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,12 +19,12 @@ jobs: variant: #- { profile: release, target: x86_64-unknown-linux-musl, glibc: "default" } - { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" } - - { profile: release, target: aarch64-apple-darwin, glibc: "default" } + #- { profile: release, target: aarch64-apple-darwin, glibc: "default" } #- { profile: release, target: aarch64-unknown-linux-gnu, glibc: "default" } #- { profile: release, target: aarch64-unknown-linux-musl, glibc: "default" } exclude: - os: ubuntu-latest - variant: { profile: release, target: x86_64-apple-darwin, glibc: "default" } + variant: { profile: release, target: aarch64-apple-darwin, glibc: "default" } - os: macos-latest variant: { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" } - os: macos-latest From 43f2963d1400b193875c992f1a060eae364749ab Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 10 May 2025 00:39:08 +0800 Subject: [PATCH 30/38] test target `aarch64-apple-darwin` --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6b22ece1f..b2532f33f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,7 +19,7 @@ jobs: variant: #- { profile: release, target: x86_64-unknown-linux-musl, glibc: "default" } - { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" } - #- { profile: release, target: aarch64-apple-darwin, glibc: "default" } + - { profile: release, target: aarch64-apple-darwin, glibc: "default" } #- { profile: release, target: aarch64-unknown-linux-gnu, glibc: "default" } #- { profile: release, target: aarch64-unknown-linux-musl, glibc: "default" } exclude: From f8f58a83558dc7340f010f81e8b5b4cfcb1029d1 Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 10 May 2025 00:52:41 +0800 Subject: [PATCH 31/38] test --- .github/actions/setup/action.yml | 2 +- .github/workflows/build.yml | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 73861f253..345eec131 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -50,5 +50,5 @@ runs: shared-key: ${{ inputs.cache-shared-key }} save-if: ${{ inputs.cache-save-if }} - - uses: mlugg/setup-zig@v1 + - uses: mlugg/setup-zig@v2 - uses: taiki-e/install-action@cargo-zigbuild diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b2532f33f..fa552ebea 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,6 +36,11 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Set up authentication + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "Authenticated with GITHUB_TOKEN" - uses: ./.github/actions/setup with: cache-shared-key: rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.${{ matrix.variant.glibc }} @@ -133,6 +138,7 @@ jobs: with: tool: dioxus-cli - name: Build and Bundle rustfs-gui + id: package run: | ls -la From 780ff5f5df9734a2081617395dffa69519164584 Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 10 May 2025 09:46:49 +0800 Subject: [PATCH 32/38] add `x86_64-unknown-linux-musl` target --- .github/workflows/build.yml | 46 ++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fa552ebea..85ca40b55 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,7 +17,7 @@ jobs: matrix: os: [ ubuntu-latest, macos-latest ] variant: - #- { profile: release, target: x86_64-unknown-linux-musl, glibc: "default" } + - { profile: release, target: x86_64-unknown-linux-musl, glibc: "default" } - { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" } - { profile: release, target: aarch64-apple-darwin, glibc: "default" } #- { profile: release, target: aarch64-unknown-linux-gnu, glibc: "default" } @@ -35,7 +35,7 @@ jobs: variant: { profile: release, target: aarch64-unknown-linux-gnu, glibc: "default" } steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4.2.2 - name: Set up authentication env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -101,17 +101,35 @@ jobs: ${{ steps.package.outputs.artifact_name }}.zip:/artifacts/rustfs/${{ steps.package.outputs.artifact_name }}.latest.zip build-rustfs-gui: - runs-on: ubuntu-latest + runs-on: ${ matrix.os } needs: build-rustfs strategy: matrix: + os: [ ubuntu-latest, macos-latest ] variant: - { profile: release, target: x86_64-unknown-linux-gnu } + - { profile: release, target: x86_64-unknown-linux-musl } + - { profile: release, target: aarch64-apple-darwin } # - { profile: release, target: x86_64-apple-darwin } + # - { profile: release, target: aarch64-unknown-linux-gnu } + # - { profile: release, target: aarch64-unknown-linux-musl } + exclude: + - os: ubuntu-latest + variant: { profile: release, target: aarch64-apple-darwin } + - os: ubuntu-latest + variant: { profile: release, target: x86_64-apple-darwin } + - os: macos-latest + variant: { profile: release, target: x86_64-unknown-linux-gnu } + - os: macos-latest + variant: { profile: release, target: x86_64-unknown-linux-musl } + - os: macos-latest + variant: { profile: release, target: aarch64-unknown-linux-musl } + - os: macos-latest + variant: { profile: release, target: aarch64-unknown-linux-gnu } if: startsWith(github.ref, 'refs/tags/') steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4.2.2 - name: Download artifact uses: actions/download-artifact@v4 with: @@ -121,24 +139,10 @@ jobs: ls -R unzip -o -j "rustfs-${{ matrix.variant.profile }}-${{ matrix.variant.target }}.zip" -d ./cli/rustfs-gui/embedded-rustfs/ ls -la cli/rustfs-gui/embedded-rustfs - # - name: Cache dioxus-cli - # uses: actions/cache@v4 - # with: - # path: ~/.cargo/bin/dx - # key: ${{ runner.os }}-dioxus-cli-${{ hashFiles('**/Cargo.lock') }} - # restore-keys: | - # ${{ runner.os }}-dioxus-cli- - # - # - name: Install dioxus-cli - # run: | - # if [ ! -f ~/.cargo/bin/dx ]; then - # cargo install dioxus-cli - # fi - uses: taiki-e/cache-cargo-install-action@v2 with: tool: dioxus-cli - name: Build and Bundle rustfs-gui - id: package run: | ls -la @@ -148,7 +152,7 @@ jobs: ls -la embedded-rustfs # Configure the linker based on the target - case "${{ matrix.target }}" in + case "${{ matrix.variant.target }}" in "x86_64-unknown-linux-gnu") # Default gcc export CC_x86_64_unknown_linux_gnu=gcc @@ -171,8 +175,8 @@ jobs: ;; esac # Validating Environment Variables (for Debugging) - echo "CC for ${{ matrix.target }}: $CC_${{ matrix.target }}" - echo "Linker for ${{ matrix.target }}: $CARGO_TARGET_${{ matrix.target }}_LINKER" + echo "CC for ${{ matrix.variant.target }}: $CC_${{ matrix.variant.target }}" + echo "Linker for ${{ matrix.variant.target }}: $CARGO_TARGET_${{ matrix.variant.target }}_LINKER" if [[ "${{ matrix.variant.target }}" == *"apple-darwin"* ]]; then dx bundle --platform macos --package-types "macos" --package-types "dmg" --package-types "ios" --release --profile release --out-dir ../../${release_path} From f6f1f3b329f5416d185d382b9266d44b97f89266 Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 10 May 2025 10:05:15 +0800 Subject: [PATCH 33/38] add GH_TOKEN --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 85ca40b55..ba37e2cbd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,6 +45,8 @@ jobs: with: cache-shared-key: rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.${{ matrix.variant.glibc }} run-os: ${{ matrix.os }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Download and Extract Static Assets run: | From 33cd4c546acec776d308e919117b6bf18ee5a3e6 Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 11 May 2025 23:41:09 +0800 Subject: [PATCH 34/38] refactor(ci): optimize build workflow for better efficiency - Integrate GUI build steps into main build-rustfs job - Add conditional GUI build execution based on tag releases - Simplify workflow by removing redundant build-rustfs-gui job - Copy binary directly to embedded-rustfs directory without downloading artifacts - Update merge job dependency to only rely on build-rustfs - Improve cross-platform compatibility for Windows binary naming (.exe) - Streamline artifact uploading and OSS publishing process - Maintain consistent conditional logic for release operations --- .github/workflows/build.yml | 431 ++++++++++++++++++++++++++---------- LICENSE | 0 2 files changed, 311 insertions(+), 120 deletions(-) create mode 100644 LICENSE diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ba37e2cbd..bd2095a17 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,7 +3,7 @@ name: Build on: workflow_dispatch: schedule: - - cron: "0 0 * * 0" # at midnight of each sunday + - cron: "0 0 * * 0" # 每周日午夜执行 push: branches: - main @@ -12,86 +12,256 @@ on: jobs: build-rustfs: runs-on: ${{ matrix.os }} - strategy: + fail-fast: false matrix: - os: [ ubuntu-latest, macos-latest ] + os: [ ubuntu-latest, macos-latest, windows-latest ] variant: - { profile: release, target: x86_64-unknown-linux-musl, glibc: "default" } - { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" } - { profile: release, target: aarch64-apple-darwin, glibc: "default" } #- { profile: release, target: aarch64-unknown-linux-gnu, glibc: "default" } #- { profile: release, target: aarch64-unknown-linux-musl, glibc: "default" } + #- { profile: release, target: x86_64-pc-windows-msvc, glibc: "default" } exclude: - - os: ubuntu-latest - variant: { profile: release, target: aarch64-apple-darwin, glibc: "default" } + # Linux targets on non-Linux systems - os: macos-latest variant: { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" } - os: macos-latest variant: { profile: release, target: x86_64-unknown-linux-musl, glibc: "default" } - - os: macos-latest - variant: { profile: release, target: aarch64-unknown-linux-musl, glibc: "default" } - os: macos-latest variant: { profile: release, target: aarch64-unknown-linux-gnu, glibc: "default" } + - os: macos-latest + variant: { profile: release, target: aarch64-unknown-linux-musl, glibc: "default" } + - os: windows-latest + variant: { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" } + - os: windows-latest + variant: { profile: release, target: x86_64-unknown-linux-musl, glibc: "default" } + - os: windows-latest + variant: { profile: release, target: aarch64-unknown-linux-gnu, glibc: "default" } + - os: windows-latest + variant: { profile: release, target: aarch64-unknown-linux-musl, glibc: "default" } + + # Apple targets on non-macOS systems + - os: ubuntu-latest + variant: { profile: release, target: aarch64-apple-darwin, glibc: "default" } + - os: windows-latest + variant: { profile: release, target: aarch64-apple-darwin, glibc: "default" } + + # Windows targets on non-Windows systems + - os: ubuntu-latest + variant: { profile: release, target: x86_64-pc-windows-msvc, glibc: "default" } + - os: macos-latest + variant: { profile: release, target: x86_64-pc-windows-msvc, glibc: "default" } steps: - - uses: actions/checkout@v4.2.2 + - name: Checkout repository + uses: actions/checkout@v4.2.2 + with: + fetch-depth: 0 + - name: Set up authentication env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - echo "Authenticated with GITHUB_TOKEN" - - uses: ./.github/actions/setup - with: - cache-shared-key: rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.${{ matrix.variant.glibc }} - run-os: ${{ matrix.os }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: echo "Authenticated with GITHUB_TOKEN" + shell: bash + # Installation system dependencies + - name: Install system dependencies (Ubuntu) + if: runner.os == 'Linux' + run: | + sudo apt update + sudo apt install -y musl-tools build-essential lld libdbus-1-dev libwayland-dev libwebkit2gtk-4.1-dev libxdo-dev + shell: bash + + #Install Rust using dtolnay/rust-toolchain + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + targets: ${{ matrix.variant.target }} + components: rustfmt, clippy + + # Setting up Protobuf and Flatbuffers + - name: Setup Protoc + uses: arduino/setup-protoc@v3 + with: + version: "30.2" + + - name: Setup Flatc + uses: Nugine/setup-flatc@v1 + with: + version: "25.2.10" + + # Cache Cargo dependencies + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + cache-all-crates: true + shared-key: rustfs-${{ matrix.os }}-${{ matrix.variant.profile }}-${{ matrix.variant.target }}-${{ matrix.variant.glibc }}-${{ hashFiles('**/Cargo.lock') }} + save-if: ${{ github.event_name != 'pull_request' }} + + # Set up Zig for cross-compilation + - uses: mlugg/setup-zig@v2 + if: matrix.variant.glibc != 'default' || contains(matrix.variant.target, 'linux') + + - uses: taiki-e/install-action@cargo-zigbuild + if: matrix.variant.glibc != 'default' || contains(matrix.variant.target, 'linux') + + # Download static resources - name: Download and Extract Static Assets run: | url="https://dl.rustfs.com/artifacts/console/rustfs-console-latest.zip" - mkdir -p static - curl -L -o static_assets.zip "$url" - unzip -o static_assets.zip -d ./rustfs/static - rm static_assets.zip - ls -la ./rustfs/static - - name: Build + # Create a static resource directory + mkdir -p ./rustfs/static + + # Download static resources + echo "::group::Downloading static assets" + curl -L -o static_assets.zip "$url" --retry 3 + + # Unzip static resources + echo "::group::Extracting static assets" + if [ "${{ runner.os }}" = "Windows" ]; then + 7z x static_assets.zip -o./rustfs/static + del static_assets.zip + else + unzip -o static_assets.zip -d ./rustfs/static + rm static_assets.zip + fi + + echo "::group::Static assets content" + ls -la ./rustfs/static + shell: bash + + # Build rustfs + - name: Build rustfs + id: build + shell: bash run: | - ./scripts/build.py \ - --profile ${{ matrix.variant.profile }} \ - --target ${{ matrix.variant.target }} \ - --glibc ${{ matrix.variant.glibc }} + echo "::group::Setting up build parameters" + PROFILE="${{ matrix.variant.profile }}" + TARGET="${{ matrix.variant.target }}" + GLIBC="${{ matrix.variant.glibc }}" + + # Determine whether to use zigbuild + USE_ZIGBUILD=false + if [[ "$GLIBC" != "default" || "$TARGET" == *"linux"* ]]; then + USE_ZIGBUILD=true + echo "Using zigbuild for cross-compilation" + fi + + # Determine the target parameters + TARGET_ARG="$TARGET" + if [[ "$GLIBC" != "default" ]]; then + TARGET_ARG="${TARGET}.${GLIBC}" + echo "Using custom glibc target: $TARGET_ARG" + fi + + # Confirm the profile directory name + if [[ "$PROFILE" == "dev" ]]; then + PROFILE_DIR="debug" + else + PROFILE_DIR="$PROFILE" + fi + + # Determine the binary suffix + BIN_SUFFIX="" + if [[ "${{ matrix.variant.target }}" == *"windows"* ]]; then + BIN_SUFFIX=".exe" + fi + + # Determine the binary name - Use the appropriate extension for Windows + BIN_NAME="rustfs.${PROFILE}.${TARGET}" + if [[ "$GLIBC" != "default" ]]; then + BIN_NAME="${BIN_NAME}.glibc${GLIBC}" + fi + + # Windows systems use exe suffix, and other systems do not have suffix + if [[ "${{ matrix.variant.target }}" == *"windows"* ]]; then + BIN_NAME="${BIN_NAME}.exe" + else + BIN_NAME="${BIN_NAME}.bin" + fi + + echo "Binary name will be: $BIN_NAME" + + echo "::group::Building rustfs" + # Refresh build information + touch rustfs/build.rs + + # Identify the build command and execute it + if [[ "$USE_ZIGBUILD" == "true" ]]; then + echo "Build command: cargo zigbuild --profile $PROFILE --target $TARGET_ARG -p rustfs --bins" + cargo zigbuild --profile $PROFILE --target $TARGET_ARG -p rustfs --bins + else + echo "Build command: cargo build --profile $PROFILE --target $TARGET_ARG -p rustfs --bins" + cargo build --profile $PROFILE --target $TARGET_ARG -p rustfs --bins + fi + + # Determine the binary path and output path + BIN_PATH="target/${TARGET_ARG}/${PROFILE_DIR}/rustfs${BIN_SUFFIX}" + OUT_PATH="target/artifacts/${BIN_NAME}" + + # Create a target directory + mkdir -p target/artifacts + + echo "Copying binary from ${BIN_PATH} to ${OUT_PATH}" + cp "${BIN_PATH}" "${OUT_PATH}" + + # Record the output path for use in the next steps + echo "bin_path=${OUT_PATH}" >> $GITHUB_OUTPUT + echo "bin_name=${BIN_NAME}" >> $GITHUB_OUTPUT - name: Package Binary and Static Assets id: package run: | - # Create artifact filename + # Create component file name ARTIFACT_NAME="rustfs-${{ matrix.variant.profile }}-${{ matrix.variant.target }}" if [ "${{ matrix.variant.glibc }}" != "default" ]; then ARTIFACT_NAME="${ARTIFACT_NAME}-glibc${{ matrix.variant.glibc }}" fi echo "artifact_name=${ARTIFACT_NAME}" >> $GITHUB_OUTPUT - # Determine binary path - bin_path="target/artifacts/rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.bin" - if [ -f "target/artifacts/rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.glibc${{ matrix.variant.glibc }}.bin" ]; then - bin_path="target/artifacts/rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.glibc${{ matrix.variant.glibc }}.bin" + # Get the binary path + BIN_PATH="${{ steps.build.outputs.bin_path }}" + + # Create a packaged directory structure - only contains bin and docs directories + mkdir -p ${ARTIFACT_NAME}/{bin,docs} + + # Copy binary files (note the difference between Windows and other systems) + if [[ "${{ matrix.variant.target }}" == *"windows"* ]]; then + cp "${BIN_PATH}" ${ARTIFACT_NAME}/bin/rustfs.exe + else + cp "${BIN_PATH}" ${ARTIFACT_NAME}/bin/rustfs fi - # Create package - mkdir -p ${ARTIFACT_NAME} - cp "$bin_path" ${ARTIFACT_NAME}/rustfs - zip -r ${ARTIFACT_NAME}.zip ${ARTIFACT_NAME} - ls -la + # copy documents and licenses + if [ -f "LICENSE" ]; then + cp LICENSE ${ARTIFACT_NAME}/docs/ + fi + if [ -f "README.md" ]; then + cp README.md ${ARTIFACT_NAME}/docs/ + fi + + # Packaged as zip + if [ "${{ runner.os }}" = "Windows" ]; then + 7z a ${ARTIFACT_NAME}.zip ${ARTIFACT_NAME} + else + zip -r ${ARTIFACT_NAME}.zip ${ARTIFACT_NAME} + fi + + echo "Created artifact: ${ARTIFACT_NAME}.zip" + ls -la ${ARTIFACT_NAME}.zip + shell: bash - uses: actions/upload-artifact@v4 with: name: ${{ steps.package.outputs.artifact_name }} path: ${{ steps.package.outputs.artifact_name }}.zip retention-days: 7 + - name: Upload to Aliyun OSS + if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' uses: JohnGuan/oss-upload-action@main with: key-id: ${{ secrets.ALICLOUDOSS_KEY_ID }} @@ -100,105 +270,124 @@ jobs: bucket: rustfs-artifacts assets: | ${{ steps.package.outputs.artifact_name }}.zip:/artifacts/rustfs/${{ steps.package.outputs.artifact_name }}.zip - ${{ steps.package.outputs.artifact_name }}.zip:/artifacts/rustfs/${{ steps.package.outputs.artifact_name }}.latest.zip + ${{ steps.package.outputs.artifact_name }}.zip:/artifacts/rustfs/${{ steps.package.outputs.artifact_name }}.latest.zip - build-rustfs-gui: - runs-on: ${ matrix.os } - needs: build-rustfs - - strategy: - matrix: - os: [ ubuntu-latest, macos-latest ] - variant: - - { profile: release, target: x86_64-unknown-linux-gnu } - - { profile: release, target: x86_64-unknown-linux-musl } - - { profile: release, target: aarch64-apple-darwin } - # - { profile: release, target: x86_64-apple-darwin } - # - { profile: release, target: aarch64-unknown-linux-gnu } - # - { profile: release, target: aarch64-unknown-linux-musl } - exclude: - - os: ubuntu-latest - variant: { profile: release, target: aarch64-apple-darwin } - - os: ubuntu-latest - variant: { profile: release, target: x86_64-apple-darwin } - - os: macos-latest - variant: { profile: release, target: x86_64-unknown-linux-gnu } - - os: macos-latest - variant: { profile: release, target: x86_64-unknown-linux-musl } - - os: macos-latest - variant: { profile: release, target: aarch64-unknown-linux-musl } - - os: macos-latest - variant: { profile: release, target: aarch64-unknown-linux-gnu } - if: startsWith(github.ref, 'refs/tags/') - steps: - - uses: actions/checkout@v4.2.2 - - name: Download artifact - uses: actions/download-artifact@v4 - with: - name: "rustfs-${{ matrix.variant.profile }}-${{ matrix.variant.target }}" - - name: Display structure of downloaded files + # Determine whether to perform GUI construction based on conditions + - name: Prepare for GUI build + if: startsWith(github.ref, 'refs/tags/') + id: prepare_gui run: | - ls -R - unzip -o -j "rustfs-${{ matrix.variant.profile }}-${{ matrix.variant.target }}.zip" -d ./cli/rustfs-gui/embedded-rustfs/ - ls -la cli/rustfs-gui/embedded-rustfs + # Create a target directory + mkdir -p ./cli/rustfs-gui/embedded-rustfs/ + + # Copy the currently built binary to the embedded-rustfs directory + if [[ "${{ matrix.variant.target }}" == *"windows"* ]]; then + cp "${{ steps.build.outputs.bin_path }}" ./cli/rustfs-gui/embedded-rustfs/rustfs.exe + else + cp "${{ steps.build.outputs.bin_path }}" ./cli/rustfs-gui/embedded-rustfs/rustfs + fi + + echo "Copied binary to embedded-rustfs directory" + ls -la ./cli/rustfs-gui/embedded-rustfs/ + shell: bash + + #Install the dioxus-cli tool - uses: taiki-e/cache-cargo-install-action@v2 + if: startsWith(github.ref, 'refs/tags/') with: tool: dioxus-cli + + # Build and package GUI applications - name: Build and Bundle rustfs-gui + if: startsWith(github.ref, 'refs/tags/') + id: build_gui + shell: bash run: | - ls -la + echo "::group::Setting up build parameters for GUI" + PROFILE="${{ matrix.variant.profile }}" + TARGET="${{ matrix.variant.target }}" + GLIBC="${{ matrix.variant.glibc }}" + RELEASE_PATH="target/artifacts/$TARGET" - release_path="target/${{ matrix.variant.target }}" - mkdir -p ${release_path} - cd cli/rustfs-gui - ls -la embedded-rustfs + # Make sure the output directory exists + mkdir -p ${RELEASE_PATH} - # Configure the linker based on the target - case "${{ matrix.variant.target }}" in + # Configure the target platform linker + echo "::group::Configuring linker for $TARGET" + case "$TARGET" in "x86_64-unknown-linux-gnu") - # Default gcc - export CC_x86_64_unknown_linux_gnu=gcc - export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=gcc - ;; + export CC_x86_64_unknown_linux_gnu=gcc + export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=gcc + ;; + "x86_64-unknown-linux-musl") + export CC_x86_64_unknown_linux_musl=musl-gcc + export CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER=musl-gcc + ;; "aarch64-unknown-linux-gnu") - # AArch64 Cross-compiler - export CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc - export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc - ;; - "x86_64-apple-darwin") - # macOS default clang - export CC_x86_64_apple_darwin=clang - export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=clang - ;; + export CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc + export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc + ;; + "aarch64-unknown-linux-musl") + export CC_aarch64_unknown_linux_musl=aarch64-linux-musl-gcc + export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-musl-gcc + ;; "aarch64-apple-darwin") - # macOS ARM64 used clang - export CC_aarch64_apple_darwin=clang - export CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER=clang - ;; + export CC_aarch64_apple_darwin=clang + export CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER=clang + ;; + "x86_64-pc-windows-msvc") + export CC_x86_64_pc_windows_msvc=cl + export CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER=link + ;; esac - # Validating Environment Variables (for Debugging) - echo "CC for ${{ matrix.variant.target }}: $CC_${{ matrix.variant.target }}" - echo "Linker for ${{ matrix.variant.target }}: $CARGO_TARGET_${{ matrix.variant.target }}_LINKER" - if [[ "${{ matrix.variant.target }}" == *"apple-darwin"* ]]; then - dx bundle --platform macos --package-types "macos" --package-types "dmg" --package-types "ios" --release --profile release --out-dir ../../${release_path} - elif [[ "${{ matrix.variant.target }}" == *"windows-msvc"* ]]; then - dx bundle --platform windows --package-types "msi" --release --profile release --out-dir ../../${release_path} - elif [[ "${{ matrix.variant.target }}" == *"unknown-linux-gnu"* ]]; then - dx bundle --platform linux --package-types "deb" --package-types "rpm" --package-types "appimage" --release --profile release --out-dir ../../${release_path} + echo "::group::Building GUI application" + cd cli/rustfs-gui + + # Building according to the target platform + if [[ "$TARGET" == *"apple-darwin"* ]]; then + echo "Building for macOS" + dx bundle --platform macos --package-types "macos" --package-types "dmg" --release --profile ${PROFILE} --out-dir ../../${RELEASE_PATH} + elif [[ "$TARGET" == *"windows-msvc"* ]]; then + echo "Building for Windows" + dx bundle --platform windows --package-types "msi" --release --profile ${PROFILE} --out-dir ../../${RELEASE_PATH} + elif [[ "$TARGET" == *"linux"* ]]; then + echo "Building for Linux" + dx bundle --platform linux --package-types "deb" --package-types "rpm" --package-types "appimage" --release --profile ${PROFILE} --out-dir ../../${RELEASE_PATH} fi - cd ../.. - GUI_ARTIFACT_NAME="rustfs-gui-${{ matrix.variant.profile }}-${{ matrix.variant.target }}" - zip -r ${GUI_ARTIFACT_NAME}.zip ${release_path}/* - echo "gui_artifact_name=${GUI_ARTIFACT_NAME}" >> $GITHUB_OUTPUT - ls -la ${release_path} + cd ../.. + + # Create component name + GUI_ARTIFACT_NAME="rustfs-gui-${PROFILE}-${TARGET}" + + if [ "$GLIBC" != "default" ]; then + GUI_ARTIFACT_NAME="${GUI_ARTIFACT_NAME}-glibc${GLIBC}" + fi + + echo "::group::Packaging GUI application" + # Select packaging method according to the operating system + if [ "${{ runner.os }}" = "Windows" ]; then + 7z a ${GUI_ARTIFACT_NAME}.zip ${RELEASE_PATH}/* + else + zip -r ${GUI_ARTIFACT_NAME}.zip ${RELEASE_PATH}/* + fi + + echo "gui_artifact_name=${GUI_ARTIFACT_NAME}" >> $GITHUB_OUTPUT + echo "Created GUI artifact: ${GUI_ARTIFACT_NAME}.zip" + ls -la ${GUI_ARTIFACT_NAME}.zip + + # Upload GUI components - uses: actions/upload-artifact@v4 + if: startsWith(github.ref, 'refs/tags/') with: - name: ${{ steps.package.outputs.gui_artifact_name }} - path: ${{ steps.package.outputs.gui_artifact_name }}.zip + name: ${{ steps.build_gui.outputs.gui_artifact_name }} + path: ${{ steps.build_gui.outputs.gui_artifact_name }}.zip retention-days: 7 - - name: Upload to Aliyun OSS + + # Upload GUI to Alibaba Cloud OSS + - name: Upload GUI to Aliyun OSS + if: startsWith(github.ref, 'refs/tags/') uses: JohnGuan/oss-upload-action@main with: key-id: ${{ secrets.ALICLOUDOSS_KEY_ID }} @@ -206,15 +395,17 @@ jobs: region: oss-cn-beijing bucket: rustfs-artifacts assets: | - ${{ steps.package.outputs.gui_artifact_name }}.zip:/artifacts/rustfs/${{ steps.package.outputs.gui_artifact_name }}.zip - ${{ steps.package.outputs.gui_artifact_name }}.zip:/artifacts/rustfs/${{ steps.package.outputs.gui_artifact_name }}.latest.zip + ${{ steps.build_gui.outputs.gui_artifact_name }}.zip:/artifacts/rustfs/${{ steps.build_gui.outputs.gui_artifact_name }}.zip + ${{ steps.build_gui.outputs.gui_artifact_name }}.zip:/artifacts/rustfs/${{ steps.build_gui.outputs.gui_artifact_name }}.latest.zip + merge: runs-on: ubuntu-latest - needs: [ build-rustfs, build-rustfs-gui ] + needs: [ build-rustfs ] + if: startsWith(github.ref, 'refs/tags/') steps: - uses: actions/upload-artifact/merge@v4 with: name: rustfs-packages pattern: "rustfs-*" - delete-merged: true + delete-merged: true \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..e69de29bb From 07c3cb3f0a03ae625c1fb66d7df5ad74f4da9d7d Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 11 May 2025 23:41:19 +0800 Subject: [PATCH 35/38] refactor(ci): optimize build workflow for better efficiency - Integrate GUI build steps into main build-rustfs job - Add conditional GUI build execution based on tag releases - Simplify workflow by removing redundant build-rustfs-gui job - Copy binary directly to embedded-rustfs directory without downloading artifacts - Update merge job dependency to only rely on build-rustfs - Improve cross-platform compatibility for Windows binary naming (.exe) - Streamline artifact uploading and OSS publishing process - Maintain consistent conditional logic for release operations --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bd2095a17..7d28cb52e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,9 +1,9 @@ -name: Build +name: Build RustFS And GUI on: workflow_dispatch: schedule: - - cron: "0 0 * * 0" # 每周日午夜执行 + - cron: "0 0 * * 0" # at midnight of each sunday push: branches: - main From 0c351965a260ae72268289ecd40fdf2ad8bedaeb Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 12 May 2025 00:36:29 +0800 Subject: [PATCH 36/38] fix(ci): add repo-token to setup-protoc action for authentication - Add GITHUB_TOKEN parameter to arduino/setup-protoc@v3 action - Ensure proper authentication for Protoc installation in CI workflow - Maintain consistent setup across different CI environments --- .github/workflows/build.yml | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7d28cb52e..bf729d21c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -60,12 +60,6 @@ jobs: with: fetch-depth: 0 - - name: Set up authentication - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: echo "Authenticated with GITHUB_TOKEN" - shell: bash - # Installation system dependencies - name: Install system dependencies (Ubuntu) if: runner.os == 'Linux' @@ -81,11 +75,22 @@ jobs: targets: ${{ matrix.variant.target }} components: rustfmt, clippy - # Setting up Protobuf and Flatbuffers - - name: Setup Protoc + # Install system dependencies + - name: Cache Protoc + id: cache-protoc + uses: actions/cache@v4.2.3 + with: + path: /Users/runner/hostedtoolcache/protoc + key: protoc-${{ runner.os }}-30.2 + restore-keys: | + protoc-${{ runner.os }}- + + - name: Install Protoc + if: steps.cache-protoc.outputs.cache-hit != 'true' uses: arduino/setup-protoc@v3 with: - version: "30.2" + version: '30.2' + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Flatc uses: Nugine/setup-flatc@v1 From dd7da015e32f063424cf5dd9cf1b661e58162709 Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 12 May 2025 01:17:31 +0800 Subject: [PATCH 37/38] Feature/rustfs config (#396) * init rustfs config * improve code for rustfs-config crate * add * improve code for comment * fix: modify rustfs-config crate name * add default fn * improve error logger * fix: modify docker config yaml * improve code for config * feat: restrict kafka feature to Linux only - Add target-specific feature configuration in Cargo.toml for obs and event-notifier crates - Implement conditional compilation for kafka feature only on Linux systems - Add appropriate error handling for non-Linux platforms - Ensure backward compatibility with existing code * refactor(ci): optimize build workflow for better efficiency - Integrate GUI build steps into main build-rustfs job - Add conditional GUI build execution based on tag releases - Simplify workflow by removing redundant build-rustfs-gui job - Copy binary directly to embedded-rustfs directory without downloading artifacts - Update merge job dependency to only rely on build-rustfs - Improve cross-platform compatibility for Windows binary naming (.exe) - Streamline artifact uploading and OSS publishing process - Maintain consistent conditional logic for release operations * refactor(ci): optimize build workflow for better efficiency - Integrate GUI build steps into main build-rustfs job - Add conditional GUI build execution based on tag releases - Simplify workflow by removing redundant build-rustfs-gui job - Copy binary directly to embedded-rustfs directory without downloading artifacts - Update merge job dependency to only rely on build-rustfs - Improve cross-platform compatibility for Windows binary naming (.exe) - Streamline artifact uploading and OSS publishing process - Maintain consistent conditional logic for release operations * fix(ci): add repo-token to setup-protoc action for authentication - Add GITHUB_TOKEN parameter to arduino/setup-protoc@v3 action - Ensure proper authentication for Protoc installation in CI workflow - Maintain consistent setup across different CI environments * modify config * improve readme.md * remove env config relation * add allow(dead_code) --- .docker/observability/config/obs-multi.toml | 38 +- .docker/observability/config/obs.toml | 38 +- .gitignore | 3 +- Cargo.lock | 26 +- Cargo.toml | 2 +- README.md | 24 +- README_ZH.md | 25 +- crates/config/src/config.rs | 6 +- crates/config/src/constants/app.rs | 19 + crates/config/src/event/adapters.rs | 27 + crates/config/src/event/config.rs | 54 +- crates/config/src/event/event.rs | 17 - crates/config/src/event/kafka.rs | 29 + crates/config/src/event/mod.rs | 5 +- crates/config/src/event/mqtt.rs | 31 ++ crates/config/src/event/webhook.rs | 51 ++ crates/config/src/lib.rs | 2 + crates/config/src/observability/config.rs | 8 +- crates/config/src/observability/file.rs | 59 +++ crates/config/src/observability/file_sink.rs | 25 - crates/config/src/observability/kafka.rs | 36 ++ crates/config/src/observability/kafka_sink.rs | 23 - crates/config/src/observability/logger.rs | 4 +- crates/config/src/observability/mod.rs | 7 +- .../config/src/observability/observability.rs | 22 - crates/config/src/observability/otel.rs | 66 ++- crates/config/src/observability/sink.rs | 25 +- crates/config/src/observability/webhook.rs | 39 ++ .../config/src/observability/webhook_sink.rs | 25 - crates/event-notifier/Cargo.toml | 7 +- .../event-notifier/examples/.env-zh.example | 28 - crates/event-notifier/examples/.env.example | 56 +- .../event-notifier/examples/.env.zh.example | 28 + crates/event-notifier/examples/simple.rs | 4 +- crates/event-notifier/src/adapter/mod.rs | 6 +- crates/event-notifier/src/config.rs | 11 +- crates/event-notifier/src/error.rs | 2 +- crates/event-notifier/src/lib.rs | 4 +- crates/obs/Cargo.toml | 8 +- crates/obs/examples/config.toml | 39 +- crates/obs/src/config.rs | 144 +++-- crates/obs/src/lib.rs | 10 +- crates/obs/src/logger.rs | 2 +- crates/obs/src/sink.rs | 497 ------------------ crates/obs/src/sinks/file.rs | 164 ++++++ crates/obs/src/sinks/kafka.rs | 165 ++++++ crates/obs/src/sinks/mod.rs | 92 ++++ crates/obs/src/sinks/webhook.rs | 70 +++ crates/obs/src/worker.rs | 2 +- crates/utils/src/certs.rs | 4 +- crypto/Cargo.toml | 2 +- deploy/config/.example.obs.env | 55 +- deploy/config/obs-zh.example.toml | 34 +- deploy/config/obs.example.toml | 42 +- deploy/config/rustfs-zh.env | 4 +- deploy/config/rustfs.env | 4 +- docker-compose-obs.yaml | 2 +- ecstore/src/config/com.rs | 2 +- ecstore/src/config/mod.rs | 4 +- ecstore/src/config/storageclass.rs | 6 +- rustfs/src/admin/handlers.rs | 1 + rustfs/src/admin/handlers/event.rs | 1 + rustfs/src/admin/mod.rs | 4 +- scripts/run.sh | 51 +- 64 files changed, 1283 insertions(+), 1008 deletions(-) create mode 100644 crates/config/src/event/adapters.rs delete mode 100644 crates/config/src/event/event.rs create mode 100644 crates/config/src/event/kafka.rs create mode 100644 crates/config/src/event/mqtt.rs create mode 100644 crates/config/src/event/webhook.rs create mode 100644 crates/config/src/observability/file.rs delete mode 100644 crates/config/src/observability/file_sink.rs create mode 100644 crates/config/src/observability/kafka.rs delete mode 100644 crates/config/src/observability/kafka_sink.rs delete mode 100644 crates/config/src/observability/observability.rs create mode 100644 crates/config/src/observability/webhook.rs delete mode 100644 crates/config/src/observability/webhook_sink.rs delete mode 100644 crates/event-notifier/examples/.env-zh.example create mode 100644 crates/event-notifier/examples/.env.zh.example delete mode 100644 crates/obs/src/sink.rs create mode 100644 crates/obs/src/sinks/file.rs create mode 100644 crates/obs/src/sinks/kafka.rs create mode 100644 crates/obs/src/sinks/mod.rs create mode 100644 crates/obs/src/sinks/webhook.rs create mode 100644 rustfs/src/admin/handlers/event.rs diff --git a/.docker/observability/config/obs-multi.toml b/.docker/observability/config/obs-multi.toml index e4ea037b1..2637a4012 100644 --- a/.docker/observability/config/obs-multi.toml +++ b/.docker/observability/config/obs-multi.toml @@ -9,26 +9,26 @@ environments = "production" logger_level = "debug" local_logging_enabled = true -[sinks] -[sinks.kafka] # Kafka sink is disabled by default -enabled = false -bootstrap_servers = "localhost:9092" -topic = "logs" -batch_size = 100 # Default is 100 if not specified -batch_timeout_ms = 1000 # Default is 1000ms if not specified +#[[sinks]] +#type = "Kafka" +#brokers = "localhost:9092" +#topic = "logs" +#batch_size = 100 # Default is 100 if not specified +#batch_timeout_ms = 1000 # Default is 1000ms if not specified +# +#[[sinks]] +#type = "Webhook" +#endpoint = "http://localhost:8080/webhook" +#auth_token = "" +#batch_size = 100 # Default is 3 if not specified +#batch_timeout_ms = 1000 # Default is 100ms if not specified -[sinks.webhook] -enabled = false -endpoint = "http://localhost:8080/webhook" -auth_token = "" -batch_size = 100 # Default is 3 if not specified -batch_timeout_ms = 1000 # Default is 100ms if not specified - -[sinks.file] -enabled = true -path = "/root/data/logs/app.log" -batch_size = 10 -batch_timeout_ms = 1000 # Default is 8192 bytes if not specified +[[sinks]] +type = "File" +path = "/root/data/logs/rustfs.log" +buffer_size = 100 # Default is 8192 bytes if not specified +flush_interval_ms = 1000 +flush_threshold = 100 [logger] queue_capacity = 10 \ No newline at end of file diff --git a/.docker/observability/config/obs.toml b/.docker/observability/config/obs.toml index f77c25d84..58069fc5c 100644 --- a/.docker/observability/config/obs.toml +++ b/.docker/observability/config/obs.toml @@ -9,26 +9,26 @@ environments = "production" logger_level = "debug" local_logging_enabled = true -[sinks] -[sinks.kafka] # Kafka sink is disabled by default -enabled = false -bootstrap_servers = "localhost:9092" -topic = "logs" -batch_size = 100 # Default is 100 if not specified -batch_timeout_ms = 1000 # Default is 1000ms if not specified +#[[sinks]] +#type = "Kafka" +#brokers = "localhost:9092" +#topic = "logs" +#batch_size = 100 # Default is 100 if not specified +#batch_timeout_ms = 1000 # Default is 1000ms if not specified +# +#[[sinks]] +#type = "Webhook" +#endpoint = "http://localhost:8080/webhook" +#auth_token = "" +#batch_size = 100 # Default is 3 if not specified +#batch_timeout_ms = 1000 # Default is 100ms if not specified -[sinks.webhook] -enabled = false -endpoint = "http://localhost:8080/webhook" -auth_token = "" -batch_size = 100 # Default is 3 if not specified -batch_timeout_ms = 1000 # Default is 100ms if not specified - -[sinks.file] -enabled = true -path = "/root/data/logs/app.log" -batch_size = 10 -batch_timeout_ms = 1000 # Default is 8192 bytes if not specified +[[sinks]] +type = "File" +path = "/root/data/logs/rustfs.log" +buffer_size = 100 # Default is 8192 bytes if not specified +flush_interval_ms = 1000 +flush_threshold = 100 [logger] queue_capacity = 10 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 8f19d5fca..7ccca205e 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ cli/rustfs-gui/embedded-rustfs/rustfs deploy/config/obs.toml *.log deploy/certs/* -*jsonl \ No newline at end of file +*jsonl +.env \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index cce1d3bcb..01ab7a994 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1746,7 +1746,7 @@ dependencies = [ "pbkdf2", "rand 0.8.5", "serde_json", - "sha2 0.10.8", + "sha2 0.10.9", "test-case", "thiserror 2.0.12", "time", @@ -2141,7 +2141,7 @@ dependencies = [ "md-5", "rand 0.8.5", "regex", - "sha2 0.10.8", + "sha2 0.10.9", "unicode-segmentation", "uuid", ] @@ -3020,6 +3020,12 @@ dependencies = [ "const-random", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "dpi" version = "0.1.1" @@ -4807,7 +4813,7 @@ dependencies = [ "nom 8.0.0", "once_cell", "serde", - "sha2 0.10.8", + "sha2 0.10.9", "thiserror 2.0.12", "uuid", ] @@ -6120,7 +6126,7 @@ checksum = "7f9f832470494906d1fca5329f8ab5791cc60beb230c74815dff541cbd2b5ca0" dependencies = [ "once_cell", "pest", - "sha2 0.10.8", + "sha2 0.10.9", ] [[package]] @@ -7209,7 +7215,7 @@ version = "8.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d55b95147fe01265d06b3955db798bdaed52e60e2211c41137701b3aba8e21" dependencies = [ - "sha2 0.10.8", + "sha2 0.10.9", "walkdir", ] @@ -7340,6 +7346,7 @@ dependencies = [ "async-trait", "axum", "config", + "dotenvy", "http", "rdkafka", "reqwest", @@ -7371,7 +7378,7 @@ dependencies = [ "rust-embed", "serde", "serde_json", - "sha2 0.10.8", + "sha2 0.10.9", "tokio", "tracing-appender", "tracing-subscriber", @@ -7394,6 +7401,7 @@ dependencies = [ "opentelemetry_sdk", "rdkafka", "reqwest", + "rustfs-config", "serde", "serde_json", "smallvec", @@ -7919,9 +7927,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.8" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures", @@ -10218,7 +10226,7 @@ dependencies = [ "once_cell", "percent-encoding", "raw-window-handle 0.6.2", - "sha2 0.10.8", + "sha2 0.10.9", "soup3", "tao-macros", "thiserror 1.0.69", diff --git a/Cargo.toml b/Cargo.toml index 2adf24d73..a18ac9d81 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -157,7 +157,7 @@ serde = { version = "1.0.219", features = ["derive"] } serde_json = "1.0.140" serde_urlencoded = "0.7.1" serde_with = "3.12.0" -sha2 = "0.10.8" +sha2 = "0.10.9" smallvec = { version = "1.15.0", features = ["serde"] } snafu = "0.8.5" socket2 = "0.5.9" diff --git a/README.md b/README.md index e4818d4dd..9bbb8be9d 100644 --- a/README.md +++ b/README.md @@ -59,30 +59,12 @@ export RUSTFS_ADDRESS="0.0.0.0:9000" export RUSTFS_CONSOLE_ENABLE=true export RUSTFS_CONSOLE_ADDRESS="0.0.0.0:9001" -# Observability config (option 1: config file) +# Observability config export RUSTFS_OBS_CONFIG="./deploy/config/obs.toml" -# Observability config (option 2: environment variables) -export RUSTFS__OBSERVABILITY__ENDPOINT=http://localhost:4317 -export RUSTFS__OBSERVABILITY__USE_STDOUT=true -export RUSTFS__OBSERVABILITY__SAMPLE_RATIO=2.0 -export RUSTFS__OBSERVABILITY__METER_INTERVAL=30 -export RUSTFS__OBSERVABILITY__SERVICE_NAME=rustfs -export RUSTFS__OBSERVABILITY__SERVICE_VERSION=0.1.0 -export RUSTFS__OBSERVABILITY__ENVIRONMENT=develop -export RUSTFS__OBSERVABILITY__LOGGER_LEVEL=info -export RUSTFS__OBSERVABILITY__LOCAL_LOGGING_ENABLED=true +# Event message configuration +#export RUSTFS_EVENT_CONFIG="./deploy/config/event.toml" -# Logging sinks -export RUSTFS__SINKS__FILE__ENABLED=true -export RUSTFS__SINKS__FILE__PATH="./deploy/logs/rustfs.log" -export RUSTFS__SINKS__WEBHOOK__ENABLED=false -export RUSTFS__SINKS__WEBHOOK__ENDPOINT="" -export RUSTFS__SINKS__WEBHOOK__AUTH_TOKEN="" -export RUSTFS__SINKS__KAFKA__ENABLED=false -export RUSTFS__SINKS__KAFKA__BOOTSTRAP_SERVERS="" -export RUSTFS__SINKS__KAFKA__TOPIC="" -export RUSTFS__LOGGER__QUEUE_CAPACITY=10 ``` #### Start the service diff --git a/README_ZH.md b/README_ZH.md index 10f05bbef..89fe320a0 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -59,30 +59,11 @@ export RUSTFS_ADDRESS="0.0.0.0:9000" export RUSTFS_CONSOLE_ENABLE=true export RUSTFS_CONSOLE_ADDRESS="0.0.0.0:9001" -# 可观测性配置（方式一：配置文件） +# 可观测性配置 export RUSTFS_OBS_CONFIG="./deploy/config/obs.toml" -# 可观测性配置（方式二：环境变量） -export RUSTFS__OBSERVABILITY__ENDPOINT=http://localhost:4317 -export RUSTFS__OBSERVABILITY__USE_STDOUT=true -export RUSTFS__OBSERVABILITY__SAMPLE_RATIO=2.0 -export RUSTFS__OBSERVABILITY__METER_INTERVAL=30 -export RUSTFS__OBSERVABILITY__SERVICE_NAME=rustfs -export RUSTFS__OBSERVABILITY__SERVICE_VERSION=0.1.0 -export RUSTFS__OBSERVABILITY__ENVIRONMENT=develop -export RUSTFS__OBSERVABILITY__LOGGER_LEVEL=info -export RUSTFS__OBSERVABILITY__LOCAL_LOGGING_ENABLED=true - -# 日志接收器 -export RUSTFS__SINKS__FILE__ENABLED=true -export RUSTFS__SINKS__FILE__PATH="./deploy/logs/rustfs.log" -export RUSTFS__SINKS__WEBHOOK__ENABLED=false -export RUSTFS__SINKS__WEBHOOK__ENDPOINT="" -export RUSTFS__SINKS__WEBHOOK__AUTH_TOKEN="" -export RUSTFS__SINKS__KAFKA__ENABLED=false -export RUSTFS__SINKS__KAFKA__BOOTSTRAP_SERVERS="" -export RUSTFS__SINKS__KAFKA__TOPIC="" -export RUSTFS__LOGGER__QUEUE_CAPACITY=10 +# 事件消息配置 +#export RUSTFS_EVENT_CONFIG="./deploy/config/event.toml" ``` #### 启动服务 diff --git a/crates/config/src/config.rs b/crates/config/src/config.rs index bebcbfd00..3d427f51c 100644 --- a/crates/config/src/config.rs +++ b/crates/config/src/config.rs @@ -1,17 +1,17 @@ -use crate::event::config::EventConfig; +use crate::event::config::NotifierConfig; use crate::ObservabilityConfig; /// RustFs configuration pub struct RustFsConfig { pub observability: ObservabilityConfig, - pub event: EventConfig, + pub event: NotifierConfig, } impl RustFsConfig { pub fn new() -> Self { Self { observability: ObservabilityConfig::new(), - event: EventConfig::new(), + event: NotifierConfig::new(), } } } diff --git a/crates/config/src/constants/app.rs b/crates/config/src/constants/app.rs index 294c7677f..467905f7b 100644 --- a/crates/config/src/constants/app.rs +++ b/crates/config/src/constants/app.rs @@ -14,6 +14,25 @@ pub const VERSION: &str = "0.0.1"; /// Environment variable: RUSTFS_LOG_LEVEL pub const DEFAULT_LOG_LEVEL: &str = "info"; +/// Default configuration use stdout +/// Default value: true +pub(crate) const USE_STDOUT: bool = true; + +/// Default configuration sample ratio +/// Default value: 1.0 +pub(crate) const SAMPLE_RATIO: f64 = 1.0; +/// Default configuration meter interval +/// Default value: 30 +pub(crate) const METER_INTERVAL: u64 = 30; + +/// Default configuration service version +/// Default value: 0.0.1 +pub(crate) const SERVICE_VERSION: &str = "0.0.1"; + +/// Default configuration environment +/// Default value: production +pub(crate) const ENVIRONMENT: &str = "production"; + /// maximum number of connections /// This is the maximum number of connections that the server will accept. /// This is used to limit the number of connections to the server. diff --git a/crates/config/src/event/adapters.rs b/crates/config/src/event/adapters.rs new file mode 100644 index 000000000..d66bf19e5 --- /dev/null +++ b/crates/config/src/event/adapters.rs @@ -0,0 +1,27 @@ +use crate::event::kafka::KafkaAdapter; +use crate::event::mqtt::MqttAdapter; +use crate::event::webhook::WebhookAdapter; +use serde::{Deserialize, Serialize}; + +/// Configuration for the notification system. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum AdapterConfig { + Webhook(WebhookAdapter), + Kafka(KafkaAdapter), + Mqtt(MqttAdapter), +} + +impl AdapterConfig { + /// create a new configuration with default values + pub fn new() -> Self { + Self::Webhook(WebhookAdapter::new()) + } +} + +impl Default for AdapterConfig { + /// create a new configuration with default values + fn default() -> Self { + Self::new() + } +} diff --git a/crates/config/src/event/config.rs b/crates/config/src/event/config.rs index a8a430688..ea364c9a0 100644 --- a/crates/config/src/event/config.rs +++ b/crates/config/src/event/config.rs @@ -1,23 +1,43 @@ -/// Event configuration module -pub struct EventConfig { - pub event_type: String, - pub event_source: String, - pub event_destination: String, +use crate::event::adapters::AdapterConfig; +use serde::{Deserialize, Serialize}; +use std::env; + +#[allow(dead_code)] +const DEFAULT_CONFIG_FILE: &str = "event"; + +/// Configuration for the notification system. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NotifierConfig { + #[serde(default = "default_store_path")] + pub store_path: String, + #[serde(default = "default_channel_capacity")] + pub channel_capacity: usize, + pub adapters: Vec, } -impl EventConfig { - /// Creates a new instance of `EventConfig` with default values. - pub fn new() -> Self { - Self { - event_type: "default".to_string(), - event_source: "default".to_string(), - event_destination: "default".to_string(), - } - } -} - -impl Default for EventConfig { +impl Default for NotifierConfig { fn default() -> Self { Self::new() } } + +impl NotifierConfig { + /// create a new configuration with default values + pub fn new() -> Self { + Self { + store_path: default_store_path(), + channel_capacity: default_channel_capacity(), + adapters: vec![AdapterConfig::new()], + } + } +} + +/// Provide temporary directories as default storage paths +fn default_store_path() -> String { + env::temp_dir().join("event-notification").to_string_lossy().to_string() +} + +/// Provides the recommended default channel capacity for high concurrency systems +fn default_channel_capacity() -> usize { + 10000 // Reasonable default values for high concurrency systems +} diff --git a/crates/config/src/event/event.rs b/crates/config/src/event/event.rs deleted file mode 100644 index 70a103690..000000000 --- a/crates/config/src/event/event.rs +++ /dev/null @@ -1,17 +0,0 @@ -/// Event configuration module -pub struct EventConfig { - pub event_type: String, - pub event_source: String, - pub event_destination: String, -} - -impl EventConfig { - /// Creates a new instance of `EventConfig` with default values. - pub fn new() -> Self { - Self { - event_type: "default".to_string(), - event_source: "default".to_string(), - event_destination: "default".to_string(), - } - } -} diff --git a/crates/config/src/event/kafka.rs b/crates/config/src/event/kafka.rs new file mode 100644 index 000000000..164113742 --- /dev/null +++ b/crates/config/src/event/kafka.rs @@ -0,0 +1,29 @@ +use serde::{Deserialize, Serialize}; + +/// Configuration for the Kafka adapter. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct KafkaAdapter { + pub brokers: String, + pub topic: String, + pub max_retries: u32, + pub timeout: u64, +} + +impl KafkaAdapter { + /// create a new configuration with default values + pub fn new() -> Self { + Self { + brokers: "localhost:9092".to_string(), + topic: "kafka_topic".to_string(), + max_retries: 3, + timeout: 1000, + } + } +} + +impl Default for KafkaAdapter { + /// create a new configuration with default values + fn default() -> Self { + Self::new() + } +} diff --git a/crates/config/src/event/mod.rs b/crates/config/src/event/mod.rs index 602809247..80dfd45e5 100644 --- a/crates/config/src/event/mod.rs +++ b/crates/config/src/event/mod.rs @@ -1,2 +1,5 @@ +pub(crate) mod adapters; pub(crate) mod config; -pub(crate) mod event; +pub(crate) mod kafka; +pub(crate) mod mqtt; +pub(crate) mod webhook; diff --git a/crates/config/src/event/mqtt.rs b/crates/config/src/event/mqtt.rs new file mode 100644 index 000000000..ee9835323 --- /dev/null +++ b/crates/config/src/event/mqtt.rs @@ -0,0 +1,31 @@ +use serde::{Deserialize, Serialize}; + +/// Configuration for the MQTT adapter. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MqttAdapter { + pub broker: String, + pub port: u16, + pub client_id: String, + pub topic: String, + pub max_retries: u32, +} + +impl MqttAdapter { + /// create a new configuration with default values + pub fn new() -> Self { + Self { + broker: "localhost".to_string(), + port: 1883, + client_id: "mqtt_client".to_string(), + topic: "mqtt_topic".to_string(), + max_retries: 3, + } + } +} + +impl Default for MqttAdapter { + /// create a new configuration with default values + fn default() -> Self { + Self::new() + } +} diff --git a/crates/config/src/event/webhook.rs b/crates/config/src/event/webhook.rs new file mode 100644 index 000000000..95b3adad1 --- /dev/null +++ b/crates/config/src/event/webhook.rs @@ -0,0 +1,51 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +/// Configuration for the notification system. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WebhookAdapter { + pub endpoint: String, + pub auth_token: Option, + pub custom_headers: Option>, + pub max_retries: u32, + pub timeout: u64, +} + +impl WebhookAdapter { + /// verify that the configuration is valid + pub fn validate(&self) -> Result<(), String> { + // verify that endpoint cannot be empty + if self.endpoint.trim().is_empty() { + return Err("Webhook endpoint cannot be empty".to_string()); + } + + // verification timeout must be reasonable + if self.timeout == 0 { + return Err("Webhook timeout must be greater than 0".to_string()); + } + + // Verify that the maximum number of retry is reasonable + if self.max_retries > 10 { + return Err("Maximum retry count cannot exceed 10".to_string()); + } + + Ok(()) + } + + /// Get the default configuration + pub fn new() -> Self { + Self { + endpoint: "".to_string(), + auth_token: None, + custom_headers: Some(HashMap::new()), + max_retries: 3, + timeout: 1000, + } + } +} + +impl Default for WebhookAdapter { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index fd7b8bec2..44a9fe3c5 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -7,3 +7,5 @@ mod observability; pub use config::RustFsConfig; pub use constants::app::*; + +pub use event::config::NotifierConfig; diff --git a/crates/config/src/observability/config.rs b/crates/config/src/observability/config.rs index 361f9a6c5..b43f3646b 100644 --- a/crates/config/src/observability/config.rs +++ b/crates/config/src/observability/config.rs @@ -1,13 +1,13 @@ use crate::observability::logger::LoggerConfig; use crate::observability::otel::OtelConfig; use crate::observability::sink::SinkConfig; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; /// Observability configuration -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Serialize, Clone)] pub struct ObservabilityConfig { pub otel: OtelConfig, - pub sinks: SinkConfig, + pub sinks: Vec, pub logger: Option, } @@ -15,7 +15,7 @@ impl ObservabilityConfig { pub fn new() -> Self { Self { otel: OtelConfig::new(), - sinks: SinkConfig::new(), + sinks: vec![SinkConfig::new()], logger: Some(LoggerConfig::new()), } } diff --git a/crates/config/src/observability/file.rs b/crates/config/src/observability/file.rs new file mode 100644 index 000000000..6ec74c8b3 --- /dev/null +++ b/crates/config/src/observability/file.rs @@ -0,0 +1,59 @@ +use serde::{Deserialize, Serialize}; +use std::env; + +/// File sink configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FileSink { + pub path: String, + #[serde(default = "default_buffer_size")] + pub buffer_size: Option, + #[serde(default = "default_flush_interval_ms")] + pub flush_interval_ms: Option, + #[serde(default = "default_flush_threshold")] + pub flush_threshold: Option, +} + +impl FileSink { + pub fn new() -> Self { + Self { + path: env::var("RUSTFS_SINKS_FILE_PATH") + .ok() + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(default_path), + buffer_size: default_buffer_size(), + flush_interval_ms: default_flush_interval_ms(), + flush_threshold: default_flush_threshold(), + } + } +} + +impl Default for FileSink { + fn default() -> Self { + Self::new() + } +} + +fn default_buffer_size() -> Option { + Some(8192) +} +fn default_flush_interval_ms() -> Option { + Some(1000) +} +fn default_flush_threshold() -> Option { + Some(100) +} + +fn default_path() -> String { + let temp_dir = env::temp_dir().join("rustfs"); + + if let Err(e) = std::fs::create_dir_all(&temp_dir) { + eprintln!("Failed to create log directory: {}", e); + return "rustfs/rustfs.log".to_string(); + } + + temp_dir + .join("rustfs.log") + .to_str() + .unwrap_or("rustfs/rustfs.log") + .to_string() +} diff --git a/crates/config/src/observability/file_sink.rs b/crates/config/src/observability/file_sink.rs deleted file mode 100644 index d475376ef..000000000 --- a/crates/config/src/observability/file_sink.rs +++ /dev/null @@ -1,25 +0,0 @@ -use serde::Deserialize; - -/// File sink configuration -#[derive(Debug, Deserialize, Clone)] -pub struct FileSinkConfig { - pub path: String, - pub max_size: u64, - pub max_backups: u64, -} - -impl FileSinkConfig { - pub fn new() -> Self { - Self { - path: "".to_string(), - max_size: 0, - max_backups: 0, - } - } -} - -impl Default for FileSinkConfig { - fn default() -> Self { - Self::new() - } -} diff --git a/crates/config/src/observability/kafka.rs b/crates/config/src/observability/kafka.rs new file mode 100644 index 000000000..104ab1da4 --- /dev/null +++ b/crates/config/src/observability/kafka.rs @@ -0,0 +1,36 @@ +use serde::{Deserialize, Serialize}; + +/// Kafka sink configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct KafkaSink { + pub brokers: String, + pub topic: String, + #[serde(default = "default_batch_size")] + pub batch_size: Option, + #[serde(default = "default_batch_timeout_ms")] + pub batch_timeout_ms: Option, +} + +impl KafkaSink { + pub fn new() -> Self { + Self { + brokers: "localhost:9092".to_string(), + topic: "rustfs".to_string(), + batch_size: default_batch_size(), + batch_timeout_ms: default_batch_timeout_ms(), + } + } +} + +impl Default for KafkaSink { + fn default() -> Self { + Self::new() + } +} + +fn default_batch_size() -> Option { + Some(100) +} +fn default_batch_timeout_ms() -> Option { + Some(1000) +} diff --git a/crates/config/src/observability/kafka_sink.rs b/crates/config/src/observability/kafka_sink.rs deleted file mode 100644 index f40a979b9..000000000 --- a/crates/config/src/observability/kafka_sink.rs +++ /dev/null @@ -1,23 +0,0 @@ -use serde::Deserialize; - -/// Kafka sink configuration -#[derive(Debug, Deserialize, Clone)] -pub struct KafkaSinkConfig { - pub brokers: Vec, - pub topic: String, -} - -impl KafkaSinkConfig { - pub fn new() -> Self { - Self { - brokers: vec!["localhost:9092".to_string()], - topic: "rustfs".to_string(), - } - } -} - -impl Default for KafkaSinkConfig { - fn default() -> Self { - Self::new() - } -} diff --git a/crates/config/src/observability/logger.rs b/crates/config/src/observability/logger.rs index f6c70682c..68a0bff45 100644 --- a/crates/config/src/observability/logger.rs +++ b/crates/config/src/observability/logger.rs @@ -1,7 +1,7 @@ -use serde::Deserialize; +use serde::{Deserialize, Serialize}; /// Logger configuration -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Serialize, Clone)] pub struct LoggerConfig { pub queue_capacity: Option, } diff --git a/crates/config/src/observability/mod.rs b/crates/config/src/observability/mod.rs index 65d9933b0..216c074e2 100644 --- a/crates/config/src/observability/mod.rs +++ b/crates/config/src/observability/mod.rs @@ -1,8 +1,7 @@ pub(crate) mod config; -pub(crate) mod file_sink; -pub(crate) mod kafka_sink; +pub(crate) mod file; +pub(crate) mod kafka; pub(crate) mod logger; -pub(crate) mod observability; pub(crate) mod otel; pub(crate) mod sink; -pub(crate) mod webhook_sink; +pub(crate) mod webhook; diff --git a/crates/config/src/observability/observability.rs b/crates/config/src/observability/observability.rs deleted file mode 100644 index 17b4e0704..000000000 --- a/crates/config/src/observability/observability.rs +++ /dev/null @@ -1,22 +0,0 @@ -use crate::observability::logger::LoggerConfig; -use crate::observability::otel::OtelConfig; -use crate::observability::sink::SinkConfig; -use serde::Deserialize; - -/// Observability configuration -#[derive(Debug, Deserialize, Clone)] -pub struct ObservabilityConfig { - pub otel: OtelConfig, - pub sinks: SinkConfig, - pub logger: Option, -} - -impl ObservabilityConfig { - pub fn new() -> Self { - Self { - otel: OtelConfig::new(), - sinks: SinkConfig::new(), - logger: Some(LoggerConfig::new()), - } - } -} diff --git a/crates/config/src/observability/otel.rs b/crates/config/src/observability/otel.rs index 4ac6618bd..77785c687 100644 --- a/crates/config/src/observability/otel.rs +++ b/crates/config/src/observability/otel.rs @@ -1,22 +1,25 @@ -use serde::Deserialize; +use crate::constants::app::{ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT}; +use crate::{APP_NAME, DEFAULT_LOG_LEVEL}; +use serde::{Deserialize, Serialize}; +use std::env; /// OpenTelemetry configuration -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Serialize, Clone)] pub struct OtelConfig { - pub endpoint: String, - pub service_name: String, - pub service_version: String, - pub resource_attributes: Vec, + pub endpoint: String, // Endpoint for metric collection + pub use_stdout: Option, // Output to stdout + pub sample_ratio: Option, // Trace sampling ratio + pub meter_interval: Option, // Metric collection interval + pub service_name: Option, // Service name + pub service_version: Option, // Service version + pub environment: Option, // Environment + pub logger_level: Option, // Logger level + pub local_logging_enabled: Option, // Local logging enabled } impl OtelConfig { pub fn new() -> Self { - Self { - endpoint: "http://localhost:4317".to_string(), - service_name: "rustfs".to_string(), - service_version: "0.1.0".to_string(), - resource_attributes: vec![], - } + extract_otel_config_from_env() } } @@ -25,3 +28,42 @@ impl Default for OtelConfig { Self::new() } } + +// Helper function: Extract observable configuration from environment variables +fn extract_otel_config_from_env() -> OtelConfig { + OtelConfig { + endpoint: env::var("RUSTFS_OBSERVABILITY_ENDPOINT").unwrap_or_else(|_| "".to_string()), + use_stdout: env::var("RUSTFS_OBSERVABILITY_USE_STDOUT") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(USE_STDOUT)), + sample_ratio: env::var("RUSTFS_OBSERVABILITY_SAMPLE_RATIO") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(SAMPLE_RATIO)), + meter_interval: env::var("RUSTFS_OBSERVABILITY_METER_INTERVAL") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(METER_INTERVAL)), + service_name: env::var("RUSTFS_OBSERVABILITY_SERVICE_NAME") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(APP_NAME.to_string())), + service_version: env::var("RUSTFS_OBSERVABILITY_SERVICE_VERSION") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(SERVICE_VERSION.to_string())), + environment: env::var("RUSTFS_OBSERVABILITY_ENVIRONMENT") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(ENVIRONMENT.to_string())), + logger_level: env::var("RUSTFS_OBSERVABILITY_LOGGER_LEVEL") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(DEFAULT_LOG_LEVEL.to_string())), + local_logging_enabled: env::var("RUSTFS_OBSERVABILITY_LOCAL_LOGGING_ENABLED") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(false)), + } +} diff --git a/crates/config/src/observability/sink.rs b/crates/config/src/observability/sink.rs index dcb37fa3b..9339e06ee 100644 --- a/crates/config/src/observability/sink.rs +++ b/crates/config/src/observability/sink.rs @@ -1,23 +1,20 @@ -use crate::observability::file_sink::FileSinkConfig; -use crate::observability::kafka_sink::KafkaSinkConfig; -use crate::observability::webhook_sink::WebhookSinkConfig; -use serde::Deserialize; +use crate::observability::file::FileSink; +use crate::observability::kafka::KafkaSink; +use crate::observability::webhook::WebhookSink; +use serde::{Deserialize, Serialize}; /// Sink configuration -#[derive(Debug, Deserialize, Clone)] -pub struct SinkConfig { - pub kafka: Option, - pub webhook: Option, - pub file: Option, +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum SinkConfig { + Kafka(KafkaSink), + Webhook(WebhookSink), + File(FileSink), } impl SinkConfig { pub fn new() -> Self { - Self { - kafka: None, - webhook: None, - file: Some(FileSinkConfig::new()), - } + Self::File(FileSink::new()) } } diff --git a/crates/config/src/observability/webhook.rs b/crates/config/src/observability/webhook.rs new file mode 100644 index 000000000..8e1d32f82 --- /dev/null +++ b/crates/config/src/observability/webhook.rs @@ -0,0 +1,39 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +/// Webhook sink configuration +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct WebhookSink { + pub endpoint: String, + pub auth_token: String, + pub headers: Option>, + #[serde(default = "default_max_retries")] + pub max_retries: Option, + #[serde(default = "default_retry_delay_ms")] + pub retry_delay_ms: Option, +} + +impl WebhookSink { + pub fn new() -> Self { + Self { + endpoint: "".to_string(), + auth_token: "".to_string(), + headers: Some(HashMap::new()), + max_retries: default_max_retries(), + retry_delay_ms: default_retry_delay_ms(), + } + } +} + +impl Default for WebhookSink { + fn default() -> Self { + Self::new() + } +} + +fn default_max_retries() -> Option { + Some(3) +} +fn default_retry_delay_ms() -> Option { + Some(100) +} diff --git a/crates/config/src/observability/webhook_sink.rs b/crates/config/src/observability/webhook_sink.rs deleted file mode 100644 index 494292039..000000000 --- a/crates/config/src/observability/webhook_sink.rs +++ /dev/null @@ -1,25 +0,0 @@ -use serde::Deserialize; - -/// Webhook sink configuration -#[derive(Debug, Deserialize, Clone)] -pub struct WebhookSinkConfig { - pub url: String, - pub method: String, - pub headers: Vec<(String, String)>, -} - -impl WebhookSinkConfig { - pub fn new() -> Self { - Self { - url: "http://localhost:8080/webhook".to_string(), - method: "POST".to_string(), - headers: vec![], - } - } -} - -impl Default for WebhookSinkConfig { - fn default() -> Self { - Self::new() - } -} diff --git a/crates/event-notifier/Cargo.toml b/crates/event-notifier/Cargo.toml index 5d1d33906..8d9acd9a0 100644 --- a/crates/event-notifier/Cargo.toml +++ b/crates/event-notifier/Cargo.toml @@ -9,13 +9,12 @@ version.workspace = true [features] default = ["webhook"] webhook = ["dep:reqwest"] -kafka = ["rdkafka"] mqtt = ["rumqttc"] +kafka = ["dep:rdkafka"] [dependencies] async-trait = { workspace = true } config = { workspace = true } -rdkafka = { workspace = true, features = ["tokio"], optional = true } reqwest = { workspace = true, optional = true } rumqttc = { workspace = true, optional = true } serde = { workspace = true } @@ -29,12 +28,16 @@ tokio = { workspace = true, features = ["sync", "net", "macros", "signal", "rt-m tokio-util = { workspace = true } uuid = { workspace = true, features = ["v4", "serde"] } +# Only enable kafka features and related dependencies on Linux +[target.'cfg(target_os = "linux")'.dependencies] +rdkafka = { workspace = true, features = ["tokio"], optional = true } [dev-dependencies] tokio = { workspace = true, features = ["test-util"] } tracing-subscriber = { workspace = true } http = { workspace = true } axum = { workspace = true } +dotenvy = "0.15.7" [lints] workspace = true diff --git a/crates/event-notifier/examples/.env-zh.example b/crates/event-notifier/examples/.env-zh.example deleted file mode 100644 index 00228e162..000000000 --- a/crates/event-notifier/examples/.env-zh.example +++ /dev/null @@ -1,28 +0,0 @@ -# ===== 全局配置 ===== -NOTIFIER__STORE_PATH=/var/log/event-notification -NOTIFIER__CHANNEL_CAPACITY=5000 - -# ===== 适配器配置（数组格式） ===== -# Webhook 适配器（索引 0） -NOTIFIER__ADAPTERS_0__type=Webhook -NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3020/webhook -NOTIFIER__ADAPTERS_0__auth_token=your-auth-token -NOTIFIER__ADAPTERS_0__max_retries=3 -NOTIFIER__ADAPTERS_0__timeout=50 -NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=value -NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=value - -# Kafka 适配器（索引 1） -NOTIFIER__ADAPTERS_1__type=Kafka -NOTIFIER__ADAPTERS_1__brokers=localhost:9092 -NOTIFIER__ADAPTERS_1__topic=notifications -NOTIFIER__ADAPTERS_1__max_retries=3 -NOTIFIER__ADAPTERS_1__timeout=60 - -# MQTT 适配器（索引 2） -NOTIFIER__ADAPTERS_2__type=Mqtt -NOTIFIER__ADAPTERS_2__broker=mqtt.example.com -NOTIFIER__ADAPTERS_2__port=1883 -NOTIFIER__ADAPTERS_2__client_id=event-notifier -NOTIFIER__ADAPTERS_2__topic=events -NOTIFIER__ADAPTERS_2__max_retries=3 \ No newline at end of file diff --git a/crates/event-notifier/examples/.env.example b/crates/event-notifier/examples/.env.example index af343863d..c6d371428 100644 --- a/crates/event-notifier/examples/.env.example +++ b/crates/event-notifier/examples/.env.example @@ -1,28 +1,28 @@ -# ===== global configuration ===== -NOTIFIER__STORE_PATH=/var/log/event-notification -NOTIFIER__CHANNEL_CAPACITY=5000 - -# ===== adapter configuration array format ===== -# webhook adapter index 0 -NOTIFIER__ADAPTERS_0__type=Webhook -NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3020/webhook -NOTIFIER__ADAPTERS_0__auth_token=your-auth-token -NOTIFIER__ADAPTERS_0__max_retries=3 -NOTIFIER__ADAPTERS_0__timeout=50 -NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=server-value -NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=client-value - -# kafka adapter index 1 -NOTIFIER__ADAPTERS_1__type=Kafka -NOTIFIER__ADAPTERS_1__brokers=localhost:9092 -NOTIFIER__ADAPTERS_1__topic=notifications -NOTIFIER__ADAPTERS_1__max_retries=3 -NOTIFIER__ADAPTERS_1__timeout=60 - -# mqtt adapter index 2 -NOTIFIER__ADAPTERS_2__type=Mqtt -NOTIFIER__ADAPTERS_2__broker=mqtt.example.com -NOTIFIER__ADAPTERS_2__port=1883 -NOTIFIER__ADAPTERS_2__client_id=event-notifier -NOTIFIER__ADAPTERS_2__topic=events -NOTIFIER__ADAPTERS_2__max_retries=3 \ No newline at end of file +## ===== global configuration ===== +#NOTIFIER__STORE_PATH=/var/log/event-notification +#NOTIFIER__CHANNEL_CAPACITY=5000 +# +## ===== adapter configuration array format ===== +## webhook adapter index 0 +#NOTIFIER__ADAPTERS_0__type=Webhook +#NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3020/webhook +#NOTIFIER__ADAPTERS_0__auth_token=your-auth-token +#NOTIFIER__ADAPTERS_0__max_retries=3 +#NOTIFIER__ADAPTERS_0__timeout=50 +#NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=server-value +#NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=client-value +# +## kafka adapter index 1 +#NOTIFIER__ADAPTERS_1__type=Kafka +#NOTIFIER__ADAPTERS_1__brokers=localhost:9092 +#NOTIFIER__ADAPTERS_1__topic=notifications +#NOTIFIER__ADAPTERS_1__max_retries=3 +#NOTIFIER__ADAPTERS_1__timeout=60 +# +## mqtt adapter index 2 +#NOTIFIER__ADAPTERS_2__type=Mqtt +#NOTIFIER__ADAPTERS_2__broker=mqtt.example.com +#NOTIFIER__ADAPTERS_2__port=1883 +#NOTIFIER__ADAPTERS_2__client_id=event-notifier +#NOTIFIER__ADAPTERS_2__topic=events +#NOTIFIER__ADAPTERS_2__max_retries=3 \ No newline at end of file diff --git a/crates/event-notifier/examples/.env.zh.example b/crates/event-notifier/examples/.env.zh.example new file mode 100644 index 000000000..47f543080 --- /dev/null +++ b/crates/event-notifier/examples/.env.zh.example @@ -0,0 +1,28 @@ +## ===== 全局配置 ===== +#NOTIFIER__STORE_PATH=/var/log/event-notification +#NOTIFIER__CHANNEL_CAPACITY=5000 +# +## ===== 适配器配置（数组格式） ===== +## Webhook 适配器（索引 0） +#NOTIFIER__ADAPTERS_0__type=Webhook +#NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3020/webhook +#NOTIFIER__ADAPTERS_0__auth_token=your-auth-token +#NOTIFIER__ADAPTERS_0__max_retries=3 +#NOTIFIER__ADAPTERS_0__timeout=50 +#NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=value +#NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=value +# +## Kafka 适配器（索引 1） +#NOTIFIER__ADAPTERS_1__type=Kafka +#NOTIFIER__ADAPTERS_1__brokers=localhost:9092 +#NOTIFIER__ADAPTERS_1__topic=notifications +#NOTIFIER__ADAPTERS_1__max_retries=3 +#NOTIFIER__ADAPTERS_1__timeout=60 +# +## MQTT 适配器（索引 2） +#NOTIFIER__ADAPTERS_2__type=Mqtt +#NOTIFIER__ADAPTERS_2__broker=mqtt.example.com +#NOTIFIER__ADAPTERS_2__port=1883 +#NOTIFIER__ADAPTERS_2__client_id=event-notifier +#NOTIFIER__ADAPTERS_2__topic=events +#NOTIFIER__ADAPTERS_2__max_retries=3 \ No newline at end of file diff --git a/crates/event-notifier/examples/simple.rs b/crates/event-notifier/examples/simple.rs index 93005f0a7..27d422b06 100644 --- a/crates/event-notifier/examples/simple.rs +++ b/crates/event-notifier/examples/simple.rs @@ -33,7 +33,9 @@ async fn main() -> Result<(), Box> { // loading configuration from environment variables let _config = NotifierConfig::event_load_config(Some("./crates/event-notifier/examples/event.toml".to_string())); tracing::info!("event_load_config config: {:?} \n", _config); - + dotenvy::dotenv()?; + let _config = NotifierConfig::event_load_config(None); + tracing::info!("event_load_config config: {:?} \n", _config); let system = Arc::new(tokio::sync::Mutex::new(NotifierSystem::new(config.clone()).await?)); let adapters = create_adapters(&config.adapters)?; diff --git a/crates/event-notifier/src/adapter/mod.rs b/crates/event-notifier/src/adapter/mod.rs index fa12aa97b..426fd2d83 100644 --- a/crates/event-notifier/src/adapter/mod.rs +++ b/crates/event-notifier/src/adapter/mod.rs @@ -4,7 +4,7 @@ use crate::Event; use async_trait::async_trait; use std::sync::Arc; -#[cfg(feature = "kafka")] +#[cfg(all(feature = "kafka", target_os = "linux"))] pub(crate) mod kafka; #[cfg(feature = "mqtt")] pub(crate) mod mqtt; @@ -31,7 +31,7 @@ pub fn create_adapters(configs: &[AdapterConfig]) -> Result { adapters.push(Arc::new(kafka::KafkaAdapter::new(kafka_config)?)); } @@ -43,7 +43,7 @@ pub fn create_adapters(configs: &[AdapterConfig]) -> Result return Err(Error::FeatureDisabled("webhook")), - #[cfg(not(feature = "kafka"))] + #[cfg(any(not(feature = "kafka"), not(target_os = "linux")))] AdapterConfig::Kafka(_) => return Err(Error::FeatureDisabled("kafka")), #[cfg(not(feature = "mqtt"))] AdapterConfig::Mqtt(_) => return Err(Error::FeatureDisabled("mqtt")), diff --git a/crates/event-notifier/src/config.rs b/crates/event-notifier/src/config.rs index 10429c34b..3414f3fa6 100644 --- a/crates/event-notifier/src/config.rs +++ b/crates/event-notifier/src/config.rs @@ -1,4 +1,4 @@ -use config::{Config, Environment, File, FileFormat}; +use config::{Config, File, FileFormat}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::env; @@ -138,15 +138,6 @@ impl NotifierConfig { let app_config = Config::builder() .add_source(File::with_name(config_dir.as_str()).format(FileFormat::Toml).required(false)) .add_source(File::with_name(config_dir.as_str()).format(FileFormat::Yaml).required(false)) - .add_source( - Environment::default() - .prefix("NOTIFIER") - .prefix_separator("__") - .separator("__") - .list_separator("_") - .with_list_parse_key("adapters") - .try_parsing(true), - ) .build() .unwrap_or_default(); match app_config.try_deserialize::() { diff --git a/crates/event-notifier/src/error.rs b/crates/event-notifier/src/error.rs index e6c061de9..ebdaf899d 100644 --- a/crates/event-notifier/src/error.rs +++ b/crates/event-notifier/src/error.rs @@ -15,7 +15,7 @@ pub enum Error { Serde(#[from] serde_json::Error), #[error("HTTP error: {0}")] Http(#[from] reqwest::Error), - #[cfg(feature = "kafka")] + #[cfg(all(feature = "kafka", target_os = "linux"))] #[error("Kafka error: {0}")] Kafka(#[from] rdkafka::error::KafkaError), #[cfg(feature = "mqtt")] diff --git a/crates/event-notifier/src/lib.rs b/crates/event-notifier/src/lib.rs index 20ef935d2..fe2e5e3da 100644 --- a/crates/event-notifier/src/lib.rs +++ b/crates/event-notifier/src/lib.rs @@ -8,7 +8,7 @@ mod notifier; mod store; pub use adapter::create_adapters; -#[cfg(feature = "kafka")] +#[cfg(all(feature = "kafka", target_os = "linux"))] pub use adapter::kafka::KafkaAdapter; #[cfg(feature = "mqtt")] pub use adapter::mqtt::MqttAdapter; @@ -16,7 +16,7 @@ pub use adapter::mqtt::MqttAdapter; pub use adapter::webhook::WebhookAdapter; pub use adapter::ChannelAdapter; pub use bus::event_bus; -#[cfg(feature = "kafka")] +#[cfg(all(feature = "kafka", target_os = "linux"))] pub use config::KafkaConfig; #[cfg(feature = "mqtt")] pub use config::MqttConfig; diff --git a/crates/obs/Cargo.toml b/crates/obs/Cargo.toml index 26c264335..65968d519 100644 --- a/crates/obs/Cargo.toml +++ b/crates/obs/Cargo.toml @@ -13,11 +13,11 @@ workspace = true default = ["file"] file = [] gpu = ["dep:nvml-wrapper"] -kafka = ["dep:rdkafka"] webhook = ["dep:reqwest"] -full = ["file", "gpu", "kafka", "webhook"] +kafka = ["dep:rdkafka"] [dependencies] +rustfs-config = { workspace = true } async-trait = { workspace = true } chrono = { workspace = true } config = { workspace = true } @@ -37,12 +37,14 @@ tracing-error = { workspace = true } tracing-opentelemetry = { workspace = true } tracing-subscriber = { workspace = true, features = ["registry", "std", "fmt", "env-filter", "tracing-log", "time", "local-time", "json"] } tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] } -rdkafka = { workspace = true, features = ["tokio"], optional = true } reqwest = { workspace = true, optional = true, default-features = false } serde_json = { workspace = true } sysinfo = { workspace = true } thiserror = { workspace = true } +# Only enable kafka features and related dependencies on Linux +[target.'cfg(target_os = "linux")'.dependencies] +rdkafka = { workspace = true, features = ["tokio"], optional = true } [dev-dependencies] diff --git a/crates/obs/examples/config.toml b/crates/obs/examples/config.toml index c1b3df148..b649f8060 100644 --- a/crates/obs/examples/config.toml +++ b/crates/obs/examples/config.toml @@ -7,27 +7,28 @@ service_name = "rustfs_obs" service_version = "0.1.0" environments = "develop" logger_level = "debug" +local_logging_enabled = true -[sinks] -[sinks.kafka] -enabled = false -bootstrap_servers = "localhost:9092" -topic = "logs" -batch_size = 100 # Default is 100 if not specified -batch_timeout_ms = 1000 # Default is 1000ms if not specified +#[[sinks]] +#type = "Kafka" +#bootstrap_servers = "localhost:9092" +#topic = "logs" +#batch_size = 100 # Default is 100 if not specified +#batch_timeout_ms = 100 # Default is 1000ms if not specified +# +#[[sinks]] +#type = "Webhook" +#endpoint = "http://localhost:8080/webhook" +#auth_token = "" +#batch_size = 100 # Default is 3 if not specified +#batch_timeout_ms = 100 # Default is 100ms if not specified -[sinks.webhook] -enabled = false -endpoint = "http://localhost:8080/webhook" -auth_token = "" -batch_size = 100 # Default is 3 if not specified -batch_timeout_ms = 1000 # Default is 100ms if not specified - -[sinks.file] -enabled = true -path = "deploy/logs/app.log" -batch_size = 100 -batch_timeout_ms = 1000 # Default is 8192 bytes if not specified +[[sinks]] +type = "File" +path = "deploy/logs/rustfs.log" +buffer_size = 102 # Default is 8192 bytes if not specified +flush_interval_ms = 1000 +flush_threshold = 100 [logger] queue_capacity = 10000 \ No newline at end of file diff --git a/crates/obs/src/config.rs b/crates/obs/src/config.rs index 737d2c0d2..989294704 100644 --- a/crates/obs/src/config.rs +++ b/crates/obs/src/config.rs @@ -1,6 +1,6 @@ use crate::global::{ENVIRONMENT, LOGGER_LEVEL, METER_INTERVAL, SAMPLE_RATIO, SERVICE_NAME, SERVICE_VERSION, USE_STDOUT}; -use config::{Config, Environment, File, FileFormat}; -use serde::Deserialize; +use config::{Config, File, FileFormat}; +use serde::{Deserialize, Serialize}; use std::env; /// OpenTelemetry Configuration @@ -11,7 +11,7 @@ use std::env; /// Add use_stdout for output to stdout /// Add logger level for log level /// Add local_logging_enabled for local logging enabled -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Serialize, Clone)] pub struct OtelConfig { pub endpoint: String, // Endpoint for metric collection pub use_stdout: Option, // Output to stdout @@ -24,7 +24,7 @@ pub struct OtelConfig { pub local_logging_enabled: Option, // Local logging enabled } -// Helper function: Extract observable configuration from environment variables +/// Helper function: Extract observable configuration from environment variables fn extract_otel_config_from_env() -> OtelConfig { OtelConfig { endpoint: env::var("RUSTFS_OBSERVABILITY_ENDPOINT").unwrap_or_else(|_| "".to_string()), @@ -63,36 +63,89 @@ fn extract_otel_config_from_env() -> OtelConfig { } } -impl Default for OtelConfig { - fn default() -> Self { +impl OtelConfig { + /// Create a new instance of OtelConfig with default values + /// + /// # Returns + /// A new instance of OtelConfig + pub fn new() -> Self { extract_otel_config_from_env() } } +impl Default for OtelConfig { + fn default() -> Self { + Self::new() + } +} + /// Kafka Sink Configuration - Add batch parameters -#[derive(Debug, Deserialize, Clone, Default)] +#[derive(Debug, Deserialize, Serialize, Clone)] pub struct KafkaSinkConfig { - pub enabled: bool, - pub bootstrap_servers: String, + pub brokers: String, pub topic: String, pub batch_size: Option, // Batch size, default 100 pub batch_timeout_ms: Option, // Batch timeout time, default 1000ms } +impl KafkaSinkConfig { + pub fn new() -> Self { + Self { + brokers: env::var("RUSTFS__SINKS_0_KAFKA_BROKERS") + .ok() + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(|| "localhost:9092".to_string()), + topic: env::var("RUSTFS__SINKS_0_KAFKA_TOPIC") + .ok() + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(|| "default_topic".to_string()), + batch_size: Some(100), + batch_timeout_ms: Some(1000), + } + } +} + +impl Default for KafkaSinkConfig { + fn default() -> Self { + Self::new() + } +} + /// Webhook Sink Configuration - Add Retry Parameters -#[derive(Debug, Deserialize, Clone, Default)] +#[derive(Debug, Deserialize, Serialize, Clone)] pub struct WebhookSinkConfig { - pub enabled: bool, pub endpoint: String, pub auth_token: String, pub max_retries: Option, // Maximum number of retry times, default 3 pub retry_delay_ms: Option, // Retry the delay cardinality, default 100ms } +impl WebhookSinkConfig { + pub fn new() -> Self { + Self { + endpoint: env::var("RUSTFS__SINKS_0_WEBHOOK_ENDPOINT") + .ok() + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(|| "http://localhost:8080".to_string()), + auth_token: env::var("RUSTFS__SINKS_0_WEBHOOK_AUTH_TOKEN") + .ok() + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(|| "default_token".to_string()), + max_retries: Some(3), + retry_delay_ms: Some(100), + } + } +} + +impl Default for WebhookSinkConfig { + fn default() -> Self { + Self::new() + } +} + /// File Sink Configuration - Add buffering parameters -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Serialize, Clone)] pub struct FileSinkConfig { - pub enabled: bool, pub path: String, pub buffer_size: Option, // Write buffer size, default 8192 pub flush_interval_ms: Option, // Refresh interval time, default 1000ms @@ -114,13 +167,9 @@ impl FileSinkConfig { .unwrap_or("rustfs/rustfs.log") .to_string() } -} - -impl Default for FileSinkConfig { - fn default() -> Self { - FileSinkConfig { - enabled: true, - path: env::var("RUSTFS_SINKS_FILE_PATH") + pub fn new() -> Self { + Self { + path: env::var("RUSTFS__SINKS_0_FILE_PATH") .ok() .filter(|s| !s.trim().is_empty()) .unwrap_or_else(Self::get_default_log_path), @@ -131,38 +180,53 @@ impl Default for FileSinkConfig { } } +impl Default for FileSinkConfig { + fn default() -> Self { + Self::new() + } +} + /// Sink configuration collection -#[derive(Debug, Deserialize, Clone)] -pub struct SinkConfig { - pub kafka: Option, - pub webhook: Option, - pub file: Option, +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum SinkConfig { + File(FileSinkConfig), + Kafka(KafkaSinkConfig), + Webhook(WebhookSinkConfig), +} + +impl SinkConfig { + pub fn new() -> Self { + Self::File(FileSinkConfig::new()) + } } impl Default for SinkConfig { fn default() -> Self { - SinkConfig { - kafka: None, - webhook: None, - file: Some(FileSinkConfig::default()), - } + Self::new() } } ///Logger Configuration -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Serialize, Clone)] pub struct LoggerConfig { pub queue_capacity: Option, } -impl Default for LoggerConfig { - fn default() -> Self { - LoggerConfig { +impl LoggerConfig { + pub fn new() -> Self { + Self { queue_capacity: Some(10000), } } } +impl Default for LoggerConfig { + fn default() -> Self { + Self::new() + } +} + /// Overall application configuration /// Add observability, sinks, and logger configuration /// @@ -180,7 +244,7 @@ impl Default for LoggerConfig { #[derive(Debug, Deserialize, Clone)] pub struct AppConfig { pub observability: OtelConfig, - pub sinks: SinkConfig, + pub sinks: Vec, pub logger: Option, } @@ -192,7 +256,7 @@ impl AppConfig { pub fn new() -> Self { Self { observability: OtelConfig::default(), - sinks: SinkConfig::default(), + sinks: vec![SinkConfig::default()], logger: Some(LoggerConfig::default()), } } @@ -258,14 +322,6 @@ pub fn load_config(config_dir: Option) -> AppConfig { let app_config = Config::builder() .add_source(File::with_name(config_dir.as_str()).format(FileFormat::Toml).required(false)) .add_source(File::with_name(config_dir.as_str()).format(FileFormat::Yaml).required(false)) - .add_source( - Environment::default() - .prefix("RUSTFS") - .prefix_separator("__") - .separator("__") - .with_list_parse_key("volumes") - .try_parsing(true), - ) .build() .unwrap_or_default(); diff --git a/crates/obs/src/lib.rs b/crates/obs/src/lib.rs index d41d0e66d..5d181e319 100644 --- a/crates/obs/src/lib.rs +++ b/crates/obs/src/lib.rs @@ -32,7 +32,7 @@ mod config; mod entry; mod global; mod logger; -mod sink; +mod sinks; mod system; mod telemetry; mod utils; @@ -40,12 +40,6 @@ mod worker; use crate::logger::InitLogStatus; pub use config::load_config; -#[cfg(feature = "file")] -pub use config::FileSinkConfig; -#[cfg(feature = "kafka")] -pub use config::KafkaSinkConfig; -#[cfg(feature = "webhook")] -pub use config::WebhookSinkConfig; pub use config::{AppConfig, LoggerConfig, OtelConfig, SinkConfig}; pub use entry::args::Args; pub use entry::audit::{ApiDetails, AuditLogEntry}; @@ -79,7 +73,7 @@ use tracing::{error, info}; /// ``` pub async fn init_obs(config: AppConfig) -> (Arc>, telemetry::OtelGuard) { let guard = init_telemetry(&config.observability); - let sinks = sink::create_sinks(&config).await; + let sinks = sinks::create_sinks(&config).await; let logger = init_global_logger(&config, sinks).await; let obs_config = config.observability.clone(); tokio::spawn(async move { diff --git a/crates/obs/src/logger.rs b/crates/obs/src/logger.rs index 6329ab8fc..92ff5365e 100644 --- a/crates/obs/src/logger.rs +++ b/crates/obs/src/logger.rs @@ -1,5 +1,5 @@ use crate::global::{ENVIRONMENT, SERVICE_NAME, SERVICE_VERSION}; -use crate::sink::Sink; +use crate::sinks::Sink; use crate::{AppConfig, AuditLogEntry, BaseLogEntry, ConsoleLogEntry, GlobalError, OtelConfig, ServerLogEntry, UnifiedLogEntry}; use std::sync::Arc; use std::time::SystemTime; diff --git a/crates/obs/src/sink.rs b/crates/obs/src/sink.rs deleted file mode 100644 index 4df212b31..000000000 --- a/crates/obs/src/sink.rs +++ /dev/null @@ -1,497 +0,0 @@ -use crate::{AppConfig, LogRecord, UnifiedLogEntry}; -use async_trait::async_trait; -use std::sync::Arc; -use tokio::fs::OpenOptions; -use tokio::io; -use tokio::io::AsyncWriteExt; - -/// Sink Trait definition, asynchronously write logs -#[async_trait] -pub trait Sink: Send + Sync { - async fn write(&self, entry: &UnifiedLogEntry); -} - -#[cfg(feature = "kafka")] -/// Kafka Sink Implementation -pub struct KafkaSink { - producer: rdkafka::producer::FutureProducer, - topic: String, - batch_size: usize, - batch_timeout_ms: u64, - entries: Arc>>, - last_flush: Arc, -} - -#[cfg(feature = "kafka")] -impl KafkaSink { - /// Create a new KafkaSink instance - pub fn new(producer: rdkafka::producer::FutureProducer, topic: String, batch_size: usize, batch_timeout_ms: u64) -> Self { - // Create Arc-wrapped values first - let entries = Arc::new(tokio::sync::Mutex::new(Vec::with_capacity(batch_size))); - let last_flush = Arc::new(std::sync::atomic::AtomicU64::new( - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() as u64, - )); - let sink = KafkaSink { - producer: producer.clone(), - topic: topic.clone(), - batch_size, - batch_timeout_ms, - entries: entries.clone(), - last_flush: last_flush.clone(), - }; - - // Start background flusher - tokio::spawn(Self::periodic_flush(producer, topic, entries, last_flush, batch_timeout_ms)); - - sink - } - - /// Add a getter method to read the batch_timeout_ms field - #[allow(dead_code)] - pub fn batch_timeout(&self) -> u64 { - self.batch_timeout_ms - } - - /// Add a method to dynamically adjust the timeout if needed - #[allow(dead_code)] - pub fn set_batch_timeout(&mut self, new_timeout_ms: u64) { - self.batch_timeout_ms = new_timeout_ms; - } - - async fn periodic_flush( - producer: rdkafka::producer::FutureProducer, - topic: String, - entries: Arc>>, - last_flush: Arc, - timeout_ms: u64, - ) { - loop { - tokio::time::sleep(tokio::time::Duration::from_millis(timeout_ms / 2)).await; - - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() as u64; - - let last = last_flush.load(std::sync::atomic::Ordering::Relaxed); - - if now - last >= timeout_ms { - let mut batch = entries.lock().await; - if !batch.is_empty() { - Self::send_batch(&producer, &topic, batch.drain(..).collect()).await; - last_flush.store(now, std::sync::atomic::Ordering::Relaxed); - } - } - } - } - - async fn send_batch(producer: &rdkafka::producer::FutureProducer, topic: &str, entries: Vec) { - for entry in entries { - let payload = match serde_json::to_string(&entry) { - Ok(p) => p, - Err(e) => { - eprintln!("Failed to serialize log entry: {}", e); - continue; - } - }; - - let span_id = entry.get_timestamp().to_rfc3339(); - - let _ = producer - .send( - rdkafka::producer::FutureRecord::to(topic).payload(&payload).key(&span_id), - std::time::Duration::from_secs(5), - ) - .await; - } - } -} - -#[cfg(feature = "kafka")] -#[async_trait] -impl Sink for KafkaSink { - async fn write(&self, entry: &UnifiedLogEntry) { - let mut batch = self.entries.lock().await; - batch.push(entry.clone()); - - let should_flush_by_size = batch.len() >= self.batch_size; - let should_flush_by_time = { - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() as u64; - let last = self.last_flush.load(std::sync::atomic::Ordering::Relaxed); - now - last >= self.batch_timeout_ms - }; - - if should_flush_by_size || should_flush_by_time { - // Existing flush logic - let entries_to_send: Vec = batch.drain(..).collect(); - let producer = self.producer.clone(); - let topic = self.topic.clone(); - - self.last_flush.store( - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() as u64, - std::sync::atomic::Ordering::Relaxed, - ); - - tokio::spawn(async move { - KafkaSink::send_batch(&producer, &topic, entries_to_send).await; - }); - } - } -} - -#[cfg(feature = "kafka")] -impl Drop for KafkaSink { - fn drop(&mut self) { - // Perform any necessary cleanup here - // For example, you might want to flush any remaining entries - let producer = self.producer.clone(); - let topic = self.topic.clone(); - let entries = self.entries.clone(); - let last_flush = self.last_flush.clone(); - - tokio::spawn(async move { - let mut batch = entries.lock().await; - if !batch.is_empty() { - KafkaSink::send_batch(&producer, &topic, batch.drain(..).collect()).await; - last_flush.store( - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() as u64, - std::sync::atomic::Ordering::Relaxed, - ); - } - }); - - eprintln!("Dropping KafkaSink with topic: {}", self.topic); - } -} - -#[cfg(feature = "webhook")] -/// Webhook Sink Implementation -pub struct WebhookSink { - endpoint: String, - auth_token: String, - client: reqwest::Client, - max_retries: usize, - retry_delay_ms: u64, -} - -#[cfg(feature = "webhook")] -impl WebhookSink { - pub fn new(endpoint: String, auth_token: String, max_retries: usize, retry_delay_ms: u64) -> Self { - WebhookSink { - endpoint, - auth_token, - client: reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(10)) - .build() - .unwrap_or_else(|_| reqwest::Client::new()), - max_retries, - retry_delay_ms, - } - } -} - -#[cfg(feature = "webhook")] -#[async_trait] -impl Sink for WebhookSink { - async fn write(&self, entry: &UnifiedLogEntry) { - let mut retries = 0; - let url = self.endpoint.clone(); - let entry_clone = entry.clone(); - let auth_value = reqwest::header::HeaderValue::from_str(format!("Bearer {}", self.auth_token.clone()).as_str()).unwrap(); - while retries < self.max_retries { - match self - .client - .post(&url) - .header(reqwest::header::AUTHORIZATION, auth_value.clone()) - .json(&entry_clone) - .send() - .await - { - Ok(response) if response.status().is_success() => { - return; - } - _ => { - retries += 1; - if retries < self.max_retries { - tokio::time::sleep(tokio::time::Duration::from_millis( - self.retry_delay_ms * (1 << retries), // Exponential backoff - )) - .await; - } - } - } - } - - eprintln!("Failed to send log to webhook after {} retries", self.max_retries); - } -} - -#[cfg(feature = "webhook")] -impl Drop for WebhookSink { - fn drop(&mut self) { - // Perform any necessary cleanup here - // For example, you might want to log that the sink is being dropped - eprintln!("Dropping WebhookSink with URL: {}", self.endpoint); - } -} - -#[cfg(feature = "file")] -/// File Sink Implementation -pub struct FileSink { - path: String, - buffer_size: usize, - writer: Arc>>, - entry_count: std::sync::atomic::AtomicUsize, - last_flush: std::sync::atomic::AtomicU64, - flush_interval_ms: u64, // Time between flushes - flush_threshold: usize, // Number of entries before flush -} - -#[cfg(feature = "file")] -impl FileSink { - /// Create a new FileSink instance - pub async fn new( - path: String, - buffer_size: usize, - flush_interval_ms: u64, - flush_threshold: usize, - ) -> Result { - // check if the file exists - let file_exists = tokio::fs::metadata(&path).await.is_ok(); - // if the file not exists, create it - if !file_exists { - tokio::fs::create_dir_all(std::path::Path::new(&path).parent().unwrap()).await?; - tracing::debug!("File does not exist, creating it. Path: {:?}", path) - } - let file = if file_exists { - // If the file exists, open it in append mode - tracing::debug!("FileSink: File exists, opening in append mode."); - OpenOptions::new().append(true).create(true).open(&path).await? - } else { - // If the file does not exist, create it - tracing::debug!("FileSink: File does not exist, creating a new file."); - // Create the file and write a header or initial content if needed - OpenOptions::new().create(true).truncate(true).write(true).open(&path).await? - }; - let writer = io::BufWriter::with_capacity(buffer_size, file); - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() as u64; - Ok(FileSink { - path, - buffer_size, - writer: Arc::new(tokio::sync::Mutex::new(writer)), - entry_count: std::sync::atomic::AtomicUsize::new(0), - last_flush: std::sync::atomic::AtomicU64::new(now), - flush_interval_ms, - flush_threshold, - }) - } - - #[allow(dead_code)] - async fn initialize_writer(&mut self) -> io::Result<()> { - let file = tokio::fs::File::create(&self.path).await?; - - // Use buffer_size to create a buffer writer with a specified capacity - let buf_writer = io::BufWriter::with_capacity(self.buffer_size, file); - - // Replace the original writer with the new Mutex - self.writer = Arc::new(tokio::sync::Mutex::new(buf_writer)); - Ok(()) - } - - // Get the current buffer size - #[allow(dead_code)] - pub fn buffer_size(&self) -> usize { - self.buffer_size - } - - // How to dynamically adjust the buffer size - #[allow(dead_code)] - pub async fn set_buffer_size(&mut self, new_size: usize) -> io::Result<()> { - if self.buffer_size != new_size { - self.buffer_size = new_size; - // Reinitialize the writer directly, without checking is_some() - self.initialize_writer().await?; - } - Ok(()) - } - - // Check if flushing is needed based on count or time - fn should_flush(&self) -> bool { - // Check entry count threshold - if self.entry_count.load(std::sync::atomic::Ordering::Relaxed) >= self.flush_threshold { - return true; - } - - // Check time threshold - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() as u64; - - let last = self.last_flush.load(std::sync::atomic::Ordering::Relaxed); - now - last >= self.flush_interval_ms - } -} - -#[cfg(feature = "file")] -#[async_trait] -impl Sink for FileSink { - async fn write(&self, entry: &UnifiedLogEntry) { - let line = format!("{:?}\n", entry); - let mut writer = self.writer.lock().await; - - if let Err(e) = writer.write_all(line.as_bytes()).await { - eprintln!( - "Failed to write log to file {}: {},entry timestamp:{:?}", - self.path, - e, - entry.get_timestamp() - ); - return; - } - - // Only flush periodically to improve performance - // Logic to determine when to flush could be added here - // Increment the entry count - self.entry_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - - // Check if we should flush - if self.should_flush() { - if let Err(e) = writer.flush().await { - eprintln!("Failed to flush log file {}: {}", self.path, e); - return; - } - - // Reset counters - self.entry_count.store(0, std::sync::atomic::Ordering::Relaxed); - - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() as u64; - - self.last_flush.store(now, std::sync::atomic::Ordering::Relaxed); - } - } -} - -#[cfg(feature = "file")] -impl Drop for FileSink { - fn drop(&mut self) { - let writer = self.writer.clone(); - let path = self.path.clone(); - - tokio::task::spawn_blocking(move || { - let rt = tokio::runtime::Runtime::new().unwrap(); - rt.block_on(async { - let mut writer = writer.lock().await; - if let Err(e) = writer.flush().await { - eprintln!("Failed to flush log file {}: {}", path, e); - } - }); - }); - } -} - -/// Create a list of Sink instances -pub async fn create_sinks(config: &AppConfig) -> Vec> { - let mut sinks: Vec> = Vec::new(); - - #[cfg(feature = "kafka")] - { - match &config.sinks.kafka { - Some(sink_kafka) => { - if sink_kafka.enabled { - match rdkafka::config::ClientConfig::new() - .set("bootstrap.servers", &sink_kafka.bootstrap_servers) - .set("message.timeout.ms", "5000") - .create() - { - Ok(producer) => { - sinks.push(Arc::new(KafkaSink::new( - producer, - sink_kafka.topic.clone(), - sink_kafka.batch_size.unwrap_or(100), - sink_kafka.batch_timeout_ms.unwrap_or(1000), - ))); - } - Err(e) => { - tracing::error!("Failed to create Kafka producer: {}", e); - } - } - } else { - tracing::info!("Kafka sink is disabled in the configuration"); - } - } - _ => { - tracing::info!("Kafka sink is not configured or disabled"); - } - } - } - #[cfg(feature = "webhook")] - { - match &config.sinks.webhook { - Some(sink_webhook) => { - if sink_webhook.enabled { - sinks.push(Arc::new(WebhookSink::new( - sink_webhook.endpoint.clone(), - sink_webhook.auth_token.clone(), - sink_webhook.max_retries.unwrap_or(3), - sink_webhook.retry_delay_ms.unwrap_or(100), - ))); - } else { - tracing::info!("Webhook sink is disabled in the configuration"); - } - } - _ => { - tracing::info!("Webhook sink is not configured or disabled"); - } - } - } - - #[cfg(feature = "file")] - { - // let config = config.clone(); - match &config.sinks.file { - Some(sink_file) => { - tracing::info!("File sink is enabled in the configuration"); - let path = if sink_file.enabled { - sink_file.path.clone() - } else { - "rustfs.log".to_string() - }; - tracing::debug!("FileSink: Using path: {}", path); - sinks.push(Arc::new( - FileSink::new( - path.clone(), - sink_file.buffer_size.unwrap_or(8192), - sink_file.flush_interval_ms.unwrap_or(1000), - sink_file.flush_threshold.unwrap_or(100), - ) - .await - .unwrap(), - )); - } - _ => { - tracing::info!("File sink is not configured or disabled"); - } - } - } - - sinks -} diff --git a/crates/obs/src/sinks/file.rs b/crates/obs/src/sinks/file.rs new file mode 100644 index 000000000..3e2b0db41 --- /dev/null +++ b/crates/obs/src/sinks/file.rs @@ -0,0 +1,164 @@ +use crate::sinks::Sink; +use crate::{LogRecord, UnifiedLogEntry}; +use async_trait::async_trait; +use std::sync::Arc; +use tokio::fs::OpenOptions; +use tokio::io; +use tokio::io::AsyncWriteExt; + +/// File Sink Implementation +pub struct FileSink { + path: String, + buffer_size: usize, + writer: Arc>>, + entry_count: std::sync::atomic::AtomicUsize, + last_flush: std::sync::atomic::AtomicU64, + flush_interval_ms: u64, // Time between flushes + flush_threshold: usize, // Number of entries before flush +} + +impl FileSink { + /// Create a new FileSink instance + pub async fn new( + path: String, + buffer_size: usize, + flush_interval_ms: u64, + flush_threshold: usize, + ) -> Result { + // check if the file exists + let file_exists = tokio::fs::metadata(&path).await.is_ok(); + // if the file not exists, create it + if !file_exists { + tokio::fs::create_dir_all(std::path::Path::new(&path).parent().unwrap()).await?; + tracing::debug!("File does not exist, creating it. Path: {:?}", path) + } + let file = if file_exists { + // If the file exists, open it in append mode + tracing::debug!("FileSink: File exists, opening in append mode."); + OpenOptions::new().append(true).create(true).open(&path).await? + } else { + // If the file does not exist, create it + tracing::debug!("FileSink: File does not exist, creating a new file."); + // Create the file and write a header or initial content if needed + OpenOptions::new().create(true).truncate(true).write(true).open(&path).await? + }; + let writer = io::BufWriter::with_capacity(buffer_size, file); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64; + Ok(FileSink { + path, + buffer_size, + writer: Arc::new(tokio::sync::Mutex::new(writer)), + entry_count: std::sync::atomic::AtomicUsize::new(0), + last_flush: std::sync::atomic::AtomicU64::new(now), + flush_interval_ms, + flush_threshold, + }) + } + + #[allow(dead_code)] + async fn initialize_writer(&mut self) -> io::Result<()> { + let file = tokio::fs::File::create(&self.path).await?; + + // Use buffer_size to create a buffer writer with a specified capacity + let buf_writer = io::BufWriter::with_capacity(self.buffer_size, file); + + // Replace the original writer with the new Mutex + self.writer = Arc::new(tokio::sync::Mutex::new(buf_writer)); + Ok(()) + } + + // Get the current buffer size + #[allow(dead_code)] + pub fn buffer_size(&self) -> usize { + self.buffer_size + } + + // How to dynamically adjust the buffer size + #[allow(dead_code)] + pub async fn set_buffer_size(&mut self, new_size: usize) -> io::Result<()> { + if self.buffer_size != new_size { + self.buffer_size = new_size; + // Reinitialize the writer directly, without checking is_some() + self.initialize_writer().await?; + } + Ok(()) + } + + // Check if flushing is needed based on count or time + fn should_flush(&self) -> bool { + // Check entry count threshold + if self.entry_count.load(std::sync::atomic::Ordering::Relaxed) >= self.flush_threshold { + return true; + } + + // Check time threshold + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64; + + let last = self.last_flush.load(std::sync::atomic::Ordering::Relaxed); + now - last >= self.flush_interval_ms + } +} + +#[async_trait] +impl Sink for FileSink { + async fn write(&self, entry: &UnifiedLogEntry) { + let line = format!("{:?}\n", entry); + let mut writer = self.writer.lock().await; + + if let Err(e) = writer.write_all(line.as_bytes()).await { + eprintln!( + "Failed to write log to file {}: {},entry timestamp:{:?}", + self.path, + e, + entry.get_timestamp() + ); + return; + } + + // Only flush periodically to improve performance + // Logic to determine when to flush could be added here + // Increment the entry count + self.entry_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + + // Check if we should flush + if self.should_flush() { + if let Err(e) = writer.flush().await { + eprintln!("Failed to flush log file {}: {}", self.path, e); + return; + } + + // Reset counters + self.entry_count.store(0, std::sync::atomic::Ordering::Relaxed); + + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64; + + self.last_flush.store(now, std::sync::atomic::Ordering::Relaxed); + } + } +} + +impl Drop for FileSink { + fn drop(&mut self) { + let writer = self.writer.clone(); + let path = self.path.clone(); + + tokio::task::spawn_blocking(move || { + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + let mut writer = writer.lock().await; + if let Err(e) = writer.flush().await { + eprintln!("Failed to flush log file {}: {}", path, e); + } + }); + }); + } +} diff --git a/crates/obs/src/sinks/kafka.rs b/crates/obs/src/sinks/kafka.rs new file mode 100644 index 000000000..e4ef34199 --- /dev/null +++ b/crates/obs/src/sinks/kafka.rs @@ -0,0 +1,165 @@ +use crate::sinks::Sink; +use crate::{LogRecord, UnifiedLogEntry}; +use async_trait::async_trait; +use std::sync::Arc; + +/// Kafka Sink Implementation +pub struct KafkaSink { + producer: rdkafka::producer::FutureProducer, + topic: String, + batch_size: usize, + batch_timeout_ms: u64, + entries: Arc>>, + last_flush: Arc, +} + +impl KafkaSink { + /// Create a new KafkaSink instance + pub fn new(producer: rdkafka::producer::FutureProducer, topic: String, batch_size: usize, batch_timeout_ms: u64) -> Self { + // Create Arc-wrapped values first + let entries = Arc::new(tokio::sync::Mutex::new(Vec::with_capacity(batch_size))); + let last_flush = Arc::new(std::sync::atomic::AtomicU64::new( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64, + )); + let sink = KafkaSink { + producer: producer.clone(), + topic: topic.clone(), + batch_size, + batch_timeout_ms, + entries: entries.clone(), + last_flush: last_flush.clone(), + }; + + // Start background flusher + tokio::spawn(Self::periodic_flush(producer, topic, entries, last_flush, batch_timeout_ms)); + + sink + } + + /// Add a getter method to read the batch_timeout_ms field + #[allow(dead_code)] + pub fn batch_timeout(&self) -> u64 { + self.batch_timeout_ms + } + + /// Add a method to dynamically adjust the timeout if needed + #[allow(dead_code)] + pub fn set_batch_timeout(&mut self, new_timeout_ms: u64) { + self.batch_timeout_ms = new_timeout_ms; + } + + async fn periodic_flush( + producer: rdkafka::producer::FutureProducer, + topic: String, + entries: Arc>>, + last_flush: Arc, + timeout_ms: u64, + ) { + loop { + tokio::time::sleep(tokio::time::Duration::from_millis(timeout_ms / 2)).await; + + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64; + + let last = last_flush.load(std::sync::atomic::Ordering::Relaxed); + + if now - last >= timeout_ms { + let mut batch = entries.lock().await; + if !batch.is_empty() { + Self::send_batch(&producer, &topic, batch.drain(..).collect()).await; + last_flush.store(now, std::sync::atomic::Ordering::Relaxed); + } + } + } + } + + async fn send_batch(producer: &rdkafka::producer::FutureProducer, topic: &str, entries: Vec) { + for entry in entries { + let payload = match serde_json::to_string(&entry) { + Ok(p) => p, + Err(e) => { + eprintln!("Failed to serialize log entry: {}", e); + continue; + } + }; + + let span_id = entry.get_timestamp().to_rfc3339(); + + let _ = producer + .send( + rdkafka::producer::FutureRecord::to(topic).payload(&payload).key(&span_id), + std::time::Duration::from_secs(5), + ) + .await; + } + } +} + +#[async_trait] +impl Sink for KafkaSink { + async fn write(&self, entry: &UnifiedLogEntry) { + let mut batch = self.entries.lock().await; + batch.push(entry.clone()); + + let should_flush_by_size = batch.len() >= self.batch_size; + let should_flush_by_time = { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64; + let last = self.last_flush.load(std::sync::atomic::Ordering::Relaxed); + now - last >= self.batch_timeout_ms + }; + + if should_flush_by_size || should_flush_by_time { + // Existing flush logic + let entries_to_send: Vec = batch.drain(..).collect(); + let producer = self.producer.clone(); + let topic = self.topic.clone(); + + self.last_flush.store( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64, + std::sync::atomic::Ordering::Relaxed, + ); + + tokio::spawn(async move { + KafkaSink::send_batch(&producer, &topic, entries_to_send).await; + }); + } + } +} + +impl Drop for KafkaSink { + fn drop(&mut self) { + // Perform any necessary cleanup here + // For example, you might want to flush any remaining entries + let producer = self.producer.clone(); + let topic = self.topic.clone(); + let entries = self.entries.clone(); + let last_flush = self.last_flush.clone(); + + tokio::spawn(async move { + let mut batch = entries.lock().await; + if !batch.is_empty() { + KafkaSink::send_batch(&producer, &topic, batch.drain(..).collect()).await; + last_flush.store( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64, + std::sync::atomic::Ordering::Relaxed, + ); + } + }); + + eprintln!("Dropping KafkaSink with topic: {}", self.topic); + } +} diff --git a/crates/obs/src/sinks/mod.rs b/crates/obs/src/sinks/mod.rs new file mode 100644 index 000000000..3abafd3fe --- /dev/null +++ b/crates/obs/src/sinks/mod.rs @@ -0,0 +1,92 @@ +use crate::{AppConfig, SinkConfig, UnifiedLogEntry}; +use async_trait::async_trait; +use std::sync::Arc; + +#[cfg(feature = "file")] +mod file; +#[cfg(all(feature = "kafka", target_os = "linux"))] +mod kafka; +#[cfg(feature = "webhook")] +mod webhook; + +/// Sink Trait definition, asynchronously write logs +#[async_trait] +pub trait Sink: Send + Sync { + async fn write(&self, entry: &UnifiedLogEntry); +} + +/// Create a list of Sink instances +pub async fn create_sinks(config: &AppConfig) -> Vec> { + let mut sinks: Vec> = Vec::new(); + + for sink_config in &config.sinks { + match sink_config { + #[cfg(all(feature = "kafka", target_os = "linux"))] + SinkConfig::Kafka(kafka_config) => { + match rdkafka::config::ClientConfig::new() + .set("bootstrap.servers", &kafka_config.brokers) + .set("message.timeout.ms", "5000") + .create() + { + Ok(producer) => { + sinks.push(Arc::new(kafka::KafkaSink::new( + producer, + kafka_config.topic.clone(), + kafka_config.batch_size.unwrap_or(100), + kafka_config.batch_timeout_ms.unwrap_or(1000), + ))); + tracing::info!("Kafka sink created for topic: {}", kafka_config.topic); + } + Err(e) => { + tracing::error!("Failed to create Kafka producer: {}", e); + } + } + } + #[cfg(feature = "webhook")] + SinkConfig::Webhook(webhook_config) => { + sinks.push(Arc::new(webhook::WebhookSink::new( + webhook_config.endpoint.clone(), + webhook_config.auth_token.clone(), + webhook_config.max_retries.unwrap_or(3), + webhook_config.retry_delay_ms.unwrap_or(100), + ))); + tracing::info!("Webhook sink created for endpoint: {}", webhook_config.endpoint); + } + + #[cfg(feature = "file")] + SinkConfig::File(file_config) => { + tracing::debug!("FileSink: Using path: {}", file_config.path); + match file::FileSink::new( + file_config.path.clone(), + file_config.buffer_size.unwrap_or(8192), + file_config.flush_interval_ms.unwrap_or(1000), + file_config.flush_threshold.unwrap_or(100), + ) + .await + { + Ok(sink) => { + sinks.push(Arc::new(sink)); + tracing::info!("File sink created for path: {}", file_config.path); + } + Err(e) => { + tracing::error!("Failed to create File sink: {}", e); + } + } + } + #[cfg(any(not(feature = "kafka"), not(target_os = "linux")))] + SinkConfig::Kafka(_) => { + tracing::warn!("Kafka sink is configured but the 'kafka' feature is not enabled"); + } + #[cfg(not(feature = "webhook"))] + SinkConfig::Webhook(_) => { + tracing::warn!("Webhook sink is configured but the 'webhook' feature is not enabled"); + } + #[cfg(not(feature = "file"))] + SinkConfig::File(_) => { + tracing::warn!("File sink is configured but the 'file' feature is not enabled"); + } + } + } + + sinks +} diff --git a/crates/obs/src/sinks/webhook.rs b/crates/obs/src/sinks/webhook.rs new file mode 100644 index 000000000..77a874d9f --- /dev/null +++ b/crates/obs/src/sinks/webhook.rs @@ -0,0 +1,70 @@ +use crate::sinks::Sink; +use crate::UnifiedLogEntry; +use async_trait::async_trait; + +/// Webhook Sink Implementation +pub struct WebhookSink { + endpoint: String, + auth_token: String, + client: reqwest::Client, + max_retries: usize, + retry_delay_ms: u64, +} + +impl WebhookSink { + pub fn new(endpoint: String, auth_token: String, max_retries: usize, retry_delay_ms: u64) -> Self { + WebhookSink { + endpoint, + auth_token, + client: reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(10)) + .build() + .unwrap_or_else(|_| reqwest::Client::new()), + max_retries, + retry_delay_ms, + } + } +} + +#[async_trait] +impl Sink for WebhookSink { + async fn write(&self, entry: &UnifiedLogEntry) { + let mut retries = 0; + let url = self.endpoint.clone(); + let entry_clone = entry.clone(); + let auth_value = reqwest::header::HeaderValue::from_str(format!("Bearer {}", self.auth_token.clone()).as_str()).unwrap(); + while retries < self.max_retries { + match self + .client + .post(&url) + .header(reqwest::header::AUTHORIZATION, auth_value.clone()) + .json(&entry_clone) + .send() + .await + { + Ok(response) if response.status().is_success() => { + return; + } + _ => { + retries += 1; + if retries < self.max_retries { + tokio::time::sleep(tokio::time::Duration::from_millis( + self.retry_delay_ms * (1 << retries), // Exponential backoff + )) + .await; + } + } + } + } + + eprintln!("Failed to send log to webhook after {} retries", self.max_retries); + } +} + +impl Drop for WebhookSink { + fn drop(&mut self) { + // Perform any necessary cleanup here + // For example, you might want to log that the sink is being dropped + eprintln!("Dropping WebhookSink with URL: {}", self.endpoint); + } +} diff --git a/crates/obs/src/worker.rs b/crates/obs/src/worker.rs index 2d7ee2e14..aee1695d6 100644 --- a/crates/obs/src/worker.rs +++ b/crates/obs/src/worker.rs @@ -1,4 +1,4 @@ -use crate::{sink::Sink, UnifiedLogEntry}; +use crate::{sinks::Sink, UnifiedLogEntry}; use std::sync::Arc; use tokio::sync::mpsc::Receiver; diff --git a/crates/utils/src/certs.rs b/crates/utils/src/certs.rs index 568fc6b69..021c5915e 100644 --- a/crates/utils/src/certs.rs +++ b/crates/utils/src/certs.rs @@ -20,7 +20,7 @@ pub fn load_certs(filename: &str) -> io::Result>> { // Load and return certificate. let certs = certs(&mut reader) .collect::, _>>() - .map_err(|_| certs_error(format!("certificate file {} format error", filename)))?; + .map_err(|e| certs_error(format!("certificate file {} format error:{:?}", filename, e)))?; if certs.is_empty() { return Err(certs_error(format!( "No valid certificate was found in the certificate file {}", @@ -165,7 +165,7 @@ pub fn create_multi_cert_resolver( for (domain, (certs, key)) in cert_key_pairs { // create a signature let signing_key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key) - .map_err(|_| certs_error(format!("unsupported private key types:{}", domain)))?; + .map_err(|e| certs_error(format!("unsupported private key types:{}, err:{:?}", domain, e)))?; // create a CertifiedKey let certified_key = CertifiedKey::new(certs, signing_key); diff --git a/crypto/Cargo.toml b/crypto/Cargo.toml index 5b0f58969..7822ee5a6 100644 --- a/crypto/Cargo.toml +++ b/crypto/Cargo.toml @@ -17,7 +17,7 @@ chacha20poly1305 = { version = "0.10.1", optional = true } jsonwebtoken = { workspace = true } pbkdf2 = { version = "0.12.2", optional = true } rand = { workspace = true, optional = true } -sha2 = { version = "0.10.8", optional = true } +sha2 = { workspace = true, optional = true } thiserror.workspace = true serde_json.workspace = true diff --git a/deploy/config/.example.obs.env b/deploy/config/.example.obs.env index a1ea67b4f..edbf3e887 100644 --- a/deploy/config/.example.obs.env +++ b/deploy/config/.example.obs.env @@ -1,27 +1,28 @@ -OBSERVABILITY__ENDPOINT=http://localhost:4317 -OBSERVABILITY__USE_STDOUT=true -OBSERVABILITY__SAMPLE_RATIO=2.0 -OBSERVABILITY__METER_INTERVAL=30 -OBSERVABILITY__SERVICE_NAME=rustfs -OBSERVABILITY__SERVICE_VERSION=0.1.0 -OBSERVABILITY__ENVIRONMENT=develop -OBSERVABILITY__LOGGER_LEVEL=debug - -SINKS__KAFKA__ENABLED=false -SINKS__KAFKA__BOOTSTRAP_SERVERS=localhost:9092 -SINKS__KAFKA__TOPIC=logs -SINKS__KAFKA__BATCH_SIZE=100 -SINKS__KAFKA__BATCH_TIMEOUT_MS=1000 - -SINKS__WEBHOOK__ENABLED=false -SINKS__WEBHOOK__ENDPOINT=http://localhost:8080/webhook -SINKS__WEBHOOK__AUTH_TOKEN= -SINKS__WEBHOOK__BATCH_SIZE=100 -SINKS__WEBHOOK__BATCH_TIMEOUT_MS=1000 - -SINKS__FILE__ENABLED=true -SINKS__FILE__PATH=./deploy/logs/rustfs.log -SINKS__FILE__BATCH_SIZE=10 -SINKS__FILE__BATCH_TIMEOUT_MS=1000 - -LOGGER__QUEUE_CAPACITY=10 \ No newline at end of file +#RUSTFS__OBSERVABILITY__ENDPOINT=http://localhost:4317 +#RUSTFS__OBSERVABILITY__USE_STDOUT=true +#RUSTFS__OBSERVABILITY__SAMPLE_RATIO=2.0 +#RUSTFS__OBSERVABILITY__METER_INTERVAL=30 +#RUSTFS__OBSERVABILITY__SERVICE_NAME=rustfs +#RUSTFS__OBSERVABILITY__SERVICE_VERSION=0.1.0 +#RUSTFS__OBSERVABILITY__ENVIRONMENT=develop +#RUSTFS__OBSERVABILITY__LOGGER_LEVEL=debug +# +#RUSTFS__SINKS_0__type=Kakfa +#RUSTFS__SINKS_0__brokers=localhost:9092 +#RUSTFS__SINKS_0__topic=logs +#RUSTFS__SINKS_0__batch_size=100 +#RUSTFS__SINKS_0__batch_timeout_ms=1000 +# +#RUSTFS__SINKS_1__type=Webhook +#RUSTFS__SINKS_1__endpoint=http://localhost:8080/webhook +#RUSTFS__SINKS_1__auth_token=you-auth-token +#RUSTFS__SINKS_1__batch_size=100 +#RUSTFS__SINKS_1__batch_timeout_ms=1000 +# +#RUSTFS__SINKS_2__type=File +#RUSTFS__SINKS_2__path=./deploy/logs/rustfs.log +#RUSTFS__SINKS_2__buffer_size=10 +#RUSTFS__SINKS_2__flush_interval_ms=1000 +#RUSTFS__SINKS_2__flush_threshold=100 +# +#RUSTFS__LOGGER__QUEUE_CAPACITY=10 \ No newline at end of file diff --git a/deploy/config/obs-zh.example.toml b/deploy/config/obs-zh.example.toml index 712ae6cef..0474391bd 100644 --- a/deploy/config/obs-zh.example.toml +++ b/deploy/config/obs-zh.example.toml @@ -9,26 +9,26 @@ environments = "develop" # 运行环境，如开发环境 (develop) logger_level = "debug" # 日志级别，可选 debug/info/warn/error 等 local_logging_enabled = true # 是否启用本地 stdout 日志记录，true 表示启用，false 表示禁用 -[sinks] -[sinks.kafka] # Kafka 接收器配置 -enabled = false # 是否启用 Kafka 接收器，默认禁用 -bootstrap_servers = "localhost:9092" # Kafka 服务器地址 -topic = "logs" # Kafka 主题名称 -batch_size = 100 # 批处理大小，每次发送的消息数量 -batch_timeout_ms = 1000 # 批处理超时时间，单位为毫秒 +#[[sinks]] # Kafka 接收器配置 +#type = "Kafka" # 是否启用 Kafka 接收器，默认禁用 +#brokers = "localhost:9092" # Kafka 服务器地址 +#topic = "logs" # Kafka 主题名称 +#batch_size = 100 # 批处理大小，每次发送的消息数量 +#batch_timeout_ms = 1000 # 批处理超时时间，单位为毫秒 -[sinks.webhook] # Webhook 接收器配置 -enabled = false # 是否启用 Webhook 接收器 -endpoint = "http://localhost:8080/webhook" # Webhook 接收地址 -auth_token = "" # 认证令牌 -batch_size = 100 # 批处理大小 -batch_timeout_ms = 1000 # 批处理超时时间，单位为毫秒 +#[[sinks]] # Webhook 接收器配置 +#type = "Webhook" # 是否启用 Webhook 接收器 +#endpoint = "http://localhost:8080/webhook" # Webhook 接收地址 +#auth_token = "" # 认证令牌 +#batch_size = 100 # 批处理大小 +#batch_timeout_ms = 1000 # 批处理超时时间，单位为毫秒 -[sinks.file] # 文件接收器配置 -enabled = true # 是否启用文件接收器 +[[sinks]] # 文件接收器配置 +type = "File" # 是否启用文件接收器 path = "./deploy/logs/rustfs.log" # 日志文件路径 -batch_size = 10 # 批处理大小 -batch_timeout_ms = 1000 # 批处理超时时间，单位为毫秒 +buffer_size = 10 # 缓冲区大小，表示每次写入的字节数 +flush_interval_ms = 100 # 批处理超时时间，单位为毫秒 +flush_threshold = 100 # 刷新阈值，表示在达到该数量时刷新日志 [logger] # 日志器配置 queue_capacity = 10 # 日志队列容量，表示可以缓存的日志条数 \ No newline at end of file diff --git a/deploy/config/obs.example.toml b/deploy/config/obs.example.toml index e38e06f81..e6c898339 100644 --- a/deploy/config/obs.example.toml +++ b/deploy/config/obs.example.toml @@ -4,31 +4,31 @@ use_stdout = false # Output with stdout, true output, false no output sample_ratio = 2.0 meter_interval = 30 service_name = "rustfs" -service_version = "0.1.0" +service_version = "0.0.1" environment = "develop" -logger_level = "error" +logger_level = "info" local_logging_enabled = true -[sinks] -[sinks.kafka] # Kafka sink is disabled by default -enabled = false -bootstrap_servers = "localhost:9092" -topic = "logs" -batch_size = 100 # Default is 100 if not specified -batch_timeout_ms = 1000 # Default is 1000ms if not specified +#[[sinks]] +#type = "Kafka" +#brokers = "localhost:9092" +#topic = "logs" +#batch_size = 100 # Default is 100 if not specified +#batch_timeout_ms = 100 # Default is 1000ms if not specified +# +#[[sinks]] +#type = "Webhook" +#endpoint = "http://localhost:8080/webhook" +#auth_token = "" +#batch_size = 100 # Default is 3 if not specified +#batch_timeout_ms = 100 # Default is 100ms if not specified -[sinks.webhook] -enabled = false -endpoint = "http://localhost:8080/webhook" -auth_token = "" -batch_size = 100 # Default is 3 if not specified -batch_timeout_ms = 1000 # Default is 100ms if not specified - -[sinks.file] -enabled = true -path = "./deploy/logs/rustfs.log" -batch_size = 100 -batch_timeout_ms = 1000 # Default is 8192 bytes if not specified +[[sinks]] +type = "File" +path = "deploy/logs/rustfs.log" +buffer_size = 101 # Default is 8192 bytes if not specified +flush_interval_ms = 1000 +flush_threshold = 100 [logger] queue_capacity = 10000 diff --git a/deploy/config/rustfs-zh.env b/deploy/config/rustfs-zh.env index b9700adb8..fd2a5178b 100644 --- a/deploy/config/rustfs-zh.env +++ b/deploy/config/rustfs-zh.env @@ -23,4 +23,6 @@ RUSTFS_LICENSE="license content" # 可观测性配置文件路径：deploy/config/obs.example.toml RUSTFS_OBS_CONFIG=/etc/default/obs.toml # TLS 证书目录路径：deploy/certs -RUSTFS_TLS_PATH=/etc/default/tls \ No newline at end of file +RUSTFS_TLS_PATH=/etc/default/tls +# 事件通知配置文件路径：deploy/config/event.example.toml +RUSTFS_EVENT_CONFIG=/etc/default/event.toml \ No newline at end of file diff --git a/deploy/config/rustfs.env b/deploy/config/rustfs.env index ec7d31b48..3f50033f4 100644 --- a/deploy/config/rustfs.env +++ b/deploy/config/rustfs.env @@ -23,4 +23,6 @@ RUSTFS_LICENSE="license content" # Observability configuration file path: deploy/config/obs.example.toml RUSTFS_OBS_CONFIG=/etc/default/obs.toml # TLS certificates directory path: deploy/certs -RUSTFS_TLS_PATH=/etc/default/tls \ No newline at end of file +RUSTFS_TLS_PATH=/etc/default/tls +# event notification configuration file path: deploy/config/event.example.toml +RUSTFS_EVENT_CONFIG=/etc/default/event.toml \ No newline at end of file diff --git a/docker-compose-obs.yaml b/docker-compose-obs.yaml index f6d85b449..a709587b8 100644 --- a/docker-compose-obs.yaml +++ b/docker-compose-obs.yaml @@ -83,7 +83,7 @@ services: dockerfile: Dockerfile.obs container_name: node2 environment: - - RUSTFS_VOLUMES=/root/data/target/volume/test{1...4} + - RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4} - RUSTFS_ADDRESS=:9000 - RUSTFS_CONSOLE_ENABLE=true - RUSTFS_CONSOLE_ADDRESS=:9002 diff --git a/ecstore/src/config/com.rs b/ecstore/src/config/com.rs index 383359ea1..43e48c6d2 100644 --- a/ecstore/src/config/com.rs +++ b/ecstore/src/config/com.rs @@ -204,7 +204,7 @@ async fn apply_dynamic_config_for_sub_sys(cfg: &mut Config, api: } } Err(err) => { - error!("init storageclass err:{:?}", &err); + error!("init storage class err:{:?}", &err); break; } } diff --git a/ecstore/src/config/mod.rs b/ecstore/src/config/mod.rs index ffe00477c..10a5d50fd 100644 --- a/ecstore/src/config/mod.rs +++ b/ecstore/src/config/mod.rs @@ -141,7 +141,7 @@ impl Config { } pub fn merge(&self) -> Config { - // TODO: merge defauls + // TODO: merge default self.clone() } } @@ -158,6 +158,6 @@ pub fn register_default_kvs(kvs: HashMap) { pub fn init() { let mut kvs = HashMap::new(); kvs.insert(STORAGE_CLASS_SUB_SYS.to_owned(), storageclass::DefaultKVS.clone()); - // TODO: other defauls + // TODO: other default register_default_kvs(kvs) } diff --git a/ecstore/src/config/storageclass.rs b/ecstore/src/config/storageclass.rs index edb2095f9..ef5199fc6 100644 --- a/ecstore/src/config/storageclass.rs +++ b/ecstore/src/config/storageclass.rs @@ -8,8 +8,8 @@ use lazy_static::lazy_static; use serde::{Deserialize, Serialize}; use tracing::warn; -// default_partiy_count 默认配置，根据磁盘总数分配校验磁盘数量 -pub fn default_partiy_count(drive: usize) -> usize { +// default_parity_count 默认配置，根据磁盘总数分配校验磁盘数量 +pub fn default_parity_count(drive: usize) -> usize { match drive { 1 => 0, 2 | 3 => 1, @@ -158,7 +158,7 @@ pub fn lookup_config(kvs: &KVS, set_drive_count: usize) -> Result { parse_storage_class(&ssc_str)? } else { StorageClass { - parity: default_partiy_count(set_drive_count), + parity: default_parity_count(set_drive_count), } } }; diff --git a/rustfs/src/admin/handlers.rs b/rustfs/src/admin/handlers.rs index dfeeeb876..45ee477a8 100644 --- a/rustfs/src/admin/handlers.rs +++ b/rustfs/src/admin/handlers.rs @@ -48,6 +48,7 @@ use tokio::{select, spawn}; use tokio_stream::wrappers::ReceiverStream; use tracing::{error, info, warn}; +pub mod event; pub mod group; pub mod policys; pub mod pools; diff --git a/rustfs/src/admin/handlers/event.rs b/rustfs/src/admin/handlers/event.rs new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/rustfs/src/admin/handlers/event.rs @@ -0,0 +1 @@ + diff --git a/rustfs/src/admin/mod.rs b/rustfs/src/admin/mod.rs index 57293d4ea..f0229f201 100644 --- a/rustfs/src/admin/mod.rs +++ b/rustfs/src/admin/mod.rs @@ -25,7 +25,7 @@ pub fn make_admin_route() -> Result { r.insert(Method::POST, "/", AdminOperation(&sts::AssumeRoleHandle {}))?; regist_rpc_route(&mut r)?; - regist_user_route(&mut r)?; + register_user_route(&mut r)?; r.insert( Method::POST, @@ -124,7 +124,7 @@ pub fn make_admin_route() -> Result { Ok(r) } -fn regist_user_route(r: &mut S3Router) -> Result<()> { +fn register_user_route(r: &mut S3Router) -> Result<()> { // 1 r.insert( Method::GET, diff --git a/scripts/run.sh b/scripts/run.sh index 59703af95..58d0f2642 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -39,24 +39,35 @@ export RUSTFS_CONSOLE_ADDRESS=":9002" export RUSTFS_OBS_CONFIG="./deploy/config/obs.example.toml" # 如下变量需要必须参数都有值才可以，以及会覆盖配置文件中的值 -export RUSTFS__OBSERVABILITY__ENDPOINT=http://localhost:4317 -export RUSTFS__OBSERVABILITY__USE_STDOUT=false -export RUSTFS__OBSERVABILITY__SAMPLE_RATIO=2.0 -export RUSTFS__OBSERVABILITY__METER_INTERVAL=30 -export RUSTFS__OBSERVABILITY__SERVICE_NAME=rustfs -export RUSTFS__OBSERVABILITY__SERVICE_VERSION=0.1.0 -export RUSTFS__OBSERVABILITY__ENVIRONMENT=develop -export RUSTFS__OBSERVABILITY__LOGGER_LEVEL=debug -export RUSTFS__OBSERVABILITY__LOCAL_LOGGING_ENABLED=true -export RUSTFS__SINKS__FILE__ENABLED=true -export RUSTFS__SINKS__FILE__PATH="./deploy/logs/rustfs.log" -export RUSTFS__SINKS__WEBHOOK__ENABLED=false -export RUSTFS__SINKS__WEBHOOK__ENDPOINT="" -export RUSTFS__SINKS__WEBHOOK__AUTH_TOKEN="" -export RUSTFS__SINKS__KAFKA__ENABLED=false -export RUSTFS__SINKS__KAFKA__BOOTSTRAP_SERVERS="" -export RUSTFS__SINKS__KAFKA__TOPIC="" -export RUSTFS__LOGGER__QUEUE_CAPACITY=10 +#export RUSTFS__OBSERVABILITY__ENDPOINT=http://localhost:4317 +#export RUSTFS__OBSERVABILITY__USE_STDOUT=false +#export RUSTFS__OBSERVABILITY__SAMPLE_RATIO=2.0 +#export RUSTFS__OBSERVABILITY__METER_INTERVAL=31 +#export RUSTFS__OBSERVABILITY__SERVICE_NAME=rustfs +#export RUSTFS__OBSERVABILITY__SERVICE_VERSION=0.1.0 +#export RUSTFS__OBSERVABILITY__ENVIRONMENT=develop +#export RUSTFS__OBSERVABILITY__LOGGER_LEVEL=debug +#export RUSTFS__OBSERVABILITY__LOCAL_LOGGING_ENABLED=true +# +#export RUSTFS__SINKS_0__type=File +#export RUSTFS__SINKS_0__path=./deploy/logs/rustfs.log +#export RUSTFS__SINKS_0__buffer_size=12 +#export RUSTFS__SINKS_0__flush_interval_ms=1000 +#export RUSTFS__SINKS_0__flush_threshold=100 +# +#export RUSTFS__SINKS_1__type=Kakfa +#export RUSTFS__SINKS_1__brokers=localhost:9092 +#export RUSTFS__SINKS_1__topic=logs +#export RUSTFS__SINKS_1__batch_size=100 +#export RUSTFS__SINKS_1__batch_timeout_ms=1000 +# +#export RUSTFS__SINKS_2__type=Webhook +#export RUSTFS__SINKS_2__endpoint=http://localhost:8080/webhook +#export RUSTFS__SINKS_2__auth_token=you-auth-token +#export RUSTFS__SINKS_2__batch_size=100 +#export RUSTFS__SINKS_2__batch_timeout_ms=1000 +# +#export RUSTFS__LOGGER__QUEUE_CAPACITY=10 export OTEL_INSTRUMENTATION_NAME="rustfs" export OTEL_INSTRUMENTATION_VERSION="0.1.1" @@ -64,13 +75,13 @@ export OTEL_INSTRUMENTATION_SCHEMA_URL="https://opentelemetry.io/schemas/1.31.0" export OTEL_INSTRUMENTATION_ATTRIBUTES="env=production" # 事件消息配置 -export RUSTFS_EVENT_CONFIG="./deploy/config/event.example.toml" +#export RUSTFS_EVENT_CONFIG="./deploy/config/event.example.toml" if [ -n "$1" ]; then export RUSTFS_VOLUMES="$1" fi # 启动 webhook 服务器 -cargo run --example webhook -p rustfs-event-notifier & +#cargo run --example webhook -p rustfs-event-notifier & # 启动主服务 cargo run --bin rustfs \ No newline at end of file From 571cedf4cec44ae60892a359df004f0bdaf5cd51 Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 12 May 2025 13:32:18 +0800 Subject: [PATCH 38/38] feat(obs): implement global OpenTelemetry guard management --- .gitignore | 3 ++- Cargo.lock | 1 + crates/config/src/constants/app.rs | 12 ++++----- crates/obs/Cargo.toml | 1 + crates/obs/src/config.rs | 18 ++++++------- crates/obs/src/global.rs | 8 ------ crates/obs/src/lib.rs | 1 - crates/obs/src/logger.rs | 6 ++--- crates/obs/src/telemetry.rs | 10 +++---- crates/obs/src/utils.rs | 42 ------------------------------ crates/obs/src/worker.rs | 2 +- crates/utils/Cargo.toml | 15 ++++++++--- crates/utils/src/lib.rs | 11 +++----- rustfs/Cargo.toml | 2 +- 14 files changed, 44 insertions(+), 88 deletions(-) delete mode 100644 crates/obs/src/utils.rs diff --git a/.gitignore b/.gitignore index 7ccca205e..0a1501ce9 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,5 @@ deploy/config/obs.toml *.log deploy/certs/* *jsonl -.env \ No newline at end of file +.env +.rustfs.sys \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 01ab7a994..fa09fb754 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7402,6 +7402,7 @@ dependencies = [ "rdkafka", "reqwest", "rustfs-config", + "rustfs-utils", "serde", "serde_json", "smallvec", diff --git a/crates/config/src/constants/app.rs b/crates/config/src/constants/app.rs index 467905f7b..c5ad125a7 100644 --- a/crates/config/src/constants/app.rs +++ b/crates/config/src/constants/app.rs @@ -16,22 +16,22 @@ pub const DEFAULT_LOG_LEVEL: &str = "info"; /// Default configuration use stdout /// Default value: true -pub(crate) const USE_STDOUT: bool = true; +pub const USE_STDOUT: bool = true; /// Default configuration sample ratio /// Default value: 1.0 -pub(crate) const SAMPLE_RATIO: f64 = 1.0; +pub const SAMPLE_RATIO: f64 = 1.0; /// Default configuration meter interval /// Default value: 30 -pub(crate) const METER_INTERVAL: u64 = 30; +pub const METER_INTERVAL: u64 = 30; /// Default configuration service version /// Default value: 0.0.1 -pub(crate) const SERVICE_VERSION: &str = "0.0.1"; +pub const SERVICE_VERSION: &str = "0.0.1"; /// Default configuration environment /// Default value: production -pub(crate) const ENVIRONMENT: &str = "production"; +pub const ENVIRONMENT: &str = "production"; /// maximum number of connections /// This is the maximum number of connections that the server will accept. @@ -63,7 +63,7 @@ pub const DEFAULT_SECRET_KEY: &str = "rustfsadmin"; /// Example: RUSTFS_OBS_CONFIG=config/obs.toml /// Example: --obs-config config/obs.toml /// Example: --obs-config /etc/rustfs/obs.toml -pub const DEFAULT_OBS_CONFIG: &str = "config/obs.toml"; +pub const DEFAULT_OBS_CONFIG: &str = "./deploy/config/obs.toml"; /// Default TLS key for rustfs /// This is the default key for TLS. diff --git a/crates/obs/Cargo.toml b/crates/obs/Cargo.toml index 65968d519..12c9831ec 100644 --- a/crates/obs/Cargo.toml +++ b/crates/obs/Cargo.toml @@ -29,6 +29,7 @@ opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] } opentelemetry-stdout = { workspace = true } opentelemetry-otlp = { workspace = true, features = ["grpc-tonic", "gzip-tonic"] } opentelemetry-semantic-conventions = { workspace = true, features = ["semconv_experimental"] } +rustfs-utils = { workspace = true, features = ["ip"] } serde = { workspace = true } smallvec = { workspace = true, features = ["serde"] } tracing = { workspace = true, features = ["std", "attributes"] } diff --git a/crates/obs/src/config.rs b/crates/obs/src/config.rs index 989294704..770203de3 100644 --- a/crates/obs/src/config.rs +++ b/crates/obs/src/config.rs @@ -1,5 +1,5 @@ -use crate::global::{ENVIRONMENT, LOGGER_LEVEL, METER_INTERVAL, SAMPLE_RATIO, SERVICE_NAME, SERVICE_VERSION, USE_STDOUT}; use config::{Config, File, FileFormat}; +use rustfs_config::{APP_NAME, DEFAULT_LOG_LEVEL, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT}; use serde::{Deserialize, Serialize}; use std::env; @@ -43,7 +43,7 @@ fn extract_otel_config_from_env() -> OtelConfig { service_name: env::var("RUSTFS_OBSERVABILITY_SERVICE_NAME") .ok() .and_then(|v| v.parse().ok()) - .or(Some(SERVICE_NAME.to_string())), + .or(Some(APP_NAME.to_string())), service_version: env::var("RUSTFS_OBSERVABILITY_SERVICE_VERSION") .ok() .and_then(|v| v.parse().ok()) @@ -55,7 +55,7 @@ fn extract_otel_config_from_env() -> OtelConfig { logger_level: env::var("RUSTFS_OBSERVABILITY_LOGGER_LEVEL") .ok() .and_then(|v| v.parse().ok()) - .or(Some(LOGGER_LEVEL.to_string())), + .or(Some(DEFAULT_LOG_LEVEL.to_string())), local_logging_enabled: env::var("RUSTFS_OBSERVABILITY_LOCAL_LOGGING_ENABLED") .ok() .and_then(|v| v.parse().ok()) @@ -91,11 +91,11 @@ pub struct KafkaSinkConfig { impl KafkaSinkConfig { pub fn new() -> Self { Self { - brokers: env::var("RUSTFS__SINKS_0_KAFKA_BROKERS") + brokers: env::var("RUSTFS_SINKS_KAFKA_BROKERS") .ok() .filter(|s| !s.trim().is_empty()) .unwrap_or_else(|| "localhost:9092".to_string()), - topic: env::var("RUSTFS__SINKS_0_KAFKA_TOPIC") + topic: env::var("RUSTFS_SINKS_KAFKA_TOPIC") .ok() .filter(|s| !s.trim().is_empty()) .unwrap_or_else(|| "default_topic".to_string()), @@ -123,11 +123,11 @@ pub struct WebhookSinkConfig { impl WebhookSinkConfig { pub fn new() -> Self { Self { - endpoint: env::var("RUSTFS__SINKS_0_WEBHOOK_ENDPOINT") + endpoint: env::var("RUSTFS_SINKS_WEBHOOK_ENDPOINT") .ok() .filter(|s| !s.trim().is_empty()) .unwrap_or_else(|| "http://localhost:8080".to_string()), - auth_token: env::var("RUSTFS__SINKS_0_WEBHOOK_AUTH_TOKEN") + auth_token: env::var("RUSTFS_SINKS_WEBHOOK_AUTH_TOKEN") .ok() .filter(|s| !s.trim().is_empty()) .unwrap_or_else(|| "default_token".to_string()), @@ -160,7 +160,7 @@ impl FileSinkConfig { eprintln!("Failed to create log directory: {}", e); return "rustfs/rustfs.log".to_string(); } - + println!("Using log directory: {:?}", temp_dir); temp_dir .join("rustfs.log") .to_str() @@ -169,7 +169,7 @@ impl FileSinkConfig { } pub fn new() -> Self { Self { - path: env::var("RUSTFS__SINKS_0_FILE_PATH") + path: env::var("RUSTFS_SINKS_FILE_PATH") .ok() .filter(|s| !s.trim().is_empty()) .unwrap_or_else(Self::get_default_log_path), diff --git a/crates/obs/src/global.rs b/crates/obs/src/global.rs index 8d392ad94..07657fb41 100644 --- a/crates/obs/src/global.rs +++ b/crates/obs/src/global.rs @@ -3,14 +3,6 @@ use std::sync::{Arc, Mutex}; use tokio::sync::{OnceCell, SetError}; use tracing::{error, info}; -pub(crate) const USE_STDOUT: bool = true; -pub(crate) const SERVICE_NAME: &str = "RustFS"; -pub(crate) const SAMPLE_RATIO: f64 = 1.0; -pub(crate) const METER_INTERVAL: u64 = 60; -pub(crate) const SERVICE_VERSION: &str = "0.1.0"; -pub(crate) const ENVIRONMENT: &str = "production"; -pub(crate) const LOGGER_LEVEL: &str = "info"; - /// Global guard for OpenTelemetry tracing static GLOBAL_GUARD: OnceCell>> = OnceCell::const_new(); diff --git a/crates/obs/src/lib.rs b/crates/obs/src/lib.rs index 5d181e319..6a8f6219a 100644 --- a/crates/obs/src/lib.rs +++ b/crates/obs/src/lib.rs @@ -35,7 +35,6 @@ mod logger; mod sinks; mod system; mod telemetry; -mod utils; mod worker; use crate::logger::InitLogStatus; diff --git a/crates/obs/src/logger.rs b/crates/obs/src/logger.rs index 92ff5365e..02e0bf9b2 100644 --- a/crates/obs/src/logger.rs +++ b/crates/obs/src/logger.rs @@ -1,6 +1,6 @@ -use crate::global::{ENVIRONMENT, SERVICE_NAME, SERVICE_VERSION}; use crate::sinks::Sink; use crate::{AppConfig, AuditLogEntry, BaseLogEntry, ConsoleLogEntry, GlobalError, OtelConfig, ServerLogEntry, UnifiedLogEntry}; +use rustfs_config::{APP_NAME, ENVIRONMENT, SERVICE_VERSION}; use std::sync::Arc; use std::time::SystemTime; use tokio::sync::mpsc::{self, Receiver, Sender}; @@ -428,7 +428,7 @@ impl Default for InitLogStatus { fn default() -> Self { Self { timestamp: SystemTime::now(), - service_name: String::from(SERVICE_NAME), + service_name: String::from(APP_NAME), version: SERVICE_VERSION.to_string(), environment: ENVIRONMENT.to_string(), } @@ -442,7 +442,7 @@ impl InitLogStatus { let version = config.service_version.unwrap_or(SERVICE_VERSION.to_string()); Self { timestamp: SystemTime::now(), - service_name: String::from(SERVICE_NAME), + service_name: String::from(APP_NAME), version, environment, } diff --git a/crates/obs/src/telemetry.rs b/crates/obs/src/telemetry.rs index 5a08321d3..cba605d3e 100644 --- a/crates/obs/src/telemetry.rs +++ b/crates/obs/src/telemetry.rs @@ -1,5 +1,3 @@ -use crate::global::{ENVIRONMENT, LOGGER_LEVEL, METER_INTERVAL, SAMPLE_RATIO, SERVICE_NAME, SERVICE_VERSION, USE_STDOUT}; -use crate::utils::get_local_ip_with_default; use crate::OtelConfig; use opentelemetry::trace::TracerProvider; use opentelemetry::{global, KeyValue}; @@ -15,6 +13,8 @@ use opentelemetry_semantic_conventions::{ attribute::{DEPLOYMENT_ENVIRONMENT_NAME, NETWORK_LOCAL_ADDRESS, SERVICE_VERSION as OTEL_SERVICE_VERSION}, SCHEMA_URL, }; +use rustfs_config::{APP_NAME, DEFAULT_LOG_LEVEL, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT}; +use rustfs_utils::get_local_ip_with_default; use smallvec::SmallVec; use std::borrow::Cow; use std::io::IsTerminal; @@ -70,7 +70,7 @@ impl Drop for OtelGuard { /// create OpenTelemetry Resource fn resource(config: &OtelConfig) -> Resource { Resource::builder() - .with_service_name(Cow::Borrowed(config.service_name.as_deref().unwrap_or(SERVICE_NAME)).to_string()) + .with_service_name(Cow::Borrowed(config.service_name.as_deref().unwrap_or(APP_NAME)).to_string()) .with_schema_url( [ KeyValue::new( @@ -101,8 +101,8 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard { let endpoint = &config.endpoint; let use_stdout = config.use_stdout.unwrap_or(USE_STDOUT); let meter_interval = config.meter_interval.unwrap_or(METER_INTERVAL); - let logger_level = config.logger_level.as_deref().unwrap_or(LOGGER_LEVEL); - let service_name = config.service_name.as_deref().unwrap_or(SERVICE_NAME); + let logger_level = config.logger_level.as_deref().unwrap_or(DEFAULT_LOG_LEVEL); + let service_name = config.service_name.as_deref().unwrap_or(APP_NAME); // Pre-create resource objects to avoid repeated construction let res = resource(config); diff --git a/crates/obs/src/utils.rs b/crates/obs/src/utils.rs deleted file mode 100644 index 774594577..000000000 --- a/crates/obs/src/utils.rs +++ /dev/null @@ -1,42 +0,0 @@ -use local_ip_address::{local_ip, local_ipv6}; -use std::net::{IpAddr, Ipv4Addr}; - -/// Get the IP address of the machine -/// -/// Priority is given to trying to get the IPv4 address, and if it fails, try to get the IPv6 address. -/// If both fail to retrieve, None is returned. -/// -/// # Returns -/// -/// * `Some(IpAddr)` - Native IP address (IPv4 or IPv6) -/// * `None` - Unable to obtain any native IP address -pub fn get_local_ip() -> Option { - local_ip().ok().or_else(|| local_ipv6().ok()) -} - -/// Get the IP address of the machine as a string -/// -/// If the IP address cannot be obtained, returns "127.0.0.1" as the default value. -/// -/// # Returns -/// -/// * `String` - Native IP address (IPv4 or IPv6) as a string, or the default value -pub fn get_local_ip_with_default() -> String { - get_local_ip() - .unwrap_or_else(|| IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))) // Provide a safe default value - .to_string() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_get_local_ip() { - match get_local_ip() { - Some(ip) => println!("the ip address of this machine:{}", ip), - None => println!("Unable to obtain the IP address of the machine"), - } - assert!(get_local_ip().is_some()); - } -} diff --git a/crates/obs/src/worker.rs b/crates/obs/src/worker.rs index aee1695d6..cfe2f26ce 100644 --- a/crates/obs/src/worker.rs +++ b/crates/obs/src/worker.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use tokio::sync::mpsc::Receiver; /// Start the log processing worker thread -pub async fn start_worker(receiver: Receiver, sinks: Vec>) { +pub(crate) async fn start_worker(receiver: Receiver, sinks: Vec>) { let mut receiver = receiver; while let Some(entry) = receiver.recv().await { for sink in &sinks { diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 13ee21e40..76cac05c7 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -7,12 +7,19 @@ rust-version.workspace = true version.workspace = true [dependencies] -local-ip-address = { workspace = true } +local-ip-address = { workspace = true, optional = true } rustfs-config = { workspace = true } -rustls = { workspace = true } -rustls-pemfile = { workspace = true } -rustls-pki-types = { workspace = true } +rustls = { workspace = true, optional = true } +rustls-pemfile = { workspace = true, optional = true } +rustls-pki-types = { workspace = true, optional = true } tracing = { workspace = true } [lints] workspace = true + +[features] +default = ["ip"] # features that are enabled by default +ip = ["dep:local-ip-address"] # ip characteristics and their dependencies +tls = ["dep:rustls", "dep:rustls-pemfile", "dep:rustls-pki-types"] # tls characteristics and their dependencies +net = ["ip"] # empty network features +full = ["ip", "tls", "net"] # all features \ No newline at end of file diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index fbb5936b6..cda53d088 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -2,10 +2,7 @@ mod certs; mod ip; mod net; -pub use certs::certs_error; -pub use certs::create_multi_cert_resolver; -pub use certs::load_all_certs_from_directory; -pub use certs::load_certs; -pub use certs::load_private_key; -pub use ip::get_local_ip; -pub use ip::get_local_ip_with_default; +#[cfg(feature = "ip")] +pub use certs::*; +#[cfg(feature = "ip")] +pub use ip::*; diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 3023a9cab..a238665d9 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -55,7 +55,7 @@ rmp-serde.workspace = true rustfs-config = { workspace = true } rustfs-event-notifier = { workspace = true } rustfs-obs = { workspace = true } -rustfs-utils = { workspace = true } +rustfs-utils = { workspace = true, features = ["full"] } rustls.workspace = true rust-embed = { workspace = true, features = ["interpolate-folder-path"] } s3s.workspace = true