feat: Add RustFS Scanner Module and Multiple Bug Fixes (#1579)

rustfs/Cargo.toml

@@ -61,6 +61,7 @@ rustfs-s3select-query = { workspace = true }
 rustfs-targets = { workspace = true }
 rustfs-utils = { workspace = true, features = ["full"] }
 rustfs-zip = { workspace = true }
+rustfs-scanner = { workspace = true }
 
 # Async Runtime and Networking
 async-trait = { workspace = true }

rustfs/src/admin/handlers.rs

@@ -696,7 +696,6 @@ impl Stream for MetricsStream {
     type Item = Result<Bytes, StdError>;
 
     fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-        info!("MetricsStream poll_next");
         let this = Pin::into_inner(self);
         this.inner.poll_next_unpin(cx)
     }
@@ -758,7 +757,6 @@ impl Operation for MetricsHandler {
         let body = Body::from(in_stream);
         spawn(async move {
             while n > 0 {
-                info!("loop, n: {n}");
                 let mut m = RealtimeMetrics::default();
                 let m_local = collect_local_metrics(types, &opts).await;
                 m.merge(m_local);
@@ -775,7 +773,6 @@ impl Operation for MetricsHandler {
                 // todo write resp
                 match serde_json::to_vec(&m) {
                     Ok(re) => {
-                        info!("got metrics, send it to client, m: {m:?}");
                         let _ = tx.send(Ok(Bytes::from(re))).await;
                     }
                     Err(e) => {

rustfs/src/main.rs

@@ -36,10 +36,7 @@ use crate::server::{
 };
 use clap::Parser;
 use license::init_license;
-use rustfs_ahm::{
-    Scanner, create_ahm_services_cancel_token, heal::storage::ECStoreHealStorage, init_heal_manager,
-    scanner::data_scanner::ScannerConfig, shutdown_ahm_services,
-};
+use rustfs_ahm::{create_ahm_services_cancel_token, heal::storage::ECStoreHealStorage, init_heal_manager, shutdown_ahm_services};
 use rustfs_common::{GlobalReadiness, SystemStage, set_global_addr};
 use rustfs_credentials::init_global_action_credentials;
 use rustfs_ecstore::{
@@ -59,6 +56,7 @@ use rustfs_ecstore::{
 };
 use rustfs_iam::init_iam_sys;
 use rustfs_obs::{init_obs, set_global_guard};
+use rustfs_scanner::init_data_scanner;
 use rustfs_utils::net::parse_and_resolve_address;
 use std::io::{Error, Result};
 use std::sync::Arc;
@@ -334,23 +332,29 @@ async fn run(opt: config::Opt) -> Result<()> {
 
     // Initialize heal manager and scanner based on environment variables
     if enable_heal || enable_scanner {
-        if enable_heal {
-            // Initialize heal manager with channel processor
-            let heal_storage = Arc::new(ECStoreHealStorage::new(store.clone()));
-            let heal_manager = init_heal_manager(heal_storage, None).await?;
+        let heal_storage = Arc::new(ECStoreHealStorage::new(store.clone()));
 
-            if enable_scanner {
-                info!(target: "rustfs::main::run","Starting scanner with heal manager...");
-                let scanner = Scanner::new(Some(ScannerConfig::default()), Some(heal_manager));
-                scanner.start().await?;
-            } else {
-                info!(target: "rustfs::main::run","Scanner disabled, but heal manager is initialized and available");
-            }
-        } else if enable_scanner {
-            info!("Starting scanner without heal manager...");
-            let scanner = Scanner::new(Some(ScannerConfig::default()), None);
-            scanner.start().await?;
-        }
+        init_heal_manager(heal_storage, None).await?;
+
+        init_data_scanner(ctx.clone(), store.clone()).await;
+
+        // if enable_heal {
+        //     // Initialize heal manager with channel processor
+        //     let heal_storage = Arc::new(ECStoreHealStorage::new(store.clone()));
+        //     let heal_manager = init_heal_manager(heal_storage, None).await?;
+
+        //     if enable_scanner {
+        //         info!(target: "rustfs::main::run","Starting scanner with heal manager...");
+        //         let scanner = Scanner::new(Some(ScannerConfig::default()), Some(heal_manager));
+        //         scanner.start().await?;
+        //     } else {
+        //         info!(target: "rustfs::main::run","Scanner disabled, but heal manager is initialized and available");
+        //     }
+        // } else if enable_scanner {
+        //     info!("Starting scanner without heal manager...");
+        //     let scanner = Scanner::new(Some(ScannerConfig::default()), None);
+        //     scanner.start().await?;
+        // }
     } else {
         info!(target: "rustfs::main::run","Both scanner and heal are disabled, skipping AHM service initialization");
     }

rustfs/src/storage/options.rs

@@ -58,7 +58,7 @@ pub async fn del_opts(
     let vid = if vid.is_none() {
         headers
             .get(RUSTFS_BUCKET_SOURCE_VERSION_ID)
-            .map(|v| v.to_str().unwrap().to_owned())
+            .and_then(|v| v.to_str().ok().map(|s| s.to_owned()))
     } else {
         vid
     };
@@ -194,7 +194,7 @@ pub async fn put_opts(
     let vid = if vid.is_none() {
         headers
             .get(RUSTFS_BUCKET_SOURCE_VERSION_ID)
-            .map(|v| v.to_str().unwrap().to_owned())
+            .and_then(|v| v.to_str().ok().map(|s| s.to_owned()))
     } else {
         vid
     };

rustfs/src/storage/tonic_service.rs

@@ -1072,6 +1072,29 @@ impl Node for NodeService {
             }))
         }
     }
+    async fn read_metadata(&self, request: Request<ReadMetadataRequest>) -> Result<Response<ReadMetadataResponse>, Status> {
+        let request = request.into_inner();
+        if let Some(disk) = self.find_disk(&request.disk).await {
+            match disk.read_metadata(&request.volume, &request.path).await {
+                Ok(data) => Ok(Response::new(ReadMetadataResponse {
+                    success: true,
+                    data,
+                    error: None,
+                })),
+                Err(err) => Ok(Response::new(ReadMetadataResponse {
+                    success: false,
+                    data: Bytes::new(),
+                    error: Some(err.into()),
+                })),
+            }
+        } else {
+            Ok(Response::new(ReadMetadataResponse {
+                success: false,
+                data: Bytes::new(),
+                error: Some(DiskError::other("can not find disk".to_string()).into()),
+            }))
+        }
+    }
 
     async fn update_metadata(&self, request: Request<UpdateMetadataRequest>) -> Result<Response<UpdateMetadataResponse>, Status> {
         let request = request.into_inner();