refactor: reorganize RPC modules into unified structure

rustfs/src/admin/handlers.rs

@@ -11,20 +11,20 @@ use ecstore::bucket::versioning_sys::BucketVersioningSys;
 use ecstore::cmd::bucket_targets::{self, GLOBAL_Bucket_Target_Sys};
 use ecstore::error::StorageError;
 use ecstore::global::GLOBAL_ALlHealState;
+use ecstore::global::get_global_action_cred;
 use ecstore::heal::data_usage::load_data_usage_from_backend;
 use ecstore::heal::heal_commands::HealOpts;
 use ecstore::heal::heal_ops::new_heal_sequence;
 use ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics};
 use ecstore::new_object_layer_fn;
-use ecstore::peer::is_reserved_or_invalid_bucket;
 use ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free};
 use ecstore::store::is_valid_object_prefix;
 use ecstore::store_api::BucketOptions;
 use ecstore::store_api::StorageAPI;
+use ecstore::store_utils::is_reserved_or_invalid_bucket;
 use futures::{Stream, StreamExt};
 use http::{HeaderMap, Uri};
 use hyper::StatusCode;
-use iam::get_global_action_cred;
 use iam::store::MappedPolicy;
 use rustfs_utils::path::path_join;
 // use lazy_static::lazy_static;
@@ -810,7 +810,7 @@ impl Operation for SetRemoteTargetHandler {
         //println!("bucket is:{}", bucket.clone());
         if let Some(bucket) = querys.get("bucket") {
             if bucket.is_empty() {
-                println!("have bucket: {}", bucket);
+                info!("have bucket: {}", bucket);
                 return Ok(S3Response::new((StatusCode::OK, Body::from("fuck".to_string()))));
             }
             let Some(store) = new_object_layer_fn() else {
@@ -824,13 +824,13 @@ impl Operation for SetRemoteTargetHandler {
                 .await
             {
                 Ok(info) => {
-                    println!("Bucket Info: {:?}", info);
+                    info!("Bucket Info: {:?}", info);
                     if !info.versionning {
                         return Ok(S3Response::new((StatusCode::FORBIDDEN, Body::from("bucket need versioned".to_string()))));
                     }
                 }
                 Err(err) => {
-                    eprintln!("Error: {:?}", err);
+                    error!("Error: {:?}", err);
                     return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from("empty bucket".to_string()))));
                 }
             }
@@ -934,7 +934,7 @@ impl Operation for ListRemoteTargetHandler {
                 .await
             {
                 Ok(info) => {
-                    println!("Bucket Info: {:?}", info);
+                    info!("Bucket Info: {:?}", info);
                     if !info.versionning {
                         return Ok(S3Response::new((
                             StatusCode::FORBIDDEN,
@@ -943,7 +943,7 @@ impl Operation for ListRemoteTargetHandler {
                     }
                 }
                 Err(err) => {
-                    eprintln!("Error fetching bucket info: {:?}", err);
+                    error!("Error fetching bucket info: {:?}", err);
                     return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from("Invalid bucket".to_string()))));
                 }
             }

rustfs/src/admin/handlers/group.rs

@@ -1,8 +1,6 @@
+use ecstore::global::get_global_action_cred;
 use http::{HeaderMap, StatusCode};
-use iam::{
-    error::{is_err_no_such_group, is_err_no_such_user},
-    get_global_action_cred,
-};
+use iam::error::{is_err_no_such_group, is_err_no_such_user};
 use madmin::GroupAddRemove;
 use matchit::Params;
 use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error};

rustfs/src/admin/handlers/policys.rs

@@ -1,6 +1,8 @@
 use crate::admin::{router::Operation, utils::has_space_be};
+use ecstore::global::get_global_action_cred;
 use http::{HeaderMap, StatusCode};
-use iam::{error::is_err_no_such_user, get_global_action_cred, store::MappedPolicy};
+use iam::error::is_err_no_such_user;
+use iam::store::MappedPolicy;
 use matchit::Params;
 use policy::policy::Policy;
 use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error};

rustfs/src/admin/handlers/service_account.rs

@@ -1,13 +1,11 @@
 use crate::admin::utils::has_space_be;
 use crate::auth::{get_condition_values, get_session_token};
 use crate::{admin::router::Operation, auth::check_key_valid};
+use ecstore::global::get_global_action_cred;
 use http::HeaderMap;
 use hyper::StatusCode;
-use iam::{
-    error::is_err_no_such_service_account,
-    get_global_action_cred,
-    sys::{NewServiceAccountOpts, UpdateServiceAccountOpts},
-};
+use iam::error::is_err_no_such_service_account;
+use iam::sys::{NewServiceAccountOpts, UpdateServiceAccountOpts};
 use madmin::{
     AddServiceAccountReq, AddServiceAccountResp, Credentials, InfoServiceAccountResp, ListServiceAccountsResp,
     ServiceAccountInfo, UpdateServiceAccountReq,

rustfs/src/admin/handlers/trace.rs

@@ -1,4 +1,4 @@
-use ecstore::{GLOBAL_Endpoints, peer_rest_client::PeerRestClient};
+use ecstore::{GLOBAL_Endpoints, rpc::PeerRestClient};
 use http::StatusCode;
 use hyper::Uri;
 use madmin::service_commands::ServiceTraceOpts;

rustfs/src/admin/handlers/user.rs

@@ -1,7 +1,9 @@
-use std::{collections::HashMap, str::from_utf8};
-
+use crate::{
+    admin::{router::Operation, utils::has_space_be},
+    auth::{check_key_valid, get_condition_values, get_session_token},
+};
+use ecstore::global::get_global_action_cred;
 use http::{HeaderMap, StatusCode};
-use iam::get_global_action_cred;
 use madmin::{AccountStatus, AddOrUpdateUserReq};
 use matchit::Params;
 use policy::policy::{
@@ -11,13 +13,9 @@ use policy::policy::{
 use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
 use serde::Deserialize;
 use serde_urlencoded::from_bytes;
+use std::{collections::HashMap, str::from_utf8};
 use tracing::warn;
 
-use crate::{
-    admin::{router::Operation, utils::has_space_be},
-    auth::{check_key_valid, get_condition_values, get_session_token},
-};
-
 #[derive(Debug, Deserialize, Default)]
 pub struct AddUserQuery {
     #[serde(rename = "accessKey")]

rustfs/src/admin/router.rs

@@ -1,5 +1,4 @@
-use base64::{Engine as _, engine::general_purpose};
-use hmac::{Hmac, Mac};
+use ecstore::rpc::verify_rpc_signature;
 use hyper::HeaderMap;
 use hyper::Method;
 use hyper::StatusCode;
@@ -14,80 +13,11 @@ use s3s::S3Result;
 use s3s::header;
 use s3s::route::S3Route;
 use s3s::s3_error;
-use sha2::Sha256;
-use std::time::{SystemTime, UNIX_EPOCH};
+use tracing::error;
 
 use super::ADMIN_PREFIX;
 use super::RUSTFS_ADMIN_PREFIX;
 use super::rpc::RPC_PREFIX;
-use iam::get_global_action_cred;
-
-type HmacSha256 = Hmac<Sha256>;
-
-const SIGNATURE_HEADER: &str = "x-rustfs-signature";
-const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
-const SIGNATURE_VALID_DURATION: u64 = 300; // 5 minutes
-
-/// Get the shared secret for HMAC signing
-fn get_shared_secret() -> String {
-    if let Some(cred) = get_global_action_cred() {
-        cred.secret_key
-    } else {
-        // Fallback to environment variable if global credentials are not available
-        std::env::var("RUSTFS_RPC_SECRET").unwrap_or_else(|_| "rustfs-default-secret".to_string())
-    }
-}
-
-/// Generate HMAC-SHA256 signature for the given data
-fn generate_signature(secret: &str, url: &str, method: &str, timestamp: u64) -> String {
-    let data = format!("{}|{}|{}", url, method, timestamp);
-    let mut mac = HmacSha256::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size");
-    mac.update(data.as_bytes());
-    let result = mac.finalize();
-    general_purpose::STANDARD.encode(result.into_bytes())
-}
-
-/// Verify the request signature for RPC requests
-fn verify_rpc_signature(req: &S3Request<Body>) -> S3Result<()> {
-    let secret = get_shared_secret();
-
-    // Get signature from header
-    let signature = req
-        .headers
-        .get(SIGNATURE_HEADER)
-        .and_then(|v| v.to_str().ok())
-        .ok_or_else(|| s3_error!(InvalidArgument, "Missing signature header"))?;
-
-    // Get timestamp from header
-    let timestamp_str = req
-        .headers
-        .get(TIMESTAMP_HEADER)
-        .and_then(|v| v.to_str().ok())
-        .ok_or_else(|| s3_error!(InvalidArgument, "Missing timestamp header"))?;
-
-    let timestamp: u64 = timestamp_str
-        .parse()
-        .map_err(|_| s3_error!(InvalidArgument, "Invalid timestamp format"))?;
-
-    // Check timestamp validity (prevent replay attacks)
-    let current_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
-
-    if current_time.saturating_sub(timestamp) > SIGNATURE_VALID_DURATION {
-        return Err(s3_error!(InvalidArgument, "Request timestamp expired"));
-    }
-
-    // Generate expected signature
-    let url = req.uri.to_string();
-    let method = req.method.as_str();
-    let expected_signature = generate_signature(&secret, &url, method, timestamp);
-
-    // Compare signatures
-    if signature != expected_signature {
-        return Err(s3_error!(AccessDenied, "Invalid signature"));
-    }
-
-    Ok(())
-}
 
 pub struct S3Router<T> {
     router: Router<T>,
@@ -160,7 +90,10 @@ where
         if req.uri.path().starts_with(RPC_PREFIX) {
             // Skip signature verification for HEAD requests (health checks)
             if req.method != Method::HEAD {
-                verify_rpc_signature(req)?;
+                verify_rpc_signature(&req.uri.to_string(), &req.method, &req.headers).map_err(|e| {
+                    error!("RPC signature verification failed: {}", e);
+                    s3_error!(AccessDenied, "{}", e)
+                })?;
             }
             return Ok(());
         }

rustfs/src/admin/rpc.rs

@@ -1,7 +1,6 @@
 use super::router::AdminOperation;
 use super::router::Operation;
 use super::router::S3Router;
-use crate::storage::ecfs::bytes_stream;
 use ecstore::disk::DiskAPI;
 use ecstore::disk::WalkDirOptions;
 use ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
@@ -10,6 +9,7 @@ use futures::StreamExt;
 use http::StatusCode;
 use hyper::Method;
 use matchit::Params;
+use rustfs_utils::net::bytes_stream;
 use s3s::Body;
 use s3s::S3Request;
 use s3s::S3Response;

rustfs/src/auth.rs

@@ -1,9 +1,7 @@
-use std::collections::HashMap;
-
+use ecstore::global::get_global_action_cred;
 use http::HeaderMap;
 use http::Uri;
 use iam::error::Error as IamError;
-use iam::get_global_action_cred;
 use iam::sys::SESSION_POLICY_NAME;
 use policy::auth;
 use policy::auth::get_claims_from_token_with_secret;
@@ -15,6 +13,7 @@ use s3s::auth::SecretKey;
 use s3s::auth::SimpleAuth;
 use s3s::s3_error;
 use serde_json::Value;
+use std::collections::HashMap;
 
 pub struct IAMAuth {
     simple_auth: SimpleAuth,

rustfs/src/main.rs

@@ -4,7 +4,7 @@ mod config;
 mod console;
 mod error;
 mod event;
-mod grpc;
+// mod grpc;
 pub mod license;
 mod logging;
 mod server;
@@ -28,6 +28,7 @@ use ecstore::cmd::bucket_replication::init_bucket_replication_pool;
 use ecstore::config as ecconfig;
 use ecstore::config::GLOBAL_ConfigSys;
 use ecstore::heal::background_heal_ops::init_auto_heal;
+use ecstore::rpc::make_server;
 use ecstore::store_api::BucketOptions;
 use ecstore::{
     endpoints::EndpointServerPools,
@@ -37,7 +38,6 @@ use ecstore::{
     update_erasure_type,
 };
 use ecstore::{global::set_global_rustfs_port, notification_sys::new_global_notification_sys};
-use grpc::make_server;
 use http::{HeaderMap, Request as HttpRequest, Response};
 use hyper_util::server::graceful::GracefulShutdown;
 use hyper_util::{
@@ -129,7 +129,7 @@ async fn run(opt: config::Opt) -> Result<()> {
     debug!("server_address {}", &server_address);
 
     // Set up AK and SK
-    iam::init_global_action_cred(Some(opt.access_key.clone()), Some(opt.secret_key.clone()))?;
+    ecstore::global::init_global_action_cred(Some(opt.access_key.clone()), Some(opt.secret_key.clone()));
 
     set_global_rustfs_port(server_port);
 

rustfs/src/storage/ecfs.rs

@@ -8,6 +8,7 @@ use crate::storage::access::ReqInfo;
 use crate::storage::options::copy_dst_opts;
 use crate::storage::options::copy_src_opts;
 use crate::storage::options::{extract_metadata_from_mime, get_opts};
+use api::object_store::bytes_stream;
 use api::query::Context;
 use api::query::Query;
 use api::server::dbms::DatabaseManagerSystem;
@@ -52,8 +53,7 @@ use ecstore::store_api::ObjectOptions;
 use ecstore::store_api::ObjectToDelete;
 use ecstore::store_api::PutObjReader;
 use ecstore::store_api::StorageAPI; // use ecstore::store_api::RESERVED_METADATA_PREFIX;
-use futures::pin_mut;
-use futures::{Stream, StreamExt};
+use futures::StreamExt;
 use http::HeaderMap;
 use lazy_static::lazy_static;
 use policy::auth;
@@ -95,7 +95,6 @@ use tracing::debug;
 use tracing::error;
 use tracing::info;
 use tracing::warn;
-use transform_stream::AsyncTryStream;
 use uuid::Uuid;
 
 macro_rules! try_ {
@@ -2414,24 +2413,3 @@ impl S3 for FS {
         }))
     }
 }
-
-#[allow(dead_code)]
-pub fn bytes_stream<S, E>(stream: S, content_length: usize) -> impl Stream<Item = std::result::Result<Bytes, E>> + Send + 'static
-where
-    S: Stream<Item = std::result::Result<Bytes, E>> + Send + 'static,
-    E: Send + 'static,
-{
-    AsyncTryStream::<Bytes, E, _>::new(|mut y| async move {
-        pin_mut!(stream);
-        let mut remaining: usize = content_length;
-        while let Some(result) = stream.next().await {
-            let mut bytes = result?;
-            if bytes.len() > remaining {
-                bytes.truncate(remaining);
-            }
-            remaining -= bytes.len();
-            y.yield_ok(bytes).await;
-        }
-        Ok(())
-    })
-}