gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-13 00:26:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/rustfs/s3-rustfs into feature/ilm

Browse Source 
# Conflicts:
#	Cargo.lock
#	Cargo.toml
#	crates/utils/Cargo.toml
#	crates/utils/src/net.rs
#	ecstore/Cargo.toml
#	ecstore/src/set_disk.rs
#	rustfs/src/storage/ecfs.rs
This commit is contained in:
likewu
2025-06-23 16:42:18 +08:00
parent 2c176dd864 dc93398768
commit 81f0c9763f
225 changed files with 14913 additions and 6941 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/Cargo.toml
+5 -2
View File
@@ -57,8 +57,8 @@ protos.workspace = true
query = { workspace = true }
regex = { workspace = true }
rmp-serde.workspace = true
rustfs-config = { workspace = true }
rustfs-event-notifier = { workspace = true }
rustfs-config = { workspace = true, features = ["constants"] }
rustfs-notify = { workspace = true }
rustfs-obs = { workspace = true }
rustfs-utils = { workspace = true, features = ["full"] }
rustls.workspace = true
@@ -95,6 +95,9 @@ urlencoding = { workspace = true }
uuid = { workspace = true }
rustfs-filemeta.workspace = true
rustfs-rio.workspace = true
base64 = { workspace = true }
hmac = { workspace = true }
sha2 = { workspace = true }
[target.'cfg(target_os = "linux")'.dependencies]
libsystemd.workspace = true
rustfs/src/admin/handlers.rs
+7 -7
View File
@@ -11,20 +11,20 @@ use ecstore::bucket::versioning_sys::BucketVersioningSys;
use ecstore::cmd::bucket_targets::{self, GLOBAL_Bucket_Target_Sys};
use ecstore::error::StorageError;
use ecstore::global::GLOBAL_ALlHealState;
use ecstore::global::get_global_action_cred;
use ecstore::heal::data_usage::load_data_usage_from_backend;
use ecstore::heal::heal_commands::HealOpts;
use ecstore::heal::heal_ops::new_heal_sequence;
use ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics};
use ecstore::new_object_layer_fn;
use ecstore::peer::is_reserved_or_invalid_bucket;
use ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free};
use ecstore::store::is_valid_object_prefix;
use ecstore::store_api::BucketOptions;
use ecstore::store_api::StorageAPI;
use ecstore::store_utils::is_reserved_or_invalid_bucket;
use futures::{Stream, StreamExt};
use http::{HeaderMap, Uri};
use hyper::StatusCode;
use iam::get_global_action_cred;
use iam::store::MappedPolicy;
use rustfs_utils::path::path_join;
// use lazy_static::lazy_static;
@@ -811,7 +811,7 @@ impl Operation for SetRemoteTargetHandler {
//println!("bucket is:{}", bucket.clone());
if let Some(bucket) = querys.get("bucket") {
if bucket.is_empty() {
println!("have bucket: {}", bucket);
info!("have bucket: {}", bucket);
return Ok(S3Response::new((StatusCode::OK, Body::from("fuck".to_string()))));
}
let Some(store) = new_object_layer_fn() else {
@@ -825,13 +825,13 @@ impl Operation for SetRemoteTargetHandler {
.await
{
Ok(info) => {
println!("Bucket Info: {:?}", info);
info!("Bucket Info: {:?}", info);
if !info.versionning {
return Ok(S3Response::new((StatusCode::FORBIDDEN, Body::from("bucket need versioned".to_string()))));
}
}
Err(err) => {
eprintln!("Error: {:?}", err);
error!("Error: {:?}", err);
return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from("empty bucket".to_string()))));
}
}
@@ -935,7 +935,7 @@ impl Operation for ListRemoteTargetHandler {
.await
{
Ok(info) => {
println!("Bucket Info: {:?}", info);
info!("Bucket Info: {:?}", info);
if !info.versionning {
return Ok(S3Response::new((
StatusCode::FORBIDDEN,
@@ -944,7 +944,7 @@ impl Operation for ListRemoteTargetHandler {
}
}
Err(err) => {
eprintln!("Error fetching bucket info: {:?}", err);
error!("Error fetching bucket info: {:?}", err);
return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from("Invalid bucket".to_string()))));
}
}
rustfs/src/admin/handlers/group.rs
+2 -4
View File
@@ -1,8 +1,6 @@
use ecstore::global::get_global_action_cred;
use http::{HeaderMap, StatusCode};
use iam::{
error::{is_err_no_such_group, is_err_no_such_user},
get_global_action_cred,
};
use iam::error::{is_err_no_such_group, is_err_no_such_user};
use madmin::GroupAddRemove;
use matchit::Params;
use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
rustfs/src/admin/handlers/policys.rs
+3 -1
View File
@@ -1,6 +1,8 @@
use crate::admin::{router::Operation, utils::has_space_be};
use ecstore::global::get_global_action_cred;
use http::{HeaderMap, StatusCode};
use iam::{error::is_err_no_such_user, get_global_action_cred, store::MappedPolicy};
use iam::error::is_err_no_such_user;
use iam::store::MappedPolicy;
use matchit::Params;
use policy::policy::Policy;
use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
rustfs/src/admin/handlers/rebalance.rs
+51 -16
View File
@@ -10,7 +10,8 @@ use http::{HeaderMap, StatusCode};
use matchit::Params;
use s3s::{Body, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
use serde::{Deserialize, Serialize};
use std::time::{Duration, SystemTime};
use std::time::Duration;
use time::OffsetDateTime;
use tracing::warn;
use crate::admin::router::Operation;
@@ -56,8 +57,8 @@ pub struct RebalanceAdminStatus {
pub id: String, // Identifies the ongoing rebalance operation by a UUID
#[serde(rename = "pools")]
pub pools: Vec<RebalancePoolStatus>, // Contains all pools, including inactive
#[serde(rename = "stoppedAt")]
pub stopped_at: Option<SystemTime>, // Optional timestamp when rebalance was stopped
#[serde(rename = "stoppedAt", with = "offsetdatetime_rfc3339")]
pub stopped_at: Option<OffsetDateTime>, // Optional timestamp when rebalance was stopped
}
pub struct RebalanceStart {}
@@ -101,11 +102,13 @@ impl Operation for RebalanceStart {
}
};
store.start_rebalance().await;
warn!("Rebalance started with id: {}", id);
if let Some(notification_sys) = get_global_notification_sys() {
warn!("Loading rebalance meta");
warn!("RebalanceStart Loading rebalance meta start");
notification_sys.load_rebalance_meta(true).await;
warn!("Rebalance meta loaded");
warn!("RebalanceStart Loading rebalance meta done");
}
let resp = RebalanceResp { id };
@@ -175,15 +178,14 @@ impl Operation for RebalanceStatus {
let total_bytes_to_rebal = ps.init_capacity as f64 * meta.percent_free_goal - ps.init_free_space as f64;
let mut elapsed = if let Some(start_time) = ps.info.start_time {
SystemTime::now()
.duration_since(start_time)
.map_err(|e| s3_error!(InternalError, "Failed to calculate elapsed time: {}", e))?
let now = OffsetDateTime::now_utc();
now - start_time
} else {
return Err(s3_error!(InternalError, "Start time is not available"));
};
let mut eta = if ps.bytes > 0 {
Duration::from_secs_f64(total_bytes_to_rebal * elapsed.as_secs_f64() / ps.bytes as f64)
Duration::from_secs_f64(total_bytes_to_rebal * elapsed.as_seconds_f64() / ps.bytes as f64)
} else {
Duration::ZERO
};
@@ -193,10 +195,8 @@ impl Operation for RebalanceStatus {
}
if let Some(stopped_at) = stop_time {
if let Ok(du) = stopped_at.duration_since(ps.info.start_time.unwrap_or(stopped_at)) {
elapsed = du;
} else {
return Err(s3_error!(InternalError, "Failed to calculate elapsed time"));
if let Some(start_time) = ps.info.start_time {
elapsed = stopped_at - start_time;
}
eta = Duration::ZERO;
@@ -208,7 +208,7 @@ impl Operation for RebalanceStatus {
bytes: ps.bytes,
bucket: ps.bucket.clone(),
object: ps.object.clone(),
elapsed: elapsed.as_secs(),
elapsed: elapsed.whole_seconds() as u64,
eta: eta.as_secs(),
});
}
@@ -244,10 +244,45 @@ impl Operation for RebalanceStop {
.await
.map_err(|e| s3_error!(InternalError, "Failed to stop rebalance: {}", e))?;
warn!("handle RebalanceStop save_rebalance_stats done ");
if let Some(notification_sys) = get_global_notification_sys() {
notification_sys.load_rebalance_meta(true).await;
warn!("handle RebalanceStop notification_sys load_rebalance_meta");
notification_sys.load_rebalance_meta(false).await;
warn!("handle RebalanceStop notification_sys load_rebalance_meta done");
}
return Err(s3_error!(NotImplemented));
Ok(S3Response::new((StatusCode::OK, Body::empty())))
}
}
mod offsetdatetime_rfc3339 {
use serde::{self, Deserialize, Deserializer, Serializer};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
pub fn serialize<S>(dt: &Option<OffsetDateTime>, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
match dt {
Some(dt) => {
let s = dt.format(&Rfc3339).map_err(serde::ser::Error::custom)?;
serializer.serialize_some(&s)
}
None => serializer.serialize_none(),
}
}
pub fn deserialize<'de, D>(deserializer: D) -> Result<Option<OffsetDateTime>, D::Error>
where
D: Deserializer<'de>,
{
let opt = Option::<String>::deserialize(deserializer)?;
match opt {
Some(s) => {
let dt = OffsetDateTime::parse(&s, &Rfc3339).map_err(serde::de::Error::custom)?;
Ok(Some(dt))
}
None => Ok(None),
}
}
}
rustfs/src/admin/handlers/service_account.rs
+3 -5
View File
@@ -1,13 +1,11 @@
use crate::admin::utils::has_space_be;
use crate::auth::{get_condition_values, get_session_token};
use crate::{admin::router::Operation, auth::check_key_valid};
use ecstore::global::get_global_action_cred;
use http::HeaderMap;
use hyper::StatusCode;
use iam::{
error::is_err_no_such_service_account,
get_global_action_cred,
sys::{NewServiceAccountOpts, UpdateServiceAccountOpts},
};
use iam::error::is_err_no_such_service_account;
use iam::sys::{NewServiceAccountOpts, UpdateServiceAccountOpts};
use madmin::{
AddServiceAccountReq, AddServiceAccountResp, Credentials, InfoServiceAccountResp, ListServiceAccountsResp,
ServiceAccountInfo, UpdateServiceAccountReq,
rustfs/src/admin/handlers/sts.rs
+4 -5
View File
@@ -1,5 +1,3 @@
use std::collections::HashMap;
use crate::{
admin::router::Operation,
auth::{check_key_valid, get_session_token},
@@ -18,6 +16,7 @@ use s3s::{
use serde::Deserialize;
use serde_json::Value;
use serde_urlencoded::from_bytes;
use std::collections::HashMap;
use time::{Duration, OffsetDateTime};
use tracing::{info, warn};
@@ -52,7 +51,7 @@ impl Operation for AssumeRoleHandle {
let (cred, _owner) =
check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &user.access_key).await?;
// // TODO: Check permissions, do not allow STS access
// TODO: Check permissions, do not allow STS access
if cred.is_temp() || cred.is_service_account() {
return Err(s3_error!(InvalidRequest, "AccessDenied"));
}
@@ -70,11 +69,11 @@ impl Operation for AssumeRoleHandle {
let body: AssumeRoleRequest = from_bytes(&bytes).map_err(|_e| s3_error!(InvalidRequest, "get body failed"))?;
if body.action.as_str() != ASSUME_ROLE_ACTION {
return Err(s3_error!(InvalidArgument, "not suport action"));
return Err(s3_error!(InvalidArgument, "not support action"));
}
if body.version.as_str() != ASSUME_ROLE_VERSION {
return Err(s3_error!(InvalidArgument, "not suport version"));
return Err(s3_error!(InvalidArgument, "not support version"));
}
let mut claims = cred.claims.unwrap_or_default();
rustfs/src/admin/handlers/trace.rs
+1 -1
View File
@@ -1,4 +1,4 @@
use ecstore::{GLOBAL_Endpoints, peer_rest_client::PeerRestClient};
use ecstore::{GLOBAL_Endpoints, rpc::PeerRestClient};
use http::StatusCode;
use hyper::Uri;
use madmin::service_commands::ServiceTraceOpts;
rustfs/src/admin/handlers/user.rs
+6 -8
View File
@@ -1,7 +1,9 @@
use std::{collections::HashMap, str::from_utf8};
use crate::{
admin::{router::Operation, utils::has_space_be},
auth::{check_key_valid, get_condition_values, get_session_token},
};
use ecstore::global::get_global_action_cred;
use http::{HeaderMap, StatusCode};
use iam::get_global_action_cred;
use madmin::{AccountStatus, AddOrUpdateUserReq};
use matchit::Params;
use policy::policy::{
@@ -11,13 +13,9 @@ use policy::policy::{
use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
use serde::Deserialize;
use serde_urlencoded::from_bytes;
use std::{collections::HashMap, str::from_utf8};
use tracing::warn;
use crate::{
admin::{router::Operation, utils::has_space_be},
auth::{check_key_valid, get_condition_values, get_session_token},
};
#[derive(Debug, Deserialize, Default)]
pub struct AddUserQuery {
#[serde(rename = "accessKey")]
rustfs/src/admin/mod.rs
+2 -2
View File
@@ -13,7 +13,7 @@ use handlers::{
use handlers::{GetReplicationMetricsHandler, ListRemoteTargetHandler, RemoveRemoteTargetHandler, SetRemoteTargetHandler};
use hyper::Method;
use router::{AdminOperation, S3Router};
use rpc::regist_rpc_route;
use rpc::register_rpc_route;
use s3s::route::S3Route;
const ADMIN_PREFIX: &str = "/rustfs/admin";
@@ -25,7 +25,7 @@ pub fn make_admin_route() -> std::io::Result<impl S3Route> {
// 1
r.insert(Method::POST, "/", AdminOperation(&sts::AssumeRoleHandle {}))?;
regist_rpc_route(&mut r)?;
register_rpc_route(&mut r)?;
register_user_route(&mut r)?;
r.insert(
rustfs/src/admin/router.rs
+12 -1
View File
@@ -1,3 +1,4 @@
use ecstore::rpc::verify_rpc_signature;
use hyper::HeaderMap;
use hyper::Method;
use hyper::StatusCode;
@@ -12,6 +13,7 @@ use s3s::S3Result;
use s3s::header;
use s3s::route::S3Route;
use s3s::s3_error;
use tracing::error;
use super::ADMIN_PREFIX;
use super::RUSTFS_ADMIN_PREFIX;
@@ -84,10 +86,19 @@ where
// check_access before call
async fn check_access(&self, req: &mut S3Request<Body>) -> S3Result<()> {
// TODO: check access by req.credentials
// Check RPC signature verification
if req.uri.path().starts_with(RPC_PREFIX) {
// Skip signature verification for HEAD requests (health checks)
if req.method != Method::HEAD {
verify_rpc_signature(&req.uri.to_string(), &req.method, &req.headers).map_err(|e| {
error!("RPC signature verification failed: {}", e);
s3_error!(AccessDenied, "{}", e)
})?;
}
return Ok(());
}
// For non-RPC admin requests, check credentials
match req.credentials {
Some(_) => Ok(()),
None => Err(s3_error!(AccessDenied, "Signature is required")),
rustfs/src/admin/rpc.rs
+89 -10
View File
@@ -1,14 +1,15 @@
use super::router::AdminOperation;
use super::router::Operation;
use super::router::S3Router;
use crate::storage::ecfs::bytes_stream;
use ecstore::disk::DiskAPI;
use ecstore::disk::WalkDirOptions;
use ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
use ecstore::store::find_local_disk;
use futures::TryStreamExt;
use futures::StreamExt;
use http::StatusCode;
use hyper::Method;
use matchit::Params;
use rustfs_utils::net::bytes_stream;
use s3s::Body;
use s3s::S3Request;
use s3s::S3Response;
@@ -16,24 +17,43 @@ use s3s::S3Result;
use s3s::dto::StreamingBlob;
use s3s::s3_error;
use serde_urlencoded::from_bytes;
use tokio::io::AsyncWriteExt;
use tokio_util::io::ReaderStream;
use tokio_util::io::StreamReader;
use tracing::warn;
pub const RPC_PREFIX: &str = "/rustfs/rpc";
pub fn regist_rpc_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
pub fn register_rpc_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
r.insert(
Method::GET,
format!("{}{}", RPC_PREFIX, "/read_file_stream").as_str(),
AdminOperation(&ReadFile {}),
)?;
r.insert(
Method::HEAD,
format!("{}{}", RPC_PREFIX, "/read_file_stream").as_str(),
AdminOperation(&ReadFile {}),
)?;
r.insert(
Method::PUT,
format!("{}{}", RPC_PREFIX, "/put_file_stream").as_str(),
AdminOperation(&PutFile {}),
)?;
r.insert(
Method::GET,
format!("{}{}", RPC_PREFIX, "/walk_dir").as_str(),
AdminOperation(&WalkDir {}),
)?;
r.insert(
Method::HEAD,
format!("{}{}", RPC_PREFIX, "/walk_dir").as_str(),
AdminOperation(&WalkDir {}),
)?;
Ok(())
}
@@ -50,6 +70,9 @@ pub struct ReadFile {}
#[async_trait::async_trait]
impl Operation for ReadFile {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
if req.method == Method::HEAD {
return Ok(S3Response::new((StatusCode::OK, Body::empty())));
}
let query = {
if let Some(query) = req.uri.query() {
let input: ReadFileQuery =
@@ -79,6 +102,61 @@ impl Operation for ReadFile {
}
}
#[derive(Debug, Default, serde::Deserialize)]
pub struct WalkDirQuery {
disk: String,
}
pub struct WalkDir {}
#[async_trait::async_trait]
impl Operation for WalkDir {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
if req.method == Method::HEAD {
return Ok(S3Response::new((StatusCode::OK, Body::empty())));
}
let query = {
if let Some(query) = req.uri.query() {
let input: WalkDirQuery =
from_bytes(query.as_bytes()).map_err(|e| s3_error!(InvalidArgument, "get query failed1 {:?}", e))?;
input
} else {
WalkDirQuery::default()
}
};
let mut input = req.input;
let body = match input.store_all_unlimited().await {
Ok(b) => b,
Err(e) => {
warn!("get body failed, e: {:?}", e);
return Err(s3_error!(InvalidRequest, "get body failed"));
}
};
// let body_bytes = decrypt_data(input_cred.secret_key.expose().as_bytes(), &body)
// .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidArgument, format!("decrypt_data err {}", e)))?;
let args: WalkDirOptions =
serde_json::from_slice(&body).map_err(|e| s3_error!(InternalError, "unmarshal body err {}", e))?;
let Some(disk) = find_local_disk(&query.disk).await else {
return Err(s3_error!(InvalidArgument, "disk not found"));
};
let (rd, mut wd) = tokio::io::duplex(DEFAULT_READ_BUFFER_SIZE);
tokio::spawn(async move {
if let Err(e) = disk.walk_dir(args, &mut wd).await {
warn!("walk dir err {}", e);
}
});
let body = Body::from(StreamingBlob::wrap(ReaderStream::with_capacity(rd, DEFAULT_READ_BUFFER_SIZE)));
Ok(S3Response::new((StatusCode::OK, body)))
}
}
// /rustfs/rpc/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}"
#[derive(Debug, Default, serde::Deserialize)]
pub struct PutFileQuery {
@@ -86,7 +164,7 @@ pub struct PutFileQuery {
volume: String,
path: String,
append: bool,
size: usize,
size: i64,
}
pub struct PutFile {}
#[async_trait::async_trait]
@@ -116,11 +194,12 @@ impl Operation for PutFile {
.map_err(|e| s3_error!(InternalError, "read file err {}", e))?
};
let mut body = StreamReader::new(req.input.into_stream().map_err(std::io::Error::other));
tokio::io::copy(&mut body, &mut file)
.await
.map_err(|e| s3_error!(InternalError, "copy err {}", e))?;
let mut body = req.input;
while let Some(item) = body.next().await {
let bytes = item.map_err(|e| s3_error!(InternalError, "body stream err {}", e))?;
let result = file.write_all(&bytes).await;
result.map_err(|e| s3_error!(InternalError, "write file err {}", e))?;
}
Ok(S3Response::new((StatusCode::OK, Body::empty())))
}
rustfs/src/auth.rs
+2 -3
View File
@@ -1,9 +1,7 @@
use std::collections::HashMap;
use ecstore::global::get_global_action_cred;
use http::HeaderMap;
use http::Uri;
use iam::error::Error as IamError;
use iam::get_global_action_cred;
use iam::sys::SESSION_POLICY_NAME;
use policy::auth;
use policy::auth::get_claims_from_token_with_secret;
@@ -15,6 +13,7 @@ use s3s::auth::SecretKey;
use s3s::auth::SimpleAuth;
use s3s::s3_error;
use serde_json::Value;
use std::collections::HashMap;
pub struct IAMAuth {
simple_auth: SimpleAuth,
rustfs/src/config/mod.rs
-4
View File
@@ -73,10 +73,6 @@ pub struct Opt {
#[arg(long, env = "RUSTFS_LICENSE")]
pub license: Option<String>,
/// event notifier config file
#[arg(long, env = "RUSTFS_EVENT_CONFIG")]
pub event_config: Option<String>,
}
// lazy_static::lazy_static! {
rustfs/src/event.rs
+29 -16
View File
@@ -1,21 +1,34 @@
use rustfs_event_notifier::NotifierConfig;
use ecstore::config::GLOBAL_ServerConfig;
use tracing::{error, info, instrument};
#[instrument]
pub(crate) async fn init_event_notifier(notifier_config: Option<String>) {
// Initialize event notifier
if notifier_config.is_some() {
info!("event_config is not empty");
tokio::spawn(async move {
let config = NotifierConfig::event_load_config(notifier_config);
let result = rustfs_event_notifier::initialize(config).await;
if let Err(e) = result {
error!("Failed to initialize event notifier: {}", e);
} else {
info!("Event notifier initialized successfully");
}
});
} else {
info!("event_config is empty");
pub(crate) async fn init_event_notifier() {
info!("Initializing event notifier...");
// 1. Get the global configuration loaded by ecstore
let server_config = match GLOBAL_ServerConfig.get() {
Some(config) => config.clone(), // Clone the config to pass ownership
None => {
error!("Event notifier initialization failed: Global server config not loaded.");
return;
}
};
// 2. Check if the notify subsystem exists in the configuration, and skip initialization if it doesn't
if server_config.get_value("notify", "_").is_none() {
info!("'notify' subsystem not configured, skipping event notifier initialization.");
return;
}
info!("Event notifier configuration found, proceeding with initialization.");
// 3. Initialize the notification system asynchronously with a global configuration
// Put it into a separate task to avoid blocking the main initialization process
tokio::spawn(async move {
if let Err(e) = rustfs_notify::initialize(server_config).await {
error!("Failed to initialize event notifier system: {}", e);
} else {
info!("Event notifier system initialized successfully.");
}
});
}
rustfs/src/grpc.rs
-3600
View File
File diff suppressed because it is too large Load Diff
rustfs/src/main.rs
+26 -28
View File
@@ -4,7 +4,7 @@ mod config;
mod console;
mod error;
mod event;
mod grpc;
// mod grpc;
pub mod license;
mod logging;
mod server;
@@ -12,9 +12,9 @@ mod service;
mod storage;
use crate::auth::IAMAuth;
use crate::console::{CONSOLE_CONFIG, init_console_cfg};
use crate::console::{init_console_cfg, CONSOLE_CONFIG};
// Ensure the correct path for parse_license is imported
use crate::server::{SHUTDOWN_TIMEOUT, ServiceState, ServiceStateManager, ShutdownSignal, wait_for_shutdown};
use crate::server::{wait_for_shutdown, ServiceState, ServiceStateManager, ShutdownSignal, SHUTDOWN_TIMEOUT};
use bytes::Bytes;
use chrono::Datelike;
use clap::Parser;
@@ -22,22 +22,22 @@ use common::{
// error::{Error, Result},
globals::set_global_addr,
};
use ecstore::StorageAPI;
use ecstore::bucket::metadata_sys::init_bucket_metadata_sys;
use ecstore::cmd::bucket_replication::init_bucket_replication_pool;
use ecstore::config as ecconfig;
use ecstore::config::GLOBAL_ConfigSys;
use ecstore::heal::background_heal_ops::init_auto_heal;
use ecstore::rpc::make_server;
use ecstore::store_api::BucketOptions;
use ecstore::StorageAPI;
use ecstore::{
endpoints::EndpointServerPools,
heal::data_scanner::init_data_scanner,
set_global_endpoints,
store::{ECStore, init_local_disks},
store::{init_local_disks, ECStore},
update_erasure_type,
};
use ecstore::{global::set_global_rustfs_port, notification_sys::new_global_notification_sys};
use grpc::make_server;
use http::{HeaderMap, Request as HttpRequest, Response};
use hyper_util::server::graceful::GracefulShutdown;
use hyper_util::{
@@ -49,7 +49,7 @@ use iam::init_iam_sys;
use license::init_license;
use protos::proto_gen::node_service::node_service_server::NodeServiceServer;
use rustfs_config::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, RUSTFS_TLS_CERT, RUSTFS_TLS_KEY};
use rustfs_obs::{SystemObserver, init_obs, set_global_guard};
use rustfs_obs::{init_obs, set_global_guard, SystemObserver};
use rustfs_utils::net::parse_and_resolve_address;
use rustls::ServerConfig;
use s3s::{host::MultiDomain, service::S3ServiceBuilder};
@@ -60,13 +60,12 @@ use std::net::SocketAddr;
use std::sync::Arc;
use std::time::Duration;
use tokio::net::TcpListener;
use tokio::signal::unix::{SignalKind, signal};
use tokio::signal::unix::{signal, SignalKind};
use tokio_rustls::TlsAcceptor;
use tonic::{Request, Status, metadata::MetadataValue};
use tonic::{metadata::MetadataValue, Request, Status};
use tower_http::cors::CorsLayer;
use tower_http::trace::TraceLayer;
use tracing::{Span, instrument};
use tracing::{debug, error, info, warn};
use tracing::{debug, error, info, instrument, warn, Span};
const MI_B: usize = 1024 * 1024;
@@ -119,9 +118,6 @@ async fn main() -> Result<()> {
async fn run(opt: config::Opt) -> Result<()> {
debug!("opt: {:?}", &opt);
// Initialize event notifier
event::init_event_notifier(opt.event_config).await;
let server_addr = parse_and_resolve_address(opt.address.as_str()).map_err(Error::other)?;
let server_port = server_addr.port();
let server_address = server_addr.to_string();
@@ -129,7 +125,7 @@ async fn run(opt: config::Opt) -> Result<()> {
debug!("server_address {}", &server_address);
// Set up AK and SK
iam::init_global_action_cred(Some(opt.access_key.clone()), Some(opt.secret_key.clone()))?;
ecstore::global::init_global_action_cred(Some(opt.access_key.clone()), Some(opt.secret_key.clone()));
set_global_rustfs_port(server_port);
@@ -502,15 +498,17 @@ async fn run(opt: config::Opt) -> Result<()> {
});
// init store
let store = ECStore::new(server_address.clone(), endpoint_pools.clone())
.await
.inspect_err(|err| {
error!("ECStore::new {:?}", err);
})?;
let store = ECStore::new(server_addr, endpoint_pools.clone()).await.inspect_err(|err| {
error!("ECStore::new {:?}", err);
})?;
ecconfig::init();
// config system configuration
GLOBAL_ConfigSys.init(store.clone()).await?;
// Initialize event notifier
event::init_event_notifier().await;
let buckets_list = store
.list_bucket(&BucketOptions {
no_metadata: true,
@@ -570,14 +568,14 @@ async fn run(opt: config::Opt) -> Result<()> {
// update the status to stopping first
state_manager.update(ServiceState::Stopping);
// Stop the notification system
if rustfs_event_notifier::is_ready() {
// stop event notifier
rustfs_event_notifier::shutdown().await.map_err(|err| {
error!("Failed to shut down the notification system: {}", err);
Error::other(err)
})?;
}
// // Stop the notification system
// if rustfs_event::is_ready() {
// // stop event notifier
// rustfs_event::shutdown().await.map_err(|err| {
// error!("Failed to shut down the notification system: {}", err);
// Error::from_string(err.to_string())
// })?;
// }
info!("Server is stopping...");
let _ = shutdown_tx.send(());
rustfs/src/storage/ecfs.rs
+131 -63
View File
@@ -8,6 +8,7 @@ use crate::storage::access::ReqInfo;
use crate::storage::options::copy_dst_opts;
use crate::storage::options::copy_src_opts;
use crate::storage::options::{extract_metadata_from_mime, get_opts};
use api::object_store::bytes_stream;
use api::query::Context;
use api::query::Query;
use api::server::dbms::DatabaseManagerSystem;
@@ -30,10 +31,15 @@ use ecstore::bucket::metadata_sys;
use ecstore::bucket::policy_sys::PolicySys;
use ecstore::bucket::tagging::decode_tags;
use ecstore::bucket::tagging::encode_tags;
use ecstore::bucket::utils::serialize;
use ecstore::bucket::versioning_sys::BucketVersioningSys;
use ecstore::cmd::bucket_replication::ReplicationStatusType;
use ecstore::cmd::bucket_replication::ReplicationType;
use ecstore::cmd::bucket_replication::get_must_replicate_options;
use ecstore::cmd::bucket_replication::must_replicate;
use ecstore::cmd::bucket_replication::schedule_replication;
use ecstore::compress::MIN_COMPRESSIBLE_SIZE;
use ecstore::compress::is_compressible;
use ecstore::error::StorageError;
use ecstore::new_object_layer_fn;
use ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
@@ -65,8 +71,13 @@ use policy::policy::Validator;
use policy::policy::action::Action;
use policy::policy::action::S3Action;
use query::instance::make_rustfsms;
use rustfs_filemeta::headers::RESERVED_METADATA_PREFIX_LOWER;
use rustfs_filemeta::headers::{AMZ_DECODED_CONTENT_LENGTH, AMZ_OBJECT_TAGGING};
use rustfs_rio::CompressReader;
use rustfs_rio::HashReader;
use rustfs_rio::Reader;
use rustfs_rio::WarpReader;
use rustfs_utils::CompressionAlgorithm;
use rustfs_utils::path::path_join_buf;
use rustfs_zip::CompressionFormat;
use s3s::S3;
@@ -92,7 +103,6 @@ use tracing::debug;
use tracing::error;
use tracing::info;
use tracing::warn;
use transform_stream::AsyncTryStream;
use uuid::Uuid;
use ecstore::bucket::{
@@ -186,14 +196,31 @@ impl FS {
fpath = format!("{}/{}", prefix, fpath);
}
let size = f.header().size().unwrap_or_default() as usize;
let mut size = f.header().size().unwrap_or_default() as i64;
println!("Extracted: {}, size {}", fpath, size);
// Wrap the tar entry with BufReader to make it compatible with Reader trait
let reader = Box::new(tokio::io::BufReader::new(f));
let hrd = HashReader::new(reader, size as i64, size as i64, None, false).map_err(ApiError::from)?;
let mut reader = PutObjReader::new(hrd, size);
let mut reader: Box<dyn Reader> = Box::new(WarpReader::new(f));
let mut metadata = HashMap::new();
let actual_size = size;
if is_compressible(&HeaderMap::new(), &fpath) && size > MIN_COMPRESSIBLE_SIZE as i64 {
metadata.insert(
format!("{}compression", RESERVED_METADATA_PREFIX_LOWER),
CompressionAlgorithm::default().to_string(),
);
metadata.insert(format!("{}actual-size", RESERVED_METADATA_PREFIX_LOWER,), size.to_string());
let hrd = HashReader::new(reader, size, actual_size, None, false).map_err(ApiError::from)?;
reader = Box::new(CompressReader::new(hrd, CompressionAlgorithm::default()));
size = -1;
}
let hrd = HashReader::new(reader, size, actual_size, None, false).map_err(ApiError::from)?;
let mut reader = PutObjReader::new(hrd);
let _obj_info = store
.put_object(&bucket, &fpath, &mut reader, &ObjectOptions::default())
@@ -208,6 +235,21 @@ impl FS {
// e_tag,
// ..Default::default()
// };
// let event_args = rustfs_notify::event::EventArgs {
// event_name: EventName::ObjectCreatedPut, // 或者其他相应的事件类型
// bucket_name: bucket.clone(),
// object: _obj_info.clone(), // clone() 或传递所需字段
// req_params: crate::storage::global::extract_req_params(&req), // 假设有一个辅助函数来提取请求参数
// resp_elements: crate::storage::global::extract_resp_elements(&output), // 假设有一个辅助函数来提取响应元素
// host: crate::storage::global::get_request_host(&req.headers), // 假设的辅助函数
// user_agent: crate::storage::global::get_request_user_agent(&req.headers), // 假设的辅助函数
// };
//
// // 异步调用，不会阻塞当前请求的响应
// tokio::spawn(async move {
// rustfs_notify::notifier::GLOBAL_NOTIFIER.notify(event_args).await;
// });
}
}
@@ -326,13 +368,10 @@ impl S3 for FS {
src_info.metadata_only = true;
}
let hrd = HashReader::new(gr.stream, gr.object_info.size as i64, gr.object_info.size as i64, None, false)
.map_err(ApiError::from)?;
let reader = Box::new(WarpReader::new(gr.stream));
let hrd = HashReader::new(reader, gr.object_info.size, gr.object_info.size, None, false).map_err(ApiError::from)?;
src_info.put_object_reader = Some(PutObjReader {
stream: hrd,
content_length: gr.object_info.size as usize,
});
src_info.put_object_reader = Some(PutObjReader::new(hrd));
// check quota
// TODO: src metadada
@@ -543,13 +582,13 @@ impl S3 for FS {
let rs = range.map(|v| match v {
Range::Int { first, last } => HTTPRangeSpec {
is_suffix_length: false,
start: first as usize,
end: last.map(|v| v as usize),
start: first as i64,
end: if let Some(last) = last { last as i64 } else { -1 },
},
Range::Suffix { length } => HTTPRangeSpec {
is_suffix_length: true,
start: length as usize,
end: None,
start: length as i64,
end: -1,
},
});
@@ -590,7 +629,7 @@ impl S3 for FS {
let body = Some(StreamingBlob::wrap(bytes_stream(
ReaderStream::with_capacity(reader.stream, DEFAULT_READ_BUFFER_SIZE),
info.size,
info.size as usize,
)));
let output = GetObjectOutput {
@@ -644,13 +683,13 @@ impl S3 for FS {
let rs = range.map(|v| match v {
Range::Int { first, last } => HTTPRangeSpec {
is_suffix_length: false,
start: first as usize,
end: last.map(|v| v as usize),
start: first as i64,
end: if let Some(last) = last { last as i64 } else { -1 },
},
Range::Suffix { length } => HTTPRangeSpec {
is_suffix_length: true,
start: length as usize,
end: None,
start: length as i64,
end: -1,
},
});
@@ -671,8 +710,8 @@ impl S3 for FS {
// warn!("head_object info {:?}", &info);
let content_type = {
if let Some(content_type) = info.content_type {
match ContentType::from_str(&content_type) {
if let Some(content_type) = &info.content_type {
match ContentType::from_str(content_type) {
Ok(res) => Some(res),
Err(err) => {
error!("parse content-type err {} {:?}", &content_type, err);
@@ -686,10 +725,14 @@ impl S3 for FS {
};
let last_modified = info.mod_time.map(Timestamp::from);
// TODO: range download
let content_length = info.get_actual_size().map_err(ApiError::from)?;
let metadata = info.user_defined;
let output = HeadObjectOutput {
content_length: Some(try_!(i64::try_from(info.size))),
content_length: Some(content_length),
content_type,
last_modified,
e_tag: info.etag,
@@ -813,7 +856,7 @@ impl S3 for FS {
let mut obj = Object {
key: Some(v.name.to_owned()),
last_modified: v.mod_time.map(Timestamp::from),
size: Some(v.size as i64),
size: Some(v.size),
e_tag: v.etag.clone(),
..Default::default()
};
@@ -892,7 +935,7 @@ impl S3 for FS {
ObjectVersion {
key: Some(v.name.to_owned()),
last_modified: v.mod_time.map(Timestamp::from),
size: Some(v.size as i64),
size: Some(v.size),
version_id: v.version_id.map(|v| v.to_string()),
is_latest: Some(v.is_latest),
e_tag: v.etag.clone(),
@@ -933,7 +976,6 @@ impl S3 for FS {
return self.put_object_extract(req).await;
}
info!("put object");
let input = req.input;
if let Some(ref storage_class) = input.storage_class {
@@ -956,7 +998,7 @@ impl S3 for FS {
let Some(body) = body else { return Err(s3_error!(IncompleteBody)) };
let content_length = match content_length {
let mut size = match content_length {
Some(c) => c,
None => {
if let Some(val) = req.headers.get(AMZ_DECODED_CONTENT_LENGTH) {
@@ -971,9 +1013,6 @@ impl S3 for FS {
};
let body = StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string()))));
let body = Box::new(tokio::io::BufReader::new(body));
let hrd = HashReader::new(body, content_length as i64, content_length as i64, None, false).map_err(ApiError::from)?;
let mut reader = PutObjReader::new(hrd, content_length as usize);
// let body = Box::new(StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))));
@@ -991,10 +1030,32 @@ impl S3 for FS {
metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags);
}
let mut reader: Box<dyn Reader> = Box::new(WarpReader::new(body));
let actual_size = size;
if is_compressible(&req.headers, &key) && size > MIN_COMPRESSIBLE_SIZE as i64 {
metadata.insert(
format!("{}compression", RESERVED_METADATA_PREFIX_LOWER),
CompressionAlgorithm::default().to_string(),
);
metadata.insert(format!("{}actual-size", RESERVED_METADATA_PREFIX_LOWER,), size.to_string());
let hrd = HashReader::new(reader, size as i64, size as i64, None, false).map_err(ApiError::from)?;
reader = Box::new(CompressReader::new(hrd, CompressionAlgorithm::default()));
size = -1;
}
// TODO: md5 check
let reader = HashReader::new(reader, size, actual_size, None, false).map_err(ApiError::from)?;
let mut reader = PutObjReader::new(reader);
let mt = metadata.clone();
let mt2 = metadata.clone();
let opts: ObjectOptions = put_opts(&bucket, &key, version_id, &req.headers, Some(mt))
let mut opts: ObjectOptions = put_opts(&bucket, &key, version_id, &req.headers, Some(mt))
.await
.map_err(ApiError::from)?;
@@ -1002,8 +1063,9 @@ impl S3 for FS {
get_must_replicate_options(&mt2, "", ReplicationStatusType::Unknown, ReplicationType::ObjectReplicationType, &opts);
let dsc = must_replicate(&bucket, &key, &repoptions).await;
warn!("dsc {}", &dsc.replicate_any().clone());
// warn!("dsc {}", &dsc.replicate_any().clone());
if dsc.replicate_any() {
if let Some(metadata) = opts.user_defined.as_mut() {
let k = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-timestamp");
let now: DateTime<Utc> = Utc::now();
let formatted_time = now.to_rfc3339();
@@ -1011,8 +1073,7 @@ impl S3 for FS {
let k = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-status");
metadata.insert(k, dsc.pending_status());
}
debug!("put_object opts {:?}", &opts);
}
let obj_info = store
.put_object(&bucket, &key, &mut reader, &opts)
@@ -1065,6 +1126,13 @@ impl S3 for FS {
metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags);
}
if is_compressible(&req.headers, &key) {
metadata.insert(
format!("{}compression", RESERVED_METADATA_PREFIX_LOWER),
CompressionAlgorithm::default().to_string(),
);
}
let opts: ObjectOptions = put_opts(&bucket, &key, version_id, &req.headers, Some(metadata))
.await
.map_err(ApiError::from)?;
@@ -1102,7 +1170,7 @@ impl S3 for FS {
// let upload_id =
let body = body.ok_or_else(|| s3_error!(IncompleteBody))?;
let content_length = match content_length {
let mut size = match content_length {
Some(c) => c,
None => {
if let Some(val) = req.headers.get(AMZ_DECODED_CONTENT_LENGTH) {
@@ -1117,21 +1185,42 @@ impl S3 for FS {
};
let body = StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string()))));
let body = Box::new(tokio::io::BufReader::new(body));
let hrd = HashReader::new(body, content_length as i64, content_length as i64, None, false).map_err(ApiError::from)?;
// mc cp step 4
let mut data = PutObjReader::new(hrd, content_length as usize);
let opts = ObjectOptions::default();
let Some(store) = new_object_layer_fn() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
// TODO: hash_reader
let fi = store
.get_multipart_info(&bucket, &key, &upload_id, &opts)
.await
.map_err(ApiError::from)?;
let is_compressible = fi
.user_defined
.contains_key(format!("{}compression", RESERVED_METADATA_PREFIX_LOWER).as_str());
let mut reader: Box<dyn Reader> = Box::new(WarpReader::new(body));
let actual_size = size;
if is_compressible {
let hrd = HashReader::new(reader, size, actual_size, None, false).map_err(ApiError::from)?;
reader = Box::new(CompressReader::new(hrd, CompressionAlgorithm::default()));
size = -1;
}
// TODO: md5 check
let reader = HashReader::new(reader, size, actual_size, None, false).map_err(ApiError::from)?;
let mut reader = PutObjReader::new(reader);
let info = store
.put_object_part(&bucket, &key, &upload_id, part_id, &mut data, &opts)
.put_object_part(&bucket, &key, &upload_id, part_id, &mut reader, &opts)
.await
.map_err(ApiError::from)?;
@@ -1749,7 +1838,7 @@ impl S3 for FS {
let object_lock_configuration = match metadata_sys::get_object_lock_config(&bucket).await {
Ok((cfg, _created)) => Some(cfg),
Err(err) => {
warn!("get_object_lock_config err {:?}", err);
debug!("get_object_lock_config err {:?}", err);
None
}
};
@@ -2399,24 +2488,3 @@ impl S3 for FS {
}))
}
}
#[allow(dead_code)]
pub fn bytes_stream<S, E>(stream: S, content_length: usize) -> impl Stream<Item = std::result::Result<Bytes, E>> + Send + 'static
where
S: Stream<Item = std::result::Result<Bytes, E>> + Send + 'static,
E: Send + 'static,
{
AsyncTryStream::<Bytes, E, _>::new(|mut y| async move {
pin_mut!(stream);
let mut remaining: usize = content_length;
while let Some(result) = stream.next().await {
let mut bytes = result?;
if bytes.len() > remaining {
bytes.truncate(remaining);
}
remaining -= bytes.len();
y.yield_ok(bytes).await;
}
Ok(())
})
}
rustfs/src/storage/event_notifier.rs
-17
View File
@@ -1,17 +0,0 @@
use rustfs_event_notifier::{Event, Metadata};
/// Create a new metadata object
#[allow(dead_code)]
pub(crate) fn create_metadata() -> Metadata {
// Create a new metadata object
let mut metadata = Metadata::new();
metadata.set_configuration_id("test-config".to_string());
// Return the created metadata object
metadata
}
/// Create a new event object
#[allow(dead_code)]
pub(crate) async fn send_event(event: Event) -> Result<(), Box<dyn std::error::Error>> {
rustfs_event_notifier::send_event(event).await.map_err(|e| e.into())
}
rustfs/src/storage/global.rs
+47
View File
@@ -0,0 +1,47 @@
use hyper::HeaderMap;
use s3s::{S3Request, S3Response};
use std::collections::HashMap;
/// Extract request parameters from S3Request, mainly header information.
#[allow(dead_code)]
pub fn extract_req_params<T>(req: &S3Request<T>) -> HashMap<String, String> {
let mut params = HashMap::new();
for (key, value) in req.headers.iter() {
if let Ok(val_str) = value.to_str() {
params.insert(key.as_str().to_string(), val_str.to_string());
}
}
params
}
/// Extract response elements from S3Response, mainly header information.
#[allow(dead_code)]
pub fn extract_resp_elements<T>(resp: &S3Response<T>) -> HashMap<String, String> {
let mut params = HashMap::new();
for (key, value) in resp.headers.iter() {
if let Ok(val_str) = value.to_str() {
params.insert(key.as_str().to_string(), val_str.to_string());
}
}
params
}
/// Get host from header information.
#[allow(dead_code)]
pub fn get_request_host(headers: &HeaderMap) -> String {
headers
.get("host")
.and_then(|v| v.to_str().ok())
.unwrap_or_default()
.to_string()
}
/// Get user-agent from header information.
#[allow(dead_code)]
pub fn get_request_user_agent(headers: &HeaderMap) -> String {
headers
.get("user-agent")
.and_then(|v| v.to_str().ok())
.unwrap_or_default()
.to_string()
}
rustfs/src/storage/mod.rs
+1 -1
View File
@@ -1,5 +1,5 @@
pub mod access;
pub mod ecfs;
// pub mod error;
mod event_notifier;
mod global;
pub mod options;