diff --git a/Cargo.lock b/Cargo.lock index 2f50fe8e3..c9bfb8548 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3626,7 +3626,6 @@ dependencies = [ "policy", "protos", "rand 0.9.1", - "reader", "reed-solomon-simd", "regex", "reqwest", @@ -3636,6 +3635,7 @@ dependencies = [ "rustfs-filemeta", "rustfs-rio", "rustfs-rsc", + "rustfs-signer", "rustfs-utils", "rustls 0.23.28", "s3s", @@ -7763,25 +7763,6 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "reader" -version = "0.0.1" -dependencies = [ - "async-trait", - "base64-simd", - "bytes", - "common", - "futures", - "hex-simd", - "md-5", - "pin-project-lite", - "s3s", - "sha2 0.11.0-pre.5", - "thiserror 2.0.12", - "tokio", - "tracing", -] - [[package]] name = "readme-rustdocifier" version = "0.1.1" @@ -8441,6 +8422,23 @@ dependencies = [ "urlencoding", ] +[[package]] +name = "rustfs-signer" +version = "0.0.1" +dependencies = [ + "bytes", + "http 1.3.1", + "hyper 1.6.0", + "lazy_static", + "rand 0.9.1", + "rustfs-utils", + "serde", + "serde_urlencoded", + "tempfile", + "time", + "tracing", +] + [[package]] name = "rustfs-utils" version = "0.0.1" diff --git a/crates/signer/src/request_signature_streaming.rs b/crates/signer/src/request_signature_streaming.rs index 08320f6f4..cd558fc11 100644 --- a/crates/signer/src/request_signature_streaming.rs +++ b/crates/signer/src/request_signature_streaming.rs @@ -1,10 +1,10 @@ -use http::{request, HeaderMap, HeaderValue}; +use http::{HeaderMap, HeaderValue, request}; use lazy_static::lazy_static; use std::collections::HashMap; use time::{OffsetDateTime, macros::format_description}; -use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; use super::request_signature_v4::{SERVICE_TYPE_S3, get_scope, get_signature, get_signing_key}; +use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; const STREAMING_SIGN_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; const STREAMING_SIGN_TRAILER_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"; @@ -67,7 +67,10 @@ pub fn streaming_sign_v4( if trailer.is_empty() { headers.append("X-Amz-Content-Sha256", HeaderValue::from_str(STREAMING_SIGN_ALGORITHM).expect("err")); } else { - headers.append("X-Amz-Content-Sha256", HeaderValue::from_str(STREAMING_SIGN_TRAILER_ALGORITHM).expect("err")); + headers.append( + "X-Amz-Content-Sha256", + HeaderValue::from_str(STREAMING_SIGN_TRAILER_ALGORITHM).expect("err"), + ); for (k, _) in &trailer { headers.append("X-Amz-Trailer", k.as_str().to_lowercase().parse().unwrap()); } diff --git a/crates/signer/src/request_signature_streaming_unsigned_trailer.rs b/crates/signer/src/request_signature_streaming_unsigned_trailer.rs index 1c096eaae..e3b903fe1 100644 --- a/crates/signer/src/request_signature_streaming_unsigned_trailer.rs +++ b/crates/signer/src/request_signature_streaming_unsigned_trailer.rs @@ -1,4 +1,4 @@ -use http::{request, HeaderValue}; +use http::{HeaderValue, request}; use time::{OffsetDateTime, macros::format_description}; pub fn streaming_unsigned_v4( diff --git a/crates/signer/src/request_signature_v2.rs b/crates/signer/src/request_signature_v2.rs index 6cd59349b..f71c32956 100644 --- a/crates/signer/src/request_signature_v2.rs +++ b/crates/signer/src/request_signature_v2.rs @@ -5,8 +5,8 @@ use std::collections::HashMap; use std::fmt::Write; use time::{OffsetDateTime, format_description}; -use rustfs_utils::crypto::{base64_encode, hex, hmac_sha1}; use super::utils::get_host_addr; +use rustfs_utils::crypto::{base64_encode, hex, hmac_sha1}; const _SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256"; const SIGN_V2_ALGORITHM: &str = "AWS"; diff --git a/crates/signer/src/request_signature_v4.rs b/crates/signer/src/request_signature_v4.rs index b989db149..30f6b989f 100644 --- a/crates/signer/src/request_signature_v4.rs +++ b/crates/signer/src/request_signature_v4.rs @@ -8,10 +8,10 @@ use std::fmt::Write; use time::{OffsetDateTime, macros::format_description}; use tracing::debug; -use rustfs_utils::crypto::{hex, hex_sha256, hmac_sha256}; +use super::constants::UNSIGNED_PAYLOAD; use super::request_signature_streaming_unsigned_trailer::streaming_unsigned_v4; use super::utils::{get_host_addr, sign_v4_trim_all}; -use super::constants::UNSIGNED_PAYLOAD; +use rustfs_utils::crypto::{hex, hex_sha256, hmac_sha256}; pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256"; pub const SERVICE_TYPE_S3: &str = "s3"; diff --git a/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 620070885..535b34ac0 100644 --- a/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -411,10 +411,6 @@ pub struct TransitionState { last_day_stats: Arc>>, } -type BoxFuture<'a, T> = Pin + Send + 'a>>; -//type RetKill = impl Future> + Send + 'static; -//type RetTransitionTask = impl Future>> + Send + 'static; - impl TransitionState { #[allow(clippy::new_ret_no_self)] pub fn new() -> Arc { diff --git a/ecstore/src/bucket/lifecycle/lifecycle.rs b/ecstore/src/bucket/lifecycle/lifecycle.rs index 3d4576eb2..0d9343878 100644 --- a/ecstore/src/bucket/lifecycle/lifecycle.rs +++ b/ecstore/src/bucket/lifecycle/lifecycle.rs @@ -25,7 +25,8 @@ pub const TRANSITION_PENDING: &str = "pending"; const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration allows a maximum of 1000 rules"; const ERR_LIFECYCLE_NO_RULE: &str = "Lifecycle configuration should have at least one rule"; const ERR_LIFECYCLE_DUPLICATE_ID: &str = "Rule ID must be unique. Found same ID for more than one rule"; -const _ERR_XML_NOT_WELL_FORMED: &str = "The XML you provided was not well-formed or did not validate against our published schema"; +const _ERR_XML_NOT_WELL_FORMED: &str = + "The XML you provided was not well-formed or did not validate against our published schema"; const ERR_LIFECYCLE_BUCKET_LOCKED: &str = "ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket"; @@ -697,11 +698,6 @@ pub struct ExpirationOptions { pub expire: bool, } -impl ExpirationOptions { - fn msg_size(&self) -> i64 { - 1 + 7 + 10 - } -} #[derive(Debug, Clone)] pub struct TransitionOptions { diff --git a/ecstore/src/checksum.rs b/ecstore/src/checksum.rs index cd2047355..d04ac5d78 100644 --- a/ecstore/src/checksum.rs +++ b/ecstore/src/checksum.rs @@ -11,8 +11,8 @@ use std::collections::HashMap; use crate::client::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart}; use crate::{disk::DiskAPI, store_api::GetObjectReader}; -use rustfs_utils::hasher::{Hasher, Sha256}; use rustfs_utils::crypto::{base64_decode, base64_encode}; +use rustfs_utils::hasher::{Hasher, Sha256}; use s3s::header::{ X_AMZ_CHECKSUM_ALGORITHM, X_AMZ_CHECKSUM_CRC32, X_AMZ_CHECKSUM_CRC32C, X_AMZ_CHECKSUM_SHA1, X_AMZ_CHECKSUM_SHA256, }; diff --git a/ecstore/src/client/api_remove.rs b/ecstore/src/client/api_remove.rs index fe0b6023c..88275b360 100644 --- a/ecstore/src/client/api_remove.rs +++ b/ecstore/src/client/api_remove.rs @@ -24,8 +24,8 @@ use crate::{ disk::DiskAPI, store_api::{GetObjectReader, ObjectInfo, StorageAPI}, }; -use rustfs_utils::hasher::{sum_md5_base64, sum_sha256_hex}; use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; +use rustfs_utils::hasher::{sum_md5_base64, sum_sha256_hex}; pub struct RemoveBucketOptions { _forced_elete: bool, diff --git a/ecstore/src/client/bucket_cache.rs b/ecstore/src/client/bucket_cache.rs index 028a726d7..e291984e8 100644 --- a/ecstore/src/client/bucket_cache.rs +++ b/ecstore/src/client/bucket_cache.rs @@ -17,8 +17,8 @@ use crate::client::{ api_error_response::{http_resp_to_error_response, to_error_response}, transition_api::{Document, TransitionClient}, }; -use rustfs_utils::hasher::{Hasher, Sha256}; use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; +use rustfs_utils::hasher::{Hasher, Sha256}; use s3s::Body; use s3s::S3ErrorCode; diff --git a/ecstore/src/client/transition_api.rs b/ecstore/src/client/transition_api.rs index 2003e17de..f33b8ad42 100644 --- a/ecstore/src/client/transition_api.rs +++ b/ecstore/src/client/transition_api.rs @@ -46,8 +46,8 @@ use crate::client::{ credentials::{CredContext, Credentials, SignatureType, Static}, }; use crate::{checksum::ChecksumMode, store_api::GetObjectReader}; -use rustfs_utils::hasher::{MD5, Sha256}; use rustfs_rio::HashReader; +use rustfs_utils::hasher::{MD5, Sha256}; use rustfs_utils::{ net::get_endpoint_url, retry::{MAX_RETRY, new_retry_timer}, @@ -431,8 +431,13 @@ impl TransitionClient { } } if signer_type == SignatureType::SignatureV2 { - req_builder = - rustfs_signer::pre_sign_v2(req_builder, &access_key_id, &secret_access_key, metadata.expires, is_virtual_host); + req_builder = rustfs_signer::pre_sign_v2( + req_builder, + &access_key_id, + &secret_access_key, + metadata.expires, + is_virtual_host, + ); } else if signer_type == SignatureType::SignatureV4 { req_builder = rustfs_signer::pre_sign_v4( req_builder, diff --git a/ecstore/src/tier/tier.rs b/ecstore/src/tier/tier.rs index c96d1e671..6a43d428c 100644 --- a/ecstore/src/tier/tier.rs +++ b/ecstore/src/tier/tier.rs @@ -61,25 +61,21 @@ lazy_static! { message: "Specified remote credentials are empty".to_string(), status_code: StatusCode::FORBIDDEN, }; - pub static ref ERR_TIER_BACKEND_IN_USE: AdminError = AdminError { code: "XRustFSAdminTierBackendInUse".to_string(), message: "Specified remote tier is already in use".to_string(), status_code: StatusCode::CONFLICT, }; - pub static ref ERR_TIER_TYPE_UNSUPPORTED: AdminError = AdminError { code: "XRustFSAdminTierTypeUnsupported".to_string(), message: "Specified tier type is unsupported".to_string(), status_code: StatusCode::BAD_REQUEST, }; - pub static ref ERR_TIER_BACKEND_NOT_EMPTY: AdminError = AdminError { code: "XRustFSAdminTierBackendNotEmpty".to_string(), message: "Specified remote backend is not empty".to_string(), status_code: StatusCode::BAD_REQUEST, }; - pub static ref ERR_TIER_INVALID_CONFIG: AdminError = AdminError { code: "XRustFSAdminTierInvalidConfig".to_string(), message: "Unable to setup remote tier, check tier configuration".to_string(), diff --git a/ecstore/src/tier/tier_handlers.rs b/ecstore/src/tier/tier_handlers.rs index 65136f52a..289799259 100644 --- a/ecstore/src/tier/tier_handlers.rs +++ b/ecstore/src/tier/tier_handlers.rs @@ -8,46 +8,39 @@ lazy_static! { message: "Specified remote tier already exists".to_string(), status_code: StatusCode::CONFLICT, }; - pub static ref ERR_TIER_NOT_FOUND: AdminError = AdminError { code: "XRustFSAdminTierNotFound".to_string(), message: "Specified remote tier was not found".to_string(), status_code: StatusCode::NOT_FOUND, }; - pub static ref ERR_TIER_NAME_NOT_UPPERCASE: AdminError = AdminError { code: "XRustFSAdminTierNameNotUpperCase".to_string(), message: "Tier name must be in uppercase".to_string(), status_code: StatusCode::BAD_REQUEST, }; - pub static ref ERR_TIER_BUCKET_NOT_FOUND: AdminError = AdminError { code: "XRustFSAdminTierBucketNotFound".to_string(), message: "Remote tier bucket not found".to_string(), status_code: StatusCode::BAD_REQUEST, }; - pub static ref ERR_TIER_INVALID_CREDENTIALS: AdminError = AdminError { code: "XRustFSAdminTierInvalidCredentials".to_string(), message: "Invalid remote tier credentials".to_string(), status_code: StatusCode::BAD_REQUEST, }; - pub static ref ERR_TIER_RESERVED_NAME: AdminError = AdminError { code: "XRustFSAdminTierReserved".to_string(), message: "Cannot use reserved tier name".to_string(), status_code: StatusCode::BAD_REQUEST, }; - pub static ref ERR_TIER_PERM_ERR: AdminError = AdminError { code: "TierPermErr".to_string(), message: "Tier Perm Err".to_string(), status_code: StatusCode::OK, }; - pub static ref ERR_TIER_CONNECT_ERR: AdminError = AdminError { code: "TierConnectErr".to_string(), message: "Tier Connect Err".to_string(), status_code: StatusCode::OK, }; -} \ No newline at end of file +} diff --git a/ecstore/src/tier/warm_backend.rs b/ecstore/src/tier/warm_backend.rs index 8cbed67bf..ac5f43c48 100644 --- a/ecstore/src/tier/warm_backend.rs +++ b/ecstore/src/tier/warm_backend.rs @@ -5,9 +5,6 @@ #![allow(unused_must_use)] #![allow(clippy::all)] -use bytes::Bytes; -use std::collections::HashMap; -use http::StatusCode; use crate::client::{ admin_handler_utils::AdminError, transition_api::{ReadCloser, ReaderImpl}, @@ -21,6 +18,9 @@ use crate::tier::{ warm_backend_rustfs::WarmBackendRustFS, warm_backend_s3::WarmBackendS3, }; +use bytes::Bytes; +use http::StatusCode; +use std::collections::HashMap; use tracing::{info, warn}; pub type WarmBackendImpl = Box; diff --git a/rustfs/src/admin/handlers/tier.rs b/rustfs/src/admin/handlers/tier.rs index f46037fc4..3b691d2ce 100644 --- a/rustfs/src/admin/handlers/tier.rs +++ b/rustfs/src/admin/handlers/tier.rs @@ -1,7 +1,7 @@ #![allow(unused_variables, unused_mut, unused_must_use)] -use time::OffsetDateTime; use http::{HeaderMap, StatusCode}; +use time::OffsetDateTime; //use iam::get_global_action_cred; use matchit::Params; use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error}; @@ -141,14 +141,12 @@ impl Operation for AddTier { "tier connect error!", )); } else if err.code == ERR_TIER_INVALID_CREDENTIALS.code { - return Err(S3Error::with_message( - S3ErrorCode::Custom(err.code.clone().into()), - err.message.clone(), - )); + return Err(S3Error::with_message(S3ErrorCode::Custom(err.code.clone().into()), err.message.clone())); } else { warn!("tier_config_mgr add failed, e: {:?}", err); return Err(S3Error::with_message( - S3ErrorCode::Custom("TierAddFailed".into()), format!("tier add failed. {}", err.to_string()) + S3ErrorCode::Custom("TierAddFailed".into()), + format!("tier add failed. {}", err.to_string()), )); } } @@ -208,9 +206,7 @@ impl Operation for EditTier { match tier_config_mgr.edit(&tier_name, creds).await { Err(err) => { if err.code == ERR_TIER_NOT_FOUND.code { - return Err(S3Error::with_message( - S3ErrorCode::Custom("TierNotFound".into()), "tier not found!" - )); + return Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found!")); } else if err.code == ERR_TIER_MISSING_CREDENTIALS.code { return Err(S3Error::with_message( S3ErrorCode::Custom("TierMissingCredentials".into()), @@ -219,7 +215,8 @@ impl Operation for EditTier { } else { warn!("tier_config_mgr edit failed, e: {:?}", err); return Err(S3Error::with_message( - S3ErrorCode::Custom("TierEditFailed".into()), format!("tier edit failed. {}", err.to_string()) + S3ErrorCode::Custom("TierEditFailed".into()), + format!("tier edit failed. {}", err.to_string()), )); } } @@ -227,9 +224,7 @@ impl Operation for EditTier { } if let Err(e) = tier_config_mgr.save().await { warn!("tier_config_mgr save failed, e: {:?}", e); - return Err(S3Error::with_message( - S3ErrorCode::Custom("TierEditFailed".into()), "tier save failed" - )); + return Err(S3Error::with_message(S3ErrorCode::Custom("TierEditFailed".into()), "tier save failed")); } let mut header = HeaderMap::new(); @@ -312,13 +307,9 @@ impl Operation for RemoveTier { match tier_config_mgr.remove(&tier_name, force).await { Err(err) => { if err.code == ERR_TIER_NOT_FOUND.code { - return Err(S3Error::with_message( - S3ErrorCode::Custom("TierNotFound".into()), "tier not found." - )); + return Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found.")); } else if err.code == ERR_TIER_BACKEND_NOT_EMPTY.code { - return Err(S3Error::with_message( - S3ErrorCode::Custom("TierNameBackendInUse".into()), "tier is used." - )); + return Err(S3Error::with_message(S3ErrorCode::Custom("TierNameBackendInUse".into()), "tier is used.")); } else { warn!("tier_config_mgr remove failed, e: {:?}", err); return Err(S3Error::with_message( @@ -331,9 +322,7 @@ impl Operation for RemoveTier { } if let Err(e) = tier_config_mgr.save().await { warn!("tier_config_mgr save failed, e: {:?}", e); - return Err(S3Error::with_message( - S3ErrorCode::Custom("TierRemoveFailed".into()), "tier save failed" - )); + return Err(S3Error::with_message(S3ErrorCode::Custom("TierRemoveFailed".into()), "tier save failed")); } let mut header = HeaderMap::new(); @@ -453,16 +442,15 @@ impl Operation for ClearTier { Err(err) => { warn!("tier_config_mgr clear failed, e: {:?}", err); return Err(S3Error::with_message( - S3ErrorCode::Custom("TierClearFailed".into()), format!("tier clear failed. {}", err.to_string()) + S3ErrorCode::Custom("TierClearFailed".into()), + format!("tier clear failed. {}", err.to_string()), )); } Ok(_) => (), } if let Err(e) = tier_config_mgr.save().await { warn!("tier_config_mgr save failed, e: {:?}", e); - return Err(S3Error::with_message( - S3ErrorCode::Custom("TierEditFailed".into()), "tier save failed" - )); + return Err(S3Error::with_message(S3ErrorCode::Custom("TierEditFailed".into()), "tier save failed")); } let mut header = HeaderMap::new();