gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-07 22:03:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix clippy

Browse Source
This commit is contained in:
likewu
2025-06-23 12:03:46 +08:00
parent cc71f40a6d
commit 2c176dd864
70 changed files with 2779 additions and 1969 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ecstore/src/tier/mod.rs
+6 -6
View File
@@ -1,9 +1,9 @@
pub mod warm_backend_s3;
pub mod warm_backend_minio;
pub mod warm_backend_rustfs;
pub mod warm_backend;
pub mod tier;
pub mod tier_admin;
pub mod tier_config;
pub mod tier;
pub mod tier_gen;
pub mod tier_handlers;
pub mod tier_handlers;
pub mod warm_backend;
pub mod warm_backend_minio;
pub mod warm_backend_rustfs;
pub mod warm_backend_s3;
ecstore/src/tier/tier.rs
+82 -70
View File
@@ -1,46 +1,49 @@
use std::{
collections::{hash_map::Entry, HashMap}, io::Cursor, sync::Arc, time::{Duration,}
};
use bytes::Bytes;
use serde::{Serialize, Deserialize};
use time::OffsetDateTime;
use tokio::{select, sync::RwLock, time::interval};
use rand::Rng;
use tracing::{info, debug, warn, error};
use http::status::StatusCode;
use lazy_static::lazy_static;
use rand::Rng;
use serde::{Deserialize, Serialize};
use std::{
collections::{HashMap, hash_map::Entry},
io::Cursor,
sync::Arc,
time::Duration,
};
use time::OffsetDateTime;
use tokio::io::BufReader;
use tokio::{select, sync::RwLock, time::interval};
use tracing::{debug, error, info, warn};
use s3s::S3ErrorCode;
use crate::error::{Error, Result, StorageError};
use rustfs_utils::path::{path_join, SLASH_SEPARATOR};
use crate::{
config::com::{read_config, CONFIG_PREFIX},
disk::RUSTFS_META_BUCKET,
store::ECStore, store_api::{ObjectOptions, PutObjReader}, StorageAPI
};
use crate::client::admin_handler_utils::AdminError;
use crate::tier::{
warm_backend::{check_warm_backend, new_warm_backend},
tier_handlers::{
ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_ALREADY_EXISTS, ERR_TIER_NOT_FOUND,
},
tier_admin::TierCreds,
tier_config::{TierType, TierConfig,},
};
use crate::error::{Error, Result, StorageError};
use crate::new_object_layer_fn;
use crate::tier::{
tier_admin::TierCreds,
tier_config::{TierConfig, TierType},
tier_handlers::{ERR_TIER_ALREADY_EXISTS, ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_NOT_FOUND},
warm_backend::{check_warm_backend, new_warm_backend},
};
use crate::{
StorageAPI,
config::com::{CONFIG_PREFIX, read_config},
disk::RUSTFS_META_BUCKET,
store::ECStore,
store_api::{ObjectOptions, PutObjReader},
};
use rustfs_rio::HashReader;
use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
use s3s::S3ErrorCode;
use super::{
tier_handlers::{ERR_TIER_PERM_ERR, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_BUCKET_NOT_FOUND},
tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_PERM_ERR},
warm_backend::WarmBackendImpl,
};
const TIER_CFG_REFRESH: Duration = Duration::from_secs(15 * 60);
pub const TIER_CONFIG_FILE: &str = "tier-config.json";
pub const TIER_CONFIG_FORMAT: u16 = 1;
pub const TIER_CONFIG_V1: u16 = 1;
pub const TIER_CONFIG_FILE: &str = "tier-config.json";
pub const TIER_CONFIG_FORMAT: u16 = 1;
pub const TIER_CONFIG_V1: u16 = 1;
pub const TIER_CONFIG_VERSION: u16 = 1;
lazy_static! {
@@ -50,32 +53,32 @@ lazy_static! {
const TIER_CFG_REFRESH_AT_HDR: &str = "X-RustFS-TierCfg-RefreshedAt";
pub const ERR_TIER_MISSING_CREDENTIALS: AdminError = AdminError {
code: "XRustFSAdminTierMissingCredentials",
message: "Specified remote credentials are empty",
code: "XRustFSAdminTierMissingCredentials",
message: "Specified remote credentials are empty",
status_code: StatusCode::FORBIDDEN,
};
pub const ERR_TIER_BACKEND_IN_USE: AdminError = AdminError {
code: "XRustFSAdminTierBackendInUse",
message: "Specified remote tier is already in use",
code: "XRustFSAdminTierBackendInUse",
message: "Specified remote tier is already in use",
status_code: StatusCode::CONFLICT,
};
pub const ERR_TIER_TYPE_UNSUPPORTED: AdminError = AdminError {
code: "XRustFSAdminTierTypeUnsupported",
message: "Specified tier type is unsupported",
code: "XRustFSAdminTierTypeUnsupported",
message: "Specified tier type is unsupported",
status_code: StatusCode::BAD_REQUEST,
};
pub const ERR_TIER_BACKEND_NOT_EMPTY: AdminError = AdminError {
code: "XRustFSAdminTierBackendNotEmpty",
message: "Specified remote backend is not empty",
code: "XRustFSAdminTierBackendNotEmpty",
message: "Specified remote backend is not empty",
status_code: StatusCode::BAD_REQUEST,
};
pub const ERR_TIER_INVALID_CONFIG: AdminError = AdminError {
code: "XRustFSAdminTierInvalidConfig",
message: "Unable to setup remote tier, check tier configuration",
code: "XRustFSAdminTierInvalidConfig",
message: "Unable to setup remote tier, check tier configuration",
status_code: StatusCode::BAD_REQUEST,
};
@@ -147,22 +150,22 @@ impl TierConfigMgr {
if !force {
let in_use = d.in_use().await;
match in_use {
Ok(b) => {
if b {
return Err(ERR_TIER_BACKEND_IN_USE);
}
}
Err(err) => {
warn!("tier add failed, err: {:?}", err);
if err.to_string().contains("connect") {
return Err(ERR_TIER_CONNECT_ERR);
} else if err.to_string().contains("authorization") {
return Err(ERR_TIER_INVALID_CREDENTIALS);
} else if err.to_string().contains("bucket") {
return Err(ERR_TIER_BUCKET_NOT_FOUND);
}
return Err(ERR_TIER_PERM_ERR);
}
Ok(b) => {
if b {
return Err(ERR_TIER_BACKEND_IN_USE);
}
}
Err(err) => {
warn!("tier add failed, err: {:?}", err);
if err.to_string().contains("connect") {
return Err(ERR_TIER_CONNECT_ERR);
} else if err.to_string().contains("authorization") {
return Err(ERR_TIER_INVALID_CREDENTIALS);
} else if err.to_string().contains("bucket") {
return Err(ERR_TIER_BUCKET_NOT_FOUND);
}
return Err(ERR_TIER_PERM_ERR);
}
}
}
@@ -279,7 +282,7 @@ impl TierConfigMgr {
minio.access_key = creds.access_key;
minio.secret_key = creds.secret_key;
}
_ => ()
_ => (),
}
let d = new_warm_backend(&cfg, true).await?;
@@ -290,9 +293,7 @@ impl TierConfigMgr {
pub async fn get_driver<'a>(&'a mut self, tier_name: &str) -> std::result::Result<&'a WarmBackendImpl, AdminError> {
Ok(match self.driver_cache.entry(tier_name.to_string()) {
Entry::Occupied(e) => {
e.into_mut()
}
Entry::Occupied(e) => e.into_mut(),
Entry::Vacant(e) => {
let t = self.tiers.get(tier_name);
if t.is_none() {
@@ -326,7 +327,9 @@ impl TierConfigMgr {
#[tracing::instrument(level = "debug", name = "tier_save", skip(self))]
pub async fn save(&self) -> std::result::Result<(), std::io::Error> {
let Some(api) = new_object_layer_fn() else { return Err(std::io::Error::other("errServerNotInitialized")) };
let Some(api) = new_object_layer_fn() else {
return Err(std::io::Error::other("errServerNotInitialized"));
};
//let (pr, opts) = GLOBAL_TierConfigMgr.write().config_reader()?;
self.save_tiering_config(api).await
@@ -340,7 +343,12 @@ impl TierConfigMgr {
self.save_config(api, &config_file, data).await
}
pub async fn save_config<S: StorageAPI>(&self, api: Arc<S>, file: &str, data: Bytes) -> std::result::Result<(), std::io::Error> {
pub async fn save_config<S: StorageAPI>(
&self,
api: Arc<S>,
file: &str,
data: Bytes,
) -> std::result::Result<(), std::io::Error> {
self.save_config_with_opts(
api,
file,
@@ -353,7 +361,13 @@ impl TierConfigMgr {
.await
}
pub async fn save_config_with_opts<S: StorageAPI>(&self, api: Arc<S>, file: &str, data: Bytes, opts: &ObjectOptions) -> std::result::Result<(), std::io::Error> {
pub async fn save_config_with_opts<S: StorageAPI>(
&self,
api: Arc<S>,
file: &str,
data: Bytes,
opts: &ObjectOptions,
) -> std::result::Result<(), std::io::Error> {
debug!("save tier config:{}", file);
let _ = api
.put_object(RUSTFS_META_BUCKET, file, &mut PutObjReader::from_vec(data.to_vec()), opts)
@@ -365,9 +379,7 @@ impl TierConfigMgr {
//let r = rand.New(rand.NewSource(time.Now().UnixNano()));
let mut rng = rand::rng();
let r = rng.random_range(0.0..1.0);
let rand_interval = || {
Duration::from_secs((r * 60_f64).round() as u64)
};
let rand_interval = || Duration::from_secs((r * 60_f64).round() as u64);
let mut t = interval(TIER_CFG_REFRESH + rand_interval());
loop {
@@ -394,10 +406,10 @@ impl TierConfigMgr {
async fn new_and_save_tiering_config<S: StorageAPI>(api: Arc<S>) -> Result<TierConfigMgr> {
let mut cfg = TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
};
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
};
//lookup_configs(&mut cfg, api.clone()).await;
cfg.save_tiering_config(api).await?;
@@ -420,7 +432,7 @@ async fn load_tier_config(api: Arc<ECStore>) -> std::result::Result<TierConfigMg
}
let cfg;
let version = 1;//LittleEndian::read_u16(&data[2..4]);
let version = 1; //LittleEndian::read_u16(&data[2..4]);
match version {
TIER_CONFIG_V1/* | TIER_CONFIG_VERSION */ => {
cfg = match TierConfigMgr::unmarshal(&data.unwrap()) {
@@ -440,4 +452,4 @@ async fn load_tier_config(api: Arc<ECStore>) -> std::result::Result<TierConfigMg
pub fn is_err_config_not_found(err: &StorageError) -> bool {
matches!(err, StorageError::ObjectNotFound(_, _))
}
}
ecstore/src/tier/tier_admin.rs
+6 -9
View File
@@ -1,13 +1,10 @@
use std::{
time::{Duration, SystemTime, UNIX_EPOCH},
};
use rand::Rng;
use tracing::warn;
use http::status::StatusCode;
use serde::{Serialize, Deserialize};
use rand::Rng;
use serde::{Deserialize, Serialize};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use tracing::warn;
#[derive(Serialize, Deserialize)]
#[derive(Default, Debug, Clone)]
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[serde(default)]
pub struct TierCreds {
#[serde(rename = "accessKey")]
@@ -26,4 +23,4 @@ pub struct TierCreds {
//#[serde(rename = "credsJson")]
pub creds_json: Vec<u8>,
}
}
ecstore/src/tier/tier_config.rs
+59 -96
View File
@@ -1,5 +1,5 @@
use serde::{Deserialize, Serialize};
use std::fmt::Display;
use serde::{Serialize, Deserialize};
use tracing::info;
const C_TierConfigVer: &str = "v1";
@@ -9,8 +9,7 @@ const ERR_TIER_INVALID_CONFIG: &str = "invalid tier config";
const ERR_TIER_INVALID_CONFIG_VERSION: &str = "invalid tier config version";
const ERR_TIER_TYPE_UNSUPPORTED: &str = "unsupported tier type";
#[derive(Serialize, Deserialize)]
#[derive(Default, Debug, Clone)]
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
pub enum TierType {
#[default]
Unsupported,
@@ -48,35 +47,19 @@ impl Display for TierType {
impl TierType {
pub fn new(sc_type: &str) -> Self {
match sc_type {
"S3" => {
TierType::S3
}
"RustFS" => {
TierType::RustFS
}
"MinIO" => {
TierType::MinIO
}
_ => {
TierType::Unsupported
}
"S3" => TierType::S3,
"RustFS" => TierType::RustFS,
"MinIO" => TierType::MinIO,
_ => TierType::Unsupported,
}
}
pub fn to_string(&self) -> String {
match self {
TierType::S3 => {
"s3".to_string()
}
TierType::RustFS => {
"rustfs".to_string()
}
TierType::MinIO => {
"minio".to_string()
}
_ => {
"unsupported".to_string()
}
TierType::S3 => "s3".to_string(),
TierType::RustFS => "rustfs".to_string(),
TierType::MinIO => "minio".to_string(),
_ => "unsupported".to_string(),
}
}
}
@@ -123,17 +106,17 @@ impl Clone for TierConfig {
m_.secret_key = "REDACTED".to_string();
m = Some(m_);
}
_ => ()
_ => (),
}
TierConfig {
version: self.version.clone(),
tier_type: self.tier_type.clone(),
name: self.name.clone(),
name: self.name.clone(),
s3: s3,
//azure: az,
//gcs: gcs,
rustfs: r,
minio: m,
rustfs: r,
minio: m,
}
}
}
@@ -154,15 +137,9 @@ impl TierConfig {
fn endpoint(&self) -> String {
match self.tier_type {
TierType::S3 => {
self.s3.as_ref().expect("err").endpoint.clone()
}
TierType::RustFS => {
self.rustfs.as_ref().expect("err").endpoint.clone()
}
TierType::MinIO => {
self.minio.as_ref().expect("err").endpoint.clone()
}
TierType::S3 => self.s3.as_ref().expect("err").endpoint.clone(),
TierType::RustFS => self.rustfs.as_ref().expect("err").endpoint.clone(),
TierType::MinIO => self.minio.as_ref().expect("err").endpoint.clone(),
_ => {
info!("unexpected tier type {}", self.tier_type);
"".to_string()
@@ -172,15 +149,9 @@ impl TierConfig {
fn bucket(&self) -> String {
match self.tier_type {
TierType::S3 => {
self.s3.as_ref().expect("err").bucket.clone()
}
TierType::RustFS => {
self.rustfs.as_ref().expect("err").bucket.clone()
}
TierType::MinIO => {
self.minio.as_ref().expect("err").bucket.clone()
}
TierType::S3 => self.s3.as_ref().expect("err").bucket.clone(),
TierType::RustFS => self.rustfs.as_ref().expect("err").bucket.clone(),
TierType::MinIO => self.minio.as_ref().expect("err").bucket.clone(),
_ => {
info!("unexpected tier type {}", self.tier_type);
"".to_string()
@@ -190,15 +161,9 @@ impl TierConfig {
fn prefix(&self) -> String {
match self.tier_type {
TierType::S3 => {
self.s3.as_ref().expect("err").prefix.clone()
}
TierType::RustFS => {
self.rustfs.as_ref().expect("err").prefix.clone()
}
TierType::MinIO => {
self.minio.as_ref().expect("err").prefix.clone()
}
TierType::S3 => self.s3.as_ref().expect("err").prefix.clone(),
TierType::RustFS => self.rustfs.as_ref().expect("err").prefix.clone(),
TierType::MinIO => self.minio.as_ref().expect("err").prefix.clone(),
_ => {
info!("unexpected tier type {}", self.tier_type);
"".to_string()
@@ -208,18 +173,12 @@ impl TierConfig {
fn region(&self) -> String {
match self.tier_type {
TierType::S3 => {
self.s3.as_ref().expect("err").region.clone()
}
TierType::RustFS => {
self.rustfs.as_ref().expect("err").region.clone()
}
TierType::MinIO => {
self.minio.as_ref().expect("err").region.clone()
}
TierType::S3 => self.s3.as_ref().expect("err").region.clone(),
TierType::RustFS => self.rustfs.as_ref().expect("err").region.clone(),
TierType::MinIO => self.minio.as_ref().expect("err").region.clone(),
_ => {
info!("unexpected tier type {}", self.tier_type);
"".to_string()
"".to_string()
}
}
}
@@ -227,8 +186,7 @@ impl TierConfig {
//type S3Options = impl Fn(TierS3) -> Pin<Box<Result<()>>> + Send + Sync + 'static;
#[derive(Serialize, Deserialize)]
#[derive(Default, Debug, Clone)]
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[serde(default)]
pub struct TierS3 {
pub name: String,
@@ -257,17 +215,17 @@ pub struct TierS3 {
impl TierS3 {
fn new<F>(name: &str, access_key: &str, secret_key: &str, bucket: &str, options: Vec<F>) -> Result<TierConfig, std::io::Error>
where
F: Fn(TierS3) -> Box<Result<(), std::io::Error>> + Send + Sync + 'static
F: Fn(TierS3) -> Box<Result<(), std::io::Error>> + Send + Sync + 'static,
{
if name == "" {
return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY));
}
let sc = TierS3 {
access_key: access_key.to_string(),
secret_key: secret_key.to_string(),
bucket: bucket.to_string(),
endpoint: "https://s3.amazonaws.com".to_string(),
region: "".to_string(),
access_key: access_key.to_string(),
secret_key: secret_key.to_string(),
bucket: bucket.to_string(),
endpoint: "https://s3.amazonaws.com".to_string(),
region: "".to_string(),
storage_class: "".to_string(),
..Default::default()
};
@@ -281,49 +239,54 @@ impl TierS3 {
Ok(TierConfig {
version: C_TierConfigVer.to_string(),
tier_type: TierType::S3,
name: name.to_string(),
s3: Some(sc),
name: name.to_string(),
s3: Some(sc),
..Default::default()
})
}
}
#[derive(Serialize, Deserialize)]
#[derive(Default, Debug, Clone)]
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[serde(default)]
pub struct TierRustFS {
pub name: String,
pub endpoint: String,
pub endpoint: String,
#[serde(rename = "accesskey")]
pub access_key: String,
#[serde(rename = "secretkey")]
pub secret_key: String,
pub bucket: String,
pub prefix: String,
pub region: String,
pub bucket: String,
pub prefix: String,
pub region: String,
#[serde(rename = "storageclass")]
pub storage_class: String,
}
#[derive(Serialize, Deserialize)]
#[derive(Default, Debug, Clone)]
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[serde(default)]
pub struct TierMinIO {
pub name: String,
pub endpoint: String,
pub endpoint: String,
#[serde(rename = "accesskey")]
pub access_key: String,
#[serde(rename = "secretkey")]
pub secret_key: String,
pub bucket: String,
pub prefix: String,
pub region: String,
pub bucket: String,
pub prefix: String,
pub region: String,
}
impl TierMinIO {
fn new<F>(name: &str, endpoint: &str, access_key: &str, secret_key: &str, bucket: &str, options: Vec<F>) -> Result<TierConfig, std::io::Error>
fn new<F>(
name: &str,
endpoint: &str,
access_key: &str,
secret_key: &str,
bucket: &str,
options: Vec<F>,
) -> Result<TierConfig, std::io::Error>
where
F: Fn(TierMinIO) -> Box<Result<(), std::io::Error>> + Send + Sync + 'static
F: Fn(TierMinIO) -> Box<Result<(), std::io::Error>> + Send + Sync + 'static,
{
if name == "" {
return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY));
@@ -331,8 +294,8 @@ impl TierMinIO {
let m = TierMinIO {
access_key: access_key.to_string(),
secret_key: secret_key.to_string(),
bucket: bucket.to_string(),
endpoint: endpoint.to_string(),
bucket: bucket.to_string(),
endpoint: endpoint.to_string(),
..Default::default()
};
@@ -345,8 +308,8 @@ impl TierMinIO {
Ok(TierConfig {
version: C_TierConfigVer.to_string(),
tier_type: TierType::MinIO,
name: name.to_string(),
minio: Some(m),
name: name.to_string(),
minio: Some(m),
..Default::default()
})
}
ecstore/src/tier/tier_handlers.rs
+18 -18
View File
@@ -1,51 +1,51 @@
use crate::client::admin_handler_utils::AdminError;
use tracing::warn;
use http::status::StatusCode;
use tracing::warn;
pub const ERR_TIER_ALREADY_EXISTS: AdminError = AdminError {
code: "XRustFSAdminTierAlreadyExists",
message: "Specified remote tier already exists",
code: "XRustFSAdminTierAlreadyExists",
message: "Specified remote tier already exists",
status_code: StatusCode::CONFLICT,
};
pub const ERR_TIER_NOT_FOUND: AdminError = AdminError {
code: "XRustFSAdminTierNotFound",
message: "Specified remote tier was not found",
code: "XRustFSAdminTierNotFound",
message: "Specified remote tier was not found",
status_code: StatusCode::NOT_FOUND,
};
pub const ERR_TIER_NAME_NOT_UPPERCASE: AdminError = AdminError {
code: "XRustFSAdminTierNameNotUpperCase",
message: "Tier name must be in uppercase",
code: "XRustFSAdminTierNameNotUpperCase",
message: "Tier name must be in uppercase",
status_code: StatusCode::BAD_REQUEST,
};
pub const ERR_TIER_BUCKET_NOT_FOUND: AdminError = AdminError {
code: "XRustFSAdminTierBucketNotFound",
message: "Remote tier bucket not found",
code: "XRustFSAdminTierBucketNotFound",
message: "Remote tier bucket not found",
status_code: StatusCode::BAD_REQUEST,
};
pub const ERR_TIER_INVALID_CREDENTIALS: AdminError = AdminError {
code: "XRustFSAdminTierInvalidCredentials",
message: "Invalid remote tier credentials",
code: "XRustFSAdminTierInvalidCredentials",
message: "Invalid remote tier credentials",
status_code: StatusCode::BAD_REQUEST,
};
pub const ERR_TIER_RESERVED_NAME: AdminError = AdminError {
code: "XRustFSAdminTierReserved",
message: "Cannot use reserved tier name",
code: "XRustFSAdminTierReserved",
message: "Cannot use reserved tier name",
status_code: StatusCode::BAD_REQUEST,
};
pub const ERR_TIER_PERM_ERR: AdminError = AdminError {
code: "TierPermErr",
message: "Tier Perm Err",
code: "TierPermErr",
message: "Tier Perm Err",
status_code: StatusCode::OK,
};
pub const ERR_TIER_CONNECT_ERR: AdminError = AdminError {
code: "TierConnectErr",
message: "Tier Connect Err",
code: "TierConnectErr",
message: "Tier Connect Err",
status_code: StatusCode::OK,
};
};
ecstore/src/tier/warm_backend.rs
+19 -11
View File
@@ -1,20 +1,20 @@
use std::collections::HashMap;
use bytes::Bytes;
use std::collections::HashMap;
use crate::client::{
admin_handler_utils::AdminError,
transition_api::{ReadCloser, ReaderImpl,},
transition_api::{ReadCloser, ReaderImpl},
};
use crate::error::is_err_bucket_not_found;
use tracing::{info, warn};
use crate::tier::{
tier_config::{TierType, TierConfig},
tier::{ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED},
tier_config::{TierConfig, TierType},
tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_PERM_ERR},
tier::{ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED,},
warm_backend_s3::WarmBackendS3,
warm_backend_rustfs::WarmBackendRustFS,
warm_backend_minio::WarmBackendMinIO,
warm_backend_rustfs::WarmBackendRustFS,
warm_backend_s3::WarmBackendS3,
};
use tracing::{info, warn};
pub type WarmBackendImpl = Box<dyn WarmBackend + Send + Sync + 'static>;
@@ -29,7 +29,13 @@ pub struct WarmBackendGetOpts {
#[async_trait::async_trait]
pub trait WarmBackend {
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error>;
async fn put_with_meta(&self, object: &str, r: ReaderImpl, length: i64, meta: HashMap<String, String>) -> Result<String, std::io::Error>;
async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
meta: HashMap<String, String>,
) -> Result<String, std::io::Error>;
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error>;
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error>;
async fn in_use(&self) -> Result<bool, std::io::Error>;
@@ -37,7 +43,9 @@ pub trait WarmBackend {
pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> {
let w = w.expect("err");
let remote_version_id = w.put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5).await;
let remote_version_id = w
.put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5)
.await;
if let Err(err) = remote_version_id {
return Err(ERR_TIER_PERM_ERR);
}
@@ -52,7 +60,7 @@ pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), Admin
return Err(ERR_TIER_MISSING_CREDENTIALS);
}*/
//else {
return Err(ERR_TIER_PERM_ERR);
return Err(ERR_TIER_PERM_ERR);
//}
}
if let Err(err) = w.remove(PROBE_OBJECT, &remote_version_id.expect("err")).await {
@@ -94,4 +102,4 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
}
Ok(d.expect("err"))
}
}
ecstore/src/tier/warm_backend_minio.rs
+37 -27
View File
@@ -1,24 +1,23 @@
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use tracing::warn;
use crate::client::{
admin_handler_utils::AdminError,
transition_api::{Options, ReaderImpl, ReadCloser, TransitionClient, TransitionCore},
credentials::{Credentials, SignatureType, Static, Value},
api_put_object::PutObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
};
use crate::tier::{
tier_config::TierMinIO,
warm_backend::{WarmBackend, WarmBackendGetOpts},
warm_backend_s3::WarmBackendS3,
};
use tracing::warn;
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
const MAX_PARTS_COUNT: i64 = 10000;
const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
const MAX_PARTS_COUNT: i64 = 10000;
const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
pub struct WarmBackendMinIO(WarmBackendS3);
@@ -40,26 +39,23 @@ impl WarmBackendMinIO {
};
let creds = Credentials::new(Static(Value {
access_key_id: conf.access_key.clone(),
access_key_id: conf.access_key.clone(),
secret_access_key: conf.secret_key.clone(),
session_token: "".to_string(),
session_token: "".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
}));
let opts = Options {
creds: creds,
secure: u.scheme() == "https",
creds: creds,
secure: u.scheme() == "https",
//transport: GLOBAL_RemoteTargetTransport,
trailing_headers: true,
..Default::default()
};
let scheme = u.scheme();
let default_port = if scheme == "https" {
443
} else {
80
};
let client = TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?;
let default_port = if scheme == "https" { 443 } else { 80 };
let client =
TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?;
//client.set_appinfo(format!("minio-tier-{}", tier), ReleaseTag);
let client = Arc::new(client);
@@ -67,8 +63,8 @@ impl WarmBackendMinIO {
Ok(Self(WarmBackendS3 {
client,
core,
bucket: conf.bucket.clone(),
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
bucket: conf.bucket.clone(),
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
storage_class: "".to_string(),
}))
}
@@ -76,16 +72,30 @@ impl WarmBackendMinIO {
#[async_trait::async_trait]
impl WarmBackend for WarmBackendMinIO {
async fn put_with_meta(&self, object: &str, r: ReaderImpl, length: i64, meta: HashMap<String, String>) -> Result<String, std::io::Error> {
async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
meta: HashMap<String, String>,
) -> Result<String, std::io::Error> {
let part_size = optimal_part_size(length)?;
let client = self.0.client.clone();
let res = client.put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &PutObjectOptions {
storage_class: self.0.storage_class.clone(),
part_size: part_size as u64,
disable_content_sha256: true,
user_metadata: meta,
..Default::default()
}).await?;
let res = client
.put_object(
&self.0.bucket,
&self.0.get_dest(object),
r,
length,
&PutObjectOptions {
storage_class: self.0.storage_class.clone(),
part_size: part_size as u64,
disable_content_sha256: true,
user_metadata: meta,
..Default::default()
},
)
.await?;
//self.ToObjectError(err, object)
Ok(res.version_id)
}
ecstore/src/tier/warm_backend_rustfs.rs
+36 -25
View File
@@ -4,9 +4,9 @@ use tracing::warn;
use crate::client::{
admin_handler_utils::AdminError,
transition_api::{Options, ReaderImpl, ReadCloser, TransitionClient, TransitionCore},
credentials::{Credentials, SignatureType, Static, Value},
api_put_object::PutObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
};
use crate::tier::{
tier_config::TierRustFS,
@@ -15,9 +15,9 @@ use crate::tier::{
};
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
const MAX_PARTS_COUNT: i64 = 10000;
const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
const MAX_PARTS_COUNT: i64 = 10000;
const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
pub struct WarmBackendRustFS(WarmBackendS3);
@@ -37,26 +37,23 @@ impl WarmBackendRustFS {
};
let creds = Credentials::new(Static(Value {
access_key_id: conf.access_key.clone(),
access_key_id: conf.access_key.clone(),
secret_access_key: conf.secret_key.clone(),
session_token: "".to_string(),
session_token: "".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
}));
let opts = Options {
creds: creds,
secure: u.scheme() == "https",
creds: creds,
secure: u.scheme() == "https",
//transport: GLOBAL_RemoteTargetTransport,
trailing_headers: true,
..Default::default()
};
let scheme = u.scheme();
let default_port = if scheme == "https" {
443
} else {
80
};
let client = TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?;
let default_port = if scheme == "https" { 443 } else { 80 };
let client =
TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?;
//client.set_appinfo(format!("rustfs-tier-{}", tier), ReleaseTag);
let client = Arc::new(client);
@@ -64,8 +61,8 @@ impl WarmBackendRustFS {
Ok(Self(WarmBackendS3 {
client,
core,
bucket: conf.bucket.clone(),
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
bucket: conf.bucket.clone(),
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
storage_class: "".to_string(),
}))
}
@@ -73,16 +70,30 @@ impl WarmBackendRustFS {
#[async_trait::async_trait]
impl WarmBackend for WarmBackendRustFS {
async fn put_with_meta(&self, object: &str, r: ReaderImpl, length: i64, meta: HashMap<String, String>) -> Result<String, std::io::Error> {
async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
meta: HashMap<String, String>,
) -> Result<String, std::io::Error> {
let part_size = optimal_part_size(length)?;
let client = self.0.client.clone();
let res = client.put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &PutObjectOptions {
storage_class: self.0.storage_class.clone(),
part_size: part_size as u64,
disable_content_sha256: true,
user_metadata: meta,
..Default::default()
}).await?;
let res = client
.put_object(
&self.0.bucket,
&self.0.get_dest(object),
r,
length,
&PutObjectOptions {
storage_class: self.0.storage_class.clone(),
part_size: part_size as u64,
disable_content_sha256: true,
user_metadata: meta,
..Default::default()
},
)
.await?;
//self.ToObjectError(err, object)
Ok(res.version_id)
}
ecstore/src/tier/warm_backend_s3.rs
+55 -33
View File
@@ -2,21 +2,21 @@ use std::collections::HashMap;
use std::sync::Arc;
use url::Url;
use crate::error::ErrorResponse;
use crate::error::error_resp_to_object_err;
use crate::client::{
api_get_options::GetObjectOptions,
credentials::{Credentials, Static, Value, SignatureType},
transition_api::{ReaderImpl, ReadCloser},
api_put_object::PutObjectOptions,
api_remove::RemoveObjectOptions,
transition_api::{Options, TransitionClient, TransitionCore,},
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{Options, TransitionClient, TransitionCore},
transition_api::{ReadCloser, ReaderImpl},
};
use rustfs_utils::path::SLASH_SEPARATOR;
use crate::error::ErrorResponse;
use crate::error::error_resp_to_object_err;
use crate::tier::{
tier_config::TierS3,
warm_backend::{WarmBackend, WarmBackendGetOpts,}
warm_backend::{WarmBackend, WarmBackendGetOpts},
};
use rustfs_utils::path::SLASH_SEPARATOR;
pub struct WarmBackendS3 {
pub client: Arc<TransitionClient>,
@@ -35,16 +35,22 @@ impl WarmBackendS3 {
}
};
if conf.aws_role_web_identity_token_file == "" && conf.aws_role_arn != "" || conf.aws_role_web_identity_token_file != "" && conf.aws_role_arn == "" {
if conf.aws_role_web_identity_token_file == "" && conf.aws_role_arn != ""
|| conf.aws_role_web_identity_token_file != "" && conf.aws_role_arn == ""
{
return Err(std::io::Error::other("both the token file and the role ARN are required"));
}
else if conf.access_key == "" && conf.secret_key != "" || conf.access_key != "" && conf.secret_key == "" {
} else if conf.access_key == "" && conf.secret_key != "" || conf.access_key != "" && conf.secret_key == "" {
return Err(std::io::Error::other("both the access and secret keys are required"));
}
else if conf.aws_role && (conf.aws_role_web_identity_token_file != "" || conf.aws_role_arn != "" || conf.access_key != "" || conf.secret_key != "") {
return Err(std::io::Error::other("AWS Role cannot be activated with static credentials or the web identity token file"));
}
else if conf.bucket == "" {
} else if conf.aws_role
&& (conf.aws_role_web_identity_token_file != ""
|| conf.aws_role_arn != ""
|| conf.access_key != ""
|| conf.secret_key != "")
{
return Err(std::io::Error::other(
"AWS Role cannot be activated with static credentials or the web identity token file",
));
} else if conf.bucket == "" {
return Err(std::io::Error::other("no bucket name was provided"));
}
@@ -53,19 +59,18 @@ impl WarmBackendS3 {
if conf.access_key != "" && conf.secret_key != "" {
//creds = Credentials::new_static_v4(conf.access_key, conf.secret_key, "");
creds = Credentials::new(Static(Value {
access_key_id: conf.access_key.clone(),
access_key_id: conf.access_key.clone(),
secret_access_key: conf.secret_key.clone(),
session_token: "".to_string(),
session_token: "".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
}));
}
else {
} else {
return Err(std::io::Error::other("insufficient parameters for S3 backend authentication"));
}
let opts = Options {
creds: creds,
secure: u.scheme() == "https",
creds: creds,
secure: u.scheme() == "https",
//transport: GLOBAL_RemoteTargetTransport,
..Default::default()
};
@@ -77,8 +82,8 @@ impl WarmBackendS3 {
Ok(Self {
client,
core,
bucket: conf.bucket.clone(),
prefix: conf.prefix.clone().trim_matches('/').to_string(),
bucket: conf.bucket.clone(),
prefix: conf.prefix.clone().trim_matches('/').to_string(),
storage_class: conf.storage_class.clone(),
})
}
@@ -103,14 +108,28 @@ impl WarmBackendS3 {
#[async_trait::async_trait]
impl WarmBackend for WarmBackendS3 {
async fn put_with_meta(&self, object: &str, r: ReaderImpl, length: i64, meta: HashMap<String, String>) -> Result<String, std::io::Error> {
async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
meta: HashMap<String, String>,
) -> Result<String, std::io::Error> {
let client = self.client.clone();
let res = client.put_object(&self.bucket, &self.get_dest(object), r, length, &PutObjectOptions {
send_content_md5: true,
storage_class: self.storage_class.clone(),
user_metadata: meta,
..Default::default()
}).await?;
let res = client
.put_object(
&self.bucket,
&self.get_dest(object),
r,
length,
&PutObjectOptions {
send_content_md5: true,
storage_class: self.storage_class.clone(),
user_metadata: meta,
..Default::default()
},
)
.await?;
Ok(res.version_id)
}
@@ -125,7 +144,7 @@ impl WarmBackend for WarmBackendS3 {
gopts.version_id = rv.to_string();
}
if opts.start_offset >= 0 && opts.length > 0 {
if let Err(err) = gopts.set_range(opts.start_offset, opts.start_offset+opts.length-1) {
if let Err(err) = gopts.set_range(opts.start_offset, opts.start_offset + opts.length - 1) {
return Err(std::io::Error::other(err));
}
}
@@ -146,8 +165,11 @@ impl WarmBackend for WarmBackendS3 {
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
let result = self.core.list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1).await?;
let result = self
.core
.list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1)
.await?;
Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)
}
}
}