mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-22 12:26:37 +00:00
refactor: Restructure project layout and clean up dependencies (#30)
This commit introduces a significant reorganization of the project structure to improve maintainability and clarity. Key changes include: - Adjusted the directory layout for a more logical module organization. - Removed unused crate dependencies, reducing the overall project size and potentially speeding up build times. - Updated import paths and configuration files to reflect the structural changes.
This commit is contained in:
houseme
@@ -0,0 +1,23 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub mod tier;
|
||||
pub mod tier_admin;
|
||||
pub mod tier_config;
|
||||
pub mod tier_gen;
|
||||
pub mod tier_handlers;
|
||||
pub mod warm_backend;
|
||||
pub mod warm_backend_minio;
|
||||
pub mod warm_backend_rustfs;
|
||||
pub mod warm_backend_s3;
|
||||
@@ -0,0 +1,471 @@
|
||||
#![allow(unused_imports)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use bytes::Bytes;
|
||||
use http::status::StatusCode;
|
||||
use lazy_static::lazy_static;
|
||||
use rand::Rng;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
collections::{HashMap, hash_map::Entry},
|
||||
io::Cursor,
|
||||
sync::Arc,
|
||||
time::Duration,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::io::BufReader;
|
||||
use tokio::{select, sync::RwLock, time::interval};
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
use crate::client::admin_handler_utils::AdminError;
|
||||
use crate::error::{Error, Result, StorageError};
|
||||
use crate::new_object_layer_fn;
|
||||
use crate::tier::{
|
||||
tier_admin::TierCreds,
|
||||
tier_config::{TierConfig, TierType},
|
||||
tier_handlers::{ERR_TIER_ALREADY_EXISTS, ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_NOT_FOUND},
|
||||
warm_backend::{check_warm_backend, new_warm_backend},
|
||||
};
|
||||
use crate::{
|
||||
StorageAPI,
|
||||
config::com::{CONFIG_PREFIX, read_config},
|
||||
disk::RUSTFS_META_BUCKET,
|
||||
store::ECStore,
|
||||
store_api::{ObjectOptions, PutObjReader},
|
||||
};
|
||||
use rustfs_rio::HashReader;
|
||||
use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
|
||||
use s3s::S3ErrorCode;
|
||||
|
||||
use super::{
|
||||
tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_PERM_ERR},
|
||||
warm_backend::WarmBackendImpl,
|
||||
};
|
||||
|
||||
const TIER_CFG_REFRESH: Duration = Duration::from_secs(15 * 60);
|
||||
|
||||
pub const TIER_CONFIG_FILE: &str = "tier-config.json";
|
||||
pub const TIER_CONFIG_FORMAT: u16 = 1;
|
||||
pub const TIER_CONFIG_V1: u16 = 1;
|
||||
pub const TIER_CONFIG_VERSION: u16 = 1;
|
||||
|
||||
const _TIER_CFG_REFRESH_AT_HDR: &str = "X-RustFS-TierCfg-RefreshedAt";
|
||||
|
||||
lazy_static! {
|
||||
pub static ref ERR_TIER_MISSING_CREDENTIALS: AdminError = AdminError {
|
||||
code: "XRustFSAdminTierMissingCredentials".to_string(),
|
||||
message: "Specified remote credentials are empty".to_string(),
|
||||
status_code: StatusCode::FORBIDDEN,
|
||||
};
|
||||
pub static ref ERR_TIER_BACKEND_IN_USE: AdminError = AdminError {
|
||||
code: "XRustFSAdminTierBackendInUse".to_string(),
|
||||
message: "Specified remote tier is already in use".to_string(),
|
||||
status_code: StatusCode::CONFLICT,
|
||||
};
|
||||
pub static ref ERR_TIER_TYPE_UNSUPPORTED: AdminError = AdminError {
|
||||
code: "XRustFSAdminTierTypeUnsupported".to_string(),
|
||||
message: "Specified tier type is unsupported".to_string(),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
};
|
||||
pub static ref ERR_TIER_BACKEND_NOT_EMPTY: AdminError = AdminError {
|
||||
code: "XRustFSAdminTierBackendNotEmpty".to_string(),
|
||||
message: "Specified remote backend is not empty".to_string(),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
};
|
||||
pub static ref ERR_TIER_INVALID_CONFIG: AdminError = AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: "Unable to setup remote tier, check tier configuration".to_string(),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
};
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct TierConfigMgr {
|
||||
#[serde(skip)]
|
||||
pub driver_cache: HashMap<String, WarmBackendImpl>,
|
||||
pub tiers: HashMap<String, TierConfig>,
|
||||
pub last_refreshed_at: OffsetDateTime,
|
||||
}
|
||||
|
||||
impl TierConfigMgr {
|
||||
pub fn new() -> Arc<RwLock<Self>> {
|
||||
Arc::new(RwLock::new(Self {
|
||||
driver_cache: HashMap::new(),
|
||||
tiers: HashMap::new(),
|
||||
last_refreshed_at: OffsetDateTime::now_utc(),
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn unmarshal(data: &[u8]) -> std::result::Result<TierConfigMgr, std::io::Error> {
|
||||
let cfg: TierConfigMgr = serde_json::from_slice(data)?;
|
||||
Ok(cfg)
|
||||
}
|
||||
|
||||
pub fn marshal(&self) -> std::result::Result<Bytes, std::io::Error> {
|
||||
let data = serde_json::to_vec(&self)?;
|
||||
let mut data = Bytes::from(data);
|
||||
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub fn refreshed_at(&self) -> OffsetDateTime {
|
||||
self.last_refreshed_at
|
||||
}
|
||||
|
||||
pub fn is_tier_valid(&self, tier_name: &str) -> bool {
|
||||
let (_, valid) = self.is_tier_name_in_use(tier_name);
|
||||
valid
|
||||
}
|
||||
|
||||
pub fn is_tier_name_in_use(&self, tier_name: &str) -> (TierType, bool) {
|
||||
if let Some(t) = self.tiers.get(tier_name) {
|
||||
return (t.tier_type.clone(), true);
|
||||
}
|
||||
(TierType::Unsupported, false)
|
||||
}
|
||||
|
||||
pub async fn add(&mut self, tier: TierConfig, force: bool) -> std::result::Result<(), AdminError> {
|
||||
let tier_name = &tier.name;
|
||||
if tier_name != tier_name.to_uppercase().as_str() {
|
||||
return Err(ERR_TIER_NAME_NOT_UPPERCASE.clone());
|
||||
}
|
||||
|
||||
let (_, b) = self.is_tier_name_in_use(tier_name);
|
||||
if b {
|
||||
return Err(ERR_TIER_ALREADY_EXISTS.clone());
|
||||
}
|
||||
|
||||
let d = new_warm_backend(&tier, true).await?;
|
||||
|
||||
if !force {
|
||||
let in_use = d.in_use().await;
|
||||
match in_use {
|
||||
Ok(b) => {
|
||||
if b {
|
||||
return Err(ERR_TIER_BACKEND_IN_USE.clone());
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
warn!("tier add failed, err: {:?}", err);
|
||||
if err.to_string().contains("connect") {
|
||||
return Err(ERR_TIER_CONNECT_ERR.clone());
|
||||
} else if err.to_string().contains("authorization") {
|
||||
return Err(ERR_TIER_INVALID_CREDENTIALS.clone());
|
||||
} else if err.to_string().contains("bucket") {
|
||||
return Err(ERR_TIER_BUCKET_NOT_FOUND.clone());
|
||||
}
|
||||
let mut e = ERR_TIER_PERM_ERR.clone();
|
||||
e.message.push('.');
|
||||
e.message.push_str(&err.to_string());
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
self.driver_cache.insert(tier_name.to_string(), d);
|
||||
self.tiers.insert(tier_name.to_string(), tier);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn remove(&mut self, tier_name: &str, force: bool) -> std::result::Result<(), AdminError> {
|
||||
let d = self.get_driver(tier_name).await;
|
||||
if let Err(err) = d {
|
||||
if err.code == ERR_TIER_NOT_FOUND.code {
|
||||
return Ok(());
|
||||
} else {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
if !force {
|
||||
let inuse = d.expect("err").in_use().await;
|
||||
if let Err(err) = inuse {
|
||||
let mut e = ERR_TIER_PERM_ERR.clone();
|
||||
e.message.push('.');
|
||||
e.message.push_str(&err.to_string());
|
||||
return Err(e);
|
||||
} else if inuse.expect("err") {
|
||||
return Err(ERR_TIER_BACKEND_NOT_EMPTY.clone());
|
||||
}
|
||||
}
|
||||
self.tiers.remove(tier_name);
|
||||
self.driver_cache.remove(tier_name);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn verify(&mut self, tier_name: &str) -> std::result::Result<(), std::io::Error> {
|
||||
let d = match self.get_driver(tier_name).await {
|
||||
Ok(d) => d,
|
||||
Err(err) => {
|
||||
return Err(std::io::Error::other(err));
|
||||
}
|
||||
};
|
||||
if let Err(err) = check_warm_backend(Some(d)).await {
|
||||
return Err(std::io::Error::other(err));
|
||||
} else {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
pub fn empty(&self) -> bool {
|
||||
self.list_tiers().len() == 0
|
||||
}
|
||||
|
||||
pub fn tier_type(&self, tier_name: &str) -> String {
|
||||
let cfg = self.tiers.get(tier_name);
|
||||
if cfg.is_none() {
|
||||
return "internal".to_string();
|
||||
}
|
||||
cfg.expect("err").tier_type.as_lowercase()
|
||||
}
|
||||
|
||||
pub fn list_tiers(&self) -> Vec<TierConfig> {
|
||||
let mut tier_cfgs = Vec::<TierConfig>::new();
|
||||
for (_, tier) in self.tiers.iter() {
|
||||
let tier = tier.clone();
|
||||
tier_cfgs.push(tier);
|
||||
}
|
||||
tier_cfgs
|
||||
}
|
||||
|
||||
pub fn get(&self, tier_name: &str) -> Option<TierConfig> {
|
||||
for (tier_name2, tier) in self.tiers.iter() {
|
||||
if tier_name == tier_name2 {
|
||||
return Some(tier.clone());
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub async fn edit(&mut self, tier_name: &str, creds: TierCreds) -> std::result::Result<(), AdminError> {
|
||||
let (tier_type, exists) = self.is_tier_name_in_use(tier_name);
|
||||
if !exists {
|
||||
return Err(ERR_TIER_NOT_FOUND.clone());
|
||||
}
|
||||
|
||||
let mut cfg = self.tiers[tier_name].clone();
|
||||
match tier_type {
|
||||
TierType::S3 => {
|
||||
let mut s3 = cfg.s3.as_mut().expect("err");
|
||||
if creds.aws_role {
|
||||
s3.aws_role = true
|
||||
}
|
||||
if creds.aws_role_web_identity_token_file != "" && creds.aws_role_arn != "" {
|
||||
s3.aws_role_arn = creds.aws_role_arn;
|
||||
s3.aws_role_web_identity_token_file = creds.aws_role_web_identity_token_file;
|
||||
}
|
||||
if creds.access_key != "" && creds.secret_key != "" {
|
||||
s3.access_key = creds.access_key;
|
||||
s3.secret_key = creds.secret_key;
|
||||
}
|
||||
}
|
||||
TierType::RustFS => {
|
||||
let mut rustfs = cfg.rustfs.as_mut().expect("err");
|
||||
if creds.access_key == "" || creds.secret_key == "" {
|
||||
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
|
||||
}
|
||||
rustfs.access_key = creds.access_key;
|
||||
rustfs.secret_key = creds.secret_key;
|
||||
}
|
||||
TierType::MinIO => {
|
||||
let mut minio = cfg.minio.as_mut().expect("err");
|
||||
if creds.access_key == "" || creds.secret_key == "" {
|
||||
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
|
||||
}
|
||||
minio.access_key = creds.access_key;
|
||||
minio.secret_key = creds.secret_key;
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
|
||||
let d = new_warm_backend(&cfg, true).await?;
|
||||
self.tiers.insert(tier_name.to_string(), cfg);
|
||||
self.driver_cache.insert(tier_name.to_string(), d);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get_driver<'a>(&'a mut self, tier_name: &str) -> std::result::Result<&'a WarmBackendImpl, AdminError> {
|
||||
Ok(match self.driver_cache.entry(tier_name.to_string()) {
|
||||
Entry::Occupied(e) => e.into_mut(),
|
||||
Entry::Vacant(e) => {
|
||||
let t = self.tiers.get(tier_name);
|
||||
if t.is_none() {
|
||||
return Err(ERR_TIER_NOT_FOUND.clone());
|
||||
}
|
||||
let d = new_warm_backend(t.expect("err"), false).await?;
|
||||
e.insert(d)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn reload(&mut self, api: Arc<ECStore>) -> std::result::Result<(), std::io::Error> {
|
||||
//let Some(api) = new_object_layer_fn() else { return Err(Error::msg("errServerNotInitialized")) };
|
||||
let new_config = load_tier_config(api).await;
|
||||
|
||||
match &new_config {
|
||||
Ok(_c) => {}
|
||||
Err(err) => {
|
||||
return Err(std::io::Error::other(err.to_string()));
|
||||
}
|
||||
}
|
||||
self.driver_cache.clear();
|
||||
self.tiers.clear();
|
||||
let new_config = new_config.expect("err");
|
||||
for (tier, cfg) in new_config.tiers {
|
||||
self.tiers.insert(tier, cfg);
|
||||
}
|
||||
self.last_refreshed_at = OffsetDateTime::now_utc();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn clear_tier(&mut self, force: bool) -> std::result::Result<(), AdminError> {
|
||||
self.tiers.clear();
|
||||
self.driver_cache.clear();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", name = "tier_save", skip(self))]
|
||||
pub async fn save(&self) -> std::result::Result<(), std::io::Error> {
|
||||
let Some(api) = new_object_layer_fn() else {
|
||||
return Err(std::io::Error::other("errServerNotInitialized"));
|
||||
};
|
||||
//let (pr, opts) = GLOBAL_TierConfigMgr.write().config_reader()?;
|
||||
|
||||
self.save_tiering_config(api).await
|
||||
}
|
||||
|
||||
pub async fn save_tiering_config<S: StorageAPI>(&self, api: Arc<S>) -> std::result::Result<(), std::io::Error> {
|
||||
let data = self.marshal()?;
|
||||
|
||||
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE);
|
||||
|
||||
self.save_config(api, &config_file, data).await
|
||||
}
|
||||
|
||||
pub async fn save_config<S: StorageAPI>(
|
||||
&self,
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
data: Bytes,
|
||||
) -> std::result::Result<(), std::io::Error> {
|
||||
self.save_config_with_opts(
|
||||
api,
|
||||
file,
|
||||
data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn save_config_with_opts<S: StorageAPI>(
|
||||
&self,
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
data: Bytes,
|
||||
opts: &ObjectOptions,
|
||||
) -> std::result::Result<(), std::io::Error> {
|
||||
debug!("save tier config:{}", file);
|
||||
let _ = api
|
||||
.put_object(RUSTFS_META_BUCKET, file, &mut PutObjReader::from_vec(data.to_vec()), opts)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn refresh_tier_config(&mut self, api: Arc<ECStore>) {
|
||||
//let r = rand.New(rand.NewSource(time.Now().UnixNano()));
|
||||
let mut rng = rand::rng();
|
||||
let r = rng.random_range(0.0..1.0);
|
||||
let rand_interval = || Duration::from_secs((r * 60_f64).round() as u64);
|
||||
|
||||
let mut t = interval(TIER_CFG_REFRESH + rand_interval());
|
||||
loop {
|
||||
select! {
|
||||
_ = t.tick() => {
|
||||
if let Err(err) = self.reload(api.clone()).await {
|
||||
info!("{}", err);
|
||||
}
|
||||
}
|
||||
else => ()
|
||||
}
|
||||
t.reset();
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn init(&mut self, api: Arc<ECStore>) -> Result<()> {
|
||||
self.reload(api).await?;
|
||||
//if globalIsDistErasure {
|
||||
// self.refresh_tier_config(api).await;
|
||||
//}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn new_and_save_tiering_config<S: StorageAPI>(api: Arc<S>) -> Result<TierConfigMgr> {
|
||||
let mut cfg = TierConfigMgr {
|
||||
driver_cache: HashMap::new(),
|
||||
tiers: HashMap::new(),
|
||||
last_refreshed_at: OffsetDateTime::now_utc(),
|
||||
};
|
||||
//lookup_configs(&mut cfg, api.clone()).await;
|
||||
cfg.save_tiering_config(api).await?;
|
||||
|
||||
Ok(cfg)
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug")]
|
||||
async fn load_tier_config(api: Arc<ECStore>) -> std::result::Result<TierConfigMgr, std::io::Error> {
|
||||
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE);
|
||||
let data = read_config(api.clone(), config_file.as_str()).await;
|
||||
if let Err(err) = data {
|
||||
if is_err_config_not_found(&err) {
|
||||
warn!("config not found, start to init");
|
||||
let cfg = new_and_save_tiering_config(api).await?;
|
||||
return Ok(cfg);
|
||||
} else {
|
||||
error!("read config err {:?}", &err);
|
||||
return Err(std::io::Error::other(err));
|
||||
}
|
||||
}
|
||||
|
||||
let cfg;
|
||||
let version = 1; //LittleEndian::read_u16(&data[2..4]);
|
||||
match version {
|
||||
TIER_CONFIG_V1/* | TIER_CONFIG_VERSION */ => {
|
||||
cfg = match TierConfigMgr::unmarshal(&data.unwrap()) {
|
||||
Ok(cfg) => cfg,
|
||||
Err(err) => {
|
||||
return Err(std::io::Error::other(err.to_string()));
|
||||
}
|
||||
};
|
||||
}
|
||||
_ => {
|
||||
return Err(std::io::Error::other(format!("tierConfigInit: unknown version: {}", version)));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(cfg)
|
||||
}
|
||||
|
||||
pub fn is_err_config_not_found(err: &StorageError) -> bool {
|
||||
matches!(err, StorageError::ObjectNotFound(_, _)) || err == &StorageError::ConfigNotFound
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
#![allow(unused_imports)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
|
||||
#[serde(default)]
|
||||
pub struct TierCreds {
|
||||
#[serde(rename = "accessKey")]
|
||||
pub access_key: String,
|
||||
#[serde(rename = "secretKey")]
|
||||
pub secret_key: String,
|
||||
|
||||
#[serde(rename = "awsRole")]
|
||||
pub aws_role: bool,
|
||||
#[serde(rename = "awsRoleWebIdentityTokenFile")]
|
||||
pub aws_role_web_identity_token_file: String,
|
||||
#[serde(rename = "awsRoleArn")]
|
||||
pub aws_role_arn: String,
|
||||
|
||||
//azsp: ServicePrincipalAuth,
|
||||
|
||||
//#[serde(rename = "credsJson")]
|
||||
pub creds_json: Vec<u8>,
|
||||
}
|
||||
@@ -0,0 +1,321 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::fmt::Display;
|
||||
use tracing::info;
|
||||
|
||||
const C_TIER_CONFIG_VER: &str = "v1";
|
||||
|
||||
const ERR_TIER_NAME_EMPTY: &str = "remote tier name empty";
|
||||
|
||||
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
|
||||
pub enum TierType {
|
||||
#[default]
|
||||
Unsupported,
|
||||
#[serde(rename = "s3")]
|
||||
S3,
|
||||
#[serde(rename = "azure")]
|
||||
Azure,
|
||||
#[serde(rename = "gcs")]
|
||||
GCS,
|
||||
#[serde(rename = "rustfs")]
|
||||
RustFS,
|
||||
#[serde(rename = "minio")]
|
||||
MinIO,
|
||||
}
|
||||
|
||||
impl Display for TierType {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
TierType::S3 => {
|
||||
write!(f, "S3")
|
||||
}
|
||||
TierType::RustFS => {
|
||||
write!(f, "RustFS")
|
||||
}
|
||||
TierType::MinIO => {
|
||||
write!(f, "MinIO")
|
||||
}
|
||||
_ => {
|
||||
write!(f, "Unsupported")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TierType {
|
||||
pub fn new(sc_type: &str) -> Self {
|
||||
match sc_type {
|
||||
"S3" => TierType::S3,
|
||||
"RustFS" => TierType::RustFS,
|
||||
"MinIO" => TierType::MinIO,
|
||||
_ => TierType::Unsupported,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_lowercase(&self) -> String {
|
||||
match self {
|
||||
TierType::S3 => "s3".to_string(),
|
||||
TierType::RustFS => "rustfs".to_string(),
|
||||
TierType::MinIO => "minio".to_string(),
|
||||
_ => "unsupported".to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Debug, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct TierConfig {
|
||||
#[serde(skip)]
|
||||
pub version: String,
|
||||
#[serde(rename = "type")]
|
||||
pub tier_type: TierType,
|
||||
#[serde(skip)]
|
||||
pub name: String,
|
||||
#[serde(rename = "s3", skip_serializing_if = "Option::is_none")]
|
||||
pub s3: Option<TierS3>,
|
||||
//TODO: azure: Option<TierAzure>,
|
||||
//TODO: gcs: Option<TierGCS>,
|
||||
#[serde(rename = "rustfs", skip_serializing_if = "Option::is_none")]
|
||||
pub rustfs: Option<TierRustFS>,
|
||||
#[serde(rename = "minio", skip_serializing_if = "Option::is_none")]
|
||||
pub minio: Option<TierMinIO>,
|
||||
}
|
||||
|
||||
impl Clone for TierConfig {
|
||||
fn clone(&self) -> TierConfig {
|
||||
let mut s3 = None;
|
||||
//az TierAzure
|
||||
//gcs TierGCS
|
||||
let mut r = None;
|
||||
let mut m = None;
|
||||
match self.tier_type {
|
||||
TierType::S3 => {
|
||||
let mut s3_ = self.s3.as_ref().expect("err").clone();
|
||||
s3_.secret_key = "REDACTED".to_string();
|
||||
s3 = Some(s3_);
|
||||
}
|
||||
TierType::RustFS => {
|
||||
let mut r_ = self.rustfs.as_ref().expect("err").clone();
|
||||
r_.secret_key = "REDACTED".to_string();
|
||||
r = Some(r_);
|
||||
}
|
||||
TierType::MinIO => {
|
||||
let mut m_ = self.minio.as_ref().expect("err").clone();
|
||||
m_.secret_key = "REDACTED".to_string();
|
||||
m = Some(m_);
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
TierConfig {
|
||||
version: self.version.clone(),
|
||||
tier_type: self.tier_type.clone(),
|
||||
name: self.name.clone(),
|
||||
s3,
|
||||
rustfs: r,
|
||||
minio: m,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl TierConfig {
|
||||
fn endpoint(&self) -> String {
|
||||
match self.tier_type {
|
||||
TierType::S3 => self.s3.as_ref().expect("err").endpoint.clone(),
|
||||
TierType::RustFS => self.rustfs.as_ref().expect("err").endpoint.clone(),
|
||||
TierType::MinIO => self.minio.as_ref().expect("err").endpoint.clone(),
|
||||
_ => {
|
||||
info!("unexpected tier type {}", self.tier_type);
|
||||
"".to_string()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn bucket(&self) -> String {
|
||||
match self.tier_type {
|
||||
TierType::S3 => self.s3.as_ref().expect("err").bucket.clone(),
|
||||
TierType::RustFS => self.rustfs.as_ref().expect("err").bucket.clone(),
|
||||
TierType::MinIO => self.minio.as_ref().expect("err").bucket.clone(),
|
||||
_ => {
|
||||
info!("unexpected tier type {}", self.tier_type);
|
||||
"".to_string()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn prefix(&self) -> String {
|
||||
match self.tier_type {
|
||||
TierType::S3 => self.s3.as_ref().expect("err").prefix.clone(),
|
||||
TierType::RustFS => self.rustfs.as_ref().expect("err").prefix.clone(),
|
||||
TierType::MinIO => self.minio.as_ref().expect("err").prefix.clone(),
|
||||
_ => {
|
||||
info!("unexpected tier type {}", self.tier_type);
|
||||
"".to_string()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn region(&self) -> String {
|
||||
match self.tier_type {
|
||||
TierType::S3 => self.s3.as_ref().expect("err").region.clone(),
|
||||
TierType::RustFS => self.rustfs.as_ref().expect("err").region.clone(),
|
||||
TierType::MinIO => self.minio.as_ref().expect("err").region.clone(),
|
||||
_ => {
|
||||
info!("unexpected tier type {}", self.tier_type);
|
||||
"".to_string()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//type S3Options = impl Fn(TierS3) -> Pin<Box<Result<()>>> + Send + Sync + 'static;
|
||||
|
||||
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
|
||||
#[serde(default)]
|
||||
pub struct TierS3 {
|
||||
pub name: String,
|
||||
pub endpoint: String,
|
||||
#[serde(rename = "accessKey")]
|
||||
pub access_key: String,
|
||||
#[serde(rename = "secretKey")]
|
||||
pub secret_key: String,
|
||||
pub bucket: String,
|
||||
pub prefix: String,
|
||||
pub region: String,
|
||||
#[serde(rename = "storageClass")]
|
||||
pub storage_class: String,
|
||||
#[serde(skip)]
|
||||
pub aws_role: bool,
|
||||
#[serde(skip)]
|
||||
pub aws_role_web_identity_token_file: String,
|
||||
#[serde(skip)]
|
||||
pub aws_role_arn: String,
|
||||
#[serde(skip)]
|
||||
pub aws_role_session_name: String,
|
||||
#[serde(skip)]
|
||||
pub aws_role_duration_seconds: i32,
|
||||
}
|
||||
|
||||
impl TierS3 {
|
||||
#[allow(dead_code)]
|
||||
fn create<F>(
|
||||
name: &str,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
bucket: &str,
|
||||
options: Vec<F>,
|
||||
) -> Result<TierConfig, std::io::Error>
|
||||
where
|
||||
F: Fn(TierS3) -> Box<Result<(), std::io::Error>> + Send + Sync + 'static,
|
||||
{
|
||||
if name.is_empty() {
|
||||
return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY));
|
||||
}
|
||||
let sc = TierS3 {
|
||||
access_key: access_key.to_string(),
|
||||
secret_key: secret_key.to_string(),
|
||||
bucket: bucket.to_string(),
|
||||
endpoint: "https://s3.amazonaws.com".to_string(),
|
||||
region: "".to_string(),
|
||||
storage_class: "".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
for option in options {
|
||||
let option = option(sc.clone());
|
||||
let option = *option;
|
||||
option?;
|
||||
}
|
||||
|
||||
Ok(TierConfig {
|
||||
version: C_TIER_CONFIG_VER.to_string(),
|
||||
tier_type: TierType::S3,
|
||||
name: name.to_string(),
|
||||
s3: Some(sc),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
|
||||
#[serde(default)]
|
||||
pub struct TierRustFS {
|
||||
pub name: String,
|
||||
pub endpoint: String,
|
||||
#[serde(rename = "accessKey")]
|
||||
pub access_key: String,
|
||||
#[serde(rename = "secretKey")]
|
||||
pub secret_key: String,
|
||||
pub bucket: String,
|
||||
pub prefix: String,
|
||||
pub region: String,
|
||||
#[serde(rename = "storageClass")]
|
||||
pub storage_class: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
|
||||
#[serde(default)]
|
||||
pub struct TierMinIO {
|
||||
pub name: String,
|
||||
pub endpoint: String,
|
||||
#[serde(rename = "accessKey")]
|
||||
pub access_key: String,
|
||||
#[serde(rename = "secretKey")]
|
||||
pub secret_key: String,
|
||||
pub bucket: String,
|
||||
pub prefix: String,
|
||||
pub region: String,
|
||||
}
|
||||
|
||||
impl TierMinIO {
|
||||
#[allow(dead_code)]
|
||||
fn create<F>(
|
||||
name: &str,
|
||||
endpoint: &str,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
bucket: &str,
|
||||
options: Vec<F>,
|
||||
) -> Result<TierConfig, std::io::Error>
|
||||
where
|
||||
F: Fn(TierMinIO) -> Box<Result<(), std::io::Error>> + Send + Sync + 'static,
|
||||
{
|
||||
if name.is_empty() {
|
||||
return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY));
|
||||
}
|
||||
let m = TierMinIO {
|
||||
access_key: access_key.to_string(),
|
||||
secret_key: secret_key.to_string(),
|
||||
bucket: bucket.to_string(),
|
||||
endpoint: endpoint.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
for option in options {
|
||||
let option = option(m.clone());
|
||||
let option = *option;
|
||||
option?;
|
||||
}
|
||||
|
||||
Ok(TierConfig {
|
||||
version: C_TIER_CONFIG_VER.to_string(),
|
||||
tier_type: TierType::MinIO,
|
||||
name: name.to_string(),
|
||||
minio: Some(m),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::tier::tier::TierConfigMgr;
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl TierConfigMgr {
|
||||
pub fn msg_size(&self) -> usize {
|
||||
100
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::client::admin_handler_utils::AdminError;
|
||||
use http::status::StatusCode;
|
||||
use lazy_static::lazy_static;
|
||||
|
||||
lazy_static! {
|
||||
pub static ref ERR_TIER_ALREADY_EXISTS: AdminError = AdminError {
|
||||
code: "XRustFSAdminTierAlreadyExists".to_string(),
|
||||
message: "Specified remote tier already exists".to_string(),
|
||||
status_code: StatusCode::CONFLICT,
|
||||
};
|
||||
pub static ref ERR_TIER_NOT_FOUND: AdminError = AdminError {
|
||||
code: "XRustFSAdminTierNotFound".to_string(),
|
||||
message: "Specified remote tier was not found".to_string(),
|
||||
status_code: StatusCode::NOT_FOUND,
|
||||
};
|
||||
pub static ref ERR_TIER_NAME_NOT_UPPERCASE: AdminError = AdminError {
|
||||
code: "XRustFSAdminTierNameNotUpperCase".to_string(),
|
||||
message: "Tier name must be in uppercase".to_string(),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
};
|
||||
pub static ref ERR_TIER_BUCKET_NOT_FOUND: AdminError = AdminError {
|
||||
code: "XRustFSAdminTierBucketNotFound".to_string(),
|
||||
message: "Remote tier bucket not found".to_string(),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
};
|
||||
pub static ref ERR_TIER_INVALID_CREDENTIALS: AdminError = AdminError {
|
||||
code: "XRustFSAdminTierInvalidCredentials".to_string(),
|
||||
message: "Invalid remote tier credentials".to_string(),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
};
|
||||
pub static ref ERR_TIER_RESERVED_NAME: AdminError = AdminError {
|
||||
code: "XRustFSAdminTierReserved".to_string(),
|
||||
message: "Cannot use reserved tier name".to_string(),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
};
|
||||
pub static ref ERR_TIER_PERM_ERR: AdminError = AdminError {
|
||||
code: "TierPermErr".to_string(),
|
||||
message: "Tier Perm Err".to_string(),
|
||||
status_code: StatusCode::OK,
|
||||
};
|
||||
pub static ref ERR_TIER_CONNECT_ERR: AdminError = AdminError {
|
||||
code: "TierConnectErr".to_string(),
|
||||
message: "Tier Connect Err".to_string(),
|
||||
status_code: StatusCode::OK,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
#![allow(unused_imports)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use crate::client::{
|
||||
admin_handler_utils::AdminError,
|
||||
transition_api::{ReadCloser, ReaderImpl},
|
||||
};
|
||||
use crate::error::is_err_bucket_not_found;
|
||||
use crate::tier::{
|
||||
tier::ERR_TIER_TYPE_UNSUPPORTED,
|
||||
tier_config::{TierConfig, TierType},
|
||||
tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_PERM_ERR},
|
||||
warm_backend_minio::WarmBackendMinIO,
|
||||
warm_backend_rustfs::WarmBackendRustFS,
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use http::StatusCode;
|
||||
use std::collections::HashMap;
|
||||
use tracing::{info, warn};
|
||||
|
||||
pub type WarmBackendImpl = Box<dyn WarmBackend + Send + Sync + 'static>;
|
||||
|
||||
const PROBE_OBJECT: &str = "probeobject";
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct WarmBackendGetOpts {
|
||||
pub start_offset: i64,
|
||||
pub length: i64,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait WarmBackend {
|
||||
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error>;
|
||||
async fn put_with_meta(
|
||||
&self,
|
||||
object: &str,
|
||||
r: ReaderImpl,
|
||||
length: i64,
|
||||
meta: HashMap<String, String>,
|
||||
) -> Result<String, std::io::Error>;
|
||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error>;
|
||||
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error>;
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error>;
|
||||
}
|
||||
|
||||
pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> {
|
||||
let w = w.expect("err");
|
||||
let remote_version_id = w
|
||||
.put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5)
|
||||
.await;
|
||||
if let Err(err) = remote_version_id {
|
||||
return Err(ERR_TIER_PERM_ERR.clone());
|
||||
}
|
||||
|
||||
let r = w.get(PROBE_OBJECT, "", WarmBackendGetOpts::default()).await;
|
||||
//xhttp.DrainBody(r);
|
||||
if let Err(err) = r {
|
||||
//if is_err_bucket_not_found(&err) {
|
||||
// return Err(ERR_TIER_BUCKET_NOT_FOUND);
|
||||
//}
|
||||
/*else if is_err_signature_does_not_match(err) {
|
||||
return Err(ERR_TIER_MISSING_CREDENTIALS);
|
||||
}*/
|
||||
//else {
|
||||
return Err(ERR_TIER_PERM_ERR.clone());
|
||||
//}
|
||||
}
|
||||
if let Err(err) = w.remove(PROBE_OBJECT, &remote_version_id.expect("err")).await {
|
||||
return Err(ERR_TIER_PERM_ERR.clone());
|
||||
};
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBackendImpl, AdminError> {
|
||||
let mut d: Option<WarmBackendImpl> = None;
|
||||
match tier.tier_type {
|
||||
TierType::S3 => {
|
||||
let dd = WarmBackendS3::new(tier.s3.as_ref().expect("err"), &tier.name).await;
|
||||
if let Err(err) = dd {
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
d = Some(Box::new(dd.expect("err")));
|
||||
}
|
||||
TierType::RustFS => {
|
||||
let dd = WarmBackendRustFS::new(tier.rustfs.as_ref().expect("err"), &tier.name).await;
|
||||
if let Err(err) = dd {
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
d = Some(Box::new(dd.expect("err")));
|
||||
}
|
||||
TierType::MinIO => {
|
||||
let dd = WarmBackendMinIO::new(tier.minio.as_ref().expect("err"), &tier.name).await;
|
||||
if let Err(err) = dd {
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
d = Some(Box::new(dd.expect("err")));
|
||||
}
|
||||
_ => {
|
||||
return Err(ERR_TIER_TYPE_UNSUPPORTED.clone());
|
||||
}
|
||||
}
|
||||
|
||||
Ok(d.expect("err"))
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
#![allow(unused_imports)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use crate::tier::{
|
||||
tier_config::TierMinIO,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
const MAX_PARTS_COUNT: i64 = 10000;
|
||||
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
|
||||
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
|
||||
|
||||
pub struct WarmBackendMinIO(WarmBackendS3);
|
||||
|
||||
impl WarmBackendMinIO {
|
||||
pub async fn new(conf: &TierMinIO, tier: &str) -> Result<Self, std::io::Error> {
|
||||
if conf.access_key == "" || conf.secret_key == "" {
|
||||
return Err(std::io::Error::other("both access and secret keys are required"));
|
||||
}
|
||||
|
||||
if conf.bucket == "" {
|
||||
return Err(std::io::Error::other("no bucket name was provided"));
|
||||
}
|
||||
|
||||
let u = match url::Url::parse(&conf.endpoint) {
|
||||
Ok(u) => u,
|
||||
Err(e) => {
|
||||
return Err(std::io::Error::other(e.to_string()));
|
||||
}
|
||||
};
|
||||
|
||||
let creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
secret_access_key: conf.secret_key.clone(),
|
||||
session_token: "".to_string(),
|
||||
signer_type: SignatureType::SignatureV4,
|
||||
..Default::default()
|
||||
}));
|
||||
let opts = Options {
|
||||
creds,
|
||||
secure: u.scheme() == "https",
|
||||
//transport: GLOBAL_RemoteTargetTransport,
|
||||
trailing_headers: true,
|
||||
..Default::default()
|
||||
};
|
||||
let scheme = u.scheme();
|
||||
let default_port = if scheme == "https" { 443 } else { 80 };
|
||||
let client =
|
||||
TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?;
|
||||
|
||||
let client = Arc::new(client);
|
||||
let core = TransitionCore(Arc::clone(&client));
|
||||
Ok(Self(WarmBackendS3 {
|
||||
client,
|
||||
core,
|
||||
bucket: conf.bucket.clone(),
|
||||
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
|
||||
storage_class: "".to_string(),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl WarmBackend for WarmBackendMinIO {
|
||||
async fn put_with_meta(
|
||||
&self,
|
||||
object: &str,
|
||||
r: ReaderImpl,
|
||||
length: i64,
|
||||
meta: HashMap<String, String>,
|
||||
) -> Result<String, std::io::Error> {
|
||||
let part_size = optimal_part_size(length)?;
|
||||
let client = self.0.client.clone();
|
||||
let res = client
|
||||
.put_object(
|
||||
&self.0.bucket,
|
||||
&self.0.get_dest(object),
|
||||
r,
|
||||
length,
|
||||
&PutObjectOptions {
|
||||
storage_class: self.0.storage_class.clone(),
|
||||
part_size: part_size as u64,
|
||||
disable_content_sha256: true,
|
||||
user_metadata: meta,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
//self.ToObjectError(err, object)
|
||||
Ok(res.version_id)
|
||||
}
|
||||
|
||||
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
|
||||
self.put_with_meta(object, r, length, HashMap::new()).await
|
||||
}
|
||||
|
||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
self.0.get(object, rv, opts).await
|
||||
}
|
||||
|
||||
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||
self.0.remove(object, rv).await
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
self.0.in_use().await
|
||||
}
|
||||
}
|
||||
|
||||
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
let mut object_size = object_size;
|
||||
if object_size == -1 {
|
||||
object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
|
||||
}
|
||||
|
||||
if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
|
||||
return Err(std::io::Error::other("entity too large"));
|
||||
}
|
||||
|
||||
let configured_part_size = MIN_PART_SIZE;
|
||||
let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64;
|
||||
part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64;
|
||||
|
||||
let part_size = part_size_flt as i64;
|
||||
if part_size == 0 {
|
||||
return Ok(MIN_PART_SIZE);
|
||||
}
|
||||
Ok(part_size)
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
#![allow(unused_imports)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use crate::tier::{
|
||||
tier_config::TierRustFS,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
const MAX_PARTS_COUNT: i64 = 10000;
|
||||
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
|
||||
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
|
||||
|
||||
pub struct WarmBackendRustFS(WarmBackendS3);
|
||||
|
||||
impl WarmBackendRustFS {
|
||||
pub async fn new(conf: &TierRustFS, tier: &str) -> Result<Self, std::io::Error> {
|
||||
if conf.access_key == "" || conf.secret_key == "" {
|
||||
return Err(std::io::Error::other("both access and secret keys are required"));
|
||||
}
|
||||
|
||||
if conf.bucket == "" {
|
||||
return Err(std::io::Error::other("no bucket name was provided"));
|
||||
}
|
||||
|
||||
let u = match url::Url::parse(&conf.endpoint) {
|
||||
Ok(u) => u,
|
||||
Err(e) => return Err(std::io::Error::other(e)),
|
||||
};
|
||||
|
||||
let creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
secret_access_key: conf.secret_key.clone(),
|
||||
session_token: "".to_string(),
|
||||
signer_type: SignatureType::SignatureV4,
|
||||
..Default::default()
|
||||
}));
|
||||
let opts = Options {
|
||||
creds,
|
||||
secure: u.scheme() == "https",
|
||||
//transport: GLOBAL_RemoteTargetTransport,
|
||||
trailing_headers: true,
|
||||
..Default::default()
|
||||
};
|
||||
let scheme = u.scheme();
|
||||
let default_port = if scheme == "https" { 443 } else { 80 };
|
||||
let client =
|
||||
TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?;
|
||||
|
||||
let client = Arc::new(client);
|
||||
let core = TransitionCore(Arc::clone(&client));
|
||||
Ok(Self(WarmBackendS3 {
|
||||
client,
|
||||
core,
|
||||
bucket: conf.bucket.clone(),
|
||||
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
|
||||
storage_class: "".to_string(),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl WarmBackend for WarmBackendRustFS {
|
||||
async fn put_with_meta(
|
||||
&self,
|
||||
object: &str,
|
||||
r: ReaderImpl,
|
||||
length: i64,
|
||||
meta: HashMap<String, String>,
|
||||
) -> Result<String, std::io::Error> {
|
||||
let part_size = optimal_part_size(length)?;
|
||||
let client = self.0.client.clone();
|
||||
let res = client
|
||||
.put_object(
|
||||
&self.0.bucket,
|
||||
&self.0.get_dest(object),
|
||||
r,
|
||||
length,
|
||||
&PutObjectOptions {
|
||||
storage_class: self.0.storage_class.clone(),
|
||||
part_size: part_size as u64,
|
||||
disable_content_sha256: true,
|
||||
user_metadata: meta,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
//self.ToObjectError(err, object)
|
||||
Ok(res.version_id)
|
||||
}
|
||||
|
||||
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
|
||||
self.put_with_meta(object, r, length, HashMap::new()).await
|
||||
}
|
||||
|
||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
self.0.get(object, rv, opts).await
|
||||
}
|
||||
|
||||
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||
self.0.remove(object, rv).await
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
self.0.in_use().await
|
||||
}
|
||||
}
|
||||
|
||||
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
let mut object_size = object_size;
|
||||
if object_size == -1 {
|
||||
object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
|
||||
}
|
||||
|
||||
if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
|
||||
return Err(std::io::Error::other("entity too large"));
|
||||
}
|
||||
|
||||
let configured_part_size = MIN_PART_SIZE;
|
||||
let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64;
|
||||
part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64;
|
||||
|
||||
let part_size = part_size_flt as i64;
|
||||
if part_size == 0 {
|
||||
return Ok(MIN_PART_SIZE);
|
||||
}
|
||||
Ok(part_size)
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
#![allow(unused_imports)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use url::Url;
|
||||
|
||||
use crate::client::{
|
||||
api_get_options::GetObjectOptions,
|
||||
api_put_object::PutObjectOptions,
|
||||
api_remove::RemoveObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{Options, TransitionClient, TransitionCore},
|
||||
transition_api::{ReadCloser, ReaderImpl},
|
||||
};
|
||||
use crate::error::ErrorResponse;
|
||||
use crate::error::error_resp_to_object_err;
|
||||
use crate::tier::{
|
||||
tier_config::TierS3,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts},
|
||||
};
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
|
||||
pub struct WarmBackendS3 {
|
||||
pub client: Arc<TransitionClient>,
|
||||
pub core: TransitionCore,
|
||||
pub bucket: String,
|
||||
pub prefix: String,
|
||||
pub storage_class: String,
|
||||
}
|
||||
|
||||
impl WarmBackendS3 {
|
||||
pub async fn new(conf: &TierS3, tier: &str) -> Result<Self, std::io::Error> {
|
||||
let u = match Url::parse(&conf.endpoint) {
|
||||
Ok(u) => u,
|
||||
Err(err) => {
|
||||
return Err(std::io::Error::other(err.to_string()));
|
||||
}
|
||||
};
|
||||
|
||||
if conf.aws_role_web_identity_token_file == "" && conf.aws_role_arn != ""
|
||||
|| conf.aws_role_web_identity_token_file != "" && conf.aws_role_arn == ""
|
||||
{
|
||||
return Err(std::io::Error::other("both the token file and the role ARN are required"));
|
||||
} else if conf.access_key == "" && conf.secret_key != "" || conf.access_key != "" && conf.secret_key == "" {
|
||||
return Err(std::io::Error::other("both the access and secret keys are required"));
|
||||
} else if conf.aws_role
|
||||
&& (conf.aws_role_web_identity_token_file != ""
|
||||
|| conf.aws_role_arn != ""
|
||||
|| conf.access_key != ""
|
||||
|| conf.secret_key != "")
|
||||
{
|
||||
return Err(std::io::Error::other(
|
||||
"AWS Role cannot be activated with static credentials or the web identity token file",
|
||||
));
|
||||
} else if conf.bucket == "" {
|
||||
return Err(std::io::Error::other("no bucket name was provided"));
|
||||
}
|
||||
|
||||
let creds: Credentials<Static>;
|
||||
|
||||
if conf.access_key != "" && conf.secret_key != "" {
|
||||
//creds = Credentials::new_static_v4(conf.access_key, conf.secret_key, "");
|
||||
creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
secret_access_key: conf.secret_key.clone(),
|
||||
session_token: "".to_string(),
|
||||
signer_type: SignatureType::SignatureV4,
|
||||
..Default::default()
|
||||
}));
|
||||
} else {
|
||||
return Err(std::io::Error::other("insufficient parameters for S3 backend authentication"));
|
||||
}
|
||||
let opts = Options {
|
||||
creds,
|
||||
secure: u.scheme() == "https",
|
||||
//transport: GLOBAL_RemoteTargetTransport,
|
||||
..Default::default()
|
||||
};
|
||||
let client = TransitionClient::new(&u.host().expect("err").to_string(), opts).await?;
|
||||
//client.set_appinfo(format!("s3-tier-{}", tier), ReleaseTag);
|
||||
|
||||
let client = Arc::new(client);
|
||||
let core = TransitionCore(Arc::clone(&client));
|
||||
Ok(Self {
|
||||
client,
|
||||
core,
|
||||
bucket: conf.bucket.clone(),
|
||||
prefix: conf.prefix.clone().trim_matches('/').to_string(),
|
||||
storage_class: conf.storage_class.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn get_dest(&self, object: &str) -> String {
|
||||
let mut dest_obj = object.to_string();
|
||||
if self.prefix != "" {
|
||||
dest_obj = format!("{}/{}", &self.prefix, object);
|
||||
}
|
||||
return dest_obj;
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl WarmBackend for WarmBackendS3 {
|
||||
async fn put_with_meta(
|
||||
&self,
|
||||
object: &str,
|
||||
r: ReaderImpl,
|
||||
length: i64,
|
||||
meta: HashMap<String, String>,
|
||||
) -> Result<String, std::io::Error> {
|
||||
let client = self.client.clone();
|
||||
let res = client
|
||||
.put_object(
|
||||
&self.bucket,
|
||||
&self.get_dest(object),
|
||||
r,
|
||||
length,
|
||||
&PutObjectOptions {
|
||||
send_content_md5: true,
|
||||
storage_class: self.storage_class.clone(),
|
||||
user_metadata: meta,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
Ok(res.version_id)
|
||||
}
|
||||
|
||||
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
|
||||
self.put_with_meta(object, r, length, HashMap::new()).await
|
||||
}
|
||||
|
||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
let mut gopts = GetObjectOptions::default();
|
||||
|
||||
if rv != "" {
|
||||
gopts.version_id = rv.to_string();
|
||||
}
|
||||
if opts.start_offset >= 0 && opts.length > 0 {
|
||||
if let Err(err) = gopts.set_range(opts.start_offset, opts.start_offset + opts.length - 1) {
|
||||
return Err(std::io::Error::other(err));
|
||||
}
|
||||
}
|
||||
let c = TransitionCore(Arc::clone(&self.client));
|
||||
let (_, _, r) = c.get_object(&self.bucket, &self.get_dest(object), &gopts).await?;
|
||||
|
||||
Ok(r)
|
||||
}
|
||||
|
||||
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||
let mut ropts = RemoveObjectOptions::default();
|
||||
if rv != "" {
|
||||
ropts.version_id = rv.to_string();
|
||||
}
|
||||
let client = self.client.clone();
|
||||
let err = client.remove_object(&self.bucket, &self.get_dest(object), ropts).await;
|
||||
Err(std::io::Error::other(err.expect("err")))
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
let result = self
|
||||
.core
|
||||
.list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1)
|
||||
.await?;
|
||||
|
||||
Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user