gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-13 00:26:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix/ilm (#721)

Browse Source 
* fix tip remote tier error
* fix transitioned_object
* fix filemeta
* add GCS R2
* add aliyun tencent huaweicloud azure gcs r2 backend tier
* fix signer
* change azure to s3
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: loverustfs <155562731+loverustfs@users.noreply.github.com>
This commit is contained in:
likewu
2025-10-27 20:23:50 +08:00
committed by GitHub
parent 6f3d2885cd
commit 2aca1f77af
38 changed files with 4054 additions and 389 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+184 -34
View File
@@ -18,14 +18,18 @@
#![allow(unused_must_use)]
#![allow(clippy::all)]
use crate::error::StorageError;
use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
use bytes::BytesMut;
use futures::Future;
use http::HeaderMap;
use lazy_static::lazy_static;
use rustfs_common::data_usage::TierStats;
use rustfs_common::heal_channel::rep_has_active_rules;
use rustfs_common::metrics::{IlmAction, Metrics};
use rustfs_filemeta::fileinfo::{NULL_VERSION_ID, RestoreStatusOps, is_restored_object_on_disk};
use rustfs_utils::path::encode_dir_object;
use rustfs_utils::string::strings_has_prefix_fold;
use s3s::Body;
use sha2::{Digest, Sha256};
use std::any::Any;
@@ -62,7 +66,11 @@ use crate::store::ECStore;
use crate::store_api::StorageAPI;
use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete};
use crate::tier::warm_backend::WarmBackendGetOpts;
use s3s::dto::{BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration};
use s3s::dto::{
BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration, RestoreRequest, RestoreRequestType, RestoreStatus,
ServerSideEncryption, Timestamp,
};
use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION, X_AMZ_STORAGE_CLASS};
pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
pub type TraceFn =
@@ -71,9 +79,12 @@ pub type ExpiryOpType = Box<dyn ExpiryOp + Send + Sync + 'static>;
static XXHASH_SEED: u64 = 0;
const _DISABLED: &str = "Disabled";
pub const AMZ_OBJECT_TAGGING: &str = "X-Amz-Tagging";
pub const AMZ_TAG_COUNT: &str = "x-amz-tagging-count";
pub const AMZ_TAG_DIRECTIVE: &str = "X-Amz-Tagging-Directive";
pub const AMZ_ENCRYPTION_AES: &str = "AES256";
pub const AMZ_ENCRYPTION_KMS: &str = "aws:kms";
//pub const ERR_INVALID_STORAGECLASS: &str = "invalid storage class.";
pub const ERR_INVALID_STORAGECLASS: &str = "invalid tier.";
lazy_static! {
@@ -762,11 +773,14 @@ pub fn gen_transition_objname(bucket: &str) -> Result<String, Error> {
pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditEvent) -> Result<(), Error> {
let time_ilm = Metrics::time_ilm(lae.event.action);
let etag = if let Some(etag) = &oi.etag { etag } else { "" };
let etag = etag.to_string();
let opts = ObjectOptions {
transition: TransitionOptions {
status: lifecycle::TRANSITION_PENDING.to_string(),
tier: lae.event.storage_class,
etag: oi.etag.clone().expect("err").to_string(),
etag,
..Default::default()
},
//lifecycle_audit_event: lae,
@@ -787,9 +801,9 @@ pub fn audit_tier_actions(_api: ECStore, _tier: &str, _bytes: i64) -> TimeFn {
pub async fn get_transitioned_object_reader(
bucket: &str,
object: &str,
rs: HTTPRangeSpec,
h: HeaderMap,
oi: ObjectInfo,
rs: &Option<HTTPRangeSpec>,
h: &HeaderMap,
oi: &ObjectInfo,
opts: &ObjectOptions,
) -> Result<GetObjectReader, std::io::Error> {
let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await;
@@ -815,19 +829,131 @@ pub async fn get_transitioned_object_reader(
let reader = tgt_client
.get(&oi.transitioned_object.name, &oi.transitioned_object.version_id, gopts)
.await?;
Ok(get_fn(reader, h))
Ok(get_fn(reader, h.clone()))
}
pub fn post_restore_opts(_r: http::Request<Body>, _bucket: &str, _object: &str) -> Result<ObjectOptions, std::io::Error> {
todo!();
pub async fn post_restore_opts(version_id: &str, bucket: &str, object: &str) -> Result<ObjectOptions, std::io::Error> {
let versioned = BucketVersioningSys::prefix_enabled(bucket, object).await;
let version_suspended = BucketVersioningSys::prefix_suspended(bucket, object).await;
let vid = version_id.trim();
if vid != "" && vid != NULL_VERSION_ID {
if let Err(err) = Uuid::parse_str(vid) {
return Err(std::io::Error::other(
StorageError::InvalidVersionID(bucket.to_string(), object.to_string(), vid.to_string()).to_string(),
));
}
if !versioned && !version_suspended {
return Err(std::io::Error::other(
StorageError::InvalidArgument(
bucket.to_string(),
object.to_string(),
format!("version-id specified {} but versioning is not enabled on {}", vid, bucket),
)
.to_string(),
));
}
}
Ok(ObjectOptions {
versioned: versioned,
version_suspended: version_suspended,
version_id: Some(vid.to_string()),
..Default::default()
})
}
pub fn put_restore_opts(_bucket: &str, _object: &str, _rreq: &RestoreObjectRequest, _oi: &ObjectInfo) -> ObjectOptions {
todo!();
pub async fn put_restore_opts(
bucket: &str,
object: &str,
rreq: &RestoreRequest,
oi: &ObjectInfo,
) -> Result<ObjectOptions, std::io::Error> {
let mut meta = HashMap::<String, String>::new();
/*let mut b = false;
let Some(Some(Some(mut sc))) = rreq.output_location.s3.storage_class else { b = true; };
if b || sc == "" {
//sc = oi.storage_class;
sc = oi.transitioned_object.tier;
}
meta.insert(X_AMZ_STORAGE_CLASS.as_str().to_lowercase(), sc);*/
if let Some(type_) = &rreq.type_
&& type_.as_str() == RestoreRequestType::SELECT
{
for v in rreq
.output_location
.as_ref()
.unwrap()
.s3
.as_ref()
.unwrap()
.user_metadata
.as_ref()
.unwrap()
{
if !strings_has_prefix_fold(&v.name.clone().unwrap(), "x-amz-meta") {
meta.insert(
format!("x-amz-meta-{}", v.name.as_ref().unwrap()),
v.value.clone().unwrap_or("".to_string()),
);
continue;
}
meta.insert(v.name.clone().unwrap(), v.value.clone().unwrap_or("".to_string()));
}
if let Some(output_location) = rreq.output_location.as_ref() {
if let Some(s3) = &output_location.s3 {
if let Some(tags) = &s3.tagging {
meta.insert(
AMZ_OBJECT_TAGGING.to_string(),
serde_urlencoded::to_string(tags.tag_set.clone()).unwrap_or("".to_string()),
);
}
}
}
if let Some(output_location) = rreq.output_location.as_ref() {
if let Some(s3) = &output_location.s3 {
if let Some(encryption) = &s3.encryption {
if encryption.encryption_type.as_str() != "" {
meta.insert(X_AMZ_SERVER_SIDE_ENCRYPTION.as_str().to_string(), AMZ_ENCRYPTION_AES.to_string());
}
}
}
}
return Ok(ObjectOptions {
versioned: BucketVersioningSys::prefix_enabled(bucket, object).await,
version_suspended: BucketVersioningSys::prefix_suspended(bucket, object).await,
user_defined: meta,
..Default::default()
});
}
for (k, v) in &oi.user_defined {
meta.insert(k.to_string(), v.clone());
}
if oi.user_tags.len() != 0 {
meta.insert(AMZ_OBJECT_TAGGING.to_string(), oi.user_tags.clone());
}
let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1));
meta.insert(
X_AMZ_RESTORE.as_str().to_string(),
RestoreStatus {
is_restore_in_progress: Some(false),
restore_expiry_date: Some(Timestamp::from(restore_expiry)),
}
.to_string(),
);
Ok(ObjectOptions {
versioned: BucketVersioningSys::prefix_enabled(bucket, object).await,
version_suspended: BucketVersioningSys::prefix_suspended(bucket, object).await,
user_defined: meta,
version_id: oi.version_id.map(|e| e.to_string()),
mod_time: oi.mod_time,
//expires: oi.expires,
..Default::default()
})
}
pub trait LifecycleOps {
fn to_lifecycle_opts(&self) -> lifecycle::ObjectOpts;
fn is_remote(&self) -> bool;
}
impl LifecycleOps for ObjectInfo {
@@ -848,29 +974,54 @@ impl LifecycleOps for ObjectInfo {
..Default::default()
}
}
fn is_remote(&self) -> bool {
if self.transitioned_object.status != lifecycle::TRANSITION_COMPLETE {
return false;
}
!is_restored_object_on_disk(&self.user_defined)
}
}
#[derive(Debug, Default, Clone)]
pub struct S3Location {
pub bucketname: String,
//pub encryption: Encryption,
pub prefix: String,
pub storage_class: String,
//pub tagging: Tags,
pub user_metadata: HashMap<String, String>,
pub trait RestoreRequestOps {
fn validate(&self, api: Arc<ECStore>) -> Result<(), std::io::Error>;
}
#[derive(Debug, Default, Clone)]
pub struct OutputLocation(pub S3Location);
impl RestoreRequestOps for RestoreRequest {
fn validate(&self, api: Arc<ECStore>) -> Result<(), std::io::Error> {
/*if self.type_.is_none() && self.select_parameters.is_some() {
return Err(std::io::Error::other("Select parameters can only be specified with SELECT request type"));
}
if let Some(type_) = self.type_ && type_ == RestoreRequestType::SELECT && self.select_parameters.is_none() {
return Err(std::io::Error::other("SELECT restore request requires select parameters to be specified"));
}
#[derive(Debug, Default, Clone)]
pub struct RestoreObjectRequest {
pub days: i64,
pub ror_type: String,
pub tier: String,
pub description: String,
//pub select_parameters: SelectParameters,
pub output_location: OutputLocation,
if self.type_.is_none() && self.output_location.is_some() {
return Err(std::io::Error::other("OutputLocation required only for SELECT request type"));
}
if let Some(type_) = self.type_ && type_ == RestoreRequestType::SELECT && self.output_location.is_none() {
return Err(std::io::Error::other("OutputLocation required for SELECT requests"));
}
if let Some(type_) = self.type_ && type_ == RestoreRequestType::SELECT && self.days != 0 {
return Err(std::io::Error::other("Days cannot be specified with SELECT restore request"));
}
if self.days == 0 && self.type_.is_none() {
return Err(std::io::Error::other("restoration days should be at least 1"));
}
if self.output_location.is_some() {
if _, err := api.get_bucket_info(self.output_location.s3.bucket_name, BucketOptions{}); err != nil {
return err
}
if self.output_location.s3.prefix == "" {
return Err(std::io::Error::other("Prefix is a required parameter in OutputLocation"));
}
if self.output_location.s3.encryption.encryption_type.as_str() != ServerSideEncryption::AES256 {
return NotImplemented{}
}
}*/
Ok(())
}
}
const _MAX_RESTORE_OBJECT_REQUEST_SIZE: i64 = 2 << 20;
@@ -975,11 +1126,10 @@ pub async fn apply_expiry_on_non_transitioned_objects(
//debug!("lc_event.action: {:?}", lc_event.action);
//debug!("opts: {:?}", opts);
let mut dobj = match api.delete_object(&oi.bucket, &encode_dir_object(&oi.name), opts).await {
Ok(obj) => obj,
Ok(dobj) => dobj,
Err(e) => {
error!("Failed to delete object {}/{}: {:?}", oi.bucket, oi.name, e);
// Return the original object info if deletion fails
oi.clone()
error!("delete_object error: {:?}", e);
return false;
}
};
//debug!("dobj: {:?}", dobj);
crates/ecstore/src/bucket/lifecycle/lifecycle.rs
+38 -30
View File
@@ -20,7 +20,7 @@
use s3s::dto::{
BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, NoncurrentVersionTransition,
ObjectLockConfiguration, ObjectLockEnabled, Transition,
ObjectLockConfiguration, ObjectLockEnabled, RestoreRequest, Transition,
};
use std::cmp::Ordering;
use std::env;
@@ -32,8 +32,6 @@ use tracing::info;
use crate::bucket::lifecycle::rule::TransitionOps;
use super::bucket_lifecycle_ops::RestoreObjectRequest;
pub const TRANSITION_COMPLETE: &str = "complete";
pub const TRANSITION_PENDING: &str = "pending";
@@ -325,7 +323,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
}
if let Some(days) = expiration.days {
let expected_expiry = expected_expiry_time(obj.mod_time.expect("err!"), days /*, date*/);
let expected_expiry = expected_expiry_time(obj.mod_time.unwrap(), days /*, date*/);
if now.unix_timestamp() >= expected_expiry.unix_timestamp() {
events.push(Event {
action: IlmAction::DeleteVersionAction,
@@ -402,19 +400,21 @@ impl Lifecycle for BucketLifecycleConfiguration {
if storage_class.as_str() != "" && !obj.delete_marker && obj.transition_status != TRANSITION_COMPLETE
{
let due = rule.noncurrent_version_transitions.as_ref().unwrap()[0].next_due(obj);
if due.is_some() && (now.unix_timestamp() >= due.unwrap().unix_timestamp()) {
events.push(Event {
action: IlmAction::TransitionVersionAction,
rule_id: rule.id.clone().expect("err!"),
due,
storage_class: rule.noncurrent_version_transitions.as_ref().unwrap()[0]
.storage_class
.clone()
.unwrap()
.as_str()
.to_string(),
..Default::default()
});
if let Some(due0) = due {
if now.unix_timestamp() == 0 || now.unix_timestamp() > due0.unix_timestamp() {
events.push(Event {
action: IlmAction::TransitionVersionAction,
rule_id: rule.id.clone().expect("err!"),
due,
storage_class: rule.noncurrent_version_transitions.as_ref().unwrap()[0]
.storage_class
.clone()
.unwrap()
.as_str()
.to_string(),
..Default::default()
});
}
}
}
}
@@ -446,7 +446,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
});
}
} else if let Some(days) = expiration.days {
let expected_expiry: OffsetDateTime = expected_expiry_time(obj.mod_time.expect("err!"), days);
let expected_expiry: OffsetDateTime = expected_expiry_time(obj.mod_time.unwrap(), days);
info!(
"eval_inner: expiration check - days={}, obj_time={:?}, expiry_time={:?}, now={:?}, should_expire={}",
days,
@@ -480,12 +480,12 @@ impl Lifecycle for BucketLifecycleConfiguration {
if obj.transition_status != TRANSITION_COMPLETE {
if let Some(ref transitions) = rule.transitions {
let due = transitions[0].next_due(obj);
if let Some(due) = due {
if due.unix_timestamp() > 0 && (now.unix_timestamp() >= due.unix_timestamp()) {
if let Some(due0) = due {
if now.unix_timestamp() == 0 || now.unix_timestamp() > due0.unix_timestamp() {
events.push(Event {
action: IlmAction::TransitionAction,
rule_id: rule.id.clone().expect("err!"),
due: Some(due),
due,
storage_class: transitions[0].storage_class.clone().expect("err!").as_str().to_string(),
noncurrent_days: 0,
newer_noncurrent_versions: 0,
@@ -580,8 +580,10 @@ impl LifecycleCalculate for LifecycleExpiration {
if !obj.is_latest || !obj.delete_marker {
return None;
}
Some(expected_expiry_time(obj.mod_time.unwrap(), self.days.unwrap()))
match self.days {
Some(days) => Some(expected_expiry_time(obj.mod_time.unwrap(), days)),
None => None,
}
}
}
@@ -591,10 +593,16 @@ impl LifecycleCalculate for NoncurrentVersionTransition {
if obj.is_latest || self.storage_class.is_none() {
return None;
}
if self.noncurrent_days.is_none() {
return obj.successor_mod_time;
match self.noncurrent_days {
Some(noncurrent_days) => {
if let Some(successor_mod_time) = obj.successor_mod_time {
Some(expected_expiry_time(successor_mod_time, noncurrent_days))
} else {
Some(expected_expiry_time(OffsetDateTime::now_utc(), noncurrent_days))
}
}
None => obj.successor_mod_time,
}
Some(expected_expiry_time(obj.successor_mod_time.unwrap(), self.noncurrent_days.unwrap()))
}
}
@@ -609,10 +617,10 @@ impl LifecycleCalculate for Transition {
return Some(date.into());
}
if self.days.is_none() {
return obj.mod_time;
match self.days {
Some(days) => Some(expected_expiry_time(obj.mod_time.unwrap(), days)),
None => obj.mod_time,
}
Some(expected_expiry_time(obj.mod_time.unwrap(), self.days.unwrap()))
}
}
@@ -692,7 +700,7 @@ pub struct TransitionOptions {
pub status: String,
pub tier: String,
pub etag: String,
pub restore_request: RestoreObjectRequest,
pub restore_request: RestoreRequest,
pub restore_expiry: OffsetDateTime,
pub expire_restored: bool,
}
crates/ecstore/src/client/api_restore.rs
+9 -65
View File
@@ -18,28 +18,23 @@
#![allow(unused_must_use)]
#![allow(clippy::all)]
use bytes::Bytes;
use http::HeaderMap;
use std::collections::HashMap;
use std::io::Cursor;
use tokio::io::BufReader;
use crate::client::{
api_error_response::{err_invalid_argument, http_resp_to_error_response},
api_get_object_acl::AccessControlList,
api_get_options::GetObjectOptions,
transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info},
};
use bytes::Bytes;
use http::HeaderMap;
use s3s::dto::RestoreRequest;
use std::collections::HashMap;
use std::io::Cursor;
use tokio::io::BufReader;
const TIER_STANDARD: &str = "Standard";
const TIER_BULK: &str = "Bulk";
const TIER_EXPEDITED: &str = "Expedited";
#[derive(Debug, Default, serde::Serialize)]
pub struct GlacierJobParameters {
pub tier: String,
}
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
pub struct Encryption {
pub encryption_type: String,
@@ -65,58 +60,6 @@ pub struct S3 {
pub user_metadata: MetadataEntry,
}
#[derive(Debug, Default, serde::Serialize)]
pub struct SelectParameters {
pub expression_type: String,
pub expression: String,
//input_serialization: SelectObjectInputSerialization,
//output_serialization: SelectObjectOutputSerialization,
}
#[derive(Debug, Default, serde::Serialize)]
pub struct OutputLocation(pub S3);
#[derive(Debug, Default, serde::Serialize)]
pub struct RestoreRequest {
pub restore_type: String,
pub tier: String,
pub days: i64,
pub glacier_job_parameters: GlacierJobParameters,
pub description: String,
pub select_parameters: SelectParameters,
pub output_location: OutputLocation,
}
impl RestoreRequest {
fn set_days(&mut self, v: i64) {
self.days = v;
}
fn set_glacier_job_parameters(&mut self, v: GlacierJobParameters) {
self.glacier_job_parameters = v;
}
fn set_type(&mut self, v: &str) {
self.restore_type = v.to_string();
}
fn set_tier(&mut self, v: &str) {
self.tier = v.to_string();
}
fn set_description(&mut self, v: &str) {
self.description = v.to_string();
}
fn set_select_parameters(&mut self, v: SelectParameters) {
self.select_parameters = v;
}
fn set_output_location(&mut self, v: OutputLocation) {
self.output_location = v;
}
}
impl TransitionClient {
pub async fn restore_object(
&self,
@@ -125,12 +68,13 @@ impl TransitionClient {
version_id: &str,
restore_req: &RestoreRequest,
) -> Result<(), std::io::Error> {
let restore_request = match quick_xml::se::to_string(restore_req) {
/*let restore_request = match quick_xml::se::to_string(restore_req) {
Ok(buf) => buf,
Err(e) => {
return Err(std::io::Error::other(e));
}
};
};*/
let restore_request = "".to_string();
let restore_request_bytes = restore_request.as_bytes().to_vec();
let mut url_values = HashMap::new();
crates/ecstore/src/client/bucket_cache.rs
+17 -5
View File
@@ -27,7 +27,7 @@ use tracing::{debug, error, info};
use crate::client::{
api_error_response::{http_resp_to_error_response, to_error_response},
transition_api::{Document, TransitionClient},
transition_api::{CreateBucketConfiguration, LocationConstraint, TransitionClient},
};
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
use s3s::Body;
@@ -82,7 +82,7 @@ impl TransitionClient {
let req = self.get_bucket_location_request(bucket_name)?;
let mut resp = self.doit(req).await?;
location = process_bucket_location_response(resp, bucket_name).await?;
location = process_bucket_location_response(resp, bucket_name, &self.tier_type).await?;
{
let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
bucket_loc_cache.set(bucket_name, &location);
@@ -175,7 +175,11 @@ impl TransitionClient {
}
}
async fn process_bucket_location_response(mut resp: http::Response<Body>, bucket_name: &str) -> Result<String, std::io::Error> {
async fn process_bucket_location_response(
mut resp: http::Response<Body>,
bucket_name: &str,
tier_type: &str,
) -> Result<String, std::io::Error> {
//if resp != nil {
if resp.status() != StatusCode::OK {
let err_resp = http_resp_to_error_response(&resp, vec![], bucket_name, "");
@@ -209,9 +213,17 @@ async fn process_bucket_location_response(mut resp: http::Response<Body>, bucket
//}
let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
let Document(location_constraint) = quick_xml::de::from_str::<Document>(&String::from_utf8(b).unwrap()).unwrap();
let mut location = "".to_string();
if tier_type == "huaweicloud" {
let d = quick_xml::de::from_str::<CreateBucketConfiguration>(&String::from_utf8(b).unwrap()).unwrap();
location = d.location_constraint;
} else {
if let Ok(LocationConstraint { field }) = quick_xml::de::from_str::<LocationConstraint>(&String::from_utf8(b).unwrap()) {
location = field;
}
}
//debug!("location: {}", location);
let mut location = location_constraint;
if location == "" {
location = "us-east-1".to_string();
}
crates/ecstore/src/client/object_api_utils.rs
+16 -11
View File
@@ -21,6 +21,7 @@
use http::HeaderMap;
use s3s::dto::ETag;
use std::pin::Pin;
use std::{collections::HashMap, io::Cursor, sync::Arc};
use tokio::io::BufReader;
@@ -54,7 +55,7 @@ impl PutObjReader {
}
}
pub type ObjReaderFn = Arc<dyn Fn(BufReader<Cursor<Vec<u8>>>, HeaderMap) -> GetObjectReader + 'static>;
pub type ObjReaderFn<'a> = Arc<dyn Fn(BufReader<Cursor<Vec<u8>>>, HeaderMap) -> GetObjectReader + Send + Sync + 'a>;
fn part_number_to_rangespec(oi: ObjectInfo, part_number: usize) -> Option<HTTPRangeSpec> {
if oi.size == 0 || oi.parts.len() == 0 {
@@ -108,19 +109,24 @@ fn get_compressed_offsets(oi: ObjectInfo, offset: i64) -> (i64, i64, i64, i64, u
(compressed_offset, part_skip, first_part_idx, decrypt_skip, seq_num)
}
pub fn new_getobjectreader(
rs: HTTPRangeSpec,
oi: &ObjectInfo,
pub fn new_getobjectreader<'a>(
rs: &Option<HTTPRangeSpec>,
oi: &'a ObjectInfo,
opts: &ObjectOptions,
h: &HeaderMap,
) -> Result<(ObjReaderFn, i64, i64), ErrorResponse> {
_h: &HeaderMap,
) -> Result<(ObjReaderFn<'a>, i64, i64), ErrorResponse> {
//let (_, mut is_encrypted) = crypto.is_encrypted(oi.user_defined)?;
let mut is_encrypted = false;
let is_compressed = false; //oi.is_compressed_ok();
let mut rs_ = None;
if rs.is_none() && opts.part_number.is_some() && opts.part_number.unwrap() > 0 {
rs_ = part_number_to_rangespec(oi.clone(), opts.part_number.unwrap());
}
let mut get_fn: ObjReaderFn;
let (off, length) = match rs.get_offset_length(oi.size) {
let (off, length) = match rs_.unwrap().get_offset_length(oi.size) {
Ok(x) => x,
Err(err) => {
return Err(ErrorResponse {
@@ -136,12 +142,11 @@ pub fn new_getobjectreader(
};
get_fn = Arc::new(move |input_reader: BufReader<Cursor<Vec<u8>>>, _: HeaderMap| {
//Box::pin({
/*let r = GetObjectReader {
let r = GetObjectReader {
object_info: oi.clone(),
stream: StreamingBlob::new(HashReader::new(input_reader, 10, None, None, 10)),
stream: Box::new(input_reader),
};
r*/
todo!();
r
//})
});
crates/ecstore/src/client/transition_api.rs
+25 -10
View File
@@ -44,7 +44,7 @@ use std::{
use time::Duration;
use time::OffsetDateTime;
use tokio::io::BufReader;
use tracing::{debug, error};
use tracing::{debug, error, warn};
use url::{Url, form_urlencoded};
use uuid::Uuid;
@@ -109,6 +109,7 @@ pub struct TransitionClient {
pub health_status: AtomicI32,
pub trailing_header_support: bool,
pub max_retries: i64,
pub tier_type: String,
}
#[derive(Debug, Default)]
@@ -132,13 +133,13 @@ pub enum BucketLookupType {
}
impl TransitionClient {
pub async fn new(endpoint: &str, opts: Options) -> Result<TransitionClient, std::io::Error> {
let clnt = Self::private_new(endpoint, opts).await?;
pub async fn new(endpoint: &str, opts: Options, tier_type: &str) -> Result<TransitionClient, std::io::Error> {
let clnt = Self::private_new(endpoint, opts, tier_type).await?;
Ok(clnt)
}
async fn private_new(endpoint: &str, opts: Options) -> Result<TransitionClient, std::io::Error> {
async fn private_new(endpoint: &str, opts: Options, tier_type: &str) -> Result<TransitionClient, std::io::Error> {
let endpoint_url = get_endpoint_url(endpoint, opts.secure)?;
//#[cfg(feature = "ring")]
@@ -175,6 +176,7 @@ impl TransitionClient {
health_status: AtomicI32::new(C_UNKNOWN),
trailing_header_support: opts.trailing_headers,
max_retries: opts.max_retries,
tier_type: tier_type.to_string(),
};
{
@@ -283,11 +285,14 @@ impl TransitionClient {
let mut resp = resp.unwrap();
debug!("http_resp: {:?}", resp);
//let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
//debug!("http_resp_body: {}", String::from_utf8(b).unwrap());
//if self.is_trace_enabled && !(self.trace_errors_only && resp.status() == StatusCode::OK) {
if resp.status() != StatusCode::OK {
//self.dump_http(&cloned_req, &resp)?;
let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
debug!("err_body: {}", String::from_utf8(b).unwrap());
warn!("err_body: {}", String::from_utf8(b).unwrap());
}
Ok(resp)
@@ -330,7 +335,8 @@ impl TransitionClient {
}
let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
let err_response = http_resp_to_error_response(&resp, b.clone(), &metadata.bucket_name, &metadata.object_name);
let mut err_response = http_resp_to_error_response(&resp, b.clone(), &metadata.bucket_name, &metadata.object_name);
err_response.message = format!("remote tier error: {}", err_response.message);
if self.region == "" {
match err_response.code {
@@ -380,9 +386,9 @@ impl TransitionClient {
method: &http::Method,
metadata: &mut RequestMetadata,
) -> Result<http::Request<Body>, std::io::Error> {
let location = metadata.bucket_location.clone();
let mut location = metadata.bucket_location.clone();
if location == "" && metadata.bucket_name != "" {
let location = self.get_bucket_location(&metadata.bucket_name).await?;
location = self.get_bucket_location(&metadata.bucket_name).await?;
}
let is_makebucket = metadata.object_name == "" && method == http::Method::PUT && metadata.query_values.len() == 0;
@@ -624,7 +630,7 @@ pub struct TransitionCore(pub Arc<TransitionClient>);
impl TransitionCore {
pub async fn new(endpoint: &str, opts: Options) -> Result<Self, std::io::Error> {
let client = TransitionClient::new(endpoint, opts).await?;
let client = TransitionClient::new(endpoint, opts, "").await?;
Ok(Self(Arc::new(client)))
}
@@ -997,4 +1003,13 @@ impl tower::Service<Request<Body>> for SendRequest {
}
#[derive(Serialize, Deserialize)]
pub struct Document(pub String);
pub struct LocationConstraint {
#[serde(rename = "$value")]
pub field: String,
}
#[derive(Serialize, Deserialize)]
pub struct CreateBucketConfiguration {
#[serde(rename = "LocationConstraint")]
pub location_constraint: String,
}
crates/ecstore/src/set_disk.rs
+86 -46
View File
@@ -31,13 +31,15 @@ use crate::disk::{
use crate::erasure_coding;
use crate::erasure_coding::bitrot_verify;
use crate::error::{Error, Result, is_err_version_not_found};
use crate::error::{ObjectApiError, is_err_object_not_found};
use crate::error::{GenericError, ObjectApiError, is_err_object_not_found};
use crate::global::{GLOBAL_LocalNodeName, GLOBAL_TierConfigMgr};
use crate::store_api::ListObjectVersionsInfo;
use crate::store_api::{ListPartsInfo, ObjectOptions, ObjectToDelete};
use crate::store_api::{ObjectInfoOrErr, WalkOptions};
use crate::{
bucket::lifecycle::bucket_lifecycle_ops::{gen_transition_objname, get_transitioned_object_reader, put_restore_opts},
bucket::lifecycle::bucket_lifecycle_ops::{
LifecycleOps, gen_transition_objname, get_transitioned_object_reader, put_restore_opts,
},
cache_value::metacache_set::{ListPathRawOptions, list_path_raw},
config::{GLOBAL_STORAGE_CLASS, storageclass},
disk::{
@@ -96,7 +98,7 @@ use std::{
};
use time::OffsetDateTime;
use tokio::{
io::AsyncWrite,
io::{AsyncReadExt, AsyncWrite, AsyncWriteExt, BufReader},
sync::{RwLock, broadcast},
};
use tokio::{
@@ -3419,7 +3421,7 @@ impl SetDisks {
oi.user_defined.remove(X_AMZ_RESTORE.as_str());
let version_id = oi.version_id.map(|v| v.to_string());
let obj = self
let _obj = self
.copy_object(
bucket,
object,
@@ -3435,8 +3437,7 @@ impl SetDisks {
..Default::default()
},
)
.await;
obj?;
.await?;
Ok(())
}
@@ -3536,7 +3537,10 @@ impl ObjectIO for SetDisks {
return Ok(reader);
}
// TODO: remote
if object_info.is_remote() {
let gr = get_transitioned_object_reader(bucket, object, &range, &h, &object_info, opts).await?;
return Ok(gr);
}
let (rd, wd) = tokio::io::duplex(DEFAULT_READ_BUFFER_SIZE);
@@ -4565,7 +4569,7 @@ impl StorageAPI for SetDisks {
let tgt_client = match tier_config_mgr.get_driver(&opts.transition.tier).await {
Ok(client) => client,
Err(err) => {
return Err(Error::other(err.to_string()));
return Err(Error::other(format!("remote tier error: {}", err)));
}
};
@@ -4594,10 +4598,10 @@ impl StorageAPI for SetDisks {
// Normalize ETags by removing quotes before comparison (PR #592 compatibility)
let transition_etag = rustfs_utils::path::trim_etag(&opts.transition.etag);
let stored_etag = rustfs_utils::path::trim_etag(&get_raw_etag(&fi.metadata));
if !opts.mod_time.expect("err").unix_timestamp() == fi.mod_time.as_ref().expect("err").unix_timestamp()
if opts.mod_time.expect("err").unix_timestamp() != fi.mod_time.as_ref().expect("err").unix_timestamp()
|| transition_etag != stored_etag
{
return Err(to_object_err(Error::from(DiskError::FileNotFound), vec![bucket, object]));
return Err(to_object_err(Error::other(DiskError::FileNotFound), vec![bucket, object]));
}
if fi.transition_status == TRANSITION_COMPLETE {
return Ok(());
@@ -4699,7 +4703,7 @@ impl StorageAPI for SetDisks {
}
#[tracing::instrument(level = "debug", skip(self))]
async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
async fn restore_transitioned_object(self: Arc<Self>, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
// Acquire write-lock early for the restore operation
// if !opts.no_lock {
// let guard_opt = self
@@ -4711,6 +4715,7 @@ impl StorageAPI for SetDisks {
// }
// _lock_guard = guard_opt;
// }
let self_ = self.clone();
let set_restore_header_fn = async move |oi: &mut ObjectInfo, rerr: Option<Error>| -> Result<()> {
if rerr.is_none() {
return Ok(());
@@ -4719,54 +4724,79 @@ impl StorageAPI for SetDisks {
Err(rerr.unwrap())
};
let mut oi = ObjectInfo::default();
let fi = self.get_object_fileinfo(bucket, object, opts, true).await;
let fi = self_.clone().get_object_fileinfo(bucket, object, opts, true).await;
if let Err(err) = fi {
return set_restore_header_fn(&mut oi, Some(to_object_err(err, vec![bucket, object]))).await;
}
let (actual_fi, _, _) = fi.unwrap();
oi = ObjectInfo::from_file_info(&actual_fi, bucket, object, opts.versioned || opts.version_suspended);
let ropts = put_restore_opts(bucket, object, &opts.transition.restore_request, &oi);
/*if oi.parts.len() == 1 {
let mut rs: HTTPRangeSpec;
let gr = get_transitioned_object_reader(bucket, object, rs, HeaderMap::new(), oi, opts);
//if err != nil {
// return set_restore_header_fn(&mut oi, Some(toObjectErr(err, bucket, object)));
//}
let hash_reader = HashReader::new(gr, gr.obj_info.size, "", "", gr.obj_info.size);
let p_reader = PutObjReader::new(StreamingBlob::from(Box::pin(hash_reader)), hash_reader.size());
if let Err(err) = self.put_object(bucket, object, &mut p_reader, &ropts).await {
return set_restore_header_fn(&mut oi, Some(to_object_err(err, vec![bucket, object])));
let ropts = put_restore_opts(bucket, object, &opts.transition.restore_request, &oi).await?;
if oi.parts.len() == 1 {
let rs: Option<HTTPRangeSpec> = None;
let gr = get_transitioned_object_reader(bucket, object, &rs, &HeaderMap::new(), &oi, opts).await;
if let Err(err) = gr {
return set_restore_header_fn(&mut oi, Some(to_object_err(err.into(), vec![bucket, object]))).await;
}
let gr = gr.unwrap();
let reader = BufReader::new(gr.stream);
let hash_reader = HashReader::new(
Box::new(WarpReader::new(reader)),
gr.object_info.size,
gr.object_info.size,
None,
None,
false,
)?;
let mut p_reader = PutObjReader::new(hash_reader);
if let Err(err) = self_.clone().put_object(bucket, object, &mut p_reader, &ropts).await {
return set_restore_header_fn(&mut oi, Some(to_object_err(err, vec![bucket, object]))).await;
} else {
return Ok(());
}
}
let res = self.new_multipart_upload(bucket, object, &ropts).await?;
let res = self_.clone().new_multipart_upload(bucket, object, &ropts).await?;
//if err != nil {
// return set_restore_header_fn(&mut oi, err);
// return set_restore_header_fn(&mut oi, err).await;
//}
let mut uploaded_parts: Vec<CompletePart> = vec![];
let mut rs: HTTPRangeSpec;
let gr = get_transitioned_object_reader(bucket, object, rs, HeaderMap::new(), oi, opts).await?;
//if err != nil {
// return set_restore_header_fn(&mut oi, err);
//}
let rs: Option<HTTPRangeSpec> = None;
let gr = get_transitioned_object_reader(bucket, object, &rs, &HeaderMap::new(), &oi, opts).await;
if let Err(err) = gr {
return set_restore_header_fn(&mut oi, Some(StorageError::Io(err))).await;
}
let gr = gr.unwrap();
for part_info in oi.parts {
//let hr = HashReader::new(LimitReader(gr, part_info.size), part_info.size, "", "", part_info.size);
let hr = HashReader::new(gr, part_info.size as i64, part_info.size as i64, None, false);
//if err != nil {
// return set_restore_header_fn(&mut oi, err);
//}
let mut p_reader = PutObjReader::new(hr, hr.size());
let p_info = self.put_object_part(bucket, object, &res.upload_id, part_info.number, &mut p_reader, &ObjectOptions::default()).await?;
for part_info in &oi.parts {
let reader = BufReader::new(Cursor::new(vec![] /*gr.stream*/));
let hash_reader = HashReader::new(
Box::new(WarpReader::new(reader)),
part_info.size as i64,
part_info.size as i64,
None,
None,
false,
)?;
let mut p_reader = PutObjReader::new(hash_reader);
let p_info = self_
.clone()
.put_object_part(bucket, object, &res.upload_id, part_info.number, &mut p_reader, &ObjectOptions::default())
.await?;
//if let Err(err) = p_info {
// return set_restore_header_fn(&mut oi, err);
// return set_restore_header_fn(&mut oi, err).await;
//}
if p_info.size != part_info.size {
return set_restore_header_fn(&mut oi, Some(Error::from(ObjectApiError::InvalidObjectState(GenericError{bucket: bucket.to_string(), object: object.to_string(), ..Default::default()}))));
return set_restore_header_fn(
&mut oi,
Some(Error::other(ObjectApiError::InvalidObjectState(GenericError {
bucket: bucket.to_string(),
object: object.to_string(),
..Default::default()
}))),
)
.await;
}
uploaded_parts.push(CompletePart {
part_num: p_info.part_num,
@@ -4778,12 +4808,22 @@ impl StorageAPI for SetDisks {
checksum_crc64nvme: None,
});
}
if let Err(err) = self.complete_multipart_upload(bucket, object, &res.upload_id, uploaded_parts, &ObjectOptions {
mod_time: oi.mod_time,
..Default::default()
}).await {
set_restore_header_fn(&mut oi, Some(err));
}*/
if let Err(err) = self_
.clone()
.complete_multipart_upload(
bucket,
object,
&res.upload_id,
uploaded_parts,
&ObjectOptions {
mod_time: oi.mod_time,
..Default::default()
},
)
.await
{
return set_restore_header_fn(&mut oi, Some(err)).await;
}
Ok(())
}
crates/ecstore/src/sets.rs
+1 -1
View File
@@ -646,7 +646,7 @@ impl StorageAPI for Sets {
}
#[tracing::instrument(skip(self))]
async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
async fn restore_transitioned_object(self: Arc<Self>, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
self.get_disks_by_key(object)
.restore_transitioned_object(bucket, object, opts)
.await
crates/ecstore/src/store.rs
+6 -3
View File
@@ -1864,17 +1864,20 @@ impl StorageAPI for ECStore {
}
#[tracing::instrument(skip(self))]
async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
async fn restore_transitioned_object(self: Arc<Self>, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
let object = encode_dir_object(object);
if self.single_pool() {
return self.pools[0].restore_transitioned_object(bucket, &object, opts).await;
return self.pools[0].clone().restore_transitioned_object(bucket, &object, opts).await;
}
//opts.skip_decommissioned = true;
//opts.nolock = true;
let idx = self.get_pool_idx_existing_with_opts(bucket, object.as_str(), opts).await?;
self.pools[idx].restore_transitioned_object(bucket, &object, opts).await
self.pools[idx]
.clone()
.restore_transitioned_object(bucket, &object, opts)
.await
}
#[tracing::instrument(skip(self))]
crates/ecstore/src/store_api.rs
+1 -1
View File
@@ -1325,7 +1325,7 @@ pub trait StorageAPI: ObjectIO + Debug {
async fn get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<String>;
async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()>;
async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()>;
async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()>;
async fn restore_transitioned_object(self: Arc<Self>, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()>;
async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result<ObjectInfo>;
async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo>;
crates/ecstore/src/tier/mod.rs
+7
View File
@@ -18,6 +18,13 @@ pub mod tier_config;
pub mod tier_gen;
pub mod tier_handlers;
pub mod warm_backend;
pub mod warm_backend_aliyun;
pub mod warm_backend_azure;
pub mod warm_backend_gcs;
pub mod warm_backend_huaweicloud;
pub mod warm_backend_minio;
pub mod warm_backend_r2;
pub mod warm_backend_rustfs;
pub mod warm_backend_s3;
pub mod warm_backend_s3sdk;
pub mod warm_backend_tencent;
crates/ecstore/src/tier/tier.rs
+57 -10
View File
@@ -141,8 +141,8 @@ impl TierConfigMgr {
(TierType::Unsupported, false)
}
pub async fn add(&mut self, tier: TierConfig, force: bool) -> std::result::Result<(), AdminError> {
let tier_name = &tier.name;
pub async fn add(&mut self, tier_config: TierConfig, force: bool) -> std::result::Result<(), AdminError> {
let tier_name = &tier_config.name;
if tier_name != tier_name.to_uppercase().as_str() {
return Err(ERR_TIER_NAME_NOT_UPPERCASE.clone());
}
@@ -152,7 +152,7 @@ impl TierConfigMgr {
return Err(ERR_TIER_ALREADY_EXISTS.clone());
}
let d = new_warm_backend(&tier, true).await?;
let d = new_warm_backend(&tier_config, true).await?;
if !force {
let in_use = d.in_use().await;
@@ -180,7 +180,7 @@ impl TierConfigMgr {
}
self.driver_cache.insert(tier_name.to_string(), d);
self.tiers.insert(tier_name.to_string(), tier);
self.tiers.insert(tier_name.to_string(), tier_config);
Ok(())
}
@@ -260,10 +260,10 @@ impl TierConfigMgr {
return Err(ERR_TIER_NOT_FOUND.clone());
}
let mut cfg = self.tiers[tier_name].clone();
let mut tier_config = self.tiers[tier_name].clone();
match tier_type {
TierType::S3 => {
let mut s3 = cfg.s3.as_mut().expect("err");
let mut s3 = tier_config.s3.as_mut().expect("err");
if creds.aws_role {
s3.aws_role = true
}
@@ -277,7 +277,7 @@ impl TierConfigMgr {
}
}
TierType::RustFS => {
let mut rustfs = cfg.rustfs.as_mut().expect("err");
let mut rustfs = tier_config.rustfs.as_mut().expect("err");
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
@@ -285,18 +285,65 @@ impl TierConfigMgr {
rustfs.secret_key = creds.secret_key;
}
TierType::MinIO => {
let mut minio = cfg.minio.as_mut().expect("err");
let mut minio = tier_config.minio.as_mut().expect("err");
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
minio.access_key = creds.access_key;
minio.secret_key = creds.secret_key;
}
TierType::Aliyun => {
let mut aliyun = tier_config.aliyun.as_mut().expect("err");
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
aliyun.access_key = creds.access_key;
aliyun.secret_key = creds.secret_key;
}
TierType::Tencent => {
let mut tencent = tier_config.tencent.as_mut().expect("err");
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
tencent.access_key = creds.access_key;
tencent.secret_key = creds.secret_key;
}
TierType::Huaweicloud => {
let mut huaweicloud = tier_config.huaweicloud.as_mut().expect("err");
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
huaweicloud.access_key = creds.access_key;
huaweicloud.secret_key = creds.secret_key;
}
TierType::Azure => {
let mut azure = tier_config.azure.as_mut().expect("err");
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
azure.access_key = creds.access_key;
azure.secret_key = creds.secret_key;
}
TierType::GCS => {
let mut gcs = tier_config.gcs.as_mut().expect("err");
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
gcs.creds = creds.access_key; //creds.creds_json
}
TierType::R2 => {
let mut r2 = tier_config.r2.as_mut().expect("err");
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
r2.access_key = creds.access_key;
r2.secret_key = creds.secret_key;
}
_ => (),
}
let d = new_warm_backend(&cfg, true).await?;
self.tiers.insert(tier_name.to_string(), cfg);
let d = new_warm_backend(&tier_config, true).await?;
self.tiers.insert(tier_name.to_string(), tier_config);
self.driver_cache.insert(tier_name.to_string(), d);
Ok(())
}
crates/ecstore/src/tier/tier_config.rs
+269 -8
View File
@@ -26,14 +26,22 @@ pub enum TierType {
Unsupported,
#[serde(rename = "s3")]
S3,
#[serde(rename = "azure")]
Azure,
#[serde(rename = "gcs")]
GCS,
#[serde(rename = "rustfs")]
RustFS,
#[serde(rename = "minio")]
MinIO,
#[serde(rename = "aliyun")]
Aliyun,
#[serde(rename = "tencent")]
Tencent,
#[serde(rename = "huaweicloud")]
Huaweicloud,
#[serde(rename = "azure")]
Azure,
#[serde(rename = "gcs")]
GCS,
#[serde(rename = "r2")]
R2,
}
impl Display for TierType {
@@ -48,6 +56,24 @@ impl Display for TierType {
TierType::MinIO => {
write!(f, "MinIO")
}
TierType::Aliyun => {
write!(f, "Aliyun")
}
TierType::Tencent => {
write!(f, "Tencent")
}
TierType::Huaweicloud => {
write!(f, "Huaweicloud")
}
TierType::Azure => {
write!(f, "Azure")
}
TierType::GCS => {
write!(f, "GCS")
}
TierType::R2 => {
write!(f, "R2")
}
_ => {
write!(f, "Unsupported")
}
@@ -61,6 +87,12 @@ impl TierType {
"S3" => TierType::S3,
"RustFS" => TierType::RustFS,
"MinIO" => TierType::MinIO,
"Aliyun" => TierType::Aliyun,
"Tencent" => TierType::Tencent,
"Huaweicloud" => TierType::Huaweicloud,
"Azure" => TierType::Azure,
"GCS" => TierType::GCS,
"R2" => TierType::R2,
_ => TierType::Unsupported,
}
}
@@ -70,6 +102,12 @@ impl TierType {
TierType::S3 => "s3".to_string(),
TierType::RustFS => "rustfs".to_string(),
TierType::MinIO => "minio".to_string(),
TierType::Aliyun => "aliyun".to_string(),
TierType::Tencent => "tencent".to_string(),
TierType::Huaweicloud => "huaweicloud".to_string(),
TierType::Azure => "azure".to_string(),
TierType::GCS => "gcs".to_string(),
TierType::R2 => "r2".to_string(),
_ => "unsupported".to_string(),
}
}
@@ -86,8 +124,18 @@ pub struct TierConfig {
pub name: String,
#[serde(rename = "s3", skip_serializing_if = "Option::is_none")]
pub s3: Option<TierS3>,
//TODO: azure: Option<TierAzure>,
//TODO: gcs: Option<TierGCS>,
#[serde(rename = "aliyun", skip_serializing_if = "Option::is_none")]
pub aliyun: Option<TierAliyun>,
#[serde(rename = "tencent", skip_serializing_if = "Option::is_none")]
pub tencent: Option<TierTencent>,
#[serde(rename = "huaweicloud", skip_serializing_if = "Option::is_none")]
pub huaweicloud: Option<TierHuaweicloud>,
#[serde(rename = "azure", skip_serializing_if = "Option::is_none")]
pub azure: Option<TierAzure>,
#[serde(rename = "gcs", skip_serializing_if = "Option::is_none")]
pub gcs: Option<TierGCS>,
#[serde(rename = "r2", skip_serializing_if = "Option::is_none")]
pub r2: Option<TierR2>,
#[serde(rename = "rustfs", skip_serializing_if = "Option::is_none")]
pub rustfs: Option<TierRustFS>,
#[serde(rename = "minio", skip_serializing_if = "Option::is_none")]
@@ -97,10 +145,14 @@ pub struct TierConfig {
impl Clone for TierConfig {
fn clone(&self) -> TierConfig {
let mut s3 = None;
//az TierAzure
//gcs TierGCS
let mut r = None;
let mut m = None;
let mut aliyun = None;
let mut tencent = None;
let mut huaweicloud = None;
let mut azure = None;
let mut gcs = None;
let mut r2 = None;
match self.tier_type {
TierType::S3 => {
let mut s3_ = self.s3.as_ref().expect("err").clone();
@@ -117,6 +169,36 @@ impl Clone for TierConfig {
m_.secret_key = "REDACTED".to_string();
m = Some(m_);
}
TierType::Aliyun => {
let mut aliyun_ = self.aliyun.as_ref().expect("err").clone();
aliyun_.secret_key = "REDACTED".to_string();
aliyun = Some(aliyun_);
}
TierType::Tencent => {
let mut tencent_ = self.tencent.as_ref().expect("err").clone();
tencent_.secret_key = "REDACTED".to_string();
tencent = Some(tencent_);
}
TierType::Huaweicloud => {
let mut huaweicloud_ = self.huaweicloud.as_ref().expect("err").clone();
huaweicloud_.secret_key = "REDACTED".to_string();
huaweicloud = Some(huaweicloud_);
}
TierType::Azure => {
let mut azure_ = self.azure.as_ref().expect("err").clone();
azure_.secret_key = "REDACTED".to_string();
azure = Some(azure_);
}
TierType::GCS => {
let mut gcs_ = self.gcs.as_ref().expect("err").clone();
gcs_.creds = "REDACTED".to_string();
gcs = Some(gcs_);
}
TierType::R2 => {
let mut r2_ = self.r2.as_ref().expect("err").clone();
r2_.secret_key = "REDACTED".to_string();
r2 = Some(r2_);
}
_ => (),
}
TierConfig {
@@ -126,6 +208,12 @@ impl Clone for TierConfig {
s3,
rustfs: r,
minio: m,
aliyun,
tencent,
huaweicloud,
azure,
gcs,
r2,
}
}
}
@@ -137,6 +225,12 @@ impl TierConfig {
TierType::S3 => self.s3.as_ref().expect("err").endpoint.clone(),
TierType::RustFS => self.rustfs.as_ref().expect("err").endpoint.clone(),
TierType::MinIO => self.minio.as_ref().expect("err").endpoint.clone(),
TierType::Aliyun => self.aliyun.as_ref().expect("err").endpoint.clone(),
TierType::Tencent => self.tencent.as_ref().expect("err").endpoint.clone(),
TierType::Huaweicloud => self.huaweicloud.as_ref().expect("err").endpoint.clone(),
TierType::Azure => self.azure.as_ref().expect("err").endpoint.clone(),
TierType::GCS => self.gcs.as_ref().expect("err").endpoint.clone(),
TierType::R2 => self.r2.as_ref().expect("err").endpoint.clone(),
_ => {
info!("unexpected tier type {}", self.tier_type);
"".to_string()
@@ -149,6 +243,12 @@ impl TierConfig {
TierType::S3 => self.s3.as_ref().expect("err").bucket.clone(),
TierType::RustFS => self.rustfs.as_ref().expect("err").bucket.clone(),
TierType::MinIO => self.minio.as_ref().expect("err").bucket.clone(),
TierType::Aliyun => self.aliyun.as_ref().expect("err").bucket.clone(),
TierType::Tencent => self.tencent.as_ref().expect("err").bucket.clone(),
TierType::Huaweicloud => self.huaweicloud.as_ref().expect("err").bucket.clone(),
TierType::Azure => self.azure.as_ref().expect("err").bucket.clone(),
TierType::GCS => self.gcs.as_ref().expect("err").bucket.clone(),
TierType::R2 => self.r2.as_ref().expect("err").bucket.clone(),
_ => {
info!("unexpected tier type {}", self.tier_type);
"".to_string()
@@ -161,6 +261,12 @@ impl TierConfig {
TierType::S3 => self.s3.as_ref().expect("err").prefix.clone(),
TierType::RustFS => self.rustfs.as_ref().expect("err").prefix.clone(),
TierType::MinIO => self.minio.as_ref().expect("err").prefix.clone(),
TierType::Aliyun => self.aliyun.as_ref().expect("err").prefix.clone(),
TierType::Tencent => self.tencent.as_ref().expect("err").prefix.clone(),
TierType::Huaweicloud => self.huaweicloud.as_ref().expect("err").prefix.clone(),
TierType::Azure => self.azure.as_ref().expect("err").prefix.clone(),
TierType::GCS => self.gcs.as_ref().expect("err").prefix.clone(),
TierType::R2 => self.r2.as_ref().expect("err").prefix.clone(),
_ => {
info!("unexpected tier type {}", self.tier_type);
"".to_string()
@@ -173,6 +279,12 @@ impl TierConfig {
TierType::S3 => self.s3.as_ref().expect("err").region.clone(),
TierType::RustFS => self.rustfs.as_ref().expect("err").region.clone(),
TierType::MinIO => self.minio.as_ref().expect("err").region.clone(),
TierType::Aliyun => self.aliyun.as_ref().expect("err").region.clone(),
TierType::Tencent => self.tencent.as_ref().expect("err").region.clone(),
TierType::Huaweicloud => self.huaweicloud.as_ref().expect("err").region.clone(),
TierType::Azure => self.azure.as_ref().expect("err").region.clone(),
TierType::GCS => self.gcs.as_ref().expect("err").region.clone(),
TierType::R2 => self.r2.as_ref().expect("err").region.clone(),
_ => {
info!("unexpected tier type {}", self.tier_type);
"".to_string()
@@ -319,3 +431,152 @@ impl TierMinIO {
})
}
}
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[serde(default)]
pub struct TierAliyun {
pub name: String,
pub endpoint: String,
#[serde(rename = "accessKey")]
pub access_key: String,
#[serde(rename = "secretKey")]
pub secret_key: String,
pub bucket: String,
pub prefix: String,
pub region: String,
}
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[serde(default)]
pub struct TierTencent {
pub name: String,
pub endpoint: String,
#[serde(rename = "accessKey")]
pub access_key: String,
#[serde(rename = "secretKey")]
pub secret_key: String,
pub bucket: String,
pub prefix: String,
pub region: String,
}
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[serde(default)]
pub struct TierHuaweicloud {
pub name: String,
pub endpoint: String,
#[serde(rename = "accessKey")]
pub access_key: String,
#[serde(rename = "secretKey")]
pub secret_key: String,
pub bucket: String,
pub prefix: String,
pub region: String,
}
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[serde(default)]
pub struct ServicePrincipalAuth {
pub tenant_id: String,
pub client_id: String,
pub client_secret: String,
}
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[serde(default)]
pub struct TierAzure {
pub name: String,
pub endpoint: String,
#[serde(rename = "accessKey")]
pub access_key: String,
#[serde(rename = "secretKey")]
pub secret_key: String,
pub bucket: String,
pub prefix: String,
pub region: String,
#[serde(rename = "storageClass")]
pub storage_class: String,
#[serde(rename = "spAuth")]
pub sp_auth: ServicePrincipalAuth,
}
impl TierAzure {
pub fn is_sp_enabled(&self) -> bool {
!self.sp_auth.tenant_id.is_empty() && !self.sp_auth.client_id.is_empty() && !self.sp_auth.client_secret.is_empty()
}
}
/*
fn AzureServicePrincipal(tenantID, clientID, clientSecret string) func(az *TierAzure) error {
return func(az *TierAzure) error {
if tenantID == "" {
return errors.New("empty tenant ID unsupported")
}
if clientID == "" {
return errors.New("empty client ID unsupported")
}
if clientSecret == "" {
return errors.New("empty client secret unsupported")
}
az.SPAuth.TenantID = tenantID
az.SPAuth.ClientID = clientID
az.SPAuth.ClientSecret = clientSecret
return nil
}
}
fn AzurePrefix(prefix string) func(az *TierAzure) error {
return func(az *TierAzure) error {
az.Prefix = prefix
return nil
}
}
fn AzureEndpoint(endpoint string) func(az *TierAzure) error {
return func(az *TierAzure) error {
az.Endpoint = endpoint
return nil
}
}
fn AzureRegion(region string) func(az *TierAzure) error {
return func(az *TierAzure) error {
az.Region = region
return nil
}
}
fn AzureStorageClass(sc string) func(az *TierAzure) error {
return func(az *TierAzure) error {
az.StorageClass = sc
return nil
}
}*/
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[serde(default)]
pub struct TierGCS {
pub name: String,
pub endpoint: String,
#[serde(rename = "creds")]
pub creds: String,
pub bucket: String,
pub prefix: String,
pub region: String,
#[serde(rename = "storageClass")]
pub storage_class: String,
}
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[serde(default)]
pub struct TierR2 {
pub name: String,
pub endpoint: String,
#[serde(rename = "accessKey")]
pub access_key: String,
#[serde(rename = "secretKey")]
pub secret_key: String,
pub bucket: String,
pub prefix: String,
pub region: String,
}
crates/ecstore/src/tier/warm_backend.rs
+78
View File
@@ -27,9 +27,15 @@ use crate::tier::{
tier::ERR_TIER_TYPE_UNSUPPORTED,
tier_config::{TierConfig, TierType},
tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_PERM_ERR},
warm_backend_aliyun::WarmBackendAliyun,
warm_backend_azure::WarmBackendAzure,
warm_backend_gcs::WarmBackendGCS,
warm_backend_huaweicloud::WarmBackendHuaweicloud,
warm_backend_minio::WarmBackendMinIO,
warm_backend_r2::WarmBackendR2,
warm_backend_rustfs::WarmBackendRustFS,
warm_backend_s3::WarmBackendS3,
warm_backend_tencent::WarmBackendTencent,
};
use bytes::Bytes;
use http::StatusCode;
@@ -128,6 +134,78 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
}
d = Some(Box::new(dd.expect("err")));
}
TierType::Aliyun => {
let dd = WarmBackendAliyun::new(tier.aliyun.as_ref().expect("err"), &tier.name).await;
if let Err(err) = dd {
warn!("{}", err);
return Err(AdminError {
code: "XRustFSAdminTierInvalidConfig".to_string(),
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
status_code: StatusCode::BAD_REQUEST,
});
}
d = Some(Box::new(dd.expect("err")));
}
TierType::Tencent => {
let dd = WarmBackendTencent::new(tier.tencent.as_ref().expect("err"), &tier.name).await;
if let Err(err) = dd {
warn!("{}", err);
return Err(AdminError {
code: "XRustFSAdminTierInvalidConfig".to_string(),
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
status_code: StatusCode::BAD_REQUEST,
});
}
d = Some(Box::new(dd.expect("err")));
}
TierType::Huaweicloud => {
let dd = WarmBackendHuaweicloud::new(tier.huaweicloud.as_ref().expect("err"), &tier.name).await;
if let Err(err) = dd {
warn!("{}", err);
return Err(AdminError {
code: "XRustFSAdminTierInvalidConfig".to_string(),
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
status_code: StatusCode::BAD_REQUEST,
});
}
d = Some(Box::new(dd.expect("err")));
}
TierType::Azure => {
let dd = WarmBackendAzure::new(tier.azure.as_ref().expect("err"), &tier.name).await;
if let Err(err) = dd {
warn!("{}", err);
return Err(AdminError {
code: "XRustFSAdminTierInvalidConfig".to_string(),
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
status_code: StatusCode::BAD_REQUEST,
});
}
d = Some(Box::new(dd.expect("err")));
}
TierType::GCS => {
let dd = WarmBackendGCS::new(tier.gcs.as_ref().expect("err"), &tier.name).await;
if let Err(err) = dd {
warn!("{}", err);
return Err(AdminError {
code: "XRustFSAdminTierInvalidConfig".to_string(),
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
status_code: StatusCode::BAD_REQUEST,
});
}
d = Some(Box::new(dd.expect("err")));
}
TierType::R2 => {
let dd = WarmBackendR2::new(tier.r2.as_ref().expect("err"), &tier.name).await;
if let Err(err) = dd {
warn!("{}", err);
return Err(AdminError {
code: "XRustFSAdminTierInvalidConfig".to_string(),
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
status_code: StatusCode::BAD_REQUEST,
});
}
d = Some(Box::new(dd.expect("err")));
}
_ => {
return Err(ERR_TIER_TYPE_UNSUPPORTED.clone());
}
crates/ecstore/src/tier/warm_backend_aliyun.rs
+164
View File
@@ -0,0 +1,164 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap;
use std::sync::Arc;
use crate::client::{
admin_handler_utils::AdminError,
api_put_object::PutObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
};
use crate::tier::{
tier_config::TierAliyun,
warm_backend::{WarmBackend, WarmBackendGetOpts},
warm_backend_s3::WarmBackendS3,
};
use tracing::warn;
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
const MAX_PARTS_COUNT: i64 = 10000;
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
pub struct WarmBackendAliyun(WarmBackendS3);
impl WarmBackendAliyun {
pub async fn new(conf: &TierAliyun, tier: &str) -> Result<Self, std::io::Error> {
if conf.access_key == "" || conf.secret_key == "" {
return Err(std::io::Error::other("both access and secret keys are required"));
}
if conf.bucket == "" {
return Err(std::io::Error::other("no bucket name was provided"));
}
let u = match url::Url::parse(&conf.endpoint) {
Ok(u) => u,
Err(e) => {
return Err(std::io::Error::other(e.to_string()));
}
};
let creds = Credentials::new(Static(Value {
access_key_id: conf.access_key.clone(),
secret_access_key: conf.secret_key.clone(),
session_token: "".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
}));
let opts = Options {
creds,
secure: u.scheme() == "https",
//transport: GLOBAL_RemoteTargetTransport,
trailing_headers: true,
region: conf.region.clone(),
bucket_lookup: BucketLookupType::BucketLookupDNS,
..Default::default()
};
let scheme = u.scheme();
let default_port = if scheme == "https" { 443 } else { 80 };
let client = TransitionClient::new(
&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)),
opts,
"aliyun",
)
.await?;
let client = Arc::new(client);
let core = TransitionCore(Arc::clone(&client));
Ok(Self(WarmBackendS3 {
client,
core,
bucket: conf.bucket.clone(),
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
storage_class: "".to_string(),
}))
}
}
#[async_trait::async_trait]
impl WarmBackend for WarmBackendAliyun {
async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
meta: HashMap<String, String>,
) -> Result<String, std::io::Error> {
let part_size = optimal_part_size(length)?;
let client = self.0.client.clone();
let res = client
.put_object(
&self.0.bucket,
&self.0.get_dest(object),
r,
length,
&PutObjectOptions {
storage_class: self.0.storage_class.clone(),
part_size: part_size as u64,
disable_content_sha256: true,
user_metadata: meta,
..Default::default()
},
)
.await?;
//self.ToObjectError(err, object)
Ok(res.version_id)
}
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
self.put_with_meta(object, r, length, HashMap::new()).await
}
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
self.0.get(object, rv, opts).await
}
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
self.0.remove(object, rv).await
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
self.0.in_use().await
}
}
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
let mut object_size = object_size;
if object_size == -1 {
object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
}
if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
return Err(std::io::Error::other("entity too large"));
}
let configured_part_size = MIN_PART_SIZE;
let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64;
part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64;
let part_size = part_size_flt as i64;
if part_size == 0 {
return Ok(MIN_PART_SIZE);
}
Ok(part_size)
}
crates/ecstore/src/tier/warm_backend_azure.rs
+164
View File
@@ -0,0 +1,164 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap;
use std::sync::Arc;
use crate::client::{
admin_handler_utils::AdminError,
api_put_object::PutObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
};
use crate::tier::{
tier_config::TierAzure,
warm_backend::{WarmBackend, WarmBackendGetOpts},
warm_backend_s3::WarmBackendS3,
};
use tracing::warn;
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
const MAX_PARTS_COUNT: i64 = 10000;
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
pub struct WarmBackendAzure(WarmBackendS3);
impl WarmBackendAzure {
pub async fn new(conf: &TierAzure, tier: &str) -> Result<Self, std::io::Error> {
if conf.access_key == "" || conf.secret_key == "" {
return Err(std::io::Error::other("both access and secret keys are required"));
}
if conf.bucket == "" {
return Err(std::io::Error::other("no bucket name was provided"));
}
let u = match url::Url::parse(&conf.endpoint) {
Ok(u) => u,
Err(e) => {
return Err(std::io::Error::other(e.to_string()));
}
};
let creds = Credentials::new(Static(Value {
access_key_id: conf.access_key.clone(),
secret_access_key: conf.secret_key.clone(),
session_token: "".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
}));
let opts = Options {
creds,
secure: u.scheme() == "https",
//transport: GLOBAL_RemoteTargetTransport,
trailing_headers: true,
region: conf.region.clone(),
bucket_lookup: BucketLookupType::BucketLookupDNS,
..Default::default()
};
let scheme = u.scheme();
let default_port = if scheme == "https" { 443 } else { 80 };
let client = TransitionClient::new(
&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)),
opts,
"azure",
)
.await?;
let client = Arc::new(client);
let core = TransitionCore(Arc::clone(&client));
Ok(Self(WarmBackendS3 {
client,
core,
bucket: conf.bucket.clone(),
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
storage_class: "".to_string(),
}))
}
}
#[async_trait::async_trait]
impl WarmBackend for WarmBackendAzure {
async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
meta: HashMap<String, String>,
) -> Result<String, std::io::Error> {
let part_size = optimal_part_size(length)?;
let client = self.0.client.clone();
let res = client
.put_object(
&self.0.bucket,
&self.0.get_dest(object),
r,
length,
&PutObjectOptions {
storage_class: self.0.storage_class.clone(),
part_size: part_size as u64,
disable_content_sha256: true,
user_metadata: meta,
..Default::default()
},
)
.await?;
//self.ToObjectError(err, object)
Ok(res.version_id)
}
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
self.put_with_meta(object, r, length, HashMap::new()).await
}
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
self.0.get(object, rv, opts).await
}
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
self.0.remove(object, rv).await
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
self.0.in_use().await
}
}
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
let mut object_size = object_size;
if object_size == -1 {
object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
}
if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
return Err(std::io::Error::other("entity too large"));
}
let configured_part_size = MIN_PART_SIZE;
let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64;
part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64;
let part_size = part_size_flt as i64;
if part_size == 0 {
return Ok(MIN_PART_SIZE);
}
Ok(part_size)
}
crates/ecstore/src/tier/warm_backend_azure2.rs
+231
View File
@@ -0,0 +1,231 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap;
use std::sync::Arc;
use azure_core::http::{Body, ClientOptions, RequestContent};
use azure_storage::StorageCredentials;
use azure_storage_blobs::prelude::*;
use crate::client::{
admin_handler_utils::AdminError,
api_put_object::PutObjectOptions,
transition_api::{Options, ReadCloser, ReaderImpl},
};
use crate::tier::{
tier_config::TierAzure,
warm_backend::{WarmBackend, WarmBackendGetOpts},
};
use tracing::warn;
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
const MAX_PARTS_COUNT: i64 = 10000;
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
pub struct WarmBackendAzure {
pub client: Arc<BlobServiceClient>,
pub bucket: String,
pub prefix: String,
pub storage_class: String,
}
impl WarmBackendAzure {
pub async fn new(conf: &TierAzure, tier: &str) -> Result<Self, std::io::Error> {
if conf.access_key == "" || conf.secret_key == "" {
return Err(std::io::Error::other("both access and secret keys are required"));
}
if conf.bucket == "" {
return Err(std::io::Error::other("no bucket name was provided"));
}
let creds = StorageCredentials::access_key(conf.access_key.clone(), conf.secret_key.clone());
let client = ClientBuilder::new(conf.access_key.clone(), creds)
//.endpoint(conf.endpoint)
.blob_service_client();
let client = Arc::new(client);
Ok(Self {
client,
bucket: conf.bucket.clone(),
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
storage_class: "".to_string(),
})
}
/*pub fn tier(&self) -> *blob.AccessTier {
if self.storage_class == "" {
return None;
}
for t in blob.PossibleAccessTierValues() {
if strings.EqualFold(self.storage_class, t) {
return &t
}
}
None
}*/
pub fn get_dest(&self, object: &str) -> String {
let mut dest_obj = object.to_string();
if self.prefix != "" {
dest_obj = format!("{}/{}", &self.prefix, object);
}
return dest_obj;
}
}
#[async_trait::async_trait]
impl WarmBackend for WarmBackendAzure {
async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
meta: HashMap<String, String>,
) -> Result<String, std::io::Error> {
let part_size = length;
let client = self.client.clone();
let container_client = client.container_client(self.bucket.clone());
let blob_client = container_client.blob_client(self.get_dest(object));
/*let res = blob_client
.upload(
RequestContent::from(match r {
ReaderImpl::Body(content_body) => content_body.to_vec(),
ReaderImpl::ObjectBody(mut content_body) => content_body.read_all().await?,
}),
false,
length as u64,
None,
)
.await
else {
return Err(std::io::Error::other("upload error"));
};*/
let Ok(res) = blob_client
.put_block_blob(match r {
ReaderImpl::Body(content_body) => content_body.to_vec(),
ReaderImpl::ObjectBody(mut content_body) => content_body.read_all().await?,
})
.content_type("text/plain")
.into_future()
.await
else {
return Err(std::io::Error::other("put_block_blob error"));
};
//self.ToObjectError(err, object)
Ok(res.request_id.to_string())
}
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
self.put_with_meta(object, r, length, HashMap::new()).await
}
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
let client = self.client.clone();
let container_client = client.container_client(self.bucket.clone());
let blob_client = container_client.blob_client(self.get_dest(object));
blob_client.get();
todo!();
}
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
let client = self.client.clone();
let container_client = client.container_client(self.bucket.clone());
let blob_client = container_client.blob_client(self.get_dest(object));
blob_client.delete();
todo!();
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
/*let result = self.client
.list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1)
.await?;
Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)*/
Ok(false)
}
}
/*fn azure_to_object_error(err: Error, params: Vec<String>) -> Option<error> {
if err == nil {
return nil
}
bucket := ""
object := ""
if len(params) >= 1 {
bucket = params[0]
}
if len(params) == 2 {
object = params[1]
}
azureErr, ok := err.(*azcore.ResponseError)
if !ok {
// We don't interpret non Azure errors. As azure errors will
// have StatusCode to help to convert to object errors.
return err
}
serviceCode := azureErr.ErrorCode
statusCode := azureErr.StatusCode
azureCodesToObjectError(err, serviceCode, statusCode, bucket, object)
}*/
/*fn azure_codes_to_object_error(err: Error, service_code: String, status_code: i32, bucket: String, object: String) -> Option<Error> {
switch serviceCode {
case "ContainerNotFound", "ContainerBeingDeleted":
err = BucketNotFound{Bucket: bucket}
case "ContainerAlreadyExists":
err = BucketExists{Bucket: bucket}
case "InvalidResourceName":
err = BucketNameInvalid{Bucket: bucket}
case "RequestBodyTooLarge":
err = PartTooBig{}
case "InvalidMetadata":
err = UnsupportedMetadata{}
case "BlobAccessTierNotSupportedForAccountType":
err = NotImplemented{}
case "OutOfRangeInput":
err = ObjectNameInvalid{
Bucket: bucket,
Object: object,
}
default:
switch statusCode {
case http.StatusNotFound:
if object != "" {
err = ObjectNotFound{
Bucket: bucket,
Object: object,
}
} else {
err = BucketNotFound{Bucket: bucket}
}
case http.StatusBadRequest:
err = BucketNameInvalid{Bucket: bucket}
}
}
return err
}*/
crates/ecstore/src/tier/warm_backend_gcs.rs
+248
View File
@@ -0,0 +1,248 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap;
use std::sync::Arc;
use bytes::Bytes;
use google_cloud_auth::credentials::Credentials;
use google_cloud_auth::credentials::user_account::Builder;
use google_cloud_storage as gcs;
use google_cloud_storage::client::Storage;
use std::convert::TryFrom;
use crate::client::{
admin_handler_utils::AdminError,
api_put_object::PutObjectOptions,
transition_api::{Options, ReadCloser, ReaderImpl},
};
use crate::tier::{
tier_config::TierGCS,
warm_backend::{WarmBackend, WarmBackendGetOpts},
};
use tracing::warn;
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
const MAX_PARTS_COUNT: i64 = 10000;
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
pub struct WarmBackendGCS {
pub client: Arc<Storage>,
pub bucket: String,
pub prefix: String,
pub storage_class: String,
}
impl WarmBackendGCS {
pub async fn new(conf: &TierGCS, tier: &str) -> Result<Self, std::io::Error> {
if conf.creds == "" {
return Err(std::io::Error::other("both access and secret keys are required"));
}
if conf.bucket == "" {
return Err(std::io::Error::other("no bucket name was provided"));
}
let authorized_user = serde_json::from_str(&conf.creds)?;
let credentials = Builder::new(authorized_user)
//.with_retry_policy(AlwaysRetry.with_attempt_limit(3))
//.with_backoff_policy(backoff)
.build()
.map_err(|e| std::io::Error::other(format!("Invalid credentials JSON: {}", e)))?;
let Ok(client) = Storage::builder()
.with_endpoint(conf.endpoint.clone())
.with_credentials(credentials)
.build()
.await
else {
return Err(std::io::Error::other("Storage::builder error"));
};
let client = Arc::new(client);
Ok(Self {
client,
bucket: conf.bucket.clone(),
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
storage_class: "".to_string(),
})
}
pub fn get_dest(&self, object: &str) -> String {
let mut dest_obj = object.to_string();
if self.prefix != "" {
dest_obj = format!("{}/{}", &self.prefix, object);
}
return dest_obj;
}
}
#[async_trait::async_trait]
impl WarmBackend for WarmBackendGCS {
async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
meta: HashMap<String, String>,
) -> Result<String, std::io::Error> {
let d = match r {
ReaderImpl::Body(content_body) => content_body.to_vec(),
ReaderImpl::ObjectBody(mut content_body) => content_body.read_all().await?,
};
let Ok(res) = self
.client
.write_object(&self.bucket, &self.get_dest(object), Bytes::from(d))
.send_buffered()
.await
else {
return Err(std::io::Error::other("write_object error"));
};
//self.ToObjectError(err, object)
Ok(res.generation.to_string())
}
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
self.put_with_meta(object, r, length, HashMap::new()).await
}
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
let Ok(mut reader) = self.client.read_object(&self.bucket, &self.get_dest(object)).send().await else {
return Err(std::io::Error::other("read_object error"));
};
let mut contents = Vec::new();
while let Ok(Some(chunk)) = reader.next().await.transpose() {
contents.extend_from_slice(&chunk);
}
Ok(ReadCloser::new(std::io::Cursor::new(contents)))
}
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
/*self.client
.delete_object()
.set_bucket(&self.bucket)
.set_object(&self.get_dest(object))
//.set_generation(object.generation)
.send()
.await?;*/
Ok(())
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
/*let result = self.client
.list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1)
.await?;
Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)*/
Ok(false)
}
}
/*fn gcs_to_object_error(err: Error, params: Vec<String>) -> Option<Error> {
if err == nil {
return nil
}
bucket := ""
object := ""
uploadID := ""
if len(params) >= 1 {
bucket = params[0]
}
if len(params) == 2 {
object = params[1]
}
if len(params) == 3 {
uploadID = params[2]
}
// in some cases just a plain error is being returned
switch err.Error() {
case "storage: bucket doesn't exist":
err = BucketNotFound{
Bucket: bucket,
}
return err
case "storage: object doesn't exist":
if uploadID != "" {
err = InvalidUploadID{
UploadID: uploadID,
}
} else {
err = ObjectNotFound{
Bucket: bucket,
Object: object,
}
}
return err
}
googleAPIErr, ok := err.(*googleapi.Error)
if !ok {
// We don't interpret non MinIO errors. As minio errors will
// have StatusCode to help to convert to object errors.
return err
}
if len(googleAPIErr.Errors) == 0 {
return err
}
reason := googleAPIErr.Errors[0].Reason
message := googleAPIErr.Errors[0].Message
switch reason {
case "required":
// Anonymous users does not have storage.xyz access to project 123.
fallthrough
case "keyInvalid":
fallthrough
case "forbidden":
err = PrefixAccessDenied{
Bucket: bucket,
Object: object,
}
case "invalid":
err = BucketNameInvalid{
Bucket: bucket,
}
case "notFound":
if object != "" {
err = ObjectNotFound{
Bucket: bucket,
Object: object,
}
break
}
err = BucketNotFound{Bucket: bucket}
case "conflict":
if message == "You already own this bucket. Please select another name." {
err = BucketAlreadyOwnedByYou{Bucket: bucket}
break
}
if message == "Sorry, that name is not available. Please try a different one." {
err = BucketAlreadyExists{Bucket: bucket}
break
}
err = BucketNotEmpty{Bucket: bucket}
}
return err
}*/
crates/ecstore/src/tier/warm_backend_huaweicloud.rs
+164
View File
@@ -0,0 +1,164 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap;
use std::sync::Arc;
use crate::client::{
admin_handler_utils::AdminError,
api_put_object::PutObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
};
use crate::tier::{
tier_config::TierHuaweicloud,
warm_backend::{WarmBackend, WarmBackendGetOpts},
warm_backend_s3::WarmBackendS3,
};
use tracing::warn;
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
const MAX_PARTS_COUNT: i64 = 10000;
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
pub struct WarmBackendHuaweicloud(WarmBackendS3);
impl WarmBackendHuaweicloud {
pub async fn new(conf: &TierHuaweicloud, tier: &str) -> Result<Self, std::io::Error> {
if conf.access_key == "" || conf.secret_key == "" {
return Err(std::io::Error::other("both access and secret keys are required"));
}
if conf.bucket == "" {
return Err(std::io::Error::other("no bucket name was provided"));
}
let u = match url::Url::parse(&conf.endpoint) {
Ok(u) => u,
Err(e) => {
return Err(std::io::Error::other(e.to_string()));
}
};
let creds = Credentials::new(Static(Value {
access_key_id: conf.access_key.clone(),
secret_access_key: conf.secret_key.clone(),
session_token: "".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
}));
let opts = Options {
creds,
secure: u.scheme() == "https",
//transport: GLOBAL_RemoteTargetTransport,
trailing_headers: true,
region: conf.region.clone(),
bucket_lookup: BucketLookupType::BucketLookupDNS,
..Default::default()
};
let scheme = u.scheme();
let default_port = if scheme == "https" { 443 } else { 80 };
let client = TransitionClient::new(
&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)),
opts,
"huaweicloud",
)
.await?;
let client = Arc::new(client);
let core = TransitionCore(Arc::clone(&client));
Ok(Self(WarmBackendS3 {
client,
core,
bucket: conf.bucket.clone(),
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
storage_class: "".to_string(),
}))
}
}
#[async_trait::async_trait]
impl WarmBackend for WarmBackendHuaweicloud {
async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
meta: HashMap<String, String>,
) -> Result<String, std::io::Error> {
let part_size = optimal_part_size(length)?;
let client = self.0.client.clone();
let res = client
.put_object(
&self.0.bucket,
&self.0.get_dest(object),
r,
length,
&PutObjectOptions {
storage_class: self.0.storage_class.clone(),
part_size: part_size as u64,
disable_content_sha256: true,
user_metadata: meta,
..Default::default()
},
)
.await?;
//self.ToObjectError(err, object)
Ok(res.version_id)
}
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
self.put_with_meta(object, r, length, HashMap::new()).await
}
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
self.0.get(object, rv, opts).await
}
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
self.0.remove(object, rv).await
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
self.0.in_use().await
}
}
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
let mut object_size = object_size;
if object_size == -1 {
object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
}
if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
return Err(std::io::Error::other("entity too large"));
}
let configured_part_size = MIN_PART_SIZE;
let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64;
part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64;
let part_size = part_size_flt as i64;
if part_size == 0 {
return Ok(MIN_PART_SIZE);
}
Ok(part_size)
}
crates/ecstore/src/tier/warm_backend_minio.rs
+8 -3
View File
@@ -1,4 +1,3 @@
#![allow(unused_imports)]
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,6 +11,7 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
@@ -70,12 +70,17 @@ impl WarmBackendMinIO {
secure: u.scheme() == "https",
//transport: GLOBAL_RemoteTargetTransport,
trailing_headers: true,
region: conf.region.clone(),
..Default::default()
};
let scheme = u.scheme();
let default_port = if scheme == "https" { 443 } else { 80 };
let client =
TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?;
let client = TransitionClient::new(
&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)),
opts,
"minio",
)
.await?;
let client = Arc::new(client);
let core = TransitionCore(Arc::clone(&client));
crates/ecstore/src/tier/warm_backend_r2.rs
+163
View File
@@ -0,0 +1,163 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap;
use std::sync::Arc;
use crate::client::{
admin_handler_utils::AdminError,
api_put_object::PutObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
};
use crate::tier::{
tier_config::TierR2,
warm_backend::{WarmBackend, WarmBackendGetOpts},
warm_backend_s3::WarmBackendS3,
};
use tracing::warn;
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
const MAX_PARTS_COUNT: i64 = 10000;
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
pub struct WarmBackendR2(WarmBackendS3);
impl WarmBackendR2 {
pub async fn new(conf: &TierR2, tier: &str) -> Result<Self, std::io::Error> {
if conf.access_key == "" || conf.secret_key == "" {
return Err(std::io::Error::other("both access and secret keys are required"));
}
if conf.bucket == "" {
return Err(std::io::Error::other("no bucket name was provided"));
}
let u = match url::Url::parse(&conf.endpoint) {
Ok(u) => u,
Err(e) => {
return Err(std::io::Error::other(e.to_string()));
}
};
let creds = Credentials::new(Static(Value {
access_key_id: conf.access_key.clone(),
secret_access_key: conf.secret_key.clone(),
session_token: "".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
}));
let opts = Options {
creds,
secure: u.scheme() == "https",
//transport: GLOBAL_RemoteTargetTransport,
trailing_headers: true,
region: conf.region.clone(),
..Default::default()
};
let scheme = u.scheme();
let default_port = if scheme == "https" { 443 } else { 80 };
let client = TransitionClient::new(
&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)),
opts,
"r2",
)
.await?;
let client = Arc::new(client);
let core = TransitionCore(Arc::clone(&client));
Ok(Self(WarmBackendS3 {
client,
core,
bucket: conf.bucket.clone(),
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
storage_class: "".to_string(),
}))
}
}
#[async_trait::async_trait]
impl WarmBackend for WarmBackendR2 {
async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
meta: HashMap<String, String>,
) -> Result<String, std::io::Error> {
let part_size = optimal_part_size(length)?;
let client = self.0.client.clone();
let res = client
.put_object(
&self.0.bucket,
&self.0.get_dest(object),
r,
length,
&PutObjectOptions {
storage_class: self.0.storage_class.clone(),
part_size: part_size as u64,
disable_content_sha256: true,
user_metadata: meta,
..Default::default()
},
)
.await?;
//self.ToObjectError(err, object)
Ok(res.version_id)
}
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
self.put_with_meta(object, r, length, HashMap::new()).await
}
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
self.0.get(object, rv, opts).await
}
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
self.0.remove(object, rv).await
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
self.0.in_use().await
}
}
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
let mut object_size = object_size;
if object_size == -1 {
object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
}
if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
return Err(std::io::Error::other("entity too large"));
}
let configured_part_size = MIN_PART_SIZE;
let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64;
part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64;
let part_size = part_size_flt as i64;
if part_size == 0 {
return Ok(MIN_PART_SIZE);
}
Ok(part_size)
}
crates/ecstore/src/tier/warm_backend_rustfs.rs
+8 -3
View File
@@ -1,4 +1,3 @@
#![allow(unused_imports)]
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,6 +11,7 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
@@ -67,12 +67,17 @@ impl WarmBackendRustFS {
secure: u.scheme() == "https",
//transport: GLOBAL_RemoteTargetTransport,
trailing_headers: true,
region: conf.region.clone(),
..Default::default()
};
let scheme = u.scheme();
let default_port = if scheme == "https" { 443 } else { 80 };
let client =
TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?;
let client = TransitionClient::new(
&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)),
opts,
"rustfs",
)
.await?;
let client = Arc::new(client);
let core = TransitionCore(Arc::clone(&client));
crates/ecstore/src/tier/warm_backend_s3.rs
+3 -2
View File
@@ -1,4 +1,3 @@
#![allow(unused_imports)]
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,6 +11,7 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
@@ -92,9 +92,10 @@ impl WarmBackendS3 {
creds,
secure: u.scheme() == "https",
//transport: GLOBAL_RemoteTargetTransport,
region: conf.region.clone(),
..Default::default()
};
let client = TransitionClient::new(&u.host().expect("err").to_string(), opts).await?;
let client = TransitionClient::new(&u.host().expect("err").to_string(), opts, "s3").await?;
let client = Arc::new(client);
let core = TransitionCore(Arc::clone(&client));
crates/ecstore/src/tier/warm_backend_s3sdk.rs
+196
View File
@@ -0,0 +1,196 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap;
use std::sync::Arc;
use url::Url;
use aws_config::meta::region::RegionProviderChain;
use aws_sdk_s3::Client;
use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::primitives::ByteStream;
use crate::client::{
api_get_options::GetObjectOptions,
api_put_object::PutObjectOptions,
api_remove::RemoveObjectOptions,
transition_api::{ReadCloser, ReaderImpl},
};
use crate::error::ErrorResponse;
use crate::error::error_resp_to_object_err;
use crate::tier::{
tier_config::TierS3,
warm_backend::{WarmBackend, WarmBackendGetOpts},
};
use rustfs_utils::path::SLASH_SEPARATOR;
pub struct WarmBackendS3 {
pub client: Arc<Client>,
pub bucket: String,
pub prefix: String,
pub storage_class: String,
}
impl WarmBackendS3 {
pub async fn new(conf: &TierS3, tier: &str) -> Result<Self, std::io::Error> {
let u = match Url::parse(&conf.endpoint) {
Ok(u) => u,
Err(err) => {
return Err(std::io::Error::other(err.to_string()));
}
};
if conf.aws_role_web_identity_token_file == "" && conf.aws_role_arn != ""
|| conf.aws_role_web_identity_token_file != "" && conf.aws_role_arn == ""
{
return Err(std::io::Error::other("both the token file and the role ARN are required"));
} else if conf.access_key == "" && conf.secret_key != "" || conf.access_key != "" && conf.secret_key == "" {
return Err(std::io::Error::other("both the access and secret keys are required"));
} else if conf.aws_role
&& (conf.aws_role_web_identity_token_file != ""
|| conf.aws_role_arn != ""
|| conf.access_key != ""
|| conf.secret_key != "")
{
return Err(std::io::Error::other(
"AWS Role cannot be activated with static credentials or the web identity token file",
));
} else if conf.bucket == "" {
return Err(std::io::Error::other("no bucket name was provided"));
}
let creds;
if conf.access_key != "" && conf.secret_key != "" {
creds = Credentials::new(
conf.access_key.clone(), // access_key_id
conf.secret_key.clone(), // secret_access_key
None, // session_token (可选)
None,
"Static",
);
} else {
return Err(std::io::Error::other("insufficient parameters for S3 backend authentication"));
}
let region_provider = RegionProviderChain::default_provider().or_else(Region::new(conf.region.clone()));
#[allow(deprecated)]
let config = aws_config::from_env()
.endpoint_url(conf.endpoint.clone())
.region(region_provider)
.credentials_provider(creds)
.load()
.await;
let client = Client::new(&config);
let client = Arc::new(client);
Ok(Self {
client,
bucket: conf.bucket.clone(),
prefix: conf.prefix.clone().trim_matches('/').to_string(),
storage_class: conf.storage_class.clone(),
})
}
pub fn get_dest(&self, object: &str) -> String {
let mut dest_obj = object.to_string();
if self.prefix != "" {
dest_obj = format!("{}/{}", &self.prefix, object);
}
return dest_obj;
}
}
#[async_trait::async_trait]
impl WarmBackend for WarmBackendS3 {
async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
meta: HashMap<String, String>,
) -> Result<String, std::io::Error> {
let client = self.client.clone();
let Ok(res) = client
.put_object()
.bucket(&self.bucket)
.key(&self.get_dest(object))
.body(match r {
ReaderImpl::Body(content_body) => ByteStream::from(content_body.to_vec()),
ReaderImpl::ObjectBody(mut content_body) => ByteStream::from(content_body.read_all().await?),
})
.send()
.await
else {
return Err(std::io::Error::other("put_object error"));
};
Ok(res.version_id().unwrap_or("").to_string())
}
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
self.put_with_meta(object, r, length, HashMap::new()).await
}
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
let client = self.client.clone();
let Ok(res) = client
.get_object()
.bucket(&self.bucket)
.key(&self.get_dest(object))
.send()
.await
else {
return Err(std::io::Error::other("get_object error"));
};
Ok(ReadCloser::new(std::io::Cursor::new(
res.body.collect().await.map(|data| data.into_bytes().to_vec())?,
)))
}
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
let client = self.client.clone();
if let Err(_) = client
.delete_object()
.bucket(&self.bucket)
.key(&self.get_dest(object))
.send()
.await
{
return Err(std::io::Error::other("delete_object error"));
}
Ok(())
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
let client = self.client.clone();
let Ok(res) = client
.list_objects_v2()
.bucket(&self.bucket)
//.max_keys(10)
//.into_paginator()
.send()
.await
else {
return Err(std::io::Error::other("list_objects_v2 error"));
};
Ok(res.common_prefixes.unwrap().len() > 0 || res.contents.unwrap().len() > 0)
}
}
crates/ecstore/src/tier/warm_backend_tencent.rs
+164
View File
@@ -0,0 +1,164 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap;
use std::sync::Arc;
use crate::client::{
admin_handler_utils::AdminError,
api_put_object::PutObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
};
use crate::tier::{
tier_config::TierTencent,
warm_backend::{WarmBackend, WarmBackendGetOpts},
warm_backend_s3::WarmBackendS3,
};
use tracing::warn;
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
const MAX_PARTS_COUNT: i64 = 10000;
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
pub struct WarmBackendTencent(WarmBackendS3);
impl WarmBackendTencent {
pub async fn new(conf: &TierTencent, tier: &str) -> Result<Self, std::io::Error> {
if conf.access_key == "" || conf.secret_key == "" {
return Err(std::io::Error::other("both access and secret keys are required"));
}
if conf.bucket == "" {
return Err(std::io::Error::other("no bucket name was provided"));
}
let u = match url::Url::parse(&conf.endpoint) {
Ok(u) => u,
Err(e) => {
return Err(std::io::Error::other(e.to_string()));
}
};
let creds = Credentials::new(Static(Value {
access_key_id: conf.access_key.clone(),
secret_access_key: conf.secret_key.clone(),
session_token: "".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
}));
let opts = Options {
creds,
secure: u.scheme() == "https",
//transport: GLOBAL_RemoteTargetTransport,
trailing_headers: true,
region: conf.region.clone(),
bucket_lookup: BucketLookupType::BucketLookupDNS,
..Default::default()
};
let scheme = u.scheme();
let default_port = if scheme == "https" { 443 } else { 80 };
let client = TransitionClient::new(
&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)),
opts,
"tencent",
)
.await?;
let client = Arc::new(client);
let core = TransitionCore(Arc::clone(&client));
Ok(Self(WarmBackendS3 {
client,
core,
bucket: conf.bucket.clone(),
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
storage_class: "".to_string(),
}))
}
}
#[async_trait::async_trait]
impl WarmBackend for WarmBackendTencent {
async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
meta: HashMap<String, String>,
) -> Result<String, std::io::Error> {
let part_size = optimal_part_size(length)?;
let client = self.0.client.clone();
let res = client
.put_object(
&self.0.bucket,
&self.0.get_dest(object),
r,
length,
&PutObjectOptions {
storage_class: self.0.storage_class.clone(),
part_size: part_size as u64,
disable_content_sha256: true,
user_metadata: meta,
..Default::default()
},
)
.await?;
//self.ToObjectError(err, object)
Ok(res.version_id)
}
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
self.put_with_meta(object, r, length, HashMap::new()).await
}
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
self.0.get(object, rv, opts).await
}
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
self.0.remove(object, rv).await
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
self.0.in_use().await
}
}
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
let mut object_size = object_size;
if object_size == -1 {
object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
}
if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
return Err(std::io::Error::other("entity too large"));
}
let configured_part_size = MIN_PART_SIZE;
let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64;
part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64;
let part_size = part_size_flt as i64;
if part_size == 0 {
return Ok(MIN_PART_SIZE);
}
Ok(part_size)
}