mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-01 09:48:20 +00:00
refactor: consolidate ecstore owner module layout (#3934)
* refactor: shrink ecstore root owner facades * refactor: remove ecstore core store root shims * refactor: move ecstore erasure owner modules * refactor: remove ecstore root rpc facade * refactor: move ecstore services domain modules
This commit is contained in:
Zhengchao An
@@ -0,0 +1,30 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub mod tier;
|
||||
pub mod tier_admin;
|
||||
pub mod tier_config;
|
||||
pub mod tier_gen;
|
||||
pub mod tier_handlers;
|
||||
pub mod warm_backend;
|
||||
pub mod warm_backend_aliyun;
|
||||
pub mod warm_backend_azure;
|
||||
pub mod warm_backend_gcs;
|
||||
pub mod warm_backend_huaweicloud;
|
||||
pub mod warm_backend_minio;
|
||||
pub mod warm_backend_r2;
|
||||
pub mod warm_backend_rustfs;
|
||||
pub mod warm_backend_s3;
|
||||
pub mod warm_backend_s3sdk;
|
||||
pub mod warm_backend_tencent;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,42 @@
|
||||
#![allow(unused_imports)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
|
||||
#[serde(default)]
|
||||
pub struct TierCreds {
|
||||
#[serde(rename = "accessKey")]
|
||||
pub access_key: String,
|
||||
#[serde(rename = "secretKey")]
|
||||
pub secret_key: String,
|
||||
|
||||
#[serde(rename = "awsRole")]
|
||||
pub aws_role: bool,
|
||||
#[serde(rename = "awsRoleWebIdentityTokenFile")]
|
||||
pub aws_role_web_identity_token_file: String,
|
||||
#[serde(rename = "awsRoleArn")]
|
||||
pub aws_role_arn: String,
|
||||
|
||||
//azsp: ServicePrincipalAuth,
|
||||
|
||||
//#[serde(rename = "credsJson")]
|
||||
pub creds_json: Vec<u8>,
|
||||
}
|
||||
@@ -0,0 +1,600 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::fmt::Display;
|
||||
use tracing::info;
|
||||
|
||||
const C_TIER_CONFIG_VER: &str = "v1";
|
||||
|
||||
const ERR_TIER_NAME_EMPTY: &str = "remote tier name empty";
|
||||
|
||||
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
|
||||
pub enum TierType {
|
||||
#[default]
|
||||
Unsupported,
|
||||
#[serde(rename = "s3")]
|
||||
S3,
|
||||
#[serde(rename = "rustfs")]
|
||||
RustFS,
|
||||
#[serde(rename = "minio")]
|
||||
MinIO,
|
||||
#[serde(rename = "aliyun")]
|
||||
Aliyun,
|
||||
#[serde(rename = "tencent")]
|
||||
Tencent,
|
||||
#[serde(rename = "huaweicloud")]
|
||||
Huaweicloud,
|
||||
#[serde(rename = "azure")]
|
||||
Azure,
|
||||
#[serde(rename = "gcs")]
|
||||
GCS,
|
||||
#[serde(rename = "r2")]
|
||||
R2,
|
||||
}
|
||||
|
||||
impl Display for TierType {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
TierType::S3 => {
|
||||
write!(f, "S3")
|
||||
}
|
||||
TierType::RustFS => {
|
||||
write!(f, "RustFS")
|
||||
}
|
||||
TierType::MinIO => {
|
||||
write!(f, "MinIO")
|
||||
}
|
||||
TierType::Aliyun => {
|
||||
write!(f, "Aliyun")
|
||||
}
|
||||
TierType::Tencent => {
|
||||
write!(f, "Tencent")
|
||||
}
|
||||
TierType::Huaweicloud => {
|
||||
write!(f, "Huaweicloud")
|
||||
}
|
||||
TierType::Azure => {
|
||||
write!(f, "Azure")
|
||||
}
|
||||
TierType::GCS => {
|
||||
write!(f, "GCS")
|
||||
}
|
||||
TierType::R2 => {
|
||||
write!(f, "R2")
|
||||
}
|
||||
_ => {
|
||||
write!(f, "Unsupported")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TierType {
|
||||
pub fn new(sc_type: &str) -> Self {
|
||||
match sc_type {
|
||||
"S3" => TierType::S3,
|
||||
"RustFS" => TierType::RustFS,
|
||||
"MinIO" => TierType::MinIO,
|
||||
"Aliyun" => TierType::Aliyun,
|
||||
"Tencent" => TierType::Tencent,
|
||||
"Huaweicloud" => TierType::Huaweicloud,
|
||||
"Azure" => TierType::Azure,
|
||||
"GCS" => TierType::GCS,
|
||||
"R2" => TierType::R2,
|
||||
_ => TierType::Unsupported,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_lowercase(&self) -> String {
|
||||
match self {
|
||||
TierType::S3 => "s3".to_string(),
|
||||
TierType::RustFS => "rustfs".to_string(),
|
||||
TierType::MinIO => "minio".to_string(),
|
||||
TierType::Aliyun => "aliyun".to_string(),
|
||||
TierType::Tencent => "tencent".to_string(),
|
||||
TierType::Huaweicloud => "huaweicloud".to_string(),
|
||||
TierType::Azure => "azure".to_string(),
|
||||
TierType::GCS => "gcs".to_string(),
|
||||
TierType::R2 => "r2".to_string(),
|
||||
_ => "unsupported".to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Debug, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct TierConfig {
|
||||
#[serde(skip)]
|
||||
pub version: String,
|
||||
#[serde(rename = "type")]
|
||||
pub tier_type: TierType,
|
||||
#[serde(skip)]
|
||||
pub name: String,
|
||||
#[serde(rename = "s3", skip_serializing_if = "Option::is_none")]
|
||||
pub s3: Option<TierS3>,
|
||||
#[serde(rename = "aliyun", skip_serializing_if = "Option::is_none")]
|
||||
pub aliyun: Option<TierAliyun>,
|
||||
#[serde(rename = "tencent", skip_serializing_if = "Option::is_none")]
|
||||
pub tencent: Option<TierTencent>,
|
||||
#[serde(rename = "huaweicloud", skip_serializing_if = "Option::is_none")]
|
||||
pub huaweicloud: Option<TierHuaweicloud>,
|
||||
#[serde(rename = "azure", skip_serializing_if = "Option::is_none")]
|
||||
pub azure: Option<TierAzure>,
|
||||
#[serde(rename = "gcs", skip_serializing_if = "Option::is_none")]
|
||||
pub gcs: Option<TierGCS>,
|
||||
#[serde(rename = "r2", skip_serializing_if = "Option::is_none")]
|
||||
pub r2: Option<TierR2>,
|
||||
#[serde(rename = "rustfs", skip_serializing_if = "Option::is_none")]
|
||||
pub rustfs: Option<TierRustFS>,
|
||||
#[serde(rename = "minio", skip_serializing_if = "Option::is_none")]
|
||||
pub minio: Option<TierMinIO>,
|
||||
}
|
||||
|
||||
impl Clone for TierConfig {
|
||||
fn clone(&self) -> TierConfig {
|
||||
let mut s3 = None;
|
||||
let mut r = None;
|
||||
let mut compatible_backend = None;
|
||||
let mut aliyun = None;
|
||||
let mut tencent = None;
|
||||
let mut huaweicloud = None;
|
||||
let mut azure = None;
|
||||
let mut gcs = None;
|
||||
let mut r2 = None;
|
||||
match self.tier_type {
|
||||
TierType::S3 => {
|
||||
if let Some(s3_) = self.s3.as_ref() {
|
||||
let mut s3_clone = s3_.clone();
|
||||
s3_clone.secret_key = "REDACTED".to_string();
|
||||
s3 = Some(s3_clone);
|
||||
}
|
||||
}
|
||||
TierType::RustFS => {
|
||||
if let Some(r_) = self.rustfs.as_ref() {
|
||||
let mut r_clone = r_.clone();
|
||||
r_clone.secret_key = "REDACTED".to_string();
|
||||
r = Some(r_clone);
|
||||
}
|
||||
}
|
||||
TierType::MinIO => {
|
||||
if let Some(compatible_backend_) = self.minio.as_ref() {
|
||||
let mut compatible_backend_clone = compatible_backend_.clone();
|
||||
compatible_backend_clone.secret_key = "REDACTED".to_string();
|
||||
compatible_backend = Some(compatible_backend_clone);
|
||||
}
|
||||
}
|
||||
TierType::Aliyun => {
|
||||
if let Some(aliyun_) = self.aliyun.as_ref() {
|
||||
let mut aliyun_clone = aliyun_.clone();
|
||||
aliyun_clone.secret_key = "REDACTED".to_string();
|
||||
aliyun = Some(aliyun_clone);
|
||||
}
|
||||
}
|
||||
TierType::Tencent => {
|
||||
if let Some(tencent_) = self.tencent.as_ref() {
|
||||
let mut tencent_clone = tencent_.clone();
|
||||
tencent_clone.secret_key = "REDACTED".to_string();
|
||||
tencent = Some(tencent_clone);
|
||||
}
|
||||
}
|
||||
TierType::Huaweicloud => {
|
||||
if let Some(huaweicloud_) = self.huaweicloud.as_ref() {
|
||||
let mut huaweicloud_clone = huaweicloud_.clone();
|
||||
huaweicloud_clone.secret_key = "REDACTED".to_string();
|
||||
huaweicloud = Some(huaweicloud_clone);
|
||||
}
|
||||
}
|
||||
TierType::Azure => {
|
||||
if let Some(azure_) = self.azure.as_ref() {
|
||||
let mut azure_clone = azure_.clone();
|
||||
azure_clone.secret_key = "REDACTED".to_string();
|
||||
azure = Some(azure_clone);
|
||||
}
|
||||
}
|
||||
TierType::GCS => {
|
||||
if let Some(gcs_) = self.gcs.as_ref() {
|
||||
let mut gcs_clone = gcs_.clone();
|
||||
gcs_clone.creds = "REDACTED".to_string();
|
||||
gcs = Some(gcs_clone);
|
||||
}
|
||||
}
|
||||
TierType::R2 => {
|
||||
if let Some(r2_) = self.r2.as_ref() {
|
||||
let mut r2_clone = r2_.clone();
|
||||
r2_clone.secret_key = "REDACTED".to_string();
|
||||
r2 = Some(r2_clone);
|
||||
}
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
TierConfig {
|
||||
version: self.version.clone(),
|
||||
tier_type: self.tier_type.clone(),
|
||||
name: self.name.clone(),
|
||||
s3,
|
||||
rustfs: r,
|
||||
minio: compatible_backend,
|
||||
aliyun,
|
||||
tencent,
|
||||
huaweicloud,
|
||||
azure,
|
||||
gcs,
|
||||
r2,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl TierConfig {
|
||||
fn endpoint(&self) -> String {
|
||||
match self.tier_type {
|
||||
TierType::S3 => self.s3.as_ref().map(|s| s.endpoint.clone()).unwrap_or_default(),
|
||||
TierType::RustFS => self.rustfs.as_ref().map(|r| r.endpoint.clone()).unwrap_or_default(),
|
||||
TierType::MinIO => self.minio.as_ref().map(|m| m.endpoint.clone()).unwrap_or_default(),
|
||||
TierType::Aliyun => self.aliyun.as_ref().map(|a| a.endpoint.clone()).unwrap_or_default(),
|
||||
TierType::Tencent => self.tencent.as_ref().map(|t| t.endpoint.clone()).unwrap_or_default(),
|
||||
TierType::Huaweicloud => self.huaweicloud.as_ref().map(|h| h.endpoint.clone()).unwrap_or_default(),
|
||||
TierType::Azure => self.azure.as_ref().map(|a| a.endpoint.clone()).unwrap_or_default(),
|
||||
TierType::GCS => self.gcs.as_ref().map(|g| g.endpoint.clone()).unwrap_or_default(),
|
||||
TierType::R2 => self.r2.as_ref().map(|r| r.endpoint.clone()).unwrap_or_default(),
|
||||
_ => {
|
||||
info!("unexpected tier type {}", self.tier_type);
|
||||
"".to_string()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn bucket(&self) -> String {
|
||||
match self.tier_type {
|
||||
TierType::S3 => self.s3.as_ref().map(|s| s.bucket.clone()).unwrap_or_default(),
|
||||
TierType::RustFS => self.rustfs.as_ref().map(|r| r.bucket.clone()).unwrap_or_default(),
|
||||
TierType::MinIO => self.minio.as_ref().map(|m| m.bucket.clone()).unwrap_or_default(),
|
||||
TierType::Aliyun => self.aliyun.as_ref().map(|a| a.bucket.clone()).unwrap_or_default(),
|
||||
TierType::Tencent => self.tencent.as_ref().map(|t| t.bucket.clone()).unwrap_or_default(),
|
||||
TierType::Huaweicloud => self.huaweicloud.as_ref().map(|h| h.bucket.clone()).unwrap_or_default(),
|
||||
TierType::Azure => self.azure.as_ref().map(|a| a.bucket.clone()).unwrap_or_default(),
|
||||
TierType::GCS => self.gcs.as_ref().map(|g| g.bucket.clone()).unwrap_or_default(),
|
||||
TierType::R2 => self.r2.as_ref().map(|r| r.bucket.clone()).unwrap_or_default(),
|
||||
_ => {
|
||||
info!("unexpected tier type {}", self.tier_type);
|
||||
"".to_string()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn prefix(&self) -> String {
|
||||
match self.tier_type {
|
||||
TierType::S3 => self.s3.as_ref().map(|s| s.prefix.clone()).unwrap_or_default(),
|
||||
TierType::RustFS => self.rustfs.as_ref().map(|r| r.prefix.clone()).unwrap_or_default(),
|
||||
TierType::MinIO => self.minio.as_ref().map(|m| m.prefix.clone()).unwrap_or_default(),
|
||||
TierType::Aliyun => self.aliyun.as_ref().map(|a| a.prefix.clone()).unwrap_or_default(),
|
||||
TierType::Tencent => self.tencent.as_ref().map(|t| t.prefix.clone()).unwrap_or_default(),
|
||||
TierType::Huaweicloud => self.huaweicloud.as_ref().map(|h| h.prefix.clone()).unwrap_or_default(),
|
||||
TierType::Azure => self.azure.as_ref().map(|a| a.prefix.clone()).unwrap_or_default(),
|
||||
TierType::GCS => self.gcs.as_ref().map(|g| g.prefix.clone()).unwrap_or_default(),
|
||||
TierType::R2 => self.r2.as_ref().map(|r| r.prefix.clone()).unwrap_or_default(),
|
||||
_ => {
|
||||
info!("unexpected tier type {}", self.tier_type);
|
||||
"".to_string()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn region(&self) -> String {
|
||||
match self.tier_type {
|
||||
TierType::S3 => self.s3.as_ref().map(|s| s.region.clone()).unwrap_or_default(),
|
||||
TierType::RustFS => self.rustfs.as_ref().map(|r| r.region.clone()).unwrap_or_default(),
|
||||
TierType::MinIO => self.minio.as_ref().map(|m| m.region.clone()).unwrap_or_default(),
|
||||
TierType::Aliyun => self.aliyun.as_ref().map(|a| a.region.clone()).unwrap_or_default(),
|
||||
TierType::Tencent => self.tencent.as_ref().map(|t| t.region.clone()).unwrap_or_default(),
|
||||
TierType::Huaweicloud => self.huaweicloud.as_ref().map(|h| h.region.clone()).unwrap_or_default(),
|
||||
TierType::Azure => self.azure.as_ref().map(|a| a.region.clone()).unwrap_or_default(),
|
||||
TierType::GCS => self.gcs.as_ref().map(|g| g.region.clone()).unwrap_or_default(),
|
||||
TierType::R2 => self.r2.as_ref().map(|r| r.region.clone()).unwrap_or_default(),
|
||||
_ => {
|
||||
info!("unexpected tier type {}", self.tier_type);
|
||||
"".to_string()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//type S3Options = impl Fn(TierS3) -> Pin<Box<Result<()>>> + Send + Sync + 'static;
|
||||
|
||||
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
|
||||
#[serde(default)]
|
||||
pub struct TierS3 {
|
||||
pub name: String,
|
||||
pub endpoint: String,
|
||||
#[serde(rename = "accessKey")]
|
||||
pub access_key: String,
|
||||
#[serde(rename = "secretKey")]
|
||||
pub secret_key: String,
|
||||
pub bucket: String,
|
||||
pub prefix: String,
|
||||
pub region: String,
|
||||
#[serde(rename = "storageClass")]
|
||||
pub storage_class: String,
|
||||
#[serde(skip)]
|
||||
pub aws_role: bool,
|
||||
#[serde(skip)]
|
||||
pub aws_role_web_identity_token_file: String,
|
||||
#[serde(skip)]
|
||||
pub aws_role_arn: String,
|
||||
#[serde(skip)]
|
||||
pub aws_role_session_name: String,
|
||||
#[serde(skip)]
|
||||
pub aws_role_duration_seconds: i32,
|
||||
}
|
||||
|
||||
impl TierS3 {
|
||||
#[allow(dead_code)]
|
||||
fn create<F>(
|
||||
name: &str,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
bucket: &str,
|
||||
options: Vec<F>,
|
||||
) -> Result<TierConfig, std::io::Error>
|
||||
where
|
||||
F: Fn(TierS3) -> Box<Result<(), std::io::Error>> + Send + Sync + 'static,
|
||||
{
|
||||
if name.is_empty() {
|
||||
return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY));
|
||||
}
|
||||
let sc = TierS3 {
|
||||
access_key: access_key.to_string(),
|
||||
secret_key: secret_key.to_string(),
|
||||
bucket: bucket.to_string(),
|
||||
endpoint: "https://s3.amazonaws.com".to_string(),
|
||||
region: "".to_string(),
|
||||
storage_class: "".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
for option in options {
|
||||
let option = option(sc.clone());
|
||||
let option = *option;
|
||||
option?;
|
||||
}
|
||||
|
||||
Ok(TierConfig {
|
||||
version: C_TIER_CONFIG_VER.to_string(),
|
||||
tier_type: TierType::S3,
|
||||
name: name.to_string(),
|
||||
s3: Some(sc),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
|
||||
#[serde(default)]
|
||||
pub struct TierRustFS {
|
||||
pub name: String,
|
||||
pub endpoint: String,
|
||||
#[serde(rename = "accessKey")]
|
||||
pub access_key: String,
|
||||
#[serde(rename = "secretKey")]
|
||||
pub secret_key: String,
|
||||
pub bucket: String,
|
||||
pub prefix: String,
|
||||
pub region: String,
|
||||
#[serde(rename = "storageClass")]
|
||||
pub storage_class: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
|
||||
#[serde(default)]
|
||||
pub struct TierMinIO {
|
||||
pub name: String,
|
||||
pub endpoint: String,
|
||||
#[serde(rename = "accessKey")]
|
||||
pub access_key: String,
|
||||
#[serde(rename = "secretKey")]
|
||||
pub secret_key: String,
|
||||
pub bucket: String,
|
||||
pub prefix: String,
|
||||
pub region: String,
|
||||
}
|
||||
|
||||
impl TierMinIO {
|
||||
#[allow(dead_code)]
|
||||
fn create<F>(
|
||||
name: &str,
|
||||
endpoint: &str,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
bucket: &str,
|
||||
options: Vec<F>,
|
||||
) -> Result<TierConfig, std::io::Error>
|
||||
where
|
||||
F: Fn(TierMinIO) -> Box<Result<(), std::io::Error>> + Send + Sync + 'static,
|
||||
{
|
||||
if name.is_empty() {
|
||||
return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY));
|
||||
}
|
||||
let backend = TierMinIO {
|
||||
access_key: access_key.to_string(),
|
||||
secret_key: secret_key.to_string(),
|
||||
bucket: bucket.to_string(),
|
||||
endpoint: endpoint.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
for option in options {
|
||||
let option = option(backend.clone());
|
||||
let option = *option;
|
||||
option?;
|
||||
}
|
||||
|
||||
Ok(TierConfig {
|
||||
version: C_TIER_CONFIG_VER.to_string(),
|
||||
tier_type: TierType::MinIO,
|
||||
name: name.to_string(),
|
||||
minio: Some(backend),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
|
||||
#[serde(default)]
|
||||
pub struct TierAliyun {
|
||||
pub name: String,
|
||||
pub endpoint: String,
|
||||
#[serde(rename = "accessKey")]
|
||||
pub access_key: String,
|
||||
#[serde(rename = "secretKey")]
|
||||
pub secret_key: String,
|
||||
pub bucket: String,
|
||||
pub prefix: String,
|
||||
pub region: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
|
||||
#[serde(default)]
|
||||
pub struct TierTencent {
|
||||
pub name: String,
|
||||
pub endpoint: String,
|
||||
#[serde(rename = "accessKey")]
|
||||
pub access_key: String,
|
||||
#[serde(rename = "secretKey")]
|
||||
pub secret_key: String,
|
||||
pub bucket: String,
|
||||
pub prefix: String,
|
||||
pub region: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
|
||||
#[serde(default)]
|
||||
pub struct TierHuaweicloud {
|
||||
pub name: String,
|
||||
pub endpoint: String,
|
||||
#[serde(rename = "accessKey")]
|
||||
pub access_key: String,
|
||||
#[serde(rename = "secretKey")]
|
||||
pub secret_key: String,
|
||||
pub bucket: String,
|
||||
pub prefix: String,
|
||||
pub region: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
|
||||
#[serde(default)]
|
||||
pub struct ServicePrincipalAuth {
|
||||
pub tenant_id: String,
|
||||
pub client_id: String,
|
||||
pub client_secret: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
|
||||
#[serde(default)]
|
||||
pub struct TierAzure {
|
||||
pub name: String,
|
||||
pub endpoint: String,
|
||||
#[serde(rename = "accessKey")]
|
||||
pub access_key: String,
|
||||
#[serde(rename = "secretKey")]
|
||||
pub secret_key: String,
|
||||
pub bucket: String,
|
||||
pub prefix: String,
|
||||
pub region: String,
|
||||
#[serde(rename = "storageClass")]
|
||||
pub storage_class: String,
|
||||
#[serde(rename = "spAuth")]
|
||||
pub sp_auth: ServicePrincipalAuth,
|
||||
}
|
||||
|
||||
impl TierAzure {
|
||||
pub fn is_sp_enabled(&self) -> bool {
|
||||
!self.sp_auth.tenant_id.is_empty() && !self.sp_auth.client_id.is_empty() && !self.sp_auth.client_secret.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
fn AzureServicePrincipal(tenantID, clientID, clientSecret string) func(az *TierAzure) error {
|
||||
return func(az *TierAzure) error {
|
||||
if tenantID == "" {
|
||||
return errors.New("empty tenant ID unsupported")
|
||||
}
|
||||
if clientID == "" {
|
||||
return errors.New("empty client ID unsupported")
|
||||
}
|
||||
if clientSecret == "" {
|
||||
return errors.New("empty client secret unsupported")
|
||||
}
|
||||
az.SPAuth.TenantID = tenantID
|
||||
az.SPAuth.ClientID = clientID
|
||||
az.SPAuth.ClientSecret = clientSecret
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
fn AzurePrefix(prefix string) func(az *TierAzure) error {
|
||||
return func(az *TierAzure) error {
|
||||
az.Prefix = prefix
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
fn AzureEndpoint(endpoint string) func(az *TierAzure) error {
|
||||
return func(az *TierAzure) error {
|
||||
az.Endpoint = endpoint
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
fn AzureRegion(region string) func(az *TierAzure) error {
|
||||
return func(az *TierAzure) error {
|
||||
az.Region = region
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
fn AzureStorageClass(sc string) func(az *TierAzure) error {
|
||||
return func(az *TierAzure) error {
|
||||
az.StorageClass = sc
|
||||
return nil
|
||||
}
|
||||
}*/
|
||||
|
||||
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
|
||||
#[serde(default)]
|
||||
pub struct TierGCS {
|
||||
pub name: String,
|
||||
pub endpoint: String,
|
||||
#[serde(rename = "creds")]
|
||||
pub creds: String,
|
||||
pub bucket: String,
|
||||
pub prefix: String,
|
||||
pub region: String,
|
||||
#[serde(rename = "storageClass")]
|
||||
pub storage_class: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
|
||||
#[serde(default)]
|
||||
pub struct TierR2 {
|
||||
pub name: String,
|
||||
pub endpoint: String,
|
||||
#[serde(rename = "accessKey")]
|
||||
pub access_key: String,
|
||||
#[serde(rename = "secretKey")]
|
||||
pub secret_key: String,
|
||||
pub bucket: String,
|
||||
pub prefix: String,
|
||||
pub region: String,
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::services::tier::tier::TierConfigMgr;
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl TierConfigMgr {
|
||||
pub fn msg_size(&self) -> usize {
|
||||
100
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::client::admin_handler_utils::AdminError;
|
||||
use http::status::StatusCode;
|
||||
use lazy_static::lazy_static;
|
||||
|
||||
lazy_static! {
|
||||
pub static ref ERR_TIER_ALREADY_EXISTS: AdminError = AdminError {
|
||||
code: "XRustFSAdminTierAlreadyExists".to_string(),
|
||||
message: "Specified remote tier already exists".to_string(),
|
||||
status_code: StatusCode::CONFLICT,
|
||||
};
|
||||
pub static ref ERR_TIER_NOT_FOUND: AdminError = AdminError {
|
||||
code: "XRustFSAdminTierNotFound".to_string(),
|
||||
message: "Specified remote tier was not found".to_string(),
|
||||
status_code: StatusCode::NOT_FOUND,
|
||||
};
|
||||
pub static ref ERR_TIER_NAME_NOT_UPPERCASE: AdminError = AdminError {
|
||||
code: "XRustFSAdminTierNameNotUpperCase".to_string(),
|
||||
message: "Tier name must be in uppercase".to_string(),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
};
|
||||
pub static ref ERR_TIER_BUCKET_NOT_FOUND: AdminError = AdminError {
|
||||
code: "XRustFSAdminTierBucketNotFound".to_string(),
|
||||
message: "Remote tier bucket not found".to_string(),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
};
|
||||
pub static ref ERR_TIER_INVALID_CREDENTIALS: AdminError = AdminError {
|
||||
code: "XRustFSAdminTierInvalidCredentials".to_string(),
|
||||
message: "Invalid remote tier credentials".to_string(),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
};
|
||||
pub static ref ERR_TIER_RESERVED_NAME: AdminError = AdminError {
|
||||
code: "XRustFSAdminTierReserved".to_string(),
|
||||
message: "Cannot use reserved tier name".to_string(),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
};
|
||||
pub static ref ERR_TIER_PERM_ERR: AdminError = AdminError {
|
||||
code: "TierPermErr".to_string(),
|
||||
message: "Tier Perm Err".to_string(),
|
||||
status_code: StatusCode::OK,
|
||||
};
|
||||
pub static ref ERR_TIER_CONNECT_ERR: AdminError = AdminError {
|
||||
code: "TierConnectErr".to_string(),
|
||||
message: "Tier Connect Err".to_string(),
|
||||
status_code: StatusCode::OK,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,432 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use crate::client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::{AdvancedPutOptions, PutObjectOptions},
|
||||
transition_api::{ReadCloser, ReaderImpl},
|
||||
};
|
||||
use crate::error::is_err_bucket_not_found;
|
||||
use crate::services::tier::{
|
||||
tier::ERR_TIER_TYPE_UNSUPPORTED,
|
||||
tier_config::{TierConfig, TierType},
|
||||
tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_NOT_FOUND, ERR_TIER_PERM_ERR},
|
||||
warm_backend_aliyun::WarmBackendAliyun,
|
||||
warm_backend_azure::WarmBackendAzure,
|
||||
warm_backend_gcs::WarmBackendGCS,
|
||||
warm_backend_huaweicloud::WarmBackendHuaweicloud,
|
||||
warm_backend_minio::WarmBackendMinIO,
|
||||
warm_backend_r2::WarmBackendR2,
|
||||
warm_backend_rustfs::WarmBackendRustFS,
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
warm_backend_tencent::WarmBackendTencent,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use http::StatusCode;
|
||||
use rustfs_utils::http::headers::{
|
||||
CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _,
|
||||
};
|
||||
use s3s::dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus};
|
||||
use s3s::header::{
|
||||
X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_REPLICATION_STATUS,
|
||||
X_AMZ_STORAGE_CLASS,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
use time::format_description::well_known::{Rfc2822, Rfc3339};
|
||||
use tracing::{info, warn};
|
||||
|
||||
pub type WarmBackendImpl = Box<dyn WarmBackend + Send + Sync + 'static>;
|
||||
|
||||
const PROBE_OBJECT: &str = "probeobject";
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct WarmBackendGetOpts {
|
||||
pub start_offset: i64,
|
||||
pub length: i64,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait WarmBackend {
|
||||
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error>;
|
||||
async fn put_with_meta(
|
||||
&self,
|
||||
object: &str,
|
||||
r: ReaderImpl,
|
||||
length: i64,
|
||||
meta: HashMap<String, String>,
|
||||
) -> Result<String, std::io::Error>;
|
||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error>;
|
||||
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error>;
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error>;
|
||||
}
|
||||
|
||||
fn parse_http_timestamp(value: &str) -> Option<OffsetDateTime> {
|
||||
OffsetDateTime::parse(value, &Rfc3339)
|
||||
.or_else(|_| OffsetDateTime::parse(value, &Rfc2822))
|
||||
.ok()
|
||||
}
|
||||
|
||||
pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap<String, String>) -> PutObjectOptions {
|
||||
let mut opts = PutObjectOptions {
|
||||
storage_class,
|
||||
send_content_md5: true,
|
||||
legalhold: ObjectLockLegalHoldStatus::from_static(""),
|
||||
internal: AdvancedPutOptions {
|
||||
replication_status: ReplicationStatus::from_static(""),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
if let Some(content_type) = metadata.lookup(CONTENT_TYPE) {
|
||||
opts.content_type = content_type.to_string();
|
||||
}
|
||||
|
||||
if let Some(content_encoding) = metadata.lookup(CONTENT_ENCODING) {
|
||||
opts.content_encoding = content_encoding.to_string();
|
||||
}
|
||||
|
||||
if let Some(content_language) = metadata.lookup(CONTENT_LANGUAGE) {
|
||||
opts.content_language = content_language.to_string();
|
||||
}
|
||||
|
||||
if let Some(content_disposition) = metadata.lookup(CONTENT_DISPOSITION) {
|
||||
opts.content_disposition = content_disposition.to_string();
|
||||
}
|
||||
|
||||
if let Some(cache_control) = metadata.lookup(CACHE_CONTROL) {
|
||||
opts.cache_control = cache_control.to_string();
|
||||
}
|
||||
|
||||
if let Some(expires) = metadata.lookup(EXPIRES).and_then(parse_http_timestamp) {
|
||||
opts.expires = expires;
|
||||
}
|
||||
|
||||
if let Some(mode) = metadata.lookup(X_AMZ_OBJECT_LOCK_MODE.as_str()) {
|
||||
opts.mode = ObjectLockRetentionMode::from(mode.to_ascii_uppercase());
|
||||
}
|
||||
|
||||
if let Some(retain_until_date) = metadata
|
||||
.lookup(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str())
|
||||
.and_then(parse_http_timestamp)
|
||||
{
|
||||
opts.retain_until_date = retain_until_date;
|
||||
}
|
||||
|
||||
if let Some(legalhold) = metadata.lookup(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()) {
|
||||
opts.legalhold = ObjectLockLegalHoldStatus::from(legalhold.to_ascii_uppercase());
|
||||
}
|
||||
|
||||
for key in [
|
||||
CONTENT_TYPE,
|
||||
CONTENT_ENCODING,
|
||||
CONTENT_LANGUAGE,
|
||||
CONTENT_DISPOSITION,
|
||||
CACHE_CONTROL,
|
||||
EXPIRES,
|
||||
X_AMZ_OBJECT_LOCK_MODE.as_str(),
|
||||
X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str(),
|
||||
X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str(),
|
||||
X_AMZ_REPLICATION_STATUS.as_str(),
|
||||
X_AMZ_STORAGE_CLASS.as_str(),
|
||||
] {
|
||||
metadata.remove(key);
|
||||
}
|
||||
|
||||
opts.user_metadata = metadata;
|
||||
opts
|
||||
}
|
||||
|
||||
pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> {
|
||||
let w = w.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
|
||||
let remote_version_id = w
|
||||
.put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5)
|
||||
.await;
|
||||
if let Err(err) = remote_version_id {
|
||||
return Err(ERR_TIER_PERM_ERR.clone());
|
||||
}
|
||||
|
||||
let r = w.get(PROBE_OBJECT, "", WarmBackendGetOpts::default()).await;
|
||||
//xhttp.DrainBody(r);
|
||||
if let Err(err) = r {
|
||||
//if is_err_bucket_not_found(&err) {
|
||||
// return Err(ERR_TIER_BUCKET_NOT_FOUND);
|
||||
//}
|
||||
/*else if is_err_signature_does_not_match(err) {
|
||||
return Err(ERR_TIER_MISSING_CREDENTIALS);
|
||||
}*/
|
||||
//else {
|
||||
return Err(ERR_TIER_PERM_ERR.clone());
|
||||
//}
|
||||
}
|
||||
if let Ok(version_id) = remote_version_id {
|
||||
if let Err(err) = w.remove(PROBE_OBJECT, &version_id).await {
|
||||
return Err(ERR_TIER_PERM_ERR.clone());
|
||||
};
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBackendImpl, AdminError> {
|
||||
let mut d: Option<WarmBackendImpl> = None;
|
||||
match tier.tier_type {
|
||||
TierType::S3 => {
|
||||
if let Some(s3_config) = tier.s3.as_ref() {
|
||||
let dd = WarmBackendS3::new(s3_config, &tier.name).await;
|
||||
if let Err(err) = dd {
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
d = Some(Box::new(dd.expect("Failed to create S3 backend")));
|
||||
} else {
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: "S3 tier configuration not found".to_string(),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
}
|
||||
TierType::RustFS => {
|
||||
if let Some(rustfs_config) = tier.rustfs.as_ref() {
|
||||
let dd = WarmBackendRustFS::new(rustfs_config, &tier.name).await;
|
||||
if let Err(err) = dd {
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
d = Some(Box::new(dd.expect("Failed to create RustFS backend")));
|
||||
} else {
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: "RustFS tier configuration not found".to_string(),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
}
|
||||
TierType::MinIO => {
|
||||
if let Some(minio_config) = tier.minio.as_ref() {
|
||||
let dd = WarmBackendMinIO::new(minio_config, &tier.name).await;
|
||||
if let Err(err) = dd {
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
d = Some(Box::new(dd.expect("Failed to create MinIO backend")));
|
||||
} else {
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: "MinIO tier configuration not found".to_string(),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
}
|
||||
TierType::Aliyun => {
|
||||
if let Some(aliyun_config) = tier.aliyun.as_ref() {
|
||||
let dd = WarmBackendAliyun::new(aliyun_config, &tier.name).await;
|
||||
if let Err(err) = dd {
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
d = Some(Box::new(dd.expect("Failed to create Aliyun backend")));
|
||||
} else {
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: "Aliyun tier configuration not found".to_string(),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
}
|
||||
TierType::Tencent => {
|
||||
if let Some(tencent_config) = tier.tencent.as_ref() {
|
||||
let dd = WarmBackendTencent::new(tencent_config, &tier.name).await;
|
||||
if let Err(err) = dd {
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
d = Some(Box::new(dd.expect("Failed to create Tencent backend")));
|
||||
} else {
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: "Tencent tier configuration not found".to_string(),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
}
|
||||
TierType::Huaweicloud => {
|
||||
if let Some(huaweicloud_config) = tier.huaweicloud.as_ref() {
|
||||
let dd = WarmBackendHuaweicloud::new(huaweicloud_config, &tier.name).await;
|
||||
if let Err(err) = dd {
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
d = Some(Box::new(dd.expect("Failed to create Huaweicloud backend")));
|
||||
} else {
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: "Huaweicloud tier configuration not found".to_string(),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
}
|
||||
TierType::Azure => {
|
||||
if let Some(azure_config) = tier.azure.as_ref() {
|
||||
let dd = WarmBackendAzure::new(azure_config, &tier.name).await;
|
||||
if let Err(err) = dd {
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
d = Some(Box::new(dd.expect("Failed to create Azure backend")));
|
||||
} else {
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: "Azure tier configuration not found".to_string(),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
}
|
||||
TierType::GCS => {
|
||||
if let Some(gcs_config) = tier.gcs.as_ref() {
|
||||
let dd = WarmBackendGCS::new(gcs_config, &tier.name).await;
|
||||
if let Err(err) = dd {
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
d = Some(Box::new(dd.expect("Failed to create GCS backend")));
|
||||
} else {
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: "GCS tier configuration not found".to_string(),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
}
|
||||
TierType::R2 => {
|
||||
if let Some(r2_config) = tier.r2.as_ref() {
|
||||
let dd = WarmBackendR2::new(r2_config, &tier.name).await;
|
||||
if let Err(err) = dd {
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
d = Some(Box::new(dd.expect("Failed to create R2 backend")));
|
||||
} else {
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: "R2 tier configuration not found".to_string(),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
return Err(ERR_TIER_TYPE_UNSUPPORTED.clone());
|
||||
}
|
||||
}
|
||||
|
||||
d.ok_or_else(|| AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: "Tier backend not initialized".to_string(),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn build_transition_put_options_preserves_content_headers() {
|
||||
let mut metadata = HashMap::new();
|
||||
metadata.insert("content-type".to_string(), "text/plain".to_string());
|
||||
metadata.insert("content-encoding".to_string(), "gzip".to_string());
|
||||
metadata.insert("cache-control".to_string(), "max-age=60".to_string());
|
||||
|
||||
let opts = build_transition_put_options("COLD".to_string(), metadata);
|
||||
|
||||
assert_eq!(opts.content_type, "text/plain");
|
||||
assert_eq!(opts.content_encoding, "gzip");
|
||||
assert_eq!(opts.cache_control, "max-age=60");
|
||||
assert_eq!(opts.internal.replication_status.as_str(), "");
|
||||
assert_eq!(opts.legalhold.as_str(), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_transition_put_options_preserves_object_lock_headers_when_present() {
|
||||
let mut metadata = HashMap::new();
|
||||
metadata.insert(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), "2026-03-23T00:00:00Z".to_string());
|
||||
metadata.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.to_string(), ObjectLockLegalHoldStatus::ON.to_string());
|
||||
metadata.insert(X_AMZ_OBJECT_LOCK_MODE.to_string(), ObjectLockRetentionMode::GOVERNANCE.to_string());
|
||||
|
||||
let opts = build_transition_put_options("COLD".to_string(), metadata);
|
||||
|
||||
assert_eq!(opts.mode.as_str(), ObjectLockRetentionMode::GOVERNANCE);
|
||||
assert_eq!(opts.legalhold.as_str(), ObjectLockLegalHoldStatus::ON);
|
||||
assert_ne!(opts.retain_until_date, OffsetDateTime::UNIX_EPOCH);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_transition_put_options_filters_promoted_headers_from_user_metadata() {
|
||||
let mut metadata = HashMap::new();
|
||||
metadata.insert("name".to_string(), "object".to_string());
|
||||
metadata.insert(CONTENT_TYPE.to_string(), "text/plain".to_string());
|
||||
metadata.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.to_string(), ObjectLockLegalHoldStatus::ON.to_string());
|
||||
metadata.insert(X_AMZ_REPLICATION_STATUS.to_string(), "PENDING".to_string());
|
||||
|
||||
let opts = build_transition_put_options("COLD".to_string(), metadata);
|
||||
|
||||
assert_eq!(opts.user_metadata.get("name"), Some(&"object".to_string()));
|
||||
assert!(!opts.user_metadata.contains_key(CONTENT_TYPE));
|
||||
assert!(!opts.user_metadata.contains_key(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()));
|
||||
assert!(!opts.user_metadata.contains_key(X_AMZ_REPLICATION_STATUS.as_str()));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use crate::services::tier::{
|
||||
tier_config::TierAliyun,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
const MAX_PARTS_COUNT: i64 = 10000;
|
||||
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
|
||||
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
|
||||
|
||||
pub struct WarmBackendAliyun(WarmBackendS3);
|
||||
|
||||
impl WarmBackendAliyun {
|
||||
pub async fn new(conf: &TierAliyun, tier: &str) -> Result<Self, std::io::Error> {
|
||||
if conf.access_key == "" || conf.secret_key == "" {
|
||||
return Err(std::io::Error::other("both access and secret keys are required"));
|
||||
}
|
||||
|
||||
if conf.bucket == "" {
|
||||
return Err(std::io::Error::other("no bucket name was provided"));
|
||||
}
|
||||
|
||||
let u = match url::Url::parse(&conf.endpoint) {
|
||||
Ok(u) => u,
|
||||
Err(e) => {
|
||||
return Err(std::io::Error::other(e.to_string()));
|
||||
}
|
||||
};
|
||||
|
||||
let creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
secret_access_key: conf.secret_key.clone(),
|
||||
session_token: "".to_string(),
|
||||
signer_type: SignatureType::SignatureV4,
|
||||
..Default::default()
|
||||
}));
|
||||
let opts = Options {
|
||||
creds,
|
||||
secure: u.scheme() == "https",
|
||||
//transport: GLOBAL_RemoteTargetTransport,
|
||||
trailing_headers: true,
|
||||
region: conf.region.clone(),
|
||||
bucket_lookup: BucketLookupType::BucketLookupDNS,
|
||||
..Default::default()
|
||||
};
|
||||
let scheme = u.scheme();
|
||||
let default_port = if scheme == "https" { 443 } else { 80 };
|
||||
let host = u
|
||||
.host_str()
|
||||
.ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
|
||||
let client = TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, "aliyun").await?;
|
||||
|
||||
let client = Arc::new(client);
|
||||
let core = TransitionCore(Arc::clone(&client));
|
||||
Ok(Self(WarmBackendS3 {
|
||||
client,
|
||||
core,
|
||||
bucket: conf.bucket.clone(),
|
||||
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
|
||||
storage_class: "".to_string(),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl WarmBackend for WarmBackendAliyun {
|
||||
async fn put_with_meta(
|
||||
&self,
|
||||
object: &str,
|
||||
r: ReaderImpl,
|
||||
length: i64,
|
||||
meta: HashMap<String, String>,
|
||||
) -> Result<String, std::io::Error> {
|
||||
let part_size = optimal_part_size(length)?;
|
||||
let client = self.0.client.clone();
|
||||
let res = client
|
||||
.put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &{
|
||||
let mut opts = build_transition_put_options(self.0.storage_class.clone(), meta);
|
||||
opts.part_size = part_size as u64;
|
||||
opts.disable_content_sha256 = true;
|
||||
opts
|
||||
})
|
||||
.await?;
|
||||
//self.ToObjectError(err, object)
|
||||
Ok(res.version_id)
|
||||
}
|
||||
|
||||
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
|
||||
self.put_with_meta(object, r, length, HashMap::new()).await
|
||||
}
|
||||
|
||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
self.0.get(object, rv, opts).await
|
||||
}
|
||||
|
||||
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||
self.0.remove(object, rv).await
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
self.0.in_use().await
|
||||
}
|
||||
}
|
||||
|
||||
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
let mut object_size = object_size;
|
||||
if object_size == -1 {
|
||||
object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
|
||||
}
|
||||
|
||||
if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
|
||||
return Err(std::io::Error::other("entity too large"));
|
||||
}
|
||||
|
||||
let configured_part_size = MIN_PART_SIZE;
|
||||
let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64;
|
||||
part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64;
|
||||
|
||||
let part_size = part_size_flt as i64;
|
||||
if part_size == 0 {
|
||||
return Ok(MIN_PART_SIZE);
|
||||
}
|
||||
Ok(part_size)
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use crate::services::tier::{
|
||||
tier_config::TierAzure,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
const MAX_PARTS_COUNT: i64 = 10000;
|
||||
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
|
||||
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
|
||||
|
||||
pub struct WarmBackendAzure(WarmBackendS3);
|
||||
|
||||
impl WarmBackendAzure {
|
||||
pub async fn new(conf: &TierAzure, tier: &str) -> Result<Self, std::io::Error> {
|
||||
if conf.access_key == "" || conf.secret_key == "" {
|
||||
return Err(std::io::Error::other("both access and secret keys are required"));
|
||||
}
|
||||
|
||||
if conf.bucket == "" {
|
||||
return Err(std::io::Error::other("no bucket name was provided"));
|
||||
}
|
||||
|
||||
let u = match url::Url::parse(&conf.endpoint) {
|
||||
Ok(u) => u,
|
||||
Err(e) => {
|
||||
return Err(std::io::Error::other(e.to_string()));
|
||||
}
|
||||
};
|
||||
|
||||
let creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
secret_access_key: conf.secret_key.clone(),
|
||||
session_token: "".to_string(),
|
||||
signer_type: SignatureType::SignatureV4,
|
||||
..Default::default()
|
||||
}));
|
||||
let opts = Options {
|
||||
creds,
|
||||
secure: u.scheme() == "https",
|
||||
//transport: GLOBAL_RemoteTargetTransport,
|
||||
trailing_headers: true,
|
||||
region: conf.region.clone(),
|
||||
bucket_lookup: BucketLookupType::BucketLookupDNS,
|
||||
..Default::default()
|
||||
};
|
||||
let scheme = u.scheme();
|
||||
let default_port = if scheme == "https" { 443 } else { 80 };
|
||||
let host = u
|
||||
.host_str()
|
||||
.ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
|
||||
let client = TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, "azure").await?;
|
||||
|
||||
let client = Arc::new(client);
|
||||
let core = TransitionCore(Arc::clone(&client));
|
||||
Ok(Self(WarmBackendS3 {
|
||||
client,
|
||||
core,
|
||||
bucket: conf.bucket.clone(),
|
||||
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
|
||||
storage_class: "".to_string(),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl WarmBackend for WarmBackendAzure {
|
||||
async fn put_with_meta(
|
||||
&self,
|
||||
object: &str,
|
||||
r: ReaderImpl,
|
||||
length: i64,
|
||||
meta: HashMap<String, String>,
|
||||
) -> Result<String, std::io::Error> {
|
||||
let part_size = optimal_part_size(length)?;
|
||||
let client = self.0.client.clone();
|
||||
let res = client
|
||||
.put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &{
|
||||
let mut opts = build_transition_put_options(self.0.storage_class.clone(), meta);
|
||||
opts.part_size = part_size as u64;
|
||||
opts.disable_content_sha256 = true;
|
||||
opts
|
||||
})
|
||||
.await?;
|
||||
//self.ToObjectError(err, object)
|
||||
Ok(res.version_id)
|
||||
}
|
||||
|
||||
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
|
||||
self.put_with_meta(object, r, length, HashMap::new()).await
|
||||
}
|
||||
|
||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
self.0.get(object, rv, opts).await
|
||||
}
|
||||
|
||||
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||
self.0.remove(object, rv).await
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
self.0.in_use().await
|
||||
}
|
||||
}
|
||||
|
||||
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
let mut object_size = object_size;
|
||||
if object_size == -1 {
|
||||
object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
|
||||
}
|
||||
|
||||
if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
|
||||
return Err(std::io::Error::other("entity too large"));
|
||||
}
|
||||
|
||||
let configured_part_size = MIN_PART_SIZE;
|
||||
let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64;
|
||||
part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64;
|
||||
|
||||
let part_size = part_size_flt as i64;
|
||||
if part_size == 0 {
|
||||
return Ok(MIN_PART_SIZE);
|
||||
}
|
||||
Ok(part_size)
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use bytes::Bytes;
|
||||
use google_cloud_auth::credentials::Credentials;
|
||||
use google_cloud_auth::credentials::user_account::Builder;
|
||||
use google_cloud_storage as gcs;
|
||||
use google_cloud_storage::client::Storage;
|
||||
use std::convert::TryFrom;
|
||||
|
||||
use crate::client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
transition_api::{Options, ReadCloser, ReaderImpl},
|
||||
};
|
||||
use crate::services::tier::{
|
||||
tier_config::TierGCS,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts},
|
||||
};
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
const MAX_PARTS_COUNT: i64 = 10000;
|
||||
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
|
||||
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
|
||||
|
||||
pub struct WarmBackendGCS {
|
||||
pub client: Arc<Storage>,
|
||||
pub bucket: String,
|
||||
pub prefix: String,
|
||||
pub storage_class: String,
|
||||
}
|
||||
|
||||
impl WarmBackendGCS {
|
||||
pub async fn new(conf: &TierGCS, tier: &str) -> Result<Self, std::io::Error> {
|
||||
if conf.creds == "" {
|
||||
return Err(std::io::Error::other("both access and secret keys are required"));
|
||||
}
|
||||
|
||||
if conf.bucket == "" {
|
||||
return Err(std::io::Error::other("no bucket name was provided"));
|
||||
}
|
||||
|
||||
let authorized_user = serde_json::from_str(&conf.creds)?;
|
||||
let credentials = Builder::new(authorized_user)
|
||||
//.with_retry_policy(AlwaysRetry.with_attempt_limit(3))
|
||||
//.with_backoff_policy(backoff)
|
||||
.build()
|
||||
.map_err(|e| std::io::Error::other(format!("Invalid credentials JSON: {}", e)))?;
|
||||
|
||||
let Ok(client) = Storage::builder()
|
||||
.with_endpoint(conf.endpoint.clone())
|
||||
.with_credentials(credentials)
|
||||
.build()
|
||||
.await
|
||||
else {
|
||||
return Err(std::io::Error::other("Storage::builder error"));
|
||||
};
|
||||
let client = Arc::new(client);
|
||||
Ok(Self {
|
||||
client,
|
||||
bucket: conf.bucket.clone(),
|
||||
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
|
||||
storage_class: "".to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn get_dest(&self, object: &str) -> String {
|
||||
let mut dest_obj = object.to_string();
|
||||
if self.prefix != "" {
|
||||
dest_obj = format!("{}/{}", &self.prefix, object);
|
||||
}
|
||||
return dest_obj;
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl WarmBackend for WarmBackendGCS {
|
||||
async fn put_with_meta(
|
||||
&self,
|
||||
object: &str,
|
||||
r: ReaderImpl,
|
||||
length: i64,
|
||||
meta: HashMap<String, String>,
|
||||
) -> Result<String, std::io::Error> {
|
||||
let d = match r {
|
||||
ReaderImpl::Body(content_body) => content_body.to_vec(),
|
||||
ReaderImpl::ObjectBody(mut content_body) => content_body.read_all().await?,
|
||||
};
|
||||
let Ok(res) = Box::pin(
|
||||
self.client
|
||||
.write_object(&self.bucket, &self.get_dest(object), Bytes::from(d))
|
||||
.send_buffered(),
|
||||
)
|
||||
.await
|
||||
else {
|
||||
return Err(std::io::Error::other("write_object error"));
|
||||
};
|
||||
//self.ToObjectError(err, object)
|
||||
Ok(res.generation.to_string())
|
||||
}
|
||||
|
||||
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
|
||||
self.put_with_meta(object, r, length, HashMap::new()).await
|
||||
}
|
||||
|
||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
let Ok(mut reader) = self.client.read_object(&self.bucket, &self.get_dest(object)).send().await else {
|
||||
return Err(std::io::Error::other("read_object error"));
|
||||
};
|
||||
let mut contents = Vec::new();
|
||||
while let Ok(Some(chunk)) = reader.next().await.transpose() {
|
||||
contents.extend_from_slice(&chunk);
|
||||
}
|
||||
Ok(ReadCloser::new(std::io::Cursor::new(contents)))
|
||||
}
|
||||
|
||||
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||
/*self.client
|
||||
.delete_object()
|
||||
.set_bucket(&self.bucket)
|
||||
.set_object(&self.get_dest(object))
|
||||
//.set_generation(object.generation)
|
||||
.send()
|
||||
.await?;*/
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
/*let result = self.client
|
||||
.list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1)
|
||||
.await?;
|
||||
|
||||
Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)*/
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
/*fn gcs_to_object_error(err: Error, params: Vec<String>) -> Option<Error> {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
bucket := ""
|
||||
object := ""
|
||||
uploadID := ""
|
||||
if len(params) >= 1 {
|
||||
bucket = params[0]
|
||||
}
|
||||
if len(params) == 2 {
|
||||
object = params[1]
|
||||
}
|
||||
if len(params) == 3 {
|
||||
uploadID = params[2]
|
||||
}
|
||||
|
||||
// in some cases just a plain error is being returned
|
||||
switch err.Error() {
|
||||
case "storage: bucket doesn't exist":
|
||||
err = BucketNotFound{
|
||||
Bucket: bucket,
|
||||
}
|
||||
return err
|
||||
case "storage: object doesn't exist":
|
||||
if uploadID != "" {
|
||||
err = InvalidUploadID{
|
||||
UploadID: uploadID,
|
||||
}
|
||||
} else {
|
||||
err = ObjectNotFound{
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
googleAPIErr, ok := err.(*googleapi.Error)
|
||||
if !ok {
|
||||
// We don't interpret non MinIO errors. As minio errors will
|
||||
// have StatusCode to help to convert to object errors.
|
||||
return err
|
||||
}
|
||||
|
||||
if len(googleAPIErr.Errors) == 0 {
|
||||
return err
|
||||
}
|
||||
|
||||
reason := googleAPIErr.Errors[0].Reason
|
||||
message := googleAPIErr.Errors[0].Message
|
||||
|
||||
switch reason {
|
||||
case "required":
|
||||
// Anonymous users does not have storage.xyz access to project 123.
|
||||
fallthrough
|
||||
case "keyInvalid":
|
||||
fallthrough
|
||||
case "forbidden":
|
||||
err = PrefixAccessDenied{
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
}
|
||||
case "invalid":
|
||||
err = BucketNameInvalid{
|
||||
Bucket: bucket,
|
||||
}
|
||||
case "notFound":
|
||||
if object != "" {
|
||||
err = ObjectNotFound{
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
}
|
||||
break
|
||||
}
|
||||
err = BucketNotFound{Bucket: bucket}
|
||||
case "conflict":
|
||||
if message == "You already own this bucket. Please select another name." {
|
||||
err = BucketAlreadyOwnedByYou{Bucket: bucket}
|
||||
break
|
||||
}
|
||||
if message == "Sorry, that name is not available. Please try a different one." {
|
||||
err = BucketAlreadyExists{Bucket: bucket}
|
||||
break
|
||||
}
|
||||
err = BucketNotEmpty{Bucket: bucket}
|
||||
}
|
||||
|
||||
return err
|
||||
}*/
|
||||
@@ -0,0 +1,156 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use crate::services::tier::{
|
||||
tier_config::TierHuaweicloud,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
const MAX_PARTS_COUNT: i64 = 10000;
|
||||
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
|
||||
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
|
||||
|
||||
pub struct WarmBackendHuaweicloud(WarmBackendS3);
|
||||
|
||||
impl WarmBackendHuaweicloud {
|
||||
pub async fn new(conf: &TierHuaweicloud, tier: &str) -> Result<Self, std::io::Error> {
|
||||
if conf.access_key == "" || conf.secret_key == "" {
|
||||
return Err(std::io::Error::other("both access and secret keys are required"));
|
||||
}
|
||||
|
||||
if conf.bucket == "" {
|
||||
return Err(std::io::Error::other("no bucket name was provided"));
|
||||
}
|
||||
|
||||
let u = match url::Url::parse(&conf.endpoint) {
|
||||
Ok(u) => u,
|
||||
Err(e) => {
|
||||
return Err(std::io::Error::other(e.to_string()));
|
||||
}
|
||||
};
|
||||
|
||||
let creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
secret_access_key: conf.secret_key.clone(),
|
||||
session_token: "".to_string(),
|
||||
signer_type: SignatureType::SignatureV4,
|
||||
..Default::default()
|
||||
}));
|
||||
let opts = Options {
|
||||
creds,
|
||||
secure: u.scheme() == "https",
|
||||
//transport: GLOBAL_RemoteTargetTransport,
|
||||
trailing_headers: true,
|
||||
region: conf.region.clone(),
|
||||
bucket_lookup: BucketLookupType::BucketLookupDNS,
|
||||
..Default::default()
|
||||
};
|
||||
let scheme = u.scheme();
|
||||
let default_port = if scheme == "https" { 443 } else { 80 };
|
||||
let host = u
|
||||
.host_str()
|
||||
.ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
|
||||
let client =
|
||||
TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, "huaweicloud").await?;
|
||||
|
||||
let client = Arc::new(client);
|
||||
let core = TransitionCore(Arc::clone(&client));
|
||||
Ok(Self(WarmBackendS3 {
|
||||
client,
|
||||
core,
|
||||
bucket: conf.bucket.clone(),
|
||||
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
|
||||
storage_class: "".to_string(),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl WarmBackend for WarmBackendHuaweicloud {
|
||||
async fn put_with_meta(
|
||||
&self,
|
||||
object: &str,
|
||||
r: ReaderImpl,
|
||||
length: i64,
|
||||
meta: HashMap<String, String>,
|
||||
) -> Result<String, std::io::Error> {
|
||||
let part_size = optimal_part_size(length)?;
|
||||
let client = self.0.client.clone();
|
||||
let res = client
|
||||
.put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &{
|
||||
let mut opts = build_transition_put_options(self.0.storage_class.clone(), meta);
|
||||
opts.part_size = part_size as u64;
|
||||
opts.disable_content_sha256 = true;
|
||||
opts
|
||||
})
|
||||
.await?;
|
||||
//self.ToObjectError(err, object)
|
||||
Ok(res.version_id)
|
||||
}
|
||||
|
||||
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
|
||||
self.put_with_meta(object, r, length, HashMap::new()).await
|
||||
}
|
||||
|
||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
self.0.get(object, rv, opts).await
|
||||
}
|
||||
|
||||
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||
self.0.remove(object, rv).await
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
self.0.in_use().await
|
||||
}
|
||||
}
|
||||
|
||||
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
let mut object_size = object_size;
|
||||
if object_size == -1 {
|
||||
object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
|
||||
}
|
||||
|
||||
if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
|
||||
return Err(std::io::Error::other("entity too large"));
|
||||
}
|
||||
|
||||
let configured_part_size = MIN_PART_SIZE;
|
||||
let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64;
|
||||
part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64;
|
||||
|
||||
let part_size = part_size_flt as i64;
|
||||
if part_size == 0 {
|
||||
return Ok(MIN_PART_SIZE);
|
||||
}
|
||||
Ok(part_size)
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use crate::services::tier::{
|
||||
tier_config::TierMinIO,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
const MAX_PARTS_COUNT: i64 = 10000;
|
||||
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
|
||||
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
|
||||
|
||||
pub struct WarmBackendMinIO(WarmBackendS3);
|
||||
|
||||
impl WarmBackendMinIO {
|
||||
pub async fn new(conf: &TierMinIO, tier: &str) -> Result<Self, std::io::Error> {
|
||||
if conf.access_key == "" || conf.secret_key == "" {
|
||||
return Err(std::io::Error::other("both access and secret keys are required"));
|
||||
}
|
||||
|
||||
if conf.bucket == "" {
|
||||
return Err(std::io::Error::other("no bucket name was provided"));
|
||||
}
|
||||
|
||||
let u = match url::Url::parse(&conf.endpoint) {
|
||||
Ok(u) => u,
|
||||
Err(e) => {
|
||||
return Err(std::io::Error::other(e.to_string()));
|
||||
}
|
||||
};
|
||||
|
||||
let creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
secret_access_key: conf.secret_key.clone(),
|
||||
session_token: "".to_string(),
|
||||
signer_type: SignatureType::SignatureV4,
|
||||
..Default::default()
|
||||
}));
|
||||
let opts = Options {
|
||||
creds,
|
||||
secure: u.scheme() == "https",
|
||||
//transport: GLOBAL_RemoteTargetTransport,
|
||||
trailing_headers: true,
|
||||
region: conf.region.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
let scheme = u.scheme();
|
||||
let default_port = if scheme == "https" { 443 } else { 80 };
|
||||
let host = u
|
||||
.host_str()
|
||||
.ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
|
||||
let client = TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, "minio").await?;
|
||||
|
||||
let client = Arc::new(client);
|
||||
let core = TransitionCore(Arc::clone(&client));
|
||||
Ok(Self(WarmBackendS3 {
|
||||
client,
|
||||
core,
|
||||
bucket: conf.bucket.clone(),
|
||||
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
|
||||
storage_class: "".to_string(),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl WarmBackend for WarmBackendMinIO {
|
||||
async fn put_with_meta(
|
||||
&self,
|
||||
object: &str,
|
||||
r: ReaderImpl,
|
||||
length: i64,
|
||||
meta: HashMap<String, String>,
|
||||
) -> Result<String, std::io::Error> {
|
||||
let part_size = optimal_part_size(length)?;
|
||||
let client = self.0.client.clone();
|
||||
let res = client
|
||||
.put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &{
|
||||
let mut opts = build_transition_put_options(self.0.storage_class.clone(), meta);
|
||||
opts.part_size = part_size as u64;
|
||||
opts.disable_content_sha256 = true;
|
||||
opts
|
||||
})
|
||||
.await?;
|
||||
//self.ToObjectError(err, object)
|
||||
Ok(res.version_id)
|
||||
}
|
||||
|
||||
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
|
||||
self.put_with_meta(object, r, length, HashMap::new()).await
|
||||
}
|
||||
|
||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
self.0.get(object, rv, opts).await
|
||||
}
|
||||
|
||||
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||
self.0.remove(object, rv).await
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
self.0.in_use().await
|
||||
}
|
||||
}
|
||||
|
||||
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
let mut object_size = object_size;
|
||||
if object_size == -1 {
|
||||
object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
|
||||
}
|
||||
|
||||
if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
|
||||
return Err(std::io::Error::other("entity too large"));
|
||||
}
|
||||
|
||||
let configured_part_size = MIN_PART_SIZE;
|
||||
let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64;
|
||||
part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64;
|
||||
|
||||
let part_size = part_size_flt as i64;
|
||||
if part_size == 0 {
|
||||
return Ok(MIN_PART_SIZE);
|
||||
}
|
||||
Ok(part_size)
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use crate::services::tier::{
|
||||
tier_config::TierR2,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
const MAX_PARTS_COUNT: i64 = 10000;
|
||||
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
|
||||
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
|
||||
|
||||
pub struct WarmBackendR2(WarmBackendS3);
|
||||
|
||||
impl WarmBackendR2 {
|
||||
pub async fn new(conf: &TierR2, tier: &str) -> Result<Self, std::io::Error> {
|
||||
if conf.access_key == "" || conf.secret_key == "" {
|
||||
return Err(std::io::Error::other("both access and secret keys are required"));
|
||||
}
|
||||
|
||||
if conf.bucket == "" {
|
||||
return Err(std::io::Error::other("no bucket name was provided"));
|
||||
}
|
||||
|
||||
let u = match url::Url::parse(&conf.endpoint) {
|
||||
Ok(u) => u,
|
||||
Err(e) => {
|
||||
return Err(std::io::Error::other(e.to_string()));
|
||||
}
|
||||
};
|
||||
|
||||
let creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
secret_access_key: conf.secret_key.clone(),
|
||||
session_token: "".to_string(),
|
||||
signer_type: SignatureType::SignatureV4,
|
||||
..Default::default()
|
||||
}));
|
||||
let opts = Options {
|
||||
creds,
|
||||
secure: u.scheme() == "https",
|
||||
//transport: GLOBAL_RemoteTargetTransport,
|
||||
trailing_headers: true,
|
||||
region: conf.region.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
let scheme = u.scheme();
|
||||
let default_port = if scheme == "https" { 443 } else { 80 };
|
||||
let host = u
|
||||
.host_str()
|
||||
.ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
|
||||
let client = TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, "r2").await?;
|
||||
|
||||
let client = Arc::new(client);
|
||||
let core = TransitionCore(Arc::clone(&client));
|
||||
Ok(Self(WarmBackendS3 {
|
||||
client,
|
||||
core,
|
||||
bucket: conf.bucket.clone(),
|
||||
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
|
||||
storage_class: "".to_string(),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl WarmBackend for WarmBackendR2 {
|
||||
async fn put_with_meta(
|
||||
&self,
|
||||
object: &str,
|
||||
r: ReaderImpl,
|
||||
length: i64,
|
||||
meta: HashMap<String, String>,
|
||||
) -> Result<String, std::io::Error> {
|
||||
let part_size = optimal_part_size(length)?;
|
||||
let client = self.0.client.clone();
|
||||
let res = client
|
||||
.put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &{
|
||||
let mut opts = build_transition_put_options(self.0.storage_class.clone(), meta);
|
||||
opts.part_size = part_size as u64;
|
||||
opts.disable_content_sha256 = true;
|
||||
opts
|
||||
})
|
||||
.await?;
|
||||
//self.ToObjectError(err, object)
|
||||
Ok(res.version_id)
|
||||
}
|
||||
|
||||
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
|
||||
self.put_with_meta(object, r, length, HashMap::new()).await
|
||||
}
|
||||
|
||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
self.0.get(object, rv, opts).await
|
||||
}
|
||||
|
||||
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||
self.0.remove(object, rv).await
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
self.0.in_use().await
|
||||
}
|
||||
}
|
||||
|
||||
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
let mut object_size = object_size;
|
||||
if object_size == -1 {
|
||||
object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
|
||||
}
|
||||
|
||||
if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
|
||||
return Err(std::io::Error::other("entity too large"));
|
||||
}
|
||||
|
||||
let configured_part_size = MIN_PART_SIZE;
|
||||
let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64;
|
||||
part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64;
|
||||
|
||||
let part_size = part_size_flt as i64;
|
||||
if part_size == 0 {
|
||||
return Ok(MIN_PART_SIZE);
|
||||
}
|
||||
Ok(part_size)
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use crate::services::tier::{
|
||||
tier_config::TierRustFS,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
const MAX_PARTS_COUNT: i64 = 10000;
|
||||
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
|
||||
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
|
||||
|
||||
pub struct WarmBackendRustFS(WarmBackendS3);
|
||||
|
||||
impl WarmBackendRustFS {
|
||||
pub async fn new(conf: &TierRustFS, tier: &str) -> Result<Self, std::io::Error> {
|
||||
if conf.access_key == "" || conf.secret_key == "" {
|
||||
return Err(std::io::Error::other("both access and secret keys are required"));
|
||||
}
|
||||
|
||||
if conf.bucket == "" {
|
||||
return Err(std::io::Error::other("no bucket name was provided"));
|
||||
}
|
||||
|
||||
let u = match url::Url::parse(&conf.endpoint) {
|
||||
Ok(u) => u,
|
||||
Err(e) => return Err(std::io::Error::other(e)),
|
||||
};
|
||||
|
||||
let creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
secret_access_key: conf.secret_key.clone(),
|
||||
session_token: "".to_string(),
|
||||
signer_type: SignatureType::SignatureV4,
|
||||
..Default::default()
|
||||
}));
|
||||
let opts = Options {
|
||||
creds,
|
||||
secure: u.scheme() == "https",
|
||||
//transport: GLOBAL_RemoteTargetTransport,
|
||||
trailing_headers: true,
|
||||
region: conf.region.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
let scheme = u.scheme();
|
||||
let default_port = if scheme == "https" { 443 } else { 80 };
|
||||
let host = u
|
||||
.host_str()
|
||||
.ok_or_else(|| std::io::Error::other("endpoint URL must include a host"))?;
|
||||
let client = TransitionClient::new(&format!("{host}:{}", u.port().unwrap_or(default_port)), opts, "rustfs").await?;
|
||||
|
||||
let client = Arc::new(client);
|
||||
let core = TransitionCore(Arc::clone(&client));
|
||||
Ok(Self(WarmBackendS3 {
|
||||
client,
|
||||
core,
|
||||
bucket: conf.bucket.clone(),
|
||||
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
|
||||
storage_class: "".to_string(),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl WarmBackend for WarmBackendRustFS {
|
||||
async fn put_with_meta(
|
||||
&self,
|
||||
object: &str,
|
||||
r: ReaderImpl,
|
||||
length: i64,
|
||||
meta: HashMap<String, String>,
|
||||
) -> Result<String, std::io::Error> {
|
||||
let part_size = optimal_part_size(length)?;
|
||||
let client = self.0.client.clone();
|
||||
let res = client
|
||||
.put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &{
|
||||
let mut opts = build_transition_put_options(self.0.storage_class.clone(), meta);
|
||||
opts.part_size = part_size as u64;
|
||||
opts.disable_content_sha256 = true;
|
||||
opts
|
||||
})
|
||||
.await?;
|
||||
//self.ToObjectError(err, object)
|
||||
Ok(res.version_id)
|
||||
}
|
||||
|
||||
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
|
||||
self.put_with_meta(object, r, length, HashMap::new()).await
|
||||
}
|
||||
|
||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
self.0.get(object, rv, opts).await
|
||||
}
|
||||
|
||||
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||
self.0.remove(object, rv).await
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
self.0.in_use().await
|
||||
}
|
||||
}
|
||||
|
||||
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
let mut object_size = object_size;
|
||||
if object_size == -1 {
|
||||
object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
|
||||
}
|
||||
|
||||
if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
|
||||
return Err(std::io::Error::other("entity too large"));
|
||||
}
|
||||
|
||||
let configured_part_size = MIN_PART_SIZE;
|
||||
let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64;
|
||||
part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64;
|
||||
|
||||
let part_size = part_size_flt as i64;
|
||||
if part_size == 0 {
|
||||
return Ok(MIN_PART_SIZE);
|
||||
}
|
||||
Ok(part_size)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use futures::FutureExt;
|
||||
use std::panic::AssertUnwindSafe;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn rustfs_tier(endpoint: &str) -> TierRustFS {
|
||||
TierRustFS {
|
||||
endpoint: endpoint.to_string(),
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
bucket: "bucket".to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_returns_error_when_endpoint_has_no_host() {
|
||||
let conf = rustfs_tier("rustfs://");
|
||||
|
||||
let outcome = AssertUnwindSafe(WarmBackendRustFS::new(&conf, "tier")).catch_unwind().await;
|
||||
|
||||
let result = outcome.expect("initialization should return an error instead of panicking");
|
||||
let err = match result {
|
||||
Ok(_) => panic!("endpoint without host must be rejected"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert!(err.to_string().contains("host"), "expected host validation error, got: {err}");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use url::Url;
|
||||
|
||||
use crate::client::{
|
||||
api_get_options::GetObjectOptions,
|
||||
api_put_object::PutObjectOptions,
|
||||
api_remove::RemoveObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{Options, TransitionClient, TransitionCore},
|
||||
transition_api::{ReadCloser, ReaderImpl},
|
||||
};
|
||||
use crate::error::ErrorResponse;
|
||||
use crate::error::error_resp_to_object_err;
|
||||
use crate::services::tier::{
|
||||
tier_config::TierS3,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
};
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
|
||||
pub struct WarmBackendS3 {
|
||||
pub client: Arc<TransitionClient>,
|
||||
pub core: TransitionCore,
|
||||
pub bucket: String,
|
||||
pub prefix: String,
|
||||
pub storage_class: String,
|
||||
}
|
||||
|
||||
impl WarmBackendS3 {
|
||||
pub async fn new(conf: &TierS3, tier: &str) -> Result<Self, std::io::Error> {
|
||||
let u = match Url::parse(&conf.endpoint) {
|
||||
Ok(u) => u,
|
||||
Err(err) => {
|
||||
return Err(std::io::Error::other(err.to_string()));
|
||||
}
|
||||
};
|
||||
validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
||||
|
||||
if conf.aws_role_web_identity_token_file == "" && conf.aws_role_arn != ""
|
||||
|| conf.aws_role_web_identity_token_file != "" && conf.aws_role_arn == ""
|
||||
{
|
||||
return Err(std::io::Error::other("both the token file and the role ARN are required"));
|
||||
} else if conf.access_key == "" && conf.secret_key != "" || conf.access_key != "" && conf.secret_key == "" {
|
||||
return Err(std::io::Error::other("both the access and secret keys are required"));
|
||||
} else if conf.aws_role
|
||||
&& (conf.aws_role_web_identity_token_file != ""
|
||||
|| conf.aws_role_arn != ""
|
||||
|| conf.access_key != ""
|
||||
|| conf.secret_key != "")
|
||||
{
|
||||
return Err(std::io::Error::other(
|
||||
"AWS Role cannot be activated with static credentials or the web identity token file",
|
||||
));
|
||||
} else if conf.bucket == "" {
|
||||
return Err(std::io::Error::other("no bucket name was provided"));
|
||||
}
|
||||
|
||||
let creds: Credentials<Static>;
|
||||
|
||||
if conf.access_key != "" && conf.secret_key != "" {
|
||||
//creds = Credentials::new_static_v4(conf.access_key, conf.secret_key, "");
|
||||
creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
secret_access_key: conf.secret_key.clone(),
|
||||
session_token: "".to_string(),
|
||||
signer_type: SignatureType::SignatureV4,
|
||||
..Default::default()
|
||||
}));
|
||||
} else {
|
||||
return Err(std::io::Error::other("insufficient parameters for S3 backend authentication"));
|
||||
}
|
||||
let opts = Options {
|
||||
creds,
|
||||
secure: u.scheme() == "https",
|
||||
//transport: GLOBAL_RemoteTargetTransport,
|
||||
region: conf.region.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
let host = u
|
||||
.host()
|
||||
.ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
|
||||
let client = TransitionClient::new(&host.to_string(), opts, "s3").await?;
|
||||
|
||||
let client = Arc::new(client);
|
||||
let core = TransitionCore(Arc::clone(&client));
|
||||
Ok(Self {
|
||||
client,
|
||||
core,
|
||||
bucket: conf.bucket.clone(),
|
||||
prefix: conf.prefix.clone().trim_matches('/').to_string(),
|
||||
storage_class: conf.storage_class.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn get_dest(&self, object: &str) -> String {
|
||||
let mut dest_obj = object.to_string();
|
||||
if self.prefix != "" {
|
||||
dest_obj = format!("{}/{}", &self.prefix, object);
|
||||
}
|
||||
return dest_obj;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_rejects_loopback_endpoint_before_network_setup() {
|
||||
let conf = TierS3 {
|
||||
endpoint: "https://127.0.0.1:9000".to_string(),
|
||||
bucket: "tier-bucket".to_string(),
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
region: "us-east-1".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
match WarmBackendS3::new(&conf, "tier").await {
|
||||
Ok(_) => panic!("loopback endpoint should be rejected"),
|
||||
Err(err) => assert!(err.to_string().contains("not allowed")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl WarmBackend for WarmBackendS3 {
|
||||
async fn put_with_meta(
|
||||
&self,
|
||||
object: &str,
|
||||
r: ReaderImpl,
|
||||
length: i64,
|
||||
meta: HashMap<String, String>,
|
||||
) -> Result<String, std::io::Error> {
|
||||
let client = self.client.clone();
|
||||
let res = client
|
||||
.put_object(&self.bucket, &self.get_dest(object), r, length, &{
|
||||
let mut opts = build_transition_put_options(self.storage_class.clone(), meta);
|
||||
opts.send_content_md5 = true;
|
||||
opts
|
||||
})
|
||||
.await?;
|
||||
Ok(res.version_id)
|
||||
}
|
||||
|
||||
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
|
||||
self.put_with_meta(object, r, length, HashMap::new()).await
|
||||
}
|
||||
|
||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
let mut gopts = GetObjectOptions::default();
|
||||
|
||||
if rv != "" {
|
||||
gopts.version_id = rv.to_string();
|
||||
}
|
||||
if opts.start_offset >= 0 && opts.length > 0 {
|
||||
if let Err(err) = gopts.set_range(opts.start_offset, opts.start_offset + opts.length - 1) {
|
||||
return Err(std::io::Error::other(err));
|
||||
}
|
||||
}
|
||||
let c = TransitionCore(Arc::clone(&self.client));
|
||||
let (_, _, r) = c.get_object(&self.bucket, &self.get_dest(object), &gopts).await?;
|
||||
|
||||
Ok(r)
|
||||
}
|
||||
|
||||
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||
let mut ropts = RemoveObjectOptions::default();
|
||||
if rv != "" {
|
||||
ropts.version_id = rv.to_string();
|
||||
}
|
||||
let client = self.client.clone();
|
||||
match client.remove_object(&self.bucket, &self.get_dest(object), ropts).await {
|
||||
None => Ok(()),
|
||||
Some(err) => Err(std::io::Error::other(err)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
let result = self
|
||||
.core
|
||||
.list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1)
|
||||
.await?;
|
||||
|
||||
Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use url::Url;
|
||||
|
||||
use aws_config::meta::region::RegionProviderChain;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
|
||||
use crate::client::{
|
||||
api_get_options::GetObjectOptions,
|
||||
api_put_object::PutObjectOptions,
|
||||
api_remove::RemoveObjectOptions,
|
||||
transition_api::{ReadCloser, ReaderImpl},
|
||||
};
|
||||
use crate::error::ErrorResponse;
|
||||
use crate::error::error_resp_to_object_err;
|
||||
use crate::services::tier::{
|
||||
tier_config::TierS3,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts},
|
||||
};
|
||||
|
||||
pub struct WarmBackendS3 {
|
||||
pub client: Arc<Client>,
|
||||
pub bucket: String,
|
||||
pub prefix: String,
|
||||
pub storage_class: String,
|
||||
}
|
||||
|
||||
impl WarmBackendS3 {
|
||||
pub async fn new(conf: &TierS3, tier: &str) -> Result<Self, std::io::Error> {
|
||||
let u = match Url::parse(&conf.endpoint) {
|
||||
Ok(u) => u,
|
||||
Err(err) => {
|
||||
return Err(std::io::Error::other(err.to_string()));
|
||||
}
|
||||
};
|
||||
|
||||
if conf.aws_role_web_identity_token_file == "" && conf.aws_role_arn != ""
|
||||
|| conf.aws_role_web_identity_token_file != "" && conf.aws_role_arn == ""
|
||||
{
|
||||
return Err(std::io::Error::other("both the token file and the role ARN are required"));
|
||||
} else if conf.access_key == "" && conf.secret_key != "" || conf.access_key != "" && conf.secret_key == "" {
|
||||
return Err(std::io::Error::other("both the access and secret keys are required"));
|
||||
} else if conf.aws_role
|
||||
&& (conf.aws_role_web_identity_token_file != ""
|
||||
|| conf.aws_role_arn != ""
|
||||
|| conf.access_key != ""
|
||||
|| conf.secret_key != "")
|
||||
{
|
||||
return Err(std::io::Error::other(
|
||||
"AWS Role cannot be activated with static credentials or the web identity token file",
|
||||
));
|
||||
} else if conf.bucket == "" {
|
||||
return Err(std::io::Error::other("no bucket name was provided"));
|
||||
}
|
||||
|
||||
let creds;
|
||||
if conf.access_key != "" && conf.secret_key != "" {
|
||||
creds = Credentials::new(
|
||||
conf.access_key.clone(), // access_key_id
|
||||
conf.secret_key.clone(), // secret_access_key
|
||||
None, // session_token (optional)
|
||||
None,
|
||||
"Static",
|
||||
);
|
||||
} else {
|
||||
return Err(std::io::Error::other("insufficient parameters for S3 backend authentication"));
|
||||
}
|
||||
let region_provider = RegionProviderChain::default_provider().or_else(Region::new(conf.region.clone()));
|
||||
#[allow(deprecated)]
|
||||
let config = aws_config::from_env()
|
||||
.endpoint_url(conf.endpoint.clone())
|
||||
.region(region_provider)
|
||||
.credentials_provider(creds)
|
||||
.load()
|
||||
.await;
|
||||
let client = Client::new(&config);
|
||||
let client = Arc::new(client);
|
||||
Ok(Self {
|
||||
client,
|
||||
bucket: conf.bucket.clone(),
|
||||
prefix: conf.prefix.clone().trim_matches('/').to_string(),
|
||||
storage_class: conf.storage_class.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn get_dest(&self, object: &str) -> String {
|
||||
let mut dest_obj = object.to_string();
|
||||
if self.prefix != "" {
|
||||
dest_obj = format!("{}/{}", &self.prefix, object);
|
||||
}
|
||||
return dest_obj;
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl WarmBackend for WarmBackendS3 {
|
||||
async fn put_with_meta(
|
||||
&self,
|
||||
object: &str,
|
||||
r: ReaderImpl,
|
||||
length: i64,
|
||||
meta: HashMap<String, String>,
|
||||
) -> Result<String, std::io::Error> {
|
||||
let client = self.client.clone();
|
||||
let Ok(res) = client
|
||||
.put_object()
|
||||
.bucket(&self.bucket)
|
||||
.key(&self.get_dest(object))
|
||||
.body(match r {
|
||||
ReaderImpl::Body(content_body) => ByteStream::from(content_body.to_vec()),
|
||||
ReaderImpl::ObjectBody(mut content_body) => ByteStream::from(content_body.read_all().await?),
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
else {
|
||||
return Err(std::io::Error::other("put_object error"));
|
||||
};
|
||||
|
||||
Ok(res.version_id().unwrap_or("").to_string())
|
||||
}
|
||||
|
||||
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
|
||||
self.put_with_meta(object, r, length, HashMap::new()).await
|
||||
}
|
||||
|
||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
let client = self.client.clone();
|
||||
let mut req = client.get_object().bucket(&self.bucket).key(&self.get_dest(object));
|
||||
|
||||
if !rv.is_empty() {
|
||||
req = req.version_id(rv);
|
||||
}
|
||||
|
||||
if opts.start_offset >= 0 && opts.length > 0 {
|
||||
let end = opts
|
||||
.start_offset
|
||||
.checked_add(opts.length)
|
||||
.and_then(|v| v.checked_sub(1))
|
||||
.ok_or_else(|| std::io::Error::other("invalid range: overflow"))?;
|
||||
req = req.range(format!("bytes={}-{}", opts.start_offset, end));
|
||||
}
|
||||
|
||||
let res = req.send().await.map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||
|
||||
Ok(ReadCloser::new(std::io::Cursor::new(
|
||||
res.body.collect().await.map(|data| data.into_bytes().to_vec())?,
|
||||
)))
|
||||
}
|
||||
|
||||
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||
let client = self.client.clone();
|
||||
let mut req = client.delete_object().bucket(&self.bucket).key(&self.get_dest(object));
|
||||
|
||||
if !rv.is_empty() {
|
||||
req = req.version_id(rv);
|
||||
}
|
||||
|
||||
req.send().await.map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
let client = self.client.clone();
|
||||
let Ok(res) = client
|
||||
.list_objects_v2()
|
||||
.bucket(&self.bucket)
|
||||
//.max_keys(10)
|
||||
//.into_paginator()
|
||||
.send()
|
||||
.await
|
||||
else {
|
||||
return Err(std::io::Error::other("list_objects_v2 error"));
|
||||
};
|
||||
|
||||
Ok(res.common_prefixes.unwrap_or_default().len() > 0 || res.contents.unwrap_or_default().len() > 0)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use crate::services::tier::{
|
||||
tier_config::TierTencent,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
const MAX_PARTS_COUNT: i64 = 10000;
|
||||
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
|
||||
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
|
||||
|
||||
pub struct WarmBackendTencent(WarmBackendS3);
|
||||
|
||||
impl WarmBackendTencent {
|
||||
pub async fn new(conf: &TierTencent, tier: &str) -> Result<Self, std::io::Error> {
|
||||
if conf.access_key == "" || conf.secret_key == "" {
|
||||
return Err(std::io::Error::other("both access and secret keys are required"));
|
||||
}
|
||||
|
||||
if conf.bucket == "" {
|
||||
return Err(std::io::Error::other("no bucket name was provided"));
|
||||
}
|
||||
|
||||
let u = match url::Url::parse(&conf.endpoint) {
|
||||
Ok(u) => u,
|
||||
Err(e) => {
|
||||
return Err(std::io::Error::other(e.to_string()));
|
||||
}
|
||||
};
|
||||
|
||||
let creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
secret_access_key: conf.secret_key.clone(),
|
||||
session_token: "".to_string(),
|
||||
signer_type: SignatureType::SignatureV4,
|
||||
..Default::default()
|
||||
}));
|
||||
let opts = Options {
|
||||
creds,
|
||||
secure: u.scheme() == "https",
|
||||
//transport: GLOBAL_RemoteTargetTransport,
|
||||
trailing_headers: true,
|
||||
region: conf.region.clone(),
|
||||
bucket_lookup: BucketLookupType::BucketLookupDNS,
|
||||
..Default::default()
|
||||
};
|
||||
let scheme = u.scheme();
|
||||
let default_port = if scheme == "https" { 443 } else { 80 };
|
||||
let host = u
|
||||
.host_str()
|
||||
.ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
|
||||
let client = TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, "tencent").await?;
|
||||
|
||||
let client = Arc::new(client);
|
||||
let core = TransitionCore(Arc::clone(&client));
|
||||
Ok(Self(WarmBackendS3 {
|
||||
client,
|
||||
core,
|
||||
bucket: conf.bucket.clone(),
|
||||
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
|
||||
storage_class: "".to_string(),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl WarmBackend for WarmBackendTencent {
|
||||
async fn put_with_meta(
|
||||
&self,
|
||||
object: &str,
|
||||
r: ReaderImpl,
|
||||
length: i64,
|
||||
meta: HashMap<String, String>,
|
||||
) -> Result<String, std::io::Error> {
|
||||
let part_size = optimal_part_size(length)?;
|
||||
let client = self.0.client.clone();
|
||||
let res = client
|
||||
.put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &{
|
||||
let mut opts = build_transition_put_options(self.0.storage_class.clone(), meta);
|
||||
opts.part_size = part_size as u64;
|
||||
opts.disable_content_sha256 = true;
|
||||
opts
|
||||
})
|
||||
.await?;
|
||||
//self.ToObjectError(err, object)
|
||||
Ok(res.version_id)
|
||||
}
|
||||
|
||||
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
|
||||
self.put_with_meta(object, r, length, HashMap::new()).await
|
||||
}
|
||||
|
||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
self.0.get(object, rv, opts).await
|
||||
}
|
||||
|
||||
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||
self.0.remove(object, rv).await
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
self.0.in_use().await
|
||||
}
|
||||
}
|
||||
|
||||
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
let mut object_size = object_size;
|
||||
if object_size == -1 {
|
||||
object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
|
||||
}
|
||||
|
||||
if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
|
||||
return Err(std::io::Error::other("entity too large"));
|
||||
}
|
||||
|
||||
let configured_part_size = MIN_PART_SIZE;
|
||||
let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64;
|
||||
part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64;
|
||||
|
||||
let part_size = part_size_flt as i64;
|
||||
if part_size == 0 {
|
||||
return Ok(MIN_PART_SIZE);
|
||||
}
|
||||
Ok(part_size)
|
||||
}
|
||||
Reference in New Issue
Block a user