gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-08 14:23:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

merge main

Browse Source
This commit is contained in:
weisd
2025-06-09 15:31:11 +08:00
parent 91c099e35f cecde068e1
commit f85ef06783
84 changed files with 10192 additions and 4976 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/Cargo.toml
+7 -1
View File
@@ -22,6 +22,7 @@ api = { workspace = true }
appauth = { workspace = true }
atoi = { workspace = true }
atomic_enum = { workspace = true }
aws-sdk-s3 = { workspace = true }
axum.workspace = true
axum-extra = { workspace = true }
axum-server = { workspace = true }
@@ -43,14 +44,18 @@ hyper-util.workspace = true
http.workspace = true
http-body.workspace = true
iam = { workspace = true }
include_dir = { workspace = true }
jsonwebtoken = { workspace = true }
lock.workspace = true
matchit = { workspace = true }
mime.workspace = true
mime_guess = { workspace = true }
opentelemetry = { workspace = true }
percent-encoding = { workspace = true }
pin-project-lite.workspace = true
protos.workspace = true
query = { workspace = true }
regex = { workspace = true }
rmp-serde.workspace = true
rustfs-config = { workspace = true }
rustfs-event-notifier = { workspace = true }
@@ -64,6 +69,7 @@ serde_json.workspace = true
serde_urlencoded = { workspace = true }
shadow-rs = { workspace = true, features = ["build", "metadata"] }
socket2 = { workspace = true }
thiserror = { workspace = true }
tracing.workspace = true
time = { workspace = true, features = ["parsing", "formatting", "serde"] }
tokio-util.workspace = true
@@ -85,9 +91,9 @@ tower-http = { workspace = true, features = [
"compression-gzip",
"cors",
] }
urlencoding = { workspace = true }
uuid = { workspace = true }
rustfs-filemeta.workspace = true
thiserror.workspace = true
rustfs-rio.workspace = true
[target.'cfg(target_os = "linux")'.dependencies]
rustfs/src/admin/handlers.rs
+281
View File
@@ -2,9 +2,13 @@ use super::router::Operation;
use crate::auth::check_key_valid;
use crate::auth::get_condition_values;
use crate::auth::get_session_token;
use crate::error::ApiError;
use bytes::Bytes;
use ecstore::admin_server_info::get_server_info;
use ecstore::bucket::metadata_sys::{self, get_replication_config};
use ecstore::bucket::target::BucketTarget;
use ecstore::bucket::versioning_sys::BucketVersioningSys;
use ecstore::cmd::bucket_targets::{self, GLOBAL_Bucket_Target_Sys};
use ecstore::error::StorageError;
use ecstore::global::GLOBAL_ALlHealState;
use ecstore::heal::data_usage::load_data_usage_from_backend;
@@ -23,9 +27,11 @@ use http::{HeaderMap, Uri};
use hyper::StatusCode;
use iam::get_global_action_cred;
use iam::store::MappedPolicy;
// use lazy_static::lazy_static;
use madmin::metrics::RealtimeMetrics;
use madmin::utils::parse_duration;
use matchit::Params;
use percent_encoding::{AsciiSet, CONTROLS, percent_encode};
use policy::policy::Args;
use policy::policy::BucketPolicy;
use policy::policy::action::Action;
@@ -36,6 +42,7 @@ use s3s::stream::{ByteStream, DynByteStream};
use s3s::{Body, S3Error, S3Request, S3Response, S3Result, s3_error};
use s3s::{S3ErrorCode, StdError};
use serde::{Deserialize, Serialize};
// use serde_json::to_vec;
use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
use std::pin::Pin;
@@ -47,6 +54,7 @@ use tokio::time::interval;
use tokio::{select, spawn};
use tokio_stream::wrappers::ReceiverStream;
use tracing::{error, info, warn};
// use url::UrlQuery;
pub mod event;
pub mod group;
@@ -57,6 +65,7 @@ pub mod service_account;
pub mod sts;
pub mod trace;
pub mod user;
use urlencoding::decode;
#[derive(Debug, Serialize, Default)]
#[serde(rename_all = "PascalCase", default)]
@@ -745,6 +754,278 @@ impl Operation for BackgroundHealStatusHandler {
}
}
fn extract_query_params(uri: &Uri) -> HashMap<String, String> {
let mut params = HashMap::new();
if let Some(query) = uri.query() {
query.split('&').for_each(|pair| {
if let Some((key, value)) = pair.split_once('=') {
params.insert(key.to_string(), value.to_string());
}
});
}
params
}
//disable encrypto from client because rustfs use len 8 Nonce but rustfs use 12 len Nonce
#[allow(dead_code)]
fn is_local_host(_host: String) -> bool {
false
}
//awscurl --service s3 --region us-east-1 --access_key rustfsadmin --secret_key rustfsadmin "http://:9000/rustfs/admin/v3/replicationmetrics?bucket=1"
pub struct GetReplicationMetricsHandler {}
#[async_trait::async_trait]
impl Operation for GetReplicationMetricsHandler {
async fn call(&self, _req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
error!("GetReplicationMetricsHandler");
let querys = extract_query_params(&_req.uri);
if let Some(bucket) = querys.get("bucket") {
error!("get bucket:{} metris", bucket);
}
//return Err(s3_error!(InvalidArgument, "Invalid bucket name"));
//Ok(S3Response::with_headers((StatusCode::OK, Body::from()), header))
return Ok(S3Response::new((StatusCode::OK, Body::from("Ok".to_string()))));
}
}
pub struct SetRemoteTargetHandler {}
#[async_trait::async_trait]
impl Operation for SetRemoteTargetHandler {
async fn call(&self, mut _req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
//return Ok(S3Response::new((StatusCode::OK, Body::from("OK".to_string()))));
// println!("handle MetricsHandler, params: {:?}", _req.input);
info!("handle MetricsHandler, params: {:?}", _req.credentials);
let querys = extract_query_params(&_req.uri);
let Some(_cred) = _req.credentials else {
error!("credentials null");
return Err(s3_error!(InvalidRequest, "get cred failed"));
};
let _is_owner = true; // 先按 true 处理，后期根据请求决定
let body = _req.input.store_all_unlimited().await.unwrap();
//println!("body: {}", std::str::from_utf8(&body.clone()).unwrap());
//println!("bucket is:{}", bucket.clone());
if let Some(bucket) = querys.get("bucket") {
if bucket.is_empty() {
println!("have bucket: {}", bucket);
return Ok(S3Response::new((StatusCode::OK, Body::from("fuck".to_string()))));
}
let Some(store) = new_object_layer_fn() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
// let binfo:BucketInfo = store
// .get_bucket_info(bucket, &ecstore::store_api::BucketOptions::default()).await;
match store
.get_bucket_info(bucket, &ecstore::store_api::BucketOptions::default())
.await
{
Ok(info) => {
println!("Bucket Info: {:?}", info);
if !info.versionning {
return Ok(S3Response::new((StatusCode::FORBIDDEN, Body::from("bucket need versioned".to_string()))));
}
}
Err(err) => {
eprintln!("Error: {:?}", err);
return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from("empty bucket".to_string()))));
}
}
let mut remote_target: BucketTarget = serde_json::from_slice(&body).map_err(ApiError::other)?; // 错误会被传播
remote_target.source_bucket = bucket.clone();
info!("remote target {} And arn is:", remote_target.source_bucket.clone());
if let Some(val) = remote_target.arn.clone() {
info!("arn is {}", val);
}
if let Some(sys) = GLOBAL_Bucket_Target_Sys.get() {
let (arn, exist) = sys.get_remote_arn(bucket, Some(&remote_target), "").await;
info!("exist: {} {}", exist, arn.clone().unwrap_or_default());
if exist && arn.is_some() {
let jsonarn = serde_json::to_string(&arn).expect("failed to serialize");
//Ok(S3Response::new)
return Ok(S3Response::new((StatusCode::OK, Body::from(jsonarn))));
} else {
remote_target.arn = arn;
match sys.set_target(bucket, &remote_target, false, false).await {
Ok(_) => {
{
//todo 各种持久化的工作
let targets = sys.list_targets(Some(bucket), None).await;
info!("targets is {}", targets.len());
match serde_json::to_vec(&targets) {
Ok(json) => {
//println!("json is:{:?}", json.clone().to_ascii_lowercase());
//metadata_sys::GLOBAL_BucketMetadataSys::
//BUCKET_TARGETS_FILE: &str = "bucket-targets.json"
let _ = metadata_sys::update(bucket, "bucket-targets.json", json).await;
// if let Err(err) = metadata_sys::GLOBAL_BucketMetadataSys.get().
// .update(ctx, bucket, "bucketTargetsFile", tgt_bytes)
// .await
// {
// write_error_response(ctx, &err)?;
// return Err(err);
// }
}
Err(e) => {
error!("序列化失败{}", e);
}
}
}
let jsonarn = serde_json::to_string(&remote_target.arn.clone()).expect("failed to serialize");
return Ok(S3Response::new((StatusCode::OK, Body::from(jsonarn))));
}
Err(e) => {
error!("set target error {}", e);
return Ok(S3Response::new((
StatusCode::BAD_REQUEST,
Body::from("remote target not ready".to_string()),
)));
}
}
}
} else {
error!("GLOBAL_BUCKET _TARGET_SYS is not initialized");
return Err(S3Error::with_message(
S3ErrorCode::InternalError,
"GLOBAL_BUCKET_TARGET_SYS is not initialized".to_string(),
));
}
}
// return Err(s3_error!(InvalidArgument));
return Ok(S3Response::new((StatusCode::OK, Body::from("Ok".to_string()))));
}
}
pub struct ListRemoteTargetHandler {}
#[async_trait::async_trait]
impl Operation for ListRemoteTargetHandler {
async fn call(&self, _req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("list GetRemoteTargetHandler, params: {:?}", _req.credentials);
let querys = extract_query_params(&_req.uri);
let Some(_cred) = _req.credentials else {
error!("credentials null");
return Err(s3_error!(InvalidRequest, "get cred failed"));
};
if let Some(bucket) = querys.get("bucket") {
if bucket.is_empty() {
error!("bucket parameter is empty");
return Ok(S3Response::new((
StatusCode::BAD_REQUEST,
Body::from("Bucket parameter is required".to_string()),
)));
}
let Some(store) = new_object_layer_fn() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not initialized".to_string()));
};
match store
.get_bucket_info(bucket, &ecstore::store_api::BucketOptions::default())
.await
{
Ok(info) => {
println!("Bucket Info: {:?}", info);
if !info.versionning {
return Ok(S3Response::new((
StatusCode::FORBIDDEN,
Body::from("Bucket needs versioning".to_string()),
)));
}
}
Err(err) => {
eprintln!("Error fetching bucket info: {:?}", err);
return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from("Invalid bucket".to_string()))));
}
}
if let Some(sys) = GLOBAL_Bucket_Target_Sys.get() {
let targets = sys.list_targets(Some(bucket), None).await;
error!("target sys len {}", targets.len());
if targets.is_empty() {
return Ok(S3Response::new((
StatusCode::NOT_FOUND,
Body::from("No remote targets found".to_string()),
)));
}
let json_targets = serde_json::to_string(&targets).map_err(|e| {
error!("Serialization error: {}", e);
S3Error::with_message(S3ErrorCode::InternalError, "Failed to serialize targets".to_string())
})?;
return Ok(S3Response::new((StatusCode::OK, Body::from(json_targets))));
} else {
println!("GLOBAL_BUCKET_TARGET_SYS is not initialized");
return Err(S3Error::with_message(
S3ErrorCode::InternalError,
"GLOBAL_BUCKET_TARGET_SYS is not initialized".to_string(),
));
}
}
println!("Bucket parameter missing in request");
Ok(S3Response::new((
StatusCode::BAD_REQUEST,
Body::from("Bucket parameter is required".to_string()),
)))
//return Err(s3_error!(NotImplemented));
}
}
const COLON: AsciiSet = CONTROLS.add(b':');
pub struct RemoveRemoteTargetHandler {}
#[async_trait::async_trait]
impl Operation for RemoveRemoteTargetHandler {
async fn call(&self, _req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
error!("remove remote target called");
let querys = extract_query_params(&_req.uri);
if let Some(arnstr) = querys.get("arn") {
if let Some(bucket) = querys.get("bucket") {
if bucket.is_empty() {
error!("bucket parameter is empty");
return Ok(S3Response::new((StatusCode::NOT_FOUND, Body::from("bucket not found".to_string()))));
}
let _arn = bucket_targets::ARN::parse(arnstr);
match get_replication_config(bucket).await {
Ok((conf, _ts)) => {
for ru in conf.rules {
let encoded = percent_encode(ru.destination.bucket.as_bytes(), &COLON);
let encoded_str = encoded.to_string();
if *arnstr == encoded_str {
error!("target in use");
return Ok(S3Response::new((StatusCode::FORBIDDEN, Body::from("Ok".to_string()))));
}
info!("bucket: {} and arn str is {} ", encoded_str, arnstr);
}
}
Err(err) => {
error!("get replication config err: {}", err);
return Ok(S3Response::new((StatusCode::NOT_FOUND, Body::from(err.to_string()))));
}
}
//percent_decode_str(&arnstr);
let decoded_str = decode(arnstr).unwrap();
error!("need delete target is {}", decoded_str);
bucket_targets::remove_bucket_target(bucket, arnstr).await;
}
}
//return Err(s3_error!(InvalidArgument, "Invalid bucket name"));
//Ok(S3Response::with_headers((StatusCode::OK, Body::from()), header))
return Ok(S3Response::new((StatusCode::OK, Body::from("Ok".to_string()))));
}
}
#[cfg(test)]
mod test {
use ecstore::heal::heal_commands::HealOpts;
rustfs/src/admin/mod.rs
+49
View File
@@ -10,12 +10,14 @@ use handlers::{
sts, user,
};
use handlers::{GetReplicationMetricsHandler, ListRemoteTargetHandler, RemoveRemoteTargetHandler, SetRemoteTargetHandler};
use hyper::Method;
use router::{AdminOperation, S3Router};
use rpc::regist_rpc_route;
use s3s::route::S3Route;
const ADMIN_PREFIX: &str = "/rustfs/admin";
const RUSTFS_ADMIN_PREFIX: &str = "/rustfs/admin";
pub fn make_admin_route() -> std::io::Result<impl S3Route> {
let mut r: S3Router<AdminOperation> = S3Router::new();
@@ -227,6 +229,53 @@ fn register_user_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()>
AdminOperation(&AddServiceAccount {}),
)?;
r.insert(
Method::GET,
format!("{}{}", RUSTFS_ADMIN_PREFIX, "/v3/list-remote-targets").as_str(),
AdminOperation(&ListRemoteTargetHandler {}),
)?;
r.insert(
Method::GET,
format!("{}{}", ADMIN_PREFIX, "/v3/list-remote-targets").as_str(),
AdminOperation(&ListRemoteTargetHandler {}),
)?;
r.insert(
Method::GET,
format!("{}{}", RUSTFS_ADMIN_PREFIX, "/v3/replicationmetrics").as_str(),
AdminOperation(&GetReplicationMetricsHandler {}),
)?;
r.insert(
Method::GET,
format!("{}{}", ADMIN_PREFIX, "/v3/replicationmetrics").as_str(),
AdminOperation(&GetReplicationMetricsHandler {}),
)?;
r.insert(
Method::PUT,
format!("{}{}", RUSTFS_ADMIN_PREFIX, "/v3/set-remote-target").as_str(),
AdminOperation(&SetRemoteTargetHandler {}),
)?;
r.insert(
Method::PUT,
format!("{}{}", ADMIN_PREFIX, "/v3/set-remote-target").as_str(),
AdminOperation(&SetRemoteTargetHandler {}),
)?;
r.insert(
Method::DELETE,
format!("{}{}", RUSTFS_ADMIN_PREFIX, "/v3/remove-remote-target").as_str(),
AdminOperation(&RemoveRemoteTargetHandler {}),
)?;
r.insert(
Method::DELETE,
format!("{}{}", ADMIN_PREFIX, "/v3/remove-remote-target").as_str(),
AdminOperation(&RemoveRemoteTargetHandler {}),
)?;
// list-canned-policies?bucket=xxx
r.insert(
Method::GET,
rustfs/src/admin/router.rs
+2 -1
View File
@@ -14,6 +14,7 @@ use s3s::route::S3Route;
use s3s::s3_error;
use super::ADMIN_PREFIX;
use super::RUSTFS_ADMIN_PREFIX;
use super::rpc::RPC_PREFIX;
pub struct S3Router<T> {
@@ -63,7 +64,7 @@ where
}
}
uri.path().starts_with(ADMIN_PREFIX) || uri.path().starts_with(RPC_PREFIX)
uri.path().starts_with(ADMIN_PREFIX) || uri.path().starts_with(RPC_PREFIX) || uri.path().starts_with(RUSTFS_ADMIN_PREFIX)
}
async fn call(&self, req: S3Request<Body>) -> S3Result<S3Response<Body>> {
rustfs/src/grpc.rs
+3
View File
@@ -224,6 +224,8 @@ impl Node for NodeService {
}));
}
};
println!("bucket info {}", bucket_info.clone());
Ok(tonic::Response::new(GetBucketInfoResponse {
success: true,
bucket_info,
@@ -231,6 +233,7 @@ impl Node for NodeService {
}))
}
// println!("vuc")
Err(err) => Ok(tonic::Response::new(GetBucketInfoResponse {
success: false,
bucket_info: String::new(),
rustfs/src/main.rs
+6
View File
@@ -24,6 +24,7 @@ use common::{
};
use ecstore::StorageAPI;
use ecstore::bucket::metadata_sys::init_bucket_metadata_sys;
use ecstore::cmd::bucket_replication::init_bucket_replication_pool;
use ecstore::config as ecconfig;
use ecstore::config::GLOBAL_ConfigSys;
use ecstore::heal::background_heal_ops::init_auto_heal;
@@ -536,6 +537,11 @@ async fn run(opt: config::Opt) -> Result<()> {
init_console_cfg(local_ip, server_port);
print_server_info();
init_bucket_replication_pool().await;
init_console_cfg(local_ip, server_port);
print_server_info();
if opt.console_enable {
rustfs/src/storage/ecfs.rs
+184 -18
View File
@@ -12,6 +12,8 @@ use api::query::Context;
use api::query::Query;
use api::server::dbms::DatabaseManagerSystem;
use bytes::Bytes;
use chrono::DateTime;
use chrono::Utc;
use datafusion::arrow::csv::WriterBuilder as CsvWriterBuilder;
use datafusion::arrow::json::WriterBuilder as JsonWriterBuilder;
use datafusion::arrow::json::writer::JsonArray;
@@ -28,6 +30,9 @@ use ecstore::bucket::policy_sys::PolicySys;
use ecstore::bucket::tagging::decode_tags;
use ecstore::bucket::tagging::encode_tags;
use ecstore::bucket::versioning_sys::BucketVersioningSys;
use ecstore::cmd::bucket_replication::get_must_replicate_options;
use ecstore::cmd::bucket_replication::must_replicate;
use ecstore::cmd::bucket_replication::schedule_replication;
use ecstore::error::StorageError;
use ecstore::new_object_layer_fn;
use ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
@@ -41,8 +46,11 @@ use ecstore::store_api::ObjectIO;
use ecstore::store_api::ObjectOptions;
use ecstore::store_api::ObjectToDelete;
use ecstore::store_api::PutObjReader;
use ecstore::store_api::RESERVED_METADATA_PREFIX_LOWER;
use ecstore::store_api::StorageAPI;
// use ecstore::store_api::RESERVED_METADATA_PREFIX;
use ecstore::cmd::bucket_replication::ReplicationStatusType;
use ecstore::cmd::bucket_replication::ReplicationType;
use ecstore::store_api::RESERVED_METADATA_PREFIX_LOWER;
use ecstore::utils::path::path_join_buf;
use ecstore::utils::xml;
use ecstore::xhttp;
@@ -918,6 +926,7 @@ impl S3 for FS {
return self.put_object_extract(req).await;
}
info!("put object");
let input = req.input;
if let Some(ref storage_class) = input.storage_class {
@@ -975,10 +984,27 @@ impl S3 for FS {
metadata.insert(xhttp::AMZ_OBJECT_TAGGING.to_owned(), tags);
}
let opts: ObjectOptions = put_opts(&bucket, &key, version_id, &req.headers, Some(metadata))
let mt = metadata.clone();
let mt2 = metadata.clone();
let opts: ObjectOptions = put_opts(&bucket, &key, version_id, &req.headers, Some(mt))
.await
.map_err(ApiError::from)?;
let repoptions =
get_must_replicate_options(&mt2, "", ReplicationStatusType::Unknown, ReplicationType::ObjectReplicationType, &opts);
let dsc = must_replicate(&bucket, &key, &repoptions).await;
warn!("dsc {}", &dsc.replicate_any().clone());
if dsc.replicate_any() {
let k = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-timestamp");
let now: DateTime<Utc> = Utc::now();
let formatted_time = now.to_rfc3339();
metadata.insert(k, formatted_time);
let k = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-status");
metadata.insert(k, dsc.pending_status());
}
debug!("put_object opts {:?}", &opts);
let obj_info = store
@@ -986,9 +1012,17 @@ impl S3 for FS {
.await
.map_err(ApiError::from)?;
let e_tag = obj_info.etag;
let e_tag = obj_info.etag.clone();
// store.put_object(bucket, object, data, opts);
let repoptions =
get_must_replicate_options(&mt2, "", ReplicationStatusType::Unknown, ReplicationType::ObjectReplicationType, &opts);
let dsc = must_replicate(&bucket, &key, &repoptions).await;
if dsc.replicate_any() {
let objectlayer = new_object_layer_fn();
schedule_replication(obj_info, objectlayer.unwrap(), dsc, 1).await;
}
let output = PutObjectOutput {
e_tag,
@@ -1155,17 +1189,30 @@ impl S3 for FS {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
let oi = store
let obj_info = store
.complete_multipart_upload(&bucket, &key, &upload_id, uploaded_parts, opts)
.await
.map_err(ApiError::from)?;
let output = CompleteMultipartUploadOutput {
bucket: Some(bucket),
key: Some(key),
e_tag: oi.etag,
bucket: Some(bucket.clone()),
key: Some(key.clone()),
e_tag: obj_info.etag.clone(),
location: Some("us-east-1".to_string()),
..Default::default()
};
let mt2 = HashMap::new();
let repoptions =
get_must_replicate_options(&mt2, "", ReplicationStatusType::Unknown, ReplicationType::ObjectReplicationType, opts);
let dsc = must_replicate(&bucket, &key, &repoptions).await;
if dsc.replicate_any() {
warn!("need multipart replication");
let objectlayer = new_object_layer_fn();
schedule_replication(obj_info, objectlayer.unwrap(), dsc, 1).await;
}
Ok(S3Response::new(output))
}
@@ -1730,17 +1777,35 @@ impl S3 for FS {
.await
.map_err(ApiError::from)?;
let replication_configuration = match metadata_sys::get_replication_config(&bucket).await {
let rcfg = match metadata_sys::get_replication_config(&bucket).await {
Ok((cfg, _created)) => Some(cfg),
Err(err) => {
warn!("get_object_lock_config err {:?}", err);
None
error!("get_replication_config err {:?}", err);
return Err(ApiError::from(err).into());
}
};
Ok(S3Response::new(GetBucketReplicationOutput {
replication_configuration,
}))
if rcfg.is_none() {
return Err(S3Error::with_message(S3ErrorCode::NoSuchBucket, "replication not found".to_string()));
}
// Ok(S3Response::new(GetBucketReplicationOutput {
// replication_configuration: rcfg,
// }))
if rcfg.is_some() {
Ok(S3Response::new(GetBucketReplicationOutput {
replication_configuration: rcfg,
}))
} else {
let rep = ReplicationConfiguration {
role: "".to_string(),
rules: vec![],
};
Ok(S3Response::new(GetBucketReplicationOutput {
replication_configuration: Some(rep),
}))
}
}
async fn put_bucket_replication(
@@ -1752,6 +1817,7 @@ impl S3 for FS {
replication_configuration,
..
} = req.input;
warn!("put bucket replication");
let Some(store) = new_object_layer_fn() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
@@ -1791,6 +1857,7 @@ impl S3 for FS {
.map_err(ApiError::from)?;
// TODO: remove targets
error!("delete bucket");
Ok(S3Response::new(DeleteBucketReplicationOutput::default()))
}
@@ -2110,13 +2177,15 @@ impl S3 for FS {
None
};
if legal_hold.is_none() {
return Err(s3_error!(InvalidRequest, "Object does not have legal hold"));
}
let status = if let Some(v) = legal_hold {
v
} else {
ObjectLockLegalHoldStatus::OFF.to_string()
};
Ok(S3Response::new(GetObjectLegalHoldOutput {
legal_hold: Some(ObjectLockLegalHold {
status: Some(ObjectLockLegalHoldStatus::from(legal_hold.unwrap_or_default())),
status: Some(ObjectLockLegalHoldStatus::from(status)),
}),
}))
}
@@ -2178,6 +2247,103 @@ impl S3 for FS {
request_charged: Some(RequestCharged::from_static(RequestCharged::REQUESTER)),
}))
}
async fn get_object_retention(
&self,
req: S3Request<GetObjectRetentionInput>,
) -> S3Result<S3Response<GetObjectRetentionOutput>> {
let GetObjectRetentionInput {
bucket, key, version_id, ..
} = req.input;
let Some(store) = new_object_layer_fn() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
// check object lock
let _ = metadata_sys::get_object_lock_config(&bucket).await.map_err(ApiError::from)?;
let opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers)
.await
.map_err(ApiError::from)?;
let object_info = store.get_object_info(&bucket, &key, &opts).await.map_err(|e| {
error!("get_object_info failed, {}", e.to_string());
s3_error!(InternalError, "{}", e.to_string())
})?;
let mode = if let Some(ref ud) = object_info.user_defined {
ud.get("x-amz-object-lock-mode")
.map(|v| ObjectLockRetentionMode::from(v.as_str().to_string()))
} else {
None
};
let retain_until_date = if let Some(ref ud) = object_info.user_defined {
ud.get("x-amz-object-lock-retain-until-date")
.and_then(|v| OffsetDateTime::parse(v.as_str(), &Rfc3339).ok())
.map(Timestamp::from)
} else {
None
};
Ok(S3Response::new(GetObjectRetentionOutput {
retention: Some(ObjectLockRetention { mode, retain_until_date }),
}))
}
async fn put_object_retention(
&self,
req: S3Request<PutObjectRetentionInput>,
) -> S3Result<S3Response<PutObjectRetentionOutput>> {
let PutObjectRetentionInput {
bucket,
key,
retention,
version_id,
..
} = req.input;
let Some(store) = new_object_layer_fn() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
// check object lock
let _ = metadata_sys::get_object_lock_config(&bucket).await.map_err(ApiError::from)?;
// TODO: check allow
let mut eval_metadata = HashMap::new();
if let Some(v) = retention {
let mode = v.mode.map(|v| v.as_str().to_string()).unwrap_or_default();
let retain_until_date = v
.retain_until_date
.map(|v| OffsetDateTime::from(v).format(&Rfc3339).unwrap())
.unwrap_or_default();
let now = OffsetDateTime::now_utc();
eval_metadata.insert("x-amz-object-lock-mode".to_string(), mode);
eval_metadata.insert("x-amz-object-lock-retain-until-date".to_string(), retain_until_date);
eval_metadata.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "objectlock-retention-timestamp"),
format!("{}.{:09}Z", now.format(&Rfc3339).unwrap(), now.nanosecond()),
);
}
let mut opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers)
.await
.map_err(ApiError::from)?;
opts.eval_metadata = Some(eval_metadata);
store.put_object_metadata(&bucket, &key, &opts).await.map_err(|e| {
error!("put_object_metadata failed, {}", e.to_string());
s3_error!(InternalError, "{}", e.to_string())
})?;
Ok(S3Response::new(PutObjectRetentionOutput {
request_charged: Some(RequestCharged::from_static(RequestCharged::REQUESTER)),
}))
}
}
#[allow(dead_code)]
rustfs/src/storage/error.rs
+485 -485
View File
@@ -1,485 +1,485 @@
// use common::error::Error;
// use ecstore::error::StorageError;
// use s3s::{S3Error, S3ErrorCode, s3_error};
// pub fn to_s3_error(err: Error) -> S3Error {
// if let Some(storage_err) = err.downcast_ref::<StorageError>() {
// return match storage_err {
// StorageError::NotImplemented => s3_error!(NotImplemented),
// StorageError::InvalidArgument(bucket, object, version_id) => {
// s3_error!(InvalidArgument, "Invalid arguments provided for {}/{}-{}", bucket, object, version_id)
// }
// StorageError::MethodNotAllowed => s3_error!(MethodNotAllowed),
// StorageError::BucketNotFound(bucket) => {
// s3_error!(NoSuchBucket, "bucket not found {}", bucket)
// }
// StorageError::BucketNotEmpty(bucket) => s3_error!(BucketNotEmpty, "bucket not empty {}", bucket),
// StorageError::BucketNameInvalid(bucket) => s3_error!(InvalidBucketName, "invalid bucket name {}", bucket),
// StorageError::ObjectNameInvalid(bucket, object) => {
// s3_error!(InvalidArgument, "invalid object name {}/{}", bucket, object)
// }
// StorageError::BucketExists(bucket) => s3_error!(BucketAlreadyExists, "{}", bucket),
// StorageError::StorageFull => s3_error!(ServiceUnavailable, "Storage reached its minimum free drive threshold."),
// StorageError::SlowDown => s3_error!(SlowDown, "Please reduce your request rate"),
// StorageError::PrefixAccessDenied(bucket, object) => {
// s3_error!(AccessDenied, "PrefixAccessDenied {}/{}", bucket, object)
// }
// StorageError::InvalidUploadIDKeyCombination(bucket, object) => {
// s3_error!(InvalidArgument, "Invalid UploadID KeyCombination: {}/{}", bucket, object)
// }
// StorageError::MalformedUploadID(bucket) => s3_error!(InvalidArgument, "Malformed UploadID: {}", bucket),
// StorageError::ObjectNameTooLong(bucket, object) => {
// s3_error!(InvalidArgument, "Object name too long: {}/{}", bucket, object)
// }
// StorageError::ObjectNamePrefixAsSlash(bucket, object) => {
// s3_error!(InvalidArgument, "Object name contains forward slash as prefix: {}/{}", bucket, object)
// }
// StorageError::ObjectNotFound(bucket, object) => s3_error!(NoSuchKey, "{}/{}", bucket, object),
// StorageError::VersionNotFound(bucket, object, version_id) => {
// s3_error!(NoSuchVersion, "{}/{}/{}", bucket, object, version_id)
// }
// StorageError::InvalidUploadID(bucket, object, version_id) => {
// s3_error!(InvalidPart, "Invalid upload id: {}/{}-{}", bucket, object, version_id)
// }
// StorageError::InvalidVersionID(bucket, object, version_id) => {
// s3_error!(InvalidArgument, "Invalid version id: {}/{}-{}", bucket, object, version_id)
// }
// // extended
// StorageError::DataMovementOverwriteErr(bucket, object, version_id) => s3_error!(
// InvalidArgument,
// "invalid data movement operation, source and destination pool are the same for : {}/{}-{}",
// bucket,
// object,
// version_id
// ),
// // extended
// StorageError::ObjectExistsAsDirectory(bucket, object) => {
// s3_error!(InvalidArgument, "Object exists on :{} as directory {}", bucket, object)
// }
// StorageError::InvalidPart(bucket, object, version_id) => {
// s3_error!(
// InvalidPart,
// "Specified part could not be found. PartNumber {}, Expected {}, got {}",
// bucket,
// object,
// version_id
// )
// }
// StorageError::DoneForNow => s3_error!(InternalError, "DoneForNow"),
// };
// }
// if is_err_file_not_found(&err) {
// return S3Error::with_message(S3ErrorCode::NoSuchKey, format!(" ec err {}", err));
// }
// S3Error::with_message(S3ErrorCode::InternalError, format!(" ec err {}", err))
// }
// #[cfg(test)]
// mod tests {
// use super::*;
// use s3s::S3ErrorCode;
// #[test]
// fn test_to_s3_error_not_implemented() {
// let storage_err = StorageError::NotImplemented;
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::NotImplemented);
// }
// #[test]
// fn test_to_s3_error_invalid_argument() {
// let storage_err =
// StorageError::InvalidArgument("test-bucket".to_string(), "test-object".to_string(), "test-version".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
// assert!(s3_err.message().unwrap().contains("Invalid arguments provided"));
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// assert!(s3_err.message().unwrap().contains("test-object"));
// assert!(s3_err.message().unwrap().contains("test-version"));
// }
// #[test]
// fn test_to_s3_error_method_not_allowed() {
// let storage_err = StorageError::MethodNotAllowed;
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::MethodNotAllowed);
// }
// #[test]
// fn test_to_s3_error_bucket_not_found() {
// let storage_err = StorageError::BucketNotFound("test-bucket".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket);
// assert!(s3_err.message().unwrap().contains("bucket not found"));
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// }
// #[test]
// fn test_to_s3_error_bucket_not_empty() {
// let storage_err = StorageError::BucketNotEmpty("test-bucket".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::BucketNotEmpty);
// assert!(s3_err.message().unwrap().contains("bucket not empty"));
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// }
// #[test]
// fn test_to_s3_error_bucket_name_invalid() {
// let storage_err = StorageError::BucketNameInvalid("invalid-bucket-name".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidBucketName);
// assert!(s3_err.message().unwrap().contains("invalid bucket name"));
// assert!(s3_err.message().unwrap().contains("invalid-bucket-name"));
// }
// #[test]
// fn test_to_s3_error_object_name_invalid() {
// let storage_err = StorageError::ObjectNameInvalid("test-bucket".to_string(), "invalid-object".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
// assert!(s3_err.message().unwrap().contains("invalid object name"));
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// assert!(s3_err.message().unwrap().contains("invalid-object"));
// }
// #[test]
// fn test_to_s3_error_bucket_exists() {
// let storage_err = StorageError::BucketExists("existing-bucket".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::BucketAlreadyExists);
// assert!(s3_err.message().unwrap().contains("existing-bucket"));
// }
// #[test]
// fn test_to_s3_error_storage_full() {
// let storage_err = StorageError::StorageFull;
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::ServiceUnavailable);
// assert!(
// s3_err
// .message()
// .unwrap()
// .contains("Storage reached its minimum free drive threshold")
// );
// }
// #[test]
// fn test_to_s3_error_slow_down() {
// let storage_err = StorageError::SlowDown;
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::SlowDown);
// assert!(s3_err.message().unwrap().contains("Please reduce your request rate"));
// }
// #[test]
// fn test_to_s3_error_prefix_access_denied() {
// let storage_err = StorageError::PrefixAccessDenied("test-bucket".to_string(), "test-prefix".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::AccessDenied);
// assert!(s3_err.message().unwrap().contains("PrefixAccessDenied"));
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// assert!(s3_err.message().unwrap().contains("test-prefix"));
// }
// #[test]
// fn test_to_s3_error_invalid_upload_id_key_combination() {
// let storage_err = StorageError::InvalidUploadIDKeyCombination("test-bucket".to_string(), "test-object".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
// assert!(s3_err.message().unwrap().contains("Invalid UploadID KeyCombination"));
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// assert!(s3_err.message().unwrap().contains("test-object"));
// }
// #[test]
// fn test_to_s3_error_malformed_upload_id() {
// let storage_err = StorageError::MalformedUploadID("malformed-id".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
// assert!(s3_err.message().unwrap().contains("Malformed UploadID"));
// assert!(s3_err.message().unwrap().contains("malformed-id"));
// }
// #[test]
// fn test_to_s3_error_object_name_too_long() {
// let storage_err = StorageError::ObjectNameTooLong("test-bucket".to_string(), "very-long-object-name".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
// assert!(s3_err.message().unwrap().contains("Object name too long"));
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// assert!(s3_err.message().unwrap().contains("very-long-object-name"));
// }
// #[test]
// fn test_to_s3_error_object_name_prefix_as_slash() {
// let storage_err = StorageError::ObjectNamePrefixAsSlash("test-bucket".to_string(), "/invalid-object".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
// assert!(
// s3_err
// .message()
// .unwrap()
// .contains("Object name contains forward slash as prefix")
// );
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// assert!(s3_err.message().unwrap().contains("/invalid-object"));
// }
// #[test]
// fn test_to_s3_error_object_not_found() {
// let storage_err = StorageError::ObjectNotFound("test-bucket".to_string(), "missing-object".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchKey);
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// assert!(s3_err.message().unwrap().contains("missing-object"));
// }
// #[test]
// fn test_to_s3_error_version_not_found() {
// let storage_err =
// StorageError::VersionNotFound("test-bucket".to_string(), "test-object".to_string(), "missing-version".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchVersion);
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// assert!(s3_err.message().unwrap().contains("test-object"));
// assert!(s3_err.message().unwrap().contains("missing-version"));
// }
// #[test]
// fn test_to_s3_error_invalid_upload_id() {
// let storage_err =
// StorageError::InvalidUploadID("test-bucket".to_string(), "test-object".to_string(), "invalid-upload-id".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidPart);
// assert!(s3_err.message().unwrap().contains("Invalid upload id"));
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// assert!(s3_err.message().unwrap().contains("test-object"));
// assert!(s3_err.message().unwrap().contains("invalid-upload-id"));
// }
// #[test]
// fn test_to_s3_error_invalid_version_id() {
// let storage_err = StorageError::InvalidVersionID(
// "test-bucket".to_string(),
// "test-object".to_string(),
// "invalid-version-id".to_string(),
// );
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
// assert!(s3_err.message().unwrap().contains("Invalid version id"));
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// assert!(s3_err.message().unwrap().contains("test-object"));
// assert!(s3_err.message().unwrap().contains("invalid-version-id"));
// }
// #[test]
// fn test_to_s3_error_data_movement_overwrite_err() {
// let storage_err = StorageError::DataMovementOverwriteErr(
// "test-bucket".to_string(),
// "test-object".to_string(),
// "test-version".to_string(),
// );
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
// assert!(s3_err.message().unwrap().contains("invalid data movement operation"));
// assert!(s3_err.message().unwrap().contains("source and destination pool are the same"));
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// assert!(s3_err.message().unwrap().contains("test-object"));
// assert!(s3_err.message().unwrap().contains("test-version"));
// }
// #[test]
// fn test_to_s3_error_object_exists_as_directory() {
// let storage_err = StorageError::ObjectExistsAsDirectory("test-bucket".to_string(), "directory-object".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
// assert!(s3_err.message().unwrap().contains("Object exists on"));
// assert!(s3_err.message().unwrap().contains("as directory"));
// assert!(s3_err.message().unwrap().contains("test-bucket"));
// assert!(s3_err.message().unwrap().contains("directory-object"));
// }
// #[test]
// fn test_to_s3_error_insufficient_read_quorum() {
// let storage_err = StorageError::InsufficientReadQuorum;
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::SlowDown);
// assert!(
// s3_err
// .message()
// .unwrap()
// .contains("Storage resources are insufficient for the read operation")
// );
// }
// #[test]
// fn test_to_s3_error_insufficient_write_quorum() {
// let storage_err = StorageError::InsufficientWriteQuorum;
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::SlowDown);
// assert!(
// s3_err
// .message()
// .unwrap()
// .contains("Storage resources are insufficient for the write operation")
// );
// }
// #[test]
// fn test_to_s3_error_decommission_not_started() {
// let storage_err = StorageError::DecommissionNotStarted;
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
// assert!(s3_err.message().unwrap().contains("Decommission Not Started"));
// }
// #[test]
// fn test_to_s3_error_decommission_already_running() {
// let storage_err = StorageError::DecommissionAlreadyRunning;
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InternalError);
// assert!(s3_err.message().unwrap().contains("Decommission already running"));
// }
// #[test]
// fn test_to_s3_error_volume_not_found() {
// let storage_err = StorageError::VolumeNotFound("test-volume".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket);
// assert!(s3_err.message().unwrap().contains("bucket not found"));
// assert!(s3_err.message().unwrap().contains("test-volume"));
// }
// #[test]
// fn test_to_s3_error_invalid_part() {
// let storage_err = StorageError::InvalidPart(1, "expected-part".to_string(), "got-part".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidPart);
// assert!(s3_err.message().unwrap().contains("Specified part could not be found"));
// assert!(s3_err.message().unwrap().contains("PartNumber"));
// assert!(s3_err.message().unwrap().contains("expected-part"));
// assert!(s3_err.message().unwrap().contains("got-part"));
// }
// #[test]
// fn test_to_s3_error_done_for_now() {
// let storage_err = StorageError::DoneForNow;
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InternalError);
// assert!(s3_err.message().unwrap().contains("DoneForNow"));
// }
// #[test]
// fn test_to_s3_error_non_storage_error() {
// // Test with a non-StorageError
// let err = Error::from_string("Generic error message".to_string());
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InternalError);
// assert!(s3_err.message().unwrap().contains("ec err"));
// assert!(s3_err.message().unwrap().contains("Generic error message"));
// }
// #[test]
// fn test_to_s3_error_with_unicode_strings() {
// let storage_err = StorageError::BucketNotFound("测试桶".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket);
// assert!(s3_err.message().unwrap().contains("bucket not found"));
// assert!(s3_err.message().unwrap().contains("测试桶"));
// }
// #[test]
// fn test_to_s3_error_with_special_characters() {
// let storage_err = StorageError::ObjectNameInvalid("bucket-with-@#$%".to_string(), "object-with-!@#$%^&*()".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
// assert!(s3_err.message().unwrap().contains("invalid object name"));
// assert!(s3_err.message().unwrap().contains("bucket-with-@#$%"));
// assert!(s3_err.message().unwrap().contains("object-with-!@#$%^&*()"));
// }
// #[test]
// fn test_to_s3_error_with_empty_strings() {
// let storage_err = StorageError::BucketNotFound("".to_string());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket);
// assert!(s3_err.message().unwrap().contains("bucket not found"));
// }
// #[test]
// fn test_to_s3_error_with_very_long_strings() {
// let long_bucket_name = "a".repeat(1000);
// let storage_err = StorageError::BucketNotFound(long_bucket_name.clone());
// let err = Error::new(storage_err);
// let s3_err = to_s3_error(err);
// assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket);
// assert!(s3_err.message().unwrap().contains("bucket not found"));
// assert!(s3_err.message().unwrap().contains(&long_bucket_name));
// }
// }
use common::error::Error;
use ecstore::error::StorageError;
use s3s::{S3Error, S3ErrorCode, s3_error};
pub fn to_s3_error(err: Error) -> S3Error {
if let Some(storage_err) = err.downcast_ref::<StorageError>() {
return match storage_err {
StorageError::NotImplemented => s3_error!(NotImplemented),
StorageError::InvalidArgument(bucket, object, version_id) => {
s3_error!(InvalidArgument, "Invalid arguments provided for {}/{}-{}", bucket, object, version_id)
}
StorageError::MethodNotAllowed => s3_error!(MethodNotAllowed),
StorageError::BucketNotFound(bucket) => {
s3_error!(NoSuchBucket, "bucket not found {}", bucket)
}
StorageError::BucketNotEmpty(bucket) => s3_error!(BucketNotEmpty, "bucket not empty {}", bucket),
StorageError::BucketNameInvalid(bucket) => s3_error!(InvalidBucketName, "invalid bucket name {}", bucket),
StorageError::ObjectNameInvalid(bucket, object) => {
s3_error!(InvalidArgument, "invalid object name {}/{}", bucket, object)
}
StorageError::BucketExists(bucket) => s3_error!(BucketAlreadyExists, "{}", bucket),
StorageError::StorageFull => s3_error!(ServiceUnavailable, "Storage reached its minimum free drive threshold."),
StorageError::SlowDown => s3_error!(SlowDown, "Please reduce your request rate"),
StorageError::PrefixAccessDenied(bucket, object) => {
s3_error!(AccessDenied, "PrefixAccessDenied {}/{}", bucket, object)
}
StorageError::InvalidUploadIDKeyCombination(bucket, object) => {
s3_error!(InvalidArgument, "Invalid UploadID KeyCombination: {}/{}", bucket, object)
}
StorageError::MalformedUploadID(bucket) => s3_error!(InvalidArgument, "Malformed UploadID: {}", bucket),
StorageError::ObjectNameTooLong(bucket, object) => {
s3_error!(InvalidArgument, "Object name too long: {}/{}", bucket, object)
}
StorageError::ObjectNamePrefixAsSlash(bucket, object) => {
s3_error!(InvalidArgument, "Object name contains forward slash as prefix: {}/{}", bucket, object)
}
StorageError::ObjectNotFound(bucket, object) => s3_error!(NoSuchKey, "{}/{}", bucket, object),
StorageError::VersionNotFound(bucket, object, version_id) => {
s3_error!(NoSuchVersion, "{}/{}/{}", bucket, object, version_id)
}
StorageError::InvalidUploadID(bucket, object, version_id) => {
s3_error!(InvalidPart, "Invalid upload id: {}/{}-{}", bucket, object, version_id)
}
StorageError::InvalidVersionID(bucket, object, version_id) => {
s3_error!(InvalidArgument, "Invalid version id: {}/{}-{}", bucket, object, version_id)
}
// extended
StorageError::DataMovementOverwriteErr(bucket, object, version_id) => s3_error!(
InvalidArgument,
"invalid data movement operation, source and destination pool are the same for : {}/{}-{}",
bucket,
object,
version_id
),
// extended
StorageError::ObjectExistsAsDirectory(bucket, object) => {
s3_error!(InvalidArgument, "Object exists on :{} as directory {}", bucket, object)
}
StorageError::InvalidPart(bucket, object, version_id) => {
s3_error!(
InvalidPart,
"Specified part could not be found. PartNumber {}, Expected {}, got {}",
bucket,
object,
version_id
)
}
StorageError::DoneForNow => s3_error!(InternalError, "DoneForNow"),
};
}
if is_err_file_not_found(&err) {
return S3Error::with_message(S3ErrorCode::NoSuchKey, format!(" ec err {}", err));
}
S3Error::with_message(S3ErrorCode::InternalError, format!(" ec err {}", err))
}
#[cfg(test)]
mod tests {
use super::*;
use s3s::S3ErrorCode;
#[test]
fn test_to_s3_error_not_implemented() {
let storage_err = StorageError::NotImplemented;
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::NotImplemented);
}
#[test]
fn test_to_s3_error_invalid_argument() {
let storage_err =
StorageError::InvalidArgument("test-bucket".to_string(), "test-object".to_string(), "test-version".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
assert!(s3_err.message().unwrap().contains("Invalid arguments provided"));
assert!(s3_err.message().unwrap().contains("test-bucket"));
assert!(s3_err.message().unwrap().contains("test-object"));
assert!(s3_err.message().unwrap().contains("test-version"));
}
#[test]
fn test_to_s3_error_method_not_allowed() {
let storage_err = StorageError::MethodNotAllowed;
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::MethodNotAllowed);
}
#[test]
fn test_to_s3_error_bucket_not_found() {
let storage_err = StorageError::BucketNotFound("test-bucket".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket);
assert!(s3_err.message().unwrap().contains("bucket not found"));
assert!(s3_err.message().unwrap().contains("test-bucket"));
}
#[test]
fn test_to_s3_error_bucket_not_empty() {
let storage_err = StorageError::BucketNotEmpty("test-bucket".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::BucketNotEmpty);
assert!(s3_err.message().unwrap().contains("bucket not empty"));
assert!(s3_err.message().unwrap().contains("test-bucket"));
}
#[test]
fn test_to_s3_error_bucket_name_invalid() {
let storage_err = StorageError::BucketNameInvalid("invalid-bucket-name".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidBucketName);
assert!(s3_err.message().unwrap().contains("invalid bucket name"));
assert!(s3_err.message().unwrap().contains("invalid-bucket-name"));
}
#[test]
fn test_to_s3_error_object_name_invalid() {
let storage_err = StorageError::ObjectNameInvalid("test-bucket".to_string(), "invalid-object".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
assert!(s3_err.message().unwrap().contains("invalid object name"));
assert!(s3_err.message().unwrap().contains("test-bucket"));
assert!(s3_err.message().unwrap().contains("invalid-object"));
}
#[test]
fn test_to_s3_error_bucket_exists() {
let storage_err = StorageError::BucketExists("existing-bucket".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::BucketAlreadyExists);
assert!(s3_err.message().unwrap().contains("existing-bucket"));
}
#[test]
fn test_to_s3_error_storage_full() {
let storage_err = StorageError::StorageFull;
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::ServiceUnavailable);
assert!(
s3_err
.message()
.unwrap()
.contains("Storage reached its minimum free drive threshold")
);
}
#[test]
fn test_to_s3_error_slow_down() {
let storage_err = StorageError::SlowDown;
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::SlowDown);
assert!(s3_err.message().unwrap().contains("Please reduce your request rate"));
}
#[test]
fn test_to_s3_error_prefix_access_denied() {
let storage_err = StorageError::PrefixAccessDenied("test-bucket".to_string(), "test-prefix".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::AccessDenied);
assert!(s3_err.message().unwrap().contains("PrefixAccessDenied"));
assert!(s3_err.message().unwrap().contains("test-bucket"));
assert!(s3_err.message().unwrap().contains("test-prefix"));
}
#[test]
fn test_to_s3_error_invalid_upload_id_key_combination() {
let storage_err = StorageError::InvalidUploadIDKeyCombination("test-bucket".to_string(), "test-object".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
assert!(s3_err.message().unwrap().contains("Invalid UploadID KeyCombination"));
assert!(s3_err.message().unwrap().contains("test-bucket"));
assert!(s3_err.message().unwrap().contains("test-object"));
}
#[test]
fn test_to_s3_error_malformed_upload_id() {
let storage_err = StorageError::MalformedUploadID("malformed-id".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
assert!(s3_err.message().unwrap().contains("Malformed UploadID"));
assert!(s3_err.message().unwrap().contains("malformed-id"));
}
#[test]
fn test_to_s3_error_object_name_too_long() {
let storage_err = StorageError::ObjectNameTooLong("test-bucket".to_string(), "very-long-object-name".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
assert!(s3_err.message().unwrap().contains("Object name too long"));
assert!(s3_err.message().unwrap().contains("test-bucket"));
assert!(s3_err.message().unwrap().contains("very-long-object-name"));
}
#[test]
fn test_to_s3_error_object_name_prefix_as_slash() {
let storage_err = StorageError::ObjectNamePrefixAsSlash("test-bucket".to_string(), "/invalid-object".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
assert!(
s3_err
.message()
.unwrap()
.contains("Object name contains forward slash as prefix")
);
assert!(s3_err.message().unwrap().contains("test-bucket"));
assert!(s3_err.message().unwrap().contains("/invalid-object"));
}
#[test]
fn test_to_s3_error_object_not_found() {
let storage_err = StorageError::ObjectNotFound("test-bucket".to_string(), "missing-object".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchKey);
assert!(s3_err.message().unwrap().contains("test-bucket"));
assert!(s3_err.message().unwrap().contains("missing-object"));
}
#[test]
fn test_to_s3_error_version_not_found() {
let storage_err =
StorageError::VersionNotFound("test-bucket".to_string(), "test-object".to_string(), "missing-version".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchVersion);
assert!(s3_err.message().unwrap().contains("test-bucket"));
assert!(s3_err.message().unwrap().contains("test-object"));
assert!(s3_err.message().unwrap().contains("missing-version"));
}
#[test]
fn test_to_s3_error_invalid_upload_id() {
let storage_err =
StorageError::InvalidUploadID("test-bucket".to_string(), "test-object".to_string(), "invalid-upload-id".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidPart);
assert!(s3_err.message().unwrap().contains("Invalid upload id"));
assert!(s3_err.message().unwrap().contains("test-bucket"));
assert!(s3_err.message().unwrap().contains("test-object"));
assert!(s3_err.message().unwrap().contains("invalid-upload-id"));
}
#[test]
fn test_to_s3_error_invalid_version_id() {
let storage_err = StorageError::InvalidVersionID(
"test-bucket".to_string(),
"test-object".to_string(),
"invalid-version-id".to_string(),
);
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
assert!(s3_err.message().unwrap().contains("Invalid version id"));
assert!(s3_err.message().unwrap().contains("test-bucket"));
assert!(s3_err.message().unwrap().contains("test-object"));
assert!(s3_err.message().unwrap().contains("invalid-version-id"));
}
#[test]
fn test_to_s3_error_data_movement_overwrite_err() {
let storage_err = StorageError::DataMovementOverwriteErr(
"test-bucket".to_string(),
"test-object".to_string(),
"test-version".to_string(),
);
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
assert!(s3_err.message().unwrap().contains("invalid data movement operation"));
assert!(s3_err.message().unwrap().contains("source and destination pool are the same"));
assert!(s3_err.message().unwrap().contains("test-bucket"));
assert!(s3_err.message().unwrap().contains("test-object"));
assert!(s3_err.message().unwrap().contains("test-version"));
}
#[test]
fn test_to_s3_error_object_exists_as_directory() {
let storage_err = StorageError::ObjectExistsAsDirectory("test-bucket".to_string(), "directory-object".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
assert!(s3_err.message().unwrap().contains("Object exists on"));
assert!(s3_err.message().unwrap().contains("as directory"));
assert!(s3_err.message().unwrap().contains("test-bucket"));
assert!(s3_err.message().unwrap().contains("directory-object"));
}
#[test]
fn test_to_s3_error_insufficient_read_quorum() {
let storage_err = StorageError::InsufficientReadQuorum;
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::SlowDown);
assert!(
s3_err
.message()
.unwrap()
.contains("Storage resources are insufficient for the read operation")
);
}
#[test]
fn test_to_s3_error_insufficient_write_quorum() {
let storage_err = StorageError::InsufficientWriteQuorum;
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::SlowDown);
assert!(
s3_err
.message()
.unwrap()
.contains("Storage resources are insufficient for the write operation")
);
}
#[test]
fn test_to_s3_error_decommission_not_started() {
let storage_err = StorageError::DecommissionNotStarted;
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
assert!(s3_err.message().unwrap().contains("Decommission Not Started"));
}
#[test]
fn test_to_s3_error_decommission_already_running() {
let storage_err = StorageError::DecommissionAlreadyRunning;
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InternalError);
assert!(s3_err.message().unwrap().contains("Decommission already running"));
}
#[test]
fn test_to_s3_error_volume_not_found() {
let storage_err = StorageError::VolumeNotFound("test-volume".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket);
assert!(s3_err.message().unwrap().contains("bucket not found"));
assert!(s3_err.message().unwrap().contains("test-volume"));
}
#[test]
fn test_to_s3_error_invalid_part() {
let storage_err = StorageError::InvalidPart(1, "expected-part".to_string(), "got-part".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidPart);
assert!(s3_err.message().unwrap().contains("Specified part could not be found"));
assert!(s3_err.message().unwrap().contains("PartNumber"));
assert!(s3_err.message().unwrap().contains("expected-part"));
assert!(s3_err.message().unwrap().contains("got-part"));
}
#[test]
fn test_to_s3_error_done_for_now() {
let storage_err = StorageError::DoneForNow;
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InternalError);
assert!(s3_err.message().unwrap().contains("DoneForNow"));
}
#[test]
fn test_to_s3_error_non_storage_error() {
// Test with a non-StorageError
let err = Error::from_string("Generic error message".to_string());
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InternalError);
assert!(s3_err.message().unwrap().contains("ec err"));
assert!(s3_err.message().unwrap().contains("Generic error message"));
}
#[test]
fn test_to_s3_error_with_unicode_strings() {
let storage_err = StorageError::BucketNotFound("测试桶".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket);
assert!(s3_err.message().unwrap().contains("bucket not found"));
assert!(s3_err.message().unwrap().contains("测试桶"));
}
#[test]
fn test_to_s3_error_with_special_characters() {
let storage_err = StorageError::ObjectNameInvalid("bucket-with-@#$%".to_string(), "object-with-!@#$%^&*()".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::InvalidArgument);
assert!(s3_err.message().unwrap().contains("invalid object name"));
assert!(s3_err.message().unwrap().contains("bucket-with-@#$%"));
assert!(s3_err.message().unwrap().contains("object-with-!@#$%^&*()"));
}
#[test]
fn test_to_s3_error_with_empty_strings() {
let storage_err = StorageError::BucketNotFound("".to_string());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket);
assert!(s3_err.message().unwrap().contains("bucket not found"));
}
#[test]
fn test_to_s3_error_with_very_long_strings() {
let long_bucket_name = "a".repeat(1000);
let storage_err = StorageError::BucketNotFound(long_bucket_name.clone());
let err = Error::new(storage_err);
let s3_err = to_s3_error(err);
assert_eq!(*s3_err.code(), S3ErrorCode::NoSuchBucket);
assert!(s3_err.message().unwrap().contains("bucket not found"));
assert!(s3_err.message().unwrap().contains(&long_bucket_name));
}
}