fix: make lint build and clippy happy (#71)

Signed-off-by: yihong0618 <zouzou0208@gmail.com>
This commit is contained in:
yihong
2025-07-07 09:55:53 +08:00
committed by GitHub
parent 040b05c318
commit abd5dff9b5
47 changed files with 1091 additions and 1379 deletions
Generated
+4
View File
@@ -8312,8 +8312,12 @@ dependencies = [
"http 1.3.1", "http 1.3.1",
"hyper 1.6.0", "hyper 1.6.0",
"lazy_static", "lazy_static",
"rand 0.9.1",
"rustfs-utils", "rustfs-utils",
"s3s",
"serde",
"serde_urlencoded", "serde_urlencoded",
"tempfile",
"time", "time",
"tracing", "tracing",
] ]
+2 -5
View File
@@ -568,14 +568,11 @@ mod tests {
let mut latency = LastMinuteLatency::default(); let mut latency = LastMinuteLatency::default();
// Add data at time 1000 // Add data at time 1000
latency.add_all( latency.add_all(1000, &AccElem {
1000,
&AccElem {
total: 10, total: 10,
size: 0, size: 0,
n: 1, n: 1,
}, });
);
// Forward to time 1030 (30 seconds later) // Forward to time 1030 (30 seconds later)
latency.forward_to(1030); latency.forward_to(1030);
+2 -5
View File
@@ -236,13 +236,10 @@ impl BucketMetadataSys {
futures.push(async move { futures.push(async move {
sleep(Duration::from_millis(30)).await; sleep(Duration::from_millis(30)).await;
let _ = api let _ = api
.heal_bucket( .heal_bucket(&bucket, &HealOpts {
&bucket,
&HealOpts {
recreate: true, recreate: true,
..Default::default() ..Default::default()
}, })
)
.await; .await;
load_bucket_metadata(self.api.clone(), bucket.as_str()).await load_bucket_metadata(self.api.clone(), bucket.as_str()).await
}); });
+4 -10
View File
@@ -74,9 +74,7 @@ impl TransitionClient {
url_values.insert("policy".to_string(), "".to_string()); url_values.insert("policy".to_string(), "".to_string());
let resp = self let resp = self
.execute_method( .execute_method(http::Method::DELETE, &mut RequestMetadata {
http::Method::DELETE,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
query_values: url_values, query_values: url_values,
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
@@ -92,8 +90,7 @@ impl TransitionClient {
extra_pre_sign_header: Default::default(), extra_pre_sign_header: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
expires: Default::default(), expires: Default::default(),
}, })
)
.await?; .await?;
//defer closeResponse(resp) //defer closeResponse(resp)
@@ -114,9 +111,7 @@ impl TransitionClient {
url_values.insert("policy".to_string(), "".to_string()); url_values.insert("policy".to_string(), "".to_string());
let resp = self let resp = self
.execute_method( .execute_method(http::Method::GET, &mut RequestMetadata {
http::Method::GET,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
query_values: url_values, query_values: url_values,
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
@@ -132,8 +127,7 @@ impl TransitionClient {
extra_pre_sign_header: Default::default(), extra_pre_sign_header: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
expires: Default::default(), expires: Default::default(),
}, })
)
.await?; .await?;
let policy = String::from_utf8_lossy(&resp.body().bytes().expect("err").to_vec()).to_string(); let policy = String::from_utf8_lossy(&resp.body().bytes().expect("err").to_vec()).to_string();
+2 -5
View File
@@ -43,9 +43,7 @@ impl TransitionClient {
opts: &GetObjectOptions, opts: &GetObjectOptions,
) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> { ) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> {
let resp = self let resp = self
.execute_method( .execute_method(http::Method::GET, &mut RequestMetadata {
http::Method::GET,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(), object_name: object_name.to_string(),
query_values: opts.to_query_values(), query_values: opts.to_query_values(),
@@ -61,8 +59,7 @@ impl TransitionClient {
extra_pre_sign_header: Default::default(), extra_pre_sign_header: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
expires: Default::default(), expires: Default::default(),
}, })
)
.await?; .await?;
let resp = &resp; let resp = &resp;
+18 -17
View File
@@ -21,28 +21,28 @@
use bytes::Bytes; use bytes::Bytes;
use http::{HeaderMap, HeaderValue}; use http::{HeaderMap, HeaderValue};
use s3s::dto::Owner; use s3s::dto::Owner;
use std::io::Cursor;
use std::collections::HashMap; use std::collections::HashMap;
use std::io::Cursor;
use tokio::io::BufReader; use tokio::io::BufReader;
use rustfs_utils::EMPTY_STRING_SHA256_HASH;
use crate::client::{ use crate::client::{
api_error_response::{err_invalid_argument, http_resp_to_error_response}, api_error_response::{err_invalid_argument, http_resp_to_error_response},
api_get_options::GetObjectOptions, api_get_options::GetObjectOptions,
transition_api::{to_object_info, ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient}, transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info},
}; };
use rustfs_utils::EMPTY_STRING_SHA256_HASH;
#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)] #[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
struct Grantee { pub struct Grantee {
id: String, pub id: String,
display_name: String, pub display_name: String,
uri: String, pub uri: String,
} }
#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)] #[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
struct Grant { pub struct Grant {
grantee: Grantee, pub grantee: Grantee,
permission: String, pub permission: String,
} }
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)] #[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
@@ -63,9 +63,7 @@ impl TransitionClient {
let mut url_values = HashMap::new(); let mut url_values = HashMap::new();
url_values.insert("acl".to_string(), "".to_string()); url_values.insert("acl".to_string(), "".to_string());
let mut resp = self let mut resp = self
.execute_method( .execute_method(http::Method::GET, &mut RequestMetadata {
http::Method::GET,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(), object_name: object_name.to_string(),
query_values: url_values, query_values: url_values,
@@ -81,8 +79,7 @@ impl TransitionClient {
extra_pre_sign_header: Default::default(), extra_pre_sign_header: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
expires: Default::default(), expires: Default::default(),
}, })
)
.await?; .await?;
if resp.status() != http::StatusCode::OK { if resp.status() != http::StatusCode::OK {
@@ -98,7 +95,9 @@ impl TransitionClient {
} }
}; };
let mut obj_info = self.stat_object(bucket_name, object_name, &GetObjectOptions::default()).await?; let mut obj_info = self
.stat_object(bucket_name, object_name, &GetObjectOptions::default())
.await?;
obj_info.owner.display_name = res.owner.display_name.clone(); obj_info.owner.display_name = res.owner.display_name.clone();
obj_info.owner.id = res.owner.id.clone(); obj_info.owner.id = res.owner.id.clone();
@@ -107,7 +106,9 @@ impl TransitionClient {
let canned_acl = get_canned_acl(&res); let canned_acl = get_canned_acl(&res);
if canned_acl != "" { if canned_acl != "" {
obj_info.metadata.insert("X-Amz-Acl", HeaderValue::from_str(&canned_acl).unwrap()); obj_info
.metadata
.insert("X-Amz-Acl", HeaderValue::from_str(&canned_acl).unwrap());
return Ok(obj_info); return Ok(obj_info);
} }
@@ -20,35 +20,37 @@
use bytes::Bytes; use bytes::Bytes;
use http::{HeaderMap, HeaderValue}; use http::{HeaderMap, HeaderValue};
use time::OffsetDateTime;
use std::io::Cursor;
use std::collections::HashMap; use std::collections::HashMap;
use std::io::Cursor;
use time::OffsetDateTime;
use tokio::io::BufReader; use tokio::io::BufReader;
use s3s::{Body, dto::Owner};
use s3s::header::{X_AMZ_OBJECT_ATTRIBUTES, X_AMZ_DELETE_MARKER, X_AMZ_METADATA_DIRECTIVE, X_AMZ_VERSION_ID,
X_AMZ_REQUEST_CHARGED, X_AMZ_RESTORE, X_AMZ_PART_NUMBER_MARKER, X_AMZ_MAX_PARTS,};
use rustfs_utils::EMPTY_STRING_SHA256_HASH;
use crate::client::constants::{GET_OBJECT_ATTRIBUTES_MAX_PARTS, GET_OBJECT_ATTRIBUTES_TAGS, ISO8601_DATEFORMAT}; use crate::client::constants::{GET_OBJECT_ATTRIBUTES_MAX_PARTS, GET_OBJECT_ATTRIBUTES_TAGS, ISO8601_DATEFORMAT};
use rustfs_utils::EMPTY_STRING_SHA256_HASH;
use s3s::header::{
X_AMZ_DELETE_MARKER, X_AMZ_MAX_PARTS, X_AMZ_METADATA_DIRECTIVE, X_AMZ_OBJECT_ATTRIBUTES, X_AMZ_PART_NUMBER_MARKER,
X_AMZ_REQUEST_CHARGED, X_AMZ_RESTORE, X_AMZ_VERSION_ID,
};
use s3s::{Body, dto::Owner};
use crate::client::{ use crate::client::{
api_error_response::err_invalid_argument, api_error_response::err_invalid_argument,
api_get_object_acl::AccessControlPolicy,
api_get_options::GetObjectOptions, api_get_options::GetObjectOptions,
transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info}, transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info},
api_get_object_acl::AccessControlPolicy,
}; };
struct ObjectAttributesOptions { pub struct ObjectAttributesOptions {
max_parts: i64, pub max_parts: i64,
version_id: String, pub version_id: String,
part_number_marker: i64, pub part_number_marker: i64,
//server_side_encryption: encrypt::ServerSide, //server_side_encryption: encrypt::ServerSide,
} }
struct ObjectAttributes { pub struct ObjectAttributes {
version_id: String, pub version_id: String,
last_modified: OffsetDateTime, pub last_modified: OffsetDateTime,
object_attributes_response: ObjectAttributesResponse, pub object_attributes_response: ObjectAttributesResponse,
} }
impl ObjectAttributes { impl ObjectAttributes {
@@ -81,11 +83,11 @@ impl Checksum {
} }
#[derive(Debug, Default, serde::Deserialize)] #[derive(Debug, Default, serde::Deserialize)]
struct ObjectParts { pub struct ObjectParts {
parts_count: i64, pub parts_count: i64,
part_number_marker: i64, pub part_number_marker: i64,
next_part_number_marker: i64, pub next_part_number_marker: i64,
max_parts: i64, pub max_parts: i64,
is_truncated: bool, is_truncated: bool,
parts: Vec<ObjectAttributePart>, parts: Vec<ObjectAttributePart>,
} }
@@ -104,12 +106,12 @@ impl ObjectParts {
} }
#[derive(Debug, Default, serde::Deserialize)] #[derive(Debug, Default, serde::Deserialize)]
struct ObjectAttributesResponse { pub struct ObjectAttributesResponse {
etag: String, pub etag: String,
storage_class: String, pub storage_class: String,
object_size: i64, pub object_size: i64,
checksum: Checksum, pub checksum: Checksum,
object_parts: ObjectParts, pub object_parts: ObjectParts,
} }
impl ObjectAttributesResponse { impl ObjectAttributesResponse {
@@ -155,7 +157,12 @@ impl ObjectAttributes {
} }
impl TransitionClient { impl TransitionClient {
pub async fn get_object_attributes(&self, bucket_name: &str, object_name: &str, opts: ObjectAttributesOptions) -> Result<ObjectAttributes, std::io::Error> { pub async fn get_object_attributes(
&self,
bucket_name: &str,
object_name: &str,
opts: ObjectAttributesOptions,
) -> Result<ObjectAttributes, std::io::Error> {
let mut url_values = HashMap::new(); let mut url_values = HashMap::new();
url_values.insert("attributes".to_string(), "".to_string()); url_values.insert("attributes".to_string(), "".to_string());
if opts.version_id != "" { if opts.version_id != "" {
@@ -166,13 +173,19 @@ impl TransitionClient {
headers.insert(X_AMZ_OBJECT_ATTRIBUTES, HeaderValue::from_str(GET_OBJECT_ATTRIBUTES_TAGS).unwrap()); headers.insert(X_AMZ_OBJECT_ATTRIBUTES, HeaderValue::from_str(GET_OBJECT_ATTRIBUTES_TAGS).unwrap());
if opts.part_number_marker > 0 { if opts.part_number_marker > 0 {
headers.insert(X_AMZ_PART_NUMBER_MARKER, HeaderValue::from_str(&opts.part_number_marker.to_string()).unwrap()); headers.insert(
X_AMZ_PART_NUMBER_MARKER,
HeaderValue::from_str(&opts.part_number_marker.to_string()).unwrap(),
);
} }
if opts.max_parts > 0 { if opts.max_parts > 0 {
headers.insert(X_AMZ_MAX_PARTS, HeaderValue::from_str(&opts.max_parts.to_string()).unwrap()); headers.insert(X_AMZ_MAX_PARTS, HeaderValue::from_str(&opts.max_parts.to_string()).unwrap());
} else { } else {
headers.insert(X_AMZ_MAX_PARTS, HeaderValue::from_str(&GET_OBJECT_ATTRIBUTES_MAX_PARTS.to_string()).unwrap()); headers.insert(
X_AMZ_MAX_PARTS,
HeaderValue::from_str(&GET_OBJECT_ATTRIBUTES_MAX_PARTS.to_string()).unwrap(),
);
} }
/*if opts.server_side_encryption.is_some() { /*if opts.server_side_encryption.is_some() {
@@ -180,9 +193,7 @@ impl TransitionClient {
}*/ }*/
let mut resp = self let mut resp = self
.execute_method( .execute_method(http::Method::HEAD, &mut RequestMetadata {
http::Method::HEAD,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(), object_name: object_name.to_string(),
query_values: url_values, query_values: url_values,
@@ -198,14 +209,15 @@ impl TransitionClient {
extra_pre_sign_header: Default::default(), extra_pre_sign_header: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
expires: Default::default(), expires: Default::default(),
}, })
)
.await?; .await?;
let h = resp.headers(); let h = resp.headers();
let has_etag = h.get("ETag").unwrap().to_str().unwrap(); let has_etag = h.get("ETag").unwrap().to_str().unwrap();
if !has_etag.is_empty() { if !has_etag.is_empty() {
return Err(std::io::Error::other("get_object_attributes is not supported by the current endpoint version")); return Err(std::io::Error::other(
"get_object_attributes is not supported by the current endpoint version",
));
} }
if resp.status() != http::StatusCode::OK { if resp.status() != http::StatusCode::OK {
@@ -21,15 +21,15 @@
use bytes::Bytes; use bytes::Bytes;
use http::HeaderMap; use http::HeaderMap;
use std::io::Cursor; use std::io::Cursor;
use tokio::io::BufReader;
#[cfg(not(windows))] #[cfg(not(windows))]
use std::os::unix::fs::PermissionsExt; use std::os::unix::fs::MetadataExt;
#[cfg(not(windows))] #[cfg(not(windows))]
use std::os::unix::fs::OpenOptionsExt; use std::os::unix::fs::OpenOptionsExt;
#[cfg(not(windows))] #[cfg(not(windows))]
use std::os::unix::fs::MetadataExt; use std::os::unix::fs::PermissionsExt;
#[cfg(windows)] #[cfg(windows)]
use std::os::windows::fs::MetadataExt; use std::os::windows::fs::MetadataExt;
use tokio::io::BufReader;
use crate::client::{ use crate::client::{
api_error_response::err_invalid_argument, api_error_response::err_invalid_argument,
@@ -38,14 +38,20 @@ use crate::client::{
}; };
impl TransitionClient { impl TransitionClient {
pub async fn fget_object(&self, bucket_name: &str, object_name: &str, file_path: &str, opts: GetObjectOptions) -> Result<(), std::io::Error> { pub async fn fget_object(
&self,
bucket_name: &str,
object_name: &str,
file_path: &str,
opts: GetObjectOptions,
) -> Result<(), std::io::Error> {
match std::fs::metadata(file_path) { match std::fs::metadata(file_path) {
Ok(file_path_stat) => { Ok(file_path_stat) => {
let ft = file_path_stat.file_type(); let ft = file_path_stat.file_type();
if ft.is_dir() { if ft.is_dir() {
return Err(std::io::Error::other(err_invalid_argument("filename is a directory."))); return Err(std::io::Error::other(err_invalid_argument("filename is a directory.")));
} }
}, }
Err(err) => { Err(err) => {
return Err(std::io::Error::other(err)); return Err(std::io::Error::other(err));
} }
+2 -5
View File
@@ -76,9 +76,7 @@ impl TransitionClient {
} }
let mut resp = self let mut resp = self
.execute_method( .execute_method(http::Method::GET, &mut RequestMetadata {
http::Method::GET,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
object_name: "".to_string(), object_name: "".to_string(),
query_values: url_values, query_values: url_values,
@@ -94,8 +92,7 @@ impl TransitionClient {
extra_pre_sign_header: Default::default(), extra_pre_sign_header: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
expires: Default::default(), expires: Default::default(),
}, })
)
.await?; .await?;
if resp.status() != StatusCode::OK { if resp.status() != StatusCode::OK {
return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, ""))); return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, "")));
+12 -31
View File
@@ -78,9 +78,7 @@ impl TransitionClient {
let headers = HeaderMap::new(); let headers = HeaderMap::new();
let resp = self let resp = self
.execute_method( .execute_method(Method::DELETE, &mut RequestMetadata {
Method::DELETE,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
custom_header: headers, custom_header: headers,
@@ -96,8 +94,7 @@ impl TransitionClient {
extra_pre_sign_header: Default::default(), extra_pre_sign_header: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
expires: Default::default(), expires: Default::default(),
}, })
)
.await?; .await?;
{ {
@@ -109,9 +106,7 @@ impl TransitionClient {
pub async fn remove_bucket(&self, bucket_name: &str) -> Result<(), std::io::Error> { pub async fn remove_bucket(&self, bucket_name: &str) -> Result<(), std::io::Error> {
let resp = self let resp = self
.execute_method( .execute_method(http::Method::DELETE, &mut RequestMetadata {
http::Method::DELETE,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
custom_header: Default::default(), custom_header: Default::default(),
@@ -127,8 +122,7 @@ impl TransitionClient {
extra_pre_sign_header: Default::default(), extra_pre_sign_header: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
expires: Default::default(), expires: Default::default(),
}, })
)
.await?; .await?;
{ {
@@ -163,9 +157,7 @@ impl TransitionClient {
} }
let resp = self let resp = self
.execute_method( .execute_method(http::Method::DELETE, &mut RequestMetadata {
http::Method::DELETE,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(), object_name: object_name.to_string(),
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
@@ -181,8 +173,7 @@ impl TransitionClient {
extra_pre_sign_header: Default::default(), extra_pre_sign_header: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
expires: Default::default(), expires: Default::default(),
}, })
)
.await?; .await?;
Ok(RemoveObjectResult { Ok(RemoveObjectResult {
@@ -277,15 +268,11 @@ impl TransitionClient {
while let Some(object) = objects_rx.recv().await { while let Some(object) = objects_rx.recv().await {
if has_invalid_xml_char(&object.name) { if has_invalid_xml_char(&object.name) {
let remove_result = self let remove_result = self
.remove_object_inner( .remove_object_inner(bucket_name, &object.name, RemoveObjectOptions {
bucket_name,
&object.name,
RemoveObjectOptions {
version_id: object.version_id.expect("err").to_string(), version_id: object.version_id.expect("err").to_string(),
governance_bypass: opts.governance_bypass, governance_bypass: opts.governance_bypass,
..Default::default() ..Default::default()
}, })
)
.await?; .await?;
let remove_result_clone = remove_result.clone(); let remove_result_clone = remove_result.clone();
if !remove_result.err.is_none() { if !remove_result.err.is_none() {
@@ -322,9 +309,7 @@ impl TransitionClient {
let remove_bytes = generate_remove_multi_objects_request(&batch); let remove_bytes = generate_remove_multi_objects_request(&batch);
let resp = self let resp = self
.execute_method( .execute_method(http::Method::POST, &mut RequestMetadata {
http::Method::POST,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
query_values: url_values.clone(), query_values: url_values.clone(),
content_body: ReaderImpl::Body(Bytes::from(remove_bytes.clone())), content_body: ReaderImpl::Body(Bytes::from(remove_bytes.clone())),
@@ -340,8 +325,7 @@ impl TransitionClient {
extra_pre_sign_header: Default::default(), extra_pre_sign_header: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
expires: Default::default(), expires: Default::default(),
}, })
)
.await?; .await?;
let body_bytes: Vec<u8> = resp.body().bytes().expect("err").to_vec(); let body_bytes: Vec<u8> = resp.body().bytes().expect("err").to_vec();
@@ -369,9 +353,7 @@ impl TransitionClient {
url_values.insert("uploadId".to_string(), upload_id.to_string()); url_values.insert("uploadId".to_string(), upload_id.to_string());
let resp = self let resp = self
.execute_method( .execute_method(http::Method::DELETE, &mut RequestMetadata {
http::Method::DELETE,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(), object_name: object_name.to_string(),
query_values: url_values, query_values: url_values,
@@ -387,8 +369,7 @@ impl TransitionClient {
extra_pre_sign_header: Default::default(), extra_pre_sign_header: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
expires: Default::default(), expires: Default::default(),
}, })
)
.await?; .await?;
//if resp.is_some() { //if resp.is_some() {
if resp.status() != StatusCode::NO_CONTENT { if resp.status() != StatusCode::NO_CONTENT {
+43 -37
View File
@@ -20,12 +20,15 @@
use bytes::Bytes; use bytes::Bytes;
use http::HeaderMap; use http::HeaderMap;
use std::collections::HashMap;
use std::io::Cursor; use std::io::Cursor;
use tokio::io::BufReader; use tokio::io::BufReader;
use std::collections::HashMap;
use crate::client::{ use crate::client::{
api_error_response::{err_invalid_argument, http_resp_to_error_response}, api_get_object_acl::AccessControlList, api_get_options::GetObjectOptions, transition_api::{to_object_info, ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient} api_error_response::{err_invalid_argument, http_resp_to_error_response},
api_get_object_acl::AccessControlList,
api_get_options::GetObjectOptions,
transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info},
}; };
const TIER_STANDARD: &str = "Standard"; const TIER_STANDARD: &str = "Standard";
@@ -33,55 +36,55 @@ const TIER_BULK: &str = "Bulk";
const TIER_EXPEDITED: &str = "Expedited"; const TIER_EXPEDITED: &str = "Expedited";
#[derive(Debug, Default, serde::Serialize)] #[derive(Debug, Default, serde::Serialize)]
struct GlacierJobParameters { pub struct GlacierJobParameters {
tier: String, pub tier: String,
} }
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)] #[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
struct Encryption { pub struct Encryption {
encryption_type: String, pub encryption_type: String,
kms_context: String, pub kms_context: String,
kms_key_id: String, pub kms_key_id: String,
} }
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)] #[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
struct MetadataEntry { pub struct MetadataEntry {
name: String, pub name: String,
value: String, pub value: String,
} }
#[derive(Debug, Default, serde::Serialize)] #[derive(Debug, Default, serde::Serialize)]
struct S3 { pub struct S3 {
access_control_list: AccessControlList, pub access_control_list: AccessControlList,
bucket_name: String, pub bucket_name: String,
prefix: String, pub prefix: String,
canned_acl: String, pub canned_acl: String,
encryption: Encryption, pub encryption: Encryption,
storage_class: String, pub storage_class: String,
//tagging: Tags, //tagging: Tags,
user_metadata: MetadataEntry, pub user_metadata: MetadataEntry,
} }
#[derive(Debug, Default, serde::Serialize)] #[derive(Debug, Default, serde::Serialize)]
struct SelectParameters { pub struct SelectParameters {
expression_type: String, pub expression_type: String,
expression: String, pub expression: String,
//input_serialization: SelectObjectInputSerialization, //input_serialization: SelectObjectInputSerialization,
//output_serialization: SelectObjectOutputSerialization, //output_serialization: SelectObjectOutputSerialization,
} }
#[derive(Debug, Default, serde::Serialize)] #[derive(Debug, Default, serde::Serialize)]
struct OutputLocation(S3); pub struct OutputLocation(pub S3);
#[derive(Debug, Default, serde::Serialize)] #[derive(Debug, Default, serde::Serialize)]
struct RestoreRequest { pub struct RestoreRequest {
restore_type: String, pub restore_type: String,
tier: String, pub tier: String,
days: i64, pub days: i64,
glacier_job_parameters: GlacierJobParameters, pub glacier_job_parameters: GlacierJobParameters,
description: String, pub description: String,
select_parameters: SelectParameters, pub select_parameters: SelectParameters,
output_location: OutputLocation, pub output_location: OutputLocation,
} }
impl RestoreRequest { impl RestoreRequest {
@@ -115,7 +118,13 @@ impl RestoreRequest {
} }
impl TransitionClient { impl TransitionClient {
pub async fn restore_object(&self, bucket_name: &str, object_name: &str, version_id: &str, restore_req: &RestoreRequest) -> Result<(), std::io::Error> { pub async fn restore_object(
&self,
bucket_name: &str,
object_name: &str,
version_id: &str,
restore_req: &RestoreRequest,
) -> Result<(), std::io::Error> {
let restore_request = match serde_xml_rs::to_string(restore_req) { let restore_request = match serde_xml_rs::to_string(restore_req) {
Ok(buf) => buf, Ok(buf) => buf,
Err(e) => { Err(e) => {
@@ -132,9 +141,7 @@ impl TransitionClient {
let restore_request_buffer = Bytes::from(restore_request_bytes.clone()); let restore_request_buffer = Bytes::from(restore_request_bytes.clone());
let resp = self let resp = self
.execute_method( .execute_method(http::Method::HEAD, &mut RequestMetadata {
http::Method::HEAD,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(), object_name: object_name.to_string(),
query_values: url_values, query_values: url_values,
@@ -150,8 +157,7 @@ impl TransitionClient {
extra_pre_sign_header: Default::default(), extra_pre_sign_header: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
expires: Default::default(), expires: Default::default(),
}, })
)
.await?; .await?;
let b = resp.body().bytes().expect("err").to_vec(); let b = resp.body().bytes().expect("err").to_vec();
+26 -19
View File
@@ -21,23 +21,21 @@
use bytes::Bytes; use bytes::Bytes;
use http::{HeaderMap, HeaderValue}; use http::{HeaderMap, HeaderValue};
use rustfs_utils::EMPTY_STRING_SHA256_HASH; use rustfs_utils::EMPTY_STRING_SHA256_HASH;
use uuid::Uuid;
use std::{collections::HashMap, str::FromStr}; use std::{collections::HashMap, str::FromStr};
use tokio::io::BufReader; use tokio::io::BufReader;
use uuid::Uuid;
use s3s::header::{X_AMZ_DELETE_MARKER, X_AMZ_VERSION_ID};
use crate::client::{ use crate::client::{
api_error_response::{err_invalid_argument, http_resp_to_error_response, ErrorResponse}, api_error_response::{ErrorResponse, err_invalid_argument, http_resp_to_error_response},
api_get_options::GetObjectOptions, api_get_options::GetObjectOptions,
transition_api::{to_object_info, ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient}, transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info},
}; };
use s3s::header::{X_AMZ_DELETE_MARKER, X_AMZ_VERSION_ID};
impl TransitionClient { impl TransitionClient {
pub async fn bucket_exists(&self, bucket_name: &str) -> Result<bool, std::io::Error> { pub async fn bucket_exists(&self, bucket_name: &str) -> Result<bool, std::io::Error> {
let resp = self let resp = self
.execute_method( .execute_method(http::Method::HEAD, &mut RequestMetadata {
http::Method::HEAD,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
object_name: "".to_string(), object_name: "".to_string(),
query_values: HashMap::new(), query_values: HashMap::new(),
@@ -53,8 +51,7 @@ impl TransitionClient {
extra_pre_sign_header: Default::default(), extra_pre_sign_header: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
expires: Default::default(), expires: Default::default(),
}, })
)
.await; .await;
if let Ok(resp) = resp { if let Ok(resp) = resp {
@@ -70,7 +67,12 @@ impl TransitionClient {
Ok(true) Ok(true)
} }
pub async fn stat_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<ObjectInfo, std::io::Error> { pub async fn stat_object(
&self,
bucket_name: &str,
object_name: &str,
opts: &GetObjectOptions,
) -> Result<ObjectInfo, std::io::Error> {
let mut headers = opts.header(); let mut headers = opts.header();
if opts.internal.replication_delete_marker { if opts.internal.replication_delete_marker {
headers.insert("X-Source-DeleteMarker", HeaderValue::from_str("true").unwrap()); headers.insert("X-Source-DeleteMarker", HeaderValue::from_str("true").unwrap());
@@ -80,9 +82,7 @@ impl TransitionClient {
} }
let resp = self let resp = self
.execute_method( .execute_method(http::Method::HEAD, &mut RequestMetadata {
http::Method::HEAD,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(), object_name: object_name.to_string(),
query_values: opts.to_query_values(), query_values: opts.to_query_values(),
@@ -98,8 +98,7 @@ impl TransitionClient {
extra_pre_sign_header: Default::default(), extra_pre_sign_header: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
expires: Default::default(), expires: Default::default(),
}, })
)
.await; .await;
match resp { match resp {
@@ -107,10 +106,14 @@ impl TransitionClient {
let h = resp.headers(); let h = resp.headers();
let delete_marker = if let Some(x_amz_delete_marker) = h.get(X_AMZ_DELETE_MARKER.as_str()) { let delete_marker = if let Some(x_amz_delete_marker) = h.get(X_AMZ_DELETE_MARKER.as_str()) {
x_amz_delete_marker.to_str().unwrap() == "true" x_amz_delete_marker.to_str().unwrap() == "true"
} else { false }; } else {
false
};
let replication_ready = if let Some(x_amz_delete_marker) = h.get("X-Replication-Ready") { let replication_ready = if let Some(x_amz_delete_marker) = h.get("X-Replication-Ready") {
x_amz_delete_marker.to_str().unwrap() == "true" x_amz_delete_marker.to_str().unwrap() == "true"
} else { false }; } else {
false
};
if resp.status() != http::StatusCode::OK && resp.status() != http::StatusCode::PARTIAL_CONTENT { if resp.status() != http::StatusCode::OK && resp.status() != http::StatusCode::PARTIAL_CONTENT {
if resp.status() == http::StatusCode::METHOD_NOT_ALLOWED && opts.version_id != "" && delete_marker { if resp.status() == http::StatusCode::METHOD_NOT_ALLOWED && opts.version_id != "" && delete_marker {
let err_resp = ErrorResponse { let err_resp = ErrorResponse {
@@ -124,7 +127,9 @@ impl TransitionClient {
return Ok(ObjectInfo { return Ok(ObjectInfo {
version_id: match Uuid::from_str(h.get(X_AMZ_VERSION_ID).unwrap().to_str().unwrap()) { version_id: match Uuid::from_str(h.get(X_AMZ_VERSION_ID).unwrap().to_str().unwrap()) {
Ok(v) => v, Ok(v) => v,
Err(e) => { return Err(std::io::Error::other(e)); } Err(e) => {
return Err(std::io::Error::other(e));
}
}, },
is_delete_marker: delete_marker, is_delete_marker: delete_marker,
..Default::default() ..Default::default()
@@ -134,7 +139,9 @@ impl TransitionClient {
return Ok(ObjectInfo { return Ok(ObjectInfo {
version_id: match Uuid::from_str(h.get(X_AMZ_VERSION_ID).unwrap().to_str().unwrap()) { version_id: match Uuid::from_str(h.get(X_AMZ_VERSION_ID).unwrap().to_str().unwrap()) {
Ok(v) => v, Ok(v) => v,
Err(e) => { return Err(std::io::Error::other(e)); } Err(e) => {
return Err(std::io::Error::other(e));
}
}, },
is_delete_marker: delete_marker, is_delete_marker: delete_marker,
replication_ready: replication_ready, replication_ready: replication_ready,
+1 -2
View File
@@ -167,8 +167,7 @@ impl TransitionClient {
content_sha256 = UNSIGNED_PAYLOAD.to_string(); content_sha256 = UNSIGNED_PAYLOAD.to_string();
} }
req req.headers_mut()
.headers_mut()
.insert("X-Amz-Content-Sha256", content_sha256.parse().unwrap()); .insert("X-Amz-Content-Sha256", content_sha256.parse().unwrap());
let req = rustfs_signer::sign_v4(req, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1"); let req = rustfs_signer::sign_v4(req, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1");
Ok(req) Ok(req)
+4 -4
View File
@@ -16,6 +16,9 @@ pub mod admin_handler_utils;
pub mod api_bucket_policy; pub mod api_bucket_policy;
pub mod api_error_response; pub mod api_error_response;
pub mod api_get_object; pub mod api_get_object;
pub mod api_get_object_acl;
pub mod api_get_object_attributes;
pub mod api_get_object_file;
pub mod api_get_options; pub mod api_get_options;
pub mod api_list; pub mod api_list;
pub mod api_put_object; pub mod api_put_object;
@@ -24,11 +27,8 @@ pub mod api_put_object_multipart;
pub mod api_put_object_streaming; pub mod api_put_object_streaming;
pub mod api_remove; pub mod api_remove;
pub mod api_restore; pub mod api_restore;
pub mod api_stat;
pub mod api_get_object_acl;
pub mod api_get_object_attributes;
pub mod api_get_object_file;
pub mod api_s3_datatypes; pub mod api_s3_datatypes;
pub mod api_stat;
pub mod bucket_cache; pub mod bucket_cache;
pub mod constants; pub mod constants;
pub mod credentials; pub mod credentials;
@@ -31,14 +31,10 @@ pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[Object
remaining = &[]; remaining = &[];
} }
let vc = BucketVersioningSys::get(bucket).await.expect("err!"); let vc = BucketVersioningSys::get(bucket).await.expect("err!");
let _deleted_objs = api.delete_objects( let _deleted_objs = api.delete_objects(bucket, to_del.to_vec(), ObjectOptions {
bucket,
to_del.to_vec(),
ObjectOptions {
//prefix_enabled_fn: vc.prefix_enabled(""), //prefix_enabled_fn: vc.prefix_enabled(""),
version_suspended: vc.suspended(), version_suspended: vc.suspended(),
..Default::default() ..Default::default()
}, });
);
} }
} }
+10 -15
View File
@@ -387,7 +387,11 @@ impl TransitionClient {
&metadata.query_values, &metadata.query_values,
)?; )?;
let Ok(mut req) = Request::builder().method(method).uri(target_url.to_string()).body(Body::empty()) else { let Ok(mut req) = Request::builder()
.method(method)
.uri(target_url.to_string())
.body(Body::empty())
else {
return Err(std::io::Error::other("create request error")); return Err(std::io::Error::other("create request error"));
}; };
@@ -428,13 +432,7 @@ impl TransitionClient {
} }
} }
if signer_type == SignatureType::SignatureV2 { if signer_type == SignatureType::SignatureV2 {
req = rustfs_signer::pre_sign_v2( req = rustfs_signer::pre_sign_v2(req, &access_key_id, &secret_access_key, metadata.expires, is_virtual_host);
req,
&access_key_id,
&secret_access_key,
metadata.expires,
is_virtual_host,
);
} else if signer_type == SignatureType::SignatureV4 { } else if signer_type == SignatureType::SignatureV4 {
req = rustfs_signer::pre_sign_v4( req = rustfs_signer::pre_sign_v4(
req, req,
@@ -458,9 +456,7 @@ impl TransitionClient {
//req.content_length = metadata.content_length; //req.content_length = metadata.content_length;
if metadata.content_length <= -1 { if metadata.content_length <= -1 {
let chunked_value = HeaderValue::from_str(&vec!["chunked"].join(",")).expect("err"); let chunked_value = HeaderValue::from_str(&vec!["chunked"].join(",")).expect("err");
req req.headers_mut().insert(http::header::TRANSFER_ENCODING, chunked_value);
.headers_mut()
.insert(http::header::TRANSFER_ENCODING, chunked_value);
} }
if metadata.content_md5_base64.len() > 0 { if metadata.content_md5_base64.len() > 0 {
@@ -473,8 +469,7 @@ impl TransitionClient {
} }
if signer_type == SignatureType::SignatureV2 { if signer_type == SignatureType::SignatureV2 {
req = req = rustfs_signer::sign_v2(req, metadata.content_length, &access_key_id, &secret_access_key, is_virtual_host);
rustfs_signer::sign_v2(req, metadata.content_length, &access_key_id, &secret_access_key, is_virtual_host);
} else if metadata.stream_sha256 && !self.secure { } else if metadata.stream_sha256 && !self.secure {
if metadata.trailer.len() > 0 { if metadata.trailer.len() > 0 {
for (_, v) in &metadata.trailer { for (_, v) in &metadata.trailer {
@@ -491,8 +486,8 @@ impl TransitionClient {
} else if metadata.trailer.len() > 0 { } else if metadata.trailer.len() > 0 {
sha_header = UNSIGNED_PAYLOAD_TRAILER.to_string(); sha_header = UNSIGNED_PAYLOAD_TRAILER.to_string();
} }
req req.headers_mut()
.headers_mut().insert("X-Amz-Content-Sha256".parse::<HeaderName>().unwrap(), sha_header.parse().expect("err")); .insert("X-Amz-Content-Sha256".parse::<HeaderName>().unwrap(), sha_header.parse().expect("err"));
req = rustfs_signer::sign_v4_trailer( req = rustfs_signer::sign_v4_trailer(
req, req,
+4 -13
View File
@@ -70,29 +70,20 @@ pub async fn read_config_with_metadata<S: StorageAPI>(
} }
pub async fn save_config<S: StorageAPI>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()> { pub async fn save_config<S: StorageAPI>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()> {
save_config_with_opts( save_config_with_opts(api, file, data, &ObjectOptions {
api,
file,
data,
&ObjectOptions {
max_parity: true, max_parity: true,
..Default::default() ..Default::default()
}, })
)
.await .await
} }
pub async fn delete_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<()> { pub async fn delete_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<()> {
match api match api
.delete_object( .delete_object(RUSTFS_META_BUCKET, file, ObjectOptions {
RUSTFS_META_BUCKET,
file,
ObjectOptions {
delete_prefix: true, delete_prefix: true,
delete_prefix_object: true, delete_prefix_object: true,
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
Ok(_) => Ok(()), Ok(_) => Ok(()),
+2 -6
View File
@@ -2026,15 +2026,11 @@ impl DiskAPI for LocalDisk {
) -> Result<()> { ) -> Result<()> {
if path.starts_with(SLASH_SEPARATOR) { if path.starts_with(SLASH_SEPARATOR) {
return self return self
.delete( .delete(volume, path, DeleteOptions {
volume,
path,
DeleteOptions {
recursive: false, recursive: false,
immediate: false, immediate: false,
..Default::default() ..Default::default()
}, })
)
.await; .await;
} }
+2 -5
View File
@@ -317,13 +317,10 @@ enum WriterType {
impl std::fmt::Debug for BitrotWriterWrapper { impl std::fmt::Debug for BitrotWriterWrapper {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("BitrotWriterWrapper") f.debug_struct("BitrotWriterWrapper")
.field( .field("writer_type", &match self.writer_type {
"writer_type",
&match self.writer_type {
WriterType::InlineBuffer => "InlineBuffer", WriterType::InlineBuffer => "InlineBuffer",
WriterType::Other => "Other", WriterType::Other => "Other",
}, })
)
.finish() .finish()
} }
} }
+6 -10
View File
@@ -174,13 +174,10 @@ pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsa
.bucket_sizes .bucket_sizes
.iter() .iter()
.map(|(bucket, &size)| { .map(|(bucket, &size)| {
( (bucket.clone(), BucketUsageInfo {
bucket.clone(),
BucketUsageInfo {
size, size,
..Default::default() ..Default::default()
}, })
)
}) })
.collect(); .collect();
} }
@@ -201,17 +198,16 @@ pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsa
{ {
if let Ok((cfg, _)) = get_replication_config(bucket).await { if let Ok((cfg, _)) = get_replication_config(bucket).await {
if !cfg.role.is_empty() { if !cfg.role.is_empty() {
data_usage_info.replication_info.insert( data_usage_info
cfg.role.clone(), .replication_info
BucketTargetUsageInfo { .insert(cfg.role.clone(), BucketTargetUsageInfo {
replication_failed_size: bui.replication_failed_size_v1, replication_failed_size: bui.replication_failed_size_v1,
replication_failed_count: bui.replication_failed_count_v1, replication_failed_count: bui.replication_failed_count_v1,
replicated_size: bui.replicated_size_v1, replicated_size: bui.replicated_size_v1,
replication_pending_count: bui.replication_pending_count_v1, replication_pending_count: bui.replication_pending_count_v1,
replication_pending_size: bui.replication_pending_size_v1, replication_pending_size: bui.replication_pending_size_v1,
..Default::default() ..Default::default()
}, });
);
} }
} }
} }
+8 -26
View File
@@ -183,14 +183,11 @@ impl AllTierStats {
fn populate_stats(&self, stats: &mut HashMap<String, TierStats>) { fn populate_stats(&self, stats: &mut HashMap<String, TierStats>) {
for (tier, st) in &self.tiers { for (tier, st) in &self.tiers {
stats.insert( stats.insert(tier.clone(), TierStats {
tier.clone(),
TierStats {
total_size: st.total_size, total_size: st.total_size,
num_versions: st.num_versions, num_versions: st.num_versions,
num_objects: st.num_objects, num_objects: st.num_objects,
}, });
);
} }
} }
} }
@@ -446,16 +443,10 @@ impl DataUsageCache {
let path = Path::new(BUCKET_META_PREFIX).join(name); let path = Path::new(BUCKET_META_PREFIX).join(name);
// warn!("Loading data usage cache from backend: {}", path.display()); // warn!("Loading data usage cache from backend: {}", path.display());
match store match store
.get_object_reader( .get_object_reader(RUSTFS_META_BUCKET, path.to_str().unwrap(), None, HeaderMap::new(), &ObjectOptions {
RUSTFS_META_BUCKET,
path.to_str().unwrap(),
None,
HeaderMap::new(),
&ObjectOptions {
no_lock: true, no_lock: true,
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
Ok(mut reader) => { Ok(mut reader) => {
@@ -469,16 +460,10 @@ impl DataUsageCache {
match err { match err {
Error::FileNotFound | Error::VolumeNotFound => { Error::FileNotFound | Error::VolumeNotFound => {
match store match store
.get_object_reader( .get_object_reader(RUSTFS_META_BUCKET, name, None, HeaderMap::new(), &ObjectOptions {
RUSTFS_META_BUCKET,
name,
None,
HeaderMap::new(),
&ObjectOptions {
no_lock: true, no_lock: true,
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
Ok(mut reader) => { Ok(mut reader) => {
@@ -819,9 +804,7 @@ impl DataUsageCache {
bui.replica_count = rs.replica_count; bui.replica_count = rs.replica_count;
for (arn, stat) in rs.targets.iter() { for (arn, stat) in rs.targets.iter() {
bui.replication_info.insert( bui.replication_info.insert(arn.clone(), BucketTargetUsageInfo {
arn.clone(),
BucketTargetUsageInfo {
replication_pending_size: stat.pending_size, replication_pending_size: stat.pending_size,
replicated_size: stat.replicated_size, replicated_size: stat.replicated_size,
replication_failed_size: stat.failed_size, replication_failed_size: stat.failed_size,
@@ -829,8 +812,7 @@ impl DataUsageCache {
replication_failed_count: stat.failed_count, replication_failed_count: stat.failed_count,
replicated_count: stat.replicated_count, replicated_count: stat.replicated_count,
..Default::default() ..Default::default()
}, });
);
} }
} }
dst.insert(bucket.name.clone(), bui); dst.insert(bucket.name.clone(), bui);
+2 -6
View File
@@ -255,15 +255,11 @@ impl HealingTracker {
pub async fn delete(&self) -> Result<()> { pub async fn delete(&self) -> Result<()> {
if let Some(disk) = &self.disk { if let Some(disk) = &self.disk {
let file_path = Path::new(BUCKET_META_PREFIX).join(HEALING_TRACKER_FILENAME); let file_path = Path::new(BUCKET_META_PREFIX).join(HEALING_TRACKER_FILENAME);
disk.delete( disk.delete(RUSTFS_META_BUCKET, file_path.to_str().unwrap(), DeleteOptions {
RUSTFS_META_BUCKET,
file_path.to_str().unwrap(),
DeleteOptions {
recursive: false, recursive: false,
immediate: false, immediate: false,
..Default::default() ..Default::default()
}, })
)
.await?; .await?;
} }
+2 -5
View File
@@ -148,14 +148,11 @@ async fn collect_local_disks_metrics(disks: &HashSet<String>) -> HashMap<String,
} }
if d.state != *DRIVE_STATE_OK && d.state != *DRIVE_STATE_UNFORMATTED { if d.state != *DRIVE_STATE_OK && d.state != *DRIVE_STATE_UNFORMATTED {
metrics.insert( metrics.insert(d.endpoint.clone(), DiskMetric {
d.endpoint.clone(),
DiskMetric {
n_disks: 1, n_disks: 1,
offline: 1, offline: 1,
..Default::default() ..Default::default()
}, });
);
continue; continue;
} }
+16 -52
View File
@@ -367,14 +367,11 @@ impl PoolMeta {
} }
decom_started = true; decom_started = true;
} }
remembered_pools.insert( remembered_pools.insert(pool.cmd_line.clone(), PoolInfo {
pool.cmd_line.clone(),
PoolInfo {
position: idx, position: idx,
completed: complete, completed: complete,
decom_started, decom_started,
}, });
);
} }
let mut specified_pools = HashMap::new(); let mut specified_pools = HashMap::new();
@@ -725,10 +722,7 @@ impl ECStore {
if version.deleted { if version.deleted {
// TODO: other params // TODO: other params
if let Err(err) = self if let Err(err) = self
.delete_object( .delete_object(bucket.as_str(), &version.name, ObjectOptions {
bucket.as_str(),
&version.name,
ObjectOptions {
versioned: true, versioned: true,
version_id: version_id.clone(), version_id: version_id.clone(),
mod_time: version.mod_time, mod_time: version.mod_time,
@@ -737,8 +731,7 @@ impl ECStore {
delete_marker: true, delete_marker: true,
skip_decommissioned: true, skip_decommissioned: true,
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) { if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) {
@@ -851,16 +844,12 @@ impl ECStore {
if decommissioned == fivs.versions.len() { if decommissioned == fivs.versions.len() {
if let Err(err) = set if let Err(err) = set
.delete_object( .delete_object(bucket.as_str(), &encode_dir_object(&entry.name), ObjectOptions {
bucket.as_str(),
&encode_dir_object(&entry.name),
ObjectOptions {
delete_prefix: true, delete_prefix: true,
delete_prefix_object: true, delete_prefix_object: true,
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
error!("decommission_pool: delete_object err {:?}", &err); error!("decommission_pool: delete_object err {:?}", &err);
@@ -1198,17 +1187,13 @@ impl ECStore {
if object_info.is_multipart() { if object_info.is_multipart() {
let res = match self let res = match self
.new_multipart_upload( .new_multipart_upload(&bucket, &object_info.name, &ObjectOptions {
&bucket,
&object_info.name,
&ObjectOptions {
version_id: object_info.version_id.as_ref().map(|v| v.to_string()), version_id: object_info.version_id.as_ref().map(|v| v.to_string()),
user_defined: object_info.user_defined.clone(), user_defined: object_info.user_defined.clone(),
src_pool_idx: pool_idx, src_pool_idx: pool_idx,
data_movement: true, data_movement: true,
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
Ok(res) => res, Ok(res) => res,
@@ -1239,17 +1224,10 @@ impl ECStore {
let mut data = PutObjReader::from_vec(chunk); let mut data = PutObjReader::from_vec(chunk);
let pi = match self let pi = match self
.put_object_part( .put_object_part(&bucket, &object_info.name, &res.upload_id, part.number, &mut data, &ObjectOptions {
&bucket,
&object_info.name,
&res.upload_id,
part.number,
&mut data,
&ObjectOptions {
preserve_etag: Some(part.etag.clone()), preserve_etag: Some(part.etag.clone()),
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
Ok(pi) => pi, Ok(pi) => pi,
@@ -1269,17 +1247,11 @@ impl ECStore {
if let Err(err) = self if let Err(err) = self
.clone() .clone()
.complete_multipart_upload( .complete_multipart_upload(&bucket, &object_info.name, &res.upload_id, parts, &ObjectOptions {
&bucket,
&object_info.name,
&res.upload_id,
parts,
&ObjectOptions {
data_movement: true, data_movement: true,
mod_time: object_info.mod_time, mod_time: object_info.mod_time,
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
error!("decommission_object: complete_multipart_upload err {:?}", &err); error!("decommission_object: complete_multipart_upload err {:?}", &err);
@@ -1295,11 +1267,7 @@ impl ECStore {
let mut data = PutObjReader::new(hrd); let mut data = PutObjReader::new(hrd);
if let Err(err) = self if let Err(err) = self
.put_object( .put_object(&bucket, &object_info.name, &mut data, &ObjectOptions {
&bucket,
&object_info.name,
&mut data,
&ObjectOptions {
src_pool_idx: pool_idx, src_pool_idx: pool_idx,
data_movement: true, data_movement: true,
version_id: object_info.version_id.as_ref().map(|v| v.to_string()), version_id: object_info.version_id.as_ref().map(|v| v.to_string()),
@@ -1308,8 +1276,7 @@ impl ECStore {
preserve_etag: object_info.etag.clone(), preserve_etag: object_info.etag.clone(),
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
error!("decommission_object: put_object err {:?}", &err); error!("decommission_object: put_object err {:?}", &err);
@@ -1349,9 +1316,7 @@ impl SetDisks {
let cb1 = cb_func.clone(); let cb1 = cb_func.clone();
list_path_raw( list_path_raw(rx, ListPathRawOptions {
rx,
ListPathRawOptions {
disks: disks.iter().cloned().map(Some).collect(), disks: disks.iter().cloned().map(Some).collect(),
bucket: bucket_info.name.clone(), bucket: bucket_info.name.clone(),
path: bucket_info.prefix.clone(), path: bucket_info.prefix.clone(),
@@ -1375,8 +1340,7 @@ impl SetDisks {
} }
})), })),
..Default::default() ..Default::default()
}, })
)
.await?; .await?;
Ok(()) Ok(())
+14 -47
View File
@@ -774,10 +774,7 @@ impl ECStore {
let mut error = None; let mut error = None;
if version.deleted { if version.deleted {
if let Err(err) = set if let Err(err) = set
.delete_object( .delete_object(&bucket, &version.name, ObjectOptions {
&bucket,
&version.name,
ObjectOptions {
versioned: true, versioned: true,
version_id: version_id.clone(), version_id: version_id.clone(),
mod_time: version.mod_time, mod_time: version.mod_time,
@@ -786,8 +783,7 @@ impl ECStore {
delete_marker: true, delete_marker: true,
skip_decommissioned: true, skip_decommissioned: true,
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) { if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) {
@@ -883,16 +879,12 @@ impl ECStore {
if rebalanced == fivs.versions.len() { if rebalanced == fivs.versions.len() {
if let Err(err) = set if let Err(err) = set
.delete_object( .delete_object(bucket.as_str(), &encode_dir_object(&entry.name), ObjectOptions {
bucket.as_str(),
&encode_dir_object(&entry.name),
ObjectOptions {
delete_prefix: true, delete_prefix: true,
delete_prefix_object: true, delete_prefix_object: true,
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
error!("rebalance_entry: delete_object err {:?}", &err); error!("rebalance_entry: delete_object err {:?}", &err);
@@ -911,17 +903,13 @@ impl ECStore {
if object_info.is_multipart() { if object_info.is_multipart() {
let res = match self let res = match self
.new_multipart_upload( .new_multipart_upload(&bucket, &object_info.name, &ObjectOptions {
&bucket,
&object_info.name,
&ObjectOptions {
version_id: object_info.version_id.as_ref().map(|v| v.to_string()), version_id: object_info.version_id.as_ref().map(|v| v.to_string()),
user_defined: object_info.user_defined.clone(), user_defined: object_info.user_defined.clone(),
src_pool_idx: pool_idx, src_pool_idx: pool_idx,
data_movement: true, data_movement: true,
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
Ok(res) => res, Ok(res) => res,
@@ -955,17 +943,10 @@ impl ECStore {
let mut data = PutObjReader::from_vec(chunk); let mut data = PutObjReader::from_vec(chunk);
let pi = match self let pi = match self
.put_object_part( .put_object_part(&bucket, &object_info.name, &res.upload_id, part.number, &mut data, &ObjectOptions {
&bucket,
&object_info.name,
&res.upload_id,
part.number,
&mut data,
&ObjectOptions {
preserve_etag: Some(part.etag.clone()), preserve_etag: Some(part.etag.clone()),
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
Ok(pi) => pi, Ok(pi) => pi,
@@ -983,17 +964,11 @@ impl ECStore {
if let Err(err) = self if let Err(err) = self
.clone() .clone()
.complete_multipart_upload( .complete_multipart_upload(&bucket, &object_info.name, &res.upload_id, parts, &ObjectOptions {
&bucket,
&object_info.name,
&res.upload_id,
parts,
&ObjectOptions {
data_movement: true, data_movement: true,
mod_time: object_info.mod_time, mod_time: object_info.mod_time,
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
error!("rebalance_object: complete_multipart_upload err {:?}", &err); error!("rebalance_object: complete_multipart_upload err {:?}", &err);
@@ -1008,11 +983,7 @@ impl ECStore {
let mut data = PutObjReader::new(hrd); let mut data = PutObjReader::new(hrd);
if let Err(err) = self if let Err(err) = self
.put_object( .put_object(&bucket, &object_info.name, &mut data, &ObjectOptions {
&bucket,
&object_info.name,
&mut data,
&ObjectOptions {
src_pool_idx: pool_idx, src_pool_idx: pool_idx,
data_movement: true, data_movement: true,
version_id: object_info.version_id.as_ref().map(|v| v.to_string()), version_id: object_info.version_id.as_ref().map(|v| v.to_string()),
@@ -1021,8 +992,7 @@ impl ECStore {
preserve_etag: object_info.etag.clone(), preserve_etag: object_info.etag.clone(),
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
error!("rebalance_object: put_object err {:?}", &err); error!("rebalance_object: put_object err {:?}", &err);
@@ -1167,9 +1137,7 @@ impl SetDisks {
}; };
let cb1 = cb.clone(); let cb1 = cb.clone();
list_path_raw( list_path_raw(rx, ListPathRawOptions {
rx,
ListPathRawOptions {
disks: disks.iter().cloned().map(Some).collect(), disks: disks.iter().cloned().map(Some).collect(),
bucket: bucket.clone(), bucket: bucket.clone(),
recursice: true, recursice: true,
@@ -1195,8 +1163,7 @@ impl SetDisks {
} }
})), })),
..Default::default() ..Default::default()
}, })
)
.await?; .await?;
info!("list_objects_to_rebalance: list_objects_to_rebalance done"); info!("list_objects_to_rebalance: list_objects_to_rebalance done");
+2 -5
View File
@@ -262,13 +262,10 @@ impl Node for NodeService {
let request = request.into_inner(); let request = request.into_inner();
match self match self
.local_peer .local_peer
.delete_bucket( .delete_bucket(&request.bucket, &DeleteBucketOptions {
&request.bucket,
&DeleteBucketOptions {
force: false, force: false,
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
Ok(_) => Ok(tonic::Response::new(DeleteBucketResponse { Ok(_) => Ok(tonic::Response::new(DeleteBucketResponse {
+22 -74
View File
@@ -406,17 +406,11 @@ impl SetDisks {
let src_object = src_object.clone(); let src_object = src_object.clone();
futures.push(tokio::spawn(async move { futures.push(tokio::spawn(async move {
let _ = disk let _ = disk
.delete_version( .delete_version(&src_bucket, &src_object, fi, false, DeleteOptions {
&src_bucket,
&src_object,
fi,
false,
DeleteOptions {
undo_write: true, undo_write: true,
old_data_dir, old_data_dir,
..Default::default() ..Default::default()
}, })
)
.await .await
.map_err(|e| { .map_err(|e| {
debug!("rename_data delete_version err {:?}", e); debug!("rename_data delete_version err {:?}", e);
@@ -489,14 +483,10 @@ impl SetDisks {
tokio::spawn(async move { tokio::spawn(async move {
if let Some(disk) = disk { if let Some(disk) = disk {
(disk (disk
.delete( .delete(&bucket, &file_path, DeleteOptions {
&bucket,
&file_path,
DeleteOptions {
recursive: true, recursive: true,
..Default::default() ..Default::default()
}, })
)
.await) .await)
.err() .err()
} else { } else {
@@ -695,14 +685,10 @@ impl SetDisks {
if let Some(disk) = disks[i].as_ref() { if let Some(disk) = disks[i].as_ref() {
let _ = disk let _ = disk
.delete( .delete(bucket, &path_join_buf(&[prefix, STORAGE_FORMAT_FILE]), DeleteOptions {
bucket,
&path_join_buf(&[prefix, STORAGE_FORMAT_FILE]),
DeleteOptions {
recursive: true, recursive: true,
..Default::default() ..Default::default()
}, })
)
.await .await
.map_err(|e| { .map_err(|e| {
warn!("write meta revert err {:?}", e); warn!("write meta revert err {:?}", e);
@@ -1681,14 +1667,10 @@ impl SetDisks {
for disk in disks.iter() { for disk in disks.iter() {
futures.push(async move { futures.push(async move {
if let Some(disk) = disk { if let Some(disk) = disk {
disk.delete( disk.delete(bucket, prefix, DeleteOptions {
bucket,
prefix,
DeleteOptions {
recursive: true, recursive: true,
..Default::default() ..Default::default()
}, })
)
.await .await
} else { } else {
Err(DiskError::DiskNotFound) Err(DiskError::DiskNotFound)
@@ -2409,17 +2391,10 @@ impl SetDisks {
// Allow for dangling deletes, on versions that have DataDir missing etc. // Allow for dangling deletes, on versions that have DataDir missing etc.
// this would end up restoring the correct readable versions. // this would end up restoring the correct readable versions.
return match self return match self
.delete_if_dang_ling( .delete_if_dang_ling(bucket, object, &parts_metadata, &errs, &data_errs_by_part, ObjectOptions {
bucket,
object,
&parts_metadata,
&errs,
&data_errs_by_part,
ObjectOptions {
version_id: version_id_op.clone(), version_id: version_id_op.clone(),
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
Ok(m) => { Ok(m) => {
@@ -2725,15 +2700,11 @@ impl SetDisks {
if parts_metadata[index].is_remote() { if parts_metadata[index].is_remote() {
let rm_data_dir = parts_metadata[index].data_dir.unwrap().to_string(); let rm_data_dir = parts_metadata[index].data_dir.unwrap().to_string();
let d_path = Path::new(&encode_dir_object(object)).join(rm_data_dir); let d_path = Path::new(&encode_dir_object(object)).join(rm_data_dir);
disk.delete( disk.delete(bucket, d_path.to_str().unwrap(), DeleteOptions {
bucket,
d_path.to_str().unwrap(),
DeleteOptions {
immediate: true, immediate: true,
recursive: true, recursive: true,
..Default::default() ..Default::default()
}, })
)
.await?; .await?;
} }
@@ -2756,17 +2727,10 @@ impl SetDisks {
Err(err) => { Err(err) => {
let data_errs_by_part = HashMap::new(); let data_errs_by_part = HashMap::new();
match self match self
.delete_if_dang_ling( .delete_if_dang_ling(bucket, object, &parts_metadata, &errs, &data_errs_by_part, ObjectOptions {
bucket,
object,
&parts_metadata,
&errs,
&data_errs_by_part,
ObjectOptions {
version_id: version_id_op.clone(), version_id: version_id_op.clone(),
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
Ok(m) => { Ok(m) => {
@@ -2822,15 +2786,11 @@ impl SetDisks {
let object = object.to_string(); let object = object.to_string();
futures.push(tokio::spawn(async move { futures.push(tokio::spawn(async move {
let _ = disk let _ = disk
.delete( .delete(&bucket, &object, DeleteOptions {
&bucket,
&object,
DeleteOptions {
recursive: false, recursive: false,
immediate: false, immediate: false,
..Default::default() ..Default::default()
}, })
)
.await; .await;
})); }));
} }
@@ -3525,16 +3485,11 @@ impl SetDisks {
Ok(fivs) => fivs, Ok(fivs) => fivs,
Err(err) => { Err(err) => {
match self_clone match self_clone
.heal_object( .heal_object(&bucket, &encoded_entry_name, "", &HealOpts {
&bucket,
&encoded_entry_name,
"",
&HealOpts {
scan_mode, scan_mode,
remove: HEAL_DELETE_DANGLING, remove: HEAL_DELETE_DANGLING,
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
Ok((res, None)) => { Ok((res, None)) => {
@@ -3651,9 +3606,7 @@ impl SetDisks {
let bucket_partial = bucket.clone(); let bucket_partial = bucket.clone();
let heal_entry_agree = heal_entry.clone(); let heal_entry_agree = heal_entry.clone();
let heal_entry_partial = heal_entry.clone(); let heal_entry_partial = heal_entry.clone();
if let Err(err) = list_path_raw( if let Err(err) = list_path_raw(rx, ListPathRawOptions {
rx,
ListPathRawOptions {
disks, disks,
fallback_disks, fallback_disks,
bucket: bucket.clone(), bucket: bucket.clone(),
@@ -3699,8 +3652,7 @@ impl SetDisks {
})), })),
finished: None, finished: None,
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
ret_err = Some(err.into()); ret_err = Some(err.into());
@@ -3743,15 +3695,11 @@ impl SetDisks {
let prefix = prefix.to_string(); let prefix = prefix.to_string();
futures.push(async move { futures.push(async move {
if let Some(disk) = disk_op { if let Some(disk) = disk_op {
disk.delete( disk.delete(&bucket, &prefix, DeleteOptions {
&bucket,
&prefix,
DeleteOptions {
recursive: true, recursive: true,
immediate: true, immediate: true,
..Default::default() ..Default::default()
}, })
)
.await .await
} else { } else {
Ok(()) Ok(())
+4 -10
View File
@@ -557,14 +557,11 @@ impl StorageAPI for Sets {
let idx = self.get_hashed_set_index(obj.object_name.as_str()); let idx = self.get_hashed_set_index(obj.object_name.as_str());
if !set_obj_map.contains_key(&idx) { if !set_obj_map.contains_key(&idx) {
set_obj_map.insert( set_obj_map.insert(idx, vec![DelObj {
idx,
vec![DelObj {
// set_idx: idx, // set_idx: idx,
orig_idx: i, orig_idx: i,
obj: obj.clone(), obj: obj.clone(),
}], }]);
);
} else if let Some(val) = set_obj_map.get_mut(&idx) { } else if let Some(val) = set_obj_map.get_mut(&idx) {
val.push(DelObj { val.push(DelObj {
// set_idx: idx, // set_idx: idx,
@@ -756,13 +753,10 @@ impl StorageAPI for Sets {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> { async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
let (disks, _) = init_storage_disks_with_errors( let (disks, _) = init_storage_disks_with_errors(&self.endpoints.endpoints, &DiskOption {
&self.endpoints.endpoints,
&DiskOption {
cleanup: false, cleanup: false,
health_check: false, health_check: false,
}, })
)
.await; .await;
let (formats, errs) = load_format_erasure_all(&disks, true).await; let (formats, errs) = load_format_erasure_all(&disks, true).await;
if let Err(err) = check_format_erasure_values(&formats, self.set_drive_count) { if let Err(err) = check_format_erasure_values(&formats, self.set_drive_count) {
+12 -34
View File
@@ -154,13 +154,10 @@ impl ECStore {
// validate_parity(partiy_count, pool_eps.drives_per_set)?; // validate_parity(partiy_count, pool_eps.drives_per_set)?;
let (disks, errs) = store_init::init_disks( let (disks, errs) = store_init::init_disks(&pool_eps.endpoints, &DiskOption {
&pool_eps.endpoints,
&DiskOption {
cleanup: true, cleanup: true,
health_check: true, health_check: true,
}, })
)
.await; .await;
check_disk_fatal_errs(&errs)?; check_disk_fatal_errs(&errs)?;
@@ -504,14 +501,10 @@ impl ECStore {
} }
async fn delete_prefix(&self, bucket: &str, object: &str) -> Result<()> { async fn delete_prefix(&self, bucket: &str, object: &str) -> Result<()> {
for pool in self.pools.iter() { for pool in self.pools.iter() {
pool.delete_object( pool.delete_object(bucket, object, ObjectOptions {
bucket,
object,
ObjectOptions {
delete_prefix: true, delete_prefix: true,
..Default::default() ..Default::default()
}, })
)
.await?; .await?;
} }
@@ -621,15 +614,11 @@ impl ECStore {
async fn get_pool_idx(&self, bucket: &str, object: &str, size: i64) -> Result<usize> { async fn get_pool_idx(&self, bucket: &str, object: &str, size: i64) -> Result<usize> {
let idx = match self let idx = match self
.get_pool_idx_existing_with_opts( .get_pool_idx_existing_with_opts(bucket, object, &ObjectOptions {
bucket,
object,
&ObjectOptions {
skip_decommissioned: true, skip_decommissioned: true,
skip_rebalancing: true, skip_rebalancing: true,
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
Ok(res) => res, Ok(res) => res,
@@ -670,16 +659,12 @@ impl ECStore {
} }
async fn get_pool_idx_existing_no_lock(&self, bucket: &str, object: &str) -> Result<usize> { async fn get_pool_idx_existing_no_lock(&self, bucket: &str, object: &str) -> Result<usize> {
self.get_pool_idx_existing_with_opts( self.get_pool_idx_existing_with_opts(bucket, object, &ObjectOptions {
bucket,
object,
&ObjectOptions {
no_lock: true, no_lock: true,
skip_decommissioned: true, skip_decommissioned: true,
skip_rebalancing: true, skip_rebalancing: true,
..Default::default() ..Default::default()
}, })
)
.await .await
} }
@@ -1374,14 +1359,11 @@ impl StorageAPI for ECStore {
if let Err(err) = self.peer_sys.make_bucket(bucket, opts).await { if let Err(err) = self.peer_sys.make_bucket(bucket, opts).await {
if !is_err_bucket_exists(&err.into()) { if !is_err_bucket_exists(&err.into()) {
let _ = self let _ = self
.delete_bucket( .delete_bucket(bucket, &DeleteBucketOptions {
bucket,
&DeleteBucketOptions {
no_lock: true, no_lock: true,
no_recreate: true, no_recreate: true,
..Default::default() ..Default::default()
}, })
)
.await; .await;
} }
}; };
@@ -1684,14 +1666,10 @@ impl StorageAPI for ECStore {
for obj in objects.iter() { for obj in objects.iter() {
futures.push(async move { futures.push(async move {
self.internal_get_pool_info_existing_with_opts( self.internal_get_pool_info_existing_with_opts(bucket, &obj.object_name, &ObjectOptions {
bucket,
&obj.object_name,
&ObjectOptions {
no_lock: true, no_lock: true,
..Default::default() ..Default::default()
}, })
)
.await .await
}); });
} }
+6 -16
View File
@@ -263,14 +263,10 @@ impl ECStore {
// use get // use get
if !opts.prefix.is_empty() && opts.limit == 1 && opts.marker.is_none() { if !opts.prefix.is_empty() && opts.limit == 1 && opts.marker.is_none() {
match self match self
.get_object_info( .get_object_info(&opts.bucket, &opts.prefix, &ObjectOptions {
&opts.bucket,
&opts.prefix,
&ObjectOptions {
no_lock: true, no_lock: true,
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
Ok(res) => { Ok(res) => {
@@ -769,9 +765,7 @@ impl ECStore {
let tx1 = sender.clone(); let tx1 = sender.clone();
let tx2 = sender.clone(); let tx2 = sender.clone();
list_path_raw( list_path_raw(rx.resubscribe(), ListPathRawOptions {
rx.resubscribe(),
ListPathRawOptions {
disks: disks.iter().cloned().map(Some).collect(), disks: disks.iter().cloned().map(Some).collect(),
fallback_disks: fallback_disks.iter().cloned().map(Some).collect(), fallback_disks: fallback_disks.iter().cloned().map(Some).collect(),
bucket: bucket.to_owned(), bucket: bucket.to_owned(),
@@ -809,8 +803,7 @@ impl ECStore {
})), })),
finished: None, finished: None,
..Default::default() ..Default::default()
}, })
)
.await .await
}); });
} }
@@ -1279,9 +1272,7 @@ impl SetDisks {
let tx1 = sender.clone(); let tx1 = sender.clone();
let tx2 = sender.clone(); let tx2 = sender.clone();
list_path_raw( list_path_raw(rx, ListPathRawOptions {
rx,
ListPathRawOptions {
disks: disks.iter().cloned().map(Some).collect(), disks: disks.iter().cloned().map(Some).collect(),
fallback_disks: fallback_disks.iter().cloned().map(Some).collect(), fallback_disks: fallback_disks.iter().cloned().map(Some).collect(),
bucket: opts.bucket, bucket: opts.bucket,
@@ -1316,8 +1307,7 @@ impl SetDisks {
})), })),
finished: None, finished: None,
..Default::default() ..Default::default()
}, })
)
.await .await
.map_err(Error::other) .map_err(Error::other)
} }
+2 -7
View File
@@ -365,15 +365,10 @@ impl TierConfigMgr {
file: &str, file: &str,
data: Bytes, data: Bytes,
) -> std::result::Result<(), std::io::Error> { ) -> std::result::Result<(), std::io::Error> {
self.save_config_with_opts( self.save_config_with_opts(api, file, data, &ObjectOptions {
api,
file,
data,
&ObjectOptions {
max_parity: true, max_parity: true,
..Default::default() ..Default::default()
}, })
)
.await .await
} }
@@ -101,19 +101,13 @@ impl WarmBackend for WarmBackendMinIO {
let part_size = optimal_part_size(length)?; let part_size = optimal_part_size(length)?;
let client = self.0.client.clone(); let client = self.0.client.clone();
let res = client let res = client
.put_object( .put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &PutObjectOptions {
&self.0.bucket,
&self.0.get_dest(object),
r,
length,
&PutObjectOptions {
storage_class: self.0.storage_class.clone(), storage_class: self.0.storage_class.clone(),
part_size: part_size as u64, part_size: part_size as u64,
disable_content_sha256: true, disable_content_sha256: true,
user_metadata: meta, user_metadata: meta,
..Default::default() ..Default::default()
}, })
)
.await?; .await?;
//self.ToObjectError(err, object) //self.ToObjectError(err, object)
Ok(res.version_id) Ok(res.version_id)
@@ -98,19 +98,13 @@ impl WarmBackend for WarmBackendRustFS {
let part_size = optimal_part_size(length)?; let part_size = optimal_part_size(length)?;
let client = self.0.client.clone(); let client = self.0.client.clone();
let res = client let res = client
.put_object( .put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &PutObjectOptions {
&self.0.bucket,
&self.0.get_dest(object),
r,
length,
&PutObjectOptions {
storage_class: self.0.storage_class.clone(), storage_class: self.0.storage_class.clone(),
part_size: part_size as u64, part_size: part_size as u64,
disable_content_sha256: true, disable_content_sha256: true,
user_metadata: meta, user_metadata: meta,
..Default::default() ..Default::default()
}, })
)
.await?; .await?;
//self.ToObjectError(err, object) //self.ToObjectError(err, object)
Ok(res.version_id) Ok(res.version_id)
+2 -8
View File
@@ -127,18 +127,12 @@ impl WarmBackend for WarmBackendS3 {
) -> Result<String, std::io::Error> { ) -> Result<String, std::io::Error> {
let client = self.client.clone(); let client = self.client.clone();
let res = client let res = client
.put_object( .put_object(&self.bucket, &self.get_dest(object), r, length, &PutObjectOptions {
&self.bucket,
&self.get_dest(object),
r,
length,
&PutObjectOptions {
send_content_md5: true, send_content_md5: true,
storage_class: self.storage_class.clone(), storage_class: self.storage_class.clone(),
user_metadata: meta, user_metadata: meta,
..Default::default() ..Default::default()
}, })
)
.await?; .await?;
Ok(res.version_id) Ok(res.version_id)
} }
+2 -5
View File
@@ -1575,14 +1575,11 @@ pub fn get_default_policyes() -> HashMap<String, PolicyDoc> {
default_policies default_policies
.iter() .iter()
.map(|(n, p)| { .map(|(n, p)| {
( (n.to_string(), PolicyDoc {
n.to_string(),
PolicyDoc {
version: 1, version: 1,
policy: p.clone(), policy: p.clone(),
..Default::default() ..Default::default()
}, })
)
}) })
.collect() .collect()
} }
+4 -10
View File
@@ -140,9 +140,7 @@ impl Locker for LocalLocker {
} }
args.resources.iter().enumerate().for_each(|(idx, resource)| { args.resources.iter().enumerate().for_each(|(idx, resource)| {
self.lock_map.insert( self.lock_map.insert(resource.to_string(), vec![LockRequesterInfo {
resource.to_string(),
vec![LockRequesterInfo {
name: resource.to_string(), name: resource.to_string(),
writer: true, writer: true,
source: args.source.to_string(), source: args.source.to_string(),
@@ -152,8 +150,7 @@ impl Locker for LocalLocker {
quorum: args.quorum, quorum: args.quorum,
idx, idx,
..Default::default() ..Default::default()
}], }]);
);
let mut uuid = args.uid.to_string(); let mut uuid = args.uid.to_string();
format_uuid(&mut uuid, &idx); format_uuid(&mut uuid, &idx);
@@ -230,9 +227,7 @@ impl Locker for LocalLocker {
} }
} }
None => { None => {
self.lock_map.insert( self.lock_map.insert(resource.to_string(), vec![LockRequesterInfo {
resource.to_string(),
vec![LockRequesterInfo {
name: resource.to_string(), name: resource.to_string(),
writer: false, writer: false,
source: args.source.to_string(), source: args.source.to_string(),
@@ -240,8 +235,7 @@ impl Locker for LocalLocker {
uid: args.uid.to_string(), uid: args.uid.to_string(),
quorum: args.quorum, quorum: args.quorum,
..Default::default() ..Default::default()
}], }]);
);
} }
} }
let mut uuid = args.uid.to_string(); let mut uuid = args.uid.to_string();
+8 -8
View File
@@ -46,10 +46,10 @@ async fn run(bucket: String, object: String, user: String, service_name: String)
// Record Metrics // Record Metrics
let meter = global::meter("rustfs"); let meter = global::meter("rustfs");
let request_duration = meter.f64_histogram("s3_request_duration_seconds").build(); let request_duration = meter.f64_histogram("s3_request_duration_seconds").build();
request_duration.record( request_duration.record(start_time.elapsed().unwrap().as_secs_f64(), &[opentelemetry::KeyValue::new(
start_time.elapsed().unwrap().as_secs_f64(), "operation",
&[opentelemetry::KeyValue::new("operation", "run")], "run",
); )]);
match SystemObserver::init_process_observer(meter).await { match SystemObserver::init_process_observer(meter).await {
Ok(_) => info!("Process observer initialized successfully"), Ok(_) => info!("Process observer initialized successfully"),
@@ -84,10 +84,10 @@ async fn put_object(bucket: String, object: String, user: String) {
let meter = global::meter("rustfs"); let meter = global::meter("rustfs");
let request_duration = meter.f64_histogram("s3_request_duration_seconds").build(); let request_duration = meter.f64_histogram("s3_request_duration_seconds").build();
request_duration.record( request_duration.record(start_time.elapsed().unwrap().as_secs_f64(), &[opentelemetry::KeyValue::new(
start_time.elapsed().unwrap().as_secs_f64(), "operation",
&[opentelemetry::KeyValue::new("operation", "put_object")], "put_object",
); )]);
info!( info!(
"Starting PUT operation content: bucket = {}, object = {}, user = {},start_time = {}", "Starting PUT operation content: bucket = {}, object = {}, user = {},start_time = {}",
+8 -14
View File
@@ -115,24 +115,18 @@ impl Collector {
let transmitted = data.transmitted() as i64; let transmitted = data.transmitted() as i64;
self.metrics.network_io_per_interface.record( self.metrics.network_io_per_interface.record(
received, received,
&[ &[&self.attributes.attributes[..], &[
&self.attributes.attributes[..],
&[
KeyValue::new(INTERFACE, interface_name.to_string()), KeyValue::new(INTERFACE, interface_name.to_string()),
KeyValue::new(DIRECTION, "received"), KeyValue::new(DIRECTION, "received"),
], ]]
]
.concat(), .concat(),
); );
self.metrics.network_io_per_interface.record( self.metrics.network_io_per_interface.record(
transmitted, transmitted,
&[ &[&self.attributes.attributes[..], &[
&self.attributes.attributes[..],
&[
KeyValue::new(INTERFACE, interface_name.to_string()), KeyValue::new(INTERFACE, interface_name.to_string()),
KeyValue::new(DIRECTION, "transmitted"), KeyValue::new(DIRECTION, "transmitted"),
], ]]
]
.concat(), .concat(),
); );
} }
@@ -155,10 +149,10 @@ impl Collector {
}; };
self.metrics.process_status.record( self.metrics.process_status.record(
status_value, status_value,
&[ &[&self.attributes.attributes[..], &[KeyValue::new(
&self.attributes.attributes[..], STATUS,
&[KeyValue::new(STATUS, format!("{:?}", process.status()))], format!("{:?}", process.status()),
] )]]
.concat(), .concat(),
); );
+12 -30
View File
@@ -276,9 +276,7 @@ pub mod default {
#[allow(clippy::incompatible_msrv)] #[allow(clippy::incompatible_msrv)]
pub static DEFAULT_POLICIES: LazyLock<[(&'static str, Policy); 6]> = LazyLock::new(|| { pub static DEFAULT_POLICIES: LazyLock<[(&'static str, Policy); 6]> = LazyLock::new(|| {
[ [
( ("readwrite", Policy {
"readwrite",
Policy {
id: "".into(), id: "".into(),
version: DEFAULT_VERSION.into(), version: DEFAULT_VERSION.into(),
statements: vec![Statement { statements: vec![Statement {
@@ -298,11 +296,8 @@ pub mod default {
conditions: Functions::default(), conditions: Functions::default(),
..Default::default() ..Default::default()
}], }],
}, }),
), ("readonly", Policy {
(
"readonly",
Policy {
id: "".into(), id: "".into(),
version: DEFAULT_VERSION.into(), version: DEFAULT_VERSION.into(),
statements: vec![Statement { statements: vec![Statement {
@@ -323,11 +318,8 @@ pub mod default {
conditions: Functions::default(), conditions: Functions::default(),
..Default::default() ..Default::default()
}], }],
}, }),
), ("writeonly", Policy {
(
"writeonly",
Policy {
id: "".into(), id: "".into(),
version: DEFAULT_VERSION.into(), version: DEFAULT_VERSION.into(),
statements: vec![Statement { statements: vec![Statement {
@@ -347,11 +339,8 @@ pub mod default {
conditions: Functions::default(), conditions: Functions::default(),
..Default::default() ..Default::default()
}], }],
}, }),
), ("writeonly", Policy {
(
"writeonly",
Policy {
id: "".into(), id: "".into(),
version: DEFAULT_VERSION.into(), version: DEFAULT_VERSION.into(),
statements: vec![Statement { statements: vec![Statement {
@@ -371,11 +360,8 @@ pub mod default {
conditions: Functions::default(), conditions: Functions::default(),
..Default::default() ..Default::default()
}], }],
}, }),
), ("diagnostics", Policy {
(
"diagnostics",
Policy {
id: "".into(), id: "".into(),
version: DEFAULT_VERSION.into(), version: DEFAULT_VERSION.into(),
statements: vec![Statement { statements: vec![Statement {
@@ -402,11 +388,8 @@ pub mod default {
conditions: Functions::default(), conditions: Functions::default(),
..Default::default() ..Default::default()
}], }],
}, }),
), ("consoleAdmin", Policy {
(
"consoleAdmin",
Policy {
id: "".into(), id: "".into(),
version: DEFAULT_VERSION.into(), version: DEFAULT_VERSION.into(),
statements: vec![ statements: vec![
@@ -454,8 +437,7 @@ pub mod default {
..Default::default() ..Default::default()
}, },
], ],
}, }),
),
] ]
}); });
} }
+3 -7
View File
@@ -85,13 +85,9 @@ fn main() -> Result<(), AnyError> {
Err(_) => "flatc".to_string(), Err(_) => "flatc".to_string(),
}; };
compile_flatbuffers_models( compile_flatbuffers_models(&mut generated_mod_rs, &flatc_path, proto_dir.clone(), flatbuffer_out_dir.clone(), vec![
&mut generated_mod_rs, "models",
&flatc_path, ])?;
proto_dir.clone(),
flatbuffer_out_dir.clone(),
vec!["models"],
)?;
fmt(); fmt();
Ok(()) Ok(())
@@ -17,9 +17,9 @@ use lazy_static::lazy_static;
use std::collections::HashMap; use std::collections::HashMap;
use time::{OffsetDateTime, macros::format_description}; use time::{OffsetDateTime, macros::format_description};
use s3s::Body;
use super::request_signature_v4::{SERVICE_TYPE_S3, get_scope, get_signature, get_signing_key}; use super::request_signature_v4::{SERVICE_TYPE_S3, get_scope, get_signature, get_signing_key};
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
use s3s::Body;
const STREAMING_SIGN_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; const STREAMING_SIGN_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD";
const STREAMING_SIGN_TRAILER_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"; const STREAMING_SIGN_TRAILER_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER";
+1 -2
View File
@@ -19,15 +19,14 @@ use std::collections::HashMap;
use std::fmt::Write; use std::fmt::Write;
use time::{OffsetDateTime, format_description}; use time::{OffsetDateTime, format_description};
use s3s::Body;
use super::utils::get_host_addr; use super::utils::get_host_addr;
use rustfs_utils::crypto::{base64_encode, hex, hmac_sha1}; use rustfs_utils::crypto::{base64_encode, hex, hmac_sha1};
use s3s::Body;
const _SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256"; const _SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
const SIGN_V2_ALGORITHM: &str = "AWS"; const SIGN_V2_ALGORITHM: &str = "AWS";
fn encode_url2path(req: &request::Request<Body>, _virtual_host: bool) -> String { fn encode_url2path(req: &request::Request<Body>, _virtual_host: bool) -> String {
req.uri().path().to_string() req.uri().path().to_string()
} }
+9 -4
View File
@@ -22,11 +22,11 @@ use std::fmt::Write;
use time::{OffsetDateTime, macros::format_description}; use time::{OffsetDateTime, macros::format_description};
use tracing::debug; use tracing::debug;
use rustfs_utils::crypto::{hex, hex_sha256, hmac_sha256};
use s3s::Body;
use super::constants::UNSIGNED_PAYLOAD; use super::constants::UNSIGNED_PAYLOAD;
use super::request_signature_streaming_unsigned_trailer::streaming_unsigned_v4; use super::request_signature_streaming_unsigned_trailer::streaming_unsigned_v4;
use super::utils::{get_host_addr, sign_v4_trim_all}; use super::utils::{get_host_addr, sign_v4_trim_all};
use rustfs_utils::crypto::{hex, hex_sha256, hmac_sha256};
use s3s::Body;
pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256"; pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
pub const SERVICE_TYPE_S3: &str = "s3"; pub const SERVICE_TYPE_S3: &str = "s3";
@@ -270,7 +270,6 @@ pub fn pre_sign_v4(
.unwrap(), .unwrap(),
); );
*req.uri_mut() = Uri::from_parts(parts).unwrap(); *req.uri_mut() = Uri::from_parts(parts).unwrap();
req req
@@ -282,10 +281,16 @@ fn _post_pre_sign_signature_v4(policy_base64: &str, t: OffsetDateTime, secret_ac
get_signature(signing_key, policy_base64) get_signature(signing_key, policy_base64)
} }
fn _sign_v4_sts(req: request::Request<Body>, access_key_id: &str, secret_access_key: &str, location: &str) -> request::Request<Body> { fn _sign_v4_sts(
req: request::Request<Body>,
access_key_id: &str,
secret_access_key: &str,
location: &str,
) -> request::Request<Body> {
sign_v4_inner(req, 0, access_key_id, secret_access_key, "", location, SERVICE_TYPE_STS, HeaderMap::new()) sign_v4_inner(req, 0, access_key_id, secret_access_key, "", location, SERVICE_TYPE_STS, HeaderMap::new())
} }
#[allow(clippy::too_many_arguments)]
fn sign_v4_inner( fn sign_v4_inner(
mut req: request::Request<Body>, mut req: request::Request<Body>,
content_len: i64, content_len: i64,
-2
View File
@@ -1016,7 +1016,6 @@ impl Operation for RemoveRemoteTargetHandler {
))); )));
}; };
let mut need_delete = true; let mut need_delete = true;
if let Some(arnstr) = querys.get("arn") { if let Some(arnstr) = querys.get("arn") {
@@ -1047,7 +1046,6 @@ impl Operation for RemoveRemoteTargetHandler {
error!("need delete target is {}", decoded_str); error!("need delete target is {}", decoded_str);
bucket_targets::remove_bucket_target(bucket, arnstr).await; bucket_targets::remove_bucket_target(bucket, arnstr).await;
} }
} }
// List bucket targets and return as JSON to client // List bucket targets and return as JSON to client
// match bucket_targets::list_bucket_targets(bucket).await { // match bucket_targets::list_bucket_targets(bucket).await {
+2 -5
View File
@@ -453,13 +453,10 @@ impl Operation for ImportBucketMetadata {
// create bucket if not exists // create bucket if not exists
if !bucket_metadatas.contains_key(bucket_name) { if !bucket_metadatas.contains_key(bucket_name) {
if let Err(e) = store if let Err(e) = store
.make_bucket( .make_bucket(bucket_name, &MakeBucketOptions {
bucket_name,
&MakeBucketOptions {
force_create: true, force_create: true,
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
warn!("create bucket failed: {e}"); warn!("create bucket failed: {e}");
+2 -5
View File
@@ -445,9 +445,7 @@ impl Operation for ExportIam {
let users: HashMap<String, AddOrUpdateUserReq> = users let users: HashMap<String, AddOrUpdateUserReq> = users
.into_iter() .into_iter()
.map(|(k, v)| { .map(|(k, v)| {
( (k, AddOrUpdateUserReq {
k,
AddOrUpdateUserReq {
secret_key: v.credentials.secret_key, secret_key: v.credentials.secret_key,
status: { status: {
if v.credentials.status == "off" { if v.credentials.status == "off" {
@@ -457,8 +455,7 @@ impl Operation for ExportIam {
} }
}, },
policy: None, policy: None,
}, })
)
}) })
.collect::<HashMap<String, AddOrUpdateUserReq>>(); .collect::<HashMap<String, AddOrUpdateUserReq>>();
+5 -11
View File
@@ -310,14 +310,11 @@ impl S3 for FS {
}; };
store store
.make_bucket( .make_bucket(&bucket, &MakeBucketOptions {
&bucket,
&MakeBucketOptions {
force_create: true, force_create: true,
lock_enabled: object_lock_enabled_for_bucket.is_some_and(|v| v), lock_enabled: object_lock_enabled_for_bucket.is_some_and(|v| v),
..Default::default() ..Default::default()
}, })
)
.await .await
.map_err(ApiError::from)?; .map_err(ApiError::from)?;
@@ -498,13 +495,10 @@ impl S3 for FS {
}; };
store store
.delete_bucket( .delete_bucket(&input.bucket, &DeleteBucketOptions {
&input.bucket,
&DeleteBucketOptions {
force: false, force: false,
..Default::default() ..Default::default()
}, })
)
.await .await
.map_err(ApiError::from)?; .map_err(ApiError::from)?;
@@ -1976,7 +1970,7 @@ impl S3 for FS {
.. ..
} = req.input; } = req.input;
let mut lr_retention = false; let lr_retention = false;
/*let rcfg = metadata_sys::get_object_lock_config(&bucket).await; /*let rcfg = metadata_sys::get_object_lock_config(&bucket).await;
if let Ok(rcfg) = rcfg { if let Ok(rcfg) = rcfg {
if let Some(rule) = rcfg.0.rule { if let Some(rule) = rcfg.0.rule {