gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-19 11:06:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: make lint build and clippy happy (#71)

Browse Source 
Signed-off-by: yihong0618 <zouzou0208@gmail.com>
This commit is contained in:
yihong
2025-07-07 09:55:53 +08:00
committed by GitHub
parent 040b05c318
commit abd5dff9b5
47 changed files with 1091 additions and 1379 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+4
View File
@@ -8312,8 +8312,12 @@ dependencies = [
"http 1.3.1", "http 1.3.1",
"hyper 1.6.0", "hyper 1.6.0",
"lazy_static", "lazy_static",
"rand 0.9.1",
"rustfs-utils", "rustfs-utils",
"s3s",
"serde",
"serde_urlencoded", "serde_urlencoded",
"tempfile",
"time", "time",
"tracing", "tracing",
] ]
crates/common/src/last_minute.rs
+5 -8
View File
@@ -568,14 +568,11 @@ mod tests {
let mut latency = LastMinuteLatency::default(); let mut latency = LastMinuteLatency::default();
// Add data at time 1000 // Add data at time 1000
latency.add_all( latency.add_all(1000, &AccElem {
1000, total: 10,
&AccElem { size: 0,
total: 10, n: 1,
size: 0, });
n: 1,
},
);
// Forward to time 1030 (30 seconds later) // Forward to time 1030 (30 seconds later)
latency.forward_to(1030); latency.forward_to(1030);
crates/ecstore/src/bucket/metadata_sys.rs
+4 -7
View File
@@ -236,13 +236,10 @@ impl BucketMetadataSys {
futures.push(async move { futures.push(async move {
sleep(Duration::from_millis(30)).await; sleep(Duration::from_millis(30)).await;
let _ = api let _ = api
.heal_bucket( .heal_bucket(&bucket, &HealOpts {
&bucket, recreate: true,
&HealOpts { ..Default::default()
recreate: true, })
..Default::default()
},
)
.await; .await;
load_bucket_metadata(self.api.clone(), bucket.as_str()).await load_bucket_metadata(self.api.clone(), bucket.as_str()).await
}); });
crates/ecstore/src/client/api_bucket_policy.rs
+34 -40
View File
@@ -74,26 +74,23 @@ impl TransitionClient {
url_values.insert("policy".to_string(), "".to_string()); url_values.insert("policy".to_string(), "".to_string());
let resp = self let resp = self
.execute_method( .execute_method(http::Method::DELETE, &mut RequestMetadata {
http::Method::DELETE, bucket_name: bucket_name.to_string(),
&mut RequestMetadata { query_values: url_values,
bucket_name: bucket_name.to_string(), content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
query_values: url_values, object_name: "".to_string(),
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), custom_header: HeaderMap::new(),
object_name: "".to_string(), content_body: ReaderImpl::Body(Bytes::new()),
custom_header: HeaderMap::new(), content_length: 0,
content_body: ReaderImpl::Body(Bytes::new()), content_md5_base64: "".to_string(),
content_length: 0, stream_sha256: false,
content_md5_base64: "".to_string(), trailer: HeaderMap::new(),
stream_sha256: false, pre_sign_url: Default::default(),
trailer: HeaderMap::new(), add_crc: Default::default(),
pre_sign_url: Default::default(), extra_pre_sign_header: Default::default(),
add_crc: Default::default(), bucket_location: Default::default(),
extra_pre_sign_header: Default::default(), expires: Default::default(),
bucket_location: Default::default(), })
expires: Default::default(),
},
)
.await?; .await?;
//defer closeResponse(resp) //defer closeResponse(resp)
@@ -114,26 +111,23 @@ impl TransitionClient {
url_values.insert("policy".to_string(), "".to_string()); url_values.insert("policy".to_string(), "".to_string());
let resp = self let resp = self
.execute_method( .execute_method(http::Method::GET, &mut RequestMetadata {
http::Method::GET, bucket_name: bucket_name.to_string(),
&mut RequestMetadata { query_values: url_values,
bucket_name: bucket_name.to_string(), content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
query_values: url_values, object_name: "".to_string(),
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), custom_header: HeaderMap::new(),
object_name: "".to_string(), content_body: ReaderImpl::Body(Bytes::new()),
custom_header: HeaderMap::new(), content_length: 0,
content_body: ReaderImpl::Body(Bytes::new()), content_md5_base64: "".to_string(),
content_length: 0, stream_sha256: false,
content_md5_base64: "".to_string(), trailer: HeaderMap::new(),
stream_sha256: false, pre_sign_url: Default::default(),
trailer: HeaderMap::new(), add_crc: Default::default(),
pre_sign_url: Default::default(), extra_pre_sign_header: Default::default(),
add_crc: Default::default(), bucket_location: Default::default(),
extra_pre_sign_header: Default::default(), expires: Default::default(),
bucket_location: Default::default(), })
expires: Default::default(),
},
)
.await?; .await?;
let policy = String::from_utf8_lossy(&resp.body().bytes().expect("err").to_vec()).to_string(); let policy = String::from_utf8_lossy(&resp.body().bytes().expect("err").to_vec()).to_string();
crates/ecstore/src/client/api_get_object.rs
+17 -20
View File
@@ -43,26 +43,23 @@ impl TransitionClient {
opts: &GetObjectOptions, opts: &GetObjectOptions,
) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> { ) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> {
let resp = self let resp = self
.execute_method( .execute_method(http::Method::GET, &mut RequestMetadata {
http::Method::GET, bucket_name: bucket_name.to_string(),
&mut RequestMetadata { object_name: object_name.to_string(),
bucket_name: bucket_name.to_string(), query_values: opts.to_query_values(),
object_name: object_name.to_string(), custom_header: opts.header(),
query_values: opts.to_query_values(), content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
custom_header: opts.header(), content_body: ReaderImpl::Body(Bytes::new()),
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), content_length: 0,
content_body: ReaderImpl::Body(Bytes::new()), content_md5_base64: "".to_string(),
content_length: 0, stream_sha256: false,
content_md5_base64: "".to_string(), trailer: HeaderMap::new(),
stream_sha256: false, pre_sign_url: Default::default(),
trailer: HeaderMap::new(), add_crc: Default::default(),
pre_sign_url: Default::default(), extra_pre_sign_header: Default::default(),
add_crc: Default::default(), bucket_location: Default::default(),
extra_pre_sign_header: Default::default(), expires: Default::default(),
bucket_location: Default::default(), })
expires: Default::default(),
},
)
.await?; .await?;
let resp = &resp; let resp = &resp;
crates/ecstore/src/client/api_get_object_acl.rs
+35 -34
View File
@@ -21,40 +21,40 @@
use bytes::Bytes; use bytes::Bytes;
use http::{HeaderMap, HeaderValue}; use http::{HeaderMap, HeaderValue};
use s3s::dto::Owner; use s3s::dto::Owner;
use std::io::Cursor;
use std::collections::HashMap; use std::collections::HashMap;
use std::io::Cursor;
use tokio::io::BufReader; use tokio::io::BufReader;
use rustfs_utils::EMPTY_STRING_SHA256_HASH;
use crate::client::{ use crate::client::{
api_error_response::{err_invalid_argument, http_resp_to_error_response}, api_error_response::{err_invalid_argument, http_resp_to_error_response},
api_get_options::GetObjectOptions, api_get_options::GetObjectOptions,
transition_api::{to_object_info, ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient}, transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info},
}; };
use rustfs_utils::EMPTY_STRING_SHA256_HASH;
#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)] #[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
struct Grantee { pub struct Grantee {
id: String, pub id: String,
display_name: String, pub display_name: String,
uri: String, pub uri: String,
} }
#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)] #[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
struct Grant { pub struct Grant {
grantee: Grantee, pub grantee: Grantee,
permission: String, pub permission: String,
} }
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)] #[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
pub struct AccessControlList { pub struct AccessControlList {
pub grant: Vec<Grant>, pub grant: Vec<Grant>,
pub permission: String, pub permission: String,
} }
#[derive(Debug, Default, serde::Deserialize)] #[derive(Debug, Default, serde::Deserialize)]
pub struct AccessControlPolicy { pub struct AccessControlPolicy {
#[serde(skip)] #[serde(skip)]
owner: Owner, owner: Owner,
pub access_control_list: AccessControlList, pub access_control_list: AccessControlList,
} }
@@ -63,26 +63,23 @@ impl TransitionClient {
let mut url_values = HashMap::new(); let mut url_values = HashMap::new();
url_values.insert("acl".to_string(), "".to_string()); url_values.insert("acl".to_string(), "".to_string());
let mut resp = self let mut resp = self
.execute_method( .execute_method(http::Method::GET, &mut RequestMetadata {
http::Method::GET, bucket_name: bucket_name.to_string(),
&mut RequestMetadata { object_name: object_name.to_string(),
bucket_name: bucket_name.to_string(), query_values: url_values,
object_name: object_name.to_string(), custom_header: HeaderMap::new(),
query_values: url_values, content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
custom_header: HeaderMap::new(), content_body: ReaderImpl::Body(Bytes::new()),
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), content_length: 0,
content_body: ReaderImpl::Body(Bytes::new()), content_md5_base64: "".to_string(),
content_length: 0, stream_sha256: false,
content_md5_base64: "".to_string(), trailer: HeaderMap::new(),
stream_sha256: false, pre_sign_url: Default::default(),
trailer: HeaderMap::new(), add_crc: Default::default(),
pre_sign_url: Default::default(), extra_pre_sign_header: Default::default(),
add_crc: Default::default(), bucket_location: Default::default(),
extra_pre_sign_header: Default::default(), expires: Default::default(),
bucket_location: Default::default(), })
expires: Default::default(),
},
)
.await?; .await?;
if resp.status() != http::StatusCode::OK { if resp.status() != http::StatusCode::OK {
@@ -98,7 +95,9 @@ impl TransitionClient {
} }
}; };
let mut obj_info = self.stat_object(bucket_name, object_name, &GetObjectOptions::default()).await?; let mut obj_info = self
.stat_object(bucket_name, object_name, &GetObjectOptions::default())
.await?;
obj_info.owner.display_name = res.owner.display_name.clone(); obj_info.owner.display_name = res.owner.display_name.clone();
obj_info.owner.id = res.owner.id.clone(); obj_info.owner.id = res.owner.id.clone();
@@ -107,7 +106,9 @@ impl TransitionClient {
let canned_acl = get_canned_acl(&res); let canned_acl = get_canned_acl(&res);
if canned_acl != "" { if canned_acl != "" {
obj_info.metadata.insert("X-Amz-Acl", HeaderValue::from_str(&canned_acl).unwrap()); obj_info
.metadata
.insert("X-Amz-Acl", HeaderValue::from_str(&canned_acl).unwrap());
return Ok(obj_info); return Ok(obj_info);
} }
crates/ecstore/src/client/api_get_object_attributes.rs
+84 -72
View File
@@ -20,41 +20,43 @@
use bytes::Bytes; use bytes::Bytes;
use http::{HeaderMap, HeaderValue}; use http::{HeaderMap, HeaderValue};
use time::OffsetDateTime;
use std::io::Cursor;
use std::collections::HashMap; use std::collections::HashMap;
use std::io::Cursor;
use time::OffsetDateTime;
use tokio::io::BufReader; use tokio::io::BufReader;
use s3s::{Body, dto::Owner};
use s3s::header::{X_AMZ_OBJECT_ATTRIBUTES, X_AMZ_DELETE_MARKER, X_AMZ_METADATA_DIRECTIVE, X_AMZ_VERSION_ID,
X_AMZ_REQUEST_CHARGED, X_AMZ_RESTORE, X_AMZ_PART_NUMBER_MARKER, X_AMZ_MAX_PARTS,};
use rustfs_utils::EMPTY_STRING_SHA256_HASH;
use crate::client::constants::{GET_OBJECT_ATTRIBUTES_MAX_PARTS, GET_OBJECT_ATTRIBUTES_TAGS, ISO8601_DATEFORMAT}; use crate::client::constants::{GET_OBJECT_ATTRIBUTES_MAX_PARTS, GET_OBJECT_ATTRIBUTES_TAGS, ISO8601_DATEFORMAT};
use rustfs_utils::EMPTY_STRING_SHA256_HASH;
use s3s::header::{
X_AMZ_DELETE_MARKER, X_AMZ_MAX_PARTS, X_AMZ_METADATA_DIRECTIVE, X_AMZ_OBJECT_ATTRIBUTES, X_AMZ_PART_NUMBER_MARKER,
X_AMZ_REQUEST_CHARGED, X_AMZ_RESTORE, X_AMZ_VERSION_ID,
};
use s3s::{Body, dto::Owner};
use crate::client::{ use crate::client::{
api_error_response::err_invalid_argument, api_error_response::err_invalid_argument,
api_get_object_acl::AccessControlPolicy,
api_get_options::GetObjectOptions, api_get_options::GetObjectOptions,
transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info}, transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info},
api_get_object_acl::AccessControlPolicy,
}; };
struct ObjectAttributesOptions { pub struct ObjectAttributesOptions {
max_parts: i64, pub max_parts: i64,
version_id: String, pub version_id: String,
part_number_marker: i64, pub part_number_marker: i64,
//server_side_encryption: encrypt::ServerSide, //server_side_encryption: encrypt::ServerSide,
} }
struct ObjectAttributes { pub struct ObjectAttributes {
version_id: String, pub version_id: String,
last_modified: OffsetDateTime, pub last_modified: OffsetDateTime,
object_attributes_response: ObjectAttributesResponse, pub object_attributes_response: ObjectAttributesResponse,
} }
impl ObjectAttributes { impl ObjectAttributes {
fn new() -> Self { fn new() -> Self {
Self { Self {
version_id: "".to_string(), version_id: "".to_string(),
last_modified: OffsetDateTime::now_utc(), last_modified: OffsetDateTime::now_utc(),
object_attributes_response: ObjectAttributesResponse::new(), object_attributes_response: ObjectAttributesResponse::new(),
} }
@@ -63,81 +65,81 @@ impl ObjectAttributes {
#[derive(Debug, Default, serde::Deserialize)] #[derive(Debug, Default, serde::Deserialize)]
pub struct Checksum { pub struct Checksum {
checksum_crc32: String, checksum_crc32: String,
checksum_crc32c: String, checksum_crc32c: String,
checksum_sha1: String, checksum_sha1: String,
checksum_sha256: String, checksum_sha256: String,
} }
impl Checksum { impl Checksum {
fn new() -> Self { fn new() -> Self {
Self { Self {
checksum_crc32: "".to_string(), checksum_crc32: "".to_string(),
checksum_crc32c: "".to_string(), checksum_crc32c: "".to_string(),
checksum_sha1: "".to_string(), checksum_sha1: "".to_string(),
checksum_sha256: "".to_string(), checksum_sha256: "".to_string(),
} }
} }
} }
#[derive(Debug, Default, serde::Deserialize)] #[derive(Debug, Default, serde::Deserialize)]
struct ObjectParts { pub struct ObjectParts {
parts_count: i64, pub parts_count: i64,
part_number_marker: i64, pub part_number_marker: i64,
next_part_number_marker: i64, pub next_part_number_marker: i64,
max_parts: i64, pub max_parts: i64,
is_truncated: bool, is_truncated: bool,
parts: Vec<ObjectAttributePart>, parts: Vec<ObjectAttributePart>,
} }
impl ObjectParts { impl ObjectParts {
fn new() -> Self { fn new() -> Self {
Self { Self {
parts_count: 0, parts_count: 0,
part_number_marker: 0, part_number_marker: 0,
next_part_number_marker: 0, next_part_number_marker: 0,
max_parts: 0, max_parts: 0,
is_truncated: false, is_truncated: false,
parts: Vec::new(), parts: Vec::new(),
} }
} }
} }
#[derive(Debug, Default, serde::Deserialize)] #[derive(Debug, Default, serde::Deserialize)]
struct ObjectAttributesResponse { pub struct ObjectAttributesResponse {
etag: String, pub etag: String,
storage_class: String, pub storage_class: String,
object_size: i64, pub object_size: i64,
checksum: Checksum, pub checksum: Checksum,
object_parts: ObjectParts, pub object_parts: ObjectParts,
} }
impl ObjectAttributesResponse { impl ObjectAttributesResponse {
fn new() -> Self { fn new() -> Self {
Self { Self {
etag: "".to_string(), etag: "".to_string(),
storage_class: "".to_string(), storage_class: "".to_string(),
object_size: 0, object_size: 0,
checksum: Checksum::new(), checksum: Checksum::new(),
object_parts: ObjectParts::new(), object_parts: ObjectParts::new(),
} }
} }
} }
#[derive(Debug, Default, serde::Deserialize)] #[derive(Debug, Default, serde::Deserialize)]
struct ObjectAttributePart { struct ObjectAttributePart {
checksum_crc32: String, checksum_crc32: String,
checksum_crc32c: String, checksum_crc32c: String,
checksum_sha1: String, checksum_sha1: String,
checksum_sha256: String, checksum_sha256: String,
part_number: i64, part_number: i64,
size: i64, size: i64,
} }
impl ObjectAttributes { impl ObjectAttributes {
pub async fn parse_response(&mut self, resp: &mut http::Response<Body>) -> Result<(), std::io::Error> { pub async fn parse_response(&mut self, resp: &mut http::Response<Body>) -> Result<(), std::io::Error> {
let h = resp.headers(); let h = resp.headers();
let mod_time = OffsetDateTime::parse(h.get("Last-Modified").unwrap().to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(); //RFC7231Time let mod_time = OffsetDateTime::parse(h.get("Last-Modified").unwrap().to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(); //RFC7231Time
self.last_modified = mod_time; self.last_modified = mod_time;
self.version_id = h.get(X_AMZ_VERSION_ID).unwrap().to_str().unwrap().to_string(); self.version_id = h.get(X_AMZ_VERSION_ID).unwrap().to_str().unwrap().to_string();
@@ -155,7 +157,12 @@ impl ObjectAttributes {
} }
impl TransitionClient { impl TransitionClient {
pub async fn get_object_attributes(&self, bucket_name: &str, object_name: &str, opts: ObjectAttributesOptions) -> Result<ObjectAttributes, std::io::Error> { pub async fn get_object_attributes(
&self,
bucket_name: &str,
object_name: &str,
opts: ObjectAttributesOptions,
) -> Result<ObjectAttributes, std::io::Error> {
let mut url_values = HashMap::new(); let mut url_values = HashMap::new();
url_values.insert("attributes".to_string(), "".to_string()); url_values.insert("attributes".to_string(), "".to_string());
if opts.version_id != "" { if opts.version_id != "" {
@@ -166,13 +173,19 @@ impl TransitionClient {
headers.insert(X_AMZ_OBJECT_ATTRIBUTES, HeaderValue::from_str(GET_OBJECT_ATTRIBUTES_TAGS).unwrap()); headers.insert(X_AMZ_OBJECT_ATTRIBUTES, HeaderValue::from_str(GET_OBJECT_ATTRIBUTES_TAGS).unwrap());
if opts.part_number_marker > 0 { if opts.part_number_marker > 0 {
headers.insert(X_AMZ_PART_NUMBER_MARKER, HeaderValue::from_str(&opts.part_number_marker.to_string()).unwrap()); headers.insert(
X_AMZ_PART_NUMBER_MARKER,
HeaderValue::from_str(&opts.part_number_marker.to_string()).unwrap(),
);
} }
if opts.max_parts > 0 { if opts.max_parts > 0 {
headers.insert(X_AMZ_MAX_PARTS, HeaderValue::from_str(&opts.max_parts.to_string()).unwrap()); headers.insert(X_AMZ_MAX_PARTS, HeaderValue::from_str(&opts.max_parts.to_string()).unwrap());
} else { } else {
headers.insert(X_AMZ_MAX_PARTS, HeaderValue::from_str(&GET_OBJECT_ATTRIBUTES_MAX_PARTS.to_string()).unwrap()); headers.insert(
X_AMZ_MAX_PARTS,
HeaderValue::from_str(&GET_OBJECT_ATTRIBUTES_MAX_PARTS.to_string()).unwrap(),
);
} }
/*if opts.server_side_encryption.is_some() { /*if opts.server_side_encryption.is_some() {
@@ -180,32 +193,31 @@ impl TransitionClient {
}*/ }*/
let mut resp = self let mut resp = self
.execute_method( .execute_method(http::Method::HEAD, &mut RequestMetadata {
http::Method::HEAD, bucket_name: bucket_name.to_string(),
&mut RequestMetadata { object_name: object_name.to_string(),
bucket_name: bucket_name.to_string(), query_values: url_values,
object_name: object_name.to_string(), custom_header: headers,
query_values: url_values, content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
custom_header: headers, content_md5_base64: "".to_string(),
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), content_body: ReaderImpl::Body(Bytes::new()),
content_md5_base64: "".to_string(), content_length: 0,
content_body: ReaderImpl::Body(Bytes::new()), stream_sha256: false,
content_length: 0, trailer: HeaderMap::new(),
stream_sha256: false, pre_sign_url: Default::default(),
trailer: HeaderMap::new(), add_crc: Default::default(),
pre_sign_url: Default::default(), extra_pre_sign_header: Default::default(),
add_crc: Default::default(), bucket_location: Default::default(),
extra_pre_sign_header: Default::default(), expires: Default::default(),
bucket_location: Default::default(), })
expires: Default::default(),
},
)
.await?; .await?;
let h = resp.headers(); let h = resp.headers();
let has_etag = h.get("ETag").unwrap().to_str().unwrap(); let has_etag = h.get("ETag").unwrap().to_str().unwrap();
if !has_etag.is_empty() { if !has_etag.is_empty() {
return Err(std::io::Error::other("get_object_attributes is not supported by the current endpoint version")); return Err(std::io::Error::other(
"get_object_attributes is not supported by the current endpoint version",
));
} }
if resp.status() != http::StatusCode::OK { if resp.status() != http::StatusCode::OK {
@@ -226,4 +238,4 @@ impl TransitionClient {
Ok(oa) Ok(oa)
} }
} }
crates/ecstore/src/client/api_get_object_file.rs
+13 -7
View File
@@ -21,15 +21,15 @@
use bytes::Bytes; use bytes::Bytes;
use http::HeaderMap; use http::HeaderMap;
use std::io::Cursor; use std::io::Cursor;
use tokio::io::BufReader;
#[cfg(not(windows))] #[cfg(not(windows))]
use std::os::unix::fs::PermissionsExt; use std::os::unix::fs::MetadataExt;
#[cfg(not(windows))] #[cfg(not(windows))]
use std::os::unix::fs::OpenOptionsExt; use std::os::unix::fs::OpenOptionsExt;
#[cfg(not(windows))] #[cfg(not(windows))]
use std::os::unix::fs::MetadataExt; use std::os::unix::fs::PermissionsExt;
#[cfg(windows)] #[cfg(windows)]
use std::os::windows::fs::MetadataExt; use std::os::windows::fs::MetadataExt;
use tokio::io::BufReader;
use crate::client::{ use crate::client::{
api_error_response::err_invalid_argument, api_error_response::err_invalid_argument,
@@ -38,14 +38,20 @@ use crate::client::{
}; };
impl TransitionClient { impl TransitionClient {
pub async fn fget_object(&self, bucket_name: &str, object_name: &str, file_path: &str, opts: GetObjectOptions) -> Result<(), std::io::Error> { pub async fn fget_object(
&self,
bucket_name: &str,
object_name: &str,
file_path: &str,
opts: GetObjectOptions,
) -> Result<(), std::io::Error> {
match std::fs::metadata(file_path) { match std::fs::metadata(file_path) {
Ok(file_path_stat) => { Ok(file_path_stat) => {
let ft = file_path_stat.file_type(); let ft = file_path_stat.file_type();
if ft.is_dir() { if ft.is_dir() {
return Err(std::io::Error::other(err_invalid_argument("filename is a directory."))); return Err(std::io::Error::other(err_invalid_argument("filename is a directory.")));
} }
}, }
Err(err) => { Err(err) => {
return Err(std::io::Error::other(err)); return Err(std::io::Error::other(err));
} }
@@ -77,7 +83,7 @@ impl TransitionClient {
}; };
let mut file_part_path = file_path.to_string(); let mut file_part_path = file_path.to_string();
file_part_path.push_str(""/*sum_sha256_hex(object_stat.etag.as_bytes())*/); file_part_path.push_str("" /*sum_sha256_hex(object_stat.etag.as_bytes())*/);
file_part_path.push_str(".part.rustfs"); file_part_path.push_str(".part.rustfs");
#[cfg(not(windows))] #[cfg(not(windows))]
@@ -138,4 +144,4 @@ impl TransitionClient {
Ok(()) Ok(())
} }
} }
crates/ecstore/src/client/api_list.rs
+17 -20
View File
@@ -76,26 +76,23 @@ impl TransitionClient {
} }
let mut resp = self let mut resp = self
.execute_method( .execute_method(http::Method::GET, &mut RequestMetadata {
http::Method::GET, bucket_name: bucket_name.to_string(),
&mut RequestMetadata { object_name: "".to_string(),
bucket_name: bucket_name.to_string(), query_values: url_values,
object_name: "".to_string(), content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
query_values: url_values, custom_header: headers,
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), content_body: ReaderImpl::Body(Bytes::new()),
custom_header: headers, content_length: 0,
content_body: ReaderImpl::Body(Bytes::new()), content_md5_base64: "".to_string(),
content_length: 0, stream_sha256: false,
content_md5_base64: "".to_string(), trailer: HeaderMap::new(),
stream_sha256: false, pre_sign_url: Default::default(),
trailer: HeaderMap::new(), add_crc: Default::default(),
pre_sign_url: Default::default(), extra_pre_sign_header: Default::default(),
add_crc: Default::default(), bucket_location: Default::default(),
extra_pre_sign_header: Default::default(), expires: Default::default(),
bucket_location: Default::default(), })
expires: Default::default(),
},
)
.await?; .await?;
if resp.status() != StatusCode::OK { if resp.status() != StatusCode::OK {
return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, ""))); return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, "")));
crates/ecstore/src/client/api_remove.rs
+90 -109
View File
@@ -78,26 +78,23 @@ impl TransitionClient {
let headers = HeaderMap::new(); let headers = HeaderMap::new();
let resp = self let resp = self
.execute_method( .execute_method(Method::DELETE, &mut RequestMetadata {
Method::DELETE, bucket_name: bucket_name.to_string(),
&mut RequestMetadata { content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
bucket_name: bucket_name.to_string(), custom_header: headers,
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), object_name: "".to_string(),
custom_header: headers, query_values: Default::default(),
object_name: "".to_string(), content_body: ReaderImpl::Body(Bytes::new()),
query_values: Default::default(), content_length: 0,
content_body: ReaderImpl::Body(Bytes::new()), content_md5_base64: "".to_string(),
content_length: 0, stream_sha256: false,
content_md5_base64: "".to_string(), trailer: HeaderMap::new(),
stream_sha256: false, pre_sign_url: Default::default(),
trailer: HeaderMap::new(), add_crc: Default::default(),
pre_sign_url: Default::default(), extra_pre_sign_header: Default::default(),
add_crc: Default::default(), bucket_location: Default::default(),
extra_pre_sign_header: Default::default(), expires: Default::default(),
bucket_location: Default::default(), })
expires: Default::default(),
},
)
.await?; .await?;
{ {
@@ -109,26 +106,23 @@ impl TransitionClient {
pub async fn remove_bucket(&self, bucket_name: &str) -> Result<(), std::io::Error> { pub async fn remove_bucket(&self, bucket_name: &str) -> Result<(), std::io::Error> {
let resp = self let resp = self
.execute_method( .execute_method(http::Method::DELETE, &mut RequestMetadata {
http::Method::DELETE, bucket_name: bucket_name.to_string(),
&mut RequestMetadata { content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
bucket_name: bucket_name.to_string(), custom_header: Default::default(),
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), object_name: "".to_string(),
custom_header: Default::default(), query_values: Default::default(),
object_name: "".to_string(), content_body: ReaderImpl::Body(Bytes::new()),
query_values: Default::default(), content_length: 0,
content_body: ReaderImpl::Body(Bytes::new()), content_md5_base64: "".to_string(),
content_length: 0, stream_sha256: false,
content_md5_base64: "".to_string(), trailer: HeaderMap::new(),
stream_sha256: false, pre_sign_url: Default::default(),
trailer: HeaderMap::new(), add_crc: Default::default(),
pre_sign_url: Default::default(), extra_pre_sign_header: Default::default(),
add_crc: Default::default(), bucket_location: Default::default(),
extra_pre_sign_header: Default::default(), expires: Default::default(),
bucket_location: Default::default(), })
expires: Default::default(),
},
)
.await?; .await?;
{ {
@@ -163,26 +157,23 @@ impl TransitionClient {
} }
let resp = self let resp = self
.execute_method( .execute_method(http::Method::DELETE, &mut RequestMetadata {
http::Method::DELETE, bucket_name: bucket_name.to_string(),
&mut RequestMetadata { object_name: object_name.to_string(),
bucket_name: bucket_name.to_string(), content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
object_name: object_name.to_string(), query_values: url_values,
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), custom_header: headers,
query_values: url_values, content_body: ReaderImpl::Body(Bytes::new()),
custom_header: headers, content_length: 0,
content_body: ReaderImpl::Body(Bytes::new()), content_md5_base64: "".to_string(),
content_length: 0, stream_sha256: false,
content_md5_base64: "".to_string(), trailer: HeaderMap::new(),
stream_sha256: false, pre_sign_url: Default::default(),
trailer: HeaderMap::new(), add_crc: Default::default(),
pre_sign_url: Default::default(), extra_pre_sign_header: Default::default(),
add_crc: Default::default(), bucket_location: Default::default(),
extra_pre_sign_header: Default::default(), expires: Default::default(),
bucket_location: Default::default(), })
expires: Default::default(),
},
)
.await?; .await?;
Ok(RemoveObjectResult { Ok(RemoveObjectResult {
@@ -277,15 +268,11 @@ impl TransitionClient {
while let Some(object) = objects_rx.recv().await { while let Some(object) = objects_rx.recv().await {
if has_invalid_xml_char(&object.name) { if has_invalid_xml_char(&object.name) {
let remove_result = self let remove_result = self
.remove_object_inner( .remove_object_inner(bucket_name, &object.name, RemoveObjectOptions {
bucket_name, version_id: object.version_id.expect("err").to_string(),
&object.name, governance_bypass: opts.governance_bypass,
RemoveObjectOptions { ..Default::default()
version_id: object.version_id.expect("err").to_string(), })
governance_bypass: opts.governance_bypass,
..Default::default()
},
)
.await?; .await?;
let remove_result_clone = remove_result.clone(); let remove_result_clone = remove_result.clone();
if !remove_result.err.is_none() { if !remove_result.err.is_none() {
@@ -322,26 +309,23 @@ impl TransitionClient {
let remove_bytes = generate_remove_multi_objects_request(&batch); let remove_bytes = generate_remove_multi_objects_request(&batch);
let resp = self let resp = self
.execute_method( .execute_method(http::Method::POST, &mut RequestMetadata {
http::Method::POST, bucket_name: bucket_name.to_string(),
&mut RequestMetadata { query_values: url_values.clone(),
bucket_name: bucket_name.to_string(), content_body: ReaderImpl::Body(Bytes::from(remove_bytes.clone())),
query_values: url_values.clone(), content_length: remove_bytes.len() as i64,
content_body: ReaderImpl::Body(Bytes::from(remove_bytes.clone())), content_md5_base64: base64_encode(&HashAlgorithm::Md5.hash_encode(&remove_bytes).as_ref()),
content_length: remove_bytes.len() as i64, content_sha256_hex: base64_encode(&HashAlgorithm::SHA256.hash_encode(&remove_bytes).as_ref()),
content_md5_base64: base64_encode(&HashAlgorithm::Md5.hash_encode(&remove_bytes).as_ref()), custom_header: headers,
content_sha256_hex: base64_encode(&HashAlgorithm::SHA256.hash_encode(&remove_bytes).as_ref()), object_name: "".to_string(),
custom_header: headers, stream_sha256: false,
object_name: "".to_string(), trailer: HeaderMap::new(),
stream_sha256: false, pre_sign_url: Default::default(),
trailer: HeaderMap::new(), add_crc: Default::default(),
pre_sign_url: Default::default(), extra_pre_sign_header: Default::default(),
add_crc: Default::default(), bucket_location: Default::default(),
extra_pre_sign_header: Default::default(), expires: Default::default(),
bucket_location: Default::default(), })
expires: Default::default(),
},
)
.await?; .await?;
let body_bytes: Vec<u8> = resp.body().bytes().expect("err").to_vec(); let body_bytes: Vec<u8> = resp.body().bytes().expect("err").to_vec();
@@ -369,26 +353,23 @@ impl TransitionClient {
url_values.insert("uploadId".to_string(), upload_id.to_string()); url_values.insert("uploadId".to_string(), upload_id.to_string());
let resp = self let resp = self
.execute_method( .execute_method(http::Method::DELETE, &mut RequestMetadata {
http::Method::DELETE, bucket_name: bucket_name.to_string(),
&mut RequestMetadata { object_name: object_name.to_string(),
bucket_name: bucket_name.to_string(), query_values: url_values,
object_name: object_name.to_string(), content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
query_values: url_values, custom_header: HeaderMap::new(),
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), content_body: ReaderImpl::Body(Bytes::new()),
custom_header: HeaderMap::new(), content_length: 0,
content_body: ReaderImpl::Body(Bytes::new()), content_md5_base64: "".to_string(),
content_length: 0, stream_sha256: false,
content_md5_base64: "".to_string(), trailer: HeaderMap::new(),
stream_sha256: false, pre_sign_url: Default::default(),
trailer: HeaderMap::new(), add_crc: Default::default(),
pre_sign_url: Default::default(), extra_pre_sign_header: Default::default(),
add_crc: Default::default(), bucket_location: Default::default(),
extra_pre_sign_header: Default::default(), expires: Default::default(),
bucket_location: Default::default(), })
expires: Default::default(),
},
)
.await?; .await?;
//if resp.is_some() { //if resp.is_some() {
if resp.status() != StatusCode::NO_CONTENT { if resp.status() != StatusCode::NO_CONTENT {
crates/ecstore/src/client/api_restore.rs
+59 -53
View File
@@ -20,12 +20,15 @@
use bytes::Bytes; use bytes::Bytes;
use http::HeaderMap; use http::HeaderMap;
use std::collections::HashMap;
use std::io::Cursor; use std::io::Cursor;
use tokio::io::BufReader; use tokio::io::BufReader;
use std::collections::HashMap;
use crate::client::{ use crate::client::{
api_error_response::{err_invalid_argument, http_resp_to_error_response}, api_get_object_acl::AccessControlList, api_get_options::GetObjectOptions, transition_api::{to_object_info, ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient} api_error_response::{err_invalid_argument, http_resp_to_error_response},
api_get_object_acl::AccessControlList,
api_get_options::GetObjectOptions,
transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info},
}; };
const TIER_STANDARD: &str = "Standard"; const TIER_STANDARD: &str = "Standard";
@@ -33,55 +36,55 @@ const TIER_BULK: &str = "Bulk";
const TIER_EXPEDITED: &str = "Expedited"; const TIER_EXPEDITED: &str = "Expedited";
#[derive(Debug, Default, serde::Serialize)] #[derive(Debug, Default, serde::Serialize)]
struct GlacierJobParameters { pub struct GlacierJobParameters {
tier: String, pub tier: String,
} }
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)] #[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
struct Encryption { pub struct Encryption {
encryption_type: String, pub encryption_type: String,
kms_context: String, pub kms_context: String,
kms_key_id: String, pub kms_key_id: String,
} }
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)] #[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
struct MetadataEntry { pub struct MetadataEntry {
name: String, pub name: String,
value: String, pub value: String,
} }
#[derive(Debug, Default, serde::Serialize)] #[derive(Debug, Default, serde::Serialize)]
struct S3 { pub struct S3 {
access_control_list: AccessControlList, pub access_control_list: AccessControlList,
bucket_name: String, pub bucket_name: String,
prefix: String, pub prefix: String,
canned_acl: String, pub canned_acl: String,
encryption: Encryption, pub encryption: Encryption,
storage_class: String, pub storage_class: String,
//tagging: Tags, //tagging: Tags,
user_metadata: MetadataEntry, pub user_metadata: MetadataEntry,
} }
#[derive(Debug, Default, serde::Serialize)] #[derive(Debug, Default, serde::Serialize)]
struct SelectParameters { pub struct SelectParameters {
expression_type: String, pub expression_type: String,
expression: String, pub expression: String,
//input_serialization: SelectObjectInputSerialization, //input_serialization: SelectObjectInputSerialization,
//output_serialization: SelectObjectOutputSerialization, //output_serialization: SelectObjectOutputSerialization,
} }
#[derive(Debug, Default, serde::Serialize)] #[derive(Debug, Default, serde::Serialize)]
struct OutputLocation(S3); pub struct OutputLocation(pub S3);
#[derive(Debug, Default, serde::Serialize)] #[derive(Debug, Default, serde::Serialize)]
struct RestoreRequest { pub struct RestoreRequest {
restore_type: String, pub restore_type: String,
tier: String, pub tier: String,
days: i64, pub days: i64,
glacier_job_parameters: GlacierJobParameters, pub glacier_job_parameters: GlacierJobParameters,
description: String, pub description: String,
select_parameters: SelectParameters, pub select_parameters: SelectParameters,
output_location: OutputLocation, pub output_location: OutputLocation,
} }
impl RestoreRequest { impl RestoreRequest {
@@ -115,7 +118,13 @@ impl RestoreRequest {
} }
impl TransitionClient { impl TransitionClient {
pub async fn restore_object(&self, bucket_name: &str, object_name: &str, version_id: &str, restore_req: &RestoreRequest) -> Result<(), std::io::Error> { pub async fn restore_object(
&self,
bucket_name: &str,
object_name: &str,
version_id: &str,
restore_req: &RestoreRequest,
) -> Result<(), std::io::Error> {
let restore_request = match serde_xml_rs::to_string(restore_req) { let restore_request = match serde_xml_rs::to_string(restore_req) {
Ok(buf) => buf, Ok(buf) => buf,
Err(e) => { Err(e) => {
@@ -132,26 +141,23 @@ impl TransitionClient {
let restore_request_buffer = Bytes::from(restore_request_bytes.clone()); let restore_request_buffer = Bytes::from(restore_request_bytes.clone());
let resp = self let resp = self
.execute_method( .execute_method(http::Method::HEAD, &mut RequestMetadata {
http::Method::HEAD, bucket_name: bucket_name.to_string(),
&mut RequestMetadata { object_name: object_name.to_string(),
bucket_name: bucket_name.to_string(), query_values: url_values,
object_name: object_name.to_string(), custom_header: HeaderMap::new(),
query_values: url_values, content_sha256_hex: "".to_string(), //sum_sha256_hex(&restore_request_bytes),
custom_header: HeaderMap::new(), content_md5_base64: "".to_string(), //sum_md5_base64(&restore_request_bytes),
content_sha256_hex: "".to_string(), //sum_sha256_hex(&restore_request_bytes), content_body: ReaderImpl::Body(restore_request_buffer),
content_md5_base64: "".to_string(), //sum_md5_base64(&restore_request_bytes), content_length: restore_request_bytes.len() as i64,
content_body: ReaderImpl::Body(restore_request_buffer), stream_sha256: false,
content_length: restore_request_bytes.len() as i64, trailer: HeaderMap::new(),
stream_sha256: false, pre_sign_url: Default::default(),
trailer: HeaderMap::new(), add_crc: Default::default(),
pre_sign_url: Default::default(), extra_pre_sign_header: Default::default(),
add_crc: Default::default(), bucket_location: Default::default(),
extra_pre_sign_header: Default::default(), expires: Default::default(),
bucket_location: Default::default(), })
expires: Default::default(),
},
)
.await?; .await?;
let b = resp.body().bytes().expect("err").to_vec(); let b = resp.body().bytes().expect("err").to_vec();
@@ -160,4 +166,4 @@ impl TransitionClient {
} }
Ok(()) Ok(())
} }
} }
crates/ecstore/src/client/api_stat.rs
+63 -56
View File
@@ -21,40 +21,37 @@
use bytes::Bytes; use bytes::Bytes;
use http::{HeaderMap, HeaderValue}; use http::{HeaderMap, HeaderValue};
use rustfs_utils::EMPTY_STRING_SHA256_HASH; use rustfs_utils::EMPTY_STRING_SHA256_HASH;
use uuid::Uuid;
use std::{collections::HashMap, str::FromStr}; use std::{collections::HashMap, str::FromStr};
use tokio::io::BufReader; use tokio::io::BufReader;
use uuid::Uuid;
use s3s::header::{X_AMZ_DELETE_MARKER, X_AMZ_VERSION_ID};
use crate::client::{ use crate::client::{
api_error_response::{err_invalid_argument, http_resp_to_error_response, ErrorResponse}, api_error_response::{ErrorResponse, err_invalid_argument, http_resp_to_error_response},
api_get_options::GetObjectOptions, api_get_options::GetObjectOptions,
transition_api::{to_object_info, ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient}, transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info},
}; };
use s3s::header::{X_AMZ_DELETE_MARKER, X_AMZ_VERSION_ID};
impl TransitionClient { impl TransitionClient {
pub async fn bucket_exists(&self, bucket_name: &str) -> Result<bool, std::io::Error> { pub async fn bucket_exists(&self, bucket_name: &str) -> Result<bool, std::io::Error> {
let resp = self let resp = self
.execute_method( .execute_method(http::Method::HEAD, &mut RequestMetadata {
http::Method::HEAD, bucket_name: bucket_name.to_string(),
&mut RequestMetadata { object_name: "".to_string(),
bucket_name: bucket_name.to_string(), query_values: HashMap::new(),
object_name: "".to_string(), custom_header: HeaderMap::new(),
query_values: HashMap::new(), content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
custom_header: HeaderMap::new(), content_md5_base64: "".to_string(),
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), content_body: ReaderImpl::Body(Bytes::new()),
content_md5_base64: "".to_string(), content_length: 0,
content_body: ReaderImpl::Body(Bytes::new()), stream_sha256: false,
content_length: 0, trailer: HeaderMap::new(),
stream_sha256: false, pre_sign_url: Default::default(),
trailer: HeaderMap::new(), add_crc: Default::default(),
pre_sign_url: Default::default(), extra_pre_sign_header: Default::default(),
add_crc: Default::default(), bucket_location: Default::default(),
extra_pre_sign_header: Default::default(), expires: Default::default(),
bucket_location: Default::default(), })
expires: Default::default(),
},
)
.await; .await;
if let Ok(resp) = resp { if let Ok(resp) = resp {
@@ -70,7 +67,12 @@ impl TransitionClient {
Ok(true) Ok(true)
} }
pub async fn stat_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<ObjectInfo, std::io::Error> { pub async fn stat_object(
&self,
bucket_name: &str,
object_name: &str,
opts: &GetObjectOptions,
) -> Result<ObjectInfo, std::io::Error> {
let mut headers = opts.header(); let mut headers = opts.header();
if opts.internal.replication_delete_marker { if opts.internal.replication_delete_marker {
headers.insert("X-Source-DeleteMarker", HeaderValue::from_str("true").unwrap()); headers.insert("X-Source-DeleteMarker", HeaderValue::from_str("true").unwrap());
@@ -80,26 +82,23 @@ impl TransitionClient {
} }
let resp = self let resp = self
.execute_method( .execute_method(http::Method::HEAD, &mut RequestMetadata {
http::Method::HEAD, bucket_name: bucket_name.to_string(),
&mut RequestMetadata { object_name: object_name.to_string(),
bucket_name: bucket_name.to_string(), query_values: opts.to_query_values(),
object_name: object_name.to_string(), custom_header: headers,
query_values: opts.to_query_values(), content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
custom_header: headers, content_md5_base64: "".to_string(),
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), content_body: ReaderImpl::Body(Bytes::new()),
content_md5_base64: "".to_string(), content_length: 0,
content_body: ReaderImpl::Body(Bytes::new()), stream_sha256: false,
content_length: 0, trailer: HeaderMap::new(),
stream_sha256: false, pre_sign_url: Default::default(),
trailer: HeaderMap::new(), add_crc: Default::default(),
pre_sign_url: Default::default(), extra_pre_sign_header: Default::default(),
add_crc: Default::default(), bucket_location: Default::default(),
extra_pre_sign_header: Default::default(), expires: Default::default(),
bucket_location: Default::default(), })
expires: Default::default(),
},
)
.await; .await;
match resp { match resp {
@@ -107,24 +106,30 @@ impl TransitionClient {
let h = resp.headers(); let h = resp.headers();
let delete_marker = if let Some(x_amz_delete_marker) = h.get(X_AMZ_DELETE_MARKER.as_str()) { let delete_marker = if let Some(x_amz_delete_marker) = h.get(X_AMZ_DELETE_MARKER.as_str()) {
x_amz_delete_marker.to_str().unwrap() == "true" x_amz_delete_marker.to_str().unwrap() == "true"
} else { false }; } else {
false
};
let replication_ready = if let Some(x_amz_delete_marker) = h.get("X-Replication-Ready") { let replication_ready = if let Some(x_amz_delete_marker) = h.get("X-Replication-Ready") {
x_amz_delete_marker.to_str().unwrap() == "true" x_amz_delete_marker.to_str().unwrap() == "true"
} else { false }; } else {
false
};
if resp.status() != http::StatusCode::OK && resp.status() != http::StatusCode::PARTIAL_CONTENT { if resp.status() != http::StatusCode::OK && resp.status() != http::StatusCode::PARTIAL_CONTENT {
if resp.status() == http::StatusCode::METHOD_NOT_ALLOWED && opts.version_id != "" && delete_marker { if resp.status() == http::StatusCode::METHOD_NOT_ALLOWED && opts.version_id != "" && delete_marker {
let err_resp = ErrorResponse { let err_resp = ErrorResponse {
status_code: resp.status(), status_code: resp.status(),
code: s3s::S3ErrorCode::MethodNotAllowed, code: s3s::S3ErrorCode::MethodNotAllowed,
message: "the specified method is not allowed against this resource.".to_string(), message: "the specified method is not allowed against this resource.".to_string(),
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
key: object_name.to_string(), key: object_name.to_string(),
..Default::default() ..Default::default()
}; };
return Ok(ObjectInfo { return Ok(ObjectInfo {
version_id: match Uuid::from_str(h.get(X_AMZ_VERSION_ID).unwrap().to_str().unwrap()) { version_id: match Uuid::from_str(h.get(X_AMZ_VERSION_ID).unwrap().to_str().unwrap()) {
Ok(v) => v, Ok(v) => v,
Err(e) => { return Err(std::io::Error::other(e)); } Err(e) => {
return Err(std::io::Error::other(e));
}
}, },
is_delete_marker: delete_marker, is_delete_marker: delete_marker,
..Default::default() ..Default::default()
@@ -132,12 +137,14 @@ impl TransitionClient {
//err_resp //err_resp
} }
return Ok(ObjectInfo { return Ok(ObjectInfo {
version_id: match Uuid::from_str(h.get(X_AMZ_VERSION_ID).unwrap().to_str().unwrap()) { version_id: match Uuid::from_str(h.get(X_AMZ_VERSION_ID).unwrap().to_str().unwrap()) {
Ok(v) => v, Ok(v) => v,
Err(e) => { return Err(std::io::Error::other(e)); } Err(e) => {
return Err(std::io::Error::other(e));
}
}, },
is_delete_marker: delete_marker, is_delete_marker: delete_marker,
replication_ready: replication_ready, replication_ready: replication_ready,
..Default::default() ..Default::default()
}); });
//http_resp_to_error_response(resp, bucket_name, object_name) //http_resp_to_error_response(resp, bucket_name, object_name)
crates/ecstore/src/client/bucket_cache.rs
+1 -2
View File
@@ -167,8 +167,7 @@ impl TransitionClient {
content_sha256 = UNSIGNED_PAYLOAD.to_string(); content_sha256 = UNSIGNED_PAYLOAD.to_string();
} }
req req.headers_mut()
.headers_mut()
.insert("X-Amz-Content-Sha256", content_sha256.parse().unwrap()); .insert("X-Amz-Content-Sha256", content_sha256.parse().unwrap());
let req = rustfs_signer::sign_v4(req, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1"); let req = rustfs_signer::sign_v4(req, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1");
Ok(req) Ok(req)
crates/ecstore/src/client/mod.rs
+4 -4
View File
@@ -16,6 +16,9 @@ pub mod admin_handler_utils;
pub mod api_bucket_policy; pub mod api_bucket_policy;
pub mod api_error_response; pub mod api_error_response;
pub mod api_get_object; pub mod api_get_object;
pub mod api_get_object_acl;
pub mod api_get_object_attributes;
pub mod api_get_object_file;
pub mod api_get_options; pub mod api_get_options;
pub mod api_list; pub mod api_list;
pub mod api_put_object; pub mod api_put_object;
@@ -24,11 +27,8 @@ pub mod api_put_object_multipart;
pub mod api_put_object_streaming; pub mod api_put_object_streaming;
pub mod api_remove; pub mod api_remove;
pub mod api_restore; pub mod api_restore;
pub mod api_stat;
pub mod api_get_object_acl;
pub mod api_get_object_attributes;
pub mod api_get_object_file;
pub mod api_s3_datatypes; pub mod api_s3_datatypes;
pub mod api_stat;
pub mod bucket_cache; pub mod bucket_cache;
pub mod constants; pub mod constants;
pub mod credentials; pub mod credentials;
crates/ecstore/src/client/object_handlers_common.rs
+5 -9
View File
@@ -31,14 +31,10 @@ pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[Object
remaining = &[]; remaining = &[];
} }
let vc = BucketVersioningSys::get(bucket).await.expect("err!"); let vc = BucketVersioningSys::get(bucket).await.expect("err!");
let _deleted_objs = api.delete_objects( let _deleted_objs = api.delete_objects(bucket, to_del.to_vec(), ObjectOptions {
bucket, //prefix_enabled_fn: vc.prefix_enabled(""),
to_del.to_vec(), version_suspended: vc.suspended(),
ObjectOptions { ..Default::default()
//prefix_enabled_fn: vc.prefix_enabled(""), });
version_suspended: vc.suspended(),
..Default::default()
},
);
} }
} }
crates/ecstore/src/client/transition_api.rs
+11 -16
View File
@@ -387,7 +387,11 @@ impl TransitionClient {
&metadata.query_values, &metadata.query_values,
)?; )?;
let Ok(mut req) = Request::builder().method(method).uri(target_url.to_string()).body(Body::empty()) else { let Ok(mut req) = Request::builder()
.method(method)
.uri(target_url.to_string())
.body(Body::empty())
else {
return Err(std::io::Error::other("create request error")); return Err(std::io::Error::other("create request error"));
}; };
@@ -428,13 +432,7 @@ impl TransitionClient {
} }
} }
if signer_type == SignatureType::SignatureV2 { if signer_type == SignatureType::SignatureV2 {
req = rustfs_signer::pre_sign_v2( req = rustfs_signer::pre_sign_v2(req, &access_key_id, &secret_access_key, metadata.expires, is_virtual_host);
req,
&access_key_id,
&secret_access_key,
metadata.expires,
is_virtual_host,
);
} else if signer_type == SignatureType::SignatureV4 { } else if signer_type == SignatureType::SignatureV4 {
req = rustfs_signer::pre_sign_v4( req = rustfs_signer::pre_sign_v4(
req, req,
@@ -458,9 +456,7 @@ impl TransitionClient {
//req.content_length = metadata.content_length; //req.content_length = metadata.content_length;
if metadata.content_length <= -1 { if metadata.content_length <= -1 {
let chunked_value = HeaderValue::from_str(&vec!["chunked"].join(",")).expect("err"); let chunked_value = HeaderValue::from_str(&vec!["chunked"].join(",")).expect("err");
req req.headers_mut().insert(http::header::TRANSFER_ENCODING, chunked_value);
.headers_mut()
.insert(http::header::TRANSFER_ENCODING, chunked_value);
} }
if metadata.content_md5_base64.len() > 0 { if metadata.content_md5_base64.len() > 0 {
@@ -473,8 +469,7 @@ impl TransitionClient {
} }
if signer_type == SignatureType::SignatureV2 { if signer_type == SignatureType::SignatureV2 {
req = req = rustfs_signer::sign_v2(req, metadata.content_length, &access_key_id, &secret_access_key, is_virtual_host);
rustfs_signer::sign_v2(req, metadata.content_length, &access_key_id, &secret_access_key, is_virtual_host);
} else if metadata.stream_sha256 && !self.secure { } else if metadata.stream_sha256 && !self.secure {
if metadata.trailer.len() > 0 { if metadata.trailer.len() > 0 {
for (_, v) in &metadata.trailer { for (_, v) in &metadata.trailer {
@@ -491,8 +486,8 @@ impl TransitionClient {
} else if metadata.trailer.len() > 0 { } else if metadata.trailer.len() > 0 {
sha_header = UNSIGNED_PAYLOAD_TRAILER.to_string(); sha_header = UNSIGNED_PAYLOAD_TRAILER.to_string();
} }
req req.headers_mut()
.headers_mut().insert("X-Amz-Content-Sha256".parse::<HeaderName>().unwrap(), sha_header.parse().expect("err")); .insert("X-Amz-Content-Sha256".parse::<HeaderName>().unwrap(), sha_header.parse().expect("err"));
req = rustfs_signer::sign_v4_trailer( req = rustfs_signer::sign_v4_trailer(
req, req,
@@ -516,7 +511,7 @@ impl TransitionClient {
} }
Ok(req) Ok(req)
} }
pub fn set_user_agent(&self, req: &mut Request<Body>) { pub fn set_user_agent(&self, req: &mut Request<Body>) {
let headers = req.headers_mut(); let headers = req.headers_mut();
crates/ecstore/src/config/com.rs
+9 -18
View File
@@ -70,29 +70,20 @@ pub async fn read_config_with_metadata<S: StorageAPI>(
} }
pub async fn save_config<S: StorageAPI>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()> { pub async fn save_config<S: StorageAPI>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()> {
save_config_with_opts( save_config_with_opts(api, file, data, &ObjectOptions {
api, max_parity: true,
file, ..Default::default()
data, })
&ObjectOptions {
max_parity: true,
..Default::default()
},
)
.await .await
} }
pub async fn delete_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<()> { pub async fn delete_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<()> {
match api match api
.delete_object( .delete_object(RUSTFS_META_BUCKET, file, ObjectOptions {
RUSTFS_META_BUCKET, delete_prefix: true,
file, delete_prefix_object: true,
ObjectOptions { ..Default::default()
delete_prefix: true, })
delete_prefix_object: true,
..Default::default()
},
)
.await .await
{ {
Ok(_) => Ok(()), Ok(_) => Ok(()),
crates/ecstore/src/disk/local.rs
+5 -9
View File
@@ -2026,15 +2026,11 @@ impl DiskAPI for LocalDisk {
) -> Result<()> { ) -> Result<()> {
if path.starts_with(SLASH_SEPARATOR) { if path.starts_with(SLASH_SEPARATOR) {
return self return self
.delete( .delete(volume, path, DeleteOptions {
volume, recursive: false,
path, immediate: false,
DeleteOptions { ..Default::default()
recursive: false, })
immediate: false,
..Default::default()
},
)
.await; .await;
} }
crates/ecstore/src/erasure_coding/bitrot.rs
+4 -7
View File
@@ -317,13 +317,10 @@ enum WriterType {
impl std::fmt::Debug for BitrotWriterWrapper { impl std::fmt::Debug for BitrotWriterWrapper {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("BitrotWriterWrapper") f.debug_struct("BitrotWriterWrapper")
.field( .field("writer_type", &match self.writer_type {
"writer_type", WriterType::InlineBuffer => "InlineBuffer",
&match self.writer_type { WriterType::Other => "Other",
WriterType::InlineBuffer => "InlineBuffer", })
WriterType::Other => "Other",
},
)
.finish() .finish()
} }
} }
crates/ecstore/src/heal/data_usage.rs
+8 -12
View File
@@ -174,13 +174,10 @@ pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsa
.bucket_sizes .bucket_sizes
.iter() .iter()
.map(|(bucket, &size)| { .map(|(bucket, &size)| {
( (bucket.clone(), BucketUsageInfo {
bucket.clone(), size,
BucketUsageInfo { ..Default::default()
size, })
..Default::default()
},
)
}) })
.collect(); .collect();
} }
@@ -201,17 +198,16 @@ pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsa
{ {
if let Ok((cfg, _)) = get_replication_config(bucket).await { if let Ok((cfg, _)) = get_replication_config(bucket).await {
if !cfg.role.is_empty() { if !cfg.role.is_empty() {
data_usage_info.replication_info.insert( data_usage_info
cfg.role.clone(), .replication_info
BucketTargetUsageInfo { .insert(cfg.role.clone(), BucketTargetUsageInfo {
replication_failed_size: bui.replication_failed_size_v1, replication_failed_size: bui.replication_failed_size_v1,
replication_failed_count: bui.replication_failed_count_v1, replication_failed_count: bui.replication_failed_count_v1,
replicated_size: bui.replicated_size_v1, replicated_size: bui.replicated_size_v1,
replication_pending_count: bui.replication_pending_count_v1, replication_pending_count: bui.replication_pending_count_v1,
replication_pending_size: bui.replication_pending_size_v1, replication_pending_size: bui.replication_pending_size_v1,
..Default::default() ..Default::default()
}, });
);
} }
} }
} }
crates/ecstore/src/heal/data_usage_cache.rs
+22 -40
View File
@@ -183,14 +183,11 @@ impl AllTierStats {
fn populate_stats(&self, stats: &mut HashMap<String, TierStats>) { fn populate_stats(&self, stats: &mut HashMap<String, TierStats>) {
for (tier, st) in &self.tiers { for (tier, st) in &self.tiers {
stats.insert( stats.insert(tier.clone(), TierStats {
tier.clone(), total_size: st.total_size,
TierStats { num_versions: st.num_versions,
total_size: st.total_size, num_objects: st.num_objects,
num_versions: st.num_versions, });
num_objects: st.num_objects,
},
);
} }
} }
} }
@@ -446,16 +443,10 @@ impl DataUsageCache {
let path = Path::new(BUCKET_META_PREFIX).join(name); let path = Path::new(BUCKET_META_PREFIX).join(name);
// warn!("Loading data usage cache from backend: {}", path.display()); // warn!("Loading data usage cache from backend: {}", path.display());
match store match store
.get_object_reader( .get_object_reader(RUSTFS_META_BUCKET, path.to_str().unwrap(), None, HeaderMap::new(), &ObjectOptions {
RUSTFS_META_BUCKET, no_lock: true,
path.to_str().unwrap(), ..Default::default()
None, })
HeaderMap::new(),
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await .await
{ {
Ok(mut reader) => { Ok(mut reader) => {
@@ -469,16 +460,10 @@ impl DataUsageCache {
match err { match err {
Error::FileNotFound | Error::VolumeNotFound => { Error::FileNotFound | Error::VolumeNotFound => {
match store match store
.get_object_reader( .get_object_reader(RUSTFS_META_BUCKET, name, None, HeaderMap::new(), &ObjectOptions {
RUSTFS_META_BUCKET, no_lock: true,
name, ..Default::default()
None, })
HeaderMap::new(),
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await .await
{ {
Ok(mut reader) => { Ok(mut reader) => {
@@ -819,18 +804,15 @@ impl DataUsageCache {
bui.replica_count = rs.replica_count; bui.replica_count = rs.replica_count;
for (arn, stat) in rs.targets.iter() { for (arn, stat) in rs.targets.iter() {
bui.replication_info.insert( bui.replication_info.insert(arn.clone(), BucketTargetUsageInfo {
arn.clone(), replication_pending_size: stat.pending_size,
BucketTargetUsageInfo { replicated_size: stat.replicated_size,
replication_pending_size: stat.pending_size, replication_failed_size: stat.failed_size,
replicated_size: stat.replicated_size, replication_pending_count: stat.pending_count,
replication_failed_size: stat.failed_size, replication_failed_count: stat.failed_count,
replication_pending_count: stat.pending_count, replicated_count: stat.replicated_count,
replication_failed_count: stat.failed_count, ..Default::default()
replicated_count: stat.replicated_count, });
..Default::default()
},
);
} }
} }
dst.insert(bucket.name.clone(), bui); dst.insert(bucket.name.clone(), bui);
crates/ecstore/src/heal/heal_commands.rs
+5 -9
View File
@@ -255,15 +255,11 @@ impl HealingTracker {
pub async fn delete(&self) -> Result<()> { pub async fn delete(&self) -> Result<()> {
if let Some(disk) = &self.disk { if let Some(disk) = &self.disk {
let file_path = Path::new(BUCKET_META_PREFIX).join(HEALING_TRACKER_FILENAME); let file_path = Path::new(BUCKET_META_PREFIX).join(HEALING_TRACKER_FILENAME);
disk.delete( disk.delete(RUSTFS_META_BUCKET, file_path.to_str().unwrap(), DeleteOptions {
RUSTFS_META_BUCKET, recursive: false,
file_path.to_str().unwrap(), immediate: false,
DeleteOptions { ..Default::default()
recursive: false, })
immediate: false,
..Default::default()
},
)
.await?; .await?;
} }
crates/ecstore/src/metrics_realtime.rs
+5 -8
View File
@@ -148,14 +148,11 @@ async fn collect_local_disks_metrics(disks: &HashSet<String>) -> HashMap<String,
} }
if d.state != *DRIVE_STATE_OK && d.state != *DRIVE_STATE_UNFORMATTED { if d.state != *DRIVE_STATE_OK && d.state != *DRIVE_STATE_UNFORMATTED {
metrics.insert( metrics.insert(d.endpoint.clone(), DiskMetric {
d.endpoint.clone(), n_disks: 1,
DiskMetric { offline: 1,
n_disks: 1, ..Default::default()
offline: 1, });
..Default::default()
},
);
continue; continue;
} }
crates/ecstore/src/pools.rs
+69 -105
View File
@@ -367,14 +367,11 @@ impl PoolMeta {
} }
decom_started = true; decom_started = true;
} }
remembered_pools.insert( remembered_pools.insert(pool.cmd_line.clone(), PoolInfo {
pool.cmd_line.clone(), position: idx,
PoolInfo { completed: complete,
position: idx, decom_started,
completed: complete, });
decom_started,
},
);
} }
let mut specified_pools = HashMap::new(); let mut specified_pools = HashMap::new();
@@ -725,20 +722,16 @@ impl ECStore {
if version.deleted { if version.deleted {
// TODO: other params // TODO: other params
if let Err(err) = self if let Err(err) = self
.delete_object( .delete_object(bucket.as_str(), &version.name, ObjectOptions {
bucket.as_str(), versioned: true,
&version.name, version_id: version_id.clone(),
ObjectOptions { mod_time: version.mod_time,
versioned: true, src_pool_idx: idx,
version_id: version_id.clone(), data_movement: true,
mod_time: version.mod_time, delete_marker: true,
src_pool_idx: idx, skip_decommissioned: true,
data_movement: true, ..Default::default()
delete_marker: true, })
skip_decommissioned: true,
..Default::default()
},
)
.await .await
{ {
if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) { if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) {
@@ -851,16 +844,12 @@ impl ECStore {
if decommissioned == fivs.versions.len() { if decommissioned == fivs.versions.len() {
if let Err(err) = set if let Err(err) = set
.delete_object( .delete_object(bucket.as_str(), &encode_dir_object(&entry.name), ObjectOptions {
bucket.as_str(), delete_prefix: true,
&encode_dir_object(&entry.name), delete_prefix_object: true,
ObjectOptions {
delete_prefix: true,
delete_prefix_object: true,
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
error!("decommission_pool: delete_object err {:?}", &err); error!("decommission_pool: delete_object err {:?}", &err);
@@ -1198,17 +1187,13 @@ impl ECStore {
if object_info.is_multipart() { if object_info.is_multipart() {
let res = match self let res = match self
.new_multipart_upload( .new_multipart_upload(&bucket, &object_info.name, &ObjectOptions {
&bucket, version_id: object_info.version_id.as_ref().map(|v| v.to_string()),
&object_info.name, user_defined: object_info.user_defined.clone(),
&ObjectOptions { src_pool_idx: pool_idx,
version_id: object_info.version_id.as_ref().map(|v| v.to_string()), data_movement: true,
user_defined: object_info.user_defined.clone(), ..Default::default()
src_pool_idx: pool_idx, })
data_movement: true,
..Default::default()
},
)
.await .await
{ {
Ok(res) => res, Ok(res) => res,
@@ -1239,17 +1224,10 @@ impl ECStore {
let mut data = PutObjReader::from_vec(chunk); let mut data = PutObjReader::from_vec(chunk);
let pi = match self let pi = match self
.put_object_part( .put_object_part(&bucket, &object_info.name, &res.upload_id, part.number, &mut data, &ObjectOptions {
&bucket, preserve_etag: Some(part.etag.clone()),
&object_info.name, ..Default::default()
&res.upload_id, })
part.number,
&mut data,
&ObjectOptions {
preserve_etag: Some(part.etag.clone()),
..Default::default()
},
)
.await .await
{ {
Ok(pi) => pi, Ok(pi) => pi,
@@ -1269,17 +1247,11 @@ impl ECStore {
if let Err(err) = self if let Err(err) = self
.clone() .clone()
.complete_multipart_upload( .complete_multipart_upload(&bucket, &object_info.name, &res.upload_id, parts, &ObjectOptions {
&bucket, data_movement: true,
&object_info.name, mod_time: object_info.mod_time,
&res.upload_id, ..Default::default()
parts, })
&ObjectOptions {
data_movement: true,
mod_time: object_info.mod_time,
..Default::default()
},
)
.await .await
{ {
error!("decommission_object: complete_multipart_upload err {:?}", &err); error!("decommission_object: complete_multipart_upload err {:?}", &err);
@@ -1295,21 +1267,16 @@ impl ECStore {
let mut data = PutObjReader::new(hrd); let mut data = PutObjReader::new(hrd);
if let Err(err) = self if let Err(err) = self
.put_object( .put_object(&bucket, &object_info.name, &mut data, &ObjectOptions {
&bucket, src_pool_idx: pool_idx,
&object_info.name, data_movement: true,
&mut data, version_id: object_info.version_id.as_ref().map(|v| v.to_string()),
&ObjectOptions { mod_time: object_info.mod_time,
src_pool_idx: pool_idx, user_defined: object_info.user_defined.clone(),
data_movement: true, preserve_etag: object_info.etag.clone(),
version_id: object_info.version_id.as_ref().map(|v| v.to_string()),
mod_time: object_info.mod_time,
user_defined: object_info.user_defined.clone(),
preserve_etag: object_info.etag.clone(),
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
error!("decommission_object: put_object err {:?}", &err); error!("decommission_object: put_object err {:?}", &err);
@@ -1349,34 +1316,31 @@ impl SetDisks {
let cb1 = cb_func.clone(); let cb1 = cb_func.clone();
list_path_raw( list_path_raw(rx, ListPathRawOptions {
rx, disks: disks.iter().cloned().map(Some).collect(),
ListPathRawOptions { bucket: bucket_info.name.clone(),
disks: disks.iter().cloned().map(Some).collect(), path: bucket_info.prefix.clone(),
bucket: bucket_info.name.clone(), recursice: true,
path: bucket_info.prefix.clone(), min_disks: listing_quorum,
recursice: true, agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry)))),
min_disks: listing_quorum, partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry)))), let resolver = resolver.clone();
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| { let cb_func = cb_func.clone();
let resolver = resolver.clone(); match entries.resolve(resolver) {
let cb_func = cb_func.clone(); Some(entry) => {
match entries.resolve(resolver) { warn!("decommission_pool: list_objects_to_decommission get {}", &entry.name);
Some(entry) => { Box::pin(async move {
warn!("decommission_pool: list_objects_to_decommission get {}", &entry.name); cb_func(entry).await;
Box::pin(async move { })
cb_func(entry).await;
})
}
None => {
warn!("decommission_pool: list_objects_to_decommission get none");
Box::pin(async {})
}
} }
})), None => {
..Default::default() warn!("decommission_pool: list_objects_to_decommission get none");
}, Box::pin(async {})
) }
}
})),
..Default::default()
})
.await?; .await?;
Ok(()) Ok(())
crates/ecstore/src/rebalance.rs
+65 -98
View File
@@ -774,20 +774,16 @@ impl ECStore {
let mut error = None; let mut error = None;
if version.deleted { if version.deleted {
if let Err(err) = set if let Err(err) = set
.delete_object( .delete_object(&bucket, &version.name, ObjectOptions {
&bucket, versioned: true,
&version.name, version_id: version_id.clone(),
ObjectOptions { mod_time: version.mod_time,
versioned: true, src_pool_idx: pool_index,
version_id: version_id.clone(), data_movement: true,
mod_time: version.mod_time, delete_marker: true,
src_pool_idx: pool_index, skip_decommissioned: true,
data_movement: true, ..Default::default()
delete_marker: true, })
skip_decommissioned: true,
..Default::default()
},
)
.await .await
{ {
if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) { if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) {
@@ -883,16 +879,12 @@ impl ECStore {
if rebalanced == fivs.versions.len() { if rebalanced == fivs.versions.len() {
if let Err(err) = set if let Err(err) = set
.delete_object( .delete_object(bucket.as_str(), &encode_dir_object(&entry.name), ObjectOptions {
bucket.as_str(), delete_prefix: true,
&encode_dir_object(&entry.name), delete_prefix_object: true,
ObjectOptions {
delete_prefix: true,
delete_prefix_object: true,
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
error!("rebalance_entry: delete_object err {:?}", &err); error!("rebalance_entry: delete_object err {:?}", &err);
@@ -911,17 +903,13 @@ impl ECStore {
if object_info.is_multipart() { if object_info.is_multipart() {
let res = match self let res = match self
.new_multipart_upload( .new_multipart_upload(&bucket, &object_info.name, &ObjectOptions {
&bucket, version_id: object_info.version_id.as_ref().map(|v| v.to_string()),
&object_info.name, user_defined: object_info.user_defined.clone(),
&ObjectOptions { src_pool_idx: pool_idx,
version_id: object_info.version_id.as_ref().map(|v| v.to_string()), data_movement: true,
user_defined: object_info.user_defined.clone(), ..Default::default()
src_pool_idx: pool_idx, })
data_movement: true,
..Default::default()
},
)
.await .await
{ {
Ok(res) => res, Ok(res) => res,
@@ -955,17 +943,10 @@ impl ECStore {
let mut data = PutObjReader::from_vec(chunk); let mut data = PutObjReader::from_vec(chunk);
let pi = match self let pi = match self
.put_object_part( .put_object_part(&bucket, &object_info.name, &res.upload_id, part.number, &mut data, &ObjectOptions {
&bucket, preserve_etag: Some(part.etag.clone()),
&object_info.name, ..Default::default()
&res.upload_id, })
part.number,
&mut data,
&ObjectOptions {
preserve_etag: Some(part.etag.clone()),
..Default::default()
},
)
.await .await
{ {
Ok(pi) => pi, Ok(pi) => pi,
@@ -983,17 +964,11 @@ impl ECStore {
if let Err(err) = self if let Err(err) = self
.clone() .clone()
.complete_multipart_upload( .complete_multipart_upload(&bucket, &object_info.name, &res.upload_id, parts, &ObjectOptions {
&bucket, data_movement: true,
&object_info.name, mod_time: object_info.mod_time,
&res.upload_id, ..Default::default()
parts, })
&ObjectOptions {
data_movement: true,
mod_time: object_info.mod_time,
..Default::default()
},
)
.await .await
{ {
error!("rebalance_object: complete_multipart_upload err {:?}", &err); error!("rebalance_object: complete_multipart_upload err {:?}", &err);
@@ -1008,21 +983,16 @@ impl ECStore {
let mut data = PutObjReader::new(hrd); let mut data = PutObjReader::new(hrd);
if let Err(err) = self if let Err(err) = self
.put_object( .put_object(&bucket, &object_info.name, &mut data, &ObjectOptions {
&bucket, src_pool_idx: pool_idx,
&object_info.name, data_movement: true,
&mut data, version_id: object_info.version_id.as_ref().map(|v| v.to_string()),
&ObjectOptions { mod_time: object_info.mod_time,
src_pool_idx: pool_idx, user_defined: object_info.user_defined.clone(),
data_movement: true, preserve_etag: object_info.etag.clone(),
version_id: object_info.version_id.as_ref().map(|v| v.to_string()),
mod_time: object_info.mod_time,
user_defined: object_info.user_defined.clone(),
preserve_etag: object_info.etag.clone(),
..Default::default() ..Default::default()
}, })
)
.await .await
{ {
error!("rebalance_object: put_object err {:?}", &err); error!("rebalance_object: put_object err {:?}", &err);
@@ -1167,36 +1137,33 @@ impl SetDisks {
}; };
let cb1 = cb.clone(); let cb1 = cb.clone();
list_path_raw( list_path_raw(rx, ListPathRawOptions {
rx, disks: disks.iter().cloned().map(Some).collect(),
ListPathRawOptions { bucket: bucket.clone(),
disks: disks.iter().cloned().map(Some).collect(), recursice: true,
bucket: bucket.clone(), min_disks: listing_quorum,
recursice: true, agreed: Some(Box::new(move |entry: MetaCacheEntry| {
min_disks: listing_quorum, info!("list_objects_to_rebalance: agreed: {:?}", &entry.name);
agreed: Some(Box::new(move |entry: MetaCacheEntry| { Box::pin(cb1(entry))
info!("list_objects_to_rebalance: agreed: {:?}", &entry.name); })),
Box::pin(cb1(entry)) partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
})), // let cb = cb.clone();
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| { let resolver = resolver.clone();
// let cb = cb.clone(); let cb = cb.clone();
let resolver = resolver.clone();
let cb = cb.clone();
match entries.resolve(resolver) { match entries.resolve(resolver) {
Some(entry) => { Some(entry) => {
info!("list_objects_to_rebalance: list_objects_to_decommission get {}", &entry.name); info!("list_objects_to_rebalance: list_objects_to_decommission get {}", &entry.name);
Box::pin(async move { cb(entry).await }) Box::pin(async move { cb(entry).await })
}
None => {
info!("list_objects_to_rebalance: list_objects_to_decommission get none");
Box::pin(async {})
}
} }
})), None => {
..Default::default() info!("list_objects_to_rebalance: list_objects_to_decommission get none");
}, Box::pin(async {})
) }
}
})),
..Default::default()
})
.await?; .await?;
info!("list_objects_to_rebalance: list_objects_to_rebalance done"); info!("list_objects_to_rebalance: list_objects_to_rebalance done");
crates/ecstore/src/rpc/tonic_service.rs
+4 -7
View File
@@ -262,13 +262,10 @@ impl Node for NodeService {
let request = request.into_inner(); let request = request.into_inner();
match self match self
.local_peer .local_peer
.delete_bucket( .delete_bucket(&request.bucket, &DeleteBucketOptions {
&request.bucket, force: false,
&DeleteBucketOptions { ..Default::default()
force: false, })
..Default::default()
},
)
.await .await
{ {
Ok(_) => Ok(tonic::Response::new(DeleteBucketResponse { Ok(_) => Ok(tonic::Response::new(DeleteBucketResponse {
crates/ecstore/src/set_disk.rs
+87 -139
View File
@@ -406,17 +406,11 @@ impl SetDisks {
let src_object = src_object.clone(); let src_object = src_object.clone();
futures.push(tokio::spawn(async move { futures.push(tokio::spawn(async move {
let _ = disk let _ = disk
.delete_version( .delete_version(&src_bucket, &src_object, fi, false, DeleteOptions {
&src_bucket, undo_write: true,
&src_object, old_data_dir,
fi, ..Default::default()
false, })
DeleteOptions {
undo_write: true,
old_data_dir,
..Default::default()
},
)
.await .await
.map_err(|e| { .map_err(|e| {
debug!("rename_data delete_version err {:?}", e); debug!("rename_data delete_version err {:?}", e);
@@ -489,14 +483,10 @@ impl SetDisks {
tokio::spawn(async move { tokio::spawn(async move {
if let Some(disk) = disk { if let Some(disk) = disk {
(disk (disk
.delete( .delete(&bucket, &file_path, DeleteOptions {
&bucket, recursive: true,
&file_path, ..Default::default()
DeleteOptions { })
recursive: true,
..Default::default()
},
)
.await) .await)
.err() .err()
} else { } else {
@@ -695,14 +685,10 @@ impl SetDisks {
if let Some(disk) = disks[i].as_ref() { if let Some(disk) = disks[i].as_ref() {
let _ = disk let _ = disk
.delete( .delete(bucket, &path_join_buf(&[prefix, STORAGE_FORMAT_FILE]), DeleteOptions {
bucket, recursive: true,
&path_join_buf(&[prefix, STORAGE_FORMAT_FILE]), ..Default::default()
DeleteOptions { })
recursive: true,
..Default::default()
},
)
.await .await
.map_err(|e| { .map_err(|e| {
warn!("write meta revert err {:?}", e); warn!("write meta revert err {:?}", e);
@@ -1681,14 +1667,10 @@ impl SetDisks {
for disk in disks.iter() { for disk in disks.iter() {
futures.push(async move { futures.push(async move {
if let Some(disk) = disk { if let Some(disk) = disk {
disk.delete( disk.delete(bucket, prefix, DeleteOptions {
bucket, recursive: true,
prefix, ..Default::default()
DeleteOptions { })
recursive: true,
..Default::default()
},
)
.await .await
} else { } else {
Err(DiskError::DiskNotFound) Err(DiskError::DiskNotFound)
@@ -2409,17 +2391,10 @@ impl SetDisks {
// Allow for dangling deletes, on versions that have DataDir missing etc. // Allow for dangling deletes, on versions that have DataDir missing etc.
// this would end up restoring the correct readable versions. // this would end up restoring the correct readable versions.
return match self return match self
.delete_if_dang_ling( .delete_if_dang_ling(bucket, object, &parts_metadata, &errs, &data_errs_by_part, ObjectOptions {
bucket, version_id: version_id_op.clone(),
object, ..Default::default()
&parts_metadata, })
&errs,
&data_errs_by_part,
ObjectOptions {
version_id: version_id_op.clone(),
..Default::default()
},
)
.await .await
{ {
Ok(m) => { Ok(m) => {
@@ -2725,15 +2700,11 @@ impl SetDisks {
if parts_metadata[index].is_remote() { if parts_metadata[index].is_remote() {
let rm_data_dir = parts_metadata[index].data_dir.unwrap().to_string(); let rm_data_dir = parts_metadata[index].data_dir.unwrap().to_string();
let d_path = Path::new(&encode_dir_object(object)).join(rm_data_dir); let d_path = Path::new(&encode_dir_object(object)).join(rm_data_dir);
disk.delete( disk.delete(bucket, d_path.to_str().unwrap(), DeleteOptions {
bucket, immediate: true,
d_path.to_str().unwrap(), recursive: true,
DeleteOptions { ..Default::default()
immediate: true, })
recursive: true,
..Default::default()
},
)
.await?; .await?;
} }
@@ -2756,17 +2727,10 @@ impl SetDisks {
Err(err) => { Err(err) => {
let data_errs_by_part = HashMap::new(); let data_errs_by_part = HashMap::new();
match self match self
.delete_if_dang_ling( .delete_if_dang_ling(bucket, object, &parts_metadata, &errs, &data_errs_by_part, ObjectOptions {
bucket, version_id: version_id_op.clone(),
object, ..Default::default()
&parts_metadata, })
&errs,
&data_errs_by_part,
ObjectOptions {
version_id: version_id_op.clone(),
..Default::default()
},
)
.await .await
{ {
Ok(m) => { Ok(m) => {
@@ -2822,15 +2786,11 @@ impl SetDisks {
let object = object.to_string(); let object = object.to_string();
futures.push(tokio::spawn(async move { futures.push(tokio::spawn(async move {
let _ = disk let _ = disk
.delete( .delete(&bucket, &object, DeleteOptions {
&bucket, recursive: false,
&object, immediate: false,
DeleteOptions { ..Default::default()
recursive: false, })
immediate: false,
..Default::default()
},
)
.await; .await;
})); }));
} }
@@ -3525,16 +3485,11 @@ impl SetDisks {
Ok(fivs) => fivs, Ok(fivs) => fivs,
Err(err) => { Err(err) => {
match self_clone match self_clone
.heal_object( .heal_object(&bucket, &encoded_entry_name, "", &HealOpts {
&bucket, scan_mode,
&encoded_entry_name, remove: HEAL_DELETE_DANGLING,
"", ..Default::default()
&HealOpts { })
scan_mode,
remove: HEAL_DELETE_DANGLING,
..Default::default()
},
)
.await .await
{ {
Ok((res, None)) => { Ok((res, None)) => {
@@ -3651,56 +3606,53 @@ impl SetDisks {
let bucket_partial = bucket.clone(); let bucket_partial = bucket.clone();
let heal_entry_agree = heal_entry.clone(); let heal_entry_agree = heal_entry.clone();
let heal_entry_partial = heal_entry.clone(); let heal_entry_partial = heal_entry.clone();
if let Err(err) = list_path_raw( if let Err(err) = list_path_raw(rx, ListPathRawOptions {
rx, disks,
ListPathRawOptions { fallback_disks,
disks, bucket: bucket.clone(),
fallback_disks, recursice: true,
bucket: bucket.clone(), forward_to,
recursice: true, min_disks: 1,
forward_to, report_not_found: false,
min_disks: 1, agreed: Some(Box::new(move |entry: MetaCacheEntry| {
report_not_found: false, let jt = jt_agree.clone();
agreed: Some(Box::new(move |entry: MetaCacheEntry| { let bucket = bucket_agree.clone();
let jt = jt_agree.clone(); let heal_entry = heal_entry_agree.clone();
let bucket = bucket_agree.clone(); Box::pin(async move {
let heal_entry = heal_entry_agree.clone(); jt.take().await;
Box::pin(async move { let bucket = bucket.clone();
tokio::spawn(async move {
heal_entry(bucket, entry).await;
});
})
})),
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
let jt = jt_partial.clone();
let bucket = bucket_partial.clone();
let heal_entry = heal_entry_partial.clone();
Box::pin({
let heal_entry = heal_entry.clone();
let resolver = resolver.clone();
async move {
let entry = if let Some(entry) = entries.resolve(resolver) {
entry
} else if let (Some(entry), _) = entries.first_found() {
entry
} else {
return;
};
jt.take().await; jt.take().await;
let bucket = bucket.clone(); let bucket = bucket.clone();
let heal_entry = heal_entry.clone();
tokio::spawn(async move { tokio::spawn(async move {
heal_entry(bucket, entry).await; heal_entry(bucket, entry).await;
}); });
}) }
})), })
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| { })),
let jt = jt_partial.clone(); finished: None,
let bucket = bucket_partial.clone(); ..Default::default()
let heal_entry = heal_entry_partial.clone(); })
Box::pin({
let heal_entry = heal_entry.clone();
let resolver = resolver.clone();
async move {
let entry = if let Some(entry) = entries.resolve(resolver) {
entry
} else if let (Some(entry), _) = entries.first_found() {
entry
} else {
return;
};
jt.take().await;
let bucket = bucket.clone();
let heal_entry = heal_entry.clone();
tokio::spawn(async move {
heal_entry(bucket, entry).await;
});
}
})
})),
finished: None,
..Default::default()
},
)
.await .await
{ {
ret_err = Some(err.into()); ret_err = Some(err.into());
@@ -3743,15 +3695,11 @@ impl SetDisks {
let prefix = prefix.to_string(); let prefix = prefix.to_string();
futures.push(async move { futures.push(async move {
if let Some(disk) = disk_op { if let Some(disk) = disk_op {
disk.delete( disk.delete(&bucket, &prefix, DeleteOptions {
&bucket, recursive: true,
&prefix, immediate: true,
DeleteOptions { ..Default::default()
recursive: true, })
immediate: true,
..Default::default()
},
)
.await .await
} else { } else {
Ok(()) Ok(())
crates/ecstore/src/sets.rs
+9 -15
View File
@@ -557,14 +557,11 @@ impl StorageAPI for Sets {
let idx = self.get_hashed_set_index(obj.object_name.as_str()); let idx = self.get_hashed_set_index(obj.object_name.as_str());
if !set_obj_map.contains_key(&idx) { if !set_obj_map.contains_key(&idx) {
set_obj_map.insert( set_obj_map.insert(idx, vec![DelObj {
idx, // set_idx: idx,
vec![DelObj { orig_idx: i,
// set_idx: idx, obj: obj.clone(),
orig_idx: i, }]);
obj: obj.clone(),
}],
);
} else if let Some(val) = set_obj_map.get_mut(&idx) { } else if let Some(val) = set_obj_map.get_mut(&idx) {
val.push(DelObj { val.push(DelObj {
// set_idx: idx, // set_idx: idx,
@@ -756,13 +753,10 @@ impl StorageAPI for Sets {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> { async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
let (disks, _) = init_storage_disks_with_errors( let (disks, _) = init_storage_disks_with_errors(&self.endpoints.endpoints, &DiskOption {
&self.endpoints.endpoints, cleanup: false,
&DiskOption { health_check: false,
cleanup: false, })
health_check: false,
},
)
.await; .await;
let (formats, errs) = load_format_erasure_all(&disks, true).await; let (formats, errs) = load_format_erasure_all(&disks, true).await;
if let Err(err) = check_format_erasure_values(&formats, self.set_drive_count) { if let Err(err) = check_format_erasure_values(&formats, self.set_drive_count) {
crates/ecstore/src/store.rs
+28 -50
View File
@@ -154,13 +154,10 @@ impl ECStore {
// validate_parity(partiy_count, pool_eps.drives_per_set)?; // validate_parity(partiy_count, pool_eps.drives_per_set)?;
let (disks, errs) = store_init::init_disks( let (disks, errs) = store_init::init_disks(&pool_eps.endpoints, &DiskOption {
&pool_eps.endpoints, cleanup: true,
&DiskOption { health_check: true,
cleanup: true, })
health_check: true,
},
)
.await; .await;
check_disk_fatal_errs(&errs)?; check_disk_fatal_errs(&errs)?;
@@ -504,14 +501,10 @@ impl ECStore {
} }
async fn delete_prefix(&self, bucket: &str, object: &str) -> Result<()> { async fn delete_prefix(&self, bucket: &str, object: &str) -> Result<()> {
for pool in self.pools.iter() { for pool in self.pools.iter() {
pool.delete_object( pool.delete_object(bucket, object, ObjectOptions {
bucket, delete_prefix: true,
object, ..Default::default()
ObjectOptions { })
delete_prefix: true,
..Default::default()
},
)
.await?; .await?;
} }
@@ -621,15 +614,11 @@ impl ECStore {
async fn get_pool_idx(&self, bucket: &str, object: &str, size: i64) -> Result<usize> { async fn get_pool_idx(&self, bucket: &str, object: &str, size: i64) -> Result<usize> {
let idx = match self let idx = match self
.get_pool_idx_existing_with_opts( .get_pool_idx_existing_with_opts(bucket, object, &ObjectOptions {
bucket, skip_decommissioned: true,
object, skip_rebalancing: true,
&ObjectOptions { ..Default::default()
skip_decommissioned: true, })
skip_rebalancing: true,
..Default::default()
},
)
.await .await
{ {
Ok(res) => res, Ok(res) => res,
@@ -670,16 +659,12 @@ impl ECStore {
} }
async fn get_pool_idx_existing_no_lock(&self, bucket: &str, object: &str) -> Result<usize> { async fn get_pool_idx_existing_no_lock(&self, bucket: &str, object: &str) -> Result<usize> {
self.get_pool_idx_existing_with_opts( self.get_pool_idx_existing_with_opts(bucket, object, &ObjectOptions {
bucket, no_lock: true,
object, skip_decommissioned: true,
&ObjectOptions { skip_rebalancing: true,
no_lock: true, ..Default::default()
skip_decommissioned: true, })
skip_rebalancing: true,
..Default::default()
},
)
.await .await
} }
@@ -1374,14 +1359,11 @@ impl StorageAPI for ECStore {
if let Err(err) = self.peer_sys.make_bucket(bucket, opts).await { if let Err(err) = self.peer_sys.make_bucket(bucket, opts).await {
if !is_err_bucket_exists(&err.into()) { if !is_err_bucket_exists(&err.into()) {
let _ = self let _ = self
.delete_bucket( .delete_bucket(bucket, &DeleteBucketOptions {
bucket, no_lock: true,
&DeleteBucketOptions { no_recreate: true,
no_lock: true, ..Default::default()
no_recreate: true, })
..Default::default()
},
)
.await; .await;
} }
}; };
@@ -1684,14 +1666,10 @@ impl StorageAPI for ECStore {
for obj in objects.iter() { for obj in objects.iter() {
futures.push(async move { futures.push(async move {
self.internal_get_pool_info_existing_with_opts( self.internal_get_pool_info_existing_with_opts(bucket, &obj.object_name, &ObjectOptions {
bucket, no_lock: true,
&obj.object_name, ..Default::default()
&ObjectOptions { })
no_lock: true,
..Default::default()
},
)
.await .await
}); });
} }
crates/ecstore/src/store_list_objects.rs
+71 -81
View File
@@ -263,14 +263,10 @@ impl ECStore {
// use get // use get
if !opts.prefix.is_empty() && opts.limit == 1 && opts.marker.is_none() { if !opts.prefix.is_empty() && opts.limit == 1 && opts.marker.is_none() {
match self match self
.get_object_info( .get_object_info(&opts.bucket, &opts.prefix, &ObjectOptions {
&opts.bucket, no_lock: true,
&opts.prefix, ..Default::default()
&ObjectOptions { })
no_lock: true,
..Default::default()
},
)
.await .await
{ {
Ok(res) => { Ok(res) => {
@@ -769,48 +765,45 @@ impl ECStore {
let tx1 = sender.clone(); let tx1 = sender.clone();
let tx2 = sender.clone(); let tx2 = sender.clone();
list_path_raw( list_path_raw(rx.resubscribe(), ListPathRawOptions {
rx.resubscribe(), disks: disks.iter().cloned().map(Some).collect(),
ListPathRawOptions { fallback_disks: fallback_disks.iter().cloned().map(Some).collect(),
disks: disks.iter().cloned().map(Some).collect(), bucket: bucket.to_owned(),
fallback_disks: fallback_disks.iter().cloned().map(Some).collect(), path,
bucket: bucket.to_owned(), recursice: true,
path, filter_prefix: Some(filter_prefix),
recursice: true, forward_to: opts.marker.clone(),
filter_prefix: Some(filter_prefix), min_disks: listing_quorum,
forward_to: opts.marker.clone(), per_disk_limit: opts.limit as i32,
min_disks: listing_quorum, agreed: Some(Box::new(move |entry: MetaCacheEntry| {
per_disk_limit: opts.limit as i32, Box::pin({
agreed: Some(Box::new(move |entry: MetaCacheEntry| { let value = tx1.clone();
Box::pin({ async move {
let value = tx1.clone(); if entry.is_dir() {
async move { return;
if entry.is_dir() { }
return; if let Err(err) = value.send(entry).await {
} error!("list_path send fail {:?}", err);
}
}
})
})),
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
Box::pin({
let value = tx2.clone();
let resolver = resolver.clone();
async move {
if let Some(entry) = entries.resolve(resolver) {
if let Err(err) = value.send(entry).await { if let Err(err) = value.send(entry).await {
error!("list_path send fail {:?}", err); error!("list_path send fail {:?}", err);
} }
} }
}) }
})), })
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| { })),
Box::pin({ finished: None,
let value = tx2.clone(); ..Default::default()
let resolver = resolver.clone(); })
async move {
if let Some(entry) = entries.resolve(resolver) {
if let Err(err) = value.send(entry).await {
error!("list_path send fail {:?}", err);
}
}
}
})
})),
finished: None,
..Default::default()
},
)
.await .await
}); });
} }
@@ -1279,45 +1272,42 @@ impl SetDisks {
let tx1 = sender.clone(); let tx1 = sender.clone();
let tx2 = sender.clone(); let tx2 = sender.clone();
list_path_raw( list_path_raw(rx, ListPathRawOptions {
rx, disks: disks.iter().cloned().map(Some).collect(),
ListPathRawOptions { fallback_disks: fallback_disks.iter().cloned().map(Some).collect(),
disks: disks.iter().cloned().map(Some).collect(), bucket: opts.bucket,
fallback_disks: fallback_disks.iter().cloned().map(Some).collect(), path: opts.base_dir,
bucket: opts.bucket, recursice: opts.recursive,
path: opts.base_dir, filter_prefix: opts.filter_prefix,
recursice: opts.recursive, forward_to: opts.marker,
filter_prefix: opts.filter_prefix, min_disks: listing_quorum,
forward_to: opts.marker, per_disk_limit: limit,
min_disks: listing_quorum, agreed: Some(Box::new(move |entry: MetaCacheEntry| {
per_disk_limit: limit, Box::pin({
agreed: Some(Box::new(move |entry: MetaCacheEntry| { let value = tx1.clone();
Box::pin({ async move {
let value = tx1.clone(); if let Err(err) = value.send(entry).await {
async move { error!("list_path send fail {:?}", err);
}
}
})
})),
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
Box::pin({
let value = tx2.clone();
let resolver = resolver.clone();
async move {
if let Some(entry) = entries.resolve(resolver) {
if let Err(err) = value.send(entry).await { if let Err(err) = value.send(entry).await {
error!("list_path send fail {:?}", err); error!("list_path send fail {:?}", err);
} }
} }
}) }
})), })
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| { })),
Box::pin({ finished: None,
let value = tx2.clone(); ..Default::default()
let resolver = resolver.clone(); })
async move {
if let Some(entry) = entries.resolve(resolver) {
if let Err(err) = value.send(entry).await {
error!("list_path send fail {:?}", err);
}
}
}
})
})),
finished: None,
..Default::default()
},
)
.await .await
.map_err(Error::other) .map_err(Error::other)
} }
crates/ecstore/src/tier/tier.rs
+4 -9
View File
@@ -365,15 +365,10 @@ impl TierConfigMgr {
file: &str, file: &str,
data: Bytes, data: Bytes,
) -> std::result::Result<(), std::io::Error> { ) -> std::result::Result<(), std::io::Error> {
self.save_config_with_opts( self.save_config_with_opts(api, file, data, &ObjectOptions {
api, max_parity: true,
file, ..Default::default()
data, })
&ObjectOptions {
max_parity: true,
..Default::default()
},
)
.await .await
} }
crates/ecstore/src/tier/warm_backend_minio.rs
+7 -13
View File
@@ -101,19 +101,13 @@ impl WarmBackend for WarmBackendMinIO {
let part_size = optimal_part_size(length)?; let part_size = optimal_part_size(length)?;
let client = self.0.client.clone(); let client = self.0.client.clone();
let res = client let res = client
.put_object( .put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &PutObjectOptions {
&self.0.bucket, storage_class: self.0.storage_class.clone(),
&self.0.get_dest(object), part_size: part_size as u64,
r, disable_content_sha256: true,
length, user_metadata: meta,
&PutObjectOptions { ..Default::default()
storage_class: self.0.storage_class.clone(), })
part_size: part_size as u64,
disable_content_sha256: true,
user_metadata: meta,
..Default::default()
},
)
.await?; .await?;
//self.ToObjectError(err, object) //self.ToObjectError(err, object)
Ok(res.version_id) Ok(res.version_id)
crates/ecstore/src/tier/warm_backend_rustfs.rs
+7 -13
View File
@@ -98,19 +98,13 @@ impl WarmBackend for WarmBackendRustFS {
let part_size = optimal_part_size(length)?; let part_size = optimal_part_size(length)?;
let client = self.0.client.clone(); let client = self.0.client.clone();
let res = client let res = client
.put_object( .put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &PutObjectOptions {
&self.0.bucket, storage_class: self.0.storage_class.clone(),
&self.0.get_dest(object), part_size: part_size as u64,
r, disable_content_sha256: true,
length, user_metadata: meta,
&PutObjectOptions { ..Default::default()
storage_class: self.0.storage_class.clone(), })
part_size: part_size as u64,
disable_content_sha256: true,
user_metadata: meta,
..Default::default()
},
)
.await?; .await?;
//self.ToObjectError(err, object) //self.ToObjectError(err, object)
Ok(res.version_id) Ok(res.version_id)
crates/ecstore/src/tier/warm_backend_s3.rs
+6 -12
View File
@@ -127,18 +127,12 @@ impl WarmBackend for WarmBackendS3 {
) -> Result<String, std::io::Error> { ) -> Result<String, std::io::Error> {
let client = self.client.clone(); let client = self.client.clone();
let res = client let res = client
.put_object( .put_object(&self.bucket, &self.get_dest(object), r, length, &PutObjectOptions {
&self.bucket, send_content_md5: true,
&self.get_dest(object), storage_class: self.storage_class.clone(),
r, user_metadata: meta,
length, ..Default::default()
&PutObjectOptions { })
send_content_md5: true,
storage_class: self.storage_class.clone(),
user_metadata: meta,
..Default::default()
},
)
.await?; .await?;
Ok(res.version_id) Ok(res.version_id)
} }
crates/iam/src/manager.rs
+5 -8
View File
@@ -1575,14 +1575,11 @@ pub fn get_default_policyes() -> HashMap<String, PolicyDoc> {
default_policies default_policies
.iter() .iter()
.map(|(n, p)| { .map(|(n, p)| {
( (n.to_string(), PolicyDoc {
n.to_string(), version: 1,
PolicyDoc { policy: p.clone(),
version: 1, ..Default::default()
policy: p.clone(), })
..Default::default()
},
)
}) })
.collect() .collect()
} }
crates/lock/src/local_locker.rs
+20 -26
View File
@@ -140,20 +140,17 @@ impl Locker for LocalLocker {
} }
args.resources.iter().enumerate().for_each(|(idx, resource)| { args.resources.iter().enumerate().for_each(|(idx, resource)| {
self.lock_map.insert( self.lock_map.insert(resource.to_string(), vec![LockRequesterInfo {
resource.to_string(), name: resource.to_string(),
vec![LockRequesterInfo { writer: true,
name: resource.to_string(), source: args.source.to_string(),
writer: true, owner: args.owner.to_string(),
source: args.source.to_string(), uid: args.uid.to_string(),
owner: args.owner.to_string(), group: args.resources.len() > 1,
uid: args.uid.to_string(), quorum: args.quorum,
group: args.resources.len() > 1, idx,
quorum: args.quorum, ..Default::default()
idx, }]);
..Default::default()
}],
);
let mut uuid = args.uid.to_string(); let mut uuid = args.uid.to_string();
format_uuid(&mut uuid, &idx); format_uuid(&mut uuid, &idx);
@@ -230,18 +227,15 @@ impl Locker for LocalLocker {
} }
} }
None => { None => {
self.lock_map.insert( self.lock_map.insert(resource.to_string(), vec![LockRequesterInfo {
resource.to_string(), name: resource.to_string(),
vec![LockRequesterInfo { writer: false,
name: resource.to_string(), source: args.source.to_string(),
writer: false, owner: args.owner.to_string(),
source: args.source.to_string(), uid: args.uid.to_string(),
owner: args.owner.to_string(), quorum: args.quorum,
uid: args.uid.to_string(), ..Default::default()
quorum: args.quorum, }]);
..Default::default()
}],
);
} }
} }
let mut uuid = args.uid.to_string(); let mut uuid = args.uid.to_string();
crates/obs/examples/server.rs
+8 -8
View File
@@ -46,10 +46,10 @@ async fn run(bucket: String, object: String, user: String, service_name: String)
// Record Metrics // Record Metrics
let meter = global::meter("rustfs"); let meter = global::meter("rustfs");
let request_duration = meter.f64_histogram("s3_request_duration_seconds").build(); let request_duration = meter.f64_histogram("s3_request_duration_seconds").build();
request_duration.record( request_duration.record(start_time.elapsed().unwrap().as_secs_f64(), &[opentelemetry::KeyValue::new(
start_time.elapsed().unwrap().as_secs_f64(), "operation",
&[opentelemetry::KeyValue::new("operation", "run")], "run",
); )]);
match SystemObserver::init_process_observer(meter).await { match SystemObserver::init_process_observer(meter).await {
Ok(_) => info!("Process observer initialized successfully"), Ok(_) => info!("Process observer initialized successfully"),
@@ -84,10 +84,10 @@ async fn put_object(bucket: String, object: String, user: String) {
let meter = global::meter("rustfs"); let meter = global::meter("rustfs");
let request_duration = meter.f64_histogram("s3_request_duration_seconds").build(); let request_duration = meter.f64_histogram("s3_request_duration_seconds").build();
request_duration.record( request_duration.record(start_time.elapsed().unwrap().as_secs_f64(), &[opentelemetry::KeyValue::new(
start_time.elapsed().unwrap().as_secs_f64(), "operation",
&[opentelemetry::KeyValue::new("operation", "put_object")], "put_object",
); )]);
info!( info!(
"Starting PUT operation content: bucket = {}, object = {}, user = {},start_time = {}", "Starting PUT operation content: bucket = {}, object = {}, user = {},start_time = {}",
crates/obs/src/system/collector.rs
+12 -18
View File
@@ -115,24 +115,18 @@ impl Collector {
let transmitted = data.transmitted() as i64; let transmitted = data.transmitted() as i64;
self.metrics.network_io_per_interface.record( self.metrics.network_io_per_interface.record(
received, received,
&[ &[&self.attributes.attributes[..], &[
&self.attributes.attributes[..], KeyValue::new(INTERFACE, interface_name.to_string()),
&[ KeyValue::new(DIRECTION, "received"),
KeyValue::new(INTERFACE, interface_name.to_string()), ]]
KeyValue::new(DIRECTION, "received"),
],
]
.concat(), .concat(),
); );
self.metrics.network_io_per_interface.record( self.metrics.network_io_per_interface.record(
transmitted, transmitted,
&[ &[&self.attributes.attributes[..], &[
&self.attributes.attributes[..], KeyValue::new(INTERFACE, interface_name.to_string()),
&[ KeyValue::new(DIRECTION, "transmitted"),
KeyValue::new(INTERFACE, interface_name.to_string()), ]]
KeyValue::new(DIRECTION, "transmitted"),
],
]
.concat(), .concat(),
); );
} }
@@ -155,10 +149,10 @@ impl Collector {
}; };
self.metrics.process_status.record( self.metrics.process_status.record(
status_value, status_value,
&[ &[&self.attributes.attributes[..], &[KeyValue::new(
&self.attributes.attributes[..], STATUS,
&[KeyValue::new(STATUS, format!("{:?}", process.status()))], format!("{:?}", process.status()),
] )]]
.concat(), .concat(),
); );
crates/policy/src/policy/policy.rs
+147 -165
View File
@@ -276,12 +276,150 @@ pub mod default {
#[allow(clippy::incompatible_msrv)] #[allow(clippy::incompatible_msrv)]
pub static DEFAULT_POLICIES: LazyLock<[(&'static str, Policy); 6]> = LazyLock::new(|| { pub static DEFAULT_POLICIES: LazyLock<[(&'static str, Policy); 6]> = LazyLock::new(|| {
[ [
( ("readwrite", Policy {
"readwrite", id: "".into(),
Policy { version: DEFAULT_VERSION.into(),
id: "".into(), statements: vec![Statement {
version: DEFAULT_VERSION.into(), sid: "".into(),
statements: vec![Statement { effect: Effect::Allow,
actions: ActionSet({
let mut hash_set = HashSet::new();
hash_set.insert(Action::S3Action(S3Action::AllActions));
hash_set
}),
not_actions: ActionSet(Default::default()),
resources: ResourceSet({
let mut hash_set = HashSet::new();
hash_set.insert(Resource::S3("*".into()));
hash_set
}),
conditions: Functions::default(),
..Default::default()
}],
}),
("readonly", Policy {
id: "".into(),
version: DEFAULT_VERSION.into(),
statements: vec![Statement {
sid: "".into(),
effect: Effect::Allow,
actions: ActionSet({
let mut hash_set = HashSet::new();
hash_set.insert(Action::S3Action(S3Action::GetBucketLocationAction));
hash_set.insert(Action::S3Action(S3Action::GetObjectAction));
hash_set
}),
not_actions: ActionSet(Default::default()),
resources: ResourceSet({
let mut hash_set = HashSet::new();
hash_set.insert(Resource::S3("*".into()));
hash_set
}),
conditions: Functions::default(),
..Default::default()
}],
}),
("writeonly", Policy {
id: "".into(),
version: DEFAULT_VERSION.into(),
statements: vec![Statement {
sid: "".into(),
effect: Effect::Allow,
actions: ActionSet({
let mut hash_set = HashSet::new();
hash_set.insert(Action::S3Action(S3Action::PutObjectAction));
hash_set
}),
not_actions: ActionSet(Default::default()),
resources: ResourceSet({
let mut hash_set = HashSet::new();
hash_set.insert(Resource::S3("*".into()));
hash_set
}),
conditions: Functions::default(),
..Default::default()
}],
}),
("writeonly", Policy {
id: "".into(),
version: DEFAULT_VERSION.into(),
statements: vec![Statement {
sid: "".into(),
effect: Effect::Allow,
actions: ActionSet({
let mut hash_set = HashSet::new();
hash_set.insert(Action::S3Action(S3Action::PutObjectAction));
hash_set
}),
not_actions: ActionSet(Default::default()),
resources: ResourceSet({
let mut hash_set = HashSet::new();
hash_set.insert(Resource::S3("*".into()));
hash_set
}),
conditions: Functions::default(),
..Default::default()
}],
}),
("diagnostics", Policy {
id: "".into(),
version: DEFAULT_VERSION.into(),
statements: vec![Statement {
sid: "".into(),
effect: Effect::Allow,
actions: ActionSet({
let mut hash_set = HashSet::new();
hash_set.insert(Action::AdminAction(AdminAction::ProfilingAdminAction));
hash_set.insert(Action::AdminAction(AdminAction::TraceAdminAction));
hash_set.insert(Action::AdminAction(AdminAction::ConsoleLogAdminAction));
hash_set.insert(Action::AdminAction(AdminAction::ServerInfoAdminAction));
hash_set.insert(Action::AdminAction(AdminAction::TopLocksAdminAction));
hash_set.insert(Action::AdminAction(AdminAction::HealthInfoAdminAction));
hash_set.insert(Action::AdminAction(AdminAction::PrometheusAdminAction));
hash_set.insert(Action::AdminAction(AdminAction::BandwidthMonitorAction));
hash_set
}),
not_actions: ActionSet(Default::default()),
resources: ResourceSet({
let mut hash_set = HashSet::new();
hash_set.insert(Resource::S3("*".into()));
hash_set
}),
conditions: Functions::default(),
..Default::default()
}],
}),
("consoleAdmin", Policy {
id: "".into(),
version: DEFAULT_VERSION.into(),
statements: vec![
Statement {
sid: "".into(),
effect: Effect::Allow,
actions: ActionSet({
let mut hash_set = HashSet::new();
hash_set.insert(Action::AdminAction(AdminAction::AllAdminActions));
hash_set
}),
not_actions: ActionSet(Default::default()),
resources: ResourceSet(HashSet::new()),
conditions: Functions::default(),
..Default::default()
},
Statement {
sid: "".into(),
effect: Effect::Allow,
actions: ActionSet({
let mut hash_set = HashSet::new();
hash_set.insert(Action::KmsAction(KmsAction::AllActions));
hash_set
}),
not_actions: ActionSet(Default::default()),
resources: ResourceSet(HashSet::new()),
conditions: Functions::default(),
..Default::default()
},
Statement {
sid: "".into(), sid: "".into(),
effect: Effect::Allow, effect: Effect::Allow,
actions: ActionSet({ actions: ActionSet({
@@ -297,165 +435,9 @@ pub mod default {
}), }),
conditions: Functions::default(), conditions: Functions::default(),
..Default::default() ..Default::default()
}], },
}, ],
), }),
(
"readonly",
Policy {
id: "".into(),
version: DEFAULT_VERSION.into(),
statements: vec![Statement {
sid: "".into(),
effect: Effect::Allow,
actions: ActionSet({
let mut hash_set = HashSet::new();
hash_set.insert(Action::S3Action(S3Action::GetBucketLocationAction));
hash_set.insert(Action::S3Action(S3Action::GetObjectAction));
hash_set
}),
not_actions: ActionSet(Default::default()),
resources: ResourceSet({
let mut hash_set = HashSet::new();
hash_set.insert(Resource::S3("*".into()));
hash_set
}),
conditions: Functions::default(),
..Default::default()
}],
},
),
(
"writeonly",
Policy {
id: "".into(),
version: DEFAULT_VERSION.into(),
statements: vec![Statement {
sid: "".into(),
effect: Effect::Allow,
actions: ActionSet({
let mut hash_set = HashSet::new();
hash_set.insert(Action::S3Action(S3Action::PutObjectAction));
hash_set
}),
not_actions: ActionSet(Default::default()),
resources: ResourceSet({
let mut hash_set = HashSet::new();
hash_set.insert(Resource::S3("*".into()));
hash_set
}),
conditions: Functions::default(),
..Default::default()
}],
},
),
(
"writeonly",
Policy {
id: "".into(),
version: DEFAULT_VERSION.into(),
statements: vec![Statement {
sid: "".into(),
effect: Effect::Allow,
actions: ActionSet({
let mut hash_set = HashSet::new();
hash_set.insert(Action::S3Action(S3Action::PutObjectAction));
hash_set
}),
not_actions: ActionSet(Default::default()),
resources: ResourceSet({
let mut hash_set = HashSet::new();
hash_set.insert(Resource::S3("*".into()));
hash_set
}),
conditions: Functions::default(),
..Default::default()
}],
},
),
(
"diagnostics",
Policy {
id: "".into(),
version: DEFAULT_VERSION.into(),
statements: vec![Statement {
sid: "".into(),
effect: Effect::Allow,
actions: ActionSet({
let mut hash_set = HashSet::new();
hash_set.insert(Action::AdminAction(AdminAction::ProfilingAdminAction));
hash_set.insert(Action::AdminAction(AdminAction::TraceAdminAction));
hash_set.insert(Action::AdminAction(AdminAction::ConsoleLogAdminAction));
hash_set.insert(Action::AdminAction(AdminAction::ServerInfoAdminAction));
hash_set.insert(Action::AdminAction(AdminAction::TopLocksAdminAction));
hash_set.insert(Action::AdminAction(AdminAction::HealthInfoAdminAction));
hash_set.insert(Action::AdminAction(AdminAction::PrometheusAdminAction));
hash_set.insert(Action::AdminAction(AdminAction::BandwidthMonitorAction));
hash_set
}),
not_actions: ActionSet(Default::default()),
resources: ResourceSet({
let mut hash_set = HashSet::new();
hash_set.insert(Resource::S3("*".into()));
hash_set
}),
conditions: Functions::default(),
..Default::default()
}],
},
),
(
"consoleAdmin",
Policy {
id: "".into(),
version: DEFAULT_VERSION.into(),
statements: vec![
Statement {
sid: "".into(),
effect: Effect::Allow,
actions: ActionSet({
let mut hash_set = HashSet::new();
hash_set.insert(Action::AdminAction(AdminAction::AllAdminActions));
hash_set
}),
not_actions: ActionSet(Default::default()),
resources: ResourceSet(HashSet::new()),
conditions: Functions::default(),
..Default::default()
},
Statement {
sid: "".into(),
effect: Effect::Allow,
actions: ActionSet({
let mut hash_set = HashSet::new();
hash_set.insert(Action::KmsAction(KmsAction::AllActions));
hash_set
}),
not_actions: ActionSet(Default::default()),
resources: ResourceSet(HashSet::new()),
conditions: Functions::default(),
..Default::default()
},
Statement {
sid: "".into(),
effect: Effect::Allow,
actions: ActionSet({
let mut hash_set = HashSet::new();
hash_set.insert(Action::S3Action(S3Action::AllActions));
hash_set
}),
not_actions: ActionSet(Default::default()),
resources: ResourceSet({
let mut hash_set = HashSet::new();
hash_set.insert(Resource::S3("*".into()));
hash_set
}),
conditions: Functions::default(),
..Default::default()
},
],
},
),
] ]
}); });
} }
crates/protos/src/main.rs
+3 -7
View File
@@ -85,13 +85,9 @@ fn main() -> Result<(), AnyError> {
Err(_) => "flatc".to_string(), Err(_) => "flatc".to_string(),
}; };
compile_flatbuffers_models( compile_flatbuffers_models(&mut generated_mod_rs, &flatc_path, proto_dir.clone(), flatbuffer_out_dir.clone(), vec![
&mut generated_mod_rs, "models",
&flatc_path, ])?;
proto_dir.clone(),
flatbuffer_out_dir.clone(),
vec!["models"],
)?;
fmt(); fmt();
Ok(()) Ok(())
crates/signer/src/request_signature_streaming.rs
+1 -1
View File
@@ -17,9 +17,9 @@ use lazy_static::lazy_static;
use std::collections::HashMap; use std::collections::HashMap;
use time::{OffsetDateTime, macros::format_description}; use time::{OffsetDateTime, macros::format_description};
use s3s::Body;
use super::request_signature_v4::{SERVICE_TYPE_S3, get_scope, get_signature, get_signing_key}; use super::request_signature_v4::{SERVICE_TYPE_S3, get_scope, get_signature, get_signing_key};
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
use s3s::Body;
const STREAMING_SIGN_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; const STREAMING_SIGN_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD";
const STREAMING_SIGN_TRAILER_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"; const STREAMING_SIGN_TRAILER_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER";
crates/signer/src/request_signature_v2.rs
+1 -2
View File
@@ -19,15 +19,14 @@ use std::collections::HashMap;
use std::fmt::Write; use std::fmt::Write;
use time::{OffsetDateTime, format_description}; use time::{OffsetDateTime, format_description};
use s3s::Body;
use super::utils::get_host_addr; use super::utils::get_host_addr;
use rustfs_utils::crypto::{base64_encode, hex, hmac_sha1}; use rustfs_utils::crypto::{base64_encode, hex, hmac_sha1};
use s3s::Body;
const _SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256"; const _SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
const SIGN_V2_ALGORITHM: &str = "AWS"; const SIGN_V2_ALGORITHM: &str = "AWS";
fn encode_url2path(req: &request::Request<Body>, _virtual_host: bool) -> String { fn encode_url2path(req: &request::Request<Body>, _virtual_host: bool) -> String {
req.uri().path().to_string() req.uri().path().to_string()
} }
crates/signer/src/request_signature_v4.rs
+9 -4
View File
@@ -22,11 +22,11 @@ use std::fmt::Write;
use time::{OffsetDateTime, macros::format_description}; use time::{OffsetDateTime, macros::format_description};
use tracing::debug; use tracing::debug;
use rustfs_utils::crypto::{hex, hex_sha256, hmac_sha256};
use s3s::Body;
use super::constants::UNSIGNED_PAYLOAD; use super::constants::UNSIGNED_PAYLOAD;
use super::request_signature_streaming_unsigned_trailer::streaming_unsigned_v4; use super::request_signature_streaming_unsigned_trailer::streaming_unsigned_v4;
use super::utils::{get_host_addr, sign_v4_trim_all}; use super::utils::{get_host_addr, sign_v4_trim_all};
use rustfs_utils::crypto::{hex, hex_sha256, hmac_sha256};
use s3s::Body;
pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256"; pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
pub const SERVICE_TYPE_S3: &str = "s3"; pub const SERVICE_TYPE_S3: &str = "s3";
@@ -270,7 +270,6 @@ pub fn pre_sign_v4(
.unwrap(), .unwrap(),
); );
*req.uri_mut() = Uri::from_parts(parts).unwrap(); *req.uri_mut() = Uri::from_parts(parts).unwrap();
req req
@@ -282,10 +281,16 @@ fn _post_pre_sign_signature_v4(policy_base64: &str, t: OffsetDateTime, secret_ac
get_signature(signing_key, policy_base64) get_signature(signing_key, policy_base64)
} }
fn _sign_v4_sts(req: request::Request<Body>, access_key_id: &str, secret_access_key: &str, location: &str) -> request::Request<Body> { fn _sign_v4_sts(
req: request::Request<Body>,
access_key_id: &str,
secret_access_key: &str,
location: &str,
) -> request::Request<Body> {
sign_v4_inner(req, 0, access_key_id, secret_access_key, "", location, SERVICE_TYPE_STS, HeaderMap::new()) sign_v4_inner(req, 0, access_key_id, secret_access_key, "", location, SERVICE_TYPE_STS, HeaderMap::new())
} }
#[allow(clippy::too_many_arguments)]
fn sign_v4_inner( fn sign_v4_inner(
mut req: request::Request<Body>, mut req: request::Request<Body>,
content_len: i64, content_len: i64,
rustfs/src/admin/handlers.rs
-2
View File
@@ -1016,7 +1016,6 @@ impl Operation for RemoveRemoteTargetHandler {
))); )));
}; };
let mut need_delete = true; let mut need_delete = true;
if let Some(arnstr) = querys.get("arn") { if let Some(arnstr) = querys.get("arn") {
@@ -1047,7 +1046,6 @@ impl Operation for RemoveRemoteTargetHandler {
error!("need delete target is {}", decoded_str); error!("need delete target is {}", decoded_str);
bucket_targets::remove_bucket_target(bucket, arnstr).await; bucket_targets::remove_bucket_target(bucket, arnstr).await;
} }
} }
// List bucket targets and return as JSON to client // List bucket targets and return as JSON to client
// match bucket_targets::list_bucket_targets(bucket).await { // match bucket_targets::list_bucket_targets(bucket).await {
rustfs/src/admin/handlers/bucket_meta.rs
+4 -7
View File
@@ -453,13 +453,10 @@ impl Operation for ImportBucketMetadata {
// create bucket if not exists // create bucket if not exists
if !bucket_metadatas.contains_key(bucket_name) { if !bucket_metadatas.contains_key(bucket_name) {
if let Err(e) = store if let Err(e) = store
.make_bucket( .make_bucket(bucket_name, &MakeBucketOptions {
bucket_name, force_create: true,
&MakeBucketOptions { ..Default::default()
force_create: true, })
..Default::default()
},
)
.await .await
{ {
warn!("create bucket failed: {e}"); warn!("create bucket failed: {e}");
rustfs/src/admin/handlers/user.rs
+10 -13
View File
@@ -445,20 +445,17 @@ impl Operation for ExportIam {
let users: HashMap<String, AddOrUpdateUserReq> = users let users: HashMap<String, AddOrUpdateUserReq> = users
.into_iter() .into_iter()
.map(|(k, v)| { .map(|(k, v)| {
( (k, AddOrUpdateUserReq {
k, secret_key: v.credentials.secret_key,
AddOrUpdateUserReq { status: {
secret_key: v.credentials.secret_key, if v.credentials.status == "off" {
status: { AccountStatus::Disabled
if v.credentials.status == "off" { } else {
AccountStatus::Disabled AccountStatus::Enabled
} else { }
AccountStatus::Enabled
}
},
policy: None,
}, },
) policy: None,
})
}) })
.collect::<HashMap<String, AddOrUpdateUserReq>>(); .collect::<HashMap<String, AddOrUpdateUserReq>>();
rustfs/src/storage/ecfs.rs
+10 -16
View File
@@ -310,14 +310,11 @@ impl S3 for FS {
}; };
store store
.make_bucket( .make_bucket(&bucket, &MakeBucketOptions {
&bucket, force_create: true,
&MakeBucketOptions { lock_enabled: object_lock_enabled_for_bucket.is_some_and(|v| v),
force_create: true, ..Default::default()
lock_enabled: object_lock_enabled_for_bucket.is_some_and(|v| v), })
..Default::default()
},
)
.await .await
.map_err(ApiError::from)?; .map_err(ApiError::from)?;
@@ -498,13 +495,10 @@ impl S3 for FS {
}; };
store store
.delete_bucket( .delete_bucket(&input.bucket, &DeleteBucketOptions {
&input.bucket, force: false,
&DeleteBucketOptions { ..Default::default()
force: false, })
..Default::default()
},
)
.await .await
.map_err(ApiError::from)?; .map_err(ApiError::from)?;
@@ -1976,7 +1970,7 @@ impl S3 for FS {
.. ..
} = req.input; } = req.input;
let mut lr_retention = false; let lr_retention = false;
/*let rcfg = metadata_sys::get_object_lock_config(&bucket).await; /*let rcfg = metadata_sys::get_object_lock_config(&bucket).await;
if let Ok(rcfg) = rcfg { if let Ok(rcfg) = rcfg {
if let Some(rule) = rcfg.0.rule { if let Some(rule) = rcfg.0.rule {