gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-20 11:32:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: some expect error (#2622)

Browse Source 
Signed-off-by: likewu <likewu@126.com>
Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
likewu
2026-04-21 11:27:50 +08:00
committed by GitHub
parent 1525143a04
commit a77be8f89b
23 changed files with 613 additions and 395 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/client/api_error_response.rs
+23 -13
View File
@@ -100,7 +100,7 @@ pub fn http_resp_to_error_response(
bucket_name: &str,
object_name: &str,
) -> ErrorResponse {
let err_body = String::from_utf8(b).unwrap();
let err_body = String::from_utf8_lossy(&b).to_string();
if h.is_empty() || resp_status.is_client_error() || resp_status.is_server_error() {
return ErrorResponse {
status_code: resp_status,
@@ -178,36 +178,46 @@ pub fn http_resp_to_error_response(
};
}
}
} else {
err_resp = err_resp_.unwrap();
} else if let Ok(parsed_resp) = err_resp_ {
err_resp = parsed_resp;
}
err_resp.status_code = resp_status;
if let Some(server_name) = h.get("Server") {
err_resp.server = server_name.to_str().expect("err").to_string();
if let Ok(server_str) = server_name.to_str() {
err_resp.server = server_str.to_string();
}
}
let code = h.get("x-minio-error-code");
if code.is_some() {
err_resp.code = S3ErrorCode::Custom(code.expect("err").to_str().expect("err").into());
if let Some(code) = h.get("x-minio-error-code") {
if let Ok(code_str) = code.to_str() {
err_resp.code = S3ErrorCode::Custom(code_str.into());
}
}
let desc = h.get("x-minio-error-desc");
if desc.is_some() {
err_resp.message = desc.expect("err").to_str().expect("err").trim_matches('"').to_string();
if let Some(desc) = h.get("x-minio-error-desc") {
if let Ok(desc_str) = desc.to_str() {
err_resp.message = desc_str.trim_matches('"').to_string();
}
}
if err_resp.request_id == "" {
if let Some(x_amz_request_id) = h.get("x-amz-request-id") {
err_resp.request_id = x_amz_request_id.to_str().expect("err").to_string();
if let Ok(request_id_str) = x_amz_request_id.to_str() {
err_resp.request_id = request_id_str.to_string();
}
}
}
if err_resp.host_id == "" {
if let Some(x_amz_id_2) = h.get("x-amz-id-2") {
err_resp.host_id = x_amz_id_2.to_str().expect("err").to_string();
if let Ok(host_id_str) = x_amz_id_2.to_str() {
err_resp.host_id = host_id_str.to_string();
}
}
}
if err_resp.region == "" {
if let Some(x_amz_bucket_region) = h.get("x-amz-bucket-region") {
err_resp.region = x_amz_bucket_region.to_str().expect("err").to_string();
if let Ok(region_str) = x_amz_bucket_region.to_str() {
err_resp.region = region_str.to_string();
}
}
}
if err_resp.code == S3ErrorCode::InvalidLocationConstraint/*InvalidRegion*/ && err_resp.region != "" {
crates/ecstore/src/client/api_get_object.rs
+1 -1
View File
@@ -1,4 +1,3 @@
#![allow(clippy::map_entry)]
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,6 +11,7 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(clippy::map_entry)]
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(unused_mut)]
crates/ecstore/src/client/api_put_object.rs
+57 -37
View File
@@ -137,21 +137,21 @@ impl Default for PutObjectOptions {
impl PutObjectOptions {
fn set_match_etag(&mut self, etag: &str) {
if etag == "*" {
self.custom_header
.insert("If-Match", HeaderValue::from_str("*").expect("err"));
self.custom_header.insert("If-Match", HeaderValue::from_static("*"));
} else {
self.custom_header
.insert("If-Match", HeaderValue::from_str(&format!("\"{}\"", etag)).expect("err"));
if let Ok(etag_value) = HeaderValue::from_str(&format!("\"{}\"", etag)) {
self.custom_header.insert("If-Match", etag_value);
}
}
}
fn set_match_etag_except(&mut self, etag: &str) {
if etag == "*" {
self.custom_header
.insert("If-None-Match", HeaderValue::from_str("*").expect("err"));
self.custom_header.insert("If-None-Match", HeaderValue::from_static("*"));
} else {
self.custom_header
.insert("If-None-Match", HeaderValue::from_str(&format!("\"{etag}\"")).expect("err"));
if let Ok(etag_value) = HeaderValue::from_str(&format!("\"{etag}\"")) {
self.custom_header.insert("If-None-Match", etag_value);
}
}
}
@@ -162,59 +162,75 @@ impl PutObjectOptions {
if content_type == "" {
content_type = "application/octet-stream".to_string();
}
header.insert("Content-Type", HeaderValue::from_str(&content_type).expect("err"));
if let Ok(content_type_value) = HeaderValue::from_str(&content_type) {
header.insert("Content-Type", content_type_value);
}
if self.content_encoding != "" {
header.insert("Content-Encoding", HeaderValue::from_str(&self.content_encoding).expect("err"));
if let Ok(encoding_value) = HeaderValue::from_str(&self.content_encoding) {
header.insert("Content-Encoding", encoding_value);
}
}
if self.content_disposition != "" {
header.insert("Content-Disposition", HeaderValue::from_str(&self.content_disposition).expect("err"));
if let Ok(disposition_value) = HeaderValue::from_str(&self.content_disposition) {
header.insert("Content-Disposition", disposition_value);
}
}
if self.content_language != "" {
header.insert("Content-Language", HeaderValue::from_str(&self.content_language).expect("err"));
if let Ok(language_value) = HeaderValue::from_str(&self.content_language) {
header.insert("Content-Language", language_value);
}
}
if self.cache_control != "" {
header.insert("Cache-Control", HeaderValue::from_str(&self.cache_control).expect("err"));
if let Ok(cache_value) = HeaderValue::from_str(&self.cache_control) {
header.insert("Cache-Control", cache_value);
}
}
if self.expires.unix_timestamp() != 0 {
header.insert(
"Expires",
HeaderValue::from_str(&self.expires.format(ISO8601_DATEFORMAT).unwrap()).expect("err"),
); //rustfs invalid header
if let Ok(expires_str) = self.expires.format(ISO8601_DATEFORMAT) {
if let Ok(expires_value) = HeaderValue::from_str(&expires_str) {
header.insert("Expires", expires_value);
}
}
}
if self.mode.as_str() != "" {
header.insert(X_AMZ_OBJECT_LOCK_MODE, HeaderValue::from_str(self.mode.as_str()).expect("err"));
if let Ok(mode_value) = HeaderValue::from_str(self.mode.as_str()) {
header.insert(X_AMZ_OBJECT_LOCK_MODE, mode_value);
}
}
if self.retain_until_date.unix_timestamp() != 0 {
header.insert(
X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE,
HeaderValue::from_str(&self.retain_until_date.format(ISO8601_DATEFORMAT).unwrap()).expect("err"),
);
if let Ok(retain_str) = self.retain_until_date.format(ISO8601_DATEFORMAT) {
if let Ok(retain_value) = HeaderValue::from_str(&retain_str) {
header.insert(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, retain_value);
}
}
}
if self.legalhold.as_str() != "" {
header.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD, HeaderValue::from_str(self.legalhold.as_str()).expect("err"));
if let Ok(legalhold_value) = HeaderValue::from_str(self.legalhold.as_str()) {
header.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD, legalhold_value);
}
}
if self.storage_class != "" {
header.insert(X_AMZ_STORAGE_CLASS, HeaderValue::from_str(&self.storage_class).expect("err"));
if let Ok(storage_class_value) = HeaderValue::from_str(&self.storage_class) {
header.insert(X_AMZ_STORAGE_CLASS, storage_class_value);
}
}
if self.website_redirect_location != "" {
header.insert(
X_AMZ_WEBSITE_REDIRECT_LOCATION,
HeaderValue::from_str(&self.website_redirect_location).expect("err"),
);
if let Ok(redirect_value) = HeaderValue::from_str(&self.website_redirect_location) {
header.insert(X_AMZ_WEBSITE_REDIRECT_LOCATION, redirect_value);
}
}
if !self.internal.replication_status.as_str().is_empty() {
header.insert(
X_AMZ_REPLICATION_STATUS,
HeaderValue::from_str(self.internal.replication_status.as_str()).expect("err"),
);
if let Ok(replication_status_value) = HeaderValue::from_str(self.internal.replication_status.as_str()) {
header.insert(X_AMZ_REPLICATION_STATUS, replication_status_value);
}
}
for (k, v) in &self.user_metadata {
@@ -360,17 +376,21 @@ impl TransitionClient {
let mut md5_base64: String = "".to_string();
if opts.send_content_md5 {
let mut md5_hasher = self.md5_hasher.lock().unwrap();
let hash = md5_hasher.as_mut().expect("err");
let hash = hash.hash_encode(&buf[..length]);
md5_base64 = base64_encode(hash.as_ref());
if let Some(mut md5_hasher) = self.md5_hasher.lock().unwrap().as_mut() {
let hash = md5_hasher.hash_encode(&buf[..length]);
md5_base64 = base64_encode(hash.as_ref());
}
} else {
let mut crc = opts.auto_checksum.hasher()?;
crc.update(&buf[..length]);
let csum = crc.finalize();
if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err"));
if let Ok(header_value) = base64_encode(csum.as_ref()).parse() {
custom_header.insert(header_name, header_value);
} else {
warn!("Failed to parse checksum value");
}
} else {
warn!("Invalid header name: {}", opts.auto_checksum.key());
}
crates/ecstore/src/client/api_put_object_multipart.rs
+12 -8
View File
@@ -127,7 +127,11 @@ impl TransitionClient {
let csum = crc.finalize();
if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err"));
if let Ok(header_value) = base64_encode(csum.as_ref()).parse() {
custom_header.insert(header_name, header_value);
} else {
warn!("Failed to parse checksum value");
}
} else {
warn!("Invalid header name: {}", opts.auto_checksum.key());
}
@@ -309,27 +313,27 @@ impl TransitionClient {
let h = resp.headers();
let mut obj_part = ObjectPart {
checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) {
h_checksum_crc32.to_str().expect("err").to_string()
h_checksum_crc32.to_str().unwrap_or("").to_string()
} else {
"".to_string()
},
checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) {
h_checksum_crc32c.to_str().expect("err").to_string()
h_checksum_crc32c.to_str().unwrap_or("").to_string()
} else {
"".to_string()
},
checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) {
h_checksum_sha1.to_str().expect("err").to_string()
h_checksum_sha1.to_str().unwrap_or("").to_string()
} else {
"".to_string()
},
checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) {
h_checksum_sha256.to_str().expect("err").to_string()
h_checksum_sha256.to_str().unwrap_or("").to_string()
} else {
"".to_string()
},
checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) {
h_checksum_crc64nvme.to_str().expect("err").to_string()
h_checksum_crc64nvme.to_str().unwrap_or("").to_string()
} else {
"".to_string()
},
@@ -338,7 +342,7 @@ impl TransitionClient {
obj_part.size = p.size;
obj_part.part_num = p.part_number;
obj_part.etag = if let Some(h_etag) = h.get("ETag") {
h_etag.to_str().expect("err").trim_matches('"').to_string()
h_etag.to_str().unwrap_or("").trim_matches('"').to_string()
} else {
"".to_string()
};
@@ -398,7 +402,7 @@ impl TransitionClient {
key: complete_multipart_upload_result.key,
etag: trim_etag(&complete_multipart_upload_result.etag),
version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) {
h_x_amz_version_id.to_str().expect("err").to_string()
h_x_amz_version_id.to_str().unwrap_or("").to_string()
} else {
"".to_string()
},
crates/ecstore/src/client/api_put_object_streaming.rs
+51 -19
View File
@@ -21,6 +21,7 @@
use bytes::Bytes;
use futures::future::join_all;
use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
use std::io::Error;
use std::sync::RwLock;
use std::{collections::HashMap, sync::Arc};
use time::{OffsetDateTime, format_description};
@@ -152,7 +153,10 @@ impl TransitionClient {
if opts.send_content_md5 {
let mut md5_hasher = self.md5_hasher.lock().unwrap();
let md5_hash = md5_hasher.as_mut().expect("err");
let md5_hash = match md5_hasher.as_mut() {
Some(hasher) => hasher,
None => return Err(std::io::Error::other("MD5 hasher not initialized")),
};
let hash = md5_hash.hash_encode(&buf[..length]);
md5_base64 = base64_encode(hash.as_ref());
} else {
@@ -161,7 +165,11 @@ impl TransitionClient {
let csum = crc.finalize();
if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err"));
if let Ok(header_value) = base64_encode(csum.as_ref()).parse() {
custom_header.insert(header_name, header_value);
} else {
warn!("Failed to parse checksum value");
}
} else {
warn!("Invalid header name: {}", opts.auto_checksum.key());
}
@@ -275,11 +283,14 @@ impl TransitionClient {
for part_number in 1..=total_parts_count {
let mut buf = Vec::<u8>::new();
select! {
buf = bufs_rx.recv() => {}
buf1 = bufs_rx.recv() => {
if let Some(buf1) = buf1 {
buf = buf1;
}
}
err = err_rx.recv() => {
//cancel_token.cancel();
//wg.Wait()
return Err(err.expect("err"));
return Err(err.unwrap_or_else(|| std::io::Error::other("Unknown error received from channel")));
}
else => (),
}
@@ -309,7 +320,11 @@ impl TransitionClient {
let csum = crc.finalize();
if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err"));
if let Ok(header_value) = base64_encode(csum.as_ref()).parse() {
custom_header.insert(header_name, header_value);
} else {
warn!("Failed to parse checksum value");
}
} else {
warn!("Invalid header name: {}", opts.auto_checksum.key());
}
@@ -319,12 +334,19 @@ impl TransitionClient {
let clone_parts_info = parts_info.clone();
let clone_upload_id = upload_id.clone();
let clone_self = self.clone();
let err_tx_clone = err_tx.clone();
futures.push(async move {
let mut md5_base64: String = "".to_string();
if opts.send_content_md5 {
let mut md5_hasher = clone_self.md5_hasher.lock().unwrap();
let md5_hash = md5_hasher.as_mut().expect("err");
let md5_hash = match md5_hasher.as_mut() {
Some(hasher) => hasher,
None => {
//let _ = err_tx_clone.send(std::io::Error::other("MD5 hasher not initialized")).await;
return Ok::<(), Error>(());
}
};
let hash = md5_hash.hash_encode(&buf[..length]);
md5_base64 = base64_encode(hash.as_ref());
}
@@ -344,12 +366,21 @@ impl TransitionClient {
sha256_hex: "".to_string(),
trailer: HeaderMap::new(),
};
let obj_part = clone_self.upload_part(&mut p).await.expect("err");
let obj_part = match clone_self.upload_part(&mut p).await {
Ok(part) => part,
Err(err) => {
let _ = err_tx_clone.send(std::io::Error::other(err.to_string())).await;
return Err::<(), Error>(err);
}
};
let mut clone_parts_info = clone_parts_info.write().unwrap();
clone_parts_info.entry(part_number).or_insert(obj_part);
{
let mut clone_parts_info = clone_parts_info.write().unwrap();
clone_parts_info.entry(part_number).or_insert(obj_part);
}
clone_bufs_tx.send(buf);
let _ = clone_bufs_tx.send(buf).await;
Ok::<(), Error>(())
});
total_uploaded_size += length as i64;
@@ -359,7 +390,7 @@ impl TransitionClient {
select! {
err = err_rx.recv() => {
return Err(err.expect("err"));
return Err(err.unwrap_or_else(|| std::io::Error::other("Unknown error received from channel")));
}
else => (),
}
@@ -504,9 +535,10 @@ impl TransitionClient {
Ok(UploadInfo {
bucket: bucket_name.to_string(),
key: object_name.to_string(),
etag: trim_etag(h.get("ETag").expect("err").to_str().expect("err")),
etag: trim_etag(h.get("ETag").and_then(|v| v.to_str().ok()).unwrap_or("")),
version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) {
h_x_amz_version_id.to_str().expect("err").to_string()
h_x_amz_version_id.to_str().unwrap_or("").to_string()
} else {
"".to_string()
},
@@ -514,27 +546,27 @@ impl TransitionClient {
expiration: exp_time,
expiration_rule_id: rule_id,
checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) {
h_checksum_crc32.to_str().expect("err").to_string()
h_checksum_crc32.to_str().unwrap_or("").to_string()
} else {
"".to_string()
},
checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) {
h_checksum_crc32c.to_str().expect("err").to_string()
h_checksum_crc32c.to_str().unwrap_or("").to_string()
} else {
"".to_string()
},
checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) {
h_checksum_sha1.to_str().expect("err").to_string()
h_checksum_sha1.to_str().unwrap_or("").to_string()
} else {
"".to_string()
},
checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) {
h_checksum_sha256.to_str().expect("err").to_string()
h_checksum_sha256.to_str().unwrap_or("").to_string()
} else {
"".to_string()
},
checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) {
h_checksum_crc64nvme.to_str().expect("err").to_string()
h_checksum_crc64nvme.to_str().unwrap_or("").to_string()
} else {
"".to_string()
},
crates/ecstore/src/client/api_remove.rs
+26 -27
View File
@@ -25,7 +25,7 @@ use hyper::body::Bytes;
use rustfs_utils::HashAlgorithm;
use s3s::S3ErrorCode;
use s3s::dto::ReplicationStatus;
use s3s::header::X_AMZ_BYPASS_GOVERNANCE_RETENTION;
use s3s::header::{X_AMZ_BYPASS_GOVERNANCE_RETENTION, X_AMZ_DELETE_MARKER, X_AMZ_VERSION_ID};
use serde::Deserialize;
use std::fmt::Display;
use std::{
@@ -111,8 +111,9 @@ impl TransitionClient {
.await?;
{
let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
bucket_loc_cache.delete(bucket_name);
if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
bucket_loc_cache.delete(bucket_name);
}
}
Ok(())
}
@@ -142,8 +143,9 @@ impl TransitionClient {
.await?;
{
let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
bucket_loc_cache.delete(bucket_name);
if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
bucket_loc_cache.delete(bucket_name);
}
}
Ok(())
@@ -168,7 +170,7 @@ impl TransitionClient {
let mut headers = HeaderMap::new();
if opts.governance_bypass {
headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err")); //amzBypassGovernance
headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, HeaderValue::from_static("true")); //amzBypassGovernance
}
let resp = self
@@ -197,13 +199,12 @@ impl TransitionClient {
Ok(RemoveObjectResult {
object_name: object_name.to_string(),
object_version_id: opts.version_id,
delete_marker: resp.headers().get("x-amz-delete-marker").expect("err") == "true",
delete_marker: resp.headers().get(X_AMZ_DELETE_MARKER).map_or(false, |v| v == "true"),
delete_marker_version_id: resp
.headers()
.get("x-amz-version-id")
.expect("err")
.to_str()
.expect("err")
.get(X_AMZ_VERSION_ID)
.and_then(|v| v.to_str().ok())
.unwrap_or_default()
.to_string(),
..Default::default()
})
@@ -290,15 +291,15 @@ impl TransitionClient {
bucket_name,
&object.name,
RemoveObjectOptions {
version_id: object.version_id.expect("err").to_string(),
version_id: object.version_id.map(|id| id.to_string()).unwrap_or_default(),
governance_bypass: opts.governance_bypass,
..Default::default()
},
)
.await?;
let remove_result_clone = remove_result.clone();
if !remove_result.err.is_none() {
match to_error_response(&remove_result.err.expect("err")).code {
if let Some(err) = &remove_result.err {
match to_error_response(err).code {
S3ErrorCode::InvalidArgument | S3ErrorCode::NoSuchVersion => {
continue;
}
@@ -326,7 +327,7 @@ impl TransitionClient {
let mut headers = HeaderMap::new();
if opts.governance_bypass {
headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err"));
headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, HeaderValue::from_static("true"));
}
let remove_bytes = generate_remove_multi_objects_request(&batch);
@@ -423,23 +424,20 @@ impl TransitionClient {
request_id: resp
.headers()
.get("x-amz-request-id")
.expect("err")
.to_str()
.expect("err")
.and_then(|v| v.to_str().ok())
.unwrap_or_default()
.to_string(),
host_id: resp
.headers()
.get("x-amz-id-2")
.expect("err")
.to_str()
.expect("err")
.and_then(|v| v.to_str().ok())
.unwrap_or_default()
.to_string(),
region: resp
.headers()
.get("x-amz-bucket-region")
.expect("err")
.to_str()
.expect("err")
.and_then(|v| v.to_str().ok())
.unwrap_or_default()
.to_string(),
..Default::default()
};
@@ -472,10 +470,11 @@ pub struct RemoveObjectError {
impl Display for RemoveObjectError {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
if self.err.is_none() {
return write!(f, "unexpected remove object error result");
if let Some(err) = &self.err {
write!(f, "{}", err.to_string())
} else {
write!(f, "unexpected remove object error result")
}
write!(f, "{}", self.err.as_ref().expect("err").to_string())
}
}
crates/ecstore/src/client/bucket_cache.rs
+34 -22
View File
@@ -70,10 +70,10 @@ impl TransitionClient {
let mut location;
{
let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
let ret = bucket_loc_cache.get(bucket_name);
if let Some(location) = ret {
return Ok(location);
if let Ok(bucket_loc_cache) = self.bucket_loc_cache.lock() {
if let Some(location) = bucket_loc_cache.get(bucket_name) {
return Ok(location);
}
}
//location = ret?;
}
@@ -83,8 +83,9 @@ impl TransitionClient {
let mut resp = self.doit(req).await?;
location = process_bucket_location_response(resp, bucket_name, &self.tier_type).await?;
{
let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
bucket_loc_cache.set(bucket_name, &location);
if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
bucket_loc_cache.set(bucket_name, &location);
}
}
Ok(location)
}
@@ -108,7 +109,11 @@ impl TransitionClient {
url_str.push_str("://");
url_str.push_str(bucket_name);
url_str.push_str(".");
url_str.push_str(target_url.host_str().expect("err"));
url_str.push_str(
target_url
.host_str()
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "host is none"))?,
);
url_str.push_str("/?location");
} else {
let mut path = bucket_name.to_string();
@@ -135,13 +140,16 @@ impl TransitionClient {
let value;
{
let mut creds_provider = self.creds_provider.lock().unwrap();
value = match creds_provider.get_with_context(Some(self.cred_context())) {
Ok(v) => v,
Err(err) => {
return Err(std::io::Error::other(err));
}
};
if let Ok(mut creds_provider) = self.creds_provider.lock() {
value = match creds_provider.get_with_context(Some(self.cred_context())) {
Ok(v) => v,
Err(err) => {
return Err(std::io::Error::other(err));
}
};
} else {
return Err(std::io::Error::other("Failed to acquire credentials provider lock"));
}
}
let mut signer_type = value.signer_type.clone();
@@ -171,8 +179,9 @@ impl TransitionClient {
content_sha256 = UNSIGNED_PAYLOAD.to_string();
}
req.headers_mut()
.insert("X-Amz-Content-Sha256", content_sha256.parse().unwrap());
if let Ok(content_sha256_value) = content_sha256.parse() {
req.headers_mut().insert("X-Amz-Content-Sha256", content_sha256_value);
}
let req = rustfs_signer::sign_v4(req, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1");
Ok(req)
}
@@ -228,13 +237,16 @@ async fn process_bucket_location_response(
}
let mut location = "".to_string();
if tier_type == "huaweicloud" {
let d = quick_xml::de::from_str::<CreateBucketConfiguration>(&String::from_utf8(body_vec).unwrap()).unwrap();
location = d.location_constraint;
if let Ok(body_str) = String::from_utf8(body_vec) {
if let Ok(d) = quick_xml::de::from_str::<CreateBucketConfiguration>(&body_str) {
location = d.location_constraint;
}
}
} else {
if let Ok(LocationConstraint { field }) =
quick_xml::de::from_str::<LocationConstraint>(&String::from_utf8(body_vec).unwrap())
{
location = field;
if let Ok(body_str) = String::from_utf8(body_vec) {
if let Ok(LocationConstraint { field }) = quick_xml::de::from_str::<LocationConstraint>(&body_str) {
location = field;
}
}
}
//debug!("location: {}", location);
crates/ecstore/src/client/credentials.rs
+4 -2
View File
@@ -1,4 +1,3 @@
#![allow(unused_imports)]
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,6 +11,7 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
@@ -57,7 +57,9 @@ impl<P: Provider + Default> Credentials<P> {
pub fn get_with_context(&mut self, mut cc: Option<CredContext>) -> Result<Value, std::io::Error> {
if self.is_expired() {
let creds = self.provider.retrieve_with_cred_context(cc.expect("err"));
let creds = self.provider.retrieve_with_cred_context(cc.unwrap_or(CredContext {
endpoint: "".to_string(),
}));
self.creds = creds;
self.force_refresh = false;
}
crates/ecstore/src/client/transition_api.rs
+64 -36
View File
@@ -251,9 +251,10 @@ impl TransitionClient {
};
{
let mut md5_hasher = client.md5_hasher.lock().unwrap();
if md5_hasher.is_none() {
*md5_hasher = Some(HashAlgorithm::Md5);
if let Ok(mut md5_hasher) = client.md5_hasher.lock() {
if md5_hasher.is_none() {
*md5_hasher = Some(HashAlgorithm::Md5);
}
}
}
if client.sha256_hasher.is_none() {
@@ -275,25 +276,30 @@ impl TransitionClient {
}
fn trace_errors_only_off(&self) {
let mut trace_errors_only = self.trace_errors_only.lock().unwrap();
*trace_errors_only = false;
if let Ok(mut trace_errors_only) = self.trace_errors_only.lock() {
*trace_errors_only = false;
}
}
fn trace_off(&self) {
let mut is_trace_enabled = self.is_trace_enabled.lock().unwrap();
*is_trace_enabled = false;
let mut trace_errors_only = self.trace_errors_only.lock().unwrap();
*trace_errors_only = false;
if let Ok(mut is_trace_enabled) = self.is_trace_enabled.lock() {
*is_trace_enabled = false;
}
if let Ok(mut trace_errors_only) = self.trace_errors_only.lock() {
*trace_errors_only = false;
}
}
fn set_s3_transfer_accelerate(&self, accelerate_endpoint: &str) {
let mut endpoint = self.s3_accelerate_endpoint.lock().unwrap();
*endpoint = accelerate_endpoint.to_string();
if let Ok(mut endpoint) = self.s3_accelerate_endpoint.lock() {
*endpoint = accelerate_endpoint.to_string();
}
}
fn set_s3_enable_dual_stack(&self, enabled: bool) {
let mut dual_stack = self.s3_dual_stack_enabled.lock().unwrap();
*dual_stack = enabled;
if let Ok(mut dual_stack) = self.s3_dual_stack_enabled.lock() {
*dual_stack = enabled;
}
}
pub fn hash_materials(
@@ -352,7 +358,6 @@ impl TransitionClient {
let resp;
let http_client = self.http_client.clone();
{
//let mut http_client = http_client.lock().unwrap();
req_method = req.method().clone();
req_uri = req.uri().clone();
req_headers = req.headers().clone();
@@ -368,7 +373,10 @@ impl TransitionClient {
return Err(std::io::Error::other(err));
}
let resp = resp.unwrap();
let resp = match resp {
Ok(r) => r,
Err(_) => return Err(std::io::Error::other("Unexpected error in response")),
};
debug!("http_resp: {:?}", resp);
//let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
@@ -455,11 +463,13 @@ impl TransitionClient {
return Err(std::io::Error::other(err_response));
}
if metadata.bucket_name != "" {
let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
let location = bucket_loc_cache.get(&metadata.bucket_name);
if location.is_some() && location.unwrap() != err_response.region {
bucket_loc_cache.set(&metadata.bucket_name, &err_response.region);
//continue;
if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
if let Some(location) = bucket_loc_cache.get(&metadata.bucket_name) {
if location != err_response.region {
bucket_loc_cache.set(&metadata.bucket_name, &err_response.region);
//continue;
}
}
}
} else if err_response.region != metadata.bucket_location {
metadata.bucket_location = err_response.region.clone();
@@ -518,8 +528,11 @@ impl TransitionClient {
let value;
{
let mut creds_provider = self.creds_provider.lock().unwrap();
value = creds_provider.get_with_context(Some(self.cred_context()))?;
if let Ok(mut creds_provider) = self.creds_provider.lock() {
value = creds_provider.get_with_context(Some(self.cred_context()))?;
} else {
return Err(std::io::Error::other("Failed to acquire credentials provider lock"));
}
}
let mut signer_type = value.signer_type.clone();
@@ -548,8 +561,10 @@ impl TransitionClient {
)));
}
let headers = req.headers_mut();
for (k, v) in metadata.extra_pre_sign_header.as_ref().unwrap() {
headers.insert(k, v.clone());
if let Some(extra_headers) = metadata.extra_pre_sign_header.as_ref() {
for (k, v) in extra_headers {
headers.insert(k, v.clone());
}
}
}
if signer_type == SignatureType::SignatureV2 {
@@ -571,18 +586,22 @@ impl TransitionClient {
self.set_user_agent(&mut req);
for (k, v) in metadata.custom_header.clone() {
req.headers_mut().insert(k.expect("err"), v);
if let Some(key) = k {
req.headers_mut().insert(key, v);
}
}
//req.content_length = metadata.content_length;
if metadata.content_length <= -1 {
let chunked_value = HeaderValue::from_str(&vec!["chunked"].join(",")).expect("err");
req.headers_mut().insert(http::header::TRANSFER_ENCODING, chunked_value);
if let Ok(chunked_value) = HeaderValue::from_str(&vec!["chunked"].join(",")) {
req.headers_mut().insert(http::header::TRANSFER_ENCODING, chunked_value);
}
}
if metadata.content_md5_base64.len() > 0 {
let md5_value = HeaderValue::from_str(&metadata.content_md5_base64).expect("err");
req.headers_mut().insert("Content-Md5", md5_value);
if let Ok(md5_value) = HeaderValue::from_str(&metadata.content_md5_base64) {
req.headers_mut().insert("Content-Md5", md5_value);
}
}
if signer_type == SignatureType::SignatureAnonymous {
@@ -607,8 +626,13 @@ impl TransitionClient {
} else if metadata.trailer.len() > 0 {
sha_header = UNSIGNED_PAYLOAD_TRAILER.to_string();
}
req.headers_mut()
.insert("X-Amz-Content-Sha256".parse::<HeaderName>().unwrap(), sha_header.parse().expect("err"));
let header_name = "X-Amz-Content-Sha256"
.parse::<HeaderName>()
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?;
let header_value = sha_header
.parse()
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?;
req.headers_mut().insert(header_name, header_value);
req = rustfs_signer::sign_v4_trailer(
req,
@@ -636,7 +660,7 @@ impl TransitionClient {
pub fn set_user_agent(&self, req: &mut Request<s3s::Body>) {
let headers = req.headers_mut();
headers.insert("User-Agent", C_USER_AGENT.parse().expect("err"));
headers.insert("User-Agent", HeaderValue::from_static(C_USER_AGENT));
}
fn make_target_url(
@@ -648,7 +672,10 @@ impl TransitionClient {
query_values: &HashMap<String, String>,
) -> Result<Url, std::io::Error> {
let scheme = self.endpoint_url.scheme();
let host = self.endpoint_url.host().unwrap();
let host = self
.endpoint_url
.host()
.ok_or_else(|| std::io::Error::other("Endpoint URL has no host"))?;
let default_port = if scheme == "https" { 443 } else { 80 };
let port = self.endpoint_url.port().unwrap_or(default_port);
@@ -1155,9 +1182,10 @@ pub fn to_object_info(bucket_name: &str, object_name: &str, h: &HeaderMap) -> Re
for (name, value) in h.iter() {
let header_name = name.as_str().to_lowercase();
if header_name.starts_with("x-amz-meta-") {
let key = header_name.strip_prefix("x-amz-meta-").unwrap().to_string();
if let Ok(value_str) = value.to_str() {
meta.insert(key, value_str.to_string());
if let Some(key) = header_name.strip_prefix("x-amz-meta-") {
if let Ok(value_str) = value.to_str() {
meta.insert(key.to_string(), value_str.to_string());
}
}
}
}