gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-12 16:16:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix clippy

Browse Source
This commit is contained in:
likewu
2025-06-23 12:03:46 +08:00
parent cc71f40a6d
commit 2c176dd864
70 changed files with 2779 additions and 1969 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/filemeta/src/fileinfo.rs
+12 -7
View File
@@ -18,8 +18,8 @@ pub const BLOCK_SIZE_V2: usize = 1024 * 1024; // 1M
pub const NULL_VERSION_ID: &str = "null"; pub const NULL_VERSION_ID: &str = "null";
// pub const RUSTFS_ERASURE_UPGRADED: &str = "x-rustfs-internal-erasure-upgraded"; // pub const RUSTFS_ERASURE_UPGRADED: &str = "x-rustfs-internal-erasure-upgraded";
pub const TIER_FV_ID: &str = "tier-free-versionID"; pub const TIER_FV_ID: &str = "tier-free-versionID";
pub const TIER_FV_MARKER: &str = "tier-free-marker"; pub const TIER_FV_MARKER: &str = "tier-free-marker";
pub const TIER_SKIP_FV_ID: &str = "tier-skip-fvid"; pub const TIER_SKIP_FV_ID: &str = "tier-skip-fvid";
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)] #[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)]
@@ -309,7 +309,8 @@ impl FileInfo {
} }
pub fn set_tier_free_version_id(&mut self, version_id: &str) { pub fn set_tier_free_version_id(&mut self, version_id: &str) {
self.metadata.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_ID), version_id.to_string()); self.metadata
.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_ID), version_id.to_string());
} }
pub fn tier_free_version_id(&self) -> String { pub fn tier_free_version_id(&self) -> String {
@@ -317,19 +318,23 @@ impl FileInfo {
} }
pub fn set_tier_free_version(&mut self) { pub fn set_tier_free_version(&mut self) {
self.metadata.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_MARKER), "".to_string()); self.metadata
.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_MARKER), "".to_string());
} }
pub fn set_skip_tier_free_version(&mut self) { pub fn set_skip_tier_free_version(&mut self) {
self.metadata.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_SKIP_FV_ID), "".to_string()); self.metadata
.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_SKIP_FV_ID), "".to_string());
} }
pub fn skip_tier_free_version(&self) -> bool { pub fn skip_tier_free_version(&self) -> bool {
self.metadata.contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_SKIP_FV_ID)) self.metadata
.contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_SKIP_FV_ID))
} }
pub fn tier_free_version(&self) -> bool { pub fn tier_free_version(&self) -> bool {
self.metadata.contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_MARKER)) self.metadata
.contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_MARKER))
} }
pub fn set_inline_data(&mut self) { pub fn set_inline_data(&mut self) {
crates/filemeta/src/filemeta.rs
+52 -17
View File
@@ -47,7 +47,7 @@ pub const TRANSITIONED_OBJECTNAME: &str = "transitioned-object";
pub const TRANSITIONED_VERSION_ID: &str = "transitioned-versionID"; pub const TRANSITIONED_VERSION_ID: &str = "transitioned-versionID";
pub const TRANSITION_TIER: &str = "transition-tier"; pub const TRANSITION_TIER: &str = "transition-tier";
const X_AMZ_RESTORE_EXPIRY_DAYS: &str = "X-Amz-Restore-Expiry-Days"; const X_AMZ_RESTORE_EXPIRY_DAYS: &str = "X-Amz-Restore-Expiry-Days";
const X_AMZ_RESTORE_REQUEST_DATE: &str = "X-Amz-Restore-Request-Date"; const X_AMZ_RESTORE_REQUEST_DATE: &str = "X-Amz-Restore-Request-Date";
// type ScanHeaderVersionFn = Box<dyn Fn(usize, &[u8], &[u8]) -> Result<()>>; // type ScanHeaderVersionFn = Box<dyn Fn(usize, &[u8], &[u8]) -> Result<()>>;
@@ -487,8 +487,10 @@ impl FileMeta {
} }
let encoded = ver.marshal_msg()?; let encoded = ver.marshal_msg()?;
if self.versions.len()+1 > 100 { if self.versions.len() + 1 > 100 {
return Err(Error::other("You've exceeded the limit on the number of versions you can create on this object")); return Err(Error::other(
"You've exceeded the limit on the number of versions you can create on this object",
));
} }
self.versions.push(FileMetaShallowVersion { self.versions.push(FileMetaShallowVersion {
@@ -502,11 +504,11 @@ impl FileMeta {
let len = self.versions.len(); let len = self.versions.len();
for (i, existing) in self.versions.iter().enumerate() { for (i, existing) in self.versions.iter().enumerate() {
if existing.header.mod_time.unwrap().nanosecond() <= mod_time { if existing.header.mod_time.unwrap().nanosecond() <= mod_time {
let vers = self.versions[i..len-1].to_vec(); let vers = self.versions[i..len - 1].to_vec();
self.versions[i+1..].clone_from_slice(vers.as_slice()); self.versions[i + 1..].clone_from_slice(vers.as_slice());
self.versions[i] = FileMetaShallowVersion { self.versions[i] = FileMetaShallowVersion {
header: ver.header(), header: ver.header(),
meta: encoded, meta: encoded,
}; };
return Ok(()); return Ok(());
} }
@@ -564,7 +566,7 @@ impl FileMeta {
ver.object.as_mut().unwrap().reset_inline_data(); ver.object.as_mut().unwrap().reset_inline_data();
self.set_idx(i, ver.clone())?; self.set_idx(i, ver.clone())?;
} else { } else {
let vers = self.versions[i+1..].to_vec(); let vers = self.versions[i + 1..].to_vec();
self.versions.extend(vers.iter().cloned()); self.versions.extend(vers.iter().cloned());
let (free_version, to_free) = ver.object.as_ref().unwrap().init_free_version(fi); let (free_version, to_free) = ver.object.as_ref().unwrap().init_free_version(fi);
if to_free { if to_free {
@@ -1927,10 +1929,22 @@ impl MetaObject {
} }
pub fn set_transition(&mut self, fi: &FileInfo) { pub fn set_transition(&mut self, fi: &FileInfo) {
self.meta_sys.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_STATUS), fi.transition_status.as_bytes().to_vec()); self.meta_sys.insert(
self.meta_sys.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_OBJECTNAME), fi.transitioned_objname.as_bytes().to_vec()); format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_STATUS),
self.meta_sys.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_VERSION_ID), fi.transition_version_id.unwrap().as_bytes().to_vec()); fi.transition_status.as_bytes().to_vec(),
self.meta_sys.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_TIER), fi.transition_tier.as_bytes().to_vec()); );
self.meta_sys.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_OBJECTNAME),
fi.transitioned_objname.as_bytes().to_vec(),
);
self.meta_sys.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_VERSION_ID),
fi.transition_version_id.unwrap().as_bytes().to_vec(),
);
self.meta_sys.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_TIER),
fi.transition_tier.as_bytes().to_vec(),
);
} }
pub fn remove_restore_hdrs(&mut self) { pub fn remove_restore_hdrs(&mut self) {
@@ -1977,11 +1991,18 @@ impl MetaObject {
if fi.skip_tier_free_version() { if fi.skip_tier_free_version() {
return (FileMetaVersion::default(), false); return (FileMetaVersion::default(), false);
} }
if let Some(status) = self.meta_sys.get(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_STATUS)) { if let Some(status) = self
.meta_sys
.get(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_STATUS))
{
if *status == TRANSITION_COMPLETE.as_bytes().to_vec() { if *status == TRANSITION_COMPLETE.as_bytes().to_vec() {
let vid = Uuid::parse_str(&fi.tier_free_version_id()); let vid = Uuid::parse_str(&fi.tier_free_version_id());
if let Err(err) = vid { if let Err(err) = vid {
panic!("Invalid Tier Object delete marker versionId {} {}", fi.tier_free_version_id(), err.to_string()); panic!(
"Invalid Tier Object delete marker versionId {} {}",
fi.tier_free_version_id(),
err.to_string()
);
} }
let vid = vid.unwrap(); let vid = vid.unwrap();
let mut free_entry = FileMetaVersion { let mut free_entry = FileMetaVersion {
@@ -1991,11 +2012,18 @@ impl MetaObject {
}; };
free_entry.delete_marker = Some(MetaDeleteMarker { free_entry.delete_marker = Some(MetaDeleteMarker {
version_id: Some(vid), version_id: Some(vid),
mod_time: self.mod_time, mod_time: self.mod_time,
meta_sys: Some(HashMap::<String, Vec<u8>>::new()), meta_sys: Some(HashMap::<String, Vec<u8>>::new()),
}); });
free_entry.delete_marker.as_mut().unwrap().meta_sys.as_mut().unwrap().insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, FREE_VERSION), vec![]); free_entry
.delete_marker
.as_mut()
.unwrap()
.meta_sys
.as_mut()
.unwrap()
.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, FREE_VERSION), vec![]);
let tier_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_TIER); let tier_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_TIER);
let tier_obj_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_OBJECTNAME); let tier_obj_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_OBJECTNAME);
let tier_obj_vid_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_VERSION_ID); let tier_obj_vid_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_VERSION_ID);
@@ -2003,7 +2031,14 @@ impl MetaObject {
let aa = [tier_key, tier_obj_key, tier_obj_vid_key]; let aa = [tier_key, tier_obj_key, tier_obj_vid_key];
for (k, v) in &self.meta_sys { for (k, v) in &self.meta_sys {
if aa.contains(&k) { if aa.contains(&k) {
free_entry.delete_marker.as_mut().unwrap().meta_sys.as_mut().unwrap().insert(k.clone(), v.clone()); free_entry
.delete_marker
.as_mut()
.unwrap()
.meta_sys
.as_mut()
.unwrap()
.insert(k.clone(), v.clone());
} }
} }
return (free_entry, true); return (free_entry, true);
crates/utils/src/net.rs
+31 -27
View File
@@ -122,7 +122,9 @@ pub fn get_endpoint_url(endpoint: &str, secure: bool) -> Result<Url, std::io::Er
} }
let endpoint_url_str = format!("{scheme}://{endpoint}"); let endpoint_url_str = format!("{scheme}://{endpoint}");
let Ok(endpoint_url) = Url::parse(&endpoint_url_str) else { return Err(std::io::Error::other("url parse error.")); }; let Ok(endpoint_url) = Url::parse(&endpoint_url_str) else {
return Err(std::io::Error::other("url parse error."));
};
//is_valid_endpoint_url(endpoint_url)?; //is_valid_endpoint_url(endpoint_url)?;
Ok(endpoint_url) Ok(endpoint_url)
@@ -137,42 +139,40 @@ pub fn new_remotetarget_http_transport(insecure: bool) -> Builder<TokioExecutor>
lazy_static! { lazy_static! {
static ref SUPPORTED_QUERY_VALUES: HashMap<String, bool> = { static ref SUPPORTED_QUERY_VALUES: HashMap<String, bool> = {
let mut m = HashMap::new(); let mut m = HashMap::new();
m.insert("attributes".to_string(), true); m.insert("attributes".to_string(), true);
m.insert("partNumber".to_string(), true); m.insert("partNumber".to_string(), true);
m.insert("versionId".to_string(), true); m.insert("versionId".to_string(), true);
m.insert("response-cache-control".to_string(), true); m.insert("response-cache-control".to_string(), true);
m.insert("response-content-disposition".to_string(), true); m.insert("response-content-disposition".to_string(), true);
m.insert("response-content-encoding".to_string(), true); m.insert("response-content-encoding".to_string(), true);
m.insert("response-content-language".to_string(), true); m.insert("response-content-language".to_string(), true);
m.insert("response-content-type".to_string(), true); m.insert("response-content-type".to_string(), true);
m.insert("response-expires".to_string(), true); m.insert("response-expires".to_string(), true);
m m
}; };
static ref SUPPORTED_HEADERS: HashMap<String, bool> = { static ref SUPPORTED_HEADERS: HashMap<String, bool> = {
let mut m = HashMap::new(); let mut m = HashMap::new();
m.insert("content-type".to_string(), true); m.insert("content-type".to_string(), true);
m.insert("cache-control".to_string(), true); m.insert("cache-control".to_string(), true);
m.insert("content-encoding".to_string(), true); m.insert("content-encoding".to_string(), true);
m.insert("content-disposition".to_string(), true); m.insert("content-disposition".to_string(), true);
m.insert("content-language".to_string(), true); m.insert("content-language".to_string(), true);
m.insert("x-amz-website-redirect-location".to_string(), true); m.insert("x-amz-website-redirect-location".to_string(), true);
m.insert("x-amz-object-lock-mode".to_string(), true); m.insert("x-amz-object-lock-mode".to_string(), true);
m.insert("x-amz-metadata-directive".to_string(), true); m.insert("x-amz-metadata-directive".to_string(), true);
m.insert("x-amz-object-lock-retain-until-date".to_string(), true); m.insert("x-amz-object-lock-retain-until-date".to_string(), true);
m.insert("expires".to_string(), true); m.insert("expires".to_string(), true);
m.insert("x-amz-replication-status".to_string(), true); m.insert("x-amz-replication-status".to_string(), true);
m m
}; };
static ref SSE_HEADERS: HashMap<String, bool> = { static ref SSE_HEADERS: HashMap<String, bool> = {
let mut m = HashMap::new(); let mut m = HashMap::new();
m.insert("x-amz-server-side-encryption".to_string(), true); m.insert("x-amz-server-side-encryption".to_string(), true);
m.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), true); m.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), true);
m.insert("x-amz-server-side-encryption-context".to_string(), true); m.insert("x-amz-server-side-encryption-context".to_string(), true);
m.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), true); m.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), true);
m.insert("x-amz-server-side-encryption-customer-key".to_string(), true); m.insert("x-amz-server-side-encryption-customer-key".to_string(), true);
m.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), true); m.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), true);
m m
}; };
} }
@@ -201,7 +201,11 @@ pub fn is_sse_header(header_key: &str) -> bool {
pub fn is_amz_header(header_key: &str) -> bool { pub fn is_amz_header(header_key: &str) -> bool {
let key = header_key.to_lowercase(); let key = header_key.to_lowercase();
key.starts_with("x-amz-meta-") || key.starts_with("x-amz-grant-") || key == "x-amz-acl" || is_sse_header(header_key) || key.starts_with("x-amz-checksum-") key.starts_with("x-amz-meta-")
|| key.starts_with("x-amz-grant-")
|| key == "x-amz-acl"
|| is_sse_header(header_key)
|| key.starts_with("x-amz-checksum-")
} }
pub fn is_rustfs_header(header_key: &str) -> bool { pub fn is_rustfs_header(header_key: &str) -> bool {
crates/utils/src/retry.rs
+1 -5
View File
@@ -1,8 +1,4 @@
//use tokio_stream::Stream; use std::time::Duration;
use std::future::Future;
use std::time::{Duration, Instant};
use std::pin::Pin;
use std::task::{Context, Poll};
// MaxRetry is the maximum number of retries before stopping. // MaxRetry is the maximum number of retries before stopping.
pub const MAX_RETRY: i64 = 10; pub const MAX_RETRY: i64 = 10;
ecstore/src/bucket/lifecycle/bucket_lifecycle_audit.rs
+1 -4
View File
@@ -24,9 +24,6 @@ pub struct LcAuditEvent {
impl LcAuditEvent { impl LcAuditEvent {
pub fn new(event: lifecycle::Event, source: LcEventSrc) -> Self { pub fn new(event: lifecycle::Event, source: LcEventSrc) -> Self {
Self { Self { event, source }
event,
source,
}
} }
} }
ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+112 -84
View File
@@ -1,53 +1,49 @@
use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
use futures::Future;
use http::HeaderMap;
use lazy_static::lazy_static;
use s3s::Body;
use sha2::{Digest, Sha256};
use std::any::{Any, TypeId}; use std::any::{Any, TypeId};
use std::collections::HashMap;
use std::env; use std::env;
use std::io::{Cursor, Write}; use std::io::{Cursor, Write};
use std::pin::Pin; use std::pin::Pin;
use std::sync::atomic::{AtomicI32, AtomicI64, Ordering}; use std::sync::atomic::{AtomicI32, AtomicI64, Ordering};
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use futures::Future;
use lazy_static::lazy_static;
use s3s::Body;
use std::collections::HashMap;
use tracing::{error, info, warn};
use sha2::{Digest, Sha256};
use xxhash_rust::xxh64;
use uuid::Uuid;
use http::HeaderMap;
use tokio::select; use tokio::select;
use tokio::sync::mpsc::{Receiver, Sender}; use tokio::sync::mpsc::{Receiver, Sender};
use tokio::sync::{mpsc, RwLock}; use tokio::sync::{RwLock, mpsc};
use async_channel::{bounded, Receiver as A_Receiver, Sender as A_Sender}; use tracing::{error, info, warn};
use uuid::Uuid;
use xxhash_rust::xxh64;
use s3s::dto::BucketLifecycleConfiguration; use super::bucket_lifecycle_audit::{LcAuditEvent, LcEventSrc};
use super::lifecycle::{self, ExpirationOptions, IlmAction, Lifecycle, TransitionOptions};
use super::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats};
use super::tier_sweeper::{Jentry, delete_object_from_remote_tier};
use crate::bucket::{metadata_sys::get_lifecycle_config, versioning_sys::BucketVersioningSys};
use crate::client::object_api_utils::new_getobjectreader;
use crate::error::Error; use crate::error::Error;
use crate::event::name::EventName;
use crate::store::ECStore;
use crate::store_api::StorageAPI;
use crate::store_api::{ObjectInfo, ObjectOptions, ObjectToDelete, GetObjectReader, HTTPRangeSpec,};
use crate::error::{error_resp_to_object_err, is_err_object_not_found, is_err_version_not_found, is_network_or_host_down}; use crate::error::{error_resp_to_object_err, is_err_object_not_found, is_err_version_not_found, is_network_or_host_down};
use crate::event::name::EventName;
use crate::event_notification::{EventArgs, send_event};
use crate::global::GLOBAL_LocalNodeName;
use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id}; use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id};
use crate::client::object_api_utils::{new_getobjectreader,};
use crate::event_notification::{send_event, EventArgs};
use crate::heal::{ use crate::heal::{
data_scanner::{apply_expiry_on_non_transitioned_objects, apply_expiry_on_transitioned_object},
data_scanner_metric::ScannerMetrics, data_scanner_metric::ScannerMetrics,
data_scanner::{
apply_expiry_on_transitioned_object, apply_expiry_on_non_transitioned_objects,
},
data_usage_cache::TierStats, data_usage_cache::TierStats,
}; };
use crate::global::GLOBAL_LocalNodeName; use crate::store::ECStore;
use crate::bucket::{ use crate::store_api::StorageAPI;
metadata_sys::get_lifecycle_config, use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete};
versioning_sys::BucketVersioningSys,
};
use crate::tier::warm_backend::WarmBackendGetOpts; use crate::tier::warm_backend::WarmBackendGetOpts;
use super::lifecycle::{self, ExpirationOptions, IlmAction, Lifecycle, TransitionOptions}; use s3s::dto::BucketLifecycleConfiguration;
use super::tier_last_day_stats::{LastDayTierStats, DailyAllTierStats};
use super::tier_sweeper::{delete_object_from_remote_tier, Jentry};
use super::bucket_lifecycle_audit::{LcEventSrc, LcAuditEvent};
pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>; pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
pub type TraceFn = Arc<dyn Fn(String, HashMap<String, String>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>; pub type TraceFn =
Arc<dyn Fn(String, HashMap<String, String>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
pub type ExpiryOpType = Box<dyn ExpiryOp + Send + Sync + 'static>; pub type ExpiryOpType = Box<dyn ExpiryOp + Send + Sync + 'static>;
static XXHASH_SEED: u64 = 0; static XXHASH_SEED: u64 = 0;
@@ -74,8 +70,7 @@ impl LifecycleSys {
Some(lc) Some(lc)
} }
pub fn trace(oi: &ObjectInfo) -> TraceFn pub fn trace(oi: &ObjectInfo) -> TraceFn {
{
todo!(); todo!();
} }
} }
@@ -132,11 +127,11 @@ pub trait ExpiryOp: 'static {
#[derive(Debug, Default, Clone)] #[derive(Debug, Default, Clone)]
pub struct TransitionedObject { pub struct TransitionedObject {
pub name: String, pub name: String,
pub version_id: String, pub version_id: String,
pub tier: String, pub tier: String,
pub free_version: bool, pub free_version: bool,
pub status: String, pub status: String,
} }
struct FreeVersionTask(ObjectInfo); struct FreeVersionTask(ObjectInfo);
@@ -181,8 +176,6 @@ pub struct ExpiryState {
stats: Option<ExpiryStats>, stats: Option<ExpiryStats>,
} }
impl ExpiryState { impl ExpiryState {
#[allow(clippy::new_ret_no_self)] #[allow(clippy::new_ret_no_self)]
pub fn new() -> Arc<RwLock<Self>> { pub fn new() -> Arc<RwLock<Self>> {
@@ -203,7 +196,7 @@ impl ExpiryState {
if rxs.len() == 0 { if rxs.len() == 0 {
return 0; return 0;
} }
let mut tasks=0; let mut tasks = 0;
for rx in rxs.iter() { for rx in rxs.iter() {
tasks += rx.lock().await.len(); tasks += rx.lock().await.len();
} }
@@ -244,7 +237,11 @@ impl ExpiryState {
} }
pub async fn enqueue_by_days(&mut self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) { pub async fn enqueue_by_days(&mut self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) {
let task = ExpiryTask {obj_info: oi.clone(), event: event.clone(), src: src.clone()}; let task = ExpiryTask {
obj_info: oi.clone(),
event: event.clone(),
src: src.clone(),
};
let wrkr = self.get_worker_ch(task.op_hash()); let wrkr = self.get_worker_ch(task.op_hash());
if wrkr.is_none() { if wrkr.is_none() {
*self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1; *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
@@ -265,7 +262,11 @@ impl ExpiryState {
return; return;
} }
let task = NewerNoncurrentTask {bucket: String::from(bucket), versions: versions, event: lc_event}; let task = NewerNoncurrentTask {
bucket: String::from(bucket),
versions: versions,
event: lc_event,
};
let wrkr = self.get_worker_ch(task.op_hash()); let wrkr = self.get_worker_ch(task.op_hash());
if wrkr.is_none() { if wrkr.is_none() {
*self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1; *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
@@ -285,7 +286,7 @@ impl ExpiryState {
if self.tasks_tx.len() == 0 { if self.tasks_tx.len() == 0 {
return None; return None;
} }
Some(self.tasks_tx[h as usize %self.tasks_tx.len()].clone()) Some(self.tasks_tx[h as usize % self.tasks_tx.len()].clone())
} }
pub async fn resize_workers(n: usize, api: Arc<ECStore>) { pub async fn resize_workers(n: usize, api: Arc<ECStore>) {
@@ -311,10 +312,10 @@ impl ExpiryState {
let mut l = state.tasks_tx.len(); let mut l = state.tasks_tx.len();
while l > n { while l > n {
let worker = state.tasks_tx[l-1].clone(); let worker = state.tasks_tx[l - 1].clone();
worker.send(None).await.unwrap_or(()); worker.send(None).await.unwrap_or(());
state.tasks_tx.remove(l-1); state.tasks_tx.remove(l - 1);
state.tasks_rx.remove(l-1); state.tasks_rx.remove(l - 1);
*state.stats.as_mut().expect("err").workers.get_mut() -= 1; *state.stats.as_mut().expect("err").workers.get_mut() -= 1;
l -= 1; l -= 1;
} }
@@ -422,7 +423,11 @@ impl TransitionState {
} }
pub async fn queue_transition_task(&self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) { pub async fn queue_transition_task(&self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) {
let task = TransitionTask {obj_info: oi.clone(), src: src.clone(), event: event.clone()}; let task = TransitionTask {
obj_info: oi.clone(),
src: src.clone(),
event: event.clone(),
};
select! { select! {
//_ -> t.ctx.Done() => (), //_ -> t.ctx.Done() => (),
_ = self.transition_tx.send(Some(task)) => (), _ = self.transition_tx.send(Some(task)) => (),
@@ -438,8 +443,8 @@ impl TransitionState {
} }
pub async fn init(api: Arc<ECStore>) { pub async fn init(api: Arc<ECStore>) {
let mut n = 10;//globalAPIConfig.getTransitionWorkers(); let mut n = 10; //globalAPIConfig.getTransitionWorkers();
let tw = 10;//globalILMConfig.getTransitionWorkers(); let tw = 10; //globalILMConfig.getTransitionWorkers();
if tw > 0 { if tw > 0 {
n = tw; n = tw;
} }
@@ -512,8 +517,10 @@ impl TransitionState {
pub fn add_lastday_stats(&self, tier: &str, ts: TierStats) { pub fn add_lastday_stats(&self, tier: &str, ts: TierStats) {
let mut tier_stats = self.last_day_stats.lock().unwrap(); let mut tier_stats = self.last_day_stats.lock().unwrap();
tier_stats.entry(tier.to_string()).and_modify(|e| e.add_stats(ts)) tier_stats
.or_insert(LastDayTierStats::default()); .entry(tier.to_string())
.and_modify(|e| e.add_stats(ts))
.or_insert(LastDayTierStats::default());
} }
pub fn get_daily_all_tier_stats(&self) -> DailyAllTierStats { pub fn get_daily_all_tier_stats(&self) -> DailyAllTierStats {
@@ -574,7 +581,10 @@ impl AuditTierOp {
} }
pub fn string(&self) -> String { pub fn string(&self) -> String {
format!("tier:{},respNS:{},tx:{},err:{}", self.tier, self.time_to_responsens, self.output_bytes, self.error) format!(
"tier:{},respNS:{},tx:{},err:{}",
self.tier, self.time_to_responsens, self.output_bytes, self.error
)
} }
} }
@@ -636,16 +646,21 @@ pub async fn enqueue_transition_immediate(oi: &ObjectInfo, src: LcEventSrc) {
} }
GLOBAL_TransitionState.queue_transition_task(oi, &event, &src).await; GLOBAL_TransitionState.queue_transition_task(oi, &event, &src).await;
} }
_ => () _ => (),
} }
} }
} }
pub async fn expire_transitioned_object(api: Arc<ECStore>, oi: &ObjectInfo, lc_event: &lifecycle::Event, src: &LcEventSrc) -> Result<ObjectInfo, std::io::Error> { pub async fn expire_transitioned_object(
api: Arc<ECStore>,
oi: &ObjectInfo,
lc_event: &lifecycle::Event,
src: &LcEventSrc,
) -> Result<ObjectInfo, std::io::Error> {
//let traceFn = GLOBAL_LifecycleSys.trace(oi); //let traceFn = GLOBAL_LifecycleSys.trace(oi);
let mut opts = ObjectOptions { let mut opts = ObjectOptions {
versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await, versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await,
expiration: ExpirationOptions {expire: true}, expiration: ExpirationOptions { expire: true },
..Default::default() ..Default::default()
}; };
if lc_event.action == IlmAction::DeleteVersionAction { if lc_event.action == IlmAction::DeleteVersionAction {
@@ -663,7 +678,12 @@ pub async fn expire_transitioned_object(api: Arc<ECStore>, oi: &ObjectInfo, lc_e
} }
} }
let ret = delete_object_from_remote_tier(&oi.transitioned_object.name, &oi.transitioned_object.version_id, &oi.transitioned_object.tier).await; let ret = delete_object_from_remote_tier(
&oi.transitioned_object.name,
&oi.transitioned_object.version_id,
&oi.transitioned_object.tier,
)
.await;
if ret.is_ok() { if ret.is_ok() {
opts.skip_decommissioned = true; opts.skip_decommissioned = true;
} else { } else {
@@ -679,8 +699,8 @@ pub async fn expire_transitioned_object(api: Arc<ECStore>, oi: &ObjectInfo, lc_e
event_name = EventName::ObjectRemovedDeleteMarkerCreated; event_name = EventName::ObjectRemovedDeleteMarkerCreated;
} }
let obj_info = ObjectInfo { let obj_info = ObjectInfo {
name: oi.name.clone(), name: oi.name.clone(),
version_id: oi.version_id, version_id: oi.version_id,
delete_marker: oi.delete_marker, delete_marker: oi.delete_marker,
..Default::default() ..Default::default()
}; };
@@ -712,15 +732,15 @@ pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditE
let opts = ObjectOptions { let opts = ObjectOptions {
transition: TransitionOptions { transition: TransitionOptions {
status: lifecycle::TRANSITION_PENDING.to_string(), status: lifecycle::TRANSITION_PENDING.to_string(),
tier: lae.event.storage_class, tier: lae.event.storage_class,
etag: oi.etag.clone().expect("err").to_string(), etag: oi.etag.clone().expect("err").to_string(),
..Default::default() ..Default::default()
}, },
//lifecycle_audit_event: lae, //lifecycle_audit_event: lae,
version_id: Some(oi.version_id.expect("err").to_string()), version_id: Some(oi.version_id.expect("err").to_string()),
versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await, versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await,
version_suspended: BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await, version_suspended: BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await,
mod_time: oi.mod_time, mod_time: oi.mod_time,
..Default::default() ..Default::default()
}; };
time_ilm(1); time_ilm(1);
@@ -731,7 +751,14 @@ pub fn audit_tier_actions(api: ECStore, tier: &str, bytes: i64) -> TimeFn {
todo!(); todo!();
} }
pub async fn get_transitioned_object_reader(bucket: &str, object: &str, rs: HTTPRangeSpec, h: HeaderMap, oi: ObjectInfo, opts: &ObjectOptions) -> Result<GetObjectReader, std::io::Error> { pub async fn get_transitioned_object_reader(
bucket: &str,
object: &str,
rs: HTTPRangeSpec,
h: HeaderMap,
oi: ObjectInfo,
opts: &ObjectOptions,
) -> Result<GetObjectReader, std::io::Error> {
let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await; let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await;
let tgt_client = match tier_config_mgr.get_driver(&oi.transitioned_object.tier).await { let tgt_client = match tier_config_mgr.get_driver(&oi.transitioned_object.tier).await {
Ok(d) => d, Ok(d) => d,
@@ -752,7 +779,9 @@ pub async fn get_transitioned_object_reader(bucket: &str, object: &str, rs: HTTP
//return Ok(HttpFileReader::new(rs, &oi, opts, &h)); //return Ok(HttpFileReader::new(rs, &oi, opts, &h));
//timeTierAction := auditTierActions(oi.transitioned_object.Tier, length) //timeTierAction := auditTierActions(oi.transitioned_object.Tier, length)
let reader = tgt_client.get(&oi.transitioned_object.name, &oi.transitioned_object.version_id, gopts).await?; let reader = tgt_client
.get(&oi.transitioned_object.name, &oi.transitioned_object.version_id, gopts)
.await?;
Ok(get_fn(reader, h)) Ok(get_fn(reader, h))
} }
@@ -771,18 +800,18 @@ pub trait LifecycleOps {
impl LifecycleOps for ObjectInfo { impl LifecycleOps for ObjectInfo {
fn to_lifecycle_opts(&self) -> lifecycle::ObjectOpts { fn to_lifecycle_opts(&self) -> lifecycle::ObjectOpts {
lifecycle::ObjectOpts { lifecycle::ObjectOpts {
name: self.name.clone(), name: self.name.clone(),
user_tags: self.user_tags.clone(), user_tags: self.user_tags.clone(),
version_id: self.version_id.expect("err").to_string(), version_id: self.version_id.expect("err").to_string(),
mod_time: self.mod_time, mod_time: self.mod_time,
size: self.size, size: self.size,
is_latest: self.is_latest, is_latest: self.is_latest,
num_versions: self.num_versions, num_versions: self.num_versions,
delete_marker: self.delete_marker, delete_marker: self.delete_marker,
successor_mod_time: self.successor_mod_time, successor_mod_time: self.successor_mod_time,
//restore_ongoing: self.restore_ongoing, //restore_ongoing: self.restore_ongoing,
//restore_expires: self.restore_expires, //restore_expires: self.restore_expires,
transition_status: self.transitioned_object.status.clone(), transition_status: self.transitioned_object.status.clone(),
..Default::default() ..Default::default()
} }
} }
@@ -790,9 +819,9 @@ impl LifecycleOps for ObjectInfo {
#[derive(Debug, Default, Clone)] #[derive(Debug, Default, Clone)]
pub struct S3Location { pub struct S3Location {
pub bucketname: String, pub bucketname: String,
//pub encryption: Encryption, //pub encryption: Encryption,
pub prefix: String, pub prefix: String,
pub storage_class: String, pub storage_class: String,
//pub tagging: Tags, //pub tagging: Tags,
pub user_metadata: HashMap<String, String>, pub user_metadata: HashMap<String, String>,
@@ -803,13 +832,12 @@ pub struct OutputLocation(pub S3Location);
#[derive(Debug, Default, Clone)] #[derive(Debug, Default, Clone)]
pub struct RestoreObjectRequest { pub struct RestoreObjectRequest {
pub days: i64, pub days: i64,
pub ror_type: String, pub ror_type: String,
pub tier: String, pub tier: String,
pub description: String, pub description: String,
//pub select_parameters: SelectParameters, //pub select_parameters: SelectParameters,
pub output_location: OutputLocation, pub output_location: OutputLocation,
} }
const MAX_RESTORE_OBJECT_REQUEST_SIZE: i64 = 2 << 20; const MAX_RESTORE_OBJECT_REQUEST_SIZE: i64 = 2 << 20;
ecstore/src/bucket/lifecycle/lifecycle.rs
+91 -63
View File
@@ -1,12 +1,12 @@
use s3s::dto::{
BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, NoncurrentVersionTransition,
ObjectLockConfiguration, ObjectLockEnabled, Transition,
};
use std::cmp::Ordering; use std::cmp::Ordering;
use std::env; use std::env;
use std::fmt::Display; use std::fmt::Display;
use s3s::dto::{
BucketLifecycleConfiguration, ExpirationStatus, LifecycleRule, ObjectLockConfiguration,
ObjectLockEnabled, LifecycleExpiration, Transition, NoncurrentVersionTransition,
};
use time::macros::{datetime, offset}; use time::macros::{datetime, offset};
use time::{self, OffsetDateTime, Duration}; use time::{self, Duration, OffsetDateTime};
use crate::bucket::lifecycle::rule::TransitionOps; use crate::bucket::lifecycle::rule::TransitionOps;
@@ -16,10 +16,11 @@ pub const TRANSITION_COMPLETE: &str = "complete";
pub const TRANSITION_PENDING: &str = "pending"; pub const TRANSITION_PENDING: &str = "pending";
const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration allows a maximum of 1000 rules"; const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration allows a maximum of 1000 rules";
const ERR_LIFECYCLE_NO_RULE: &str = "Lifecycle configuration should have at least one rule"; const ERR_LIFECYCLE_NO_RULE: &str = "Lifecycle configuration should have at least one rule";
const ERR_LIFECYCLE_DUPLICATE_ID: &str = "Rule ID must be unique. Found same ID for more than one rule"; const ERR_LIFECYCLE_DUPLICATE_ID: &str = "Rule ID must be unique. Found same ID for more than one rule";
const ERR_XML_NOT_WELL_FORMED: &str = "The XML you provided was not well-formed or did not validate against our published schema"; const ERR_XML_NOT_WELL_FORMED: &str = "The XML you provided was not well-formed or did not validate against our published schema";
const ERR_LIFECYCLE_BUCKET_LOCKED: &str = "ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket"; const ERR_LIFECYCLE_BUCKET_LOCKED: &str =
"ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket";
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
pub enum IlmAction { pub enum IlmAction {
@@ -52,7 +53,10 @@ impl IlmAction {
if self.delete_restored() { if self.delete_restored() {
return true; return true;
} }
*self == Self::DeleteVersionAction || *self == Self::DeleteAction || *self == Self::DeleteAllVersionsAction || *self == Self::DelMarkerDeleteAllVersionsAction *self == Self::DeleteVersionAction
|| *self == Self::DeleteAction
|| *self == Self::DeleteAllVersionsAction
|| *self == Self::DelMarkerDeleteAllVersionsAction
} }
} }
@@ -204,7 +208,10 @@ impl Lifecycle for BucketLifecycleConfiguration {
return true; return true;
} }
let rule_expiration = rule.expiration.as_ref().expect("err!"); let rule_expiration = rule.expiration.as_ref().expect("err!");
if !rule_expiration.date.is_none() && OffsetDateTime::from(rule_expiration.date.clone().expect("err!")).unix_timestamp() < OffsetDateTime::now_utc().unix_timestamp() { if !rule_expiration.date.is_none()
&& OffsetDateTime::from(rule_expiration.date.clone().expect("err!")).unix_timestamp()
< OffsetDateTime::now_utc().unix_timestamp()
{
return true; return true;
} }
if !rule_expiration.date.is_none() { if !rule_expiration.date.is_none() {
@@ -213,9 +220,12 @@ impl Lifecycle for BucketLifecycleConfiguration {
if rule_expiration.expired_object_delete_marker.expect("err!") { if rule_expiration.expired_object_delete_marker.expect("err!") {
return true; return true;
} }
let rule_transitions: &[Transition]= &rule.transitions.as_ref().expect("err!"); let rule_transitions: &[Transition] = &rule.transitions.as_ref().expect("err!");
let rule_transitions_0 = rule_transitions[0].clone(); let rule_transitions_0 = rule_transitions[0].clone();
if !rule_transitions_0.date.is_none() && OffsetDateTime::from(rule_transitions_0.date.expect("err!")).unix_timestamp() < OffsetDateTime::now_utc().unix_timestamp() { if !rule_transitions_0.date.is_none()
&& OffsetDateTime::from(rule_transitions_0.date.expect("err!")).unix_timestamp()
< OffsetDateTime::now_utc().unix_timestamp()
{
return true; return true;
} }
if !rule.transitions.is_none() { if !rule.transitions.is_none() {
@@ -242,18 +252,18 @@ impl Lifecycle for BucketLifecycleConfiguration {
return Err(std::io::Error::other(ERR_LIFECYCLE_BUCKET_LOCKED)); return Err(std::io::Error::other(ERR_LIFECYCLE_BUCKET_LOCKED));
} }
} /*else { } /*else {
if object_lock_enabled.as_str() == ObjectLockEnabled::ENABLED { if object_lock_enabled.as_str() == ObjectLockEnabled::ENABLED {
return Err(Error::msg(ERR_LIFECYCLE_BUCKET_LOCKED)); return Err(Error::msg(ERR_LIFECYCLE_BUCKET_LOCKED));
} }
}*/ }*/
} }
} }
} }
for (i,_) in self.rules.iter().enumerate() { for (i, _) in self.rules.iter().enumerate() {
if i == self.rules.len()-1 { if i == self.rules.len() - 1 {
break; break;
} }
let other_rules = &self.rules[i+1..]; let other_rules = &self.rules[i + 1..];
for other_rule in other_rules { for other_rule in other_rules {
if self.rules[i].id == other_rule.id { if self.rules[i].id == other_rule.id {
return Err(std::io::Error::other(ERR_LIFECYCLE_DUPLICATE_ID)); return Err(std::io::Error::other(ERR_LIFECYCLE_DUPLICATE_ID));
@@ -281,7 +291,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
continue; continue;
}*/ }*/
//if !obj.delete_marker && !rule.filter.BySize(obj.size) { //if !obj.delete_marker && !rule.filter.BySize(obj.size) {
if !obj.delete_marker && false{ if !obj.delete_marker && false {
continue; continue;
} }
rules.push(rule.clone()); rules.push(rule.clone());
@@ -306,9 +316,9 @@ impl Lifecycle for BucketLifecycleConfiguration {
action = IlmAction::DeleteRestoredVersionAction; action = IlmAction::DeleteRestoredVersionAction;
} }
events.push(Event{ events.push(Event {
action: action, action: action,
due: Some(now), due: Some(now),
rule_id: "".into(), rule_id: "".into(),
noncurrent_days: 0, noncurrent_days: 0,
newer_noncurrent_versions: 0, newer_noncurrent_versions: 0,
@@ -322,10 +332,10 @@ impl Lifecycle for BucketLifecycleConfiguration {
if obj.expired_object_deletemarker() { if obj.expired_object_deletemarker() {
if let Some(expiration) = rule.expiration.as_ref() { if let Some(expiration) = rule.expiration.as_ref() {
if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker { if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker {
events.push(Event{ events.push(Event {
action: IlmAction::DeleteVersionAction, action: IlmAction::DeleteVersionAction,
rule_id: rule.id.clone().expect("err!"), rule_id: rule.id.clone().expect("err!"),
due: Some(now), due: Some(now),
noncurrent_days: 0, noncurrent_days: 0,
newer_noncurrent_versions: 0, newer_noncurrent_versions: 0,
storage_class: "".into(), storage_class: "".into(),
@@ -336,12 +346,12 @@ impl Lifecycle for BucketLifecycleConfiguration {
if let Some(expiration) = rule.expiration.as_ref() { if let Some(expiration) = rule.expiration.as_ref() {
if let Some(days) = expiration.days { if let Some(days) = expiration.days {
let expected_expiry = expected_expiry_time(obj.mod_time.expect("err!"), days/*, date*/); let expected_expiry = expected_expiry_time(obj.mod_time.expect("err!"), days /*, date*/);
if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() { if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() {
events.push(Event{ events.push(Event {
action: IlmAction::DeleteVersionAction, action: IlmAction::DeleteVersionAction,
rule_id: rule.id.clone().expect("err!"), rule_id: rule.id.clone().expect("err!"),
due: Some(expected_expiry), due: Some(expected_expiry),
noncurrent_days: 0, noncurrent_days: 0,
newer_noncurrent_versions: 0, newer_noncurrent_versions: 0,
storage_class: "".into(), storage_class: "".into(),
@@ -359,10 +369,10 @@ impl Lifecycle for BucketLifecycleConfiguration {
let due = expiration.next_due(obj); let due = expiration.next_due(obj);
if let Some(due) = due { if let Some(due) = due {
if now.unix_timestamp() == 0 || now.unix_timestamp() > due.unix_timestamp() { if now.unix_timestamp() == 0 || now.unix_timestamp() > due.unix_timestamp() {
events.push(Event{ events.push(Event {
action: IlmAction::DelMarkerDeleteAllVersionsAction, action: IlmAction::DelMarkerDeleteAllVersionsAction,
rule_id: rule.id.clone().expect("err!"), rule_id: rule.id.clone().expect("err!"),
due: Some(due), due: Some(due),
noncurrent_days: 0, noncurrent_days: 0,
newer_noncurrent_versions: 0, newer_noncurrent_versions: 0,
storage_class: "".into(), storage_class: "".into(),
@@ -392,10 +402,10 @@ impl Lifecycle for BucketLifecycleConfiguration {
if let Some(successor_mod_time) = obj.successor_mod_time { if let Some(successor_mod_time) = obj.successor_mod_time {
let expected_expiry = expected_expiry_time(successor_mod_time, noncurrent_days); let expected_expiry = expected_expiry_time(successor_mod_time, noncurrent_days);
if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() { if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() {
events.push(Event{ events.push(Event {
action: IlmAction::DeleteVersionAction, action: IlmAction::DeleteVersionAction,
rule_id: rule.id.clone().expect("err!"), rule_id: rule.id.clone().expect("err!"),
due: Some(expected_expiry), due: Some(expected_expiry),
noncurrent_days: 0, noncurrent_days: 0,
newer_noncurrent_versions: 0, newer_noncurrent_versions: 0,
storage_class: "".into(), storage_class: "".into(),
@@ -413,12 +423,19 @@ impl Lifecycle for BucketLifecycleConfiguration {
if storage_class.as_str() != "" { if storage_class.as_str() != "" {
if !obj.delete_marker && obj.transition_status != TRANSITION_COMPLETE { if !obj.delete_marker && obj.transition_status != TRANSITION_COMPLETE {
let due = rule.noncurrent_version_transitions.as_ref().unwrap()[0].next_due(obj); let due = rule.noncurrent_version_transitions.as_ref().unwrap()[0].next_due(obj);
if due.is_some() && (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unwrap().unix_timestamp()) { if due.is_some()
&& (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unwrap().unix_timestamp())
{
events.push(Event { events.push(Event {
action: IlmAction::TransitionVersionAction, action: IlmAction::TransitionVersionAction,
rule_id: rule.id.clone().expect("err!"), rule_id: rule.id.clone().expect("err!"),
due, due,
storage_class: rule.noncurrent_version_transitions.as_ref().unwrap()[0].storage_class.clone().unwrap().as_str().to_string(), storage_class: rule.noncurrent_version_transitions.as_ref().unwrap()[0]
.storage_class
.clone()
.unwrap()
.as_str()
.to_string(),
..Default::default() ..Default::default()
}); });
} }
@@ -434,10 +451,10 @@ impl Lifecycle for BucketLifecycleConfiguration {
let date0 = OffsetDateTime::from(date.clone()); let date0 = OffsetDateTime::from(date.clone());
if date0.unix_timestamp() != 0 { if date0.unix_timestamp() != 0 {
if now.unix_timestamp() == 0 || now.unix_timestamp() > date0.unix_timestamp() { if now.unix_timestamp() == 0 || now.unix_timestamp() > date0.unix_timestamp() {
events.push(Event{ events.push(Event {
action: IlmAction::DeleteAction, action: IlmAction::DeleteAction,
rule_id: rule.id.clone().expect("err!"), rule_id: rule.id.clone().expect("err!"),
due: Some(date0), due: Some(date0),
noncurrent_days: 0, noncurrent_days: 0,
newer_noncurrent_versions: 0, newer_noncurrent_versions: 0,
storage_class: "".into(), storage_class: "".into(),
@@ -448,10 +465,10 @@ impl Lifecycle for BucketLifecycleConfiguration {
if days != 0 { if days != 0 {
let expected_expiry: OffsetDateTime = expected_expiry_time(obj.mod_time.expect("err!"), days); let expected_expiry: OffsetDateTime = expected_expiry_time(obj.mod_time.expect("err!"), days);
if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() { if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() {
let mut event = Event{ let mut event = Event {
action: IlmAction::DeleteAction, action: IlmAction::DeleteAction,
rule_id: rule.id.clone().expect("err!"), rule_id: rule.id.clone().expect("err!"),
due: Some(expected_expiry), due: Some(expected_expiry),
noncurrent_days: 0, noncurrent_days: 0,
newer_noncurrent_versions: 0, newer_noncurrent_versions: 0,
storage_class: "".into(), storage_class: "".into(),
@@ -469,10 +486,12 @@ impl Lifecycle for BucketLifecycleConfiguration {
if let Some(ref transitions) = rule.transitions { if let Some(ref transitions) = rule.transitions {
let due = transitions[0].next_due(obj); let due = transitions[0].next_due(obj);
if let Some(due) = due { if let Some(due) = due {
if due.unix_timestamp() > 0 && (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unix_timestamp()) { if due.unix_timestamp() > 0
events.push(Event{ && (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unix_timestamp())
action: IlmAction::TransitionAction, {
rule_id: rule.id.clone().expect("err!"), events.push(Event {
action: IlmAction::TransitionAction,
rule_id: rule.id.clone().expect("err!"),
due: Some(due), due: Some(due),
storage_class: transitions[0].storage_class.clone().expect("err!").as_str().to_string(), storage_class: transitions[0].storage_class.clone().expect("err!").as_str().to_string(),
noncurrent_days: 0, noncurrent_days: 0,
@@ -488,20 +507,27 @@ impl Lifecycle for BucketLifecycleConfiguration {
if events.len() > 0 { if events.len() > 0 {
events.sort_by(|a, b| { events.sort_by(|a, b| {
if now.unix_timestamp() > a.due.expect("err!").unix_timestamp() && now.unix_timestamp() > b.due.expect("err").unix_timestamp() || a.due.expect("err").unix_timestamp() == b.due.expect("err").unix_timestamp() { if now.unix_timestamp() > a.due.expect("err!").unix_timestamp()
&& now.unix_timestamp() > b.due.expect("err").unix_timestamp()
|| a.due.expect("err").unix_timestamp() == b.due.expect("err").unix_timestamp()
{
match a.action { match a.action {
IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction IlmAction::DeleteAllVersionsAction
| IlmAction::DeleteAction | IlmAction::DeleteVersionAction => { | IlmAction::DelMarkerDeleteAllVersionsAction
| IlmAction::DeleteAction
| IlmAction::DeleteVersionAction => {
return Ordering::Less; return Ordering::Less;
} }
_ => () _ => (),
} }
match b.action { match b.action {
IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction IlmAction::DeleteAllVersionsAction
| IlmAction::DeleteAction | IlmAction::DeleteVersionAction => { | IlmAction::DelMarkerDeleteAllVersionsAction
| IlmAction::DeleteAction
| IlmAction::DeleteVersionAction => {
return Ordering::Greater; return Ordering::Greater;
} }
_ => () _ => (),
} }
return Ordering::Less; return Ordering::Less;
} }
@@ -526,18 +552,18 @@ impl Lifecycle for BucketLifecycleConfiguration {
continue; continue;
} }
return Event { return Event {
action: IlmAction::DeleteVersionAction, action: IlmAction::DeleteVersionAction,
rule_id: rule.id.clone().expect("err"), rule_id: rule.id.clone().expect("err"),
noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32, noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32,
newer_noncurrent_versions: newer_noncurrent_versions as usize, newer_noncurrent_versions: newer_noncurrent_versions as usize,
due: Some(OffsetDateTime::UNIX_EPOCH), due: Some(OffsetDateTime::UNIX_EPOCH),
storage_class: "".into(), storage_class: "".into(),
}; };
} else { } else {
return Event { return Event {
action: IlmAction::DeleteVersionAction, action: IlmAction::DeleteVersionAction,
rule_id: rule.id.clone().expect("err"), rule_id: rule.id.clone().expect("err"),
noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32, noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32,
newer_noncurrent_versions: 0, newer_noncurrent_versions: 0,
due: Some(OffsetDateTime::UNIX_EPOCH), due: Some(OffsetDateTime::UNIX_EPOCH),
storage_class: "".into(), storage_class: "".into(),
@@ -601,7 +627,9 @@ pub fn expected_expiry_time(mod_time: OffsetDateTime, days: i32) -> OffsetDateTi
if days == 0 { if days == 0 {
return mod_time; return mod_time;
} }
let t = mod_time.to_offset(offset!(-0:00:00)).saturating_add(Duration::days(0/*days as i64*/)); //debug let t = mod_time
.to_offset(offset!(-0:00:00))
.saturating_add(Duration::days(0 /*days as i64*/)); //debug
let mut hour = 3600; let mut hour = 3600;
if let Ok(env_ilm_hour) = env::var("_RUSTFS_ILM_HOUR") { if let Ok(env_ilm_hour) = env::var("_RUSTFS_ILM_HOUR") {
if let Ok(num_hour) = env_ilm_hour.parse::<usize>() { if let Ok(num_hour) = env_ilm_hour.parse::<usize>() {
@@ -661,7 +689,7 @@ impl Default for Event {
#[derive(Debug, Clone, Default)] #[derive(Debug, Clone, Default)]
pub struct ExpirationOptions { pub struct ExpirationOptions {
pub expire: bool pub expire: bool,
} }
impl ExpirationOptions { impl ExpirationOptions {
ecstore/src/bucket/lifecycle/mod.rs
+5 -5
View File
@@ -1,6 +1,6 @@
pub mod rule;
pub mod lifecycle;
pub mod tier_sweeper;
pub mod tier_last_day_stats;
pub mod bucket_lifecycle_ops;
pub mod bucket_lifecycle_audit; pub mod bucket_lifecycle_audit;
pub mod bucket_lifecycle_ops;
pub mod lifecycle;
pub mod rule;
pub mod tier_last_day_stats;
pub mod tier_sweeper;
ecstore/src/bucket/lifecycle/rule.rs
+5 -7
View File
@@ -1,10 +1,9 @@
use s3s::dto::{ use s3s::dto::{LifecycleRuleFilter, Transition};
LifecycleRuleFilter, Transition,
};
const ERR_TRANSITION_INVALID_DAYS: &str = "Days must be 0 or greater when used with Transition"; const ERR_TRANSITION_INVALID_DAYS: &str = "Days must be 0 or greater when used with Transition";
const ERR_TRANSITION_INVALID_DATE: &str = "Date must be provided in ISO 8601 format"; const ERR_TRANSITION_INVALID_DATE: &str = "Date must be provided in ISO 8601 format";
const ERR_TRANSITION_INVALID: &str = "Exactly one of Days (0 or greater) or Date (positive ISO 8601 format) should be present in Transition."; const ERR_TRANSITION_INVALID: &str =
"Exactly one of Days (0 or greater) or Date (positive ISO 8601 format) should be present in Transition.";
const ERR_TRANSITION_DATE_NOT_MIDNIGHT: &str = "'Date' must be at midnight GMT"; const ERR_TRANSITION_DATE_NOT_MIDNIGHT: &str = "'Date' must be at midnight GMT";
pub trait Filter { pub trait Filter {
@@ -39,7 +38,6 @@ impl TransitionOps for Transition {
} }
} }
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use super::*; use super::*;
ecstore/src/bucket/lifecycle/tier_last_day_stats.rs
+2 -5
View File
@@ -1,9 +1,9 @@
use sha2::Sha256; use sha2::Sha256;
use std::collections::HashMap; use std::collections::HashMap;
use std::ops::Sub;
use time::OffsetDateTime; use time::OffsetDateTime;
use tracing::{error, warn}; use tracing::{error, warn};
use std::ops::Sub;
use crate::heal::data_usage_cache::TierStats; use crate::heal::data_usage_cache::TierStats;
@@ -79,8 +79,5 @@ impl LastDayTierStats {
} }
} }
#[cfg(test)] #[cfg(test)]
mod test { mod test {}
}
ecstore/src/bucket/lifecycle/tier_sweeper.rs
+12 -13
View File
@@ -1,11 +1,11 @@
use sha2::{Digest, Sha256}; use sha2::{Digest, Sha256};
use xxhash_rust::xxh64;
use std::any::Any; use std::any::Any;
use std::io::{Cursor, Write}; use std::io::{Cursor, Write};
use xxhash_rust::xxh64;
use crate::global::GLOBAL_TierConfigMgr;
use super::bucket_lifecycle_ops::{ExpiryOp, GLOBAL_ExpiryState, TransitionedObject}; use super::bucket_lifecycle_ops::{ExpiryOp, GLOBAL_ExpiryState, TransitionedObject};
use super::lifecycle::{self, ObjectOpts}; use super::lifecycle::{self, ObjectOpts};
use crate::global::GLOBAL_TierConfigMgr;
static XXHASH_SEED: u64 = 0; static XXHASH_SEED: u64 = 0;
@@ -44,9 +44,9 @@ impl ObjSweeper {
} }
pub fn get_opts(&self) -> lifecycle::ObjectOpts { pub fn get_opts(&self) -> lifecycle::ObjectOpts {
let mut opts = ObjectOpts{ let mut opts = ObjectOpts {
version_id: self.version_id.clone(), version_id: self.version_id.clone(),
versioned: self.versioned, versioned: self.versioned,
version_suspended: self.suspended, version_suspended: self.suspended,
..Default::default() ..Default::default()
}; };
@@ -69,16 +69,18 @@ impl ObjSweeper {
} }
let mut del_tier = false; let mut del_tier = false;
if !self.versioned || self.suspended { // 1, 2.a, 2.b if !self.versioned || self.suspended {
// 1, 2.a, 2.b
del_tier = true; del_tier = true;
} else if self.versioned && self.version_id != "" { // 3.a } else if self.versioned && self.version_id != "" {
// 3.a
del_tier = true; del_tier = true;
} }
if del_tier { if del_tier {
return Some(Jentry { return Some(Jentry {
obj_name: self.remote_object.clone(), obj_name: self.remote_object.clone(),
version_id: self.transition_version_id.clone(), version_id: self.transition_version_id.clone(),
tier_name: self.transition_tier.clone(), tier_name: self.transition_tier.clone(),
}); });
} }
None None
@@ -123,8 +125,5 @@ pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_na
w.remove(obj_name, rv_id).await w.remove(obj_name, rv_id).await
} }
#[cfg(test)] #[cfg(test)]
mod test { mod test {}
}
ecstore/src/bucket/mod.rs
+1 -1
View File
@@ -1,4 +1,5 @@
pub mod error; pub mod error;
pub mod lifecycle;
pub mod metadata; pub mod metadata;
pub mod metadata_sys; pub mod metadata_sys;
pub mod object_lock; pub mod object_lock;
@@ -10,4 +11,3 @@ pub mod target;
pub mod utils; pub mod utils;
pub mod versioning; pub mod versioning;
pub mod versioning_sys; pub mod versioning_sys;
pub mod lifecycle;
ecstore/src/bucket/object_lock/objectlock.rs
+25 -20
View File
@@ -3,13 +3,8 @@ use std::collections::HashMap;
use time::{OffsetDateTime, format_description}; use time::{OffsetDateTime, format_description};
use tracing::{error, warn}; use tracing::{error, warn};
use s3s::dto::{ use s3s::dto::{Date, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockRetention, ObjectLockRetentionMode};
ObjectLockRetentionMode, ObjectLockRetention, ObjectLockLegalHoldStatus, ObjectLockLegalHold, use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
Date,
};
use s3s::header::{
X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_OBJECT_LOCK_LEGAL_HOLD,
};
//const AMZ_OBJECTLOCK_BYPASS_RET_GOVERNANCE: &str = "X-Amz-Bypass-Governance-Retention"; //const AMZ_OBJECTLOCK_BYPASS_RET_GOVERNANCE: &str = "X-Amz-Bypass-Governance-Retention";
//const AMZ_OBJECTLOCK_RETAIN_UNTIL_DATE: &str = "X-Amz-Object-Lock-Retain-Until-Date"; //const AMZ_OBJECTLOCK_RETAIN_UNTIL_DATE: &str = "X-Amz-Object-Lock-Retain-Until-Date";
@@ -17,12 +12,14 @@ use s3s::header::{
//const AMZ_OBJECTLOCK_LEGALHOLD: &str = "X-Amz-Object-Lock-Legal-Hold"; //const AMZ_OBJECTLOCK_LEGALHOLD: &str = "X-Amz-Object-Lock-Legal-Hold";
const ERR_MALFORMED_BUCKET_OBJECT_CONFIG: &str = "invalid bucket object lock config"; const ERR_MALFORMED_BUCKET_OBJECT_CONFIG: &str = "invalid bucket object lock config";
const ERR_INVALID_RETENTION_DATE: &str = "date must be provided in ISO 8601 format"; const ERR_INVALID_RETENTION_DATE: &str = "date must be provided in ISO 8601 format";
const ERR_PAST_OBJECTLOCK_RETAIN_DATE: &str = "the retain until date must be in the future"; const ERR_PAST_OBJECTLOCK_RETAIN_DATE: &str = "the retain until date must be in the future";
const ERR_UNKNOWN_WORMMODE_DIRECTIVE: &str = "unknown WORM mode directive"; const ERR_UNKNOWN_WORMMODE_DIRECTIVE: &str = "unknown WORM mode directive";
const ERR_OBJECTLOCK_MISSING_CONTENT_MD5: &str = "content-MD5 HTTP header is required for Put Object requests with Object Lock parameters"; const ERR_OBJECTLOCK_MISSING_CONTENT_MD5: &str =
const ERR_OBJECTLOCK_INVALID_HEADERS: &str = "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied"; "content-MD5 HTTP header is required for Put Object requests with Object Lock parameters";
const ERR_MALFORMED_XML: &str = "the XML you provided was not well-formed or did not validate against our published schema"; const ERR_OBJECTLOCK_INVALID_HEADERS: &str =
"x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied";
const ERR_MALFORMED_XML: &str = "the XML you provided was not well-formed or did not validate against our published schema";
pub fn utc_now_ntp() -> OffsetDateTime { pub fn utc_now_ntp() -> OffsetDateTime {
return OffsetDateTime::now_utc(); return OffsetDateTime::now_utc();
@@ -39,7 +36,10 @@ pub fn get_object_retention_meta(meta: HashMap<String, String>) -> ObjectLockRet
if let Some(mode_str) = mode_str { if let Some(mode_str) = mode_str {
mode = parse_ret_mode(mode_str.as_str()); mode = parse_ret_mode(mode_str.as_str());
} else { } else {
return ObjectLockRetention {mode: None, retain_until_date: None}; return ObjectLockRetention {
mode: None,
retain_until_date: None,
};
} }
let mut till_str = meta.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_lowercase().as_str()); let mut till_str = meta.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_lowercase().as_str());
@@ -49,10 +49,13 @@ pub fn get_object_retention_meta(meta: HashMap<String, String>) -> ObjectLockRet
if let Some(till_str) = till_str { if let Some(till_str) = till_str {
let t = OffsetDateTime::parse(till_str, &format_description::well_known::Iso8601::DEFAULT); let t = OffsetDateTime::parse(till_str, &format_description::well_known::Iso8601::DEFAULT);
if t.is_err() { if t.is_err() {
retain_until_date = Date::from(t.expect("err")); //TODO: utc retain_until_date = Date::from(t.expect("err")); //TODO: utc
} }
} }
ObjectLockRetention {mode: Some(mode), retain_until_date: Some(retain_until_date)} ObjectLockRetention {
mode: Some(mode),
retain_until_date: Some(retain_until_date),
}
} }
pub fn get_object_legalhold_meta(meta: HashMap<String, String>) -> ObjectLockLegalHold { pub fn get_object_legalhold_meta(meta: HashMap<String, String>) -> ObjectLockLegalHold {
@@ -61,9 +64,11 @@ pub fn get_object_legalhold_meta(meta: HashMap<String, String>) -> ObjectLockLeg
hold_str = Some(&meta[X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()]); hold_str = Some(&meta[X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()]);
} }
if let Some(hold_str) = hold_str { if let Some(hold_str) = hold_str {
return ObjectLockLegalHold {status: Some(parse_legalhold_status(hold_str))}; return ObjectLockLegalHold {
status: Some(parse_legalhold_status(hold_str)),
};
} }
ObjectLockLegalHold {status: None} ObjectLockLegalHold { status: None }
} }
pub fn parse_ret_mode(mode_str: &str) -> ObjectLockRetentionMode { pub fn parse_ret_mode(mode_str: &str) -> ObjectLockRetentionMode {
@@ -75,7 +80,7 @@ pub fn parse_ret_mode(mode_str: &str) -> ObjectLockRetentionMode {
"COMPLIANCE" => { "COMPLIANCE" => {
mode = ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE); mode = ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE);
} }
_ => unreachable!() _ => unreachable!(),
} }
mode mode
} }
@@ -89,7 +94,7 @@ pub fn parse_legalhold_status(hold_str: &str) -> ObjectLockLegalHoldStatus {
"OFF" => { "OFF" => {
st = ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF); st = ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF);
} }
_ => unreachable!() _ => unreachable!(),
} }
st st
} }
ecstore/src/bucket/object_lock/objectlock_sys.rs
+11 -8
View File
@@ -3,12 +3,10 @@ use std::sync::Arc;
use time::OffsetDateTime; use time::OffsetDateTime;
use tracing::{error, warn}; use tracing::{error, warn};
use s3s::dto::{ use s3s::dto::{DefaultRetention, ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
DefaultRetention, ObjectLockRetentionMode, ObjectLockLegalHoldStatus,
};
use crate::store_api::ObjectInfo;
use crate::bucket::metadata_sys::get_object_lock_config; use crate::bucket::metadata_sys::get_object_lock_config;
use crate::store_api::ObjectInfo;
use super::objectlock; use super::objectlock;
@@ -21,7 +19,12 @@ impl BucketObjectLockSys {
} }
pub async fn get(bucket: &str) -> Option<DefaultRetention> { pub async fn get(bucket: &str) -> Option<DefaultRetention> {
if let Some(object_lock_rule) = get_object_lock_config(bucket).await.expect("get_object_lock_config err!").0.rule { if let Some(object_lock_rule) = get_object_lock_config(bucket)
.await
.expect("get_object_lock_config err!")
.0
.rule
{
return object_lock_rule.default_retention; return object_lock_rule.default_retention;
} }
None None
@@ -35,10 +38,10 @@ pub fn enforce_retention_for_deletion(obj_info: &ObjectInfo) -> bool {
let lhold = objectlock::get_object_legalhold_meta(obj_info.user_defined.clone().expect("err")); let lhold = objectlock::get_object_legalhold_meta(obj_info.user_defined.clone().expect("err"));
match lhold.status { match lhold.status {
Some(st) if st.as_str()==ObjectLockLegalHoldStatus::ON => { Some(st) if st.as_str() == ObjectLockLegalHoldStatus::ON => {
return true; return true;
} }
_ => () _ => (),
} }
let ret = objectlock::get_object_retention_meta(obj_info.user_defined.clone().expect("err")); let ret = objectlock::get_object_retention_meta(obj_info.user_defined.clone().expect("err"));
@@ -49,7 +52,7 @@ pub fn enforce_retention_for_deletion(obj_info: &ObjectInfo) -> bool {
return true; return true;
} }
} }
_ => () _ => (),
} }
false false
} }
ecstore/src/checksum.rs
+49 -61
View File
@@ -1,23 +1,17 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use std::{collections::HashMap, sync::Arc};
use std::ops::{BitAnd, BitOr};
use lazy_static::lazy_static; use lazy_static::lazy_static;
use std::ops::{BitAnd, BitOr};
use std::{collections::HashMap, sync::Arc};
use crate::client::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart};
use crate::{disk::DiskAPI, store_api::GetObjectReader};
use reader::hasher::{Hasher, Sha256}; use reader::hasher::{Hasher, Sha256};
use rustfs_utils::crypto::{base64_decode, base64_encode};
use s3s::header::{ use s3s::header::{
X_AMZ_CHECKSUM_ALGORITHM, X_AMZ_CHECKSUM_CRC32, X_AMZ_CHECKSUM_CRC32C, X_AMZ_CHECKSUM_SHA1, X_AMZ_CHECKSUM_SHA256, X_AMZ_CHECKSUM_ALGORITHM, X_AMZ_CHECKSUM_CRC32, X_AMZ_CHECKSUM_CRC32C, X_AMZ_CHECKSUM_SHA1, X_AMZ_CHECKSUM_SHA256,
}; };
use crate::client::{
api_put_object::PutObjectOptions,
api_s3_datatypes::ObjectPart,
};
use rustfs_utils::crypto::{base64_decode, base64_encode};
use crate::{
disk::DiskAPI,
store_api::GetObjectReader,
};
use enumset::{enum_set, EnumSet, EnumSetType}; use enumset::{EnumSet, EnumSetType, enum_set};
#[derive(Debug, EnumSetType, Default)] #[derive(Debug, EnumSetType, Default)]
#[enumset(repr = "u8")] #[enumset(repr = "u8")]
@@ -38,8 +32,10 @@ lazy_static! {
s.remove(ChecksumMode::ChecksumFullObject); s.remove(ChecksumMode::ChecksumFullObject);
s s
}; };
static ref C_ChecksumFullObjectCRC32: EnumSet<ChecksumMode> = enum_set!(ChecksumMode::ChecksumCRC32 | ChecksumMode::ChecksumFullObject); static ref C_ChecksumFullObjectCRC32: EnumSet<ChecksumMode> =
static ref C_ChecksumFullObjectCRC32C: EnumSet<ChecksumMode> = enum_set!(ChecksumMode::ChecksumCRC32C | ChecksumMode::ChecksumFullObject); enum_set!(ChecksumMode::ChecksumCRC32 | ChecksumMode::ChecksumFullObject);
static ref C_ChecksumFullObjectCRC32C: EnumSet<ChecksumMode> =
enum_set!(ChecksumMode::ChecksumCRC32C | ChecksumMode::ChecksumFullObject);
} }
const AMZ_CHECKSUM_CRC64NVME: &str = "x-amz-checksum-crc64nvme"; const AMZ_CHECKSUM_CRC64NVME: &str = "x-amz-checksum-crc64nvme";
@@ -49,24 +45,12 @@ impl ChecksumMode {
pub fn base(&self) -> ChecksumMode { pub fn base(&self) -> ChecksumMode {
let s = EnumSet::from(*self).intersection(*C_ChecksumMask); let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
match s.as_u8() { match s.as_u8() {
1_u8 => { 1_u8 => ChecksumMode::ChecksumNone,
ChecksumMode::ChecksumNone 2_u8 => ChecksumMode::ChecksumSHA256,
} 4_u8 => ChecksumMode::ChecksumSHA1,
2_u8 => { 8_u8 => ChecksumMode::ChecksumCRC32,
ChecksumMode::ChecksumSHA256 16_u8 => ChecksumMode::ChecksumCRC32C,
} 32_u8 => ChecksumMode::ChecksumCRC64NVME,
4_u8 => {
ChecksumMode::ChecksumSHA1
}
8_u8 => {
ChecksumMode::ChecksumCRC32
}
16_u8 => {
ChecksumMode::ChecksumCRC32C
}
32_u8 => {
ChecksumMode::ChecksumCRC64NVME
}
_ => panic!("enum err."), _ => panic!("enum err."),
} }
} }
@@ -119,17 +103,13 @@ impl ChecksumMode {
let u = EnumSet::from(*self).intersection(*C_ChecksumMask).as_u8(); let u = EnumSet::from(*self).intersection(*C_ChecksumMask).as_u8();
if u == ChecksumMode::ChecksumCRC32 as u8 || u == ChecksumMode::ChecksumCRC32C as u8 { if u == ChecksumMode::ChecksumCRC32 as u8 || u == ChecksumMode::ChecksumCRC32C as u8 {
4 4
} } else if u == ChecksumMode::ChecksumSHA1 as u8 {
else if u == ChecksumMode::ChecksumSHA1 as u8 { 4 //sha1.size
4//sha1.size } else if u == ChecksumMode::ChecksumSHA256 as u8 {
} 4 //sha256.size
else if u == ChecksumMode::ChecksumSHA256 as u8 { } else if u == ChecksumMode::ChecksumCRC64NVME as u8 {
4//sha256.size 4 //crc64.size
} } else {
else if u == ChecksumMode::ChecksumCRC64NVME as u8 {
4//crc64.size
}
else {
0 0
} }
} }
@@ -196,7 +176,7 @@ impl ChecksumMode {
ChecksumMode::ChecksumCRC64NVME => { ChecksumMode::ChecksumCRC64NVME => {
return "CRC64NVME".to_string(); return "CRC64NVME".to_string();
} }
_=> { _ => {
return "<invalid>".to_string(); return "<invalid>".to_string();
} }
} }
@@ -226,10 +206,13 @@ impl ChecksumMode {
} }
}); });
let c = self.base(); let c = self.base();
let mut crc_bytes = Vec::<u8>::with_capacity(p.len()*self.raw_byte_len() as usize); let mut crc_bytes = Vec::<u8>::with_capacity(p.len() * self.raw_byte_len() as usize);
let mut h = self.hasher()?; let mut h = self.hasher()?;
h.write(&crc_bytes); h.write(&crc_bytes);
Ok(Checksum {checksum_type: self.clone(), r: h.sum().as_bytes().to_vec()}) Ok(Checksum {
checksum_type: self.clone(),
r: h.sum().as_bytes().to_vec(),
})
} }
pub fn full_object_checksum(&self, p: &mut [ObjectPart]) -> Result<Checksum, std::io::Error> { pub fn full_object_checksum(&self, p: &mut [ObjectPart]) -> Result<Checksum, std::io::Error> {
@@ -246,7 +229,10 @@ struct Checksum {
impl Checksum { impl Checksum {
fn new(t: ChecksumMode, b: &[u8]) -> Checksum { fn new(t: ChecksumMode, b: &[u8]) -> Checksum {
if t.is_set() && b.len() == t.raw_byte_len() { if t.is_set() && b.len() == t.raw_byte_len() {
return Checksum {checksum_type: t, r: b.to_vec()}; return Checksum {
checksum_type: t,
r: b.to_vec(),
};
} }
Checksum::default() Checksum::default()
} }
@@ -257,7 +243,7 @@ impl Checksum {
Err(err) => return Err(std::io::Error::other(err.to_string())), Err(err) => return Err(std::io::Error::other(err.to_string())),
}; };
if t.is_set() && b.len() == t.raw_byte_len() { if t.is_set() && b.len() == t.raw_byte_len() {
return Ok(Checksum {checksum_type: t, r: b}); return Ok(Checksum { checksum_type: t, r: b });
} }
Ok(Checksum::default()) Ok(Checksum::default())
} }
@@ -282,28 +268,30 @@ impl Checksum {
} }
pub fn add_auto_checksum_headers(opts: &mut PutObjectOptions) { pub fn add_auto_checksum_headers(opts: &mut PutObjectOptions) {
opts.user_metadata.insert("X-Amz-Checksum-Algorithm".to_string(), opts.auto_checksum.to_string()); opts.user_metadata
.insert("X-Amz-Checksum-Algorithm".to_string(), opts.auto_checksum.to_string());
if opts.auto_checksum.full_object_requested() { if opts.auto_checksum.full_object_requested() {
opts.user_metadata.insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string()); opts.user_metadata
.insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string());
} }
} }
pub fn apply_auto_checksum(opts: &mut PutObjectOptions, all_parts: &mut [ObjectPart]) -> Result<(), std::io::Error> { pub fn apply_auto_checksum(opts: &mut PutObjectOptions, all_parts: &mut [ObjectPart]) -> Result<(), std::io::Error> {
if opts.auto_checksum.can_composite() && !opts.auto_checksum.is(ChecksumMode::ChecksumFullObject) { if opts.auto_checksum.can_composite() && !opts.auto_checksum.is(ChecksumMode::ChecksumFullObject) {
let crc = opts.auto_checksum.composite_checksum(all_parts)?; let crc = opts.auto_checksum.composite_checksum(all_parts)?;
opts.user_metadata = { opts.user_metadata = {
let mut hm = HashMap::new(); let mut hm = HashMap::new();
hm.insert(opts.auto_checksum.key(), crc.encoded()); hm.insert(opts.auto_checksum.key(), crc.encoded());
hm hm
} }
} else if opts.auto_checksum.can_merge_crc() { } else if opts.auto_checksum.can_merge_crc() {
let crc = opts.auto_checksum.full_object_checksum(all_parts)?; let crc = opts.auto_checksum.full_object_checksum(all_parts)?;
opts.user_metadata = { opts.user_metadata = {
let mut hm = HashMap::new(); let mut hm = HashMap::new();
hm.insert(opts.auto_checksum.key_capitalized(), crc.encoded()); hm.insert(opts.auto_checksum.key_capitalized(), crc.encoded());
hm.insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string()); hm.insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string());
hm hm
} }
} }
Ok(()) Ok(())
ecstore/src/client/api_bucket_policy.rs
+53 -43
View File
@@ -1,11 +1,11 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use std::collections::HashMap;
use http::{HeaderMap, StatusCode};
use bytes::Bytes; use bytes::Bytes;
use http::{HeaderMap, StatusCode};
use std::collections::HashMap;
use crate::client::{ use crate::client::{
api_error_response::http_resp_to_error_response, api_error_response::http_resp_to_error_response,
transition_api::{RequestMetadata, TransitionClient, ReaderImpl} transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
}; };
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
@@ -23,9 +23,9 @@ impl TransitionClient {
url_values.insert("policy".to_string(), "".to_string()); url_values.insert("policy".to_string(), "".to_string());
let mut req_metadata = RequestMetadata { let mut req_metadata = RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
query_values: url_values, query_values: url_values,
content_body: ReaderImpl::Body(Bytes::from(policy.as_bytes().to_vec())), content_body: ReaderImpl::Body(Bytes::from(policy.as_bytes().to_vec())),
content_length: policy.len() as i64, content_length: policy.len() as i64,
object_name: "".to_string(), object_name: "".to_string(),
custom_header: HeaderMap::new(), custom_header: HeaderMap::new(),
@@ -43,9 +43,9 @@ impl TransitionClient {
let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?; let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
//defer closeResponse(resp) //defer closeResponse(resp)
//if resp != nil { //if resp != nil {
if resp.status() != StatusCode::NO_CONTENT && resp.status() != StatusCode::OK { if resp.status() != StatusCode::NO_CONTENT && resp.status() != StatusCode::OK {
return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, ""))); return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, "")));
} }
//} //}
Ok(()) Ok(())
} }
@@ -54,23 +54,28 @@ impl TransitionClient {
let mut url_values = HashMap::new(); let mut url_values = HashMap::new();
url_values.insert("policy".to_string(), "".to_string()); url_values.insert("policy".to_string(), "".to_string());
let resp = self.execute_method(http::Method::DELETE, &mut RequestMetadata { let resp = self
bucket_name: bucket_name.to_string(), .execute_method(
query_values: url_values, http::Method::DELETE,
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), &mut RequestMetadata {
object_name: "".to_string(), bucket_name: bucket_name.to_string(),
custom_header: HeaderMap::new(), query_values: url_values,
content_body: ReaderImpl::Body(Bytes::new()), content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
content_length: 0, object_name: "".to_string(),
content_md5_base64: "".to_string(), custom_header: HeaderMap::new(),
stream_sha256: false, content_body: ReaderImpl::Body(Bytes::new()),
trailer: HeaderMap::new(), content_length: 0,
pre_sign_url: Default::default(), content_md5_base64: "".to_string(),
add_crc: Default::default(), stream_sha256: false,
extra_pre_sign_header: Default::default(), trailer: HeaderMap::new(),
bucket_location: Default::default(), pre_sign_url: Default::default(),
expires: Default::default(), add_crc: Default::default(),
}).await?; extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
},
)
.await?;
//defer closeResponse(resp) //defer closeResponse(resp)
if resp.status() != StatusCode::NO_CONTENT { if resp.status() != StatusCode::NO_CONTENT {
@@ -89,23 +94,28 @@ impl TransitionClient {
let mut url_values = HashMap::new(); let mut url_values = HashMap::new();
url_values.insert("policy".to_string(), "".to_string()); url_values.insert("policy".to_string(), "".to_string());
let resp = self.execute_method(http::Method::GET, &mut RequestMetadata { let resp = self
bucket_name: bucket_name.to_string(), .execute_method(
query_values: url_values, http::Method::GET,
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), &mut RequestMetadata {
object_name: "".to_string(), bucket_name: bucket_name.to_string(),
custom_header: HeaderMap::new(), query_values: url_values,
content_body: ReaderImpl::Body(Bytes::new()), content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
content_length: 0, object_name: "".to_string(),
content_md5_base64: "".to_string(), custom_header: HeaderMap::new(),
stream_sha256: false, content_body: ReaderImpl::Body(Bytes::new()),
trailer: HeaderMap::new(), content_length: 0,
pre_sign_url: Default::default(), content_md5_base64: "".to_string(),
add_crc: Default::default(), stream_sha256: false,
extra_pre_sign_header: Default::default(), trailer: HeaderMap::new(),
bucket_location: Default::default(), pre_sign_url: Default::default(),
expires: Default::default(), add_crc: Default::default(),
}).await?; extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
},
)
.await?;
let policy = String::from_utf8_lossy(&resp.body().bytes().expect("err").to_vec()).to_string(); let policy = String::from_utf8_lossy(&resp.body().bytes().expect("err").to_vec()).to_string();
Ok(policy) Ok(policy)
ecstore/src/client/api_error_response.rs
+72 -58
View File
@@ -1,26 +1,25 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use std::fmt::Display;
use http::StatusCode; use http::StatusCode;
use serde::{Serialize, Deserialize}; use serde::{Deserialize, Serialize};
use serde::{ser::Serializer, de::Deserializer}; use serde::{de::Deserializer, ser::Serializer};
use std::fmt::Display;
use s3s::S3ErrorCode;
use s3s::Body; use s3s::Body;
use s3s::S3ErrorCode;
const REPORT_ISSUE: &str = "Please report this issue at https://github.com/rustfs/rustfs/issues."; const REPORT_ISSUE: &str = "Please report this issue at https://github.com/rustfs/rustfs/issues.";
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize, Debug, Clone, thiserror::Error, PartialEq, Eq)]
#[derive(Debug, Clone, thiserror::Error, PartialEq, Eq)]
#[serde(default, rename_all = "PascalCase")] #[serde(default, rename_all = "PascalCase")]
pub struct ErrorResponse { pub struct ErrorResponse {
#[serde(serialize_with = "serialize_code", deserialize_with = "deserialize_code")] #[serde(serialize_with = "serialize_code", deserialize_with = "deserialize_code")]
pub code: S3ErrorCode, pub code: S3ErrorCode,
pub message: String, pub message: String,
pub bucket_name: String, pub bucket_name: String,
pub key: String, pub key: String,
pub resource: String, pub resource: String,
pub request_id: String, pub request_id: String,
pub host_id: String, pub host_id: String,
pub region: String, pub region: String,
pub server: String, pub server: String,
#[serde(skip)] #[serde(skip)]
@@ -29,14 +28,14 @@ pub struct ErrorResponse {
fn serialize_code<S>(data: &S3ErrorCode, s: S) -> Result<S::Ok, S::Error> fn serialize_code<S>(data: &S3ErrorCode, s: S) -> Result<S::Ok, S::Error>
where where
S: Serializer S: Serializer,
{ {
s.serialize_str("") s.serialize_str("")
} }
fn deserialize_code<'de, D>(d: D) -> Result<S3ErrorCode, D::Error> fn deserialize_code<'de, D>(d: D) -> Result<S3ErrorCode, D::Error>
where where
D: Deserializer<'de> D: Deserializer<'de>,
{ {
Ok(S3ErrorCode::from_bytes(String::deserialize(d)?.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into()))) Ok(S3ErrorCode::from_bytes(String::deserialize(d)?.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into())))
} }
@@ -45,14 +44,14 @@ impl Default for ErrorResponse {
fn default() -> Self { fn default() -> Self {
ErrorResponse { ErrorResponse {
code: S3ErrorCode::Custom("".into()), code: S3ErrorCode::Custom("".into()),
message: Default::default(), message: Default::default(),
bucket_name: Default::default(), bucket_name: Default::default(),
key: Default::default(), key: Default::default(),
resource: Default::default(), resource: Default::default(),
request_id: Default::default(), request_id: Default::default(),
host_id: Default::default(), host_id: Default::default(),
region: Default::default(), region: Default::default(),
server: Default::default(), server: Default::default(),
status_code: Default::default(), status_code: Default::default(),
} }
} }
@@ -76,7 +75,12 @@ pub fn to_error_response(err: &std::io::Error) -> ErrorResponse {
} }
} }
pub fn http_resp_to_error_response(resp: http::Response<Body>, b: Vec<u8>, bucket_name: &str, object_name: &str) -> ErrorResponse { pub fn http_resp_to_error_response(
resp: http::Response<Body>,
b: Vec<u8>,
bucket_name: &str,
object_name: &str,
) -> ErrorResponse {
let err_body = String::from_utf8(b).unwrap(); let err_body = String::from_utf8(b).unwrap();
//let err_body = xml_decode_and_body(resp.body, &err_resp); //let err_body = xml_decode_and_body(resp.body, &err_resp);
let err_resp_ = serde_xml_rs::from_str::<ErrorResponse>(&err_body); let err_resp_ = serde_xml_rs::from_str::<ErrorResponse>(&err_body);
@@ -87,18 +91,18 @@ pub fn http_resp_to_error_response(resp: http::Response<Body>, b: Vec<u8>, bucke
if object_name == "" { if object_name == "" {
err_resp = ErrorResponse { err_resp = ErrorResponse {
status_code: resp.status(), status_code: resp.status(),
code: S3ErrorCode::NoSuchBucket, code: S3ErrorCode::NoSuchBucket,
message: "The specified bucket does not exist.".to_string(), message: "The specified bucket does not exist.".to_string(),
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
..Default::default() ..Default::default()
}; };
} else { } else {
err_resp = ErrorResponse { err_resp = ErrorResponse {
status_code: resp.status(), status_code: resp.status(),
code: S3ErrorCode::NoSuchKey, code: S3ErrorCode::NoSuchKey,
message: "The specified key does not exist.".to_string(), message: "The specified key does not exist.".to_string(),
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
key: object_name.to_string(), key: object_name.to_string(),
..Default::default() ..Default::default()
}; };
} }
@@ -106,18 +110,18 @@ pub fn http_resp_to_error_response(resp: http::Response<Body>, b: Vec<u8>, bucke
StatusCode::FORBIDDEN => { StatusCode::FORBIDDEN => {
err_resp = ErrorResponse { err_resp = ErrorResponse {
status_code: resp.status(), status_code: resp.status(),
code: S3ErrorCode::AccessDenied, code: S3ErrorCode::AccessDenied,
message: "Access Denied.".to_string(), message: "Access Denied.".to_string(),
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
key: object_name.to_string(), key: object_name.to_string(),
..Default::default() ..Default::default()
}; };
} }
StatusCode::CONFLICT => { StatusCode::CONFLICT => {
err_resp = ErrorResponse { err_resp = ErrorResponse {
status_code: resp.status(), status_code: resp.status(),
code: S3ErrorCode::BucketNotEmpty, code: S3ErrorCode::BucketNotEmpty,
message: "Bucket not empty.".to_string(), message: "Bucket not empty.".to_string(),
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
..Default::default() ..Default::default()
}; };
@@ -125,10 +129,10 @@ pub fn http_resp_to_error_response(resp: http::Response<Body>, b: Vec<u8>, bucke
StatusCode::PRECONDITION_FAILED => { StatusCode::PRECONDITION_FAILED => {
err_resp = ErrorResponse { err_resp = ErrorResponse {
status_code: resp.status(), status_code: resp.status(),
code: S3ErrorCode::PreconditionFailed, code: S3ErrorCode::PreconditionFailed,
message: "Pre condition failed.".to_string(), message: "Pre condition failed.".to_string(),
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
key: object_name.to_string(), key: object_name.to_string(),
..Default::default() ..Default::default()
}; };
} }
@@ -137,10 +141,10 @@ pub fn http_resp_to_error_response(resp: http::Response<Body>, b: Vec<u8>, bucke
if err_body.len() > 0 { if err_body.len() > 0 {
msg = err_body; msg = err_body;
} }
err_resp = ErrorResponse{ err_resp = ErrorResponse {
status_code: resp.status(), status_code: resp.status(),
code: S3ErrorCode::Custom(resp.status().to_string().into()), code: S3ErrorCode::Custom(resp.status().to_string().into()),
message: msg, message: msg,
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
..Default::default() ..Default::default()
}; };
@@ -188,45 +192,55 @@ pub fn http_resp_to_error_response(resp: http::Response<Body>, b: Vec<u8>, bucke
pub fn err_transfer_acceleration_bucket(bucket_name: &str) -> ErrorResponse { pub fn err_transfer_acceleration_bucket(bucket_name: &str) -> ErrorResponse {
ErrorResponse { ErrorResponse {
status_code: StatusCode::BAD_REQUEST, status_code: StatusCode::BAD_REQUEST,
code: S3ErrorCode::InvalidArgument, code: S3ErrorCode::InvalidArgument,
message: "The name of the bucket used for Transfer Acceleration must be DNS-compliant and must not contain periods ‘.’.".to_string(), message: "The name of the bucket used for Transfer Acceleration must be DNS-compliant and must not contain periods ‘.’."
.to_string(),
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
..Default::default() ..Default::default()
} }
} }
pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse { pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
let msg = format!("Your proposed upload size ‘{}’ exceeds the maximum allowed object size ‘{}’ for single PUT operation.", total_size, max_object_size); let msg = format!(
"Your proposed upload size ‘{}’ exceeds the maximum allowed object size ‘{}’ for single PUT operation.",
total_size, max_object_size
);
ErrorResponse { ErrorResponse {
status_code: StatusCode::BAD_REQUEST, status_code: StatusCode::BAD_REQUEST,
code: S3ErrorCode::EntityTooLarge, code: S3ErrorCode::EntityTooLarge,
message: msg, message: msg,
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
key: object_name.to_string(), key: object_name.to_string(),
..Default::default() ..Default::default()
} }
} }
pub fn err_entity_too_small(total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse { pub fn err_entity_too_small(total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
let msg = format!("Your proposed upload size ‘{}’ is below the minimum allowed object size ‘0B’ for single PUT operation.", total_size); let msg = format!(
"Your proposed upload size ‘{}’ is below the minimum allowed object size ‘0B’ for single PUT operation.",
total_size
);
ErrorResponse { ErrorResponse {
status_code: StatusCode::BAD_REQUEST, status_code: StatusCode::BAD_REQUEST,
code: S3ErrorCode::EntityTooSmall, code: S3ErrorCode::EntityTooSmall,
message: msg, message: msg,
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
key: object_name.to_string(), key: object_name.to_string(),
..Default::default() ..Default::default()
} }
} }
pub fn err_unexpected_eof(total_read: i64, total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse { pub fn err_unexpected_eof(total_read: i64, total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
let msg = format!("Data read ‘{}’ is not equal to the size ‘{}’ of the input Reader.", total_read, total_size); let msg = format!(
"Data read ‘{}’ is not equal to the size ‘{}’ of the input Reader.",
total_read, total_size
);
ErrorResponse { ErrorResponse {
status_code: StatusCode::BAD_REQUEST, status_code: StatusCode::BAD_REQUEST,
code: S3ErrorCode::Custom("UnexpectedEOF".into()), code: S3ErrorCode::Custom("UnexpectedEOF".into()),
message: msg, message: msg,
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
key: object_name.to_string(), key: object_name.to_string(),
..Default::default() ..Default::default()
} }
} }
@@ -234,9 +248,9 @@ pub fn err_unexpected_eof(total_read: i64, total_size: i64, bucket_name: &str, o
pub fn err_invalid_argument(message: &str) -> ErrorResponse { pub fn err_invalid_argument(message: &str) -> ErrorResponse {
ErrorResponse { ErrorResponse {
status_code: StatusCode::BAD_REQUEST, status_code: StatusCode::BAD_REQUEST,
code: S3ErrorCode::InvalidArgument, code: S3ErrorCode::InvalidArgument,
message: message.to_string(), message: message.to_string(),
request_id: "rustfs".to_string(), request_id: "rustfs".to_string(),
..Default::default() ..Default::default()
} }
} }
@@ -244,9 +258,9 @@ pub fn err_invalid_argument(message: &str) -> ErrorResponse {
pub fn err_api_not_supported(message: &str) -> ErrorResponse { pub fn err_api_not_supported(message: &str) -> ErrorResponse {
ErrorResponse { ErrorResponse {
status_code: StatusCode::NOT_IMPLEMENTED, status_code: StatusCode::NOT_IMPLEMENTED,
code: S3ErrorCode::Custom("APINotSupported".into()), code: S3ErrorCode::Custom("APINotSupported".into()),
message: message.to_string(), message: message.to_string(),
request_id: "rustfs".to_string(), request_id: "rustfs".to_string(),
..Default::default() ..Default::default()
} }
} }
ecstore/src/client/api_get_object.rs
+47 -39
View File
@@ -1,13 +1,13 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use bytes::Bytes; use bytes::Bytes;
use http::HeaderMap; use http::HeaderMap;
use tokio::io::BufReader;
use std::io::Cursor; use std::io::Cursor;
use tokio::io::BufReader;
use crate::client::{ use crate::client::{
transition_api::{ObjectInfo, to_object_info, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient},
api_error_response::err_invalid_argument, api_error_response::err_invalid_argument,
api_get_options::GetObjectOptions, api_get_options::GetObjectOptions,
transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info},
}; };
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
@@ -16,24 +16,34 @@ impl TransitionClient {
todo!(); todo!();
} }
pub async fn get_object_inner(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> { pub async fn get_object_inner(
let resp = self.execute_method(http::Method::GET, &mut RequestMetadata { &self,
bucket_name: bucket_name.to_string(), bucket_name: &str,
object_name: object_name.to_string(), object_name: &str,
query_values: opts.to_query_values(), opts: &GetObjectOptions,
custom_header: opts.header(), ) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> {
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), let resp = self
content_body: ReaderImpl::Body(Bytes::new()), .execute_method(
content_length: 0, http::Method::GET,
content_md5_base64: "".to_string(), &mut RequestMetadata {
stream_sha256: false, bucket_name: bucket_name.to_string(),
trailer: HeaderMap::new(), object_name: object_name.to_string(),
pre_sign_url: Default::default(), query_values: opts.to_query_values(),
add_crc: Default::default(), custom_header: opts.header(),
extra_pre_sign_header: Default::default(), content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
bucket_location: Default::default(), content_body: ReaderImpl::Body(Bytes::new()),
expires: Default::default(), content_length: 0,
}).await?; content_md5_base64: "".to_string(),
stream_sha256: false,
trailer: HeaderMap::new(),
pre_sign_url: Default::default(),
add_crc: Default::default(),
extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
},
)
.await?;
let resp = &resp; let resp = &resp;
let object_stat = to_object_info(bucket_name, object_name, resp.headers())?; let object_stat = to_object_info(bucket_name, object_name, resp.headers())?;
@@ -45,20 +55,20 @@ impl TransitionClient {
#[derive(Default)] #[derive(Default)]
struct GetRequest { struct GetRequest {
pub buffer: Vec<u8>, pub buffer: Vec<u8>,
pub offset: i64, pub offset: i64,
pub did_offset_change: bool, pub did_offset_change: bool,
pub been_read: bool, pub been_read: bool,
pub is_read_at: bool, pub is_read_at: bool,
pub is_read_op: bool, pub is_read_op: bool,
pub is_first_req: bool, pub is_first_req: bool,
pub setting_object_info: bool, pub setting_object_info: bool,
} }
struct GetResponse { struct GetResponse {
pub size: i64, pub size: i64,
//pub error: error, //pub error: error,
pub did_read: bool, pub did_read: bool,
pub object_info: ObjectInfo, pub object_info: ObjectInfo,
} }
@@ -79,9 +89,7 @@ struct Object {
impl Object { impl Object {
pub fn new() -> Object { pub fn new() -> Object {
Self { Self { ..Default::default() }
..Default::default()
}
} }
fn do_get_request(&self, request: &GetRequest) -> Result<GetResponse, std::io::Error> { fn do_get_request(&self, request: &GetRequest) -> Result<GetResponse, std::io::Error> {
@@ -121,7 +129,7 @@ impl Object {
fn stat(&self) -> Result<ObjectInfo, std::io::Error> { fn stat(&self) -> Result<ObjectInfo, std::io::Error> {
if !self.is_started || !self.object_info_set { if !self.is_started || !self.object_info_set {
let _ = self.do_get_request(&GetRequest { let _ = self.do_get_request(&GetRequest {
is_first_req: !self.is_started, is_first_req: !self.is_started,
setting_object_info: !self.object_info_set, setting_object_info: !self.object_info_set,
..Default::default() ..Default::default()
})?; })?;
@@ -134,12 +142,12 @@ impl Object {
self.curr_offset = offset; self.curr_offset = offset;
let mut read_at_req = GetRequest { let mut read_at_req = GetRequest {
is_read_op: true, is_read_op: true,
is_read_at: true, is_read_at: true,
did_offset_change: true, did_offset_change: true,
been_read: self.been_read, been_read: self.been_read,
offset, offset,
buffer: b.to_vec(), buffer: b.to_vec(),
..Default::default() ..Default::default()
}; };
@@ -160,8 +168,8 @@ impl Object {
fn seek(&mut self, offset: i64, whence: i64) -> Result<i64, std::io::Error> { fn seek(&mut self, offset: i64, whence: i64) -> Result<i64, std::io::Error> {
if !self.is_started || !self.object_info_set { if !self.is_started || !self.object_info_set {
let seek_req = GetRequest { let seek_req = GetRequest {
is_read_op: false, is_read_op: false,
offset: offset, offset: offset,
is_first_req: true, is_first_req: true,
..Default::default() ..Default::default()
}; };
ecstore/src/client/api_get_options.rs
+11 -11
View File
@@ -1,6 +1,6 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use std::collections::HashMap;
use http::{HeaderMap, HeaderName, HeaderValue}; use http::{HeaderMap, HeaderName, HeaderValue};
use std::collections::HashMap;
use time::OffsetDateTime; use time::OffsetDateTime;
use tracing::warn; use tracing::warn;
@@ -8,9 +8,9 @@ use crate::client::api_error_response::err_invalid_argument;
#[derive(Default)] #[derive(Default)]
pub struct AdvancedGetOptions { pub struct AdvancedGetOptions {
replication_deletemarker: bool, replication_deletemarker: bool,
is_replication_ready_for_deletemarker: bool, is_replication_ready_for_deletemarker: bool,
replication_proxy_request: String, replication_proxy_request: String,
} }
pub struct GetObjectOptions { pub struct GetObjectOptions {
@@ -50,7 +50,7 @@ impl GetObjectOptions {
} }
} }
if self.checksum { if self.checksum {
headers.insert("x-amz-checksum-mode", "ENABLED".parse().expect("err")); headers.insert("x-amz-checksum-mode", "ENABLED".parse().expect("err"));
} }
headers headers
} }
@@ -96,15 +96,15 @@ impl GetObjectOptions {
pub fn set_range(&mut self, start: i64, end: i64) -> Result<(), std::io::Error> { pub fn set_range(&mut self, start: i64, end: i64) -> Result<(), std::io::Error> {
if start == 0 && end < 0 { if start == 0 && end < 0 {
self.set("Range", &format!("bytes={}", end)); self.set("Range", &format!("bytes={}", end));
} } else if 0 < start && end == 0 {
else if 0 < start && end == 0 {
self.set("Range", &format!("bytes={}-", start)); self.set("Range", &format!("bytes={}-", start));
} } else if 0 <= start && start <= end {
else if 0 <= start && start <= end {
self.set("Range", &format!("bytes={}-{}", start, end)); self.set("Range", &format!("bytes={}-{}", start, end));
} } else {
else { return Err(std::io::Error::other(err_invalid_argument(&format!(
return Err(std::io::Error::other(err_invalid_argument(&format!("Invalid range specified: start={} end={}", start, end)))); "Invalid range specified: start={} end={}",
start, end
))));
} }
Ok(()) Ok(())
} }
ecstore/src/client/api_list.rs
+86 -33
View File
@@ -1,23 +1,36 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use std::collections::HashMap;
use bytes::Bytes; use bytes::Bytes;
use http::{HeaderMap, StatusCode}; use http::{HeaderMap, StatusCode};
use std::collections::HashMap;
use crate::client::{ use crate::client::{
api_error_response::http_resp_to_error_response, api_error_response::http_resp_to_error_response,
api_s3_datatypes::{
ListBucketResult, ListBucketV2Result, ListMultipartUploadsResult, ListObjectPartsResult, ListVersionsResult, ObjectPart,
},
credentials, credentials,
api_s3_datatypes::{ListBucketV2Result, ListMultipartUploadsResult, ListBucketResult, ListObjectPartsResult, ListVersionsResult, ObjectPart}, transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
transition_api::{ReaderImpl, TransitionClient, RequestMetadata,},
}; };
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
use crate::store_api::BucketInfo; use crate::store_api::BucketInfo;
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
impl TransitionClient { impl TransitionClient {
pub fn list_buckets(&self) -> Result<Vec<BucketInfo>, std::io::Error> { pub fn list_buckets(&self) -> Result<Vec<BucketInfo>, std::io::Error> {
todo!(); todo!();
} }
pub async fn list_objects_v2_query(&self, bucket_name: &str, object_prefix: &str, continuation_token: &str, fetch_owner: bool, metadata: bool, delimiter: &str, start_after: &str, max_keys: i64, headers: HeaderMap) -> Result<ListBucketV2Result, std::io::Error> { pub async fn list_objects_v2_query(
&self,
bucket_name: &str,
object_prefix: &str,
continuation_token: &str,
fetch_owner: bool,
metadata: bool,
delimiter: &str,
start_after: &str,
max_keys: i64,
headers: HeaderMap,
) -> Result<ListBucketV2Result, std::io::Error> {
let mut url_values = HashMap::new(); let mut url_values = HashMap::new();
url_values.insert("list-type".to_string(), "2".to_string()); url_values.insert("list-type".to_string(), "2".to_string());
@@ -43,23 +56,28 @@ impl TransitionClient {
url_values.insert("max-keys".to_string(), max_keys.to_string()); url_values.insert("max-keys".to_string(), max_keys.to_string());
} }
let mut resp = self.execute_method(http::Method::GET, &mut RequestMetadata { let mut resp = self
bucket_name: bucket_name.to_string(), .execute_method(
object_name: "".to_string(), http::Method::GET,
query_values: url_values, &mut RequestMetadata {
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), bucket_name: bucket_name.to_string(),
custom_header: headers, object_name: "".to_string(),
content_body: ReaderImpl::Body(Bytes::new()), query_values: url_values,
content_length: 0, content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
content_md5_base64: "".to_string(), custom_header: headers,
stream_sha256: false, content_body: ReaderImpl::Body(Bytes::new()),
trailer: HeaderMap::new(), content_length: 0,
pre_sign_url: Default::default(), content_md5_base64: "".to_string(),
add_crc: Default::default(), stream_sha256: false,
extra_pre_sign_header: Default::default(), trailer: HeaderMap::new(),
bucket_location: Default::default(), pre_sign_url: Default::default(),
expires: Default::default(), add_crc: Default::default(),
}).await?; extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
},
)
.await?;
if resp.status() != StatusCode::OK { if resp.status() != StatusCode::OK {
return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, ""))); return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, "")));
} }
@@ -76,12 +94,12 @@ impl TransitionClient {
if list_bucket_result.is_truncated && list_bucket_result.next_continuation_token == "" { if list_bucket_result.is_truncated && list_bucket_result.next_continuation_token == "" {
return Err(std::io::Error::other(credentials::ErrorResponse { return Err(std::io::Error::other(credentials::ErrorResponse {
sts_error: credentials::STSError { sts_error: credentials::STSError {
r#type: "".to_string(), r#type: "".to_string(),
code: "NotImplemented".to_string(), code: "NotImplemented".to_string(),
message: "Truncated response should have continuation token set".to_string(), message: "Truncated response should have continuation token set".to_string(),
}, },
request_id: "".to_string(), request_id: "".to_string(),
})); }));
} }
@@ -97,7 +115,14 @@ impl TransitionClient {
Ok(list_bucket_result) Ok(list_bucket_result)
} }
pub fn list_object_versions_query(&self, bucket_name: &str, opts: &ListObjectsOptions, key_marker: &str, version_id_marker: &str, delimiter: &str) -> Result<ListVersionsResult, std::io::Error> { pub fn list_object_versions_query(
&self,
bucket_name: &str,
opts: &ListObjectsOptions,
key_marker: &str,
version_id_marker: &str,
delimiter: &str,
) -> Result<ListVersionsResult, std::io::Error> {
/*if err := s3utils.CheckValidBucketName(bucketName); err != nil { /*if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return ListVersionsResult{}, err return ListVersionsResult{}, err
} }
@@ -177,15 +202,36 @@ impl TransitionClient {
todo!(); todo!();
} }
pub fn list_objects_query(&self, bucket_name: &str, object_prefix: &str, object_marker: &str, delimiter: &str, max_keys: i64, headers: HeaderMap) -> Result<ListBucketResult, std::io::Error> { pub fn list_objects_query(
&self,
bucket_name: &str,
object_prefix: &str,
object_marker: &str,
delimiter: &str,
max_keys: i64,
headers: HeaderMap,
) -> Result<ListBucketResult, std::io::Error> {
todo!(); todo!();
} }
pub fn list_multipart_uploads_query(&self, bucket_name: &str, key_marker: &str, upload_id_marker: &str, prefix: &str, delimiter: &str, max_uploads: i64) -> Result<ListMultipartUploadsResult, std::io::Error> { pub fn list_multipart_uploads_query(
&self,
bucket_name: &str,
key_marker: &str,
upload_id_marker: &str,
prefix: &str,
delimiter: &str,
max_uploads: i64,
) -> Result<ListMultipartUploadsResult, std::io::Error> {
todo!(); todo!();
} }
pub fn list_object_parts(&self, bucket_name: &str, object_name: &str, upload_id: &str) -> Result<HashMap<i64, ObjectPart>, std::io::Error> { pub fn list_object_parts(
&self,
bucket_name: &str,
object_name: &str,
upload_id: &str,
) -> Result<HashMap<i64, ObjectPart>, std::io::Error> {
todo!(); todo!();
} }
@@ -193,7 +239,14 @@ impl TransitionClient {
todo!(); todo!();
} }
pub async fn list_object_parts_query(&self, bucket_name: &str, object_name: &str, upload_id: &str, part_number_marker: i64, max_parts: i64) -> Result<ListObjectPartsResult, std::io::Error> { pub async fn list_object_parts_query(
&self,
bucket_name: &str,
object_name: &str,
upload_id: &str,
part_number_marker: i64,
max_parts: i64,
) -> Result<ListObjectPartsResult, std::io::Error> {
todo!(); todo!();
} }
} }
ecstore/src/client/api_put_object.rs
+97 -56
View File
@@ -1,33 +1,30 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use std::{collections::HashMap, sync::Arc};
use bytes::Bytes; use bytes::Bytes;
use time::{OffsetDateTime, macros::format_description, Duration};
use http::{HeaderMap, HeaderName, HeaderValue}; use http::{HeaderMap, HeaderName, HeaderValue};
use std::{collections::HashMap, sync::Arc};
use time::{Duration, OffsetDateTime, macros::format_description};
use tracing::{error, info, warn}; use tracing::{error, info, warn};
use s3s::dto::{
ObjectLockRetentionMode, ObjectLockLegalHoldStatus,
ReplicationStatus,
};
use s3s::header::{
X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_OBJECT_LOCK_LEGAL_HOLD,
X_AMZ_STORAGE_CLASS, X_AMZ_WEBSITE_REDIRECT_LOCATION, X_AMZ_REPLICATION_STATUS,
};
use reader::hasher::Hasher; use reader::hasher::Hasher;
//use crate::disk::{BufferReader, Reader}; use s3s::dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus};
use rustfs_utils::{ use s3s::header::{
crypto::base64_encode, X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_REPLICATION_STATUS,
net::{is_amz_header, is_standard_header, is_storageclass_header, is_rustfs_header, is_minio_header}, X_AMZ_STORAGE_CLASS, X_AMZ_WEBSITE_REDIRECT_LOCATION,
}; };
//use crate::disk::{BufferReader, Reader};
use crate::checksum::ChecksumMode; use crate::checksum::ChecksumMode;
use crate::client::{ use crate::client::{
api_s3_datatypes::{CompletePart, ObjectPart, CompleteMultipartUpload}, api_error_response::{err_entity_too_large, err_invalid_argument},
api_put_object_common::optimal_part_info, api_put_object_common::optimal_part_info,
transition_api::{TransitionClient, UploadInfo, ReaderImpl},
api_error_response::{err_invalid_argument, err_entity_too_large},
api_put_object_multipart::UploadPartParams, api_put_object_multipart::UploadPartParams,
api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart},
constants::{ISO8601_DATEFORMAT, MAX_MULTIPART_PUT_OBJECT_SIZE, MIN_PART_SIZE, TOTAL_WORKERS},
credentials::SignatureType, credentials::SignatureType,
constants::{MAX_MULTIPART_PUT_OBJECT_SIZE, TOTAL_WORKERS, MIN_PART_SIZE, ISO8601_DATEFORMAT,}, transition_api::{ReaderImpl, TransitionClient, UploadInfo},
};
use rustfs_utils::{
crypto::base64_encode,
net::{is_amz_header, is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header},
}; };
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
@@ -123,17 +120,21 @@ impl Default for PutObjectOptions {
impl PutObjectOptions { impl PutObjectOptions {
fn set_matche_tag(&mut self, etag: &str) { fn set_matche_tag(&mut self, etag: &str) {
if etag == "*" { if etag == "*" {
self.custom_header.insert("If-Match", HeaderValue::from_str("*").expect("err")); self.custom_header
.insert("If-Match", HeaderValue::from_str("*").expect("err"));
} else { } else {
self.custom_header.insert("If-Match", HeaderValue::from_str(&format!("\"{}\"", etag)).expect("err")); self.custom_header
.insert("If-Match", HeaderValue::from_str(&format!("\"{}\"", etag)).expect("err"));
} }
} }
fn set_matche_tag_except(&mut self, etag: &str) { fn set_matche_tag_except(&mut self, etag: &str) {
if etag == "*" { if etag == "*" {
self.custom_header.insert("If-None-Match", HeaderValue::from_str("*").expect("err")); self.custom_header
.insert("If-None-Match", HeaderValue::from_str("*").expect("err"));
} else { } else {
self.custom_header.insert("If-None-Match", HeaderValue::from_str(&format!("\"{etag}\"")).expect("err")); self.custom_header
.insert("If-None-Match", HeaderValue::from_str(&format!("\"{etag}\"")).expect("err"));
} }
} }
@@ -160,7 +161,10 @@ impl PutObjectOptions {
} }
if self.expires.unix_timestamp() != 0 { if self.expires.unix_timestamp() != 0 {
header.insert("Expires", HeaderValue::from_str(&self.expires.format(ISO8601_DATEFORMAT).unwrap()).expect("err")); //rustfs invalid heade header.insert(
"Expires",
HeaderValue::from_str(&self.expires.format(ISO8601_DATEFORMAT).unwrap()).expect("err"),
); //rustfs invalid heade
} }
if self.mode.as_str() != "" { if self.mode.as_str() != "" {
@@ -168,7 +172,10 @@ impl PutObjectOptions {
} }
if self.retain_until_date.unix_timestamp() != 0 { if self.retain_until_date.unix_timestamp() != 0 {
header.insert(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, HeaderValue::from_str(&self.retain_until_date.format(ISO8601_DATEFORMAT).unwrap()).expect("err")); header.insert(
X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE,
HeaderValue::from_str(&self.retain_until_date.format(ISO8601_DATEFORMAT).unwrap()).expect("err"),
);
} }
if self.legalhold.as_str() != "" { if self.legalhold.as_str() != "" {
@@ -180,15 +187,22 @@ impl PutObjectOptions {
} }
if self.website_redirect_location != "" { if self.website_redirect_location != "" {
header.insert(X_AMZ_WEBSITE_REDIRECT_LOCATION, HeaderValue::from_str(&self.website_redirect_location).expect("err")); header.insert(
X_AMZ_WEBSITE_REDIRECT_LOCATION,
HeaderValue::from_str(&self.website_redirect_location).expect("err"),
);
} }
if !self.internal.replication_status.as_str().is_empty() { if !self.internal.replication_status.as_str().is_empty() {
header.insert(X_AMZ_REPLICATION_STATUS, HeaderValue::from_str(self.internal.replication_status.as_str()).expect("err")); header.insert(
X_AMZ_REPLICATION_STATUS,
HeaderValue::from_str(self.internal.replication_status.as_str()).expect("err"),
);
} }
for (k, v) in &self.user_metadata { for (k, v) in &self.user_metadata {
if is_amz_header(k) || is_standard_header(k) || is_storageclass_header(k) || is_rustfs_header(k) || is_minio_header(k) { if is_amz_header(k) || is_standard_header(k) || is_storageclass_header(k) || is_rustfs_header(k) || is_minio_header(k)
{
if let Ok(header_name) = HeaderName::from_bytes(k.as_bytes()) { if let Ok(header_name) = HeaderName::from_bytes(k.as_bytes()) {
header.insert(header_name, HeaderValue::from_str(&v).unwrap()); header.insert(header_name, HeaderValue::from_str(&v).unwrap());
} }
@@ -208,12 +222,12 @@ impl PutObjectOptions {
fn validate(&self, c: TransitionClient) -> Result<(), std::io::Error> { fn validate(&self, c: TransitionClient) -> Result<(), std::io::Error> {
//if self.checksum.is_set() { //if self.checksum.is_set() {
/*if !self.trailing_header_support { /*if !self.trailing_header_support {
return Err(Error::from(err_invalid_argument("Checksum requires Client with TrailingHeaders enabled"))); return Err(Error::from(err_invalid_argument("Checksum requires Client with TrailingHeaders enabled")));
}*/ }*/
/*else if self.override_signer_type == SignatureType::SignatureV2 { /*else if self.override_signer_type == SignatureType::SignatureV2 {
return Err(Error::from(err_invalid_argument("Checksum cannot be used with v2 signatures"))); return Err(Error::from(err_invalid_argument("Checksum cannot be used with v2 signatures")));
}*/ }*/
//} //}
Ok(()) Ok(())
@@ -221,19 +235,37 @@ impl PutObjectOptions {
} }
impl TransitionClient { impl TransitionClient {
pub async fn put_object(self: Arc<Self>, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, object_size: i64, pub async fn put_object(
opts: &PutObjectOptions self: Arc<Self>,
bucket_name: &str,
object_name: &str,
mut reader: ReaderImpl,
object_size: i64,
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> { ) -> Result<UploadInfo, std::io::Error> {
if object_size < 0 && opts.disable_multipart { if object_size < 0 && opts.disable_multipart {
return Err(std::io::Error::other("object size must be provided with disable multipart upload")); return Err(std::io::Error::other("object size must be provided with disable multipart upload"));
} }
self.put_object_common(bucket_name, object_name, reader, object_size, opts).await self.put_object_common(bucket_name, object_name, reader, object_size, opts)
.await
} }
pub async fn put_object_common(self: Arc<Self>, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions) -> Result<UploadInfo, std::io::Error> { pub async fn put_object_common(
self: Arc<Self>,
bucket_name: &str,
object_name: &str,
mut reader: ReaderImpl,
size: i64,
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> {
if size > MAX_MULTIPART_PUT_OBJECT_SIZE { if size > MAX_MULTIPART_PUT_OBJECT_SIZE {
return Err(std::io::Error::other(err_entity_too_large(size, MAX_MULTIPART_PUT_OBJECT_SIZE, bucket_name, object_name))); return Err(std::io::Error::other(err_entity_too_large(
size,
MAX_MULTIPART_PUT_OBJECT_SIZE,
bucket_name,
object_name,
)));
} }
let mut opts = opts.clone(); let mut opts = opts.clone();
opts.auto_checksum.set_default(ChecksumMode::ChecksumCRC32C); opts.auto_checksum.set_default(ChecksumMode::ChecksumCRC32C);
@@ -255,19 +287,30 @@ impl TransitionClient {
return Err(std::io::Error::other("no length provided and multipart disabled")); return Err(std::io::Error::other("no length provided and multipart disabled"));
} }
if opts.concurrent_stream_parts && opts.num_threads > 1 { if opts.concurrent_stream_parts && opts.num_threads > 1 {
return self.put_object_multipart_stream_parallel(bucket_name, object_name, reader, &opts).await; return self
.put_object_multipart_stream_parallel(bucket_name, object_name, reader, &opts)
.await;
} }
return self.put_object_multipart_stream_no_length(bucket_name, object_name, reader, &opts).await; return self
.put_object_multipart_stream_no_length(bucket_name, object_name, reader, &opts)
.await;
} }
if size <= part_size || opts.disable_multipart { if size <= part_size || opts.disable_multipart {
return self.put_object_gcs(bucket_name, object_name, reader, size, &opts).await; return self.put_object_gcs(bucket_name, object_name, reader, size, &opts).await;
} }
self.put_object_multipart_stream(bucket_name, object_name, reader, size, &opts).await self.put_object_multipart_stream(bucket_name, object_name, reader, size, &opts)
.await
} }
pub async fn put_object_multipart_stream_no_length(&self, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, opts: &PutObjectOptions) -> Result<UploadInfo, std::io::Error> { pub async fn put_object_multipart_stream_no_length(
&self,
bucket_name: &str,
object_name: &str,
mut reader: ReaderImpl,
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> {
let mut total_uploaded_size: i64 = 0; let mut total_uploaded_size: i64 = 0;
let mut compl_multipart_upload = CompleteMultipartUpload::default(); let mut compl_multipart_upload = CompleteMultipartUpload::default();
@@ -295,12 +338,8 @@ impl TransitionClient {
while part_number <= total_parts_count { while part_number <= total_parts_count {
buf = match &mut reader { buf = match &mut reader {
ReaderImpl::Body(content_body) => { ReaderImpl::Body(content_body) => content_body.to_vec(),
content_body.to_vec() ReaderImpl::ObjectBody(content_body) => content_body.read_all().await?,
}
ReaderImpl::ObjectBody(content_body) => {
content_body.read_all().await?
}
}; };
let length = buf.len(); let length = buf.len();
@@ -354,13 +393,13 @@ impl TransitionClient {
let part = parts_info[&i].clone(); let part = parts_info[&i].clone();
all_parts.push(part.clone()); all_parts.push(part.clone());
compl_multipart_upload.parts.push(CompletePart { compl_multipart_upload.parts.push(CompletePart {
etag: part.etag, etag: part.etag,
part_num: part.part_num, part_num: part.part_num,
checksum_crc32: part.checksum_crc32, checksum_crc32: part.checksum_crc32,
checksum_crc32c: part.checksum_crc32c, checksum_crc32c: part.checksum_crc32c,
checksum_sha1: part.checksum_sha1, checksum_sha1: part.checksum_sha1,
checksum_sha256: part.checksum_sha256, checksum_sha256: part.checksum_sha256,
checksum_crc64nvme: part.checksum_crc64nvme, checksum_crc64nvme: part.checksum_crc64nvme,
..Default::default() ..Default::default()
}); });
} }
@@ -374,7 +413,9 @@ impl TransitionClient {
}; };
//apply_auto_checksum(&mut opts, all_parts); //apply_auto_checksum(&mut opts, all_parts);
let mut upload_info = self.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts).await?; let mut upload_info = self
.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts)
.await?;
upload_info.size = total_uploaded_size; upload_info.size = total_uploaded_size;
Ok(upload_info) Ok(upload_info)
ecstore/src/client/api_put_object_common.rs
+32 -11
View File
@@ -1,10 +1,10 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use crate::client::{ use crate::client::{
api_put_object::PutObjectOptions,
constants::{ABS_MIN_PART_SIZE, MAX_MULTIPART_PUT_OBJECT_SIZE, MAX_PARTS_COUNT, MAX_PART_SIZE, MIN_PART_SIZE},
transition_api::TransitionClient,
transition_api::ReaderImpl,
api_error_response::{err_entity_too_large, err_invalid_argument}, api_error_response::{err_entity_too_large, err_invalid_argument},
api_put_object::PutObjectOptions,
constants::{ABS_MIN_PART_SIZE, MAX_MULTIPART_PUT_OBJECT_SIZE, MAX_PART_SIZE, MAX_PARTS_COUNT, MIN_PART_SIZE},
transition_api::ReaderImpl,
transition_api::TransitionClient,
}; };
const NULL_VERSION_ID: &str = "null"; const NULL_VERSION_ID: &str = "null";
@@ -28,27 +28,43 @@ pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<
} }
if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE { if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
return Err(std::io::Error::other(err_entity_too_large(object_size, MAX_MULTIPART_PUT_OBJECT_SIZE, "", ""))); return Err(std::io::Error::other(err_entity_too_large(
object_size,
MAX_MULTIPART_PUT_OBJECT_SIZE,
"",
"",
)));
} }
let mut part_size_flt: f64; let mut part_size_flt: f64;
if configured_part_size > 0 { if configured_part_size > 0 {
if configured_part_size as i64 > object_size { if configured_part_size as i64 > object_size {
return Err(std::io::Error::other(err_entity_too_large(configured_part_size as i64, object_size, "", ""))); return Err(std::io::Error::other(err_entity_too_large(
configured_part_size as i64,
object_size,
"",
"",
)));
} }
if !unknown_size { if !unknown_size {
if object_size > (configured_part_size as i64 * MAX_PARTS_COUNT) { if object_size > (configured_part_size as i64 * MAX_PARTS_COUNT) {
return Err(std::io::Error::other(err_invalid_argument("Part size * max_parts(10000) is lesser than input objectSize."))); return Err(std::io::Error::other(err_invalid_argument(
"Part size * max_parts(10000) is lesser than input objectSize.",
)));
} }
} }
if (configured_part_size as i64) < ABS_MIN_PART_SIZE { if (configured_part_size as i64) < ABS_MIN_PART_SIZE {
return Err(std::io::Error::other(err_invalid_argument("Input part size is smaller than allowed minimum of 5MiB."))); return Err(std::io::Error::other(err_invalid_argument(
"Input part size is smaller than allowed minimum of 5MiB.",
)));
} }
if configured_part_size as i64 > MAX_PART_SIZE { if configured_part_size as i64 > MAX_PART_SIZE {
return Err(std::io::Error::other(err_invalid_argument("Input part size is bigger than allowed maximum of 5GiB."))); return Err(std::io::Error::other(err_invalid_argument(
"Input part size is bigger than allowed maximum of 5GiB.",
)));
} }
part_size_flt = configured_part_size as f64; part_size_flt = configured_part_size as f64;
@@ -64,12 +80,17 @@ pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<
let total_parts_count = (object_size as f64 / part_size_flt).ceil() as i64; let total_parts_count = (object_size as f64 / part_size_flt).ceil() as i64;
let part_size = part_size_flt.ceil() as i64; let part_size = part_size_flt.ceil() as i64;
let last_part_size = object_size - (total_parts_count-1) * part_size; let last_part_size = object_size - (total_parts_count - 1) * part_size;
Ok((total_parts_count, part_size, last_part_size)) Ok((total_parts_count, part_size, last_part_size))
} }
impl TransitionClient { impl TransitionClient {
pub async fn new_upload_id(&self ,bucket_name: &str, object_name: &str, opts: &PutObjectOptions) -> Result<String, std::io::Error> { pub async fn new_upload_id(
&self,
bucket_name: &str,
object_name: &str,
opts: &PutObjectOptions,
) -> Result<String, std::io::Error> {
let init_multipart_upload_result = self.initiate_multipart_upload(bucket_name, object_name, opts).await?; let init_multipart_upload_result = self.initiate_multipart_upload(bucket_name, object_name, opts).await?;
Ok(init_multipart_upload_result.upload_id) Ok(init_multipart_upload_result.upload_id)
} }
ecstore/src/client/api_put_object_multipart.rs
+157 -92
View File
@@ -1,50 +1,61 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use bytes::Bytes;
use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
use s3s::S3ErrorCode;
use std::io::Read; use std::io::Read;
use std::{collections::HashMap, sync::Arc}; use std::{collections::HashMap, sync::Arc};
use bytes::Bytes; use time::{OffsetDateTime, format_description};
use s3s::S3ErrorCode;
use time::{format_description, OffsetDateTime};
use uuid::Uuid;
use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
use url::form_urlencoded::Serializer;
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
use tracing::warn; use tracing::warn;
use tracing::{error, info}; use tracing::{error, info};
use url::form_urlencoded::Serializer;
use uuid::Uuid;
use s3s::{dto::StreamingBlob, Body};
use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};
use reader::hasher::Hasher; use reader::hasher::Hasher;
use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};
use s3s::{Body, dto::StreamingBlob};
//use crate::disk::{Reader, BufferReader}; //use crate::disk::{Reader, BufferReader};
use crate::client::{ use crate::client::{
transition_api::{RequestMetadata, TransitionClient, UploadInfo, ReaderImpl,}, api_error_response::{
api_error_response::{err_entity_too_large, err_entity_too_small, err_invalid_argument, http_resp_to_error_response, to_error_response}, err_entity_too_large, err_entity_too_small, err_invalid_argument, http_resp_to_error_response, to_error_response,
},
api_put_object::PutObjectOptions, api_put_object::PutObjectOptions,
api_put_object_common::optimal_part_info, api_put_object_common::optimal_part_info,
api_s3_datatypes::{CompleteMultipartUpload, CompleteMultipartUploadResult, CompletePart, InitiateMultipartUploadResult, ObjectPart}, api_s3_datatypes::{
constants::{ABS_MIN_PART_SIZE, MAX_PART_SIZE, MAX_SINGLE_PUT_OBJECT_SIZE, ISO8601_DATEFORMAT, }, CompleteMultipartUpload, CompleteMultipartUploadResult, CompletePart, InitiateMultipartUploadResult, ObjectPart,
}; },
use rustfs_utils::{ constants::{ABS_MIN_PART_SIZE, ISO8601_DATEFORMAT, MAX_PART_SIZE, MAX_SINGLE_PUT_OBJECT_SIZE},
path::trim_etag, transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo},
crypto::base64_encode,
}; };
use crate::{ use crate::{
disk::DiskAPI,
store_api::{
GetObjectReader, StorageAPI,
},
checksum::ChecksumMode, checksum::ChecksumMode,
disk::DiskAPI,
store_api::{GetObjectReader, StorageAPI},
}; };
use rustfs_utils::{crypto::base64_encode, path::trim_etag};
impl TransitionClient { impl TransitionClient {
pub async fn put_object_multipart(&self, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, size: i64, pub async fn put_object_multipart(
opts: &PutObjectOptions &self,
bucket_name: &str,
object_name: &str,
mut reader: ReaderImpl,
size: i64,
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> { ) -> Result<UploadInfo, std::io::Error> {
let info = self.put_object_multipart_no_stream(bucket_name, object_name, &mut reader, opts).await; let info = self
.put_object_multipart_no_stream(bucket_name, object_name, &mut reader, opts)
.await;
if let Err(err) = &info { if let Err(err) = &info {
let err_resp = to_error_response(err); let err_resp = to_error_response(err);
if err_resp.code == S3ErrorCode::AccessDenied && err_resp.message.contains("Access Denied") { if err_resp.code == S3ErrorCode::AccessDenied && err_resp.message.contains("Access Denied") {
if size > MAX_SINGLE_PUT_OBJECT_SIZE { if size > MAX_SINGLE_PUT_OBJECT_SIZE {
return Err(std::io::Error::other(err_entity_too_large(size, MAX_SINGLE_PUT_OBJECT_SIZE, bucket_name, object_name))); return Err(std::io::Error::other(err_entity_too_large(
size,
MAX_SINGLE_PUT_OBJECT_SIZE,
bucket_name,
object_name,
)));
} }
return self.put_object_gcs(bucket_name, object_name, reader, size, opts).await; return self.put_object_gcs(bucket_name, object_name, reader, size, opts).await;
} }
@@ -52,7 +63,13 @@ impl TransitionClient {
Ok(info?) Ok(info?)
} }
pub async fn put_object_multipart_no_stream(&self, bucket_name: &str, object_name: &str, reader: &mut ReaderImpl, opts: &PutObjectOptions) -> Result<UploadInfo, std::io::Error> { pub async fn put_object_multipart_no_stream(
&self,
bucket_name: &str,
object_name: &str,
reader: &mut ReaderImpl,
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> {
let mut total_uploaded_size: i64 = 0; let mut total_uploaded_size: i64 = 0;
let mut compl_multipart_upload = CompleteMultipartUpload::default(); let mut compl_multipart_upload = CompleteMultipartUpload::default();
@@ -91,10 +108,10 @@ impl TransitionClient {
let mut sha256_hex: String; let mut sha256_hex: String;
//if hash_sums["md5"] != nil { //if hash_sums["md5"] != nil {
md5_base64 = base64_encode(&hash_sums["md5"]); md5_base64 = base64_encode(&hash_sums["md5"]);
//} //}
//if hash_sums["sha256"] != nil { //if hash_sums["sha256"] != nil {
sha256_hex = hex_simd::encode_to_string(hash_sums["sha256"].clone(), hex_simd::AsciiCase::Lower); sha256_hex = hex_simd::encode_to_string(hash_sums["sha256"].clone(), hex_simd::AsciiCase::Lower);
//} //}
if hash_sums.len() == 0 { if hash_sums.len() == 0 {
let csum; let csum;
@@ -137,12 +154,12 @@ impl TransitionClient {
let part = parts_info[&i].clone(); let part = parts_info[&i].clone();
all_parts.push(part.clone()); all_parts.push(part.clone());
compl_multipart_upload.parts.push(CompletePart { compl_multipart_upload.parts.push(CompletePart {
etag: part.etag, etag: part.etag,
part_num: part.part_num, part_num: part.part_num,
checksum_crc32: part.checksum_crc32, checksum_crc32: part.checksum_crc32,
checksum_crc32c: part.checksum_crc32c, checksum_crc32c: part.checksum_crc32c,
checksum_sha1: part.checksum_sha1, checksum_sha1: part.checksum_sha1,
checksum_sha256: part.checksum_sha256, checksum_sha256: part.checksum_sha256,
checksum_crc64nvme: part.checksum_crc64nvme, checksum_crc64nvme: part.checksum_crc64nvme,
..Default::default() ..Default::default()
}); });
@@ -151,18 +168,25 @@ impl TransitionClient {
compl_multipart_upload.parts.sort(); compl_multipart_upload.parts.sort();
let mut opts = PutObjectOptions { let mut opts = PutObjectOptions {
//server_side_encryption: opts.server_side_encryption, //server_side_encryption: opts.server_side_encryption,
auto_checksum: opts.auto_checksum, auto_checksum: opts.auto_checksum,
..Default::default() ..Default::default()
}; };
//apply_auto_checksum(&mut opts, all_parts); //apply_auto_checksum(&mut opts, all_parts);
let mut upload_info = self.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts).await?; let mut upload_info = self
.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts)
.await?;
upload_info.size = total_uploaded_size; upload_info.size = total_uploaded_size;
Ok(upload_info) Ok(upload_info)
} }
pub async fn initiate_multipart_upload(&self, bucket_name: &str, object_name: &str, opts: &PutObjectOptions) -> Result<InitiateMultipartUploadResult, std::io::Error> { pub async fn initiate_multipart_upload(
&self,
bucket_name: &str,
object_name: &str,
opts: &PutObjectOptions,
) -> Result<InitiateMultipartUploadResult, std::io::Error> {
let mut url_values = HashMap::new(); let mut url_values = HashMap::new();
url_values.insert("uploads".to_string(), "".to_string()); url_values.insert("uploads".to_string(), "".to_string());
@@ -180,9 +204,9 @@ impl TransitionClient {
let mut req_metadata = RequestMetadata { let mut req_metadata = RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(), object_name: object_name.to_string(),
query_values: url_values, query_values: url_values,
custom_header, custom_header,
content_body: ReaderImpl::Body(Bytes::new()), content_body: ReaderImpl::Body(Bytes::new()),
content_length: 0, content_length: 0,
content_md5_base64: "".to_string(), content_md5_base64: "".to_string(),
content_sha256_hex: "".to_string(), content_sha256_hex: "".to_string(),
@@ -197,9 +221,9 @@ impl TransitionClient {
let resp = self.execute_method(http::Method::POST, &mut req_metadata).await?; let resp = self.execute_method(http::Method::POST, &mut req_metadata).await?;
//if resp.is_none() { //if resp.is_none() {
if resp.status() != StatusCode::OK { if resp.status() != StatusCode::OK {
return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name))); return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name)));
} }
//} //}
let initiate_multipart_upload_result = InitiateMultipartUploadResult::default(); let initiate_multipart_upload_result = InitiateMultipartUploadResult::default();
Ok(initiate_multipart_upload_result) Ok(initiate_multipart_upload_result)
@@ -207,13 +231,20 @@ impl TransitionClient {
pub async fn upload_part(&self, p: &mut UploadPartParams) -> Result<ObjectPart, std::io::Error> { pub async fn upload_part(&self, p: &mut UploadPartParams) -> Result<ObjectPart, std::io::Error> {
if p.size > MAX_PART_SIZE { if p.size > MAX_PART_SIZE {
return Err(std::io::Error::other(err_entity_too_large(p.size, MAX_PART_SIZE, &p.bucket_name, &p.object_name))); return Err(std::io::Error::other(err_entity_too_large(
p.size,
MAX_PART_SIZE,
&p.bucket_name,
&p.object_name,
)));
} }
if p.size <= -1 { if p.size <= -1 {
return Err(std::io::Error::other(err_entity_too_small(p.size, &p.bucket_name, &p.object_name))); return Err(std::io::Error::other(err_entity_too_small(p.size, &p.bucket_name, &p.object_name)));
} }
if p.part_number <= 0 { if p.part_number <= 0 {
return Err(std::io::Error::other(err_invalid_argument("Part number cannot be negative or equal to zero."))); return Err(std::io::Error::other(err_invalid_argument(
"Part number cannot be negative or equal to zero.",
)));
} }
if p.upload_id == "" { if p.upload_id == "" {
return Err(std::io::Error::other(err_invalid_argument("UploadID cannot be empty."))); return Err(std::io::Error::other(err_invalid_argument("UploadID cannot be empty.")));
@@ -224,55 +255,85 @@ impl TransitionClient {
url_values.insert("uploadId".to_string(), p.upload_id.clone()); url_values.insert("uploadId".to_string(), p.upload_id.clone());
let buf = match &mut p.reader { let buf = match &mut p.reader {
ReaderImpl::Body(content_body) => { ReaderImpl::Body(content_body) => content_body.to_vec(),
content_body.to_vec() ReaderImpl::ObjectBody(content_body) => content_body.read_all().await?,
}
ReaderImpl::ObjectBody(content_body) => {
content_body.read_all().await?
}
}; };
let mut req_metadata = RequestMetadata { let mut req_metadata = RequestMetadata {
bucket_name: p.bucket_name.clone(), bucket_name: p.bucket_name.clone(),
object_name: p.object_name.clone(), object_name: p.object_name.clone(),
query_values: url_values, query_values: url_values,
custom_header: p.custom_header.clone(), custom_header: p.custom_header.clone(),
content_body: ReaderImpl::Body(Bytes::from(buf)), content_body: ReaderImpl::Body(Bytes::from(buf)),
content_length: p.size, content_length: p.size,
content_md5_base64: p.md5_base64.clone(), content_md5_base64: p.md5_base64.clone(),
content_sha256_hex: p.sha256_hex.clone(), content_sha256_hex: p.sha256_hex.clone(),
stream_sha256: p.stream_sha256, stream_sha256: p.stream_sha256,
trailer: p.trailer.clone(), trailer: p.trailer.clone(),
pre_sign_url: Default::default(), pre_sign_url: Default::default(),
add_crc: Default::default(), add_crc: Default::default(),
extra_pre_sign_header: Default::default(), extra_pre_sign_header: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
expires: Default::default(), expires: Default::default(),
}; };
let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?; let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
//defer closeResponse(resp) //defer closeResponse(resp)
//if resp.is_none() { //if resp.is_none() {
if resp.status() != StatusCode::OK { if resp.status() != StatusCode::OK {
return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], &p.bucket_name.clone(), &p.object_name))); return Err(std::io::Error::other(http_resp_to_error_response(
} resp,
vec![],
&p.bucket_name.clone(),
&p.object_name,
)));
}
//} //}
let h = resp.headers(); let h = resp.headers();
let mut obj_part = ObjectPart { let mut obj_part = ObjectPart {
checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) { h_checksum_crc32.to_str().expect("err").to_string() } else { "".to_string() }, checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) {
checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) { h_checksum_crc32c.to_str().expect("err").to_string() } else { "".to_string() }, h_checksum_crc32.to_str().expect("err").to_string()
checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) { h_checksum_sha1.to_str().expect("err").to_string() } else { "".to_string() }, } else {
checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) { h_checksum_sha256.to_str().expect("err").to_string() } else { "".to_string() }, "".to_string()
checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) { h_checksum_crc64nvme.to_str().expect("err").to_string() } else { "".to_string() }, },
checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) {
h_checksum_crc32c.to_str().expect("err").to_string()
} else {
"".to_string()
},
checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) {
h_checksum_sha1.to_str().expect("err").to_string()
} else {
"".to_string()
},
checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) {
h_checksum_sha256.to_str().expect("err").to_string()
} else {
"".to_string()
},
checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) {
h_checksum_crc64nvme.to_str().expect("err").to_string()
} else {
"".to_string()
},
..Default::default() ..Default::default()
}; };
obj_part.size = p.size; obj_part.size = p.size;
obj_part.part_num = p.part_number; obj_part.part_num = p.part_number;
obj_part.etag = if let Some(h_etag) = h.get("ETag") { h_etag.to_str().expect("err").trim_matches('"').to_string() } else { "".to_string() }; obj_part.etag = if let Some(h_etag) = h.get("ETag") {
h_etag.to_str().expect("err").trim_matches('"').to_string()
} else {
"".to_string()
};
Ok(obj_part) Ok(obj_part)
} }
pub async fn complete_multipart_upload(&self, bucket_name: &str, object_name: &str, upload_id: &str, pub async fn complete_multipart_upload(
complete: CompleteMultipartUpload, opts: &PutObjectOptions &self,
bucket_name: &str,
object_name: &str,
upload_id: &str,
complete: CompleteMultipartUpload,
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> { ) -> Result<UploadInfo, std::io::Error> {
let mut url_values = HashMap::new(); let mut url_values = HashMap::new();
url_values.insert("uploadId".to_string(), upload_id.to_string()); url_values.insert("uploadId".to_string(), upload_id.to_string());
@@ -284,19 +345,19 @@ impl TransitionClient {
let mut req_metadata = RequestMetadata { let mut req_metadata = RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(), object_name: object_name.to_string(),
query_values: url_values, query_values: url_values,
content_body: ReaderImpl::Body(complete_multipart_upload_buffer), content_body: ReaderImpl::Body(complete_multipart_upload_buffer),
content_length: 100,//complete_multipart_upload_bytes.len(), content_length: 100, //complete_multipart_upload_bytes.len(),
content_sha256_hex: "".to_string(),//hex_simd::encode_to_string(complete_multipart_upload_bytes, hex_simd::AsciiCase::Lower), content_sha256_hex: "".to_string(), //hex_simd::encode_to_string(complete_multipart_upload_bytes, hex_simd::AsciiCase::Lower),
custom_header: headers, custom_header: headers,
stream_sha256: Default::default(), stream_sha256: Default::default(),
trailer: Default::default(), trailer: Default::default(),
content_md5_base64: "".to_string(), content_md5_base64: "".to_string(),
pre_sign_url: Default::default(), pre_sign_url: Default::default(),
add_crc: Default::default(), add_crc: Default::default(),
extra_pre_sign_header: Default::default(), extra_pre_sign_header: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
expires: Default::default(), expires: Default::default(),
}; };
let resp = self.execute_method(http::Method::POST, &mut req_metadata).await?; let resp = self.execute_method(http::Method::POST, &mut req_metadata).await?;
@@ -307,7 +368,7 @@ impl TransitionClient {
let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) { let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) {
( (
OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(), OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(),
"".to_string() "".to_string(),
) )
} else { } else {
(OffsetDateTime::now_utc(), "".to_string()) (OffsetDateTime::now_utc(), "".to_string())
@@ -315,17 +376,21 @@ impl TransitionClient {
let h = resp.headers(); let h = resp.headers();
Ok(UploadInfo { Ok(UploadInfo {
bucket: complete_multipart_upload_result.bucket, bucket: complete_multipart_upload_result.bucket,
key: complete_multipart_upload_result.key, key: complete_multipart_upload_result.key,
etag: trim_etag(&complete_multipart_upload_result.etag), etag: trim_etag(&complete_multipart_upload_result.etag),
version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) { h_x_amz_version_id.to_str().expect("err").to_string() } else { "".to_string() }, version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) {
location: complete_multipart_upload_result.location, h_x_amz_version_id.to_str().expect("err").to_string()
expiration: exp_time, } else {
"".to_string()
},
location: complete_multipart_upload_result.location,
expiration: exp_time,
expiration_rule_id: rule_id, expiration_rule_id: rule_id,
checksum_sha256: complete_multipart_upload_result.checksum_sha256, checksum_sha256: complete_multipart_upload_result.checksum_sha256,
checksum_sha1: complete_multipart_upload_result.checksum_sha1, checksum_sha1: complete_multipart_upload_result.checksum_sha1,
checksum_crc32: complete_multipart_upload_result.checksum_crc32, checksum_crc32: complete_multipart_upload_result.checksum_crc32,
checksum_crc32c: complete_multipart_upload_result.checksum_crc32c, checksum_crc32c: complete_multipart_upload_result.checksum_crc32c,
checksum_crc64nvme: complete_multipart_upload_result.checksum_crc64nvme, checksum_crc64nvme: complete_multipart_upload_result.checksum_crc64nvme,
..Default::default() ..Default::default()
}) })
ecstore/src/client/api_put_object_streaming.rs
+147 -68
View File
@@ -1,28 +1,28 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use std::sync::RwLock;
use std::{collections::HashMap, sync::Arc};
use bytes::Bytes; use bytes::Bytes;
use futures::future::join_all; use futures::future::join_all;
use http::{HeaderMap, HeaderName, HeaderValue, StatusCode}; use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
use time::{format_description, OffsetDateTime}; use std::sync::RwLock;
use std::{collections::HashMap, sync::Arc};
use time::{OffsetDateTime, format_description};
use tokio::{select, sync::mpsc}; use tokio::{select, sync::mpsc};
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
use tracing::warn; use tracing::warn;
use uuid::Uuid; use uuid::Uuid;
use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID}; use crate::checksum::{ChecksumMode, add_auto_checksum_headers, apply_auto_checksum};
use reader::hasher::Hasher;
use crate::client::{ use crate::client::{
constants::ISO8601_DATEFORMAT, api_error_response::{err_invalid_argument, err_unexpected_eof, http_resp_to_error_response},
api_put_object::PutObjectOptions, api_put_object::PutObjectOptions,
api_put_object_common::{is_object, optimal_part_info},
api_put_object_multipart::UploadPartParams, api_put_object_multipart::UploadPartParams,
api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart}, api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart},
transition_api::{TransitionClient, RequestMetadata, UploadInfo, ReaderImpl}, constants::ISO8601_DATEFORMAT,
api_put_object_common::{is_object, optimal_part_info,}, transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo},
api_error_response::{err_invalid_argument, http_resp_to_error_response, err_unexpected_eof},
}; };
use reader::hasher::Hasher;
use rustfs_utils::{crypto::base64_encode, path::trim_etag}; use rustfs_utils::{crypto::base64_encode, path::trim_etag};
use crate::checksum::{add_auto_checksum_headers, apply_auto_checksum, ChecksumMode}; use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};
pub struct UploadedPartRes { pub struct UploadedPartRes {
pub error: std::io::Error, pub error: std::io::Error,
@@ -37,23 +37,39 @@ pub struct UploadPartReq {
} }
impl TransitionClient { impl TransitionClient {
pub async fn put_object_multipart_stream(self: Arc<Self>, bucket_name: &str, object_name: &str, pub async fn put_object_multipart_stream(
mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions self: Arc<Self>,
bucket_name: &str,
object_name: &str,
mut reader: ReaderImpl,
size: i64,
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> { ) -> Result<UploadInfo, std::io::Error> {
let info: UploadInfo; let info: UploadInfo;
if opts.concurrent_stream_parts && opts.num_threads > 1 { if opts.concurrent_stream_parts && opts.num_threads > 1 {
info = self.put_object_multipart_stream_parallel(bucket_name, object_name, reader, opts).await?; info = self
.put_object_multipart_stream_parallel(bucket_name, object_name, reader, opts)
.await?;
} else if !is_object(&reader) && !opts.send_content_md5 { } else if !is_object(&reader) && !opts.send_content_md5 {
info = self.put_object_multipart_stream_from_readat(bucket_name, object_name, reader, size, opts).await?; info = self
.put_object_multipart_stream_from_readat(bucket_name, object_name, reader, size, opts)
.await?;
} else { } else {
info = self.put_object_multipart_stream_optional_checksum(bucket_name, object_name, reader, size, opts).await?; info = self
.put_object_multipart_stream_optional_checksum(bucket_name, object_name, reader, size, opts)
.await?;
} }
Ok(info) Ok(info)
} }
pub async fn put_object_multipart_stream_from_readat(&self, bucket_name: &str, object_name: &str, pub async fn put_object_multipart_stream_from_readat(
mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions &self,
bucket_name: &str,
object_name: &str,
mut reader: ReaderImpl,
size: i64,
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> { ) -> Result<UploadInfo, std::io::Error> {
let ret = optimal_part_info(size, opts.part_size)?; let ret = optimal_part_info(size, opts.part_size)?;
let (total_parts_count, part_size, lastpart_size) = ret; let (total_parts_count, part_size, lastpart_size) = ret;
@@ -68,8 +84,13 @@ impl TransitionClient {
todo!(); todo!();
} }
pub async fn put_object_multipart_stream_optional_checksum(&self, bucket_name: &str, object_name: &str, pub async fn put_object_multipart_stream_optional_checksum(
mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions &self,
bucket_name: &str,
object_name: &str,
mut reader: ReaderImpl,
size: i64,
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> { ) -> Result<UploadInfo, std::io::Error> {
let mut opts = opts.clone(); let mut opts = opts.clone();
if opts.checksum.is_set() { if opts.checksum.is_set() {
@@ -130,7 +151,7 @@ impl TransitionClient {
} }
} }
let hooked = ReaderImpl::Body(Bytes::from(buf));//newHook(BufferReader::new(buf), opts.progress); let hooked = ReaderImpl::Body(Bytes::from(buf)); //newHook(BufferReader::new(buf), opts.progress);
let mut p = UploadPartParams { let mut p = UploadPartParams {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(), object_name: object_name.to_string(),
@@ -154,7 +175,12 @@ impl TransitionClient {
if size > 0 { if size > 0 {
if total_uploaded_size != size { if total_uploaded_size != size {
return Err(std::io::Error::other(err_unexpected_eof(total_uploaded_size, size, bucket_name, object_name))); return Err(std::io::Error::other(err_unexpected_eof(
total_uploaded_size,
size,
bucket_name,
object_name,
)));
} }
} }
@@ -167,12 +193,12 @@ impl TransitionClient {
all_parts.push(part.clone()); all_parts.push(part.clone());
compl_multipart_upload.parts.push(CompletePart { compl_multipart_upload.parts.push(CompletePart {
etag: part.etag, etag: part.etag,
part_num: part.part_num, part_num: part.part_num,
checksum_crc32: part.checksum_crc32, checksum_crc32: part.checksum_crc32,
checksum_crc32c: part.checksum_crc32c, checksum_crc32c: part.checksum_crc32c,
checksum_sha1: part.checksum_sha1, checksum_sha1: part.checksum_sha1,
checksum_sha256: part.checksum_sha256, checksum_sha256: part.checksum_sha256,
checksum_crc64nvme: part.checksum_crc64nvme, checksum_crc64nvme: part.checksum_crc64nvme,
}); });
} }
@@ -181,18 +207,24 @@ impl TransitionClient {
let mut opts = PutObjectOptions { let mut opts = PutObjectOptions {
//server_side_encryption: opts.server_side_encryption, //server_side_encryption: opts.server_side_encryption,
auto_checksum: opts.auto_checksum, auto_checksum: opts.auto_checksum,
..Default::default() ..Default::default()
}; };
apply_auto_checksum(&mut opts, &mut all_parts); apply_auto_checksum(&mut opts, &mut all_parts);
let mut upload_info = self.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts).await?; let mut upload_info = self
.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts)
.await?;
upload_info.size = total_uploaded_size; upload_info.size = total_uploaded_size;
Ok(upload_info) Ok(upload_info)
} }
pub async fn put_object_multipart_stream_parallel(self: Arc<Self>, bucket_name: &str, object_name: &str, pub async fn put_object_multipart_stream_parallel(
mut reader: ReaderImpl/*GetObjectReader*/, opts: &PutObjectOptions self: Arc<Self>,
bucket_name: &str,
object_name: &str,
mut reader: ReaderImpl, /*GetObjectReader*/
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> { ) -> Result<UploadInfo, std::io::Error> {
let mut opts = opts.clone(); let mut opts = opts.clone();
if opts.checksum.is_set() { if opts.checksum.is_set() {
@@ -239,7 +271,11 @@ impl TransitionClient {
} }
if buf.len() != part_size as usize { if buf.len() != part_size as usize {
return Err(std::io::Error::other(format!("read buffer < {} than expected partSize: {}", buf.len(), part_size))); return Err(std::io::Error::other(format!(
"read buffer < {} than expected partSize: {}",
buf.len(),
part_size
)));
} }
match &mut reader { match &mut reader {
@@ -286,18 +322,18 @@ impl TransitionClient {
//defer wg.Done() //defer wg.Done()
let mut p = UploadPartParams { let mut p = UploadPartParams {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(), object_name: object_name.to_string(),
upload_id: clone_upload_id, upload_id: clone_upload_id,
reader: ReaderImpl::Body(Bytes::from(buf.clone())), reader: ReaderImpl::Body(Bytes::from(buf.clone())),
part_number, part_number,
md5_base64, md5_base64,
size: length as i64, size: length as i64,
//sse: opts.server_side_encryption, //sse: opts.server_side_encryption,
stream_sha256: !opts.disable_content_sha256, stream_sha256: !opts.disable_content_sha256,
custom_header, custom_header,
sha256_hex: "".to_string(), sha256_hex: "".to_string(),
trailer: HeaderMap::new(), trailer: HeaderMap::new(),
}; };
let obj_part = clone_self.upload_part(&mut p).await.expect("err"); let obj_part = clone_self.upload_part(&mut p).await.expect("err");
@@ -328,12 +364,12 @@ impl TransitionClient {
all_parts.push(part.clone()); all_parts.push(part.clone());
compl_multipart_upload.parts.push(CompletePart { compl_multipart_upload.parts.push(CompletePart {
etag: part.etag, etag: part.etag,
part_num: part.part_num, part_num: part.part_num,
checksum_crc32: part.checksum_crc32, checksum_crc32: part.checksum_crc32,
checksum_crc32c: part.checksum_crc32c, checksum_crc32c: part.checksum_crc32c,
checksum_sha1: part.checksum_sha1, checksum_sha1: part.checksum_sha1,
checksum_sha256: part.checksum_sha256, checksum_sha256: part.checksum_sha256,
checksum_crc64nvme: part.checksum_crc64nvme, checksum_crc64nvme: part.checksum_crc64nvme,
..Default::default() ..Default::default()
}); });
@@ -343,41 +379,60 @@ impl TransitionClient {
let mut opts = PutObjectOptions { let mut opts = PutObjectOptions {
//server_side_encryption: opts.server_side_encryption, //server_side_encryption: opts.server_side_encryption,
auto_checksum: opts.auto_checksum, auto_checksum: opts.auto_checksum,
..Default::default() ..Default::default()
}; };
apply_auto_checksum(&mut opts, &mut all_parts); apply_auto_checksum(&mut opts, &mut all_parts);
let mut upload_info = self.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts).await?; let mut upload_info = self
.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts)
.await?;
upload_info.size = total_uploaded_size; upload_info.size = total_uploaded_size;
Ok(upload_info) Ok(upload_info)
} }
pub async fn put_object_gcs(&self, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions) -> Result<UploadInfo, std::io::Error> { pub async fn put_object_gcs(
&self,
bucket_name: &str,
object_name: &str,
mut reader: ReaderImpl,
size: i64,
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> {
let mut opts = opts.clone(); let mut opts = opts.clone();
if opts.checksum.is_set() { if opts.checksum.is_set() {
opts.send_content_md5 = false; opts.send_content_md5 = false;
} }
let mut md5_base64: String = "".to_string(); let mut md5_base64: String = "".to_string();
let progress_reader = reader;//newHook(reader, opts.progress); let progress_reader = reader; //newHook(reader, opts.progress);
self.put_object_do(bucket_name, object_name, progress_reader, &md5_base64, "", size, &opts).await self.put_object_do(bucket_name, object_name, progress_reader, &md5_base64, "", size, &opts)
.await
} }
pub async fn put_object_do(&self, bucket_name: &str, object_name: &str, reader: ReaderImpl, md5_base64: &str, sha256_hex: &str, size: i64, opts: &PutObjectOptions) -> Result<UploadInfo, std::io::Error> { pub async fn put_object_do(
&self,
bucket_name: &str,
object_name: &str,
reader: ReaderImpl,
md5_base64: &str,
sha256_hex: &str,
size: i64,
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> {
let custom_header = opts.header(); let custom_header = opts.header();
let mut req_metadata = RequestMetadata { let mut req_metadata = RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(), object_name: object_name.to_string(),
custom_header, custom_header,
content_body: reader, content_body: reader,
content_length: size, content_length: size,
content_md5_base64: md5_base64.to_string(), content_md5_base64: md5_base64.to_string(),
content_sha256_hex: sha256_hex.to_string(), content_sha256_hex: sha256_hex.to_string(),
stream_sha256: !opts.disable_content_sha256, stream_sha256: !opts.disable_content_sha256,
add_crc: Default::default(), add_crc: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
pre_sign_url: Default::default(), pre_sign_url: Default::default(),
@@ -386,7 +441,7 @@ impl TransitionClient {
expires: Default::default(), expires: Default::default(),
trailer: Default::default(), trailer: Default::default(),
}; };
let mut add_crc = false;//self.trailing_header_support && md5_base64 == "" && !s3utils.IsGoogleEndpoint(self.endpoint_url) && (opts.disable_content_sha256 || self.secure); let mut add_crc = false; //self.trailing_header_support && md5_base64 == "" && !s3utils.IsGoogleEndpoint(self.endpoint_url) && (opts.disable_content_sha256 || self.secure);
let mut opts = opts.clone(); let mut opts = opts.clone();
if opts.checksum.is_set() { if opts.checksum.is_set() {
req_metadata.add_crc = opts.checksum; req_metadata.add_crc = opts.checksum;
@@ -422,25 +477,49 @@ impl TransitionClient {
let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) { let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) {
( (
OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(), OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(),
"".to_string() "".to_string(),
) )
} else { } else {
(OffsetDateTime::now_utc(), "".to_string()) (OffsetDateTime::now_utc(), "".to_string())
}; };
let h = resp.headers(); let h = resp.headers();
Ok(UploadInfo { Ok(UploadInfo {
bucket: bucket_name.to_string(), bucket: bucket_name.to_string(),
key: object_name.to_string(), key: object_name.to_string(),
etag: trim_etag(h.get("ETag").expect("err").to_str().expect("err")), etag: trim_etag(h.get("ETag").expect("err").to_str().expect("err")),
version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) { h_x_amz_version_id.to_str().expect("err").to_string() } else { "".to_string() }, version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) {
size: size, h_x_amz_version_id.to_str().expect("err").to_string()
expiration: exp_time, } else {
"".to_string()
},
size: size,
expiration: exp_time,
expiration_rule_id: rule_id, expiration_rule_id: rule_id,
checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) { h_checksum_crc32.to_str().expect("err").to_string() } else { "".to_string() }, checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) {
checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) { h_checksum_crc32c.to_str().expect("err").to_string() } else { "".to_string() }, h_checksum_crc32.to_str().expect("err").to_string()
checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) { h_checksum_sha1.to_str().expect("err").to_string() } else { "".to_string() }, } else {
checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) { h_checksum_sha256.to_str().expect("err").to_string() } else { "".to_string() }, "".to_string()
checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) { h_checksum_crc64nvme.to_str().expect("err").to_string() } else { "".to_string() }, },
checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) {
h_checksum_crc32c.to_str().expect("err").to_string()
} else {
"".to_string()
},
checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) {
h_checksum_sha1.to_str().expect("err").to_string()
} else {
"".to_string()
},
checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) {
h_checksum_sha256.to_str().expect("err").to_string()
} else {
"".to_string()
},
checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) {
h_checksum_crc64nvme.to_str().expect("err").to_string()
} else {
"".to_string()
},
..Default::default() ..Default::default()
}) })
} }
ecstore/src/client/api_remove.rs
+227 -142
View File
@@ -1,26 +1,24 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use bytes::Bytes;
use http::{HeaderMap, HeaderValue, Method, StatusCode};
use s3s::S3ErrorCode;
use s3s::dto::ReplicationStatus;
use s3s::header::X_AMZ_BYPASS_GOVERNANCE_RETENTION;
use std::fmt::Display; use std::fmt::Display;
use std::{collections::HashMap, sync::Arc}; use std::{collections::HashMap, sync::Arc};
use bytes::Bytes;
use s3s::header::X_AMZ_BYPASS_GOVERNANCE_RETENTION;
use s3s::S3ErrorCode;
use time::OffsetDateTime; use time::OffsetDateTime;
use s3s::dto::ReplicationStatus;
use tokio::sync::mpsc::{self, Receiver, Sender}; use tokio::sync::mpsc::{self, Receiver, Sender};
use http::{HeaderMap, HeaderValue, Method, StatusCode};
use reader::hasher::{sum_sha256_hex, sum_md5_base64}; use crate::client::{
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; api_error_response::{ErrorResponse, http_resp_to_error_response, to_error_response},
transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
};
use crate::{ use crate::{
disk::DiskAPI, disk::DiskAPI,
store_api::{ store_api::{GetObjectReader, ObjectInfo, StorageAPI},
GetObjectReader, ObjectInfo, StorageAPI,
},
};
use crate::client::{
transition_api::{TransitionClient, RequestMetadata, ReaderImpl},
api_error_response::{http_resp_to_error_response, to_error_response, ErrorResponse,},
}; };
use reader::hasher::{sum_md5_base64, sum_sha256_hex};
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
struct RemoveBucketOptions { struct RemoveBucketOptions {
forced_elete: bool, forced_elete: bool,
@@ -62,23 +60,28 @@ impl TransitionClient {
headers.insert(rustFSForceDelete, "true"); headers.insert(rustFSForceDelete, "true");
}*/ }*/
let resp = self.execute_method(Method::DELETE, &mut RequestMetadata { let resp = self
bucket_name: bucket_name.to_string(), .execute_method(
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), Method::DELETE,
custom_header: headers, &mut RequestMetadata {
object_name: "".to_string(), bucket_name: bucket_name.to_string(),
query_values: Default::default(), content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
content_body: ReaderImpl::Body(Bytes::new()), custom_header: headers,
content_length: 0, object_name: "".to_string(),
content_md5_base64: "".to_string(), query_values: Default::default(),
stream_sha256: false, content_body: ReaderImpl::Body(Bytes::new()),
trailer: HeaderMap::new(), content_length: 0,
pre_sign_url: Default::default(), content_md5_base64: "".to_string(),
add_crc: Default::default(), stream_sha256: false,
extra_pre_sign_header: Default::default(), trailer: HeaderMap::new(),
bucket_location: Default::default(), pre_sign_url: Default::default(),
expires: Default::default(), add_crc: Default::default(),
}).await?; extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
},
)
.await?;
{ {
let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap(); let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
@@ -88,23 +91,28 @@ impl TransitionClient {
} }
pub async fn remove_bucket(&self, bucket_name: &str) -> Result<(), std::io::Error> { pub async fn remove_bucket(&self, bucket_name: &str) -> Result<(), std::io::Error> {
let resp = self.execute_method(http::Method::DELETE, &mut RequestMetadata { let resp = self
bucket_name: bucket_name.to_string(), .execute_method(
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), http::Method::DELETE,
custom_header: Default::default(), &mut RequestMetadata {
object_name: "".to_string(), bucket_name: bucket_name.to_string(),
query_values: Default::default(), content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
content_body: ReaderImpl::Body(Bytes::new()), custom_header: Default::default(),
content_length: 0, object_name: "".to_string(),
content_md5_base64: "".to_string(), query_values: Default::default(),
stream_sha256: false, content_body: ReaderImpl::Body(Bytes::new()),
trailer: HeaderMap::new(), content_length: 0,
pre_sign_url: Default::default(), content_md5_base64: "".to_string(),
add_crc: Default::default(), stream_sha256: false,
extra_pre_sign_header: Default::default(), trailer: HeaderMap::new(),
bucket_location: Default::default(), pre_sign_url: Default::default(),
expires: Default::default(), add_crc: Default::default(),
}).await?; extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
},
)
.await?;
{ {
let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap(); let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
@@ -119,7 +127,12 @@ impl TransitionClient {
res.err res.err
} }
pub async fn remove_object_inner(&self, bucket_name: &str, object_name: &str, opts: RemoveObjectOptions) -> Result<RemoveObjectResult, std::io::Error> { pub async fn remove_object_inner(
&self,
bucket_name: &str,
object_name: &str,
opts: RemoveObjectOptions,
) -> Result<RemoveObjectResult, std::io::Error> {
let mut url_values = HashMap::new(); let mut url_values = HashMap::new();
if opts.version_id != "" { if opts.version_id != "" {
@@ -129,49 +142,72 @@ impl TransitionClient {
let mut headers = HeaderMap::new(); let mut headers = HeaderMap::new();
if opts.governance_bypass { if opts.governance_bypass {
headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err"));//amzBypassGovernance headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err")); //amzBypassGovernance
} }
let resp = self.execute_method(http::Method::DELETE, &mut RequestMetadata { let resp = self
bucket_name: bucket_name.to_string(), .execute_method(
object_name: object_name.to_string(), http::Method::DELETE,
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), &mut RequestMetadata {
query_values: url_values, bucket_name: bucket_name.to_string(),
custom_header: headers, object_name: object_name.to_string(),
content_body: ReaderImpl::Body(Bytes::new()), content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
content_length: 0, query_values: url_values,
content_md5_base64: "".to_string(), custom_header: headers,
stream_sha256: false, content_body: ReaderImpl::Body(Bytes::new()),
trailer: HeaderMap::new(), content_length: 0,
pre_sign_url: Default::default(), content_md5_base64: "".to_string(),
add_crc: Default::default(), stream_sha256: false,
extra_pre_sign_header: Default::default(), trailer: HeaderMap::new(),
bucket_location: Default::default(), pre_sign_url: Default::default(),
expires: Default::default(), add_crc: Default::default(),
}).await?; extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
},
)
.await?;
Ok(RemoveObjectResult { Ok(RemoveObjectResult {
object_name: object_name.to_string(), object_name: object_name.to_string(),
object_version_id: opts.version_id, object_version_id: opts.version_id,
delete_marker: resp.headers().get("x-amz-delete-marker").expect("err") == "true", delete_marker: resp.headers().get("x-amz-delete-marker").expect("err") == "true",
delete_marker_version_id: resp.headers().get("x-amz-version-id").expect("err").to_str().expect("err").to_string(), delete_marker_version_id: resp
.headers()
.get("x-amz-version-id")
.expect("err")
.to_str()
.expect("err")
.to_string(),
..Default::default() ..Default::default()
}) })
} }
pub async fn remove_objects_with_result(self: Arc<Self>, bucket_name: &str, objects_rx: Receiver<ObjectInfo>, opts: RemoveObjectsOptions) -> Receiver<RemoveObjectResult> { pub async fn remove_objects_with_result(
self: Arc<Self>,
bucket_name: &str,
objects_rx: Receiver<ObjectInfo>,
opts: RemoveObjectsOptions,
) -> Receiver<RemoveObjectResult> {
let (result_tx, mut result_rx) = mpsc::channel(1); let (result_tx, mut result_rx) = mpsc::channel(1);
let self_clone = Arc::clone(&self); let self_clone = Arc::clone(&self);
let bucket_name_owned = bucket_name.to_string(); let bucket_name_owned = bucket_name.to_string();
tokio::spawn(async move { tokio::spawn(async move {
self_clone.remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts).await; self_clone
.remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts)
.await;
}); });
result_rx result_rx
} }
pub async fn remove_objects(self: Arc<Self>, bucket_name: &str, objects_rx: Receiver<ObjectInfo>, opts: RemoveObjectsOptions) -> Receiver<RemoveObjectError> { pub async fn remove_objects(
self: Arc<Self>,
bucket_name: &str,
objects_rx: Receiver<ObjectInfo>,
opts: RemoveObjectsOptions,
) -> Receiver<RemoveObjectError> {
let (error_tx, mut error_rx) = mpsc::channel(1); let (error_tx, mut error_rx) = mpsc::channel(1);
let self_clone = Arc::clone(&self); let self_clone = Arc::clone(&self);
@@ -179,26 +215,36 @@ impl TransitionClient {
let (result_tx, mut result_rx) = mpsc::channel(1); let (result_tx, mut result_rx) = mpsc::channel(1);
tokio::spawn(async move { tokio::spawn(async move {
self_clone.remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts).await; self_clone
.remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts)
.await;
}); });
tokio::spawn(async move { tokio::spawn(async move {
while let Some(res) = result_rx.recv().await { while let Some(res) = result_rx.recv().await {
if res.err.is_none() { if res.err.is_none() {
continue; continue;
} }
error_tx.send(RemoveObjectError { error_tx
object_name: res.object_name, .send(RemoveObjectError {
version_id: res.object_version_id, object_name: res.object_name,
err: res.err, version_id: res.object_version_id,
..Default::default() err: res.err,
}).await; ..Default::default()
})
.await;
} }
}); });
error_rx error_rx
} }
pub async fn remove_objects_inner(&self, bucket_name: &str, mut objects_rx: Receiver<ObjectInfo>, result_tx: &Sender<RemoveObjectResult>, opts: RemoveObjectsOptions) -> Result<(), std::io::Error> { pub async fn remove_objects_inner(
&self,
bucket_name: &str,
mut objects_rx: Receiver<ObjectInfo>,
result_tx: &Sender<RemoveObjectResult>,
opts: RemoveObjectsOptions,
) -> Result<(), std::io::Error> {
let max_entries = 1000; let max_entries = 1000;
let mut finish = false; let mut finish = false;
let mut url_values = HashMap::new(); let mut url_values = HashMap::new();
@@ -213,11 +259,17 @@ impl TransitionClient {
while let Some(object) = objects_rx.recv().await { while let Some(object) = objects_rx.recv().await {
if has_invalid_xml_char(&object.name) { if has_invalid_xml_char(&object.name) {
let remove_result = self.remove_object_inner(bucket_name, &object.name, RemoveObjectOptions { let remove_result = self
version_id: object.version_id.expect("err").to_string(), .remove_object_inner(
governance_bypass: opts.governance_bypass, bucket_name,
..Default::default() &object.name,
}).await?; RemoveObjectOptions {
version_id: object.version_id.expect("err").to_string(),
governance_bypass: opts.governance_bypass,
..Default::default()
},
)
.await?;
let remove_result_clone = remove_result.clone(); let remove_result_clone = remove_result.clone();
if !remove_result.err.is_none() { if !remove_result.err.is_none() {
match to_error_response(&remove_result.err.expect("err")).code { match to_error_response(&remove_result.err.expect("err")).code {
@@ -252,23 +304,28 @@ impl TransitionClient {
} }
let remove_bytes = generate_remove_multi_objects_request(&batch); let remove_bytes = generate_remove_multi_objects_request(&batch);
let resp = self.execute_method(http::Method::POST, &mut RequestMetadata { let resp = self
bucket_name: bucket_name.to_string(), .execute_method(
query_values: url_values.clone(), http::Method::POST,
content_body: ReaderImpl::Body(Bytes::from(remove_bytes.clone())), &mut RequestMetadata {
content_length: remove_bytes.len() as i64, bucket_name: bucket_name.to_string(),
content_md5_base64: sum_md5_base64(&remove_bytes), query_values: url_values.clone(),
content_sha256_hex: sum_sha256_hex(&remove_bytes), content_body: ReaderImpl::Body(Bytes::from(remove_bytes.clone())),
custom_header: headers, content_length: remove_bytes.len() as i64,
object_name: "".to_string(), content_md5_base64: sum_md5_base64(&remove_bytes),
stream_sha256: false, content_sha256_hex: sum_sha256_hex(&remove_bytes),
trailer: HeaderMap::new(), custom_header: headers,
pre_sign_url: Default::default(), object_name: "".to_string(),
add_crc: Default::default(), stream_sha256: false,
extra_pre_sign_header: Default::default(), trailer: HeaderMap::new(),
bucket_location: Default::default(), pre_sign_url: Default::default(),
expires: Default::default(), add_crc: Default::default(),
}).await?; extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
},
)
.await?;
let body_bytes: Vec<u8> = resp.body().bytes().expect("err").to_vec(); let body_bytes: Vec<u8> = resp.body().bytes().expect("err").to_vec();
process_remove_multi_objects_response(ReaderImpl::Body(Bytes::from(body_bytes)), result_tx.clone()); process_remove_multi_objects_response(ReaderImpl::Body(Bytes::from(body_bytes)), result_tx.clone());
@@ -285,49 +342,77 @@ impl TransitionClient {
Ok(()) Ok(())
} }
pub async fn abort_multipart_upload(&self, bucket_name: &str, object_name: &str, upload_id: &str) -> Result<(), std::io::Error> { pub async fn abort_multipart_upload(
&self,
bucket_name: &str,
object_name: &str,
upload_id: &str,
) -> Result<(), std::io::Error> {
let mut url_values = HashMap::new(); let mut url_values = HashMap::new();
url_values.insert("uploadId".to_string(), upload_id.to_string()); url_values.insert("uploadId".to_string(), upload_id.to_string());
let resp = self.execute_method(http::Method::DELETE, &mut RequestMetadata { let resp = self
bucket_name: bucket_name.to_string(), .execute_method(
object_name: object_name.to_string(), http::Method::DELETE,
query_values: url_values, &mut RequestMetadata {
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), bucket_name: bucket_name.to_string(),
custom_header: HeaderMap::new(), object_name: object_name.to_string(),
content_body: ReaderImpl::Body(Bytes::new()), query_values: url_values,
content_length: 0, content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
content_md5_base64: "".to_string(), custom_header: HeaderMap::new(),
stream_sha256: false, content_body: ReaderImpl::Body(Bytes::new()),
trailer: HeaderMap::new(), content_length: 0,
pre_sign_url: Default::default(), content_md5_base64: "".to_string(),
add_crc: Default::default(), stream_sha256: false,
extra_pre_sign_header: Default::default(), trailer: HeaderMap::new(),
bucket_location: Default::default(), pre_sign_url: Default::default(),
expires: Default::default(), add_crc: Default::default(),
}).await?; extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
},
)
.await?;
//if resp.is_some() { //if resp.is_some() {
if resp.status() != StatusCode::NO_CONTENT { if resp.status() != StatusCode::NO_CONTENT {
let error_response: ErrorResponse; let error_response: ErrorResponse;
match resp.status() { match resp.status() {
StatusCode::NOT_FOUND => { StatusCode::NOT_FOUND => {
error_response = ErrorResponse { error_response = ErrorResponse {
code: S3ErrorCode::NoSuchUpload, code: S3ErrorCode::NoSuchUpload,
message: "The specified multipart upload does not exist.".to_string(), message: "The specified multipart upload does not exist.".to_string(),
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
key: object_name.to_string(), key: object_name.to_string(),
request_id: resp.headers().get("x-amz-request-id").expect("err").to_str().expect("err").to_string(), request_id: resp
host_id: resp.headers().get("x-amz-id-2").expect("err").to_str().expect("err").to_string(), .headers()
region: resp.headers().get("x-amz-bucket-region").expect("err").to_str().expect("err").to_string(), .get("x-amz-request-id")
..Default::default() .expect("err")
}; .to_str()
} .expect("err")
_ => { .to_string(),
return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name))); host_id: resp
} .headers()
.get("x-amz-id-2")
.expect("err")
.to_str()
.expect("err")
.to_string(),
region: resp
.headers()
.get("x-amz-bucket-region")
.expect("err")
.to_str()
.expect("err")
.to_string(),
..Default::default()
};
}
_ => {
return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name)));
} }
return Err(std::io::Error::other(error_response));
} }
return Err(std::io::Error::other(error_response));
}
//} //}
Ok(()) Ok(())
} }
@@ -336,8 +421,8 @@ impl TransitionClient {
#[derive(Debug, Default)] #[derive(Debug, Default)]
struct RemoveObjectError { struct RemoveObjectError {
object_name: String, object_name: String,
version_id: String, version_id: String,
err: Option<std::io::Error>, err: Option<std::io::Error>,
} }
impl Display for RemoveObjectError { impl Display for RemoveObjectError {
ecstore/src/client/api_s3_datatypes.rs
+59 -59
View File
@@ -1,12 +1,12 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use std::collections::HashMap;
use s3s::dto::Owner; use s3s::dto::Owner;
use serde::{Serialize, Deserialize}; use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use time::OffsetDateTime; use time::OffsetDateTime;
use rustfs_utils::crypto::base64_decode;
use crate::checksum::ChecksumMode; use crate::checksum::ChecksumMode;
use crate::client::transition_api::ObjectMultipartInfo; use crate::client::transition_api::ObjectMultipartInfo;
use rustfs_utils::crypto::base64_decode;
use super::transition_api; use super::transition_api;
@@ -31,15 +31,15 @@ pub struct ListBucketV2Result {
pub name: String, pub name: String,
pub next_continuation_token: String, pub next_continuation_token: String,
pub continuation_token: String, pub continuation_token: String,
pub prefix: String, pub prefix: String,
pub fetch_owner: String, pub fetch_owner: String,
pub start_after: String, pub start_after: String,
} }
pub struct Version { pub struct Version {
etag: String, etag: String,
is_latest: bool, is_latest: bool,
key: String, key: String,
last_modified: OffsetDateTime, last_modified: OffsetDateTime,
owner: Owner, owner: Owner,
size: i64, size: i64,
@@ -53,15 +53,15 @@ pub struct Version {
pub struct ListVersionsResult { pub struct ListVersionsResult {
versions: Vec<Version>, versions: Vec<Version>,
common_prefixes: Vec<CommonPrefix>, common_prefixes: Vec<CommonPrefix>,
name: String, name: String,
prefix: String, prefix: String,
delimiter: String, delimiter: String,
max_keys: i64, max_keys: i64,
encoding_type: String, encoding_type: String,
is_truncated: bool, is_truncated: bool,
key_marker: String, key_marker: String,
version_id_marker: String, version_id_marker: String,
next_key_marker: String, next_key_marker: String,
next_version_id_marker: String, next_version_id_marker: String,
} }
@@ -77,35 +77,35 @@ pub struct ListBucketResult {
delimiter: String, delimiter: String,
encoding_type: String, encoding_type: String,
is_truncated: bool, is_truncated: bool,
marker: String, marker: String,
max_keys: i64, max_keys: i64,
name: String, name: String,
next_marker: String, next_marker: String,
prefix: String, prefix: String,
} }
pub struct ListMultipartUploadsResult { pub struct ListMultipartUploadsResult {
bucket: String, bucket: String,
key_marker: String, key_marker: String,
upload_id_marker: String, upload_id_marker: String,
next_key_marker: String, next_key_marker: String,
next_upload_id_marker: String, next_upload_id_marker: String,
encoding_type: String, encoding_type: String,
max_uploads: i64, max_uploads: i64,
is_truncated: bool, is_truncated: bool,
uploads: Vec<ObjectMultipartInfo>, uploads: Vec<ObjectMultipartInfo>,
prefix: String, prefix: String,
delimiter: String, delimiter: String,
common_prefixes: Vec<CommonPrefix>, common_prefixes: Vec<CommonPrefix>,
} }
pub struct Initiator { pub struct Initiator {
id: String, id: String,
display_name: String, display_name: String,
} }
pub struct CopyObjectResult { pub struct CopyObjectResult {
pub etag: String, pub etag: String,
pub last_modified: OffsetDateTime, pub last_modified: OffsetDateTime,
} }
@@ -115,10 +115,10 @@ pub struct ObjectPart {
pub part_num: i64, pub part_num: i64,
pub last_modified: OffsetDateTime, pub last_modified: OffsetDateTime,
pub size: i64, pub size: i64,
pub checksum_crc32: String, pub checksum_crc32: String,
pub checksum_crc32c: String, pub checksum_crc32c: String,
pub checksum_sha1: String, pub checksum_sha1: String,
pub checksum_sha256: String, pub checksum_sha256: String,
pub checksum_crc64nvme: String, pub checksum_crc64nvme: String,
} }
@@ -129,14 +129,13 @@ impl Default for ObjectPart {
part_num: 0, part_num: 0,
last_modified: OffsetDateTime::now_utc(), last_modified: OffsetDateTime::now_utc(),
size: 0, size: 0,
checksum_crc32: Default::default(), checksum_crc32: Default::default(),
checksum_crc32c: Default::default(), checksum_crc32c: Default::default(),
checksum_sha1: Default::default(), checksum_sha1: Default::default(),
checksum_sha256: Default::default(), checksum_sha256: Default::default(),
checksum_crc64nvme: Default::default(), checksum_crc64nvme: Default::default(),
} }
} }
} }
impl ObjectPart { impl ObjectPart {
@@ -216,15 +215,15 @@ pub struct CompleteMultipartUploadResult {
pub checksum_crc64nvme: String, pub checksum_crc64nvme: String,
} }
#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] #[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, serde::Serialize)]
#[derive(serde::Serialize)] pub struct CompletePart {
pub struct CompletePart { //api has //api has
pub etag: String, pub etag: String,
pub part_num: i64, pub part_num: i64,
pub checksum_crc32: String, pub checksum_crc32: String,
pub checksum_crc32c: String, pub checksum_crc32c: String,
pub checksum_sha1: String, pub checksum_sha1: String,
pub checksum_sha256: String, pub checksum_sha256: String,
pub checksum_crc64nvme: String, pub checksum_crc64nvme: String,
} }
@@ -258,8 +257,7 @@ pub struct CopyObjectPartResult {
pub last_modified: OffsetDateTime, pub last_modified: OffsetDateTime,
} }
#[derive(Debug, Default)] #[derive(Debug, Default, serde::Serialize)]
#[derive(serde::Serialize)]
pub struct CompleteMultipartUpload { pub struct CompleteMultipartUpload {
pub parts: Vec<CompletePart>, pub parts: Vec<CompletePart>,
} }
@@ -287,12 +285,14 @@ pub struct CreateBucketConfiguration {
} }
#[derive(serde::Serialize)] #[derive(serde::Serialize)]
pub struct DeleteObject { //api has pub struct DeleteObject {
//api has
pub key: String, pub key: String,
pub version_id: String, pub version_id: String,
} }
pub struct DeletedObject { //s3s has pub struct DeletedObject {
//s3s has
pub key: String, pub key: String,
pub version_id: String, pub version_id: String,
pub deletemarker: bool, pub deletemarker: bool,
@@ -300,15 +300,15 @@ pub struct DeletedObject { //s3s has
} }
pub struct NonDeletedObject { pub struct NonDeletedObject {
pub key: String, pub key: String,
pub code: String, pub code: String,
pub message: String, pub message: String,
pub version_id: String, pub version_id: String,
} }
#[derive(serde::Serialize)] #[derive(serde::Serialize)]
pub struct DeleteMultiObjects { pub struct DeleteMultiObjects {
pub quiet: bool, pub quiet: bool,
pub objects: Vec<DeleteObject>, pub objects: Vec<DeleteObject>,
} }
@@ -331,6 +331,6 @@ impl DeleteMultiObjects {
} }
pub struct DeleteMultiObjectsResult { pub struct DeleteMultiObjectsResult {
pub deleted_objects: Vec<DeletedObject>, pub deleted_objects: Vec<DeletedObject>,
pub undeleted_objects: Vec<NonDeletedObject>, pub undeleted_objects: Vec<NonDeletedObject>,
} }
ecstore/src/client/bucket_cache.rs
+21 -24
View File
@@ -1,20 +1,20 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use http::Request; use http::Request;
use hyper::StatusCode;
use hyper::body::Incoming; use hyper::body::Incoming;
use std::{collections::HashMap, sync::Arc}; use std::{collections::HashMap, sync::Arc};
use tracing::warn; use tracing::warn;
use tracing::{error, info, debug}; use tracing::{debug, error, info};
use hyper::StatusCode;
use reader::hasher::{Hasher, Sha256};
use s3s::S3ErrorCode;
use s3s::Body;
use crate::client::{ use crate::client::{
api_error_response::{http_resp_to_error_response, to_error_response}, api_error_response::{http_resp_to_error_response, to_error_response},
transition_api::{TransitionClient, Document}, transition_api::{Document, TransitionClient},
}; };
use crate::signer; use crate::signer;
use reader::hasher::{Hasher, Sha256};
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
use s3s::Body;
use s3s::S3ErrorCode;
use super::constants::UNSIGNED_PAYLOAD; use super::constants::UNSIGNED_PAYLOAD;
use super::credentials::SignatureType; use super::credentials::SignatureType;
@@ -25,9 +25,7 @@ pub struct BucketLocationCache {
impl BucketLocationCache { impl BucketLocationCache {
pub fn new() -> BucketLocationCache { pub fn new() -> BucketLocationCache {
BucketLocationCache{ BucketLocationCache { items: HashMap::new() }
items: HashMap::new(),
}
} }
pub fn get(&self, bucket_name: &str) -> Option<String> { pub fn get(&self, bucket_name: &str) -> Option<String> {
@@ -50,7 +48,7 @@ impl TransitionClient {
async fn get_bucket_location_inner(&self, bucket_name: &str) -> Result<String, std::io::Error> { async fn get_bucket_location_inner(&self, bucket_name: &str) -> Result<String, std::io::Error> {
if self.region != "" { if self.region != "" {
return Ok(self.region.clone()) return Ok(self.region.clone());
} }
let mut location; let mut location;
@@ -81,11 +79,7 @@ impl TransitionClient {
let mut target_url = self.endpoint_url.clone(); let mut target_url = self.endpoint_url.clone();
let scheme = self.endpoint_url.scheme(); let scheme = self.endpoint_url.scheme();
let h = target_url.host().expect("host is none."); let h = target_url.host().expect("host is none.");
let default_port = if scheme == "https" { let default_port = if scheme == "https" { 443 } else { 80 };
443
} else {
80
};
let p = target_url.port().unwrap_or(default_port); let p = target_url.port().unwrap_or(default_port);
let is_virtual_style = self.is_virtual_host_style_request(&target_url, bucket_name); let is_virtual_style = self.is_virtual_host_style_request(&target_url, bucket_name);
@@ -128,9 +122,9 @@ impl TransitionClient {
} }
let mut signer_type = value.signer_type.clone(); let mut signer_type = value.signer_type.clone();
let mut access_key_id = value.access_key_id; let mut access_key_id = value.access_key_id;
let mut secret_access_key = value.secret_access_key; let mut secret_access_key = value.secret_access_key;
let mut session_token = value.session_token; let mut session_token = value.session_token;
if self.override_signer_type != SignatureType::SignatureDefault { if self.override_signer_type != SignatureType::SignatureDefault {
signer_type = self.override_signer_type.clone(); signer_type = self.override_signer_type.clone();
@@ -164,7 +158,10 @@ impl TransitionClient {
content_sha256 = UNSIGNED_PAYLOAD.to_string(); content_sha256 = UNSIGNED_PAYLOAD.to_string();
} }
req_builder.headers_mut().expect("err").insert("X-Amz-Content-Sha256", content_sha256.parse().unwrap()); req_builder
.headers_mut()
.expect("err")
.insert("X-Amz-Content-Sha256", content_sha256.parse().unwrap());
let req_builder = signer::sign_v4(req_builder, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1"); let req_builder = signer::sign_v4(req_builder, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1");
let req = match req_builder.body(Body::empty()) { let req = match req_builder.body(Body::empty()) {
Ok(req) => return Ok(req), Ok(req) => return Ok(req),
@@ -177,9 +174,9 @@ impl TransitionClient {
async fn process_bucket_location_response(mut resp: http::Response<Body>, bucket_name: &str) -> Result<String, std::io::Error> { async fn process_bucket_location_response(mut resp: http::Response<Body>, bucket_name: &str) -> Result<String, std::io::Error> {
//if resp != nil { //if resp != nil {
if resp.status() != StatusCode::OK { if resp.status() != StatusCode::OK {
let err_resp = http_resp_to_error_response(resp, vec![], bucket_name, ""); let err_resp = http_resp_to_error_response(resp, vec![], bucket_name, "");
match err_resp.code { match err_resp.code {
S3ErrorCode::NotImplemented => { S3ErrorCode::NotImplemented => {
match err_resp.server.as_str() { match err_resp.server.as_str() {
"AmazonSnowball" => { "AmazonSnowball" => {
@@ -205,13 +202,13 @@ async fn process_bucket_location_response(mut resp: http::Response<Body>, bucket
return Err(std::io::Error::other(err_resp)); return Err(std::io::Error::other(err_resp));
} }
} }
} }
//} //}
let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec(); let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
let Document(location_constraint) = serde_xml_rs::from_str::<Document>(&String::from_utf8(b).unwrap()).unwrap(); let Document(location_constraint) = serde_xml_rs::from_str::<Document>(&String::from_utf8(b).unwrap()).unwrap();
let mut location = location_constraint; let mut location = location_constraint;
if location == "" { if location == "" {
location = "us-east-1".to_string(); location = "us-east-1".to_string();
} }
ecstore/src/client/constants.rs
+5 -4
View File
@@ -1,7 +1,7 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use std::{collections::HashMap, sync::Arc};
use time::{macros::format_description, format_description::FormatItem};
use lazy_static::lazy_static; use lazy_static::lazy_static;
use std::{collections::HashMap, sync::Arc};
use time::{format_description::FormatItem, macros::format_description};
pub const ABS_MIN_PART_SIZE: i64 = 1024 * 1024 * 5; pub const ABS_MIN_PART_SIZE: i64 = 1024 * 1024 * 5;
pub const MAX_PARTS_COUNT: i64 = 10000; pub const MAX_PARTS_COUNT: i64 = 10000;
@@ -16,8 +16,9 @@ pub const UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER";
pub const TOTAL_WORKERS: i64 = 4; pub const TOTAL_WORKERS: i64 = 4;
pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256"; pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] = format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z"); pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] =
format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z");
const GetObjectAttributesTags: &str = "ETag,Checksum,StorageClass,ObjectSize,ObjectParts"; const GetObjectAttributesTags: &str = "ETag,Checksum,StorageClass,ObjectSize,ObjectParts";
const GetObjectAttributesMaxParts: i64 = 1000; const GetObjectAttributesMaxParts: i64 = 1000;
ecstore/src/client/credentials.rs
+13 -14
View File
@@ -19,8 +19,7 @@ pub struct Credentials<P: Provider + Default> {
provider: P, provider: P,
} }
impl<P: Provider + Default> Credentials<P> impl<P: Provider + Default> Credentials<P> {
{
pub fn new(provider: P) -> Self { pub fn new(provider: P) -> Self {
Self { Self {
provider: provider, provider: provider,
@@ -109,8 +108,8 @@ impl Provider for Static {
#[derive(Debug, Clone, Default)] #[derive(Debug, Clone, Default)]
pub struct STSError { pub struct STSError {
pub r#type: String, pub r#type: String,
pub code: String, pub code: String,
pub message: String, pub message: String,
} }
@@ -128,21 +127,21 @@ impl Display for ErrorResponse {
impl ErrorResponse { impl ErrorResponse {
fn error(&self) -> String { fn error(&self) -> String {
if self.sts_error.message == "" { if self.sts_error.message == "" {
return format!("Error response code {}.", self.sts_error.code); return format!("Error response code {}.", self.sts_error.code);
} }
return self.sts_error.message.clone(); return self.sts_error.message.clone();
} }
} }
struct Error { struct Error {
code: String, code: String,
message: String, message: String,
bucket_name: String, bucket_name: String,
key: String, key: String,
resource: String, resource: String,
request_id: String, request_id: String,
host_id: String, host_id: String,
region: String, region: String,
server: String, server: String,
status_code: i64, status_code: i64,
ecstore/src/client/mod.rs
+11 -11
View File
@@ -1,18 +1,18 @@
pub mod constants; pub mod admin_handler_utils;
pub mod transition_api;
pub mod api_list;
pub mod api_error_response;
pub mod api_s3_datatypes;
pub mod api_bucket_policy; pub mod api_bucket_policy;
pub mod api_put_object_common; pub mod api_error_response;
pub mod api_get_options;
pub mod api_get_object; pub mod api_get_object;
pub mod api_get_options;
pub mod api_list;
pub mod api_put_object; pub mod api_put_object;
pub mod api_put_object_streaming; pub mod api_put_object_common;
pub mod api_put_object_multipart; pub mod api_put_object_multipart;
pub mod api_put_object_streaming;
pub mod api_remove; pub mod api_remove;
pub mod api_s3_datatypes;
pub mod bucket_cache;
pub mod constants;
pub mod credentials;
pub mod object_api_utils; pub mod object_api_utils;
pub mod object_handlers_common; pub mod object_handlers_common;
pub mod admin_handler_utils; pub mod transition_api;
pub mod credentials;
pub mod bucket_cache;
ecstore/src/client/object_api_utils.rs
+41 -30
View File
@@ -1,17 +1,14 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use std::{collections::HashMap, sync::Arc};
use http::HeaderMap; use http::HeaderMap;
use tokio::io::BufReader;
use std::io::Cursor; use std::io::Cursor;
use std::{collections::HashMap, sync::Arc};
use tokio::io::BufReader;
use s3s::S3ErrorCode; use crate::error::ErrorResponse;
use crate::store_api::{ use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions};
GetObjectReader, HTTPRangeSpec,
ObjectInfo, ObjectOptions,
};
use rustfs_filemeta::fileinfo::ObjectPartInfo; use rustfs_filemeta::fileinfo::ObjectPartInfo;
use rustfs_rio::HashReader; use rustfs_rio::HashReader;
use crate::error::ErrorResponse; use s3s::S3ErrorCode;
//#[derive(Clone)] //#[derive(Clone)]
pub struct PutObjReader { pub struct PutObjReader {
@@ -57,7 +54,11 @@ fn part_number_to_rangespec(oi: ObjectInfo, part_number: usize) -> Option<HTTPRa
i += 1; i += 1;
} }
Some(HTTPRangeSpec {start: start as usize, end: Some(end as usize), is_suffix_length: false}) Some(HTTPRangeSpec {
start: start as usize,
end: Some(end as usize),
is_suffix_length: false,
})
} }
fn get_compressed_offsets(oi: ObjectInfo, offset: i64) -> (i64, i64, i64, i64, u64) { fn get_compressed_offsets(oi: ObjectInfo, offset: i64) -> (i64, i64, i64, i64, u64) {
@@ -71,7 +72,7 @@ fn get_compressed_offsets(oi: ObjectInfo, offset: i64) -> (i64, i64, i64, i64, u
for (i, part) in oi.parts.iter().enumerate() { for (i, part) in oi.parts.iter().enumerate() {
cumulative_actual_size += part.actual_size as i64; cumulative_actual_size += part.actual_size as i64;
if cumulative_actual_size <= offset { if cumulative_actual_size <= offset {
compressed_offset += part.size as i64; compressed_offset += part.size as i64;
} else { } else {
first_part_idx = i as i64; first_part_idx = i as i64;
skip_length = cumulative_actual_size - part.actual_size as i64; skip_length = cumulative_actual_size - part.actual_size as i64;
@@ -81,23 +82,32 @@ fn get_compressed_offsets(oi: ObjectInfo, offset: i64) -> (i64, i64, i64, i64, u
skip_length = offset - skip_length; skip_length = offset - skip_length;
let parts: &[ObjectPartInfo] = &oi.parts; let parts: &[ObjectPartInfo] = &oi.parts;
if skip_length > 0 && parts.len() > first_part_idx as usize && parts[first_part_idx as usize].index.as_ref().expect("err").len() > 0 { if skip_length > 0
&& parts.len() > first_part_idx as usize
&& parts[first_part_idx as usize].index.as_ref().expect("err").len() > 0
{
todo!(); todo!();
} }
(compressed_offset, part_skip, first_part_idx, decrypt_skip, seq_num) (compressed_offset, part_skip, first_part_idx, decrypt_skip, seq_num)
} }
pub fn new_getobjectreader(rs: HTTPRangeSpec, oi: &ObjectInfo, opts: &ObjectOptions, h: &HeaderMap) -> Result<(ObjReaderFn, i64, i64), ErrorResponse> { pub fn new_getobjectreader(
rs: HTTPRangeSpec,
oi: &ObjectInfo,
opts: &ObjectOptions,
h: &HeaderMap,
) -> Result<(ObjReaderFn, i64, i64), ErrorResponse> {
//let (_, mut is_encrypted) = crypto.is_encrypted(oi.user_defined)?; //let (_, mut is_encrypted) = crypto.is_encrypted(oi.user_defined)?;
let mut is_encrypted = false; let mut is_encrypted = false;
let is_compressed = false;//oi.is_compressed_ok(); let is_compressed = false; //oi.is_compressed_ok();
let mut get_fn: ObjReaderFn; let mut get_fn: ObjReaderFn;
let (off, length) = match rs.get_offset_length(oi.size) { let (off, length) = match rs.get_offset_length(oi.size) {
Ok(x) => x, Ok(x) => x,
Err(err) => return Err(ErrorResponse { Err(err) => {
return Err(ErrorResponse {
code: S3ErrorCode::InvalidRange, code: S3ErrorCode::InvalidRange,
message: err.to_string(), message: err.to_string(),
key: None, key: None,
@@ -105,26 +115,27 @@ pub fn new_getobjectreader(rs: HTTPRangeSpec, oi: &ObjectInfo, opts: &ObjectOpti
region: None, region: None,
request_id: None, request_id: None,
host_id: "".to_string(), host_id: "".to_string(),
}), });
}
};
get_fn = Arc::new(move |input_reader: BufReader<Cursor<Vec<u8>>>, _: HeaderMap| {
//Box::pin({
/*let r = GetObjectReader {
object_info: oi.clone(),
stream: StreamingBlob::new(HashReader::new(input_reader, 10, None, None, 10)),
}; };
get_fn = Arc::new(move |input_reader: BufReader<Cursor<Vec<u8>>>, _: HeaderMap| { r*/
//Box::pin({ todo!();
/*let r = GetObjectReader { //})
object_info: oi.clone(), });
stream: StreamingBlob::new(HashReader::new(input_reader, 10, None, None, 10)),
};
r*/
todo!();
//})
});
Ok((get_fn, off as i64, length as i64)) Ok((get_fn, off as i64, length as i64))
} }
pub fn extract_etag(metadata: &HashMap<String, String>) -> String { pub fn extract_etag(metadata: &HashMap<String, String>) -> String {
if let Some(etag) = metadata.get("etag") { if let Some(etag) = metadata.get("etag") {
etag.clone() etag.clone()
} else { } else {
metadata["md5Sum"].clone() metadata["md5Sum"].clone()
} }
} }
ecstore/src/client/object_handlers_common.rs
+16 -10
View File
@@ -1,15 +1,17 @@
use lock::local_locker::MAX_DELETE_LIST; use crate::StorageAPI;
use crate::bucket::lifecycle::lifecycle;
use crate::bucket::versioning::VersioningApi; use crate::bucket::versioning::VersioningApi;
use crate::bucket::versioning_sys::BucketVersioningSys; use crate::bucket::versioning_sys::BucketVersioningSys;
use crate::store::ECStore; use crate::store::ECStore;
use crate::store_api::{ObjectOptions, ObjectToDelete,}; use crate::store_api::{ObjectOptions, ObjectToDelete};
use crate::StorageAPI; use lock::local_locker::MAX_DELETE_LIST;
use crate::bucket::lifecycle::lifecycle;
pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], lc_event: lifecycle::Event) { pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], lc_event: lifecycle::Event) {
let mut remaining = to_del; let mut remaining = to_del;
loop { loop {
if remaining.len() <= 0 {break}; if remaining.len() <= 0 {
break;
};
let mut to_del = remaining; let mut to_del = remaining;
if to_del.len() > MAX_DELETE_LIST { if to_del.len() > MAX_DELETE_LIST {
remaining = &to_del[MAX_DELETE_LIST..]; remaining = &to_del[MAX_DELETE_LIST..];
@@ -18,10 +20,14 @@ pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[Object
remaining = &[]; remaining = &[];
} }
let vc = BucketVersioningSys::get(bucket).await.expect("err!"); let vc = BucketVersioningSys::get(bucket).await.expect("err!");
let deleted_objs = api.delete_objects(bucket, to_del.to_vec(), ObjectOptions { let deleted_objs = api.delete_objects(
//prefix_enabled_fn: vc.prefix_enabled(""), bucket,
version_suspended: vc.suspended(), to_del.to_vec(),
..Default::default() ObjectOptions {
}); //prefix_enabled_fn: vc.prefix_enabled(""),
version_suspended: vc.suspended(),
..Default::default()
},
);
} }
} }
ecstore/src/client/transition_api.rs
+281 -144
View File
@@ -1,61 +1,63 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use std::pin::Pin;
use bytes::Bytes; use bytes::Bytes;
use futures::Future; use futures::Future;
use http::{HeaderMap, HeaderName}; use http::{HeaderMap, HeaderName};
use serde::{Serialize, Deserialize}; use http::{
use uuid::Uuid; HeaderValue, Response, StatusCode,
request::{Builder, Request},
};
use hyper_rustls::{ConfigBuilderExt, HttpsConnector};
use hyper_util::{client::legacy::Client, client::legacy::connect::HttpConnector, rt::TokioExecutor};
use rand::Rng; use rand::Rng;
use std::{collections::HashMap, sync::{Arc, Mutex}}; use serde::{Deserialize, Serialize};
use std::io::Cursor;
use std::pin::Pin;
use std::sync::atomic::{AtomicI32, Ordering}; use std::sync::atomic::{AtomicI32, Ordering};
use std::task::{Context, Poll}; use std::task::{Context, Poll};
use std::{
collections::HashMap,
sync::{Arc, Mutex},
};
use time::Duration; use time::Duration;
use time::OffsetDateTime; use time::OffsetDateTime;
use hyper_rustls::{ConfigBuilderExt, HttpsConnector};
use hyper_util::{client::legacy::Client, rt::TokioExecutor, client::legacy::connect::HttpConnector};
use http::{StatusCode, HeaderValue, request::{Request, Builder}, Response};
use tracing::{error, debug};
use url::{form_urlencoded, Url};
use tokio::io::BufReader; use tokio::io::BufReader;
use std::io::Cursor; use tracing::{debug, error};
use url::{Url, form_urlencoded};
use uuid::Uuid;
use s3s::S3ErrorCode;
use s3s::{dto::Owner, Body};
use s3s::dto::ReplicationStatus;
use crate::client::bucket_cache::BucketLocationCache; use crate::client::bucket_cache::BucketLocationCache;
use reader::hasher::{Sha256, MD5,}; use crate::client::{
api_error_response::{err_invalid_argument, http_resp_to_error_response, to_error_response},
api_get_options::GetObjectOptions,
api_put_object::PutObjectOptions,
api_put_object_multipart::UploadPartParams,
api_s3_datatypes::{
CompleteMultipartUpload, CompletePart, ListBucketResult, ListBucketV2Result, ListMultipartUploadsResult,
ListObjectPartsResult, ObjectPart,
},
constants::{UNSIGNED_PAYLOAD, UNSIGNED_PAYLOAD_TRAILER},
credentials::{CredContext, Credentials, SignatureType, Static},
};
use crate::signer;
use crate::{checksum::ChecksumMode, store_api::GetObjectReader};
use reader::hasher::{MD5, Sha256};
use rustfs_rio::HashReader; use rustfs_rio::HashReader;
use rustfs_utils::{ use rustfs_utils::{
net::get_endpoint_url, net::get_endpoint_url,
retry::{new_retry_timer, MAX_RETRY}, retry::{MAX_RETRY, new_retry_timer},
};
use crate::{
store_api::GetObjectReader,
checksum::ChecksumMode,
};
use crate::signer;
use crate::client::{
constants::{UNSIGNED_PAYLOAD, UNSIGNED_PAYLOAD_TRAILER},
credentials::{Credentials, SignatureType, CredContext, Static,},
api_error_response::{to_error_response, http_resp_to_error_response, err_invalid_argument},
api_put_object_multipart::UploadPartParams,
api_put_object::PutObjectOptions,
api_get_options::GetObjectOptions,
api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ListBucketResult, ListBucketV2Result, ListMultipartUploadsResult, ListObjectPartsResult, ObjectPart},
}; };
use s3s::S3ErrorCode;
use s3s::dto::ReplicationStatus;
use s3s::{Body, dto::Owner};
const C_USER_AGENT_PREFIX: &str = "RustFS (linux; x86)"; const C_USER_AGENT_PREFIX: &str = "RustFS (linux; x86)";
const C_USER_AGENT: &str = "RustFS (linux; x86)"; const C_USER_AGENT: &str = "RustFS (linux; x86)";
const SUCCESS_STATUS: [StatusCode; 3] = [ const SUCCESS_STATUS: [StatusCode; 3] = [StatusCode::OK, StatusCode::NO_CONTENT, StatusCode::PARTIAL_CONTENT];
StatusCode::OK,
StatusCode::NO_CONTENT,
StatusCode::PARTIAL_CONTENT,
];
const C_UNKNOWN: i32 = -1; const C_UNKNOWN: i32 = -1;
const C_OFFLINE: i32 = 0; const C_OFFLINE: i32 = 0;
const C_ONLINE: i32 = 1; const C_ONLINE: i32 = 1;
//pub type ReaderImpl = Box<dyn Reader + Send + Sync + 'static>; //pub type ReaderImpl = Box<dyn Reader + Send + Sync + 'static>;
pub enum ReaderImpl { pub enum ReaderImpl {
@@ -71,7 +73,7 @@ pub struct TransitionClient {
pub override_signer_type: SignatureType, pub override_signer_type: SignatureType,
/*app_info: TODO*/ /*app_info: TODO*/
pub secure: bool, pub secure: bool,
pub http_client: Client<HttpsConnector<HttpConnector>, Body>, pub http_client: Client<HttpsConnector<HttpConnector>, Body>,
//pub http_trace: Httptrace.ClientTrace, //pub http_trace: Httptrace.ClientTrace,
pub bucket_loc_cache: Arc<Mutex<BucketLocationCache>>, pub bucket_loc_cache: Arc<Mutex<BucketLocationCache>>,
pub is_trace_enabled: Arc<Mutex<bool>>, pub is_trace_enabled: Arc<Mutex<bool>>,
@@ -83,7 +85,7 @@ pub struct TransitionClient {
pub random: u64, pub random: u64,
pub lookup: BucketLookupType, pub lookup: BucketLookupType,
//pub lookupFn: func(u url.URL, bucketName string) BucketLookupType, //pub lookupFn: func(u url.URL, bucketName string) BucketLookupType,
pub md5_hasher: Arc<Mutex<Option<MD5>>>, pub md5_hasher: Arc<Mutex<Option<MD5>>>,
pub sha256_hasher: Option<Sha256>, pub sha256_hasher: Option<Sha256>,
pub health_status: AtomicI32, pub health_status: AtomicI32,
pub trailing_header_support: bool, pub trailing_header_support: bool,
@@ -92,16 +94,16 @@ pub struct TransitionClient {
#[derive(Debug, Default)] #[derive(Debug, Default)]
pub struct Options { pub struct Options {
pub creds: Credentials<Static>, pub creds: Credentials<Static>,
pub secure: bool, pub secure: bool,
//pub transport: http.RoundTripper, //pub transport: http.RoundTripper,
//pub trace: *httptrace.ClientTrace, //pub trace: *httptrace.ClientTrace,
pub region: String, pub region: String,
pub bucket_lookup: BucketLookupType, pub bucket_lookup: BucketLookupType,
//pub custom_region_via_url: func(u url.URL) string, //pub custom_region_via_url: func(u url.URL) string,
//pub bucket_lookup_via_url: func(u url.URL, bucketName string) BucketLookupType, //pub bucket_lookup_via_url: func(u url.URL, bucketName string) BucketLookupType,
pub trailing_headers: bool, pub trailing_headers: bool,
pub custom_md5: Option<MD5>, pub custom_md5: Option<MD5>,
pub custom_sha256: Option<Sha256>, pub custom_sha256: Option<Sha256>,
pub max_retries: i64, pub max_retries: i64,
} }
@@ -136,15 +138,13 @@ impl TransitionClient {
//if scheme == "https" { //if scheme == "https" {
// client = Client::builder(TokioExecutor::new()).build_http(); // client = Client::builder(TokioExecutor::new()).build_http();
//} else { //} else {
let tls = rustls::ClientConfig::builder() let tls = rustls::ClientConfig::builder().with_native_roots()?.with_no_client_auth();
.with_native_roots()? let https = hyper_rustls::HttpsConnectorBuilder::new()
.with_no_client_auth(); .with_tls_config(tls)
let https = hyper_rustls::HttpsConnectorBuilder::new() .https_or_http()
.with_tls_config(tls) .enable_http1()
.https_or_http() .build();
.enable_http1() client = Client::builder(TokioExecutor::new()).build(https);
.build();
client = Client::builder(TokioExecutor::new()).build(https);
//} //}
let mut clnt = TransitionClient { let mut clnt = TransitionClient {
@@ -218,7 +218,11 @@ impl TransitionClient {
todo!(); todo!();
} }
pub fn hash_materials(&self, is_md5_requested: bool, is_sha256_requested: bool) -> (HashMap<String, MD5>, HashMap<String, Vec<u8>>) { pub fn hash_materials(
&self,
is_md5_requested: bool,
is_sha256_requested: bool,
) -> (HashMap<String, MD5>, HashMap<String, Vec<u8>>) {
todo!(); todo!();
} }
@@ -227,7 +231,8 @@ impl TransitionClient {
} }
fn mark_offline(&self) { fn mark_offline(&self) {
self.health_status.compare_exchange(C_ONLINE, C_OFFLINE, Ordering::SeqCst, Ordering::SeqCst); self.health_status
.compare_exchange(C_ONLINE, C_OFFLINE, Ordering::SeqCst, Ordering::SeqCst);
} }
fn is_offline(&self) -> bool { fn is_offline(&self) -> bool {
@@ -261,7 +266,9 @@ impl TransitionClient {
debug!("endpoint_url: {}", self.endpoint_url.as_str().to_string()); debug!("endpoint_url: {}", self.endpoint_url.as_str().to_string());
resp = http_client.request(req); resp = http_client.request(req);
} }
let resp = resp.await/*.map_err(Into::into)*/.map(|res| res.map(Body::from)); let resp = resp
.await /*.map_err(Into::into)*/
.map(|res| res.map(Body::from));
debug!("http_client url: {} {}", req_method, req_uri); debug!("http_client url: {} {}", req_method, req_uri);
debug!("http_client headers: {:?}", req_headers); debug!("http_client headers: {:?}", req_headers);
if let Err(err) = resp { if let Err(err) = resp {
@@ -282,7 +289,11 @@ impl TransitionClient {
Ok(resp) Ok(resp)
} }
pub async fn execute_method(&self, method: http::Method, metadata: &mut RequestMetadata) -> Result<http::Response<Body>, std::io::Error> { pub async fn execute_method(
&self,
method: http::Method,
metadata: &mut RequestMetadata,
) -> Result<http::Response<Body>, std::io::Error> {
if self.is_offline() { if self.is_offline() {
let mut s = self.endpoint_url.to_string(); let mut s = self.endpoint_url.to_string();
s.push_str(" is offline."); s.push_str(" is offline.");
@@ -295,16 +306,18 @@ impl TransitionClient {
let mut resp: http::Response<Body>; let mut resp: http::Response<Body>;
//if metadata.content_body != nil { //if metadata.content_body != nil {
//body_seeker = BufferReader::new(metadata.content_body.read_all().await?); //body_seeker = BufferReader::new(metadata.content_body.read_all().await?);
retryable = true; retryable = true;
if !retryable { if !retryable {
req_retry = 1; req_retry = 1;
} }
//} //}
//let mut retry_timer = RetryTimer::new(); //let mut retry_timer = RetryTimer::new();
//while let Some(v) = retry_timer.next().await { //while let Some(v) = retry_timer.next().await {
for _ in [1;1]/*new_retry_timer(req_retry, DefaultRetryUnit, DefaultRetryCap, MaxJitter)*/ { for _ in [1; 1]
/*new_retry_timer(req_retry, DefaultRetryUnit, DefaultRetryCap, MaxJitter)*/
{
let req = self.new_request(method, metadata).await?; let req = self.new_request(method, metadata).await?;
resp = self.doit(req).await?; resp = self.doit(req).await?;
@@ -355,7 +368,11 @@ impl TransitionClient {
Err(std::io::Error::other("resp err")) Err(std::io::Error::other("resp err"))
} }
async fn new_request(&self, method: http::Method, metadata: &mut RequestMetadata) -> Result<http::Request<Body>, std::io::Error> { async fn new_request(
&self,
method: http::Method,
metadata: &mut RequestMetadata,
) -> Result<http::Request<Body>, std::io::Error> {
let location = metadata.bucket_location.clone(); let location = metadata.bucket_location.clone();
if location == "" { if location == "" {
if metadata.bucket_name != "" { if metadata.bucket_name != "" {
@@ -366,8 +383,13 @@ impl TransitionClient {
let is_makebucket = metadata.object_name == "" && method == http::Method::PUT && metadata.query_values.len() == 0; let is_makebucket = metadata.object_name == "" && method == http::Method::PUT && metadata.query_values.len() == 0;
let is_virtual_host = self.is_virtual_host_style_request(&self.endpoint_url, &metadata.bucket_name) && !is_makebucket; let is_virtual_host = self.is_virtual_host_style_request(&self.endpoint_url, &metadata.bucket_name) && !is_makebucket;
let target_url = self.make_target_url(&metadata.bucket_name, &metadata.object_name, &location, let target_url = self.make_target_url(
is_virtual_host, &metadata.query_values)?; &metadata.bucket_name,
&metadata.object_name,
&location,
is_virtual_host,
&metadata.query_values,
)?;
let mut req_builder = Request::builder().method(method).uri(target_url.to_string()); let mut req_builder = Request::builder().method(method).uri(target_url.to_string());
@@ -377,10 +399,10 @@ impl TransitionClient {
value = creds_provider.get_with_context(Some(self.cred_context()))?; value = creds_provider.get_with_context(Some(self.cred_context()))?;
} }
let mut signer_type = value.signer_type.clone(); let mut signer_type = value.signer_type.clone();
let access_key_id = value.access_key_id; let access_key_id = value.access_key_id;
let secret_access_key = value.secret_access_key; let secret_access_key = value.secret_access_key;
let session_token = value.session_token; let session_token = value.session_token;
if self.override_signer_type != SignatureType::SignatureDefault { if self.override_signer_type != SignatureType::SignatureDefault {
signer_type = self.override_signer_type.clone(); signer_type = self.override_signer_type.clone();
@@ -392,24 +414,39 @@ impl TransitionClient {
if metadata.expires != 0 && metadata.pre_sign_url { if metadata.expires != 0 && metadata.pre_sign_url {
if signer_type == SignatureType::SignatureAnonymous { if signer_type == SignatureType::SignatureAnonymous {
return Err(std::io::Error::other(err_invalid_argument("Presigned URLs cannot be generated with anonymous credentials."))); return Err(std::io::Error::other(err_invalid_argument(
"Presigned URLs cannot be generated with anonymous credentials.",
)));
} }
if metadata.extra_pre_sign_header.is_some() { if metadata.extra_pre_sign_header.is_some() {
if signer_type == SignatureType::SignatureV2 { if signer_type == SignatureType::SignatureV2 {
return Err(std::io::Error::other(err_invalid_argument("Extra signed headers for Presign with Signature V2 is not supported."))); return Err(std::io::Error::other(err_invalid_argument(
"Extra signed headers for Presign with Signature V2 is not supported.",
)));
} }
for (k, v) in metadata.extra_pre_sign_header.as_ref().unwrap() { for (k, v) in metadata.extra_pre_sign_header.as_ref().unwrap() {
req_builder = req_builder.header(k, v); req_builder = req_builder.header(k, v);
} }
} }
if signer_type == SignatureType::SignatureV2 { if signer_type == SignatureType::SignatureV2 {
req_builder = signer::pre_sign_v2(req_builder, &access_key_id, &secret_access_key, metadata.expires, is_virtual_host); req_builder =
signer::pre_sign_v2(req_builder, &access_key_id, &secret_access_key, metadata.expires, is_virtual_host);
} else if signer_type == SignatureType::SignatureV4 { } else if signer_type == SignatureType::SignatureV4 {
req_builder = signer::pre_sign_v4(req_builder, &access_key_id, &secret_access_key, &session_token, &location, metadata.expires, OffsetDateTime::now_utc()); req_builder = signer::pre_sign_v4(
req_builder,
&access_key_id,
&secret_access_key,
&session_token,
&location,
metadata.expires,
OffsetDateTime::now_utc(),
);
} }
let req = match req_builder.body(Body::empty()) { let req = match req_builder.body(Body::empty()) {
Ok(req) => req, Ok(req) => req,
Err(err) => { return Err(std::io::Error::other(err)); } Err(err) => {
return Err(std::io::Error::other(err));
}
}; };
return Ok(req); return Ok(req);
} }
@@ -423,7 +460,10 @@ impl TransitionClient {
//req.content_length = metadata.content_length; //req.content_length = metadata.content_length;
if metadata.content_length <= -1 { if metadata.content_length <= -1 {
let chunked_value = HeaderValue::from_str(&vec!["chunked"].join(",")).expect("err"); let chunked_value = HeaderValue::from_str(&vec!["chunked"].join(",")).expect("err");
req_builder.headers_mut().expect("err").insert(http::header::TRANSFER_ENCODING, chunked_value); req_builder
.headers_mut()
.expect("err")
.insert(http::header::TRANSFER_ENCODING, chunked_value);
} }
if metadata.content_md5_base64.len() > 0 { if metadata.content_md5_base64.len() > 0 {
@@ -434,15 +474,17 @@ impl TransitionClient {
if signer_type == SignatureType::SignatureAnonymous { if signer_type == SignatureType::SignatureAnonymous {
let req = match req_builder.body(Body::empty()) { let req = match req_builder.body(Body::empty()) {
Ok(req) => req, Ok(req) => req,
Err(err) => { return Err(std::io::Error::other(err)); } Err(err) => {
return Err(std::io::Error::other(err));
}
}; };
return Ok(req); return Ok(req);
} }
if signer_type == SignatureType::SignatureV2 { if signer_type == SignatureType::SignatureV2 {
req_builder = signer::sign_v2(req_builder, metadata.content_length, &access_key_id, &secret_access_key, is_virtual_host); req_builder =
} signer::sign_v2(req_builder, metadata.content_length, &access_key_id, &secret_access_key, is_virtual_host);
else if metadata.stream_sha256 && !self.secure { } else if metadata.stream_sha256 && !self.secure {
if metadata.trailer.len() > 0 { if metadata.trailer.len() > 0 {
//req.Trailer = metadata.trailer; //req.Trailer = metadata.trailer;
for (_, v) in &metadata.trailer { for (_, v) in &metadata.trailer {
@@ -451,8 +493,7 @@ impl TransitionClient {
} }
//req_builder = signer::streaming_sign_v4(req_builder, &access_key_id, //req_builder = signer::streaming_sign_v4(req_builder, &access_key_id,
// &secret_access_key, &session_token, &location, metadata.content_length, OffsetDateTime::now_utc(), self.sha256_hasher()); // &secret_access_key, &session_token, &location, metadata.content_length, OffsetDateTime::now_utc(), self.sha256_hasher());
} } else {
else {
let mut sha_header = UNSIGNED_PAYLOAD.to_string(); let mut sha_header = UNSIGNED_PAYLOAD.to_string();
if metadata.content_sha256_hex != "" { if metadata.content_sha256_hex != "" {
sha_header = metadata.content_sha256_hex.clone(); sha_header = metadata.content_sha256_hex.clone();
@@ -462,9 +503,17 @@ impl TransitionClient {
} else if metadata.trailer.len() > 0 { } else if metadata.trailer.len() > 0 {
sha_header = UNSIGNED_PAYLOAD_TRAILER.to_string(); sha_header = UNSIGNED_PAYLOAD_TRAILER.to_string();
} }
req_builder = req_builder.header::<HeaderName, HeaderValue>("X-Amz-Content-Sha256".parse().unwrap(), sha_header.parse().expect("err")); req_builder = req_builder
.header::<HeaderName, HeaderValue>("X-Amz-Content-Sha256".parse().unwrap(), sha_header.parse().expect("err"));
req_builder = signer::sign_v4_trailer(req_builder, &access_key_id, &secret_access_key, &session_token, &location, metadata.trailer.clone()); req_builder = signer::sign_v4_trailer(
req_builder,
&access_key_id,
&secret_access_key,
&session_token,
&location,
metadata.trailer.clone(),
);
} }
let req; let req;
@@ -484,9 +533,7 @@ impl TransitionClient {
match req { match req {
Ok(req) => Ok(req), Ok(req) => Ok(req),
Err(err) => { Err(err) => Err(std::io::Error::other(err)),
Err(std::io::Error::other(err))
}
} }
} }
@@ -498,14 +545,17 @@ impl TransitionClient {
}*/ }*/
} }
fn make_target_url(&self, bucket_name: &str, object_name: &str, bucket_location: &str, is_virtual_host_style: bool, query_values: &HashMap<String, String>) -> Result<Url, std::io::Error> { fn make_target_url(
&self,
bucket_name: &str,
object_name: &str,
bucket_location: &str,
is_virtual_host_style: bool,
query_values: &HashMap<String, String>,
) -> Result<Url, std::io::Error> {
let scheme = self.endpoint_url.scheme(); let scheme = self.endpoint_url.scheme();
let host = self.endpoint_url.host().unwrap(); let host = self.endpoint_url.host().unwrap();
let default_port = if scheme == "https" { let default_port = if scheme == "https" { 443 } else { 80 };
443
} else {
80
};
let port = self.endpoint_url.port().unwrap_or(default_port); let port = self.endpoint_url.port().unwrap_or(default_port);
let mut url_str = format!("{scheme}://{host}:{port}/"); let mut url_str = format!("{scheme}://{host}:{port}/");
@@ -562,7 +612,7 @@ impl TransitionClient {
} }
struct LockedRandSource { struct LockedRandSource {
src: u64,//rand.Source, src: u64, //rand.Source,
} }
impl LockedRandSource { impl LockedRandSource {
@@ -604,76 +654,159 @@ impl TransitionCore {
Ok(Self(Arc::new(client))) Ok(Self(Arc::new(client)))
} }
pub fn list_objects(&self, bucket: &str, prefix: &str, marker: &str, delimiter: &str, max_keys: i64) -> Result<ListBucketResult, std::io::Error> { pub fn list_objects(
&self,
bucket: &str,
prefix: &str,
marker: &str,
delimiter: &str,
max_keys: i64,
) -> Result<ListBucketResult, std::io::Error> {
let client = self.0.clone(); let client = self.0.clone();
client.list_objects_query(bucket, prefix, marker, delimiter, max_keys, HeaderMap::new()) client.list_objects_query(bucket, prefix, marker, delimiter, max_keys, HeaderMap::new())
} }
pub async fn list_objects_v2(&self, bucket_name: &str, object_prefix: &str, start_after: &str, continuation_token: &str, delimiter: &str, max_keys: i64) -> Result<ListBucketV2Result, std::io::Error> { pub async fn list_objects_v2(
&self,
bucket_name: &str,
object_prefix: &str,
start_after: &str,
continuation_token: &str,
delimiter: &str,
max_keys: i64,
) -> Result<ListBucketV2Result, std::io::Error> {
let client = self.0.clone(); let client = self.0.clone();
client.list_objects_v2_query(bucket_name, object_prefix, continuation_token, true, false, delimiter, start_after, max_keys, HeaderMap::new()).await client
.list_objects_v2_query(
bucket_name,
object_prefix,
continuation_token,
true,
false,
delimiter,
start_after,
max_keys,
HeaderMap::new(),
)
.await
} }
/*pub fn copy_object(&self, source_bucket: &str, source_object: &str, dest_bucket: &str, dest_object: &str, metadata: HashMap<String, String>, src_opts: CopySrcOptions, dst_opts: PutObjectOptions) -> Result<ObjectInfo> { /*pub fn copy_object(&self, source_bucket: &str, source_object: &str, dest_bucket: &str, dest_object: &str, metadata: HashMap<String, String>, src_opts: CopySrcOptions, dst_opts: PutObjectOptions) -> Result<ObjectInfo> {
self.0.copy_object_do(source_bucket, source_object, dest_bucket, dest_object, metadata, src_opts, dst_opts) self.0.copy_object_do(source_bucket, source_object, dest_bucket, dest_object, metadata, src_opts, dst_opts)
}*/ }*/
pub fn copy_object_part(&self, src_bucket: &str, src_object: &str, dest_bucket: &str, dest_object: &str, upload_id: &str, pub fn copy_object_part(
part_id: i32, start_offset: i32, length: i64, metadata: HashMap<String, String>, &self,
src_bucket: &str,
src_object: &str,
dest_bucket: &str,
dest_object: &str,
upload_id: &str,
part_id: i32,
start_offset: i32,
length: i64,
metadata: HashMap<String, String>,
) -> Result<CompletePart, std::io::Error> { ) -> Result<CompletePart, std::io::Error> {
//self.0.copy_object_part_do(src_bucket, src_object, dest_bucket, dest_object, upload_id, //self.0.copy_object_part_do(src_bucket, src_object, dest_bucket, dest_object, upload_id,
// part_id, start_offset, length, metadata) // part_id, start_offset, length, metadata)
todo!(); todo!();
} }
pub async fn put_object(&self, bucket: &str, object: &str, data: ReaderImpl, size: i64, md5_base64: &str, sha256_hex: &str, opts: &PutObjectOptions) -> Result<UploadInfo, std::io::Error> { pub async fn put_object(
let hook_reader = data;//newHook(data, opts.progress); &self,
bucket: &str,
object: &str,
data: ReaderImpl,
size: i64,
md5_base64: &str,
sha256_hex: &str,
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> {
let hook_reader = data; //newHook(data, opts.progress);
let client = self.0.clone(); let client = self.0.clone();
client.put_object_do(bucket, object, hook_reader, md5_base64, sha256_hex, size, opts).await client
.put_object_do(bucket, object, hook_reader, md5_base64, sha256_hex, size, opts)
.await
} }
pub async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: PutObjectOptions) -> Result<String, std::io::Error> { pub async fn new_multipart_upload(
&self,
bucket: &str,
object: &str,
opts: PutObjectOptions,
) -> Result<String, std::io::Error> {
let client = self.0.clone(); let client = self.0.clone();
let result = client.initiate_multipart_upload(bucket, object, &opts).await?; let result = client.initiate_multipart_upload(bucket, object, &opts).await?;
Ok(result.upload_id) Ok(result.upload_id)
} }
pub fn list_multipart_uploads(&self, bucket: &str, prefix: &str, key_marker: &str, upload_id_marker: &str, delimiter: &str, max_uploads: i64) -> Result<ListMultipartUploadsResult, std::io::Error> { pub fn list_multipart_uploads(
&self,
bucket: &str,
prefix: &str,
key_marker: &str,
upload_id_marker: &str,
delimiter: &str,
max_uploads: i64,
) -> Result<ListMultipartUploadsResult, std::io::Error> {
let client = self.0.clone(); let client = self.0.clone();
client.list_multipart_uploads_query(bucket, key_marker, upload_id_marker, prefix, delimiter, max_uploads) client.list_multipart_uploads_query(bucket, key_marker, upload_id_marker, prefix, delimiter, max_uploads)
} }
pub async fn put_object_part(&self, bucket: &str, object: &str, upload_id: &str, part_id: i64, pub async fn put_object_part(
data: ReaderImpl, size: i64, opts: PutObjectPartOptions &self,
bucket: &str,
object: &str,
upload_id: &str,
part_id: i64,
data: ReaderImpl,
size: i64,
opts: PutObjectPartOptions,
) -> Result<ObjectPart, std::io::Error> { ) -> Result<ObjectPart, std::io::Error> {
let mut p = UploadPartParams { let mut p = UploadPartParams {
bucket_name: bucket.to_string(), bucket_name: bucket.to_string(),
object_name: object.to_string(), object_name: object.to_string(),
upload_id: upload_id.to_string(), upload_id: upload_id.to_string(),
reader: data, reader: data,
part_number: part_id, part_number: part_id,
md5_base64: opts.md5_base64, md5_base64: opts.md5_base64,
sha256_hex: opts.sha256_hex, sha256_hex: opts.sha256_hex,
size: size, size: size,
//sse: opts.sse, //sse: opts.sse,
stream_sha256: !opts.disable_content_sha256, stream_sha256: !opts.disable_content_sha256,
custom_header: opts.custom_header, custom_header: opts.custom_header,
trailer: opts.trailer, trailer: opts.trailer,
}; };
let client = self.0.clone(); let client = self.0.clone();
client.upload_part(&mut p).await client.upload_part(&mut p).await
} }
pub async fn list_object_parts(&self, bucket: &str, object: &str, upload_id: &str, part_number_marker: i64, max_parts: i64) -> Result<ListObjectPartsResult, std::io::Error> { pub async fn list_object_parts(
&self,
bucket: &str,
object: &str,
upload_id: &str,
part_number_marker: i64,
max_parts: i64,
) -> Result<ListObjectPartsResult, std::io::Error> {
let client = self.0.clone(); let client = self.0.clone();
client.list_object_parts_query(bucket, object, upload_id, part_number_marker, max_parts).await client
.list_object_parts_query(bucket, object, upload_id, part_number_marker, max_parts)
.await
} }
pub async fn complete_multipart_upload(&self, bucket: &str, object: &str, upload_id: &str, parts: &[CompletePart], opts: PutObjectOptions) -> Result<UploadInfo, std::io::Error> { pub async fn complete_multipart_upload(
&self,
bucket: &str,
object: &str,
upload_id: &str,
parts: &[CompletePart],
opts: PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> {
let client = self.0.clone(); let client = self.0.clone();
let res = client.complete_multipart_upload(bucket, object, upload_id, CompleteMultipartUpload { let res = client
parts: parts.to_vec(), .complete_multipart_upload(bucket, object, upload_id, CompleteMultipartUpload { parts: parts.to_vec() }, &opts)
}, &opts).await?; .await?;
Ok(res) Ok(res)
} }
@@ -692,7 +825,12 @@ impl TransitionCore {
client.put_bucket_policy(bucket_name, bucket_policy).await client.put_bucket_policy(bucket_name, bucket_policy).await
} }
pub async fn get_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> { pub async fn get_object(
&self,
bucket_name: &str,
object_name: &str,
opts: &GetObjectOptions,
) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> {
let client = self.0.clone(); let client = self.0.clone();
client.get_object_inner(bucket_name, object_name, opts).await client.get_object_inner(bucket_name, object_name, opts).await
} }
@@ -709,8 +847,8 @@ pub struct PutObjectPartOptions {
#[derive(Debug, Clone, Deserialize, Serialize)] #[derive(Debug, Clone, Deserialize, Serialize)]
pub struct ObjectInfo { pub struct ObjectInfo {
pub etag: String, pub etag: String,
pub name: String, pub name: String,
pub mod_time: OffsetDateTime, pub mod_time: OffsetDateTime,
pub size: usize, pub size: usize,
pub content_type: Option<String>, pub content_type: Option<String>,
@@ -730,16 +868,16 @@ pub struct ObjectInfo {
#[serde(skip, default = "replication_status_default")] #[serde(skip, default = "replication_status_default")]
pub replication_status: ReplicationStatus, pub replication_status: ReplicationStatus,
pub replication_ready: bool, pub replication_ready: bool,
pub expiration: OffsetDateTime, pub expiration: OffsetDateTime,
pub expiration_rule_id: String, pub expiration_rule_id: String,
pub num_versions: usize, pub num_versions: usize,
pub restore: RestoreInfo, pub restore: RestoreInfo,
pub checksum_crc32: String, pub checksum_crc32: String,
pub checksum_crc32c: String, pub checksum_crc32c: String,
pub checksum_sha1: String, pub checksum_sha1: String,
pub checksum_sha256: String, pub checksum_sha256: String,
pub checksum_crc64nvme: String, pub checksum_crc64nvme: String,
pub checksum_mode: String, pub checksum_mode: String,
} }
@@ -767,22 +905,21 @@ impl Default for ObjectInfo {
version_id: Uuid::nil(), version_id: Uuid::nil(),
replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING), replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING),
replication_ready: false, replication_ready: false,
expiration: OffsetDateTime::now_utc(), expiration: OffsetDateTime::now_utc(),
expiration_rule_id: "".to_string(), expiration_rule_id: "".to_string(),
num_versions: 0, num_versions: 0,
restore: RestoreInfo::default(), restore: RestoreInfo::default(),
checksum_crc32: "".to_string(), checksum_crc32: "".to_string(),
checksum_crc32c: "".to_string(), checksum_crc32c: "".to_string(),
checksum_sha1: "".to_string(), checksum_sha1: "".to_string(),
checksum_sha256: "".to_string(), checksum_sha256: "".to_string(),
checksum_crc64nvme: "".to_string(), checksum_crc64nvme: "".to_string(),
checksum_mode: "".to_string(), checksum_mode: "".to_string(),
} }
} }
} }
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize, Debug, Clone)]
#[derive(Debug, Clone)]
pub struct RestoreInfo { pub struct RestoreInfo {
ongoing_restore: bool, ongoing_restore: bool,
expiry_time: OffsetDateTime, expiry_time: OffsetDateTime,
@@ -809,19 +946,19 @@ pub struct ObjectMultipartInfo {
} }
pub struct UploadInfo { pub struct UploadInfo {
pub bucket: String, pub bucket: String,
pub key: String, pub key: String,
pub etag: String, pub etag: String,
pub size: i64, pub size: i64,
pub last_modified: OffsetDateTime, pub last_modified: OffsetDateTime,
pub location: String, pub location: String,
pub version_id: String, pub version_id: String,
pub expiration: OffsetDateTime, pub expiration: OffsetDateTime,
pub expiration_rule_id: String, pub expiration_rule_id: String,
pub checksum_crc32: String, pub checksum_crc32: String,
pub checksum_crc32c: String, pub checksum_crc32c: String,
pub checksum_sha1: String, pub checksum_sha1: String,
pub checksum_sha256: String, pub checksum_sha256: String,
pub checksum_crc64nvme: String, pub checksum_crc64nvme: String,
pub checksum_mode: String, pub checksum_mode: String,
} }
ecstore/src/error.rs
+22 -16
View File
@@ -1,4 +1,4 @@
use s3s::{S3ErrorCode, S3Error}; use s3s::{S3Error, S3ErrorCode};
use rustfs_utils::path::decode_dir_object; use rustfs_utils::path::decode_dir_object;
@@ -727,16 +727,16 @@ pub fn to_object_err(err: Error, params: Vec<&str>) -> Error {
} }
pub fn is_network_or_host_down(err: &str, expect_timeouts: bool) -> bool { pub fn is_network_or_host_down(err: &str, expect_timeouts: bool) -> bool {
err.contains("Connection closed by foreign host") || err.contains("Connection closed by foreign host")
err.contains("TLS handshake timeout") || || err.contains("TLS handshake timeout")
err.contains("i/o timeout") || || err.contains("i/o timeout")
err.contains("connection timed out") || || err.contains("connection timed out")
err.contains("connection reset by peer") || || err.contains("connection reset by peer")
err.contains("broken pipe") || || err.contains("broken pipe")
err.to_lowercase().contains("503 service unavailable") || || err.to_lowercase().contains("503 service unavailable")
err.contains("use of closed network connection") || || err.contains("use of closed network connection")
err.contains("An existing connection was forcibly closed by the remote host") || || err.contains("An existing connection was forcibly closed by the remote host")
err.contains("client error (Connect)") || err.contains("client error (Connect)")
} }
#[derive(Debug, Default, PartialEq, Eq)] #[derive(Debug, Default, PartialEq, Eq)]
@@ -834,7 +834,7 @@ pub fn error_resp_to_object_err(err: ErrorResponse, params: Vec<&str>) -> std::i
err = std::io::Error::other(StorageError::BucketNameInvalid(bucket)); err = std::io::Error::other(StorageError::BucketNameInvalid(bucket));
} }
S3ErrorCode::InvalidPart => { S3ErrorCode::InvalidPart => {
err = std::io::Error::other(StorageError::InvalidPart(0, bucket, object/* , version_id */)); err = std::io::Error::other(StorageError::InvalidPart(0, bucket, object /* , version_id */));
} }
S3ErrorCode::NoSuchBucket => { S3ErrorCode::NoSuchBucket => {
err = std::io::Error::other(StorageError::BucketNotFound(bucket)); err = std::io::Error::other(StorageError::BucketNotFound(bucket));
@@ -848,7 +848,7 @@ pub fn error_resp_to_object_err(err: ErrorResponse, params: Vec<&str>) -> std::i
} }
S3ErrorCode::NoSuchVersion => { S3ErrorCode::NoSuchVersion => {
if object != "" { if object != "" {
err = std::io::Error::other(StorageError::ObjectNotFound(bucket, object));//, version_id); err = std::io::Error::other(StorageError::ObjectNotFound(bucket, object)); //, version_id);
} else { } else {
err = std::io::Error::other(StorageError::BucketNotFound(bucket)); err = std::io::Error::other(StorageError::BucketNotFound(bucket));
} }
@@ -894,9 +894,15 @@ pub fn storage_to_object_err(err: Error, params: Vec<&str>) -> S3Error {
StorageError::InvalidArgument(bucket, object, version_id) => { StorageError::InvalidArgument(bucket, object, version_id) => {
s3_error!(InvalidArgument, "Invalid arguments provided for {}/{}-{}", bucket, object, version_id) s3_error!(InvalidArgument, "Invalid arguments provided for {}/{}-{}", bucket, object, version_id)
}*/ }*/
StorageError::MethodNotAllowed => { StorageError::MethodNotAllowed => S3Error::with_message(
S3Error::with_message(S3ErrorCode::MethodNotAllowed, ObjectApiError::MethodNotAllowed(GenericError {bucket: bucket, object: object, ..Default::default()}).to_string()) S3ErrorCode::MethodNotAllowed,
} ObjectApiError::MethodNotAllowed(GenericError {
bucket: bucket,
object: object,
..Default::default()
})
.to_string(),
),
/*StorageError::BucketNotFound(bucket) => { /*StorageError::BucketNotFound(bucket) => {
s3_error!(NoSuchBucket, "bucket not found {}", bucket) s3_error!(NoSuchBucket, "bucket not found {}", bucket)
} }
ecstore/src/event/name.rs
+40 -118
View File
@@ -55,7 +55,7 @@ impl EventName {
} }
impl AsRef<str> for EventName { impl AsRef<str> for EventName {
fn as_ref(&self) -> &str { fn as_ref(&self) -> &str {
match self { match self {
EventName::BucketCreated => "s3:BucketCreated:*", EventName::BucketCreated => "s3:BucketCreated:*",
EventName::BucketRemoved => "s3:BucketRemoved:*", EventName::BucketRemoved => "s3:BucketRemoved:*",
@@ -103,123 +103,45 @@ impl AsRef<str> for EventName {
impl From<&str> for EventName { impl From<&str> for EventName {
fn from(s: &str) -> Self { fn from(s: &str) -> Self {
match s { match s {
"s3:BucketCreated:*" => { "s3:BucketCreated:*" => EventName::BucketCreated,
EventName::BucketCreated "s3:BucketRemoved:*" => EventName::BucketRemoved,
} "s3:ObjectAccessed:*" => EventName::ObjectAccessedAll,
"s3:BucketRemoved:*" => { "s3:ObjectAccessed:Get" => EventName::ObjectAccessedGet,
EventName::BucketRemoved "s3:ObjectAccessed:GetRetention" => EventName::ObjectAccessedGetRetention,
} "s3:ObjectAccessed:GetLegalHold" => EventName::ObjectAccessedGetLegalHold,
"s3:ObjectAccessed:*" => { "s3:ObjectAccessed:Head" => EventName::ObjectAccessedHead,
EventName::ObjectAccessedAll "s3:ObjectAccessed:Attributes" => EventName::ObjectAccessedAttributes,
} "s3:ObjectCreated:*" => EventName::ObjectCreatedAll,
"s3:ObjectAccessed:Get" => { "s3:ObjectCreated:CompleteMultipartUpload" => EventName::ObjectCreatedCompleteMultipartUpload,
EventName::ObjectAccessedGet "s3:ObjectCreated:Copy" => EventName::ObjectCreatedCopy,
} "s3:ObjectCreated:Post" => EventName::ObjectCreatedPost,
"s3:ObjectAccessed:GetRetention" => { "s3:ObjectCreated:Put" => EventName::ObjectCreatedPut,
EventName::ObjectAccessedGetRetention "s3:ObjectCreated:PutRetention" => EventName::ObjectCreatedPutRetention,
} "s3:ObjectCreated:PutLegalHold" => EventName::ObjectCreatedPutLegalHold,
"s3:ObjectAccessed:GetLegalHold" => { "s3:ObjectCreated:PutTagging" => EventName::ObjectCreatedPutTagging,
EventName::ObjectAccessedGetLegalHold "s3:ObjectCreated:DeleteTagging" => EventName::ObjectCreatedDeleteTagging,
} "s3:ObjectRemoved:*" => EventName::ObjectRemovedAll,
"s3:ObjectAccessed:Head" => { "s3:ObjectRemoved:Delete" => EventName::ObjectRemovedDelete,
EventName::ObjectAccessedHead "s3:ObjectRemoved:DeleteMarkerCreated" => EventName::ObjectRemovedDeleteMarkerCreated,
} "s3:ObjectRemoved:NoOP" => EventName::ObjectRemovedNoOP,
"s3:ObjectAccessed:Attributes" => { "s3:ObjectRemoved:DeleteAllVersions" => EventName::ObjectRemovedDeleteAllVersions,
EventName::ObjectAccessedAttributes "s3:LifecycleDelMarkerExpiration:Delete" => EventName::ILMDelMarkerExpirationDelete,
} "s3:Replication:*" => EventName::ObjectReplicationAll,
"s3:ObjectCreated:*" => { "s3:Replication:OperationFailedReplication" => EventName::ObjectReplicationFailed,
EventName::ObjectCreatedAll "s3:Replication:OperationCompletedReplication" => EventName::ObjectReplicationComplete,
} "s3:Replication:OperationMissedThreshold" => EventName::ObjectReplicationMissedThreshold,
"s3:ObjectCreated:CompleteMultipartUpload" => { "s3:Replication:OperationReplicatedAfterThreshold" => EventName::ObjectReplicationReplicatedAfterThreshold,
EventName::ObjectCreatedCompleteMultipartUpload "s3:Replication:OperationNotTracked" => EventName::ObjectReplicationNotTracked,
} "s3:ObjectRestore:*" => EventName::ObjectRestoreAll,
"s3:ObjectCreated:Copy" => { "s3:ObjectRestore:Post" => EventName::ObjectRestorePost,
EventName::ObjectCreatedCopy "s3:ObjectRestore:Completed" => EventName::ObjectRestoreCompleted,
} "s3:ObjectTransition:Failed" => EventName::ObjectTransitionFailed,
"s3:ObjectCreated:Post" => { "s3:ObjectTransition:Complete" => EventName::ObjectTransitionComplete,
EventName::ObjectCreatedPost "s3:ObjectTransition:*" => EventName::ObjectTransitionAll,
} "s3:Scanner:ManyVersions" => EventName::ObjectManyVersions,
"s3:ObjectCreated:Put" => { "s3:Scanner:LargeVersions" => EventName::ObjectLargeVersions,
EventName::ObjectCreatedPut "s3:Scanner:BigPrefix" => EventName::PrefixManyFolders,
} _ => EventName::Everything,
"s3:ObjectCreated:PutRetention" => {
EventName::ObjectCreatedPutRetention
}
"s3:ObjectCreated:PutLegalHold" => {
EventName::ObjectCreatedPutLegalHold
}
"s3:ObjectCreated:PutTagging" => {
EventName::ObjectCreatedPutTagging
}
"s3:ObjectCreated:DeleteTagging" => {
EventName::ObjectCreatedDeleteTagging
}
"s3:ObjectRemoved:*" => {
EventName::ObjectRemovedAll
}
"s3:ObjectRemoved:Delete" => {
EventName::ObjectRemovedDelete
}
"s3:ObjectRemoved:DeleteMarkerCreated" => {
EventName::ObjectRemovedDeleteMarkerCreated
}
"s3:ObjectRemoved:NoOP" => {
EventName::ObjectRemovedNoOP
}
"s3:ObjectRemoved:DeleteAllVersions" => {
EventName::ObjectRemovedDeleteAllVersions
}
"s3:LifecycleDelMarkerExpiration:Delete" => {
EventName::ILMDelMarkerExpirationDelete
}
"s3:Replication:*" => {
EventName::ObjectReplicationAll
}
"s3:Replication:OperationFailedReplication" => {
EventName::ObjectReplicationFailed
}
"s3:Replication:OperationCompletedReplication" => {
EventName::ObjectReplicationComplete
}
"s3:Replication:OperationMissedThreshold" => {
EventName::ObjectReplicationMissedThreshold
}
"s3:Replication:OperationReplicatedAfterThreshold" => {
EventName::ObjectReplicationReplicatedAfterThreshold
}
"s3:Replication:OperationNotTracked" => {
EventName::ObjectReplicationNotTracked
}
"s3:ObjectRestore:*" => {
EventName::ObjectRestoreAll
}
"s3:ObjectRestore:Post" => {
EventName::ObjectRestorePost
}
"s3:ObjectRestore:Completed" => {
EventName::ObjectRestoreCompleted
}
"s3:ObjectTransition:Failed" => {
EventName::ObjectTransitionFailed
}
"s3:ObjectTransition:Complete" => {
EventName::ObjectTransitionComplete
}
"s3:ObjectTransition:*" => {
EventName::ObjectTransitionAll
}
"s3:Scanner:ManyVersions" => {
EventName::ObjectManyVersions
}
"s3:Scanner:LargeVersions" => {
EventName::ObjectLargeVersions
}
"s3:Scanner:BigPrefix" => {
EventName::PrefixManyFolders
}
_ => {
EventName::Everything
}
} }
} }
} }
ecstore/src/event/targetid.rs
+2 -2
View File
@@ -1,6 +1,6 @@
pub struct TargetID { pub struct TargetID {
id: String, id: String,
name: String, name: String,
} }
impl TargetID { impl TargetID {
ecstore/src/event/targetlist.rs
+4 -4
View File
@@ -4,9 +4,9 @@ use super::targetid::TargetID;
#[derive(Default)] #[derive(Default)]
pub struct TargetList { pub struct TargetList {
pub current_send_calls: AtomicI64, pub current_send_calls: AtomicI64,
pub total_events: AtomicI64, pub total_events: AtomicI64,
pub events_skipped: AtomicI64, pub events_skipped: AtomicI64,
pub events_errors_total: AtomicI64, pub events_errors_total: AtomicI64,
//pub targets: HashMap<TargetID, Target>, //pub targets: HashMap<TargetID, Target>,
//pub queue: AsyncEvent, //pub queue: AsyncEvent,
@@ -26,6 +26,6 @@ struct TargetStat {
} }
struct TargetIDResult { struct TargetIDResult {
id: TargetID, id: TargetID,
err: std::io::Error, err: std::io::Error,
} }
ecstore/src/event_notification.rs
+5 -7
View File
@@ -2,11 +2,11 @@ use std::collections::HashMap;
use std::sync::Arc; use std::sync::Arc;
use tokio::sync::RwLock; use tokio::sync::RwLock;
use crate::store_api::ObjectInfo; use crate::bucket::metadata::BucketMetadata;
use crate::event::name::EventName; use crate::event::name::EventName;
use crate::event::targetlist::TargetList; use crate::event::targetlist::TargetList;
use crate::store::ECStore; use crate::store::ECStore;
use crate::bucket::metadata::BucketMetadata; use crate::store_api::ObjectInfo;
pub struct EventNotifier { pub struct EventNotifier {
target_list: TargetList, target_list: TargetList,
@@ -16,7 +16,7 @@ pub struct EventNotifier {
impl EventNotifier { impl EventNotifier {
pub fn new() -> Arc<RwLock<Self>> { pub fn new() -> Arc<RwLock<Self>> {
Arc::new(RwLock::new(Self { Arc::new(RwLock::new(Self {
target_list: TargetList::new(), target_list: TargetList::new(),
//bucket_rules_map: HashMap::new(), //bucket_rules_map: HashMap::new(),
})) }))
} }
@@ -54,8 +54,6 @@ pub struct EventArgs {
pub user_agent: String, pub user_agent: String,
} }
impl EventArgs { impl EventArgs {}
}
pub fn send_event(args: EventArgs) { pub fn send_event(args: EventArgs) {}
}
ecstore/src/global.rs
+26 -26
View File
@@ -9,13 +9,13 @@ use uuid::Uuid;
use crate::heal::mrf::MRFState; use crate::heal::mrf::MRFState;
use crate::{ use crate::{
bucket::lifecycle::bucket_lifecycle_ops::LifecycleSys,
disk::DiskStore, disk::DiskStore,
endpoints::{EndpointServerPools, PoolEndpoints, SetupType}, endpoints::{EndpointServerPools, PoolEndpoints, SetupType},
heal::{background_heal_ops::HealRoutine, heal_ops::AllHealState},
bucket::lifecycle::bucket_lifecycle_ops::LifecycleSys,
tier::tier::TierConfigMgr,
event_notification::EventNotifier, event_notification::EventNotifier,
heal::{background_heal_ops::HealRoutine, heal_ops::AllHealState},
store::ECStore, store::ECStore,
tier::tier::TierConfigMgr,
}; };
pub const DISK_ASSUME_UNKNOWN_SIZE: u64 = 1 << 30; pub const DISK_ASSUME_UNKNOWN_SIZE: u64 = 1 << 30;
@@ -26,29 +26,29 @@ pub const DISK_RESERVE_FRACTION: f64 = 0.15;
pub const DEFAULT_PORT: u16 = 9000; pub const DEFAULT_PORT: u16 = 9000;
lazy_static! { lazy_static! {
static ref GLOBAL_RUSTFS_PORT: OnceLock<u16> = OnceLock::new(); static ref GLOBAL_RUSTFS_PORT: OnceLock<u16> = OnceLock::new();
pub static ref GLOBAL_OBJECT_API: OnceLock<Arc<ECStore>> = OnceLock::new(); pub static ref GLOBAL_OBJECT_API: OnceLock<Arc<ECStore>> = OnceLock::new();
pub static ref GLOBAL_LOCAL_DISK: Arc<RwLock<Vec<Option<DiskStore>>>> = Arc::new(RwLock::new(Vec::new())); pub static ref GLOBAL_LOCAL_DISK: Arc<RwLock<Vec<Option<DiskStore>>>> = Arc::new(RwLock::new(Vec::new()));
pub static ref GLOBAL_IsErasure: RwLock<bool> = RwLock::new(false); pub static ref GLOBAL_IsErasure: RwLock<bool> = RwLock::new(false);
pub static ref GLOBAL_IsDistErasure: RwLock<bool> = RwLock::new(false); pub static ref GLOBAL_IsDistErasure: RwLock<bool> = RwLock::new(false);
pub static ref GLOBAL_IsErasureSD: RwLock<bool> = RwLock::new(false); pub static ref GLOBAL_IsErasureSD: RwLock<bool> = RwLock::new(false);
pub static ref GLOBAL_LOCAL_DISK_MAP: Arc<RwLock<HashMap<String, Option<DiskStore>>>> = Arc::new(RwLock::new(HashMap::new())); pub static ref GLOBAL_LOCAL_DISK_MAP: Arc<RwLock<HashMap<String, Option<DiskStore>>>> = Arc::new(RwLock::new(HashMap::new()));
pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc<RwLock<TypeLocalDiskSetDrives>> = Arc::new(RwLock::new(Vec::new())); pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc<RwLock<TypeLocalDiskSetDrives>> = Arc::new(RwLock::new(Vec::new()));
pub static ref GLOBAL_Endpoints: OnceLock<EndpointServerPools> = OnceLock::new(); pub static ref GLOBAL_Endpoints: OnceLock<EndpointServerPools> = OnceLock::new();
pub static ref GLOBAL_RootDiskThreshold: RwLock<u64> = RwLock::new(0); pub static ref GLOBAL_RootDiskThreshold: RwLock<u64> = RwLock::new(0);
pub static ref GLOBAL_BackgroundHealRoutine: Arc<HealRoutine> = HealRoutine::new(); pub static ref GLOBAL_BackgroundHealRoutine: Arc<HealRoutine> = HealRoutine::new();
pub static ref GLOBAL_BackgroundHealState: Arc<AllHealState> = AllHealState::new(false); pub static ref GLOBAL_BackgroundHealState: Arc<AllHealState> = AllHealState::new(false);
pub static ref GLOBAL_TierConfigMgr: Arc<RwLock<TierConfigMgr>> = TierConfigMgr::new(); pub static ref GLOBAL_TierConfigMgr: Arc<RwLock<TierConfigMgr>> = TierConfigMgr::new();
pub static ref GLOBAL_LifecycleSys: Arc<LifecycleSys> = LifecycleSys::new(); pub static ref GLOBAL_LifecycleSys: Arc<LifecycleSys> = LifecycleSys::new();
pub static ref GLOBAL_EventNotifier: Arc<RwLock<EventNotifier>> = EventNotifier::new(); pub static ref GLOBAL_EventNotifier: Arc<RwLock<EventNotifier>> = EventNotifier::new();
//pub static ref GLOBAL_RemoteTargetTransport //pub static ref GLOBAL_RemoteTargetTransport
pub static ref GLOBAL_ALlHealState: Arc<AllHealState> = AllHealState::new(false); pub static ref GLOBAL_ALlHealState: Arc<AllHealState> = AllHealState::new(false);
pub static ref GLOBAL_MRFState: Arc<MRFState> = Arc::new(MRFState::new()); pub static ref GLOBAL_MRFState: Arc<MRFState> = Arc::new(MRFState::new());
static ref globalDeploymentIDPtr: OnceLock<Uuid> = OnceLock::new(); static ref globalDeploymentIDPtr: OnceLock<Uuid> = OnceLock::new();
pub static ref GLOBAL_BOOT_TIME: OnceCell<SystemTime> = OnceCell::new(); pub static ref GLOBAL_BOOT_TIME: OnceCell<SystemTime> = OnceCell::new();
pub static ref GLOBAL_LocalNodeName: String = "127.0.0.1:9000".to_string(); pub static ref GLOBAL_LocalNodeName: String = "127.0.0.1:9000".to_string();
pub static ref GLOBAL_LocalNodeNameHex: String = rustfs_utils::crypto::hex(GLOBAL_LocalNodeName.as_bytes()); pub static ref GLOBAL_LocalNodeNameHex: String = rustfs_utils::crypto::hex(GLOBAL_LocalNodeName.as_bytes());
pub static ref GLOBAL_NodeNamesHex: HashMap<String, ()> = HashMap::new();} pub static ref GLOBAL_NodeNamesHex: HashMap<String, ()> = HashMap::new();}
/// Get the global rustfs port /// Get the global rustfs port
pub fn global_rustfs_port() -> u16 { pub fn global_rustfs_port() -> u16 {
ecstore/src/heal/data_scanner.rs
+108 -62
View File
@@ -6,16 +6,15 @@ use std::{
path::{Path, PathBuf}, path::{Path, PathBuf},
pin::Pin, pin::Pin,
sync::{ sync::{
atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering},
Arc, Arc,
atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering},
}, },
time::{Duration, SystemTime}, time::{Duration, SystemTime},
}; };
use time::{self, OffsetDateTime};
use common::defer; use common::defer;
use time::{self, OffsetDateTime};
use rustfs_utils::path::encode_dir_object;
use super::{ use super::{
data_scanner_metric::{ScannerMetric, ScannerMetrics, globalScannerMetrics}, data_scanner_metric::{ScannerMetric, ScannerMetrics, globalScannerMetrics},
data_usage::{DATA_USAGE_BLOOM_NAME_PATH, store_data_usage_in_backend}, data_usage::{DATA_USAGE_BLOOM_NAME_PATH, store_data_usage_in_backend},
@@ -23,28 +22,25 @@ use super::{
heal_commands::{HEAL_DEEP_SCAN, HEAL_NORMAL_SCAN, HealScanMode}, heal_commands::{HEAL_DEEP_SCAN, HEAL_NORMAL_SCAN, HealScanMode},
}; };
use crate::bucket::{ use crate::bucket::{
object_lock::objectlock_sys::{ object_lock::objectlock_sys::{BucketObjectLockSys, enforce_retention_for_deletion},
enforce_retention_for_deletion, utils::is_meta_bucketname,
BucketObjectLockSys,
}, utils::is_meta_bucketname,
}; };
use crate::cmd::bucket_replication::queue_replication_heal;
use crate::event::name::EventName;
use crate::{ use crate::{
bucket::{ bucket::{
lifecycle::{ lifecycle::{
bucket_lifecycle_audit::{LcAuditEvent, LcEventSrc}, bucket_lifecycle_audit::{LcAuditEvent, LcEventSrc},
lifecycle::{self, Lifecycle, ExpirationOptions}, bucket_lifecycle_ops::{self, GLOBAL_ExpiryState, GLOBAL_TransitionState, LifecycleOps, expire_transitioned_object},
bucket_lifecycle_ops::{ lifecycle::{self, ExpirationOptions, Lifecycle},
self, GLOBAL_ExpiryState, GLOBAL_TransitionState, LifecycleOps,
expire_transitioned_object,
},
}, },
metadata_sys metadata_sys,
}, },
event_notification::{send_event, EventArgs}, event_notification::{EventArgs, send_event},
global::GLOBAL_LocalNodeName, heal::{data_scanner}, global::GLOBAL_LocalNodeName,
heal::data_scanner,
store_api::{ObjectOptions, ObjectToDelete, StorageAPI}, store_api::{ObjectOptions, ObjectToDelete, StorageAPI},
}; };
use crate::cmd::bucket_replication::queue_replication_heal;
use crate::{ use crate::{
bucket::{versioning::VersioningApi, versioning_sys::BucketVersioningSys}, bucket::{versioning::VersioningApi, versioning_sys::BucketVersioningSys},
cmd::bucket_replication::ReplicationStatusType, cmd::bucket_replication::ReplicationStatusType,
@@ -70,7 +66,6 @@ use crate::{
peer::is_reserved_or_invalid_bucket, peer::is_reserved_or_invalid_bucket,
store::ECStore, store::ECStore,
}; };
use crate::event::name::EventName;
use crate::{disk::DiskAPI, store_api::ObjectInfo}; use crate::{disk::DiskAPI, store_api::ObjectInfo};
use crate::{ use crate::{
disk::error::DiskError, disk::error::DiskError,
@@ -82,8 +77,12 @@ use lazy_static::lazy_static;
use rand::Rng; use rand::Rng;
use rmp_serde::{Deserializer, Serializer}; use rmp_serde::{Deserializer, Serializer};
use rustfs_filemeta::{FileInfo, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; use rustfs_filemeta::{FileInfo, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
use rustfs_utils::path::encode_dir_object;
use rustfs_utils::path::{SLASH_SEPARATOR, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path}; use rustfs_utils::path::{SLASH_SEPARATOR, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path};
use s3s::dto::{BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleRule, ReplicationConfiguration, ReplicationRuleStatus, VersioningConfiguration}; use s3s::dto::{
BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleRule, ReplicationConfiguration,
ReplicationRuleStatus, VersioningConfiguration,
};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tokio::{ use tokio::{
sync::{ sync::{
@@ -561,20 +560,33 @@ impl ScannerItem {
lr = BucketObjectLockSys::get(&self.bucket).await; lr = BucketObjectLockSys::get(&self.bucket).await;
rcfg = if let Ok(replication_config) = metadata_sys::get_replication_config(&self.bucket).await { rcfg = if let Ok(replication_config) = metadata_sys::get_replication_config(&self.bucket).await {
Some(replication_config) Some(replication_config)
} else { None }; } else {
None
};
} }
let lc_evt = eval_action_from_lifecycle(self.lifecycle.as_ref().expect("err"), lr, rcfg, oi).await; let lc_evt = eval_action_from_lifecycle(self.lifecycle.as_ref().expect("err"), lr, rcfg, oi).await;
if self.debug { if self.debug {
if !version_id.is_none() { if !version_id.is_none() {
info!("lifecycle: {} (version-id={}), Initial scan: {}", self.object_path().to_string_lossy().to_string(), version_id.expect("err"), lc_evt.action); info!(
"lifecycle: {} (version-id={}), Initial scan: {}",
self.object_path().to_string_lossy().to_string(),
version_id.expect("err"),
lc_evt.action
);
} else { } else {
info!("lifecycle: {} Initial scan: {}", self.object_path().to_string_lossy().to_string(), lc_evt.action); info!(
"lifecycle: {} Initial scan: {}",
self.object_path().to_string_lossy().to_string(),
lc_evt.action
);
} }
} }
match lc_evt.action { match lc_evt.action {
lifecycle::IlmAction::DeleteVersionAction | lifecycle::IlmAction::DeleteAllVersionsAction | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { lifecycle::IlmAction::DeleteVersionAction
| lifecycle::IlmAction::DeleteAllVersionsAction
| lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => {
size = 0; size = 0;
} }
lifecycle::IlmAction::DeleteAction => { lifecycle::IlmAction::DeleteAction => {
@@ -582,7 +594,7 @@ impl ScannerItem {
size = 0 size = 0
} }
} }
_ => () _ => (),
} }
apply_lifecycle_action(&lc_evt, &LcEventSrc::Scanner, oi).await; apply_lifecycle_action(&lc_evt, &LcEventSrc::Scanner, oi).await;
@@ -612,7 +624,9 @@ impl ScannerItem {
let lock_enabled = if let Some(rcfg) = BucketObjectLockSys::get(&self.bucket).await { let lock_enabled = if let Some(rcfg) = BucketObjectLockSys::get(&self.bucket).await {
rcfg.mode.is_some() rcfg.mode.is_some()
} else { false }; } else {
false
};
let vcfg = BucketVersioningSys::get(&self.bucket).await?; let vcfg = BucketVersioningSys::get(&self.bucket).await?;
let versioned = match BucketVersioningSys::get(&self.bucket).await { let versioned = match BucketVersioningSys::get(&self.bucket).await {
@@ -633,28 +647,43 @@ impl ScannerItem {
return Ok(object_infos); return Ok(object_infos);
} }
let event = self.lifecycle.as_ref().expect("lifecycle err.").noncurrent_versions_expiration_limit(&lifecycle::ObjectOpts { let event = self
name: self.object_path().to_string_lossy().to_string(), .lifecycle
..Default::default() .as_ref()
}).await; .expect("lifecycle err.")
.noncurrent_versions_expiration_limit(&lifecycle::ObjectOpts {
name: self.object_path().to_string_lossy().to_string(),
..Default::default()
})
.await;
let lim = event.newer_noncurrent_versions; let lim = event.newer_noncurrent_versions;
if lim == 0 || fivs.len() <= lim+1 { if lim == 0 || fivs.len() <= lim + 1 {
for fi in fivs.iter() { for fi in fivs.iter() {
object_infos.push(ObjectInfo::from_file_info(fi, &self.bucket, &self.object_path().to_string_lossy(), versioned)); object_infos.push(ObjectInfo::from_file_info(
fi,
&self.bucket,
&self.object_path().to_string_lossy(),
versioned,
));
} }
return Ok(object_infos); return Ok(object_infos);
} }
let overflow_versions = &fivs[lim+1..]; let overflow_versions = &fivs[lim + 1..];
for fi in fivs[..lim+1].iter() { for fi in fivs[..lim + 1].iter() {
object_infos.push(ObjectInfo::from_file_info(fi, &self.bucket, &self.object_path().to_string_lossy(), versioned)); object_infos.push(ObjectInfo::from_file_info(
fi,
&self.bucket,
&self.object_path().to_string_lossy(),
versioned,
));
} }
let mut to_del = Vec::<ObjectToDelete>::with_capacity(overflow_versions.len()); let mut to_del = Vec::<ObjectToDelete>::with_capacity(overflow_versions.len());
for fi in overflow_versions.iter() { for fi in overflow_versions.iter() {
let obj = ObjectInfo::from_file_info(fi, &self.bucket, &self.object_path().to_string_lossy(), versioned); let obj = ObjectInfo::from_file_info(fi, &self.bucket, &self.object_path().to_string_lossy(), versioned);
if lock_enabled && enforce_retention_for_deletion(&obj) { if lock_enabled && enforce_retention_for_deletion(&obj) {
//if enforce_retention_for_deletion(&obj) { //if enforce_retention_for_deletion(&obj) {
if self.debug { if self.debug {
if obj.version_id.is_some() { if obj.version_id.is_some() {
info!("lifecycle: {} v({}) is locked, not deleting\n", obj.name, obj.version_id.expect("err")); info!("lifecycle: {} v({}) is locked, not deleting\n", obj.name, obj.version_id.expect("err"));
@@ -666,7 +695,10 @@ impl ScannerItem {
continue; continue;
} }
if OffsetDateTime::now_utc().unix_timestamp() < lifecycle::expected_expiry_time(obj.successor_mod_time.expect("err"), event.noncurrent_days as i32).unix_timestamp() { if OffsetDateTime::now_utc().unix_timestamp()
< lifecycle::expected_expiry_time(obj.successor_mod_time.expect("err"), event.noncurrent_days as i32)
.unix_timestamp()
{
object_infos.push(obj); object_infos.push(obj);
continue; continue;
} }
@@ -1535,15 +1567,18 @@ pub async fn scan_data_folder(
Ok(s.new_cache) Ok(s.new_cache)
} }
pub async fn eval_action_from_lifecycle(lc: &BucketLifecycleConfiguration, lr: Option<DefaultRetention>, rcfg: Option<(ReplicationConfiguration, OffsetDateTime)>, oi: &ObjectInfo) -> lifecycle::Event { pub async fn eval_action_from_lifecycle(
lc: &BucketLifecycleConfiguration,
lr: Option<DefaultRetention>,
rcfg: Option<(ReplicationConfiguration, OffsetDateTime)>,
oi: &ObjectInfo,
) -> lifecycle::Event {
let event = lc.eval(&oi.to_lifecycle_opts()).await; let event = lc.eval(&oi.to_lifecycle_opts()).await;
//if serverDebugLog { //if serverDebugLog {
info!("lifecycle: Secondary scan: {}", event.action); info!("lifecycle: Secondary scan: {}", event.action);
//} //}
let lock_enabled = if let Some(lr) = lr { let lock_enabled = if let Some(lr) = lr { lr.mode.is_some() } else { false };
lr.mode.is_some()
} else { false };
match event.action { match event.action {
lifecycle::IlmAction::DeleteAllVersionsAction | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { lifecycle::IlmAction::DeleteAllVersionsAction | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => {
@@ -1557,11 +1592,11 @@ pub async fn eval_action_from_lifecycle(lc: &BucketLifecycleConfiguration, lr: O
} }
if lock_enabled && enforce_retention_for_deletion(oi) { if lock_enabled && enforce_retention_for_deletion(oi) {
//if serverDebugLog { //if serverDebugLog {
if !oi.version_id.is_none() { if !oi.version_id.is_none() {
info!("lifecycle: {} v({}) is locked, not deleting", oi.name, oi.version_id.expect("err")); info!("lifecycle: {} v({}) is locked, not deleting", oi.name, oi.version_id.expect("err"));
} else { } else {
info!("lifecycle: {} is locked, not deleting", oi.name); info!("lifecycle: {} is locked, not deleting", oi.name);
} }
//} //}
return lifecycle::Event::default(); return lifecycle::Event::default();
} }
@@ -1571,7 +1606,7 @@ pub async fn eval_action_from_lifecycle(lc: &BucketLifecycleConfiguration, lr: O
} }
} }
} }
_ => () _ => (),
} }
event event
@@ -1585,7 +1620,12 @@ async fn apply_transition_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &
true true
} }
pub async fn apply_expiry_on_transitioned_object(api: Arc<ECStore>, oi: &ObjectInfo, lc_event: &lifecycle::Event, src: &LcEventSrc) -> bool { pub async fn apply_expiry_on_transitioned_object(
api: Arc<ECStore>,
oi: &ObjectInfo,
lc_event: &lifecycle::Event,
src: &LcEventSrc,
) -> bool {
let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone()); let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone());
if let Err(_err) = expire_transitioned_object(api, oi, lc_event, src).await { if let Err(_err) = expire_transitioned_object(api, oi, lc_event, src).await {
return false; return false;
@@ -1595,9 +1635,14 @@ pub async fn apply_expiry_on_transitioned_object(api: Arc<ECStore>, oi: &ObjectI
true true
} }
pub async fn apply_expiry_on_non_transitioned_objects(api: Arc<ECStore>, oi: &ObjectInfo, lc_event: &lifecycle::Event, src: &LcEventSrc) -> bool { pub async fn apply_expiry_on_non_transitioned_objects(
api: Arc<ECStore>,
oi: &ObjectInfo,
lc_event: &lifecycle::Event,
src: &LcEventSrc,
) -> bool {
let mut opts = ObjectOptions { let mut opts = ObjectOptions {
expiration: ExpirationOptions {expire: true}, expiration: ExpirationOptions { expire: true },
..Default::default() ..Default::default()
}; };
@@ -1617,7 +1662,10 @@ pub async fn apply_expiry_on_non_transitioned_objects(api: Arc<ECStore>, oi: &Ob
let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone()); let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone());
let mut dobj = api.delete_object(&oi.bucket, &encode_dir_object(&oi.name), opts).await.unwrap(); let mut dobj = api
.delete_object(&oi.bucket, &encode_dir_object(&oi.name), opts)
.await
.unwrap();
if dobj.name == "" { if dobj.name == "" {
dobj = oi.clone(); dobj = oi.clone();
} }
@@ -1630,13 +1678,9 @@ pub async fn apply_expiry_on_non_transitioned_objects(api: Arc<ECStore>, oi: &Ob
event_name = EventName::ObjectRemovedDeleteMarkerCreated; event_name = EventName::ObjectRemovedDeleteMarkerCreated;
} }
match lc_event.action { match lc_event.action {
lifecycle::IlmAction::DeleteAllVersionsAction => { lifecycle::IlmAction::DeleteAllVersionsAction => event_name = EventName::ObjectRemovedDeleteAllVersions,
event_name = EventName::ObjectRemovedDeleteAllVersions lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => event_name = EventName::ILMDelMarkerExpirationDelete,
} _ => (),
lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => {
event_name = EventName::ILMDelMarkerExpirationDelete
}
_ => ()
} }
send_event(EventArgs { send_event(EventArgs {
event_name: event_name.as_ref().to_string(), event_name: event_name.as_ref().to_string(),
@@ -1667,20 +1711,22 @@ async fn apply_expiry_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &Obje
pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool { pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
let mut success = false; let mut success = false;
match event.action { match event.action {
lifecycle::IlmAction::DeleteVersionAction | lifecycle::IlmAction::DeleteAction lifecycle::IlmAction::DeleteVersionAction
| lifecycle::IlmAction::DeleteRestoredAction | lifecycle::IlmAction::DeleteRestoredVersionAction | lifecycle::IlmAction::DeleteAction
| lifecycle::IlmAction::DeleteAllVersionsAction | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { | lifecycle::IlmAction::DeleteRestoredAction
| lifecycle::IlmAction::DeleteRestoredVersionAction
| lifecycle::IlmAction::DeleteAllVersionsAction
| lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => {
success = apply_expiry_rule(event, src, oi).await; success = apply_expiry_rule(event, src, oi).await;
} }
lifecycle::IlmAction::TransitionAction | lifecycle::IlmAction::TransitionVersionAction => { lifecycle::IlmAction::TransitionAction | lifecycle::IlmAction::TransitionVersionAction => {
success = apply_transition_rule(event, src, oi).await; success = apply_transition_rule(event, src, oi).await;
} }
_ => () _ => (),
} }
success success
} }
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use std::io::Cursor; use std::io::Cursor;
ecstore/src/heal/data_scanner_metric.rs
+1 -3
View File
@@ -312,9 +312,7 @@ impl ScannerMetrics {
pub fn time_ilm(a: lifecycle::IlmAction) -> Box<dyn Fn(u64) -> Box<dyn Fn() + Send + Sync> + Send + Sync> { pub fn time_ilm(a: lifecycle::IlmAction) -> Box<dyn Fn(u64) -> Box<dyn Fn() + Send + Sync> + Send + Sync> {
let a_clone = a as usize; let a_clone = a as usize;
if a_clone == lifecycle::IlmAction::NoneAction as usize || a_clone >= lifecycle::IlmAction::ActionCount as usize { if a_clone == lifecycle::IlmAction::NoneAction as usize || a_clone >= lifecycle::IlmAction::ActionCount as usize {
return Box::new(move |_: u64| { return Box::new(move |_: u64| Box::new(move || {}));
Box::new(move || {})
});
} }
let start = SystemTime::now(); let start = SystemTime::now();
Box::new(move |versions: u64| { Box::new(move |versions: u64| {
ecstore/src/heal/data_usage_cache.rs
+11 -10
View File
@@ -139,9 +139,9 @@ pub struct TierStats {
impl TierStats { impl TierStats {
pub fn add(&self, u: &TierStats) -> TierStats { pub fn add(&self, u: &TierStats) -> TierStats {
TierStats { TierStats {
total_size: self.total_size + u.total_size, total_size: self.total_size + u.total_size,
num_versions: self.num_versions + u.num_versions, num_versions: self.num_versions + u.num_versions,
num_objects: self.num_objects + u.num_objects, num_objects: self.num_objects + u.num_objects,
} }
} }
} }
@@ -153,9 +153,7 @@ struct AllTierStats {
impl AllTierStats { impl AllTierStats {
pub fn new() -> Self { pub fn new() -> Self {
Self { Self { tiers: HashMap::new() }
tiers: HashMap::new(),
}
} }
fn add_sizes(&mut self, tiers: HashMap<String, TierStats>) { fn add_sizes(&mut self, tiers: HashMap<String, TierStats>) {
@@ -172,11 +170,14 @@ impl AllTierStats {
fn populate_stats(&self, stats: &mut HashMap<String, TierStats>) { fn populate_stats(&self, stats: &mut HashMap<String, TierStats>) {
for (tier, st) in &self.tiers { for (tier, st) in &self.tiers {
stats.insert(tier.clone(), TierStats { stats.insert(
total_size: st.total_size.clone(), tier.clone(),
num_versions: st.num_versions.clone(), TierStats {
num_objects: st.num_objects.clone(), total_size: st.total_size.clone(),
}); num_versions: st.num_versions.clone(),
num_objects: st.num_objects.clone(),
},
);
} }
} }
} }
ecstore/src/lib.rs
+2 -2
View File
@@ -27,11 +27,11 @@ pub mod store_list_objects;
mod store_utils; mod store_utils;
pub mod checksum; pub mod checksum;
pub mod client;
pub mod event; pub mod event;
pub mod event_notification; pub mod event_notification;
pub mod client;
pub mod tier;
pub mod signer; pub mod signer;
pub mod tier;
pub use global::new_object_layer_fn; pub use global::new_object_layer_fn;
pub use global::set_global_endpoints; pub use global::set_global_endpoints;
ecstore/src/set_disk.rs
+77 -42
View File
@@ -1,6 +1,6 @@
use s3s::header::X_AMZ_RESTORE;
use crate::error::ObjectApiError;
use crate::bitrot::{create_bitrot_reader, create_bitrot_writer}; use crate::bitrot::{create_bitrot_reader, create_bitrot_writer};
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
use crate::client::{object_api_utils::extract_etag, transition_api::ReaderImpl};
use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_read_quorum_errs, reduce_write_quorum_errs}; use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_read_quorum_errs, reduce_write_quorum_errs};
use crate::disk::{ use crate::disk::{
self, CHECK_PART_DISK_NOT_FOUND, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, self, CHECK_PART_DISK_NOT_FOUND, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS,
@@ -8,12 +8,15 @@ use crate::disk::{
}; };
use crate::erasure_coding; use crate::erasure_coding;
use crate::erasure_coding::bitrot_verify; use crate::erasure_coding::bitrot_verify;
use crate::error::ObjectApiError;
use crate::error::{Error, Result}; use crate::error::{Error, Result};
use crate::global::GLOBAL_MRFState; use crate::global::GLOBAL_MRFState;
use crate::global::{GLOBAL_LocalNodeName, GLOBAL_TierConfigMgr};
use crate::heal::data_usage_cache::DataUsageCache; use crate::heal::data_usage_cache::DataUsageCache;
use crate::heal::heal_ops::{HealEntryFn, HealSequence}; use crate::heal::heal_ops::{HealEntryFn, HealSequence};
use crate::store_api::ObjectToDelete; use crate::store_api::ObjectToDelete;
use crate::{ use crate::{
bucket::lifecycle::bucket_lifecycle_ops::{gen_transition_objname, get_transitioned_object_reader, put_restore_opts},
cache_value::metacache_set::{ListPathRawOptions, list_path_raw}, cache_value::metacache_set::{ListPathRawOptions, list_path_raw},
config::{GLOBAL_StorageClass, storageclass}, config::{GLOBAL_StorageClass, storageclass},
disk::{ disk::{
@@ -22,8 +25,8 @@ use crate::{
UpdateMetadataOpts, endpoint::Endpoint, error::DiskError, format::FormatV3, new_disk, UpdateMetadataOpts, endpoint::Endpoint, error::DiskError, format::FormatV3, new_disk,
}, },
error::{StorageError, to_object_err}, error::{StorageError, to_object_err},
event::name::EventName, event_notification::{send_event, EventArgs}, event::name::EventName,
bucket::lifecycle::bucket_lifecycle_ops::{gen_transition_objname, put_restore_opts, get_transitioned_object_reader,}, event_notification::{EventArgs, send_event},
global::{ global::{
GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id, GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id,
is_dist_erasure, is_dist_erasure,
@@ -44,16 +47,11 @@ use crate::{
}, },
store_init::load_format_erasure, store_init::load_format_erasure,
}; };
use crate::client::{
object_api_utils::extract_etag,
transition_api::ReaderImpl,
};
use crate::{disk::STORAGE_FORMAT_FILE, heal::mrf::PartialOperation}; use crate::{disk::STORAGE_FORMAT_FILE, heal::mrf::PartialOperation};
use crate::{ use crate::{
heal::data_scanner::{HEAL_DELETE_DANGLING, globalHealConfig}, heal::data_scanner::{HEAL_DELETE_DANGLING, globalHealConfig},
store_api::ListObjectVersionsInfo, store_api::ListObjectVersionsInfo,
}; };
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
use bytesize::ByteSize; use bytesize::ByteSize;
use chrono::Utc; use chrono::Utc;
use futures::future::join_all; use futures::future::join_all;
@@ -75,6 +73,7 @@ use rustfs_utils::{
crypto::{base64_decode, base64_encode, hex}, crypto::{base64_decode, base64_encode, hex},
path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, path_join_buf}, path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, path_join_buf},
}; };
use s3s::header::X_AMZ_RESTORE;
use sha2::{Digest, Sha256}; use sha2::{Digest, Sha256};
use std::hash::Hash; use std::hash::Hash;
use std::mem; use std::mem;
@@ -100,7 +99,6 @@ use tracing::error;
use tracing::{debug, info, warn}; use tracing::{debug, info, warn};
use uuid::Uuid; use uuid::Uuid;
use workers::workers::Workers; use workers::workers::Workers;
use crate::global::{GLOBAL_TierConfigMgr, GLOBAL_LocalNodeName};
pub const DEFAULT_READ_BUFFER_SIZE: usize = 1024 * 1024; pub const DEFAULT_READ_BUFFER_SIZE: usize = 1024 * 1024;
@@ -3677,7 +3675,13 @@ impl SetDisks {
Ok(()) Ok(())
} }
pub async fn update_restore_metadata(&self, bucket: &str, object: &str, obj_info: &ObjectInfo, opts: &ObjectOptions) -> Result<()> { pub async fn update_restore_metadata(
&self,
bucket: &str,
object: &str,
obj_info: &ObjectInfo,
opts: &ObjectOptions,
) -> Result<()> {
let mut oi = obj_info.clone(); let mut oi = obj_info.clone();
oi.metadata_only = true; oi.metadata_only = true;
@@ -3686,13 +3690,23 @@ impl SetDisks {
} }
let version_id = oi.version_id.clone().map(|v| v.to_string()); let version_id = oi.version_id.clone().map(|v| v.to_string());
let obj = self.copy_object(bucket, object, bucket, object, &mut oi, &ObjectOptions { let obj = self
version_id: version_id.clone(), .copy_object(
..Default::default() bucket,
}, &ObjectOptions { object,
version_id: version_id, bucket,
..Default::default() object,
}).await; &mut oi,
&ObjectOptions {
version_id: version_id.clone(),
..Default::default()
},
&ObjectOptions {
version_id: version_id,
..Default::default()
},
)
.await;
if let Err(err) = obj { if let Err(err) = obj {
//storagelogif(ctx, fmt.Errorf("Unable to update transition restore metadata for %s/%s(%s): %s", bucket, object, oi.VersionID, err)) //storagelogif(ctx, fmt.Errorf("Unable to update transition restore metadata for %s/%s(%s): %s", bucket, object, oi.VersionID, err))
return Err(err); return Err(err);
@@ -4169,7 +4183,10 @@ impl StorageAPI for SetDisks {
for disk in disks.iter() { for disk in disks.iter() {
futures.push(async move { futures.push(async move {
if let Some(disk) = disk { if let Some(disk) = disk {
match disk.delete_version(&bucket, &object, fi.clone(), force_del_marker, DeleteOptions::default()).await { match disk
.delete_version(&bucket, &object, fi.clone(), force_del_marker, DeleteOptions::default())
.await
{
Ok(r) => Ok(r), Ok(r) => Ok(r),
Err(e) => Err(e), Err(e) => Err(e),
} }
@@ -4392,16 +4409,17 @@ impl StorageAPI for SetDisks {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> { async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
GLOBAL_MRFState.add_partial(PartialOperation { GLOBAL_MRFState
bucket: bucket.to_string(), .add_partial(PartialOperation {
object: object.to_string(), bucket: bucket.to_string(),
version_id: Some(version_id.to_string()), object: object.to_string(),
queued: Utc::now(), version_id: Some(version_id.to_string()),
set_index: self.set_index, queued: Utc::now(),
pool_index: self.pool_index, set_index: self.set_index,
..Default::default() pool_index: self.pool_index,
}) ..Default::default()
.await; })
.await;
Ok(()) Ok(())
} }
@@ -4518,7 +4536,9 @@ impl StorageAPI for SetDisks {
} }
return Err(to_object_err(ERR_METHOD_NOT_ALLOWED, vec![bucket, object])); return Err(to_object_err(ERR_METHOD_NOT_ALLOWED, vec![bucket, object]));
}*/ }*/
if !opts.mod_time.expect("err").unix_timestamp() == fi.mod_time.as_ref().expect("err").unix_timestamp() || !(opts.transition.etag == extract_etag(&fi.metadata)) { if !opts.mod_time.expect("err").unix_timestamp() == fi.mod_time.as_ref().expect("err").unix_timestamp()
|| !(opts.transition.etag == extract_etag(&fi.metadata))
{
return Err(to_object_err(Error::from(DiskError::FileNotFound), vec![bucket, object])); return Err(to_object_err(Error::from(DiskError::FileNotFound), vec![bucket, object]));
} }
if fi.transition_status == TRANSITION_COMPLETE { if fi.transition_status == TRANSITION_COMPLETE {
@@ -4548,7 +4568,10 @@ impl StorageAPI for SetDisks {
let (pr, mut pw) = tokio::io::duplex(fi.erasure.block_size); let (pr, mut pw) = tokio::io::duplex(fi.erasure.block_size);
//let h = HeaderMap::new(); //let h = HeaderMap::new();
//let reader = ReaderImpl::ObjectBody(GetObjectReader {stream: StreamingBlob::wrap(tokio_util::io::ReaderStream::new(pr)), object_info: oi}); //let reader = ReaderImpl::ObjectBody(GetObjectReader {stream: StreamingBlob::wrap(tokio_util::io::ReaderStream::new(pr)), object_info: oi});
let reader = ReaderImpl::ObjectBody(GetObjectReader {stream: Box::new(pr), object_info: oi}); let reader = ReaderImpl::ObjectBody(GetObjectReader {
stream: Box::new(pr),
object_info: oi,
});
let cloned_bucket = bucket.to_string(); let cloned_bucket = bucket.to_string();
let cloned_object = object.to_string(); let cloned_object = object.to_string();
@@ -4557,7 +4580,16 @@ impl StorageAPI for SetDisks {
let pool_index = self.pool_index; let pool_index = self.pool_index;
tokio::spawn(async move { tokio::spawn(async move {
if let Err(e) = Self::get_object_with_fileinfo( if let Err(e) = Self::get_object_with_fileinfo(
&cloned_bucket, &cloned_object, 0, cloned_fi.size, &mut pw, cloned_fi, meta_arr, &online_disks, set_index, pool_index &cloned_bucket,
&cloned_object,
0,
cloned_fi.size,
&mut pw,
cloned_fi,
meta_arr,
&online_disks,
set_index,
pool_index,
) )
.await .await
{ {
@@ -4565,11 +4597,13 @@ impl StorageAPI for SetDisks {
}; };
}); });
let rv = tgt_client.put_with_meta(&dest_obj, reader, fi.size as i64, { let rv = tgt_client
let mut m = HashMap::<String, String>::new(); .put_with_meta(&dest_obj, reader, fi.size as i64, {
m.insert("name".to_string(), object.to_string()); let mut m = HashMap::<String, String>::new();
m m.insert("name".to_string(), object.to_string());
}).await; m
})
.await;
//pr.CloseWithError(err); //pr.CloseWithError(err);
if let Err(err) = rv { if let Err(err) = rv {
//traceFn(ILMTransition, nil, err) //traceFn(ILMTransition, nil, err)
@@ -4579,7 +4613,7 @@ impl StorageAPI for SetDisks {
fi.transition_status = TRANSITION_COMPLETE.to_string(); fi.transition_status = TRANSITION_COMPLETE.to_string();
fi.transitioned_objname = dest_obj; fi.transitioned_objname = dest_obj;
fi.transition_tier = opts.transition.tier.clone(); fi.transition_tier = opts.transition.tier.clone();
fi.transition_version_id = if rv=="" { None } else { Some(Uuid::parse_str(&rv)?) }; fi.transition_version_id = if rv == "" { None } else { Some(Uuid::parse_str(&rv)?) };
let mut event_name = EventName::ObjectTransitionComplete.as_ref(); let mut event_name = EventName::ObjectTransitionComplete.as_ref();
let disks = self.get_disks(0, 0).await?; let disks = self.get_disks(0, 0).await?;
@@ -4594,7 +4628,8 @@ impl StorageAPI for SetDisks {
continue; continue;
} }
} }
self.add_partial(bucket, object, &opts.version_id.as_ref().expect("err")).await; self.add_partial(bucket, object, &opts.version_id.as_ref().expect("err"))
.await;
break; break;
} }
@@ -4602,9 +4637,9 @@ impl StorageAPI for SetDisks {
send_event(EventArgs { send_event(EventArgs {
event_name: event_name.to_string(), event_name: event_name.to_string(),
bucket_name: bucket.to_string(), bucket_name: bucket.to_string(),
object: obj_info, object: obj_info,
user_agent: "Internal: [ILM-Transition]".to_string(), user_agent: "Internal: [ILM-Transition]".to_string(),
host: GLOBAL_LocalNodeName.to_string(), host: GLOBAL_LocalNodeName.to_string(),
..Default::default() ..Default::default()
}); });
//let tags = opts.lifecycle_audit_event.tags(); //let tags = opts.lifecycle_audit_event.tags();
ecstore/src/sets.rs
+4 -4
View File
@@ -1,7 +1,6 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use std::{collections::HashMap, sync::Arc}; use std::{collections::HashMap, sync::Arc};
use rustfs_filemeta::FileInfo;
use crate::disk::error_reduce::count_errs; use crate::disk::error_reduce::count_errs;
use crate::error::{Error, Result}; use crate::error::{Error, Result};
use crate::{ use crate::{
@@ -30,6 +29,7 @@ use futures::future::join_all;
use http::HeaderMap; use http::HeaderMap;
use lock::{LockApi, namespace_lock::NsLockMap, new_lock_api}; use lock::{LockApi, namespace_lock::NsLockMap, new_lock_api};
use madmin::heal_commands::{HealDriveInfo, HealResultItem}; use madmin::heal_commands::{HealDriveInfo, HealResultItem};
use rustfs_filemeta::FileInfo;
use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash}; use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash};
use tokio::sync::RwLock; use tokio::sync::RwLock;
use uuid::Uuid; use uuid::Uuid;
@@ -591,7 +591,9 @@ impl StorageAPI for Sets {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
self.get_disks_by_key(object).restore_transitioned_object(bucket, object, opts).await self.get_disks_by_key(object)
.restore_transitioned_object(bucket, object, opts)
.await
} }
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
@@ -688,8 +690,6 @@ impl StorageAPI for Sets {
.await .await
} }
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> { async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
self.get_disks_by_key(object).delete_object_tags(bucket, object, opts).await self.get_disks_by_key(object).delete_object_tags(bucket, object, opts).await
ecstore/src/signer/mod.rs
+9 -9
View File
@@ -1,13 +1,13 @@
pub mod utils; pub mod ordered_qs;
pub mod request_signature_v2;
pub mod request_signature_v4;
pub mod request_signature_streaming; pub mod request_signature_streaming;
pub mod request_signature_streaming_unsigned_trailer; pub mod request_signature_streaming_unsigned_trailer;
pub mod ordered_qs; pub mod request_signature_v2;
pub mod request_signature_v4;
pub mod utils;
pub use request_signature_v2::sign_v2;
pub use request_signature_v2::pre_sign_v2;
pub use request_signature_v4::sign_v4;
pub use request_signature_v4::pre_sign_v4;
pub use request_signature_v4::sign_v4_trailer;
pub use request_signature_streaming::streaming_sign_v4; pub use request_signature_streaming::streaming_sign_v4;
pub use request_signature_v2::pre_sign_v2;
pub use request_signature_v2::sign_v2;
pub use request_signature_v4::pre_sign_v4;
pub use request_signature_v4::sign_v4;
pub use request_signature_v4::sign_v4_trailer;
ecstore/src/signer/request_signature_streaming.rs
+37 -28
View File
@@ -1,37 +1,37 @@
use std::pin::Pin; use bytes::{Bytes, BytesMut};
use std::sync::Mutex;
use futures::prelude::*; use futures::prelude::*;
use futures::task; use futures::task;
use bytes::{Bytes, BytesMut}; use http::HeaderMap;
use http::header::TRAILER;
use http::Uri; use http::Uri;
use http::header::TRAILER;
use http::request::{self, Request};
use hyper::Method;
use lazy_static::lazy_static; use lazy_static::lazy_static;
use std::collections::HashMap; use std::collections::HashMap;
use stdx::str::StrExt;
use std::fmt::Write; use std::fmt::Write;
use tracing::{error, info, warn, debug}; use std::pin::Pin;
use time::{OffsetDateTime, macros::datetime, macros::format_description, format_description}; use std::sync::Mutex;
use http::request::{self, Request}; use stdx::str::StrExt;
use http::HeaderMap; use time::{OffsetDateTime, format_description, macros::datetime, macros::format_description};
use hyper::Method; use tracing::{debug, error, info, warn};
use super::request_signature_v4::{SERVICE_TYPE_S3, get_scope, get_signature, get_signing_key};
use crate::client::constants::UNSIGNED_PAYLOAD;
use rustfs_utils::{ use rustfs_utils::{
crypto::{hex, hex_sha256, hex_sha256_chunk, hmac_sha256}, crypto::{hex, hex_sha256, hex_sha256_chunk, hmac_sha256},
hash::EMPTY_STRING_SHA256_HASH hash::EMPTY_STRING_SHA256_HASH,
}; };
use crate::client::constants::UNSIGNED_PAYLOAD;
use super::request_signature_v4::{get_scope, get_signing_key, get_signature, SERVICE_TYPE_S3};
const STREAMING_SIGN_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; const STREAMING_SIGN_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD";
const STREAMING_SIGN_TRAILER_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"; const STREAMING_SIGN_TRAILER_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER";
const STREAMING_PAYLOAD_HDR: &str = "AWS4-HMAC-SHA256-PAYLOAD"; const STREAMING_PAYLOAD_HDR: &str = "AWS4-HMAC-SHA256-PAYLOAD";
const STREAMING_TRAILER_HDR: &str = "AWS4-HMAC-SHA256-TRAILER"; const STREAMING_TRAILER_HDR: &str = "AWS4-HMAC-SHA256-TRAILER";
const PAYLOAD_CHUNK_SIZE: i64 = 64 * 1024; const PAYLOAD_CHUNK_SIZE: i64 = 64 * 1024;
const CHUNK_SIGCONST_LEN: i64 = 17; const CHUNK_SIGCONST_LEN: i64 = 17;
const SIGNATURESTR_LEN: i64 = 64; const SIGNATURESTR_LEN: i64 = 64;
const CRLF_LEN: i64 = 2; const CRLF_LEN: i64 = 2;
const TRAILER_KV_SEPARATOR: &str = ":"; const TRAILER_KV_SEPARATOR: &str = ":";
const TRAILER_SIGNATURE: &str = "x-amz-trailer-signature"; const TRAILER_SIGNATURE: &str = "x-amz-trailer-signature";
lazy_static! { lazy_static! {
static ref ignored_streaming_headers: HashMap<String, bool> = { static ref ignored_streaming_headers: HashMap<String, bool> = {
@@ -55,17 +55,26 @@ fn build_chunk_string_to_sign(t: OffsetDateTime, region: &str, previous_sig: &st
string_to_sign_parts.join("\n") string_to_sign_parts.join("\n")
} }
fn build_chunk_signature(chunk_check_sum: &str, req_time: OffsetDateTime, region: &str, fn build_chunk_signature(
previous_signature: &str, secret_access_key: &str chunk_check_sum: &str,
req_time: OffsetDateTime,
region: &str,
previous_signature: &str,
secret_access_key: &str,
) -> String { ) -> String {
let chunk_string_to_sign = build_chunk_string_to_sign(req_time, region, let chunk_string_to_sign = build_chunk_string_to_sign(req_time, region, previous_signature, chunk_check_sum);
previous_signature, chunk_check_sum);
let signing_key = get_signing_key(secret_access_key, region, req_time, SERVICE_TYPE_S3); let signing_key = get_signing_key(secret_access_key, region, req_time, SERVICE_TYPE_S3);
get_signature(signing_key, &chunk_string_to_sign) get_signature(signing_key, &chunk_string_to_sign)
} }
pub fn streaming_sign_v4(req: request::Builder, access_key_id: &str, secret_access_key: &str, session_token: &str, pub fn streaming_sign_v4(
region: &str, data_len: i64, req_time: OffsetDateTime/*, sh256: md5simd.Hasher*/ req: request::Builder,
access_key_id: &str,
secret_access_key: &str,
session_token: &str,
region: &str,
data_len: i64,
req_time: OffsetDateTime, /*, sh256: md5simd.Hasher*/
) -> request::Builder { ) -> request::Builder {
todo!(); todo!();
} }
ecstore/src/signer/request_signature_streaming_unsigned_trailer.rs
+6 -1
View File
@@ -1,6 +1,11 @@
use http::request; use http::request;
use time::OffsetDateTime; use time::OffsetDateTime;
pub fn streaming_unsigned_v4(mut req: request::Builder, session_token: &str, data_len: i64, req_time: OffsetDateTime) -> request::Builder{ pub fn streaming_unsigned_v4(
mut req: request::Builder,
session_token: &str,
data_len: i64,
req_time: OffsetDateTime,
) -> request::Builder {
todo!(); todo!();
} }
ecstore/src/signer/request_signature_v2.rs
+48 -19
View File
@@ -1,15 +1,15 @@
use std::collections::HashMap; use bytes::{Bytes, BytesMut};
use http::request; use http::request;
use hyper::Uri; use hyper::Uri;
use bytes::{Bytes, BytesMut}; use std::collections::HashMap;
use std::fmt::Write; use std::fmt::Write;
use time::{OffsetDateTime, macros::format_description, format_description}; use time::{OffsetDateTime, format_description, macros::format_description};
use rustfs_utils::crypto::{base64_encode, hex, hmac_sha1}; use rustfs_utils::crypto::{base64_encode, hex, hmac_sha1};
use super::utils::get_host_addr; use super::utils::get_host_addr;
const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256"; const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
const SIGN_V2_ALGORITHM: &str = "AWS"; const SIGN_V2_ALGORITHM: &str = "AWS";
fn encode_url2path(req: &request::Builder, virtual_host: bool) -> String { fn encode_url2path(req: &request::Builder, virtual_host: bool) -> String {
@@ -20,7 +20,13 @@ fn encode_url2path(req: &request::Builder, virtual_host: bool) -> String {
path path
} }
pub fn pre_sign_v2(mut req: request::Builder, access_key_id: &str, secret_access_key: &str, expires: i64, virtual_host: bool) -> request::Builder { pub fn pre_sign_v2(
mut req: request::Builder,
access_key_id: &str,
secret_access_key: &str,
expires: i64,
virtual_host: bool,
) -> request::Builder {
if access_key_id == "" || secret_access_key == "" { if access_key_id == "" || secret_access_key == "" {
return req; return req;
} }
@@ -50,7 +56,11 @@ pub fn pre_sign_v2(mut req: request::Builder, access_key_id: &str, secret_access
let uri = req.uri_ref().unwrap().clone(); let uri = req.uri_ref().unwrap().clone();
let mut parts = req.uri_ref().unwrap().clone().into_parts(); let mut parts = req.uri_ref().unwrap().clone().into_parts();
parts.path_and_query = Some(format!("{}?{}&Signature={}", uri.path(), serde_urlencoded::to_string(&query).unwrap(), signature).parse().unwrap()); parts.path_and_query = Some(
format!("{}?{}&Signature={}", uri.path(), serde_urlencoded::to_string(&query).unwrap(), signature)
.parse()
.unwrap(),
);
let req = req.uri(Uri::from_parts(parts).unwrap()); let req = req.uri(Uri::from_parts(parts).unwrap());
req req
@@ -61,7 +71,13 @@ fn post_pre_sign_signature_v2(policy_base64: &str, secret_access_key: &str) -> S
signature signature
} }
pub fn sign_v2(mut req: request::Builder, content_len: i64, access_key_id: &str, secret_access_key: &str, virtual_host: bool) -> request::Builder { pub fn sign_v2(
mut req: request::Builder,
content_len: i64,
access_key_id: &str,
secret_access_key: &str,
virtual_host: bool,
) -> request::Builder {
if access_key_id == "" || secret_access_key == "" { if access_key_id == "" || secret_access_key == "" {
return req; return req;
} }
@@ -74,7 +90,14 @@ pub fn sign_v2(mut req: request::Builder, content_len: i64, access_key_id: &str,
let date = headers.get("Date").unwrap(); let date = headers.get("Date").unwrap();
if date.to_str().unwrap() == "" { if date.to_str().unwrap() == "" {
headers.insert("Date", d2.format(&format_description::well_known::Rfc2822).unwrap().to_string().parse().unwrap()); headers.insert(
"Date",
d2.format(&format_description::well_known::Rfc2822)
.unwrap()
.to_string()
.parse()
.unwrap(),
);
} }
let mut auth_header = format!("{} {}:", SIGN_V2_ALGORITHM, access_key_id); let mut auth_header = format!("{} {}:", SIGN_V2_ALGORITHM, access_key_id);
@@ -130,21 +153,27 @@ fn write_canonicalized_headers(buf: &mut BytesMut, req: &request::Builder) {
let lk = k.as_str().to_lowercase(); let lk = k.as_str().to_lowercase();
if lk.starts_with("x-amz") { if lk.starts_with("x-amz") {
proto_headers.push(lk.clone()); proto_headers.push(lk.clone());
let vv = req.headers_ref().expect("err").get_all(k).iter().map(|e| e.to_str().unwrap().to_string()).collect(); let vv = req
.headers_ref()
.expect("err")
.get_all(k)
.iter()
.map(|e| e.to_str().unwrap().to_string())
.collect();
vals.insert(lk, vv); vals.insert(lk, vv);
} }
} }
proto_headers.sort(); proto_headers.sort();
for k in proto_headers { for k in proto_headers {
buf.write_str(&k); buf.write_str(&k);
buf.write_char(':'); buf.write_char(':');
for (idx, v) in vals[&k].iter().enumerate() { for (idx, v) in vals[&k].iter().enumerate() {
if idx > 0 { if idx > 0 {
buf.write_char(','); buf.write_char(',');
} }
buf.write_str(v); buf.write_str(v);
} }
buf.write_char('\n'); buf.write_char('\n');
} }
} }
@@ -181,7 +210,7 @@ fn write_canonicalized_resource(buf: &mut BytesMut, req: &request::Builder, virt
let mut vals = result.unwrap_or_default(); let mut vals = result.unwrap_or_default();
for resource in INCLUDED_QUERY { for resource in INCLUDED_QUERY {
let vv = &vals[*resource]; let vv = &vals[*resource];
if vv.len() > 0 { if vv.len() > 0 {
n += 1; n += 1;
match n { match n {
1 => { 1 => {
ecstore/src/signer/request_signature_v4.rs
+144 -35
View File
@@ -1,21 +1,21 @@
use bytes::{Bytes, BytesMut}; use bytes::{Bytes, BytesMut};
use http::header::TRAILER; use http::HeaderMap;
use http::Uri; use http::Uri;
use http::header::TRAILER;
use http::request::{self, Request};
use lazy_static::lazy_static; use lazy_static::lazy_static;
use std::collections::HashMap; use std::collections::HashMap;
use std::fmt::Write; use std::fmt::Write;
use tracing::{error, info, warn, debug}; use time::{OffsetDateTime, format_description, macros::datetime, macros::format_description};
use time::{OffsetDateTime, macros::datetime, macros::format_description, format_description}; use tracing::{debug, error, info, warn};
use http::request::{self, Request};
use http::HeaderMap;
use rustfs_utils::crypto::{hex, hex_sha256, hmac_sha256};
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
use crate::client::constants::UNSIGNED_PAYLOAD;
use super::ordered_qs::OrderedQs; use super::ordered_qs::OrderedQs;
use super::request_signature_streaming_unsigned_trailer::streaming_unsigned_v4; use super::request_signature_streaming_unsigned_trailer::streaming_unsigned_v4;
use super::utils::stable_sort_by_first; use super::utils::stable_sort_by_first;
use super::utils::{get_host_addr, sign_v4_trim_all}; use super::utils::{get_host_addr, sign_v4_trim_all};
use crate::client::constants::UNSIGNED_PAYLOAD;
use rustfs_utils::crypto::{hex, hex_sha256, hmac_sha256};
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256"; pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
pub const SERVICE_TYPE_S3: &str = "s3"; pub const SERVICE_TYPE_S3: &str = "s3";
@@ -69,7 +69,7 @@ fn get_credential(access_key_id: &str, location: &str, t: OffsetDateTime, servic
fn get_hashed_payload(req: &request::Builder) -> String { fn get_hashed_payload(req: &request::Builder) -> String {
let headers = req.headers_ref().unwrap(); let headers = req.headers_ref().unwrap();
let mut hashed_payload = ""; let mut hashed_payload = "";
if let Some(payload)= headers.get("X-Amz-Content-Sha256") { if let Some(payload) = headers.get("X-Amz-Content-Sha256") {
hashed_payload = payload.to_str().unwrap(); hashed_payload = payload.to_str().unwrap();
} }
if hashed_payload == "" { if hashed_payload == "" {
@@ -86,7 +86,13 @@ fn get_canonical_headers(req: &request::Builder, ignored_headers: &HashMap<Strin
continue; continue;
} }
headers.push(k.as_str().to_lowercase()); headers.push(k.as_str().to_lowercase());
let vv = req.headers_ref().expect("err").get_all(k).iter().map(|e| e.to_str().unwrap().to_string()).collect(); let vv = req
.headers_ref()
.expect("err")
.get_all(k)
.iter()
.map(|e| e.to_str().unwrap().to_string())
.collect();
vals.insert(k.as_str().to_lowercase(), vv); vals.insert(k.as_str().to_lowercase(), vv);
} }
if !header_exists("host", &headers) { if !header_exists("host", &headers) {
@@ -178,7 +184,15 @@ fn get_string_to_sign_v4(t: OffsetDateTime, location: &str, canonical_request: &
string_to_sign string_to_sign
} }
pub fn pre_sign_v4(req: request::Builder, access_key_id: &str, secret_access_key: &str, session_token: &str, location: &str, expires: i64, t: OffsetDateTime) -> request::Builder { pub fn pre_sign_v4(
req: request::Builder,
access_key_id: &str,
secret_access_key: &str,
session_token: &str,
location: &str,
expires: i64,
t: OffsetDateTime,
) -> request::Builder {
if access_key_id == "" || secret_access_key == "" { if access_key_id == "" || secret_access_key == "" {
return req; return req;
} }
@@ -208,7 +222,11 @@ pub fn pre_sign_v4(req: request::Builder, access_key_id: &str, secret_access_key
let uri = req.uri_ref().unwrap().clone(); let uri = req.uri_ref().unwrap().clone();
let mut parts = req.uri_ref().unwrap().clone().into_parts(); let mut parts = req.uri_ref().unwrap().clone().into_parts();
parts.path_and_query = Some(format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap()).parse().unwrap()); parts.path_and_query = Some(
format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap())
.parse()
.unwrap(),
);
let req = req.uri(Uri::from_parts(parts).unwrap()); let req = req.uri(Uri::from_parts(parts).unwrap());
let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req)); let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req));
@@ -220,7 +238,16 @@ pub fn pre_sign_v4(req: request::Builder, access_key_id: &str, secret_access_key
let uri = req.uri_ref().unwrap().clone(); let uri = req.uri_ref().unwrap().clone();
let mut parts = req.uri_ref().unwrap().clone().into_parts(); let mut parts = req.uri_ref().unwrap().clone().into_parts();
parts.path_and_query = Some(format!("{}?{}&X-Amz-Signature={}", uri.path(), serde_urlencoded::to_string(&query).unwrap(), signature).parse().unwrap()); parts.path_and_query = Some(
format!(
"{}?{}&X-Amz-Signature={}",
uri.path(),
serde_urlencoded::to_string(&query).unwrap(),
signature
)
.parse()
.unwrap(),
);
let req = req.uri(Uri::from_parts(parts).unwrap()); let req = req.uri(Uri::from_parts(parts).unwrap());
req req
@@ -236,7 +263,16 @@ fn sign_v4_sts(mut req: request::Builder, access_key_id: &str, secret_access_key
sign_v4_inner(req, 0, access_key_id, secret_access_key, "", location, SERVICE_TYPE_STS, HeaderMap::new()) sign_v4_inner(req, 0, access_key_id, secret_access_key, "", location, SERVICE_TYPE_STS, HeaderMap::new())
} }
fn sign_v4_inner(mut req: request::Builder, content_len: i64, access_key_id: &str, secret_access_key: &str, session_token: &str, location: &str, service_type: &str, trailer: HeaderMap) -> request::Builder { fn sign_v4_inner(
mut req: request::Builder,
content_len: i64,
access_key_id: &str,
secret_access_key: &str,
session_token: &str,
location: &str,
service_type: &str,
trailer: HeaderMap,
) -> request::Builder {
if access_key_id == "" || secret_access_key == "" { if access_key_id == "" || secret_access_key == "" {
return req; return req;
} }
@@ -252,7 +288,7 @@ fn sign_v4_inner(mut req: request::Builder, content_len: i64, access_key_id: &st
headers.insert("X-Amz-Security-Token", session_token.parse().unwrap()); headers.insert("X-Amz-Security-Token", session_token.parse().unwrap());
} }
if trailer.len() > 0{ if trailer.len() > 0 {
for (k, _) in &trailer { for (k, _) in &trailer {
headers.append("X-Amz-Trailer", k.as_str().to_lowercase().parse().unwrap()); headers.append("X-Amz-Trailer", k.as_str().to_lowercase().parse().unwrap());
} }
@@ -276,7 +312,10 @@ fn sign_v4_inner(mut req: request::Builder, content_len: i64, access_key_id: &st
let mut headers = req.headers_mut().expect("err"); let mut headers = req.headers_mut().expect("err");
let auth = format!("{} Credential={}, SignedHeaders={}, Signature={}", SIGN_V4_ALGORITHM, credential, signed_headers, signature); let auth = format!(
"{} Credential={}, SignedHeaders={}, Signature={}",
SIGN_V4_ALGORITHM, credential, signed_headers, signature
);
headers.insert("Authorization", auth.parse().unwrap()); headers.insert("Authorization", auth.parse().unwrap());
if trailer.len() > 0 { if trailer.len() > 0 {
@@ -315,12 +354,44 @@ fn unsigned_trailer(mut req: request::Builder, content_len: i64, trailer: Header
streaming_unsigned_v4(req, "", content_len, t); streaming_unsigned_v4(req, "", content_len, t);
} }
pub fn sign_v4(mut req: request::Builder, content_len: i64, access_key_id: &str, secret_access_key: &str, session_token: &str, location: &str) -> request::Builder { pub fn sign_v4(
sign_v4_inner(req, content_len, access_key_id, secret_access_key, session_token, location, SERVICE_TYPE_S3, HeaderMap::new()) mut req: request::Builder,
content_len: i64,
access_key_id: &str,
secret_access_key: &str,
session_token: &str,
location: &str,
) -> request::Builder {
sign_v4_inner(
req,
content_len,
access_key_id,
secret_access_key,
session_token,
location,
SERVICE_TYPE_S3,
HeaderMap::new(),
)
} }
pub fn sign_v4_trailer(req: request::Builder, access_key_id: &str, secret_access_key: &str, session_token: &str, location: &str, trailer: HeaderMap) -> request::Builder { pub fn sign_v4_trailer(
sign_v4_inner(req, 0, access_key_id, secret_access_key, session_token, location, SERVICE_TYPE_S3, trailer) req: request::Builder,
access_key_id: &str,
secret_access_key: &str,
session_token: &str,
location: &str,
trailer: HeaderMap,
) -> request::Builder {
sign_v4_inner(
req,
0,
access_key_id,
secret_access_key,
session_token,
location,
SERVICE_TYPE_S3,
trailer,
)
} }
#[cfg(test)] #[cfg(test)]
@@ -338,10 +409,17 @@ mod tests {
let service = "s3"; let service = "s3";
let path = "/"; let path = "/";
let mut req = Request::builder().method(http::Method::GET).uri("http://examplebucket.s3.amazonaws.com/?"); let mut req = Request::builder()
.method(http::Method::GET)
.uri("http://examplebucket.s3.amazonaws.com/?");
let mut headers = req.headers_mut().expect("err"); let mut headers = req.headers_mut().expect("err");
headers.insert("host", "examplebucket.s3.amazonaws.com".parse().unwrap()); headers.insert("host", "examplebucket.s3.amazonaws.com".parse().unwrap());
headers.insert("x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855".parse().unwrap()); headers.insert(
"x-amz-content-sha256",
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
.parse()
.unwrap(),
);
headers.insert("x-amz-date", timestamp.parse().unwrap()); headers.insert("x-amz-date", timestamp.parse().unwrap());
let mut query = <Vec<(String, String)>>::new(); let mut query = <Vec<(String, String)>>::new();
@@ -349,7 +427,11 @@ mod tests {
query.push(("prefix".to_string(), "J".to_string())); query.push(("prefix".to_string(), "J".to_string()));
let uri = req.uri_ref().unwrap().clone(); let uri = req.uri_ref().unwrap().clone();
let mut parts = req.uri_ref().unwrap().clone().into_parts(); let mut parts = req.uri_ref().unwrap().clone().into_parts();
parts.path_and_query = Some(format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap()).parse().unwrap()); parts.path_and_query = Some(
format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap())
.parse()
.unwrap(),
);
let req = req.uri(Uri::from_parts(parts).unwrap()); let req = req.uri(Uri::from_parts(parts).unwrap());
let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req)); let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req));
@@ -361,7 +443,9 @@ mod tests {
"max-keys=2&prefix=J\n", "max-keys=2&prefix=J\n",
"host:examplebucket.s3.amazonaws.com\n", "host:examplebucket.s3.amazonaws.com\n",
"x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n", "x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n",
"x-amz-date:", "20130524T000000Z", "\n", "x-amz-date:",
"20130524T000000Z",
"\n",
"\n", "\n",
"host;x-amz-content-sha256;x-amz-date\n", "host;x-amz-content-sha256;x-amz-date\n",
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
@@ -373,7 +457,8 @@ mod tests {
string_to_sign, string_to_sign,
concat!( concat!(
"AWS4-HMAC-SHA256\n", "AWS4-HMAC-SHA256\n",
"20130524T000000Z", "\n", "20130524T000000Z",
"\n",
"20130524/us-east-1/s3/aws4_request\n", "20130524/us-east-1/s3/aws4_request\n",
"df57d21db20da04d7fa30298dd4488ba3a2b47ca3a489c74750e0f1e7df1b9b7", "df57d21db20da04d7fa30298dd4488ba3a2b47ca3a489c74750e0f1e7df1b9b7",
) )
@@ -396,17 +481,28 @@ mod tests {
let service = "s3"; let service = "s3";
let path = "/mblock2/"; let path = "/mblock2/";
let mut req = Request::builder().method(http::Method::GET).uri("http://192.168.1.11:9020/mblock2/?"); let mut req = Request::builder()
.method(http::Method::GET)
.uri("http://192.168.1.11:9020/mblock2/?");
let mut headers = req.headers_mut().expect("err"); let mut headers = req.headers_mut().expect("err");
headers.insert("host", "192.168.1.11:9020".parse().unwrap()); headers.insert("host", "192.168.1.11:9020".parse().unwrap());
headers.insert("x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855".parse().unwrap()); headers.insert(
"x-amz-content-sha256",
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
.parse()
.unwrap(),
);
headers.insert("x-amz-date", timestamp.parse().unwrap()); headers.insert("x-amz-date", timestamp.parse().unwrap());
let mut query: Vec<(String, String)> = Vec::new(); let mut query: Vec<(String, String)> = Vec::new();
let uri = req.uri_ref().unwrap().clone(); let uri = req.uri_ref().unwrap().clone();
let mut parts = req.uri_ref().unwrap().clone().into_parts(); let mut parts = req.uri_ref().unwrap().clone().into_parts();
parts.path_and_query = Some(format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap()).parse().unwrap()); parts.path_and_query = Some(
format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap())
.parse()
.unwrap(),
);
let req = req.uri(Uri::from_parts(parts).unwrap()); let req = req.uri(Uri::from_parts(parts).unwrap());
let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req)); let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req));
@@ -419,7 +515,9 @@ mod tests {
"\n", "\n",
"host:192.168.1.11:9020\n", "host:192.168.1.11:9020\n",
"x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n", "x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n",
"x-amz-date:", "20250505T011054Z", "\n", "x-amz-date:",
"20250505T011054Z",
"\n",
"\n", "\n",
"host;x-amz-content-sha256;x-amz-date\n", "host;x-amz-content-sha256;x-amz-date\n",
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
@@ -432,7 +530,8 @@ mod tests {
string_to_sign, string_to_sign,
concat!( concat!(
"AWS4-HMAC-SHA256\n", "AWS4-HMAC-SHA256\n",
"20250505T011054Z", "\n", "20250505T011054Z",
"\n",
"20250505/us-east-1/s3/aws4_request\n", "20250505/us-east-1/s3/aws4_request\n",
"c2960d00cc7de7bed3e2e2d1330ec298ded8f78a231c1d32dedac72ebec7f9b0", "c2960d00cc7de7bed3e2e2d1330ec298ded8f78a231c1d32dedac72ebec7f9b0",
) )
@@ -459,7 +558,12 @@ mod tests {
let mut headers = req.headers_mut().expect("err"); let mut headers = req.headers_mut().expect("err");
headers.insert("host", "192.168.1.11:9020".parse().unwrap()); headers.insert("host", "192.168.1.11:9020".parse().unwrap());
headers.insert("x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855".parse().unwrap()); headers.insert(
"x-amz-content-sha256",
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
.parse()
.unwrap(),
);
headers.insert("x-amz-date", timestamp.parse().unwrap()); headers.insert("x-amz-date", timestamp.parse().unwrap());
/*let uri = req.uri_ref().unwrap().clone(); /*let uri = req.uri_ref().unwrap().clone();
@@ -475,7 +579,7 @@ mod tests {
let mut parts = req.uri_ref().unwrap().clone().into_parts(); let mut parts = req.uri_ref().unwrap().clone().into_parts();
parts.path_and_query = Some(format!("{}?{}", uri.path(), canonical_query_string).parse().unwrap()); parts.path_and_query = Some(format!("{}?{}", uri.path(), canonical_query_string).parse().unwrap());
let req = req.uri(Uri::from_parts(parts).unwrap());*/ let req = req.uri(Uri::from_parts(parts).unwrap());*/
println!("{:?}", req.uri_ref().unwrap().query()); println!("{:?}", req.uri_ref().unwrap().query());
let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req)); let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req));
println!("canonical_request: \n{}\n", canonical_request); println!("canonical_request: \n{}\n", canonical_request);
assert_eq!( assert_eq!(
@@ -486,7 +590,9 @@ println!("{:?}", req.uri_ref().unwrap().query());
"delimiter=%2F&encoding-type=url&fetch-owner=true&list-type=2&max-keys=1&prefix=mypre\n", "delimiter=%2F&encoding-type=url&fetch-owner=true&list-type=2&max-keys=1&prefix=mypre\n",
"host:192.168.1.11:9020\n", "host:192.168.1.11:9020\n",
"x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n", "x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n",
"x-amz-date:", "20250507T051030Z", "\n", "x-amz-date:",
"20250507T051030Z",
"\n",
"\n", "\n",
"host;x-amz-content-sha256;x-amz-date\n", "host;x-amz-content-sha256;x-amz-date\n",
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
@@ -499,7 +605,8 @@ println!("{:?}", req.uri_ref().unwrap().query());
string_to_sign, string_to_sign,
concat!( concat!(
"AWS4-HMAC-SHA256\n", "AWS4-HMAC-SHA256\n",
"20250507T051030Z", "\n", "20250507T051030Z",
"\n",
"20250507/us-east-1/s3/aws4_request\n", "20250507/us-east-1/s3/aws4_request\n",
"e6db9e09e9c873aff0b9ca170998b4753f6a6c36c90bc2dca80613affb47f999", "e6db9e09e9c873aff0b9ca170998b4753f6a6c36c90bc2dca80613affb47f999",
) )
@@ -525,7 +632,9 @@ println!("{:?}", req.uri_ref().unwrap().query());
let path = "/"; let path = "/";
let session_token = ""; let session_token = "";
let mut req = Request::builder().method(http::Method::GET).uri("http://examplebucket.s3.amazonaws.com/test.txt"); let mut req = Request::builder()
.method(http::Method::GET)
.uri("http://examplebucket.s3.amazonaws.com/test.txt");
let mut headers = req.headers_mut().expect("err"); let mut headers = req.headers_mut().expect("err");
headers.insert("host", "examplebucket.s3.amazonaws.com".parse().unwrap()); headers.insert("host", "examplebucket.s3.amazonaws.com".parse().unwrap());
ecstore/src/store.rs
+15 -14
View File
@@ -1,20 +1,21 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use rustfs_filemeta::FileInfo; use crate::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry;
use crate::bucket::metadata_sys::{self, set_bucket_metadata}; use crate::bucket::metadata_sys::{self, set_bucket_metadata};
use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname}; use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname};
use crate::config::GLOBAL_StorageClass; use crate::config::GLOBAL_StorageClass;
use crate::config::storageclass; use crate::config::storageclass;
use crate::disk::endpoint::{Endpoint, EndpointType}; use crate::disk::endpoint::{Endpoint, EndpointType};
use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions}; use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions};
use crate::error::{Error, Result};
use crate::error::{ use crate::error::{
StorageError, is_err_bucket_exists, is_err_invalid_upload_id, is_err_object_not_found, is_err_read_quorum, StorageError, is_err_bucket_exists, is_err_invalid_upload_id, is_err_object_not_found, is_err_read_quorum,
is_err_version_not_found, to_object_err, is_err_version_not_found, to_object_err,
}; };
use crate::global::{ use crate::global::{
DISK_ASSUME_UNKNOWN_SIZE, DISK_FILL_FRACTION, DISK_MIN_INODES, DISK_RESERVE_FRACTION, GLOBAL_BOOT_TIME, DISK_ASSUME_UNKNOWN_SIZE, DISK_FILL_FRACTION, DISK_MIN_INODES, DISK_RESERVE_FRACTION, GLOBAL_BOOT_TIME,
GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_endpoints, is_dist_erasure, is_erasure_sd, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, GLOBAL_TierConfigMgr, get_global_endpoints, is_dist_erasure,
set_global_deployment_id, set_object_layer, GLOBAL_TierConfigMgr, is_erasure_sd, set_global_deployment_id, set_object_layer,
}; };
use crate::heal::data_usage::{DATA_USAGE_ROOT, DataUsageInfo}; use crate::heal::data_usage::{DATA_USAGE_ROOT, DataUsageInfo};
use crate::heal::data_usage_cache::{DataUsageCache, DataUsageCacheInfo}; use crate::heal::data_usage_cache::{DataUsageCache, DataUsageCacheInfo};
@@ -27,10 +28,7 @@ use crate::rebalance::RebalanceMeta;
use crate::store_api::{ListMultipartsInfo, ListObjectVersionsInfo, MultipartInfo, ObjectIO}; use crate::store_api::{ListMultipartsInfo, ListObjectVersionsInfo, MultipartInfo, ObjectIO};
use crate::store_init::{check_disk_fatal_errs, ec_drives_no_config}; use crate::store_init::{check_disk_fatal_errs, ec_drives_no_config};
use crate::{ use crate::{
bucket::{ bucket::{lifecycle::bucket_lifecycle_ops::TransitionState, metadata::BucketMetadata},
metadata::BucketMetadata,
lifecycle::bucket_lifecycle_ops::TransitionState
},
disk::{BUCKET_META_PREFIX, DiskOption, DiskStore, RUSTFS_META_BUCKET, new_disk}, disk::{BUCKET_META_PREFIX, DiskOption, DiskStore, RUSTFS_META_BUCKET, new_disk},
endpoints::EndpointServerPools, endpoints::EndpointServerPools,
peer::S3PeerSys, peer::S3PeerSys,
@@ -42,10 +40,6 @@ use crate::{
}, },
store_init, store_init,
}; };
use rustfs_utils::crypto::base64_decode;
use rustfs_utils::path::{SLASH_SEPARATOR, decode_dir_object, encode_dir_object, path_join_buf};
use crate::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry;
use crate::error::{Error, Result};
use common::globals::{GLOBAL_Local_Node_Name, GLOBAL_Rustfs_Host, GLOBAL_Rustfs_Port}; use common::globals::{GLOBAL_Local_Node_Name, GLOBAL_Rustfs_Host, GLOBAL_Rustfs_Port};
use futures::future::join_all; use futures::future::join_all;
use glob::Pattern; use glob::Pattern;
@@ -53,7 +47,10 @@ use http::HeaderMap;
use lazy_static::lazy_static; use lazy_static::lazy_static;
use madmin::heal_commands::HealResultItem; use madmin::heal_commands::HealResultItem;
use rand::Rng as _; use rand::Rng as _;
use rustfs_filemeta::FileInfo;
use rustfs_filemeta::MetaCacheEntry; use rustfs_filemeta::MetaCacheEntry;
use rustfs_utils::crypto::base64_decode;
use rustfs_utils::path::{SLASH_SEPARATOR, decode_dir_object, encode_dir_object, path_join_buf};
use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration}; use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration};
use std::cmp::Ordering; use std::cmp::Ordering;
use std::process::exit; use std::process::exit;
@@ -332,7 +329,7 @@ impl ECStore {
TransitionState::init(self.clone()).await; TransitionState::init(self.clone()).await;
if let Err(err) = GLOBAL_TierConfigMgr.write().await.init(self.clone()).await { if let Err(err) = GLOBAL_TierConfigMgr.write().await.init(self.clone()).await {
info!("TierConfigMgr init error: {}", err); info!("TierConfigMgr init error: {}", err);
} }
@@ -1868,7 +1865,9 @@ impl StorageAPI for ECStore {
self.pools[0].add_partial(bucket, object.as_str(), version_id).await; self.pools[0].add_partial(bucket, object.as_str(), version_id).await;
} }
let idx = self.get_pool_idx_existing_with_opts(bucket, object.as_str(), &ObjectOptions::default()).await?; let idx = self
.get_pool_idx_existing_with_opts(bucket, object.as_str(), &ObjectOptions::default())
.await?;
self.pools[idx].add_partial(bucket, object.as_str(), version_id).await; self.pools[idx].add_partial(bucket, object.as_str(), version_id).await;
Ok(()) Ok(())
@@ -2134,7 +2133,9 @@ impl StorageAPI for ECStore {
let object = rustfs_utils::path::encode_dir_object(object); let object = rustfs_utils::path::encode_dir_object(object);
if self.single_pool() { if self.single_pool() {
return self.pools[0].delete_object_version(bucket, object.as_str(), fi, force_del_marker).await; return self.pools[0]
.delete_object_version(bucket, object.as_str(), fi, force_del_marker)
.await;
} }
Ok(()) Ok(())
} }
ecstore/src/store_api.rs
+12 -12
View File
@@ -2,22 +2,19 @@ use crate::bucket::metadata_sys::get_versioning_config;
use crate::bucket::versioning::VersioningApi as _; use crate::bucket::versioning::VersioningApi as _;
use crate::cmd::bucket_replication::{ReplicationStatusType, VersionPurgeStatusType}; use crate::cmd::bucket_replication::{ReplicationStatusType, VersionPurgeStatusType};
use crate::error::{Error, Result}; use crate::error::{Error, Result};
use rustfs_utils::path::decode_dir_object;
use crate::heal::heal_ops::HealSequence; use crate::heal::heal_ops::HealSequence;
use crate::store_utils::clean_metadata; use crate::store_utils::clean_metadata;
use crate::{disk::DiskStore, heal::heal_commands::HealOpts,};
use crate::{ use crate::{
bucket::lifecycle::{
lifecycle::TransitionOptions,
bucket_lifecycle_ops::TransitionedObject,
},
bucket::lifecycle::bucket_lifecycle_audit::LcAuditEvent, bucket::lifecycle::bucket_lifecycle_audit::LcAuditEvent,
bucket::lifecycle::lifecycle::ExpirationOptions, bucket::lifecycle::lifecycle::ExpirationOptions,
bucket::lifecycle::{bucket_lifecycle_ops::TransitionedObject, lifecycle::TransitionOptions},
}; };
use crate::{disk::DiskStore, heal::heal_commands::HealOpts};
use http::{HeaderMap, HeaderValue}; use http::{HeaderMap, HeaderValue};
use madmin::heal_commands::HealResultItem; use madmin::heal_commands::HealResultItem;
use rustfs_filemeta::{FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, headers::AMZ_OBJECT_TAGGING}; use rustfs_filemeta::{FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, headers::AMZ_OBJECT_TAGGING};
use rustfs_rio::{HashReader, Reader}; use rustfs_rio::{HashReader, Reader};
use rustfs_utils::path::decode_dir_object;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::collections::HashMap; use std::collections::HashMap;
use std::fmt::Debug; use std::fmt::Debug;
@@ -446,14 +443,17 @@ impl ObjectInfo {
// TODO:ReplicationState // TODO:ReplicationState
let transitioned_object = TransitionedObject { let transitioned_object = TransitionedObject {
name: fi.transitioned_objname.clone(), name: fi.transitioned_objname.clone(),
version_id: if let Some(transition_version_id) = fi.transition_version_id { transition_version_id.to_string() } else { "".to_string() }, version_id: if let Some(transition_version_id) = fi.transition_version_id {
status: fi.transition_status.clone(), transition_version_id.to_string()
free_version: fi.tier_free_version(), } else {
tier: fi.transition_tier.clone(), "".to_string()
},
status: fi.transition_status.clone(),
free_version: fi.tier_free_version(),
tier: fi.transition_tier.clone(),
}; };
let metadata = { let metadata = {
let mut v = fi.metadata.clone(); let mut v = fi.metadata.clone();
clean_metadata(&mut v); clean_metadata(&mut v);
ecstore/src/tier/mod.rs
+5 -5
View File
@@ -1,9 +1,9 @@
pub mod warm_backend_s3; pub mod tier;
pub mod warm_backend_minio;
pub mod warm_backend_rustfs;
pub mod warm_backend;
pub mod tier_admin; pub mod tier_admin;
pub mod tier_config; pub mod tier_config;
pub mod tier;
pub mod tier_gen; pub mod tier_gen;
pub mod tier_handlers; pub mod tier_handlers;
pub mod warm_backend;
pub mod warm_backend_minio;
pub mod warm_backend_rustfs;
pub mod warm_backend_s3;
ecstore/src/tier/tier.rs
+81 -69
View File
@@ -1,46 +1,49 @@
use std::{
collections::{hash_map::Entry, HashMap}, io::Cursor, sync::Arc, time::{Duration,}
};
use bytes::Bytes; use bytes::Bytes;
use serde::{Serialize, Deserialize};
use time::OffsetDateTime;
use tokio::{select, sync::RwLock, time::interval};
use rand::Rng;
use tracing::{info, debug, warn, error};
use http::status::StatusCode; use http::status::StatusCode;
use lazy_static::lazy_static; use lazy_static::lazy_static;
use rand::Rng;
use serde::{Deserialize, Serialize};
use std::{
collections::{HashMap, hash_map::Entry},
io::Cursor,
sync::Arc,
time::Duration,
};
use time::OffsetDateTime;
use tokio::io::BufReader; use tokio::io::BufReader;
use tokio::{select, sync::RwLock, time::interval};
use tracing::{debug, error, info, warn};
use s3s::S3ErrorCode;
use crate::error::{Error, Result, StorageError};
use rustfs_utils::path::{path_join, SLASH_SEPARATOR};
use crate::{
config::com::{read_config, CONFIG_PREFIX},
disk::RUSTFS_META_BUCKET,
store::ECStore, store_api::{ObjectOptions, PutObjReader}, StorageAPI
};
use crate::client::admin_handler_utils::AdminError; use crate::client::admin_handler_utils::AdminError;
use crate::tier::{ use crate::error::{Error, Result, StorageError};
warm_backend::{check_warm_backend, new_warm_backend},
tier_handlers::{
ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_ALREADY_EXISTS, ERR_TIER_NOT_FOUND,
},
tier_admin::TierCreds,
tier_config::{TierType, TierConfig,},
};
use crate::new_object_layer_fn; use crate::new_object_layer_fn;
use crate::tier::{
tier_admin::TierCreds,
tier_config::{TierConfig, TierType},
tier_handlers::{ERR_TIER_ALREADY_EXISTS, ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_NOT_FOUND},
warm_backend::{check_warm_backend, new_warm_backend},
};
use crate::{
StorageAPI,
config::com::{CONFIG_PREFIX, read_config},
disk::RUSTFS_META_BUCKET,
store::ECStore,
store_api::{ObjectOptions, PutObjReader},
};
use rustfs_rio::HashReader; use rustfs_rio::HashReader;
use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
use s3s::S3ErrorCode;
use super::{ use super::{
tier_handlers::{ERR_TIER_PERM_ERR, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_BUCKET_NOT_FOUND}, tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_PERM_ERR},
warm_backend::WarmBackendImpl, warm_backend::WarmBackendImpl,
}; };
const TIER_CFG_REFRESH: Duration = Duration::from_secs(15 * 60); const TIER_CFG_REFRESH: Duration = Duration::from_secs(15 * 60);
pub const TIER_CONFIG_FILE: &str = "tier-config.json"; pub const TIER_CONFIG_FILE: &str = "tier-config.json";
pub const TIER_CONFIG_FORMAT: u16 = 1; pub const TIER_CONFIG_FORMAT: u16 = 1;
pub const TIER_CONFIG_V1: u16 = 1; pub const TIER_CONFIG_V1: u16 = 1;
pub const TIER_CONFIG_VERSION: u16 = 1; pub const TIER_CONFIG_VERSION: u16 = 1;
lazy_static! { lazy_static! {
@@ -50,32 +53,32 @@ lazy_static! {
const TIER_CFG_REFRESH_AT_HDR: &str = "X-RustFS-TierCfg-RefreshedAt"; const TIER_CFG_REFRESH_AT_HDR: &str = "X-RustFS-TierCfg-RefreshedAt";
pub const ERR_TIER_MISSING_CREDENTIALS: AdminError = AdminError { pub const ERR_TIER_MISSING_CREDENTIALS: AdminError = AdminError {
code: "XRustFSAdminTierMissingCredentials", code: "XRustFSAdminTierMissingCredentials",
message: "Specified remote credentials are empty", message: "Specified remote credentials are empty",
status_code: StatusCode::FORBIDDEN, status_code: StatusCode::FORBIDDEN,
}; };
pub const ERR_TIER_BACKEND_IN_USE: AdminError = AdminError { pub const ERR_TIER_BACKEND_IN_USE: AdminError = AdminError {
code: "XRustFSAdminTierBackendInUse", code: "XRustFSAdminTierBackendInUse",
message: "Specified remote tier is already in use", message: "Specified remote tier is already in use",
status_code: StatusCode::CONFLICT, status_code: StatusCode::CONFLICT,
}; };
pub const ERR_TIER_TYPE_UNSUPPORTED: AdminError = AdminError { pub const ERR_TIER_TYPE_UNSUPPORTED: AdminError = AdminError {
code: "XRustFSAdminTierTypeUnsupported", code: "XRustFSAdminTierTypeUnsupported",
message: "Specified tier type is unsupported", message: "Specified tier type is unsupported",
status_code: StatusCode::BAD_REQUEST, status_code: StatusCode::BAD_REQUEST,
}; };
pub const ERR_TIER_BACKEND_NOT_EMPTY: AdminError = AdminError { pub const ERR_TIER_BACKEND_NOT_EMPTY: AdminError = AdminError {
code: "XRustFSAdminTierBackendNotEmpty", code: "XRustFSAdminTierBackendNotEmpty",
message: "Specified remote backend is not empty", message: "Specified remote backend is not empty",
status_code: StatusCode::BAD_REQUEST, status_code: StatusCode::BAD_REQUEST,
}; };
pub const ERR_TIER_INVALID_CONFIG: AdminError = AdminError { pub const ERR_TIER_INVALID_CONFIG: AdminError = AdminError {
code: "XRustFSAdminTierInvalidConfig", code: "XRustFSAdminTierInvalidConfig",
message: "Unable to setup remote tier, check tier configuration", message: "Unable to setup remote tier, check tier configuration",
status_code: StatusCode::BAD_REQUEST, status_code: StatusCode::BAD_REQUEST,
}; };
@@ -147,22 +150,22 @@ impl TierConfigMgr {
if !force { if !force {
let in_use = d.in_use().await; let in_use = d.in_use().await;
match in_use { match in_use {
Ok(b) => { Ok(b) => {
if b { if b {
return Err(ERR_TIER_BACKEND_IN_USE); return Err(ERR_TIER_BACKEND_IN_USE);
} }
} }
Err(err) => { Err(err) => {
warn!("tier add failed, err: {:?}", err); warn!("tier add failed, err: {:?}", err);
if err.to_string().contains("connect") { if err.to_string().contains("connect") {
return Err(ERR_TIER_CONNECT_ERR); return Err(ERR_TIER_CONNECT_ERR);
} else if err.to_string().contains("authorization") { } else if err.to_string().contains("authorization") {
return Err(ERR_TIER_INVALID_CREDENTIALS); return Err(ERR_TIER_INVALID_CREDENTIALS);
} else if err.to_string().contains("bucket") { } else if err.to_string().contains("bucket") {
return Err(ERR_TIER_BUCKET_NOT_FOUND); return Err(ERR_TIER_BUCKET_NOT_FOUND);
} }
return Err(ERR_TIER_PERM_ERR); return Err(ERR_TIER_PERM_ERR);
} }
} }
} }
@@ -279,7 +282,7 @@ impl TierConfigMgr {
minio.access_key = creds.access_key; minio.access_key = creds.access_key;
minio.secret_key = creds.secret_key; minio.secret_key = creds.secret_key;
} }
_ => () _ => (),
} }
let d = new_warm_backend(&cfg, true).await?; let d = new_warm_backend(&cfg, true).await?;
@@ -290,9 +293,7 @@ impl TierConfigMgr {
pub async fn get_driver<'a>(&'a mut self, tier_name: &str) -> std::result::Result<&'a WarmBackendImpl, AdminError> { pub async fn get_driver<'a>(&'a mut self, tier_name: &str) -> std::result::Result<&'a WarmBackendImpl, AdminError> {
Ok(match self.driver_cache.entry(tier_name.to_string()) { Ok(match self.driver_cache.entry(tier_name.to_string()) {
Entry::Occupied(e) => { Entry::Occupied(e) => e.into_mut(),
e.into_mut()
}
Entry::Vacant(e) => { Entry::Vacant(e) => {
let t = self.tiers.get(tier_name); let t = self.tiers.get(tier_name);
if t.is_none() { if t.is_none() {
@@ -326,7 +327,9 @@ impl TierConfigMgr {
#[tracing::instrument(level = "debug", name = "tier_save", skip(self))] #[tracing::instrument(level = "debug", name = "tier_save", skip(self))]
pub async fn save(&self) -> std::result::Result<(), std::io::Error> { pub async fn save(&self) -> std::result::Result<(), std::io::Error> {
let Some(api) = new_object_layer_fn() else { return Err(std::io::Error::other("errServerNotInitialized")) }; let Some(api) = new_object_layer_fn() else {
return Err(std::io::Error::other("errServerNotInitialized"));
};
//let (pr, opts) = GLOBAL_TierConfigMgr.write().config_reader()?; //let (pr, opts) = GLOBAL_TierConfigMgr.write().config_reader()?;
self.save_tiering_config(api).await self.save_tiering_config(api).await
@@ -340,7 +343,12 @@ impl TierConfigMgr {
self.save_config(api, &config_file, data).await self.save_config(api, &config_file, data).await
} }
pub async fn save_config<S: StorageAPI>(&self, api: Arc<S>, file: &str, data: Bytes) -> std::result::Result<(), std::io::Error> { pub async fn save_config<S: StorageAPI>(
&self,
api: Arc<S>,
file: &str,
data: Bytes,
) -> std::result::Result<(), std::io::Error> {
self.save_config_with_opts( self.save_config_with_opts(
api, api,
file, file,
@@ -353,7 +361,13 @@ impl TierConfigMgr {
.await .await
} }
pub async fn save_config_with_opts<S: StorageAPI>(&self, api: Arc<S>, file: &str, data: Bytes, opts: &ObjectOptions) -> std::result::Result<(), std::io::Error> { pub async fn save_config_with_opts<S: StorageAPI>(
&self,
api: Arc<S>,
file: &str,
data: Bytes,
opts: &ObjectOptions,
) -> std::result::Result<(), std::io::Error> {
debug!("save tier config:{}", file); debug!("save tier config:{}", file);
let _ = api let _ = api
.put_object(RUSTFS_META_BUCKET, file, &mut PutObjReader::from_vec(data.to_vec()), opts) .put_object(RUSTFS_META_BUCKET, file, &mut PutObjReader::from_vec(data.to_vec()), opts)
@@ -365,9 +379,7 @@ impl TierConfigMgr {
//let r = rand.New(rand.NewSource(time.Now().UnixNano())); //let r = rand.New(rand.NewSource(time.Now().UnixNano()));
let mut rng = rand::rng(); let mut rng = rand::rng();
let r = rng.random_range(0.0..1.0); let r = rng.random_range(0.0..1.0);
let rand_interval = || { let rand_interval = || Duration::from_secs((r * 60_f64).round() as u64);
Duration::from_secs((r * 60_f64).round() as u64)
};
let mut t = interval(TIER_CFG_REFRESH + rand_interval()); let mut t = interval(TIER_CFG_REFRESH + rand_interval());
loop { loop {
@@ -394,10 +406,10 @@ impl TierConfigMgr {
async fn new_and_save_tiering_config<S: StorageAPI>(api: Arc<S>) -> Result<TierConfigMgr> { async fn new_and_save_tiering_config<S: StorageAPI>(api: Arc<S>) -> Result<TierConfigMgr> {
let mut cfg = TierConfigMgr { let mut cfg = TierConfigMgr {
driver_cache: HashMap::new(), driver_cache: HashMap::new(),
tiers: HashMap::new(), tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(), last_refreshed_at: OffsetDateTime::now_utc(),
}; };
//lookup_configs(&mut cfg, api.clone()).await; //lookup_configs(&mut cfg, api.clone()).await;
cfg.save_tiering_config(api).await?; cfg.save_tiering_config(api).await?;
@@ -420,7 +432,7 @@ async fn load_tier_config(api: Arc<ECStore>) -> std::result::Result<TierConfigMg
} }
let cfg; let cfg;
let version = 1;//LittleEndian::read_u16(&data[2..4]); let version = 1; //LittleEndian::read_u16(&data[2..4]);
match version { match version {
TIER_CONFIG_V1/* | TIER_CONFIG_VERSION */ => { TIER_CONFIG_V1/* | TIER_CONFIG_VERSION */ => {
cfg = match TierConfigMgr::unmarshal(&data.unwrap()) { cfg = match TierConfigMgr::unmarshal(&data.unwrap()) {
ecstore/src/tier/tier_admin.rs
+5 -8
View File
@@ -1,13 +1,10 @@
use std::{
time::{Duration, SystemTime, UNIX_EPOCH},
};
use rand::Rng;
use tracing::warn;
use http::status::StatusCode; use http::status::StatusCode;
use serde::{Serialize, Deserialize}; use rand::Rng;
use serde::{Deserialize, Serialize};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use tracing::warn;
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[derive(Default, Debug, Clone)]
#[serde(default)] #[serde(default)]
pub struct TierCreds { pub struct TierCreds {
#[serde(rename = "accessKey")] #[serde(rename = "accessKey")]
ecstore/src/tier/tier_config.rs
+58 -95
View File
@@ -1,5 +1,5 @@
use serde::{Deserialize, Serialize};
use std::fmt::Display; use std::fmt::Display;
use serde::{Serialize, Deserialize};
use tracing::info; use tracing::info;
const C_TierConfigVer: &str = "v1"; const C_TierConfigVer: &str = "v1";
@@ -9,8 +9,7 @@ const ERR_TIER_INVALID_CONFIG: &str = "invalid tier config";
const ERR_TIER_INVALID_CONFIG_VERSION: &str = "invalid tier config version"; const ERR_TIER_INVALID_CONFIG_VERSION: &str = "invalid tier config version";
const ERR_TIER_TYPE_UNSUPPORTED: &str = "unsupported tier type"; const ERR_TIER_TYPE_UNSUPPORTED: &str = "unsupported tier type";
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[derive(Default, Debug, Clone)]
pub enum TierType { pub enum TierType {
#[default] #[default]
Unsupported, Unsupported,
@@ -48,35 +47,19 @@ impl Display for TierType {
impl TierType { impl TierType {
pub fn new(sc_type: &str) -> Self { pub fn new(sc_type: &str) -> Self {
match sc_type { match sc_type {
"S3" => { "S3" => TierType::S3,
TierType::S3 "RustFS" => TierType::RustFS,
} "MinIO" => TierType::MinIO,
"RustFS" => { _ => TierType::Unsupported,
TierType::RustFS
}
"MinIO" => {
TierType::MinIO
}
_ => {
TierType::Unsupported
}
} }
} }
pub fn to_string(&self) -> String { pub fn to_string(&self) -> String {
match self { match self {
TierType::S3 => { TierType::S3 => "s3".to_string(),
"s3".to_string() TierType::RustFS => "rustfs".to_string(),
} TierType::MinIO => "minio".to_string(),
TierType::RustFS => { _ => "unsupported".to_string(),
"rustfs".to_string()
}
TierType::MinIO => {
"minio".to_string()
}
_ => {
"unsupported".to_string()
}
} }
} }
} }
@@ -123,17 +106,17 @@ impl Clone for TierConfig {
m_.secret_key = "REDACTED".to_string(); m_.secret_key = "REDACTED".to_string();
m = Some(m_); m = Some(m_);
} }
_ => () _ => (),
} }
TierConfig { TierConfig {
version: self.version.clone(), version: self.version.clone(),
tier_type: self.tier_type.clone(), tier_type: self.tier_type.clone(),
name: self.name.clone(), name: self.name.clone(),
s3: s3, s3: s3,
//azure: az, //azure: az,
//gcs: gcs, //gcs: gcs,
rustfs: r, rustfs: r,
minio: m, minio: m,
} }
} }
} }
@@ -154,15 +137,9 @@ impl TierConfig {
fn endpoint(&self) -> String { fn endpoint(&self) -> String {
match self.tier_type { match self.tier_type {
TierType::S3 => { TierType::S3 => self.s3.as_ref().expect("err").endpoint.clone(),
self.s3.as_ref().expect("err").endpoint.clone() TierType::RustFS => self.rustfs.as_ref().expect("err").endpoint.clone(),
} TierType::MinIO => self.minio.as_ref().expect("err").endpoint.clone(),
TierType::RustFS => {
self.rustfs.as_ref().expect("err").endpoint.clone()
}
TierType::MinIO => {
self.minio.as_ref().expect("err").endpoint.clone()
}
_ => { _ => {
info!("unexpected tier type {}", self.tier_type); info!("unexpected tier type {}", self.tier_type);
"".to_string() "".to_string()
@@ -172,15 +149,9 @@ impl TierConfig {
fn bucket(&self) -> String { fn bucket(&self) -> String {
match self.tier_type { match self.tier_type {
TierType::S3 => { TierType::S3 => self.s3.as_ref().expect("err").bucket.clone(),
self.s3.as_ref().expect("err").bucket.clone() TierType::RustFS => self.rustfs.as_ref().expect("err").bucket.clone(),
} TierType::MinIO => self.minio.as_ref().expect("err").bucket.clone(),
TierType::RustFS => {
self.rustfs.as_ref().expect("err").bucket.clone()
}
TierType::MinIO => {
self.minio.as_ref().expect("err").bucket.clone()
}
_ => { _ => {
info!("unexpected tier type {}", self.tier_type); info!("unexpected tier type {}", self.tier_type);
"".to_string() "".to_string()
@@ -190,15 +161,9 @@ impl TierConfig {
fn prefix(&self) -> String { fn prefix(&self) -> String {
match self.tier_type { match self.tier_type {
TierType::S3 => { TierType::S3 => self.s3.as_ref().expect("err").prefix.clone(),
self.s3.as_ref().expect("err").prefix.clone() TierType::RustFS => self.rustfs.as_ref().expect("err").prefix.clone(),
} TierType::MinIO => self.minio.as_ref().expect("err").prefix.clone(),
TierType::RustFS => {
self.rustfs.as_ref().expect("err").prefix.clone()
}
TierType::MinIO => {
self.minio.as_ref().expect("err").prefix.clone()
}
_ => { _ => {
info!("unexpected tier type {}", self.tier_type); info!("unexpected tier type {}", self.tier_type);
"".to_string() "".to_string()
@@ -208,15 +173,9 @@ impl TierConfig {
fn region(&self) -> String { fn region(&self) -> String {
match self.tier_type { match self.tier_type {
TierType::S3 => { TierType::S3 => self.s3.as_ref().expect("err").region.clone(),
self.s3.as_ref().expect("err").region.clone() TierType::RustFS => self.rustfs.as_ref().expect("err").region.clone(),
} TierType::MinIO => self.minio.as_ref().expect("err").region.clone(),
TierType::RustFS => {
self.rustfs.as_ref().expect("err").region.clone()
}
TierType::MinIO => {
self.minio.as_ref().expect("err").region.clone()
}
_ => { _ => {
info!("unexpected tier type {}", self.tier_type); info!("unexpected tier type {}", self.tier_type);
"".to_string() "".to_string()
@@ -227,8 +186,7 @@ impl TierConfig {
//type S3Options = impl Fn(TierS3) -> Pin<Box<Result<()>>> + Send + Sync + 'static; //type S3Options = impl Fn(TierS3) -> Pin<Box<Result<()>>> + Send + Sync + 'static;
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[derive(Default, Debug, Clone)]
#[serde(default)] #[serde(default)]
pub struct TierS3 { pub struct TierS3 {
pub name: String, pub name: String,
@@ -257,17 +215,17 @@ pub struct TierS3 {
impl TierS3 { impl TierS3 {
fn new<F>(name: &str, access_key: &str, secret_key: &str, bucket: &str, options: Vec<F>) -> Result<TierConfig, std::io::Error> fn new<F>(name: &str, access_key: &str, secret_key: &str, bucket: &str, options: Vec<F>) -> Result<TierConfig, std::io::Error>
where where
F: Fn(TierS3) -> Box<Result<(), std::io::Error>> + Send + Sync + 'static F: Fn(TierS3) -> Box<Result<(), std::io::Error>> + Send + Sync + 'static,
{ {
if name == "" { if name == "" {
return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY)); return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY));
} }
let sc = TierS3 { let sc = TierS3 {
access_key: access_key.to_string(), access_key: access_key.to_string(),
secret_key: secret_key.to_string(), secret_key: secret_key.to_string(),
bucket: bucket.to_string(), bucket: bucket.to_string(),
endpoint: "https://s3.amazonaws.com".to_string(), endpoint: "https://s3.amazonaws.com".to_string(),
region: "".to_string(), region: "".to_string(),
storage_class: "".to_string(), storage_class: "".to_string(),
..Default::default() ..Default::default()
}; };
@@ -281,49 +239,54 @@ impl TierS3 {
Ok(TierConfig { Ok(TierConfig {
version: C_TierConfigVer.to_string(), version: C_TierConfigVer.to_string(),
tier_type: TierType::S3, tier_type: TierType::S3,
name: name.to_string(), name: name.to_string(),
s3: Some(sc), s3: Some(sc),
..Default::default() ..Default::default()
}) })
} }
} }
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[derive(Default, Debug, Clone)]
#[serde(default)] #[serde(default)]
pub struct TierRustFS { pub struct TierRustFS {
pub name: String, pub name: String,
pub endpoint: String, pub endpoint: String,
#[serde(rename = "accesskey")] #[serde(rename = "accesskey")]
pub access_key: String, pub access_key: String,
#[serde(rename = "secretkey")] #[serde(rename = "secretkey")]
pub secret_key: String, pub secret_key: String,
pub bucket: String, pub bucket: String,
pub prefix: String, pub prefix: String,
pub region: String, pub region: String,
#[serde(rename = "storageclass")] #[serde(rename = "storageclass")]
pub storage_class: String, pub storage_class: String,
} }
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[derive(Default, Debug, Clone)]
#[serde(default)] #[serde(default)]
pub struct TierMinIO { pub struct TierMinIO {
pub name: String, pub name: String,
pub endpoint: String, pub endpoint: String,
#[serde(rename = "accesskey")] #[serde(rename = "accesskey")]
pub access_key: String, pub access_key: String,
#[serde(rename = "secretkey")] #[serde(rename = "secretkey")]
pub secret_key: String, pub secret_key: String,
pub bucket: String, pub bucket: String,
pub prefix: String, pub prefix: String,
pub region: String, pub region: String,
} }
impl TierMinIO { impl TierMinIO {
fn new<F>(name: &str, endpoint: &str, access_key: &str, secret_key: &str, bucket: &str, options: Vec<F>) -> Result<TierConfig, std::io::Error> fn new<F>(
name: &str,
endpoint: &str,
access_key: &str,
secret_key: &str,
bucket: &str,
options: Vec<F>,
) -> Result<TierConfig, std::io::Error>
where where
F: Fn(TierMinIO) -> Box<Result<(), std::io::Error>> + Send + Sync + 'static F: Fn(TierMinIO) -> Box<Result<(), std::io::Error>> + Send + Sync + 'static,
{ {
if name == "" { if name == "" {
return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY)); return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY));
@@ -331,8 +294,8 @@ impl TierMinIO {
let m = TierMinIO { let m = TierMinIO {
access_key: access_key.to_string(), access_key: access_key.to_string(),
secret_key: secret_key.to_string(), secret_key: secret_key.to_string(),
bucket: bucket.to_string(), bucket: bucket.to_string(),
endpoint: endpoint.to_string(), endpoint: endpoint.to_string(),
..Default::default() ..Default::default()
}; };
@@ -345,8 +308,8 @@ impl TierMinIO {
Ok(TierConfig { Ok(TierConfig {
version: C_TierConfigVer.to_string(), version: C_TierConfigVer.to_string(),
tier_type: TierType::MinIO, tier_type: TierType::MinIO,
name: name.to_string(), name: name.to_string(),
minio: Some(m), minio: Some(m),
..Default::default() ..Default::default()
}) })
} }
ecstore/src/tier/tier_handlers.rs
+17 -17
View File
@@ -1,51 +1,51 @@
use crate::client::admin_handler_utils::AdminError; use crate::client::admin_handler_utils::AdminError;
use tracing::warn;
use http::status::StatusCode; use http::status::StatusCode;
use tracing::warn;
pub const ERR_TIER_ALREADY_EXISTS: AdminError = AdminError { pub const ERR_TIER_ALREADY_EXISTS: AdminError = AdminError {
code: "XRustFSAdminTierAlreadyExists", code: "XRustFSAdminTierAlreadyExists",
message: "Specified remote tier already exists", message: "Specified remote tier already exists",
status_code: StatusCode::CONFLICT, status_code: StatusCode::CONFLICT,
}; };
pub const ERR_TIER_NOT_FOUND: AdminError = AdminError { pub const ERR_TIER_NOT_FOUND: AdminError = AdminError {
code: "XRustFSAdminTierNotFound", code: "XRustFSAdminTierNotFound",
message: "Specified remote tier was not found", message: "Specified remote tier was not found",
status_code: StatusCode::NOT_FOUND, status_code: StatusCode::NOT_FOUND,
}; };
pub const ERR_TIER_NAME_NOT_UPPERCASE: AdminError = AdminError { pub const ERR_TIER_NAME_NOT_UPPERCASE: AdminError = AdminError {
code: "XRustFSAdminTierNameNotUpperCase", code: "XRustFSAdminTierNameNotUpperCase",
message: "Tier name must be in uppercase", message: "Tier name must be in uppercase",
status_code: StatusCode::BAD_REQUEST, status_code: StatusCode::BAD_REQUEST,
}; };
pub const ERR_TIER_BUCKET_NOT_FOUND: AdminError = AdminError { pub const ERR_TIER_BUCKET_NOT_FOUND: AdminError = AdminError {
code: "XRustFSAdminTierBucketNotFound", code: "XRustFSAdminTierBucketNotFound",
message: "Remote tier bucket not found", message: "Remote tier bucket not found",
status_code: StatusCode::BAD_REQUEST, status_code: StatusCode::BAD_REQUEST,
}; };
pub const ERR_TIER_INVALID_CREDENTIALS: AdminError = AdminError { pub const ERR_TIER_INVALID_CREDENTIALS: AdminError = AdminError {
code: "XRustFSAdminTierInvalidCredentials", code: "XRustFSAdminTierInvalidCredentials",
message: "Invalid remote tier credentials", message: "Invalid remote tier credentials",
status_code: StatusCode::BAD_REQUEST, status_code: StatusCode::BAD_REQUEST,
}; };
pub const ERR_TIER_RESERVED_NAME: AdminError = AdminError { pub const ERR_TIER_RESERVED_NAME: AdminError = AdminError {
code: "XRustFSAdminTierReserved", code: "XRustFSAdminTierReserved",
message: "Cannot use reserved tier name", message: "Cannot use reserved tier name",
status_code: StatusCode::BAD_REQUEST, status_code: StatusCode::BAD_REQUEST,
}; };
pub const ERR_TIER_PERM_ERR: AdminError = AdminError { pub const ERR_TIER_PERM_ERR: AdminError = AdminError {
code: "TierPermErr", code: "TierPermErr",
message: "Tier Perm Err", message: "Tier Perm Err",
status_code: StatusCode::OK, status_code: StatusCode::OK,
}; };
pub const ERR_TIER_CONNECT_ERR: AdminError = AdminError { pub const ERR_TIER_CONNECT_ERR: AdminError = AdminError {
code: "TierConnectErr", code: "TierConnectErr",
message: "Tier Connect Err", message: "Tier Connect Err",
status_code: StatusCode::OK, status_code: StatusCode::OK,
}; };
ecstore/src/tier/warm_backend.rs
+18 -10
View File
@@ -1,20 +1,20 @@
use std::collections::HashMap;
use bytes::Bytes; use bytes::Bytes;
use std::collections::HashMap;
use crate::client::{ use crate::client::{
admin_handler_utils::AdminError, admin_handler_utils::AdminError,
transition_api::{ReadCloser, ReaderImpl,}, transition_api::{ReadCloser, ReaderImpl},
}; };
use crate::error::is_err_bucket_not_found; use crate::error::is_err_bucket_not_found;
use tracing::{info, warn};
use crate::tier::{ use crate::tier::{
tier_config::{TierType, TierConfig}, tier::{ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED},
tier_config::{TierConfig, TierType},
tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_PERM_ERR}, tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_PERM_ERR},
tier::{ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED,},
warm_backend_s3::WarmBackendS3,
warm_backend_rustfs::WarmBackendRustFS,
warm_backend_minio::WarmBackendMinIO, warm_backend_minio::WarmBackendMinIO,
warm_backend_rustfs::WarmBackendRustFS,
warm_backend_s3::WarmBackendS3,
}; };
use tracing::{info, warn};
pub type WarmBackendImpl = Box<dyn WarmBackend + Send + Sync + 'static>; pub type WarmBackendImpl = Box<dyn WarmBackend + Send + Sync + 'static>;
@@ -29,7 +29,13 @@ pub struct WarmBackendGetOpts {
#[async_trait::async_trait] #[async_trait::async_trait]
pub trait WarmBackend { pub trait WarmBackend {
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error>; async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error>;
async fn put_with_meta(&self, object: &str, r: ReaderImpl, length: i64, meta: HashMap<String, String>) -> Result<String, std::io::Error>; async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
meta: HashMap<String, String>,
) -> Result<String, std::io::Error>;
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error>; async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error>;
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error>; async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error>;
async fn in_use(&self) -> Result<bool, std::io::Error>; async fn in_use(&self) -> Result<bool, std::io::Error>;
@@ -37,7 +43,9 @@ pub trait WarmBackend {
pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> { pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> {
let w = w.expect("err"); let w = w.expect("err");
let remote_version_id = w.put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5).await; let remote_version_id = w
.put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5)
.await;
if let Err(err) = remote_version_id { if let Err(err) = remote_version_id {
return Err(ERR_TIER_PERM_ERR); return Err(ERR_TIER_PERM_ERR);
} }
@@ -52,7 +60,7 @@ pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), Admin
return Err(ERR_TIER_MISSING_CREDENTIALS); return Err(ERR_TIER_MISSING_CREDENTIALS);
}*/ }*/
//else { //else {
return Err(ERR_TIER_PERM_ERR); return Err(ERR_TIER_PERM_ERR);
//} //}
} }
if let Err(err) = w.remove(PROBE_OBJECT, &remote_version_id.expect("err")).await { if let Err(err) = w.remove(PROBE_OBJECT, &remote_version_id.expect("err")).await {
ecstore/src/tier/warm_backend_minio.rs
+37 -27
View File
@@ -1,24 +1,23 @@
use std::collections::HashMap; use std::collections::HashMap;
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use tracing::warn;
use crate::client::{ use crate::client::{
admin_handler_utils::AdminError, admin_handler_utils::AdminError,
transition_api::{Options, ReaderImpl, ReadCloser, TransitionClient, TransitionCore},
credentials::{Credentials, SignatureType, Static, Value},
api_put_object::PutObjectOptions, api_put_object::PutObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
}; };
use crate::tier::{ use crate::tier::{
tier_config::TierMinIO, tier_config::TierMinIO,
warm_backend::{WarmBackend, WarmBackendGetOpts}, warm_backend::{WarmBackend, WarmBackendGetOpts},
warm_backend_s3::WarmBackendS3, warm_backend_s3::WarmBackendS3,
}; };
use tracing::warn;
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
const MAX_PARTS_COUNT: i64 = 10000; const MAX_PARTS_COUNT: i64 = 10000;
const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5; const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128; const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
pub struct WarmBackendMinIO(WarmBackendS3); pub struct WarmBackendMinIO(WarmBackendS3);
@@ -40,26 +39,23 @@ impl WarmBackendMinIO {
}; };
let creds = Credentials::new(Static(Value { let creds = Credentials::new(Static(Value {
access_key_id: conf.access_key.clone(), access_key_id: conf.access_key.clone(),
secret_access_key: conf.secret_key.clone(), secret_access_key: conf.secret_key.clone(),
session_token: "".to_string(), session_token: "".to_string(),
signer_type: SignatureType::SignatureV4, signer_type: SignatureType::SignatureV4,
..Default::default() ..Default::default()
})); }));
let opts = Options { let opts = Options {
creds: creds, creds: creds,
secure: u.scheme() == "https", secure: u.scheme() == "https",
//transport: GLOBAL_RemoteTargetTransport, //transport: GLOBAL_RemoteTargetTransport,
trailing_headers: true, trailing_headers: true,
..Default::default() ..Default::default()
}; };
let scheme = u.scheme(); let scheme = u.scheme();
let default_port = if scheme == "https" { let default_port = if scheme == "https" { 443 } else { 80 };
443 let client =
} else { TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?;
80
};
let client = TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?;
//client.set_appinfo(format!("minio-tier-{}", tier), ReleaseTag); //client.set_appinfo(format!("minio-tier-{}", tier), ReleaseTag);
let client = Arc::new(client); let client = Arc::new(client);
@@ -67,8 +63,8 @@ impl WarmBackendMinIO {
Ok(Self(WarmBackendS3 { Ok(Self(WarmBackendS3 {
client, client,
core, core,
bucket: conf.bucket.clone(), bucket: conf.bucket.clone(),
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(), prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
storage_class: "".to_string(), storage_class: "".to_string(),
})) }))
} }
@@ -76,16 +72,30 @@ impl WarmBackendMinIO {
#[async_trait::async_trait] #[async_trait::async_trait]
impl WarmBackend for WarmBackendMinIO { impl WarmBackend for WarmBackendMinIO {
async fn put_with_meta(&self, object: &str, r: ReaderImpl, length: i64, meta: HashMap<String, String>) -> Result<String, std::io::Error> { async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
meta: HashMap<String, String>,
) -> Result<String, std::io::Error> {
let part_size = optimal_part_size(length)?; let part_size = optimal_part_size(length)?;
let client = self.0.client.clone(); let client = self.0.client.clone();
let res = client.put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &PutObjectOptions { let res = client
storage_class: self.0.storage_class.clone(), .put_object(
part_size: part_size as u64, &self.0.bucket,
disable_content_sha256: true, &self.0.get_dest(object),
user_metadata: meta, r,
..Default::default() length,
}).await?; &PutObjectOptions {
storage_class: self.0.storage_class.clone(),
part_size: part_size as u64,
disable_content_sha256: true,
user_metadata: meta,
..Default::default()
},
)
.await?;
//self.ToObjectError(err, object) //self.ToObjectError(err, object)
Ok(res.version_id) Ok(res.version_id)
} }
ecstore/src/tier/warm_backend_rustfs.rs
+36 -25
View File
@@ -4,9 +4,9 @@ use tracing::warn;
use crate::client::{ use crate::client::{
admin_handler_utils::AdminError, admin_handler_utils::AdminError,
transition_api::{Options, ReaderImpl, ReadCloser, TransitionClient, TransitionCore},
credentials::{Credentials, SignatureType, Static, Value},
api_put_object::PutObjectOptions, api_put_object::PutObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
}; };
use crate::tier::{ use crate::tier::{
tier_config::TierRustFS, tier_config::TierRustFS,
@@ -15,9 +15,9 @@ use crate::tier::{
}; };
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
const MAX_PARTS_COUNT: i64 = 10000; const MAX_PARTS_COUNT: i64 = 10000;
const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5; const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128; const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
pub struct WarmBackendRustFS(WarmBackendS3); pub struct WarmBackendRustFS(WarmBackendS3);
@@ -37,26 +37,23 @@ impl WarmBackendRustFS {
}; };
let creds = Credentials::new(Static(Value { let creds = Credentials::new(Static(Value {
access_key_id: conf.access_key.clone(), access_key_id: conf.access_key.clone(),
secret_access_key: conf.secret_key.clone(), secret_access_key: conf.secret_key.clone(),
session_token: "".to_string(), session_token: "".to_string(),
signer_type: SignatureType::SignatureV4, signer_type: SignatureType::SignatureV4,
..Default::default() ..Default::default()
})); }));
let opts = Options { let opts = Options {
creds: creds, creds: creds,
secure: u.scheme() == "https", secure: u.scheme() == "https",
//transport: GLOBAL_RemoteTargetTransport, //transport: GLOBAL_RemoteTargetTransport,
trailing_headers: true, trailing_headers: true,
..Default::default() ..Default::default()
}; };
let scheme = u.scheme(); let scheme = u.scheme();
let default_port = if scheme == "https" { let default_port = if scheme == "https" { 443 } else { 80 };
443 let client =
} else { TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?;
80
};
let client = TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?;
//client.set_appinfo(format!("rustfs-tier-{}", tier), ReleaseTag); //client.set_appinfo(format!("rustfs-tier-{}", tier), ReleaseTag);
let client = Arc::new(client); let client = Arc::new(client);
@@ -64,8 +61,8 @@ impl WarmBackendRustFS {
Ok(Self(WarmBackendS3 { Ok(Self(WarmBackendS3 {
client, client,
core, core,
bucket: conf.bucket.clone(), bucket: conf.bucket.clone(),
prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(), prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
storage_class: "".to_string(), storage_class: "".to_string(),
})) }))
} }
@@ -73,16 +70,30 @@ impl WarmBackendRustFS {
#[async_trait::async_trait] #[async_trait::async_trait]
impl WarmBackend for WarmBackendRustFS { impl WarmBackend for WarmBackendRustFS {
async fn put_with_meta(&self, object: &str, r: ReaderImpl, length: i64, meta: HashMap<String, String>) -> Result<String, std::io::Error> { async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
meta: HashMap<String, String>,
) -> Result<String, std::io::Error> {
let part_size = optimal_part_size(length)?; let part_size = optimal_part_size(length)?;
let client = self.0.client.clone(); let client = self.0.client.clone();
let res = client.put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &PutObjectOptions { let res = client
storage_class: self.0.storage_class.clone(), .put_object(
part_size: part_size as u64, &self.0.bucket,
disable_content_sha256: true, &self.0.get_dest(object),
user_metadata: meta, r,
..Default::default() length,
}).await?; &PutObjectOptions {
storage_class: self.0.storage_class.clone(),
part_size: part_size as u64,
disable_content_sha256: true,
user_metadata: meta,
..Default::default()
},
)
.await?;
//self.ToObjectError(err, object) //self.ToObjectError(err, object)
Ok(res.version_id) Ok(res.version_id)
} }
ecstore/src/tier/warm_backend_s3.rs
+54 -32
View File
@@ -2,21 +2,21 @@ use std::collections::HashMap;
use std::sync::Arc; use std::sync::Arc;
use url::Url; use url::Url;
use crate::error::ErrorResponse;
use crate::error::error_resp_to_object_err;
use crate::client::{ use crate::client::{
api_get_options::GetObjectOptions, api_get_options::GetObjectOptions,
credentials::{Credentials, Static, Value, SignatureType},
transition_api::{ReaderImpl, ReadCloser},
api_put_object::PutObjectOptions, api_put_object::PutObjectOptions,
api_remove::RemoveObjectOptions, api_remove::RemoveObjectOptions,
transition_api::{Options, TransitionClient, TransitionCore,}, credentials::{Credentials, SignatureType, Static, Value},
transition_api::{Options, TransitionClient, TransitionCore},
transition_api::{ReadCloser, ReaderImpl},
}; };
use rustfs_utils::path::SLASH_SEPARATOR; use crate::error::ErrorResponse;
use crate::error::error_resp_to_object_err;
use crate::tier::{ use crate::tier::{
tier_config::TierS3, tier_config::TierS3,
warm_backend::{WarmBackend, WarmBackendGetOpts,} warm_backend::{WarmBackend, WarmBackendGetOpts},
}; };
use rustfs_utils::path::SLASH_SEPARATOR;
pub struct WarmBackendS3 { pub struct WarmBackendS3 {
pub client: Arc<TransitionClient>, pub client: Arc<TransitionClient>,
@@ -35,16 +35,22 @@ impl WarmBackendS3 {
} }
}; };
if conf.aws_role_web_identity_token_file == "" && conf.aws_role_arn != "" || conf.aws_role_web_identity_token_file != "" && conf.aws_role_arn == "" { if conf.aws_role_web_identity_token_file == "" && conf.aws_role_arn != ""
|| conf.aws_role_web_identity_token_file != "" && conf.aws_role_arn == ""
{
return Err(std::io::Error::other("both the token file and the role ARN are required")); return Err(std::io::Error::other("both the token file and the role ARN are required"));
} } else if conf.access_key == "" && conf.secret_key != "" || conf.access_key != "" && conf.secret_key == "" {
else if conf.access_key == "" && conf.secret_key != "" || conf.access_key != "" && conf.secret_key == "" {
return Err(std::io::Error::other("both the access and secret keys are required")); return Err(std::io::Error::other("both the access and secret keys are required"));
} } else if conf.aws_role
else if conf.aws_role && (conf.aws_role_web_identity_token_file != "" || conf.aws_role_arn != "" || conf.access_key != "" || conf.secret_key != "") { && (conf.aws_role_web_identity_token_file != ""
return Err(std::io::Error::other("AWS Role cannot be activated with static credentials or the web identity token file")); || conf.aws_role_arn != ""
} || conf.access_key != ""
else if conf.bucket == "" { || conf.secret_key != "")
{
return Err(std::io::Error::other(
"AWS Role cannot be activated with static credentials or the web identity token file",
));
} else if conf.bucket == "" {
return Err(std::io::Error::other("no bucket name was provided")); return Err(std::io::Error::other("no bucket name was provided"));
} }
@@ -53,19 +59,18 @@ impl WarmBackendS3 {
if conf.access_key != "" && conf.secret_key != "" { if conf.access_key != "" && conf.secret_key != "" {
//creds = Credentials::new_static_v4(conf.access_key, conf.secret_key, ""); //creds = Credentials::new_static_v4(conf.access_key, conf.secret_key, "");
creds = Credentials::new(Static(Value { creds = Credentials::new(Static(Value {
access_key_id: conf.access_key.clone(), access_key_id: conf.access_key.clone(),
secret_access_key: conf.secret_key.clone(), secret_access_key: conf.secret_key.clone(),
session_token: "".to_string(), session_token: "".to_string(),
signer_type: SignatureType::SignatureV4, signer_type: SignatureType::SignatureV4,
..Default::default() ..Default::default()
})); }));
} } else {
else {
return Err(std::io::Error::other("insufficient parameters for S3 backend authentication")); return Err(std::io::Error::other("insufficient parameters for S3 backend authentication"));
} }
let opts = Options { let opts = Options {
creds: creds, creds: creds,
secure: u.scheme() == "https", secure: u.scheme() == "https",
//transport: GLOBAL_RemoteTargetTransport, //transport: GLOBAL_RemoteTargetTransport,
..Default::default() ..Default::default()
}; };
@@ -77,8 +82,8 @@ impl WarmBackendS3 {
Ok(Self { Ok(Self {
client, client,
core, core,
bucket: conf.bucket.clone(), bucket: conf.bucket.clone(),
prefix: conf.prefix.clone().trim_matches('/').to_string(), prefix: conf.prefix.clone().trim_matches('/').to_string(),
storage_class: conf.storage_class.clone(), storage_class: conf.storage_class.clone(),
}) })
} }
@@ -103,14 +108,28 @@ impl WarmBackendS3 {
#[async_trait::async_trait] #[async_trait::async_trait]
impl WarmBackend for WarmBackendS3 { impl WarmBackend for WarmBackendS3 {
async fn put_with_meta(&self, object: &str, r: ReaderImpl, length: i64, meta: HashMap<String, String>) -> Result<String, std::io::Error> { async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
meta: HashMap<String, String>,
) -> Result<String, std::io::Error> {
let client = self.client.clone(); let client = self.client.clone();
let res = client.put_object(&self.bucket, &self.get_dest(object), r, length, &PutObjectOptions { let res = client
send_content_md5: true, .put_object(
storage_class: self.storage_class.clone(), &self.bucket,
user_metadata: meta, &self.get_dest(object),
..Default::default() r,
}).await?; length,
&PutObjectOptions {
send_content_md5: true,
storage_class: self.storage_class.clone(),
user_metadata: meta,
..Default::default()
},
)
.await?;
Ok(res.version_id) Ok(res.version_id)
} }
@@ -125,7 +144,7 @@ impl WarmBackend for WarmBackendS3 {
gopts.version_id = rv.to_string(); gopts.version_id = rv.to_string();
} }
if opts.start_offset >= 0 && opts.length > 0 { if opts.start_offset >= 0 && opts.length > 0 {
if let Err(err) = gopts.set_range(opts.start_offset, opts.start_offset+opts.length-1) { if let Err(err) = gopts.set_range(opts.start_offset, opts.start_offset + opts.length - 1) {
return Err(std::io::Error::other(err)); return Err(std::io::Error::other(err));
} }
} }
@@ -146,7 +165,10 @@ impl WarmBackend for WarmBackendS3 {
} }
async fn in_use(&self) -> Result<bool, std::io::Error> { async fn in_use(&self) -> Result<bool, std::io::Error> {
let result = self.core.list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1).await?; let result = self
.core
.list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1)
.await?;
Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0) Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)
} }
reader/src/hasher.rs
+1 -1
View File
@@ -1,7 +1,7 @@
use md5::{Digest as Md5Digest, Md5}; use md5::{Digest as Md5Digest, Md5};
use sha2::{ use sha2::{
digest::{Reset, Update},
Digest, Sha256 as sha_sha256, Digest, Sha256 as sha_sha256,
digest::{Reset, Update},
}; };
pub trait Hasher { pub trait Hasher {
fn write(&mut self, bytes: &[u8]); fn write(&mut self, bytes: &[u8]);
reader/src/reader.rs
+4 -16
View File
@@ -13,9 +13,9 @@ use crate::{
}; };
// use futures::stream::Stream; // use futures::stream::Stream;
use std::io::{Error, Result}; use super::hasher::{Hasher, MD5, Sha256};
use super::hasher::{Hasher, Sha256, MD5};
use futures::Stream; use futures::Stream;
use std::io::{Error, Result};
pin_project_lite::pin_project! { pin_project_lite::pin_project! {
#[derive(Default)] #[derive(Default)]
@@ -101,20 +101,8 @@ pin_project_lite::pin_project! {
impl<S> HashReader<S> { impl<S> HashReader<S> {
pub fn new(inner: S, size: usize, md5_hex: Option<String>, sha256_hex: Option<String>, actual_size: usize) -> Self { pub fn new(inner: S, size: usize, md5_hex: Option<String>, sha256_hex: Option<String>, actual_size: usize) -> Self {
let md5 = { let md5 = { if md5_hex.is_some() { Some(MD5::new()) } else { None } };
if md5_hex.is_some() { let sha256 = { if sha256_hex.is_some() { Some(Sha256::new()) } else { None } };
Some(MD5::new())
} else {
None
}
};
let sha256 = {
if sha256_hex.is_some() {
Some(Sha256::new())
} else {
None
}
};
Self { Self {
inner, inner,
size, size,
rustfs/src/admin/handlers.rs
+1 -1
View File
@@ -63,9 +63,9 @@ pub mod pools;
pub mod rebalance; pub mod rebalance;
pub mod service_account; pub mod service_account;
pub mod sts; pub mod sts;
pub mod tier;
pub mod trace; pub mod trace;
pub mod user; pub mod user;
pub mod tier;
use urlencoding::decode; use urlencoding::decode;
#[derive(Debug, Serialize, Default)] #[derive(Debug, Serialize, Default)]
rustfs/src/admin/handlers/tier.rs
+37 -16
View File
@@ -3,28 +3,28 @@ use std::str::from_utf8;
use http::{HeaderMap, StatusCode}; use http::{HeaderMap, StatusCode};
use iam::get_global_action_cred; use iam::get_global_action_cred;
use matchit::Params; use matchit::Params;
use s3s::{header::CONTENT_TYPE, s3_error, Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result}; use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
use serde::Deserialize; use serde::Deserialize;
use serde_urlencoded::from_bytes; use serde_urlencoded::from_bytes;
use tracing::{warn, debug, info}; use tracing::{debug, info, warn};
use crypto::{encrypt_data, decrypt_data};
use crate::{ use crate::{
admin::{router::Operation, utils::has_space_be}, admin::{router::Operation, utils::has_space_be},
auth::{check_key_valid, get_session_token}, auth::{check_key_valid, get_session_token},
}; };
use crypto::{decrypt_data, encrypt_data};
use ecstore::{ use ecstore::{
client::admin_handler_utils::AdminError, client::admin_handler_utils::AdminError,
config::storageclass, config::storageclass,
global::GLOBAL_TierConfigMgr, global::GLOBAL_TierConfigMgr,
tier::{ tier::{
tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_MISSING_CREDENTIALS},
tier_admin::TierCreds, tier_admin::TierCreds,
tier_config::{TierConfig, TierType}, tier_config::{TierConfig, TierType},
tier_handlers::{ tier_handlers::{
ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_ALREADY_EXISTS, ERR_TIER_NOT_FOUND, ERR_TIER_CONNECT_ERR, ERR_TIER_ALREADY_EXISTS, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_NAME_NOT_UPPERCASE,
ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_NOT_FOUND,
}, },
tier::{ERR_TIER_MISSING_CREDENTIALS, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_BACKEND_IN_USE},
}, },
}; };
@@ -92,29 +92,44 @@ impl Operation for AddTier {
} }
match args.name.as_str() { match args.name.as_str() {
storageclass::STANDARD | storageclass::RRS => { storageclass::STANDARD | storageclass::RRS => {
warn!("tier reserved name, args.name: {}", args.name); warn!("tier reserved name, args.name: {}", args.name);
return Err(s3_error!(InvalidRequest, "Cannot use reserved tier name")); return Err(s3_error!(InvalidRequest, "Cannot use reserved tier name"));
} }
&_ => () &_ => (),
} }
let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await; let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await;
//tier_config_mgr.reload(api); //tier_config_mgr.reload(api);
match tier_config_mgr.add(args, force).await { match tier_config_mgr.add(args, force).await {
Err(ERR_TIER_ALREADY_EXISTS) => { Err(ERR_TIER_ALREADY_EXISTS) => {
return Err(S3Error::with_message(S3ErrorCode::Custom("TierNameAlreadyExist".into()), "tier name already exists!")); return Err(S3Error::with_message(
S3ErrorCode::Custom("TierNameAlreadyExist".into()),
"tier name already exists!",
));
} }
Err(ERR_TIER_NAME_NOT_UPPERCASE) => { Err(ERR_TIER_NAME_NOT_UPPERCASE) => {
return Err(S3Error::with_message(S3ErrorCode::Custom("TierNameNotUppercase".into()), "tier name not uppercase!")); return Err(S3Error::with_message(
S3ErrorCode::Custom("TierNameNotUppercase".into()),
"tier name not uppercase!",
));
} }
Err(ERR_TIER_BACKEND_IN_USE) => { Err(ERR_TIER_BACKEND_IN_USE) => {
return Err(S3Error::with_message(S3ErrorCode::Custom("TierNameBackendInUse!".into()), "tier name backend in use!")); return Err(S3Error::with_message(
S3ErrorCode::Custom("TierNameBackendInUse!".into()),
"tier name backend in use!",
));
} }
Err(ERR_TIER_CONNECT_ERR) => { Err(ERR_TIER_CONNECT_ERR) => {
return Err(S3Error::with_message(S3ErrorCode::Custom("TierConnectError".into()), "tier connect error!")); return Err(S3Error::with_message(
S3ErrorCode::Custom("TierConnectError".into()),
"tier connect error!",
));
} }
Err(ERR_TIER_INVALID_CREDENTIALS) => { Err(ERR_TIER_INVALID_CREDENTIALS) => {
return Err(S3Error::with_message(S3ErrorCode::Custom(ERR_TIER_INVALID_CREDENTIALS.code.into()), ERR_TIER_INVALID_CREDENTIALS.message)); return Err(S3Error::with_message(
S3ErrorCode::Custom(ERR_TIER_INVALID_CREDENTIALS.code.into()),
ERR_TIER_INVALID_CREDENTIALS.message,
));
} }
Err(e) => { Err(e) => {
warn!("tier_config_mgr add failed, e: {:?}", e); warn!("tier_config_mgr add failed, e: {:?}", e);
@@ -182,7 +197,10 @@ impl Operation for EditTier {
return Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found!")); return Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found!"));
} }
Err(ERR_TIER_MISSING_CREDENTIALS) => { Err(ERR_TIER_MISSING_CREDENTIALS) => {
return Err(S3Error::with_message(S3ErrorCode::Custom("TierMissingCredentials".into()), "tier missing credentials!")); return Err(S3Error::with_message(
S3ErrorCode::Custom("TierMissingCredentials".into()),
"tier missing credentials!",
));
} }
Err(e) => { Err(e) => {
warn!("tier_config_mgr edit failed, e: {:?}", e); warn!("tier_config_mgr edit failed, e: {:?}", e);
@@ -281,7 +299,10 @@ impl Operation for RemoveTier {
} }
Err(e) => { Err(e) => {
warn!("tier_config_mgr remove failed, e: {:?}", e); warn!("tier_config_mgr remove failed, e: {:?}", e);
return Err(S3Error::with_message(S3ErrorCode::Custom("TierRemoveFailed".into()), "tier remove failed")); return Err(S3Error::with_message(
S3ErrorCode::Custom("TierRemoveFailed".into()),
"tier remove failed",
));
} }
Ok(_) => (), Ok(_) => (),
} }
rustfs/src/admin/mod.rs
+1 -1
View File
@@ -7,7 +7,7 @@ pub mod utils;
use handlers::{ use handlers::{
group, policys, pools, rebalance, group, policys, pools, rebalance,
service_account::{AddServiceAccount, DeleteServiceAccount, InfoServiceAccount, ListServiceAccount, UpdateServiceAccount}, service_account::{AddServiceAccount, DeleteServiceAccount, InfoServiceAccount, ListServiceAccount, UpdateServiceAccount},
sts, user, tier, sts, tier, user,
}; };
use handlers::{GetReplicationMetricsHandler, ListRemoteTargetHandler, RemoveRemoteTargetHandler, SetRemoteTargetHandler}; use handlers::{GetReplicationMetricsHandler, ListRemoteTargetHandler, RemoveRemoteTargetHandler, SetRemoteTargetHandler};
rustfs/src/storage/ecfs.rs
+18 -9
View File
@@ -49,11 +49,11 @@ use ecstore::store_api::ObjectToDelete;
use ecstore::store_api::PutObjReader; use ecstore::store_api::PutObjReader;
use ecstore::store_api::StorageAPI; use ecstore::store_api::StorageAPI;
// use ecstore::store_api::RESERVED_METADATA_PREFIX; // use ecstore::store_api::RESERVED_METADATA_PREFIX;
use ecstore::bucket::lifecycle::bucket_lifecycle_ops::validate_transition_tier;
use ecstore::bucket::utils::serialize; use ecstore::bucket::utils::serialize;
use ecstore::cmd::bucket_replication::ReplicationStatusType; use ecstore::cmd::bucket_replication::ReplicationStatusType;
use ecstore::cmd::bucket_replication::ReplicationType; use ecstore::cmd::bucket_replication::ReplicationType;
use ecstore::store_api::RESERVED_METADATA_PREFIX_LOWER; use ecstore::store_api::RESERVED_METADATA_PREFIX_LOWER;
use ecstore::bucket::lifecycle::bucket_lifecycle_ops::validate_transition_tier;
use futures::pin_mut; use futures::pin_mut;
use futures::{Stream, StreamExt}; use futures::{Stream, StreamExt};
use http::HeaderMap; use http::HeaderMap;
@@ -96,10 +96,7 @@ use transform_stream::AsyncTryStream;
use uuid::Uuid; use uuid::Uuid;
use ecstore::bucket::{ use ecstore::bucket::{
lifecycle::{ lifecycle::{bucket_lifecycle_ops::ERR_INVALID_STORAGECLASS, lifecycle::Lifecycle},
lifecycle::Lifecycle,
bucket_lifecycle_ops::ERR_INVALID_STORAGECLASS
},
object_lock::objectlock_sys::BucketObjectLockSys, object_lock::objectlock_sys::BucketObjectLockSys,
}; };
@@ -1602,7 +1599,10 @@ impl S3 for FS {
let rcfg = metadata_sys::get_object_lock_config(&bucket).await; let rcfg = metadata_sys::get_object_lock_config(&bucket).await;
if rcfg.is_err() { if rcfg.is_err() {
return Err(S3Error::with_message(S3ErrorCode::Custom("BucketLockIsNotExist".into()), "bucket lock is not exist.")); return Err(S3Error::with_message(
S3ErrorCode::Custom("BucketLockIsNotExist".into()),
"bucket lock is not exist.",
));
} }
let rcfg = rcfg.expect("get_lifecycle_config err!").0; let rcfg = rcfg.expect("get_lifecycle_config err!").0;
@@ -1612,12 +1612,18 @@ impl S3 for FS {
if let Err(err) = input_cfg.validate(&rcfg).await { if let Err(err) = input_cfg.validate(&rcfg).await {
//return Err(S3Error::with_message(S3ErrorCode::Custom("BucketLockValidateFailed".into()), "bucket lock validate failed.")); //return Err(S3Error::with_message(S3ErrorCode::Custom("BucketLockValidateFailed".into()), "bucket lock validate failed."));
return Err(S3Error::with_message(S3ErrorCode::Custom("ValidateFailed".into()), format!("{}", err.to_string()))); return Err(S3Error::with_message(
S3ErrorCode::Custom("ValidateFailed".into()),
format!("{}", err.to_string()),
));
} }
if let Err(err) = validate_transition_tier(&input_cfg).await { if let Err(err) = validate_transition_tier(&input_cfg).await {
//warn!("lifecycle_configuration add failed, err: {:?}", err); //warn!("lifecycle_configuration add failed, err: {:?}", err);
return Err(S3Error::with_message(S3ErrorCode::Custom("CustomError".into()), format!("{}", err.to_string()))); return Err(S3Error::with_message(
S3ErrorCode::Custom("CustomError".into()),
format!("{}", err.to_string()),
));
} }
let data = try_!(serialize(&input_cfg)); let data = try_!(serialize(&input_cfg));
@@ -2070,7 +2076,10 @@ impl S3 for FS {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
}; };
if let Err(e) = store.get_object_reader(&bucket, &key, None, HeaderMap::new(), &ObjectOptions::default()).await { if let Err(e) = store
.get_object_reader(&bucket, &key, None, HeaderMap::new(), &ObjectOptions::default())
.await
{
return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{}", e))); return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{}", e)));
} }