fix clippy

This commit is contained in:
likewu
2025-06-23 12:03:46 +08:00
parent cc71f40a6d
commit 2c176dd864
70 changed files with 2779 additions and 1969 deletions
+10 -5
View File
@@ -309,7 +309,8 @@ impl FileInfo {
} }
pub fn set_tier_free_version_id(&mut self, version_id: &str) { pub fn set_tier_free_version_id(&mut self, version_id: &str) {
self.metadata.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_ID), version_id.to_string()); self.metadata
.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_ID), version_id.to_string());
} }
pub fn tier_free_version_id(&self) -> String { pub fn tier_free_version_id(&self) -> String {
@@ -317,19 +318,23 @@ impl FileInfo {
} }
pub fn set_tier_free_version(&mut self) { pub fn set_tier_free_version(&mut self) {
self.metadata.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_MARKER), "".to_string()); self.metadata
.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_MARKER), "".to_string());
} }
pub fn set_skip_tier_free_version(&mut self) { pub fn set_skip_tier_free_version(&mut self) {
self.metadata.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_SKIP_FV_ID), "".to_string()); self.metadata
.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_SKIP_FV_ID), "".to_string());
} }
pub fn skip_tier_free_version(&self) -> bool { pub fn skip_tier_free_version(&self) -> bool {
self.metadata.contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_SKIP_FV_ID)) self.metadata
.contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_SKIP_FV_ID))
} }
pub fn tier_free_version(&self) -> bool { pub fn tier_free_version(&self) -> bool {
self.metadata.contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_MARKER)) self.metadata
.contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_MARKER))
} }
pub fn set_inline_data(&mut self) { pub fn set_inline_data(&mut self) {
+44 -9
View File
@@ -488,7 +488,9 @@ impl FileMeta {
let encoded = ver.marshal_msg()?; let encoded = ver.marshal_msg()?;
if self.versions.len() + 1 > 100 { if self.versions.len() + 1 > 100 {
return Err(Error::other("You've exceeded the limit on the number of versions you can create on this object")); return Err(Error::other(
"You've exceeded the limit on the number of versions you can create on this object",
));
} }
self.versions.push(FileMetaShallowVersion { self.versions.push(FileMetaShallowVersion {
@@ -1927,10 +1929,22 @@ impl MetaObject {
} }
pub fn set_transition(&mut self, fi: &FileInfo) { pub fn set_transition(&mut self, fi: &FileInfo) {
self.meta_sys.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_STATUS), fi.transition_status.as_bytes().to_vec()); self.meta_sys.insert(
self.meta_sys.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_OBJECTNAME), fi.transitioned_objname.as_bytes().to_vec()); format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_STATUS),
self.meta_sys.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_VERSION_ID), fi.transition_version_id.unwrap().as_bytes().to_vec()); fi.transition_status.as_bytes().to_vec(),
self.meta_sys.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_TIER), fi.transition_tier.as_bytes().to_vec()); );
self.meta_sys.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_OBJECTNAME),
fi.transitioned_objname.as_bytes().to_vec(),
);
self.meta_sys.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_VERSION_ID),
fi.transition_version_id.unwrap().as_bytes().to_vec(),
);
self.meta_sys.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_TIER),
fi.transition_tier.as_bytes().to_vec(),
);
} }
pub fn remove_restore_hdrs(&mut self) { pub fn remove_restore_hdrs(&mut self) {
@@ -1977,11 +1991,18 @@ impl MetaObject {
if fi.skip_tier_free_version() { if fi.skip_tier_free_version() {
return (FileMetaVersion::default(), false); return (FileMetaVersion::default(), false);
} }
if let Some(status) = self.meta_sys.get(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_STATUS)) { if let Some(status) = self
.meta_sys
.get(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_STATUS))
{
if *status == TRANSITION_COMPLETE.as_bytes().to_vec() { if *status == TRANSITION_COMPLETE.as_bytes().to_vec() {
let vid = Uuid::parse_str(&fi.tier_free_version_id()); let vid = Uuid::parse_str(&fi.tier_free_version_id());
if let Err(err) = vid { if let Err(err) = vid {
panic!("Invalid Tier Object delete marker versionId {} {}", fi.tier_free_version_id(), err.to_string()); panic!(
"Invalid Tier Object delete marker versionId {} {}",
fi.tier_free_version_id(),
err.to_string()
);
} }
let vid = vid.unwrap(); let vid = vid.unwrap();
let mut free_entry = FileMetaVersion { let mut free_entry = FileMetaVersion {
@@ -1995,7 +2016,14 @@ impl MetaObject {
meta_sys: Some(HashMap::<String, Vec<u8>>::new()), meta_sys: Some(HashMap::<String, Vec<u8>>::new()),
}); });
free_entry.delete_marker.as_mut().unwrap().meta_sys.as_mut().unwrap().insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, FREE_VERSION), vec![]); free_entry
.delete_marker
.as_mut()
.unwrap()
.meta_sys
.as_mut()
.unwrap()
.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, FREE_VERSION), vec![]);
let tier_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_TIER); let tier_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_TIER);
let tier_obj_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_OBJECTNAME); let tier_obj_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_OBJECTNAME);
let tier_obj_vid_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_VERSION_ID); let tier_obj_vid_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_VERSION_ID);
@@ -2003,7 +2031,14 @@ impl MetaObject {
let aa = [tier_key, tier_obj_key, tier_obj_vid_key]; let aa = [tier_key, tier_obj_key, tier_obj_vid_key];
for (k, v) in &self.meta_sys { for (k, v) in &self.meta_sys {
if aa.contains(&k) { if aa.contains(&k) {
free_entry.delete_marker.as_mut().unwrap().meta_sys.as_mut().unwrap().insert(k.clone(), v.clone()); free_entry
.delete_marker
.as_mut()
.unwrap()
.meta_sys
.as_mut()
.unwrap()
.insert(k.clone(), v.clone());
} }
} }
return (free_entry, true); return (free_entry, true);
+8 -4
View File
@@ -122,7 +122,9 @@ pub fn get_endpoint_url(endpoint: &str, secure: bool) -> Result<Url, std::io::Er
} }
let endpoint_url_str = format!("{scheme}://{endpoint}"); let endpoint_url_str = format!("{scheme}://{endpoint}");
let Ok(endpoint_url) = Url::parse(&endpoint_url_str) else { return Err(std::io::Error::other("url parse error.")); }; let Ok(endpoint_url) = Url::parse(&endpoint_url_str) else {
return Err(std::io::Error::other("url parse error."));
};
//is_valid_endpoint_url(endpoint_url)?; //is_valid_endpoint_url(endpoint_url)?;
Ok(endpoint_url) Ok(endpoint_url)
@@ -148,7 +150,6 @@ lazy_static! {
m.insert("response-expires".to_string(), true); m.insert("response-expires".to_string(), true);
m m
}; };
static ref SUPPORTED_HEADERS: HashMap<String, bool> = { static ref SUPPORTED_HEADERS: HashMap<String, bool> = {
let mut m = HashMap::new(); let mut m = HashMap::new();
m.insert("content-type".to_string(), true); m.insert("content-type".to_string(), true);
@@ -164,7 +165,6 @@ lazy_static! {
m.insert("x-amz-replication-status".to_string(), true); m.insert("x-amz-replication-status".to_string(), true);
m m
}; };
static ref SSE_HEADERS: HashMap<String, bool> = { static ref SSE_HEADERS: HashMap<String, bool> = {
let mut m = HashMap::new(); let mut m = HashMap::new();
m.insert("x-amz-server-side-encryption".to_string(), true); m.insert("x-amz-server-side-encryption".to_string(), true);
@@ -201,7 +201,11 @@ pub fn is_sse_header(header_key: &str) -> bool {
pub fn is_amz_header(header_key: &str) -> bool { pub fn is_amz_header(header_key: &str) -> bool {
let key = header_key.to_lowercase(); let key = header_key.to_lowercase();
key.starts_with("x-amz-meta-") || key.starts_with("x-amz-grant-") || key == "x-amz-acl" || is_sse_header(header_key) || key.starts_with("x-amz-checksum-") key.starts_with("x-amz-meta-")
|| key.starts_with("x-amz-grant-")
|| key == "x-amz-acl"
|| is_sse_header(header_key)
|| key.starts_with("x-amz-checksum-")
} }
pub fn is_rustfs_header(header_key: &str) -> bool { pub fn is_rustfs_header(header_key: &str) -> bool {
+1 -5
View File
@@ -1,8 +1,4 @@
//use tokio_stream::Stream; use std::time::Duration;
use std::future::Future;
use std::time::{Duration, Instant};
use std::pin::Pin;
use std::task::{Context, Poll};
// MaxRetry is the maximum number of retries before stopping. // MaxRetry is the maximum number of retries before stopping.
pub const MAX_RETRY: i64 = 10; pub const MAX_RETRY: i64 = 10;
@@ -24,9 +24,6 @@ pub struct LcAuditEvent {
impl LcAuditEvent { impl LcAuditEvent {
pub fn new(event: lifecycle::Event, source: LcEventSrc) -> Self { pub fn new(event: lifecycle::Event, source: LcEventSrc) -> Self {
Self { Self { event, source }
event,
source,
}
} }
} }
@@ -1,53 +1,49 @@
use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
use futures::Future;
use http::HeaderMap;
use lazy_static::lazy_static;
use s3s::Body;
use sha2::{Digest, Sha256};
use std::any::{Any, TypeId}; use std::any::{Any, TypeId};
use std::collections::HashMap;
use std::env; use std::env;
use std::io::{Cursor, Write}; use std::io::{Cursor, Write};
use std::pin::Pin; use std::pin::Pin;
use std::sync::atomic::{AtomicI32, AtomicI64, Ordering}; use std::sync::atomic::{AtomicI32, AtomicI64, Ordering};
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use futures::Future;
use lazy_static::lazy_static;
use s3s::Body;
use std::collections::HashMap;
use tracing::{error, info, warn};
use sha2::{Digest, Sha256};
use xxhash_rust::xxh64;
use uuid::Uuid;
use http::HeaderMap;
use tokio::select; use tokio::select;
use tokio::sync::mpsc::{Receiver, Sender}; use tokio::sync::mpsc::{Receiver, Sender};
use tokio::sync::{mpsc, RwLock}; use tokio::sync::{RwLock, mpsc};
use async_channel::{bounded, Receiver as A_Receiver, Sender as A_Sender}; use tracing::{error, info, warn};
use uuid::Uuid;
use xxhash_rust::xxh64;
use s3s::dto::BucketLifecycleConfiguration; use super::bucket_lifecycle_audit::{LcAuditEvent, LcEventSrc};
use super::lifecycle::{self, ExpirationOptions, IlmAction, Lifecycle, TransitionOptions};
use super::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats};
use super::tier_sweeper::{Jentry, delete_object_from_remote_tier};
use crate::bucket::{metadata_sys::get_lifecycle_config, versioning_sys::BucketVersioningSys};
use crate::client::object_api_utils::new_getobjectreader;
use crate::error::Error; use crate::error::Error;
use crate::event::name::EventName;
use crate::store::ECStore;
use crate::store_api::StorageAPI;
use crate::store_api::{ObjectInfo, ObjectOptions, ObjectToDelete, GetObjectReader, HTTPRangeSpec,};
use crate::error::{error_resp_to_object_err, is_err_object_not_found, is_err_version_not_found, is_network_or_host_down}; use crate::error::{error_resp_to_object_err, is_err_object_not_found, is_err_version_not_found, is_network_or_host_down};
use crate::event::name::EventName;
use crate::event_notification::{EventArgs, send_event};
use crate::global::GLOBAL_LocalNodeName;
use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id}; use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id};
use crate::client::object_api_utils::{new_getobjectreader,};
use crate::event_notification::{send_event, EventArgs};
use crate::heal::{ use crate::heal::{
data_scanner::{apply_expiry_on_non_transitioned_objects, apply_expiry_on_transitioned_object},
data_scanner_metric::ScannerMetrics, data_scanner_metric::ScannerMetrics,
data_scanner::{
apply_expiry_on_transitioned_object, apply_expiry_on_non_transitioned_objects,
},
data_usage_cache::TierStats, data_usage_cache::TierStats,
}; };
use crate::global::GLOBAL_LocalNodeName; use crate::store::ECStore;
use crate::bucket::{ use crate::store_api::StorageAPI;
metadata_sys::get_lifecycle_config, use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete};
versioning_sys::BucketVersioningSys,
};
use crate::tier::warm_backend::WarmBackendGetOpts; use crate::tier::warm_backend::WarmBackendGetOpts;
use super::lifecycle::{self, ExpirationOptions, IlmAction, Lifecycle, TransitionOptions}; use s3s::dto::BucketLifecycleConfiguration;
use super::tier_last_day_stats::{LastDayTierStats, DailyAllTierStats};
use super::tier_sweeper::{delete_object_from_remote_tier, Jentry};
use super::bucket_lifecycle_audit::{LcEventSrc, LcAuditEvent};
pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>; pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
pub type TraceFn = Arc<dyn Fn(String, HashMap<String, String>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>; pub type TraceFn =
Arc<dyn Fn(String, HashMap<String, String>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
pub type ExpiryOpType = Box<dyn ExpiryOp + Send + Sync + 'static>; pub type ExpiryOpType = Box<dyn ExpiryOp + Send + Sync + 'static>;
static XXHASH_SEED: u64 = 0; static XXHASH_SEED: u64 = 0;
@@ -74,8 +70,7 @@ impl LifecycleSys {
Some(lc) Some(lc)
} }
pub fn trace(oi: &ObjectInfo) -> TraceFn pub fn trace(oi: &ObjectInfo) -> TraceFn {
{
todo!(); todo!();
} }
} }
@@ -181,8 +176,6 @@ pub struct ExpiryState {
stats: Option<ExpiryStats>, stats: Option<ExpiryStats>,
} }
impl ExpiryState { impl ExpiryState {
#[allow(clippy::new_ret_no_self)] #[allow(clippy::new_ret_no_self)]
pub fn new() -> Arc<RwLock<Self>> { pub fn new() -> Arc<RwLock<Self>> {
@@ -244,7 +237,11 @@ impl ExpiryState {
} }
pub async fn enqueue_by_days(&mut self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) { pub async fn enqueue_by_days(&mut self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) {
let task = ExpiryTask {obj_info: oi.clone(), event: event.clone(), src: src.clone()}; let task = ExpiryTask {
obj_info: oi.clone(),
event: event.clone(),
src: src.clone(),
};
let wrkr = self.get_worker_ch(task.op_hash()); let wrkr = self.get_worker_ch(task.op_hash());
if wrkr.is_none() { if wrkr.is_none() {
*self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1; *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
@@ -265,7 +262,11 @@ impl ExpiryState {
return; return;
} }
let task = NewerNoncurrentTask {bucket: String::from(bucket), versions: versions, event: lc_event}; let task = NewerNoncurrentTask {
bucket: String::from(bucket),
versions: versions,
event: lc_event,
};
let wrkr = self.get_worker_ch(task.op_hash()); let wrkr = self.get_worker_ch(task.op_hash());
if wrkr.is_none() { if wrkr.is_none() {
*self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1; *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
@@ -422,7 +423,11 @@ impl TransitionState {
} }
pub async fn queue_transition_task(&self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) { pub async fn queue_transition_task(&self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) {
let task = TransitionTask {obj_info: oi.clone(), src: src.clone(), event: event.clone()}; let task = TransitionTask {
obj_info: oi.clone(),
src: src.clone(),
event: event.clone(),
};
select! { select! {
//_ -> t.ctx.Done() => (), //_ -> t.ctx.Done() => (),
_ = self.transition_tx.send(Some(task)) => (), _ = self.transition_tx.send(Some(task)) => (),
@@ -512,7 +517,9 @@ impl TransitionState {
pub fn add_lastday_stats(&self, tier: &str, ts: TierStats) { pub fn add_lastday_stats(&self, tier: &str, ts: TierStats) {
let mut tier_stats = self.last_day_stats.lock().unwrap(); let mut tier_stats = self.last_day_stats.lock().unwrap();
tier_stats.entry(tier.to_string()).and_modify(|e| e.add_stats(ts)) tier_stats
.entry(tier.to_string())
.and_modify(|e| e.add_stats(ts))
.or_insert(LastDayTierStats::default()); .or_insert(LastDayTierStats::default());
} }
@@ -574,7 +581,10 @@ impl AuditTierOp {
} }
pub fn string(&self) -> String { pub fn string(&self) -> String {
format!("tier:{},respNS:{},tx:{},err:{}", self.tier, self.time_to_responsens, self.output_bytes, self.error) format!(
"tier:{},respNS:{},tx:{},err:{}",
self.tier, self.time_to_responsens, self.output_bytes, self.error
)
} }
} }
@@ -636,12 +646,17 @@ pub async fn enqueue_transition_immediate(oi: &ObjectInfo, src: LcEventSrc) {
} }
GLOBAL_TransitionState.queue_transition_task(oi, &event, &src).await; GLOBAL_TransitionState.queue_transition_task(oi, &event, &src).await;
} }
_ => () _ => (),
} }
} }
} }
pub async fn expire_transitioned_object(api: Arc<ECStore>, oi: &ObjectInfo, lc_event: &lifecycle::Event, src: &LcEventSrc) -> Result<ObjectInfo, std::io::Error> { pub async fn expire_transitioned_object(
api: Arc<ECStore>,
oi: &ObjectInfo,
lc_event: &lifecycle::Event,
src: &LcEventSrc,
) -> Result<ObjectInfo, std::io::Error> {
//let traceFn = GLOBAL_LifecycleSys.trace(oi); //let traceFn = GLOBAL_LifecycleSys.trace(oi);
let mut opts = ObjectOptions { let mut opts = ObjectOptions {
versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await, versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await,
@@ -663,7 +678,12 @@ pub async fn expire_transitioned_object(api: Arc<ECStore>, oi: &ObjectInfo, lc_e
} }
} }
let ret = delete_object_from_remote_tier(&oi.transitioned_object.name, &oi.transitioned_object.version_id, &oi.transitioned_object.tier).await; let ret = delete_object_from_remote_tier(
&oi.transitioned_object.name,
&oi.transitioned_object.version_id,
&oi.transitioned_object.tier,
)
.await;
if ret.is_ok() { if ret.is_ok() {
opts.skip_decommissioned = true; opts.skip_decommissioned = true;
} else { } else {
@@ -731,7 +751,14 @@ pub fn audit_tier_actions(api: ECStore, tier: &str, bytes: i64) -> TimeFn {
todo!(); todo!();
} }
pub async fn get_transitioned_object_reader(bucket: &str, object: &str, rs: HTTPRangeSpec, h: HeaderMap, oi: ObjectInfo, opts: &ObjectOptions) -> Result<GetObjectReader, std::io::Error> { pub async fn get_transitioned_object_reader(
bucket: &str,
object: &str,
rs: HTTPRangeSpec,
h: HeaderMap,
oi: ObjectInfo,
opts: &ObjectOptions,
) -> Result<GetObjectReader, std::io::Error> {
let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await; let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await;
let tgt_client = match tier_config_mgr.get_driver(&oi.transitioned_object.tier).await { let tgt_client = match tier_config_mgr.get_driver(&oi.transitioned_object.tier).await {
Ok(d) => d, Ok(d) => d,
@@ -752,7 +779,9 @@ pub async fn get_transitioned_object_reader(bucket: &str, object: &str, rs: HTTP
//return Ok(HttpFileReader::new(rs, &oi, opts, &h)); //return Ok(HttpFileReader::new(rs, &oi, opts, &h));
//timeTierAction := auditTierActions(oi.transitioned_object.Tier, length) //timeTierAction := auditTierActions(oi.transitioned_object.Tier, length)
let reader = tgt_client.get(&oi.transitioned_object.name, &oi.transitioned_object.version_id, gopts).await?; let reader = tgt_client
.get(&oi.transitioned_object.name, &oi.transitioned_object.version_id, gopts)
.await?;
Ok(get_fn(reader, h)) Ok(get_fn(reader, h))
} }
@@ -812,4 +841,3 @@ pub struct RestoreObjectRequest {
} }
const MAX_RESTORE_OBJECT_REQUEST_SIZE: i64 = 2 << 20; const MAX_RESTORE_OBJECT_REQUEST_SIZE: i64 = 2 << 20;
+49 -21
View File
@@ -1,12 +1,12 @@
use s3s::dto::{
BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, NoncurrentVersionTransition,
ObjectLockConfiguration, ObjectLockEnabled, Transition,
};
use std::cmp::Ordering; use std::cmp::Ordering;
use std::env; use std::env;
use std::fmt::Display; use std::fmt::Display;
use s3s::dto::{
BucketLifecycleConfiguration, ExpirationStatus, LifecycleRule, ObjectLockConfiguration,
ObjectLockEnabled, LifecycleExpiration, Transition, NoncurrentVersionTransition,
};
use time::macros::{datetime, offset}; use time::macros::{datetime, offset};
use time::{self, OffsetDateTime, Duration}; use time::{self, Duration, OffsetDateTime};
use crate::bucket::lifecycle::rule::TransitionOps; use crate::bucket::lifecycle::rule::TransitionOps;
@@ -19,7 +19,8 @@ const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration allows a max
const ERR_LIFECYCLE_NO_RULE: &str = "Lifecycle configuration should have at least one rule"; const ERR_LIFECYCLE_NO_RULE: &str = "Lifecycle configuration should have at least one rule";
const ERR_LIFECYCLE_DUPLICATE_ID: &str = "Rule ID must be unique. Found same ID for more than one rule"; const ERR_LIFECYCLE_DUPLICATE_ID: &str = "Rule ID must be unique. Found same ID for more than one rule";
const ERR_XML_NOT_WELL_FORMED: &str = "The XML you provided was not well-formed or did not validate against our published schema"; const ERR_XML_NOT_WELL_FORMED: &str = "The XML you provided was not well-formed or did not validate against our published schema";
const ERR_LIFECYCLE_BUCKET_LOCKED: &str = "ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket"; const ERR_LIFECYCLE_BUCKET_LOCKED: &str =
"ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket";
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
pub enum IlmAction { pub enum IlmAction {
@@ -52,7 +53,10 @@ impl IlmAction {
if self.delete_restored() { if self.delete_restored() {
return true; return true;
} }
*self == Self::DeleteVersionAction || *self == Self::DeleteAction || *self == Self::DeleteAllVersionsAction || *self == Self::DelMarkerDeleteAllVersionsAction *self == Self::DeleteVersionAction
|| *self == Self::DeleteAction
|| *self == Self::DeleteAllVersionsAction
|| *self == Self::DelMarkerDeleteAllVersionsAction
} }
} }
@@ -204,7 +208,10 @@ impl Lifecycle for BucketLifecycleConfiguration {
return true; return true;
} }
let rule_expiration = rule.expiration.as_ref().expect("err!"); let rule_expiration = rule.expiration.as_ref().expect("err!");
if !rule_expiration.date.is_none() && OffsetDateTime::from(rule_expiration.date.clone().expect("err!")).unix_timestamp() < OffsetDateTime::now_utc().unix_timestamp() { if !rule_expiration.date.is_none()
&& OffsetDateTime::from(rule_expiration.date.clone().expect("err!")).unix_timestamp()
< OffsetDateTime::now_utc().unix_timestamp()
{
return true; return true;
} }
if !rule_expiration.date.is_none() { if !rule_expiration.date.is_none() {
@@ -215,7 +222,10 @@ impl Lifecycle for BucketLifecycleConfiguration {
} }
let rule_transitions: &[Transition] = &rule.transitions.as_ref().expect("err!"); let rule_transitions: &[Transition] = &rule.transitions.as_ref().expect("err!");
let rule_transitions_0 = rule_transitions[0].clone(); let rule_transitions_0 = rule_transitions[0].clone();
if !rule_transitions_0.date.is_none() && OffsetDateTime::from(rule_transitions_0.date.expect("err!")).unix_timestamp() < OffsetDateTime::now_utc().unix_timestamp() { if !rule_transitions_0.date.is_none()
&& OffsetDateTime::from(rule_transitions_0.date.expect("err!")).unix_timestamp()
< OffsetDateTime::now_utc().unix_timestamp()
{
return true; return true;
} }
if !rule.transitions.is_none() { if !rule.transitions.is_none() {
@@ -413,12 +423,19 @@ impl Lifecycle for BucketLifecycleConfiguration {
if storage_class.as_str() != "" { if storage_class.as_str() != "" {
if !obj.delete_marker && obj.transition_status != TRANSITION_COMPLETE { if !obj.delete_marker && obj.transition_status != TRANSITION_COMPLETE {
let due = rule.noncurrent_version_transitions.as_ref().unwrap()[0].next_due(obj); let due = rule.noncurrent_version_transitions.as_ref().unwrap()[0].next_due(obj);
if due.is_some() && (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unwrap().unix_timestamp()) { if due.is_some()
&& (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unwrap().unix_timestamp())
{
events.push(Event { events.push(Event {
action: IlmAction::TransitionVersionAction, action: IlmAction::TransitionVersionAction,
rule_id: rule.id.clone().expect("err!"), rule_id: rule.id.clone().expect("err!"),
due, due,
storage_class: rule.noncurrent_version_transitions.as_ref().unwrap()[0].storage_class.clone().unwrap().as_str().to_string(), storage_class: rule.noncurrent_version_transitions.as_ref().unwrap()[0]
.storage_class
.clone()
.unwrap()
.as_str()
.to_string(),
..Default::default() ..Default::default()
}); });
} }
@@ -469,7 +486,9 @@ impl Lifecycle for BucketLifecycleConfiguration {
if let Some(ref transitions) = rule.transitions { if let Some(ref transitions) = rule.transitions {
let due = transitions[0].next_due(obj); let due = transitions[0].next_due(obj);
if let Some(due) = due { if let Some(due) = due {
if due.unix_timestamp() > 0 && (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unix_timestamp()) { if due.unix_timestamp() > 0
&& (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unix_timestamp())
{
events.push(Event { events.push(Event {
action: IlmAction::TransitionAction, action: IlmAction::TransitionAction,
rule_id: rule.id.clone().expect("err!"), rule_id: rule.id.clone().expect("err!"),
@@ -488,20 +507,27 @@ impl Lifecycle for BucketLifecycleConfiguration {
if events.len() > 0 { if events.len() > 0 {
events.sort_by(|a, b| { events.sort_by(|a, b| {
if now.unix_timestamp() > a.due.expect("err!").unix_timestamp() && now.unix_timestamp() > b.due.expect("err").unix_timestamp() || a.due.expect("err").unix_timestamp() == b.due.expect("err").unix_timestamp() { if now.unix_timestamp() > a.due.expect("err!").unix_timestamp()
&& now.unix_timestamp() > b.due.expect("err").unix_timestamp()
|| a.due.expect("err").unix_timestamp() == b.due.expect("err").unix_timestamp()
{
match a.action { match a.action {
IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction IlmAction::DeleteAllVersionsAction
| IlmAction::DeleteAction | IlmAction::DeleteVersionAction => { | IlmAction::DelMarkerDeleteAllVersionsAction
| IlmAction::DeleteAction
| IlmAction::DeleteVersionAction => {
return Ordering::Less; return Ordering::Less;
} }
_ => () _ => (),
} }
match b.action { match b.action {
IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction IlmAction::DeleteAllVersionsAction
| IlmAction::DeleteAction | IlmAction::DeleteVersionAction => { | IlmAction::DelMarkerDeleteAllVersionsAction
| IlmAction::DeleteAction
| IlmAction::DeleteVersionAction => {
return Ordering::Greater; return Ordering::Greater;
} }
_ => () _ => (),
} }
return Ordering::Less; return Ordering::Less;
} }
@@ -601,7 +627,9 @@ pub fn expected_expiry_time(mod_time: OffsetDateTime, days: i32) -> OffsetDateTi
if days == 0 { if days == 0 {
return mod_time; return mod_time;
} }
let t = mod_time.to_offset(offset!(-0:00:00)).saturating_add(Duration::days(0/*days as i64*/)); //debug let t = mod_time
.to_offset(offset!(-0:00:00))
.saturating_add(Duration::days(0 /*days as i64*/)); //debug
let mut hour = 3600; let mut hour = 3600;
if let Ok(env_ilm_hour) = env::var("_RUSTFS_ILM_HOUR") { if let Ok(env_ilm_hour) = env::var("_RUSTFS_ILM_HOUR") {
if let Ok(num_hour) = env_ilm_hour.parse::<usize>() { if let Ok(num_hour) = env_ilm_hour.parse::<usize>() {
@@ -661,7 +689,7 @@ impl Default for Event {
#[derive(Debug, Clone, Default)] #[derive(Debug, Clone, Default)]
pub struct ExpirationOptions { pub struct ExpirationOptions {
pub expire: bool pub expire: bool,
} }
impl ExpirationOptions { impl ExpirationOptions {
+5 -5
View File
@@ -1,6 +1,6 @@
pub mod rule;
pub mod lifecycle;
pub mod tier_sweeper;
pub mod tier_last_day_stats;
pub mod bucket_lifecycle_ops;
pub mod bucket_lifecycle_audit; pub mod bucket_lifecycle_audit;
pub mod bucket_lifecycle_ops;
pub mod lifecycle;
pub mod rule;
pub mod tier_last_day_stats;
pub mod tier_sweeper;
+3 -5
View File
@@ -1,10 +1,9 @@
use s3s::dto::{ use s3s::dto::{LifecycleRuleFilter, Transition};
LifecycleRuleFilter, Transition,
};
const ERR_TRANSITION_INVALID_DAYS: &str = "Days must be 0 or greater when used with Transition"; const ERR_TRANSITION_INVALID_DAYS: &str = "Days must be 0 or greater when used with Transition";
const ERR_TRANSITION_INVALID_DATE: &str = "Date must be provided in ISO 8601 format"; const ERR_TRANSITION_INVALID_DATE: &str = "Date must be provided in ISO 8601 format";
const ERR_TRANSITION_INVALID: &str = "Exactly one of Days (0 or greater) or Date (positive ISO 8601 format) should be present in Transition."; const ERR_TRANSITION_INVALID: &str =
"Exactly one of Days (0 or greater) or Date (positive ISO 8601 format) should be present in Transition.";
const ERR_TRANSITION_DATE_NOT_MIDNIGHT: &str = "'Date' must be at midnight GMT"; const ERR_TRANSITION_DATE_NOT_MIDNIGHT: &str = "'Date' must be at midnight GMT";
pub trait Filter { pub trait Filter {
@@ -39,7 +38,6 @@ impl TransitionOps for Transition {
} }
} }
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use super::*; use super::*;
@@ -1,9 +1,9 @@
use sha2::Sha256; use sha2::Sha256;
use std::collections::HashMap; use std::collections::HashMap;
use std::ops::Sub;
use time::OffsetDateTime; use time::OffsetDateTime;
use tracing::{error, warn}; use tracing::{error, warn};
use std::ops::Sub;
use crate::heal::data_usage_cache::TierStats; use crate::heal::data_usage_cache::TierStats;
@@ -79,8 +79,5 @@ impl LastDayTierStats {
} }
} }
#[cfg(test)] #[cfg(test)]
mod test { mod test {}
}
+7 -8
View File
@@ -1,11 +1,11 @@
use sha2::{Digest, Sha256}; use sha2::{Digest, Sha256};
use xxhash_rust::xxh64;
use std::any::Any; use std::any::Any;
use std::io::{Cursor, Write}; use std::io::{Cursor, Write};
use xxhash_rust::xxh64;
use crate::global::GLOBAL_TierConfigMgr;
use super::bucket_lifecycle_ops::{ExpiryOp, GLOBAL_ExpiryState, TransitionedObject}; use super::bucket_lifecycle_ops::{ExpiryOp, GLOBAL_ExpiryState, TransitionedObject};
use super::lifecycle::{self, ObjectOpts}; use super::lifecycle::{self, ObjectOpts};
use crate::global::GLOBAL_TierConfigMgr;
static XXHASH_SEED: u64 = 0; static XXHASH_SEED: u64 = 0;
@@ -69,9 +69,11 @@ impl ObjSweeper {
} }
let mut del_tier = false; let mut del_tier = false;
if !self.versioned || self.suspended { // 1, 2.a, 2.b if !self.versioned || self.suspended {
// 1, 2.a, 2.b
del_tier = true; del_tier = true;
} else if self.versioned && self.version_id != "" { // 3.a } else if self.versioned && self.version_id != "" {
// 3.a
del_tier = true; del_tier = true;
} }
if del_tier { if del_tier {
@@ -123,8 +125,5 @@ pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_na
w.remove(obj_name, rv_id).await w.remove(obj_name, rv_id).await
} }
#[cfg(test)] #[cfg(test)]
mod test { mod test {}
}
+1 -1
View File
@@ -1,4 +1,5 @@
pub mod error; pub mod error;
pub mod lifecycle;
pub mod metadata; pub mod metadata;
pub mod metadata_sys; pub mod metadata_sys;
pub mod object_lock; pub mod object_lock;
@@ -10,4 +11,3 @@ pub mod target;
pub mod utils; pub mod utils;
pub mod versioning; pub mod versioning;
pub mod versioning_sys; pub mod versioning_sys;
pub mod lifecycle;
+19 -14
View File
@@ -3,13 +3,8 @@ use std::collections::HashMap;
use time::{OffsetDateTime, format_description}; use time::{OffsetDateTime, format_description};
use tracing::{error, warn}; use tracing::{error, warn};
use s3s::dto::{ use s3s::dto::{Date, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockRetention, ObjectLockRetentionMode};
ObjectLockRetentionMode, ObjectLockRetention, ObjectLockLegalHoldStatus, ObjectLockLegalHold, use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
Date,
};
use s3s::header::{
X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_OBJECT_LOCK_LEGAL_HOLD,
};
//const AMZ_OBJECTLOCK_BYPASS_RET_GOVERNANCE: &str = "X-Amz-Bypass-Governance-Retention"; //const AMZ_OBJECTLOCK_BYPASS_RET_GOVERNANCE: &str = "X-Amz-Bypass-Governance-Retention";
//const AMZ_OBJECTLOCK_RETAIN_UNTIL_DATE: &str = "X-Amz-Object-Lock-Retain-Until-Date"; //const AMZ_OBJECTLOCK_RETAIN_UNTIL_DATE: &str = "X-Amz-Object-Lock-Retain-Until-Date";
@@ -20,8 +15,10 @@ const ERR_MALFORMED_BUCKET_OBJECT_CONFIG: &str = "invalid bucket object lock con
const ERR_INVALID_RETENTION_DATE: &str = "date must be provided in ISO 8601 format"; const ERR_INVALID_RETENTION_DATE: &str = "date must be provided in ISO 8601 format";
const ERR_PAST_OBJECTLOCK_RETAIN_DATE: &str = "the retain until date must be in the future"; const ERR_PAST_OBJECTLOCK_RETAIN_DATE: &str = "the retain until date must be in the future";
const ERR_UNKNOWN_WORMMODE_DIRECTIVE: &str = "unknown WORM mode directive"; const ERR_UNKNOWN_WORMMODE_DIRECTIVE: &str = "unknown WORM mode directive";
const ERR_OBJECTLOCK_MISSING_CONTENT_MD5: &str = "content-MD5 HTTP header is required for Put Object requests with Object Lock parameters"; const ERR_OBJECTLOCK_MISSING_CONTENT_MD5: &str =
const ERR_OBJECTLOCK_INVALID_HEADERS: &str = "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied"; "content-MD5 HTTP header is required for Put Object requests with Object Lock parameters";
const ERR_OBJECTLOCK_INVALID_HEADERS: &str =
"x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied";
const ERR_MALFORMED_XML: &str = "the XML you provided was not well-formed or did not validate against our published schema"; const ERR_MALFORMED_XML: &str = "the XML you provided was not well-formed or did not validate against our published schema";
pub fn utc_now_ntp() -> OffsetDateTime { pub fn utc_now_ntp() -> OffsetDateTime {
@@ -39,7 +36,10 @@ pub fn get_object_retention_meta(meta: HashMap<String, String>) -> ObjectLockRet
if let Some(mode_str) = mode_str { if let Some(mode_str) = mode_str {
mode = parse_ret_mode(mode_str.as_str()); mode = parse_ret_mode(mode_str.as_str());
} else { } else {
return ObjectLockRetention {mode: None, retain_until_date: None}; return ObjectLockRetention {
mode: None,
retain_until_date: None,
};
} }
let mut till_str = meta.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_lowercase().as_str()); let mut till_str = meta.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_lowercase().as_str());
@@ -52,7 +52,10 @@ pub fn get_object_retention_meta(meta: HashMap<String, String>) -> ObjectLockRet
retain_until_date = Date::from(t.expect("err")); //TODO: utc retain_until_date = Date::from(t.expect("err")); //TODO: utc
} }
} }
ObjectLockRetention {mode: Some(mode), retain_until_date: Some(retain_until_date)} ObjectLockRetention {
mode: Some(mode),
retain_until_date: Some(retain_until_date),
}
} }
pub fn get_object_legalhold_meta(meta: HashMap<String, String>) -> ObjectLockLegalHold { pub fn get_object_legalhold_meta(meta: HashMap<String, String>) -> ObjectLockLegalHold {
@@ -61,7 +64,9 @@ pub fn get_object_legalhold_meta(meta: HashMap<String, String>) -> ObjectLockLeg
hold_str = Some(&meta[X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()]); hold_str = Some(&meta[X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()]);
} }
if let Some(hold_str) = hold_str { if let Some(hold_str) = hold_str {
return ObjectLockLegalHold {status: Some(parse_legalhold_status(hold_str))}; return ObjectLockLegalHold {
status: Some(parse_legalhold_status(hold_str)),
};
} }
ObjectLockLegalHold { status: None } ObjectLockLegalHold { status: None }
} }
@@ -75,7 +80,7 @@ pub fn parse_ret_mode(mode_str: &str) -> ObjectLockRetentionMode {
"COMPLIANCE" => { "COMPLIANCE" => {
mode = ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE); mode = ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE);
} }
_ => unreachable!() _ => unreachable!(),
} }
mode mode
} }
@@ -89,7 +94,7 @@ pub fn parse_legalhold_status(hold_str: &str) -> ObjectLockLegalHoldStatus {
"OFF" => { "OFF" => {
st = ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF); st = ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF);
} }
_ => unreachable!() _ => unreachable!(),
} }
st st
} }
@@ -3,12 +3,10 @@ use std::sync::Arc;
use time::OffsetDateTime; use time::OffsetDateTime;
use tracing::{error, warn}; use tracing::{error, warn};
use s3s::dto::{ use s3s::dto::{DefaultRetention, ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
DefaultRetention, ObjectLockRetentionMode, ObjectLockLegalHoldStatus,
};
use crate::store_api::ObjectInfo;
use crate::bucket::metadata_sys::get_object_lock_config; use crate::bucket::metadata_sys::get_object_lock_config;
use crate::store_api::ObjectInfo;
use super::objectlock; use super::objectlock;
@@ -21,7 +19,12 @@ impl BucketObjectLockSys {
} }
pub async fn get(bucket: &str) -> Option<DefaultRetention> { pub async fn get(bucket: &str) -> Option<DefaultRetention> {
if let Some(object_lock_rule) = get_object_lock_config(bucket).await.expect("get_object_lock_config err!").0.rule { if let Some(object_lock_rule) = get_object_lock_config(bucket)
.await
.expect("get_object_lock_config err!")
.0
.rule
{
return object_lock_rule.default_retention; return object_lock_rule.default_retention;
} }
None None
@@ -38,7 +41,7 @@ pub fn enforce_retention_for_deletion(obj_info: &ObjectInfo) -> bool {
Some(st) if st.as_str() == ObjectLockLegalHoldStatus::ON => { Some(st) if st.as_str() == ObjectLockLegalHoldStatus::ON => {
return true; return true;
} }
_ => () _ => (),
} }
let ret = objectlock::get_object_retention_meta(obj_info.user_defined.clone().expect("err")); let ret = objectlock::get_object_retention_meta(obj_info.user_defined.clone().expect("err"));
@@ -49,7 +52,7 @@ pub fn enforce_retention_for_deletion(obj_info: &ObjectInfo) -> bool {
return true; return true;
} }
} }
_ => () _ => (),
} }
false false
} }
+32 -44
View File
@@ -1,23 +1,17 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use std::{collections::HashMap, sync::Arc};
use std::ops::{BitAnd, BitOr};
use lazy_static::lazy_static; use lazy_static::lazy_static;
use std::ops::{BitAnd, BitOr};
use std::{collections::HashMap, sync::Arc};
use crate::client::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart};
use crate::{disk::DiskAPI, store_api::GetObjectReader};
use reader::hasher::{Hasher, Sha256}; use reader::hasher::{Hasher, Sha256};
use rustfs_utils::crypto::{base64_decode, base64_encode};
use s3s::header::{ use s3s::header::{
X_AMZ_CHECKSUM_ALGORITHM, X_AMZ_CHECKSUM_CRC32, X_AMZ_CHECKSUM_CRC32C, X_AMZ_CHECKSUM_SHA1, X_AMZ_CHECKSUM_SHA256, X_AMZ_CHECKSUM_ALGORITHM, X_AMZ_CHECKSUM_CRC32, X_AMZ_CHECKSUM_CRC32C, X_AMZ_CHECKSUM_SHA1, X_AMZ_CHECKSUM_SHA256,
}; };
use crate::client::{
api_put_object::PutObjectOptions,
api_s3_datatypes::ObjectPart,
};
use rustfs_utils::crypto::{base64_decode, base64_encode};
use crate::{
disk::DiskAPI,
store_api::GetObjectReader,
};
use enumset::{enum_set, EnumSet, EnumSetType}; use enumset::{EnumSet, EnumSetType, enum_set};
#[derive(Debug, EnumSetType, Default)] #[derive(Debug, EnumSetType, Default)]
#[enumset(repr = "u8")] #[enumset(repr = "u8")]
@@ -38,8 +32,10 @@ lazy_static! {
s.remove(ChecksumMode::ChecksumFullObject); s.remove(ChecksumMode::ChecksumFullObject);
s s
}; };
static ref C_ChecksumFullObjectCRC32: EnumSet<ChecksumMode> = enum_set!(ChecksumMode::ChecksumCRC32 | ChecksumMode::ChecksumFullObject); static ref C_ChecksumFullObjectCRC32: EnumSet<ChecksumMode> =
static ref C_ChecksumFullObjectCRC32C: EnumSet<ChecksumMode> = enum_set!(ChecksumMode::ChecksumCRC32C | ChecksumMode::ChecksumFullObject); enum_set!(ChecksumMode::ChecksumCRC32 | ChecksumMode::ChecksumFullObject);
static ref C_ChecksumFullObjectCRC32C: EnumSet<ChecksumMode> =
enum_set!(ChecksumMode::ChecksumCRC32C | ChecksumMode::ChecksumFullObject);
} }
const AMZ_CHECKSUM_CRC64NVME: &str = "x-amz-checksum-crc64nvme"; const AMZ_CHECKSUM_CRC64NVME: &str = "x-amz-checksum-crc64nvme";
@@ -49,24 +45,12 @@ impl ChecksumMode {
pub fn base(&self) -> ChecksumMode { pub fn base(&self) -> ChecksumMode {
let s = EnumSet::from(*self).intersection(*C_ChecksumMask); let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
match s.as_u8() { match s.as_u8() {
1_u8 => { 1_u8 => ChecksumMode::ChecksumNone,
ChecksumMode::ChecksumNone 2_u8 => ChecksumMode::ChecksumSHA256,
} 4_u8 => ChecksumMode::ChecksumSHA1,
2_u8 => { 8_u8 => ChecksumMode::ChecksumCRC32,
ChecksumMode::ChecksumSHA256 16_u8 => ChecksumMode::ChecksumCRC32C,
} 32_u8 => ChecksumMode::ChecksumCRC64NVME,
4_u8 => {
ChecksumMode::ChecksumSHA1
}
8_u8 => {
ChecksumMode::ChecksumCRC32
}
16_u8 => {
ChecksumMode::ChecksumCRC32C
}
32_u8 => {
ChecksumMode::ChecksumCRC64NVME
}
_ => panic!("enum err."), _ => panic!("enum err."),
} }
} }
@@ -119,17 +103,13 @@ impl ChecksumMode {
let u = EnumSet::from(*self).intersection(*C_ChecksumMask).as_u8(); let u = EnumSet::from(*self).intersection(*C_ChecksumMask).as_u8();
if u == ChecksumMode::ChecksumCRC32 as u8 || u == ChecksumMode::ChecksumCRC32C as u8 { if u == ChecksumMode::ChecksumCRC32 as u8 || u == ChecksumMode::ChecksumCRC32C as u8 {
4 4
} } else if u == ChecksumMode::ChecksumSHA1 as u8 {
else if u == ChecksumMode::ChecksumSHA1 as u8 {
4 //sha1.size 4 //sha1.size
} } else if u == ChecksumMode::ChecksumSHA256 as u8 {
else if u == ChecksumMode::ChecksumSHA256 as u8 {
4 //sha256.size 4 //sha256.size
} } else if u == ChecksumMode::ChecksumCRC64NVME as u8 {
else if u == ChecksumMode::ChecksumCRC64NVME as u8 {
4 //crc64.size 4 //crc64.size
} } else {
else {
0 0
} }
} }
@@ -229,7 +209,10 @@ impl ChecksumMode {
let mut crc_bytes = Vec::<u8>::with_capacity(p.len() * self.raw_byte_len() as usize); let mut crc_bytes = Vec::<u8>::with_capacity(p.len() * self.raw_byte_len() as usize);
let mut h = self.hasher()?; let mut h = self.hasher()?;
h.write(&crc_bytes); h.write(&crc_bytes);
Ok(Checksum {checksum_type: self.clone(), r: h.sum().as_bytes().to_vec()}) Ok(Checksum {
checksum_type: self.clone(),
r: h.sum().as_bytes().to_vec(),
})
} }
pub fn full_object_checksum(&self, p: &mut [ObjectPart]) -> Result<Checksum, std::io::Error> { pub fn full_object_checksum(&self, p: &mut [ObjectPart]) -> Result<Checksum, std::io::Error> {
@@ -246,7 +229,10 @@ struct Checksum {
impl Checksum { impl Checksum {
fn new(t: ChecksumMode, b: &[u8]) -> Checksum { fn new(t: ChecksumMode, b: &[u8]) -> Checksum {
if t.is_set() && b.len() == t.raw_byte_len() { if t.is_set() && b.len() == t.raw_byte_len() {
return Checksum {checksum_type: t, r: b.to_vec()}; return Checksum {
checksum_type: t,
r: b.to_vec(),
};
} }
Checksum::default() Checksum::default()
} }
@@ -282,9 +268,11 @@ impl Checksum {
} }
pub fn add_auto_checksum_headers(opts: &mut PutObjectOptions) { pub fn add_auto_checksum_headers(opts: &mut PutObjectOptions) {
opts.user_metadata.insert("X-Amz-Checksum-Algorithm".to_string(), opts.auto_checksum.to_string()); opts.user_metadata
.insert("X-Amz-Checksum-Algorithm".to_string(), opts.auto_checksum.to_string());
if opts.auto_checksum.full_object_requested() { if opts.auto_checksum.full_object_requested() {
opts.user_metadata.insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string()); opts.user_metadata
.insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string());
} }
} }
+17 -7
View File
@@ -1,11 +1,11 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use std::collections::HashMap;
use http::{HeaderMap, StatusCode};
use bytes::Bytes; use bytes::Bytes;
use http::{HeaderMap, StatusCode};
use std::collections::HashMap;
use crate::client::{ use crate::client::{
api_error_response::http_resp_to_error_response, api_error_response::http_resp_to_error_response,
transition_api::{RequestMetadata, TransitionClient, ReaderImpl} transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
}; };
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
@@ -54,7 +54,10 @@ impl TransitionClient {
let mut url_values = HashMap::new(); let mut url_values = HashMap::new();
url_values.insert("policy".to_string(), "".to_string()); url_values.insert("policy".to_string(), "".to_string());
let resp = self.execute_method(http::Method::DELETE, &mut RequestMetadata { let resp = self
.execute_method(
http::Method::DELETE,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
query_values: url_values, query_values: url_values,
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
@@ -70,7 +73,9 @@ impl TransitionClient {
extra_pre_sign_header: Default::default(), extra_pre_sign_header: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
expires: Default::default(), expires: Default::default(),
}).await?; },
)
.await?;
//defer closeResponse(resp) //defer closeResponse(resp)
if resp.status() != StatusCode::NO_CONTENT { if resp.status() != StatusCode::NO_CONTENT {
@@ -89,7 +94,10 @@ impl TransitionClient {
let mut url_values = HashMap::new(); let mut url_values = HashMap::new();
url_values.insert("policy".to_string(), "".to_string()); url_values.insert("policy".to_string(), "".to_string());
let resp = self.execute_method(http::Method::GET, &mut RequestMetadata { let resp = self
.execute_method(
http::Method::GET,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
query_values: url_values, query_values: url_values,
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
@@ -105,7 +113,9 @@ impl TransitionClient {
extra_pre_sign_header: Default::default(), extra_pre_sign_header: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
expires: Default::default(), expires: Default::default(),
}).await?; },
)
.await?;
let policy = String::from_utf8_lossy(&resp.body().bytes().expect("err").to_vec()).to_string(); let policy = String::from_utf8_lossy(&resp.body().bytes().expect("err").to_vec()).to_string();
Ok(policy) Ok(policy)
+27 -13
View File
@@ -1,16 +1,15 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use std::fmt::Display;
use http::StatusCode; use http::StatusCode;
use serde::{Serialize, Deserialize}; use serde::{Deserialize, Serialize};
use serde::{ser::Serializer, de::Deserializer}; use serde::{de::Deserializer, ser::Serializer};
use std::fmt::Display;
use s3s::S3ErrorCode;
use s3s::Body; use s3s::Body;
use s3s::S3ErrorCode;
const REPORT_ISSUE: &str = "Please report this issue at https://github.com/rustfs/rustfs/issues."; const REPORT_ISSUE: &str = "Please report this issue at https://github.com/rustfs/rustfs/issues.";
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize, Debug, Clone, thiserror::Error, PartialEq, Eq)]
#[derive(Debug, Clone, thiserror::Error, PartialEq, Eq)]
#[serde(default, rename_all = "PascalCase")] #[serde(default, rename_all = "PascalCase")]
pub struct ErrorResponse { pub struct ErrorResponse {
#[serde(serialize_with = "serialize_code", deserialize_with = "deserialize_code")] #[serde(serialize_with = "serialize_code", deserialize_with = "deserialize_code")]
@@ -29,14 +28,14 @@ pub struct ErrorResponse {
fn serialize_code<S>(data: &S3ErrorCode, s: S) -> Result<S::Ok, S::Error> fn serialize_code<S>(data: &S3ErrorCode, s: S) -> Result<S::Ok, S::Error>
where where
S: Serializer S: Serializer,
{ {
s.serialize_str("") s.serialize_str("")
} }
fn deserialize_code<'de, D>(d: D) -> Result<S3ErrorCode, D::Error> fn deserialize_code<'de, D>(d: D) -> Result<S3ErrorCode, D::Error>
where where
D: Deserializer<'de> D: Deserializer<'de>,
{ {
Ok(S3ErrorCode::from_bytes(String::deserialize(d)?.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into()))) Ok(S3ErrorCode::from_bytes(String::deserialize(d)?.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into())))
} }
@@ -76,7 +75,12 @@ pub fn to_error_response(err: &std::io::Error) -> ErrorResponse {
} }
} }
pub fn http_resp_to_error_response(resp: http::Response<Body>, b: Vec<u8>, bucket_name: &str, object_name: &str) -> ErrorResponse { pub fn http_resp_to_error_response(
resp: http::Response<Body>,
b: Vec<u8>,
bucket_name: &str,
object_name: &str,
) -> ErrorResponse {
let err_body = String::from_utf8(b).unwrap(); let err_body = String::from_utf8(b).unwrap();
//let err_body = xml_decode_and_body(resp.body, &err_resp); //let err_body = xml_decode_and_body(resp.body, &err_resp);
let err_resp_ = serde_xml_rs::from_str::<ErrorResponse>(&err_body); let err_resp_ = serde_xml_rs::from_str::<ErrorResponse>(&err_body);
@@ -189,14 +193,18 @@ pub fn err_transfer_acceleration_bucket(bucket_name: &str) -> ErrorResponse {
ErrorResponse { ErrorResponse {
status_code: StatusCode::BAD_REQUEST, status_code: StatusCode::BAD_REQUEST,
code: S3ErrorCode::InvalidArgument, code: S3ErrorCode::InvalidArgument,
message: "The name of the bucket used for Transfer Acceleration must be DNS-compliant and must not contain periods ..".to_string(), message: "The name of the bucket used for Transfer Acceleration must be DNS-compliant and must not contain periods .."
.to_string(),
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
..Default::default() ..Default::default()
} }
} }
pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse { pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
let msg = format!("Your proposed upload size {} exceeds the maximum allowed object size {} for single PUT operation.", total_size, max_object_size); let msg = format!(
"Your proposed upload size {} exceeds the maximum allowed object size {} for single PUT operation.",
total_size, max_object_size
);
ErrorResponse { ErrorResponse {
status_code: StatusCode::BAD_REQUEST, status_code: StatusCode::BAD_REQUEST,
code: S3ErrorCode::EntityTooLarge, code: S3ErrorCode::EntityTooLarge,
@@ -208,7 +216,10 @@ pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name:
} }
pub fn err_entity_too_small(total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse { pub fn err_entity_too_small(total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
let msg = format!("Your proposed upload size {} is below the minimum allowed object size 0B for single PUT operation.", total_size); let msg = format!(
"Your proposed upload size {} is below the minimum allowed object size 0B for single PUT operation.",
total_size
);
ErrorResponse { ErrorResponse {
status_code: StatusCode::BAD_REQUEST, status_code: StatusCode::BAD_REQUEST,
code: S3ErrorCode::EntityTooSmall, code: S3ErrorCode::EntityTooSmall,
@@ -220,7 +231,10 @@ pub fn err_entity_too_small(total_size: i64, bucket_name: &str, object_name: &st
} }
pub fn err_unexpected_eof(total_read: i64, total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse { pub fn err_unexpected_eof(total_read: i64, total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
let msg = format!("Data read {} is not equal to the size {} of the input Reader.", total_read, total_size); let msg = format!(
"Data read {} is not equal to the size {} of the input Reader.",
total_read, total_size
);
ErrorResponse { ErrorResponse {
status_code: StatusCode::BAD_REQUEST, status_code: StatusCode::BAD_REQUEST,
code: S3ErrorCode::Custom("UnexpectedEOF".into()), code: S3ErrorCode::Custom("UnexpectedEOF".into()),
+16 -8
View File
@@ -1,13 +1,13 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use bytes::Bytes; use bytes::Bytes;
use http::HeaderMap; use http::HeaderMap;
use tokio::io::BufReader;
use std::io::Cursor; use std::io::Cursor;
use tokio::io::BufReader;
use crate::client::{ use crate::client::{
transition_api::{ObjectInfo, to_object_info, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient},
api_error_response::err_invalid_argument, api_error_response::err_invalid_argument,
api_get_options::GetObjectOptions, api_get_options::GetObjectOptions,
transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info},
}; };
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
@@ -16,8 +16,16 @@ impl TransitionClient {
todo!(); todo!();
} }
pub async fn get_object_inner(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> { pub async fn get_object_inner(
let resp = self.execute_method(http::Method::GET, &mut RequestMetadata { &self,
bucket_name: &str,
object_name: &str,
opts: &GetObjectOptions,
) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> {
let resp = self
.execute_method(
http::Method::GET,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(), object_name: object_name.to_string(),
query_values: opts.to_query_values(), query_values: opts.to_query_values(),
@@ -33,7 +41,9 @@ impl TransitionClient {
extra_pre_sign_header: Default::default(), extra_pre_sign_header: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
expires: Default::default(), expires: Default::default(),
}).await?; },
)
.await?;
let resp = &resp; let resp = &resp;
let object_stat = to_object_info(bucket_name, object_name, resp.headers())?; let object_stat = to_object_info(bucket_name, object_name, resp.headers())?;
@@ -79,9 +89,7 @@ struct Object {
impl Object { impl Object {
pub fn new() -> Object { pub fn new() -> Object {
Self { Self { ..Default::default() }
..Default::default()
}
} }
fn do_get_request(&self, request: &GetRequest) -> Result<GetResponse, std::io::Error> { fn do_get_request(&self, request: &GetRequest) -> Result<GetResponse, std::io::Error> {
+8 -8
View File
@@ -1,6 +1,6 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use std::collections::HashMap;
use http::{HeaderMap, HeaderName, HeaderValue}; use http::{HeaderMap, HeaderName, HeaderValue};
use std::collections::HashMap;
use time::OffsetDateTime; use time::OffsetDateTime;
use tracing::warn; use tracing::warn;
@@ -96,15 +96,15 @@ impl GetObjectOptions {
pub fn set_range(&mut self, start: i64, end: i64) -> Result<(), std::io::Error> { pub fn set_range(&mut self, start: i64, end: i64) -> Result<(), std::io::Error> {
if start == 0 && end < 0 { if start == 0 && end < 0 {
self.set("Range", &format!("bytes={}", end)); self.set("Range", &format!("bytes={}", end));
} } else if 0 < start && end == 0 {
else if 0 < start && end == 0 {
self.set("Range", &format!("bytes={}-", start)); self.set("Range", &format!("bytes={}-", start));
} } else if 0 <= start && start <= end {
else if 0 <= start && start <= end {
self.set("Range", &format!("bytes={}-{}", start, end)); self.set("Range", &format!("bytes={}-{}", start, end));
} } else {
else { return Err(std::io::Error::other(err_invalid_argument(&format!(
return Err(std::io::Error::other(err_invalid_argument(&format!("Invalid range specified: start={} end={}", start, end)))); "Invalid range specified: start={} end={}",
start, end
))));
} }
Ok(()) Ok(())
} }
+65 -12
View File
@@ -1,23 +1,36 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use std::collections::HashMap;
use bytes::Bytes; use bytes::Bytes;
use http::{HeaderMap, StatusCode}; use http::{HeaderMap, StatusCode};
use std::collections::HashMap;
use crate::client::{ use crate::client::{
api_error_response::http_resp_to_error_response, api_error_response::http_resp_to_error_response,
api_s3_datatypes::{
ListBucketResult, ListBucketV2Result, ListMultipartUploadsResult, ListObjectPartsResult, ListVersionsResult, ObjectPart,
},
credentials, credentials,
api_s3_datatypes::{ListBucketV2Result, ListMultipartUploadsResult, ListBucketResult, ListObjectPartsResult, ListVersionsResult, ObjectPart}, transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
transition_api::{ReaderImpl, TransitionClient, RequestMetadata,},
}; };
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
use crate::store_api::BucketInfo; use crate::store_api::BucketInfo;
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
impl TransitionClient { impl TransitionClient {
pub fn list_buckets(&self) -> Result<Vec<BucketInfo>, std::io::Error> { pub fn list_buckets(&self) -> Result<Vec<BucketInfo>, std::io::Error> {
todo!(); todo!();
} }
pub async fn list_objects_v2_query(&self, bucket_name: &str, object_prefix: &str, continuation_token: &str, fetch_owner: bool, metadata: bool, delimiter: &str, start_after: &str, max_keys: i64, headers: HeaderMap) -> Result<ListBucketV2Result, std::io::Error> { pub async fn list_objects_v2_query(
&self,
bucket_name: &str,
object_prefix: &str,
continuation_token: &str,
fetch_owner: bool,
metadata: bool,
delimiter: &str,
start_after: &str,
max_keys: i64,
headers: HeaderMap,
) -> Result<ListBucketV2Result, std::io::Error> {
let mut url_values = HashMap::new(); let mut url_values = HashMap::new();
url_values.insert("list-type".to_string(), "2".to_string()); url_values.insert("list-type".to_string(), "2".to_string());
@@ -43,7 +56,10 @@ impl TransitionClient {
url_values.insert("max-keys".to_string(), max_keys.to_string()); url_values.insert("max-keys".to_string(), max_keys.to_string());
} }
let mut resp = self.execute_method(http::Method::GET, &mut RequestMetadata { let mut resp = self
.execute_method(
http::Method::GET,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
object_name: "".to_string(), object_name: "".to_string(),
query_values: url_values, query_values: url_values,
@@ -59,7 +75,9 @@ impl TransitionClient {
extra_pre_sign_header: Default::default(), extra_pre_sign_header: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
expires: Default::default(), expires: Default::default(),
}).await?; },
)
.await?;
if resp.status() != StatusCode::OK { if resp.status() != StatusCode::OK {
return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, ""))); return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, "")));
} }
@@ -97,7 +115,14 @@ impl TransitionClient {
Ok(list_bucket_result) Ok(list_bucket_result)
} }
pub fn list_object_versions_query(&self, bucket_name: &str, opts: &ListObjectsOptions, key_marker: &str, version_id_marker: &str, delimiter: &str) -> Result<ListVersionsResult, std::io::Error> { pub fn list_object_versions_query(
&self,
bucket_name: &str,
opts: &ListObjectsOptions,
key_marker: &str,
version_id_marker: &str,
delimiter: &str,
) -> Result<ListVersionsResult, std::io::Error> {
/*if err := s3utils.CheckValidBucketName(bucketName); err != nil { /*if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return ListVersionsResult{}, err return ListVersionsResult{}, err
} }
@@ -177,15 +202,36 @@ impl TransitionClient {
todo!(); todo!();
} }
pub fn list_objects_query(&self, bucket_name: &str, object_prefix: &str, object_marker: &str, delimiter: &str, max_keys: i64, headers: HeaderMap) -> Result<ListBucketResult, std::io::Error> { pub fn list_objects_query(
&self,
bucket_name: &str,
object_prefix: &str,
object_marker: &str,
delimiter: &str,
max_keys: i64,
headers: HeaderMap,
) -> Result<ListBucketResult, std::io::Error> {
todo!(); todo!();
} }
pub fn list_multipart_uploads_query(&self, bucket_name: &str, key_marker: &str, upload_id_marker: &str, prefix: &str, delimiter: &str, max_uploads: i64) -> Result<ListMultipartUploadsResult, std::io::Error> { pub fn list_multipart_uploads_query(
&self,
bucket_name: &str,
key_marker: &str,
upload_id_marker: &str,
prefix: &str,
delimiter: &str,
max_uploads: i64,
) -> Result<ListMultipartUploadsResult, std::io::Error> {
todo!(); todo!();
} }
pub fn list_object_parts(&self, bucket_name: &str, object_name: &str, upload_id: &str) -> Result<HashMap<i64, ObjectPart>, std::io::Error> { pub fn list_object_parts(
&self,
bucket_name: &str,
object_name: &str,
upload_id: &str,
) -> Result<HashMap<i64, ObjectPart>, std::io::Error> {
todo!(); todo!();
} }
@@ -193,7 +239,14 @@ impl TransitionClient {
todo!(); todo!();
} }
pub async fn list_object_parts_query(&self, bucket_name: &str, object_name: &str, upload_id: &str, part_number_marker: i64, max_parts: i64) -> Result<ListObjectPartsResult, std::io::Error> { pub async fn list_object_parts_query(
&self,
bucket_name: &str,
object_name: &str,
upload_id: &str,
part_number_marker: i64,
max_parts: i64,
) -> Result<ListObjectPartsResult, std::io::Error> {
todo!(); todo!();
} }
} }
+84 -43
View File
@@ -1,33 +1,30 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use std::{collections::HashMap, sync::Arc};
use bytes::Bytes; use bytes::Bytes;
use time::{OffsetDateTime, macros::format_description, Duration};
use http::{HeaderMap, HeaderName, HeaderValue}; use http::{HeaderMap, HeaderName, HeaderValue};
use std::{collections::HashMap, sync::Arc};
use time::{Duration, OffsetDateTime, macros::format_description};
use tracing::{error, info, warn}; use tracing::{error, info, warn};
use s3s::dto::{
ObjectLockRetentionMode, ObjectLockLegalHoldStatus,
ReplicationStatus,
};
use s3s::header::{
X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_OBJECT_LOCK_LEGAL_HOLD,
X_AMZ_STORAGE_CLASS, X_AMZ_WEBSITE_REDIRECT_LOCATION, X_AMZ_REPLICATION_STATUS,
};
use reader::hasher::Hasher; use reader::hasher::Hasher;
//use crate::disk::{BufferReader, Reader}; use s3s::dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus};
use rustfs_utils::{ use s3s::header::{
crypto::base64_encode, X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_REPLICATION_STATUS,
net::{is_amz_header, is_standard_header, is_storageclass_header, is_rustfs_header, is_minio_header}, X_AMZ_STORAGE_CLASS, X_AMZ_WEBSITE_REDIRECT_LOCATION,
}; };
//use crate::disk::{BufferReader, Reader};
use crate::checksum::ChecksumMode; use crate::checksum::ChecksumMode;
use crate::client::{ use crate::client::{
api_s3_datatypes::{CompletePart, ObjectPart, CompleteMultipartUpload}, api_error_response::{err_entity_too_large, err_invalid_argument},
api_put_object_common::optimal_part_info, api_put_object_common::optimal_part_info,
transition_api::{TransitionClient, UploadInfo, ReaderImpl},
api_error_response::{err_invalid_argument, err_entity_too_large},
api_put_object_multipart::UploadPartParams, api_put_object_multipart::UploadPartParams,
api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart},
constants::{ISO8601_DATEFORMAT, MAX_MULTIPART_PUT_OBJECT_SIZE, MIN_PART_SIZE, TOTAL_WORKERS},
credentials::SignatureType, credentials::SignatureType,
constants::{MAX_MULTIPART_PUT_OBJECT_SIZE, TOTAL_WORKERS, MIN_PART_SIZE, ISO8601_DATEFORMAT,}, transition_api::{ReaderImpl, TransitionClient, UploadInfo},
};
use rustfs_utils::{
crypto::base64_encode,
net::{is_amz_header, is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header},
}; };
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
@@ -123,17 +120,21 @@ impl Default for PutObjectOptions {
impl PutObjectOptions { impl PutObjectOptions {
fn set_matche_tag(&mut self, etag: &str) { fn set_matche_tag(&mut self, etag: &str) {
if etag == "*" { if etag == "*" {
self.custom_header.insert("If-Match", HeaderValue::from_str("*").expect("err")); self.custom_header
.insert("If-Match", HeaderValue::from_str("*").expect("err"));
} else { } else {
self.custom_header.insert("If-Match", HeaderValue::from_str(&format!("\"{}\"", etag)).expect("err")); self.custom_header
.insert("If-Match", HeaderValue::from_str(&format!("\"{}\"", etag)).expect("err"));
} }
} }
fn set_matche_tag_except(&mut self, etag: &str) { fn set_matche_tag_except(&mut self, etag: &str) {
if etag == "*" { if etag == "*" {
self.custom_header.insert("If-None-Match", HeaderValue::from_str("*").expect("err")); self.custom_header
.insert("If-None-Match", HeaderValue::from_str("*").expect("err"));
} else { } else {
self.custom_header.insert("If-None-Match", HeaderValue::from_str(&format!("\"{etag}\"")).expect("err")); self.custom_header
.insert("If-None-Match", HeaderValue::from_str(&format!("\"{etag}\"")).expect("err"));
} }
} }
@@ -160,7 +161,10 @@ impl PutObjectOptions {
} }
if self.expires.unix_timestamp() != 0 { if self.expires.unix_timestamp() != 0 {
header.insert("Expires", HeaderValue::from_str(&self.expires.format(ISO8601_DATEFORMAT).unwrap()).expect("err")); //rustfs invalid heade header.insert(
"Expires",
HeaderValue::from_str(&self.expires.format(ISO8601_DATEFORMAT).unwrap()).expect("err"),
); //rustfs invalid heade
} }
if self.mode.as_str() != "" { if self.mode.as_str() != "" {
@@ -168,7 +172,10 @@ impl PutObjectOptions {
} }
if self.retain_until_date.unix_timestamp() != 0 { if self.retain_until_date.unix_timestamp() != 0 {
header.insert(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, HeaderValue::from_str(&self.retain_until_date.format(ISO8601_DATEFORMAT).unwrap()).expect("err")); header.insert(
X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE,
HeaderValue::from_str(&self.retain_until_date.format(ISO8601_DATEFORMAT).unwrap()).expect("err"),
);
} }
if self.legalhold.as_str() != "" { if self.legalhold.as_str() != "" {
@@ -180,15 +187,22 @@ impl PutObjectOptions {
} }
if self.website_redirect_location != "" { if self.website_redirect_location != "" {
header.insert(X_AMZ_WEBSITE_REDIRECT_LOCATION, HeaderValue::from_str(&self.website_redirect_location).expect("err")); header.insert(
X_AMZ_WEBSITE_REDIRECT_LOCATION,
HeaderValue::from_str(&self.website_redirect_location).expect("err"),
);
} }
if !self.internal.replication_status.as_str().is_empty() { if !self.internal.replication_status.as_str().is_empty() {
header.insert(X_AMZ_REPLICATION_STATUS, HeaderValue::from_str(self.internal.replication_status.as_str()).expect("err")); header.insert(
X_AMZ_REPLICATION_STATUS,
HeaderValue::from_str(self.internal.replication_status.as_str()).expect("err"),
);
} }
for (k, v) in &self.user_metadata { for (k, v) in &self.user_metadata {
if is_amz_header(k) || is_standard_header(k) || is_storageclass_header(k) || is_rustfs_header(k) || is_minio_header(k) { if is_amz_header(k) || is_standard_header(k) || is_storageclass_header(k) || is_rustfs_header(k) || is_minio_header(k)
{
if let Ok(header_name) = HeaderName::from_bytes(k.as_bytes()) { if let Ok(header_name) = HeaderName::from_bytes(k.as_bytes()) {
header.insert(header_name, HeaderValue::from_str(&v).unwrap()); header.insert(header_name, HeaderValue::from_str(&v).unwrap());
} }
@@ -221,19 +235,37 @@ impl PutObjectOptions {
} }
impl TransitionClient { impl TransitionClient {
pub async fn put_object(self: Arc<Self>, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, object_size: i64, pub async fn put_object(
opts: &PutObjectOptions self: Arc<Self>,
bucket_name: &str,
object_name: &str,
mut reader: ReaderImpl,
object_size: i64,
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> { ) -> Result<UploadInfo, std::io::Error> {
if object_size < 0 && opts.disable_multipart { if object_size < 0 && opts.disable_multipart {
return Err(std::io::Error::other("object size must be provided with disable multipart upload")); return Err(std::io::Error::other("object size must be provided with disable multipart upload"));
} }
self.put_object_common(bucket_name, object_name, reader, object_size, opts).await self.put_object_common(bucket_name, object_name, reader, object_size, opts)
.await
} }
pub async fn put_object_common(self: Arc<Self>, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions) -> Result<UploadInfo, std::io::Error> { pub async fn put_object_common(
self: Arc<Self>,
bucket_name: &str,
object_name: &str,
mut reader: ReaderImpl,
size: i64,
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> {
if size > MAX_MULTIPART_PUT_OBJECT_SIZE { if size > MAX_MULTIPART_PUT_OBJECT_SIZE {
return Err(std::io::Error::other(err_entity_too_large(size, MAX_MULTIPART_PUT_OBJECT_SIZE, bucket_name, object_name))); return Err(std::io::Error::other(err_entity_too_large(
size,
MAX_MULTIPART_PUT_OBJECT_SIZE,
bucket_name,
object_name,
)));
} }
let mut opts = opts.clone(); let mut opts = opts.clone();
opts.auto_checksum.set_default(ChecksumMode::ChecksumCRC32C); opts.auto_checksum.set_default(ChecksumMode::ChecksumCRC32C);
@@ -255,19 +287,30 @@ impl TransitionClient {
return Err(std::io::Error::other("no length provided and multipart disabled")); return Err(std::io::Error::other("no length provided and multipart disabled"));
} }
if opts.concurrent_stream_parts && opts.num_threads > 1 { if opts.concurrent_stream_parts && opts.num_threads > 1 {
return self.put_object_multipart_stream_parallel(bucket_name, object_name, reader, &opts).await; return self
.put_object_multipart_stream_parallel(bucket_name, object_name, reader, &opts)
.await;
} }
return self.put_object_multipart_stream_no_length(bucket_name, object_name, reader, &opts).await; return self
.put_object_multipart_stream_no_length(bucket_name, object_name, reader, &opts)
.await;
} }
if size <= part_size || opts.disable_multipart { if size <= part_size || opts.disable_multipart {
return self.put_object_gcs(bucket_name, object_name, reader, size, &opts).await; return self.put_object_gcs(bucket_name, object_name, reader, size, &opts).await;
} }
self.put_object_multipart_stream(bucket_name, object_name, reader, size, &opts).await self.put_object_multipart_stream(bucket_name, object_name, reader, size, &opts)
.await
} }
pub async fn put_object_multipart_stream_no_length(&self, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, opts: &PutObjectOptions) -> Result<UploadInfo, std::io::Error> { pub async fn put_object_multipart_stream_no_length(
&self,
bucket_name: &str,
object_name: &str,
mut reader: ReaderImpl,
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> {
let mut total_uploaded_size: i64 = 0; let mut total_uploaded_size: i64 = 0;
let mut compl_multipart_upload = CompleteMultipartUpload::default(); let mut compl_multipart_upload = CompleteMultipartUpload::default();
@@ -295,12 +338,8 @@ impl TransitionClient {
while part_number <= total_parts_count { while part_number <= total_parts_count {
buf = match &mut reader { buf = match &mut reader {
ReaderImpl::Body(content_body) => { ReaderImpl::Body(content_body) => content_body.to_vec(),
content_body.to_vec() ReaderImpl::ObjectBody(content_body) => content_body.read_all().await?,
}
ReaderImpl::ObjectBody(content_body) => {
content_body.read_all().await?
}
}; };
let length = buf.len(); let length = buf.len();
@@ -374,7 +413,9 @@ impl TransitionClient {
}; };
//apply_auto_checksum(&mut opts, all_parts); //apply_auto_checksum(&mut opts, all_parts);
let mut upload_info = self.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts).await?; let mut upload_info = self
.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts)
.await?;
upload_info.size = total_uploaded_size; upload_info.size = total_uploaded_size;
Ok(upload_info) Ok(upload_info)
+31 -10
View File
@@ -1,10 +1,10 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use crate::client::{ use crate::client::{
api_put_object::PutObjectOptions,
constants::{ABS_MIN_PART_SIZE, MAX_MULTIPART_PUT_OBJECT_SIZE, MAX_PARTS_COUNT, MAX_PART_SIZE, MIN_PART_SIZE},
transition_api::TransitionClient,
transition_api::ReaderImpl,
api_error_response::{err_entity_too_large, err_invalid_argument}, api_error_response::{err_entity_too_large, err_invalid_argument},
api_put_object::PutObjectOptions,
constants::{ABS_MIN_PART_SIZE, MAX_MULTIPART_PUT_OBJECT_SIZE, MAX_PART_SIZE, MAX_PARTS_COUNT, MIN_PART_SIZE},
transition_api::ReaderImpl,
transition_api::TransitionClient,
}; };
const NULL_VERSION_ID: &str = "null"; const NULL_VERSION_ID: &str = "null";
@@ -28,27 +28,43 @@ pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<
} }
if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE { if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
return Err(std::io::Error::other(err_entity_too_large(object_size, MAX_MULTIPART_PUT_OBJECT_SIZE, "", ""))); return Err(std::io::Error::other(err_entity_too_large(
object_size,
MAX_MULTIPART_PUT_OBJECT_SIZE,
"",
"",
)));
} }
let mut part_size_flt: f64; let mut part_size_flt: f64;
if configured_part_size > 0 { if configured_part_size > 0 {
if configured_part_size as i64 > object_size { if configured_part_size as i64 > object_size {
return Err(std::io::Error::other(err_entity_too_large(configured_part_size as i64, object_size, "", ""))); return Err(std::io::Error::other(err_entity_too_large(
configured_part_size as i64,
object_size,
"",
"",
)));
} }
if !unknown_size { if !unknown_size {
if object_size > (configured_part_size as i64 * MAX_PARTS_COUNT) { if object_size > (configured_part_size as i64 * MAX_PARTS_COUNT) {
return Err(std::io::Error::other(err_invalid_argument("Part size * max_parts(10000) is lesser than input objectSize."))); return Err(std::io::Error::other(err_invalid_argument(
"Part size * max_parts(10000) is lesser than input objectSize.",
)));
} }
} }
if (configured_part_size as i64) < ABS_MIN_PART_SIZE { if (configured_part_size as i64) < ABS_MIN_PART_SIZE {
return Err(std::io::Error::other(err_invalid_argument("Input part size is smaller than allowed minimum of 5MiB."))); return Err(std::io::Error::other(err_invalid_argument(
"Input part size is smaller than allowed minimum of 5MiB.",
)));
} }
if configured_part_size as i64 > MAX_PART_SIZE { if configured_part_size as i64 > MAX_PART_SIZE {
return Err(std::io::Error::other(err_invalid_argument("Input part size is bigger than allowed maximum of 5GiB."))); return Err(std::io::Error::other(err_invalid_argument(
"Input part size is bigger than allowed maximum of 5GiB.",
)));
} }
part_size_flt = configured_part_size as f64; part_size_flt = configured_part_size as f64;
@@ -69,7 +85,12 @@ pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<
} }
impl TransitionClient { impl TransitionClient {
pub async fn new_upload_id(&self ,bucket_name: &str, object_name: &str, opts: &PutObjectOptions) -> Result<String, std::io::Error> { pub async fn new_upload_id(
&self,
bucket_name: &str,
object_name: &str,
opts: &PutObjectOptions,
) -> Result<String, std::io::Error> {
let init_multipart_upload_result = self.initiate_multipart_upload(bucket_name, object_name, opts).await?; let init_multipart_upload_result = self.initiate_multipart_upload(bucket_name, object_name, opts).await?;
Ok(init_multipart_upload_result.upload_id) Ok(init_multipart_upload_result.upload_id)
} }
+111 -46
View File
@@ -1,50 +1,61 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use bytes::Bytes;
use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
use s3s::S3ErrorCode;
use std::io::Read; use std::io::Read;
use std::{collections::HashMap, sync::Arc}; use std::{collections::HashMap, sync::Arc};
use bytes::Bytes; use time::{OffsetDateTime, format_description};
use s3s::S3ErrorCode;
use time::{format_description, OffsetDateTime};
use uuid::Uuid;
use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
use url::form_urlencoded::Serializer;
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
use tracing::warn; use tracing::warn;
use tracing::{error, info}; use tracing::{error, info};
use url::form_urlencoded::Serializer;
use uuid::Uuid;
use s3s::{dto::StreamingBlob, Body};
use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};
use reader::hasher::Hasher; use reader::hasher::Hasher;
use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};
use s3s::{Body, dto::StreamingBlob};
//use crate::disk::{Reader, BufferReader}; //use crate::disk::{Reader, BufferReader};
use crate::client::{ use crate::client::{
transition_api::{RequestMetadata, TransitionClient, UploadInfo, ReaderImpl,}, api_error_response::{
api_error_response::{err_entity_too_large, err_entity_too_small, err_invalid_argument, http_resp_to_error_response, to_error_response}, err_entity_too_large, err_entity_too_small, err_invalid_argument, http_resp_to_error_response, to_error_response,
},
api_put_object::PutObjectOptions, api_put_object::PutObjectOptions,
api_put_object_common::optimal_part_info, api_put_object_common::optimal_part_info,
api_s3_datatypes::{CompleteMultipartUpload, CompleteMultipartUploadResult, CompletePart, InitiateMultipartUploadResult, ObjectPart}, api_s3_datatypes::{
constants::{ABS_MIN_PART_SIZE, MAX_PART_SIZE, MAX_SINGLE_PUT_OBJECT_SIZE, ISO8601_DATEFORMAT, }, CompleteMultipartUpload, CompleteMultipartUploadResult, CompletePart, InitiateMultipartUploadResult, ObjectPart,
}; },
use rustfs_utils::{ constants::{ABS_MIN_PART_SIZE, ISO8601_DATEFORMAT, MAX_PART_SIZE, MAX_SINGLE_PUT_OBJECT_SIZE},
path::trim_etag, transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo},
crypto::base64_encode,
}; };
use crate::{ use crate::{
disk::DiskAPI,
store_api::{
GetObjectReader, StorageAPI,
},
checksum::ChecksumMode, checksum::ChecksumMode,
disk::DiskAPI,
store_api::{GetObjectReader, StorageAPI},
}; };
use rustfs_utils::{crypto::base64_encode, path::trim_etag};
impl TransitionClient { impl TransitionClient {
pub async fn put_object_multipart(&self, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, size: i64, pub async fn put_object_multipart(
opts: &PutObjectOptions &self,
bucket_name: &str,
object_name: &str,
mut reader: ReaderImpl,
size: i64,
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> { ) -> Result<UploadInfo, std::io::Error> {
let info = self.put_object_multipart_no_stream(bucket_name, object_name, &mut reader, opts).await; let info = self
.put_object_multipart_no_stream(bucket_name, object_name, &mut reader, opts)
.await;
if let Err(err) = &info { if let Err(err) = &info {
let err_resp = to_error_response(err); let err_resp = to_error_response(err);
if err_resp.code == S3ErrorCode::AccessDenied && err_resp.message.contains("Access Denied") { if err_resp.code == S3ErrorCode::AccessDenied && err_resp.message.contains("Access Denied") {
if size > MAX_SINGLE_PUT_OBJECT_SIZE { if size > MAX_SINGLE_PUT_OBJECT_SIZE {
return Err(std::io::Error::other(err_entity_too_large(size, MAX_SINGLE_PUT_OBJECT_SIZE, bucket_name, object_name))); return Err(std::io::Error::other(err_entity_too_large(
size,
MAX_SINGLE_PUT_OBJECT_SIZE,
bucket_name,
object_name,
)));
} }
return self.put_object_gcs(bucket_name, object_name, reader, size, opts).await; return self.put_object_gcs(bucket_name, object_name, reader, size, opts).await;
} }
@@ -52,7 +63,13 @@ impl TransitionClient {
Ok(info?) Ok(info?)
} }
pub async fn put_object_multipart_no_stream(&self, bucket_name: &str, object_name: &str, reader: &mut ReaderImpl, opts: &PutObjectOptions) -> Result<UploadInfo, std::io::Error> { pub async fn put_object_multipart_no_stream(
&self,
bucket_name: &str,
object_name: &str,
reader: &mut ReaderImpl,
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> {
let mut total_uploaded_size: i64 = 0; let mut total_uploaded_size: i64 = 0;
let mut compl_multipart_upload = CompleteMultipartUpload::default(); let mut compl_multipart_upload = CompleteMultipartUpload::default();
@@ -156,13 +173,20 @@ impl TransitionClient {
}; };
//apply_auto_checksum(&mut opts, all_parts); //apply_auto_checksum(&mut opts, all_parts);
let mut upload_info = self.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts).await?; let mut upload_info = self
.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts)
.await?;
upload_info.size = total_uploaded_size; upload_info.size = total_uploaded_size;
Ok(upload_info) Ok(upload_info)
} }
pub async fn initiate_multipart_upload(&self, bucket_name: &str, object_name: &str, opts: &PutObjectOptions) -> Result<InitiateMultipartUploadResult, std::io::Error> { pub async fn initiate_multipart_upload(
&self,
bucket_name: &str,
object_name: &str,
opts: &PutObjectOptions,
) -> Result<InitiateMultipartUploadResult, std::io::Error> {
let mut url_values = HashMap::new(); let mut url_values = HashMap::new();
url_values.insert("uploads".to_string(), "".to_string()); url_values.insert("uploads".to_string(), "".to_string());
@@ -207,13 +231,20 @@ impl TransitionClient {
pub async fn upload_part(&self, p: &mut UploadPartParams) -> Result<ObjectPart, std::io::Error> { pub async fn upload_part(&self, p: &mut UploadPartParams) -> Result<ObjectPart, std::io::Error> {
if p.size > MAX_PART_SIZE { if p.size > MAX_PART_SIZE {
return Err(std::io::Error::other(err_entity_too_large(p.size, MAX_PART_SIZE, &p.bucket_name, &p.object_name))); return Err(std::io::Error::other(err_entity_too_large(
p.size,
MAX_PART_SIZE,
&p.bucket_name,
&p.object_name,
)));
} }
if p.size <= -1 { if p.size <= -1 {
return Err(std::io::Error::other(err_entity_too_small(p.size, &p.bucket_name, &p.object_name))); return Err(std::io::Error::other(err_entity_too_small(p.size, &p.bucket_name, &p.object_name)));
} }
if p.part_number <= 0 { if p.part_number <= 0 {
return Err(std::io::Error::other(err_invalid_argument("Part number cannot be negative or equal to zero."))); return Err(std::io::Error::other(err_invalid_argument(
"Part number cannot be negative or equal to zero.",
)));
} }
if p.upload_id == "" { if p.upload_id == "" {
return Err(std::io::Error::other(err_invalid_argument("UploadID cannot be empty."))); return Err(std::io::Error::other(err_invalid_argument("UploadID cannot be empty.")));
@@ -224,12 +255,8 @@ impl TransitionClient {
url_values.insert("uploadId".to_string(), p.upload_id.clone()); url_values.insert("uploadId".to_string(), p.upload_id.clone());
let buf = match &mut p.reader { let buf = match &mut p.reader {
ReaderImpl::Body(content_body) => { ReaderImpl::Body(content_body) => content_body.to_vec(),
content_body.to_vec() ReaderImpl::ObjectBody(content_body) => content_body.read_all().await?,
}
ReaderImpl::ObjectBody(content_body) => {
content_body.read_all().await?
}
}; };
let mut req_metadata = RequestMetadata { let mut req_metadata = RequestMetadata {
bucket_name: p.bucket_name.clone(), bucket_name: p.bucket_name.clone(),
@@ -253,26 +280,60 @@ impl TransitionClient {
//defer closeResponse(resp) //defer closeResponse(resp)
//if resp.is_none() { //if resp.is_none() {
if resp.status() != StatusCode::OK { if resp.status() != StatusCode::OK {
return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], &p.bucket_name.clone(), &p.object_name))); return Err(std::io::Error::other(http_resp_to_error_response(
resp,
vec![],
&p.bucket_name.clone(),
&p.object_name,
)));
} }
//} //}
let h = resp.headers(); let h = resp.headers();
let mut obj_part = ObjectPart { let mut obj_part = ObjectPart {
checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) { h_checksum_crc32.to_str().expect("err").to_string() } else { "".to_string() }, checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) {
checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) { h_checksum_crc32c.to_str().expect("err").to_string() } else { "".to_string() }, h_checksum_crc32.to_str().expect("err").to_string()
checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) { h_checksum_sha1.to_str().expect("err").to_string() } else { "".to_string() }, } else {
checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) { h_checksum_sha256.to_str().expect("err").to_string() } else { "".to_string() }, "".to_string()
checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) { h_checksum_crc64nvme.to_str().expect("err").to_string() } else { "".to_string() }, },
checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) {
h_checksum_crc32c.to_str().expect("err").to_string()
} else {
"".to_string()
},
checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) {
h_checksum_sha1.to_str().expect("err").to_string()
} else {
"".to_string()
},
checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) {
h_checksum_sha256.to_str().expect("err").to_string()
} else {
"".to_string()
},
checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) {
h_checksum_crc64nvme.to_str().expect("err").to_string()
} else {
"".to_string()
},
..Default::default() ..Default::default()
}; };
obj_part.size = p.size; obj_part.size = p.size;
obj_part.part_num = p.part_number; obj_part.part_num = p.part_number;
obj_part.etag = if let Some(h_etag) = h.get("ETag") { h_etag.to_str().expect("err").trim_matches('"').to_string() } else { "".to_string() }; obj_part.etag = if let Some(h_etag) = h.get("ETag") {
h_etag.to_str().expect("err").trim_matches('"').to_string()
} else {
"".to_string()
};
Ok(obj_part) Ok(obj_part)
} }
pub async fn complete_multipart_upload(&self, bucket_name: &str, object_name: &str, upload_id: &str, pub async fn complete_multipart_upload(
complete: CompleteMultipartUpload, opts: &PutObjectOptions &self,
bucket_name: &str,
object_name: &str,
upload_id: &str,
complete: CompleteMultipartUpload,
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> { ) -> Result<UploadInfo, std::io::Error> {
let mut url_values = HashMap::new(); let mut url_values = HashMap::new();
url_values.insert("uploadId".to_string(), upload_id.to_string()); url_values.insert("uploadId".to_string(), upload_id.to_string());
@@ -307,7 +368,7 @@ impl TransitionClient {
let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) { let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) {
( (
OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(), OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(),
"".to_string() "".to_string(),
) )
} else { } else {
(OffsetDateTime::now_utc(), "".to_string()) (OffsetDateTime::now_utc(), "".to_string())
@@ -318,7 +379,11 @@ impl TransitionClient {
bucket: complete_multipart_upload_result.bucket, bucket: complete_multipart_upload_result.bucket,
key: complete_multipart_upload_result.key, key: complete_multipart_upload_result.key,
etag: trim_etag(&complete_multipart_upload_result.etag), etag: trim_etag(&complete_multipart_upload_result.etag),
version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) { h_x_amz_version_id.to_str().expect("err").to_string() } else { "".to_string() }, version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) {
h_x_amz_version_id.to_str().expect("err").to_string()
} else {
"".to_string()
},
location: complete_multipart_upload_result.location, location: complete_multipart_upload_result.location,
expiration: exp_time, expiration: exp_time,
expiration_rule_id: rule_id, expiration_rule_id: rule_id,
+114 -35
View File
@@ -1,28 +1,28 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use std::sync::RwLock;
use std::{collections::HashMap, sync::Arc};
use bytes::Bytes; use bytes::Bytes;
use futures::future::join_all; use futures::future::join_all;
use http::{HeaderMap, HeaderName, HeaderValue, StatusCode}; use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
use time::{format_description, OffsetDateTime}; use std::sync::RwLock;
use std::{collections::HashMap, sync::Arc};
use time::{OffsetDateTime, format_description};
use tokio::{select, sync::mpsc}; use tokio::{select, sync::mpsc};
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
use tracing::warn; use tracing::warn;
use uuid::Uuid; use uuid::Uuid;
use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID}; use crate::checksum::{ChecksumMode, add_auto_checksum_headers, apply_auto_checksum};
use reader::hasher::Hasher;
use crate::client::{ use crate::client::{
constants::ISO8601_DATEFORMAT, api_error_response::{err_invalid_argument, err_unexpected_eof, http_resp_to_error_response},
api_put_object::PutObjectOptions, api_put_object::PutObjectOptions,
api_put_object_common::{is_object, optimal_part_info},
api_put_object_multipart::UploadPartParams, api_put_object_multipart::UploadPartParams,
api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart}, api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart},
transition_api::{TransitionClient, RequestMetadata, UploadInfo, ReaderImpl}, constants::ISO8601_DATEFORMAT,
api_put_object_common::{is_object, optimal_part_info,}, transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo},
api_error_response::{err_invalid_argument, http_resp_to_error_response, err_unexpected_eof},
}; };
use reader::hasher::Hasher;
use rustfs_utils::{crypto::base64_encode, path::trim_etag}; use rustfs_utils::{crypto::base64_encode, path::trim_etag};
use crate::checksum::{add_auto_checksum_headers, apply_auto_checksum, ChecksumMode}; use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};
pub struct UploadedPartRes { pub struct UploadedPartRes {
pub error: std::io::Error, pub error: std::io::Error,
@@ -37,23 +37,39 @@ pub struct UploadPartReq {
} }
impl TransitionClient { impl TransitionClient {
pub async fn put_object_multipart_stream(self: Arc<Self>, bucket_name: &str, object_name: &str, pub async fn put_object_multipart_stream(
mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions self: Arc<Self>,
bucket_name: &str,
object_name: &str,
mut reader: ReaderImpl,
size: i64,
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> { ) -> Result<UploadInfo, std::io::Error> {
let info: UploadInfo; let info: UploadInfo;
if opts.concurrent_stream_parts && opts.num_threads > 1 { if opts.concurrent_stream_parts && opts.num_threads > 1 {
info = self.put_object_multipart_stream_parallel(bucket_name, object_name, reader, opts).await?; info = self
.put_object_multipart_stream_parallel(bucket_name, object_name, reader, opts)
.await?;
} else if !is_object(&reader) && !opts.send_content_md5 { } else if !is_object(&reader) && !opts.send_content_md5 {
info = self.put_object_multipart_stream_from_readat(bucket_name, object_name, reader, size, opts).await?; info = self
.put_object_multipart_stream_from_readat(bucket_name, object_name, reader, size, opts)
.await?;
} else { } else {
info = self.put_object_multipart_stream_optional_checksum(bucket_name, object_name, reader, size, opts).await?; info = self
.put_object_multipart_stream_optional_checksum(bucket_name, object_name, reader, size, opts)
.await?;
} }
Ok(info) Ok(info)
} }
pub async fn put_object_multipart_stream_from_readat(&self, bucket_name: &str, object_name: &str, pub async fn put_object_multipart_stream_from_readat(
mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions &self,
bucket_name: &str,
object_name: &str,
mut reader: ReaderImpl,
size: i64,
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> { ) -> Result<UploadInfo, std::io::Error> {
let ret = optimal_part_info(size, opts.part_size)?; let ret = optimal_part_info(size, opts.part_size)?;
let (total_parts_count, part_size, lastpart_size) = ret; let (total_parts_count, part_size, lastpart_size) = ret;
@@ -68,8 +84,13 @@ impl TransitionClient {
todo!(); todo!();
} }
pub async fn put_object_multipart_stream_optional_checksum(&self, bucket_name: &str, object_name: &str, pub async fn put_object_multipart_stream_optional_checksum(
mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions &self,
bucket_name: &str,
object_name: &str,
mut reader: ReaderImpl,
size: i64,
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> { ) -> Result<UploadInfo, std::io::Error> {
let mut opts = opts.clone(); let mut opts = opts.clone();
if opts.checksum.is_set() { if opts.checksum.is_set() {
@@ -154,7 +175,12 @@ impl TransitionClient {
if size > 0 { if size > 0 {
if total_uploaded_size != size { if total_uploaded_size != size {
return Err(std::io::Error::other(err_unexpected_eof(total_uploaded_size, size, bucket_name, object_name))); return Err(std::io::Error::other(err_unexpected_eof(
total_uploaded_size,
size,
bucket_name,
object_name,
)));
} }
} }
@@ -185,14 +211,20 @@ impl TransitionClient {
..Default::default() ..Default::default()
}; };
apply_auto_checksum(&mut opts, &mut all_parts); apply_auto_checksum(&mut opts, &mut all_parts);
let mut upload_info = self.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts).await?; let mut upload_info = self
.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts)
.await?;
upload_info.size = total_uploaded_size; upload_info.size = total_uploaded_size;
Ok(upload_info) Ok(upload_info)
} }
pub async fn put_object_multipart_stream_parallel(self: Arc<Self>, bucket_name: &str, object_name: &str, pub async fn put_object_multipart_stream_parallel(
mut reader: ReaderImpl/*GetObjectReader*/, opts: &PutObjectOptions self: Arc<Self>,
bucket_name: &str,
object_name: &str,
mut reader: ReaderImpl, /*GetObjectReader*/
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> { ) -> Result<UploadInfo, std::io::Error> {
let mut opts = opts.clone(); let mut opts = opts.clone();
if opts.checksum.is_set() { if opts.checksum.is_set() {
@@ -239,7 +271,11 @@ impl TransitionClient {
} }
if buf.len() != part_size as usize { if buf.len() != part_size as usize {
return Err(std::io::Error::other(format!("read buffer < {} than expected partSize: {}", buf.len(), part_size))); return Err(std::io::Error::other(format!(
"read buffer < {} than expected partSize: {}",
buf.len(),
part_size
)));
} }
match &mut reader { match &mut reader {
@@ -348,13 +384,22 @@ impl TransitionClient {
}; };
apply_auto_checksum(&mut opts, &mut all_parts); apply_auto_checksum(&mut opts, &mut all_parts);
let mut upload_info = self.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts).await?; let mut upload_info = self
.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts)
.await?;
upload_info.size = total_uploaded_size; upload_info.size = total_uploaded_size;
Ok(upload_info) Ok(upload_info)
} }
pub async fn put_object_gcs(&self, bucket_name: &str, object_name: &str, mut reader: ReaderImpl, size: i64, opts: &PutObjectOptions) -> Result<UploadInfo, std::io::Error> { pub async fn put_object_gcs(
&self,
bucket_name: &str,
object_name: &str,
mut reader: ReaderImpl,
size: i64,
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> {
let mut opts = opts.clone(); let mut opts = opts.clone();
if opts.checksum.is_set() { if opts.checksum.is_set() {
opts.send_content_md5 = false; opts.send_content_md5 = false;
@@ -363,10 +408,20 @@ impl TransitionClient {
let mut md5_base64: String = "".to_string(); let mut md5_base64: String = "".to_string();
let progress_reader = reader; //newHook(reader, opts.progress); let progress_reader = reader; //newHook(reader, opts.progress);
self.put_object_do(bucket_name, object_name, progress_reader, &md5_base64, "", size, &opts).await self.put_object_do(bucket_name, object_name, progress_reader, &md5_base64, "", size, &opts)
.await
} }
pub async fn put_object_do(&self, bucket_name: &str, object_name: &str, reader: ReaderImpl, md5_base64: &str, sha256_hex: &str, size: i64, opts: &PutObjectOptions) -> Result<UploadInfo, std::io::Error> { pub async fn put_object_do(
&self,
bucket_name: &str,
object_name: &str,
reader: ReaderImpl,
md5_base64: &str,
sha256_hex: &str,
size: i64,
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> {
let custom_header = opts.header(); let custom_header = opts.header();
let mut req_metadata = RequestMetadata { let mut req_metadata = RequestMetadata {
@@ -422,7 +477,7 @@ impl TransitionClient {
let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) { let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) {
( (
OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(), OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(),
"".to_string() "".to_string(),
) )
} else { } else {
(OffsetDateTime::now_utc(), "".to_string()) (OffsetDateTime::now_utc(), "".to_string())
@@ -432,15 +487,39 @@ impl TransitionClient {
bucket: bucket_name.to_string(), bucket: bucket_name.to_string(),
key: object_name.to_string(), key: object_name.to_string(),
etag: trim_etag(h.get("ETag").expect("err").to_str().expect("err")), etag: trim_etag(h.get("ETag").expect("err").to_str().expect("err")),
version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) { h_x_amz_version_id.to_str().expect("err").to_string() } else { "".to_string() }, version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) {
h_x_amz_version_id.to_str().expect("err").to_string()
} else {
"".to_string()
},
size: size, size: size,
expiration: exp_time, expiration: exp_time,
expiration_rule_id: rule_id, expiration_rule_id: rule_id,
checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) { h_checksum_crc32.to_str().expect("err").to_string() } else { "".to_string() }, checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) {
checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) { h_checksum_crc32c.to_str().expect("err").to_string() } else { "".to_string() }, h_checksum_crc32.to_str().expect("err").to_string()
checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) { h_checksum_sha1.to_str().expect("err").to_string() } else { "".to_string() }, } else {
checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) { h_checksum_sha256.to_str().expect("err").to_string() } else { "".to_string() }, "".to_string()
checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) { h_checksum_crc64nvme.to_str().expect("err").to_string() } else { "".to_string() }, },
checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) {
h_checksum_crc32c.to_str().expect("err").to_string()
} else {
"".to_string()
},
checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) {
h_checksum_sha1.to_str().expect("err").to_string()
} else {
"".to_string()
},
checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) {
h_checksum_sha256.to_str().expect("err").to_string()
} else {
"".to_string()
},
checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) {
h_checksum_crc64nvme.to_str().expect("err").to_string()
} else {
"".to_string()
},
..Default::default() ..Default::default()
}) })
} }
+124 -39
View File
@@ -1,26 +1,24 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use bytes::Bytes;
use http::{HeaderMap, HeaderValue, Method, StatusCode};
use s3s::S3ErrorCode;
use s3s::dto::ReplicationStatus;
use s3s::header::X_AMZ_BYPASS_GOVERNANCE_RETENTION;
use std::fmt::Display; use std::fmt::Display;
use std::{collections::HashMap, sync::Arc}; use std::{collections::HashMap, sync::Arc};
use bytes::Bytes;
use s3s::header::X_AMZ_BYPASS_GOVERNANCE_RETENTION;
use s3s::S3ErrorCode;
use time::OffsetDateTime; use time::OffsetDateTime;
use s3s::dto::ReplicationStatus;
use tokio::sync::mpsc::{self, Receiver, Sender}; use tokio::sync::mpsc::{self, Receiver, Sender};
use http::{HeaderMap, HeaderValue, Method, StatusCode};
use reader::hasher::{sum_sha256_hex, sum_md5_base64}; use crate::client::{
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; api_error_response::{ErrorResponse, http_resp_to_error_response, to_error_response},
transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
};
use crate::{ use crate::{
disk::DiskAPI, disk::DiskAPI,
store_api::{ store_api::{GetObjectReader, ObjectInfo, StorageAPI},
GetObjectReader, ObjectInfo, StorageAPI,
},
};
use crate::client::{
transition_api::{TransitionClient, RequestMetadata, ReaderImpl},
api_error_response::{http_resp_to_error_response, to_error_response, ErrorResponse,},
}; };
use reader::hasher::{sum_md5_base64, sum_sha256_hex};
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
struct RemoveBucketOptions { struct RemoveBucketOptions {
forced_elete: bool, forced_elete: bool,
@@ -62,7 +60,10 @@ impl TransitionClient {
headers.insert(rustFSForceDelete, "true"); headers.insert(rustFSForceDelete, "true");
}*/ }*/
let resp = self.execute_method(Method::DELETE, &mut RequestMetadata { let resp = self
.execute_method(
Method::DELETE,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
custom_header: headers, custom_header: headers,
@@ -78,7 +79,9 @@ impl TransitionClient {
extra_pre_sign_header: Default::default(), extra_pre_sign_header: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
expires: Default::default(), expires: Default::default(),
}).await?; },
)
.await?;
{ {
let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap(); let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
@@ -88,7 +91,10 @@ impl TransitionClient {
} }
pub async fn remove_bucket(&self, bucket_name: &str) -> Result<(), std::io::Error> { pub async fn remove_bucket(&self, bucket_name: &str) -> Result<(), std::io::Error> {
let resp = self.execute_method(http::Method::DELETE, &mut RequestMetadata { let resp = self
.execute_method(
http::Method::DELETE,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
custom_header: Default::default(), custom_header: Default::default(),
@@ -104,7 +110,9 @@ impl TransitionClient {
extra_pre_sign_header: Default::default(), extra_pre_sign_header: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
expires: Default::default(), expires: Default::default(),
}).await?; },
)
.await?;
{ {
let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap(); let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
@@ -119,7 +127,12 @@ impl TransitionClient {
res.err res.err
} }
pub async fn remove_object_inner(&self, bucket_name: &str, object_name: &str, opts: RemoveObjectOptions) -> Result<RemoveObjectResult, std::io::Error> { pub async fn remove_object_inner(
&self,
bucket_name: &str,
object_name: &str,
opts: RemoveObjectOptions,
) -> Result<RemoveObjectResult, std::io::Error> {
let mut url_values = HashMap::new(); let mut url_values = HashMap::new();
if opts.version_id != "" { if opts.version_id != "" {
@@ -132,7 +145,10 @@ impl TransitionClient {
headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err")); //amzBypassGovernance headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err")); //amzBypassGovernance
} }
let resp = self.execute_method(http::Method::DELETE, &mut RequestMetadata { let resp = self
.execute_method(
http::Method::DELETE,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(), object_name: object_name.to_string(),
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
@@ -148,30 +164,50 @@ impl TransitionClient {
extra_pre_sign_header: Default::default(), extra_pre_sign_header: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
expires: Default::default(), expires: Default::default(),
}).await?; },
)
.await?;
Ok(RemoveObjectResult { Ok(RemoveObjectResult {
object_name: object_name.to_string(), object_name: object_name.to_string(),
object_version_id: opts.version_id, object_version_id: opts.version_id,
delete_marker: resp.headers().get("x-amz-delete-marker").expect("err") == "true", delete_marker: resp.headers().get("x-amz-delete-marker").expect("err") == "true",
delete_marker_version_id: resp.headers().get("x-amz-version-id").expect("err").to_str().expect("err").to_string(), delete_marker_version_id: resp
.headers()
.get("x-amz-version-id")
.expect("err")
.to_str()
.expect("err")
.to_string(),
..Default::default() ..Default::default()
}) })
} }
pub async fn remove_objects_with_result(self: Arc<Self>, bucket_name: &str, objects_rx: Receiver<ObjectInfo>, opts: RemoveObjectsOptions) -> Receiver<RemoveObjectResult> { pub async fn remove_objects_with_result(
self: Arc<Self>,
bucket_name: &str,
objects_rx: Receiver<ObjectInfo>,
opts: RemoveObjectsOptions,
) -> Receiver<RemoveObjectResult> {
let (result_tx, mut result_rx) = mpsc::channel(1); let (result_tx, mut result_rx) = mpsc::channel(1);
let self_clone = Arc::clone(&self); let self_clone = Arc::clone(&self);
let bucket_name_owned = bucket_name.to_string(); let bucket_name_owned = bucket_name.to_string();
tokio::spawn(async move { tokio::spawn(async move {
self_clone.remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts).await; self_clone
.remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts)
.await;
}); });
result_rx result_rx
} }
pub async fn remove_objects(self: Arc<Self>, bucket_name: &str, objects_rx: Receiver<ObjectInfo>, opts: RemoveObjectsOptions) -> Receiver<RemoveObjectError> { pub async fn remove_objects(
self: Arc<Self>,
bucket_name: &str,
objects_rx: Receiver<ObjectInfo>,
opts: RemoveObjectsOptions,
) -> Receiver<RemoveObjectError> {
let (error_tx, mut error_rx) = mpsc::channel(1); let (error_tx, mut error_rx) = mpsc::channel(1);
let self_clone = Arc::clone(&self); let self_clone = Arc::clone(&self);
@@ -179,26 +215,36 @@ impl TransitionClient {
let (result_tx, mut result_rx) = mpsc::channel(1); let (result_tx, mut result_rx) = mpsc::channel(1);
tokio::spawn(async move { tokio::spawn(async move {
self_clone.remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts).await; self_clone
.remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts)
.await;
}); });
tokio::spawn(async move { tokio::spawn(async move {
while let Some(res) = result_rx.recv().await { while let Some(res) = result_rx.recv().await {
if res.err.is_none() { if res.err.is_none() {
continue; continue;
} }
error_tx.send(RemoveObjectError { error_tx
.send(RemoveObjectError {
object_name: res.object_name, object_name: res.object_name,
version_id: res.object_version_id, version_id: res.object_version_id,
err: res.err, err: res.err,
..Default::default() ..Default::default()
}).await; })
.await;
} }
}); });
error_rx error_rx
} }
pub async fn remove_objects_inner(&self, bucket_name: &str, mut objects_rx: Receiver<ObjectInfo>, result_tx: &Sender<RemoveObjectResult>, opts: RemoveObjectsOptions) -> Result<(), std::io::Error> { pub async fn remove_objects_inner(
&self,
bucket_name: &str,
mut objects_rx: Receiver<ObjectInfo>,
result_tx: &Sender<RemoveObjectResult>,
opts: RemoveObjectsOptions,
) -> Result<(), std::io::Error> {
let max_entries = 1000; let max_entries = 1000;
let mut finish = false; let mut finish = false;
let mut url_values = HashMap::new(); let mut url_values = HashMap::new();
@@ -213,11 +259,17 @@ impl TransitionClient {
while let Some(object) = objects_rx.recv().await { while let Some(object) = objects_rx.recv().await {
if has_invalid_xml_char(&object.name) { if has_invalid_xml_char(&object.name) {
let remove_result = self.remove_object_inner(bucket_name, &object.name, RemoveObjectOptions { let remove_result = self
.remove_object_inner(
bucket_name,
&object.name,
RemoveObjectOptions {
version_id: object.version_id.expect("err").to_string(), version_id: object.version_id.expect("err").to_string(),
governance_bypass: opts.governance_bypass, governance_bypass: opts.governance_bypass,
..Default::default() ..Default::default()
}).await?; },
)
.await?;
let remove_result_clone = remove_result.clone(); let remove_result_clone = remove_result.clone();
if !remove_result.err.is_none() { if !remove_result.err.is_none() {
match to_error_response(&remove_result.err.expect("err")).code { match to_error_response(&remove_result.err.expect("err")).code {
@@ -252,7 +304,10 @@ impl TransitionClient {
} }
let remove_bytes = generate_remove_multi_objects_request(&batch); let remove_bytes = generate_remove_multi_objects_request(&batch);
let resp = self.execute_method(http::Method::POST, &mut RequestMetadata { let resp = self
.execute_method(
http::Method::POST,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
query_values: url_values.clone(), query_values: url_values.clone(),
content_body: ReaderImpl::Body(Bytes::from(remove_bytes.clone())), content_body: ReaderImpl::Body(Bytes::from(remove_bytes.clone())),
@@ -268,7 +323,9 @@ impl TransitionClient {
extra_pre_sign_header: Default::default(), extra_pre_sign_header: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
expires: Default::default(), expires: Default::default(),
}).await?; },
)
.await?;
let body_bytes: Vec<u8> = resp.body().bytes().expect("err").to_vec(); let body_bytes: Vec<u8> = resp.body().bytes().expect("err").to_vec();
process_remove_multi_objects_response(ReaderImpl::Body(Bytes::from(body_bytes)), result_tx.clone()); process_remove_multi_objects_response(ReaderImpl::Body(Bytes::from(body_bytes)), result_tx.clone());
@@ -285,11 +342,19 @@ impl TransitionClient {
Ok(()) Ok(())
} }
pub async fn abort_multipart_upload(&self, bucket_name: &str, object_name: &str, upload_id: &str) -> Result<(), std::io::Error> { pub async fn abort_multipart_upload(
&self,
bucket_name: &str,
object_name: &str,
upload_id: &str,
) -> Result<(), std::io::Error> {
let mut url_values = HashMap::new(); let mut url_values = HashMap::new();
url_values.insert("uploadId".to_string(), upload_id.to_string()); url_values.insert("uploadId".to_string(), upload_id.to_string());
let resp = self.execute_method(http::Method::DELETE, &mut RequestMetadata { let resp = self
.execute_method(
http::Method::DELETE,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(), object_name: object_name.to_string(),
query_values: url_values, query_values: url_values,
@@ -305,7 +370,9 @@ impl TransitionClient {
extra_pre_sign_header: Default::default(), extra_pre_sign_header: Default::default(),
bucket_location: Default::default(), bucket_location: Default::default(),
expires: Default::default(), expires: Default::default(),
}).await?; },
)
.await?;
//if resp.is_some() { //if resp.is_some() {
if resp.status() != StatusCode::NO_CONTENT { if resp.status() != StatusCode::NO_CONTENT {
let error_response: ErrorResponse; let error_response: ErrorResponse;
@@ -316,9 +383,27 @@ impl TransitionClient {
message: "The specified multipart upload does not exist.".to_string(), message: "The specified multipart upload does not exist.".to_string(),
bucket_name: bucket_name.to_string(), bucket_name: bucket_name.to_string(),
key: object_name.to_string(), key: object_name.to_string(),
request_id: resp.headers().get("x-amz-request-id").expect("err").to_str().expect("err").to_string(), request_id: resp
host_id: resp.headers().get("x-amz-id-2").expect("err").to_str().expect("err").to_string(), .headers()
region: resp.headers().get("x-amz-bucket-region").expect("err").to_str().expect("err").to_string(), .get("x-amz-request-id")
.expect("err")
.to_str()
.expect("err")
.to_string(),
host_id: resp
.headers()
.get("x-amz-id-2")
.expect("err")
.to_str()
.expect("err")
.to_string(),
region: resp
.headers()
.get("x-amz-bucket-region")
.expect("err")
.to_str()
.expect("err")
.to_string(),
..Default::default() ..Default::default()
}; };
} }
+11 -11
View File
@@ -1,12 +1,12 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use std::collections::HashMap;
use s3s::dto::Owner; use s3s::dto::Owner;
use serde::{Serialize, Deserialize}; use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use time::OffsetDateTime; use time::OffsetDateTime;
use rustfs_utils::crypto::base64_decode;
use crate::checksum::ChecksumMode; use crate::checksum::ChecksumMode;
use crate::client::transition_api::ObjectMultipartInfo; use crate::client::transition_api::ObjectMultipartInfo;
use rustfs_utils::crypto::base64_decode;
use super::transition_api; use super::transition_api;
@@ -136,7 +136,6 @@ impl Default for ObjectPart {
checksum_crc64nvme: Default::default(), checksum_crc64nvme: Default::default(),
} }
} }
} }
impl ObjectPart { impl ObjectPart {
@@ -216,9 +215,9 @@ pub struct CompleteMultipartUploadResult {
pub checksum_crc64nvme: String, pub checksum_crc64nvme: String,
} }
#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] #[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, serde::Serialize)]
#[derive(serde::Serialize)] pub struct CompletePart {
pub struct CompletePart { //api has //api has
pub etag: String, pub etag: String,
pub part_num: i64, pub part_num: i64,
pub checksum_crc32: String, pub checksum_crc32: String,
@@ -258,8 +257,7 @@ pub struct CopyObjectPartResult {
pub last_modified: OffsetDateTime, pub last_modified: OffsetDateTime,
} }
#[derive(Debug, Default)] #[derive(Debug, Default, serde::Serialize)]
#[derive(serde::Serialize)]
pub struct CompleteMultipartUpload { pub struct CompleteMultipartUpload {
pub parts: Vec<CompletePart>, pub parts: Vec<CompletePart>,
} }
@@ -287,12 +285,14 @@ pub struct CreateBucketConfiguration {
} }
#[derive(serde::Serialize)] #[derive(serde::Serialize)]
pub struct DeleteObject { //api has pub struct DeleteObject {
//api has
pub key: String, pub key: String,
pub version_id: String, pub version_id: String,
} }
pub struct DeletedObject { //s3s has pub struct DeletedObject {
//s3s has
pub key: String, pub key: String,
pub version_id: String, pub version_id: String,
pub deletemarker: bool, pub deletemarker: bool,
+13 -16
View File
@@ -1,20 +1,20 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use http::Request; use http::Request;
use hyper::StatusCode;
use hyper::body::Incoming; use hyper::body::Incoming;
use std::{collections::HashMap, sync::Arc}; use std::{collections::HashMap, sync::Arc};
use tracing::warn; use tracing::warn;
use tracing::{error, info, debug}; use tracing::{debug, error, info};
use hyper::StatusCode;
use reader::hasher::{Hasher, Sha256};
use s3s::S3ErrorCode;
use s3s::Body;
use crate::client::{ use crate::client::{
api_error_response::{http_resp_to_error_response, to_error_response}, api_error_response::{http_resp_to_error_response, to_error_response},
transition_api::{TransitionClient, Document}, transition_api::{Document, TransitionClient},
}; };
use crate::signer; use crate::signer;
use reader::hasher::{Hasher, Sha256};
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
use s3s::Body;
use s3s::S3ErrorCode;
use super::constants::UNSIGNED_PAYLOAD; use super::constants::UNSIGNED_PAYLOAD;
use super::credentials::SignatureType; use super::credentials::SignatureType;
@@ -25,9 +25,7 @@ pub struct BucketLocationCache {
impl BucketLocationCache { impl BucketLocationCache {
pub fn new() -> BucketLocationCache { pub fn new() -> BucketLocationCache {
BucketLocationCache{ BucketLocationCache { items: HashMap::new() }
items: HashMap::new(),
}
} }
pub fn get(&self, bucket_name: &str) -> Option<String> { pub fn get(&self, bucket_name: &str) -> Option<String> {
@@ -50,7 +48,7 @@ impl TransitionClient {
async fn get_bucket_location_inner(&self, bucket_name: &str) -> Result<String, std::io::Error> { async fn get_bucket_location_inner(&self, bucket_name: &str) -> Result<String, std::io::Error> {
if self.region != "" { if self.region != "" {
return Ok(self.region.clone()) return Ok(self.region.clone());
} }
let mut location; let mut location;
@@ -81,11 +79,7 @@ impl TransitionClient {
let mut target_url = self.endpoint_url.clone(); let mut target_url = self.endpoint_url.clone();
let scheme = self.endpoint_url.scheme(); let scheme = self.endpoint_url.scheme();
let h = target_url.host().expect("host is none."); let h = target_url.host().expect("host is none.");
let default_port = if scheme == "https" { let default_port = if scheme == "https" { 443 } else { 80 };
443
} else {
80
};
let p = target_url.port().unwrap_or(default_port); let p = target_url.port().unwrap_or(default_port);
let is_virtual_style = self.is_virtual_host_style_request(&target_url, bucket_name); let is_virtual_style = self.is_virtual_host_style_request(&target_url, bucket_name);
@@ -164,7 +158,10 @@ impl TransitionClient {
content_sha256 = UNSIGNED_PAYLOAD.to_string(); content_sha256 = UNSIGNED_PAYLOAD.to_string();
} }
req_builder.headers_mut().expect("err").insert("X-Amz-Content-Sha256", content_sha256.parse().unwrap()); req_builder
.headers_mut()
.expect("err")
.insert("X-Amz-Content-Sha256", content_sha256.parse().unwrap());
let req_builder = signer::sign_v4(req_builder, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1"); let req_builder = signer::sign_v4(req_builder, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1");
let req = match req_builder.body(Body::empty()) { let req = match req_builder.body(Body::empty()) {
Ok(req) => return Ok(req), Ok(req) => return Ok(req),
+4 -3
View File
@@ -1,7 +1,7 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use std::{collections::HashMap, sync::Arc};
use time::{macros::format_description, format_description::FormatItem};
use lazy_static::lazy_static; use lazy_static::lazy_static;
use std::{collections::HashMap, sync::Arc};
use time::{format_description::FormatItem, macros::format_description};
pub const ABS_MIN_PART_SIZE: i64 = 1024 * 1024 * 5; pub const ABS_MIN_PART_SIZE: i64 = 1024 * 1024 * 5;
pub const MAX_PARTS_COUNT: i64 = 10000; pub const MAX_PARTS_COUNT: i64 = 10000;
@@ -17,7 +17,8 @@ pub const UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER";
pub const TOTAL_WORKERS: i64 = 4; pub const TOTAL_WORKERS: i64 = 4;
pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256"; pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] = format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z"); pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] =
format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z");
const GetObjectAttributesTags: &str = "ETag,Checksum,StorageClass,ObjectSize,ObjectParts"; const GetObjectAttributesTags: &str = "ETag,Checksum,StorageClass,ObjectSize,ObjectParts";
const GetObjectAttributesMaxParts: i64 = 1000; const GetObjectAttributesMaxParts: i64 = 1000;
+1 -2
View File
@@ -19,8 +19,7 @@ pub struct Credentials<P: Provider + Default> {
provider: P, provider: P,
} }
impl<P: Provider + Default> Credentials<P> impl<P: Provider + Default> Credentials<P> {
{
pub fn new(provider: P) -> Self { pub fn new(provider: P) -> Self {
Self { Self {
provider: provider, provider: provider,
+11 -11
View File
@@ -1,18 +1,18 @@
pub mod constants; pub mod admin_handler_utils;
pub mod transition_api;
pub mod api_list;
pub mod api_error_response;
pub mod api_s3_datatypes;
pub mod api_bucket_policy; pub mod api_bucket_policy;
pub mod api_put_object_common; pub mod api_error_response;
pub mod api_get_options;
pub mod api_get_object; pub mod api_get_object;
pub mod api_get_options;
pub mod api_list;
pub mod api_put_object; pub mod api_put_object;
pub mod api_put_object_streaming; pub mod api_put_object_common;
pub mod api_put_object_multipart; pub mod api_put_object_multipart;
pub mod api_put_object_streaming;
pub mod api_remove; pub mod api_remove;
pub mod api_s3_datatypes;
pub mod bucket_cache;
pub mod constants;
pub mod credentials;
pub mod object_api_utils; pub mod object_api_utils;
pub mod object_handlers_common; pub mod object_handlers_common;
pub mod admin_handler_utils; pub mod transition_api;
pub mod credentials;
pub mod bucket_cache;
+24 -13
View File
@@ -1,17 +1,14 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use std::{collections::HashMap, sync::Arc};
use http::HeaderMap; use http::HeaderMap;
use tokio::io::BufReader;
use std::io::Cursor; use std::io::Cursor;
use std::{collections::HashMap, sync::Arc};
use tokio::io::BufReader;
use s3s::S3ErrorCode; use crate::error::ErrorResponse;
use crate::store_api::{ use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions};
GetObjectReader, HTTPRangeSpec,
ObjectInfo, ObjectOptions,
};
use rustfs_filemeta::fileinfo::ObjectPartInfo; use rustfs_filemeta::fileinfo::ObjectPartInfo;
use rustfs_rio::HashReader; use rustfs_rio::HashReader;
use crate::error::ErrorResponse; use s3s::S3ErrorCode;
//#[derive(Clone)] //#[derive(Clone)]
pub struct PutObjReader { pub struct PutObjReader {
@@ -57,7 +54,11 @@ fn part_number_to_rangespec(oi: ObjectInfo, part_number: usize) -> Option<HTTPRa
i += 1; i += 1;
} }
Some(HTTPRangeSpec {start: start as usize, end: Some(end as usize), is_suffix_length: false}) Some(HTTPRangeSpec {
start: start as usize,
end: Some(end as usize),
is_suffix_length: false,
})
} }
fn get_compressed_offsets(oi: ObjectInfo, offset: i64) -> (i64, i64, i64, i64, u64) { fn get_compressed_offsets(oi: ObjectInfo, offset: i64) -> (i64, i64, i64, i64, u64) {
@@ -81,14 +82,22 @@ fn get_compressed_offsets(oi: ObjectInfo, offset: i64) -> (i64, i64, i64, i64, u
skip_length = offset - skip_length; skip_length = offset - skip_length;
let parts: &[ObjectPartInfo] = &oi.parts; let parts: &[ObjectPartInfo] = &oi.parts;
if skip_length > 0 && parts.len() > first_part_idx as usize && parts[first_part_idx as usize].index.as_ref().expect("err").len() > 0 { if skip_length > 0
&& parts.len() > first_part_idx as usize
&& parts[first_part_idx as usize].index.as_ref().expect("err").len() > 0
{
todo!(); todo!();
} }
(compressed_offset, part_skip, first_part_idx, decrypt_skip, seq_num) (compressed_offset, part_skip, first_part_idx, decrypt_skip, seq_num)
} }
pub fn new_getobjectreader(rs: HTTPRangeSpec, oi: &ObjectInfo, opts: &ObjectOptions, h: &HeaderMap) -> Result<(ObjReaderFn, i64, i64), ErrorResponse> { pub fn new_getobjectreader(
rs: HTTPRangeSpec,
oi: &ObjectInfo,
opts: &ObjectOptions,
h: &HeaderMap,
) -> Result<(ObjReaderFn, i64, i64), ErrorResponse> {
//let (_, mut is_encrypted) = crypto.is_encrypted(oi.user_defined)?; //let (_, mut is_encrypted) = crypto.is_encrypted(oi.user_defined)?;
let mut is_encrypted = false; let mut is_encrypted = false;
let is_compressed = false; //oi.is_compressed_ok(); let is_compressed = false; //oi.is_compressed_ok();
@@ -97,7 +106,8 @@ pub fn new_getobjectreader(rs: HTTPRangeSpec, oi: &ObjectInfo, opts: &ObjectOpti
let (off, length) = match rs.get_offset_length(oi.size) { let (off, length) = match rs.get_offset_length(oi.size) {
Ok(x) => x, Ok(x) => x,
Err(err) => return Err(ErrorResponse { Err(err) => {
return Err(ErrorResponse {
code: S3ErrorCode::InvalidRange, code: S3ErrorCode::InvalidRange,
message: err.to_string(), message: err.to_string(),
key: None, key: None,
@@ -105,7 +115,8 @@ pub fn new_getobjectreader(rs: HTTPRangeSpec, oi: &ObjectInfo, opts: &ObjectOpti
region: None, region: None,
request_id: None, request_id: None,
host_id: "".to_string(), host_id: "".to_string(),
}), });
}
}; };
get_fn = Arc::new(move |input_reader: BufReader<Cursor<Vec<u8>>>, _: HeaderMap| { get_fn = Arc::new(move |input_reader: BufReader<Cursor<Vec<u8>>>, _: HeaderMap| {
//Box::pin({ //Box::pin({
+13 -7
View File
@@ -1,15 +1,17 @@
use lock::local_locker::MAX_DELETE_LIST; use crate::StorageAPI;
use crate::bucket::lifecycle::lifecycle;
use crate::bucket::versioning::VersioningApi; use crate::bucket::versioning::VersioningApi;
use crate::bucket::versioning_sys::BucketVersioningSys; use crate::bucket::versioning_sys::BucketVersioningSys;
use crate::store::ECStore; use crate::store::ECStore;
use crate::store_api::{ObjectOptions, ObjectToDelete,}; use crate::store_api::{ObjectOptions, ObjectToDelete};
use crate::StorageAPI; use lock::local_locker::MAX_DELETE_LIST;
use crate::bucket::lifecycle::lifecycle;
pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], lc_event: lifecycle::Event) { pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], lc_event: lifecycle::Event) {
let mut remaining = to_del; let mut remaining = to_del;
loop { loop {
if remaining.len() <= 0 {break}; if remaining.len() <= 0 {
break;
};
let mut to_del = remaining; let mut to_del = remaining;
if to_del.len() > MAX_DELETE_LIST { if to_del.len() > MAX_DELETE_LIST {
remaining = &to_del[MAX_DELETE_LIST..]; remaining = &to_del[MAX_DELETE_LIST..];
@@ -18,10 +20,14 @@ pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[Object
remaining = &[]; remaining = &[];
} }
let vc = BucketVersioningSys::get(bucket).await.expect("err!"); let vc = BucketVersioningSys::get(bucket).await.expect("err!");
let deleted_objs = api.delete_objects(bucket, to_del.to_vec(), ObjectOptions { let deleted_objs = api.delete_objects(
bucket,
to_del.to_vec(),
ObjectOptions {
//prefix_enabled_fn: vc.prefix_enabled(""), //prefix_enabled_fn: vc.prefix_enabled(""),
version_suspended: vc.suspended(), version_suspended: vc.suspended(),
..Default::default() ..Default::default()
}); },
);
} }
} }
+225 -88
View File
@@ -1,57 +1,59 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use std::pin::Pin;
use bytes::Bytes; use bytes::Bytes;
use futures::Future; use futures::Future;
use http::{HeaderMap, HeaderName}; use http::{HeaderMap, HeaderName};
use serde::{Serialize, Deserialize}; use http::{
use uuid::Uuid; HeaderValue, Response, StatusCode,
request::{Builder, Request},
};
use hyper_rustls::{ConfigBuilderExt, HttpsConnector};
use hyper_util::{client::legacy::Client, client::legacy::connect::HttpConnector, rt::TokioExecutor};
use rand::Rng; use rand::Rng;
use std::{collections::HashMap, sync::{Arc, Mutex}}; use serde::{Deserialize, Serialize};
use std::io::Cursor;
use std::pin::Pin;
use std::sync::atomic::{AtomicI32, Ordering}; use std::sync::atomic::{AtomicI32, Ordering};
use std::task::{Context, Poll}; use std::task::{Context, Poll};
use std::{
collections::HashMap,
sync::{Arc, Mutex},
};
use time::Duration; use time::Duration;
use time::OffsetDateTime; use time::OffsetDateTime;
use hyper_rustls::{ConfigBuilderExt, HttpsConnector};
use hyper_util::{client::legacy::Client, rt::TokioExecutor, client::legacy::connect::HttpConnector};
use http::{StatusCode, HeaderValue, request::{Request, Builder}, Response};
use tracing::{error, debug};
use url::{form_urlencoded, Url};
use tokio::io::BufReader; use tokio::io::BufReader;
use std::io::Cursor; use tracing::{debug, error};
use url::{Url, form_urlencoded};
use uuid::Uuid;
use s3s::S3ErrorCode;
use s3s::{dto::Owner, Body};
use s3s::dto::ReplicationStatus;
use crate::client::bucket_cache::BucketLocationCache; use crate::client::bucket_cache::BucketLocationCache;
use reader::hasher::{Sha256, MD5,}; use crate::client::{
api_error_response::{err_invalid_argument, http_resp_to_error_response, to_error_response},
api_get_options::GetObjectOptions,
api_put_object::PutObjectOptions,
api_put_object_multipart::UploadPartParams,
api_s3_datatypes::{
CompleteMultipartUpload, CompletePart, ListBucketResult, ListBucketV2Result, ListMultipartUploadsResult,
ListObjectPartsResult, ObjectPart,
},
constants::{UNSIGNED_PAYLOAD, UNSIGNED_PAYLOAD_TRAILER},
credentials::{CredContext, Credentials, SignatureType, Static},
};
use crate::signer;
use crate::{checksum::ChecksumMode, store_api::GetObjectReader};
use reader::hasher::{MD5, Sha256};
use rustfs_rio::HashReader; use rustfs_rio::HashReader;
use rustfs_utils::{ use rustfs_utils::{
net::get_endpoint_url, net::get_endpoint_url,
retry::{new_retry_timer, MAX_RETRY}, retry::{MAX_RETRY, new_retry_timer},
};
use crate::{
store_api::GetObjectReader,
checksum::ChecksumMode,
};
use crate::signer;
use crate::client::{
constants::{UNSIGNED_PAYLOAD, UNSIGNED_PAYLOAD_TRAILER},
credentials::{Credentials, SignatureType, CredContext, Static,},
api_error_response::{to_error_response, http_resp_to_error_response, err_invalid_argument},
api_put_object_multipart::UploadPartParams,
api_put_object::PutObjectOptions,
api_get_options::GetObjectOptions,
api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ListBucketResult, ListBucketV2Result, ListMultipartUploadsResult, ListObjectPartsResult, ObjectPart},
}; };
use s3s::S3ErrorCode;
use s3s::dto::ReplicationStatus;
use s3s::{Body, dto::Owner};
const C_USER_AGENT_PREFIX: &str = "RustFS (linux; x86)"; const C_USER_AGENT_PREFIX: &str = "RustFS (linux; x86)";
const C_USER_AGENT: &str = "RustFS (linux; x86)"; const C_USER_AGENT: &str = "RustFS (linux; x86)";
const SUCCESS_STATUS: [StatusCode; 3] = [ const SUCCESS_STATUS: [StatusCode; 3] = [StatusCode::OK, StatusCode::NO_CONTENT, StatusCode::PARTIAL_CONTENT];
StatusCode::OK,
StatusCode::NO_CONTENT,
StatusCode::PARTIAL_CONTENT,
];
const C_UNKNOWN: i32 = -1; const C_UNKNOWN: i32 = -1;
const C_OFFLINE: i32 = 0; const C_OFFLINE: i32 = 0;
@@ -136,9 +138,7 @@ impl TransitionClient {
//if scheme == "https" { //if scheme == "https" {
// client = Client::builder(TokioExecutor::new()).build_http(); // client = Client::builder(TokioExecutor::new()).build_http();
//} else { //} else {
let tls = rustls::ClientConfig::builder() let tls = rustls::ClientConfig::builder().with_native_roots()?.with_no_client_auth();
.with_native_roots()?
.with_no_client_auth();
let https = hyper_rustls::HttpsConnectorBuilder::new() let https = hyper_rustls::HttpsConnectorBuilder::new()
.with_tls_config(tls) .with_tls_config(tls)
.https_or_http() .https_or_http()
@@ -218,7 +218,11 @@ impl TransitionClient {
todo!(); todo!();
} }
pub fn hash_materials(&self, is_md5_requested: bool, is_sha256_requested: bool) -> (HashMap<String, MD5>, HashMap<String, Vec<u8>>) { pub fn hash_materials(
&self,
is_md5_requested: bool,
is_sha256_requested: bool,
) -> (HashMap<String, MD5>, HashMap<String, Vec<u8>>) {
todo!(); todo!();
} }
@@ -227,7 +231,8 @@ impl TransitionClient {
} }
fn mark_offline(&self) { fn mark_offline(&self) {
self.health_status.compare_exchange(C_ONLINE, C_OFFLINE, Ordering::SeqCst, Ordering::SeqCst); self.health_status
.compare_exchange(C_ONLINE, C_OFFLINE, Ordering::SeqCst, Ordering::SeqCst);
} }
fn is_offline(&self) -> bool { fn is_offline(&self) -> bool {
@@ -261,7 +266,9 @@ impl TransitionClient {
debug!("endpoint_url: {}", self.endpoint_url.as_str().to_string()); debug!("endpoint_url: {}", self.endpoint_url.as_str().to_string());
resp = http_client.request(req); resp = http_client.request(req);
} }
let resp = resp.await/*.map_err(Into::into)*/.map(|res| res.map(Body::from)); let resp = resp
.await /*.map_err(Into::into)*/
.map(|res| res.map(Body::from));
debug!("http_client url: {} {}", req_method, req_uri); debug!("http_client url: {} {}", req_method, req_uri);
debug!("http_client headers: {:?}", req_headers); debug!("http_client headers: {:?}", req_headers);
if let Err(err) = resp { if let Err(err) = resp {
@@ -282,7 +289,11 @@ impl TransitionClient {
Ok(resp) Ok(resp)
} }
pub async fn execute_method(&self, method: http::Method, metadata: &mut RequestMetadata) -> Result<http::Response<Body>, std::io::Error> { pub async fn execute_method(
&self,
method: http::Method,
metadata: &mut RequestMetadata,
) -> Result<http::Response<Body>, std::io::Error> {
if self.is_offline() { if self.is_offline() {
let mut s = self.endpoint_url.to_string(); let mut s = self.endpoint_url.to_string();
s.push_str(" is offline."); s.push_str(" is offline.");
@@ -304,7 +315,9 @@ impl TransitionClient {
//let mut retry_timer = RetryTimer::new(); //let mut retry_timer = RetryTimer::new();
//while let Some(v) = retry_timer.next().await { //while let Some(v) = retry_timer.next().await {
for _ in [1;1]/*new_retry_timer(req_retry, DefaultRetryUnit, DefaultRetryCap, MaxJitter)*/ { for _ in [1; 1]
/*new_retry_timer(req_retry, DefaultRetryUnit, DefaultRetryCap, MaxJitter)*/
{
let req = self.new_request(method, metadata).await?; let req = self.new_request(method, metadata).await?;
resp = self.doit(req).await?; resp = self.doit(req).await?;
@@ -355,7 +368,11 @@ impl TransitionClient {
Err(std::io::Error::other("resp err")) Err(std::io::Error::other("resp err"))
} }
async fn new_request(&self, method: http::Method, metadata: &mut RequestMetadata) -> Result<http::Request<Body>, std::io::Error> { async fn new_request(
&self,
method: http::Method,
metadata: &mut RequestMetadata,
) -> Result<http::Request<Body>, std::io::Error> {
let location = metadata.bucket_location.clone(); let location = metadata.bucket_location.clone();
if location == "" { if location == "" {
if metadata.bucket_name != "" { if metadata.bucket_name != "" {
@@ -366,8 +383,13 @@ impl TransitionClient {
let is_makebucket = metadata.object_name == "" && method == http::Method::PUT && metadata.query_values.len() == 0; let is_makebucket = metadata.object_name == "" && method == http::Method::PUT && metadata.query_values.len() == 0;
let is_virtual_host = self.is_virtual_host_style_request(&self.endpoint_url, &metadata.bucket_name) && !is_makebucket; let is_virtual_host = self.is_virtual_host_style_request(&self.endpoint_url, &metadata.bucket_name) && !is_makebucket;
let target_url = self.make_target_url(&metadata.bucket_name, &metadata.object_name, &location, let target_url = self.make_target_url(
is_virtual_host, &metadata.query_values)?; &metadata.bucket_name,
&metadata.object_name,
&location,
is_virtual_host,
&metadata.query_values,
)?;
let mut req_builder = Request::builder().method(method).uri(target_url.to_string()); let mut req_builder = Request::builder().method(method).uri(target_url.to_string());
@@ -392,24 +414,39 @@ impl TransitionClient {
if metadata.expires != 0 && metadata.pre_sign_url { if metadata.expires != 0 && metadata.pre_sign_url {
if signer_type == SignatureType::SignatureAnonymous { if signer_type == SignatureType::SignatureAnonymous {
return Err(std::io::Error::other(err_invalid_argument("Presigned URLs cannot be generated with anonymous credentials."))); return Err(std::io::Error::other(err_invalid_argument(
"Presigned URLs cannot be generated with anonymous credentials.",
)));
} }
if metadata.extra_pre_sign_header.is_some() { if metadata.extra_pre_sign_header.is_some() {
if signer_type == SignatureType::SignatureV2 { if signer_type == SignatureType::SignatureV2 {
return Err(std::io::Error::other(err_invalid_argument("Extra signed headers for Presign with Signature V2 is not supported."))); return Err(std::io::Error::other(err_invalid_argument(
"Extra signed headers for Presign with Signature V2 is not supported.",
)));
} }
for (k, v) in metadata.extra_pre_sign_header.as_ref().unwrap() { for (k, v) in metadata.extra_pre_sign_header.as_ref().unwrap() {
req_builder = req_builder.header(k, v); req_builder = req_builder.header(k, v);
} }
} }
if signer_type == SignatureType::SignatureV2 { if signer_type == SignatureType::SignatureV2 {
req_builder = signer::pre_sign_v2(req_builder, &access_key_id, &secret_access_key, metadata.expires, is_virtual_host); req_builder =
signer::pre_sign_v2(req_builder, &access_key_id, &secret_access_key, metadata.expires, is_virtual_host);
} else if signer_type == SignatureType::SignatureV4 { } else if signer_type == SignatureType::SignatureV4 {
req_builder = signer::pre_sign_v4(req_builder, &access_key_id, &secret_access_key, &session_token, &location, metadata.expires, OffsetDateTime::now_utc()); req_builder = signer::pre_sign_v4(
req_builder,
&access_key_id,
&secret_access_key,
&session_token,
&location,
metadata.expires,
OffsetDateTime::now_utc(),
);
} }
let req = match req_builder.body(Body::empty()) { let req = match req_builder.body(Body::empty()) {
Ok(req) => req, Ok(req) => req,
Err(err) => { return Err(std::io::Error::other(err)); } Err(err) => {
return Err(std::io::Error::other(err));
}
}; };
return Ok(req); return Ok(req);
} }
@@ -423,7 +460,10 @@ impl TransitionClient {
//req.content_length = metadata.content_length; //req.content_length = metadata.content_length;
if metadata.content_length <= -1 { if metadata.content_length <= -1 {
let chunked_value = HeaderValue::from_str(&vec!["chunked"].join(",")).expect("err"); let chunked_value = HeaderValue::from_str(&vec!["chunked"].join(",")).expect("err");
req_builder.headers_mut().expect("err").insert(http::header::TRANSFER_ENCODING, chunked_value); req_builder
.headers_mut()
.expect("err")
.insert(http::header::TRANSFER_ENCODING, chunked_value);
} }
if metadata.content_md5_base64.len() > 0 { if metadata.content_md5_base64.len() > 0 {
@@ -434,15 +474,17 @@ impl TransitionClient {
if signer_type == SignatureType::SignatureAnonymous { if signer_type == SignatureType::SignatureAnonymous {
let req = match req_builder.body(Body::empty()) { let req = match req_builder.body(Body::empty()) {
Ok(req) => req, Ok(req) => req,
Err(err) => { return Err(std::io::Error::other(err)); } Err(err) => {
return Err(std::io::Error::other(err));
}
}; };
return Ok(req); return Ok(req);
} }
if signer_type == SignatureType::SignatureV2 { if signer_type == SignatureType::SignatureV2 {
req_builder = signer::sign_v2(req_builder, metadata.content_length, &access_key_id, &secret_access_key, is_virtual_host); req_builder =
} signer::sign_v2(req_builder, metadata.content_length, &access_key_id, &secret_access_key, is_virtual_host);
else if metadata.stream_sha256 && !self.secure { } else if metadata.stream_sha256 && !self.secure {
if metadata.trailer.len() > 0 { if metadata.trailer.len() > 0 {
//req.Trailer = metadata.trailer; //req.Trailer = metadata.trailer;
for (_, v) in &metadata.trailer { for (_, v) in &metadata.trailer {
@@ -451,8 +493,7 @@ impl TransitionClient {
} }
//req_builder = signer::streaming_sign_v4(req_builder, &access_key_id, //req_builder = signer::streaming_sign_v4(req_builder, &access_key_id,
// &secret_access_key, &session_token, &location, metadata.content_length, OffsetDateTime::now_utc(), self.sha256_hasher()); // &secret_access_key, &session_token, &location, metadata.content_length, OffsetDateTime::now_utc(), self.sha256_hasher());
} } else {
else {
let mut sha_header = UNSIGNED_PAYLOAD.to_string(); let mut sha_header = UNSIGNED_PAYLOAD.to_string();
if metadata.content_sha256_hex != "" { if metadata.content_sha256_hex != "" {
sha_header = metadata.content_sha256_hex.clone(); sha_header = metadata.content_sha256_hex.clone();
@@ -462,9 +503,17 @@ impl TransitionClient {
} else if metadata.trailer.len() > 0 { } else if metadata.trailer.len() > 0 {
sha_header = UNSIGNED_PAYLOAD_TRAILER.to_string(); sha_header = UNSIGNED_PAYLOAD_TRAILER.to_string();
} }
req_builder = req_builder.header::<HeaderName, HeaderValue>("X-Amz-Content-Sha256".parse().unwrap(), sha_header.parse().expect("err")); req_builder = req_builder
.header::<HeaderName, HeaderValue>("X-Amz-Content-Sha256".parse().unwrap(), sha_header.parse().expect("err"));
req_builder = signer::sign_v4_trailer(req_builder, &access_key_id, &secret_access_key, &session_token, &location, metadata.trailer.clone()); req_builder = signer::sign_v4_trailer(
req_builder,
&access_key_id,
&secret_access_key,
&session_token,
&location,
metadata.trailer.clone(),
);
} }
let req; let req;
@@ -484,9 +533,7 @@ impl TransitionClient {
match req { match req {
Ok(req) => Ok(req), Ok(req) => Ok(req),
Err(err) => { Err(err) => Err(std::io::Error::other(err)),
Err(std::io::Error::other(err))
}
} }
} }
@@ -498,14 +545,17 @@ impl TransitionClient {
}*/ }*/
} }
fn make_target_url(&self, bucket_name: &str, object_name: &str, bucket_location: &str, is_virtual_host_style: bool, query_values: &HashMap<String, String>) -> Result<Url, std::io::Error> { fn make_target_url(
&self,
bucket_name: &str,
object_name: &str,
bucket_location: &str,
is_virtual_host_style: bool,
query_values: &HashMap<String, String>,
) -> Result<Url, std::io::Error> {
let scheme = self.endpoint_url.scheme(); let scheme = self.endpoint_url.scheme();
let host = self.endpoint_url.host().unwrap(); let host = self.endpoint_url.host().unwrap();
let default_port = if scheme == "https" { let default_port = if scheme == "https" { 443 } else { 80 };
443
} else {
80
};
let port = self.endpoint_url.port().unwrap_or(default_port); let port = self.endpoint_url.port().unwrap_or(default_port);
let mut url_str = format!("{scheme}://{host}:{port}/"); let mut url_str = format!("{scheme}://{host}:{port}/");
@@ -604,47 +654,114 @@ impl TransitionCore {
Ok(Self(Arc::new(client))) Ok(Self(Arc::new(client)))
} }
pub fn list_objects(&self, bucket: &str, prefix: &str, marker: &str, delimiter: &str, max_keys: i64) -> Result<ListBucketResult, std::io::Error> { pub fn list_objects(
&self,
bucket: &str,
prefix: &str,
marker: &str,
delimiter: &str,
max_keys: i64,
) -> Result<ListBucketResult, std::io::Error> {
let client = self.0.clone(); let client = self.0.clone();
client.list_objects_query(bucket, prefix, marker, delimiter, max_keys, HeaderMap::new()) client.list_objects_query(bucket, prefix, marker, delimiter, max_keys, HeaderMap::new())
} }
pub async fn list_objects_v2(&self, bucket_name: &str, object_prefix: &str, start_after: &str, continuation_token: &str, delimiter: &str, max_keys: i64) -> Result<ListBucketV2Result, std::io::Error> { pub async fn list_objects_v2(
&self,
bucket_name: &str,
object_prefix: &str,
start_after: &str,
continuation_token: &str,
delimiter: &str,
max_keys: i64,
) -> Result<ListBucketV2Result, std::io::Error> {
let client = self.0.clone(); let client = self.0.clone();
client.list_objects_v2_query(bucket_name, object_prefix, continuation_token, true, false, delimiter, start_after, max_keys, HeaderMap::new()).await client
.list_objects_v2_query(
bucket_name,
object_prefix,
continuation_token,
true,
false,
delimiter,
start_after,
max_keys,
HeaderMap::new(),
)
.await
} }
/*pub fn copy_object(&self, source_bucket: &str, source_object: &str, dest_bucket: &str, dest_object: &str, metadata: HashMap<String, String>, src_opts: CopySrcOptions, dst_opts: PutObjectOptions) -> Result<ObjectInfo> { /*pub fn copy_object(&self, source_bucket: &str, source_object: &str, dest_bucket: &str, dest_object: &str, metadata: HashMap<String, String>, src_opts: CopySrcOptions, dst_opts: PutObjectOptions) -> Result<ObjectInfo> {
self.0.copy_object_do(source_bucket, source_object, dest_bucket, dest_object, metadata, src_opts, dst_opts) self.0.copy_object_do(source_bucket, source_object, dest_bucket, dest_object, metadata, src_opts, dst_opts)
}*/ }*/
pub fn copy_object_part(&self, src_bucket: &str, src_object: &str, dest_bucket: &str, dest_object: &str, upload_id: &str, pub fn copy_object_part(
part_id: i32, start_offset: i32, length: i64, metadata: HashMap<String, String>, &self,
src_bucket: &str,
src_object: &str,
dest_bucket: &str,
dest_object: &str,
upload_id: &str,
part_id: i32,
start_offset: i32,
length: i64,
metadata: HashMap<String, String>,
) -> Result<CompletePart, std::io::Error> { ) -> Result<CompletePart, std::io::Error> {
//self.0.copy_object_part_do(src_bucket, src_object, dest_bucket, dest_object, upload_id, //self.0.copy_object_part_do(src_bucket, src_object, dest_bucket, dest_object, upload_id,
// part_id, start_offset, length, metadata) // part_id, start_offset, length, metadata)
todo!(); todo!();
} }
pub async fn put_object(&self, bucket: &str, object: &str, data: ReaderImpl, size: i64, md5_base64: &str, sha256_hex: &str, opts: &PutObjectOptions) -> Result<UploadInfo, std::io::Error> { pub async fn put_object(
&self,
bucket: &str,
object: &str,
data: ReaderImpl,
size: i64,
md5_base64: &str,
sha256_hex: &str,
opts: &PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> {
let hook_reader = data; //newHook(data, opts.progress); let hook_reader = data; //newHook(data, opts.progress);
let client = self.0.clone(); let client = self.0.clone();
client.put_object_do(bucket, object, hook_reader, md5_base64, sha256_hex, size, opts).await client
.put_object_do(bucket, object, hook_reader, md5_base64, sha256_hex, size, opts)
.await
} }
pub async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: PutObjectOptions) -> Result<String, std::io::Error> { pub async fn new_multipart_upload(
&self,
bucket: &str,
object: &str,
opts: PutObjectOptions,
) -> Result<String, std::io::Error> {
let client = self.0.clone(); let client = self.0.clone();
let result = client.initiate_multipart_upload(bucket, object, &opts).await?; let result = client.initiate_multipart_upload(bucket, object, &opts).await?;
Ok(result.upload_id) Ok(result.upload_id)
} }
pub fn list_multipart_uploads(&self, bucket: &str, prefix: &str, key_marker: &str, upload_id_marker: &str, delimiter: &str, max_uploads: i64) -> Result<ListMultipartUploadsResult, std::io::Error> { pub fn list_multipart_uploads(
&self,
bucket: &str,
prefix: &str,
key_marker: &str,
upload_id_marker: &str,
delimiter: &str,
max_uploads: i64,
) -> Result<ListMultipartUploadsResult, std::io::Error> {
let client = self.0.clone(); let client = self.0.clone();
client.list_multipart_uploads_query(bucket, key_marker, upload_id_marker, prefix, delimiter, max_uploads) client.list_multipart_uploads_query(bucket, key_marker, upload_id_marker, prefix, delimiter, max_uploads)
} }
pub async fn put_object_part(&self, bucket: &str, object: &str, upload_id: &str, part_id: i64, pub async fn put_object_part(
data: ReaderImpl, size: i64, opts: PutObjectPartOptions &self,
bucket: &str,
object: &str,
upload_id: &str,
part_id: i64,
data: ReaderImpl,
size: i64,
opts: PutObjectPartOptions,
) -> Result<ObjectPart, std::io::Error> { ) -> Result<ObjectPart, std::io::Error> {
let mut p = UploadPartParams { let mut p = UploadPartParams {
bucket_name: bucket.to_string(), bucket_name: bucket.to_string(),
@@ -664,16 +781,32 @@ impl TransitionCore {
client.upload_part(&mut p).await client.upload_part(&mut p).await
} }
pub async fn list_object_parts(&self, bucket: &str, object: &str, upload_id: &str, part_number_marker: i64, max_parts: i64) -> Result<ListObjectPartsResult, std::io::Error> { pub async fn list_object_parts(
&self,
bucket: &str,
object: &str,
upload_id: &str,
part_number_marker: i64,
max_parts: i64,
) -> Result<ListObjectPartsResult, std::io::Error> {
let client = self.0.clone(); let client = self.0.clone();
client.list_object_parts_query(bucket, object, upload_id, part_number_marker, max_parts).await client
.list_object_parts_query(bucket, object, upload_id, part_number_marker, max_parts)
.await
} }
pub async fn complete_multipart_upload(&self, bucket: &str, object: &str, upload_id: &str, parts: &[CompletePart], opts: PutObjectOptions) -> Result<UploadInfo, std::io::Error> { pub async fn complete_multipart_upload(
&self,
bucket: &str,
object: &str,
upload_id: &str,
parts: &[CompletePart],
opts: PutObjectOptions,
) -> Result<UploadInfo, std::io::Error> {
let client = self.0.clone(); let client = self.0.clone();
let res = client.complete_multipart_upload(bucket, object, upload_id, CompleteMultipartUpload { let res = client
parts: parts.to_vec(), .complete_multipart_upload(bucket, object, upload_id, CompleteMultipartUpload { parts: parts.to_vec() }, &opts)
}, &opts).await?; .await?;
Ok(res) Ok(res)
} }
@@ -692,7 +825,12 @@ impl TransitionCore {
client.put_bucket_policy(bucket_name, bucket_policy).await client.put_bucket_policy(bucket_name, bucket_policy).await
} }
pub async fn get_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> { pub async fn get_object(
&self,
bucket_name: &str,
object_name: &str,
opts: &GetObjectOptions,
) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> {
let client = self.0.clone(); let client = self.0.clone();
client.get_object_inner(bucket_name, object_name, opts).await client.get_object_inner(bucket_name, object_name, opts).await
} }
@@ -781,8 +919,7 @@ impl Default for ObjectInfo {
} }
} }
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize, Debug, Clone)]
#[derive(Debug, Clone)]
pub struct RestoreInfo { pub struct RestoreInfo {
ongoing_restore: bool, ongoing_restore: bool,
expiry_time: OffsetDateTime, expiry_time: OffsetDateTime,
+20 -14
View File
@@ -1,4 +1,4 @@
use s3s::{S3ErrorCode, S3Error}; use s3s::{S3Error, S3ErrorCode};
use rustfs_utils::path::decode_dir_object; use rustfs_utils::path::decode_dir_object;
@@ -727,16 +727,16 @@ pub fn to_object_err(err: Error, params: Vec<&str>) -> Error {
} }
pub fn is_network_or_host_down(err: &str, expect_timeouts: bool) -> bool { pub fn is_network_or_host_down(err: &str, expect_timeouts: bool) -> bool {
err.contains("Connection closed by foreign host") || err.contains("Connection closed by foreign host")
err.contains("TLS handshake timeout") || || err.contains("TLS handshake timeout")
err.contains("i/o timeout") || || err.contains("i/o timeout")
err.contains("connection timed out") || || err.contains("connection timed out")
err.contains("connection reset by peer") || || err.contains("connection reset by peer")
err.contains("broken pipe") || || err.contains("broken pipe")
err.to_lowercase().contains("503 service unavailable") || || err.to_lowercase().contains("503 service unavailable")
err.contains("use of closed network connection") || || err.contains("use of closed network connection")
err.contains("An existing connection was forcibly closed by the remote host") || || err.contains("An existing connection was forcibly closed by the remote host")
err.contains("client error (Connect)") || err.contains("client error (Connect)")
} }
#[derive(Debug, Default, PartialEq, Eq)] #[derive(Debug, Default, PartialEq, Eq)]
@@ -894,9 +894,15 @@ pub fn storage_to_object_err(err: Error, params: Vec<&str>) -> S3Error {
StorageError::InvalidArgument(bucket, object, version_id) => { StorageError::InvalidArgument(bucket, object, version_id) => {
s3_error!(InvalidArgument, "Invalid arguments provided for {}/{}-{}", bucket, object, version_id) s3_error!(InvalidArgument, "Invalid arguments provided for {}/{}-{}", bucket, object, version_id)
}*/ }*/
StorageError::MethodNotAllowed => { StorageError::MethodNotAllowed => S3Error::with_message(
S3Error::with_message(S3ErrorCode::MethodNotAllowed, ObjectApiError::MethodNotAllowed(GenericError {bucket: bucket, object: object, ..Default::default()}).to_string()) S3ErrorCode::MethodNotAllowed,
} ObjectApiError::MethodNotAllowed(GenericError {
bucket: bucket,
object: object,
..Default::default()
})
.to_string(),
),
/*StorageError::BucketNotFound(bucket) => { /*StorageError::BucketNotFound(bucket) => {
s3_error!(NoSuchBucket, "bucket not found {}", bucket) s3_error!(NoSuchBucket, "bucket not found {}", bucket)
} }
+39 -117
View File
@@ -103,123 +103,45 @@ impl AsRef<str> for EventName {
impl From<&str> for EventName { impl From<&str> for EventName {
fn from(s: &str) -> Self { fn from(s: &str) -> Self {
match s { match s {
"s3:BucketCreated:*" => { "s3:BucketCreated:*" => EventName::BucketCreated,
EventName::BucketCreated "s3:BucketRemoved:*" => EventName::BucketRemoved,
} "s3:ObjectAccessed:*" => EventName::ObjectAccessedAll,
"s3:BucketRemoved:*" => { "s3:ObjectAccessed:Get" => EventName::ObjectAccessedGet,
EventName::BucketRemoved "s3:ObjectAccessed:GetRetention" => EventName::ObjectAccessedGetRetention,
} "s3:ObjectAccessed:GetLegalHold" => EventName::ObjectAccessedGetLegalHold,
"s3:ObjectAccessed:*" => { "s3:ObjectAccessed:Head" => EventName::ObjectAccessedHead,
EventName::ObjectAccessedAll "s3:ObjectAccessed:Attributes" => EventName::ObjectAccessedAttributes,
} "s3:ObjectCreated:*" => EventName::ObjectCreatedAll,
"s3:ObjectAccessed:Get" => { "s3:ObjectCreated:CompleteMultipartUpload" => EventName::ObjectCreatedCompleteMultipartUpload,
EventName::ObjectAccessedGet "s3:ObjectCreated:Copy" => EventName::ObjectCreatedCopy,
} "s3:ObjectCreated:Post" => EventName::ObjectCreatedPost,
"s3:ObjectAccessed:GetRetention" => { "s3:ObjectCreated:Put" => EventName::ObjectCreatedPut,
EventName::ObjectAccessedGetRetention "s3:ObjectCreated:PutRetention" => EventName::ObjectCreatedPutRetention,
} "s3:ObjectCreated:PutLegalHold" => EventName::ObjectCreatedPutLegalHold,
"s3:ObjectAccessed:GetLegalHold" => { "s3:ObjectCreated:PutTagging" => EventName::ObjectCreatedPutTagging,
EventName::ObjectAccessedGetLegalHold "s3:ObjectCreated:DeleteTagging" => EventName::ObjectCreatedDeleteTagging,
} "s3:ObjectRemoved:*" => EventName::ObjectRemovedAll,
"s3:ObjectAccessed:Head" => { "s3:ObjectRemoved:Delete" => EventName::ObjectRemovedDelete,
EventName::ObjectAccessedHead "s3:ObjectRemoved:DeleteMarkerCreated" => EventName::ObjectRemovedDeleteMarkerCreated,
} "s3:ObjectRemoved:NoOP" => EventName::ObjectRemovedNoOP,
"s3:ObjectAccessed:Attributes" => { "s3:ObjectRemoved:DeleteAllVersions" => EventName::ObjectRemovedDeleteAllVersions,
EventName::ObjectAccessedAttributes "s3:LifecycleDelMarkerExpiration:Delete" => EventName::ILMDelMarkerExpirationDelete,
} "s3:Replication:*" => EventName::ObjectReplicationAll,
"s3:ObjectCreated:*" => { "s3:Replication:OperationFailedReplication" => EventName::ObjectReplicationFailed,
EventName::ObjectCreatedAll "s3:Replication:OperationCompletedReplication" => EventName::ObjectReplicationComplete,
} "s3:Replication:OperationMissedThreshold" => EventName::ObjectReplicationMissedThreshold,
"s3:ObjectCreated:CompleteMultipartUpload" => { "s3:Replication:OperationReplicatedAfterThreshold" => EventName::ObjectReplicationReplicatedAfterThreshold,
EventName::ObjectCreatedCompleteMultipartUpload "s3:Replication:OperationNotTracked" => EventName::ObjectReplicationNotTracked,
} "s3:ObjectRestore:*" => EventName::ObjectRestoreAll,
"s3:ObjectCreated:Copy" => { "s3:ObjectRestore:Post" => EventName::ObjectRestorePost,
EventName::ObjectCreatedCopy "s3:ObjectRestore:Completed" => EventName::ObjectRestoreCompleted,
} "s3:ObjectTransition:Failed" => EventName::ObjectTransitionFailed,
"s3:ObjectCreated:Post" => { "s3:ObjectTransition:Complete" => EventName::ObjectTransitionComplete,
EventName::ObjectCreatedPost "s3:ObjectTransition:*" => EventName::ObjectTransitionAll,
} "s3:Scanner:ManyVersions" => EventName::ObjectManyVersions,
"s3:ObjectCreated:Put" => { "s3:Scanner:LargeVersions" => EventName::ObjectLargeVersions,
EventName::ObjectCreatedPut "s3:Scanner:BigPrefix" => EventName::PrefixManyFolders,
} _ => EventName::Everything,
"s3:ObjectCreated:PutRetention" => {
EventName::ObjectCreatedPutRetention
}
"s3:ObjectCreated:PutLegalHold" => {
EventName::ObjectCreatedPutLegalHold
}
"s3:ObjectCreated:PutTagging" => {
EventName::ObjectCreatedPutTagging
}
"s3:ObjectCreated:DeleteTagging" => {
EventName::ObjectCreatedDeleteTagging
}
"s3:ObjectRemoved:*" => {
EventName::ObjectRemovedAll
}
"s3:ObjectRemoved:Delete" => {
EventName::ObjectRemovedDelete
}
"s3:ObjectRemoved:DeleteMarkerCreated" => {
EventName::ObjectRemovedDeleteMarkerCreated
}
"s3:ObjectRemoved:NoOP" => {
EventName::ObjectRemovedNoOP
}
"s3:ObjectRemoved:DeleteAllVersions" => {
EventName::ObjectRemovedDeleteAllVersions
}
"s3:LifecycleDelMarkerExpiration:Delete" => {
EventName::ILMDelMarkerExpirationDelete
}
"s3:Replication:*" => {
EventName::ObjectReplicationAll
}
"s3:Replication:OperationFailedReplication" => {
EventName::ObjectReplicationFailed
}
"s3:Replication:OperationCompletedReplication" => {
EventName::ObjectReplicationComplete
}
"s3:Replication:OperationMissedThreshold" => {
EventName::ObjectReplicationMissedThreshold
}
"s3:Replication:OperationReplicatedAfterThreshold" => {
EventName::ObjectReplicationReplicatedAfterThreshold
}
"s3:Replication:OperationNotTracked" => {
EventName::ObjectReplicationNotTracked
}
"s3:ObjectRestore:*" => {
EventName::ObjectRestoreAll
}
"s3:ObjectRestore:Post" => {
EventName::ObjectRestorePost
}
"s3:ObjectRestore:Completed" => {
EventName::ObjectRestoreCompleted
}
"s3:ObjectTransition:Failed" => {
EventName::ObjectTransitionFailed
}
"s3:ObjectTransition:Complete" => {
EventName::ObjectTransitionComplete
}
"s3:ObjectTransition:*" => {
EventName::ObjectTransitionAll
}
"s3:Scanner:ManyVersions" => {
EventName::ObjectManyVersions
}
"s3:Scanner:LargeVersions" => {
EventName::ObjectLargeVersions
}
"s3:Scanner:BigPrefix" => {
EventName::PrefixManyFolders
}
_ => {
EventName::Everything
}
} }
} }
} }
+4 -6
View File
@@ -2,11 +2,11 @@ use std::collections::HashMap;
use std::sync::Arc; use std::sync::Arc;
use tokio::sync::RwLock; use tokio::sync::RwLock;
use crate::store_api::ObjectInfo; use crate::bucket::metadata::BucketMetadata;
use crate::event::name::EventName; use crate::event::name::EventName;
use crate::event::targetlist::TargetList; use crate::event::targetlist::TargetList;
use crate::store::ECStore; use crate::store::ECStore;
use crate::bucket::metadata::BucketMetadata; use crate::store_api::ObjectInfo;
pub struct EventNotifier { pub struct EventNotifier {
target_list: TargetList, target_list: TargetList,
@@ -54,8 +54,6 @@ pub struct EventArgs {
pub user_agent: String, pub user_agent: String,
} }
impl EventArgs { impl EventArgs {}
}
pub fn send_event(args: EventArgs) { pub fn send_event(args: EventArgs) {}
}
+3 -3
View File
@@ -9,13 +9,13 @@ use uuid::Uuid;
use crate::heal::mrf::MRFState; use crate::heal::mrf::MRFState;
use crate::{ use crate::{
bucket::lifecycle::bucket_lifecycle_ops::LifecycleSys,
disk::DiskStore, disk::DiskStore,
endpoints::{EndpointServerPools, PoolEndpoints, SetupType}, endpoints::{EndpointServerPools, PoolEndpoints, SetupType},
heal::{background_heal_ops::HealRoutine, heal_ops::AllHealState},
bucket::lifecycle::bucket_lifecycle_ops::LifecycleSys,
tier::tier::TierConfigMgr,
event_notification::EventNotifier, event_notification::EventNotifier,
heal::{background_heal_ops::HealRoutine, heal_ops::AllHealState},
store::ECStore, store::ECStore,
tier::tier::TierConfigMgr,
}; };
pub const DISK_ASSUME_UNKNOWN_SIZE: u64 = 1 << 30; pub const DISK_ASSUME_UNKNOWN_SIZE: u64 = 1 << 30;
+95 -49
View File
@@ -6,16 +6,15 @@ use std::{
path::{Path, PathBuf}, path::{Path, PathBuf},
pin::Pin, pin::Pin,
sync::{ sync::{
atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering},
Arc, Arc,
atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering},
}, },
time::{Duration, SystemTime}, time::{Duration, SystemTime},
}; };
use time::{self, OffsetDateTime};
use common::defer; use common::defer;
use time::{self, OffsetDateTime};
use rustfs_utils::path::encode_dir_object;
use super::{ use super::{
data_scanner_metric::{ScannerMetric, ScannerMetrics, globalScannerMetrics}, data_scanner_metric::{ScannerMetric, ScannerMetrics, globalScannerMetrics},
data_usage::{DATA_USAGE_BLOOM_NAME_PATH, store_data_usage_in_backend}, data_usage::{DATA_USAGE_BLOOM_NAME_PATH, store_data_usage_in_backend},
@@ -23,28 +22,25 @@ use super::{
heal_commands::{HEAL_DEEP_SCAN, HEAL_NORMAL_SCAN, HealScanMode}, heal_commands::{HEAL_DEEP_SCAN, HEAL_NORMAL_SCAN, HealScanMode},
}; };
use crate::bucket::{ use crate::bucket::{
object_lock::objectlock_sys::{ object_lock::objectlock_sys::{BucketObjectLockSys, enforce_retention_for_deletion},
enforce_retention_for_deletion, utils::is_meta_bucketname,
BucketObjectLockSys,
}, utils::is_meta_bucketname,
}; };
use crate::cmd::bucket_replication::queue_replication_heal;
use crate::event::name::EventName;
use crate::{ use crate::{
bucket::{ bucket::{
lifecycle::{ lifecycle::{
bucket_lifecycle_audit::{LcAuditEvent, LcEventSrc}, bucket_lifecycle_audit::{LcAuditEvent, LcEventSrc},
lifecycle::{self, Lifecycle, ExpirationOptions}, bucket_lifecycle_ops::{self, GLOBAL_ExpiryState, GLOBAL_TransitionState, LifecycleOps, expire_transitioned_object},
bucket_lifecycle_ops::{ lifecycle::{self, ExpirationOptions, Lifecycle},
self, GLOBAL_ExpiryState, GLOBAL_TransitionState, LifecycleOps,
expire_transitioned_object,
}, },
metadata_sys,
}, },
metadata_sys event_notification::{EventArgs, send_event},
}, global::GLOBAL_LocalNodeName,
event_notification::{send_event, EventArgs}, heal::data_scanner,
global::GLOBAL_LocalNodeName, heal::{data_scanner},
store_api::{ObjectOptions, ObjectToDelete, StorageAPI}, store_api::{ObjectOptions, ObjectToDelete, StorageAPI},
}; };
use crate::cmd::bucket_replication::queue_replication_heal;
use crate::{ use crate::{
bucket::{versioning::VersioningApi, versioning_sys::BucketVersioningSys}, bucket::{versioning::VersioningApi, versioning_sys::BucketVersioningSys},
cmd::bucket_replication::ReplicationStatusType, cmd::bucket_replication::ReplicationStatusType,
@@ -70,7 +66,6 @@ use crate::{
peer::is_reserved_or_invalid_bucket, peer::is_reserved_or_invalid_bucket,
store::ECStore, store::ECStore,
}; };
use crate::event::name::EventName;
use crate::{disk::DiskAPI, store_api::ObjectInfo}; use crate::{disk::DiskAPI, store_api::ObjectInfo};
use crate::{ use crate::{
disk::error::DiskError, disk::error::DiskError,
@@ -82,8 +77,12 @@ use lazy_static::lazy_static;
use rand::Rng; use rand::Rng;
use rmp_serde::{Deserializer, Serializer}; use rmp_serde::{Deserializer, Serializer};
use rustfs_filemeta::{FileInfo, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; use rustfs_filemeta::{FileInfo, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
use rustfs_utils::path::encode_dir_object;
use rustfs_utils::path::{SLASH_SEPARATOR, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path}; use rustfs_utils::path::{SLASH_SEPARATOR, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path};
use s3s::dto::{BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleRule, ReplicationConfiguration, ReplicationRuleStatus, VersioningConfiguration}; use s3s::dto::{
BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleRule, ReplicationConfiguration,
ReplicationRuleStatus, VersioningConfiguration,
};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tokio::{ use tokio::{
sync::{ sync::{
@@ -561,20 +560,33 @@ impl ScannerItem {
lr = BucketObjectLockSys::get(&self.bucket).await; lr = BucketObjectLockSys::get(&self.bucket).await;
rcfg = if let Ok(replication_config) = metadata_sys::get_replication_config(&self.bucket).await { rcfg = if let Ok(replication_config) = metadata_sys::get_replication_config(&self.bucket).await {
Some(replication_config) Some(replication_config)
} else { None }; } else {
None
};
} }
let lc_evt = eval_action_from_lifecycle(self.lifecycle.as_ref().expect("err"), lr, rcfg, oi).await; let lc_evt = eval_action_from_lifecycle(self.lifecycle.as_ref().expect("err"), lr, rcfg, oi).await;
if self.debug { if self.debug {
if !version_id.is_none() { if !version_id.is_none() {
info!("lifecycle: {} (version-id={}), Initial scan: {}", self.object_path().to_string_lossy().to_string(), version_id.expect("err"), lc_evt.action); info!(
"lifecycle: {} (version-id={}), Initial scan: {}",
self.object_path().to_string_lossy().to_string(),
version_id.expect("err"),
lc_evt.action
);
} else { } else {
info!("lifecycle: {} Initial scan: {}", self.object_path().to_string_lossy().to_string(), lc_evt.action); info!(
"lifecycle: {} Initial scan: {}",
self.object_path().to_string_lossy().to_string(),
lc_evt.action
);
} }
} }
match lc_evt.action { match lc_evt.action {
lifecycle::IlmAction::DeleteVersionAction | lifecycle::IlmAction::DeleteAllVersionsAction | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { lifecycle::IlmAction::DeleteVersionAction
| lifecycle::IlmAction::DeleteAllVersionsAction
| lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => {
size = 0; size = 0;
} }
lifecycle::IlmAction::DeleteAction => { lifecycle::IlmAction::DeleteAction => {
@@ -582,7 +594,7 @@ impl ScannerItem {
size = 0 size = 0
} }
} }
_ => () _ => (),
} }
apply_lifecycle_action(&lc_evt, &LcEventSrc::Scanner, oi).await; apply_lifecycle_action(&lc_evt, &LcEventSrc::Scanner, oi).await;
@@ -612,7 +624,9 @@ impl ScannerItem {
let lock_enabled = if let Some(rcfg) = BucketObjectLockSys::get(&self.bucket).await { let lock_enabled = if let Some(rcfg) = BucketObjectLockSys::get(&self.bucket).await {
rcfg.mode.is_some() rcfg.mode.is_some()
} else { false }; } else {
false
};
let vcfg = BucketVersioningSys::get(&self.bucket).await?; let vcfg = BucketVersioningSys::get(&self.bucket).await?;
let versioned = match BucketVersioningSys::get(&self.bucket).await { let versioned = match BucketVersioningSys::get(&self.bucket).await {
@@ -633,21 +647,36 @@ impl ScannerItem {
return Ok(object_infos); return Ok(object_infos);
} }
let event = self.lifecycle.as_ref().expect("lifecycle err.").noncurrent_versions_expiration_limit(&lifecycle::ObjectOpts { let event = self
.lifecycle
.as_ref()
.expect("lifecycle err.")
.noncurrent_versions_expiration_limit(&lifecycle::ObjectOpts {
name: self.object_path().to_string_lossy().to_string(), name: self.object_path().to_string_lossy().to_string(),
..Default::default() ..Default::default()
}).await; })
.await;
let lim = event.newer_noncurrent_versions; let lim = event.newer_noncurrent_versions;
if lim == 0 || fivs.len() <= lim + 1 { if lim == 0 || fivs.len() <= lim + 1 {
for fi in fivs.iter() { for fi in fivs.iter() {
object_infos.push(ObjectInfo::from_file_info(fi, &self.bucket, &self.object_path().to_string_lossy(), versioned)); object_infos.push(ObjectInfo::from_file_info(
fi,
&self.bucket,
&self.object_path().to_string_lossy(),
versioned,
));
} }
return Ok(object_infos); return Ok(object_infos);
} }
let overflow_versions = &fivs[lim + 1..]; let overflow_versions = &fivs[lim + 1..];
for fi in fivs[..lim + 1].iter() { for fi in fivs[..lim + 1].iter() {
object_infos.push(ObjectInfo::from_file_info(fi, &self.bucket, &self.object_path().to_string_lossy(), versioned)); object_infos.push(ObjectInfo::from_file_info(
fi,
&self.bucket,
&self.object_path().to_string_lossy(),
versioned,
));
} }
let mut to_del = Vec::<ObjectToDelete>::with_capacity(overflow_versions.len()); let mut to_del = Vec::<ObjectToDelete>::with_capacity(overflow_versions.len());
@@ -666,7 +695,10 @@ impl ScannerItem {
continue; continue;
} }
if OffsetDateTime::now_utc().unix_timestamp() < lifecycle::expected_expiry_time(obj.successor_mod_time.expect("err"), event.noncurrent_days as i32).unix_timestamp() { if OffsetDateTime::now_utc().unix_timestamp()
< lifecycle::expected_expiry_time(obj.successor_mod_time.expect("err"), event.noncurrent_days as i32)
.unix_timestamp()
{
object_infos.push(obj); object_infos.push(obj);
continue; continue;
} }
@@ -1535,15 +1567,18 @@ pub async fn scan_data_folder(
Ok(s.new_cache) Ok(s.new_cache)
} }
pub async fn eval_action_from_lifecycle(lc: &BucketLifecycleConfiguration, lr: Option<DefaultRetention>, rcfg: Option<(ReplicationConfiguration, OffsetDateTime)>, oi: &ObjectInfo) -> lifecycle::Event { pub async fn eval_action_from_lifecycle(
lc: &BucketLifecycleConfiguration,
lr: Option<DefaultRetention>,
rcfg: Option<(ReplicationConfiguration, OffsetDateTime)>,
oi: &ObjectInfo,
) -> lifecycle::Event {
let event = lc.eval(&oi.to_lifecycle_opts()).await; let event = lc.eval(&oi.to_lifecycle_opts()).await;
//if serverDebugLog { //if serverDebugLog {
info!("lifecycle: Secondary scan: {}", event.action); info!("lifecycle: Secondary scan: {}", event.action);
//} //}
let lock_enabled = if let Some(lr) = lr { let lock_enabled = if let Some(lr) = lr { lr.mode.is_some() } else { false };
lr.mode.is_some()
} else { false };
match event.action { match event.action {
lifecycle::IlmAction::DeleteAllVersionsAction | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { lifecycle::IlmAction::DeleteAllVersionsAction | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => {
@@ -1571,7 +1606,7 @@ pub async fn eval_action_from_lifecycle(lc: &BucketLifecycleConfiguration, lr: O
} }
} }
} }
_ => () _ => (),
} }
event event
@@ -1585,7 +1620,12 @@ async fn apply_transition_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &
true true
} }
pub async fn apply_expiry_on_transitioned_object(api: Arc<ECStore>, oi: &ObjectInfo, lc_event: &lifecycle::Event, src: &LcEventSrc) -> bool { pub async fn apply_expiry_on_transitioned_object(
api: Arc<ECStore>,
oi: &ObjectInfo,
lc_event: &lifecycle::Event,
src: &LcEventSrc,
) -> bool {
let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone()); let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone());
if let Err(_err) = expire_transitioned_object(api, oi, lc_event, src).await { if let Err(_err) = expire_transitioned_object(api, oi, lc_event, src).await {
return false; return false;
@@ -1595,7 +1635,12 @@ pub async fn apply_expiry_on_transitioned_object(api: Arc<ECStore>, oi: &ObjectI
true true
} }
pub async fn apply_expiry_on_non_transitioned_objects(api: Arc<ECStore>, oi: &ObjectInfo, lc_event: &lifecycle::Event, src: &LcEventSrc) -> bool { pub async fn apply_expiry_on_non_transitioned_objects(
api: Arc<ECStore>,
oi: &ObjectInfo,
lc_event: &lifecycle::Event,
src: &LcEventSrc,
) -> bool {
let mut opts = ObjectOptions { let mut opts = ObjectOptions {
expiration: ExpirationOptions { expire: true }, expiration: ExpirationOptions { expire: true },
..Default::default() ..Default::default()
@@ -1617,7 +1662,10 @@ pub async fn apply_expiry_on_non_transitioned_objects(api: Arc<ECStore>, oi: &Ob
let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone()); let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone());
let mut dobj = api.delete_object(&oi.bucket, &encode_dir_object(&oi.name), opts).await.unwrap(); let mut dobj = api
.delete_object(&oi.bucket, &encode_dir_object(&oi.name), opts)
.await
.unwrap();
if dobj.name == "" { if dobj.name == "" {
dobj = oi.clone(); dobj = oi.clone();
} }
@@ -1630,13 +1678,9 @@ pub async fn apply_expiry_on_non_transitioned_objects(api: Arc<ECStore>, oi: &Ob
event_name = EventName::ObjectRemovedDeleteMarkerCreated; event_name = EventName::ObjectRemovedDeleteMarkerCreated;
} }
match lc_event.action { match lc_event.action {
lifecycle::IlmAction::DeleteAllVersionsAction => { lifecycle::IlmAction::DeleteAllVersionsAction => event_name = EventName::ObjectRemovedDeleteAllVersions,
event_name = EventName::ObjectRemovedDeleteAllVersions lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => event_name = EventName::ILMDelMarkerExpirationDelete,
} _ => (),
lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => {
event_name = EventName::ILMDelMarkerExpirationDelete
}
_ => ()
} }
send_event(EventArgs { send_event(EventArgs {
event_name: event_name.as_ref().to_string(), event_name: event_name.as_ref().to_string(),
@@ -1667,20 +1711,22 @@ async fn apply_expiry_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &Obje
pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool { pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
let mut success = false; let mut success = false;
match event.action { match event.action {
lifecycle::IlmAction::DeleteVersionAction | lifecycle::IlmAction::DeleteAction lifecycle::IlmAction::DeleteVersionAction
| lifecycle::IlmAction::DeleteRestoredAction | lifecycle::IlmAction::DeleteRestoredVersionAction | lifecycle::IlmAction::DeleteAction
| lifecycle::IlmAction::DeleteAllVersionsAction | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { | lifecycle::IlmAction::DeleteRestoredAction
| lifecycle::IlmAction::DeleteRestoredVersionAction
| lifecycle::IlmAction::DeleteAllVersionsAction
| lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => {
success = apply_expiry_rule(event, src, oi).await; success = apply_expiry_rule(event, src, oi).await;
} }
lifecycle::IlmAction::TransitionAction | lifecycle::IlmAction::TransitionVersionAction => { lifecycle::IlmAction::TransitionAction | lifecycle::IlmAction::TransitionVersionAction => {
success = apply_transition_rule(event, src, oi).await; success = apply_transition_rule(event, src, oi).await;
} }
_ => () _ => (),
} }
success success
} }
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use std::io::Cursor; use std::io::Cursor;
+1 -3
View File
@@ -312,9 +312,7 @@ impl ScannerMetrics {
pub fn time_ilm(a: lifecycle::IlmAction) -> Box<dyn Fn(u64) -> Box<dyn Fn() + Send + Sync> + Send + Sync> { pub fn time_ilm(a: lifecycle::IlmAction) -> Box<dyn Fn(u64) -> Box<dyn Fn() + Send + Sync> + Send + Sync> {
let a_clone = a as usize; let a_clone = a as usize;
if a_clone == lifecycle::IlmAction::NoneAction as usize || a_clone >= lifecycle::IlmAction::ActionCount as usize { if a_clone == lifecycle::IlmAction::NoneAction as usize || a_clone >= lifecycle::IlmAction::ActionCount as usize {
return Box::new(move |_: u64| { return Box::new(move |_: u64| Box::new(move || {}));
Box::new(move || {})
});
} }
let start = SystemTime::now(); let start = SystemTime::now();
Box::new(move |versions: u64| { Box::new(move |versions: u64| {
+6 -5
View File
@@ -153,9 +153,7 @@ struct AllTierStats {
impl AllTierStats { impl AllTierStats {
pub fn new() -> Self { pub fn new() -> Self {
Self { Self { tiers: HashMap::new() }
tiers: HashMap::new(),
}
} }
fn add_sizes(&mut self, tiers: HashMap<String, TierStats>) { fn add_sizes(&mut self, tiers: HashMap<String, TierStats>) {
@@ -172,11 +170,14 @@ impl AllTierStats {
fn populate_stats(&self, stats: &mut HashMap<String, TierStats>) { fn populate_stats(&self, stats: &mut HashMap<String, TierStats>) {
for (tier, st) in &self.tiers { for (tier, st) in &self.tiers {
stats.insert(tier.clone(), TierStats { stats.insert(
tier.clone(),
TierStats {
total_size: st.total_size.clone(), total_size: st.total_size.clone(),
num_versions: st.num_versions.clone(), num_versions: st.num_versions.clone(),
num_objects: st.num_objects.clone(), num_objects: st.num_objects.clone(),
}); },
);
} }
} }
} }
+2 -2
View File
@@ -27,11 +27,11 @@ pub mod store_list_objects;
mod store_utils; mod store_utils;
pub mod checksum; pub mod checksum;
pub mod client;
pub mod event; pub mod event;
pub mod event_notification; pub mod event_notification;
pub mod client;
pub mod tier;
pub mod signer; pub mod signer;
pub mod tier;
pub use global::new_object_layer_fn; pub use global::new_object_layer_fn;
pub use global::set_global_endpoints; pub use global::set_global_endpoints;
+57 -22
View File
@@ -1,6 +1,6 @@
use s3s::header::X_AMZ_RESTORE;
use crate::error::ObjectApiError;
use crate::bitrot::{create_bitrot_reader, create_bitrot_writer}; use crate::bitrot::{create_bitrot_reader, create_bitrot_writer};
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
use crate::client::{object_api_utils::extract_etag, transition_api::ReaderImpl};
use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_read_quorum_errs, reduce_write_quorum_errs}; use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_read_quorum_errs, reduce_write_quorum_errs};
use crate::disk::{ use crate::disk::{
self, CHECK_PART_DISK_NOT_FOUND, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, self, CHECK_PART_DISK_NOT_FOUND, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS,
@@ -8,12 +8,15 @@ use crate::disk::{
}; };
use crate::erasure_coding; use crate::erasure_coding;
use crate::erasure_coding::bitrot_verify; use crate::erasure_coding::bitrot_verify;
use crate::error::ObjectApiError;
use crate::error::{Error, Result}; use crate::error::{Error, Result};
use crate::global::GLOBAL_MRFState; use crate::global::GLOBAL_MRFState;
use crate::global::{GLOBAL_LocalNodeName, GLOBAL_TierConfigMgr};
use crate::heal::data_usage_cache::DataUsageCache; use crate::heal::data_usage_cache::DataUsageCache;
use crate::heal::heal_ops::{HealEntryFn, HealSequence}; use crate::heal::heal_ops::{HealEntryFn, HealSequence};
use crate::store_api::ObjectToDelete; use crate::store_api::ObjectToDelete;
use crate::{ use crate::{
bucket::lifecycle::bucket_lifecycle_ops::{gen_transition_objname, get_transitioned_object_reader, put_restore_opts},
cache_value::metacache_set::{ListPathRawOptions, list_path_raw}, cache_value::metacache_set::{ListPathRawOptions, list_path_raw},
config::{GLOBAL_StorageClass, storageclass}, config::{GLOBAL_StorageClass, storageclass},
disk::{ disk::{
@@ -22,8 +25,8 @@ use crate::{
UpdateMetadataOpts, endpoint::Endpoint, error::DiskError, format::FormatV3, new_disk, UpdateMetadataOpts, endpoint::Endpoint, error::DiskError, format::FormatV3, new_disk,
}, },
error::{StorageError, to_object_err}, error::{StorageError, to_object_err},
event::name::EventName, event_notification::{send_event, EventArgs}, event::name::EventName,
bucket::lifecycle::bucket_lifecycle_ops::{gen_transition_objname, put_restore_opts, get_transitioned_object_reader,}, event_notification::{EventArgs, send_event},
global::{ global::{
GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id, GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id,
is_dist_erasure, is_dist_erasure,
@@ -44,16 +47,11 @@ use crate::{
}, },
store_init::load_format_erasure, store_init::load_format_erasure,
}; };
use crate::client::{
object_api_utils::extract_etag,
transition_api::ReaderImpl,
};
use crate::{disk::STORAGE_FORMAT_FILE, heal::mrf::PartialOperation}; use crate::{disk::STORAGE_FORMAT_FILE, heal::mrf::PartialOperation};
use crate::{ use crate::{
heal::data_scanner::{HEAL_DELETE_DANGLING, globalHealConfig}, heal::data_scanner::{HEAL_DELETE_DANGLING, globalHealConfig},
store_api::ListObjectVersionsInfo, store_api::ListObjectVersionsInfo,
}; };
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
use bytesize::ByteSize; use bytesize::ByteSize;
use chrono::Utc; use chrono::Utc;
use futures::future::join_all; use futures::future::join_all;
@@ -75,6 +73,7 @@ use rustfs_utils::{
crypto::{base64_decode, base64_encode, hex}, crypto::{base64_decode, base64_encode, hex},
path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, path_join_buf}, path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, path_join_buf},
}; };
use s3s::header::X_AMZ_RESTORE;
use sha2::{Digest, Sha256}; use sha2::{Digest, Sha256};
use std::hash::Hash; use std::hash::Hash;
use std::mem; use std::mem;
@@ -100,7 +99,6 @@ use tracing::error;
use tracing::{debug, info, warn}; use tracing::{debug, info, warn};
use uuid::Uuid; use uuid::Uuid;
use workers::workers::Workers; use workers::workers::Workers;
use crate::global::{GLOBAL_TierConfigMgr, GLOBAL_LocalNodeName};
pub const DEFAULT_READ_BUFFER_SIZE: usize = 1024 * 1024; pub const DEFAULT_READ_BUFFER_SIZE: usize = 1024 * 1024;
@@ -3677,7 +3675,13 @@ impl SetDisks {
Ok(()) Ok(())
} }
pub async fn update_restore_metadata(&self, bucket: &str, object: &str, obj_info: &ObjectInfo, opts: &ObjectOptions) -> Result<()> { pub async fn update_restore_metadata(
&self,
bucket: &str,
object: &str,
obj_info: &ObjectInfo,
opts: &ObjectOptions,
) -> Result<()> {
let mut oi = obj_info.clone(); let mut oi = obj_info.clone();
oi.metadata_only = true; oi.metadata_only = true;
@@ -3686,13 +3690,23 @@ impl SetDisks {
} }
let version_id = oi.version_id.clone().map(|v| v.to_string()); let version_id = oi.version_id.clone().map(|v| v.to_string());
let obj = self.copy_object(bucket, object, bucket, object, &mut oi, &ObjectOptions { let obj = self
.copy_object(
bucket,
object,
bucket,
object,
&mut oi,
&ObjectOptions {
version_id: version_id.clone(), version_id: version_id.clone(),
..Default::default() ..Default::default()
}, &ObjectOptions { },
&ObjectOptions {
version_id: version_id, version_id: version_id,
..Default::default() ..Default::default()
}).await; },
)
.await;
if let Err(err) = obj { if let Err(err) = obj {
//storagelogif(ctx, fmt.Errorf("Unable to update transition restore metadata for %s/%s(%s): %s", bucket, object, oi.VersionID, err)) //storagelogif(ctx, fmt.Errorf("Unable to update transition restore metadata for %s/%s(%s): %s", bucket, object, oi.VersionID, err))
return Err(err); return Err(err);
@@ -4169,7 +4183,10 @@ impl StorageAPI for SetDisks {
for disk in disks.iter() { for disk in disks.iter() {
futures.push(async move { futures.push(async move {
if let Some(disk) = disk { if let Some(disk) = disk {
match disk.delete_version(&bucket, &object, fi.clone(), force_del_marker, DeleteOptions::default()).await { match disk
.delete_version(&bucket, &object, fi.clone(), force_del_marker, DeleteOptions::default())
.await
{
Ok(r) => Ok(r), Ok(r) => Ok(r),
Err(e) => Err(e), Err(e) => Err(e),
} }
@@ -4392,7 +4409,8 @@ impl StorageAPI for SetDisks {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> { async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
GLOBAL_MRFState.add_partial(PartialOperation { GLOBAL_MRFState
.add_partial(PartialOperation {
bucket: bucket.to_string(), bucket: bucket.to_string(),
object: object.to_string(), object: object.to_string(),
version_id: Some(version_id.to_string()), version_id: Some(version_id.to_string()),
@@ -4518,7 +4536,9 @@ impl StorageAPI for SetDisks {
} }
return Err(to_object_err(ERR_METHOD_NOT_ALLOWED, vec![bucket, object])); return Err(to_object_err(ERR_METHOD_NOT_ALLOWED, vec![bucket, object]));
}*/ }*/
if !opts.mod_time.expect("err").unix_timestamp() == fi.mod_time.as_ref().expect("err").unix_timestamp() || !(opts.transition.etag == extract_etag(&fi.metadata)) { if !opts.mod_time.expect("err").unix_timestamp() == fi.mod_time.as_ref().expect("err").unix_timestamp()
|| !(opts.transition.etag == extract_etag(&fi.metadata))
{
return Err(to_object_err(Error::from(DiskError::FileNotFound), vec![bucket, object])); return Err(to_object_err(Error::from(DiskError::FileNotFound), vec![bucket, object]));
} }
if fi.transition_status == TRANSITION_COMPLETE { if fi.transition_status == TRANSITION_COMPLETE {
@@ -4548,7 +4568,10 @@ impl StorageAPI for SetDisks {
let (pr, mut pw) = tokio::io::duplex(fi.erasure.block_size); let (pr, mut pw) = tokio::io::duplex(fi.erasure.block_size);
//let h = HeaderMap::new(); //let h = HeaderMap::new();
//let reader = ReaderImpl::ObjectBody(GetObjectReader {stream: StreamingBlob::wrap(tokio_util::io::ReaderStream::new(pr)), object_info: oi}); //let reader = ReaderImpl::ObjectBody(GetObjectReader {stream: StreamingBlob::wrap(tokio_util::io::ReaderStream::new(pr)), object_info: oi});
let reader = ReaderImpl::ObjectBody(GetObjectReader {stream: Box::new(pr), object_info: oi}); let reader = ReaderImpl::ObjectBody(GetObjectReader {
stream: Box::new(pr),
object_info: oi,
});
let cloned_bucket = bucket.to_string(); let cloned_bucket = bucket.to_string();
let cloned_object = object.to_string(); let cloned_object = object.to_string();
@@ -4557,7 +4580,16 @@ impl StorageAPI for SetDisks {
let pool_index = self.pool_index; let pool_index = self.pool_index;
tokio::spawn(async move { tokio::spawn(async move {
if let Err(e) = Self::get_object_with_fileinfo( if let Err(e) = Self::get_object_with_fileinfo(
&cloned_bucket, &cloned_object, 0, cloned_fi.size, &mut pw, cloned_fi, meta_arr, &online_disks, set_index, pool_index &cloned_bucket,
&cloned_object,
0,
cloned_fi.size,
&mut pw,
cloned_fi,
meta_arr,
&online_disks,
set_index,
pool_index,
) )
.await .await
{ {
@@ -4565,11 +4597,13 @@ impl StorageAPI for SetDisks {
}; };
}); });
let rv = tgt_client.put_with_meta(&dest_obj, reader, fi.size as i64, { let rv = tgt_client
.put_with_meta(&dest_obj, reader, fi.size as i64, {
let mut m = HashMap::<String, String>::new(); let mut m = HashMap::<String, String>::new();
m.insert("name".to_string(), object.to_string()); m.insert("name".to_string(), object.to_string());
m m
}).await; })
.await;
//pr.CloseWithError(err); //pr.CloseWithError(err);
if let Err(err) = rv { if let Err(err) = rv {
//traceFn(ILMTransition, nil, err) //traceFn(ILMTransition, nil, err)
@@ -4594,7 +4628,8 @@ impl StorageAPI for SetDisks {
continue; continue;
} }
} }
self.add_partial(bucket, object, &opts.version_id.as_ref().expect("err")).await; self.add_partial(bucket, object, &opts.version_id.as_ref().expect("err"))
.await;
break; break;
} }
+4 -4
View File
@@ -1,7 +1,6 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use std::{collections::HashMap, sync::Arc}; use std::{collections::HashMap, sync::Arc};
use rustfs_filemeta::FileInfo;
use crate::disk::error_reduce::count_errs; use crate::disk::error_reduce::count_errs;
use crate::error::{Error, Result}; use crate::error::{Error, Result};
use crate::{ use crate::{
@@ -30,6 +29,7 @@ use futures::future::join_all;
use http::HeaderMap; use http::HeaderMap;
use lock::{LockApi, namespace_lock::NsLockMap, new_lock_api}; use lock::{LockApi, namespace_lock::NsLockMap, new_lock_api};
use madmin::heal_commands::{HealDriveInfo, HealResultItem}; use madmin::heal_commands::{HealDriveInfo, HealResultItem};
use rustfs_filemeta::FileInfo;
use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash}; use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash};
use tokio::sync::RwLock; use tokio::sync::RwLock;
use uuid::Uuid; use uuid::Uuid;
@@ -591,7 +591,9 @@ impl StorageAPI for Sets {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
self.get_disks_by_key(object).restore_transitioned_object(bucket, object, opts).await self.get_disks_by_key(object)
.restore_transitioned_object(bucket, object, opts)
.await
} }
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
@@ -688,8 +690,6 @@ impl StorageAPI for Sets {
.await .await
} }
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> { async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
self.get_disks_by_key(object).delete_object_tags(bucket, object, opts).await self.get_disks_by_key(object).delete_object_tags(bucket, object, opts).await
+9 -9
View File
@@ -1,13 +1,13 @@
pub mod utils; pub mod ordered_qs;
pub mod request_signature_v2;
pub mod request_signature_v4;
pub mod request_signature_streaming; pub mod request_signature_streaming;
pub mod request_signature_streaming_unsigned_trailer; pub mod request_signature_streaming_unsigned_trailer;
pub mod ordered_qs; pub mod request_signature_v2;
pub mod request_signature_v4;
pub mod utils;
pub use request_signature_v2::sign_v2;
pub use request_signature_v2::pre_sign_v2;
pub use request_signature_v4::sign_v4;
pub use request_signature_v4::pre_sign_v4;
pub use request_signature_v4::sign_v4_trailer;
pub use request_signature_streaming::streaming_sign_v4; pub use request_signature_streaming::streaming_sign_v4;
pub use request_signature_v2::pre_sign_v2;
pub use request_signature_v2::sign_v2;
pub use request_signature_v4::pre_sign_v4;
pub use request_signature_v4::sign_v4;
pub use request_signature_v4::sign_v4_trailer;
@@ -1,26 +1,26 @@
use std::pin::Pin; use bytes::{Bytes, BytesMut};
use std::sync::Mutex;
use futures::prelude::*; use futures::prelude::*;
use futures::task; use futures::task;
use bytes::{Bytes, BytesMut}; use http::HeaderMap;
use http::header::TRAILER;
use http::Uri; use http::Uri;
use http::header::TRAILER;
use http::request::{self, Request};
use hyper::Method;
use lazy_static::lazy_static; use lazy_static::lazy_static;
use std::collections::HashMap; use std::collections::HashMap;
use stdx::str::StrExt;
use std::fmt::Write; use std::fmt::Write;
use tracing::{error, info, warn, debug}; use std::pin::Pin;
use time::{OffsetDateTime, macros::datetime, macros::format_description, format_description}; use std::sync::Mutex;
use http::request::{self, Request}; use stdx::str::StrExt;
use http::HeaderMap; use time::{OffsetDateTime, format_description, macros::datetime, macros::format_description};
use hyper::Method; use tracing::{debug, error, info, warn};
use super::request_signature_v4::{SERVICE_TYPE_S3, get_scope, get_signature, get_signing_key};
use crate::client::constants::UNSIGNED_PAYLOAD;
use rustfs_utils::{ use rustfs_utils::{
crypto::{hex, hex_sha256, hex_sha256_chunk, hmac_sha256}, crypto::{hex, hex_sha256, hex_sha256_chunk, hmac_sha256},
hash::EMPTY_STRING_SHA256_HASH hash::EMPTY_STRING_SHA256_HASH,
}; };
use crate::client::constants::UNSIGNED_PAYLOAD;
use super::request_signature_v4::{get_scope, get_signing_key, get_signature, SERVICE_TYPE_S3};
const STREAMING_SIGN_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; const STREAMING_SIGN_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD";
const STREAMING_SIGN_TRAILER_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"; const STREAMING_SIGN_TRAILER_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER";
@@ -55,17 +55,26 @@ fn build_chunk_string_to_sign(t: OffsetDateTime, region: &str, previous_sig: &st
string_to_sign_parts.join("\n") string_to_sign_parts.join("\n")
} }
fn build_chunk_signature(chunk_check_sum: &str, req_time: OffsetDateTime, region: &str, fn build_chunk_signature(
previous_signature: &str, secret_access_key: &str chunk_check_sum: &str,
req_time: OffsetDateTime,
region: &str,
previous_signature: &str,
secret_access_key: &str,
) -> String { ) -> String {
let chunk_string_to_sign = build_chunk_string_to_sign(req_time, region, let chunk_string_to_sign = build_chunk_string_to_sign(req_time, region, previous_signature, chunk_check_sum);
previous_signature, chunk_check_sum);
let signing_key = get_signing_key(secret_access_key, region, req_time, SERVICE_TYPE_S3); let signing_key = get_signing_key(secret_access_key, region, req_time, SERVICE_TYPE_S3);
get_signature(signing_key, &chunk_string_to_sign) get_signature(signing_key, &chunk_string_to_sign)
} }
pub fn streaming_sign_v4(req: request::Builder, access_key_id: &str, secret_access_key: &str, session_token: &str, pub fn streaming_sign_v4(
region: &str, data_len: i64, req_time: OffsetDateTime/*, sh256: md5simd.Hasher*/ req: request::Builder,
access_key_id: &str,
secret_access_key: &str,
session_token: &str,
region: &str,
data_len: i64,
req_time: OffsetDateTime, /*, sh256: md5simd.Hasher*/
) -> request::Builder { ) -> request::Builder {
todo!(); todo!();
} }
@@ -1,6 +1,11 @@
use http::request; use http::request;
use time::OffsetDateTime; use time::OffsetDateTime;
pub fn streaming_unsigned_v4(mut req: request::Builder, session_token: &str, data_len: i64, req_time: OffsetDateTime) -> request::Builder{ pub fn streaming_unsigned_v4(
mut req: request::Builder,
session_token: &str,
data_len: i64,
req_time: OffsetDateTime,
) -> request::Builder {
todo!(); todo!();
} }
+37 -8
View File
@@ -1,9 +1,9 @@
use std::collections::HashMap; use bytes::{Bytes, BytesMut};
use http::request; use http::request;
use hyper::Uri; use hyper::Uri;
use bytes::{Bytes, BytesMut}; use std::collections::HashMap;
use std::fmt::Write; use std::fmt::Write;
use time::{OffsetDateTime, macros::format_description, format_description}; use time::{OffsetDateTime, format_description, macros::format_description};
use rustfs_utils::crypto::{base64_encode, hex, hmac_sha1}; use rustfs_utils::crypto::{base64_encode, hex, hmac_sha1};
@@ -20,7 +20,13 @@ fn encode_url2path(req: &request::Builder, virtual_host: bool) -> String {
path path
} }
pub fn pre_sign_v2(mut req: request::Builder, access_key_id: &str, secret_access_key: &str, expires: i64, virtual_host: bool) -> request::Builder { pub fn pre_sign_v2(
mut req: request::Builder,
access_key_id: &str,
secret_access_key: &str,
expires: i64,
virtual_host: bool,
) -> request::Builder {
if access_key_id == "" || secret_access_key == "" { if access_key_id == "" || secret_access_key == "" {
return req; return req;
} }
@@ -50,7 +56,11 @@ pub fn pre_sign_v2(mut req: request::Builder, access_key_id: &str, secret_access
let uri = req.uri_ref().unwrap().clone(); let uri = req.uri_ref().unwrap().clone();
let mut parts = req.uri_ref().unwrap().clone().into_parts(); let mut parts = req.uri_ref().unwrap().clone().into_parts();
parts.path_and_query = Some(format!("{}?{}&Signature={}", uri.path(), serde_urlencoded::to_string(&query).unwrap(), signature).parse().unwrap()); parts.path_and_query = Some(
format!("{}?{}&Signature={}", uri.path(), serde_urlencoded::to_string(&query).unwrap(), signature)
.parse()
.unwrap(),
);
let req = req.uri(Uri::from_parts(parts).unwrap()); let req = req.uri(Uri::from_parts(parts).unwrap());
req req
@@ -61,7 +71,13 @@ fn post_pre_sign_signature_v2(policy_base64: &str, secret_access_key: &str) -> S
signature signature
} }
pub fn sign_v2(mut req: request::Builder, content_len: i64, access_key_id: &str, secret_access_key: &str, virtual_host: bool) -> request::Builder { pub fn sign_v2(
mut req: request::Builder,
content_len: i64,
access_key_id: &str,
secret_access_key: &str,
virtual_host: bool,
) -> request::Builder {
if access_key_id == "" || secret_access_key == "" { if access_key_id == "" || secret_access_key == "" {
return req; return req;
} }
@@ -74,7 +90,14 @@ pub fn sign_v2(mut req: request::Builder, content_len: i64, access_key_id: &str,
let date = headers.get("Date").unwrap(); let date = headers.get("Date").unwrap();
if date.to_str().unwrap() == "" { if date.to_str().unwrap() == "" {
headers.insert("Date", d2.format(&format_description::well_known::Rfc2822).unwrap().to_string().parse().unwrap()); headers.insert(
"Date",
d2.format(&format_description::well_known::Rfc2822)
.unwrap()
.to_string()
.parse()
.unwrap(),
);
} }
let mut auth_header = format!("{} {}:", SIGN_V2_ALGORITHM, access_key_id); let mut auth_header = format!("{} {}:", SIGN_V2_ALGORITHM, access_key_id);
@@ -130,7 +153,13 @@ fn write_canonicalized_headers(buf: &mut BytesMut, req: &request::Builder) {
let lk = k.as_str().to_lowercase(); let lk = k.as_str().to_lowercase();
if lk.starts_with("x-amz") { if lk.starts_with("x-amz") {
proto_headers.push(lk.clone()); proto_headers.push(lk.clone());
let vv = req.headers_ref().expect("err").get_all(k).iter().map(|e| e.to_str().unwrap().to_string()).collect(); let vv = req
.headers_ref()
.expect("err")
.get_all(k)
.iter()
.map(|e| e.to_str().unwrap().to_string())
.collect();
vals.insert(lk, vv); vals.insert(lk, vv);
} }
} }
+141 -32
View File
@@ -1,21 +1,21 @@
use bytes::{Bytes, BytesMut}; use bytes::{Bytes, BytesMut};
use http::header::TRAILER; use http::HeaderMap;
use http::Uri; use http::Uri;
use http::header::TRAILER;
use http::request::{self, Request};
use lazy_static::lazy_static; use lazy_static::lazy_static;
use std::collections::HashMap; use std::collections::HashMap;
use std::fmt::Write; use std::fmt::Write;
use tracing::{error, info, warn, debug}; use time::{OffsetDateTime, format_description, macros::datetime, macros::format_description};
use time::{OffsetDateTime, macros::datetime, macros::format_description, format_description}; use tracing::{debug, error, info, warn};
use http::request::{self, Request};
use http::HeaderMap;
use rustfs_utils::crypto::{hex, hex_sha256, hmac_sha256};
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
use crate::client::constants::UNSIGNED_PAYLOAD;
use super::ordered_qs::OrderedQs; use super::ordered_qs::OrderedQs;
use super::request_signature_streaming_unsigned_trailer::streaming_unsigned_v4; use super::request_signature_streaming_unsigned_trailer::streaming_unsigned_v4;
use super::utils::stable_sort_by_first; use super::utils::stable_sort_by_first;
use super::utils::{get_host_addr, sign_v4_trim_all}; use super::utils::{get_host_addr, sign_v4_trim_all};
use crate::client::constants::UNSIGNED_PAYLOAD;
use rustfs_utils::crypto::{hex, hex_sha256, hmac_sha256};
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256"; pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
pub const SERVICE_TYPE_S3: &str = "s3"; pub const SERVICE_TYPE_S3: &str = "s3";
@@ -86,7 +86,13 @@ fn get_canonical_headers(req: &request::Builder, ignored_headers: &HashMap<Strin
continue; continue;
} }
headers.push(k.as_str().to_lowercase()); headers.push(k.as_str().to_lowercase());
let vv = req.headers_ref().expect("err").get_all(k).iter().map(|e| e.to_str().unwrap().to_string()).collect(); let vv = req
.headers_ref()
.expect("err")
.get_all(k)
.iter()
.map(|e| e.to_str().unwrap().to_string())
.collect();
vals.insert(k.as_str().to_lowercase(), vv); vals.insert(k.as_str().to_lowercase(), vv);
} }
if !header_exists("host", &headers) { if !header_exists("host", &headers) {
@@ -178,7 +184,15 @@ fn get_string_to_sign_v4(t: OffsetDateTime, location: &str, canonical_request: &
string_to_sign string_to_sign
} }
pub fn pre_sign_v4(req: request::Builder, access_key_id: &str, secret_access_key: &str, session_token: &str, location: &str, expires: i64, t: OffsetDateTime) -> request::Builder { pub fn pre_sign_v4(
req: request::Builder,
access_key_id: &str,
secret_access_key: &str,
session_token: &str,
location: &str,
expires: i64,
t: OffsetDateTime,
) -> request::Builder {
if access_key_id == "" || secret_access_key == "" { if access_key_id == "" || secret_access_key == "" {
return req; return req;
} }
@@ -208,7 +222,11 @@ pub fn pre_sign_v4(req: request::Builder, access_key_id: &str, secret_access_key
let uri = req.uri_ref().unwrap().clone(); let uri = req.uri_ref().unwrap().clone();
let mut parts = req.uri_ref().unwrap().clone().into_parts(); let mut parts = req.uri_ref().unwrap().clone().into_parts();
parts.path_and_query = Some(format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap()).parse().unwrap()); parts.path_and_query = Some(
format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap())
.parse()
.unwrap(),
);
let req = req.uri(Uri::from_parts(parts).unwrap()); let req = req.uri(Uri::from_parts(parts).unwrap());
let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req)); let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req));
@@ -220,7 +238,16 @@ pub fn pre_sign_v4(req: request::Builder, access_key_id: &str, secret_access_key
let uri = req.uri_ref().unwrap().clone(); let uri = req.uri_ref().unwrap().clone();
let mut parts = req.uri_ref().unwrap().clone().into_parts(); let mut parts = req.uri_ref().unwrap().clone().into_parts();
parts.path_and_query = Some(format!("{}?{}&X-Amz-Signature={}", uri.path(), serde_urlencoded::to_string(&query).unwrap(), signature).parse().unwrap()); parts.path_and_query = Some(
format!(
"{}?{}&X-Amz-Signature={}",
uri.path(),
serde_urlencoded::to_string(&query).unwrap(),
signature
)
.parse()
.unwrap(),
);
let req = req.uri(Uri::from_parts(parts).unwrap()); let req = req.uri(Uri::from_parts(parts).unwrap());
req req
@@ -236,7 +263,16 @@ fn sign_v4_sts(mut req: request::Builder, access_key_id: &str, secret_access_key
sign_v4_inner(req, 0, access_key_id, secret_access_key, "", location, SERVICE_TYPE_STS, HeaderMap::new()) sign_v4_inner(req, 0, access_key_id, secret_access_key, "", location, SERVICE_TYPE_STS, HeaderMap::new())
} }
fn sign_v4_inner(mut req: request::Builder, content_len: i64, access_key_id: &str, secret_access_key: &str, session_token: &str, location: &str, service_type: &str, trailer: HeaderMap) -> request::Builder { fn sign_v4_inner(
mut req: request::Builder,
content_len: i64,
access_key_id: &str,
secret_access_key: &str,
session_token: &str,
location: &str,
service_type: &str,
trailer: HeaderMap,
) -> request::Builder {
if access_key_id == "" || secret_access_key == "" { if access_key_id == "" || secret_access_key == "" {
return req; return req;
} }
@@ -276,7 +312,10 @@ fn sign_v4_inner(mut req: request::Builder, content_len: i64, access_key_id: &st
let mut headers = req.headers_mut().expect("err"); let mut headers = req.headers_mut().expect("err");
let auth = format!("{} Credential={}, SignedHeaders={}, Signature={}", SIGN_V4_ALGORITHM, credential, signed_headers, signature); let auth = format!(
"{} Credential={}, SignedHeaders={}, Signature={}",
SIGN_V4_ALGORITHM, credential, signed_headers, signature
);
headers.insert("Authorization", auth.parse().unwrap()); headers.insert("Authorization", auth.parse().unwrap());
if trailer.len() > 0 { if trailer.len() > 0 {
@@ -315,12 +354,44 @@ fn unsigned_trailer(mut req: request::Builder, content_len: i64, trailer: Header
streaming_unsigned_v4(req, "", content_len, t); streaming_unsigned_v4(req, "", content_len, t);
} }
pub fn sign_v4(mut req: request::Builder, content_len: i64, access_key_id: &str, secret_access_key: &str, session_token: &str, location: &str) -> request::Builder { pub fn sign_v4(
sign_v4_inner(req, content_len, access_key_id, secret_access_key, session_token, location, SERVICE_TYPE_S3, HeaderMap::new()) mut req: request::Builder,
content_len: i64,
access_key_id: &str,
secret_access_key: &str,
session_token: &str,
location: &str,
) -> request::Builder {
sign_v4_inner(
req,
content_len,
access_key_id,
secret_access_key,
session_token,
location,
SERVICE_TYPE_S3,
HeaderMap::new(),
)
} }
pub fn sign_v4_trailer(req: request::Builder, access_key_id: &str, secret_access_key: &str, session_token: &str, location: &str, trailer: HeaderMap) -> request::Builder { pub fn sign_v4_trailer(
sign_v4_inner(req, 0, access_key_id, secret_access_key, session_token, location, SERVICE_TYPE_S3, trailer) req: request::Builder,
access_key_id: &str,
secret_access_key: &str,
session_token: &str,
location: &str,
trailer: HeaderMap,
) -> request::Builder {
sign_v4_inner(
req,
0,
access_key_id,
secret_access_key,
session_token,
location,
SERVICE_TYPE_S3,
trailer,
)
} }
#[cfg(test)] #[cfg(test)]
@@ -338,10 +409,17 @@ mod tests {
let service = "s3"; let service = "s3";
let path = "/"; let path = "/";
let mut req = Request::builder().method(http::Method::GET).uri("http://examplebucket.s3.amazonaws.com/?"); let mut req = Request::builder()
.method(http::Method::GET)
.uri("http://examplebucket.s3.amazonaws.com/?");
let mut headers = req.headers_mut().expect("err"); let mut headers = req.headers_mut().expect("err");
headers.insert("host", "examplebucket.s3.amazonaws.com".parse().unwrap()); headers.insert("host", "examplebucket.s3.amazonaws.com".parse().unwrap());
headers.insert("x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855".parse().unwrap()); headers.insert(
"x-amz-content-sha256",
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
.parse()
.unwrap(),
);
headers.insert("x-amz-date", timestamp.parse().unwrap()); headers.insert("x-amz-date", timestamp.parse().unwrap());
let mut query = <Vec<(String, String)>>::new(); let mut query = <Vec<(String, String)>>::new();
@@ -349,7 +427,11 @@ mod tests {
query.push(("prefix".to_string(), "J".to_string())); query.push(("prefix".to_string(), "J".to_string()));
let uri = req.uri_ref().unwrap().clone(); let uri = req.uri_ref().unwrap().clone();
let mut parts = req.uri_ref().unwrap().clone().into_parts(); let mut parts = req.uri_ref().unwrap().clone().into_parts();
parts.path_and_query = Some(format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap()).parse().unwrap()); parts.path_and_query = Some(
format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap())
.parse()
.unwrap(),
);
let req = req.uri(Uri::from_parts(parts).unwrap()); let req = req.uri(Uri::from_parts(parts).unwrap());
let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req)); let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req));
@@ -361,7 +443,9 @@ mod tests {
"max-keys=2&prefix=J\n", "max-keys=2&prefix=J\n",
"host:examplebucket.s3.amazonaws.com\n", "host:examplebucket.s3.amazonaws.com\n",
"x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n", "x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n",
"x-amz-date:", "20130524T000000Z", "\n", "x-amz-date:",
"20130524T000000Z",
"\n",
"\n", "\n",
"host;x-amz-content-sha256;x-amz-date\n", "host;x-amz-content-sha256;x-amz-date\n",
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
@@ -373,7 +457,8 @@ mod tests {
string_to_sign, string_to_sign,
concat!( concat!(
"AWS4-HMAC-SHA256\n", "AWS4-HMAC-SHA256\n",
"20130524T000000Z", "\n", "20130524T000000Z",
"\n",
"20130524/us-east-1/s3/aws4_request\n", "20130524/us-east-1/s3/aws4_request\n",
"df57d21db20da04d7fa30298dd4488ba3a2b47ca3a489c74750e0f1e7df1b9b7", "df57d21db20da04d7fa30298dd4488ba3a2b47ca3a489c74750e0f1e7df1b9b7",
) )
@@ -396,17 +481,28 @@ mod tests {
let service = "s3"; let service = "s3";
let path = "/mblock2/"; let path = "/mblock2/";
let mut req = Request::builder().method(http::Method::GET).uri("http://192.168.1.11:9020/mblock2/?"); let mut req = Request::builder()
.method(http::Method::GET)
.uri("http://192.168.1.11:9020/mblock2/?");
let mut headers = req.headers_mut().expect("err"); let mut headers = req.headers_mut().expect("err");
headers.insert("host", "192.168.1.11:9020".parse().unwrap()); headers.insert("host", "192.168.1.11:9020".parse().unwrap());
headers.insert("x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855".parse().unwrap()); headers.insert(
"x-amz-content-sha256",
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
.parse()
.unwrap(),
);
headers.insert("x-amz-date", timestamp.parse().unwrap()); headers.insert("x-amz-date", timestamp.parse().unwrap());
let mut query: Vec<(String, String)> = Vec::new(); let mut query: Vec<(String, String)> = Vec::new();
let uri = req.uri_ref().unwrap().clone(); let uri = req.uri_ref().unwrap().clone();
let mut parts = req.uri_ref().unwrap().clone().into_parts(); let mut parts = req.uri_ref().unwrap().clone().into_parts();
parts.path_and_query = Some(format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap()).parse().unwrap()); parts.path_and_query = Some(
format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap())
.parse()
.unwrap(),
);
let req = req.uri(Uri::from_parts(parts).unwrap()); let req = req.uri(Uri::from_parts(parts).unwrap());
let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req)); let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req));
@@ -419,7 +515,9 @@ mod tests {
"\n", "\n",
"host:192.168.1.11:9020\n", "host:192.168.1.11:9020\n",
"x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n", "x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n",
"x-amz-date:", "20250505T011054Z", "\n", "x-amz-date:",
"20250505T011054Z",
"\n",
"\n", "\n",
"host;x-amz-content-sha256;x-amz-date\n", "host;x-amz-content-sha256;x-amz-date\n",
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
@@ -432,7 +530,8 @@ mod tests {
string_to_sign, string_to_sign,
concat!( concat!(
"AWS4-HMAC-SHA256\n", "AWS4-HMAC-SHA256\n",
"20250505T011054Z", "\n", "20250505T011054Z",
"\n",
"20250505/us-east-1/s3/aws4_request\n", "20250505/us-east-1/s3/aws4_request\n",
"c2960d00cc7de7bed3e2e2d1330ec298ded8f78a231c1d32dedac72ebec7f9b0", "c2960d00cc7de7bed3e2e2d1330ec298ded8f78a231c1d32dedac72ebec7f9b0",
) )
@@ -459,7 +558,12 @@ mod tests {
let mut headers = req.headers_mut().expect("err"); let mut headers = req.headers_mut().expect("err");
headers.insert("host", "192.168.1.11:9020".parse().unwrap()); headers.insert("host", "192.168.1.11:9020".parse().unwrap());
headers.insert("x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855".parse().unwrap()); headers.insert(
"x-amz-content-sha256",
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
.parse()
.unwrap(),
);
headers.insert("x-amz-date", timestamp.parse().unwrap()); headers.insert("x-amz-date", timestamp.parse().unwrap());
/*let uri = req.uri_ref().unwrap().clone(); /*let uri = req.uri_ref().unwrap().clone();
@@ -486,7 +590,9 @@ println!("{:?}", req.uri_ref().unwrap().query());
"delimiter=%2F&encoding-type=url&fetch-owner=true&list-type=2&max-keys=1&prefix=mypre\n", "delimiter=%2F&encoding-type=url&fetch-owner=true&list-type=2&max-keys=1&prefix=mypre\n",
"host:192.168.1.11:9020\n", "host:192.168.1.11:9020\n",
"x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n", "x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n",
"x-amz-date:", "20250507T051030Z", "\n", "x-amz-date:",
"20250507T051030Z",
"\n",
"\n", "\n",
"host;x-amz-content-sha256;x-amz-date\n", "host;x-amz-content-sha256;x-amz-date\n",
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
@@ -499,7 +605,8 @@ println!("{:?}", req.uri_ref().unwrap().query());
string_to_sign, string_to_sign,
concat!( concat!(
"AWS4-HMAC-SHA256\n", "AWS4-HMAC-SHA256\n",
"20250507T051030Z", "\n", "20250507T051030Z",
"\n",
"20250507/us-east-1/s3/aws4_request\n", "20250507/us-east-1/s3/aws4_request\n",
"e6db9e09e9c873aff0b9ca170998b4753f6a6c36c90bc2dca80613affb47f999", "e6db9e09e9c873aff0b9ca170998b4753f6a6c36c90bc2dca80613affb47f999",
) )
@@ -525,7 +632,9 @@ println!("{:?}", req.uri_ref().unwrap().query());
let path = "/"; let path = "/";
let session_token = ""; let session_token = "";
let mut req = Request::builder().method(http::Method::GET).uri("http://examplebucket.s3.amazonaws.com/test.txt"); let mut req = Request::builder()
.method(http::Method::GET)
.uri("http://examplebucket.s3.amazonaws.com/test.txt");
let mut headers = req.headers_mut().expect("err"); let mut headers = req.headers_mut().expect("err");
headers.insert("host", "examplebucket.s3.amazonaws.com".parse().unwrap()); headers.insert("host", "examplebucket.s3.amazonaws.com".parse().unwrap());
+14 -13
View File
@@ -1,20 +1,21 @@
#![allow(clippy::map_entry)] #![allow(clippy::map_entry)]
use rustfs_filemeta::FileInfo; use crate::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry;
use crate::bucket::metadata_sys::{self, set_bucket_metadata}; use crate::bucket::metadata_sys::{self, set_bucket_metadata};
use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname}; use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname};
use crate::config::GLOBAL_StorageClass; use crate::config::GLOBAL_StorageClass;
use crate::config::storageclass; use crate::config::storageclass;
use crate::disk::endpoint::{Endpoint, EndpointType}; use crate::disk::endpoint::{Endpoint, EndpointType};
use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions}; use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions};
use crate::error::{Error, Result};
use crate::error::{ use crate::error::{
StorageError, is_err_bucket_exists, is_err_invalid_upload_id, is_err_object_not_found, is_err_read_quorum, StorageError, is_err_bucket_exists, is_err_invalid_upload_id, is_err_object_not_found, is_err_read_quorum,
is_err_version_not_found, to_object_err, is_err_version_not_found, to_object_err,
}; };
use crate::global::{ use crate::global::{
DISK_ASSUME_UNKNOWN_SIZE, DISK_FILL_FRACTION, DISK_MIN_INODES, DISK_RESERVE_FRACTION, GLOBAL_BOOT_TIME, DISK_ASSUME_UNKNOWN_SIZE, DISK_FILL_FRACTION, DISK_MIN_INODES, DISK_RESERVE_FRACTION, GLOBAL_BOOT_TIME,
GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_endpoints, is_dist_erasure, is_erasure_sd, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, GLOBAL_TierConfigMgr, get_global_endpoints, is_dist_erasure,
set_global_deployment_id, set_object_layer, GLOBAL_TierConfigMgr, is_erasure_sd, set_global_deployment_id, set_object_layer,
}; };
use crate::heal::data_usage::{DATA_USAGE_ROOT, DataUsageInfo}; use crate::heal::data_usage::{DATA_USAGE_ROOT, DataUsageInfo};
use crate::heal::data_usage_cache::{DataUsageCache, DataUsageCacheInfo}; use crate::heal::data_usage_cache::{DataUsageCache, DataUsageCacheInfo};
@@ -27,10 +28,7 @@ use crate::rebalance::RebalanceMeta;
use crate::store_api::{ListMultipartsInfo, ListObjectVersionsInfo, MultipartInfo, ObjectIO}; use crate::store_api::{ListMultipartsInfo, ListObjectVersionsInfo, MultipartInfo, ObjectIO};
use crate::store_init::{check_disk_fatal_errs, ec_drives_no_config}; use crate::store_init::{check_disk_fatal_errs, ec_drives_no_config};
use crate::{ use crate::{
bucket::{ bucket::{lifecycle::bucket_lifecycle_ops::TransitionState, metadata::BucketMetadata},
metadata::BucketMetadata,
lifecycle::bucket_lifecycle_ops::TransitionState
},
disk::{BUCKET_META_PREFIX, DiskOption, DiskStore, RUSTFS_META_BUCKET, new_disk}, disk::{BUCKET_META_PREFIX, DiskOption, DiskStore, RUSTFS_META_BUCKET, new_disk},
endpoints::EndpointServerPools, endpoints::EndpointServerPools,
peer::S3PeerSys, peer::S3PeerSys,
@@ -42,10 +40,6 @@ use crate::{
}, },
store_init, store_init,
}; };
use rustfs_utils::crypto::base64_decode;
use rustfs_utils::path::{SLASH_SEPARATOR, decode_dir_object, encode_dir_object, path_join_buf};
use crate::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry;
use crate::error::{Error, Result};
use common::globals::{GLOBAL_Local_Node_Name, GLOBAL_Rustfs_Host, GLOBAL_Rustfs_Port}; use common::globals::{GLOBAL_Local_Node_Name, GLOBAL_Rustfs_Host, GLOBAL_Rustfs_Port};
use futures::future::join_all; use futures::future::join_all;
use glob::Pattern; use glob::Pattern;
@@ -53,7 +47,10 @@ use http::HeaderMap;
use lazy_static::lazy_static; use lazy_static::lazy_static;
use madmin::heal_commands::HealResultItem; use madmin::heal_commands::HealResultItem;
use rand::Rng as _; use rand::Rng as _;
use rustfs_filemeta::FileInfo;
use rustfs_filemeta::MetaCacheEntry; use rustfs_filemeta::MetaCacheEntry;
use rustfs_utils::crypto::base64_decode;
use rustfs_utils::path::{SLASH_SEPARATOR, decode_dir_object, encode_dir_object, path_join_buf};
use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration}; use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration};
use std::cmp::Ordering; use std::cmp::Ordering;
use std::process::exit; use std::process::exit;
@@ -1868,7 +1865,9 @@ impl StorageAPI for ECStore {
self.pools[0].add_partial(bucket, object.as_str(), version_id).await; self.pools[0].add_partial(bucket, object.as_str(), version_id).await;
} }
let idx = self.get_pool_idx_existing_with_opts(bucket, object.as_str(), &ObjectOptions::default()).await?; let idx = self
.get_pool_idx_existing_with_opts(bucket, object.as_str(), &ObjectOptions::default())
.await?;
self.pools[idx].add_partial(bucket, object.as_str(), version_id).await; self.pools[idx].add_partial(bucket, object.as_str(), version_id).await;
Ok(()) Ok(())
@@ -2134,7 +2133,9 @@ impl StorageAPI for ECStore {
let object = rustfs_utils::path::encode_dir_object(object); let object = rustfs_utils::path::encode_dir_object(object);
if self.single_pool() { if self.single_pool() {
return self.pools[0].delete_object_version(bucket, object.as_str(), fi, force_del_marker).await; return self.pools[0]
.delete_object_version(bucket, object.as_str(), fi, force_del_marker)
.await;
} }
Ok(()) Ok(())
} }
+8 -8
View File
@@ -2,22 +2,19 @@ use crate::bucket::metadata_sys::get_versioning_config;
use crate::bucket::versioning::VersioningApi as _; use crate::bucket::versioning::VersioningApi as _;
use crate::cmd::bucket_replication::{ReplicationStatusType, VersionPurgeStatusType}; use crate::cmd::bucket_replication::{ReplicationStatusType, VersionPurgeStatusType};
use crate::error::{Error, Result}; use crate::error::{Error, Result};
use rustfs_utils::path::decode_dir_object;
use crate::heal::heal_ops::HealSequence; use crate::heal::heal_ops::HealSequence;
use crate::store_utils::clean_metadata; use crate::store_utils::clean_metadata;
use crate::{disk::DiskStore, heal::heal_commands::HealOpts,};
use crate::{ use crate::{
bucket::lifecycle::{
lifecycle::TransitionOptions,
bucket_lifecycle_ops::TransitionedObject,
},
bucket::lifecycle::bucket_lifecycle_audit::LcAuditEvent, bucket::lifecycle::bucket_lifecycle_audit::LcAuditEvent,
bucket::lifecycle::lifecycle::ExpirationOptions, bucket::lifecycle::lifecycle::ExpirationOptions,
bucket::lifecycle::{bucket_lifecycle_ops::TransitionedObject, lifecycle::TransitionOptions},
}; };
use crate::{disk::DiskStore, heal::heal_commands::HealOpts};
use http::{HeaderMap, HeaderValue}; use http::{HeaderMap, HeaderValue};
use madmin::heal_commands::HealResultItem; use madmin::heal_commands::HealResultItem;
use rustfs_filemeta::{FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, headers::AMZ_OBJECT_TAGGING}; use rustfs_filemeta::{FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, headers::AMZ_OBJECT_TAGGING};
use rustfs_rio::{HashReader, Reader}; use rustfs_rio::{HashReader, Reader};
use rustfs_utils::path::decode_dir_object;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::collections::HashMap; use std::collections::HashMap;
use std::fmt::Debug; use std::fmt::Debug;
@@ -447,13 +444,16 @@ impl ObjectInfo {
let transitioned_object = TransitionedObject { let transitioned_object = TransitionedObject {
name: fi.transitioned_objname.clone(), name: fi.transitioned_objname.clone(),
version_id: if let Some(transition_version_id) = fi.transition_version_id { transition_version_id.to_string() } else { "".to_string() }, version_id: if let Some(transition_version_id) = fi.transition_version_id {
transition_version_id.to_string()
} else {
"".to_string()
},
status: fi.transition_status.clone(), status: fi.transition_status.clone(),
free_version: fi.tier_free_version(), free_version: fi.tier_free_version(),
tier: fi.transition_tier.clone(), tier: fi.transition_tier.clone(),
}; };
let metadata = { let metadata = {
let mut v = fi.metadata.clone(); let mut v = fi.metadata.clone();
clean_metadata(&mut v); clean_metadata(&mut v);
+5 -5
View File
@@ -1,9 +1,9 @@
pub mod warm_backend_s3; pub mod tier;
pub mod warm_backend_minio;
pub mod warm_backend_rustfs;
pub mod warm_backend;
pub mod tier_admin; pub mod tier_admin;
pub mod tier_config; pub mod tier_config;
pub mod tier;
pub mod tier_gen; pub mod tier_gen;
pub mod tier_handlers; pub mod tier_handlers;
pub mod warm_backend;
pub mod warm_backend_minio;
pub mod warm_backend_rustfs;
pub mod warm_backend_s3;
+47 -35
View File
@@ -1,38 +1,41 @@
use std::{
collections::{hash_map::Entry, HashMap}, io::Cursor, sync::Arc, time::{Duration,}
};
use bytes::Bytes; use bytes::Bytes;
use serde::{Serialize, Deserialize};
use time::OffsetDateTime;
use tokio::{select, sync::RwLock, time::interval};
use rand::Rng;
use tracing::{info, debug, warn, error};
use http::status::StatusCode; use http::status::StatusCode;
use lazy_static::lazy_static; use lazy_static::lazy_static;
use rand::Rng;
use serde::{Deserialize, Serialize};
use std::{
collections::{HashMap, hash_map::Entry},
io::Cursor,
sync::Arc,
time::Duration,
};
use time::OffsetDateTime;
use tokio::io::BufReader; use tokio::io::BufReader;
use tokio::{select, sync::RwLock, time::interval};
use tracing::{debug, error, info, warn};
use s3s::S3ErrorCode;
use crate::error::{Error, Result, StorageError};
use rustfs_utils::path::{path_join, SLASH_SEPARATOR};
use crate::{
config::com::{read_config, CONFIG_PREFIX},
disk::RUSTFS_META_BUCKET,
store::ECStore, store_api::{ObjectOptions, PutObjReader}, StorageAPI
};
use crate::client::admin_handler_utils::AdminError; use crate::client::admin_handler_utils::AdminError;
use crate::tier::{ use crate::error::{Error, Result, StorageError};
warm_backend::{check_warm_backend, new_warm_backend},
tier_handlers::{
ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_ALREADY_EXISTS, ERR_TIER_NOT_FOUND,
},
tier_admin::TierCreds,
tier_config::{TierType, TierConfig,},
};
use crate::new_object_layer_fn; use crate::new_object_layer_fn;
use crate::tier::{
tier_admin::TierCreds,
tier_config::{TierConfig, TierType},
tier_handlers::{ERR_TIER_ALREADY_EXISTS, ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_NOT_FOUND},
warm_backend::{check_warm_backend, new_warm_backend},
};
use crate::{
StorageAPI,
config::com::{CONFIG_PREFIX, read_config},
disk::RUSTFS_META_BUCKET,
store::ECStore,
store_api::{ObjectOptions, PutObjReader},
};
use rustfs_rio::HashReader; use rustfs_rio::HashReader;
use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
use s3s::S3ErrorCode;
use super::{ use super::{
tier_handlers::{ERR_TIER_PERM_ERR, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_BUCKET_NOT_FOUND}, tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_PERM_ERR},
warm_backend::WarmBackendImpl, warm_backend::WarmBackendImpl,
}; };
@@ -279,7 +282,7 @@ impl TierConfigMgr {
minio.access_key = creds.access_key; minio.access_key = creds.access_key;
minio.secret_key = creds.secret_key; minio.secret_key = creds.secret_key;
} }
_ => () _ => (),
} }
let d = new_warm_backend(&cfg, true).await?; let d = new_warm_backend(&cfg, true).await?;
@@ -290,9 +293,7 @@ impl TierConfigMgr {
pub async fn get_driver<'a>(&'a mut self, tier_name: &str) -> std::result::Result<&'a WarmBackendImpl, AdminError> { pub async fn get_driver<'a>(&'a mut self, tier_name: &str) -> std::result::Result<&'a WarmBackendImpl, AdminError> {
Ok(match self.driver_cache.entry(tier_name.to_string()) { Ok(match self.driver_cache.entry(tier_name.to_string()) {
Entry::Occupied(e) => { Entry::Occupied(e) => e.into_mut(),
e.into_mut()
}
Entry::Vacant(e) => { Entry::Vacant(e) => {
let t = self.tiers.get(tier_name); let t = self.tiers.get(tier_name);
if t.is_none() { if t.is_none() {
@@ -326,7 +327,9 @@ impl TierConfigMgr {
#[tracing::instrument(level = "debug", name = "tier_save", skip(self))] #[tracing::instrument(level = "debug", name = "tier_save", skip(self))]
pub async fn save(&self) -> std::result::Result<(), std::io::Error> { pub async fn save(&self) -> std::result::Result<(), std::io::Error> {
let Some(api) = new_object_layer_fn() else { return Err(std::io::Error::other("errServerNotInitialized")) }; let Some(api) = new_object_layer_fn() else {
return Err(std::io::Error::other("errServerNotInitialized"));
};
//let (pr, opts) = GLOBAL_TierConfigMgr.write().config_reader()?; //let (pr, opts) = GLOBAL_TierConfigMgr.write().config_reader()?;
self.save_tiering_config(api).await self.save_tiering_config(api).await
@@ -340,7 +343,12 @@ impl TierConfigMgr {
self.save_config(api, &config_file, data).await self.save_config(api, &config_file, data).await
} }
pub async fn save_config<S: StorageAPI>(&self, api: Arc<S>, file: &str, data: Bytes) -> std::result::Result<(), std::io::Error> { pub async fn save_config<S: StorageAPI>(
&self,
api: Arc<S>,
file: &str,
data: Bytes,
) -> std::result::Result<(), std::io::Error> {
self.save_config_with_opts( self.save_config_with_opts(
api, api,
file, file,
@@ -353,7 +361,13 @@ impl TierConfigMgr {
.await .await
} }
pub async fn save_config_with_opts<S: StorageAPI>(&self, api: Arc<S>, file: &str, data: Bytes, opts: &ObjectOptions) -> std::result::Result<(), std::io::Error> { pub async fn save_config_with_opts<S: StorageAPI>(
&self,
api: Arc<S>,
file: &str,
data: Bytes,
opts: &ObjectOptions,
) -> std::result::Result<(), std::io::Error> {
debug!("save tier config:{}", file); debug!("save tier config:{}", file);
let _ = api let _ = api
.put_object(RUSTFS_META_BUCKET, file, &mut PutObjReader::from_vec(data.to_vec()), opts) .put_object(RUSTFS_META_BUCKET, file, &mut PutObjReader::from_vec(data.to_vec()), opts)
@@ -365,9 +379,7 @@ impl TierConfigMgr {
//let r = rand.New(rand.NewSource(time.Now().UnixNano())); //let r = rand.New(rand.NewSource(time.Now().UnixNano()));
let mut rng = rand::rng(); let mut rng = rand::rng();
let r = rng.random_range(0.0..1.0); let r = rng.random_range(0.0..1.0);
let rand_interval = || { let rand_interval = || Duration::from_secs((r * 60_f64).round() as u64);
Duration::from_secs((r * 60_f64).round() as u64)
};
let mut t = interval(TIER_CFG_REFRESH + rand_interval()); let mut t = interval(TIER_CFG_REFRESH + rand_interval());
loop { loop {
+5 -8
View File
@@ -1,13 +1,10 @@
use std::{
time::{Duration, SystemTime, UNIX_EPOCH},
};
use rand::Rng;
use tracing::warn;
use http::status::StatusCode; use http::status::StatusCode;
use serde::{Serialize, Deserialize}; use rand::Rng;
use serde::{Deserialize, Serialize};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use tracing::warn;
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[derive(Default, Debug, Clone)]
#[serde(default)] #[serde(default)]
pub struct TierCreds { pub struct TierCreds {
#[serde(rename = "accessKey")] #[serde(rename = "accessKey")]
+36 -73
View File
@@ -1,5 +1,5 @@
use serde::{Deserialize, Serialize};
use std::fmt::Display; use std::fmt::Display;
use serde::{Serialize, Deserialize};
use tracing::info; use tracing::info;
const C_TierConfigVer: &str = "v1"; const C_TierConfigVer: &str = "v1";
@@ -9,8 +9,7 @@ const ERR_TIER_INVALID_CONFIG: &str = "invalid tier config";
const ERR_TIER_INVALID_CONFIG_VERSION: &str = "invalid tier config version"; const ERR_TIER_INVALID_CONFIG_VERSION: &str = "invalid tier config version";
const ERR_TIER_TYPE_UNSUPPORTED: &str = "unsupported tier type"; const ERR_TIER_TYPE_UNSUPPORTED: &str = "unsupported tier type";
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[derive(Default, Debug, Clone)]
pub enum TierType { pub enum TierType {
#[default] #[default]
Unsupported, Unsupported,
@@ -48,35 +47,19 @@ impl Display for TierType {
impl TierType { impl TierType {
pub fn new(sc_type: &str) -> Self { pub fn new(sc_type: &str) -> Self {
match sc_type { match sc_type {
"S3" => { "S3" => TierType::S3,
TierType::S3 "RustFS" => TierType::RustFS,
} "MinIO" => TierType::MinIO,
"RustFS" => { _ => TierType::Unsupported,
TierType::RustFS
}
"MinIO" => {
TierType::MinIO
}
_ => {
TierType::Unsupported
}
} }
} }
pub fn to_string(&self) -> String { pub fn to_string(&self) -> String {
match self { match self {
TierType::S3 => { TierType::S3 => "s3".to_string(),
"s3".to_string() TierType::RustFS => "rustfs".to_string(),
} TierType::MinIO => "minio".to_string(),
TierType::RustFS => { _ => "unsupported".to_string(),
"rustfs".to_string()
}
TierType::MinIO => {
"minio".to_string()
}
_ => {
"unsupported".to_string()
}
} }
} }
} }
@@ -123,7 +106,7 @@ impl Clone for TierConfig {
m_.secret_key = "REDACTED".to_string(); m_.secret_key = "REDACTED".to_string();
m = Some(m_); m = Some(m_);
} }
_ => () _ => (),
} }
TierConfig { TierConfig {
version: self.version.clone(), version: self.version.clone(),
@@ -154,15 +137,9 @@ impl TierConfig {
fn endpoint(&self) -> String { fn endpoint(&self) -> String {
match self.tier_type { match self.tier_type {
TierType::S3 => { TierType::S3 => self.s3.as_ref().expect("err").endpoint.clone(),
self.s3.as_ref().expect("err").endpoint.clone() TierType::RustFS => self.rustfs.as_ref().expect("err").endpoint.clone(),
} TierType::MinIO => self.minio.as_ref().expect("err").endpoint.clone(),
TierType::RustFS => {
self.rustfs.as_ref().expect("err").endpoint.clone()
}
TierType::MinIO => {
self.minio.as_ref().expect("err").endpoint.clone()
}
_ => { _ => {
info!("unexpected tier type {}", self.tier_type); info!("unexpected tier type {}", self.tier_type);
"".to_string() "".to_string()
@@ -172,15 +149,9 @@ impl TierConfig {
fn bucket(&self) -> String { fn bucket(&self) -> String {
match self.tier_type { match self.tier_type {
TierType::S3 => { TierType::S3 => self.s3.as_ref().expect("err").bucket.clone(),
self.s3.as_ref().expect("err").bucket.clone() TierType::RustFS => self.rustfs.as_ref().expect("err").bucket.clone(),
} TierType::MinIO => self.minio.as_ref().expect("err").bucket.clone(),
TierType::RustFS => {
self.rustfs.as_ref().expect("err").bucket.clone()
}
TierType::MinIO => {
self.minio.as_ref().expect("err").bucket.clone()
}
_ => { _ => {
info!("unexpected tier type {}", self.tier_type); info!("unexpected tier type {}", self.tier_type);
"".to_string() "".to_string()
@@ -190,15 +161,9 @@ impl TierConfig {
fn prefix(&self) -> String { fn prefix(&self) -> String {
match self.tier_type { match self.tier_type {
TierType::S3 => { TierType::S3 => self.s3.as_ref().expect("err").prefix.clone(),
self.s3.as_ref().expect("err").prefix.clone() TierType::RustFS => self.rustfs.as_ref().expect("err").prefix.clone(),
} TierType::MinIO => self.minio.as_ref().expect("err").prefix.clone(),
TierType::RustFS => {
self.rustfs.as_ref().expect("err").prefix.clone()
}
TierType::MinIO => {
self.minio.as_ref().expect("err").prefix.clone()
}
_ => { _ => {
info!("unexpected tier type {}", self.tier_type); info!("unexpected tier type {}", self.tier_type);
"".to_string() "".to_string()
@@ -208,15 +173,9 @@ impl TierConfig {
fn region(&self) -> String { fn region(&self) -> String {
match self.tier_type { match self.tier_type {
TierType::S3 => { TierType::S3 => self.s3.as_ref().expect("err").region.clone(),
self.s3.as_ref().expect("err").region.clone() TierType::RustFS => self.rustfs.as_ref().expect("err").region.clone(),
} TierType::MinIO => self.minio.as_ref().expect("err").region.clone(),
TierType::RustFS => {
self.rustfs.as_ref().expect("err").region.clone()
}
TierType::MinIO => {
self.minio.as_ref().expect("err").region.clone()
}
_ => { _ => {
info!("unexpected tier type {}", self.tier_type); info!("unexpected tier type {}", self.tier_type);
"".to_string() "".to_string()
@@ -227,8 +186,7 @@ impl TierConfig {
//type S3Options = impl Fn(TierS3) -> Pin<Box<Result<()>>> + Send + Sync + 'static; //type S3Options = impl Fn(TierS3) -> Pin<Box<Result<()>>> + Send + Sync + 'static;
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[derive(Default, Debug, Clone)]
#[serde(default)] #[serde(default)]
pub struct TierS3 { pub struct TierS3 {
pub name: String, pub name: String,
@@ -257,7 +215,7 @@ pub struct TierS3 {
impl TierS3 { impl TierS3 {
fn new<F>(name: &str, access_key: &str, secret_key: &str, bucket: &str, options: Vec<F>) -> Result<TierConfig, std::io::Error> fn new<F>(name: &str, access_key: &str, secret_key: &str, bucket: &str, options: Vec<F>) -> Result<TierConfig, std::io::Error>
where where
F: Fn(TierS3) -> Box<Result<(), std::io::Error>> + Send + Sync + 'static F: Fn(TierS3) -> Box<Result<(), std::io::Error>> + Send + Sync + 'static,
{ {
if name == "" { if name == "" {
return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY)); return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY));
@@ -288,8 +246,7 @@ impl TierS3 {
} }
} }
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[derive(Default, Debug, Clone)]
#[serde(default)] #[serde(default)]
pub struct TierRustFS { pub struct TierRustFS {
pub name: String, pub name: String,
@@ -305,8 +262,7 @@ pub struct TierRustFS {
pub storage_class: String, pub storage_class: String,
} }
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[derive(Default, Debug, Clone)]
#[serde(default)] #[serde(default)]
pub struct TierMinIO { pub struct TierMinIO {
pub name: String, pub name: String,
@@ -321,9 +277,16 @@ pub struct TierMinIO {
} }
impl TierMinIO { impl TierMinIO {
fn new<F>(name: &str, endpoint: &str, access_key: &str, secret_key: &str, bucket: &str, options: Vec<F>) -> Result<TierConfig, std::io::Error> fn new<F>(
name: &str,
endpoint: &str,
access_key: &str,
secret_key: &str,
bucket: &str,
options: Vec<F>,
) -> Result<TierConfig, std::io::Error>
where where
F: Fn(TierMinIO) -> Box<Result<(), std::io::Error>> + Send + Sync + 'static F: Fn(TierMinIO) -> Box<Result<(), std::io::Error>> + Send + Sync + 'static,
{ {
if name == "" { if name == "" {
return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY)); return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY));
+1 -1
View File
@@ -1,6 +1,6 @@
use crate::client::admin_handler_utils::AdminError; use crate::client::admin_handler_utils::AdminError;
use tracing::warn;
use http::status::StatusCode; use http::status::StatusCode;
use tracing::warn;
pub const ERR_TIER_ALREADY_EXISTS: AdminError = AdminError { pub const ERR_TIER_ALREADY_EXISTS: AdminError = AdminError {
code: "XRustFSAdminTierAlreadyExists", code: "XRustFSAdminTierAlreadyExists",
+17 -9
View File
@@ -1,20 +1,20 @@
use std::collections::HashMap;
use bytes::Bytes; use bytes::Bytes;
use std::collections::HashMap;
use crate::client::{ use crate::client::{
admin_handler_utils::AdminError, admin_handler_utils::AdminError,
transition_api::{ReadCloser, ReaderImpl,}, transition_api::{ReadCloser, ReaderImpl},
}; };
use crate::error::is_err_bucket_not_found; use crate::error::is_err_bucket_not_found;
use tracing::{info, warn};
use crate::tier::{ use crate::tier::{
tier_config::{TierType, TierConfig}, tier::{ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED},
tier_config::{TierConfig, TierType},
tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_PERM_ERR}, tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_PERM_ERR},
tier::{ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED,},
warm_backend_s3::WarmBackendS3,
warm_backend_rustfs::WarmBackendRustFS,
warm_backend_minio::WarmBackendMinIO, warm_backend_minio::WarmBackendMinIO,
warm_backend_rustfs::WarmBackendRustFS,
warm_backend_s3::WarmBackendS3,
}; };
use tracing::{info, warn};
pub type WarmBackendImpl = Box<dyn WarmBackend + Send + Sync + 'static>; pub type WarmBackendImpl = Box<dyn WarmBackend + Send + Sync + 'static>;
@@ -29,7 +29,13 @@ pub struct WarmBackendGetOpts {
#[async_trait::async_trait] #[async_trait::async_trait]
pub trait WarmBackend { pub trait WarmBackend {
async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error>; async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error>;
async fn put_with_meta(&self, object: &str, r: ReaderImpl, length: i64, meta: HashMap<String, String>) -> Result<String, std::io::Error>; async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
meta: HashMap<String, String>,
) -> Result<String, std::io::Error>;
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error>; async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error>;
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error>; async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error>;
async fn in_use(&self) -> Result<bool, std::io::Error>; async fn in_use(&self) -> Result<bool, std::io::Error>;
@@ -37,7 +43,9 @@ pub trait WarmBackend {
pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> { pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> {
let w = w.expect("err"); let w = w.expect("err");
let remote_version_id = w.put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5).await; let remote_version_id = w
.put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5)
.await;
if let Err(err) = remote_version_id { if let Err(err) = remote_version_id {
return Err(ERR_TIER_PERM_ERR); return Err(ERR_TIER_PERM_ERR);
} }
+23 -13
View File
@@ -1,19 +1,18 @@
use std::collections::HashMap; use std::collections::HashMap;
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use tracing::warn;
use crate::client::{ use crate::client::{
admin_handler_utils::AdminError, admin_handler_utils::AdminError,
transition_api::{Options, ReaderImpl, ReadCloser, TransitionClient, TransitionCore},
credentials::{Credentials, SignatureType, Static, Value},
api_put_object::PutObjectOptions, api_put_object::PutObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
}; };
use crate::tier::{ use crate::tier::{
tier_config::TierMinIO, tier_config::TierMinIO,
warm_backend::{WarmBackend, WarmBackendGetOpts}, warm_backend::{WarmBackend, WarmBackendGetOpts},
warm_backend_s3::WarmBackendS3, warm_backend_s3::WarmBackendS3,
}; };
use tracing::warn;
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
const MAX_PARTS_COUNT: i64 = 10000; const MAX_PARTS_COUNT: i64 = 10000;
@@ -54,12 +53,9 @@ impl WarmBackendMinIO {
..Default::default() ..Default::default()
}; };
let scheme = u.scheme(); let scheme = u.scheme();
let default_port = if scheme == "https" { let default_port = if scheme == "https" { 443 } else { 80 };
443 let client =
} else { TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?;
80
};
let client = TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?;
//client.set_appinfo(format!("minio-tier-{}", tier), ReleaseTag); //client.set_appinfo(format!("minio-tier-{}", tier), ReleaseTag);
let client = Arc::new(client); let client = Arc::new(client);
@@ -76,16 +72,30 @@ impl WarmBackendMinIO {
#[async_trait::async_trait] #[async_trait::async_trait]
impl WarmBackend for WarmBackendMinIO { impl WarmBackend for WarmBackendMinIO {
async fn put_with_meta(&self, object: &str, r: ReaderImpl, length: i64, meta: HashMap<String, String>) -> Result<String, std::io::Error> { async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
meta: HashMap<String, String>,
) -> Result<String, std::io::Error> {
let part_size = optimal_part_size(length)?; let part_size = optimal_part_size(length)?;
let client = self.0.client.clone(); let client = self.0.client.clone();
let res = client.put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &PutObjectOptions { let res = client
.put_object(
&self.0.bucket,
&self.0.get_dest(object),
r,
length,
&PutObjectOptions {
storage_class: self.0.storage_class.clone(), storage_class: self.0.storage_class.clone(),
part_size: part_size as u64, part_size: part_size as u64,
disable_content_sha256: true, disable_content_sha256: true,
user_metadata: meta, user_metadata: meta,
..Default::default() ..Default::default()
}).await?; },
)
.await?;
//self.ToObjectError(err, object) //self.ToObjectError(err, object)
Ok(res.version_id) Ok(res.version_id)
} }
+22 -11
View File
@@ -4,9 +4,9 @@ use tracing::warn;
use crate::client::{ use crate::client::{
admin_handler_utils::AdminError, admin_handler_utils::AdminError,
transition_api::{Options, ReaderImpl, ReadCloser, TransitionClient, TransitionCore},
credentials::{Credentials, SignatureType, Static, Value},
api_put_object::PutObjectOptions, api_put_object::PutObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
}; };
use crate::tier::{ use crate::tier::{
tier_config::TierRustFS, tier_config::TierRustFS,
@@ -51,12 +51,9 @@ impl WarmBackendRustFS {
..Default::default() ..Default::default()
}; };
let scheme = u.scheme(); let scheme = u.scheme();
let default_port = if scheme == "https" { let default_port = if scheme == "https" { 443 } else { 80 };
443 let client =
} else { TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?;
80
};
let client = TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?;
//client.set_appinfo(format!("rustfs-tier-{}", tier), ReleaseTag); //client.set_appinfo(format!("rustfs-tier-{}", tier), ReleaseTag);
let client = Arc::new(client); let client = Arc::new(client);
@@ -73,16 +70,30 @@ impl WarmBackendRustFS {
#[async_trait::async_trait] #[async_trait::async_trait]
impl WarmBackend for WarmBackendRustFS { impl WarmBackend for WarmBackendRustFS {
async fn put_with_meta(&self, object: &str, r: ReaderImpl, length: i64, meta: HashMap<String, String>) -> Result<String, std::io::Error> { async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
meta: HashMap<String, String>,
) -> Result<String, std::io::Error> {
let part_size = optimal_part_size(length)?; let part_size = optimal_part_size(length)?;
let client = self.0.client.clone(); let client = self.0.client.clone();
let res = client.put_object(&self.0.bucket, &self.0.get_dest(object), r, length, &PutObjectOptions { let res = client
.put_object(
&self.0.bucket,
&self.0.get_dest(object),
r,
length,
&PutObjectOptions {
storage_class: self.0.storage_class.clone(), storage_class: self.0.storage_class.clone(),
part_size: part_size as u64, part_size: part_size as u64,
disable_content_sha256: true, disable_content_sha256: true,
user_metadata: meta, user_metadata: meta,
..Default::default() ..Default::default()
}).await?; },
)
.await?;
//self.ToObjectError(err, object) //self.ToObjectError(err, object)
Ok(res.version_id) Ok(res.version_id)
} }
+43 -21
View File
@@ -2,21 +2,21 @@ use std::collections::HashMap;
use std::sync::Arc; use std::sync::Arc;
use url::Url; use url::Url;
use crate::error::ErrorResponse;
use crate::error::error_resp_to_object_err;
use crate::client::{ use crate::client::{
api_get_options::GetObjectOptions, api_get_options::GetObjectOptions,
credentials::{Credentials, Static, Value, SignatureType},
transition_api::{ReaderImpl, ReadCloser},
api_put_object::PutObjectOptions, api_put_object::PutObjectOptions,
api_remove::RemoveObjectOptions, api_remove::RemoveObjectOptions,
transition_api::{Options, TransitionClient, TransitionCore,}, credentials::{Credentials, SignatureType, Static, Value},
transition_api::{Options, TransitionClient, TransitionCore},
transition_api::{ReadCloser, ReaderImpl},
}; };
use rustfs_utils::path::SLASH_SEPARATOR; use crate::error::ErrorResponse;
use crate::error::error_resp_to_object_err;
use crate::tier::{ use crate::tier::{
tier_config::TierS3, tier_config::TierS3,
warm_backend::{WarmBackend, WarmBackendGetOpts,} warm_backend::{WarmBackend, WarmBackendGetOpts},
}; };
use rustfs_utils::path::SLASH_SEPARATOR;
pub struct WarmBackendS3 { pub struct WarmBackendS3 {
pub client: Arc<TransitionClient>, pub client: Arc<TransitionClient>,
@@ -35,16 +35,22 @@ impl WarmBackendS3 {
} }
}; };
if conf.aws_role_web_identity_token_file == "" && conf.aws_role_arn != "" || conf.aws_role_web_identity_token_file != "" && conf.aws_role_arn == "" { if conf.aws_role_web_identity_token_file == "" && conf.aws_role_arn != ""
|| conf.aws_role_web_identity_token_file != "" && conf.aws_role_arn == ""
{
return Err(std::io::Error::other("both the token file and the role ARN are required")); return Err(std::io::Error::other("both the token file and the role ARN are required"));
} } else if conf.access_key == "" && conf.secret_key != "" || conf.access_key != "" && conf.secret_key == "" {
else if conf.access_key == "" && conf.secret_key != "" || conf.access_key != "" && conf.secret_key == "" {
return Err(std::io::Error::other("both the access and secret keys are required")); return Err(std::io::Error::other("both the access and secret keys are required"));
} } else if conf.aws_role
else if conf.aws_role && (conf.aws_role_web_identity_token_file != "" || conf.aws_role_arn != "" || conf.access_key != "" || conf.secret_key != "") { && (conf.aws_role_web_identity_token_file != ""
return Err(std::io::Error::other("AWS Role cannot be activated with static credentials or the web identity token file")); || conf.aws_role_arn != ""
} || conf.access_key != ""
else if conf.bucket == "" { || conf.secret_key != "")
{
return Err(std::io::Error::other(
"AWS Role cannot be activated with static credentials or the web identity token file",
));
} else if conf.bucket == "" {
return Err(std::io::Error::other("no bucket name was provided")); return Err(std::io::Error::other("no bucket name was provided"));
} }
@@ -59,8 +65,7 @@ impl WarmBackendS3 {
signer_type: SignatureType::SignatureV4, signer_type: SignatureType::SignatureV4,
..Default::default() ..Default::default()
})); }));
} } else {
else {
return Err(std::io::Error::other("insufficient parameters for S3 backend authentication")); return Err(std::io::Error::other("insufficient parameters for S3 backend authentication"));
} }
let opts = Options { let opts = Options {
@@ -103,14 +108,28 @@ impl WarmBackendS3 {
#[async_trait::async_trait] #[async_trait::async_trait]
impl WarmBackend for WarmBackendS3 { impl WarmBackend for WarmBackendS3 {
async fn put_with_meta(&self, object: &str, r: ReaderImpl, length: i64, meta: HashMap<String, String>) -> Result<String, std::io::Error> { async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
meta: HashMap<String, String>,
) -> Result<String, std::io::Error> {
let client = self.client.clone(); let client = self.client.clone();
let res = client.put_object(&self.bucket, &self.get_dest(object), r, length, &PutObjectOptions { let res = client
.put_object(
&self.bucket,
&self.get_dest(object),
r,
length,
&PutObjectOptions {
send_content_md5: true, send_content_md5: true,
storage_class: self.storage_class.clone(), storage_class: self.storage_class.clone(),
user_metadata: meta, user_metadata: meta,
..Default::default() ..Default::default()
}).await?; },
)
.await?;
Ok(res.version_id) Ok(res.version_id)
} }
@@ -146,7 +165,10 @@ impl WarmBackend for WarmBackendS3 {
} }
async fn in_use(&self) -> Result<bool, std::io::Error> { async fn in_use(&self) -> Result<bool, std::io::Error> {
let result = self.core.list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1).await?; let result = self
.core
.list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1)
.await?;
Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0) Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)
} }
+1 -1
View File
@@ -1,7 +1,7 @@
use md5::{Digest as Md5Digest, Md5}; use md5::{Digest as Md5Digest, Md5};
use sha2::{ use sha2::{
digest::{Reset, Update},
Digest, Sha256 as sha_sha256, Digest, Sha256 as sha_sha256,
digest::{Reset, Update},
}; };
pub trait Hasher { pub trait Hasher {
fn write(&mut self, bytes: &[u8]); fn write(&mut self, bytes: &[u8]);
+4 -16
View File
@@ -13,9 +13,9 @@ use crate::{
}; };
// use futures::stream::Stream; // use futures::stream::Stream;
use std::io::{Error, Result}; use super::hasher::{Hasher, MD5, Sha256};
use super::hasher::{Hasher, Sha256, MD5};
use futures::Stream; use futures::Stream;
use std::io::{Error, Result};
pin_project_lite::pin_project! { pin_project_lite::pin_project! {
#[derive(Default)] #[derive(Default)]
@@ -101,20 +101,8 @@ pin_project_lite::pin_project! {
impl<S> HashReader<S> { impl<S> HashReader<S> {
pub fn new(inner: S, size: usize, md5_hex: Option<String>, sha256_hex: Option<String>, actual_size: usize) -> Self { pub fn new(inner: S, size: usize, md5_hex: Option<String>, sha256_hex: Option<String>, actual_size: usize) -> Self {
let md5 = { let md5 = { if md5_hex.is_some() { Some(MD5::new()) } else { None } };
if md5_hex.is_some() { let sha256 = { if sha256_hex.is_some() { Some(Sha256::new()) } else { None } };
Some(MD5::new())
} else {
None
}
};
let sha256 = {
if sha256_hex.is_some() {
Some(Sha256::new())
} else {
None
}
};
Self { Self {
inner, inner,
size, size,
+1 -1
View File
@@ -63,9 +63,9 @@ pub mod pools;
pub mod rebalance; pub mod rebalance;
pub mod service_account; pub mod service_account;
pub mod sts; pub mod sts;
pub mod tier;
pub mod trace; pub mod trace;
pub mod user; pub mod user;
pub mod tier;
use urlencoding::decode; use urlencoding::decode;
#[derive(Debug, Serialize, Default)] #[derive(Debug, Serialize, Default)]
+35 -14
View File
@@ -3,28 +3,28 @@ use std::str::from_utf8;
use http::{HeaderMap, StatusCode}; use http::{HeaderMap, StatusCode};
use iam::get_global_action_cred; use iam::get_global_action_cred;
use matchit::Params; use matchit::Params;
use s3s::{header::CONTENT_TYPE, s3_error, Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result}; use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
use serde::Deserialize; use serde::Deserialize;
use serde_urlencoded::from_bytes; use serde_urlencoded::from_bytes;
use tracing::{warn, debug, info}; use tracing::{debug, info, warn};
use crypto::{encrypt_data, decrypt_data};
use crate::{ use crate::{
admin::{router::Operation, utils::has_space_be}, admin::{router::Operation, utils::has_space_be},
auth::{check_key_valid, get_session_token}, auth::{check_key_valid, get_session_token},
}; };
use crypto::{decrypt_data, encrypt_data};
use ecstore::{ use ecstore::{
client::admin_handler_utils::AdminError, client::admin_handler_utils::AdminError,
config::storageclass, config::storageclass,
global::GLOBAL_TierConfigMgr, global::GLOBAL_TierConfigMgr,
tier::{ tier::{
tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_MISSING_CREDENTIALS},
tier_admin::TierCreds, tier_admin::TierCreds,
tier_config::{TierConfig, TierType}, tier_config::{TierConfig, TierType},
tier_handlers::{ tier_handlers::{
ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_ALREADY_EXISTS, ERR_TIER_NOT_FOUND, ERR_TIER_CONNECT_ERR, ERR_TIER_ALREADY_EXISTS, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_NAME_NOT_UPPERCASE,
ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_NOT_FOUND,
}, },
tier::{ERR_TIER_MISSING_CREDENTIALS, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_BACKEND_IN_USE},
}, },
}; };
@@ -95,26 +95,41 @@ impl Operation for AddTier {
warn!("tier reserved name, args.name: {}", args.name); warn!("tier reserved name, args.name: {}", args.name);
return Err(s3_error!(InvalidRequest, "Cannot use reserved tier name")); return Err(s3_error!(InvalidRequest, "Cannot use reserved tier name"));
} }
&_ => () &_ => (),
} }
let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await; let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await;
//tier_config_mgr.reload(api); //tier_config_mgr.reload(api);
match tier_config_mgr.add(args, force).await { match tier_config_mgr.add(args, force).await {
Err(ERR_TIER_ALREADY_EXISTS) => { Err(ERR_TIER_ALREADY_EXISTS) => {
return Err(S3Error::with_message(S3ErrorCode::Custom("TierNameAlreadyExist".into()), "tier name already exists!")); return Err(S3Error::with_message(
S3ErrorCode::Custom("TierNameAlreadyExist".into()),
"tier name already exists!",
));
} }
Err(ERR_TIER_NAME_NOT_UPPERCASE) => { Err(ERR_TIER_NAME_NOT_UPPERCASE) => {
return Err(S3Error::with_message(S3ErrorCode::Custom("TierNameNotUppercase".into()), "tier name not uppercase!")); return Err(S3Error::with_message(
S3ErrorCode::Custom("TierNameNotUppercase".into()),
"tier name not uppercase!",
));
} }
Err(ERR_TIER_BACKEND_IN_USE) => { Err(ERR_TIER_BACKEND_IN_USE) => {
return Err(S3Error::with_message(S3ErrorCode::Custom("TierNameBackendInUse!".into()), "tier name backend in use!")); return Err(S3Error::with_message(
S3ErrorCode::Custom("TierNameBackendInUse!".into()),
"tier name backend in use!",
));
} }
Err(ERR_TIER_CONNECT_ERR) => { Err(ERR_TIER_CONNECT_ERR) => {
return Err(S3Error::with_message(S3ErrorCode::Custom("TierConnectError".into()), "tier connect error!")); return Err(S3Error::with_message(
S3ErrorCode::Custom("TierConnectError".into()),
"tier connect error!",
));
} }
Err(ERR_TIER_INVALID_CREDENTIALS) => { Err(ERR_TIER_INVALID_CREDENTIALS) => {
return Err(S3Error::with_message(S3ErrorCode::Custom(ERR_TIER_INVALID_CREDENTIALS.code.into()), ERR_TIER_INVALID_CREDENTIALS.message)); return Err(S3Error::with_message(
S3ErrorCode::Custom(ERR_TIER_INVALID_CREDENTIALS.code.into()),
ERR_TIER_INVALID_CREDENTIALS.message,
));
} }
Err(e) => { Err(e) => {
warn!("tier_config_mgr add failed, e: {:?}", e); warn!("tier_config_mgr add failed, e: {:?}", e);
@@ -182,7 +197,10 @@ impl Operation for EditTier {
return Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found!")); return Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found!"));
} }
Err(ERR_TIER_MISSING_CREDENTIALS) => { Err(ERR_TIER_MISSING_CREDENTIALS) => {
return Err(S3Error::with_message(S3ErrorCode::Custom("TierMissingCredentials".into()), "tier missing credentials!")); return Err(S3Error::with_message(
S3ErrorCode::Custom("TierMissingCredentials".into()),
"tier missing credentials!",
));
} }
Err(e) => { Err(e) => {
warn!("tier_config_mgr edit failed, e: {:?}", e); warn!("tier_config_mgr edit failed, e: {:?}", e);
@@ -281,7 +299,10 @@ impl Operation for RemoveTier {
} }
Err(e) => { Err(e) => {
warn!("tier_config_mgr remove failed, e: {:?}", e); warn!("tier_config_mgr remove failed, e: {:?}", e);
return Err(S3Error::with_message(S3ErrorCode::Custom("TierRemoveFailed".into()), "tier remove failed")); return Err(S3Error::with_message(
S3ErrorCode::Custom("TierRemoveFailed".into()),
"tier remove failed",
));
} }
Ok(_) => (), Ok(_) => (),
} }
+1 -1
View File
@@ -7,7 +7,7 @@ pub mod utils;
use handlers::{ use handlers::{
group, policys, pools, rebalance, group, policys, pools, rebalance,
service_account::{AddServiceAccount, DeleteServiceAccount, InfoServiceAccount, ListServiceAccount, UpdateServiceAccount}, service_account::{AddServiceAccount, DeleteServiceAccount, InfoServiceAccount, ListServiceAccount, UpdateServiceAccount},
sts, user, tier, sts, tier, user,
}; };
use handlers::{GetReplicationMetricsHandler, ListRemoteTargetHandler, RemoveRemoteTargetHandler, SetRemoteTargetHandler}; use handlers::{GetReplicationMetricsHandler, ListRemoteTargetHandler, RemoveRemoteTargetHandler, SetRemoteTargetHandler};
+18 -9
View File
@@ -49,11 +49,11 @@ use ecstore::store_api::ObjectToDelete;
use ecstore::store_api::PutObjReader; use ecstore::store_api::PutObjReader;
use ecstore::store_api::StorageAPI; use ecstore::store_api::StorageAPI;
// use ecstore::store_api::RESERVED_METADATA_PREFIX; // use ecstore::store_api::RESERVED_METADATA_PREFIX;
use ecstore::bucket::lifecycle::bucket_lifecycle_ops::validate_transition_tier;
use ecstore::bucket::utils::serialize; use ecstore::bucket::utils::serialize;
use ecstore::cmd::bucket_replication::ReplicationStatusType; use ecstore::cmd::bucket_replication::ReplicationStatusType;
use ecstore::cmd::bucket_replication::ReplicationType; use ecstore::cmd::bucket_replication::ReplicationType;
use ecstore::store_api::RESERVED_METADATA_PREFIX_LOWER; use ecstore::store_api::RESERVED_METADATA_PREFIX_LOWER;
use ecstore::bucket::lifecycle::bucket_lifecycle_ops::validate_transition_tier;
use futures::pin_mut; use futures::pin_mut;
use futures::{Stream, StreamExt}; use futures::{Stream, StreamExt};
use http::HeaderMap; use http::HeaderMap;
@@ -96,10 +96,7 @@ use transform_stream::AsyncTryStream;
use uuid::Uuid; use uuid::Uuid;
use ecstore::bucket::{ use ecstore::bucket::{
lifecycle::{ lifecycle::{bucket_lifecycle_ops::ERR_INVALID_STORAGECLASS, lifecycle::Lifecycle},
lifecycle::Lifecycle,
bucket_lifecycle_ops::ERR_INVALID_STORAGECLASS
},
object_lock::objectlock_sys::BucketObjectLockSys, object_lock::objectlock_sys::BucketObjectLockSys,
}; };
@@ -1602,7 +1599,10 @@ impl S3 for FS {
let rcfg = metadata_sys::get_object_lock_config(&bucket).await; let rcfg = metadata_sys::get_object_lock_config(&bucket).await;
if rcfg.is_err() { if rcfg.is_err() {
return Err(S3Error::with_message(S3ErrorCode::Custom("BucketLockIsNotExist".into()), "bucket lock is not exist.")); return Err(S3Error::with_message(
S3ErrorCode::Custom("BucketLockIsNotExist".into()),
"bucket lock is not exist.",
));
} }
let rcfg = rcfg.expect("get_lifecycle_config err!").0; let rcfg = rcfg.expect("get_lifecycle_config err!").0;
@@ -1612,12 +1612,18 @@ impl S3 for FS {
if let Err(err) = input_cfg.validate(&rcfg).await { if let Err(err) = input_cfg.validate(&rcfg).await {
//return Err(S3Error::with_message(S3ErrorCode::Custom("BucketLockValidateFailed".into()), "bucket lock validate failed.")); //return Err(S3Error::with_message(S3ErrorCode::Custom("BucketLockValidateFailed".into()), "bucket lock validate failed."));
return Err(S3Error::with_message(S3ErrorCode::Custom("ValidateFailed".into()), format!("{}", err.to_string()))); return Err(S3Error::with_message(
S3ErrorCode::Custom("ValidateFailed".into()),
format!("{}", err.to_string()),
));
} }
if let Err(err) = validate_transition_tier(&input_cfg).await { if let Err(err) = validate_transition_tier(&input_cfg).await {
//warn!("lifecycle_configuration add failed, err: {:?}", err); //warn!("lifecycle_configuration add failed, err: {:?}", err);
return Err(S3Error::with_message(S3ErrorCode::Custom("CustomError".into()), format!("{}", err.to_string()))); return Err(S3Error::with_message(
S3ErrorCode::Custom("CustomError".into()),
format!("{}", err.to_string()),
));
} }
let data = try_!(serialize(&input_cfg)); let data = try_!(serialize(&input_cfg));
@@ -2070,7 +2076,10 @@ impl S3 for FS {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
}; };
if let Err(e) = store.get_object_reader(&bucket, &key, None, HeaderMap::new(), &ObjectOptions::default()).await { if let Err(e) = store
.get_object_reader(&bucket, &key, None, HeaderMap::new(), &ObjectOptions::default())
.await
{
return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{}", e))); return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{}", e)));
} }