gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-13 00:26:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix clippy

Browse Source
This commit is contained in:
likewu
2025-06-23 12:03:46 +08:00
parent cc71f40a6d
commit 2c176dd864
70 changed files with 2779 additions and 1969 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ecstore/src/bucket/lifecycle/bucket_lifecycle_audit.rs
+2 -5
View File
@@ -24,9 +24,6 @@ pub struct LcAuditEvent {
impl LcAuditEvent {
pub fn new(event: lifecycle::Event, source: LcEventSrc) -> Self {
Self {
event,
source,
}
Self { event, source }
}
}
}
ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+114 -86
View File
@@ -1,53 +1,49 @@
use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
use futures::Future;
use http::HeaderMap;
use lazy_static::lazy_static;
use s3s::Body;
use sha2::{Digest, Sha256};
use std::any::{Any, TypeId};
use std::collections::HashMap;
use std::env;
use std::io::{Cursor, Write};
use std::pin::Pin;
use std::sync::atomic::{AtomicI32, AtomicI64, Ordering};
use std::sync::{Arc, Mutex};
use futures::Future;
use lazy_static::lazy_static;
use s3s::Body;
use std::collections::HashMap;
use tracing::{error, info, warn};
use sha2::{Digest, Sha256};
use xxhash_rust::xxh64;
use uuid::Uuid;
use http::HeaderMap;
use tokio::select;
use tokio::sync::mpsc::{Receiver, Sender};
use tokio::sync::{mpsc, RwLock};
use async_channel::{bounded, Receiver as A_Receiver, Sender as A_Sender};
use tokio::sync::{RwLock, mpsc};
use tracing::{error, info, warn};
use uuid::Uuid;
use xxhash_rust::xxh64;
use s3s::dto::BucketLifecycleConfiguration;
use super::bucket_lifecycle_audit::{LcAuditEvent, LcEventSrc};
use super::lifecycle::{self, ExpirationOptions, IlmAction, Lifecycle, TransitionOptions};
use super::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats};
use super::tier_sweeper::{Jentry, delete_object_from_remote_tier};
use crate::bucket::{metadata_sys::get_lifecycle_config, versioning_sys::BucketVersioningSys};
use crate::client::object_api_utils::new_getobjectreader;
use crate::error::Error;
use crate::event::name::EventName;
use crate::store::ECStore;
use crate::store_api::StorageAPI;
use crate::store_api::{ObjectInfo, ObjectOptions, ObjectToDelete, GetObjectReader, HTTPRangeSpec,};
use crate::error::{error_resp_to_object_err, is_err_object_not_found, is_err_version_not_found, is_network_or_host_down};
use crate::event::name::EventName;
use crate::event_notification::{EventArgs, send_event};
use crate::global::GLOBAL_LocalNodeName;
use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id};
use crate::client::object_api_utils::{new_getobjectreader,};
use crate::event_notification::{send_event, EventArgs};
use crate::heal::{
data_scanner::{apply_expiry_on_non_transitioned_objects, apply_expiry_on_transitioned_object},
data_scanner_metric::ScannerMetrics,
data_scanner::{
apply_expiry_on_transitioned_object, apply_expiry_on_non_transitioned_objects,
},
data_usage_cache::TierStats,
};
use crate::global::GLOBAL_LocalNodeName;
use crate::bucket::{
metadata_sys::get_lifecycle_config,
versioning_sys::BucketVersioningSys,
};
use crate::store::ECStore;
use crate::store_api::StorageAPI;
use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete};
use crate::tier::warm_backend::WarmBackendGetOpts;
use super::lifecycle::{self, ExpirationOptions, IlmAction, Lifecycle, TransitionOptions};
use super::tier_last_day_stats::{LastDayTierStats, DailyAllTierStats};
use super::tier_sweeper::{delete_object_from_remote_tier, Jentry};
use super::bucket_lifecycle_audit::{LcEventSrc, LcAuditEvent};
use s3s::dto::BucketLifecycleConfiguration;
pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
pub type TraceFn = Arc<dyn Fn(String, HashMap<String, String>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
pub type TraceFn =
Arc<dyn Fn(String, HashMap<String, String>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
pub type ExpiryOpType = Box<dyn ExpiryOp + Send + Sync + 'static>;
static XXHASH_SEED: u64 = 0;
@@ -74,8 +70,7 @@ impl LifecycleSys {
Some(lc)
}
pub fn trace(oi: &ObjectInfo) -> TraceFn
{
pub fn trace(oi: &ObjectInfo) -> TraceFn {
todo!();
}
}
@@ -132,11 +127,11 @@ pub trait ExpiryOp: 'static {
#[derive(Debug, Default, Clone)]
pub struct TransitionedObject {
pub name: String,
pub version_id: String,
pub tier: String,
pub name: String,
pub version_id: String,
pub tier: String,
pub free_version: bool,
pub status: String,
pub status: String,
}
struct FreeVersionTask(ObjectInfo);
@@ -181,8 +176,6 @@ pub struct ExpiryState {
stats: Option<ExpiryStats>,
}
impl ExpiryState {
#[allow(clippy::new_ret_no_self)]
pub fn new() -> Arc<RwLock<Self>> {
@@ -203,7 +196,7 @@ impl ExpiryState {
if rxs.len() == 0 {
return 0;
}
let mut tasks=0;
let mut tasks = 0;
for rx in rxs.iter() {
tasks += rx.lock().await.len();
}
@@ -244,7 +237,11 @@ impl ExpiryState {
}
pub async fn enqueue_by_days(&mut self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) {
let task = ExpiryTask {obj_info: oi.clone(), event: event.clone(), src: src.clone()};
let task = ExpiryTask {
obj_info: oi.clone(),
event: event.clone(),
src: src.clone(),
};
let wrkr = self.get_worker_ch(task.op_hash());
if wrkr.is_none() {
*self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
@@ -265,7 +262,11 @@ impl ExpiryState {
return;
}
let task = NewerNoncurrentTask {bucket: String::from(bucket), versions: versions, event: lc_event};
let task = NewerNoncurrentTask {
bucket: String::from(bucket),
versions: versions,
event: lc_event,
};
let wrkr = self.get_worker_ch(task.op_hash());
if wrkr.is_none() {
*self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
@@ -285,7 +286,7 @@ impl ExpiryState {
if self.tasks_tx.len() == 0 {
return None;
}
Some(self.tasks_tx[h as usize %self.tasks_tx.len()].clone())
Some(self.tasks_tx[h as usize % self.tasks_tx.len()].clone())
}
pub async fn resize_workers(n: usize, api: Arc<ECStore>) {
@@ -311,10 +312,10 @@ impl ExpiryState {
let mut l = state.tasks_tx.len();
while l > n {
let worker = state.tasks_tx[l-1].clone();
let worker = state.tasks_tx[l - 1].clone();
worker.send(None).await.unwrap_or(());
state.tasks_tx.remove(l-1);
state.tasks_rx.remove(l-1);
state.tasks_tx.remove(l - 1);
state.tasks_rx.remove(l - 1);
*state.stats.as_mut().expect("err").workers.get_mut() -= 1;
l -= 1;
}
@@ -357,7 +358,7 @@ impl ExpiryState {
else if v.as_any().is::<FreeVersionTask>() {
let v = v.as_any().downcast_ref::<FreeVersionTask>().expect("err!");
let oi = v.0.clone();
}
else {
//info!("Invalid work type - {:?}", v);
@@ -422,7 +423,11 @@ impl TransitionState {
}
pub async fn queue_transition_task(&self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) {
let task = TransitionTask {obj_info: oi.clone(), src: src.clone(), event: event.clone()};
let task = TransitionTask {
obj_info: oi.clone(),
src: src.clone(),
event: event.clone(),
};
select! {
//_ -> t.ctx.Done() => (),
_ = self.transition_tx.send(Some(task)) => (),
@@ -438,8 +443,8 @@ impl TransitionState {
}
pub async fn init(api: Arc<ECStore>) {
let mut n = 10;//globalAPIConfig.getTransitionWorkers();
let tw = 10;//globalILMConfig.getTransitionWorkers();
let mut n = 10; //globalAPIConfig.getTransitionWorkers();
let tw = 10; //globalILMConfig.getTransitionWorkers();
if tw > 0 {
n = tw;
}
@@ -512,8 +517,10 @@ impl TransitionState {
pub fn add_lastday_stats(&self, tier: &str, ts: TierStats) {
let mut tier_stats = self.last_day_stats.lock().unwrap();
tier_stats.entry(tier.to_string()).and_modify(|e| e.add_stats(ts))
.or_insert(LastDayTierStats::default());
tier_stats
.entry(tier.to_string())
.and_modify(|e| e.add_stats(ts))
.or_insert(LastDayTierStats::default());
}
pub fn get_daily_all_tier_stats(&self) -> DailyAllTierStats {
@@ -574,7 +581,10 @@ impl AuditTierOp {
}
pub fn string(&self) -> String {
format!("tier:{},respNS:{},tx:{},err:{}", self.tier, self.time_to_responsens, self.output_bytes, self.error)
format!(
"tier:{},respNS:{},tx:{},err:{}",
self.tier, self.time_to_responsens, self.output_bytes, self.error
)
}
}
@@ -636,16 +646,21 @@ pub async fn enqueue_transition_immediate(oi: &ObjectInfo, src: LcEventSrc) {
}
GLOBAL_TransitionState.queue_transition_task(oi, &event, &src).await;
}
_ => ()
_ => (),
}
}
}
pub async fn expire_transitioned_object(api: Arc<ECStore>, oi: &ObjectInfo, lc_event: &lifecycle::Event, src: &LcEventSrc) -> Result<ObjectInfo, std::io::Error> {
pub async fn expire_transitioned_object(
api: Arc<ECStore>,
oi: &ObjectInfo,
lc_event: &lifecycle::Event,
src: &LcEventSrc,
) -> Result<ObjectInfo, std::io::Error> {
//let traceFn = GLOBAL_LifecycleSys.trace(oi);
let mut opts = ObjectOptions {
versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await,
expiration: ExpirationOptions {expire: true},
versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await,
expiration: ExpirationOptions { expire: true },
..Default::default()
};
if lc_event.action == IlmAction::DeleteVersionAction {
@@ -660,10 +675,15 @@ pub async fn expire_transitioned_object(api: Arc<ECStore>, oi: &ObjectInfo, lc_e
return Ok(dobj);
}
Err(err) => return Err(std::io::Error::other(err)),
}
}
}
let ret = delete_object_from_remote_tier(&oi.transitioned_object.name, &oi.transitioned_object.version_id, &oi.transitioned_object.tier).await;
let ret = delete_object_from_remote_tier(
&oi.transitioned_object.name,
&oi.transitioned_object.version_id,
&oi.transitioned_object.tier,
)
.await;
if ret.is_ok() {
opts.skip_decommissioned = true;
} else {
@@ -679,8 +699,8 @@ pub async fn expire_transitioned_object(api: Arc<ECStore>, oi: &ObjectInfo, lc_e
event_name = EventName::ObjectRemovedDeleteMarkerCreated;
}
let obj_info = ObjectInfo {
name: oi.name.clone(),
version_id: oi.version_id,
name: oi.name.clone(),
version_id: oi.version_id,
delete_marker: oi.delete_marker,
..Default::default()
};
@@ -712,15 +732,15 @@ pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditE
let opts = ObjectOptions {
transition: TransitionOptions {
status: lifecycle::TRANSITION_PENDING.to_string(),
tier: lae.event.storage_class,
etag: oi.etag.clone().expect("err").to_string(),
tier: lae.event.storage_class,
etag: oi.etag.clone().expect("err").to_string(),
..Default::default()
},
//lifecycle_audit_event: lae,
version_id: Some(oi.version_id.expect("err").to_string()),
versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await,
version_suspended: BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await,
mod_time: oi.mod_time,
version_id: Some(oi.version_id.expect("err").to_string()),
versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await,
version_suspended: BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await,
mod_time: oi.mod_time,
..Default::default()
};
time_ilm(1);
@@ -731,7 +751,14 @@ pub fn audit_tier_actions(api: ECStore, tier: &str, bytes: i64) -> TimeFn {
todo!();
}
pub async fn get_transitioned_object_reader(bucket: &str, object: &str, rs: HTTPRangeSpec, h: HeaderMap, oi: ObjectInfo, opts: &ObjectOptions) -> Result<GetObjectReader, std::io::Error> {
pub async fn get_transitioned_object_reader(
bucket: &str,
object: &str,
rs: HTTPRangeSpec,
h: HeaderMap,
oi: ObjectInfo,
opts: &ObjectOptions,
) -> Result<GetObjectReader, std::io::Error> {
let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await;
let tgt_client = match tier_config_mgr.get_driver(&oi.transitioned_object.tier).await {
Ok(d) => d,
@@ -752,7 +779,9 @@ pub async fn get_transitioned_object_reader(bucket: &str, object: &str, rs: HTTP
//return Ok(HttpFileReader::new(rs, &oi, opts, &h));
//timeTierAction := auditTierActions(oi.transitioned_object.Tier, length)
let reader = tgt_client.get(&oi.transitioned_object.name, &oi.transitioned_object.version_id, gopts).await?;
let reader = tgt_client
.get(&oi.transitioned_object.name, &oi.transitioned_object.version_id, gopts)
.await?;
Ok(get_fn(reader, h))
}
@@ -771,18 +800,18 @@ pub trait LifecycleOps {
impl LifecycleOps for ObjectInfo {
fn to_lifecycle_opts(&self) -> lifecycle::ObjectOpts {
lifecycle::ObjectOpts {
name: self.name.clone(),
user_tags: self.user_tags.clone(),
version_id: self.version_id.expect("err").to_string(),
mod_time: self.mod_time,
size: self.size,
is_latest: self.is_latest,
num_versions: self.num_versions,
delete_marker: self.delete_marker,
name: self.name.clone(),
user_tags: self.user_tags.clone(),
version_id: self.version_id.expect("err").to_string(),
mod_time: self.mod_time,
size: self.size,
is_latest: self.is_latest,
num_versions: self.num_versions,
delete_marker: self.delete_marker,
successor_mod_time: self.successor_mod_time,
//restore_ongoing: self.restore_ongoing,
//restore_expires: self.restore_expires,
transition_status: self.transitioned_object.status.clone(),
transition_status: self.transitioned_object.status.clone(),
..Default::default()
}
}
@@ -790,9 +819,9 @@ impl LifecycleOps for ObjectInfo {
#[derive(Debug, Default, Clone)]
pub struct S3Location {
pub bucketname: String,
pub bucketname: String,
//pub encryption: Encryption,
pub prefix: String,
pub prefix: String,
pub storage_class: String,
//pub tagging: Tags,
pub user_metadata: HashMap<String, String>,
@@ -803,13 +832,12 @@ pub struct OutputLocation(pub S3Location);
#[derive(Debug, Default, Clone)]
pub struct RestoreObjectRequest {
pub days: i64,
pub ror_type: String,
pub tier: String,
pub description: String,
pub days: i64,
pub ror_type: String,
pub tier: String,
pub description: String,
//pub select_parameters: SelectParameters,
pub output_location: OutputLocation,
pub output_location: OutputLocation,
}
const MAX_RESTORE_OBJECT_REQUEST_SIZE: i64 = 2 << 20;
ecstore/src/bucket/lifecycle/lifecycle.rs
+91 -63
View File
@@ -1,12 +1,12 @@
use s3s::dto::{
BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, NoncurrentVersionTransition,
ObjectLockConfiguration, ObjectLockEnabled, Transition,
};
use std::cmp::Ordering;
use std::env;
use std::fmt::Display;
use s3s::dto::{
BucketLifecycleConfiguration, ExpirationStatus, LifecycleRule, ObjectLockConfiguration,
ObjectLockEnabled, LifecycleExpiration, Transition, NoncurrentVersionTransition,
};
use time::macros::{datetime, offset};
use time::{self, OffsetDateTime, Duration};
use time::{self, Duration, OffsetDateTime};
use crate::bucket::lifecycle::rule::TransitionOps;
@@ -16,10 +16,11 @@ pub const TRANSITION_COMPLETE: &str = "complete";
pub const TRANSITION_PENDING: &str = "pending";
const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration allows a maximum of 1000 rules";
const ERR_LIFECYCLE_NO_RULE: &str = "Lifecycle configuration should have at least one rule";
const ERR_LIFECYCLE_DUPLICATE_ID: &str = "Rule ID must be unique. Found same ID for more than one rule";
const ERR_XML_NOT_WELL_FORMED: &str = "The XML you provided was not well-formed or did not validate against our published schema";
const ERR_LIFECYCLE_BUCKET_LOCKED: &str = "ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket";
const ERR_LIFECYCLE_NO_RULE: &str = "Lifecycle configuration should have at least one rule";
const ERR_LIFECYCLE_DUPLICATE_ID: &str = "Rule ID must be unique. Found same ID for more than one rule";
const ERR_XML_NOT_WELL_FORMED: &str = "The XML you provided was not well-formed or did not validate against our published schema";
const ERR_LIFECYCLE_BUCKET_LOCKED: &str =
"ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket";
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum IlmAction {
@@ -52,7 +53,10 @@ impl IlmAction {
if self.delete_restored() {
return true;
}
*self == Self::DeleteVersionAction || *self == Self::DeleteAction || *self == Self::DeleteAllVersionsAction || *self == Self::DelMarkerDeleteAllVersionsAction
*self == Self::DeleteVersionAction
|| *self == Self::DeleteAction
|| *self == Self::DeleteAllVersionsAction
|| *self == Self::DelMarkerDeleteAllVersionsAction
}
}
@@ -204,7 +208,10 @@ impl Lifecycle for BucketLifecycleConfiguration {
return true;
}
let rule_expiration = rule.expiration.as_ref().expect("err!");
if !rule_expiration.date.is_none() && OffsetDateTime::from(rule_expiration.date.clone().expect("err!")).unix_timestamp() < OffsetDateTime::now_utc().unix_timestamp() {
if !rule_expiration.date.is_none()
&& OffsetDateTime::from(rule_expiration.date.clone().expect("err!")).unix_timestamp()
< OffsetDateTime::now_utc().unix_timestamp()
{
return true;
}
if !rule_expiration.date.is_none() {
@@ -213,9 +220,12 @@ impl Lifecycle for BucketLifecycleConfiguration {
if rule_expiration.expired_object_delete_marker.expect("err!") {
return true;
}
let rule_transitions: &[Transition]= &rule.transitions.as_ref().expect("err!");
let rule_transitions: &[Transition] = &rule.transitions.as_ref().expect("err!");
let rule_transitions_0 = rule_transitions[0].clone();
if !rule_transitions_0.date.is_none() && OffsetDateTime::from(rule_transitions_0.date.expect("err!")).unix_timestamp() < OffsetDateTime::now_utc().unix_timestamp() {
if !rule_transitions_0.date.is_none()
&& OffsetDateTime::from(rule_transitions_0.date.expect("err!")).unix_timestamp()
< OffsetDateTime::now_utc().unix_timestamp()
{
return true;
}
if !rule.transitions.is_none() {
@@ -242,18 +252,18 @@ impl Lifecycle for BucketLifecycleConfiguration {
return Err(std::io::Error::other(ERR_LIFECYCLE_BUCKET_LOCKED));
}
} /*else {
if object_lock_enabled.as_str() == ObjectLockEnabled::ENABLED {
return Err(Error::msg(ERR_LIFECYCLE_BUCKET_LOCKED));
}
if object_lock_enabled.as_str() == ObjectLockEnabled::ENABLED {
return Err(Error::msg(ERR_LIFECYCLE_BUCKET_LOCKED));
}
}*/
}
}
}
for (i,_) in self.rules.iter().enumerate() {
if i == self.rules.len()-1 {
for (i, _) in self.rules.iter().enumerate() {
if i == self.rules.len() - 1 {
break;
}
let other_rules = &self.rules[i+1..];
let other_rules = &self.rules[i + 1..];
for other_rule in other_rules {
if self.rules[i].id == other_rule.id {
return Err(std::io::Error::other(ERR_LIFECYCLE_DUPLICATE_ID));
@@ -281,7 +291,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
continue;
}*/
//if !obj.delete_marker && !rule.filter.BySize(obj.size) {
if !obj.delete_marker && false{
if !obj.delete_marker && false {
continue;
}
rules.push(rule.clone());
@@ -306,9 +316,9 @@ impl Lifecycle for BucketLifecycleConfiguration {
action = IlmAction::DeleteRestoredVersionAction;
}
events.push(Event{
events.push(Event {
action: action,
due: Some(now),
due: Some(now),
rule_id: "".into(),
noncurrent_days: 0,
newer_noncurrent_versions: 0,
@@ -322,10 +332,10 @@ impl Lifecycle for BucketLifecycleConfiguration {
if obj.expired_object_deletemarker() {
if let Some(expiration) = rule.expiration.as_ref() {
if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker {
events.push(Event{
action: IlmAction::DeleteVersionAction,
events.push(Event {
action: IlmAction::DeleteVersionAction,
rule_id: rule.id.clone().expect("err!"),
due: Some(now),
due: Some(now),
noncurrent_days: 0,
newer_noncurrent_versions: 0,
storage_class: "".into(),
@@ -336,12 +346,12 @@ impl Lifecycle for BucketLifecycleConfiguration {
if let Some(expiration) = rule.expiration.as_ref() {
if let Some(days) = expiration.days {
let expected_expiry = expected_expiry_time(obj.mod_time.expect("err!"), days/*, date*/);
let expected_expiry = expected_expiry_time(obj.mod_time.expect("err!"), days /*, date*/);
if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() {
events.push(Event{
action: IlmAction::DeleteVersionAction,
events.push(Event {
action: IlmAction::DeleteVersionAction,
rule_id: rule.id.clone().expect("err!"),
due: Some(expected_expiry),
due: Some(expected_expiry),
noncurrent_days: 0,
newer_noncurrent_versions: 0,
storage_class: "".into(),
@@ -359,10 +369,10 @@ impl Lifecycle for BucketLifecycleConfiguration {
let due = expiration.next_due(obj);
if let Some(due) = due {
if now.unix_timestamp() == 0 || now.unix_timestamp() > due.unix_timestamp() {
events.push(Event{
events.push(Event {
action: IlmAction::DelMarkerDeleteAllVersionsAction,
rule_id: rule.id.clone().expect("err!"),
due: Some(due),
due: Some(due),
noncurrent_days: 0,
newer_noncurrent_versions: 0,
storage_class: "".into(),
@@ -392,10 +402,10 @@ impl Lifecycle for BucketLifecycleConfiguration {
if let Some(successor_mod_time) = obj.successor_mod_time {
let expected_expiry = expected_expiry_time(successor_mod_time, noncurrent_days);
if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() {
events.push(Event{
action: IlmAction::DeleteVersionAction,
events.push(Event {
action: IlmAction::DeleteVersionAction,
rule_id: rule.id.clone().expect("err!"),
due: Some(expected_expiry),
due: Some(expected_expiry),
noncurrent_days: 0,
newer_noncurrent_versions: 0,
storage_class: "".into(),
@@ -413,12 +423,19 @@ impl Lifecycle for BucketLifecycleConfiguration {
if storage_class.as_str() != "" {
if !obj.delete_marker && obj.transition_status != TRANSITION_COMPLETE {
let due = rule.noncurrent_version_transitions.as_ref().unwrap()[0].next_due(obj);
if due.is_some() && (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unwrap().unix_timestamp()) {
if due.is_some()
&& (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unwrap().unix_timestamp())
{
events.push(Event {
action: IlmAction::TransitionVersionAction,
rule_id: rule.id.clone().expect("err!"),
action: IlmAction::TransitionVersionAction,
rule_id: rule.id.clone().expect("err!"),
due,
storage_class: rule.noncurrent_version_transitions.as_ref().unwrap()[0].storage_class.clone().unwrap().as_str().to_string(),
storage_class: rule.noncurrent_version_transitions.as_ref().unwrap()[0]
.storage_class
.clone()
.unwrap()
.as_str()
.to_string(),
..Default::default()
});
}
@@ -434,10 +451,10 @@ impl Lifecycle for BucketLifecycleConfiguration {
let date0 = OffsetDateTime::from(date.clone());
if date0.unix_timestamp() != 0 {
if now.unix_timestamp() == 0 || now.unix_timestamp() > date0.unix_timestamp() {
events.push(Event{
action: IlmAction::DeleteAction,
events.push(Event {
action: IlmAction::DeleteAction,
rule_id: rule.id.clone().expect("err!"),
due: Some(date0),
due: Some(date0),
noncurrent_days: 0,
newer_noncurrent_versions: 0,
storage_class: "".into(),
@@ -448,10 +465,10 @@ impl Lifecycle for BucketLifecycleConfiguration {
if days != 0 {
let expected_expiry: OffsetDateTime = expected_expiry_time(obj.mod_time.expect("err!"), days);
if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() {
let mut event = Event{
action: IlmAction::DeleteAction,
let mut event = Event {
action: IlmAction::DeleteAction,
rule_id: rule.id.clone().expect("err!"),
due: Some(expected_expiry),
due: Some(expected_expiry),
noncurrent_days: 0,
newer_noncurrent_versions: 0,
storage_class: "".into(),
@@ -469,10 +486,12 @@ impl Lifecycle for BucketLifecycleConfiguration {
if let Some(ref transitions) = rule.transitions {
let due = transitions[0].next_due(obj);
if let Some(due) = due {
if due.unix_timestamp() > 0 && (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unix_timestamp()) {
events.push(Event{
action: IlmAction::TransitionAction,
rule_id: rule.id.clone().expect("err!"),
if due.unix_timestamp() > 0
&& (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unix_timestamp())
{
events.push(Event {
action: IlmAction::TransitionAction,
rule_id: rule.id.clone().expect("err!"),
due: Some(due),
storage_class: transitions[0].storage_class.clone().expect("err!").as_str().to_string(),
noncurrent_days: 0,
@@ -488,20 +507,27 @@ impl Lifecycle for BucketLifecycleConfiguration {
if events.len() > 0 {
events.sort_by(|a, b| {
if now.unix_timestamp() > a.due.expect("err!").unix_timestamp() && now.unix_timestamp() > b.due.expect("err").unix_timestamp() || a.due.expect("err").unix_timestamp() == b.due.expect("err").unix_timestamp() {
if now.unix_timestamp() > a.due.expect("err!").unix_timestamp()
&& now.unix_timestamp() > b.due.expect("err").unix_timestamp()
|| a.due.expect("err").unix_timestamp() == b.due.expect("err").unix_timestamp()
{
match a.action {
IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction
| IlmAction::DeleteAction | IlmAction::DeleteVersionAction => {
IlmAction::DeleteAllVersionsAction
| IlmAction::DelMarkerDeleteAllVersionsAction
| IlmAction::DeleteAction
| IlmAction::DeleteVersionAction => {
return Ordering::Less;
}
_ => ()
_ => (),
}
match b.action {
IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction
| IlmAction::DeleteAction | IlmAction::DeleteVersionAction => {
IlmAction::DeleteAllVersionsAction
| IlmAction::DelMarkerDeleteAllVersionsAction
| IlmAction::DeleteAction
| IlmAction::DeleteVersionAction => {
return Ordering::Greater;
}
_ => ()
_ => (),
}
return Ordering::Less;
}
@@ -526,18 +552,18 @@ impl Lifecycle for BucketLifecycleConfiguration {
continue;
}
return Event {
action: IlmAction::DeleteVersionAction,
rule_id: rule.id.clone().expect("err"),
noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32,
action: IlmAction::DeleteVersionAction,
rule_id: rule.id.clone().expect("err"),
noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32,
newer_noncurrent_versions: newer_noncurrent_versions as usize,
due: Some(OffsetDateTime::UNIX_EPOCH),
storage_class: "".into(),
};
} else {
return Event {
action: IlmAction::DeleteVersionAction,
rule_id: rule.id.clone().expect("err"),
noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32,
action: IlmAction::DeleteVersionAction,
rule_id: rule.id.clone().expect("err"),
noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32,
newer_noncurrent_versions: 0,
due: Some(OffsetDateTime::UNIX_EPOCH),
storage_class: "".into(),
@@ -601,7 +627,9 @@ pub fn expected_expiry_time(mod_time: OffsetDateTime, days: i32) -> OffsetDateTi
if days == 0 {
return mod_time;
}
let t = mod_time.to_offset(offset!(-0:00:00)).saturating_add(Duration::days(0/*days as i64*/)); //debug
let t = mod_time
.to_offset(offset!(-0:00:00))
.saturating_add(Duration::days(0 /*days as i64*/)); //debug
let mut hour = 3600;
if let Ok(env_ilm_hour) = env::var("_RUSTFS_ILM_HOUR") {
if let Ok(num_hour) = env_ilm_hour.parse::<usize>() {
@@ -661,7 +689,7 @@ impl Default for Event {
#[derive(Debug, Clone, Default)]
pub struct ExpirationOptions {
pub expire: bool
pub expire: bool,
}
impl ExpirationOptions {
ecstore/src/bucket/lifecycle/mod.rs
+5 -5
View File
@@ -1,6 +1,6 @@
pub mod rule;
pub mod lifecycle;
pub mod tier_sweeper;
pub mod tier_last_day_stats;
pub mod bucket_lifecycle_audit;
pub mod bucket_lifecycle_ops;
pub mod bucket_lifecycle_audit;
pub mod lifecycle;
pub mod rule;
pub mod tier_last_day_stats;
pub mod tier_sweeper;
ecstore/src/bucket/lifecycle/rule.rs
+5 -7
View File
@@ -1,10 +1,9 @@
use s3s::dto::{
LifecycleRuleFilter, Transition,
};
use s3s::dto::{LifecycleRuleFilter, Transition};
const ERR_TRANSITION_INVALID_DAYS: &str = "Days must be 0 or greater when used with Transition";
const ERR_TRANSITION_INVALID_DATE: &str = "Date must be provided in ISO 8601 format";
const ERR_TRANSITION_INVALID: &str = "Exactly one of Days (0 or greater) or Date (positive ISO 8601 format) should be present in Transition.";
const ERR_TRANSITION_INVALID_DAYS: &str = "Days must be 0 or greater when used with Transition";
const ERR_TRANSITION_INVALID_DATE: &str = "Date must be provided in ISO 8601 format";
const ERR_TRANSITION_INVALID: &str =
"Exactly one of Days (0 or greater) or Date (positive ISO 8601 format) should be present in Transition.";
const ERR_TRANSITION_DATE_NOT_MIDNIGHT: &str = "'Date' must be at midnight GMT";
pub trait Filter {
@@ -39,7 +38,6 @@ impl TransitionOps for Transition {
}
}
#[cfg(test)]
mod test {
use super::*;
ecstore/src/bucket/lifecycle/tier_last_day_stats.rs
+2 -5
View File
@@ -1,9 +1,9 @@
use sha2::Sha256;
use std::collections::HashMap;
use std::ops::Sub;
use time::OffsetDateTime;
use tracing::{error, warn};
use std::ops::Sub;
use crate::heal::data_usage_cache::TierStats;
@@ -79,8 +79,5 @@ impl LastDayTierStats {
}
}
#[cfg(test)]
mod test {
}
mod test {}
ecstore/src/bucket/lifecycle/tier_sweeper.rs
+12 -13
View File
@@ -1,11 +1,11 @@
use sha2::{Digest, Sha256};
use xxhash_rust::xxh64;
use std::any::Any;
use std::io::{Cursor, Write};
use xxhash_rust::xxh64;
use crate::global::GLOBAL_TierConfigMgr;
use super::bucket_lifecycle_ops::{ExpiryOp, GLOBAL_ExpiryState, TransitionedObject};
use super::lifecycle::{self, ObjectOpts};
use crate::global::GLOBAL_TierConfigMgr;
static XXHASH_SEED: u64 = 0;
@@ -44,9 +44,9 @@ impl ObjSweeper {
}
pub fn get_opts(&self) -> lifecycle::ObjectOpts {
let mut opts = ObjectOpts{
version_id: self.version_id.clone(),
versioned: self.versioned,
let mut opts = ObjectOpts {
version_id: self.version_id.clone(),
versioned: self.versioned,
version_suspended: self.suspended,
..Default::default()
};
@@ -69,16 +69,18 @@ impl ObjSweeper {
}
let mut del_tier = false;
if !self.versioned || self.suspended { // 1, 2.a, 2.b
if !self.versioned || self.suspended {
// 1, 2.a, 2.b
del_tier = true;
} else if self.versioned && self.version_id != "" { // 3.a
} else if self.versioned && self.version_id != "" {
// 3.a
del_tier = true;
}
if del_tier {
return Some(Jentry {
obj_name: self.remote_object.clone(),
obj_name: self.remote_object.clone(),
version_id: self.transition_version_id.clone(),
tier_name: self.transition_tier.clone(),
tier_name: self.transition_tier.clone(),
});
}
None
@@ -123,8 +125,5 @@ pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_na
w.remove(obj_name, rv_id).await
}
#[cfg(test)]
mod test {
}
mod test {}
ecstore/src/bucket/mod.rs
+1 -1
View File
@@ -1,4 +1,5 @@
pub mod error;
pub mod lifecycle;
pub mod metadata;
pub mod metadata_sys;
pub mod object_lock;
@@ -10,4 +11,3 @@ pub mod target;
pub mod utils;
pub mod versioning;
pub mod versioning_sys;
pub mod lifecycle;
ecstore/src/bucket/object_lock/objectlock.rs
+25 -20
View File
@@ -3,13 +3,8 @@ use std::collections::HashMap;
use time::{OffsetDateTime, format_description};
use tracing::{error, warn};
use s3s::dto::{
ObjectLockRetentionMode, ObjectLockRetention, ObjectLockLegalHoldStatus, ObjectLockLegalHold,
Date,
};
use s3s::header::{
X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_OBJECT_LOCK_LEGAL_HOLD,
};
use s3s::dto::{Date, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockRetention, ObjectLockRetentionMode};
use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
//const AMZ_OBJECTLOCK_BYPASS_RET_GOVERNANCE: &str = "X-Amz-Bypass-Governance-Retention";
//const AMZ_OBJECTLOCK_RETAIN_UNTIL_DATE: &str = "X-Amz-Object-Lock-Retain-Until-Date";
@@ -17,12 +12,14 @@ use s3s::header::{
//const AMZ_OBJECTLOCK_LEGALHOLD: &str = "X-Amz-Object-Lock-Legal-Hold";
const ERR_MALFORMED_BUCKET_OBJECT_CONFIG: &str = "invalid bucket object lock config";
const ERR_INVALID_RETENTION_DATE: &str = "date must be provided in ISO 8601 format";
const ERR_PAST_OBJECTLOCK_RETAIN_DATE: &str = "the retain until date must be in the future";
const ERR_UNKNOWN_WORMMODE_DIRECTIVE: &str = "unknown WORM mode directive";
const ERR_OBJECTLOCK_MISSING_CONTENT_MD5: &str = "content-MD5 HTTP header is required for Put Object requests with Object Lock parameters";
const ERR_OBJECTLOCK_INVALID_HEADERS: &str = "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied";
const ERR_MALFORMED_XML: &str = "the XML you provided was not well-formed or did not validate against our published schema";
const ERR_INVALID_RETENTION_DATE: &str = "date must be provided in ISO 8601 format";
const ERR_PAST_OBJECTLOCK_RETAIN_DATE: &str = "the retain until date must be in the future";
const ERR_UNKNOWN_WORMMODE_DIRECTIVE: &str = "unknown WORM mode directive";
const ERR_OBJECTLOCK_MISSING_CONTENT_MD5: &str =
"content-MD5 HTTP header is required for Put Object requests with Object Lock parameters";
const ERR_OBJECTLOCK_INVALID_HEADERS: &str =
"x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied";
const ERR_MALFORMED_XML: &str = "the XML you provided was not well-formed or did not validate against our published schema";
pub fn utc_now_ntp() -> OffsetDateTime {
return OffsetDateTime::now_utc();
@@ -39,7 +36,10 @@ pub fn get_object_retention_meta(meta: HashMap<String, String>) -> ObjectLockRet
if let Some(mode_str) = mode_str {
mode = parse_ret_mode(mode_str.as_str());
} else {
return ObjectLockRetention {mode: None, retain_until_date: None};
return ObjectLockRetention {
mode: None,
retain_until_date: None,
};
}
let mut till_str = meta.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_lowercase().as_str());
@@ -49,10 +49,13 @@ pub fn get_object_retention_meta(meta: HashMap<String, String>) -> ObjectLockRet
if let Some(till_str) = till_str {
let t = OffsetDateTime::parse(till_str, &format_description::well_known::Iso8601::DEFAULT);
if t.is_err() {
retain_until_date = Date::from(t.expect("err")); //TODO: utc
retain_until_date = Date::from(t.expect("err")); //TODO: utc
}
}
ObjectLockRetention {mode: Some(mode), retain_until_date: Some(retain_until_date)}
ObjectLockRetention {
mode: Some(mode),
retain_until_date: Some(retain_until_date),
}
}
pub fn get_object_legalhold_meta(meta: HashMap<String, String>) -> ObjectLockLegalHold {
@@ -61,9 +64,11 @@ pub fn get_object_legalhold_meta(meta: HashMap<String, String>) -> ObjectLockLeg
hold_str = Some(&meta[X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()]);
}
if let Some(hold_str) = hold_str {
return ObjectLockLegalHold {status: Some(parse_legalhold_status(hold_str))};
return ObjectLockLegalHold {
status: Some(parse_legalhold_status(hold_str)),
};
}
ObjectLockLegalHold {status: None}
ObjectLockLegalHold { status: None }
}
pub fn parse_ret_mode(mode_str: &str) -> ObjectLockRetentionMode {
@@ -75,7 +80,7 @@ pub fn parse_ret_mode(mode_str: &str) -> ObjectLockRetentionMode {
"COMPLIANCE" => {
mode = ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE);
}
_ => unreachable!()
_ => unreachable!(),
}
mode
}
@@ -89,7 +94,7 @@ pub fn parse_legalhold_status(hold_str: &str) -> ObjectLockLegalHoldStatus {
"OFF" => {
st = ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF);
}
_ => unreachable!()
_ => unreachable!(),
}
st
}
ecstore/src/bucket/object_lock/objectlock_sys.rs
+11 -8
View File
@@ -3,12 +3,10 @@ use std::sync::Arc;
use time::OffsetDateTime;
use tracing::{error, warn};
use s3s::dto::{
DefaultRetention, ObjectLockRetentionMode, ObjectLockLegalHoldStatus,
};
use s3s::dto::{DefaultRetention, ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
use crate::store_api::ObjectInfo;
use crate::bucket::metadata_sys::get_object_lock_config;
use crate::store_api::ObjectInfo;
use super::objectlock;
@@ -21,7 +19,12 @@ impl BucketObjectLockSys {
}
pub async fn get(bucket: &str) -> Option<DefaultRetention> {
if let Some(object_lock_rule) = get_object_lock_config(bucket).await.expect("get_object_lock_config err!").0.rule {
if let Some(object_lock_rule) = get_object_lock_config(bucket)
.await
.expect("get_object_lock_config err!")
.0
.rule
{
return object_lock_rule.default_retention;
}
None
@@ -35,10 +38,10 @@ pub fn enforce_retention_for_deletion(obj_info: &ObjectInfo) -> bool {
let lhold = objectlock::get_object_legalhold_meta(obj_info.user_defined.clone().expect("err"));
match lhold.status {
Some(st) if st.as_str()==ObjectLockLegalHoldStatus::ON => {
Some(st) if st.as_str() == ObjectLockLegalHoldStatus::ON => {
return true;
}
_ => ()
_ => (),
}
let ret = objectlock::get_object_retention_meta(obj_info.user_defined.clone().expect("err"));
@@ -49,7 +52,7 @@ pub fn enforce_retention_for_deletion(obj_info: &ObjectInfo) -> bool {
return true;
}
}
_ => ()
_ => (),
}
false
}