From 6ce24f3b632f86c456a95d6e8b6ba9d05c8e4ae5 Mon Sep 17 00:00:00 2001 From: houseme Date: Thu, 16 Apr 2026 18:28:03 +0800 Subject: [PATCH] feat(lifecycle): improve ILM compatibility and scanner runtime config (#2534) Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> Co-authored-by: cxymds --- Cargo.lock | 1 + crates/common/src/heal_channel.rs | 2 +- crates/config/README.md | 3 + crates/config/src/constants/scanner.rs | 8 + crates/e2e_test/src/reliant/lifecycle.rs | 13 +- .../ecstore/src/bucket/bucket_target_sys.rs | 115 +- .../bucket/lifecycle/bucket_lifecycle_ops.rs | 533 ++++++-- .../lifecycle/{lifecycle.rs => core.rs} | 1075 +++++++++++++---- crates/ecstore/src/bucket/lifecycle/mod.rs | 3 +- .../src/bucket/lifecycle/tier_sweeper.rs | 17 +- .../ecstore/src/bucket/replication/config.rs | 5 + .../replication/replication_resyncer.rs | 239 +++- .../src/client/object_handlers_common.rs | 46 +- crates/ecstore/src/set_disk.rs | 225 +++- crates/ecstore/src/store.rs | 7 +- crates/ecstore/src/store_api/types.rs | 22 + crates/filemeta/src/filemeta.rs | 48 + crates/scanner/Cargo.toml | 1 + crates/scanner/src/scanner.rs | 54 +- crates/scanner/src/scanner_folder.rs | 12 +- crates/scanner/src/sleeper.rs | 40 +- .../tests/lifecycle_integration_test.rs | 334 +++++ crates/utils/src/envs.rs | 2 + rustfs/src/app/bucket_usecase.rs | 23 +- .../src/app/lifecycle_transition_api_test.rs | 87 ++ rustfs/src/app/object_usecase.rs | 284 ++++- 26 files changed, 2672 insertions(+), 527 deletions(-) rename crates/ecstore/src/bucket/lifecycle/{lifecycle.rs => core.rs} (63%) diff --git a/Cargo.lock b/Cargo.lock index e36849dbe..6579c4dcb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8507,6 +8507,7 @@ dependencies = [ "serde", "serde_json", "serial_test", + "temp-env", "thiserror 2.0.18", "time", "tokio", diff --git a/crates/common/src/heal_channel.rs b/crates/common/src/heal_channel.rs index 7b7d585ef..e76f84180 100644 --- a/crates/common/src/heal_channel.rs +++ b/crates/common/src/heal_channel.rs @@ -511,7 +511,7 @@ pub fn lc_has_active_rules(config: &BucketLifecycleConfiguration, prefix: &str) } if let Some(e) = &rule.noncurrent_version_expiration { - if let Some(true) = e.noncurrent_days.map(|d| d > 0) { + if e.noncurrent_days.is_some() { return true; } if let Some(true) = e.newer_noncurrent_versions.map(|d| d > 0) { diff --git a/crates/config/README.md b/crates/config/README.md index c28d9f5f7..b8b2383dd 100644 --- a/crates/config/README.md +++ b/crates/config/README.md @@ -53,8 +53,11 @@ Current guidance: ## Scanner environment aliases +- `RUSTFS_SCANNER_SPEED` (canonical, also accepts `MINIO_SCANNER_SPEED`) +- `RUSTFS_SCANNER_CYCLE` (canonical, also accepts `MINIO_SCANNER_CYCLE`) - `RUSTFS_SCANNER_START_DELAY_SECS` (canonical) - `RUSTFS_DATA_SCANNER_START_DELAY_SECS` (deprecated alias for compatibility) +- `RUSTFS_SCANNER_IDLE_MODE` (canonical) ## Drive timeout environment variables diff --git a/crates/config/src/constants/scanner.rs b/crates/config/src/constants/scanner.rs index e0802e8d8..a94fd754c 100644 --- a/crates/config/src/constants/scanner.rs +++ b/crates/config/src/constants/scanner.rs @@ -25,6 +25,12 @@ pub const ENV_SCANNER_START_DELAY_SECS: &str = "RUSTFS_SCANNER_START_DELAY_SECS" #[deprecated(note = "Use RUSTFS_SCANNER_START_DELAY_SECS instead")] pub const ENV_DATA_SCANNER_START_DELAY_SECS: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS"; +/// Environment variable that specifies the scanner cycle interval in seconds. +/// If set, this overrides the cycle interval derived from `RUSTFS_SCANNER_SPEED`. +/// - Unit: seconds (u64). +/// - Example: `export RUSTFS_SCANNER_CYCLE=3600` (1 hour) +pub const ENV_SCANNER_CYCLE: &str = "RUSTFS_SCANNER_CYCLE"; + /// Environment variable that selects the scanner speed preset. /// Valid values: `fastest`, `fast`, `default`, `slow`, `slowest`. /// Controls the sleep factor, maximum sleep duration, and cycle interval. @@ -65,6 +71,8 @@ pub const DEFAULT_SCANNER_INLINE_HEAL_ENABLE: bool = false; /// | `default` | 2x | 1 second | 1 minute | /// | `slow` | 10x | 15 seconds| 1 minute | /// | `slowest` | 100x | 15 seconds| 30 minutes | +/// +/// The cycle interval can be overridden by `RUSTFS_SCANNER_CYCLE`. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum ScannerSpeed { Fastest, diff --git a/crates/e2e_test/src/reliant/lifecycle.rs b/crates/e2e_test/src/reliant/lifecycle.rs index 2302eb306..ffa73eba9 100644 --- a/crates/e2e_test/src/reliant/lifecycle.rs +++ b/crates/e2e_test/src/reliant/lifecycle.rs @@ -160,7 +160,7 @@ async fn test_bucket_lifecycle_configuration() -> Result<(), Box Result<(), Box> { +async fn test_bucket_lifecycle_accepts_zero_days() -> Result<(), Box> { use aws_sdk_s3::types::{BucketLifecycleConfiguration, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter}; let client = create_aws_s3_client().await?; @@ -176,19 +176,12 @@ async fn test_bucket_lifecycle_rejects_zero_days() -> Result<(), Box, opts: &RemoveObjectOptions) -> HeaderMap { + let mut headers = HeaderMap::new(); + if opts.force_delete { + insert_header(&mut headers, SUFFIX_FORCE_DELETE, "true"); + } + if opts.governance_bypass { + headers.insert(AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, "true".parse().unwrap()); + } + + if opts.replication_delete_marker { + insert_header(&mut headers, SUFFIX_SOURCE_DELETEMARKER, "true"); + } + + if let Some(t) = opts.replication_mtime { + insert_header(&mut headers, SUFFIX_SOURCE_MTIME, t.format(&Rfc3339).unwrap_or_default()); + } + + if !opts.replication_status.is_empty() { + headers.insert(AMZ_BUCKET_REPLICATION_STATUS, opts.replication_status.as_str().parse().unwrap()); + } + + if let Some(version_id) = version_id { + insert_header(&mut headers, SUFFIX_SOURCE_VERSION_ID, version_id); + } + + if opts.replication_request { + insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); + } + if opts.replication_validity_check { + insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true"); + } + + headers +} + #[derive(Debug, Clone)] pub struct AdvancedPutOptions { pub source_version_id: String, @@ -1427,39 +1462,15 @@ impl TargetClient { version_id: Option, opts: RemoveObjectOptions, ) -> Result<(), S3ClientError> { - let mut headers = HeaderMap::new(); - if opts.force_delete { - insert_header(&mut headers, SUFFIX_FORCE_DELETE, "true"); - } - if opts.governance_bypass { - headers.insert(AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, "true".parse().unwrap()); - } - - if opts.replication_delete_marker { - insert_header(&mut headers, SUFFIX_SOURCE_DELETEMARKER, "true"); - } - - if let Some(t) = opts.replication_mtime { - insert_header(&mut headers, SUFFIX_SOURCE_MTIME, t.format(&Rfc3339).unwrap_or_default()); - } - - if !opts.replication_status.is_empty() { - headers.insert(AMZ_BUCKET_REPLICATION_STATUS, opts.replication_status.as_str().parse().unwrap()); - } - - if opts.replication_request { - insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); - } - if opts.replication_validity_check { - insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true"); - } + let headers = build_remove_object_headers(version_id.as_deref(), &opts); + let api_version_id = if opts.replication_request { None } else { version_id }; match self .client .delete_object() .bucket(bucket) .key(object) - .set_version_id(version_id) + .set_version_id(api_version_id) .customize() .map_request(move |mut req| { for (k, v) in headers.clone().into_iter() { @@ -1553,3 +1564,53 @@ impl From for BucketTargetError { } impl Error for BucketTargetError {} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn build_remove_object_headers_includes_internal_version_id_for_replication_delete() { + let version_id = Uuid::new_v4().to_string(); + let headers = build_remove_object_headers( + Some(version_id.as_str()), + &RemoveObjectOptions { + force_delete: false, + governance_bypass: false, + replication_delete_marker: true, + replication_mtime: None, + replication_status: ReplicationStatusType::Replica, + replication_request: true, + replication_validity_check: false, + }, + ); + + assert_eq!( + rustfs_utils::http::get_header(&headers, SUFFIX_SOURCE_VERSION_ID).as_deref(), + Some(version_id.as_str()), + "replication delete requests must preserve the version id in internal headers" + ); + } + + #[test] + fn build_remove_object_headers_omits_delete_marker_flag_for_marker_version_purge() { + let version_id = Uuid::new_v4().to_string(); + let headers = build_remove_object_headers( + Some(version_id.as_str()), + &RemoveObjectOptions { + force_delete: false, + governance_bypass: false, + replication_delete_marker: false, + replication_mtime: None, + replication_status: ReplicationStatusType::Replica, + replication_request: true, + replication_validity_check: false, + }, + ); + + assert!( + rustfs_utils::http::get_header(&headers, SUFFIX_SOURCE_DELETEMARKER).is_none(), + "delete-marker version purges must not masquerade as delete-marker creations" + ); + } +} diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index cab42a946..72724ff02 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -11,20 +11,18 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -#![allow(unused_imports)] -#![allow(unused_variables)] -#![allow(unused_mut)] -#![allow(unused_assignments)] -#![allow(unused_must_use)] -#![allow(clippy::all)] use crate::bucket::lifecycle::bucket_lifecycle_audit::{LcAuditEvent, LcEventSrc}; +use crate::bucket::lifecycle::evaluator::Evaluator; use crate::bucket::lifecycle::lifecycle::{ self, ExpirationOptions, Lifecycle, ObjectOpts, TransitionOptions, abort_incomplete_multipart_upload_due, }; use crate::bucket::lifecycle::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats}; use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier}; use crate::bucket::object_lock::objectlock_sys::check_object_lock_for_deletion; +use crate::bucket::replication::{ + DeletedObjectReplicationInfo, ReplicationConfig, check_replicate_delete, schedule_replication_delete, +}; use crate::bucket::{metadata_sys, metadata_sys::get_lifecycle_config, versioning_sys::BucketVersioningSys}; use crate::client::object_api_utils::new_getobjectreader; use crate::disk::error::DiskError; @@ -37,34 +35,33 @@ use crate::global::GLOBAL_LocalNodeName; use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id}; use crate::set_disk::{MAX_PARTS_COUNT, RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, SetDisks}; use crate::store::ECStore; -use crate::store_api::StorageAPI; use crate::store_api::{ GetObjectReader, HTTPRangeSpec, ListOperations, MultipartOperations, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, }; use crate::tier::warm_backend::WarmBackendGetOpts; use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded}; -use bytes::BytesMut; use futures::Future; use http::HeaderMap; use lazy_static::lazy_static; use rustfs_common::data_usage::TierStats; use rustfs_common::heal_channel::rep_has_active_rules; use rustfs_common::metrics::{IlmAction, Metrics}; -use rustfs_filemeta::{FileInfo, FileInfoOpts, NULL_VERSION_ID, RestoreStatusOps, get_file_info, is_restored_object_on_disk}; +use rustfs_filemeta::{ + FileInfo, FileInfoOpts, NULL_VERSION_ID, REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicationState, RestoreStatusOps, + VersionPurgeStatusType, get_file_info, is_restored_object_on_disk, +}; use rustfs_s3_common::EventName; use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::strings_has_prefix_fold}; -use s3s::Body; use s3s::dto::{ BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration, RestoreRequest, RestoreRequestType, RestoreStatus, - ServerSideEncryption, Timestamp, + Timestamp, }; -use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION, X_AMZ_STORAGE_CLASS}; +use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION}; use sha2::{Digest, Sha256}; use std::any::Any; use std::collections::HashMap; use std::env; -use std::io::Write; use std::pin::Pin; use std::sync::atomic::{AtomicI64, Ordering}; use std::sync::{Arc, Mutex, Weak}; @@ -73,7 +70,7 @@ use time::OffsetDateTime; use tokio::select; use tokio::sync::mpsc::{Receiver, Sender}; use tokio::sync::{RwLock, mpsc}; -use tracing::{debug, error, info, warn}; +use tracing::{error, info, warn}; use uuid::Uuid; use xxhash_rust::xxh64; @@ -111,7 +108,7 @@ impl LifecycleSys { pub async fn get(&self, bucket: &str) -> Option { match get_lifecycle_config(bucket).await { Ok((lc, _)) => Some(lc), - Err(err) if err == Error::ConfigNotFound => None, + Err(Error::ConfigNotFound) => None, Err(err) => { warn!(bucket, error = ?err, "failed to load lifecycle config"); None @@ -119,8 +116,25 @@ impl LifecycleSys { } } - pub fn trace(_oi: &ObjectInfo) -> TraceFn { - Arc::new(|_oi, _ctx| Box::pin(async move {})) + pub fn trace(oi: &ObjectInfo) -> TraceFn { + let bucket = oi.bucket.clone(); + let name = oi.name.clone(); + let version_id = oi.version_id.map(|v| v.to_string()).unwrap_or_default(); + Arc::new(move |_action: String, _ctx: HashMap| { + let bucket = bucket.clone(); + let name = name.clone(); + let version_id = version_id.clone(); + Box::pin(async move { + info!( + bucket = %bucket, + object = %name, + version_id = %version_id, + action = %_action, + "ILM lifecycle trace: {} on {}/{} (version: {})", + _action, bucket, name, version_id + ); + }) + }) } } @@ -133,8 +147,8 @@ struct ExpiryTask { impl ExpiryOp for ExpiryTask { fn op_hash(&self) -> u64 { let mut hasher = Sha256::new(); - hasher.update(format!("{}", self.obj_info.bucket).as_bytes()); - hasher.update(format!("{}", self.obj_info.name).as_bytes()); + hasher.update(self.obj_info.bucket.as_bytes()); + hasher.update(self.obj_info.name.as_bytes()); xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED) } @@ -188,8 +202,8 @@ struct FreeVersionTask(ObjectInfo); impl ExpiryOp for FreeVersionTask { fn op_hash(&self) -> u64 { let mut hasher = Sha256::new(); - hasher.update(format!("{}", self.0.transitioned_object.tier).as_bytes()); - hasher.update(format!("{}", self.0.transitioned_object.name).as_bytes()); + hasher.update(self.0.transitioned_object.tier.as_bytes()); + hasher.update(self.0.transitioned_object.name.as_bytes()); xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED) } @@ -207,8 +221,8 @@ struct NewerNoncurrentTask { impl ExpiryOp for NewerNoncurrentTask { fn op_hash(&self) -> u64 { let mut hasher = Sha256::new(); - hasher.update(format!("{}", self.bucket).as_bytes()); - hasher.update(format!("{}", self.versions[0].object_name).as_bytes()); + hasher.update(self.bucket.as_bytes()); + hasher.update(self.versions[0].object_name.as_bytes()); xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED) } @@ -240,7 +254,7 @@ impl ExpiryState { pub async fn pending_tasks(&self) -> usize { let rxs = &self.tasks_rx; - if rxs.len() == 0 { + if rxs.is_empty() { return 0; } let mut tasks = 0; @@ -253,32 +267,33 @@ impl ExpiryState { pub async fn enqueue_tier_journal_entry(&mut self, je: &Jentry) -> Result<(), std::io::Error> { let wrkr = self.get_worker_ch(je.op_hash()); if wrkr.is_none() { - *self.stats.as_mut().expect("err").missed_tier_journal_tasks.get_mut() += 1; + *self.stats.as_mut().expect("stats lock").missed_tier_journal_tasks.get_mut() += 1; + return Ok(()); } - let wrkr = wrkr.expect("err"); + let wrkr = wrkr.expect("worker channel should exist after None check"); select! { //_ -> GlobalContext.Done() => () _ = wrkr.send(Some(Box::new(je.clone()))) => (), else => { - *self.stats.as_mut().expect("err").missed_tier_journal_tasks.get_mut() += 1; + *self.stats.as_mut().expect("stats lock").missed_tier_journal_tasks.get_mut() += 1; } } - return Ok(()); + Ok(()) } pub async fn enqueue_free_version(&mut self, oi: ObjectInfo) { let task = FreeVersionTask(oi); let wrkr = self.get_worker_ch(task.op_hash()); if wrkr.is_none() { - *self.stats.as_mut().expect("err").missed_freevers_tasks.get_mut() += 1; + *self.stats.as_mut().expect("stats lock").missed_freevers_tasks.get_mut() += 1; return; } - let wrkr = wrkr.expect("err!"); + let wrkr = wrkr.expect("worker channel should exist after None check"); select! { //_ -> GlobalContext.Done() => {} _ = wrkr.send(Some(Box::new(task))) => (), else => { - *self.stats.as_mut().expect("err").missed_freevers_tasks.get_mut() += 1; + *self.stats.as_mut().expect("stats lock").missed_freevers_tasks.get_mut() += 1; } } } @@ -291,21 +306,21 @@ impl ExpiryState { }; let wrkr = self.get_worker_ch(task.op_hash()); if wrkr.is_none() { - *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1; + *self.stats.as_mut().expect("stats lock").missed_expiry_tasks.get_mut() += 1; return; } - let wrkr = wrkr.expect("err!"); + let wrkr = wrkr.expect("worker channel should exist after None check"); select! { //_ -> GlobalContext.Done() => {} _ = wrkr.send(Some(Box::new(task))) => (), else => { - *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1; + *self.stats.as_mut().expect("stats lock").missed_expiry_tasks.get_mut() += 1; } } } pub async fn enqueue_by_newer_noncurrent(&mut self, bucket: &str, versions: Vec, lc_event: lifecycle::Event) { - if versions.len() == 0 { + if versions.is_empty() { return; } @@ -316,26 +331,30 @@ impl ExpiryState { }; let wrkr = self.get_worker_ch(task.op_hash()); if wrkr.is_none() { - *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1; + *self.stats.as_mut().expect("stats lock").missed_expiry_tasks.get_mut() += 1; return; } - let wrkr = wrkr.expect("err!"); + let wrkr = wrkr.expect("worker channel should exist after None check"); select! { //_ -> GlobalContext.Done() => {} _ = wrkr.send(Some(Box::new(task))) => (), else => { - *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1; + *self.stats.as_mut().expect("stats lock").missed_expiry_tasks.get_mut() += 1; } } } pub fn get_worker_ch(&self, h: u64) -> Option>> { - if self.tasks_tx.len() == 0 { + if self.tasks_tx.is_empty() { return None; } Some(self.tasks_tx[h as usize % self.tasks_tx.len()].clone()) } + pub fn increment_missed_tier_journal_tasks(&mut self) { + *self.stats.as_mut().expect("stats lock").missed_tier_journal_tasks.get_mut() += 1; + } + pub async fn resize_workers(n: usize, api: Arc) { if n == GLOBAL_ExpiryState.read().await.tasks_tx.len() || n < 1 { return; @@ -349,11 +368,11 @@ impl ExpiryState { let rx = Arc::new(tokio::sync::Mutex::new(rx)); state.tasks_tx.push(tx); state.tasks_rx.push(rx.clone()); - *state.stats.as_mut().expect("err").workers.get_mut() += 1; + *state.stats.as_mut().expect("stats lock").workers.get_mut() += 1; tokio::spawn(async move { let mut rx = rx.lock().await; //let mut expiry_state = GLOBAL_ExpiryState.read().await; - ExpiryState::worker(&mut *rx, api).await; + ExpiryState::worker(&mut rx, api).await; }); } @@ -363,48 +382,49 @@ impl ExpiryState { worker.send(None).await.unwrap_or(()); state.tasks_tx.remove(l - 1); state.tasks_rx.remove(l - 1); - *state.stats.as_mut().expect("err").workers.get_mut() -= 1; + *state.stats.as_mut().expect("stats lock").workers.get_mut() -= 1; l -= 1; } } pub async fn worker(rx: &mut Receiver>, api: Arc) { - //let cancel_token = - // get_background_services_cancel_token().ok_or_else(|| Error::other("Background services not initialized"))?; + let cancel_token = crate::global::get_background_services_cancel_token().unwrap_or_else(|| { + static FALLBACK: std::sync::OnceLock = std::sync::OnceLock::new(); + FALLBACK.get_or_init(tokio_util::sync::CancellationToken::new) + }); loop { select! { - //_ = cancel_token.cancelled() => { - _ = tokio::signal::ctrl_c() => { - info!("got ctrl+c, exits"); + _ = cancel_token.cancelled() => { + info!("lifecycle expiry worker received shutdown signal, exiting"); break; } v = rx.recv() => { if v.is_none() { break; } - let v = v.expect("err!"); + let v = v.expect("channel closed unexpectedly"); if v.is_none() { //rx.close(); //drop(rx); let _ = rx; return; } - let v = v.expect("err!"); + let v = v.expect("received None after None check"); if v.as_any().is::() { - let v = v.as_any().downcast_ref::().expect("err!"); - if v.obj_info.transitioned_object.status != "" { + let v = v.as_any().downcast_ref::().expect("ExpiryTask downcast failed"); + if !v.obj_info.transitioned_object.status.is_empty() { apply_expiry_on_transitioned_object(api.clone(), &v.obj_info, &v.event, &v.src).await; } else { apply_expiry_on_non_transitioned_objects(api.clone(), &v.obj_info, &v.event, &v.src).await; } } else if v.as_any().is::() { - let _v = v.as_any().downcast_ref::().expect("err!"); - //delete_object_versions(api, &v.bucket, &v.versions, v.event).await; + let v = v.as_any().downcast_ref::().expect("NewerNoncurrentTask downcast failed"); + crate::client::object_handlers_common::delete_object_versions(&api, &v.bucket, &v.versions, v.event.clone()).await; } else if v.as_any().is::() { - let v = v.as_any().downcast_ref::().expect("err!"); + let v = v.as_any().downcast_ref::().expect("Jentry downcast failed"); if let Err(err) = delete_object_from_remote_tier(&v.obj_name, &v.version_id, &v.tier_name).await { warn!( object = %v.obj_name, @@ -416,7 +436,7 @@ impl ExpiryState { } } else if v.as_any().is::() { - let v = v.as_any().downcast_ref::().expect("err!"); + let v = v.as_any().downcast_ref::().expect("FreeVersionTask downcast failed"); let oi = v.0.clone(); if let Err(err) = delete_object_from_remote_tier( &oi.transitioned_object.name, @@ -499,7 +519,7 @@ struct TransitionTask { impl ExpiryOp for TransitionTask { fn op_hash(&self) -> u64 { let mut hasher = Sha256::new(); - hasher.update(format!("{}", self.obj_info.bucket).as_bytes()); + hasher.update(self.obj_info.bucket.as_bytes()); // hasher.update(format!("{}", self.obj_info.versions[0].object_name).as_bytes()); xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED) } @@ -594,22 +614,40 @@ impl TransitionState { if task.is_err() { break; } - let task = task.expect("err!"); + let task = task.expect("channel recv should succeed after error check"); if task.is_none() { //self.transition_rx.close(); //drop(self.transition_rx); return; } - let task = task.expect("err!"); + let task = task.expect("received None after None check"); if task.as_any().is::() { - let task = task.as_any().downcast_ref::().expect("err!"); + let task = task.as_any().downcast_ref::().expect("TransitionTask downcast failed"); GLOBAL_TransitionState.active_tasks.fetch_add(1, Ordering::SeqCst); + + let obj_info_for_event = ObjectInfo { + bucket: task.obj_info.bucket.clone(), + name: task.obj_info.name.clone(), + size: task.obj_info.size, + version_id: task.obj_info.version_id, + ..Default::default() + }; + if let Err(err) = transition_object(api.clone(), &task.obj_info, LcAuditEvent::new(task.event.clone(), task.src.clone())).await { if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) && !err.to_string().contains("use of closed network connection") { error!("Transition to {} failed for {}/{} version:{} with {}", task.event.storage_class, task.obj_info.bucket, task.obj_info.name, task.obj_info.version_id.map(|v| v.to_string()).unwrap_or_default(), err.to_string()); } + // Send s3:ObjectTransition:Failed event + send_event(EventArgs { + event_name: EventName::ObjectTransitionFailed.to_string(), + bucket_name: obj_info_for_event.bucket.clone(), + object: obj_info_for_event, + user_agent: "Internal: [ILM-Transition]".to_string(), + host: GLOBAL_LocalNodeName.to_string(), + ..Default::default() + }); } else { let mut ts = TierStats { total_size: task.obj_info.size as u64, @@ -620,6 +658,16 @@ impl TransitionState { ts.num_objects = 1; } GLOBAL_TransitionState.add_lastday_stats(&task.event.storage_class, ts); + + // Send s3:ObjectTransition:Complete event + send_event(EventArgs { + event_name: EventName::ObjectTransitionComplete.to_string(), + bucket_name: obj_info_for_event.bucket.clone(), + object: obj_info_for_event, + user_agent: "Internal: [ILM-Transition]".to_string(), + host: GLOBAL_LocalNodeName.to_string(), + ..Default::default() + }); } GLOBAL_TransitionState.active_tasks.fetch_add(-1, Ordering::SeqCst); } @@ -634,7 +682,7 @@ impl TransitionState { tier_stats .entry(tier.to_string()) .and_modify(|e| e.add_stats(ts)) - .or_insert(LastDayTierStats::default()); + .or_default(); } pub fn get_daily_all_tier_stats(&self) -> DailyAllTierStats { @@ -666,15 +714,15 @@ impl TransitionState { tokio::spawn(async move { TransitionState::worker(clone_api).await; }); - num_workers = num_workers + 1; + num_workers += 1; GLOBAL_TransitionState.num_workers.fetch_add(1, Ordering::SeqCst); } let mut num_workers = GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst); while num_workers > n { let worker = GLOBAL_TransitionState.kill_tx.clone(); - worker.send(()).await; - num_workers = num_workers - 1; + let _ = worker.send(()).await; + num_workers -= 1; GLOBAL_TransitionState.num_workers.fetch_add(-1, Ordering::SeqCst); } } @@ -683,10 +731,10 @@ impl TransitionState { pub async fn init_background_expiry(api: Arc) { let mut workers = get_env_usize("RUSTFS_MAX_EXPIRY_WORKERS", std::cmp::min(num_cpus::get(), 16)); //globalILMConfig.getExpirationWorkers() - if let Ok(env_expiration_workers) = env::var("_RUSTFS_ILM_EXPIRATION_WORKERS") { - if let Ok(num_expirations) = env_expiration_workers.parse::() { - workers = num_expirations; - } + if let Ok(env_expiration_workers) = env::var("_RUSTFS_ILM_EXPIRATION_WORKERS") + && let Ok(num_expirations) = env_expiration_workers.parse::() + { + workers = num_expirations; } if workers == 0 { @@ -1029,24 +1077,24 @@ pub async fn validate_transition_tier(lc: &BucketLifecycleConfiguration) -> Resu for rule in &lc.rules { if let Some(transitions) = &rule.transitions { for transition in transitions { - if let Some(storage_class) = &transition.storage_class { - if storage_class.as_str() != "" { - let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str()); - if !valid { - return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS)); - } + if let Some(storage_class) = &transition.storage_class + && storage_class.as_str() != "" + { + let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str()); + if !valid { + return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS)); } } } } if let Some(noncurrent_version_transitions) = &rule.noncurrent_version_transitions { for noncurrent_version_transition in noncurrent_version_transitions { - if let Some(storage_class) = &noncurrent_version_transition.storage_class { - if storage_class.as_str() != "" { - let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str()); - if !valid { - return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS)); - } + if let Some(storage_class) = &noncurrent_version_transition.storage_class + && storage_class.as_str() != "" + { + let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str()); + if !valid { + return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS)); } } } @@ -1067,6 +1115,104 @@ pub async fn enqueue_transition_immediate(oi: &ObjectInfo, src: LcEventSrc) { } } +pub async fn enqueue_immediate_expiry(oi: &ObjectInfo, src: LcEventSrc) { + let Some(lifecycle) = GLOBAL_LifecycleSys.get(&oi.bucket).await else { + return; + }; + let Some(api) = crate::new_object_layer_fn() else { + return; + }; + + let mut marker = None; + let mut version_marker = None; + let mut object_infos = Vec::new(); + + loop { + let Ok(page) = api + .clone() + .list_object_versions(&oi.bucket, &oi.name, marker.clone(), version_marker.clone(), None, 1000) + .await + else { + return; + }; + + object_infos.extend(page.objects.into_iter().filter(|object| object.name == oi.name)); + + if !page.is_truncated { + break; + } + + marker = page.next_marker; + version_marker = page.next_version_idmarker; + } + + if object_infos.is_empty() { + object_infos.push(oi.clone()); + } + + let lock_config = match metadata_sys::get_object_lock_config(&oi.bucket).await { + Ok((cfg, _)) => Some(Arc::new(cfg)), + Err(_) => None, + }; + let replication = match metadata_sys::get_replication_config(&oi.bucket).await { + Ok((cfg, _)) if !cfg.rules.is_empty() => Some(Arc::new(ReplicationConfig::new(Some(cfg), None))), + _ => None, + }; + + let object_opts = object_infos + .iter() + .map(|object| object.to_lifecycle_opts()) + .collect::>(); + let Ok(events) = Evaluator::new(Arc::new(lifecycle)) + .with_lock_retention(lock_config) + .with_replication_config(replication) + .eval(&object_opts) + .await + else { + return; + }; + + let mut to_delete_objs = Vec::new(); + let mut noncurrent_event = None; + + for (object, event) in object_infos.iter().zip(events.iter()) { + if event.due != Some(OffsetDateTime::UNIX_EPOCH) { + continue; + } + + match event.action { + IlmAction::DeleteAction + | IlmAction::DeleteRestoredAction + | IlmAction::DeleteRestoredVersionAction + | IlmAction::DeleteAllVersionsAction + | IlmAction::DelMarkerDeleteAllVersionsAction => { + apply_expiry_rule(event, &src, object).await; + } + IlmAction::DeleteVersionAction => { + to_delete_objs.push(ObjectToDelete { + object_name: object.name.clone(), + version_id: object.version_id, + ..Default::default() + }); + if noncurrent_event.is_none() { + noncurrent_event = Some(event.clone()); + } + } + _ => {} + } + } + + if !to_delete_objs.is_empty() + && let Some(event) = noncurrent_event + { + GLOBAL_ExpiryState + .write() + .await + .enqueue_by_newer_noncurrent(&oi.bucket, to_delete_objs, event) + .await; + } +} + pub async fn enqueue_transition_for_existing_objects(api: Arc, bucket: &str) -> Result<(), Error> { let Some(lc) = GLOBAL_LifecycleSys.get(bucket).await else { return Ok(()); @@ -1094,6 +1240,60 @@ pub async fn enqueue_transition_for_existing_objects(api: Arc, bucket: } } +pub async fn enqueue_expiry_for_existing_objects(api: Arc, bucket: &str) -> Result<(), Error> { + let Ok((lc, _)) = metadata_sys::get_lifecycle_config(bucket).await else { + return Ok(()); + }; + let lock_retention = metadata_sys::get_object_lock_config(bucket) + .await + .ok() + .and_then(|(cfg, _)| cfg.rule.and_then(|rule| rule.default_retention)); + let replication_config = metadata_sys::get_replication_config(bucket).await.ok(); + let mut marker = None; + let mut version_marker = None; + let src = LcEventSrc::Scanner; + + loop { + let page = api + .clone() + .list_object_versions(bucket, "", marker.clone(), version_marker.clone(), None, 1000) + .await?; + + for object in &page.objects { + let event = eval_action_from_lifecycle(&lc, lock_retention.clone(), replication_config.clone(), object).await; + match event.action { + IlmAction::DeleteAction + | IlmAction::DeleteVersionAction + | IlmAction::DeleteRestoredAction + | IlmAction::DeleteRestoredVersionAction + | IlmAction::DeleteAllVersionsAction + | IlmAction::DelMarkerDeleteAllVersionsAction => { + if event + .due + .is_some_and(|due| due.unix_timestamp() <= OffsetDateTime::now_utc().unix_timestamp()) + { + if object.is_remote() { + apply_expiry_on_transitioned_object(api.clone(), object, &event, &src).await; + } else { + apply_expiry_on_non_transitioned_objects(api.clone(), object, &event, &src).await; + } + } else { + apply_expiry_rule(&event, &src, object).await; + } + } + _ => {} + } + } + + if !page.is_truncated { + return Ok(()); + } + + marker = page.next_marker; + version_marker = page.next_version_idmarker; + } +} + async fn enqueue_transition_with_lifecycle(oi: &ObjectInfo, lc: &BucketLifecycleConfiguration, src: &LcEventSrc) { let event = lc.eval(&oi.to_lifecycle_opts()).await; match event.action { @@ -1154,6 +1354,8 @@ pub async fn expire_transitioned_object( } }; + schedule_lifecycle_replication_delete_if_needed(oi).await; + //defer auditLogLifecycle(ctx, *oi, ILMExpiry, tags, traceFn) let event_name = if oi.delete_marker { @@ -1227,8 +1429,19 @@ pub async fn transition_object(api: Arc, oi: &ObjectInfo, lae: LcAuditE result } -pub fn audit_tier_actions(_api: ECStore, _tier: &str, _bytes: i64) -> TimeFn { - Arc::new(|| Box::pin(async move {})) +pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn { + let tier = _tier.to_string(); + Arc::new(move || { + let tier = tier.clone(); + Box::pin(async move { + info!( + tier = %tier, + bytes = bytes, + "ILM tier transition audit: completed transition of {} bytes to tier '{}'", + bytes, tier + ); + }) + }) } pub async fn get_transitioned_object_reader( @@ -1245,11 +1458,11 @@ pub async fn get_transitioned_object_reader( Err(err) => return Err(std::io::Error::other(err)), }; - let ret = new_getobjectreader(rs, &oi, opts, &h); + let ret = new_getobjectreader(rs, oi, opts, h); if let Err(err) = ret { return Err(error_resp_to_object_err(err, vec![bucket, object])); } - let (get_fn, off, length) = ret.expect("err"); + let (get_fn, off, length) = ret.expect("get_transitioned_object_reader should succeed after error check"); let mut gopts = WarmBackendGetOpts::default(); if off >= 0 && length >= 0 { @@ -1269,8 +1482,8 @@ pub async fn post_restore_opts(version_id: &str, bucket: &str, object: &str) -> let versioned = BucketVersioningSys::prefix_enabled(bucket, object).await; let version_suspended = BucketVersioningSys::prefix_suspended(bucket, object).await; let vid = version_id.trim(); - if vid != "" && vid != NULL_VERSION_ID { - if let Err(err) = Uuid::parse_str(vid) { + if !vid.is_empty() && vid != NULL_VERSION_ID { + if let Err(_err) = Uuid::parse_str(vid) { return Err(std::io::Error::other( StorageError::InvalidVersionID(bucket.to_string(), object.to_string(), vid.to_string()).to_string(), )); @@ -1332,24 +1545,21 @@ pub async fn put_restore_opts( } meta.insert(v.name.clone().unwrap(), v.value.clone().unwrap_or_else(|| "".to_string())); } - if let Some(output_location) = rreq.output_location.as_ref() { - if let Some(s3) = &output_location.s3 { - if let Some(tags) = &s3.tagging { - meta.insert( - AMZ_OBJECT_TAGGING.to_string(), - serde_urlencoded::to_string(tags.tag_set.clone()).unwrap_or_else(|_| "".to_string()), - ); - } - } + if let Some(output_location) = rreq.output_location.as_ref() + && let Some(s3) = &output_location.s3 + && let Some(tags) = &s3.tagging + { + meta.insert( + AMZ_OBJECT_TAGGING.to_string(), + serde_urlencoded::to_string(tags.tag_set.clone()).unwrap_or_else(|_| "".to_string()), + ); } - if let Some(output_location) = rreq.output_location.as_ref() { - if let Some(s3) = &output_location.s3 { - if let Some(encryption) = &s3.encryption { - if encryption.encryption_type.as_str() != "" { - meta.insert(X_AMZ_SERVER_SIDE_ENCRYPTION.as_str().to_string(), AMZ_ENCRYPTION_AES.to_string()); - } - } - } + if let Some(output_location) = rreq.output_location.as_ref() + && let Some(s3) = &output_location.s3 + && let Some(encryption) = &s3.encryption + && encryption.encryption_type.as_str() != "" + { + meta.insert(X_AMZ_SERVER_SIDE_ENCRYPTION.as_str().to_string(), AMZ_ENCRYPTION_AES.to_string()); } return Ok(ObjectOptions { versioned: BucketVersioningSys::prefix_enabled(bucket, object).await, @@ -1361,7 +1571,7 @@ pub async fn put_restore_opts( for (k, v) in &oi.user_defined { meta.insert(k.to_string(), v.clone()); } - if oi.user_tags.len() != 0 { + if !oi.user_tags.is_empty() { meta.insert(AMZ_OBJECT_TAGGING.to_string(), oi.user_tags.clone()); } let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1)); @@ -1394,7 +1604,7 @@ impl LifecycleOps for ObjectInfo { lifecycle::ObjectOpts { name: self.name.clone(), user_tags: self.user_tags.clone(), - version_id: self.version_id.clone(), + version_id: self.version_id, mod_time: self.mod_time, size: self.size as usize, is_latest: self.is_latest, @@ -1421,38 +1631,42 @@ pub trait RestoreRequestOps { } impl RestoreRequestOps for RestoreRequest { - fn validate(&self, api: Arc) -> Result<(), std::io::Error> { - /*if self.type_.is_none() && self.select_parameters.is_some() { + fn validate(&self, _api: Arc) -> Result<(), std::io::Error> { + // SELECT type requires select_parameters, and vice versa + if self.type_.as_ref().is_none_or(|t| t.as_str() != RestoreRequestType::SELECT) && self.select_parameters.is_some() { return Err(std::io::Error::other("Select parameters can only be specified with SELECT request type")); } - if let Some(type_) = self.type_ && type_ == RestoreRequestType::SELECT && self.select_parameters.is_none() { + if let Some(type_) = &self.type_ + && type_.as_str() == RestoreRequestType::SELECT + && self.select_parameters.is_none() + { return Err(std::io::Error::other("SELECT restore request requires select parameters to be specified")); } - if self.type_.is_none() && self.output_location.is_some() { - return Err(std::io::Error::other("OutputLocation required only for SELECT request type")); + // OutputLocation is only valid for SELECT requests + if self.type_.as_ref().is_none_or(|t| t.as_str() != RestoreRequestType::SELECT) && self.output_location.is_some() { + return Err(std::io::Error::other("OutputLocation can only be specified with SELECT request type")); } - if let Some(type_) = self.type_ && type_ == RestoreRequestType::SELECT && self.output_location.is_none() { + if let Some(type_) = &self.type_ + && type_.as_str() == RestoreRequestType::SELECT + && self.output_location.is_none() + { return Err(std::io::Error::other("OutputLocation required for SELECT requests")); } - if let Some(type_) = self.type_ && type_ == RestoreRequestType::SELECT && self.days != 0 { + // Days must not be specified with SELECT requests + if let Some(type_) = &self.type_ + && type_.as_str() == RestoreRequestType::SELECT + && self.days.is_some_and(|d| d > 0) + { return Err(std::io::Error::other("Days cannot be specified with SELECT restore request")); } - if self.days == 0 && self.type_.is_none() { + + // For non-SELECT requests, days must be at least 1 + if self.type_.is_none() && self.days.is_none_or(|d| d <= 0) { return Err(std::io::Error::other("restoration days should be at least 1")); } - if self.output_location.is_some() { - if _, err := api.get_bucket_info(self.output_location.s3.bucket_name, BucketOptions{}); err != nil { - return err - } - if self.output_location.s3.prefix == "" { - return Err(std::io::Error::other("Prefix is a required parameter in OutputLocation")); - } - if self.output_location.s3.encryption.encryption_type.as_str() != ServerSideEncryption::AES256 { - return NotImplemented{} - } - }*/ + Ok(()) } } @@ -1497,10 +1711,10 @@ pub async fn eval_action_from_lifecycle( //} return lifecycle::Event::default(); } - if let Some(rcfg) = rcfg { - if rep_has_active_rules(&rcfg.0, &oi.name, true) { - return lifecycle::Event::default(); - } + if let Some(rcfg) = rcfg + && rep_has_active_rules(&rcfg.0, &oi.name, true) + { + return lifecycle::Event::default(); } } _ => (), @@ -1566,6 +1780,7 @@ pub async fn apply_expiry_on_non_transitioned_objects( return false; } }; + schedule_lifecycle_replication_delete_if_needed(oi).await; //debug!("dobj: {:?}", dobj); if dobj.name.is_empty() { dobj = oi.clone(); @@ -1606,6 +1821,70 @@ pub async fn apply_expiry_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: & true } +async fn schedule_lifecycle_replication_delete_if_needed(oi: &ObjectInfo) { + if !oi.delete_marker || oi.version_id.is_none() { + return; + } + + let replication_state = lifecycle_delete_replication_state(oi).await; + if replication_state.is_none() { + return; + } + + schedule_replication_delete(DeletedObjectReplicationInfo { + delete_object: crate::store_api::DeletedObject { + object_name: oi.name.clone(), + delete_marker_version_id: oi.version_id, + delete_marker: false, + delete_marker_mtime: oi.mod_time, + replication_state, + ..Default::default() + }, + bucket: oi.bucket.clone(), + event_type: REPLICATE_INCOMING_DELETE.to_string(), + ..Default::default() + }) + .await; +} + +async fn lifecycle_delete_replication_state(oi: &ObjectInfo) -> Option { + if !oi.replication_decision.is_empty() || oi.version_purge_status == VersionPurgeStatusType::Pending { + return Some(oi.replication_state()); + } + + let dsc = check_replicate_delete( + &oi.bucket, + &ObjectToDelete { + object_name: oi.name.clone(), + version_id: oi.version_id, + ..Default::default() + }, + oi, + &ObjectOptions { + version_id: oi.version_id.map(|v| v.to_string()), + versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await, + ..Default::default() + }, + None, + ) + .await; + if !dsc.replicate_any() { + return None; + } + + Some(replication_state_for_version_delete(dsc)) +} + +fn replication_state_for_version_delete(dsc: ReplicateDecision) -> ReplicationState { + let pending_status = dsc.pending_status(); + ReplicationState { + replicate_decision_str: dsc.to_string(), + version_purge_status_internal: pending_status.clone(), + purge_targets: rustfs_filemeta::version_purge_statuses_map(pending_status.as_deref().unwrap_or_default()), + ..Default::default() + } +} + pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool { let mut success = false; match event.action { diff --git a/crates/ecstore/src/bucket/lifecycle/lifecycle.rs b/crates/ecstore/src/bucket/lifecycle/core.rs similarity index 63% rename from crates/ecstore/src/bucket/lifecycle/lifecycle.rs rename to crates/ecstore/src/bucket/lifecycle/core.rs index c56e2544d..a9456ca97 100644 --- a/crates/ecstore/src/bucket/lifecycle/lifecycle.rs +++ b/crates/ecstore/src/bucket/lifecycle/core.rs @@ -11,30 +11,21 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -#![allow(unused_imports)] -#![allow(unused_variables)] -#![allow(unused_mut)] -#![allow(unused_assignments)] -#![allow(unused_must_use)] -#![allow(clippy::all)] use rustfs_filemeta::{ReplicationStatusType, VersionPurgeStatusType}; use s3s::dto::{ - BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleAndOperator, - LifecycleRuleFilter, NoncurrentVersionTransition, ObjectLockConfiguration, ObjectLockEnabled, RestoreRequest, Transition, - TransitionStorageClass, + BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter, + NoncurrentVersionTransition, ObjectLockConfiguration, ObjectLockEnabled, RestoreRequest, Transition, TransitionStorageClass, }; use std::cmp::Ordering; use std::collections::HashMap; use std::env; -use std::fmt::Display; use std::sync::Arc; -use time::macros::{datetime, offset}; +use time::macros::offset; use time::{self, Duration, OffsetDateTime}; -use tracing::info; +use tracing::{debug, info}; use uuid::Uuid; -use crate::bucket::lifecycle::rule::TransitionOps; use crate::store_api::ObjectInfo; pub const TRANSITION_COMPLETE: &str = "complete"; @@ -43,15 +34,16 @@ const ERR_LIFECYCLE_NO_RULE: &str = "Lifecycle configuration should have at leas const ERR_LIFECYCLE_DUPLICATE_ID: &str = "Rule ID must be unique. Found same ID for more than one rule"; const _ERR_XML_NOT_WELL_FORMED: &str = "The XML you provided was not well-formed or did not validate against our published schema"; -const ERR_LIFECYCLE_BUCKET_LOCKED: &str = - "ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an retention bucket"; +const ERR_LIFECYCLE_BUCKET_LOCKED: &str = "ExpiredObjectDeleteMarker is not allowed on a bucket with Object Lock enabled"; const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration should have at most 1000 rules"; -const ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS: &str = "Lifecycle expiration days must be greater than 0"; +const ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS: &str = "Lifecycle expiration days must not be negative"; +const ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS: &str = "Lifecycle noncurrent expiration days must not be negative"; const ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT: &str = "Expiration.Date must be at midnight UTC"; const ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG: &str = "Rule ID must be at most 255 characters"; const ERR_LIFECYCLE_INVALID_RULE_STATUS: &str = "Rule status must be either Enabled or Disabled"; const ERR_LIFECYCLE_DEL_MARKER_WITH_TAGS: &str = "Rule with DelMarkerExpiration cannot have tags based filtering"; const ERR_LIFECYCLE_RULE_MUST_HAVE_ACTION: &str = "Rule must have at least one of Expiration, Transition, NoncurrentVersionExpiration, NoncurrentVersionTransition, or DelMarkerExpiration"; +const ERR_LIFECYCLE_PREFIX_FILTER_CONFLICT: &str = "Legacy Prefix and Filter cannot both be present in a lifecycle rule. Use Filter.Prefix instead of the top-level Prefix element."; pub use rustfs_common::metrics::IlmAction; @@ -120,22 +112,30 @@ impl RuleValidate for LifecycleRule { }*/ fn validate(&self) -> Result<(), std::io::Error> { + // S3 standard: Legacy Prefix and Filter cannot both be present. + // An empty prefix is treated as "not set" and is allowed with a Filter. + let has_legacy_prefix = self.prefix.as_deref().is_some_and(|p| !p.is_empty()); + let has_filter = self.filter.is_some(); + if has_legacy_prefix && has_filter { + return Err(std::io::Error::other(ERR_LIFECYCLE_PREFIX_FILTER_CONFLICT)); + } + // Rule with DelMarkerExpiration cannot have tags based filtering let has_tag_filter = self .filter .as_ref() - .map_or(false, |f| f.tag.is_some() || f.and.as_ref().and_then(|a| a.tags.as_ref()).is_some()); + .is_some_and(|f| f.tag.is_some() || f.and.as_ref().and_then(|a| a.tags.as_ref()).is_some()); if has_tag_filter && self.del_marker_expiration.is_some() { return Err(std::io::Error::other(ERR_LIFECYCLE_DEL_MARKER_WITH_TAGS)); } // Rule must have at least one action let has_expiration = self.expiration.is_some(); - let has_transition = self.transitions.as_ref().map_or(false, |t| !t.is_empty()); + let has_transition = self.transitions.as_ref().is_some_and(|t| !t.is_empty()); let has_noncurrent_expiration = self .noncurrent_version_expiration .as_ref() .and_then(|e| e.noncurrent_days) - .map_or(false, |d| d != 0); + .is_some(); let has_noncurrent_transition = self .noncurrent_version_transitions .as_ref() @@ -147,7 +147,7 @@ impl RuleValidate for LifecycleRule { .del_marker_expiration .as_ref() .and_then(|d| d.days) - .map_or(false, |d| d > 0); + .is_some_and(|d| d > 0); if !has_expiration && !has_transition && !has_noncurrent_expiration @@ -163,15 +163,13 @@ impl RuleValidate for LifecycleRule { fn lifecycle_rule_prefix(rule: &LifecycleRule) -> Option<&str> { // Prefer a non-empty legacy prefix; treat an empty legacy prefix as if it were not set - if let Some(p) = rule.prefix.as_deref() { - if !p.is_empty() { - return Some(p); - } + if let Some(p) = rule.prefix.as_deref() + && !p.is_empty() + { + return Some(p); } - let Some(filter) = rule.filter.as_ref() else { - return None; - }; + let filter = rule.filter.as_ref()?; if let Some(p) = filter.prefix.as_deref() { return Some(p); @@ -198,7 +196,7 @@ pub trait Lifecycle { impl Lifecycle for BucketLifecycleConfiguration { async fn has_transition(&self) -> bool { for rule in self.rules.iter() { - if !rule.transitions.is_none() { + if rule.transitions.is_some() { return true; } } @@ -207,7 +205,7 @@ impl Lifecycle for BucketLifecycleConfiguration { fn has_expiry(&self) -> bool { for rule in self.rules.iter() { - if !rule.expiration.is_none() || !rule.noncurrent_version_expiration.is_none() { + if rule.expiration.is_some() || rule.noncurrent_version_expiration.is_some() { return true; } } @@ -215,7 +213,7 @@ impl Lifecycle for BucketLifecycleConfiguration { } fn has_active_rules(&self, prefix: &str) -> bool { - if self.rules.len() == 0 { + if self.rules.is_empty() { return false; } for rule in self.rules.iter() { @@ -233,29 +231,32 @@ impl Lifecycle for BucketLifecycleConfiguration { } if let Some(rule_noncurrent_version_expiration) = &rule.noncurrent_version_expiration { - if let Some(noncurrent_days) = rule_noncurrent_version_expiration.noncurrent_days { - if noncurrent_days > 0 { - return true; - } + if let Some(noncurrent_days) = rule_noncurrent_version_expiration.noncurrent_days + && noncurrent_days >= 0 + { + return true; } - if let Some(newer_noncurrent_versions) = rule_noncurrent_version_expiration.newer_noncurrent_versions { - if newer_noncurrent_versions > 0 { - return true; - } + if let Some(newer_noncurrent_versions) = rule_noncurrent_version_expiration.newer_noncurrent_versions + && newer_noncurrent_versions > 0 + { + return true; } } if rule.noncurrent_version_transitions.is_some() { return true; } if let Some(rule_expiration) = &rule.expiration { - if let Some(date1) = rule_expiration.date.clone() { - if OffsetDateTime::from(date1).unix_timestamp() < OffsetDateTime::now_utc().unix_timestamp() { - return true; - } + if let Some(date1) = rule_expiration.date.clone() + && OffsetDateTime::from(date1).unix_timestamp() < OffsetDateTime::now_utc().unix_timestamp() + { + return true; } if rule_expiration.date.is_some() { return true; } + if rule_expiration.days.is_some() { + return true; + } if let Some(expired_object_delete_marker) = rule_expiration.expired_object_delete_marker && expired_object_delete_marker { @@ -264,10 +265,10 @@ impl Lifecycle for BucketLifecycleConfiguration { } if let Some(rule_transitions) = &rule.transitions { let rule_transitions_0 = rule_transitions[0].clone(); - if let Some(date1) = rule_transitions_0.date { - if OffsetDateTime::from(date1).unix_timestamp() < OffsetDateTime::now_utc().unix_timestamp() { - return true; - } + if let Some(date1) = rule_transitions_0.date + && OffsetDateTime::from(date1).unix_timestamp() < OffsetDateTime::now_utc().unix_timestamp() + { + return true; } } if rule.transitions.is_some() { @@ -281,7 +282,7 @@ impl Lifecycle for BucketLifecycleConfiguration { if self.rules.len() > 1000 { return Err(std::io::Error::other(ERR_LIFECYCLE_TOO_MANY_RULES)); } - if self.rules.len() == 0 { + if self.rules.is_empty() { return Err(std::io::Error::other(ERR_LIFECYCLE_NO_RULE)); } @@ -299,27 +300,37 @@ impl Lifecycle for BucketLifecycleConfiguration { return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT)); } } - if let Some(days) = expiration.days { - if days <= 0 { - return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS)); - } + if let Some(days) = expiration.days + && days < 0 + { + return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS)); } } - if let Some(id) = &r.id { - if id.len() > 255 { - return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG)); - } + if let Some(noncurrent_version_expiration) = &r.noncurrent_version_expiration + && let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days + && noncurrent_days < 0 + { + return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS)); + } + if let Some(id) = &r.id + && id.len() > 255 + { + return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG)); } r.validate()?; - /*if let Some(object_lock_enabled) = lr.object_lock_enabled.as_ref() { - if let Some(expiration) = r.expiration.as_ref() { - if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker { - if object_lock_enabled.as_str() == ObjectLockEnabled::ENABLED && (expired_object_delete_marker) { - return Err(std::io::Error::other(ERR_LIFECYCLE_BUCKET_LOCKED)); - } - } + if let Some(object_lock_enabled) = lr.object_lock_enabled.as_ref() + && object_lock_enabled.as_str() == ObjectLockEnabled::ENABLED + && let Some(expiration) = r.expiration.as_ref() + { + // Object Lock + ExpiredObjectDeleteMarker conflict + if expiration.expired_object_delete_marker.is_some_and(|v| v) { + return Err(std::io::Error::other(ERR_LIFECYCLE_BUCKET_LOCKED)); } - }*/ + // Object Lock + ExpiredObjectAllVersions conflict (MinIO extension) + if expiration.expired_object_all_versions.is_some_and(|v| v) { + return Err(std::io::Error::other(ERR_LIFECYCLE_BUCKET_LOCKED)); + } + } } for (i, _) in self.rules.iter().enumerate() { if i == self.rules.len() - 1 { @@ -327,10 +338,10 @@ impl Lifecycle for BucketLifecycleConfiguration { } let other_rules = &self.rules[i + 1..]; for other_rule in other_rules { - if let (Some(id1), Some(id2)) = (&self.rules[i].id, &other_rule.id) { - if id1 == id2 { - return Err(std::io::Error::other(ERR_LIFECYCLE_DUPLICATE_ID)); - } + if let (Some(id1), Some(id2)) = (&self.rules[i].id, &other_rule.id) + && id1 == id2 + { + return Err(std::io::Error::other(ERR_LIFECYCLE_DUPLICATE_ID)); } } } @@ -338,7 +349,7 @@ impl Lifecycle for BucketLifecycleConfiguration { } async fn filter_rules(&self, obj: &ObjectOpts) -> Option> { - if obj.name == "" { + if obj.name.is_empty() { return None; } let mut rules = Vec::::new(); @@ -346,10 +357,10 @@ impl Lifecycle for BucketLifecycleConfiguration { if rule.status.as_str() == ExpirationStatus::DISABLED { continue; } - if let Some(rule_prefix) = lifecycle_rule_prefix(rule) { - if !obj.name.starts_with(rule_prefix) { - continue; - } + if let Some(rule_prefix) = lifecycle_rule_prefix(rule) + && !obj.name.starts_with(rule_prefix) + { + continue; } if let Some(filter) = rule.filter.as_ref() { if !::test_tags(filter, &obj.user_tags) { @@ -413,10 +424,10 @@ impl Lifecycle for BucketLifecycleConfiguration { newer_noncurrent_versions: 0, storage_class: "".into(), }; - let predicted_due = predicted.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp(); + let predicted_due = predicted.due.unwrap_or(OffsetDateTime::UNIX_EPOCH).unix_timestamp(); let should_replace = event .as_ref() - .is_none_or(|e| predicted_due < e.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp()); + .is_none_or(|e| predicted_due < e.due.unwrap_or(OffsetDateTime::UNIX_EPOCH).unix_timestamp()); if should_replace { event = Some(predicted); } @@ -425,9 +436,9 @@ impl Lifecycle for BucketLifecycleConfiguration { event.unwrap_or_default() } - async fn eval_inner(&self, obj: &ObjectOpts, now: OffsetDateTime, newer_noncurrent_versions: usize) -> Event { + async fn eval_inner(&self, obj: &ObjectOpts, now: OffsetDateTime, _newer_noncurrent_versions: usize) -> Event { let mut events = Vec::::new(); - info!( + debug!( "eval_inner: object={}, mod_time={:?}, now={:?}, is_latest={}, delete_marker={}", obj.name, obj.mod_time, now, obj.is_latest, obj.delete_marker ); @@ -436,139 +447,131 @@ impl Lifecycle for BucketLifecycleConfiguration { let mod_time = match obj.mod_time { Some(t) => t, None => { - info!("eval_inner: mod_time is None for object={}, returning default event", obj.name); + debug!("eval_inner: mod_time is None for object={}, returning default event", obj.name); return Event::default(); } }; if mod_time.unix_timestamp() == 0 { - info!("eval_inner: mod_time is 0, returning default event"); + debug!("eval_inner: mod_time is 0, returning default event"); return Event::default(); } - if let Some(restore_expires) = obj.restore_expires { - if !restore_expires.unix_timestamp() == 0 && now.unix_timestamp() > restore_expires.unix_timestamp() { - let mut action = IlmAction::DeleteRestoredAction; - if !obj.is_latest { - action = IlmAction::DeleteRestoredVersionAction; - } - - events.push(Event { - action, - due: Some(now), - rule_id: "".into(), - noncurrent_days: 0, - newer_noncurrent_versions: 0, - storage_class: "".into(), - }); + if let Some(restore_expires) = obj.restore_expires + && restore_expires.unix_timestamp() != 0 + && now.unix_timestamp() > restore_expires.unix_timestamp() + { + let mut action = IlmAction::DeleteRestoredAction; + if !obj.is_latest { + action = IlmAction::DeleteRestoredVersionAction; } + + events.push(Event { + action, + due: Some(now), + rule_id: "".into(), + noncurrent_days: 0, + newer_noncurrent_versions: 0, + storage_class: "".into(), + }); } if let Some(ref lc_rules) = self.filter_rules(obj).await { for rule in lc_rules.iter() { if obj.is_latest && obj.expired_object_deletemarker() { - if let Some(expiration) = rule.expiration.as_ref() { - if expiration.expired_object_delete_marker.is_some_and(|v| v) { - // Preserve explicit date/days scheduling when configured. - // If only ExpiredObjectDeleteMarker=true is set, delete immediately. - let due = expiration.next_due(obj).unwrap_or(now); - if now.unix_timestamp() >= due.unix_timestamp() { - events.push(Event { - action: IlmAction::DeleteVersionAction, - rule_id: rule.id.clone().unwrap_or_default(), - due: Some(due), - noncurrent_days: 0, - newer_noncurrent_versions: 0, - storage_class: "".into(), - }); - // Stop after scheduling an expired delete-marker event. - break; - } + if let Some(expiration) = rule.expiration.as_ref() + && expiration.expired_object_delete_marker.is_some_and(|v| v) + { + // Preserve explicit date/days scheduling when configured. + // If only ExpiredObjectDeleteMarker=true is set, delete immediately. + let due = expiration.next_due(obj).unwrap_or(now); + if now.unix_timestamp() >= due.unix_timestamp() { + events.push(Event { + action: IlmAction::DeleteVersionAction, + rule_id: rule.id.clone().unwrap_or_default(), + due: Some(due), + noncurrent_days: 0, + newer_noncurrent_versions: 0, + storage_class: "".into(), + }); + // Stop after scheduling an expired delete-marker event. + break; } } // DelMarkerExpiration: expire delete marker after N days from mod_time - if obj.delete_marker { - if let Some(ref dme) = rule.del_marker_expiration { - if let Some(days) = dme.days { - if days > 0 { - let due = expected_expiry_time(mod_time, days); - if now.unix_timestamp() >= due.unix_timestamp() { - events.push(Event { - action: IlmAction::DelMarkerDeleteAllVersionsAction, - rule_id: rule.id.clone().unwrap_or_default(), - due: Some(due), - noncurrent_days: 0, - newer_noncurrent_versions: 0, - storage_class: "".into(), - }); - } - continue; - } - } + if obj.delete_marker + && let Some(ref dme) = rule.del_marker_expiration + && let Some(days) = dme.days + && days > 0 + { + let due = expected_expiry_time(mod_time, days); + if now.unix_timestamp() >= due.unix_timestamp() { + events.push(Event { + action: IlmAction::DelMarkerDeleteAllVersionsAction, + rule_id: rule.id.clone().unwrap_or_default(), + due: Some(due), + noncurrent_days: 0, + newer_noncurrent_versions: 0, + storage_class: "".into(), + }); } + continue; } } - if !obj.is_latest { - if let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration { - if let Some(newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions { - if newer_noncurrent_versions > 0 { - continue; - } - } + if !obj.is_latest + && let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration + && let Some(newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions + && newer_noncurrent_versions > 0 + { + continue; + } + + if !obj.is_latest + && let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration + && let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days + && let Some(successor_mod_time) = obj.successor_mod_time + { + let expected_expiry = expected_expiry_time(successor_mod_time, noncurrent_days); + if now.unix_timestamp() >= expected_expiry.unix_timestamp() { + events.push(Event { + action: IlmAction::DeleteVersionAction, + rule_id: rule.id.clone().unwrap_or_default(), + due: Some(expected_expiry), + noncurrent_days: 0, + newer_noncurrent_versions: 0, + storage_class: "".into(), + }); } } - if !obj.is_latest { - if let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration { - if let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days { - if noncurrent_days != 0 { - if let Some(successor_mod_time) = obj.successor_mod_time { - let expected_expiry = expected_expiry_time(successor_mod_time, noncurrent_days); - if now.unix_timestamp() >= expected_expiry.unix_timestamp() { - events.push(Event { - action: IlmAction::DeleteVersionAction, - rule_id: rule.id.clone().unwrap_or_default(), - due: Some(expected_expiry), - noncurrent_days: 0, - newer_noncurrent_versions: 0, - storage_class: "".into(), - }); - } - } - } - } + if !obj.is_latest + && let Some(ref noncurrent_version_transitions) = rule.noncurrent_version_transitions + && let Some(ref storage_class) = noncurrent_version_transitions[0].storage_class + && storage_class.as_str() != "" + && !obj.delete_marker + && obj.transition_status != TRANSITION_COMPLETE + { + let due = rule.noncurrent_version_transitions.as_ref().unwrap()[0].next_due(obj); + if let Some(due0) = due + && (now.unix_timestamp() == 0 || now.unix_timestamp() > due0.unix_timestamp()) + { + events.push(Event { + action: IlmAction::TransitionVersionAction, + rule_id: rule.id.clone().unwrap_or_default(), + due, + storage_class: rule.noncurrent_version_transitions.as_ref().unwrap()[0] + .storage_class + .clone() + .unwrap() + .as_str() + .to_string(), + ..Default::default() + }); } } - if !obj.is_latest { - if let Some(ref noncurrent_version_transitions) = rule.noncurrent_version_transitions { - if let Some(ref storage_class) = noncurrent_version_transitions[0].storage_class { - if storage_class.as_str() != "" && !obj.delete_marker && obj.transition_status != TRANSITION_COMPLETE - { - let due = rule.noncurrent_version_transitions.as_ref().unwrap()[0].next_due(obj); - if let Some(due0) = due { - if now.unix_timestamp() == 0 || now.unix_timestamp() > due0.unix_timestamp() { - events.push(Event { - action: IlmAction::TransitionVersionAction, - rule_id: rule.id.clone().unwrap_or_default(), - due, - storage_class: rule.noncurrent_version_transitions.as_ref().unwrap()[0] - .storage_class - .clone() - .unwrap() - .as_str() - .to_string(), - ..Default::default() - }); - } - } - } - } - } - } - - info!( + debug!( "eval_inner: checking expiration condition - is_latest={}, delete_marker={}, version_id={:?}, condition_met={}", obj.is_latest, obj.delete_marker, @@ -577,12 +580,12 @@ impl Lifecycle for BucketLifecycleConfiguration { ); // Allow expiration for latest objects OR non-versioned objects (empty version_id) if (obj.is_latest || obj.version_id.is_none_or(|v| v.is_nil())) && !obj.delete_marker { - info!("eval_inner: entering expiration check"); + debug!("eval_inner: entering expiration check"); if let Some(ref expiration) = rule.expiration { if let Some(ref date) = expiration.date { let date0 = OffsetDateTime::from(date.clone()); if date0.unix_timestamp() != 0 && (now.unix_timestamp() >= date0.unix_timestamp()) { - info!("eval_inner: expiration by date - date0={:?}", date0); + debug!("eval_inner: expiration by date - date0={:?}", date0); events.push(Event { action: IlmAction::DeleteAction, rule_id: rule.id.clone().unwrap_or_default(), @@ -594,7 +597,7 @@ impl Lifecycle for BucketLifecycleConfiguration { } } else if let Some(days) = expiration.days { let expected_expiry: OffsetDateTime = expected_expiry_time(mod_time, days); - info!( + debug!( "eval_inner: expiration check - days={}, obj_time={:?}, expiry_time={:?}, now={:?}, should_expire={}", days, mod_time, @@ -603,7 +606,7 @@ impl Lifecycle for BucketLifecycleConfiguration { now.unix_timestamp() > expected_expiry.unix_timestamp() ); if now.unix_timestamp() >= expected_expiry.unix_timestamp() { - info!("eval_inner: object should expire, adding DeleteAction"); + debug!("eval_inner: object should expire, adding DeleteAction"); let mut event = Event { action: IlmAction::DeleteAction, rule_id: rule.id.clone().unwrap_or_default(), @@ -612,50 +615,56 @@ impl Lifecycle for BucketLifecycleConfiguration { newer_noncurrent_versions: 0, storage_class: "".into(), }; - /*if rule.expiration.expect("err!").delete_all.val { - event.action = IlmAction::DeleteAllVersionsAction - }*/ + // MinIO extension: ExpiredObjectAllVersions deletes all versions + if rule + .expiration + .as_ref() + .and_then(|e| e.expired_object_all_versions) + .unwrap_or(false) + { + event.action = IlmAction::DeleteAllVersionsAction; + } events.push(event); } } else { - info!("eval_inner: expiration.days is None"); + debug!("eval_inner: expiration.days is None"); } } else { - info!("eval_inner: rule.expiration is None"); + debug!("eval_inner: rule.expiration is None"); } - if obj.transition_status != TRANSITION_COMPLETE { - if let Some(ref transitions) = rule.transitions { - let due = transitions[0].next_due(obj); - if let Some(due0) = due { - if now.unix_timestamp() == 0 || now.unix_timestamp() > due0.unix_timestamp() { - events.push(Event { - action: IlmAction::TransitionAction, - rule_id: rule.id.clone().unwrap_or_default(), - due, - storage_class: transitions[0] - .storage_class - .clone() - .unwrap_or_else(|| TransitionStorageClass::from_static("")) - .as_str() - .to_string(), - noncurrent_days: 0, - newer_noncurrent_versions: 0, - }); - } - } + if obj.transition_status != TRANSITION_COMPLETE + && let Some(ref transitions) = rule.transitions + { + let due = transitions[0].next_due(obj); + if let Some(due0) = due + && (now.unix_timestamp() == 0 || now.unix_timestamp() > due0.unix_timestamp()) + { + events.push(Event { + action: IlmAction::TransitionAction, + rule_id: rule.id.clone().unwrap_or_default(), + due, + storage_class: transitions[0] + .storage_class + .clone() + .unwrap_or_else(|| TransitionStorageClass::from_static("")) + .as_str() + .to_string(), + noncurrent_days: 0, + newer_noncurrent_versions: 0, + }); } } } } } - if events.len() > 0 { + if !events.is_empty() { events.sort_by(|a, b| { - if now.unix_timestamp() > a.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp() - && now.unix_timestamp() > b.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp() - || a.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp() - == b.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp() + if now.unix_timestamp() > a.due.unwrap_or(OffsetDateTime::UNIX_EPOCH).unix_timestamp() + && now.unix_timestamp() > b.due.unwrap_or(OffsetDateTime::UNIX_EPOCH).unix_timestamp() + || a.due.unwrap_or(OffsetDateTime::UNIX_EPOCH).unix_timestamp() + == b.due.unwrap_or(OffsetDateTime::UNIX_EPOCH).unix_timestamp() { match a.action { IlmAction::DeleteAllVersionsAction @@ -678,12 +687,12 @@ impl Lifecycle for BucketLifecycleConfiguration { return Ordering::Less; } - if a.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp() - < b.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp() + if a.due.unwrap_or(OffsetDateTime::UNIX_EPOCH).unix_timestamp() + < b.due.unwrap_or(OffsetDateTime::UNIX_EPOCH).unix_timestamp() { return Ordering::Less; } - return Ordering::Greater; + Ordering::Greater }); return events[0].clone(); } @@ -695,28 +704,28 @@ impl Lifecycle for BucketLifecycleConfiguration { if let Some(filter_rules) = self.filter_rules(obj).await { for rule in filter_rules.iter() { if let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration { - if let Some(newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions { + return if let Some(newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions { if newer_noncurrent_versions == 0 { continue; } - return Event { + Event { action: IlmAction::DeleteVersionAction, rule_id: rule.id.clone().unwrap_or_default(), noncurrent_days: noncurrent_version_expiration.noncurrent_days.unwrap_or(0) as u32, newer_noncurrent_versions: newer_noncurrent_versions as usize, due: Some(OffsetDateTime::UNIX_EPOCH), storage_class: "".into(), - }; + } } else { - return Event { + Event { action: IlmAction::DeleteVersionAction, rule_id: rule.id.clone().unwrap_or_default(), noncurrent_days: noncurrent_version_expiration.noncurrent_days.unwrap_or(0) as u32, newer_noncurrent_versions: 0, due: Some(OffsetDateTime::UNIX_EPOCH), storage_class: "".into(), - }; - } + } + }; } } } @@ -795,15 +804,17 @@ pub fn expected_expiry_time(mod_time: OffsetDateTime, days: i32) -> OffsetDateTi let t = mod_time .to_offset(offset!(-0:00:00)) .saturating_add(Duration::days(days as i64)); - let mut hour = 3600; - if let Ok(env_ilm_hour) = env::var("_RUSTFS_ILM_PROCESS_TIME") { - if let Ok(num_hour) = env_ilm_hour.parse::() { - hour = num_hour; - } - } - //t.Truncate(24 * hour) - info!("expected_expiry_time: mod_time={:?}, days={}, result={:?}", mod_time, days, t); - t + + // Truncate to midnight UTC per S3 standard, unless overridden by env var. + // _RUSTFS_ILM_PROCESS_TIME controls the truncation granularity in seconds. + let truncation_secs = env::var("_RUSTFS_ILM_PROCESS_TIME") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(86400); // default: truncate to midnight (24h) + + let unix_secs = t.unix_timestamp(); + let truncated_secs = (unix_secs / truncation_secs as i64) * truncation_secs as i64; + OffsetDateTime::from_unix_timestamp(truncated_secs).unwrap_or(t) } pub async fn abort_incomplete_multipart_upload_due( @@ -849,7 +860,7 @@ pub struct ObjectOpts { impl ObjectOpts { pub fn expired_object_deletemarker(&self) -> bool { - self.delete_marker && self.num_versions == 1 + self.delete_marker && self.is_latest } pub fn from_object_info(oi: &ObjectInfo) -> Self { @@ -858,7 +869,7 @@ impl ObjectOpts { user_tags: oi.user_tags.clone(), mod_time: oi.mod_time, size: oi.size as usize, - version_id: oi.version_id.clone(), + version_id: oi.version_id, is_latest: oi.is_latest, delete_marker: oi.delete_marker, num_versions: oi.num_versions, @@ -932,10 +943,11 @@ mod tests { use s3s::dto::LifecycleRuleFilter; use serial_test::serial; use std::sync::Arc; + use time::macros::datetime; #[tokio::test] #[serial] - async fn validate_rejects_non_positive_expiration_days() { + async fn validate_accepts_zero_expiration_days() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, rules: vec![LifecycleRule { @@ -955,6 +967,33 @@ mod tests { }], }; + lc.validate(&ObjectLockConfiguration::default()) + .await + .expect("zero-day expiration should be accepted"); + } + + #[tokio::test] + #[serial] + async fn validate_rejects_negative_expiration_days() { + let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: Some(LifecycleExpiration { + days: Some(-1), + ..Default::default() + }), + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: None, + id: None, + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + }; + let err = lc .validate(&ObjectLockConfiguration::default()) .await @@ -990,6 +1029,88 @@ mod tests { .expect("expected validation to pass"); } + #[tokio::test] + #[serial] + async fn has_active_rules_accepts_zero_day_expiration() { + let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: Some(LifecycleExpiration { + days: Some(0), + ..Default::default() + }), + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: None, + id: Some("zero-day-active".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: Some("test/".to_string()), + transitions: None, + }], + }; + + assert!(lc.has_active_rules("test/")); + } + + #[tokio::test] + #[serial] + async fn validate_accepts_zero_noncurrent_expiration_days() { + let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: None, + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: None, + id: None, + noncurrent_version_expiration: Some(s3s::dto::NoncurrentVersionExpiration { + noncurrent_days: Some(0), + newer_noncurrent_versions: None, + }), + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + }; + + lc.validate(&ObjectLockConfiguration::default()) + .await + .expect("zero-day noncurrent expiration should be accepted"); + } + + #[tokio::test] + #[serial] + async fn validate_rejects_negative_noncurrent_expiration_days() { + let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: None, + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: None, + id: None, + noncurrent_version_expiration: Some(s3s::dto::NoncurrentVersionExpiration { + noncurrent_days: Some(-1), + newer_noncurrent_versions: None, + }), + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + }; + + let err = lc + .validate(&ObjectLockConfiguration::default()) + .await + .expect_err("expected validation error"); + + assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS); + } + #[tokio::test] #[serial] async fn validate_accepts_abort_incomplete_multipart_upload_only_rule() { @@ -1024,7 +1145,7 @@ mod tests { rules: vec![LifecycleRule { status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), expiration: Some(LifecycleExpiration { - date: Some(time::OffsetDateTime::from_unix_timestamp(20_000_101).unwrap().into()), + date: Some(OffsetDateTime::from_unix_timestamp(20_000_101).unwrap().into()), ..Default::default() }), abort_incomplete_multipart_upload: None, @@ -1398,6 +1519,44 @@ mod tests { assert_eq!(event.due, Some(expected_expiry_time(base_time, 1))); } + #[tokio::test] + #[serial] + async fn eval_inner_expires_noncurrent_version_immediately_when_zero_days() { + let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); + let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: None, + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: None, + id: Some("noncurrent-expire-immediate".to_string()), + noncurrent_version_expiration: Some(s3s::dto::NoncurrentVersionExpiration { + noncurrent_days: Some(0), + newer_noncurrent_versions: None, + }), + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + }; + + let opts = ObjectOpts { + name: "obj".to_string(), + mod_time: Some(base_time), + successor_mod_time: Some(base_time), + is_latest: false, + version_id: Some(Uuid::new_v4()), + ..Default::default() + }; + let event = lc.eval_inner(&opts, base_time, 0).await; + + assert_eq!(event.action, IlmAction::DeleteVersionAction); + assert_eq!(event.rule_id, "noncurrent-expire-immediate"); + assert_eq!(event.due, Some(expected_expiry_time(base_time, 0))); + } + #[tokio::test] #[serial] async fn eval_inner_transitions_noncurrent_version_after_due() { @@ -1505,8 +1664,10 @@ mod tests { #[tokio::test] #[serial] async fn filter_rules_respects_filter_prefix() { - let mut filter = LifecycleRuleFilter::default(); - filter.prefix = Some("prefix".to_string()); + let filter = LifecycleRuleFilter { + prefix: Some("prefix".to_string()), + ..Default::default() + }; let lc = BucketLifecycleConfiguration { expiry_updated_at: None, rules: vec![LifecycleRule { @@ -1548,11 +1709,14 @@ mod tests { #[tokio::test] #[serial] async fn filter_rules_respects_filter_and_prefix() { - let mut filter = LifecycleRuleFilter::default(); - - let mut and = LifecycleRuleAndOperator::default(); - and.prefix = Some("prefix".to_string()); - filter.and = Some(and); + let and = s3s::dto::LifecycleRuleAndOperator { + prefix: Some("prefix".to_string()), + ..Default::default() + }; + let filter = LifecycleRuleFilter { + and: Some(and), + ..Default::default() + }; let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -1649,20 +1813,22 @@ mod tests { #[tokio::test] #[serial] async fn filter_rules_respects_filter_and_tags() { - let mut filter = LifecycleRuleFilter::default(); - filter.and = Some(LifecycleRuleAndOperator { - tags: Some(vec![ - s3s::dto::Tag { - key: Some("env".to_string()), - value: Some("prod".to_string()), - }, - s3s::dto::Tag { - key: Some("team".to_string()), - value: Some("storage".to_string()), - }, - ]), + let filter = LifecycleRuleFilter { + and: Some(s3s::dto::LifecycleRuleAndOperator { + tags: Some(vec![ + s3s::dto::Tag { + key: Some("env".to_string()), + value: Some("prod".to_string()), + }, + s3s::dto::Tag { + key: Some("team".to_string()), + value: Some("storage".to_string()), + }, + ]), + ..Default::default() + }), ..Default::default() - }); + }; let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -1710,7 +1876,7 @@ mod tests { #[tokio::test] #[serial] - async fn expired_object_delete_marker_requires_single_version() { + async fn expired_object_delete_marker_applies_with_noncurrent_versions_present() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -1744,7 +1910,8 @@ mod tests { let now = base_time + Duration::days(2); let event = lc.eval_inner(&opts, now, 0).await; - assert_eq!(event.action, IlmAction::NoneAction); + assert_eq!(event.action, IlmAction::DeleteVersionAction); + assert_eq!(event.due, Some(expected_expiry_time(base_time, 1))); } #[tokio::test] @@ -1873,4 +2040,392 @@ mod tests { assert_eq!(event_after.action, IlmAction::DeleteVersionAction); assert_eq!(event_after.due, Some(future_date)); } + + // --- TASK-002 tests: Object Lock + ExpiredObjectDeleteMarker conflict --- + + #[tokio::test] + #[serial] + async fn validate_rejects_expired_object_delete_marker_on_locked_bucket() { + let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: Some(LifecycleExpiration { + expired_object_delete_marker: Some(true), + ..Default::default() + }), + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: None, + id: Some("test-rule".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + }; + + let locked_config = ObjectLockConfiguration { + object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), + ..Default::default() + }; + + let err = lc.validate(&locked_config).await.unwrap_err(); + assert_eq!(err.to_string(), ERR_LIFECYCLE_BUCKET_LOCKED); + } + + #[tokio::test] + #[serial] + async fn validate_allows_expired_object_delete_marker_on_unlocked_bucket() { + let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: Some(LifecycleExpiration { + expired_object_delete_marker: Some(true), + ..Default::default() + }), + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: None, + id: Some("test-rule".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + }; + + // Default ObjectLockConfiguration (no lock enabled) should pass + lc.validate(&ObjectLockConfiguration::default()) + .await + .expect("expected validation to pass on unlocked bucket"); + } + + #[tokio::test] + #[serial] + async fn validate_allows_non_delete_marker_expiration_on_locked_bucket() { + let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: Some(LifecycleExpiration { + days: Some(30), + ..Default::default() + }), + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: None, + id: Some("test-rule".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + }; + + let locked_config = ObjectLockConfiguration { + object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), + ..Default::default() + }; + + // Days-based expiration (not DeleteMarker) should be allowed on locked bucket + lc.validate(&locked_config) + .await + .expect("expected days-based expiration to pass on locked bucket"); + } + + // --- TASK-003 tests: Midnight UTC truncation --- + + #[test] + fn expected_expiry_time_truncates_to_midnight_utc() { + // Object created at 2025-01-15T10:30:45Z, expire in 30 days + let mod_time = datetime!(2025-01-15 10:30:45 UTC); + let result = expected_expiry_time(mod_time, 30); + + // Should be truncated to midnight: 2025-02-14T00:00:00Z + assert_eq!(result.hour(), 0); + assert_eq!(result.minute(), 0); + assert_eq!(result.second(), 0); + assert_eq!(result, datetime!(2025-02-14 00:00:00 UTC)); + } + + #[test] + fn expected_expiry_time_immediate_expiry_returns_epoch() { + let mod_time = datetime!(2025-06-01 12:00:00 UTC); + let result = expected_expiry_time(mod_time, 0); + assert_eq!(result, OffsetDateTime::UNIX_EPOCH); + } + + #[test] + fn expected_expiry_time_truncates_already_midnight() { + let mod_time = datetime!(2025-03-01 00:00:00 UTC); + let result = expected_expiry_time(mod_time, 1); + assert_eq!(result, datetime!(2025-03-02 00:00:00 UTC)); + } + + #[test] + fn expected_expiry_time_truncates_end_of_day() { + let mod_time = datetime!(2025-06-15 23:59:59 UTC); + let result = expected_expiry_time(mod_time, 1); + assert_eq!(result, datetime!(2025-06-16 00:00:00 UTC)); + } + + // --- TASK-007 tests: Legacy Prefix/Filter conflict --- + + #[tokio::test] + #[serial] + async fn validate_rejects_prefix_and_filter_both_present() { + let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: Some(LifecycleExpiration { + days: Some(30), + ..Default::default() + }), + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: Some(LifecycleRuleFilter { + prefix: Some("logs/".to_string()), + ..Default::default() + }), + id: Some("test-rule".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: Some("archive/".to_string()), + transitions: None, + }], + }; + + let err = lc.validate(&ObjectLockConfiguration::default()).await.unwrap_err(); + assert_eq!(err.to_string(), ERR_LIFECYCLE_PREFIX_FILTER_CONFLICT); + } + + #[tokio::test] + #[serial] + async fn validate_allows_prefix_without_filter() { + let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: Some(LifecycleExpiration { + days: Some(30), + ..Default::default() + }), + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: None, + id: Some("test-rule".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: Some("logs/".to_string()), + transitions: None, + }], + }; + + lc.validate(&ObjectLockConfiguration::default()) + .await + .expect("prefix without filter should be valid"); + } + + #[tokio::test] + #[serial] + async fn validate_allows_filter_without_prefix() { + let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: Some(LifecycleExpiration { + days: Some(30), + ..Default::default() + }), + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: Some(LifecycleRuleFilter { + prefix: Some("logs/".to_string()), + ..Default::default() + }), + id: Some("test-rule".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + }; + + lc.validate(&ObjectLockConfiguration::default()) + .await + .expect("filter without prefix should be valid"); + } + + #[tokio::test] + #[serial] + async fn validate_allows_empty_prefix_with_filter() { + // Empty prefix should be treated as "not set" + let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: Some(LifecycleExpiration { + days: Some(30), + ..Default::default() + }), + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: Some(LifecycleRuleFilter { + prefix: Some("logs/".to_string()), + ..Default::default() + }), + id: Some("test-rule".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: Some("".to_string()), // empty = not set + transitions: None, + }], + }; + + lc.validate(&ObjectLockConfiguration::default()) + .await + .expect("empty prefix with filter should be valid"); + } + + // --- TASK-004 tests: ExpiredObjectAllVersions --- + + #[tokio::test] + #[serial] + async fn validate_rejects_expired_object_all_versions_on_locked_bucket() { + let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: Some(LifecycleExpiration { + days: Some(30), + expired_object_all_versions: Some(true), + ..Default::default() + }), + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: None, + id: Some("all-versions-rule".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + }; + + let locked_config = ObjectLockConfiguration { + object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), + ..Default::default() + }; + + let err = lc.validate(&locked_config).await.unwrap_err(); + assert_eq!(err.to_string(), ERR_LIFECYCLE_BUCKET_LOCKED); + } + + #[tokio::test] + #[serial] + async fn validate_allows_expired_object_all_versions_on_unlocked_bucket() { + let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: Some(LifecycleExpiration { + days: Some(30), + expired_object_all_versions: Some(true), + ..Default::default() + }), + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: None, + id: Some("all-versions-rule".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + }; + + lc.validate(&ObjectLockConfiguration::default()) + .await + .expect("ExpiredObjectAllVersions should be allowed on unlocked bucket"); + } + + #[tokio::test] + #[serial] + async fn eval_inner_triggers_delete_all_versions_when_expired_object_all_versions_set() { + let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); + let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: Some(LifecycleExpiration { + days: Some(1), + expired_object_all_versions: Some(true), + ..Default::default() + }), + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: None, + id: Some("all-versions-rule".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + }; + + let opts = ObjectOpts { + name: "obj".to_string(), + mod_time: Some(base_time), + is_latest: true, + version_id: None, + ..Default::default() + }; + + // now is after the expiry time + let now = base_time + Duration::days(2); + let event = lc.eval_inner(&opts, now, 0).await; + assert_eq!(event.action, IlmAction::DeleteAllVersionsAction); + assert_eq!(event.rule_id, "all-versions-rule"); + } + + #[tokio::test] + #[serial] + async fn eval_inner_uses_delete_action_when_all_versions_not_set() { + let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); + let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: Some(LifecycleExpiration { + days: Some(1), + expired_object_all_versions: None, // not set + ..Default::default() + }), + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: None, + id: Some("normal-rule".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + }; + + let opts = ObjectOpts { + name: "obj".to_string(), + mod_time: Some(base_time), + is_latest: true, + version_id: None, + ..Default::default() + }; + + let now = base_time + Duration::days(2); + let event = lc.eval_inner(&opts, now, 0).await; + // Without ExpiredObjectAllVersions, should use normal DeleteAction + assert_eq!(event.action, IlmAction::DeleteAction); + } } diff --git a/crates/ecstore/src/bucket/lifecycle/mod.rs b/crates/ecstore/src/bucket/lifecycle/mod.rs index 2eed2bfc3..6a5233811 100644 --- a/crates/ecstore/src/bucket/lifecycle/mod.rs +++ b/crates/ecstore/src/bucket/lifecycle/mod.rs @@ -14,8 +14,9 @@ pub mod bucket_lifecycle_audit; pub mod bucket_lifecycle_ops; +pub mod core; pub mod evaluator; -pub mod lifecycle; +pub use self::core as lifecycle; pub mod rule; pub mod tier_last_day_stats; pub mod tier_sweeper; diff --git a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs index 896ec8d1f..daadc84a5 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs @@ -109,10 +109,19 @@ impl ObjSweeper { } pub async fn sweep(&self) { - let je = self.should_remove_remote_object(); - if !je.is_none() { - let mut expiry_state = GLOBAL_ExpiryState.write().await; - expiry_state.enqueue_tier_journal_entry(&je.expect("err!")); + let Some(je) = self.should_remove_remote_object() else { + return; + }; + let hash = je.op_hash(); + // Grab the sender under a short read lock, then release the lock so we + // don't hold it across the async send. + let wrkr = GLOBAL_ExpiryState.read().await.get_worker_ch(hash); + let Some(wrkr) = wrkr else { + GLOBAL_ExpiryState.write().await.increment_missed_tier_journal_tasks(); + return; + }; + if wrkr.send(Some(Box::new(je))).await.is_err() { + GLOBAL_ExpiryState.write().await.increment_missed_tier_journal_tasks(); } } } diff --git a/crates/ecstore/src/bucket/replication/config.rs b/crates/ecstore/src/bucket/replication/config.rs index ed137698f..abc8cdc76 100644 --- a/crates/ecstore/src/bucket/replication/config.rs +++ b/crates/ecstore/src/bucket/replication/config.rs @@ -153,6 +153,11 @@ impl ReplicationConfigurationExt for ReplicationConfiguration { if obj.op_type == ReplicationType::Delete { if obj.version_id.is_some() { + if obj.delete_marker { + return rule.delete_marker_replication.clone().is_some_and(|d| { + d.status == Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)) + }); + } return rule .delete_replication .clone() diff --git a/crates/ecstore/src/bucket/replication/replication_resyncer.rs b/crates/ecstore/src/bucket/replication/replication_resyncer.rs index 8379f1767..d2b85cbde 100644 --- a/crates/ecstore/src/bucket/replication/replication_resyncer.rs +++ b/crates/ecstore/src/bucket/replication/replication_resyncer.rs @@ -34,7 +34,7 @@ use crate::global::get_global_bucket_monitor; use crate::set_disk::get_lock_acquire_timeout; use crate::store_api::{DeletedObject, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete, WalkOptions}; use crate::{StorageAPI, new_object_layer_fn}; -use aws_sdk_s3::error::SdkError; +use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError}; use aws_sdk_s3::operation::head_object::HeadObjectOutput; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{CompletedPart, ObjectLockLegalHoldStatus}; @@ -86,6 +86,7 @@ use tokio::time::Duration as TokioDuration; use tokio_util::io::ReaderStream; use tokio_util::sync::CancellationToken; use tracing::{error, info, instrument, warn}; +use uuid::Uuid; pub(crate) const REPLICATION_DIR: &str = ".replication"; pub(crate) const RESYNC_FILE_NAME: &str = "resync.bin"; @@ -1497,6 +1498,54 @@ pub async fn replicate_delete(dobj: DeletedObjectReplicationInfo, } }; + if dobj.delete_object.delete_marker + && let Some(delete_marker_version_id) = dobj.delete_object.delete_marker_version_id + { + let source_marker_state = storage + .get_object_info( + &bucket, + &dobj.delete_object.object_name, + &ObjectOptions { + version_id: Some(delete_marker_version_id.to_string()), + versioned: BucketVersioningSys::prefix_enabled(&bucket, &dobj.delete_object.object_name).await, + version_suspended: BucketVersioningSys::prefix_suspended(&bucket, &dobj.delete_object.object_name).await, + ..Default::default() + }, + ) + .await; + + match source_marker_state { + Ok(info) if info.delete_marker && info.version_id == Some(delete_marker_version_id) => {} + Ok(_) => { + warn!( + bucket, + object = dobj.delete_object.object_name, + version_id = %delete_marker_version_id, + "skipping stale delete-marker replication because source version is no longer a delete marker" + ); + return; + } + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => { + warn!( + bucket, + object = dobj.delete_object.object_name, + version_id = %delete_marker_version_id, + "skipping stale delete-marker replication because source version no longer exists" + ); + return; + } + Err(err) => { + warn!( + bucket, + object = dobj.delete_object.object_name, + version_id = %delete_marker_version_id, + error = %err, + "failed to verify source delete-marker state before replication" + ); + } + } + } + let dsc = match parse_replicate_decision( &bucket, &dobj @@ -1529,7 +1578,6 @@ pub async fn replicate_delete(dobj: DeletedObjectReplicationInfo, return; } }; - let ns_lock = match storage .new_ns_lock(&bucket, format!("/[replicate]/{}", dobj.delete_object.object_name).as_str()) .await @@ -1653,7 +1701,33 @@ pub async fn replicate_delete(dobj: DeletedObjectReplicationInfo, } } - let (replication_status, prev_status) = if dobj.delete_object.version_id.is_none() { + let is_version_purge = is_version_delete_replication(&dobj.delete_object); + + if !is_version_purge && dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() { + let bucket_clone = bucket.clone(); + let dobj_clone = dobj.clone(); + let dsc_clone = dsc.clone(); + let storage_clone = storage.clone(); + tokio::spawn(async move { + for _ in 0..5 { + if let Some(delete_marker_version_id) = dobj_clone.delete_object.delete_marker_version_id + && source_delete_marker_missing( + &*storage_clone, + &bucket_clone, + &dobj_clone.delete_object.object_name, + delete_marker_version_id, + ) + .await + { + replicate_delete_marker_purge_to_targets(&bucket_clone, &dobj_clone, &dsc_clone).await; + break; + } + tokio::time::sleep(TokioDuration::from_secs(1)).await; + } + }); + } + + let (replication_status, prev_status) = if !is_version_purge { ( rinfos.replication_status(), dobj.delete_object @@ -1741,6 +1815,65 @@ pub async fn replicate_delete(dobj: DeletedObjectReplicationInfo, } } +async fn source_delete_marker_missing( + storage: &S, + bucket: &str, + object_name: &str, + delete_marker_version_id: Uuid, +) -> bool { + match storage + .get_object_info( + bucket, + object_name, + &ObjectOptions { + version_id: Some(delete_marker_version_id.to_string()), + versioned: BucketVersioningSys::prefix_enabled(bucket, object_name).await, + version_suspended: BucketVersioningSys::prefix_suspended(bucket, object_name).await, + ..Default::default() + }, + ) + .await + { + Ok(info) => !info.delete_marker || info.version_id != Some(delete_marker_version_id), + Err(err) => is_err_object_not_found(&err) || is_err_version_not_found(&err), + } +} + +async fn replicate_delete_marker_purge_to_targets(bucket: &str, dobj: &DeletedObjectReplicationInfo, dsc: &ReplicateDecision) { + let Some(delete_marker_version_id) = dobj.delete_object.delete_marker_version_id else { + return; + }; + + for tgt_entry in dsc.targets_map.values() { + if !tgt_entry.replicate { + continue; + } + if !dobj.target_arn.is_empty() && dobj.target_arn != tgt_entry.arn { + continue; + } + let Some(tgt_client) = BucketTargetSys::get().get_remote_target_client(bucket, &tgt_entry.arn).await else { + continue; + }; + + let _ = tgt_client + .remove_object( + &tgt_client.bucket, + &dobj.delete_object.object_name, + Some(delete_marker_version_id.to_string()), + RemoveObjectOptions { + force_delete: false, + governance_bypass: false, + replication_delete_marker: false, + replication_mtime: dobj.delete_object.delete_marker_mtime, + replication_status: ReplicationStatusType::Replica, + replication_request: true, + replication_validity_check: false, + }, + ) + .await; + } +} + async fn replicate_force_delete_to_targets(dobj: &DeletedObjectReplicationInfo, storage: Arc) { let bucket = &dobj.bucket; let object_name = &dobj.delete_object.object_name; @@ -1924,6 +2057,14 @@ async fn replicate_force_delete_to_targets(dobj: &DeletedObjectRe } } +fn is_version_delete_replication(dobj: &DeletedObject) -> bool { + dobj.version_id.is_some() || (dobj.delete_marker_version_id.is_some() && !dobj.delete_marker) +} + +fn is_retryable_delete_replication_head_error(is_not_found: bool, code: Option<&str>) -> bool { + !is_not_found && !matches!(code, Some("MethodNotAllowed" | "405")) +} + async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_client: Arc) -> ReplicatedTargetInfo { let version_id = if let Some(version_id) = &dobj.delete_object.delete_marker_version_id { version_id.to_owned() @@ -1941,7 +2082,8 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli rinfo.endpoint = tgt_client.endpoint.clone(); rinfo.secure = tgt_client.secure; - if dobj.delete_object.version_id.is_none() + let is_version_purge = is_version_delete_replication(&dobj.delete_object); + if !is_version_purge && rinfo.prev_replication_status == ReplicationStatusType::Completed && dobj.op_type != ReplicationType::ExistingObject { @@ -1949,12 +2091,12 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli return rinfo; } - if dobj.delete_object.version_id.is_some() && rinfo.version_purge_status == VersionPurgeStatusType::Complete { + if is_version_purge && rinfo.version_purge_status == VersionPurgeStatusType::Complete { return rinfo; } if BucketTargetSys::get().is_offline(&tgt_client.to_url()).await { - if dobj.delete_object.version_id.is_none() { + if !is_version_purge { rinfo.replication_status = ReplicationStatusType::Failed; } else { rinfo.version_purge_status = VersionPurgeStatusType::Failed; @@ -1968,18 +2110,29 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli Some(version_id.to_string()) }; - if dobj.delete_object.delete_marker_version_id.is_some() - && let Err(e) = tgt_client + if dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() { + match tgt_client .head_object(&tgt_client.bucket, &dobj.delete_object.object_name, version_id.clone()) .await - && let SdkError::ServiceError(service_err) = &e - && !service_err.err().is_not_found() - { - rinfo.replication_status = ReplicationStatusType::Failed; - rinfo.error = Some(e.to_string()); - - return rinfo; - }; + { + Ok(_) => {} + Err(e) => { + let non_retryable = matches!( + &e, + SdkError::ServiceError(service_err) + if is_retryable_delete_replication_head_error( + service_err.err().is_not_found(), + service_err.err().code(), + ) + ); + if non_retryable { + rinfo.replication_status = ReplicationStatusType::Failed; + rinfo.error = Some(e.to_string()); + return rinfo; + } + } + } + } match tgt_client .remove_object( @@ -1989,7 +2142,7 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli RemoveObjectOptions { force_delete: false, governance_bypass: false, - replication_delete_marker: dobj.delete_object.delete_marker_version_id.is_some(), + replication_delete_marker: dobj.delete_object.delete_marker, replication_mtime: dobj.delete_object.delete_marker_mtime, replication_status: ReplicationStatusType::Replica, replication_request: true, @@ -1999,7 +2152,7 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli .await { Ok(_) => { - if dobj.delete_object.version_id.is_none() { + if !is_version_purge { rinfo.replication_status = ReplicationStatusType::Completed; } else { rinfo.version_purge_status = VersionPurgeStatusType::Complete; @@ -2007,7 +2160,7 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli } Err(e) => { rinfo.error = Some(e.to_string()); - if dobj.delete_object.version_id.is_none() { + if !is_version_purge { rinfo.replication_status = ReplicationStatusType::Failed; } else { rinfo.version_purge_status = VersionPurgeStatusType::Failed; @@ -3450,6 +3603,54 @@ mod tests { ); } + #[test] + fn test_is_version_delete_replication_for_delete_marker_version_purge() { + let dobj = DeletedObject { + delete_marker: false, + delete_marker_version_id: Some(Uuid::new_v4()), + ..Default::default() + }; + + assert!( + is_version_delete_replication(&dobj), + "delete-marker version purges must be tracked as version purge replication, not delete-marker creation replication" + ); + } + + #[test] + fn test_is_version_delete_replication_for_delete_marker_creation() { + let dobj = DeletedObject { + delete_marker: true, + delete_marker_version_id: Some(Uuid::new_v4()), + ..Default::default() + }; + + assert!( + !is_version_delete_replication(&dobj), + "delete-marker creation should remain on the delete-marker replication path" + ); + } + + #[test] + fn test_is_retryable_delete_replication_head_error_allows_delete_marker_head_responses() { + assert!( + !is_retryable_delete_replication_head_error(false, Some("405")), + "numeric 405 responses should not block delete-marker purge replication" + ); + assert!( + !is_retryable_delete_replication_head_error(false, Some("MethodNotAllowed")), + "MethodNotAllowed responses should not block delete-marker purge replication" + ); + assert!( + !is_retryable_delete_replication_head_error(true, Some("NoSuchKey")), + "not-found responses should not block delete-marker purge replication" + ); + assert!( + is_retryable_delete_replication_head_error(false, Some("AccessDenied")), + "unexpected head errors should still fail fast" + ); + } + #[tokio::test] async fn test_get_heal_replicate_object_info_failed_object_returns_heal_roi() { let oi = ObjectInfo { diff --git a/crates/ecstore/src/client/object_handlers_common.rs b/crates/ecstore/src/client/object_handlers_common.rs index 41e68134c..2fef505f4 100644 --- a/crates/ecstore/src/client/object_handlers_common.rs +++ b/crates/ecstore/src/client/object_handlers_common.rs @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; +use tracing::warn; + use crate::bucket::lifecycle::lifecycle; use crate::bucket::versioning::VersioningApi; use crate::bucket::versioning_sys::BucketVersioningSys; @@ -19,7 +22,14 @@ use crate::store::ECStore; use crate::store_api::{ObjectOperations, ObjectOptions, ObjectToDelete}; use rustfs_lock::MAX_DELETE_LIST; -pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) { +pub async fn delete_object_versions(api: &Arc, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) { + let version_suspended = match BucketVersioningSys::get(bucket).await { + Ok(vc) => vc.suspended(), + Err(err) => { + warn!(bucket, error = ?err, "failed to get versioning config during lifecycle noncurrent version cleanup"); + return; + } + }; let mut remaining = to_del; loop { let mut to_del = remaining; @@ -29,15 +39,29 @@ pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[Object } else { remaining = &[]; } - let vc = BucketVersioningSys::get(bucket).await.expect("err!"); - let _deleted_objs = api.delete_objects( - bucket, - to_del.to_vec(), - ObjectOptions { - //prefix_enabled_fn: vc.prefix_enabled(""), - version_suspended: vc.suspended(), - ..Default::default() - }, - ); + let (_deleted_objs, errors) = api + .delete_objects( + bucket, + to_del.to_vec(), + ObjectOptions { + version_suspended, + ..Default::default() + }, + ) + .await; + for (i, err) in errors.iter().enumerate() { + if let Some(e) = err { + let obj_name = to_del.get(i).map(|o| o.object_name.as_str()).unwrap_or(""); + let vid = to_del + .get(i) + .and_then(|o| o.version_id) + .map(|v| v.to_string()) + .unwrap_or_default(); + warn!(bucket, object = obj_name, version_id = %vid, error = ?e, "failed to delete noncurrent version during lifecycle cleanup"); + } + } + if remaining.is_empty() { + break; + } } } diff --git a/crates/ecstore/src/set_disk.rs b/crates/ecstore/src/set_disk.rs index 5e88ee7c8..1c84edce4 100644 --- a/crates/ecstore/src/set_disk.rs +++ b/crates/ecstore/src/set_disk.rs @@ -1771,13 +1771,7 @@ impl ObjectOperations for SetDisks { ..Default::default() }; - let dsc = if opts - .delete_replication - .as_ref() - .map(|v| v.replica_status == ReplicationStatusType::Replica) - == Some(true) - || opts.version_purge_status() == VersionPurgeStatusType::Complete - { + let dsc = if should_preserve_delete_replication_state(&opts) { ReplicateDecision::default() } else { check_replicate_delete(bucket, &otd, &goi, &opts, gerr.map(|e| e.to_string())).await @@ -1792,33 +1786,7 @@ impl ObjectOperations for SetDisks { .unwrap_or_default(); } - let mut mark_delete = goi.version_id.is_some(); - - let mut delete_marker = opts.versioned; - - if opts.version_id.is_some() { - // Decommission/rebalance may recreate a delete marker on a new pool before that - // exact version exists there, so we must still treat it as a mark-delete write. - if opts.data_movement && opts.delete_marker && !version_found { - mark_delete = true; - } - - if version_found && opts.delete_marker_replication_status() == ReplicationStatusType::Replica { - mark_delete = false; - } - - if opts.version_purge_status().is_empty() && opts.delete_marker_replication_status().is_empty() { - mark_delete = false; - } - - if opts.version_purge_status() == VersionPurgeStatusType::Complete { - mark_delete = false; - } - - if version_found && (!goi.version_purge_status.is_empty() || !goi.delete_marker) { - delete_marker = false; - } - } + let (mark_delete, mut delete_marker) = resolve_delete_version_state(&opts, &goi, version_found); let mod_time = if let Some(mt) = opts.mod_time { mt @@ -2427,6 +2395,63 @@ impl ObjectOperations for SetDisks { } } +fn should_preserve_delete_replication_state(opts: &ObjectOptions) -> bool { + opts.delete_replication.as_ref().is_some_and(|state| { + state.replica_status == ReplicationStatusType::Replica + || (!state.replicate_decision_str.is_empty() + && (!state.composite_replication_status().is_empty() || !state.composite_version_purge_status().is_empty())) + }) || opts.version_purge_status() == VersionPurgeStatusType::Complete +} + +fn resolve_delete_version_state(opts: &ObjectOptions, goi: &ObjectInfo, version_found: bool) -> (bool, bool) { + let mut mark_delete = goi.version_id.is_some(); + let mut delete_marker = opts.versioned; + + if opts.version_id.is_some() { + // Decommission/rebalance may recreate a delete marker on a new pool before that + // exact version exists there, so we must still treat it as a mark-delete write. + if opts.data_movement && opts.delete_marker && !version_found { + mark_delete = true; + } + + let delete_marker_version_purge = version_found && goi.delete_marker && !opts.version_purge_status().is_empty(); + + if version_found && opts.delete_marker_replication_status() == ReplicationStatusType::Replica { + mark_delete = false; + } + + if opts.version_purge_status().is_empty() && opts.delete_marker_replication_status().is_empty() { + mark_delete = false; + } + + if opts.version_purge_status() == VersionPurgeStatusType::Complete { + mark_delete = false; + } + + let replica_delete_marker_version_purge = + version_found && goi.delete_marker && opts.delete_marker_replication_status() == ReplicationStatusType::Replica; + + if delete_marker_version_purge { + mark_delete = false; + } + + if !version_found && !opts.delete_marker && opts.delete_marker_replication_status() == ReplicationStatusType::Replica { + delete_marker = false; + } + + if version_found + && (!goi.version_purge_status.is_empty() + || !goi.delete_marker + || replica_delete_marker_version_purge + || delete_marker_version_purge) + { + delete_marker = false; + } + } + + (mark_delete, delete_marker) +} + impl SetDisks { #[tracing::instrument(skip(self, fi, opts))] pub(crate) async fn decommission_tiered_object( @@ -4337,6 +4362,7 @@ mod tests { use crate::store_api::{CompletePart, ObjectInfo}; use crate::store_init::save_format_file; use rustfs_filemeta::ErasureInfo; + use rustfs_filemeta::ReplicationState; use rustfs_lock::client::local::LocalClient; use rustfs_lock::{LockError, LockInfo, LockResponse, LockStats}; use serial_test::serial; @@ -4564,6 +4590,137 @@ mod tests { assert!(is_min_allowed_part_size(100 * 1024 * 1024)); // 100MB - allowed } + #[test] + fn resolve_delete_version_state_clears_delete_marker_for_replica_marker_version_purge() { + let opts = ObjectOptions { + versioned: true, + version_id: Some(Uuid::new_v4().to_string()), + delete_replication: Some(ReplicationState { + replica_status: ReplicationStatusType::Replica, + ..Default::default() + }), + ..Default::default() + }; + let current = ObjectInfo { + version_id: Some(Uuid::new_v4()), + delete_marker: true, + ..Default::default() + }; + + let (mark_delete, delete_marker) = resolve_delete_version_state(&opts, ¤t, true); + + assert!(!mark_delete); + assert!( + !delete_marker, + "replica purge of an existing delete marker version must remove that version, not preserve delete-marker semantics" + ); + } + + #[test] + fn resolve_delete_version_state_keeps_delete_marker_for_replica_marker_creation() { + let opts = ObjectOptions { + versioned: true, + version_id: Some(Uuid::new_v4().to_string()), + delete_marker: true, + delete_replication: Some(ReplicationState { + replica_status: ReplicationStatusType::Replica, + ..Default::default() + }), + ..Default::default() + }; + + let (mark_delete, delete_marker) = resolve_delete_version_state(&opts, &ObjectInfo::default(), false); + + assert!(!mark_delete); + assert!(delete_marker); + } + + #[test] + fn resolve_delete_version_state_skips_marker_creation_for_replica_purge_when_version_missing() { + let opts = ObjectOptions { + versioned: true, + version_id: Some(Uuid::new_v4().to_string()), + delete_replication: Some(ReplicationState { + replica_status: ReplicationStatusType::Replica, + ..Default::default() + }), + ..Default::default() + }; + + let (mark_delete, delete_marker) = resolve_delete_version_state(&opts, &ObjectInfo::default(), false); + + assert!( + !mark_delete, + "replica delete-marker purges should not schedule mark-delete writes when the target version is absent" + ); + assert!( + !delete_marker, + "replica delete-marker purges must become no-ops when the marker version has not arrived on the target yet" + ); + } + + #[test] + fn should_preserve_delete_replication_state_for_completed_delete_marker_replication_update() { + let opts = ObjectOptions { + version_id: Some(Uuid::new_v4().to_string()), + delete_replication: Some(ReplicationState { + replicate_decision_str: "target=true;false;target;".to_string(), + replication_status_internal: Some("target=COMPLETED;".to_string()), + targets: rustfs_filemeta::replication_statuses_map("target=COMPLETED;"), + ..Default::default() + }), + ..Default::default() + }; + + assert!( + should_preserve_delete_replication_state(&opts), + "source delete-marker replication status updates must not be re-evaluated as fresh delete replication requests" + ); + } + + #[test] + fn should_not_preserve_delete_replication_state_for_new_version_delete_request() { + let opts = ObjectOptions { + version_id: Some(Uuid::new_v4().to_string()), + ..Default::default() + }; + + assert!( + !should_preserve_delete_replication_state(&opts), + "fresh versioned deletes still need replication eligibility checks" + ); + } + + #[test] + fn resolve_delete_version_state_removes_source_delete_marker_version_during_purge_replication() { + let opts = ObjectOptions { + versioned: true, + version_id: Some(Uuid::new_v4().to_string()), + delete_replication: Some(ReplicationState { + version_purge_status_internal: Some("target=PENDING;".to_string()), + purge_targets: rustfs_filemeta::version_purge_statuses_map("target=PENDING;"), + ..Default::default() + }), + ..Default::default() + }; + let current = ObjectInfo { + version_id: Some(Uuid::new_v4()), + delete_marker: true, + ..Default::default() + }; + + let (mark_delete, delete_marker) = resolve_delete_version_state(&opts, ¤t, true); + + assert!( + !mark_delete, + "source delete-marker version purge should delete the local marker instead of rewriting it with purge metadata" + ); + assert!( + !delete_marker, + "source delete-marker version purge should not leave delete-marker semantics behind locally" + ); + } + #[test] fn test_get_complete_multipart_md5() { // Test MD5 calculation for multipart upload diff --git a/crates/ecstore/src/store.rs b/crates/ecstore/src/store.rs index 07e16baa4..03ed09d96 100644 --- a/crates/ecstore/src/store.rs +++ b/crates/ecstore/src/store.rs @@ -15,7 +15,9 @@ #![allow(clippy::map_entry)] use crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc; -use crate::bucket::lifecycle::bucket_lifecycle_ops::{enqueue_transition_immediate, init_background_expiry}; +use crate::bucket::lifecycle::bucket_lifecycle_ops::{ + enqueue_immediate_expiry, enqueue_transition_immediate, init_background_expiry, +}; use crate::bucket::metadata_sys::{self, set_bucket_metadata}; use crate::bucket::utils::check_abort_multipart_args; use crate::bucket::utils::check_complete_multipart_args; @@ -134,7 +136,8 @@ async fn enqueue_transition_after_write(result: Result, src: LcEvent match result { Ok(oi) => { if should_enqueue_transition_immediately(&oi) { - enqueue_transition_immediate(&oi, src).await; + enqueue_transition_immediate(&oi, src.clone()).await; + enqueue_immediate_expiry(&oi, src).await; } Ok(oi) } diff --git a/crates/ecstore/src/store_api/types.rs b/crates/ecstore/src/store_api/types.rs index 31446b84d..4b5b9f87d 100644 --- a/crates/ecstore/src/store_api/types.rs +++ b/crates/ecstore/src/store_api/types.rs @@ -446,6 +446,11 @@ impl ObjectInfo { .replication_state_internal .as_ref() .and_then(|v| v.version_purge_status_internal.clone()); + let replication_decision = fi + .replication_state_internal + .as_ref() + .map(|v| v.replicate_decision_str.clone()) + .unwrap_or_default(); let mut replication_status = fi.replication_status(); if replication_status.is_empty() @@ -542,6 +547,7 @@ impl ObjectInfo { replication_status, version_purge_status_internal, version_purge_status, + replication_decision, ..Default::default() } } @@ -1010,6 +1016,7 @@ pub struct ObjectInfoOrErr { #[cfg(test)] mod tests { use super::*; + use rustfs_filemeta::ReplicationState; #[test] fn get_actual_size_prefers_actual_size_field() { @@ -1059,6 +1066,21 @@ mod tests { assert_eq!(info.get_actual_size().unwrap(), 77); } + #[test] + fn from_file_info_preserves_replication_decision() { + let fi = rustfs_filemeta::FileInfo { + replication_state_internal: Some(ReplicationState { + replicate_decision_str: "arn=true;false;arn:replication::1:dest;rule-id".to_string(), + ..Default::default() + }), + ..Default::default() + }; + + let info = ObjectInfo::from_file_info(&fi, "bucket", "object", true); + + assert_eq!(info.replication_decision, "arn=true;false;arn:replication::1:dest;rule-id"); + } + #[test] fn get_actual_size_uses_compressed_parts_actual_size_when_metadata_missing() { let user_defined = { diff --git a/crates/filemeta/src/filemeta.rs b/crates/filemeta/src/filemeta.rs index 8978b0e99..b40d1b9be 100644 --- a/crates/filemeta/src/filemeta.rs +++ b/crates/filemeta/src/filemeta.rs @@ -382,6 +382,11 @@ impl FileMeta { #[tracing::instrument(level = "debug", skip(self))] pub fn delete_version(&mut self, fi: &FileInfo) -> Result> { let vid = Some(fi.version_id.unwrap_or(Uuid::nil())); + let target_is_delete_marker = self + .versions + .iter() + .find(|ver| ver.header.version_id == vid) + .is_some_and(|ver| ver.header.version_type == VersionType::Delete); let mut ventry = FileMetaVersion::default(); if fi.deleted { @@ -416,6 +421,10 @@ impl FileMeta { } } + if target_is_delete_marker && !fi.deleted && !fi.version_purge_status().is_empty() { + update_version = false; + } + if fi.deleted { if !fi.delete_marker_replication_status().is_empty() && let Some(delete_marker) = ventry.delete_marker.as_mut() @@ -1659,6 +1668,45 @@ mod test { } } + #[test] + fn delete_version_removes_delete_marker_during_version_purge_replication() { + let version_id = Uuid::new_v4(); + let mut fm = FileMeta::new(); + fm.add_version_filemata(FileMetaVersion { + version_type: VersionType::Delete, + legacy_object: None, + object: None, + delete_marker: Some(MetaDeleteMarker { + version_id: Some(version_id), + mod_time: Some(OffsetDateTime::now_utc()), + meta_sys: HashMap::new(), + }), + write_version: 1, + uses_legacy_checksum: false, + }) + .unwrap(); + + let fi = FileInfo { + deleted: false, + mark_deleted: false, + version_id: Some(version_id), + replication_state_internal: Some(ReplicationState { + version_purge_status_internal: Some("target=PENDING;".to_string()), + purge_targets: version_purge_statuses_map("target=PENDING;"), + ..Default::default() + }), + ..Default::default() + }; + + let result = fm.delete_version(&fi).unwrap(); + + assert!(result.is_none()); + assert!( + fm.versions.is_empty(), + "delete-marker version purge should remove the local marker instead of rewriting purge metadata onto it" + ); + } + #[test] fn test_data_integrity_validation() { // Test data integrity checks diff --git a/crates/scanner/Cargo.toml b/crates/scanner/Cargo.toml index 42f5555f5..d5b939ea5 100644 --- a/crates/scanner/Cargo.toml +++ b/crates/scanner/Cargo.toml @@ -55,6 +55,7 @@ metrics = { workspace = true } [dev-dependencies] tracing-subscriber = { workspace = true } serial_test = { workspace = true } +temp-env = { workspace = true } uuid = { workspace = true, features = ["v4", "serde"] } tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index 99bcb43bf..a8a2ce161 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -22,10 +22,8 @@ use crate::{DataUsageInfo, ScannerError}; use chrono::{DateTime, Utc}; use rustfs_common::heal_channel::HealScanMode; use rustfs_common::metrics::{CurrentCycle, Metric, Metrics, emit_scan_cycle_complete, global_metrics}; -use rustfs_config::DEFAULT_SCANNER_SPEED; -use rustfs_config::ENV_SCANNER_SPEED; -use rustfs_config::ENV_SCANNER_START_DELAY_SECS; use rustfs_config::ScannerSpeed; +use rustfs_config::{DEFAULT_SCANNER_SPEED, ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS}; use rustfs_ecstore::StorageAPI as _; use rustfs_ecstore::config::com::{read_config, save_config}; use rustfs_ecstore::disk::RUSTFS_META_BUCKET; @@ -40,11 +38,15 @@ use tracing::{debug, error, info, instrument, warn}; const ENV_SCANNER_START_DELAY_SECS_DEPRECATED: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS"; -/// Returns the base cycle interval. If `RUSTFS_SCANNER_START_DELAY_SECS` -/// is set (or `RUSTFS_DATA_SCANNER_START_DELAY_SECS` as deprecated alias), -/// it takes precedence; otherwise the value is derived from the -/// `RUSTFS_SCANNER_SPEED` preset. +/// Returns the base cycle interval. +/// Priority order: +/// 1. RUSTFS_SCANNER_CYCLE (if set, overrides everything) +/// 2. RUSTFS_SCANNER_START_DELAY_SECS (for backward compatibility) +/// 3. RUSTFS_SCANNER_SPEED preset fn cycle_interval() -> Duration { + if let Some(secs) = rustfs_utils::get_env_opt_u64(ENV_SCANNER_CYCLE) { + return Duration::from_secs(secs); + } if let Some(secs) = scanner_start_delay_secs() { return Duration::from_secs(secs); } @@ -181,6 +183,7 @@ fn get_lock_acquire_timeout() -> Duration { #[instrument(skip_all)] async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc, cycle_info: &mut CurrentCycle) { + SCANNER_SLEEPER.refresh_from_env(); info!("Start run data scanner cycle"); cycle_info.current = cycle_info.next; let now = Instant::now(); @@ -356,6 +359,7 @@ pub async fn store_data_usage_in_backend( mod tests { use super::*; use serial_test::serial; + use temp_env::{with_var, with_var_unset}; #[test] #[serial] @@ -376,6 +380,42 @@ mod tests { assert!(delay <= Duration::from_secs(132)); } + #[test] + #[serial] + fn test_cycle_interval_prefers_explicit_cycle_override() { + with_var(ENV_SCANNER_SPEED, Some("slowest"), || { + with_var(ENV_SCANNER_CYCLE, Some("42"), || { + assert_eq!(cycle_interval(), Duration::from_secs(42)); + }); + }); + } + + #[test] + #[serial] + fn test_cycle_interval_supports_minio_speed_alias() { + with_var_unset(ENV_SCANNER_SPEED, || { + with_var_unset(ENV_SCANNER_CYCLE, || { + with_var_unset(ENV_SCANNER_START_DELAY_SECS, || { + with_var("MINIO_SCANNER_SPEED", Some("slowest"), || { + assert_eq!(cycle_interval(), Duration::from_secs(30 * 60)); + }); + }); + }); + }); + } + + #[test] + #[serial] + fn test_cycle_interval_supports_minio_cycle_alias() { + with_var_unset(ENV_SCANNER_CYCLE, || { + with_var_unset(ENV_SCANNER_START_DELAY_SECS, || { + with_var("MINIO_SCANNER_CYCLE", Some("90"), || { + assert_eq!(cycle_interval(), Duration::from_secs(90)); + }); + }); + }); + } + #[test] #[serial] fn test_randomized_cycle_delay_handles_small_start_delay() { diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index 9ea5e3214..cf33a87bb 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -30,7 +30,7 @@ use rustfs_common::heal_channel::{ }; use rustfs_common::metrics::{IlmAction, Metric, Metrics, UpdateCurrentPathFn, current_path_updater}; use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc; -use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_rule; +use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{GLOBAL_ExpiryState, apply_expiry_rule}; use rustfs_ecstore::bucket::lifecycle::evaluator::Evaluator; use rustfs_ecstore::bucket::lifecycle::{ bucket_lifecycle_ops::apply_transition_rule, @@ -444,8 +444,14 @@ impl ScannerItem { } } - if !to_delete_objs.is_empty() { - // TODO: enqueueNoncurrentVersions + if !to_delete_objs.is_empty() + && let Some(event) = noncurrent_events.first().cloned() + { + GLOBAL_ExpiryState + .write() + .await + .enqueue_by_newer_noncurrent(&self.bucket, to_delete_objs, event) + .await; } self.alert_excessive_versions(remaining_versions, cumulative_size); } diff --git a/crates/scanner/src/sleeper.rs b/crates/scanner/src/sleeper.rs index 0b82599fd..43434da14 100644 --- a/crates/scanner/src/sleeper.rs +++ b/crates/scanner/src/sleeper.rs @@ -21,6 +21,13 @@ use tokio::time::Duration; const MIN_SLEEP: Duration = Duration::from_millis(1); +fn scanner_env_config() -> (ScannerSpeed, bool) { + let speed_str = rustfs_utils::get_env_str(ENV_SCANNER_SPEED, DEFAULT_SCANNER_SPEED); + let speed = ScannerSpeed::from_env_str(&speed_str); + let idle_mode = rustfs_utils::get_env_bool(ENV_SCANNER_IDLE_MODE, DEFAULT_SCANNER_IDLE_MODE); + (speed, idle_mode) +} + /// When `true` (default), the scanner throttles itself between operations. /// When `false`, all sleeps are skipped and the scanner runs at full speed. pub static SCANNER_IDLE_MODE: AtomicBool = AtomicBool::new(DEFAULT_SCANNER_IDLE_MODE); @@ -28,10 +35,7 @@ pub static SCANNER_IDLE_MODE: AtomicBool = AtomicBool::new(DEFAULT_SCANNER_IDLE_ /// Global scanner sleeper initialized from the `RUSTFS_SCANNER_SPEED` and /// `RUSTFS_SCANNER_IDLE_MODE` environment variables. pub static SCANNER_SLEEPER: LazyLock = LazyLock::new(|| { - let speed_str = rustfs_utils::get_env_str(ENV_SCANNER_SPEED, DEFAULT_SCANNER_SPEED); - let speed = ScannerSpeed::from_env_str(&speed_str); - - let idle_mode = rustfs_utils::get_env_bool(ENV_SCANNER_IDLE_MODE, DEFAULT_SCANNER_IDLE_MODE); + let (speed, idle_mode) = scanner_env_config(); SCANNER_IDLE_MODE.store(idle_mode, Ordering::Relaxed); DynamicSleeper::new(speed) @@ -104,6 +108,13 @@ impl DynamicSleeper { let mut m = self.inner.max_sleep.write().unwrap_or_else(|e| e.into_inner()); *m = speed.max_sleep(); } + + /// Reload speed and idle-mode settings from the current environment. + pub fn refresh_from_env(&self) { + let (speed, idle_mode) = scanner_env_config(); + self.update(speed); + SCANNER_IDLE_MODE.store(idle_mode, Ordering::Relaxed); + } } /// A timer returned by [`DynamicSleeper::timer`]. Records the instant it @@ -138,6 +149,7 @@ impl SleepTimer { mod tests { use super::*; use serial_test::serial; + use temp_env::with_var; #[test] fn test_scanner_speed_presets() { @@ -166,6 +178,26 @@ mod tests { assert_eq!(max_sleep, Duration::from_secs(15)); } + #[test] + #[serial] + fn test_refresh_from_env_applies_speed_and_idle_mode_for_next_cycle() { + let prev_mode = SCANNER_IDLE_MODE.load(Ordering::Relaxed); + SCANNER_IDLE_MODE.store(true, Ordering::Relaxed); + + let s = DynamicSleeper::new(ScannerSpeed::Fastest); + with_var(ENV_SCANNER_SPEED, Some("slow"), || { + with_var(ENV_SCANNER_IDLE_MODE, Some("false"), || { + s.refresh_from_env(); + let (factor, max_sleep) = s.read_params(); + assert_eq!(factor, 10.0); + assert_eq!(max_sleep, Duration::from_secs(15)); + assert!(!SCANNER_IDLE_MODE.load(Ordering::Relaxed)); + }); + }); + + SCANNER_IDLE_MODE.store(prev_mode, Ordering::Relaxed); + } + #[tokio::test(start_paused = true)] #[serial] async fn test_fastest_never_sleeps() { diff --git a/crates/scanner/tests/lifecycle_integration_test.rs b/crates/scanner/tests/lifecycle_integration_test.rs index b8e8304f7..1bdbea61f 100644 --- a/crates/scanner/tests/lifecycle_integration_test.rs +++ b/crates/scanner/tests/lifecycle_integration_test.rs @@ -487,6 +487,89 @@ async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) -> usi .len() } +async fn object_version_count(disk_path: &Path, bucket: &str, object: &str) -> usize { + let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap(); + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(0); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("failed to open local disk"); + let data = disk + .read_metadata(bucket, &path_join_buf(&[object, STORAGE_FORMAT_FILE])) + .await + .expect("failed to read object metadata"); + let meta = FileMeta::load(&data).expect("failed to load file metadata"); + meta.get_file_info_versions(bucket, object, false) + .expect("failed to decode file info versions") + .versions + .len() +} + +async fn wait_for_version_count(disk_path: &Path, bucket: &str, object: &str, expected: usize, timeout: Duration) -> bool { + let deadline = tokio::time::Instant::now() + timeout; + + loop { + if object_version_count(disk_path, bucket, object).await == expected { + return true; + } + + if tokio::time::Instant::now() >= deadline { + return false; + } + + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +async fn scan_object_with_lifecycle(disk_path: &Path, bucket: &str, object: &str) { + let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap(); + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(0); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("failed to open local disk"); + let metadata_path = disk_path.join(bucket).join(object).join(STORAGE_FORMAT_FILE); + let relative_path = metadata_path.to_string_lossy().to_string(); + let (_, scanner_path) = path2_bucket_object_with_base_path(disk_path.to_string_lossy().as_ref(), relative_path.as_str()); + let file_type = fs::metadata(&metadata_path) + .await + .expect("failed to stat object metadata") + .file_type(); + let lifecycle = metadata_sys::get(bucket) + .await + .expect("failed to load bucket metadata") + .lifecycle_config + .clone() + .map(Arc::new); + let item = ScannerItem { + path: scanner_path.clone(), + bucket: bucket.to_string(), + prefix: object.to_string(), + object_name: STORAGE_FORMAT_FILE.to_string(), + file_type, + lifecycle, + replication: None, + heal_enabled: false, + heal_bitrot: false, + debug: false, + }; + disk.get_size(item).await.expect("scanner get_size should succeed"); +} + async fn scan_object_metadata(disk_path: &Path, bucket: &str, object: &str) { let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap(); endpoint.set_pool_index(0); @@ -1131,4 +1214,255 @@ mod serial_tests { "deleted object should remain absent after scanner cleanup" ); } + + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[serial] + async fn test_scanner_expires_zero_day_current_version() { + let (disk_paths, ecstore) = setup_isolated_test_env(true).await; + + let bucket_name = format!("test-zero-day-expire-{}", &Uuid::new_v4().simple().to_string()[..8]); + let object_name = "test/object.txt"; + + create_test_bucket(&ecstore, bucket_name.as_str()).await; + set_bucket_lifecycle(bucket_name.as_str()) + .await + .expect("Failed to set lifecycle configuration"); + upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await; + + assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await); + + scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await; + + assert!( + wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(3)).await, + "scanner should delete zero-day current version after enqueueing expiry" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[serial] + async fn test_put_object_immediately_enqueues_zero_day_current_expiry() { + let (_disk_paths, ecstore) = setup_isolated_test_env(true).await; + + let bucket_name = format!("test-put-zero-day-expire-{}", &Uuid::new_v4().simple().to_string()[..8]); + let object_name = "expire-now.txt"; + + create_test_bucket(&ecstore, bucket_name.as_str()).await; + + let lifecycle_xml = format!( + r#" + + + test-rule + Enabled + + {object_name} + + + 0 + + +"# + ); + metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()) + .await + .expect("Failed to set lifecycle configuration"); + + upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await; + + assert!( + wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(2)).await, + "put_object should enqueue zero-day current expiry without waiting for scanner" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[serial] + async fn test_scanner_expires_zero_day_noncurrent_version() { + let (disk_paths, ecstore) = setup_isolated_test_env(false).await; + + let bucket_name = format!("test-zero-day-noncurrent-{}", &Uuid::new_v4().simple().to_string()[..8]); + let object_name = "test/object.txt"; + + create_test_lock_bucket(&ecstore, bucket_name.as_str()).await; + + let mut reader = PutObjReader::from_vec(b"v1".to_vec()); + ecstore + .put_object( + bucket_name.as_str(), + object_name, + &mut reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("failed to upload v1"); + let mut reader = PutObjReader::from_vec(b"v2".to_vec()); + ecstore + .put_object( + bucket_name.as_str(), + object_name, + &mut reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("failed to upload v2"); + + assert_eq!(object_version_count(&disk_paths[0], bucket_name.as_str(), object_name).await, 2); + + let lifecycle_xml = r#" + + + test-rule + Enabled + + test/ + + + 0 + + +"#; + metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()) + .await + .expect("Failed to set noncurrent lifecycle configuration"); + + rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await; + + scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await; + + assert!( + wait_for_version_count(&disk_paths[0], bucket_name.as_str(), object_name, 1, Duration::from_secs(3)).await, + "scanner should delete zero-day noncurrent versions after enqueueing expiry" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[serial] + async fn test_put_object_immediately_enqueues_zero_day_noncurrent_expiry() { + let (disk_paths, ecstore) = setup_isolated_test_env(true).await; + + let bucket_name = format!("test-put-zero-day-noncurrent-{}", &Uuid::new_v4().simple().to_string()[..8]); + let object_name = "test/object.txt"; + + create_test_lock_bucket(&ecstore, bucket_name.as_str()).await; + + let lifecycle_xml = r#" + + + test-rule + Enabled + + test/ + + + 0 + + +"#; + metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()) + .await + .expect("Failed to set noncurrent lifecycle configuration"); + + let mut reader = PutObjReader::from_vec(b"v1".to_vec()); + ecstore + .put_object( + bucket_name.as_str(), + object_name, + &mut reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("failed to upload v1"); + let mut reader = PutObjReader::from_vec(b"v2".to_vec()); + ecstore + .put_object( + bucket_name.as_str(), + object_name, + &mut reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("failed to upload v2"); + + assert!( + wait_for_version_count(&disk_paths[0], bucket_name.as_str(), object_name, 1, Duration::from_secs(2)).await, + "put_object should enqueue zero-day noncurrent expiry without waiting for scanner" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[serial] + async fn test_background_scanner_expires_zero_day_current_version() { + let (_disk_paths, ecstore) = setup_isolated_test_env(true).await; + + let bucket_name = format!("test-bg-zero-day-expire-{}", &Uuid::new_v4().simple().to_string()[..8]); + let object_name = "test/object.txt"; + + create_test_bucket(&ecstore, bucket_name.as_str()).await; + set_bucket_lifecycle(bucket_name.as_str()) + .await + .expect("Failed to set lifecycle configuration"); + upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await; + + let ctx = CancellationToken::new(); + init_data_scanner(ctx.clone(), ecstore.clone()).await; + + let deleted = wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(12)).await; + + ctx.cancel(); + + assert!(deleted, "background scanner should delete zero-day current version after startup delay"); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[serial] + async fn test_background_scanner_expires_zero_day_current_version_for_exact_key_prefix() { + let (_disk_paths, ecstore) = setup_isolated_test_env(true).await; + + let bucket_name = format!("test-bg-zero-day-exact-{}", &Uuid::new_v4().simple().to_string()[..8]); + let object_name = "expire-now.txt"; + + create_test_bucket(&ecstore, bucket_name.as_str()).await; + + let lifecycle_xml = format!( + r#" + + + test-rule + Enabled + + {object_name} + + + 0 + + +"# + ); + metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()) + .await + .expect("Failed to set lifecycle configuration"); + upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await; + + let ctx = CancellationToken::new(); + init_data_scanner(ctx.clone(), ecstore.clone()).await; + + let deleted = wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(12)).await; + + ctx.cancel(); + + assert!(deleted, "background scanner should delete zero-day exact-key lifecycle targets"); + } } diff --git a/crates/utils/src/envs.rs b/crates/utils/src/envs.rs index c21734f93..74607f855 100644 --- a/crates/utils/src/envs.rs +++ b/crates/utils/src/envs.rs @@ -173,6 +173,8 @@ const EXTERNAL_COMPATIBLE_SUFFIXES: &[&str] = &[ "REGION", "ROOT_PASSWORD", "ROOT_USER", + "SCANNER_CYCLE", + "SCANNER_SPEED", "SECRET_KEY", "SECRET_KEY_FILE", "STORAGE_CLASS_INLINE_BLOCK", diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index 7d92ade6f..92e58d689 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -34,7 +34,9 @@ use http::StatusCode; use metrics::counter; use rustfs_config::RUSTFS_REGION; use rustfs_ecstore::bucket::{ - lifecycle::bucket_lifecycle_ops::{enqueue_transition_for_existing_objects, validate_transition_tier}, + lifecycle::bucket_lifecycle_ops::{ + enqueue_expiry_for_existing_objects, enqueue_transition_for_existing_objects, validate_transition_tier, + }, metadata::{ BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_REPLICATION_CONFIG, BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, @@ -494,6 +496,13 @@ fn lifecycle_has_transition_rules(config: &BucketLifecycleConfiguration) -> bool }) } +fn lifecycle_has_expiry_rules(config: &BucketLifecycleConfiguration) -> bool { + config.rules.iter().any(|rule| { + rule.status == ExpirationStatus::from_static(ExpirationStatus::ENABLED) + && (rule.expiration.is_some() || rule.del_marker_expiration.is_some() || rule.noncurrent_version_expiration.is_some()) + }) +} + #[derive(Clone, Default)] pub struct DefaultBucketUsecase { context: Option>, @@ -1342,6 +1351,18 @@ impl DefaultBucketUsecase { }); } + if lifecycle_has_expiry_rules(&input_cfg) + && let Some(store) = new_object_layer_fn() + { + let bucket_name = bucket.clone(); + let request_context = req.extensions.get::().cloned(); + spawn_background_with_context(request_context, async move { + if let Err(err) = enqueue_expiry_for_existing_objects(store, &bucket_name).await { + warn!(bucket = %bucket_name, error = ?err, "failed to enqueue expiry for existing objects"); + } + }); + } + Ok(S3Response::new(PutBucketLifecycleConfigurationOutput::default())) } diff --git a/rustfs/src/app/lifecycle_transition_api_test.rs b/rustfs/src/app/lifecycle_transition_api_test.rs index 7c8ec80b0..3e759121c 100644 --- a/rustfs/src/app/lifecycle_transition_api_test.rs +++ b/rustfs/src/app/lifecycle_transition_api_test.rs @@ -13,6 +13,7 @@ // limitations under the License. use super::{multipart_usecase::DefaultMultipartUsecase, object_usecase::DefaultObjectUsecase}; +use crate::app::bucket_usecase::DefaultBucketUsecase; use crate::storage::ecfs::FS; use bytes::Bytes; use futures::stream; @@ -182,6 +183,40 @@ async fn set_bucket_lifecycle_transition_with_tier( Ok(()) } +fn expiration_lifecycle_configuration(prefix: &str) -> BucketLifecycleConfiguration { + BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + expiration: Some(LifecycleExpiration { + date: Some(Timestamp::from( + time::OffsetDateTime::now_utc() + .replace_time(time::Time::MIDNIGHT) + .saturating_sub(time::Duration::days(1)), + )), + days: None, + expired_object_delete_marker: None, + ..Default::default() + }), + filter: Some(LifecycleRuleFilter { + and: None, + object_size_greater_than: None, + object_size_less_than: None, + prefix: Some(prefix.to_string()), + tag: None, + ..Default::default() + }), + id: Some("expire-existing".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + } +} + #[derive(Clone, Default)] struct MockWarmBackend { objects: Arc>>>, @@ -326,6 +361,24 @@ async fn wait_for_object_absence(ecstore: &Arc, bucket: &str, object: & } } +async fn wait_for_delete_marker(ecstore: &Arc, bucket: &str, object: &str, timeout: Duration) -> bool { + let deadline = tokio::time::Instant::now() + timeout; + + loop { + if let Ok(info) = ecstore.get_object_info(bucket, object, &ObjectOptions::default()).await + && info.delete_marker + { + return true; + } + + if tokio::time::Instant::now() >= deadline { + return false; + } + + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + fn build_request(input: T, method: Method) -> S3Request { S3Request { input, @@ -583,3 +636,37 @@ async fn lifecycle_transition_marks_dirty_disks_for_capacity_manager() { .collect(); assert_eq!(actual_paths, expected_paths); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[serial] +#[ignore = "requires isolated global object layer state"] +async fn put_bucket_lifecycle_configuration_expires_existing_objects() { + let (_disk_paths, ecstore) = setup_test_env().await; + let usecase = DefaultBucketUsecase::without_context(); + + let bucket = format!("test-api-expire-existing-{}", &Uuid::new_v4().simple().to_string()[..8]); + let object = "test/existing.txt"; + let payload = b"expire existing object after lifecycle update"; + + create_test_bucket(&ecstore, bucket.as_str()).await; + let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await; + + let req = build_request( + PutBucketLifecycleConfigurationInput::builder() + .bucket(bucket.clone()) + .lifecycle_configuration(Some(expiration_lifecycle_configuration("test/"))) + .build() + .unwrap(), + Method::PUT, + ); + + usecase + .execute_put_bucket_lifecycle_configuration(req) + .await + .expect("Failed to update lifecycle configuration"); + + assert!( + wait_for_delete_marker(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT).await, + "existing object should be lifecycle-deleted after lifecycle update" + ); +} diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 2cf304e7e..bf8f09ba8 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -58,8 +58,8 @@ use rustfs_ecstore::bucket::{ }, quota::QuotaOperation, replication::{ - DeletedObjectReplicationInfo, check_replicate_delete, get_must_replicate_options, must_replicate, schedule_replication, - schedule_replication_delete, + DeletedObjectReplicationInfo, ObjectOpts as ReplicationObjectOpts, ReplicationConfigurationExt, check_replicate_delete, + get_must_replicate_options, must_replicate, schedule_replication, schedule_replication_delete, }, tagging::decode_tags, versioning::VersioningApi, @@ -75,8 +75,9 @@ use rustfs_ecstore::store_api::{ HTTPRangeSpec, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, PutObjReader, }; use rustfs_filemeta::{ - REPLICATE_INCOMING_DELETE, ReplicationStatusType, ReplicationType, RestoreStatusOps, VersionPurgeStatusType, - parse_restore_obj_status, + REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateTargetDecision, ReplicationState, ReplicationStatusType, + ReplicationType, RestoreStatusOps, VersionPurgeStatusType, parse_restore_obj_status, replication_statuses_map, + version_purge_statuses_map, }; use rustfs_io_metrics; use rustfs_notify::EventArgsBuilder; @@ -460,6 +461,100 @@ fn build_put_object_expiration_header(event: &lifecycle::Event) -> Option, + replica: bool, +) -> Option { + let opts = ReplicationObjectOpts { + name: obj_info.name.clone(), + user_tags: obj_info.user_tags.clone(), + version_id, + delete_marker: obj_info.delete_marker, + op_type: ReplicationType::Delete, + replica, + ..Default::default() + }; + let target_arns = config.filter_target_arns(&opts); + if target_arns.is_empty() { + return None; + } + + let mut decision = ReplicateDecision::new(); + for target_arn in target_arns { + let mut target_opts = opts.clone(); + target_opts.target_arn = target_arn.clone(); + decision.set(ReplicateTargetDecision::new(target_arn, config.replicate(&target_opts), false)); + } + if !decision.replicate_any() { + return None; + } + + let pending_status = decision.pending_status(); + let mut state = ReplicationState { + replicate_decision_str: decision.to_string(), + ..Default::default() + }; + if version_id.is_some() { + state.version_purge_status_internal = pending_status.clone(); + state.purge_targets = version_purge_statuses_map(pending_status.as_deref().unwrap_or_default()); + } else { + state.replication_status_internal = pending_status.clone(); + state.targets = replication_statuses_map(pending_status.as_deref().unwrap_or_default()); + } + Some(state) +} + +async fn enrich_delete_replication_state_if_needed( + bucket: &str, + delete_object: &mut rustfs_ecstore::store_api::DeletedObject, + obj_info: &ObjectInfo, +) { + let Some(replication_state) = delete_object.replication_state.as_ref() else { + return; + }; + if !replication_state.replicate_decision_str.is_empty() + && (!replication_state.targets.is_empty() || !replication_state.purge_targets.is_empty()) + { + return; + } + + let Ok((config, _)) = metadata_sys::get_replication_config(bucket).await else { + return; + }; + let version_id = if delete_object.delete_marker { + None + } else if delete_object.delete_marker_version_id.is_some() { + delete_object.delete_marker_version_id + } else { + delete_object.version_id + }; + if let Some(local_state) = delete_replication_state_from_config( + &config, + obj_info, + version_id, + obj_info.replication_status == ReplicationStatusType::Replica, + ) { + delete_object.replication_state = Some(local_state); + } +} + +fn should_schedule_delete_replication( + opts: &ObjectOptions, + replication_source: &ObjectInfo, + deleted_delete_marker_version: bool, +) -> bool { + if opts.replication_request { + return false; + } + + replication_source.replication_status == ReplicationStatusType::Replica + || replication_source.replication_status == ReplicationStatusType::Pending + || replication_source.version_purge_status == VersionPurgeStatusType::Pending + || (deleted_delete_marker_version && replication_source.replication_status == ReplicationStatusType::Completed) +} + const AMZ_SNOWBALL_EXTRACT_COMPAT: &str = "X-Amz-Snowball-Auto-Extract"; #[cfg(test)] const AMZ_SNOWBALL_PREFIX_INTERNAL: &str = "X-Amz-Meta-Rustfs-Snowball-Prefix"; @@ -1904,6 +1999,15 @@ impl DefaultObjectUsecase { None }; + // x-amz-restore: extract from object metadata + let restore = info.user_defined.get(X_AMZ_RESTORE.as_str()).and_then(|v| { + let rs = parse_restore_obj_status(v).ok()?; + Some(rs.to_string2()) + }); + + // x-amz-expiration: predict from lifecycle configuration + let expiration = resolve_put_object_expiration(bucket, &info).await; + let output = GetObjectOutput { body, content_length: Some(response_content_length), @@ -1925,6 +2029,8 @@ impl DefaultObjectUsecase { checksum_crc64nvme: checksums.crc64nvme, checksum_type: checksums.checksum_type, version_id: output_version_id, + restore, + expiration, ..Default::default() }; @@ -3126,25 +3232,35 @@ impl DefaultObjectUsecase { return result; } - if obj_info.replication_status == ReplicationStatusType::Replica - || obj_info.replication_status == ReplicationStatusType::Pending - || obj_info.version_purge_status == VersionPurgeStatusType::Pending - { - schedule_replication_delete(DeletedObjectReplicationInfo { + let deleted_replication_info = existing_object_info.as_ref().filter(|_| opts.version_id.is_some()); + let replication_source = deleted_replication_info.unwrap_or(&obj_info); + let deleted_delete_marker_version = deleted_replication_info.is_some_and(|info| info.delete_marker); + + if should_schedule_delete_replication(&opts, replication_source, deleted_delete_marker_version) { + let mut deleted_object = DeletedObjectReplicationInfo { delete_object: rustfs_ecstore::store_api::DeletedObject { - delete_marker: obj_info.delete_marker, - delete_marker_version_id: if obj_info.delete_marker { obj_info.version_id } else { None }, + delete_marker: replication_source.delete_marker && !deleted_delete_marker_version, + delete_marker_version_id: if replication_source.delete_marker { + replication_source.version_id + } else { + None + }, object_name: key.clone(), - version_id: if obj_info.delete_marker { None } else { obj_info.version_id }, - delete_marker_mtime: obj_info.mod_time, - replication_state: Some(obj_info.replication_state()), + version_id: if replication_source.delete_marker { + None + } else { + replication_source.version_id + }, + delete_marker_mtime: replication_source.mod_time, + replication_state: Some(replication_source.replication_state()), ..Default::default() }, bucket: bucket.clone(), event_type: REPLICATE_INCOMING_DELETE.to_string(), ..Default::default() - }) - .await; + }; + enrich_delete_replication_state_if_needed(&bucket, &mut deleted_object.delete_object, replication_source).await; + schedule_replication_delete(deleted_object).await; } let delete_marker = obj_info.delete_marker; @@ -3299,6 +3415,8 @@ impl DefaultObjectUsecase { req.input.sse_customer_key_md5.as_ref(), )?; + // Compute x-amz-expiration header from lifecycle prediction (before info is partially moved) + let expiration_header = resolve_put_object_expiration(&bucket, &info).await; let event_info = info.clone(); let content_type = { if let Some(content_type) = &info.content_type { @@ -3411,6 +3529,13 @@ impl DefaultObjectUsecase { checksum_crc64nvme, checksum_type, storage_class, + // x-amz-restore from object metadata + restore: metadata_map.get(X_AMZ_RESTORE.as_str()).and_then(|v| { + let rs = parse_restore_obj_status(v).ok()?; + Some(rs.to_string2()) + }), + // x-amz-expiration from lifecycle prediction + expiration: expiration_header, // metadata: object_metadata, ..Default::default() }; @@ -4197,6 +4322,10 @@ fn object_attributes_requested(object_attributes: &[ObjectAttributes], name: &'s mod tests { use super::*; use http::{Extensions, HeaderMap, HeaderName, HeaderValue, Method, Uri}; + use s3s::dto::{ + DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ExistingObjectReplication, + ExistingObjectReplicationStatus, ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus, + }; fn build_request(input: T, method: Method) -> S3Request { S3Request { @@ -4606,6 +4735,44 @@ mod tests { assert_eq!(err.code(), &S3ErrorCode::InternalError); } + #[test] + fn should_schedule_delete_replication_skips_replica_requests() { + let opts = ObjectOptions { + replication_request: true, + version_id: Some(Uuid::new_v4().to_string()), + ..Default::default() + }; + let replication_source = ObjectInfo { + delete_marker: true, + replication_status: ReplicationStatusType::Completed, + ..Default::default() + }; + + assert!( + !should_schedule_delete_replication(&opts, &replication_source, true), + "replica delete requests on target sites must not enqueue a second replication delete task" + ); + } + + #[test] + fn should_schedule_delete_replication_keeps_delete_marker_version_purge_from_source() { + let opts = ObjectOptions { + replication_request: false, + version_id: Some(Uuid::new_v4().to_string()), + ..Default::default() + }; + let replication_source = ObjectInfo { + delete_marker: true, + replication_status: ReplicationStatusType::Completed, + ..Default::default() + }; + + assert!( + should_schedule_delete_replication(&opts, &replication_source, true), + "source-side delete-marker version purge still needs replication scheduling" + ); + } + #[tokio::test] async fn execute_get_object_attributes_returns_internal_error_when_store_uninitialized() { let input = GetObjectAttributesInput::builder() @@ -4772,4 +4939,89 @@ mod tests { let err = usecase.execute_restore_object(req).await.unwrap_err(); assert_eq!(err.code(), &S3ErrorCode::InternalError); } + + #[test] + fn delete_replication_state_from_config_tracks_downstream_delete_marker_targets() { + let arn = "arn:aws:s3:::target-bucket".to_string(); + let config = ReplicationConfiguration { + role: arn.clone(), + rules: vec![ReplicationRule { + delete_marker_replication: Some(DeleteMarkerReplication { + status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)), + }), + delete_replication: None, + destination: Destination { + bucket: arn.clone(), + ..Default::default() + }, + existing_object_replication: Some(ExistingObjectReplication { + status: ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::ENABLED), + }), + filter: None, + id: Some("rule-1".to_string()), + prefix: Some("test/".to_string()), + priority: Some(1), + source_selection_criteria: None, + status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED), + }], + }; + let obj_info = ObjectInfo { + bucket: "bucket".to_string(), + name: "test/object.txt".to_string(), + delete_marker: true, + replication_status: ReplicationStatusType::Replica, + ..Default::default() + }; + + let state = delete_replication_state_from_config(&config, &obj_info, None, true) + .expect("replica delete marker should be forwarded to downstream targets"); + let pending = format!("{arn}=PENDING;"); + + assert_eq!(state.replication_status_internal.as_deref(), Some(pending.as_str())); + assert_eq!(state.replicate_decision_str, format!("{arn}=true;false;{arn};")); + assert!(state.targets.contains_key(&arn)); + } + + #[test] + fn delete_replication_state_from_config_tracks_delete_marker_version_purges() { + let arn = "arn:aws:s3:::target-bucket".to_string(); + let config = ReplicationConfiguration { + role: arn.clone(), + rules: vec![ReplicationRule { + delete_marker_replication: Some(DeleteMarkerReplication { + status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)), + }), + delete_replication: None, + destination: Destination { + bucket: arn.clone(), + ..Default::default() + }, + existing_object_replication: Some(ExistingObjectReplication { + status: ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::ENABLED), + }), + filter: None, + id: Some("rule-1".to_string()), + prefix: Some("test/".to_string()), + priority: Some(1), + source_selection_criteria: None, + status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED), + }], + }; + let obj_info = ObjectInfo { + bucket: "bucket".to_string(), + name: "test/object.txt".to_string(), + delete_marker: true, + replication_status: ReplicationStatusType::Completed, + ..Default::default() + }; + + let version_id = Some(Uuid::new_v4()); + let state = delete_replication_state_from_config(&config, &obj_info, version_id, false) + .expect("delete-marker version purge should honor delete-marker replication rules"); + let pending = format!("{arn}=PENDING;"); + + assert_eq!(state.version_purge_status_internal.as_deref(), Some(pending.as_str())); + assert_eq!(state.replicate_decision_str, format!("{arn}=true;false;{arn};")); + assert!(state.purge_targets.contains_key(&arn)); + } }