diff --git a/Cargo.lock b/Cargo.lock index f7584d0b7..045893824 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7518,6 +7518,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "serial_test", "sha1 0.11.0-rc.5", "sha2 0.11.0-rc.5", "shadow-rs", diff --git a/crates/e2e_test/src/reliant/lifecycle.rs b/crates/e2e_test/src/reliant/lifecycle.rs index 9f8b55ded..2302eb306 100644 --- a/crates/e2e_test/src/reliant/lifecycle.rs +++ b/crates/e2e_test/src/reliant/lifecycle.rs @@ -19,7 +19,6 @@ use aws_sdk_s3::config::{Credentials, Region}; use bytes::Bytes; use serial_test::serial; use std::error::Error; -use tokio::time::sleep; const ENDPOINT: &str = "http://localhost:9000"; const ACCESS_KEY: &str = "rustfsadmin"; @@ -62,6 +61,7 @@ async fn setup_test_bucket(client: &Client) -> Result<(), Box> { #[ignore = "requires running RustFS server at localhost:9000"] async fn test_bucket_lifecycle_configuration() -> Result<(), Box> { use aws_sdk_s3::types::{BucketLifecycleConfiguration, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter}; + use chrono::{Duration as ChronoDuration, Utc}; use tokio::time::Duration; let client = create_aws_s3_client().await?; @@ -70,6 +70,7 @@ async fn test_bucket_lifecycle_configuration() -> Result<(), Box Result<(), Box 0); + let untouched_resp = client.get_object().bucket(BUCKET).key(untouched_object_key).send().await?; + assert!(untouched_resp.content_length().unwrap_or(0) > 0); - // Configure lifecycle rule: expire after current time + 3 seconds - let expiration = LifecycleExpiration::builder().days(0).build(); + // Use a past midnight UTC date to trigger immediate lifecycle expiry without requiring days=0. + let yesterday_midnight_utc = Utc::now() + .date_naive() + .and_hms_opt(0, 0, 0) + .expect("midnight should always be valid") + - ChronoDuration::days(1); + let expiration = LifecycleExpiration::builder() + .date(aws_sdk_s3::primitives::DateTime::from_secs(yesterday_midnight_utc.and_utc().timestamp())) + .build(); let filter = LifecycleRuleFilter::builder().prefix(lifecycle_object_key).build(); let rule = LifecycleRule::builder() .id("expire-test-object") @@ -105,29 +122,73 @@ async fn test_bucket_lifecycle_configuration() -> Result<(), Box { - panic!("Expected object to be deleted by lifecycle rule, but it still exists"); - } - Err(e) => { - if let Some(service_error) = e.as_service_error() { - if service_error.is_no_such_key() { - println!("Lifecycle configuration test completed - object was successfully deleted by lifecycle rule"); - } else { - panic!("Expected NoSuchKey error, but got: {e:?}"); + // Poll for deletion instead of using a fixed sleep to keep the test deterministic. + // Default scanner cycle interval is 60s with jitter, so allow enough time for one full cycle. + let deadline = tokio::time::Instant::now() + Duration::from_secs(150); + loop { + let get_result = client.get_object().bucket(BUCKET).key(lifecycle_object_key).send().await; + match get_result { + Ok(_) => { + if tokio::time::Instant::now() >= deadline { + panic!("Expected object to be deleted by lifecycle rule within 150s, but it still exists"); + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + Err(e) => { + if let Some(service_error) = e.as_service_error() { + if service_error.is_no_such_key() { + println!("Lifecycle configuration test completed - object was successfully deleted by lifecycle rule"); + break; + } + panic!("Expected NoSuchKey error, but got: {e:?}"); + } else { + panic!("Expected service error, but got: {e:?}"); } - } else { - panic!("Expected service error, but got: {e:?}"); } } } println!("Lifecycle configuration test completed."); + + // Non-matching prefix object should remain available. + let untouched_after = client.get_object().bucket(BUCKET).key(untouched_object_key).send().await?; + assert!(untouched_after.content_length().unwrap_or(0) > 0); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +#[ignore = "requires running RustFS server at localhost:9000"] +async fn test_bucket_lifecycle_rejects_zero_days() -> Result<(), Box> { + use aws_sdk_s3::types::{BucketLifecycleConfiguration, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter}; + + let client = create_aws_s3_client().await?; + setup_test_bucket(&client).await?; + + let expiration = LifecycleExpiration::builder().days(0).build(); + let filter = LifecycleRuleFilter::builder().prefix("zero-days/").build(); + let rule = LifecycleRule::builder() + .id("expire-zero-days") + .filter(filter) + .expiration(expiration) + .status(aws_sdk_s3::types::ExpirationStatus::Enabled) + .build()?; + let lifecycle = BucketLifecycleConfiguration::builder().rules(rule).build()?; + + let err = client + .put_bucket_lifecycle_configuration() + .bucket(BUCKET) + .lifecycle_configuration(lifecycle) + .send() + .await + .expect_err("zero-day lifecycle expiration should be rejected"); + + let err_msg = format!("{err:?}"); + assert!( + err_msg.contains("InvalidArgument") && err_msg.contains("greater than 0"), + "unexpected error: {err_msg}" + ); + Ok(()) } diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index 763fa5d7b..4bfe3c130 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -120,6 +120,7 @@ tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } criterion = { workspace = true, features = ["html_reports"] } temp-env = { workspace = true } tracing-subscriber = { workspace = true } +serial_test = { workspace = true } [build-dependencies] shadow-rs = { workspace = true, features = ["build", "metadata"] } diff --git a/crates/ecstore/src/bucket/lifecycle/lifecycle.rs b/crates/ecstore/src/bucket/lifecycle/lifecycle.rs index 5c5b26cc3..b27789b9f 100644 --- a/crates/ecstore/src/bucket/lifecycle/lifecycle.rs +++ b/crates/ecstore/src/bucket/lifecycle/lifecycle.rs @@ -859,8 +859,11 @@ impl Default for TransitionOptions { mod tests { use super::*; use s3s::dto::LifecycleRuleFilter; + use serial_test::serial; + use std::sync::Arc; #[tokio::test] + #[serial] async fn validate_rejects_non_positive_expiration_days() { let lc = BucketLifecycleConfiguration { rules: vec![LifecycleRule { @@ -888,6 +891,7 @@ mod tests { } #[tokio::test] + #[serial] async fn validate_accepts_positive_expiration_days() { let lc = BucketLifecycleConfiguration { rules: vec![LifecycleRule { @@ -912,6 +916,7 @@ mod tests { } #[tokio::test] + #[serial] async fn validate_rejects_non_midnight_expiration_date() { let lc = BucketLifecycleConfiguration { rules: vec![LifecycleRule { @@ -936,6 +941,7 @@ mod tests { } #[tokio::test] + #[serial] async fn predict_expiration_selects_closest_expiry_for_put_object() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = BucketLifecycleConfiguration { @@ -987,6 +993,7 @@ mod tests { } #[tokio::test] + #[serial] async fn validate_accepts_multiple_rules_without_ids() { let lc = BucketLifecycleConfiguration { rules: vec![ @@ -1027,6 +1034,7 @@ mod tests { } #[tokio::test] + #[serial] async fn validate_rejects_rule_id_too_long() { let lc = BucketLifecycleConfiguration { rules: vec![LifecycleRule { @@ -1051,6 +1059,258 @@ mod tests { } #[tokio::test] + #[serial] + async fn validate_rejects_duplicate_rule_ids() { + let lc = BucketLifecycleConfiguration { + rules: vec![ + LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: Some(LifecycleExpiration { + days: Some(1), + ..Default::default() + }), + abort_incomplete_multipart_upload: None, + filter: None, + id: Some("dup-rule".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }, + LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: Some(LifecycleExpiration { + days: Some(2), + ..Default::default() + }), + abort_incomplete_multipart_upload: None, + filter: None, + id: Some("dup-rule".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }, + ], + }; + + let err = lc.validate(&ObjectLockConfiguration::default()).await.unwrap_err(); + assert_eq!(err.to_string(), ERR_LIFECYCLE_DUPLICATE_ID); + } + + #[tokio::test] + #[serial] + async fn eval_inner_expires_latest_object_after_days_due() { + let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); + let lc = BucketLifecycleConfiguration { + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: Some(LifecycleExpiration { + days: Some(1), + ..Default::default() + }), + abort_incomplete_multipart_upload: None, + filter: None, + id: Some("expire-days".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + }; + + let opts = ObjectOpts { + name: "obj".to_string(), + mod_time: Some(base_time), + is_latest: true, + ..Default::default() + }; + let event = lc.eval_inner(&opts, base_time + Duration::days(2), 0).await; + + assert_eq!(event.action, IlmAction::DeleteAction); + assert_eq!(event.rule_id, "expire-days"); + assert_eq!(event.due, Some(expected_expiry_time(base_time, 1))); + } + + #[tokio::test] + #[serial] + async fn eval_inner_keeps_latest_object_before_days_due() { + let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); + let lc = BucketLifecycleConfiguration { + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: Some(LifecycleExpiration { + days: Some(2), + ..Default::default() + }), + abort_incomplete_multipart_upload: None, + filter: None, + id: Some("expire-days".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + }; + + let opts = ObjectOpts { + name: "obj".to_string(), + mod_time: Some(base_time), + is_latest: true, + ..Default::default() + }; + let event = lc.eval_inner(&opts, base_time + Duration::hours(12), 0).await; + + assert_eq!(event.action, IlmAction::NoneAction); + } + + #[tokio::test] + #[serial] + async fn eval_inner_transitions_latest_object_after_days_due() { + let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); + let lc = BucketLifecycleConfiguration { + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: None, + abort_incomplete_multipart_upload: None, + filter: None, + id: Some("transition-days".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: Some(vec![Transition { + days: Some(1), + date: None, + storage_class: Some(TransitionStorageClass::from_static("COLDTIER44")), + }]), + }], + }; + + let opts = ObjectOpts { + name: "obj".to_string(), + mod_time: Some(base_time), + is_latest: true, + transition_status: "".to_string(), + ..Default::default() + }; + let event = lc.eval_inner(&opts, base_time + Duration::days(2), 0).await; + + assert_eq!(event.action, IlmAction::TransitionAction); + assert_eq!(event.rule_id, "transition-days"); + assert_eq!(event.storage_class, "COLDTIER44"); + } + + #[tokio::test] + #[serial] + async fn eval_inner_expires_noncurrent_version_after_due() { + let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); + let lc = BucketLifecycleConfiguration { + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: None, + abort_incomplete_multipart_upload: None, + filter: None, + id: Some("noncurrent-expire".to_string()), + noncurrent_version_expiration: Some(s3s::dto::NoncurrentVersionExpiration { + noncurrent_days: Some(1), + newer_noncurrent_versions: None, + }), + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + }; + + let opts = ObjectOpts { + name: "obj".to_string(), + mod_time: Some(base_time), + successor_mod_time: Some(base_time), + is_latest: false, + version_id: Some(Uuid::new_v4()), + ..Default::default() + }; + let event = lc.eval_inner(&opts, base_time + Duration::days(2), 0).await; + + assert_eq!(event.action, IlmAction::DeleteVersionAction); + assert_eq!(event.rule_id, "noncurrent-expire"); + assert_eq!(event.due, Some(expected_expiry_time(base_time, 1))); + } + + #[tokio::test] + #[serial] + async fn eval_inner_transitions_noncurrent_version_after_due() { + let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); + let lc = BucketLifecycleConfiguration { + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: None, + abort_incomplete_multipart_upload: None, + filter: None, + id: Some("noncurrent-transition".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: Some(vec![NoncurrentVersionTransition { + noncurrent_days: Some(1), + newer_noncurrent_versions: None, + storage_class: Some(TransitionStorageClass::from_static("COLDTIER44")), + }]), + prefix: None, + transitions: None, + }], + }; + + let opts = ObjectOpts { + name: "obj".to_string(), + mod_time: Some(base_time), + successor_mod_time: Some(base_time), + is_latest: false, + transition_status: "".to_string(), + version_id: Some(Uuid::new_v4()), + ..Default::default() + }; + let event = lc.eval_inner(&opts, base_time + Duration::days(2), 0).await; + + assert_eq!(event.action, IlmAction::TransitionVersionAction); + assert_eq!(event.rule_id, "noncurrent-transition"); + assert_eq!(event.storage_class, "COLDTIER44"); + } + + #[tokio::test] + #[serial] + async fn noncurrent_versions_expiration_limit_returns_configured_limits() { + let lc = Arc::new(BucketLifecycleConfiguration { + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: None, + abort_incomplete_multipart_upload: None, + filter: None, + id: Some("noncurrent-limit".to_string()), + noncurrent_version_expiration: Some(s3s::dto::NoncurrentVersionExpiration { + noncurrent_days: Some(7), + newer_noncurrent_versions: Some(3), + }), + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + }); + + let opts = ObjectOpts { + name: "obj".to_string(), + mod_time: Some(OffsetDateTime::from_unix_timestamp(1_000_000).unwrap()), + is_latest: false, + version_id: Some(Uuid::new_v4()), + ..Default::default() + }; + let event = lc.noncurrent_versions_expiration_limit(&opts).await; + + assert_eq!(event.action, IlmAction::DeleteVersionAction); + assert_eq!(event.rule_id, "noncurrent-limit"); + assert_eq!(event.noncurrent_days, 7); + assert_eq!(event.newer_noncurrent_versions, 3); + } + + #[tokio::test] + #[serial] async fn validate_rejects_invalid_status_case_sensitive() { let lc = BucketLifecycleConfiguration { rules: vec![LifecycleRule { @@ -1075,6 +1335,7 @@ mod tests { } #[tokio::test] + #[serial] async fn filter_rules_respects_filter_prefix() { let mut filter = LifecycleRuleFilter::default(); filter.prefix = Some("prefix".to_string()); @@ -1115,6 +1376,7 @@ mod tests { } #[tokio::test] + #[serial] async fn filter_rules_respects_filter_and_prefix() { let mut filter = LifecycleRuleFilter::default(); @@ -1159,6 +1421,7 @@ mod tests { } #[tokio::test] + #[serial] async fn expired_object_delete_marker_requires_single_version() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = BucketLifecycleConfiguration { @@ -1195,6 +1458,7 @@ mod tests { } #[tokio::test] + #[serial] async fn expired_object_delete_marker_deletes_only_delete_marker_after_due() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = BucketLifecycleConfiguration {