mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-25 13:36:50 +00:00
fix(lifecycle): honor bucket default retention (#6324)
* fix(lifecycle): honor bucket retention during scanner expiry * fix(lifecycle): reject zero object lock retention * test(lifecycle): cover malformed retention metadata --------- Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
@@ -66,9 +66,10 @@ impl Evaluator {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// IsObjectLocked checks if it is appropriate to remove an
|
/// IsObjectLocked checks if it is appropriate to remove an
|
||||||
/// object according to its persisted object-lock metadata.
|
/// object according to its persisted object-lock metadata and the bucket
|
||||||
|
/// default retention.
|
||||||
pub fn is_object_locked(&self, obj: &ObjectOpts) -> bool {
|
pub fn is_object_locked(&self, obj: &ObjectOpts) -> bool {
|
||||||
object_lock::is_object_locked_by_metadata(&obj.user_defined, obj.delete_marker)
|
object_lock::is_object_locked(&obj.user_defined, obj.delete_marker, self.lock_retention.as_deref(), obj.mod_time)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// eval will return a lifecycle event for each object in objs for a given time.
|
/// eval will return a lifecycle event for each object in objs for a given time.
|
||||||
@@ -198,8 +199,9 @@ mod tests {
|
|||||||
|
|
||||||
use rustfs_common::metrics::IlmAction;
|
use rustfs_common::metrics::IlmAction;
|
||||||
use s3s::dto::{
|
use s3s::dto::{
|
||||||
BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, ObjectLockConfiguration,
|
BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleExpiration, LifecycleRule,
|
||||||
ObjectLockEnabled, Transition, TransitionStorageClass,
|
NoncurrentVersionExpiration, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule,
|
||||||
|
Transition, TransitionStorageClass,
|
||||||
};
|
};
|
||||||
use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
|
use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
|
||||||
use time::OffsetDateTime;
|
use time::OffsetDateTime;
|
||||||
@@ -300,6 +302,40 @@ mod tests {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn lock_enabled_with_default_retention(days: i32) -> Arc<ObjectLockConfiguration> {
|
||||||
|
Arc::new(ObjectLockConfiguration {
|
||||||
|
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||||
|
rule: Some(ObjectLockRule {
|
||||||
|
default_retention: Some(DefaultRetention {
|
||||||
|
days: Some(days),
|
||||||
|
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::GOVERNANCE)),
|
||||||
|
years: None,
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn noncurrent_expiration_lifecycle() -> Arc<BucketLifecycleConfiguration> {
|
||||||
|
Arc::new(BucketLifecycleConfiguration {
|
||||||
|
expiry_updated_at: None,
|
||||||
|
rules: vec![LifecycleRule {
|
||||||
|
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||||
|
expiration: None,
|
||||||
|
abort_incomplete_multipart_upload: None,
|
||||||
|
del_marker_expiration: None,
|
||||||
|
filter: None,
|
||||||
|
id: Some("expire-noncurrent".to_string()),
|
||||||
|
noncurrent_version_expiration: Some(NoncurrentVersionExpiration {
|
||||||
|
noncurrent_days: Some(1),
|
||||||
|
newer_noncurrent_versions: None,
|
||||||
|
}),
|
||||||
|
noncurrent_version_transitions: None,
|
||||||
|
prefix: None,
|
||||||
|
transitions: None,
|
||||||
|
}],
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn object_opts(replication_status: ReplicationStatusType, version_purge_status: VersionPurgeStatusType) -> ObjectOpts {
|
fn object_opts(replication_status: ReplicationStatusType, version_purge_status: VersionPurgeStatusType) -> ObjectOpts {
|
||||||
ObjectOpts {
|
ObjectOpts {
|
||||||
name: "logs/object".to_string(),
|
name: "logs/object".to_string(),
|
||||||
@@ -459,6 +495,52 @@ mod tests {
|
|||||||
assert_eq!(events[0].action, IlmAction::NoneAction);
|
assert_eq!(events[0].action, IlmAction::NoneAction);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn evaluator_skips_noncurrent_expiration_during_default_retention() {
|
||||||
|
let evaluator =
|
||||||
|
Evaluator::new(noncurrent_expiration_lifecycle()).with_lock_retention(Some(lock_enabled_with_default_retention(30)));
|
||||||
|
let successor_time = OffsetDateTime::now_utc() - time::Duration::days(2);
|
||||||
|
let noncurrent = ObjectOpts {
|
||||||
|
name: "logs/object".to_string(),
|
||||||
|
mod_time: Some(successor_time - time::Duration::days(1)),
|
||||||
|
successor_mod_time: Some(successor_time),
|
||||||
|
version_id: Some(Uuid::new_v4()),
|
||||||
|
is_latest: false,
|
||||||
|
num_versions: 1,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let events = evaluator
|
||||||
|
.eval(&[noncurrent])
|
||||||
|
.await
|
||||||
|
.expect("lifecycle evaluation should succeed");
|
||||||
|
|
||||||
|
assert_eq!(events[0].action, IlmAction::NoneAction);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn evaluator_allows_noncurrent_expiration_after_default_retention() {
|
||||||
|
let evaluator =
|
||||||
|
Evaluator::new(noncurrent_expiration_lifecycle()).with_lock_retention(Some(lock_enabled_with_default_retention(1)));
|
||||||
|
let successor_time = OffsetDateTime::now_utc() - time::Duration::days(2);
|
||||||
|
let noncurrent = ObjectOpts {
|
||||||
|
name: "logs/object".to_string(),
|
||||||
|
mod_time: Some(successor_time - time::Duration::days(1)),
|
||||||
|
successor_mod_time: Some(successor_time),
|
||||||
|
version_id: Some(Uuid::new_v4()),
|
||||||
|
is_latest: false,
|
||||||
|
num_versions: 1,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let events = evaluator
|
||||||
|
.eval(&[noncurrent])
|
||||||
|
.await
|
||||||
|
.expect("lifecycle evaluation should succeed");
|
||||||
|
|
||||||
|
assert_eq!(events[0].action, IlmAction::DeleteVersionAction);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn evaluator_skips_transition_while_replication_pending() {
|
async fn evaluator_skips_transition_while_replication_pending() {
|
||||||
let evaluator = Evaluator::new(latest_transition_lifecycle());
|
let evaluator = Evaluator::new(latest_transition_lifecycle());
|
||||||
|
|||||||
@@ -14,7 +14,7 @@
|
|||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
|
||||||
use s3s::dto::ObjectLockRetentionMode;
|
use s3s::dto::{ObjectLockConfiguration, ObjectLockRetentionMode};
|
||||||
use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
|
use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
|
||||||
use time::{OffsetDateTime, format_description};
|
use time::{OffsetDateTime, format_description};
|
||||||
|
|
||||||
@@ -43,6 +43,90 @@ pub fn is_object_locked_by_metadata(user_defined: &HashMap<String, String>, is_d
|
|||||||
.is_some_and(|retain_until| retain_until.unix_timestamp() > OffsetDateTime::now_utc().unix_timestamp())
|
.is_some_and(|retain_until| retain_until.unix_timestamp() > OffsetDateTime::now_utc().unix_timestamp())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Check persisted object-lock metadata and the bucket default retention.
|
||||||
|
///
|
||||||
|
/// A configured default retention with missing or malformed input is treated
|
||||||
|
/// as locked so a lifecycle worker cannot turn incomplete metadata into an
|
||||||
|
/// unsafe delete.
|
||||||
|
pub fn is_object_locked(
|
||||||
|
user_defined: &HashMap<String, String>,
|
||||||
|
is_delete_marker: bool,
|
||||||
|
config: Option<&ObjectLockConfiguration>,
|
||||||
|
mod_time: Option<OffsetDateTime>,
|
||||||
|
) -> bool {
|
||||||
|
if is_delete_marker {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if is_object_locked_by_metadata(user_defined, false) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if has_explicit_lock_metadata(user_defined) {
|
||||||
|
return !explicit_lock_metadata_is_well_formed(user_defined);
|
||||||
|
}
|
||||||
|
|
||||||
|
let Some(default_retention) = config.and_then(|config| config.rule.as_ref()?.default_retention.as_ref()) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let Some(mode) = default_retention.mode.as_ref() else {
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
if !is_retention_mode(mode.as_str()) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
let Some(mod_time) = mod_time else {
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
let Some(retain_until) = default_retention_until(mod_time, default_retention) else {
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
|
||||||
|
retain_until.unix_timestamp() > OffsetDateTime::now_utc().unix_timestamp()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn has_explicit_lock_metadata(user_defined: &HashMap<String, String>) -> bool {
|
||||||
|
user_defined.contains_key(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str())
|
||||||
|
|| user_defined.contains_key(X_AMZ_OBJECT_LOCK_MODE.as_str())
|
||||||
|
|| user_defined.contains_key(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn explicit_lock_metadata_is_well_formed(user_defined: &HashMap<String, String>) -> bool {
|
||||||
|
if user_defined
|
||||||
|
.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str())
|
||||||
|
.is_some_and(|value| !value.eq_ignore_ascii_case("ON") && !value.eq_ignore_ascii_case("OFF"))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
match (
|
||||||
|
user_defined.get(X_AMZ_OBJECT_LOCK_MODE.as_str()),
|
||||||
|
user_defined.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str()),
|
||||||
|
) {
|
||||||
|
(None, None) => true,
|
||||||
|
(Some(mode), Some(retain_until)) => {
|
||||||
|
is_retention_mode(mode)
|
||||||
|
&& OffsetDateTime::parse(retain_until, &format_description::well_known::Iso8601::DEFAULT).is_ok()
|
||||||
|
}
|
||||||
|
_ => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_retention_until(mod_time: OffsetDateTime, retention: &s3s::dto::DefaultRetention) -> Option<OffsetDateTime> {
|
||||||
|
match (retention.days, retention.years) {
|
||||||
|
(Some(days), None) if days > 0 => Some(mod_time.saturating_add(time::Duration::days(i64::from(days)))),
|
||||||
|
(None, Some(years)) if years > 0 => add_years(mod_time, years),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn add_years(mod_time: OffsetDateTime, years: i32) -> Option<OffsetDateTime> {
|
||||||
|
let target_year = mod_time.year().checked_add(years)?;
|
||||||
|
mod_time
|
||||||
|
.replace_year(target_year)
|
||||||
|
.or_else(|_| mod_time.replace_day(28).and_then(|date| date.replace_year(target_year)))
|
||||||
|
.ok()
|
||||||
|
}
|
||||||
|
|
||||||
fn is_retention_mode(mode: &str) -> bool {
|
fn is_retention_mode(mode: &str) -> bool {
|
||||||
mode.eq_ignore_ascii_case(ObjectLockRetentionMode::COMPLIANCE)
|
mode.eq_ignore_ascii_case(ObjectLockRetentionMode::COMPLIANCE)
|
||||||
|| mode.eq_ignore_ascii_case(ObjectLockRetentionMode::GOVERNANCE)
|
|| mode.eq_ignore_ascii_case(ObjectLockRetentionMode::GOVERNANCE)
|
||||||
@@ -52,6 +136,9 @@ fn is_retention_mode(mode: &str) -> bool {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
use s3s::dto::{DefaultRetention, ObjectLockEnabled, ObjectLockRule};
|
||||||
|
use time::Duration;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn is_object_locked_by_metadata_preserves_object_lock_parser_behavior() {
|
fn is_object_locked_by_metadata_preserves_object_lock_parser_behavior() {
|
||||||
let mut user_defined = HashMap::new();
|
let mut user_defined = HashMap::new();
|
||||||
@@ -60,4 +147,120 @@ mod tests {
|
|||||||
assert!(is_object_locked_by_metadata(&user_defined, false));
|
assert!(is_object_locked_by_metadata(&user_defined, false));
|
||||||
assert!(!is_object_locked_by_metadata(&user_defined, true));
|
assert!(!is_object_locked_by_metadata(&user_defined, true));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn default_retention_config(days: i32) -> ObjectLockConfiguration {
|
||||||
|
ObjectLockConfiguration {
|
||||||
|
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||||
|
rule: Some(ObjectLockRule {
|
||||||
|
default_retention: Some(DefaultRetention {
|
||||||
|
days: Some(days),
|
||||||
|
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::GOVERNANCE)),
|
||||||
|
years: None,
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn default_retention_blocks_lifecycle_delete_until_expired() {
|
||||||
|
let config = default_retention_config(30);
|
||||||
|
let created = OffsetDateTime::now_utc() - Duration::days(1);
|
||||||
|
|
||||||
|
assert!(is_object_locked(&HashMap::new(), false, Some(&config), Some(created)));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn expired_default_retention_allows_lifecycle_delete() {
|
||||||
|
let config = default_retention_config(1);
|
||||||
|
let created = OffsetDateTime::now_utc() - Duration::days(2);
|
||||||
|
|
||||||
|
assert!(!is_object_locked(&HashMap::new(), false, Some(&config), Some(created)));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn missing_mod_time_blocks_default_retention_delete() {
|
||||||
|
let config = default_retention_config(30);
|
||||||
|
|
||||||
|
assert!(is_object_locked(&HashMap::new(), false, Some(&config), None));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn zero_default_retention_days_fail_closed() {
|
||||||
|
let config = default_retention_config(0);
|
||||||
|
let created = OffsetDateTime::now_utc() - Duration::days(2);
|
||||||
|
|
||||||
|
assert!(is_object_locked(&HashMap::new(), false, Some(&config), Some(created)));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn zero_default_retention_years_fail_closed() {
|
||||||
|
let config = ObjectLockConfiguration {
|
||||||
|
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||||
|
rule: Some(ObjectLockRule {
|
||||||
|
default_retention: Some(DefaultRetention {
|
||||||
|
days: None,
|
||||||
|
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::GOVERNANCE)),
|
||||||
|
years: Some(0),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
let created = OffsetDateTime::now_utc() - Duration::days(2);
|
||||||
|
|
||||||
|
assert!(is_object_locked(&HashMap::new(), false, Some(&config), Some(created)));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn default_retention_years_block_lifecycle_delete_until_expired() {
|
||||||
|
let config = ObjectLockConfiguration {
|
||||||
|
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||||
|
rule: Some(ObjectLockRule {
|
||||||
|
default_retention: Some(DefaultRetention {
|
||||||
|
days: None,
|
||||||
|
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)),
|
||||||
|
years: Some(1),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
let created = OffsetDateTime::now_utc() - Duration::days(1);
|
||||||
|
|
||||||
|
assert!(is_object_locked(&HashMap::new(), false, Some(&config), Some(created)));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn expired_explicit_retention_does_not_reapply_default_retention() {
|
||||||
|
let config = default_retention_config(30);
|
||||||
|
let mut user_defined = HashMap::new();
|
||||||
|
user_defined.insert(
|
||||||
|
X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
|
||||||
|
ObjectLockRetentionMode::GOVERNANCE.to_string(),
|
||||||
|
);
|
||||||
|
user_defined.insert(
|
||||||
|
X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(),
|
||||||
|
(OffsetDateTime::now_utc() - Duration::days(1))
|
||||||
|
.format(&format_description::well_known::Iso8601::DEFAULT)
|
||||||
|
.expect("expired retention date should format"),
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(!is_object_locked(&user_defined, false, Some(&config), Some(OffsetDateTime::now_utc())));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn malformed_explicit_retention_fails_closed() {
|
||||||
|
let config = default_retention_config(1);
|
||||||
|
let mut user_defined = HashMap::new();
|
||||||
|
user_defined.insert(
|
||||||
|
X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
|
||||||
|
ObjectLockRetentionMode::GOVERNANCE.to_string(),
|
||||||
|
);
|
||||||
|
let created = OffsetDateTime::now_utc() - Duration::days(2);
|
||||||
|
|
||||||
|
assert!(is_object_locked(&user_defined, false, Some(&config), Some(created)));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn delete_markers_are_not_locked_by_default_retention() {
|
||||||
|
let config = default_retention_config(30);
|
||||||
|
|
||||||
|
assert!(!is_object_locked(&HashMap::new(), true, Some(&config), None));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user