gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 08:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(lifecycle): extract core rule contracts (#4258)

Browse Source
This commit is contained in:
Zhengchao An
2026-07-04 20:05:56 +08:00
committed by GitHub
parent e3a8234bc9
commit d0792b87be
22 changed files with 3900 additions and 3571 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+20
View File
@@ -9073,6 +9073,7 @@ dependencies = [
"rustfs-filemeta",
"rustfs-io-metrics",
"rustfs-kms",
"rustfs-lifecycle",
"rustfs-lock",
"rustfs-madmin",
"rustfs-object-capacity",
@@ -9360,6 +9361,25 @@ dependencies = [
"zeroize",
]
[[package]]
name = "rustfs-lifecycle"
version = "1.0.0-beta.8"
dependencies = [
"async-trait",
"rustfs-common",
"rustfs-config",
"rustfs-replication",
"rustfs-storage-api",
"s3s",
"serial_test",
"temp-env",
"time",
"tokio",
"tracing",
"url",
"uuid",
]
[[package]]
name = "rustfs-lock"
version = "1.0.0-beta.8"
Cargo.toml
+2
View File
@@ -28,6 +28,7 @@ members = [
"crates/heal", # Erasure set and object healing
"crates/iam", # Identity and Access Management
"crates/keystone", # OpenStack Keystone integration
"crates/lifecycle", # Lifecycle rule evaluation contracts
"crates/kms", # Key Management Service
"crates/lock", # Distributed locking implementation
"crates/madmin", # Management dashboard and admin API interface
@@ -97,6 +98,7 @@ rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.8" }
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.8" }
rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.8" }
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.8" }
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-beta.8" }
rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.8" }
rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.8" }
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.8" }
crates/ecstore/Cargo.toml
+1
View File
@@ -50,6 +50,7 @@ rustfs-common.workspace = true
rustfs-policy.workspace = true
rustfs-protos.workspace = true
rustfs-replication.workspace = true
rustfs-lifecycle.workspace = true
rustfs-kms.workspace = true
rustfs-s3-types = { workspace = true }
rustfs-data-usage.workspace = true
crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+3 -3
View File
@@ -1965,7 +1965,7 @@ pub async fn enqueue_immediate_expiry(oi: &ObjectInfo, src: LcEventSrc) {
};
let object_opts = object_infos
.iter()
.map(ObjectOpts::from_object_info)
.map(lifecycle::object_opts_from_object_info)
.collect::<Vec<ObjectOpts>>();
let Ok(events) = Evaluator::new(Arc::new(lifecycle))
.with_lock_retention(lock_config)
@@ -2086,7 +2086,7 @@ async fn enqueue_expiry_for_existing_object_group(
let object_opts = object_infos
.iter()
.map(ObjectOpts::from_object_info)
.map(lifecycle::object_opts_from_object_info)
.collect::<Vec<ObjectOpts>>();
let events = match Evaluator::new(context.lc.clone())
.with_lock_retention(context.lock_config.clone())
@@ -2559,7 +2559,7 @@ pub trait LifecycleOps {
impl LifecycleOps for ObjectInfo {
fn to_lifecycle_opts(&self) -> lifecycle::ObjectOpts {
lifecycle::ObjectOpts::from_object_info(self)
lifecycle::object_opts_from_object_info(self)
}
fn is_remote(&self) -> bool {
crates/ecstore/src/bucket/lifecycle/core.rs
+23 -2950
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/bucket/lifecycle/evaluator.rs
+1 -403
View File
@@ -12,406 +12,4 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, ObjectLockEnabled};
use time::OffsetDateTime;
use tracing::info;
use super::object_lock_boundary;
use crate::bucket::lifecycle::lifecycle::{Event, Lifecycle, ObjectOpts};
use crate::bucket::lifecycle::replication_sink::{self, LifecycleReplicationConfig};
use rustfs_common::metrics::IlmAction;
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
const EVENT_LIFECYCLE_VERSION_SCAN_SKIPPED: &str = "lifecycle_version_scan_skipped";
/// Evaluator - evaluates lifecycle policy on objects for the given lifecycle
/// configuration and lock retention configuration.
pub struct Evaluator {
policy: Arc<BucketLifecycleConfiguration>,
lock_retention: Option<Arc<ObjectLockConfiguration>>,
}
impl Evaluator {
/// NewEvaluator - creates a new evaluator with the given lifecycle
pub fn new(policy: Arc<BucketLifecycleConfiguration>) -> Self {
Self {
policy,
lock_retention: None,
}
}
/// WithLockRetention - sets the lock retention configuration for the evaluator
pub fn with_lock_retention(mut self, lr: Option<Arc<ObjectLockConfiguration>>) -> Self {
self.lock_retention = lr;
self
}
/// WithReplicationConfig is retained for caller compatibility.
/// Lifecycle replication guards are evaluated from per-object replication state.
pub fn with_replication_config(self, _rcfg: Option<Arc<LifecycleReplicationConfig>>) -> Self {
self
}
/// IsPendingReplication checks if the object is pending replication.
pub fn is_pending_replication(&self, obj: &ObjectOpts) -> bool {
replication_sink::has_pending_lifecycle_replication(obj)
}
fn any_version_has_pending_replication(&self, objs: &[ObjectOpts]) -> bool {
objs.iter().any(|obj| self.is_pending_replication(obj))
}
/// IsObjectLocked checks if it is appropriate to remove an
/// object according to locking configuration when this is lifecycle/bucket quota asking.
/// Uses the common `is_object_locked_by_metadata` function for consistency.
pub fn is_object_locked(&self, obj: &ObjectOpts) -> bool {
// First check if object lock is enabled for this bucket
if self.lock_retention.as_ref().is_none_or(|v| {
v.object_lock_enabled
.as_ref()
.is_none_or(|v| v.as_str() != ObjectLockEnabled::ENABLED)
}) {
return false;
}
// Use the common function to check if the object is locked
object_lock_boundary::is_object_locked_by_metadata(&obj.user_defined, obj.delete_marker)
}
/// eval will return a lifecycle event for each object in objs for a given time.
async fn eval_inner(&self, objs: &[ObjectOpts], now: OffsetDateTime) -> Vec<Event> {
let mut events = vec![Event::default(); objs.len()];
let mut newer_noncurrent_versions = 0;
'top_loop: {
for (i, obj) in objs.iter().enumerate() {
let mut event = self.policy.eval_inner(obj, now, newer_noncurrent_versions).await;
if replication_sink::lifecycle_action_waits_for_replication(event.action) && self.is_pending_replication(obj) {
event = Event::default();
}
match event.action {
IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction => {
// Skip if bucket has object locking enabled; To prevent the
// possibility of violating an object retention on one of the
// noncurrent versions of this object.
if self.lock_retention.as_ref().is_some_and(|v| {
v.object_lock_enabled
.as_ref()
.is_some_and(|v| v.as_str() == ObjectLockEnabled::ENABLED)
}) || self.any_version_has_pending_replication(objs)
{
event = Event::default();
} else {
// No need to evaluate remaining versions' lifecycle
// events after DeleteAllVersionsAction*
events[i] = event;
info!(
event = EVENT_LIFECYCLE_VERSION_SCAN_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
reason = "delete_all_versions_action",
action = ?events[i].action,
"Skipped remaining lifecycle version scan"
);
break 'top_loop;
}
}
IlmAction::DeleteVersionAction | IlmAction::DeleteRestoredVersionAction => {
// Defensive code, should never happen
if obj.version_id.is_none_or(|v| v.is_nil()) {
event.action = IlmAction::NoneAction;
}
if self.is_object_locked(obj) {
event = Event::default();
}
}
_ => {}
}
if !obj.is_latest {
match event.action {
IlmAction::DeleteVersionAction => {
// this noncurrent version will be expired, nothing to add
}
_ => {
// this noncurrent version will be spared
newer_noncurrent_versions += 1;
}
}
}
events[i] = event;
}
}
events
}
/// Eval will return a lifecycle event for each object in objs
pub async fn eval(&self, objs: &[ObjectOpts]) -> Result<Vec<Event>, std::io::Error> {
if objs.is_empty() {
return Ok(vec![]);
}
if objs.len() != objs[0].num_versions {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
format!("number of versions mismatch, expected {}, got {}", objs[0].num_versions, objs.len()),
));
}
Ok(self.eval_inner(objs, OffsetDateTime::now_utc()).await)
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use rustfs_common::metrics::IlmAction;
use s3s::dto::{
BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, Transition, TransitionStorageClass,
};
use time::OffsetDateTime;
use uuid::Uuid;
use super::*;
use crate::bucket::lifecycle::replication_sink::{ReplicationStatusType, VersionPurgeStatusType};
fn expired_marker_lifecycle() -> Arc<BucketLifecycleConfiguration> {
Arc::new(BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
expired_object_delete_marker: Some(true),
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("expired-marker".to_string()),
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
}],
})
}
fn latest_expiration_lifecycle() -> Arc<BucketLifecycleConfiguration> {
Arc::new(BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
days: Some(1),
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("expire-current".to_string()),
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
}],
})
}
fn latest_transition_lifecycle() -> Arc<BucketLifecycleConfiguration> {
Arc::new(BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: None,
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("transition-current".to_string()),
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
transitions: Some(vec![Transition {
days: Some(1),
date: None,
storage_class: Some(TransitionStorageClass::from_static("WARM")),
}]),
}],
})
}
fn all_versions_expiration_lifecycle() -> Arc<BucketLifecycleConfiguration> {
Arc::new(BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
days: Some(1),
expired_object_all_versions: Some(true),
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("delete-all".to_string()),
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
}],
})
}
fn object_opts(replication_status: ReplicationStatusType, version_purge_status: VersionPurgeStatusType) -> ObjectOpts {
ObjectOpts {
name: "logs/object".to_string(),
mod_time: Some(OffsetDateTime::from_unix_timestamp(1_000_000).expect("valid fixed test timestamp")),
version_id: Some(Uuid::new_v4()),
is_latest: true,
delete_marker: true,
num_versions: 1,
replication_status,
version_purge_status,
..Default::default()
}
}
fn current_object_opts(replication_status: ReplicationStatusType) -> ObjectOpts {
ObjectOpts {
name: "logs/object".to_string(),
mod_time: Some(OffsetDateTime::from_unix_timestamp(1_000_000).expect("valid fixed test timestamp")),
version_id: Some(Uuid::new_v4()),
is_latest: true,
num_versions: 1,
replication_status,
..Default::default()
}
}
fn versioned_object_opts(replication_status: ReplicationStatusType, is_latest: bool) -> ObjectOpts {
ObjectOpts {
num_versions: 2,
is_latest,
..current_object_opts(replication_status)
}
}
#[tokio::test]
async fn evaluator_allows_expired_delete_marker_after_replication_completed() {
let evaluator = Evaluator::new(expired_marker_lifecycle());
let events = evaluator
.eval(&[object_opts(
ReplicationStatusType::Completed,
VersionPurgeStatusType::Complete,
)])
.await
.expect("completed replication should allow lifecycle evaluation");
assert_eq!(events[0].action, IlmAction::DeleteVersionAction);
}
#[tokio::test]
async fn evaluator_skips_expired_delete_marker_while_replication_pending() {
let evaluator = Evaluator::new(expired_marker_lifecycle());
let events = evaluator
.eval(&[object_opts(ReplicationStatusType::Pending, VersionPurgeStatusType::default())])
.await
.expect("pending replication should still return a lifecycle decision");
assert_eq!(events[0].action, IlmAction::NoneAction);
}
#[tokio::test]
async fn evaluator_skips_expired_delete_marker_while_version_purge_pending() {
let evaluator = Evaluator::new(expired_marker_lifecycle());
let events = evaluator
.eval(&[object_opts(ReplicationStatusType::Completed, VersionPurgeStatusType::Pending)])
.await
.expect("pending version purge should still return a lifecycle decision");
assert_eq!(events[0].action, IlmAction::NoneAction);
}
#[tokio::test]
async fn evaluator_skips_latest_expiration_while_replication_failed() {
let evaluator = Evaluator::new(latest_expiration_lifecycle());
let events = evaluator
.eval(&[current_object_opts(ReplicationStatusType::Failed)])
.await
.expect("failed replication should still return a lifecycle decision");
assert_eq!(events[0].action, IlmAction::NoneAction);
}
#[tokio::test]
async fn evaluator_allows_latest_expiration_after_replication_completed() {
let evaluator = Evaluator::new(latest_expiration_lifecycle());
let events = evaluator
.eval(&[current_object_opts(ReplicationStatusType::Completed)])
.await
.expect("completed replication should allow latest expiration");
assert_eq!(events[0].action, IlmAction::DeleteAction);
}
#[tokio::test]
async fn evaluator_skips_transition_while_replication_pending() {
let evaluator = Evaluator::new(latest_transition_lifecycle());
let events = evaluator
.eval(&[current_object_opts(ReplicationStatusType::Pending)])
.await
.expect("pending replication should still return a lifecycle decision");
assert_eq!(events[0].action, IlmAction::NoneAction);
}
#[tokio::test]
async fn evaluator_allows_transition_after_replication_completed() {
let evaluator = Evaluator::new(latest_transition_lifecycle());
let events = evaluator
.eval(&[current_object_opts(ReplicationStatusType::Completed)])
.await
.expect("completed replication should allow transition");
assert_eq!(events[0].action, IlmAction::TransitionAction);
}
#[tokio::test]
async fn evaluator_skips_delete_all_versions_when_any_version_replication_pending() {
let evaluator = Evaluator::new(all_versions_expiration_lifecycle());
let latest = versioned_object_opts(ReplicationStatusType::Completed, true);
let noncurrent = versioned_object_opts(ReplicationStatusType::Pending, false);
let events = evaluator
.eval(&[latest, noncurrent])
.await
.expect("pending noncurrent replication should still return lifecycle decisions");
assert_eq!(events[0].action, IlmAction::NoneAction);
assert_eq!(events[1].action, IlmAction::NoneAction);
}
#[tokio::test]
async fn evaluator_allows_delete_all_versions_when_all_versions_replication_completed() {
let evaluator = Evaluator::new(all_versions_expiration_lifecycle());
let latest = versioned_object_opts(ReplicationStatusType::Completed, true);
let noncurrent = versioned_object_opts(ReplicationStatusType::Completed, false);
let events = evaluator
.eval(&[latest, noncurrent])
.await
.expect("completed replication should allow delete-all lifecycle decision");
assert_eq!(events[0].action, IlmAction::DeleteAllVersionsAction);
assert_eq!(events[1].action, IlmAction::NoneAction);
}
}
pub use rustfs_lifecycle::Evaluator;
crates/ecstore/src/bucket/lifecycle/object_lock_boundary.rs
+1 -1
View File
@@ -18,7 +18,7 @@ use crate::bucket::object_lock::objectlock_sys::{self, ObjectLockBlockReason};
use crate::object_api::ObjectInfo;
pub(crate) fn is_object_locked_by_metadata(user_defined: &HashMap<String, String>, is_delete_marker: bool) -> bool {
objectlock_sys::is_object_locked_by_metadata(user_defined, is_delete_marker)
rustfs_lifecycle::object_lock::is_object_locked_by_metadata(user_defined, is_delete_marker)
}
pub(crate) async fn check_object_lock_for_deletion(
crates/ecstore/src/bucket/lifecycle/rule.rs
+1 -189
View File
@@ -11,193 +11,5 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::tagging_boundary;
use s3s::dto::{LifecycleRuleAndOperator, LifecycleRuleFilter, NoncurrentVersionTransition, Tag, Transition};
use time::OffsetDateTime;
const _ERR_TRANSITION_INVALID_DAYS: &str = "Days must be 0 or greater when used with Transition";
const _ERR_TRANSITION_INVALID_DATE: &str = "Date must be provided in ISO 8601 format";
const ERR_TRANSITION_INVALID: &str =
"Exactly one of Days (0 or greater) or Date (positive ISO 8601 format) should be present in Transition.";
const _ERR_TRANSITION_DATE_NOT_MIDNIGHT: &str = "'Date' must be at midnight GMT";
pub trait Filter {
fn test_tags(&self, user_tags: &str) -> bool;
fn by_size(&self, sz: i64) -> bool;
}
impl Filter for LifecycleRuleFilter {
fn test_tags(&self, user_tags: &str) -> bool {
if !requires_tag_matching(self) {
return true;
}
let user_tags = tagging_boundary::decode_tags_to_map(user_tags);
self.tag.as_ref().is_none_or(|tag| tag_matches(tag, &user_tags))
&& self.and.as_ref().is_none_or(|and| and_tags_match(and, &user_tags))
}
fn by_size(&self, sz: i64) -> bool {
let sz = sz.max(0);
self.object_size_greater_than.is_none_or(|min| sz > min)
&& self.object_size_less_than.is_none_or(|max| sz < max)
&& self.and.as_ref().is_none_or(|and| and_size_matches(and, sz))
}
}
fn requires_tag_matching(filter: &LifecycleRuleFilter) -> bool {
filter.tag.is_some()
|| filter
.and
.as_ref()
.and_then(|and| and.tags.as_ref())
.is_some_and(|tags| !tags.is_empty())
}
fn tag_matches(tag: &Tag, user_tags: &std::collections::HashMap<String, String>) -> bool {
let Some(key) = tag.key.as_deref() else {
return false;
};
let Some(value) = tag.value.as_deref() else {
return false;
};
user_tags.get(key).is_some_and(|actual| actual == value)
}
fn and_tags_match(and: &LifecycleRuleAndOperator, user_tags: &std::collections::HashMap<String, String>) -> bool {
and.tags
.as_ref()
.is_none_or(|tags| tags.iter().all(|tag| tag_matches(tag, user_tags)))
}
fn and_size_matches(and: &LifecycleRuleAndOperator, sz: i64) -> bool {
and.object_size_greater_than.is_none_or(|min| sz > min) && and.object_size_less_than.is_none_or(|max| sz < max)
}
pub trait TransitionOps {
fn validate(&self) -> Result<(), std::io::Error>;
}
pub trait NoncurrentVersionTransitionOps {
fn validate(&self) -> Result<(), std::io::Error>;
}
fn is_midnight(date: OffsetDateTime) -> bool {
date.hour() == 0 && date.minute() == 0 && date.second() == 0 && date.nanosecond() == 0
}
impl TransitionOps for Transition {
fn validate(&self) -> Result<(), std::io::Error> {
if self
.storage_class
.as_ref()
.is_none_or(|storage_class| storage_class.as_str().is_empty())
{
return Err(std::io::Error::other("ERR_XML_NOT_WELL_FORMED"));
}
if self.date.is_some() == self.days.is_some() {
return Err(std::io::Error::other(ERR_TRANSITION_INVALID));
}
if self.days.is_some_and(|days| days < 0) {
return Err(std::io::Error::other(ERR_TRANSITION_INVALID));
}
if let Some(date) = self.date.clone() {
let date = OffsetDateTime::from(date);
if date.unix_timestamp() <= 0 || !is_midnight(date) {
return Err(std::io::Error::other(ERR_TRANSITION_INVALID));
}
}
Ok(())
}
}
impl NoncurrentVersionTransitionOps for NoncurrentVersionTransition {
fn validate(&self) -> Result<(), std::io::Error> {
if self
.storage_class
.as_ref()
.is_none_or(|storage_class| storage_class.as_str().is_empty())
{
return Err(std::io::Error::other("ERR_XML_NOT_WELL_FORMED"));
}
if self.noncurrent_days.is_none() {
return Err(std::io::Error::other(ERR_TRANSITION_INVALID));
}
if self.noncurrent_days.is_some_and(|days| days < 0)
|| self.newer_noncurrent_versions.is_some_and(|versions| versions < 0)
{
return Err(std::io::Error::other(ERR_TRANSITION_INVALID));
}
Ok(())
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn lifecycle_rule_filter_matches_single_tag() {
let filter = LifecycleRuleFilter {
tag: Some(Tag {
key: Some("env".to_string()),
value: Some("prod".to_string()),
}),
..Default::default()
};
assert!(<LifecycleRuleFilter as Filter>::test_tags(&filter, "env=prod&team=storage"));
assert!(!<LifecycleRuleFilter as Filter>::test_tags(&filter, "env=dev&team=storage"));
assert!(!<LifecycleRuleFilter as Filter>::test_tags(&filter, "team=storage"));
}
#[test]
fn lifecycle_rule_filter_matches_all_and_tags() {
let filter = LifecycleRuleFilter {
and: Some(LifecycleRuleAndOperator {
tags: Some(vec![
Tag {
key: Some("env".to_string()),
value: Some("prod".to_string()),
},
Tag {
key: Some("team".to_string()),
value: Some("storage".to_string()),
},
]),
..Default::default()
}),
..Default::default()
};
assert!(<LifecycleRuleFilter as Filter>::test_tags(&filter, "env=prod&team=storage"));
assert!(!<LifecycleRuleFilter as Filter>::test_tags(&filter, "env=prod&team=platform"));
}
#[test]
fn lifecycle_rule_filter_respects_size_bounds() {
let filter = LifecycleRuleFilter {
object_size_greater_than: Some(5),
object_size_less_than: Some(10),
..Default::default()
};
assert!(!filter.by_size(5));
assert!(filter.by_size(6));
assert!(!filter.by_size(10));
}
#[test]
fn lifecycle_rule_filter_without_tag_constraints_accepts_any_tags() {
let filter = LifecycleRuleFilter {
object_size_greater_than: Some(5),
..Default::default()
};
assert!(<LifecycleRuleFilter as Filter>::test_tags(&filter, "env=prod&team=storage"));
assert!(<LifecycleRuleFilter as Filter>::test_tags(&filter, ""));
}
}
pub use rustfs_lifecycle::rule::{Filter, NoncurrentVersionTransitionOps, TransitionOps};
crates/lifecycle/Cargo.toml
+49
View File
@@ -0,0 +1,49 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[package]
name = "rustfs-lifecycle"
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
version.workspace = true
homepage.workspace = true
description = "Lifecycle rule evaluation contracts for RustFS."
keywords = ["lifecycle", "storage", "rustfs", "Minio"]
categories = ["web-programming", "development-tools", "filesystem"]
documentation = "https://docs.rs/rustfs-lifecycle/latest/rustfs_lifecycle/"
[dependencies]
async-trait.workspace = true
rustfs-common.workspace = true
rustfs-config = { workspace = true, features = ["constants"] }
rustfs-replication.workspace = true
rustfs-storage-api.workspace = true
s3s.workspace = true
time.workspace = true
tracing.workspace = true
url.workspace = true
uuid = { workspace = true, features = ["v4", "serde"] }
[dev-dependencies]
serial_test.workspace = true
temp-env.workspace = true
tokio = { workspace = true, features = ["macros", "rt"] }
[lints]
workspace = true
[lib]
doctest = false
crates/lifecycle/src/core.rs
+2951
View File
File diff suppressed because it is too large Load Diff
crates/lifecycle/src/evaluator.rs
+448
View File
@@ -0,0 +1,448 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, ObjectLockEnabled};
use time::OffsetDateTime;
use tracing::info;
use rustfs_common::metrics::IlmAction;
use rustfs_replication::ReplicationStatusType;
use crate::object_lock;
use crate::{Event, Lifecycle, ObjectOpts};
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
const EVENT_LIFECYCLE_VERSION_SCAN_SKIPPED: &str = "lifecycle_version_scan_skipped";
/// Evaluator - evaluates lifecycle policy on objects for the given lifecycle
/// configuration and lock retention configuration.
pub struct Evaluator {
policy: Arc<BucketLifecycleConfiguration>,
lock_retention: Option<Arc<ObjectLockConfiguration>>,
}
impl Evaluator {
/// NewEvaluator - creates a new evaluator with the given lifecycle
pub fn new(policy: Arc<BucketLifecycleConfiguration>) -> Self {
Self {
policy,
lock_retention: None,
}
}
/// WithLockRetention - sets the lock retention configuration for the evaluator
pub fn with_lock_retention(mut self, lr: Option<Arc<ObjectLockConfiguration>>) -> Self {
self.lock_retention = lr;
self
}
/// WithReplicationConfig is retained for caller compatibility.
/// Lifecycle replication guards are evaluated from per-object replication state.
pub fn with_replication_config<T>(self, _rcfg: Option<Arc<T>>) -> Self {
self
}
/// IsPendingReplication checks if the object is pending replication.
pub fn is_pending_replication(&self, obj: &ObjectOpts) -> bool {
has_pending_lifecycle_replication(obj)
}
fn any_version_has_pending_replication(&self, objs: &[ObjectOpts]) -> bool {
objs.iter().any(|obj| self.is_pending_replication(obj))
}
/// IsObjectLocked checks if it is appropriate to remove an
/// object according to locking configuration when this is lifecycle/bucket quota asking.
/// Uses the common `is_object_locked_by_metadata` function for consistency.
pub fn is_object_locked(&self, obj: &ObjectOpts) -> bool {
// First check if object lock is enabled for this bucket
if self.lock_retention.as_ref().is_none_or(|v| {
v.object_lock_enabled
.as_ref()
.is_none_or(|v| v.as_str() != ObjectLockEnabled::ENABLED)
}) {
return false;
}
// Use the common function to check if the object is locked
object_lock::is_object_locked_by_metadata(&obj.user_defined, obj.delete_marker)
}
/// eval will return a lifecycle event for each object in objs for a given time.
async fn eval_inner(&self, objs: &[ObjectOpts], now: OffsetDateTime) -> Vec<Event> {
let mut events = vec![Event::default(); objs.len()];
let mut newer_noncurrent_versions = 0;
'top_loop: {
for (i, obj) in objs.iter().enumerate() {
let mut event = self.policy.eval_inner(obj, now, newer_noncurrent_versions).await;
if lifecycle_action_waits_for_replication(event.action) && self.is_pending_replication(obj) {
event = Event::default();
}
match event.action {
IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction => {
// Skip if bucket has object locking enabled; To prevent the
// possibility of violating an object retention on one of the
// noncurrent versions of this object.
if self.lock_retention.as_ref().is_some_and(|v| {
v.object_lock_enabled
.as_ref()
.is_some_and(|v| v.as_str() == ObjectLockEnabled::ENABLED)
}) || self.any_version_has_pending_replication(objs)
{
event = Event::default();
} else {
// No need to evaluate remaining versions' lifecycle
// events after DeleteAllVersionsAction*
events[i] = event;
info!(
event = EVENT_LIFECYCLE_VERSION_SCAN_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
reason = "delete_all_versions_action",
action = ?events[i].action,
"Skipped remaining lifecycle version scan"
);
break 'top_loop;
}
}
IlmAction::DeleteVersionAction | IlmAction::DeleteRestoredVersionAction => {
// Defensive code, should never happen
if obj.version_id.is_none_or(|v| v.is_nil()) {
event.action = IlmAction::NoneAction;
}
if self.is_object_locked(obj) {
event = Event::default();
}
}
_ => {}
}
if !obj.is_latest {
match event.action {
IlmAction::DeleteVersionAction => {
// this noncurrent version will be expired, nothing to add
}
_ => {
// this noncurrent version will be spared
newer_noncurrent_versions += 1;
}
}
}
events[i] = event;
}
}
events
}
/// Eval will return a lifecycle event for each object in objs
pub async fn eval(&self, objs: &[ObjectOpts]) -> Result<Vec<Event>, std::io::Error> {
if objs.is_empty() {
return Ok(vec![]);
}
if objs.len() != objs[0].num_versions {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
format!("number of versions mismatch, expected {}, got {}", objs[0].num_versions, objs.len()),
));
}
Ok(self.eval_inner(objs, OffsetDateTime::now_utc()).await)
}
}
fn has_pending_version_purge(obj: &ObjectOpts) -> bool {
obj.version_purge_status.is_pending()
}
fn has_pending_object_replication(obj: &ObjectOpts) -> bool {
replication_status_blocks_lifecycle(&obj.replication_status)
}
fn has_pending_lifecycle_replication(obj: &ObjectOpts) -> bool {
has_pending_object_replication(obj) || has_pending_version_purge(obj)
}
fn replication_status_blocks_lifecycle(status: &ReplicationStatusType) -> bool {
matches!(status, ReplicationStatusType::Pending | ReplicationStatusType::Failed)
}
fn lifecycle_action_waits_for_replication(action: IlmAction) -> bool {
matches!(
action,
IlmAction::DeleteAction
| IlmAction::DeleteVersionAction
| IlmAction::DeleteRestoredAction
| IlmAction::DeleteRestoredVersionAction
| IlmAction::DeleteAllVersionsAction
| IlmAction::DelMarkerDeleteAllVersionsAction
| IlmAction::TransitionAction
| IlmAction::TransitionVersionAction
)
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use rustfs_common::metrics::IlmAction;
use s3s::dto::{
BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, Transition, TransitionStorageClass,
};
use time::OffsetDateTime;
use uuid::Uuid;
use super::*;
use rustfs_replication::{ReplicationStatusType, VersionPurgeStatusType};
fn expired_marker_lifecycle() -> Arc<BucketLifecycleConfiguration> {
Arc::new(BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
expired_object_delete_marker: Some(true),
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("expired-marker".to_string()),
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
}],
})
}
fn latest_expiration_lifecycle() -> Arc<BucketLifecycleConfiguration> {
Arc::new(BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
days: Some(1),
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("expire-current".to_string()),
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
}],
})
}
fn latest_transition_lifecycle() -> Arc<BucketLifecycleConfiguration> {
Arc::new(BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: None,
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("transition-current".to_string()),
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
transitions: Some(vec![Transition {
days: Some(1),
date: None,
storage_class: Some(TransitionStorageClass::from_static("WARM")),
}]),
}],
})
}
fn all_versions_expiration_lifecycle() -> Arc<BucketLifecycleConfiguration> {
Arc::new(BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
days: Some(1),
expired_object_all_versions: Some(true),
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("delete-all".to_string()),
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
}],
})
}
fn object_opts(replication_status: ReplicationStatusType, version_purge_status: VersionPurgeStatusType) -> ObjectOpts {
ObjectOpts {
name: "logs/object".to_string(),
mod_time: Some(OffsetDateTime::from_unix_timestamp(1_000_000).expect("valid fixed test timestamp")),
version_id: Some(Uuid::new_v4()),
is_latest: true,
delete_marker: true,
num_versions: 1,
replication_status,
version_purge_status,
..Default::default()
}
}
fn current_object_opts(replication_status: ReplicationStatusType) -> ObjectOpts {
ObjectOpts {
name: "logs/object".to_string(),
mod_time: Some(OffsetDateTime::from_unix_timestamp(1_000_000).expect("valid fixed test timestamp")),
version_id: Some(Uuid::new_v4()),
is_latest: true,
num_versions: 1,
replication_status,
..Default::default()
}
}
fn versioned_object_opts(replication_status: ReplicationStatusType, is_latest: bool) -> ObjectOpts {
ObjectOpts {
num_versions: 2,
is_latest,
..current_object_opts(replication_status)
}
}
#[tokio::test]
async fn evaluator_allows_expired_delete_marker_after_replication_completed() {
let evaluator = Evaluator::new(expired_marker_lifecycle());
let events = evaluator
.eval(&[object_opts(
ReplicationStatusType::Completed,
VersionPurgeStatusType::Complete,
)])
.await
.expect("completed replication should allow lifecycle evaluation");
assert_eq!(events[0].action, IlmAction::DeleteVersionAction);
}
#[tokio::test]
async fn evaluator_skips_expired_delete_marker_while_replication_pending() {
let evaluator = Evaluator::new(expired_marker_lifecycle());
let events = evaluator
.eval(&[object_opts(ReplicationStatusType::Pending, VersionPurgeStatusType::default())])
.await
.expect("pending replication should still return a lifecycle decision");
assert_eq!(events[0].action, IlmAction::NoneAction);
}
#[tokio::test]
async fn evaluator_skips_expired_delete_marker_while_version_purge_pending() {
let evaluator = Evaluator::new(expired_marker_lifecycle());
let events = evaluator
.eval(&[object_opts(ReplicationStatusType::Completed, VersionPurgeStatusType::Pending)])
.await
.expect("pending version purge should still return a lifecycle decision");
assert_eq!(events[0].action, IlmAction::NoneAction);
}
#[tokio::test]
async fn evaluator_skips_latest_expiration_while_replication_failed() {
let evaluator = Evaluator::new(latest_expiration_lifecycle());
let events = evaluator
.eval(&[current_object_opts(ReplicationStatusType::Failed)])
.await
.expect("failed replication should still return a lifecycle decision");
assert_eq!(events[0].action, IlmAction::NoneAction);
}
#[tokio::test]
async fn evaluator_allows_latest_expiration_after_replication_completed() {
let evaluator = Evaluator::new(latest_expiration_lifecycle());
let events = evaluator
.eval(&[current_object_opts(ReplicationStatusType::Completed)])
.await
.expect("completed replication should allow latest expiration");
assert_eq!(events[0].action, IlmAction::DeleteAction);
}
#[tokio::test]
async fn evaluator_skips_transition_while_replication_pending() {
let evaluator = Evaluator::new(latest_transition_lifecycle());
let events = evaluator
.eval(&[current_object_opts(ReplicationStatusType::Pending)])
.await
.expect("pending replication should still return a lifecycle decision");
assert_eq!(events[0].action, IlmAction::NoneAction);
}
#[tokio::test]
async fn evaluator_allows_transition_after_replication_completed() {
let evaluator = Evaluator::new(latest_transition_lifecycle());
let events = evaluator
.eval(&[current_object_opts(ReplicationStatusType::Completed)])
.await
.expect("completed replication should allow transition");
assert_eq!(events[0].action, IlmAction::TransitionAction);
}
#[tokio::test]
async fn evaluator_skips_delete_all_versions_when_any_version_replication_pending() {
let evaluator = Evaluator::new(all_versions_expiration_lifecycle());
let latest = versioned_object_opts(ReplicationStatusType::Completed, true);
let noncurrent = versioned_object_opts(ReplicationStatusType::Pending, false);
let events = evaluator
.eval(&[latest, noncurrent])
.await
.expect("pending noncurrent replication should still return lifecycle decisions");
assert_eq!(events[0].action, IlmAction::NoneAction);
assert_eq!(events[1].action, IlmAction::NoneAction);
}
#[tokio::test]
async fn evaluator_allows_delete_all_versions_when_all_versions_replication_completed() {
let evaluator = Evaluator::new(all_versions_expiration_lifecycle());
let latest = versioned_object_opts(ReplicationStatusType::Completed, true);
let noncurrent = versioned_object_opts(ReplicationStatusType::Completed, false);
let events = evaluator
.eval(&[latest, noncurrent])
.await
.expect("completed replication should allow delete-all lifecycle decision");
assert_eq!(events[0].action, IlmAction::DeleteAllVersionsAction);
assert_eq!(events[1].action, IlmAction::NoneAction);
}
}
crates/lifecycle/src/lib.rs
+24
View File
@@ -0,0 +1,24 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub mod core;
pub mod evaluator;
pub mod object_lock;
pub mod rule;
mod tagging;
pub use core::*;
pub use evaluator::Evaluator;
pub use rustfs_common::metrics::IlmAction;
pub use rustfs_replication::{ReplicationStatusType, VersionPurgeStatusType};
crates/lifecycle/src/object_lock.rs
+63
View File
@@ -0,0 +1,63 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::HashMap;
use s3s::dto::ObjectLockRetentionMode;
use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
use time::{OffsetDateTime, format_description};
pub fn is_object_locked_by_metadata(user_defined: &HashMap<String, String>, is_delete_marker: bool) -> bool {
if is_delete_marker {
return false;
}
if user_defined
.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str())
.is_some_and(|value| value.eq_ignore_ascii_case("ON"))
{
return true;
}
let Some(mode) = user_defined.get(X_AMZ_OBJECT_LOCK_MODE.as_str()) else {
return false;
};
if !is_retention_mode(mode) {
return false;
}
user_defined
.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str())
.and_then(|value| OffsetDateTime::parse(value, &format_description::well_known::Iso8601::DEFAULT).ok())
.is_some_and(|retain_until| retain_until.unix_timestamp() > OffsetDateTime::now_utc().unix_timestamp())
}
fn is_retention_mode(mode: &str) -> bool {
mode.eq_ignore_ascii_case(ObjectLockRetentionMode::COMPLIANCE)
|| mode.eq_ignore_ascii_case(ObjectLockRetentionMode::GOVERNANCE)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn is_object_locked_by_metadata_preserves_object_lock_parser_behavior() {
let mut user_defined = HashMap::new();
user_defined.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), "ON".to_string());
assert!(is_object_locked_by_metadata(&user_defined, false));
assert!(!is_object_locked_by_metadata(&user_defined, true));
}
}
crates/lifecycle/src/rule.rs
+204
View File
@@ -0,0 +1,204 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use s3s::dto::{LifecycleRuleAndOperator, LifecycleRuleFilter, NoncurrentVersionTransition, Tag, Transition};
use time::OffsetDateTime;
use crate::tagging;
const _ERR_TRANSITION_INVALID_DAYS: &str = "Days must be 0 or greater when used with Transition";
const _ERR_TRANSITION_INVALID_DATE: &str = "Date must be provided in ISO 8601 format";
const ERR_TRANSITION_INVALID: &str =
"Exactly one of Days (0 or greater) or Date (positive ISO 8601 format) should be present in Transition.";
const _ERR_TRANSITION_DATE_NOT_MIDNIGHT: &str = "'Date' must be at midnight GMT";
pub trait Filter {
fn test_tags(&self, user_tags: &str) -> bool;
fn by_size(&self, sz: i64) -> bool;
}
impl Filter for LifecycleRuleFilter {
fn test_tags(&self, user_tags: &str) -> bool {
if !requires_tag_matching(self) {
return true;
}
let user_tags = tagging::decode_tags_to_map(user_tags);
self.tag.as_ref().is_none_or(|tag| tag_matches(tag, &user_tags))
&& self.and.as_ref().is_none_or(|and| and_tags_match(and, &user_tags))
}
fn by_size(&self, sz: i64) -> bool {
let sz = sz.max(0);
self.object_size_greater_than.is_none_or(|min| sz > min)
&& self.object_size_less_than.is_none_or(|max| sz < max)
&& self.and.as_ref().is_none_or(|and| and_size_matches(and, sz))
}
}
fn requires_tag_matching(filter: &LifecycleRuleFilter) -> bool {
filter.tag.is_some()
|| filter
.and
.as_ref()
.and_then(|and| and.tags.as_ref())
.is_some_and(|tags| !tags.is_empty())
}
fn tag_matches(tag: &Tag, user_tags: &std::collections::HashMap<String, String>) -> bool {
let Some(key) = tag.key.as_deref() else {
return false;
};
let Some(value) = tag.value.as_deref() else {
return false;
};
user_tags.get(key).is_some_and(|actual| actual == value)
}
fn and_tags_match(and: &LifecycleRuleAndOperator, user_tags: &std::collections::HashMap<String, String>) -> bool {
and.tags
.as_ref()
.is_none_or(|tags| tags.iter().all(|tag| tag_matches(tag, user_tags)))
}
fn and_size_matches(and: &LifecycleRuleAndOperator, sz: i64) -> bool {
and.object_size_greater_than.is_none_or(|min| sz > min) && and.object_size_less_than.is_none_or(|max| sz < max)
}
pub trait TransitionOps {
fn validate(&self) -> Result<(), std::io::Error>;
}
pub trait NoncurrentVersionTransitionOps {
fn validate(&self) -> Result<(), std::io::Error>;
}
fn is_midnight(date: OffsetDateTime) -> bool {
date.hour() == 0 && date.minute() == 0 && date.second() == 0 && date.nanosecond() == 0
}
impl TransitionOps for Transition {
fn validate(&self) -> Result<(), std::io::Error> {
if self
.storage_class
.as_ref()
.is_none_or(|storage_class| storage_class.as_str().is_empty())
{
return Err(std::io::Error::other("ERR_XML_NOT_WELL_FORMED"));
}
if self.date.is_some() == self.days.is_some() {
return Err(std::io::Error::other(ERR_TRANSITION_INVALID));
}
if self.days.is_some_and(|days| days < 0) {
return Err(std::io::Error::other(ERR_TRANSITION_INVALID));
}
if let Some(date) = self.date.clone() {
let date = OffsetDateTime::from(date);
if date.unix_timestamp() <= 0 || !is_midnight(date) {
return Err(std::io::Error::other(ERR_TRANSITION_INVALID));
}
}
Ok(())
}
}
impl NoncurrentVersionTransitionOps for NoncurrentVersionTransition {
fn validate(&self) -> Result<(), std::io::Error> {
if self
.storage_class
.as_ref()
.is_none_or(|storage_class| storage_class.as_str().is_empty())
{
return Err(std::io::Error::other("ERR_XML_NOT_WELL_FORMED"));
}
if self.noncurrent_days.is_none() {
return Err(std::io::Error::other(ERR_TRANSITION_INVALID));
}
if self.noncurrent_days.is_some_and(|days| days < 0)
|| self.newer_noncurrent_versions.is_some_and(|versions| versions < 0)
{
return Err(std::io::Error::other(ERR_TRANSITION_INVALID));
}
Ok(())
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn lifecycle_rule_filter_matches_single_tag() {
let filter = LifecycleRuleFilter {
tag: Some(Tag {
key: Some("env".to_string()),
value: Some("prod".to_string()),
}),
..Default::default()
};
assert!(<LifecycleRuleFilter as Filter>::test_tags(&filter, "env=prod&team=storage"));
assert!(!<LifecycleRuleFilter as Filter>::test_tags(&filter, "env=dev&team=storage"));
assert!(!<LifecycleRuleFilter as Filter>::test_tags(&filter, "team=storage"));
}
#[test]
fn lifecycle_rule_filter_matches_all_and_tags() {
let filter = LifecycleRuleFilter {
and: Some(LifecycleRuleAndOperator {
tags: Some(vec![
Tag {
key: Some("env".to_string()),
value: Some("prod".to_string()),
},
Tag {
key: Some("team".to_string()),
value: Some("storage".to_string()),
},
]),
..Default::default()
}),
..Default::default()
};
assert!(<LifecycleRuleFilter as Filter>::test_tags(&filter, "env=prod&team=storage"));
assert!(!<LifecycleRuleFilter as Filter>::test_tags(&filter, "env=prod&team=platform"));
}
#[test]
fn lifecycle_rule_filter_respects_size_bounds() {
let filter = LifecycleRuleFilter {
object_size_greater_than: Some(5),
object_size_less_than: Some(10),
..Default::default()
};
assert!(!filter.by_size(5));
assert!(filter.by_size(6));
assert!(!filter.by_size(10));
}
#[test]
fn lifecycle_rule_filter_without_tag_constraints_accepts_any_tags() {
let filter = LifecycleRuleFilter {
object_size_greater_than: Some(5),
..Default::default()
};
assert!(<LifecycleRuleFilter as Filter>::test_tags(&filter, "env=prod&team=storage"));
assert!(<LifecycleRuleFilter as Filter>::test_tags(&filter, ""));
}
}
crates/lifecycle/src/tagging.rs
+45
View File
@@ -0,0 +1,45 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::HashMap;
use url::form_urlencoded;
pub(crate) fn decode_tags_to_map(tags: &str) -> HashMap<String, String> {
let mut list = HashMap::new();
for (k, v) in form_urlencoded::parse(tags.as_bytes()) {
if k.is_empty() {
continue;
}
list.insert(k.to_string(), v.to_string());
}
list
}
#[cfg(test)]
mod tests {
use super::decode_tags_to_map;
#[test]
fn decode_tags_to_map_preserves_bucket_tagging_parser_behavior() {
let tags = decode_tags_to_map("env=prod&encoded=a%2Fb&=ignored");
assert_eq!(tags.get("env").map(String::as_str), Some("prod"));
assert_eq!(tags.get("encoded").map(String::as_str), Some("a/b"));
assert!(!tags.contains_key(""));
}
}
crates/scanner/src/lib.rs
+4 -3
View File
@@ -35,9 +35,10 @@ use storage_api::owner::{
ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule, ecstore_apply_transition_rule,
ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config,
ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_is_erasure, ecstore_is_erasure_sd,
ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_path2_bucket_object,
ecstore_path2_bucket_object_with_base_path, ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info,
ecstore_resolve_object_store_handle, ecstore_save_config, scanner_replication_config_for_lifecycle_eval,
ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info,
ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config,
ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, ecstore_save_config,
scanner_replication_config_for_lifecycle_eval,
};
#[cfg(test)]
use storage_api::owner::{EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, ecstore_config_init, ecstore_new_disk};
crates/scanner/src/scanner_folder.rs
+1 -1
View File
@@ -795,7 +795,7 @@ impl ScannerItem {
let object_opts = object_infos
.iter()
.map(ObjectOpts::from_object_info)
.map(crate::ecstore_object_opts_from_object_info)
.collect::<Vec<ObjectOpts>>();
let events = match Evaluator::new(lifecycle.clone())
crates/scanner/src/storage_api.rs
+4 -4
View File
@@ -25,7 +25,7 @@ pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::{
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::evaluator::Evaluator as EcstoreEvaluator;
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::lifecycle::{
Event as EcstoreEvent, Lifecycle as EcstoreLifecycle, ObjectOpts as EcstoreObjectOpts,
TRANSITION_COMPLETE as ECSTORE_TRANSITION_COMPLETE,
TRANSITION_COMPLETE as ECSTORE_TRANSITION_COMPLETE, object_opts_from_object_info as ecstore_object_opts_from_object_info,
};
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
get_lifecycle_config as ecstore_get_lifecycle_config, get_object_lock_config as ecstore_get_object_lock_config,
@@ -96,9 +96,9 @@ pub(crate) mod owner {
ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule, ecstore_apply_transition_rule, ecstore_expiry_state_handle,
ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config, ecstore_get_object_lock_config,
ecstore_get_replication_config, ecstore_is_erasure, ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket,
ecstore_list_path_raw, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config,
ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, ecstore_save_config,
scanner_replication_config_for_lifecycle_eval,
ecstore_list_path_raw, ecstore_object_opts_from_object_info, ecstore_path2_bucket_object,
ecstore_path2_bucket_object_with_base_path, ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info,
ecstore_resolve_object_store_handle, ecstore_save_config, scanner_replication_config_for_lifecycle_eval,
};
#[cfg(test)]
docs/architecture/ecstore-module-split-plan.md
+28 -13
View File
@@ -1,15 +1,15 @@
# ECStore Module Split Plan
This plan records the remaining ECStore split work after the final audit
remediation pass. It is intentionally design-only: no crate movement should
start until each candidate boundary has explicit contracts, compatibility
coverage, dependency evidence, and rollback steps.
remediation pass. Runtime movement must still wait until each candidate
boundary has explicit contracts, compatibility coverage, dependency evidence,
and rollback steps.
## Current Shape
| Area | Current owner | Size | Split status |
|---|---|---:|---|
| Bucket lifecycle | `crates/ecstore/src/bucket/lifecycle/` | 8,657 lines | Proposal only |
| Bucket lifecycle | `crates/lifecycle/` + `crates/ecstore/src/bucket/lifecycle/` | core contracts + ECStore runtime | Core contract extracted |
| Bucket replication | `crates/ecstore/src/bucket/replication/` | 8,730 lines | Proposal only |
| Set disks | `crates/ecstore/src/set_disk/` | state carrier plus operation modules | Keep in ECStore |
| Public ECStore facade | `crates/ecstore/src/api/mod.rs` | broad compatibility surface | Shrink only through guarded PRs |
@@ -50,7 +50,10 @@ not a runtime split PR.
## Lifecycle Candidate
`bucket/lifecycle` is not ready for a standalone crate yet.
`rustfs-lifecycle` now owns the pure lifecycle rule, event, evaluator, tag
filtering, object-lock metadata check, and expiry-time contracts. ECStore keeps
the object-store runtime, queues, tiering, audit/notification, metadata, and
replication scheduling adapters.
Current coupling:
@@ -61,9 +64,20 @@ Current coupling:
through the lifecycle metadata boundary;
- lifecycle expiry schedules bucket replication delete work through the
replication lifecycle bridge contract;
- lifecycle evaluation shares S3 DTOs, object metadata, object lock, replication
config reads through lifecycle-local boundaries, scanner metrics,
notification/audit side effects, and tier services.
- lifecycle evaluation uses S3 DTOs and replication status contracts from the
independent `rustfs-lifecycle`/`rustfs-replication` crates, while ECStore maps
`ObjectInfo` into lifecycle object options at the compatibility boundary;
- lifecycle runtime still coordinates scanner metrics, notification/audit side
effects, metadata access, replication delete scheduling, and tier services.
Current extracted contracts:
- `LifecycleCrateCoreIndependence`: lifecycle rule validation, filtering,
event evaluation, transition/expiration options, tag decoding, object-lock
metadata checks, and ILM expiry-time rounding live in `rustfs-lifecycle`.
`rustfs-lifecycle` must not import ECStore internals, file metadata, or
`rustfs-utils`; ECStore owns the `ObjectInfo` adapter in
`crates/ecstore/src/bucket/lifecycle/core.rs`.
Required contracts before crate movement:
@@ -79,12 +93,13 @@ Required contracts before crate movement:
without depending on the replication implementation module.
- `LifecycleAuditSink`: lifecycle audit and notification emission boundary.
First safe PR:
Next safe PR:
- add a lifecycle extraction inventory section or module-level README;
- list current ECStore/runtime dependencies and the target contract owner for
each dependency;
- add no code movement and no behavior changes.
- move one runtime-facing dependency behind a trait or adapter owned by
`rustfs-lifecycle` without changing queue, transition, or delete behavior;
- keep ECStore compatibility shims until scanner and RustFS app consumers stop
depending on `rustfs_ecstore::api::bucket::lifecycle` paths;
- add focused tests for the moved contract and keep architecture guard coverage.
The module-level inventory lives in
`crates/ecstore/src/bucket/lifecycle/README.md`.
rustfs/src/app/object_usecase.rs
+1 -1
View File
@@ -2017,7 +2017,7 @@ async fn resolve_put_object_expiration(bucket: &str, obj_info: &ObjectInfo) -> O
return None;
};
let obj_opts = lifecycle::ObjectOpts::from_object_info(obj_info);
let obj_opts = lifecycle::object_opts_from_object_info(obj_info);
let event = predict_lifecycle_expiration(&lifecycle_config, &obj_opts).await;
debug!(
bucket,
rustfs/src/app/storage_api.rs
+3
View File
@@ -374,6 +374,9 @@ pub(crate) mod bucket {
#[cfg(test)]
pub(crate) const TRANSITION_PENDING: &str =
crate::storage::storage_api::ecstore_bucket::lifecycle::lifecycle::TRANSITION_PENDING;
pub(crate) fn object_opts_from_object_info(obj_info: &crate::storage::storage_api::StorageObjectInfo) -> ObjectOpts {
crate::storage::storage_api::ecstore_bucket::lifecycle::lifecycle::object_opts_from_object_info(obj_info)
}
pub(crate) fn expected_expiry_time(mod_time: time::OffsetDateTime, days: i32) -> time::OffsetDateTime {
crate::storage::storage_api::ecstore_bucket::lifecycle::lifecycle::expected_expiry_time(mod_time, days)
}
scripts/check_architecture_migration_rules.sh
+23 -3
View File
@@ -89,6 +89,7 @@ require_source_contains "docs/architecture/ecstore-module-split-plan.md" "Runtim
require_source_contains "docs/architecture/ecstore-module-split-plan.md" "ReplicationCrateFileMetaIndependence" "ECStore split plan replication crate filemeta independence section"
require_source_contains "docs/architecture/ecstore-module-split-plan.md" "ReplicationCrateStorageApiIndependence" "ECStore split plan replication crate storage-api independence section"
require_source_contains "docs/architecture/ecstore-module-split-plan.md" "ReplicationCrateUtilsIndependence" "ECStore split plan replication crate utils independence section"
require_source_contains "docs/architecture/ecstore-module-split-plan.md" "LifecycleCrateCoreIndependence" "ECStore split plan lifecycle crate core independence section"
require_source_contains "docs/architecture/ecstore-module-split-plan.md" "StorageApiReplicationContracts" "ECStore split plan storage-api replication contract section"
require_source_contains "docs/architecture/ecstore-api-facade-inventory.md" "## Facade Group Inventory" "ECStore facade inventory group section"
require_source_contains "docs/architecture/ecstore-api-facade-inventory.md" "## External Consumer Boundaries" "ECStore facade inventory consumer boundary section"
@@ -143,6 +144,7 @@ LIFECYCLE_TAGGING_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/lifecycle_tagging_bounda
LIFECYCLE_OBJECT_LOCK_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/lifecycle_object_lock_boundary_bypass_hits.txt"
LIFECYCLE_REPLICATION_SINK_BYPASS_HITS_FILE="${TMP_DIR}/lifecycle_replication_sink_bypass_hits.txt"
LIFECYCLE_REPLICATION_CRATE_BYPASS_HITS_FILE="${TMP_DIR}/lifecycle_replication_crate_bypass_hits.txt"
LIFECYCLE_CRATE_CORE_BYPASS_HITS_FILE="${TMP_DIR}/lifecycle_crate_core_bypass_hits.txt"
ECSTORE_REPLICATION_CONTRACT_BYPASS_HITS_FILE="${TMP_DIR}/ecstore_replication_contract_bypass_hits.txt"
STORAGE_API_REPLICATION_CONTRACT_BYPASS_HITS_FILE="${TMP_DIR}/storage_api_replication_contract_bypass_hits.txt"
STORE_API_EXTERNAL_LIST_CONSUMER_HITS_FILE="${TMP_DIR}/store_api_external_list_consumer_hits.txt"
@@ -863,9 +865,9 @@ if [[ -s "$STORE_API_LIFECYCLE_HELPER_DEFINITION_HITS_FILE" ]]; then
fi
require_source_line \
"crates/ecstore/src/bucket/lifecycle/core.rs" \
"pub use crate::storage_api_contracts::lifecycle::ExpirationOptions;" \
"ECStore ExpirationOptions compatibility re-export"
"crates/lifecycle/src/core.rs" \
"pub use rustfs_storage_api::ExpirationOptions;" \
"rustfs-lifecycle ExpirationOptions storage-api re-export"
require_source_line \
"crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs" \
"pub use crate::storage_api_contracts::lifecycle::TransitionedObject;" \
@@ -969,6 +971,24 @@ if [[ -s "$LIFECYCLE_REPLICATION_CRATE_BYPASS_HITS_FILE" ]]; then
report_failure "lifecycle replication status/state contracts must stay behind lifecycle replication_sink: $(paste -sd '; ' "$LIFECYCLE_REPLICATION_CRATE_BYPASS_HITS_FILE")"
fi
if [[ -d "${ROOT_DIR}/crates/lifecycle" ]]; then
(
cd "$ROOT_DIR"
{
rg -n --with-filename 'rustfs_ecstore::|use\s+rustfs_ecstore\b|rustfs_filemeta::|use\s+rustfs_filemeta\b|rustfs_utils::|use\s+rustfs_utils\b' \
crates/lifecycle/Cargo.toml crates/lifecycle/src \
--glob '*.rs' || true
rg -n --with-filename 'crate::bucket::|crate::object_api|crate::storage_api_contracts' \
crates/lifecycle/src \
--glob '*.rs' || true
}
) >"$LIFECYCLE_CRATE_CORE_BYPASS_HITS_FILE"
if [[ -s "$LIFECYCLE_CRATE_CORE_BYPASS_HITS_FILE" ]]; then
report_failure "rustfs-lifecycle core contracts must not import ECStore/filemeta/utils internals: $(paste -sd '; ' "$LIFECYCLE_CRATE_CORE_BYPASS_HITS_FILE")"
fi
fi
(
cd "$ROOT_DIR"
{