gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-17 18:27:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(lifecycle): Implement object lifecycle management functionality (#358)

Browse Source 
* feat(lifecycle): Implement object lifecycle management functionality

Add a lifecycle module to automatically handle object expiration and transition during scanning
Modify the file metadata cache module to be publicly visible to support lifecycle operations
Adjust the scanning interval to a shorter time for testing lifecycle rules
Implement the parsing and execution logic for S3 lifecycle configurations
Add integration tests to verify the lifecycle expiration functionality
Update dependencies to support the new lifecycle features

Signed-off-by: junxiang Mu <1948535941@qq.com>

* fix cargo dependencies

Signed-off-by: junxiang Mu <1948535941@qq.com>

* fix fmt

Signed-off-by: junxiang Mu <1948535941@qq.com>

---------

Signed-off-by: junxiang Mu <1948535941@qq.com>
Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
guojidan
2025-08-07 19:51:02 -07:00
committed by GitHub
parent 48a9707110
commit d987686c14
12 changed files with 756 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ahm/src/scanner/data_scanner.rs
+29 -6
View File
@@ -30,15 +30,16 @@ use tracing::{debug, error, info, warn};
use super::metrics::{BucketMetrics, DiskMetrics, MetricsCollector, ScannerMetrics};
use crate::heal::HealManager;
use crate::scanner::lifecycle::ScannerItem;
use crate::{
HealRequest,
error::{Error, Result},
get_ahm_services_cancel_token,
};
use rustfs_common::{
data_usage::DataUsageInfo,
metrics::{Metric, Metrics, globalMetrics},
};
use rustfs_common::data_usage::DataUsageInfo;
use rustfs_common::metrics::{Metric, Metrics, globalMetrics};
use rustfs_ecstore::cmd::bucket_targets::VersioningConfig;
use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
@@ -290,7 +291,7 @@ impl Scanner {
/// Get global metrics from common crate
pub async fn get_global_metrics(&self) -> rustfs_madmin::metrics::ScannerMetrics {
globalMetrics.report().await
(*globalMetrics).report().await
}
/// Perform a single scan cycle
@@ -317,7 +318,7 @@ impl Scanner {
cycle_completed: vec![chrono::Utc::now()],
started: chrono::Utc::now(),
};
globalMetrics.set_cycle(Some(cycle_info)).await;
(*globalMetrics).set_cycle(Some(cycle_info)).await;
self.metrics.set_current_cycle(self.state.read().await.current_cycle);
self.metrics.increment_total_cycles();
@@ -1160,6 +1161,19 @@ impl Scanner {
/// This method collects all objects from a disk for a specific bucket.
/// It returns a map of object names to their metadata for later analysis.
async fn scan_volume(&self, disk: &DiskStore, bucket: &str) -> Result<HashMap<String, rustfs_filemeta::FileMeta>> {
let ecstore = match rustfs_ecstore::new_object_layer_fn() {
Some(ecstore) => ecstore,
None => {
error!("ECStore not available");
return Err(Error::Other("ECStore not available".to_string()));
}
};
let bucket_info = ecstore.get_bucket_info(bucket, &Default::default()).await.ok();
let versioning_config = bucket_info.map(|bi| Arc::new(VersioningConfig { enabled: bi.versioning }));
let lifecycle_config = rustfs_ecstore::bucket::metadata_sys::get_lifecycle_config(bucket)
.await
.ok()
.map(|(c, _)| Arc::new(c));
// Start global metrics collection for volume scan
let stop_fn = Metrics::time(Metric::ScanObject);
@@ -1247,6 +1261,15 @@ impl Scanner {
}
}
} else {
// Apply lifecycle actions
if let Some(lifecycle_config) = &lifecycle_config {
let mut scanner_item =
ScannerItem::new(bucket.to_string(), Some(lifecycle_config.clone()), versioning_config.clone());
if let Err(e) = scanner_item.apply_actions(&entry.name, entry.clone()).await {
error!("Failed to apply lifecycle actions for {}/{}: {}", bucket, entry.name, e);
}
}
// Store object metadata for later analysis
object_metadata.insert(entry.name.clone(), file_meta.clone());
}
crates/ahm/src/scanner/lifecycle.rs
+125
View File
@@ -0,0 +1,125 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use rustfs_common::metrics::IlmAction;
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{apply_lifecycle_action, eval_action_from_lifecycle};
use rustfs_ecstore::bucket::metadata_sys::get_object_lock_config;
use rustfs_ecstore::cmd::bucket_targets::VersioningConfig;
use rustfs_ecstore::store_api::ObjectInfo;
use rustfs_filemeta::FileMetaVersion;
use rustfs_filemeta::metacache::MetaCacheEntry;
use s3s::dto::BucketLifecycleConfiguration as LifecycleConfig;
use tracing::info;
#[derive(Clone)]
pub struct ScannerItem {
bucket: String,
lifecycle: Option<Arc<LifecycleConfig>>,
versioning: Option<Arc<VersioningConfig>>,
}
impl ScannerItem {
pub fn new(bucket: String, lifecycle: Option<Arc<LifecycleConfig>>, versioning: Option<Arc<VersioningConfig>>) -> Self {
Self {
bucket,
lifecycle,
versioning,
}
}
pub async fn apply_actions(&mut self, object: &str, mut meta: MetaCacheEntry) -> anyhow::Result<()> {
info!("apply_actions called for object: {}", object);
if self.lifecycle.is_none() {
info!("No lifecycle config for object: {}", object);
return Ok(());
}
info!("Lifecycle config exists for object: {}", object);
let file_meta = match meta.xl_meta() {
Ok(meta) => meta,
Err(e) => {
tracing::error!("Failed to get xl_meta for {}: {}", object, e);
return Ok(());
}
};
let latest_version = file_meta.versions.first().cloned().unwrap_or_default();
let file_meta_version = FileMetaVersion::try_from(latest_version.meta.as_slice()).unwrap_or_default();
let obj_info = ObjectInfo {
bucket: self.bucket.clone(),
name: object.to_string(),
version_id: latest_version.header.version_id,
mod_time: latest_version.header.mod_time,
size: file_meta_version.object.as_ref().map_or(0, |o| o.size),
user_defined: serde_json::from_slice(file_meta.data.as_slice()).unwrap_or_default(),
..Default::default()
};
self.apply_lifecycle(&obj_info).await;
Ok(())
}
async fn apply_lifecycle(&mut self, oi: &ObjectInfo) -> (IlmAction, i64) {
let size = oi.size;
if self.lifecycle.is_none() {
return (IlmAction::NoneAction, size);
}
let (olcfg, rcfg) = if self.bucket != ".minio.sys" {
(
get_object_lock_config(&self.bucket).await.ok(),
None, // FIXME: replication config
)
} else {
(None, None)
};
let lc_evt = eval_action_from_lifecycle(
self.lifecycle.as_ref().unwrap(),
olcfg
.as_ref()
.and_then(|(c, _)| c.rule.as_ref().and_then(|r| r.default_retention.clone())),
rcfg.clone(),
oi,
)
.await;
info!("lifecycle: {} Initial scan: {}", oi.name, lc_evt.action);
let mut new_size = size;
match lc_evt.action {
IlmAction::DeleteVersionAction | IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction => {
new_size = 0;
}
IlmAction::DeleteAction => {
if let Some(vcfg) = &self.versioning {
if !vcfg.is_enabled() {
new_size = 0;
}
} else {
new_size = 0;
}
}
_ => (),
}
apply_lifecycle_action(&lc_evt, &LcEventSrc::Scanner, oi).await;
(lc_evt.action, new_size)
}
}
crates/ahm/src/scanner/mod.rs
+1
View File
@@ -14,6 +14,7 @@
pub mod data_scanner;
pub mod histogram;
pub mod lifecycle;
pub mod metrics;
pub use data_scanner::Scanner;