feat(lifecycle): improve ILM compatibility and scanner runtime config (#2534)

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>
Co-authored-by: cxymds <Cxymds@qq.com>
This commit is contained in:
houseme
2026-04-16 18:28:03 +08:00
committed by GitHub
parent af93d2daba
commit 6ce24f3b63
26 changed files with 2672 additions and 527 deletions
Generated
+1
View File
@@ -8507,6 +8507,7 @@ dependencies = [
"serde",
"serde_json",
"serial_test",
"temp-env",
"thiserror 2.0.18",
"time",
"tokio",
+1 -1
View File
@@ -511,7 +511,7 @@ pub fn lc_has_active_rules(config: &BucketLifecycleConfiguration, prefix: &str)
}
if let Some(e) = &rule.noncurrent_version_expiration {
if let Some(true) = e.noncurrent_days.map(|d| d > 0) {
if e.noncurrent_days.is_some() {
return true;
}
if let Some(true) = e.newer_noncurrent_versions.map(|d| d > 0) {
+3
View File
@@ -53,8 +53,11 @@ Current guidance:
## Scanner environment aliases
- `RUSTFS_SCANNER_SPEED` (canonical, also accepts `MINIO_SCANNER_SPEED`)
- `RUSTFS_SCANNER_CYCLE` (canonical, also accepts `MINIO_SCANNER_CYCLE`)
- `RUSTFS_SCANNER_START_DELAY_SECS` (canonical)
- `RUSTFS_DATA_SCANNER_START_DELAY_SECS` (deprecated alias for compatibility)
- `RUSTFS_SCANNER_IDLE_MODE` (canonical)
## Drive timeout environment variables
+8
View File
@@ -25,6 +25,12 @@ pub const ENV_SCANNER_START_DELAY_SECS: &str = "RUSTFS_SCANNER_START_DELAY_SECS"
#[deprecated(note = "Use RUSTFS_SCANNER_START_DELAY_SECS instead")]
pub const ENV_DATA_SCANNER_START_DELAY_SECS: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS";
/// Environment variable that specifies the scanner cycle interval in seconds.
/// If set, this overrides the cycle interval derived from `RUSTFS_SCANNER_SPEED`.
/// - Unit: seconds (u64).
/// - Example: `export RUSTFS_SCANNER_CYCLE=3600` (1 hour)
pub const ENV_SCANNER_CYCLE: &str = "RUSTFS_SCANNER_CYCLE";
/// Environment variable that selects the scanner speed preset.
/// Valid values: `fastest`, `fast`, `default`, `slow`, `slowest`.
/// Controls the sleep factor, maximum sleep duration, and cycle interval.
@@ -65,6 +71,8 @@ pub const DEFAULT_SCANNER_INLINE_HEAL_ENABLE: bool = false;
/// | `default` | 2x | 1 second | 1 minute |
/// | `slow` | 10x | 15 seconds| 1 minute |
/// | `slowest` | 100x | 15 seconds| 30 minutes |
///
/// The cycle interval can be overridden by `RUSTFS_SCANNER_CYCLE`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum ScannerSpeed {
Fastest,
+3 -10
View File
@@ -160,7 +160,7 @@ async fn test_bucket_lifecycle_configuration() -> Result<(), Box<dyn std::error:
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_bucket_lifecycle_rejects_zero_days() -> Result<(), Box<dyn std::error::Error>> {
async fn test_bucket_lifecycle_accepts_zero_days() -> Result<(), Box<dyn std::error::Error>> {
use aws_sdk_s3::types::{BucketLifecycleConfiguration, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter};
let client = create_aws_s3_client().await?;
@@ -176,19 +176,12 @@ async fn test_bucket_lifecycle_rejects_zero_days() -> Result<(), Box<dyn std::er
.build()?;
let lifecycle = BucketLifecycleConfiguration::builder().rules(rule).build()?;
let err = client
client
.put_bucket_lifecycle_configuration()
.bucket(BUCKET)
.lifecycle_configuration(lifecycle)
.send()
.await
.expect_err("zero-day lifecycle expiration should be rejected");
let err_msg = format!("{err:?}");
assert!(
err_msg.contains("InvalidArgument") && err_msg.contains("greater than 0"),
"unexpected error: {err_msg}"
);
.await?;
Ok(())
}
+88 -27
View File
@@ -897,6 +897,41 @@ pub struct RemoveObjectOptions {
pub replication_validity_check: bool,
}
fn build_remove_object_headers(version_id: Option<&str>, opts: &RemoveObjectOptions) -> HeaderMap {
let mut headers = HeaderMap::new();
if opts.force_delete {
insert_header(&mut headers, SUFFIX_FORCE_DELETE, "true");
}
if opts.governance_bypass {
headers.insert(AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, "true".parse().unwrap());
}
if opts.replication_delete_marker {
insert_header(&mut headers, SUFFIX_SOURCE_DELETEMARKER, "true");
}
if let Some(t) = opts.replication_mtime {
insert_header(&mut headers, SUFFIX_SOURCE_MTIME, t.format(&Rfc3339).unwrap_or_default());
}
if !opts.replication_status.is_empty() {
headers.insert(AMZ_BUCKET_REPLICATION_STATUS, opts.replication_status.as_str().parse().unwrap());
}
if let Some(version_id) = version_id {
insert_header(&mut headers, SUFFIX_SOURCE_VERSION_ID, version_id);
}
if opts.replication_request {
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
}
if opts.replication_validity_check {
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true");
}
headers
}
#[derive(Debug, Clone)]
pub struct AdvancedPutOptions {
pub source_version_id: String,
@@ -1427,39 +1462,15 @@ impl TargetClient {
version_id: Option<String>,
opts: RemoveObjectOptions,
) -> Result<(), S3ClientError> {
let mut headers = HeaderMap::new();
if opts.force_delete {
insert_header(&mut headers, SUFFIX_FORCE_DELETE, "true");
}
if opts.governance_bypass {
headers.insert(AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, "true".parse().unwrap());
}
if opts.replication_delete_marker {
insert_header(&mut headers, SUFFIX_SOURCE_DELETEMARKER, "true");
}
if let Some(t) = opts.replication_mtime {
insert_header(&mut headers, SUFFIX_SOURCE_MTIME, t.format(&Rfc3339).unwrap_or_default());
}
if !opts.replication_status.is_empty() {
headers.insert(AMZ_BUCKET_REPLICATION_STATUS, opts.replication_status.as_str().parse().unwrap());
}
if opts.replication_request {
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
}
if opts.replication_validity_check {
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true");
}
let headers = build_remove_object_headers(version_id.as_deref(), &opts);
let api_version_id = if opts.replication_request { None } else { version_id };
match self
.client
.delete_object()
.bucket(bucket)
.key(object)
.set_version_id(version_id)
.set_version_id(api_version_id)
.customize()
.map_request(move |mut req| {
for (k, v) in headers.clone().into_iter() {
@@ -1553,3 +1564,53 @@ impl From<std::io::Error> for BucketTargetError {
}
impl Error for BucketTargetError {}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn build_remove_object_headers_includes_internal_version_id_for_replication_delete() {
let version_id = Uuid::new_v4().to_string();
let headers = build_remove_object_headers(
Some(version_id.as_str()),
&RemoveObjectOptions {
force_delete: false,
governance_bypass: false,
replication_delete_marker: true,
replication_mtime: None,
replication_status: ReplicationStatusType::Replica,
replication_request: true,
replication_validity_check: false,
},
);
assert_eq!(
rustfs_utils::http::get_header(&headers, SUFFIX_SOURCE_VERSION_ID).as_deref(),
Some(version_id.as_str()),
"replication delete requests must preserve the version id in internal headers"
);
}
#[test]
fn build_remove_object_headers_omits_delete_marker_flag_for_marker_version_purge() {
let version_id = Uuid::new_v4().to_string();
let headers = build_remove_object_headers(
Some(version_id.as_str()),
&RemoveObjectOptions {
force_delete: false,
governance_bypass: false,
replication_delete_marker: false,
replication_mtime: None,
replication_status: ReplicationStatusType::Replica,
replication_request: true,
replication_validity_check: false,
},
);
assert!(
rustfs_utils::http::get_header(&headers, SUFFIX_SOURCE_DELETEMARKER).is_none(),
"delete-marker version purges must not masquerade as delete-marker creations"
);
}
}
@@ -11,20 +11,18 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use crate::bucket::lifecycle::bucket_lifecycle_audit::{LcAuditEvent, LcEventSrc};
use crate::bucket::lifecycle::evaluator::Evaluator;
use crate::bucket::lifecycle::lifecycle::{
self, ExpirationOptions, Lifecycle, ObjectOpts, TransitionOptions, abort_incomplete_multipart_upload_due,
};
use crate::bucket::lifecycle::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats};
use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier};
use crate::bucket::object_lock::objectlock_sys::check_object_lock_for_deletion;
use crate::bucket::replication::{
DeletedObjectReplicationInfo, ReplicationConfig, check_replicate_delete, schedule_replication_delete,
};
use crate::bucket::{metadata_sys, metadata_sys::get_lifecycle_config, versioning_sys::BucketVersioningSys};
use crate::client::object_api_utils::new_getobjectreader;
use crate::disk::error::DiskError;
@@ -37,34 +35,33 @@ use crate::global::GLOBAL_LocalNodeName;
use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id};
use crate::set_disk::{MAX_PARTS_COUNT, RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, SetDisks};
use crate::store::ECStore;
use crate::store_api::StorageAPI;
use crate::store_api::{
GetObjectReader, HTTPRangeSpec, ListOperations, MultipartOperations, ObjectInfo, ObjectOperations, ObjectOptions,
ObjectToDelete,
};
use crate::tier::warm_backend::WarmBackendGetOpts;
use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
use bytes::BytesMut;
use futures::Future;
use http::HeaderMap;
use lazy_static::lazy_static;
use rustfs_common::data_usage::TierStats;
use rustfs_common::heal_channel::rep_has_active_rules;
use rustfs_common::metrics::{IlmAction, Metrics};
use rustfs_filemeta::{FileInfo, FileInfoOpts, NULL_VERSION_ID, RestoreStatusOps, get_file_info, is_restored_object_on_disk};
use rustfs_filemeta::{
FileInfo, FileInfoOpts, NULL_VERSION_ID, REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicationState, RestoreStatusOps,
VersionPurgeStatusType, get_file_info, is_restored_object_on_disk,
};
use rustfs_s3_common::EventName;
use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::strings_has_prefix_fold};
use s3s::Body;
use s3s::dto::{
BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration, RestoreRequest, RestoreRequestType, RestoreStatus,
ServerSideEncryption, Timestamp,
Timestamp,
};
use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION, X_AMZ_STORAGE_CLASS};
use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION};
use sha2::{Digest, Sha256};
use std::any::Any;
use std::collections::HashMap;
use std::env;
use std::io::Write;
use std::pin::Pin;
use std::sync::atomic::{AtomicI64, Ordering};
use std::sync::{Arc, Mutex, Weak};
@@ -73,7 +70,7 @@ use time::OffsetDateTime;
use tokio::select;
use tokio::sync::mpsc::{Receiver, Sender};
use tokio::sync::{RwLock, mpsc};
use tracing::{debug, error, info, warn};
use tracing::{error, info, warn};
use uuid::Uuid;
use xxhash_rust::xxh64;
@@ -111,7 +108,7 @@ impl LifecycleSys {
pub async fn get(&self, bucket: &str) -> Option<BucketLifecycleConfiguration> {
match get_lifecycle_config(bucket).await {
Ok((lc, _)) => Some(lc),
Err(err) if err == Error::ConfigNotFound => None,
Err(Error::ConfigNotFound) => None,
Err(err) => {
warn!(bucket, error = ?err, "failed to load lifecycle config");
None
@@ -119,8 +116,25 @@ impl LifecycleSys {
}
}
pub fn trace(_oi: &ObjectInfo) -> TraceFn {
Arc::new(|_oi, _ctx| Box::pin(async move {}))
pub fn trace(oi: &ObjectInfo) -> TraceFn {
let bucket = oi.bucket.clone();
let name = oi.name.clone();
let version_id = oi.version_id.map(|v| v.to_string()).unwrap_or_default();
Arc::new(move |_action: String, _ctx: HashMap<String, String>| {
let bucket = bucket.clone();
let name = name.clone();
let version_id = version_id.clone();
Box::pin(async move {
info!(
bucket = %bucket,
object = %name,
version_id = %version_id,
action = %_action,
"ILM lifecycle trace: {} on {}/{} (version: {})",
_action, bucket, name, version_id
);
})
})
}
}
@@ -133,8 +147,8 @@ struct ExpiryTask {
impl ExpiryOp for ExpiryTask {
fn op_hash(&self) -> u64 {
let mut hasher = Sha256::new();
hasher.update(format!("{}", self.obj_info.bucket).as_bytes());
hasher.update(format!("{}", self.obj_info.name).as_bytes());
hasher.update(self.obj_info.bucket.as_bytes());
hasher.update(self.obj_info.name.as_bytes());
xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED)
}
@@ -188,8 +202,8 @@ struct FreeVersionTask(ObjectInfo);
impl ExpiryOp for FreeVersionTask {
fn op_hash(&self) -> u64 {
let mut hasher = Sha256::new();
hasher.update(format!("{}", self.0.transitioned_object.tier).as_bytes());
hasher.update(format!("{}", self.0.transitioned_object.name).as_bytes());
hasher.update(self.0.transitioned_object.tier.as_bytes());
hasher.update(self.0.transitioned_object.name.as_bytes());
xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED)
}
@@ -207,8 +221,8 @@ struct NewerNoncurrentTask {
impl ExpiryOp for NewerNoncurrentTask {
fn op_hash(&self) -> u64 {
let mut hasher = Sha256::new();
hasher.update(format!("{}", self.bucket).as_bytes());
hasher.update(format!("{}", self.versions[0].object_name).as_bytes());
hasher.update(self.bucket.as_bytes());
hasher.update(self.versions[0].object_name.as_bytes());
xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED)
}
@@ -240,7 +254,7 @@ impl ExpiryState {
pub async fn pending_tasks(&self) -> usize {
let rxs = &self.tasks_rx;
if rxs.len() == 0 {
if rxs.is_empty() {
return 0;
}
let mut tasks = 0;
@@ -253,32 +267,33 @@ impl ExpiryState {
pub async fn enqueue_tier_journal_entry(&mut self, je: &Jentry) -> Result<(), std::io::Error> {
let wrkr = self.get_worker_ch(je.op_hash());
if wrkr.is_none() {
*self.stats.as_mut().expect("err").missed_tier_journal_tasks.get_mut() += 1;
*self.stats.as_mut().expect("stats lock").missed_tier_journal_tasks.get_mut() += 1;
return Ok(());
}
let wrkr = wrkr.expect("err");
let wrkr = wrkr.expect("worker channel should exist after None check");
select! {
//_ -> GlobalContext.Done() => ()
_ = wrkr.send(Some(Box::new(je.clone()))) => (),
else => {
*self.stats.as_mut().expect("err").missed_tier_journal_tasks.get_mut() += 1;
*self.stats.as_mut().expect("stats lock").missed_tier_journal_tasks.get_mut() += 1;
}
}
return Ok(());
Ok(())
}
pub async fn enqueue_free_version(&mut self, oi: ObjectInfo) {
let task = FreeVersionTask(oi);
let wrkr = self.get_worker_ch(task.op_hash());
if wrkr.is_none() {
*self.stats.as_mut().expect("err").missed_freevers_tasks.get_mut() += 1;
*self.stats.as_mut().expect("stats lock").missed_freevers_tasks.get_mut() += 1;
return;
}
let wrkr = wrkr.expect("err!");
let wrkr = wrkr.expect("worker channel should exist after None check");
select! {
//_ -> GlobalContext.Done() => {}
_ = wrkr.send(Some(Box::new(task))) => (),
else => {
*self.stats.as_mut().expect("err").missed_freevers_tasks.get_mut() += 1;
*self.stats.as_mut().expect("stats lock").missed_freevers_tasks.get_mut() += 1;
}
}
}
@@ -291,21 +306,21 @@ impl ExpiryState {
};
let wrkr = self.get_worker_ch(task.op_hash());
if wrkr.is_none() {
*self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
*self.stats.as_mut().expect("stats lock").missed_expiry_tasks.get_mut() += 1;
return;
}
let wrkr = wrkr.expect("err!");
let wrkr = wrkr.expect("worker channel should exist after None check");
select! {
//_ -> GlobalContext.Done() => {}
_ = wrkr.send(Some(Box::new(task))) => (),
else => {
*self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
*self.stats.as_mut().expect("stats lock").missed_expiry_tasks.get_mut() += 1;
}
}
}
pub async fn enqueue_by_newer_noncurrent(&mut self, bucket: &str, versions: Vec<ObjectToDelete>, lc_event: lifecycle::Event) {
if versions.len() == 0 {
if versions.is_empty() {
return;
}
@@ -316,26 +331,30 @@ impl ExpiryState {
};
let wrkr = self.get_worker_ch(task.op_hash());
if wrkr.is_none() {
*self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
*self.stats.as_mut().expect("stats lock").missed_expiry_tasks.get_mut() += 1;
return;
}
let wrkr = wrkr.expect("err!");
let wrkr = wrkr.expect("worker channel should exist after None check");
select! {
//_ -> GlobalContext.Done() => {}
_ = wrkr.send(Some(Box::new(task))) => (),
else => {
*self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
*self.stats.as_mut().expect("stats lock").missed_expiry_tasks.get_mut() += 1;
}
}
}
pub fn get_worker_ch(&self, h: u64) -> Option<Sender<Option<ExpiryOpType>>> {
if self.tasks_tx.len() == 0 {
if self.tasks_tx.is_empty() {
return None;
}
Some(self.tasks_tx[h as usize % self.tasks_tx.len()].clone())
}
pub fn increment_missed_tier_journal_tasks(&mut self) {
*self.stats.as_mut().expect("stats lock").missed_tier_journal_tasks.get_mut() += 1;
}
pub async fn resize_workers(n: usize, api: Arc<ECStore>) {
if n == GLOBAL_ExpiryState.read().await.tasks_tx.len() || n < 1 {
return;
@@ -349,11 +368,11 @@ impl ExpiryState {
let rx = Arc::new(tokio::sync::Mutex::new(rx));
state.tasks_tx.push(tx);
state.tasks_rx.push(rx.clone());
*state.stats.as_mut().expect("err").workers.get_mut() += 1;
*state.stats.as_mut().expect("stats lock").workers.get_mut() += 1;
tokio::spawn(async move {
let mut rx = rx.lock().await;
//let mut expiry_state = GLOBAL_ExpiryState.read().await;
ExpiryState::worker(&mut *rx, api).await;
ExpiryState::worker(&mut rx, api).await;
});
}
@@ -363,48 +382,49 @@ impl ExpiryState {
worker.send(None).await.unwrap_or(());
state.tasks_tx.remove(l - 1);
state.tasks_rx.remove(l - 1);
*state.stats.as_mut().expect("err").workers.get_mut() -= 1;
*state.stats.as_mut().expect("stats lock").workers.get_mut() -= 1;
l -= 1;
}
}
pub async fn worker(rx: &mut Receiver<Option<ExpiryOpType>>, api: Arc<ECStore>) {
//let cancel_token =
// get_background_services_cancel_token().ok_or_else(|| Error::other("Background services not initialized"))?;
let cancel_token = crate::global::get_background_services_cancel_token().unwrap_or_else(|| {
static FALLBACK: std::sync::OnceLock<tokio_util::sync::CancellationToken> = std::sync::OnceLock::new();
FALLBACK.get_or_init(tokio_util::sync::CancellationToken::new)
});
loop {
select! {
//_ = cancel_token.cancelled() => {
_ = tokio::signal::ctrl_c() => {
info!("got ctrl+c, exits");
_ = cancel_token.cancelled() => {
info!("lifecycle expiry worker received shutdown signal, exiting");
break;
}
v = rx.recv() => {
if v.is_none() {
break;
}
let v = v.expect("err!");
let v = v.expect("channel closed unexpectedly");
if v.is_none() {
//rx.close();
//drop(rx);
let _ = rx;
return;
}
let v = v.expect("err!");
let v = v.expect("received None after None check");
if v.as_any().is::<ExpiryTask>() {
let v = v.as_any().downcast_ref::<ExpiryTask>().expect("err!");
if v.obj_info.transitioned_object.status != "" {
let v = v.as_any().downcast_ref::<ExpiryTask>().expect("ExpiryTask downcast failed");
if !v.obj_info.transitioned_object.status.is_empty() {
apply_expiry_on_transitioned_object(api.clone(), &v.obj_info, &v.event, &v.src).await;
} else {
apply_expiry_on_non_transitioned_objects(api.clone(), &v.obj_info, &v.event, &v.src).await;
}
}
else if v.as_any().is::<NewerNoncurrentTask>() {
let _v = v.as_any().downcast_ref::<NewerNoncurrentTask>().expect("err!");
//delete_object_versions(api, &v.bucket, &v.versions, v.event).await;
let v = v.as_any().downcast_ref::<NewerNoncurrentTask>().expect("NewerNoncurrentTask downcast failed");
crate::client::object_handlers_common::delete_object_versions(&api, &v.bucket, &v.versions, v.event.clone()).await;
}
else if v.as_any().is::<Jentry>() {
let v = v.as_any().downcast_ref::<Jentry>().expect("err!");
let v = v.as_any().downcast_ref::<Jentry>().expect("Jentry downcast failed");
if let Err(err) = delete_object_from_remote_tier(&v.obj_name, &v.version_id, &v.tier_name).await {
warn!(
object = %v.obj_name,
@@ -416,7 +436,7 @@ impl ExpiryState {
}
}
else if v.as_any().is::<FreeVersionTask>() {
let v = v.as_any().downcast_ref::<FreeVersionTask>().expect("err!");
let v = v.as_any().downcast_ref::<FreeVersionTask>().expect("FreeVersionTask downcast failed");
let oi = v.0.clone();
if let Err(err) = delete_object_from_remote_tier(
&oi.transitioned_object.name,
@@ -499,7 +519,7 @@ struct TransitionTask {
impl ExpiryOp for TransitionTask {
fn op_hash(&self) -> u64 {
let mut hasher = Sha256::new();
hasher.update(format!("{}", self.obj_info.bucket).as_bytes());
hasher.update(self.obj_info.bucket.as_bytes());
// hasher.update(format!("{}", self.obj_info.versions[0].object_name).as_bytes());
xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED)
}
@@ -594,22 +614,40 @@ impl TransitionState {
if task.is_err() {
break;
}
let task = task.expect("err!");
let task = task.expect("channel recv should succeed after error check");
if task.is_none() {
//self.transition_rx.close();
//drop(self.transition_rx);
return;
}
let task = task.expect("err!");
let task = task.expect("received None after None check");
if task.as_any().is::<TransitionTask>() {
let task = task.as_any().downcast_ref::<TransitionTask>().expect("err!");
let task = task.as_any().downcast_ref::<TransitionTask>().expect("TransitionTask downcast failed");
GLOBAL_TransitionState.active_tasks.fetch_add(1, Ordering::SeqCst);
let obj_info_for_event = ObjectInfo {
bucket: task.obj_info.bucket.clone(),
name: task.obj_info.name.clone(),
size: task.obj_info.size,
version_id: task.obj_info.version_id,
..Default::default()
};
if let Err(err) = transition_object(api.clone(), &task.obj_info, LcAuditEvent::new(task.event.clone(), task.src.clone())).await {
if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) && !err.to_string().contains("use of closed network connection") {
error!("Transition to {} failed for {}/{} version:{} with {}",
task.event.storage_class, task.obj_info.bucket, task.obj_info.name, task.obj_info.version_id.map(|v| v.to_string()).unwrap_or_default(), err.to_string());
}
// Send s3:ObjectTransition:Failed event
send_event(EventArgs {
event_name: EventName::ObjectTransitionFailed.to_string(),
bucket_name: obj_info_for_event.bucket.clone(),
object: obj_info_for_event,
user_agent: "Internal: [ILM-Transition]".to_string(),
host: GLOBAL_LocalNodeName.to_string(),
..Default::default()
});
} else {
let mut ts = TierStats {
total_size: task.obj_info.size as u64,
@@ -620,6 +658,16 @@ impl TransitionState {
ts.num_objects = 1;
}
GLOBAL_TransitionState.add_lastday_stats(&task.event.storage_class, ts);
// Send s3:ObjectTransition:Complete event
send_event(EventArgs {
event_name: EventName::ObjectTransitionComplete.to_string(),
bucket_name: obj_info_for_event.bucket.clone(),
object: obj_info_for_event,
user_agent: "Internal: [ILM-Transition]".to_string(),
host: GLOBAL_LocalNodeName.to_string(),
..Default::default()
});
}
GLOBAL_TransitionState.active_tasks.fetch_add(-1, Ordering::SeqCst);
}
@@ -634,7 +682,7 @@ impl TransitionState {
tier_stats
.entry(tier.to_string())
.and_modify(|e| e.add_stats(ts))
.or_insert(LastDayTierStats::default());
.or_default();
}
pub fn get_daily_all_tier_stats(&self) -> DailyAllTierStats {
@@ -666,15 +714,15 @@ impl TransitionState {
tokio::spawn(async move {
TransitionState::worker(clone_api).await;
});
num_workers = num_workers + 1;
num_workers += 1;
GLOBAL_TransitionState.num_workers.fetch_add(1, Ordering::SeqCst);
}
let mut num_workers = GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst);
while num_workers > n {
let worker = GLOBAL_TransitionState.kill_tx.clone();
worker.send(()).await;
num_workers = num_workers - 1;
let _ = worker.send(()).await;
num_workers -= 1;
GLOBAL_TransitionState.num_workers.fetch_add(-1, Ordering::SeqCst);
}
}
@@ -683,10 +731,10 @@ impl TransitionState {
pub async fn init_background_expiry(api: Arc<ECStore>) {
let mut workers = get_env_usize("RUSTFS_MAX_EXPIRY_WORKERS", std::cmp::min(num_cpus::get(), 16));
//globalILMConfig.getExpirationWorkers()
if let Ok(env_expiration_workers) = env::var("_RUSTFS_ILM_EXPIRATION_WORKERS") {
if let Ok(num_expirations) = env_expiration_workers.parse::<usize>() {
workers = num_expirations;
}
if let Ok(env_expiration_workers) = env::var("_RUSTFS_ILM_EXPIRATION_WORKERS")
&& let Ok(num_expirations) = env_expiration_workers.parse::<usize>()
{
workers = num_expirations;
}
if workers == 0 {
@@ -1029,24 +1077,24 @@ pub async fn validate_transition_tier(lc: &BucketLifecycleConfiguration) -> Resu
for rule in &lc.rules {
if let Some(transitions) = &rule.transitions {
for transition in transitions {
if let Some(storage_class) = &transition.storage_class {
if storage_class.as_str() != "" {
let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str());
if !valid {
return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS));
}
if let Some(storage_class) = &transition.storage_class
&& storage_class.as_str() != ""
{
let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str());
if !valid {
return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS));
}
}
}
}
if let Some(noncurrent_version_transitions) = &rule.noncurrent_version_transitions {
for noncurrent_version_transition in noncurrent_version_transitions {
if let Some(storage_class) = &noncurrent_version_transition.storage_class {
if storage_class.as_str() != "" {
let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str());
if !valid {
return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS));
}
if let Some(storage_class) = &noncurrent_version_transition.storage_class
&& storage_class.as_str() != ""
{
let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str());
if !valid {
return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS));
}
}
}
@@ -1067,6 +1115,104 @@ pub async fn enqueue_transition_immediate(oi: &ObjectInfo, src: LcEventSrc) {
}
}
pub async fn enqueue_immediate_expiry(oi: &ObjectInfo, src: LcEventSrc) {
let Some(lifecycle) = GLOBAL_LifecycleSys.get(&oi.bucket).await else {
return;
};
let Some(api) = crate::new_object_layer_fn() else {
return;
};
let mut marker = None;
let mut version_marker = None;
let mut object_infos = Vec::new();
loop {
let Ok(page) = api
.clone()
.list_object_versions(&oi.bucket, &oi.name, marker.clone(), version_marker.clone(), None, 1000)
.await
else {
return;
};
object_infos.extend(page.objects.into_iter().filter(|object| object.name == oi.name));
if !page.is_truncated {
break;
}
marker = page.next_marker;
version_marker = page.next_version_idmarker;
}
if object_infos.is_empty() {
object_infos.push(oi.clone());
}
let lock_config = match metadata_sys::get_object_lock_config(&oi.bucket).await {
Ok((cfg, _)) => Some(Arc::new(cfg)),
Err(_) => None,
};
let replication = match metadata_sys::get_replication_config(&oi.bucket).await {
Ok((cfg, _)) if !cfg.rules.is_empty() => Some(Arc::new(ReplicationConfig::new(Some(cfg), None))),
_ => None,
};
let object_opts = object_infos
.iter()
.map(|object| object.to_lifecycle_opts())
.collect::<Vec<ObjectOpts>>();
let Ok(events) = Evaluator::new(Arc::new(lifecycle))
.with_lock_retention(lock_config)
.with_replication_config(replication)
.eval(&object_opts)
.await
else {
return;
};
let mut to_delete_objs = Vec::new();
let mut noncurrent_event = None;
for (object, event) in object_infos.iter().zip(events.iter()) {
if event.due != Some(OffsetDateTime::UNIX_EPOCH) {
continue;
}
match event.action {
IlmAction::DeleteAction
| IlmAction::DeleteRestoredAction
| IlmAction::DeleteRestoredVersionAction
| IlmAction::DeleteAllVersionsAction
| IlmAction::DelMarkerDeleteAllVersionsAction => {
apply_expiry_rule(event, &src, object).await;
}
IlmAction::DeleteVersionAction => {
to_delete_objs.push(ObjectToDelete {
object_name: object.name.clone(),
version_id: object.version_id,
..Default::default()
});
if noncurrent_event.is_none() {
noncurrent_event = Some(event.clone());
}
}
_ => {}
}
}
if !to_delete_objs.is_empty()
&& let Some(event) = noncurrent_event
{
GLOBAL_ExpiryState
.write()
.await
.enqueue_by_newer_noncurrent(&oi.bucket, to_delete_objs, event)
.await;
}
}
pub async fn enqueue_transition_for_existing_objects(api: Arc<ECStore>, bucket: &str) -> Result<(), Error> {
let Some(lc) = GLOBAL_LifecycleSys.get(bucket).await else {
return Ok(());
@@ -1094,6 +1240,60 @@ pub async fn enqueue_transition_for_existing_objects(api: Arc<ECStore>, bucket:
}
}
pub async fn enqueue_expiry_for_existing_objects(api: Arc<ECStore>, bucket: &str) -> Result<(), Error> {
let Ok((lc, _)) = metadata_sys::get_lifecycle_config(bucket).await else {
return Ok(());
};
let lock_retention = metadata_sys::get_object_lock_config(bucket)
.await
.ok()
.and_then(|(cfg, _)| cfg.rule.and_then(|rule| rule.default_retention));
let replication_config = metadata_sys::get_replication_config(bucket).await.ok();
let mut marker = None;
let mut version_marker = None;
let src = LcEventSrc::Scanner;
loop {
let page = api
.clone()
.list_object_versions(bucket, "", marker.clone(), version_marker.clone(), None, 1000)
.await?;
for object in &page.objects {
let event = eval_action_from_lifecycle(&lc, lock_retention.clone(), replication_config.clone(), object).await;
match event.action {
IlmAction::DeleteAction
| IlmAction::DeleteVersionAction
| IlmAction::DeleteRestoredAction
| IlmAction::DeleteRestoredVersionAction
| IlmAction::DeleteAllVersionsAction
| IlmAction::DelMarkerDeleteAllVersionsAction => {
if event
.due
.is_some_and(|due| due.unix_timestamp() <= OffsetDateTime::now_utc().unix_timestamp())
{
if object.is_remote() {
apply_expiry_on_transitioned_object(api.clone(), object, &event, &src).await;
} else {
apply_expiry_on_non_transitioned_objects(api.clone(), object, &event, &src).await;
}
} else {
apply_expiry_rule(&event, &src, object).await;
}
}
_ => {}
}
}
if !page.is_truncated {
return Ok(());
}
marker = page.next_marker;
version_marker = page.next_version_idmarker;
}
}
async fn enqueue_transition_with_lifecycle(oi: &ObjectInfo, lc: &BucketLifecycleConfiguration, src: &LcEventSrc) {
let event = lc.eval(&oi.to_lifecycle_opts()).await;
match event.action {
@@ -1154,6 +1354,8 @@ pub async fn expire_transitioned_object(
}
};
schedule_lifecycle_replication_delete_if_needed(oi).await;
//defer auditLogLifecycle(ctx, *oi, ILMExpiry, tags, traceFn)
let event_name = if oi.delete_marker {
@@ -1227,8 +1429,19 @@ pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditE
result
}
pub fn audit_tier_actions(_api: ECStore, _tier: &str, _bytes: i64) -> TimeFn {
Arc::new(|| Box::pin(async move {}))
pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn {
let tier = _tier.to_string();
Arc::new(move || {
let tier = tier.clone();
Box::pin(async move {
info!(
tier = %tier,
bytes = bytes,
"ILM tier transition audit: completed transition of {} bytes to tier '{}'",
bytes, tier
);
})
})
}
pub async fn get_transitioned_object_reader(
@@ -1245,11 +1458,11 @@ pub async fn get_transitioned_object_reader(
Err(err) => return Err(std::io::Error::other(err)),
};
let ret = new_getobjectreader(rs, &oi, opts, &h);
let ret = new_getobjectreader(rs, oi, opts, h);
if let Err(err) = ret {
return Err(error_resp_to_object_err(err, vec![bucket, object]));
}
let (get_fn, off, length) = ret.expect("err");
let (get_fn, off, length) = ret.expect("get_transitioned_object_reader should succeed after error check");
let mut gopts = WarmBackendGetOpts::default();
if off >= 0 && length >= 0 {
@@ -1269,8 +1482,8 @@ pub async fn post_restore_opts(version_id: &str, bucket: &str, object: &str) ->
let versioned = BucketVersioningSys::prefix_enabled(bucket, object).await;
let version_suspended = BucketVersioningSys::prefix_suspended(bucket, object).await;
let vid = version_id.trim();
if vid != "" && vid != NULL_VERSION_ID {
if let Err(err) = Uuid::parse_str(vid) {
if !vid.is_empty() && vid != NULL_VERSION_ID {
if let Err(_err) = Uuid::parse_str(vid) {
return Err(std::io::Error::other(
StorageError::InvalidVersionID(bucket.to_string(), object.to_string(), vid.to_string()).to_string(),
));
@@ -1332,24 +1545,21 @@ pub async fn put_restore_opts(
}
meta.insert(v.name.clone().unwrap(), v.value.clone().unwrap_or_else(|| "".to_string()));
}
if let Some(output_location) = rreq.output_location.as_ref() {
if let Some(s3) = &output_location.s3 {
if let Some(tags) = &s3.tagging {
meta.insert(
AMZ_OBJECT_TAGGING.to_string(),
serde_urlencoded::to_string(tags.tag_set.clone()).unwrap_or_else(|_| "".to_string()),
);
}
}
if let Some(output_location) = rreq.output_location.as_ref()
&& let Some(s3) = &output_location.s3
&& let Some(tags) = &s3.tagging
{
meta.insert(
AMZ_OBJECT_TAGGING.to_string(),
serde_urlencoded::to_string(tags.tag_set.clone()).unwrap_or_else(|_| "".to_string()),
);
}
if let Some(output_location) = rreq.output_location.as_ref() {
if let Some(s3) = &output_location.s3 {
if let Some(encryption) = &s3.encryption {
if encryption.encryption_type.as_str() != "" {
meta.insert(X_AMZ_SERVER_SIDE_ENCRYPTION.as_str().to_string(), AMZ_ENCRYPTION_AES.to_string());
}
}
}
if let Some(output_location) = rreq.output_location.as_ref()
&& let Some(s3) = &output_location.s3
&& let Some(encryption) = &s3.encryption
&& encryption.encryption_type.as_str() != ""
{
meta.insert(X_AMZ_SERVER_SIDE_ENCRYPTION.as_str().to_string(), AMZ_ENCRYPTION_AES.to_string());
}
return Ok(ObjectOptions {
versioned: BucketVersioningSys::prefix_enabled(bucket, object).await,
@@ -1361,7 +1571,7 @@ pub async fn put_restore_opts(
for (k, v) in &oi.user_defined {
meta.insert(k.to_string(), v.clone());
}
if oi.user_tags.len() != 0 {
if !oi.user_tags.is_empty() {
meta.insert(AMZ_OBJECT_TAGGING.to_string(), oi.user_tags.clone());
}
let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1));
@@ -1394,7 +1604,7 @@ impl LifecycleOps for ObjectInfo {
lifecycle::ObjectOpts {
name: self.name.clone(),
user_tags: self.user_tags.clone(),
version_id: self.version_id.clone(),
version_id: self.version_id,
mod_time: self.mod_time,
size: self.size as usize,
is_latest: self.is_latest,
@@ -1421,38 +1631,42 @@ pub trait RestoreRequestOps {
}
impl RestoreRequestOps for RestoreRequest {
fn validate(&self, api: Arc<ECStore>) -> Result<(), std::io::Error> {
/*if self.type_.is_none() && self.select_parameters.is_some() {
fn validate(&self, _api: Arc<ECStore>) -> Result<(), std::io::Error> {
// SELECT type requires select_parameters, and vice versa
if self.type_.as_ref().is_none_or(|t| t.as_str() != RestoreRequestType::SELECT) && self.select_parameters.is_some() {
return Err(std::io::Error::other("Select parameters can only be specified with SELECT request type"));
}
if let Some(type_) = self.type_ && type_ == RestoreRequestType::SELECT && self.select_parameters.is_none() {
if let Some(type_) = &self.type_
&& type_.as_str() == RestoreRequestType::SELECT
&& self.select_parameters.is_none()
{
return Err(std::io::Error::other("SELECT restore request requires select parameters to be specified"));
}
if self.type_.is_none() && self.output_location.is_some() {
return Err(std::io::Error::other("OutputLocation required only for SELECT request type"));
// OutputLocation is only valid for SELECT requests
if self.type_.as_ref().is_none_or(|t| t.as_str() != RestoreRequestType::SELECT) && self.output_location.is_some() {
return Err(std::io::Error::other("OutputLocation can only be specified with SELECT request type"));
}
if let Some(type_) = self.type_ && type_ == RestoreRequestType::SELECT && self.output_location.is_none() {
if let Some(type_) = &self.type_
&& type_.as_str() == RestoreRequestType::SELECT
&& self.output_location.is_none()
{
return Err(std::io::Error::other("OutputLocation required for SELECT requests"));
}
if let Some(type_) = self.type_ && type_ == RestoreRequestType::SELECT && self.days != 0 {
// Days must not be specified with SELECT requests
if let Some(type_) = &self.type_
&& type_.as_str() == RestoreRequestType::SELECT
&& self.days.is_some_and(|d| d > 0)
{
return Err(std::io::Error::other("Days cannot be specified with SELECT restore request"));
}
if self.days == 0 && self.type_.is_none() {
// For non-SELECT requests, days must be at least 1
if self.type_.is_none() && self.days.is_none_or(|d| d <= 0) {
return Err(std::io::Error::other("restoration days should be at least 1"));
}
if self.output_location.is_some() {
if _, err := api.get_bucket_info(self.output_location.s3.bucket_name, BucketOptions{}); err != nil {
return err
}
if self.output_location.s3.prefix == "" {
return Err(std::io::Error::other("Prefix is a required parameter in OutputLocation"));
}
if self.output_location.s3.encryption.encryption_type.as_str() != ServerSideEncryption::AES256 {
return NotImplemented{}
}
}*/
Ok(())
}
}
@@ -1497,10 +1711,10 @@ pub async fn eval_action_from_lifecycle(
//}
return lifecycle::Event::default();
}
if let Some(rcfg) = rcfg {
if rep_has_active_rules(&rcfg.0, &oi.name, true) {
return lifecycle::Event::default();
}
if let Some(rcfg) = rcfg
&& rep_has_active_rules(&rcfg.0, &oi.name, true)
{
return lifecycle::Event::default();
}
}
_ => (),
@@ -1566,6 +1780,7 @@ pub async fn apply_expiry_on_non_transitioned_objects(
return false;
}
};
schedule_lifecycle_replication_delete_if_needed(oi).await;
//debug!("dobj: {:?}", dobj);
if dobj.name.is_empty() {
dobj = oi.clone();
@@ -1606,6 +1821,70 @@ pub async fn apply_expiry_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &
true
}
async fn schedule_lifecycle_replication_delete_if_needed(oi: &ObjectInfo) {
if !oi.delete_marker || oi.version_id.is_none() {
return;
}
let replication_state = lifecycle_delete_replication_state(oi).await;
if replication_state.is_none() {
return;
}
schedule_replication_delete(DeletedObjectReplicationInfo {
delete_object: crate::store_api::DeletedObject {
object_name: oi.name.clone(),
delete_marker_version_id: oi.version_id,
delete_marker: false,
delete_marker_mtime: oi.mod_time,
replication_state,
..Default::default()
},
bucket: oi.bucket.clone(),
event_type: REPLICATE_INCOMING_DELETE.to_string(),
..Default::default()
})
.await;
}
async fn lifecycle_delete_replication_state(oi: &ObjectInfo) -> Option<ReplicationState> {
if !oi.replication_decision.is_empty() || oi.version_purge_status == VersionPurgeStatusType::Pending {
return Some(oi.replication_state());
}
let dsc = check_replicate_delete(
&oi.bucket,
&ObjectToDelete {
object_name: oi.name.clone(),
version_id: oi.version_id,
..Default::default()
},
oi,
&ObjectOptions {
version_id: oi.version_id.map(|v| v.to_string()),
versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await,
..Default::default()
},
None,
)
.await;
if !dsc.replicate_any() {
return None;
}
Some(replication_state_for_version_delete(dsc))
}
fn replication_state_for_version_delete(dsc: ReplicateDecision) -> ReplicationState {
let pending_status = dsc.pending_status();
ReplicationState {
replicate_decision_str: dsc.to_string(),
version_purge_status_internal: pending_status.clone(),
purge_targets: rustfs_filemeta::version_purge_statuses_map(pending_status.as_deref().unwrap_or_default()),
..Default::default()
}
}
pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
let mut success = false;
match event.action {
+2 -1
View File
@@ -14,8 +14,9 @@
pub mod bucket_lifecycle_audit;
pub mod bucket_lifecycle_ops;
pub mod core;
pub mod evaluator;
pub mod lifecycle;
pub use self::core as lifecycle;
pub mod rule;
pub mod tier_last_day_stats;
pub mod tier_sweeper;
@@ -109,10 +109,19 @@ impl ObjSweeper {
}
pub async fn sweep(&self) {
let je = self.should_remove_remote_object();
if !je.is_none() {
let mut expiry_state = GLOBAL_ExpiryState.write().await;
expiry_state.enqueue_tier_journal_entry(&je.expect("err!"));
let Some(je) = self.should_remove_remote_object() else {
return;
};
let hash = je.op_hash();
// Grab the sender under a short read lock, then release the lock so we
// don't hold it across the async send.
let wrkr = GLOBAL_ExpiryState.read().await.get_worker_ch(hash);
let Some(wrkr) = wrkr else {
GLOBAL_ExpiryState.write().await.increment_missed_tier_journal_tasks();
return;
};
if wrkr.send(Some(Box::new(je))).await.is_err() {
GLOBAL_ExpiryState.write().await.increment_missed_tier_journal_tasks();
}
}
}
@@ -153,6 +153,11 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
if obj.op_type == ReplicationType::Delete {
if obj.version_id.is_some() {
if obj.delete_marker {
return rule.delete_marker_replication.clone().is_some_and(|d| {
d.status == Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED))
});
}
return rule
.delete_replication
.clone()
@@ -34,7 +34,7 @@ use crate::global::get_global_bucket_monitor;
use crate::set_disk::get_lock_acquire_timeout;
use crate::store_api::{DeletedObject, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete, WalkOptions};
use crate::{StorageAPI, new_object_layer_fn};
use aws_sdk_s3::error::SdkError;
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
use aws_sdk_s3::operation::head_object::HeadObjectOutput;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedPart, ObjectLockLegalHoldStatus};
@@ -86,6 +86,7 @@ use tokio::time::Duration as TokioDuration;
use tokio_util::io::ReaderStream;
use tokio_util::sync::CancellationToken;
use tracing::{error, info, instrument, warn};
use uuid::Uuid;
pub(crate) const REPLICATION_DIR: &str = ".replication";
pub(crate) const RESYNC_FILE_NAME: &str = "resync.bin";
@@ -1497,6 +1498,54 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
}
};
if dobj.delete_object.delete_marker
&& let Some(delete_marker_version_id) = dobj.delete_object.delete_marker_version_id
{
let source_marker_state = storage
.get_object_info(
&bucket,
&dobj.delete_object.object_name,
&ObjectOptions {
version_id: Some(delete_marker_version_id.to_string()),
versioned: BucketVersioningSys::prefix_enabled(&bucket, &dobj.delete_object.object_name).await,
version_suspended: BucketVersioningSys::prefix_suspended(&bucket, &dobj.delete_object.object_name).await,
..Default::default()
},
)
.await;
match source_marker_state {
Ok(info) if info.delete_marker && info.version_id == Some(delete_marker_version_id) => {}
Ok(_) => {
warn!(
bucket,
object = dobj.delete_object.object_name,
version_id = %delete_marker_version_id,
"skipping stale delete-marker replication because source version is no longer a delete marker"
);
return;
}
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {
warn!(
bucket,
object = dobj.delete_object.object_name,
version_id = %delete_marker_version_id,
"skipping stale delete-marker replication because source version no longer exists"
);
return;
}
Err(err) => {
warn!(
bucket,
object = dobj.delete_object.object_name,
version_id = %delete_marker_version_id,
error = %err,
"failed to verify source delete-marker state before replication"
);
}
}
}
let dsc = match parse_replicate_decision(
&bucket,
&dobj
@@ -1529,7 +1578,6 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
return;
}
};
let ns_lock = match storage
.new_ns_lock(&bucket, format!("/[replicate]/{}", dobj.delete_object.object_name).as_str())
.await
@@ -1653,7 +1701,33 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
}
}
let (replication_status, prev_status) = if dobj.delete_object.version_id.is_none() {
let is_version_purge = is_version_delete_replication(&dobj.delete_object);
if !is_version_purge && dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
let bucket_clone = bucket.clone();
let dobj_clone = dobj.clone();
let dsc_clone = dsc.clone();
let storage_clone = storage.clone();
tokio::spawn(async move {
for _ in 0..5 {
if let Some(delete_marker_version_id) = dobj_clone.delete_object.delete_marker_version_id
&& source_delete_marker_missing(
&*storage_clone,
&bucket_clone,
&dobj_clone.delete_object.object_name,
delete_marker_version_id,
)
.await
{
replicate_delete_marker_purge_to_targets(&bucket_clone, &dobj_clone, &dsc_clone).await;
break;
}
tokio::time::sleep(TokioDuration::from_secs(1)).await;
}
});
}
let (replication_status, prev_status) = if !is_version_purge {
(
rinfos.replication_status(),
dobj.delete_object
@@ -1741,6 +1815,65 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
}
}
async fn source_delete_marker_missing<S: StorageAPI>(
storage: &S,
bucket: &str,
object_name: &str,
delete_marker_version_id: Uuid,
) -> bool {
match storage
.get_object_info(
bucket,
object_name,
&ObjectOptions {
version_id: Some(delete_marker_version_id.to_string()),
versioned: BucketVersioningSys::prefix_enabled(bucket, object_name).await,
version_suspended: BucketVersioningSys::prefix_suspended(bucket, object_name).await,
..Default::default()
},
)
.await
{
Ok(info) => !info.delete_marker || info.version_id != Some(delete_marker_version_id),
Err(err) => is_err_object_not_found(&err) || is_err_version_not_found(&err),
}
}
async fn replicate_delete_marker_purge_to_targets(bucket: &str, dobj: &DeletedObjectReplicationInfo, dsc: &ReplicateDecision) {
let Some(delete_marker_version_id) = dobj.delete_object.delete_marker_version_id else {
return;
};
for tgt_entry in dsc.targets_map.values() {
if !tgt_entry.replicate {
continue;
}
if !dobj.target_arn.is_empty() && dobj.target_arn != tgt_entry.arn {
continue;
}
let Some(tgt_client) = BucketTargetSys::get().get_remote_target_client(bucket, &tgt_entry.arn).await else {
continue;
};
let _ = tgt_client
.remove_object(
&tgt_client.bucket,
&dobj.delete_object.object_name,
Some(delete_marker_version_id.to_string()),
RemoveObjectOptions {
force_delete: false,
governance_bypass: false,
replication_delete_marker: false,
replication_mtime: dobj.delete_object.delete_marker_mtime,
replication_status: ReplicationStatusType::Replica,
replication_request: true,
replication_validity_check: false,
},
)
.await;
}
}
async fn replicate_force_delete_to_targets<S: StorageAPI>(dobj: &DeletedObjectReplicationInfo, storage: Arc<S>) {
let bucket = &dobj.bucket;
let object_name = &dobj.delete_object.object_name;
@@ -1924,6 +2057,14 @@ async fn replicate_force_delete_to_targets<S: StorageAPI>(dobj: &DeletedObjectRe
}
}
fn is_version_delete_replication(dobj: &DeletedObject) -> bool {
dobj.version_id.is_some() || (dobj.delete_marker_version_id.is_some() && !dobj.delete_marker)
}
fn is_retryable_delete_replication_head_error(is_not_found: bool, code: Option<&str>) -> bool {
!is_not_found && !matches!(code, Some("MethodNotAllowed" | "405"))
}
async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
let version_id = if let Some(version_id) = &dobj.delete_object.delete_marker_version_id {
version_id.to_owned()
@@ -1941,7 +2082,8 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
rinfo.endpoint = tgt_client.endpoint.clone();
rinfo.secure = tgt_client.secure;
if dobj.delete_object.version_id.is_none()
let is_version_purge = is_version_delete_replication(&dobj.delete_object);
if !is_version_purge
&& rinfo.prev_replication_status == ReplicationStatusType::Completed
&& dobj.op_type != ReplicationType::ExistingObject
{
@@ -1949,12 +2091,12 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
return rinfo;
}
if dobj.delete_object.version_id.is_some() && rinfo.version_purge_status == VersionPurgeStatusType::Complete {
if is_version_purge && rinfo.version_purge_status == VersionPurgeStatusType::Complete {
return rinfo;
}
if BucketTargetSys::get().is_offline(&tgt_client.to_url()).await {
if dobj.delete_object.version_id.is_none() {
if !is_version_purge {
rinfo.replication_status = ReplicationStatusType::Failed;
} else {
rinfo.version_purge_status = VersionPurgeStatusType::Failed;
@@ -1968,18 +2110,29 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
Some(version_id.to_string())
};
if dobj.delete_object.delete_marker_version_id.is_some()
&& let Err(e) = tgt_client
if dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
match tgt_client
.head_object(&tgt_client.bucket, &dobj.delete_object.object_name, version_id.clone())
.await
&& let SdkError::ServiceError(service_err) = &e
&& !service_err.err().is_not_found()
{
rinfo.replication_status = ReplicationStatusType::Failed;
rinfo.error = Some(e.to_string());
return rinfo;
};
{
Ok(_) => {}
Err(e) => {
let non_retryable = matches!(
&e,
SdkError::ServiceError(service_err)
if is_retryable_delete_replication_head_error(
service_err.err().is_not_found(),
service_err.err().code(),
)
);
if non_retryable {
rinfo.replication_status = ReplicationStatusType::Failed;
rinfo.error = Some(e.to_string());
return rinfo;
}
}
}
}
match tgt_client
.remove_object(
@@ -1989,7 +2142,7 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
RemoveObjectOptions {
force_delete: false,
governance_bypass: false,
replication_delete_marker: dobj.delete_object.delete_marker_version_id.is_some(),
replication_delete_marker: dobj.delete_object.delete_marker,
replication_mtime: dobj.delete_object.delete_marker_mtime,
replication_status: ReplicationStatusType::Replica,
replication_request: true,
@@ -1999,7 +2152,7 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
.await
{
Ok(_) => {
if dobj.delete_object.version_id.is_none() {
if !is_version_purge {
rinfo.replication_status = ReplicationStatusType::Completed;
} else {
rinfo.version_purge_status = VersionPurgeStatusType::Complete;
@@ -2007,7 +2160,7 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
}
Err(e) => {
rinfo.error = Some(e.to_string());
if dobj.delete_object.version_id.is_none() {
if !is_version_purge {
rinfo.replication_status = ReplicationStatusType::Failed;
} else {
rinfo.version_purge_status = VersionPurgeStatusType::Failed;
@@ -3450,6 +3603,54 @@ mod tests {
);
}
#[test]
fn test_is_version_delete_replication_for_delete_marker_version_purge() {
let dobj = DeletedObject {
delete_marker: false,
delete_marker_version_id: Some(Uuid::new_v4()),
..Default::default()
};
assert!(
is_version_delete_replication(&dobj),
"delete-marker version purges must be tracked as version purge replication, not delete-marker creation replication"
);
}
#[test]
fn test_is_version_delete_replication_for_delete_marker_creation() {
let dobj = DeletedObject {
delete_marker: true,
delete_marker_version_id: Some(Uuid::new_v4()),
..Default::default()
};
assert!(
!is_version_delete_replication(&dobj),
"delete-marker creation should remain on the delete-marker replication path"
);
}
#[test]
fn test_is_retryable_delete_replication_head_error_allows_delete_marker_head_responses() {
assert!(
!is_retryable_delete_replication_head_error(false, Some("405")),
"numeric 405 responses should not block delete-marker purge replication"
);
assert!(
!is_retryable_delete_replication_head_error(false, Some("MethodNotAllowed")),
"MethodNotAllowed responses should not block delete-marker purge replication"
);
assert!(
!is_retryable_delete_replication_head_error(true, Some("NoSuchKey")),
"not-found responses should not block delete-marker purge replication"
);
assert!(
is_retryable_delete_replication_head_error(false, Some("AccessDenied")),
"unexpected head errors should still fail fast"
);
}
#[tokio::test]
async fn test_get_heal_replicate_object_info_failed_object_returns_heal_roi() {
let oi = ObjectInfo {
@@ -12,6 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use tracing::warn;
use crate::bucket::lifecycle::lifecycle;
use crate::bucket::versioning::VersioningApi;
use crate::bucket::versioning_sys::BucketVersioningSys;
@@ -19,7 +22,14 @@ use crate::store::ECStore;
use crate::store_api::{ObjectOperations, ObjectOptions, ObjectToDelete};
use rustfs_lock::MAX_DELETE_LIST;
pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) {
pub async fn delete_object_versions(api: &Arc<ECStore>, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) {
let version_suspended = match BucketVersioningSys::get(bucket).await {
Ok(vc) => vc.suspended(),
Err(err) => {
warn!(bucket, error = ?err, "failed to get versioning config during lifecycle noncurrent version cleanup");
return;
}
};
let mut remaining = to_del;
loop {
let mut to_del = remaining;
@@ -29,15 +39,29 @@ pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[Object
} else {
remaining = &[];
}
let vc = BucketVersioningSys::get(bucket).await.expect("err!");
let _deleted_objs = api.delete_objects(
bucket,
to_del.to_vec(),
ObjectOptions {
//prefix_enabled_fn: vc.prefix_enabled(""),
version_suspended: vc.suspended(),
..Default::default()
},
);
let (_deleted_objs, errors) = api
.delete_objects(
bucket,
to_del.to_vec(),
ObjectOptions {
version_suspended,
..Default::default()
},
)
.await;
for (i, err) in errors.iter().enumerate() {
if let Some(e) = err {
let obj_name = to_del.get(i).map(|o| o.object_name.as_str()).unwrap_or("<unknown>");
let vid = to_del
.get(i)
.and_then(|o| o.version_id)
.map(|v| v.to_string())
.unwrap_or_default();
warn!(bucket, object = obj_name, version_id = %vid, error = ?e, "failed to delete noncurrent version during lifecycle cleanup");
}
}
if remaining.is_empty() {
break;
}
}
}
+191 -34
View File
@@ -1771,13 +1771,7 @@ impl ObjectOperations for SetDisks {
..Default::default()
};
let dsc = if opts
.delete_replication
.as_ref()
.map(|v| v.replica_status == ReplicationStatusType::Replica)
== Some(true)
|| opts.version_purge_status() == VersionPurgeStatusType::Complete
{
let dsc = if should_preserve_delete_replication_state(&opts) {
ReplicateDecision::default()
} else {
check_replicate_delete(bucket, &otd, &goi, &opts, gerr.map(|e| e.to_string())).await
@@ -1792,33 +1786,7 @@ impl ObjectOperations for SetDisks {
.unwrap_or_default();
}
let mut mark_delete = goi.version_id.is_some();
let mut delete_marker = opts.versioned;
if opts.version_id.is_some() {
// Decommission/rebalance may recreate a delete marker on a new pool before that
// exact version exists there, so we must still treat it as a mark-delete write.
if opts.data_movement && opts.delete_marker && !version_found {
mark_delete = true;
}
if version_found && opts.delete_marker_replication_status() == ReplicationStatusType::Replica {
mark_delete = false;
}
if opts.version_purge_status().is_empty() && opts.delete_marker_replication_status().is_empty() {
mark_delete = false;
}
if opts.version_purge_status() == VersionPurgeStatusType::Complete {
mark_delete = false;
}
if version_found && (!goi.version_purge_status.is_empty() || !goi.delete_marker) {
delete_marker = false;
}
}
let (mark_delete, mut delete_marker) = resolve_delete_version_state(&opts, &goi, version_found);
let mod_time = if let Some(mt) = opts.mod_time {
mt
@@ -2427,6 +2395,63 @@ impl ObjectOperations for SetDisks {
}
}
fn should_preserve_delete_replication_state(opts: &ObjectOptions) -> bool {
opts.delete_replication.as_ref().is_some_and(|state| {
state.replica_status == ReplicationStatusType::Replica
|| (!state.replicate_decision_str.is_empty()
&& (!state.composite_replication_status().is_empty() || !state.composite_version_purge_status().is_empty()))
}) || opts.version_purge_status() == VersionPurgeStatusType::Complete
}
fn resolve_delete_version_state(opts: &ObjectOptions, goi: &ObjectInfo, version_found: bool) -> (bool, bool) {
let mut mark_delete = goi.version_id.is_some();
let mut delete_marker = opts.versioned;
if opts.version_id.is_some() {
// Decommission/rebalance may recreate a delete marker on a new pool before that
// exact version exists there, so we must still treat it as a mark-delete write.
if opts.data_movement && opts.delete_marker && !version_found {
mark_delete = true;
}
let delete_marker_version_purge = version_found && goi.delete_marker && !opts.version_purge_status().is_empty();
if version_found && opts.delete_marker_replication_status() == ReplicationStatusType::Replica {
mark_delete = false;
}
if opts.version_purge_status().is_empty() && opts.delete_marker_replication_status().is_empty() {
mark_delete = false;
}
if opts.version_purge_status() == VersionPurgeStatusType::Complete {
mark_delete = false;
}
let replica_delete_marker_version_purge =
version_found && goi.delete_marker && opts.delete_marker_replication_status() == ReplicationStatusType::Replica;
if delete_marker_version_purge {
mark_delete = false;
}
if !version_found && !opts.delete_marker && opts.delete_marker_replication_status() == ReplicationStatusType::Replica {
delete_marker = false;
}
if version_found
&& (!goi.version_purge_status.is_empty()
|| !goi.delete_marker
|| replica_delete_marker_version_purge
|| delete_marker_version_purge)
{
delete_marker = false;
}
}
(mark_delete, delete_marker)
}
impl SetDisks {
#[tracing::instrument(skip(self, fi, opts))]
pub(crate) async fn decommission_tiered_object(
@@ -4337,6 +4362,7 @@ mod tests {
use crate::store_api::{CompletePart, ObjectInfo};
use crate::store_init::save_format_file;
use rustfs_filemeta::ErasureInfo;
use rustfs_filemeta::ReplicationState;
use rustfs_lock::client::local::LocalClient;
use rustfs_lock::{LockError, LockInfo, LockResponse, LockStats};
use serial_test::serial;
@@ -4564,6 +4590,137 @@ mod tests {
assert!(is_min_allowed_part_size(100 * 1024 * 1024)); // 100MB - allowed
}
#[test]
fn resolve_delete_version_state_clears_delete_marker_for_replica_marker_version_purge() {
let opts = ObjectOptions {
versioned: true,
version_id: Some(Uuid::new_v4().to_string()),
delete_replication: Some(ReplicationState {
replica_status: ReplicationStatusType::Replica,
..Default::default()
}),
..Default::default()
};
let current = ObjectInfo {
version_id: Some(Uuid::new_v4()),
delete_marker: true,
..Default::default()
};
let (mark_delete, delete_marker) = resolve_delete_version_state(&opts, &current, true);
assert!(!mark_delete);
assert!(
!delete_marker,
"replica purge of an existing delete marker version must remove that version, not preserve delete-marker semantics"
);
}
#[test]
fn resolve_delete_version_state_keeps_delete_marker_for_replica_marker_creation() {
let opts = ObjectOptions {
versioned: true,
version_id: Some(Uuid::new_v4().to_string()),
delete_marker: true,
delete_replication: Some(ReplicationState {
replica_status: ReplicationStatusType::Replica,
..Default::default()
}),
..Default::default()
};
let (mark_delete, delete_marker) = resolve_delete_version_state(&opts, &ObjectInfo::default(), false);
assert!(!mark_delete);
assert!(delete_marker);
}
#[test]
fn resolve_delete_version_state_skips_marker_creation_for_replica_purge_when_version_missing() {
let opts = ObjectOptions {
versioned: true,
version_id: Some(Uuid::new_v4().to_string()),
delete_replication: Some(ReplicationState {
replica_status: ReplicationStatusType::Replica,
..Default::default()
}),
..Default::default()
};
let (mark_delete, delete_marker) = resolve_delete_version_state(&opts, &ObjectInfo::default(), false);
assert!(
!mark_delete,
"replica delete-marker purges should not schedule mark-delete writes when the target version is absent"
);
assert!(
!delete_marker,
"replica delete-marker purges must become no-ops when the marker version has not arrived on the target yet"
);
}
#[test]
fn should_preserve_delete_replication_state_for_completed_delete_marker_replication_update() {
let opts = ObjectOptions {
version_id: Some(Uuid::new_v4().to_string()),
delete_replication: Some(ReplicationState {
replicate_decision_str: "target=true;false;target;".to_string(),
replication_status_internal: Some("target=COMPLETED;".to_string()),
targets: rustfs_filemeta::replication_statuses_map("target=COMPLETED;"),
..Default::default()
}),
..Default::default()
};
assert!(
should_preserve_delete_replication_state(&opts),
"source delete-marker replication status updates must not be re-evaluated as fresh delete replication requests"
);
}
#[test]
fn should_not_preserve_delete_replication_state_for_new_version_delete_request() {
let opts = ObjectOptions {
version_id: Some(Uuid::new_v4().to_string()),
..Default::default()
};
assert!(
!should_preserve_delete_replication_state(&opts),
"fresh versioned deletes still need replication eligibility checks"
);
}
#[test]
fn resolve_delete_version_state_removes_source_delete_marker_version_during_purge_replication() {
let opts = ObjectOptions {
versioned: true,
version_id: Some(Uuid::new_v4().to_string()),
delete_replication: Some(ReplicationState {
version_purge_status_internal: Some("target=PENDING;".to_string()),
purge_targets: rustfs_filemeta::version_purge_statuses_map("target=PENDING;"),
..Default::default()
}),
..Default::default()
};
let current = ObjectInfo {
version_id: Some(Uuid::new_v4()),
delete_marker: true,
..Default::default()
};
let (mark_delete, delete_marker) = resolve_delete_version_state(&opts, &current, true);
assert!(
!mark_delete,
"source delete-marker version purge should delete the local marker instead of rewriting it with purge metadata"
);
assert!(
!delete_marker,
"source delete-marker version purge should not leave delete-marker semantics behind locally"
);
}
#[test]
fn test_get_complete_multipart_md5() {
// Test MD5 calculation for multipart upload
+5 -2
View File
@@ -15,7 +15,9 @@
#![allow(clippy::map_entry)]
use crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
use crate::bucket::lifecycle::bucket_lifecycle_ops::{enqueue_transition_immediate, init_background_expiry};
use crate::bucket::lifecycle::bucket_lifecycle_ops::{
enqueue_immediate_expiry, enqueue_transition_immediate, init_background_expiry,
};
use crate::bucket::metadata_sys::{self, set_bucket_metadata};
use crate::bucket::utils::check_abort_multipart_args;
use crate::bucket::utils::check_complete_multipart_args;
@@ -134,7 +136,8 @@ async fn enqueue_transition_after_write(result: Result<ObjectInfo>, src: LcEvent
match result {
Ok(oi) => {
if should_enqueue_transition_immediately(&oi) {
enqueue_transition_immediate(&oi, src).await;
enqueue_transition_immediate(&oi, src.clone()).await;
enqueue_immediate_expiry(&oi, src).await;
}
Ok(oi)
}
+22
View File
@@ -446,6 +446,11 @@ impl ObjectInfo {
.replication_state_internal
.as_ref()
.and_then(|v| v.version_purge_status_internal.clone());
let replication_decision = fi
.replication_state_internal
.as_ref()
.map(|v| v.replicate_decision_str.clone())
.unwrap_or_default();
let mut replication_status = fi.replication_status();
if replication_status.is_empty()
@@ -542,6 +547,7 @@ impl ObjectInfo {
replication_status,
version_purge_status_internal,
version_purge_status,
replication_decision,
..Default::default()
}
}
@@ -1010,6 +1016,7 @@ pub struct ObjectInfoOrErr {
#[cfg(test)]
mod tests {
use super::*;
use rustfs_filemeta::ReplicationState;
#[test]
fn get_actual_size_prefers_actual_size_field() {
@@ -1059,6 +1066,21 @@ mod tests {
assert_eq!(info.get_actual_size().unwrap(), 77);
}
#[test]
fn from_file_info_preserves_replication_decision() {
let fi = rustfs_filemeta::FileInfo {
replication_state_internal: Some(ReplicationState {
replicate_decision_str: "arn=true;false;arn:replication::1:dest;rule-id".to_string(),
..Default::default()
}),
..Default::default()
};
let info = ObjectInfo::from_file_info(&fi, "bucket", "object", true);
assert_eq!(info.replication_decision, "arn=true;false;arn:replication::1:dest;rule-id");
}
#[test]
fn get_actual_size_uses_compressed_parts_actual_size_when_metadata_missing() {
let user_defined = {
+48
View File
@@ -382,6 +382,11 @@ impl FileMeta {
#[tracing::instrument(level = "debug", skip(self))]
pub fn delete_version(&mut self, fi: &FileInfo) -> Result<Option<Uuid>> {
let vid = Some(fi.version_id.unwrap_or(Uuid::nil()));
let target_is_delete_marker = self
.versions
.iter()
.find(|ver| ver.header.version_id == vid)
.is_some_and(|ver| ver.header.version_type == VersionType::Delete);
let mut ventry = FileMetaVersion::default();
if fi.deleted {
@@ -416,6 +421,10 @@ impl FileMeta {
}
}
if target_is_delete_marker && !fi.deleted && !fi.version_purge_status().is_empty() {
update_version = false;
}
if fi.deleted {
if !fi.delete_marker_replication_status().is_empty()
&& let Some(delete_marker) = ventry.delete_marker.as_mut()
@@ -1659,6 +1668,45 @@ mod test {
}
}
#[test]
fn delete_version_removes_delete_marker_during_version_purge_replication() {
let version_id = Uuid::new_v4();
let mut fm = FileMeta::new();
fm.add_version_filemata(FileMetaVersion {
version_type: VersionType::Delete,
legacy_object: None,
object: None,
delete_marker: Some(MetaDeleteMarker {
version_id: Some(version_id),
mod_time: Some(OffsetDateTime::now_utc()),
meta_sys: HashMap::new(),
}),
write_version: 1,
uses_legacy_checksum: false,
})
.unwrap();
let fi = FileInfo {
deleted: false,
mark_deleted: false,
version_id: Some(version_id),
replication_state_internal: Some(ReplicationState {
version_purge_status_internal: Some("target=PENDING;".to_string()),
purge_targets: version_purge_statuses_map("target=PENDING;"),
..Default::default()
}),
..Default::default()
};
let result = fm.delete_version(&fi).unwrap();
assert!(result.is_none());
assert!(
fm.versions.is_empty(),
"delete-marker version purge should remove the local marker instead of rewriting purge metadata onto it"
);
}
#[test]
fn test_data_integrity_validation() {
// Test data integrity checks
+1
View File
@@ -55,6 +55,7 @@ metrics = { workspace = true }
[dev-dependencies]
tracing-subscriber = { workspace = true }
serial_test = { workspace = true }
temp-env = { workspace = true }
uuid = { workspace = true, features = ["v4", "serde"] }
tokio = { workspace = true, features = ["test-util"] }
+47 -7
View File
@@ -22,10 +22,8 @@ use crate::{DataUsageInfo, ScannerError};
use chrono::{DateTime, Utc};
use rustfs_common::heal_channel::HealScanMode;
use rustfs_common::metrics::{CurrentCycle, Metric, Metrics, emit_scan_cycle_complete, global_metrics};
use rustfs_config::DEFAULT_SCANNER_SPEED;
use rustfs_config::ENV_SCANNER_SPEED;
use rustfs_config::ENV_SCANNER_START_DELAY_SECS;
use rustfs_config::ScannerSpeed;
use rustfs_config::{DEFAULT_SCANNER_SPEED, ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS};
use rustfs_ecstore::StorageAPI as _;
use rustfs_ecstore::config::com::{read_config, save_config};
use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
@@ -40,11 +38,15 @@ use tracing::{debug, error, info, instrument, warn};
const ENV_SCANNER_START_DELAY_SECS_DEPRECATED: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS";
/// Returns the base cycle interval. If `RUSTFS_SCANNER_START_DELAY_SECS`
/// is set (or `RUSTFS_DATA_SCANNER_START_DELAY_SECS` as deprecated alias),
/// it takes precedence; otherwise the value is derived from the
/// `RUSTFS_SCANNER_SPEED` preset.
/// Returns the base cycle interval.
/// Priority order:
/// 1. RUSTFS_SCANNER_CYCLE (if set, overrides everything)
/// 2. RUSTFS_SCANNER_START_DELAY_SECS (for backward compatibility)
/// 3. RUSTFS_SCANNER_SPEED preset
fn cycle_interval() -> Duration {
if let Some(secs) = rustfs_utils::get_env_opt_u64(ENV_SCANNER_CYCLE) {
return Duration::from_secs(secs);
}
if let Some(secs) = scanner_start_delay_secs() {
return Duration::from_secs(secs);
}
@@ -181,6 +183,7 @@ fn get_lock_acquire_timeout() -> Duration {
#[instrument(skip_all)]
async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>, cycle_info: &mut CurrentCycle) {
SCANNER_SLEEPER.refresh_from_env();
info!("Start run data scanner cycle");
cycle_info.current = cycle_info.next;
let now = Instant::now();
@@ -356,6 +359,7 @@ pub async fn store_data_usage_in_backend(
mod tests {
use super::*;
use serial_test::serial;
use temp_env::{with_var, with_var_unset};
#[test]
#[serial]
@@ -376,6 +380,42 @@ mod tests {
assert!(delay <= Duration::from_secs(132));
}
#[test]
#[serial]
fn test_cycle_interval_prefers_explicit_cycle_override() {
with_var(ENV_SCANNER_SPEED, Some("slowest"), || {
with_var(ENV_SCANNER_CYCLE, Some("42"), || {
assert_eq!(cycle_interval(), Duration::from_secs(42));
});
});
}
#[test]
#[serial]
fn test_cycle_interval_supports_minio_speed_alias() {
with_var_unset(ENV_SCANNER_SPEED, || {
with_var_unset(ENV_SCANNER_CYCLE, || {
with_var_unset(ENV_SCANNER_START_DELAY_SECS, || {
with_var("MINIO_SCANNER_SPEED", Some("slowest"), || {
assert_eq!(cycle_interval(), Duration::from_secs(30 * 60));
});
});
});
});
}
#[test]
#[serial]
fn test_cycle_interval_supports_minio_cycle_alias() {
with_var_unset(ENV_SCANNER_CYCLE, || {
with_var_unset(ENV_SCANNER_START_DELAY_SECS, || {
with_var("MINIO_SCANNER_CYCLE", Some("90"), || {
assert_eq!(cycle_interval(), Duration::from_secs(90));
});
});
});
}
#[test]
#[serial]
fn test_randomized_cycle_delay_handles_small_start_delay() {
+9 -3
View File
@@ -30,7 +30,7 @@ use rustfs_common::heal_channel::{
};
use rustfs_common::metrics::{IlmAction, Metric, Metrics, UpdateCurrentPathFn, current_path_updater};
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_rule;
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{GLOBAL_ExpiryState, apply_expiry_rule};
use rustfs_ecstore::bucket::lifecycle::evaluator::Evaluator;
use rustfs_ecstore::bucket::lifecycle::{
bucket_lifecycle_ops::apply_transition_rule,
@@ -444,8 +444,14 @@ impl ScannerItem {
}
}
if !to_delete_objs.is_empty() {
// TODO: enqueueNoncurrentVersions
if !to_delete_objs.is_empty()
&& let Some(event) = noncurrent_events.first().cloned()
{
GLOBAL_ExpiryState
.write()
.await
.enqueue_by_newer_noncurrent(&self.bucket, to_delete_objs, event)
.await;
}
self.alert_excessive_versions(remaining_versions, cumulative_size);
}
+36 -4
View File
@@ -21,6 +21,13 @@ use tokio::time::Duration;
const MIN_SLEEP: Duration = Duration::from_millis(1);
fn scanner_env_config() -> (ScannerSpeed, bool) {
let speed_str = rustfs_utils::get_env_str(ENV_SCANNER_SPEED, DEFAULT_SCANNER_SPEED);
let speed = ScannerSpeed::from_env_str(&speed_str);
let idle_mode = rustfs_utils::get_env_bool(ENV_SCANNER_IDLE_MODE, DEFAULT_SCANNER_IDLE_MODE);
(speed, idle_mode)
}
/// When `true` (default), the scanner throttles itself between operations.
/// When `false`, all sleeps are skipped and the scanner runs at full speed.
pub static SCANNER_IDLE_MODE: AtomicBool = AtomicBool::new(DEFAULT_SCANNER_IDLE_MODE);
@@ -28,10 +35,7 @@ pub static SCANNER_IDLE_MODE: AtomicBool = AtomicBool::new(DEFAULT_SCANNER_IDLE_
/// Global scanner sleeper initialized from the `RUSTFS_SCANNER_SPEED` and
/// `RUSTFS_SCANNER_IDLE_MODE` environment variables.
pub static SCANNER_SLEEPER: LazyLock<DynamicSleeper> = LazyLock::new(|| {
let speed_str = rustfs_utils::get_env_str(ENV_SCANNER_SPEED, DEFAULT_SCANNER_SPEED);
let speed = ScannerSpeed::from_env_str(&speed_str);
let idle_mode = rustfs_utils::get_env_bool(ENV_SCANNER_IDLE_MODE, DEFAULT_SCANNER_IDLE_MODE);
let (speed, idle_mode) = scanner_env_config();
SCANNER_IDLE_MODE.store(idle_mode, Ordering::Relaxed);
DynamicSleeper::new(speed)
@@ -104,6 +108,13 @@ impl DynamicSleeper {
let mut m = self.inner.max_sleep.write().unwrap_or_else(|e| e.into_inner());
*m = speed.max_sleep();
}
/// Reload speed and idle-mode settings from the current environment.
pub fn refresh_from_env(&self) {
let (speed, idle_mode) = scanner_env_config();
self.update(speed);
SCANNER_IDLE_MODE.store(idle_mode, Ordering::Relaxed);
}
}
/// A timer returned by [`DynamicSleeper::timer`]. Records the instant it
@@ -138,6 +149,7 @@ impl SleepTimer {
mod tests {
use super::*;
use serial_test::serial;
use temp_env::with_var;
#[test]
fn test_scanner_speed_presets() {
@@ -166,6 +178,26 @@ mod tests {
assert_eq!(max_sleep, Duration::from_secs(15));
}
#[test]
#[serial]
fn test_refresh_from_env_applies_speed_and_idle_mode_for_next_cycle() {
let prev_mode = SCANNER_IDLE_MODE.load(Ordering::Relaxed);
SCANNER_IDLE_MODE.store(true, Ordering::Relaxed);
let s = DynamicSleeper::new(ScannerSpeed::Fastest);
with_var(ENV_SCANNER_SPEED, Some("slow"), || {
with_var(ENV_SCANNER_IDLE_MODE, Some("false"), || {
s.refresh_from_env();
let (factor, max_sleep) = s.read_params();
assert_eq!(factor, 10.0);
assert_eq!(max_sleep, Duration::from_secs(15));
assert!(!SCANNER_IDLE_MODE.load(Ordering::Relaxed));
});
});
SCANNER_IDLE_MODE.store(prev_mode, Ordering::Relaxed);
}
#[tokio::test(start_paused = true)]
#[serial]
async fn test_fastest_never_sleeps() {
@@ -487,6 +487,89 @@ async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) -> usi
.len()
}
async fn object_version_count(disk_path: &Path, bucket: &str, object: &str) -> usize {
let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("failed to open local disk");
let data = disk
.read_metadata(bucket, &path_join_buf(&[object, STORAGE_FORMAT_FILE]))
.await
.expect("failed to read object metadata");
let meta = FileMeta::load(&data).expect("failed to load file metadata");
meta.get_file_info_versions(bucket, object, false)
.expect("failed to decode file info versions")
.versions
.len()
}
async fn wait_for_version_count(disk_path: &Path, bucket: &str, object: &str, expected: usize, timeout: Duration) -> bool {
let deadline = tokio::time::Instant::now() + timeout;
loop {
if object_version_count(disk_path, bucket, object).await == expected {
return true;
}
if tokio::time::Instant::now() >= deadline {
return false;
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
}
async fn scan_object_with_lifecycle(disk_path: &Path, bucket: &str, object: &str) {
let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("failed to open local disk");
let metadata_path = disk_path.join(bucket).join(object).join(STORAGE_FORMAT_FILE);
let relative_path = metadata_path.to_string_lossy().to_string();
let (_, scanner_path) = path2_bucket_object_with_base_path(disk_path.to_string_lossy().as_ref(), relative_path.as_str());
let file_type = fs::metadata(&metadata_path)
.await
.expect("failed to stat object metadata")
.file_type();
let lifecycle = metadata_sys::get(bucket)
.await
.expect("failed to load bucket metadata")
.lifecycle_config
.clone()
.map(Arc::new);
let item = ScannerItem {
path: scanner_path.clone(),
bucket: bucket.to_string(),
prefix: object.to_string(),
object_name: STORAGE_FORMAT_FILE.to_string(),
file_type,
lifecycle,
replication: None,
heal_enabled: false,
heal_bitrot: false,
debug: false,
};
disk.get_size(item).await.expect("scanner get_size should succeed");
}
async fn scan_object_metadata(disk_path: &Path, bucket: &str, object: &str) {
let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
endpoint.set_pool_index(0);
@@ -1131,4 +1214,255 @@ mod serial_tests {
"deleted object should remain absent after scanner cleanup"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
async fn test_scanner_expires_zero_day_current_version() {
let (disk_paths, ecstore) = setup_isolated_test_env(true).await;
let bucket_name = format!("test-zero-day-expire-{}", &Uuid::new_v4().simple().to_string()[..8]);
let object_name = "test/object.txt";
create_test_bucket(&ecstore, bucket_name.as_str()).await;
set_bucket_lifecycle(bucket_name.as_str())
.await
.expect("Failed to set lifecycle configuration");
upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await;
assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await);
scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;
assert!(
wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(3)).await,
"scanner should delete zero-day current version after enqueueing expiry"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
async fn test_put_object_immediately_enqueues_zero_day_current_expiry() {
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
let bucket_name = format!("test-put-zero-day-expire-{}", &Uuid::new_v4().simple().to_string()[..8]);
let object_name = "expire-now.txt";
create_test_bucket(&ecstore, bucket_name.as_str()).await;
let lifecycle_xml = format!(
r#"<?xml version="1.0" encoding="UTF-8"?>
<LifecycleConfiguration>
<Rule>
<ID>test-rule</ID>
<Status>Enabled</Status>
<Filter>
<Prefix>{object_name}</Prefix>
</Filter>
<Expiration>
<Days>0</Days>
</Expiration>
</Rule>
</LifecycleConfiguration>"#
);
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
.await
.expect("Failed to set lifecycle configuration");
upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await;
assert!(
wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(2)).await,
"put_object should enqueue zero-day current expiry without waiting for scanner"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
async fn test_scanner_expires_zero_day_noncurrent_version() {
let (disk_paths, ecstore) = setup_isolated_test_env(false).await;
let bucket_name = format!("test-zero-day-noncurrent-{}", &Uuid::new_v4().simple().to_string()[..8]);
let object_name = "test/object.txt";
create_test_lock_bucket(&ecstore, bucket_name.as_str()).await;
let mut reader = PutObjReader::from_vec(b"v1".to_vec());
ecstore
.put_object(
bucket_name.as_str(),
object_name,
&mut reader,
&ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("failed to upload v1");
let mut reader = PutObjReader::from_vec(b"v2".to_vec());
ecstore
.put_object(
bucket_name.as_str(),
object_name,
&mut reader,
&ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("failed to upload v2");
assert_eq!(object_version_count(&disk_paths[0], bucket_name.as_str(), object_name).await, 2);
let lifecycle_xml = r#"<?xml version="1.0" encoding="UTF-8"?>
<LifecycleConfiguration>
<Rule>
<ID>test-rule</ID>
<Status>Enabled</Status>
<Filter>
<Prefix>test/</Prefix>
</Filter>
<NoncurrentVersionExpiration>
<NoncurrentDays>0</NoncurrentDays>
</NoncurrentVersionExpiration>
</Rule>
</LifecycleConfiguration>"#;
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
.await
.expect("Failed to set noncurrent lifecycle configuration");
rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;
assert!(
wait_for_version_count(&disk_paths[0], bucket_name.as_str(), object_name, 1, Duration::from_secs(3)).await,
"scanner should delete zero-day noncurrent versions after enqueueing expiry"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
async fn test_put_object_immediately_enqueues_zero_day_noncurrent_expiry() {
let (disk_paths, ecstore) = setup_isolated_test_env(true).await;
let bucket_name = format!("test-put-zero-day-noncurrent-{}", &Uuid::new_v4().simple().to_string()[..8]);
let object_name = "test/object.txt";
create_test_lock_bucket(&ecstore, bucket_name.as_str()).await;
let lifecycle_xml = r#"<?xml version="1.0" encoding="UTF-8"?>
<LifecycleConfiguration>
<Rule>
<ID>test-rule</ID>
<Status>Enabled</Status>
<Filter>
<Prefix>test/</Prefix>
</Filter>
<NoncurrentVersionExpiration>
<NoncurrentDays>0</NoncurrentDays>
</NoncurrentVersionExpiration>
</Rule>
</LifecycleConfiguration>"#;
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
.await
.expect("Failed to set noncurrent lifecycle configuration");
let mut reader = PutObjReader::from_vec(b"v1".to_vec());
ecstore
.put_object(
bucket_name.as_str(),
object_name,
&mut reader,
&ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("failed to upload v1");
let mut reader = PutObjReader::from_vec(b"v2".to_vec());
ecstore
.put_object(
bucket_name.as_str(),
object_name,
&mut reader,
&ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("failed to upload v2");
assert!(
wait_for_version_count(&disk_paths[0], bucket_name.as_str(), object_name, 1, Duration::from_secs(2)).await,
"put_object should enqueue zero-day noncurrent expiry without waiting for scanner"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
async fn test_background_scanner_expires_zero_day_current_version() {
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
let bucket_name = format!("test-bg-zero-day-expire-{}", &Uuid::new_v4().simple().to_string()[..8]);
let object_name = "test/object.txt";
create_test_bucket(&ecstore, bucket_name.as_str()).await;
set_bucket_lifecycle(bucket_name.as_str())
.await
.expect("Failed to set lifecycle configuration");
upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await;
let ctx = CancellationToken::new();
init_data_scanner(ctx.clone(), ecstore.clone()).await;
let deleted = wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(12)).await;
ctx.cancel();
assert!(deleted, "background scanner should delete zero-day current version after startup delay");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
async fn test_background_scanner_expires_zero_day_current_version_for_exact_key_prefix() {
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
let bucket_name = format!("test-bg-zero-day-exact-{}", &Uuid::new_v4().simple().to_string()[..8]);
let object_name = "expire-now.txt";
create_test_bucket(&ecstore, bucket_name.as_str()).await;
let lifecycle_xml = format!(
r#"<?xml version="1.0" encoding="UTF-8"?>
<LifecycleConfiguration>
<Rule>
<ID>test-rule</ID>
<Status>Enabled</Status>
<Filter>
<Prefix>{object_name}</Prefix>
</Filter>
<Expiration>
<Days>0</Days>
</Expiration>
</Rule>
</LifecycleConfiguration>"#
);
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
.await
.expect("Failed to set lifecycle configuration");
upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await;
let ctx = CancellationToken::new();
init_data_scanner(ctx.clone(), ecstore.clone()).await;
let deleted = wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(12)).await;
ctx.cancel();
assert!(deleted, "background scanner should delete zero-day exact-key lifecycle targets");
}
}
+2
View File
@@ -173,6 +173,8 @@ const EXTERNAL_COMPATIBLE_SUFFIXES: &[&str] = &[
"REGION",
"ROOT_PASSWORD",
"ROOT_USER",
"SCANNER_CYCLE",
"SCANNER_SPEED",
"SECRET_KEY",
"SECRET_KEY_FILE",
"STORAGE_CLASS_INLINE_BLOCK",
+22 -1
View File
@@ -34,7 +34,9 @@ use http::StatusCode;
use metrics::counter;
use rustfs_config::RUSTFS_REGION;
use rustfs_ecstore::bucket::{
lifecycle::bucket_lifecycle_ops::{enqueue_transition_for_existing_objects, validate_transition_tier},
lifecycle::bucket_lifecycle_ops::{
enqueue_expiry_for_existing_objects, enqueue_transition_for_existing_objects, validate_transition_tier,
},
metadata::{
BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG,
BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_REPLICATION_CONFIG, BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG,
@@ -494,6 +496,13 @@ fn lifecycle_has_transition_rules(config: &BucketLifecycleConfiguration) -> bool
})
}
fn lifecycle_has_expiry_rules(config: &BucketLifecycleConfiguration) -> bool {
config.rules.iter().any(|rule| {
rule.status == ExpirationStatus::from_static(ExpirationStatus::ENABLED)
&& (rule.expiration.is_some() || rule.del_marker_expiration.is_some() || rule.noncurrent_version_expiration.is_some())
})
}
#[derive(Clone, Default)]
pub struct DefaultBucketUsecase {
context: Option<Arc<AppContext>>,
@@ -1342,6 +1351,18 @@ impl DefaultBucketUsecase {
});
}
if lifecycle_has_expiry_rules(&input_cfg)
&& let Some(store) = new_object_layer_fn()
{
let bucket_name = bucket.clone();
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
spawn_background_with_context(request_context, async move {
if let Err(err) = enqueue_expiry_for_existing_objects(store, &bucket_name).await {
warn!(bucket = %bucket_name, error = ?err, "failed to enqueue expiry for existing objects");
}
});
}
Ok(S3Response::new(PutBucketLifecycleConfigurationOutput::default()))
}
@@ -13,6 +13,7 @@
// limitations under the License.
use super::{multipart_usecase::DefaultMultipartUsecase, object_usecase::DefaultObjectUsecase};
use crate::app::bucket_usecase::DefaultBucketUsecase;
use crate::storage::ecfs::FS;
use bytes::Bytes;
use futures::stream;
@@ -182,6 +183,40 @@ async fn set_bucket_lifecycle_transition_with_tier(
Ok(())
}
fn expiration_lifecycle_configuration(prefix: &str) -> BucketLifecycleConfiguration {
BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
expiration: Some(LifecycleExpiration {
date: Some(Timestamp::from(
time::OffsetDateTime::now_utc()
.replace_time(time::Time::MIDNIGHT)
.saturating_sub(time::Duration::days(1)),
)),
days: None,
expired_object_delete_marker: None,
..Default::default()
}),
filter: Some(LifecycleRuleFilter {
and: None,
object_size_greater_than: None,
object_size_less_than: None,
prefix: Some(prefix.to_string()),
tag: None,
..Default::default()
}),
id: Some("expire-existing".to_string()),
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
}],
}
}
#[derive(Clone, Default)]
struct MockWarmBackend {
objects: Arc<Mutex<HashMap<String, Vec<u8>>>>,
@@ -326,6 +361,24 @@ async fn wait_for_object_absence(ecstore: &Arc<ECStore>, bucket: &str, object: &
}
}
async fn wait_for_delete_marker(ecstore: &Arc<ECStore>, bucket: &str, object: &str, timeout: Duration) -> bool {
let deadline = tokio::time::Instant::now() + timeout;
loop {
if let Ok(info) = ecstore.get_object_info(bucket, object, &ObjectOptions::default()).await
&& info.delete_marker
{
return true;
}
if tokio::time::Instant::now() >= deadline {
return false;
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
}
fn build_request<T>(input: T, method: Method) -> S3Request<T> {
S3Request {
input,
@@ -583,3 +636,37 @@ async fn lifecycle_transition_marks_dirty_disks_for_capacity_manager() {
.collect();
assert_eq!(actual_paths, expected_paths);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
#[ignore = "requires isolated global object layer state"]
async fn put_bucket_lifecycle_configuration_expires_existing_objects() {
let (_disk_paths, ecstore) = setup_test_env().await;
let usecase = DefaultBucketUsecase::without_context();
let bucket = format!("test-api-expire-existing-{}", &Uuid::new_v4().simple().to_string()[..8]);
let object = "test/existing.txt";
let payload = b"expire existing object after lifecycle update";
create_test_bucket(&ecstore, bucket.as_str()).await;
let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
let req = build_request(
PutBucketLifecycleConfigurationInput::builder()
.bucket(bucket.clone())
.lifecycle_configuration(Some(expiration_lifecycle_configuration("test/")))
.build()
.unwrap(),
Method::PUT,
);
usecase
.execute_put_bucket_lifecycle_configuration(req)
.await
.expect("Failed to update lifecycle configuration");
assert!(
wait_for_delete_marker(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT).await,
"existing object should be lifecycle-deleted after lifecycle update"
);
}
+268 -16
View File
@@ -58,8 +58,8 @@ use rustfs_ecstore::bucket::{
},
quota::QuotaOperation,
replication::{
DeletedObjectReplicationInfo, check_replicate_delete, get_must_replicate_options, must_replicate, schedule_replication,
schedule_replication_delete,
DeletedObjectReplicationInfo, ObjectOpts as ReplicationObjectOpts, ReplicationConfigurationExt, check_replicate_delete,
get_must_replicate_options, must_replicate, schedule_replication, schedule_replication_delete,
},
tagging::decode_tags,
versioning::VersioningApi,
@@ -75,8 +75,9 @@ use rustfs_ecstore::store_api::{
HTTPRangeSpec, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, PutObjReader,
};
use rustfs_filemeta::{
REPLICATE_INCOMING_DELETE, ReplicationStatusType, ReplicationType, RestoreStatusOps, VersionPurgeStatusType,
parse_restore_obj_status,
REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateTargetDecision, ReplicationState, ReplicationStatusType,
ReplicationType, RestoreStatusOps, VersionPurgeStatusType, parse_restore_obj_status, replication_statuses_map,
version_purge_statuses_map,
};
use rustfs_io_metrics;
use rustfs_notify::EventArgsBuilder;
@@ -460,6 +461,100 @@ fn build_put_object_expiration_header(event: &lifecycle::Event) -> Option<String
Some(format!("expiry-date=\"{}\", rule-id=\"{}\"", expiry_date, event.rule_id))
}
fn delete_replication_state_from_config(
config: &ReplicationConfiguration,
obj_info: &ObjectInfo,
version_id: Option<Uuid>,
replica: bool,
) -> Option<ReplicationState> {
let opts = ReplicationObjectOpts {
name: obj_info.name.clone(),
user_tags: obj_info.user_tags.clone(),
version_id,
delete_marker: obj_info.delete_marker,
op_type: ReplicationType::Delete,
replica,
..Default::default()
};
let target_arns = config.filter_target_arns(&opts);
if target_arns.is_empty() {
return None;
}
let mut decision = ReplicateDecision::new();
for target_arn in target_arns {
let mut target_opts = opts.clone();
target_opts.target_arn = target_arn.clone();
decision.set(ReplicateTargetDecision::new(target_arn, config.replicate(&target_opts), false));
}
if !decision.replicate_any() {
return None;
}
let pending_status = decision.pending_status();
let mut state = ReplicationState {
replicate_decision_str: decision.to_string(),
..Default::default()
};
if version_id.is_some() {
state.version_purge_status_internal = pending_status.clone();
state.purge_targets = version_purge_statuses_map(pending_status.as_deref().unwrap_or_default());
} else {
state.replication_status_internal = pending_status.clone();
state.targets = replication_statuses_map(pending_status.as_deref().unwrap_or_default());
}
Some(state)
}
async fn enrich_delete_replication_state_if_needed(
bucket: &str,
delete_object: &mut rustfs_ecstore::store_api::DeletedObject,
obj_info: &ObjectInfo,
) {
let Some(replication_state) = delete_object.replication_state.as_ref() else {
return;
};
if !replication_state.replicate_decision_str.is_empty()
&& (!replication_state.targets.is_empty() || !replication_state.purge_targets.is_empty())
{
return;
}
let Ok((config, _)) = metadata_sys::get_replication_config(bucket).await else {
return;
};
let version_id = if delete_object.delete_marker {
None
} else if delete_object.delete_marker_version_id.is_some() {
delete_object.delete_marker_version_id
} else {
delete_object.version_id
};
if let Some(local_state) = delete_replication_state_from_config(
&config,
obj_info,
version_id,
obj_info.replication_status == ReplicationStatusType::Replica,
) {
delete_object.replication_state = Some(local_state);
}
}
fn should_schedule_delete_replication(
opts: &ObjectOptions,
replication_source: &ObjectInfo,
deleted_delete_marker_version: bool,
) -> bool {
if opts.replication_request {
return false;
}
replication_source.replication_status == ReplicationStatusType::Replica
|| replication_source.replication_status == ReplicationStatusType::Pending
|| replication_source.version_purge_status == VersionPurgeStatusType::Pending
|| (deleted_delete_marker_version && replication_source.replication_status == ReplicationStatusType::Completed)
}
const AMZ_SNOWBALL_EXTRACT_COMPAT: &str = "X-Amz-Snowball-Auto-Extract";
#[cfg(test)]
const AMZ_SNOWBALL_PREFIX_INTERNAL: &str = "X-Amz-Meta-Rustfs-Snowball-Prefix";
@@ -1904,6 +1999,15 @@ impl DefaultObjectUsecase {
None
};
// x-amz-restore: extract from object metadata
let restore = info.user_defined.get(X_AMZ_RESTORE.as_str()).and_then(|v| {
let rs = parse_restore_obj_status(v).ok()?;
Some(rs.to_string2())
});
// x-amz-expiration: predict from lifecycle configuration
let expiration = resolve_put_object_expiration(bucket, &info).await;
let output = GetObjectOutput {
body,
content_length: Some(response_content_length),
@@ -1925,6 +2029,8 @@ impl DefaultObjectUsecase {
checksum_crc64nvme: checksums.crc64nvme,
checksum_type: checksums.checksum_type,
version_id: output_version_id,
restore,
expiration,
..Default::default()
};
@@ -3126,25 +3232,35 @@ impl DefaultObjectUsecase {
return result;
}
if obj_info.replication_status == ReplicationStatusType::Replica
|| obj_info.replication_status == ReplicationStatusType::Pending
|| obj_info.version_purge_status == VersionPurgeStatusType::Pending
{
schedule_replication_delete(DeletedObjectReplicationInfo {
let deleted_replication_info = existing_object_info.as_ref().filter(|_| opts.version_id.is_some());
let replication_source = deleted_replication_info.unwrap_or(&obj_info);
let deleted_delete_marker_version = deleted_replication_info.is_some_and(|info| info.delete_marker);
if should_schedule_delete_replication(&opts, replication_source, deleted_delete_marker_version) {
let mut deleted_object = DeletedObjectReplicationInfo {
delete_object: rustfs_ecstore::store_api::DeletedObject {
delete_marker: obj_info.delete_marker,
delete_marker_version_id: if obj_info.delete_marker { obj_info.version_id } else { None },
delete_marker: replication_source.delete_marker && !deleted_delete_marker_version,
delete_marker_version_id: if replication_source.delete_marker {
replication_source.version_id
} else {
None
},
object_name: key.clone(),
version_id: if obj_info.delete_marker { None } else { obj_info.version_id },
delete_marker_mtime: obj_info.mod_time,
replication_state: Some(obj_info.replication_state()),
version_id: if replication_source.delete_marker {
None
} else {
replication_source.version_id
},
delete_marker_mtime: replication_source.mod_time,
replication_state: Some(replication_source.replication_state()),
..Default::default()
},
bucket: bucket.clone(),
event_type: REPLICATE_INCOMING_DELETE.to_string(),
..Default::default()
})
.await;
};
enrich_delete_replication_state_if_needed(&bucket, &mut deleted_object.delete_object, replication_source).await;
schedule_replication_delete(deleted_object).await;
}
let delete_marker = obj_info.delete_marker;
@@ -3299,6 +3415,8 @@ impl DefaultObjectUsecase {
req.input.sse_customer_key_md5.as_ref(),
)?;
// Compute x-amz-expiration header from lifecycle prediction (before info is partially moved)
let expiration_header = resolve_put_object_expiration(&bucket, &info).await;
let event_info = info.clone();
let content_type = {
if let Some(content_type) = &info.content_type {
@@ -3411,6 +3529,13 @@ impl DefaultObjectUsecase {
checksum_crc64nvme,
checksum_type,
storage_class,
// x-amz-restore from object metadata
restore: metadata_map.get(X_AMZ_RESTORE.as_str()).and_then(|v| {
let rs = parse_restore_obj_status(v).ok()?;
Some(rs.to_string2())
}),
// x-amz-expiration from lifecycle prediction
expiration: expiration_header,
// metadata: object_metadata,
..Default::default()
};
@@ -4197,6 +4322,10 @@ fn object_attributes_requested(object_attributes: &[ObjectAttributes], name: &'s
mod tests {
use super::*;
use http::{Extensions, HeaderMap, HeaderName, HeaderValue, Method, Uri};
use s3s::dto::{
DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ExistingObjectReplication,
ExistingObjectReplicationStatus, ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus,
};
fn build_request<T>(input: T, method: Method) -> S3Request<T> {
S3Request {
@@ -4606,6 +4735,44 @@ mod tests {
assert_eq!(err.code(), &S3ErrorCode::InternalError);
}
#[test]
fn should_schedule_delete_replication_skips_replica_requests() {
let opts = ObjectOptions {
replication_request: true,
version_id: Some(Uuid::new_v4().to_string()),
..Default::default()
};
let replication_source = ObjectInfo {
delete_marker: true,
replication_status: ReplicationStatusType::Completed,
..Default::default()
};
assert!(
!should_schedule_delete_replication(&opts, &replication_source, true),
"replica delete requests on target sites must not enqueue a second replication delete task"
);
}
#[test]
fn should_schedule_delete_replication_keeps_delete_marker_version_purge_from_source() {
let opts = ObjectOptions {
replication_request: false,
version_id: Some(Uuid::new_v4().to_string()),
..Default::default()
};
let replication_source = ObjectInfo {
delete_marker: true,
replication_status: ReplicationStatusType::Completed,
..Default::default()
};
assert!(
should_schedule_delete_replication(&opts, &replication_source, true),
"source-side delete-marker version purge still needs replication scheduling"
);
}
#[tokio::test]
async fn execute_get_object_attributes_returns_internal_error_when_store_uninitialized() {
let input = GetObjectAttributesInput::builder()
@@ -4772,4 +4939,89 @@ mod tests {
let err = usecase.execute_restore_object(req).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InternalError);
}
#[test]
fn delete_replication_state_from_config_tracks_downstream_delete_marker_targets() {
let arn = "arn:aws:s3:::target-bucket".to_string();
let config = ReplicationConfiguration {
role: arn.clone(),
rules: vec![ReplicationRule {
delete_marker_replication: Some(DeleteMarkerReplication {
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
}),
delete_replication: None,
destination: Destination {
bucket: arn.clone(),
..Default::default()
},
existing_object_replication: Some(ExistingObjectReplication {
status: ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::ENABLED),
}),
filter: None,
id: Some("rule-1".to_string()),
prefix: Some("test/".to_string()),
priority: Some(1),
source_selection_criteria: None,
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
}],
};
let obj_info = ObjectInfo {
bucket: "bucket".to_string(),
name: "test/object.txt".to_string(),
delete_marker: true,
replication_status: ReplicationStatusType::Replica,
..Default::default()
};
let state = delete_replication_state_from_config(&config, &obj_info, None, true)
.expect("replica delete marker should be forwarded to downstream targets");
let pending = format!("{arn}=PENDING;");
assert_eq!(state.replication_status_internal.as_deref(), Some(pending.as_str()));
assert_eq!(state.replicate_decision_str, format!("{arn}=true;false;{arn};"));
assert!(state.targets.contains_key(&arn));
}
#[test]
fn delete_replication_state_from_config_tracks_delete_marker_version_purges() {
let arn = "arn:aws:s3:::target-bucket".to_string();
let config = ReplicationConfiguration {
role: arn.clone(),
rules: vec![ReplicationRule {
delete_marker_replication: Some(DeleteMarkerReplication {
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
}),
delete_replication: None,
destination: Destination {
bucket: arn.clone(),
..Default::default()
},
existing_object_replication: Some(ExistingObjectReplication {
status: ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::ENABLED),
}),
filter: None,
id: Some("rule-1".to_string()),
prefix: Some("test/".to_string()),
priority: Some(1),
source_selection_criteria: None,
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
}],
};
let obj_info = ObjectInfo {
bucket: "bucket".to_string(),
name: "test/object.txt".to_string(),
delete_marker: true,
replication_status: ReplicationStatusType::Completed,
..Default::default()
};
let version_id = Some(Uuid::new_v4());
let state = delete_replication_state_from_config(&config, &obj_info, version_id, false)
.expect("delete-marker version purge should honor delete-marker replication rules");
let pending = format!("{arn}=PENDING;");
assert_eq!(state.version_purge_status_internal.as_deref(), Some(pending.as_str()));
assert_eq!(state.replicate_decision_str, format!("{arn}=true;false;{arn};"));
assert!(state.purge_targets.contains_key(&arn));
}
}