mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
feat(lifecycle): improve ILM compatibility and scanner runtime config (#2534)
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> Co-authored-by: cxymds <Cxymds@qq.com>
This commit is contained in:
Generated
+1
@@ -8507,6 +8507,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serial_test",
|
||||
"temp-env",
|
||||
"thiserror 2.0.18",
|
||||
"time",
|
||||
"tokio",
|
||||
|
||||
@@ -511,7 +511,7 @@ pub fn lc_has_active_rules(config: &BucketLifecycleConfiguration, prefix: &str)
|
||||
}
|
||||
|
||||
if let Some(e) = &rule.noncurrent_version_expiration {
|
||||
if let Some(true) = e.noncurrent_days.map(|d| d > 0) {
|
||||
if e.noncurrent_days.is_some() {
|
||||
return true;
|
||||
}
|
||||
if let Some(true) = e.newer_noncurrent_versions.map(|d| d > 0) {
|
||||
|
||||
@@ -53,8 +53,11 @@ Current guidance:
|
||||
|
||||
## Scanner environment aliases
|
||||
|
||||
- `RUSTFS_SCANNER_SPEED` (canonical, also accepts `MINIO_SCANNER_SPEED`)
|
||||
- `RUSTFS_SCANNER_CYCLE` (canonical, also accepts `MINIO_SCANNER_CYCLE`)
|
||||
- `RUSTFS_SCANNER_START_DELAY_SECS` (canonical)
|
||||
- `RUSTFS_DATA_SCANNER_START_DELAY_SECS` (deprecated alias for compatibility)
|
||||
- `RUSTFS_SCANNER_IDLE_MODE` (canonical)
|
||||
|
||||
## Drive timeout environment variables
|
||||
|
||||
|
||||
@@ -25,6 +25,12 @@ pub const ENV_SCANNER_START_DELAY_SECS: &str = "RUSTFS_SCANNER_START_DELAY_SECS"
|
||||
#[deprecated(note = "Use RUSTFS_SCANNER_START_DELAY_SECS instead")]
|
||||
pub const ENV_DATA_SCANNER_START_DELAY_SECS: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS";
|
||||
|
||||
/// Environment variable that specifies the scanner cycle interval in seconds.
|
||||
/// If set, this overrides the cycle interval derived from `RUSTFS_SCANNER_SPEED`.
|
||||
/// - Unit: seconds (u64).
|
||||
/// - Example: `export RUSTFS_SCANNER_CYCLE=3600` (1 hour)
|
||||
pub const ENV_SCANNER_CYCLE: &str = "RUSTFS_SCANNER_CYCLE";
|
||||
|
||||
/// Environment variable that selects the scanner speed preset.
|
||||
/// Valid values: `fastest`, `fast`, `default`, `slow`, `slowest`.
|
||||
/// Controls the sleep factor, maximum sleep duration, and cycle interval.
|
||||
@@ -65,6 +71,8 @@ pub const DEFAULT_SCANNER_INLINE_HEAL_ENABLE: bool = false;
|
||||
/// | `default` | 2x | 1 second | 1 minute |
|
||||
/// | `slow` | 10x | 15 seconds| 1 minute |
|
||||
/// | `slowest` | 100x | 15 seconds| 30 minutes |
|
||||
///
|
||||
/// The cycle interval can be overridden by `RUSTFS_SCANNER_CYCLE`.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum ScannerSpeed {
|
||||
Fastest,
|
||||
|
||||
@@ -160,7 +160,7 @@ async fn test_bucket_lifecycle_configuration() -> Result<(), Box<dyn std::error:
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
#[ignore = "requires running RustFS server at localhost:9000"]
|
||||
async fn test_bucket_lifecycle_rejects_zero_days() -> Result<(), Box<dyn std::error::Error>> {
|
||||
async fn test_bucket_lifecycle_accepts_zero_days() -> Result<(), Box<dyn std::error::Error>> {
|
||||
use aws_sdk_s3::types::{BucketLifecycleConfiguration, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter};
|
||||
|
||||
let client = create_aws_s3_client().await?;
|
||||
@@ -176,19 +176,12 @@ async fn test_bucket_lifecycle_rejects_zero_days() -> Result<(), Box<dyn std::er
|
||||
.build()?;
|
||||
let lifecycle = BucketLifecycleConfiguration::builder().rules(rule).build()?;
|
||||
|
||||
let err = client
|
||||
client
|
||||
.put_bucket_lifecycle_configuration()
|
||||
.bucket(BUCKET)
|
||||
.lifecycle_configuration(lifecycle)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("zero-day lifecycle expiration should be rejected");
|
||||
|
||||
let err_msg = format!("{err:?}");
|
||||
assert!(
|
||||
err_msg.contains("InvalidArgument") && err_msg.contains("greater than 0"),
|
||||
"unexpected error: {err_msg}"
|
||||
);
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -897,6 +897,41 @@ pub struct RemoveObjectOptions {
|
||||
pub replication_validity_check: bool,
|
||||
}
|
||||
|
||||
fn build_remove_object_headers(version_id: Option<&str>, opts: &RemoveObjectOptions) -> HeaderMap {
|
||||
let mut headers = HeaderMap::new();
|
||||
if opts.force_delete {
|
||||
insert_header(&mut headers, SUFFIX_FORCE_DELETE, "true");
|
||||
}
|
||||
if opts.governance_bypass {
|
||||
headers.insert(AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, "true".parse().unwrap());
|
||||
}
|
||||
|
||||
if opts.replication_delete_marker {
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_DELETEMARKER, "true");
|
||||
}
|
||||
|
||||
if let Some(t) = opts.replication_mtime {
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_MTIME, t.format(&Rfc3339).unwrap_or_default());
|
||||
}
|
||||
|
||||
if !opts.replication_status.is_empty() {
|
||||
headers.insert(AMZ_BUCKET_REPLICATION_STATUS, opts.replication_status.as_str().parse().unwrap());
|
||||
}
|
||||
|
||||
if let Some(version_id) = version_id {
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_VERSION_ID, version_id);
|
||||
}
|
||||
|
||||
if opts.replication_request {
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||
}
|
||||
if opts.replication_validity_check {
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true");
|
||||
}
|
||||
|
||||
headers
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AdvancedPutOptions {
|
||||
pub source_version_id: String,
|
||||
@@ -1427,39 +1462,15 @@ impl TargetClient {
|
||||
version_id: Option<String>,
|
||||
opts: RemoveObjectOptions,
|
||||
) -> Result<(), S3ClientError> {
|
||||
let mut headers = HeaderMap::new();
|
||||
if opts.force_delete {
|
||||
insert_header(&mut headers, SUFFIX_FORCE_DELETE, "true");
|
||||
}
|
||||
if opts.governance_bypass {
|
||||
headers.insert(AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, "true".parse().unwrap());
|
||||
}
|
||||
|
||||
if opts.replication_delete_marker {
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_DELETEMARKER, "true");
|
||||
}
|
||||
|
||||
if let Some(t) = opts.replication_mtime {
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_MTIME, t.format(&Rfc3339).unwrap_or_default());
|
||||
}
|
||||
|
||||
if !opts.replication_status.is_empty() {
|
||||
headers.insert(AMZ_BUCKET_REPLICATION_STATUS, opts.replication_status.as_str().parse().unwrap());
|
||||
}
|
||||
|
||||
if opts.replication_request {
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||
}
|
||||
if opts.replication_validity_check {
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true");
|
||||
}
|
||||
let headers = build_remove_object_headers(version_id.as_deref(), &opts);
|
||||
let api_version_id = if opts.replication_request { None } else { version_id };
|
||||
|
||||
match self
|
||||
.client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.set_version_id(version_id)
|
||||
.set_version_id(api_version_id)
|
||||
.customize()
|
||||
.map_request(move |mut req| {
|
||||
for (k, v) in headers.clone().into_iter() {
|
||||
@@ -1553,3 +1564,53 @@ impl From<std::io::Error> for BucketTargetError {
|
||||
}
|
||||
|
||||
impl Error for BucketTargetError {}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn build_remove_object_headers_includes_internal_version_id_for_replication_delete() {
|
||||
let version_id = Uuid::new_v4().to_string();
|
||||
let headers = build_remove_object_headers(
|
||||
Some(version_id.as_str()),
|
||||
&RemoveObjectOptions {
|
||||
force_delete: false,
|
||||
governance_bypass: false,
|
||||
replication_delete_marker: true,
|
||||
replication_mtime: None,
|
||||
replication_status: ReplicationStatusType::Replica,
|
||||
replication_request: true,
|
||||
replication_validity_check: false,
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_header(&headers, SUFFIX_SOURCE_VERSION_ID).as_deref(),
|
||||
Some(version_id.as_str()),
|
||||
"replication delete requests must preserve the version id in internal headers"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_remove_object_headers_omits_delete_marker_flag_for_marker_version_purge() {
|
||||
let version_id = Uuid::new_v4().to_string();
|
||||
let headers = build_remove_object_headers(
|
||||
Some(version_id.as_str()),
|
||||
&RemoveObjectOptions {
|
||||
force_delete: false,
|
||||
governance_bypass: false,
|
||||
replication_delete_marker: false,
|
||||
replication_mtime: None,
|
||||
replication_status: ReplicationStatusType::Replica,
|
||||
replication_request: true,
|
||||
replication_validity_check: false,
|
||||
},
|
||||
);
|
||||
|
||||
assert!(
|
||||
rustfs_utils::http::get_header(&headers, SUFFIX_SOURCE_DELETEMARKER).is_none(),
|
||||
"delete-marker version purges must not masquerade as delete-marker creations"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,20 +11,18 @@
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use crate::bucket::lifecycle::bucket_lifecycle_audit::{LcAuditEvent, LcEventSrc};
|
||||
use crate::bucket::lifecycle::evaluator::Evaluator;
|
||||
use crate::bucket::lifecycle::lifecycle::{
|
||||
self, ExpirationOptions, Lifecycle, ObjectOpts, TransitionOptions, abort_incomplete_multipart_upload_due,
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats};
|
||||
use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier};
|
||||
use crate::bucket::object_lock::objectlock_sys::check_object_lock_for_deletion;
|
||||
use crate::bucket::replication::{
|
||||
DeletedObjectReplicationInfo, ReplicationConfig, check_replicate_delete, schedule_replication_delete,
|
||||
};
|
||||
use crate::bucket::{metadata_sys, metadata_sys::get_lifecycle_config, versioning_sys::BucketVersioningSys};
|
||||
use crate::client::object_api_utils::new_getobjectreader;
|
||||
use crate::disk::error::DiskError;
|
||||
@@ -37,34 +35,33 @@ use crate::global::GLOBAL_LocalNodeName;
|
||||
use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id};
|
||||
use crate::set_disk::{MAX_PARTS_COUNT, RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, SetDisks};
|
||||
use crate::store::ECStore;
|
||||
use crate::store_api::StorageAPI;
|
||||
use crate::store_api::{
|
||||
GetObjectReader, HTTPRangeSpec, ListOperations, MultipartOperations, ObjectInfo, ObjectOperations, ObjectOptions,
|
||||
ObjectToDelete,
|
||||
};
|
||||
use crate::tier::warm_backend::WarmBackendGetOpts;
|
||||
use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
|
||||
use bytes::BytesMut;
|
||||
use futures::Future;
|
||||
use http::HeaderMap;
|
||||
use lazy_static::lazy_static;
|
||||
use rustfs_common::data_usage::TierStats;
|
||||
use rustfs_common::heal_channel::rep_has_active_rules;
|
||||
use rustfs_common::metrics::{IlmAction, Metrics};
|
||||
use rustfs_filemeta::{FileInfo, FileInfoOpts, NULL_VERSION_ID, RestoreStatusOps, get_file_info, is_restored_object_on_disk};
|
||||
use rustfs_filemeta::{
|
||||
FileInfo, FileInfoOpts, NULL_VERSION_ID, REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicationState, RestoreStatusOps,
|
||||
VersionPurgeStatusType, get_file_info, is_restored_object_on_disk,
|
||||
};
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::strings_has_prefix_fold};
|
||||
use s3s::Body;
|
||||
use s3s::dto::{
|
||||
BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration, RestoreRequest, RestoreRequestType, RestoreStatus,
|
||||
ServerSideEncryption, Timestamp,
|
||||
Timestamp,
|
||||
};
|
||||
use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION, X_AMZ_STORAGE_CLASS};
|
||||
use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::any::Any;
|
||||
use std::collections::HashMap;
|
||||
use std::env;
|
||||
use std::io::Write;
|
||||
use std::pin::Pin;
|
||||
use std::sync::atomic::{AtomicI64, Ordering};
|
||||
use std::sync::{Arc, Mutex, Weak};
|
||||
@@ -73,7 +70,7 @@ use time::OffsetDateTime;
|
||||
use tokio::select;
|
||||
use tokio::sync::mpsc::{Receiver, Sender};
|
||||
use tokio::sync::{RwLock, mpsc};
|
||||
use tracing::{debug, error, info, warn};
|
||||
use tracing::{error, info, warn};
|
||||
use uuid::Uuid;
|
||||
use xxhash_rust::xxh64;
|
||||
|
||||
@@ -111,7 +108,7 @@ impl LifecycleSys {
|
||||
pub async fn get(&self, bucket: &str) -> Option<BucketLifecycleConfiguration> {
|
||||
match get_lifecycle_config(bucket).await {
|
||||
Ok((lc, _)) => Some(lc),
|
||||
Err(err) if err == Error::ConfigNotFound => None,
|
||||
Err(Error::ConfigNotFound) => None,
|
||||
Err(err) => {
|
||||
warn!(bucket, error = ?err, "failed to load lifecycle config");
|
||||
None
|
||||
@@ -119,8 +116,25 @@ impl LifecycleSys {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn trace(_oi: &ObjectInfo) -> TraceFn {
|
||||
Arc::new(|_oi, _ctx| Box::pin(async move {}))
|
||||
pub fn trace(oi: &ObjectInfo) -> TraceFn {
|
||||
let bucket = oi.bucket.clone();
|
||||
let name = oi.name.clone();
|
||||
let version_id = oi.version_id.map(|v| v.to_string()).unwrap_or_default();
|
||||
Arc::new(move |_action: String, _ctx: HashMap<String, String>| {
|
||||
let bucket = bucket.clone();
|
||||
let name = name.clone();
|
||||
let version_id = version_id.clone();
|
||||
Box::pin(async move {
|
||||
info!(
|
||||
bucket = %bucket,
|
||||
object = %name,
|
||||
version_id = %version_id,
|
||||
action = %_action,
|
||||
"ILM lifecycle trace: {} on {}/{} (version: {})",
|
||||
_action, bucket, name, version_id
|
||||
);
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,8 +147,8 @@ struct ExpiryTask {
|
||||
impl ExpiryOp for ExpiryTask {
|
||||
fn op_hash(&self) -> u64 {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(format!("{}", self.obj_info.bucket).as_bytes());
|
||||
hasher.update(format!("{}", self.obj_info.name).as_bytes());
|
||||
hasher.update(self.obj_info.bucket.as_bytes());
|
||||
hasher.update(self.obj_info.name.as_bytes());
|
||||
xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED)
|
||||
}
|
||||
|
||||
@@ -188,8 +202,8 @@ struct FreeVersionTask(ObjectInfo);
|
||||
impl ExpiryOp for FreeVersionTask {
|
||||
fn op_hash(&self) -> u64 {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(format!("{}", self.0.transitioned_object.tier).as_bytes());
|
||||
hasher.update(format!("{}", self.0.transitioned_object.name).as_bytes());
|
||||
hasher.update(self.0.transitioned_object.tier.as_bytes());
|
||||
hasher.update(self.0.transitioned_object.name.as_bytes());
|
||||
xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED)
|
||||
}
|
||||
|
||||
@@ -207,8 +221,8 @@ struct NewerNoncurrentTask {
|
||||
impl ExpiryOp for NewerNoncurrentTask {
|
||||
fn op_hash(&self) -> u64 {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(format!("{}", self.bucket).as_bytes());
|
||||
hasher.update(format!("{}", self.versions[0].object_name).as_bytes());
|
||||
hasher.update(self.bucket.as_bytes());
|
||||
hasher.update(self.versions[0].object_name.as_bytes());
|
||||
xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED)
|
||||
}
|
||||
|
||||
@@ -240,7 +254,7 @@ impl ExpiryState {
|
||||
|
||||
pub async fn pending_tasks(&self) -> usize {
|
||||
let rxs = &self.tasks_rx;
|
||||
if rxs.len() == 0 {
|
||||
if rxs.is_empty() {
|
||||
return 0;
|
||||
}
|
||||
let mut tasks = 0;
|
||||
@@ -253,32 +267,33 @@ impl ExpiryState {
|
||||
pub async fn enqueue_tier_journal_entry(&mut self, je: &Jentry) -> Result<(), std::io::Error> {
|
||||
let wrkr = self.get_worker_ch(je.op_hash());
|
||||
if wrkr.is_none() {
|
||||
*self.stats.as_mut().expect("err").missed_tier_journal_tasks.get_mut() += 1;
|
||||
*self.stats.as_mut().expect("stats lock").missed_tier_journal_tasks.get_mut() += 1;
|
||||
return Ok(());
|
||||
}
|
||||
let wrkr = wrkr.expect("err");
|
||||
let wrkr = wrkr.expect("worker channel should exist after None check");
|
||||
select! {
|
||||
//_ -> GlobalContext.Done() => ()
|
||||
_ = wrkr.send(Some(Box::new(je.clone()))) => (),
|
||||
else => {
|
||||
*self.stats.as_mut().expect("err").missed_tier_journal_tasks.get_mut() += 1;
|
||||
*self.stats.as_mut().expect("stats lock").missed_tier_journal_tasks.get_mut() += 1;
|
||||
}
|
||||
}
|
||||
return Ok(());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn enqueue_free_version(&mut self, oi: ObjectInfo) {
|
||||
let task = FreeVersionTask(oi);
|
||||
let wrkr = self.get_worker_ch(task.op_hash());
|
||||
if wrkr.is_none() {
|
||||
*self.stats.as_mut().expect("err").missed_freevers_tasks.get_mut() += 1;
|
||||
*self.stats.as_mut().expect("stats lock").missed_freevers_tasks.get_mut() += 1;
|
||||
return;
|
||||
}
|
||||
let wrkr = wrkr.expect("err!");
|
||||
let wrkr = wrkr.expect("worker channel should exist after None check");
|
||||
select! {
|
||||
//_ -> GlobalContext.Done() => {}
|
||||
_ = wrkr.send(Some(Box::new(task))) => (),
|
||||
else => {
|
||||
*self.stats.as_mut().expect("err").missed_freevers_tasks.get_mut() += 1;
|
||||
*self.stats.as_mut().expect("stats lock").missed_freevers_tasks.get_mut() += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -291,21 +306,21 @@ impl ExpiryState {
|
||||
};
|
||||
let wrkr = self.get_worker_ch(task.op_hash());
|
||||
if wrkr.is_none() {
|
||||
*self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
|
||||
*self.stats.as_mut().expect("stats lock").missed_expiry_tasks.get_mut() += 1;
|
||||
return;
|
||||
}
|
||||
let wrkr = wrkr.expect("err!");
|
||||
let wrkr = wrkr.expect("worker channel should exist after None check");
|
||||
select! {
|
||||
//_ -> GlobalContext.Done() => {}
|
||||
_ = wrkr.send(Some(Box::new(task))) => (),
|
||||
else => {
|
||||
*self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
|
||||
*self.stats.as_mut().expect("stats lock").missed_expiry_tasks.get_mut() += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn enqueue_by_newer_noncurrent(&mut self, bucket: &str, versions: Vec<ObjectToDelete>, lc_event: lifecycle::Event) {
|
||||
if versions.len() == 0 {
|
||||
if versions.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -316,26 +331,30 @@ impl ExpiryState {
|
||||
};
|
||||
let wrkr = self.get_worker_ch(task.op_hash());
|
||||
if wrkr.is_none() {
|
||||
*self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
|
||||
*self.stats.as_mut().expect("stats lock").missed_expiry_tasks.get_mut() += 1;
|
||||
return;
|
||||
}
|
||||
let wrkr = wrkr.expect("err!");
|
||||
let wrkr = wrkr.expect("worker channel should exist after None check");
|
||||
select! {
|
||||
//_ -> GlobalContext.Done() => {}
|
||||
_ = wrkr.send(Some(Box::new(task))) => (),
|
||||
else => {
|
||||
*self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
|
||||
*self.stats.as_mut().expect("stats lock").missed_expiry_tasks.get_mut() += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_worker_ch(&self, h: u64) -> Option<Sender<Option<ExpiryOpType>>> {
|
||||
if self.tasks_tx.len() == 0 {
|
||||
if self.tasks_tx.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(self.tasks_tx[h as usize % self.tasks_tx.len()].clone())
|
||||
}
|
||||
|
||||
pub fn increment_missed_tier_journal_tasks(&mut self) {
|
||||
*self.stats.as_mut().expect("stats lock").missed_tier_journal_tasks.get_mut() += 1;
|
||||
}
|
||||
|
||||
pub async fn resize_workers(n: usize, api: Arc<ECStore>) {
|
||||
if n == GLOBAL_ExpiryState.read().await.tasks_tx.len() || n < 1 {
|
||||
return;
|
||||
@@ -349,11 +368,11 @@ impl ExpiryState {
|
||||
let rx = Arc::new(tokio::sync::Mutex::new(rx));
|
||||
state.tasks_tx.push(tx);
|
||||
state.tasks_rx.push(rx.clone());
|
||||
*state.stats.as_mut().expect("err").workers.get_mut() += 1;
|
||||
*state.stats.as_mut().expect("stats lock").workers.get_mut() += 1;
|
||||
tokio::spawn(async move {
|
||||
let mut rx = rx.lock().await;
|
||||
//let mut expiry_state = GLOBAL_ExpiryState.read().await;
|
||||
ExpiryState::worker(&mut *rx, api).await;
|
||||
ExpiryState::worker(&mut rx, api).await;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -363,48 +382,49 @@ impl ExpiryState {
|
||||
worker.send(None).await.unwrap_or(());
|
||||
state.tasks_tx.remove(l - 1);
|
||||
state.tasks_rx.remove(l - 1);
|
||||
*state.stats.as_mut().expect("err").workers.get_mut() -= 1;
|
||||
*state.stats.as_mut().expect("stats lock").workers.get_mut() -= 1;
|
||||
l -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn worker(rx: &mut Receiver<Option<ExpiryOpType>>, api: Arc<ECStore>) {
|
||||
//let cancel_token =
|
||||
// get_background_services_cancel_token().ok_or_else(|| Error::other("Background services not initialized"))?;
|
||||
let cancel_token = crate::global::get_background_services_cancel_token().unwrap_or_else(|| {
|
||||
static FALLBACK: std::sync::OnceLock<tokio_util::sync::CancellationToken> = std::sync::OnceLock::new();
|
||||
FALLBACK.get_or_init(tokio_util::sync::CancellationToken::new)
|
||||
});
|
||||
|
||||
loop {
|
||||
select! {
|
||||
//_ = cancel_token.cancelled() => {
|
||||
_ = tokio::signal::ctrl_c() => {
|
||||
info!("got ctrl+c, exits");
|
||||
_ = cancel_token.cancelled() => {
|
||||
info!("lifecycle expiry worker received shutdown signal, exiting");
|
||||
break;
|
||||
}
|
||||
v = rx.recv() => {
|
||||
if v.is_none() {
|
||||
break;
|
||||
}
|
||||
let v = v.expect("err!");
|
||||
let v = v.expect("channel closed unexpectedly");
|
||||
if v.is_none() {
|
||||
//rx.close();
|
||||
//drop(rx);
|
||||
let _ = rx;
|
||||
return;
|
||||
}
|
||||
let v = v.expect("err!");
|
||||
let v = v.expect("received None after None check");
|
||||
if v.as_any().is::<ExpiryTask>() {
|
||||
let v = v.as_any().downcast_ref::<ExpiryTask>().expect("err!");
|
||||
if v.obj_info.transitioned_object.status != "" {
|
||||
let v = v.as_any().downcast_ref::<ExpiryTask>().expect("ExpiryTask downcast failed");
|
||||
if !v.obj_info.transitioned_object.status.is_empty() {
|
||||
apply_expiry_on_transitioned_object(api.clone(), &v.obj_info, &v.event, &v.src).await;
|
||||
} else {
|
||||
apply_expiry_on_non_transitioned_objects(api.clone(), &v.obj_info, &v.event, &v.src).await;
|
||||
}
|
||||
}
|
||||
else if v.as_any().is::<NewerNoncurrentTask>() {
|
||||
let _v = v.as_any().downcast_ref::<NewerNoncurrentTask>().expect("err!");
|
||||
//delete_object_versions(api, &v.bucket, &v.versions, v.event).await;
|
||||
let v = v.as_any().downcast_ref::<NewerNoncurrentTask>().expect("NewerNoncurrentTask downcast failed");
|
||||
crate::client::object_handlers_common::delete_object_versions(&api, &v.bucket, &v.versions, v.event.clone()).await;
|
||||
}
|
||||
else if v.as_any().is::<Jentry>() {
|
||||
let v = v.as_any().downcast_ref::<Jentry>().expect("err!");
|
||||
let v = v.as_any().downcast_ref::<Jentry>().expect("Jentry downcast failed");
|
||||
if let Err(err) = delete_object_from_remote_tier(&v.obj_name, &v.version_id, &v.tier_name).await {
|
||||
warn!(
|
||||
object = %v.obj_name,
|
||||
@@ -416,7 +436,7 @@ impl ExpiryState {
|
||||
}
|
||||
}
|
||||
else if v.as_any().is::<FreeVersionTask>() {
|
||||
let v = v.as_any().downcast_ref::<FreeVersionTask>().expect("err!");
|
||||
let v = v.as_any().downcast_ref::<FreeVersionTask>().expect("FreeVersionTask downcast failed");
|
||||
let oi = v.0.clone();
|
||||
if let Err(err) = delete_object_from_remote_tier(
|
||||
&oi.transitioned_object.name,
|
||||
@@ -499,7 +519,7 @@ struct TransitionTask {
|
||||
impl ExpiryOp for TransitionTask {
|
||||
fn op_hash(&self) -> u64 {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(format!("{}", self.obj_info.bucket).as_bytes());
|
||||
hasher.update(self.obj_info.bucket.as_bytes());
|
||||
// hasher.update(format!("{}", self.obj_info.versions[0].object_name).as_bytes());
|
||||
xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED)
|
||||
}
|
||||
@@ -594,22 +614,40 @@ impl TransitionState {
|
||||
if task.is_err() {
|
||||
break;
|
||||
}
|
||||
let task = task.expect("err!");
|
||||
let task = task.expect("channel recv should succeed after error check");
|
||||
if task.is_none() {
|
||||
//self.transition_rx.close();
|
||||
//drop(self.transition_rx);
|
||||
return;
|
||||
}
|
||||
let task = task.expect("err!");
|
||||
let task = task.expect("received None after None check");
|
||||
if task.as_any().is::<TransitionTask>() {
|
||||
let task = task.as_any().downcast_ref::<TransitionTask>().expect("err!");
|
||||
let task = task.as_any().downcast_ref::<TransitionTask>().expect("TransitionTask downcast failed");
|
||||
|
||||
GLOBAL_TransitionState.active_tasks.fetch_add(1, Ordering::SeqCst);
|
||||
|
||||
let obj_info_for_event = ObjectInfo {
|
||||
bucket: task.obj_info.bucket.clone(),
|
||||
name: task.obj_info.name.clone(),
|
||||
size: task.obj_info.size,
|
||||
version_id: task.obj_info.version_id,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
if let Err(err) = transition_object(api.clone(), &task.obj_info, LcAuditEvent::new(task.event.clone(), task.src.clone())).await {
|
||||
if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) && !err.to_string().contains("use of closed network connection") {
|
||||
error!("Transition to {} failed for {}/{} version:{} with {}",
|
||||
task.event.storage_class, task.obj_info.bucket, task.obj_info.name, task.obj_info.version_id.map(|v| v.to_string()).unwrap_or_default(), err.to_string());
|
||||
}
|
||||
// Send s3:ObjectTransition:Failed event
|
||||
send_event(EventArgs {
|
||||
event_name: EventName::ObjectTransitionFailed.to_string(),
|
||||
bucket_name: obj_info_for_event.bucket.clone(),
|
||||
object: obj_info_for_event,
|
||||
user_agent: "Internal: [ILM-Transition]".to_string(),
|
||||
host: GLOBAL_LocalNodeName.to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
} else {
|
||||
let mut ts = TierStats {
|
||||
total_size: task.obj_info.size as u64,
|
||||
@@ -620,6 +658,16 @@ impl TransitionState {
|
||||
ts.num_objects = 1;
|
||||
}
|
||||
GLOBAL_TransitionState.add_lastday_stats(&task.event.storage_class, ts);
|
||||
|
||||
// Send s3:ObjectTransition:Complete event
|
||||
send_event(EventArgs {
|
||||
event_name: EventName::ObjectTransitionComplete.to_string(),
|
||||
bucket_name: obj_info_for_event.bucket.clone(),
|
||||
object: obj_info_for_event,
|
||||
user_agent: "Internal: [ILM-Transition]".to_string(),
|
||||
host: GLOBAL_LocalNodeName.to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
GLOBAL_TransitionState.active_tasks.fetch_add(-1, Ordering::SeqCst);
|
||||
}
|
||||
@@ -634,7 +682,7 @@ impl TransitionState {
|
||||
tier_stats
|
||||
.entry(tier.to_string())
|
||||
.and_modify(|e| e.add_stats(ts))
|
||||
.or_insert(LastDayTierStats::default());
|
||||
.or_default();
|
||||
}
|
||||
|
||||
pub fn get_daily_all_tier_stats(&self) -> DailyAllTierStats {
|
||||
@@ -666,15 +714,15 @@ impl TransitionState {
|
||||
tokio::spawn(async move {
|
||||
TransitionState::worker(clone_api).await;
|
||||
});
|
||||
num_workers = num_workers + 1;
|
||||
num_workers += 1;
|
||||
GLOBAL_TransitionState.num_workers.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
let mut num_workers = GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst);
|
||||
while num_workers > n {
|
||||
let worker = GLOBAL_TransitionState.kill_tx.clone();
|
||||
worker.send(()).await;
|
||||
num_workers = num_workers - 1;
|
||||
let _ = worker.send(()).await;
|
||||
num_workers -= 1;
|
||||
GLOBAL_TransitionState.num_workers.fetch_add(-1, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
@@ -683,10 +731,10 @@ impl TransitionState {
|
||||
pub async fn init_background_expiry(api: Arc<ECStore>) {
|
||||
let mut workers = get_env_usize("RUSTFS_MAX_EXPIRY_WORKERS", std::cmp::min(num_cpus::get(), 16));
|
||||
//globalILMConfig.getExpirationWorkers()
|
||||
if let Ok(env_expiration_workers) = env::var("_RUSTFS_ILM_EXPIRATION_WORKERS") {
|
||||
if let Ok(num_expirations) = env_expiration_workers.parse::<usize>() {
|
||||
workers = num_expirations;
|
||||
}
|
||||
if let Ok(env_expiration_workers) = env::var("_RUSTFS_ILM_EXPIRATION_WORKERS")
|
||||
&& let Ok(num_expirations) = env_expiration_workers.parse::<usize>()
|
||||
{
|
||||
workers = num_expirations;
|
||||
}
|
||||
|
||||
if workers == 0 {
|
||||
@@ -1029,24 +1077,24 @@ pub async fn validate_transition_tier(lc: &BucketLifecycleConfiguration) -> Resu
|
||||
for rule in &lc.rules {
|
||||
if let Some(transitions) = &rule.transitions {
|
||||
for transition in transitions {
|
||||
if let Some(storage_class) = &transition.storage_class {
|
||||
if storage_class.as_str() != "" {
|
||||
let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str());
|
||||
if !valid {
|
||||
return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS));
|
||||
}
|
||||
if let Some(storage_class) = &transition.storage_class
|
||||
&& storage_class.as_str() != ""
|
||||
{
|
||||
let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str());
|
||||
if !valid {
|
||||
return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(noncurrent_version_transitions) = &rule.noncurrent_version_transitions {
|
||||
for noncurrent_version_transition in noncurrent_version_transitions {
|
||||
if let Some(storage_class) = &noncurrent_version_transition.storage_class {
|
||||
if storage_class.as_str() != "" {
|
||||
let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str());
|
||||
if !valid {
|
||||
return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS));
|
||||
}
|
||||
if let Some(storage_class) = &noncurrent_version_transition.storage_class
|
||||
&& storage_class.as_str() != ""
|
||||
{
|
||||
let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str());
|
||||
if !valid {
|
||||
return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1067,6 +1115,104 @@ pub async fn enqueue_transition_immediate(oi: &ObjectInfo, src: LcEventSrc) {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn enqueue_immediate_expiry(oi: &ObjectInfo, src: LcEventSrc) {
|
||||
let Some(lifecycle) = GLOBAL_LifecycleSys.get(&oi.bucket).await else {
|
||||
return;
|
||||
};
|
||||
let Some(api) = crate::new_object_layer_fn() else {
|
||||
return;
|
||||
};
|
||||
|
||||
let mut marker = None;
|
||||
let mut version_marker = None;
|
||||
let mut object_infos = Vec::new();
|
||||
|
||||
loop {
|
||||
let Ok(page) = api
|
||||
.clone()
|
||||
.list_object_versions(&oi.bucket, &oi.name, marker.clone(), version_marker.clone(), None, 1000)
|
||||
.await
|
||||
else {
|
||||
return;
|
||||
};
|
||||
|
||||
object_infos.extend(page.objects.into_iter().filter(|object| object.name == oi.name));
|
||||
|
||||
if !page.is_truncated {
|
||||
break;
|
||||
}
|
||||
|
||||
marker = page.next_marker;
|
||||
version_marker = page.next_version_idmarker;
|
||||
}
|
||||
|
||||
if object_infos.is_empty() {
|
||||
object_infos.push(oi.clone());
|
||||
}
|
||||
|
||||
let lock_config = match metadata_sys::get_object_lock_config(&oi.bucket).await {
|
||||
Ok((cfg, _)) => Some(Arc::new(cfg)),
|
||||
Err(_) => None,
|
||||
};
|
||||
let replication = match metadata_sys::get_replication_config(&oi.bucket).await {
|
||||
Ok((cfg, _)) if !cfg.rules.is_empty() => Some(Arc::new(ReplicationConfig::new(Some(cfg), None))),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
let object_opts = object_infos
|
||||
.iter()
|
||||
.map(|object| object.to_lifecycle_opts())
|
||||
.collect::<Vec<ObjectOpts>>();
|
||||
let Ok(events) = Evaluator::new(Arc::new(lifecycle))
|
||||
.with_lock_retention(lock_config)
|
||||
.with_replication_config(replication)
|
||||
.eval(&object_opts)
|
||||
.await
|
||||
else {
|
||||
return;
|
||||
};
|
||||
|
||||
let mut to_delete_objs = Vec::new();
|
||||
let mut noncurrent_event = None;
|
||||
|
||||
for (object, event) in object_infos.iter().zip(events.iter()) {
|
||||
if event.due != Some(OffsetDateTime::UNIX_EPOCH) {
|
||||
continue;
|
||||
}
|
||||
|
||||
match event.action {
|
||||
IlmAction::DeleteAction
|
||||
| IlmAction::DeleteRestoredAction
|
||||
| IlmAction::DeleteRestoredVersionAction
|
||||
| IlmAction::DeleteAllVersionsAction
|
||||
| IlmAction::DelMarkerDeleteAllVersionsAction => {
|
||||
apply_expiry_rule(event, &src, object).await;
|
||||
}
|
||||
IlmAction::DeleteVersionAction => {
|
||||
to_delete_objs.push(ObjectToDelete {
|
||||
object_name: object.name.clone(),
|
||||
version_id: object.version_id,
|
||||
..Default::default()
|
||||
});
|
||||
if noncurrent_event.is_none() {
|
||||
noncurrent_event = Some(event.clone());
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
if !to_delete_objs.is_empty()
|
||||
&& let Some(event) = noncurrent_event
|
||||
{
|
||||
GLOBAL_ExpiryState
|
||||
.write()
|
||||
.await
|
||||
.enqueue_by_newer_noncurrent(&oi.bucket, to_delete_objs, event)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn enqueue_transition_for_existing_objects(api: Arc<ECStore>, bucket: &str) -> Result<(), Error> {
|
||||
let Some(lc) = GLOBAL_LifecycleSys.get(bucket).await else {
|
||||
return Ok(());
|
||||
@@ -1094,6 +1240,60 @@ pub async fn enqueue_transition_for_existing_objects(api: Arc<ECStore>, bucket:
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn enqueue_expiry_for_existing_objects(api: Arc<ECStore>, bucket: &str) -> Result<(), Error> {
|
||||
let Ok((lc, _)) = metadata_sys::get_lifecycle_config(bucket).await else {
|
||||
return Ok(());
|
||||
};
|
||||
let lock_retention = metadata_sys::get_object_lock_config(bucket)
|
||||
.await
|
||||
.ok()
|
||||
.and_then(|(cfg, _)| cfg.rule.and_then(|rule| rule.default_retention));
|
||||
let replication_config = metadata_sys::get_replication_config(bucket).await.ok();
|
||||
let mut marker = None;
|
||||
let mut version_marker = None;
|
||||
let src = LcEventSrc::Scanner;
|
||||
|
||||
loop {
|
||||
let page = api
|
||||
.clone()
|
||||
.list_object_versions(bucket, "", marker.clone(), version_marker.clone(), None, 1000)
|
||||
.await?;
|
||||
|
||||
for object in &page.objects {
|
||||
let event = eval_action_from_lifecycle(&lc, lock_retention.clone(), replication_config.clone(), object).await;
|
||||
match event.action {
|
||||
IlmAction::DeleteAction
|
||||
| IlmAction::DeleteVersionAction
|
||||
| IlmAction::DeleteRestoredAction
|
||||
| IlmAction::DeleteRestoredVersionAction
|
||||
| IlmAction::DeleteAllVersionsAction
|
||||
| IlmAction::DelMarkerDeleteAllVersionsAction => {
|
||||
if event
|
||||
.due
|
||||
.is_some_and(|due| due.unix_timestamp() <= OffsetDateTime::now_utc().unix_timestamp())
|
||||
{
|
||||
if object.is_remote() {
|
||||
apply_expiry_on_transitioned_object(api.clone(), object, &event, &src).await;
|
||||
} else {
|
||||
apply_expiry_on_non_transitioned_objects(api.clone(), object, &event, &src).await;
|
||||
}
|
||||
} else {
|
||||
apply_expiry_rule(&event, &src, object).await;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
if !page.is_truncated {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
marker = page.next_marker;
|
||||
version_marker = page.next_version_idmarker;
|
||||
}
|
||||
}
|
||||
|
||||
async fn enqueue_transition_with_lifecycle(oi: &ObjectInfo, lc: &BucketLifecycleConfiguration, src: &LcEventSrc) {
|
||||
let event = lc.eval(&oi.to_lifecycle_opts()).await;
|
||||
match event.action {
|
||||
@@ -1154,6 +1354,8 @@ pub async fn expire_transitioned_object(
|
||||
}
|
||||
};
|
||||
|
||||
schedule_lifecycle_replication_delete_if_needed(oi).await;
|
||||
|
||||
//defer auditLogLifecycle(ctx, *oi, ILMExpiry, tags, traceFn)
|
||||
|
||||
let event_name = if oi.delete_marker {
|
||||
@@ -1227,8 +1429,19 @@ pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditE
|
||||
result
|
||||
}
|
||||
|
||||
pub fn audit_tier_actions(_api: ECStore, _tier: &str, _bytes: i64) -> TimeFn {
|
||||
Arc::new(|| Box::pin(async move {}))
|
||||
pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn {
|
||||
let tier = _tier.to_string();
|
||||
Arc::new(move || {
|
||||
let tier = tier.clone();
|
||||
Box::pin(async move {
|
||||
info!(
|
||||
tier = %tier,
|
||||
bytes = bytes,
|
||||
"ILM tier transition audit: completed transition of {} bytes to tier '{}'",
|
||||
bytes, tier
|
||||
);
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn get_transitioned_object_reader(
|
||||
@@ -1245,11 +1458,11 @@ pub async fn get_transitioned_object_reader(
|
||||
Err(err) => return Err(std::io::Error::other(err)),
|
||||
};
|
||||
|
||||
let ret = new_getobjectreader(rs, &oi, opts, &h);
|
||||
let ret = new_getobjectreader(rs, oi, opts, h);
|
||||
if let Err(err) = ret {
|
||||
return Err(error_resp_to_object_err(err, vec![bucket, object]));
|
||||
}
|
||||
let (get_fn, off, length) = ret.expect("err");
|
||||
let (get_fn, off, length) = ret.expect("get_transitioned_object_reader should succeed after error check");
|
||||
let mut gopts = WarmBackendGetOpts::default();
|
||||
|
||||
if off >= 0 && length >= 0 {
|
||||
@@ -1269,8 +1482,8 @@ pub async fn post_restore_opts(version_id: &str, bucket: &str, object: &str) ->
|
||||
let versioned = BucketVersioningSys::prefix_enabled(bucket, object).await;
|
||||
let version_suspended = BucketVersioningSys::prefix_suspended(bucket, object).await;
|
||||
let vid = version_id.trim();
|
||||
if vid != "" && vid != NULL_VERSION_ID {
|
||||
if let Err(err) = Uuid::parse_str(vid) {
|
||||
if !vid.is_empty() && vid != NULL_VERSION_ID {
|
||||
if let Err(_err) = Uuid::parse_str(vid) {
|
||||
return Err(std::io::Error::other(
|
||||
StorageError::InvalidVersionID(bucket.to_string(), object.to_string(), vid.to_string()).to_string(),
|
||||
));
|
||||
@@ -1332,24 +1545,21 @@ pub async fn put_restore_opts(
|
||||
}
|
||||
meta.insert(v.name.clone().unwrap(), v.value.clone().unwrap_or_else(|| "".to_string()));
|
||||
}
|
||||
if let Some(output_location) = rreq.output_location.as_ref() {
|
||||
if let Some(s3) = &output_location.s3 {
|
||||
if let Some(tags) = &s3.tagging {
|
||||
meta.insert(
|
||||
AMZ_OBJECT_TAGGING.to_string(),
|
||||
serde_urlencoded::to_string(tags.tag_set.clone()).unwrap_or_else(|_| "".to_string()),
|
||||
);
|
||||
}
|
||||
}
|
||||
if let Some(output_location) = rreq.output_location.as_ref()
|
||||
&& let Some(s3) = &output_location.s3
|
||||
&& let Some(tags) = &s3.tagging
|
||||
{
|
||||
meta.insert(
|
||||
AMZ_OBJECT_TAGGING.to_string(),
|
||||
serde_urlencoded::to_string(tags.tag_set.clone()).unwrap_or_else(|_| "".to_string()),
|
||||
);
|
||||
}
|
||||
if let Some(output_location) = rreq.output_location.as_ref() {
|
||||
if let Some(s3) = &output_location.s3 {
|
||||
if let Some(encryption) = &s3.encryption {
|
||||
if encryption.encryption_type.as_str() != "" {
|
||||
meta.insert(X_AMZ_SERVER_SIDE_ENCRYPTION.as_str().to_string(), AMZ_ENCRYPTION_AES.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(output_location) = rreq.output_location.as_ref()
|
||||
&& let Some(s3) = &output_location.s3
|
||||
&& let Some(encryption) = &s3.encryption
|
||||
&& encryption.encryption_type.as_str() != ""
|
||||
{
|
||||
meta.insert(X_AMZ_SERVER_SIDE_ENCRYPTION.as_str().to_string(), AMZ_ENCRYPTION_AES.to_string());
|
||||
}
|
||||
return Ok(ObjectOptions {
|
||||
versioned: BucketVersioningSys::prefix_enabled(bucket, object).await,
|
||||
@@ -1361,7 +1571,7 @@ pub async fn put_restore_opts(
|
||||
for (k, v) in &oi.user_defined {
|
||||
meta.insert(k.to_string(), v.clone());
|
||||
}
|
||||
if oi.user_tags.len() != 0 {
|
||||
if !oi.user_tags.is_empty() {
|
||||
meta.insert(AMZ_OBJECT_TAGGING.to_string(), oi.user_tags.clone());
|
||||
}
|
||||
let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1));
|
||||
@@ -1394,7 +1604,7 @@ impl LifecycleOps for ObjectInfo {
|
||||
lifecycle::ObjectOpts {
|
||||
name: self.name.clone(),
|
||||
user_tags: self.user_tags.clone(),
|
||||
version_id: self.version_id.clone(),
|
||||
version_id: self.version_id,
|
||||
mod_time: self.mod_time,
|
||||
size: self.size as usize,
|
||||
is_latest: self.is_latest,
|
||||
@@ -1421,38 +1631,42 @@ pub trait RestoreRequestOps {
|
||||
}
|
||||
|
||||
impl RestoreRequestOps for RestoreRequest {
|
||||
fn validate(&self, api: Arc<ECStore>) -> Result<(), std::io::Error> {
|
||||
/*if self.type_.is_none() && self.select_parameters.is_some() {
|
||||
fn validate(&self, _api: Arc<ECStore>) -> Result<(), std::io::Error> {
|
||||
// SELECT type requires select_parameters, and vice versa
|
||||
if self.type_.as_ref().is_none_or(|t| t.as_str() != RestoreRequestType::SELECT) && self.select_parameters.is_some() {
|
||||
return Err(std::io::Error::other("Select parameters can only be specified with SELECT request type"));
|
||||
}
|
||||
if let Some(type_) = self.type_ && type_ == RestoreRequestType::SELECT && self.select_parameters.is_none() {
|
||||
if let Some(type_) = &self.type_
|
||||
&& type_.as_str() == RestoreRequestType::SELECT
|
||||
&& self.select_parameters.is_none()
|
||||
{
|
||||
return Err(std::io::Error::other("SELECT restore request requires select parameters to be specified"));
|
||||
}
|
||||
|
||||
if self.type_.is_none() && self.output_location.is_some() {
|
||||
return Err(std::io::Error::other("OutputLocation required only for SELECT request type"));
|
||||
// OutputLocation is only valid for SELECT requests
|
||||
if self.type_.as_ref().is_none_or(|t| t.as_str() != RestoreRequestType::SELECT) && self.output_location.is_some() {
|
||||
return Err(std::io::Error::other("OutputLocation can only be specified with SELECT request type"));
|
||||
}
|
||||
if let Some(type_) = self.type_ && type_ == RestoreRequestType::SELECT && self.output_location.is_none() {
|
||||
if let Some(type_) = &self.type_
|
||||
&& type_.as_str() == RestoreRequestType::SELECT
|
||||
&& self.output_location.is_none()
|
||||
{
|
||||
return Err(std::io::Error::other("OutputLocation required for SELECT requests"));
|
||||
}
|
||||
|
||||
if let Some(type_) = self.type_ && type_ == RestoreRequestType::SELECT && self.days != 0 {
|
||||
// Days must not be specified with SELECT requests
|
||||
if let Some(type_) = &self.type_
|
||||
&& type_.as_str() == RestoreRequestType::SELECT
|
||||
&& self.days.is_some_and(|d| d > 0)
|
||||
{
|
||||
return Err(std::io::Error::other("Days cannot be specified with SELECT restore request"));
|
||||
}
|
||||
if self.days == 0 && self.type_.is_none() {
|
||||
|
||||
// For non-SELECT requests, days must be at least 1
|
||||
if self.type_.is_none() && self.days.is_none_or(|d| d <= 0) {
|
||||
return Err(std::io::Error::other("restoration days should be at least 1"));
|
||||
}
|
||||
if self.output_location.is_some() {
|
||||
if _, err := api.get_bucket_info(self.output_location.s3.bucket_name, BucketOptions{}); err != nil {
|
||||
return err
|
||||
}
|
||||
if self.output_location.s3.prefix == "" {
|
||||
return Err(std::io::Error::other("Prefix is a required parameter in OutputLocation"));
|
||||
}
|
||||
if self.output_location.s3.encryption.encryption_type.as_str() != ServerSideEncryption::AES256 {
|
||||
return NotImplemented{}
|
||||
}
|
||||
}*/
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1497,10 +1711,10 @@ pub async fn eval_action_from_lifecycle(
|
||||
//}
|
||||
return lifecycle::Event::default();
|
||||
}
|
||||
if let Some(rcfg) = rcfg {
|
||||
if rep_has_active_rules(&rcfg.0, &oi.name, true) {
|
||||
return lifecycle::Event::default();
|
||||
}
|
||||
if let Some(rcfg) = rcfg
|
||||
&& rep_has_active_rules(&rcfg.0, &oi.name, true)
|
||||
{
|
||||
return lifecycle::Event::default();
|
||||
}
|
||||
}
|
||||
_ => (),
|
||||
@@ -1566,6 +1780,7 @@ pub async fn apply_expiry_on_non_transitioned_objects(
|
||||
return false;
|
||||
}
|
||||
};
|
||||
schedule_lifecycle_replication_delete_if_needed(oi).await;
|
||||
//debug!("dobj: {:?}", dobj);
|
||||
if dobj.name.is_empty() {
|
||||
dobj = oi.clone();
|
||||
@@ -1606,6 +1821,70 @@ pub async fn apply_expiry_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &
|
||||
true
|
||||
}
|
||||
|
||||
async fn schedule_lifecycle_replication_delete_if_needed(oi: &ObjectInfo) {
|
||||
if !oi.delete_marker || oi.version_id.is_none() {
|
||||
return;
|
||||
}
|
||||
|
||||
let replication_state = lifecycle_delete_replication_state(oi).await;
|
||||
if replication_state.is_none() {
|
||||
return;
|
||||
}
|
||||
|
||||
schedule_replication_delete(DeletedObjectReplicationInfo {
|
||||
delete_object: crate::store_api::DeletedObject {
|
||||
object_name: oi.name.clone(),
|
||||
delete_marker_version_id: oi.version_id,
|
||||
delete_marker: false,
|
||||
delete_marker_mtime: oi.mod_time,
|
||||
replication_state,
|
||||
..Default::default()
|
||||
},
|
||||
bucket: oi.bucket.clone(),
|
||||
event_type: REPLICATE_INCOMING_DELETE.to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn lifecycle_delete_replication_state(oi: &ObjectInfo) -> Option<ReplicationState> {
|
||||
if !oi.replication_decision.is_empty() || oi.version_purge_status == VersionPurgeStatusType::Pending {
|
||||
return Some(oi.replication_state());
|
||||
}
|
||||
|
||||
let dsc = check_replicate_delete(
|
||||
&oi.bucket,
|
||||
&ObjectToDelete {
|
||||
object_name: oi.name.clone(),
|
||||
version_id: oi.version_id,
|
||||
..Default::default()
|
||||
},
|
||||
oi,
|
||||
&ObjectOptions {
|
||||
version_id: oi.version_id.map(|v| v.to_string()),
|
||||
versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await,
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
if !dsc.replicate_any() {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(replication_state_for_version_delete(dsc))
|
||||
}
|
||||
|
||||
fn replication_state_for_version_delete(dsc: ReplicateDecision) -> ReplicationState {
|
||||
let pending_status = dsc.pending_status();
|
||||
ReplicationState {
|
||||
replicate_decision_str: dsc.to_string(),
|
||||
version_purge_status_internal: pending_status.clone(),
|
||||
purge_targets: rustfs_filemeta::version_purge_statuses_map(pending_status.as_deref().unwrap_or_default()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
|
||||
let mut success = false;
|
||||
match event.action {
|
||||
|
||||
+815
-260
File diff suppressed because it is too large
Load Diff
@@ -14,8 +14,9 @@
|
||||
|
||||
pub mod bucket_lifecycle_audit;
|
||||
pub mod bucket_lifecycle_ops;
|
||||
pub mod core;
|
||||
pub mod evaluator;
|
||||
pub mod lifecycle;
|
||||
pub use self::core as lifecycle;
|
||||
pub mod rule;
|
||||
pub mod tier_last_day_stats;
|
||||
pub mod tier_sweeper;
|
||||
|
||||
@@ -109,10 +109,19 @@ impl ObjSweeper {
|
||||
}
|
||||
|
||||
pub async fn sweep(&self) {
|
||||
let je = self.should_remove_remote_object();
|
||||
if !je.is_none() {
|
||||
let mut expiry_state = GLOBAL_ExpiryState.write().await;
|
||||
expiry_state.enqueue_tier_journal_entry(&je.expect("err!"));
|
||||
let Some(je) = self.should_remove_remote_object() else {
|
||||
return;
|
||||
};
|
||||
let hash = je.op_hash();
|
||||
// Grab the sender under a short read lock, then release the lock so we
|
||||
// don't hold it across the async send.
|
||||
let wrkr = GLOBAL_ExpiryState.read().await.get_worker_ch(hash);
|
||||
let Some(wrkr) = wrkr else {
|
||||
GLOBAL_ExpiryState.write().await.increment_missed_tier_journal_tasks();
|
||||
return;
|
||||
};
|
||||
if wrkr.send(Some(Box::new(je))).await.is_err() {
|
||||
GLOBAL_ExpiryState.write().await.increment_missed_tier_journal_tasks();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -153,6 +153,11 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
|
||||
|
||||
if obj.op_type == ReplicationType::Delete {
|
||||
if obj.version_id.is_some() {
|
||||
if obj.delete_marker {
|
||||
return rule.delete_marker_replication.clone().is_some_and(|d| {
|
||||
d.status == Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED))
|
||||
});
|
||||
}
|
||||
return rule
|
||||
.delete_replication
|
||||
.clone()
|
||||
|
||||
@@ -34,7 +34,7 @@ use crate::global::get_global_bucket_monitor;
|
||||
use crate::set_disk::get_lock_acquire_timeout;
|
||||
use crate::store_api::{DeletedObject, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete, WalkOptions};
|
||||
use crate::{StorageAPI, new_object_layer_fn};
|
||||
use aws_sdk_s3::error::SdkError;
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::operation::head_object::HeadObjectOutput;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedPart, ObjectLockLegalHoldStatus};
|
||||
@@ -86,6 +86,7 @@ use tokio::time::Duration as TokioDuration;
|
||||
use tokio_util::io::ReaderStream;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{error, info, instrument, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub(crate) const REPLICATION_DIR: &str = ".replication";
|
||||
pub(crate) const RESYNC_FILE_NAME: &str = "resync.bin";
|
||||
@@ -1497,6 +1498,54 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
|
||||
}
|
||||
};
|
||||
|
||||
if dobj.delete_object.delete_marker
|
||||
&& let Some(delete_marker_version_id) = dobj.delete_object.delete_marker_version_id
|
||||
{
|
||||
let source_marker_state = storage
|
||||
.get_object_info(
|
||||
&bucket,
|
||||
&dobj.delete_object.object_name,
|
||||
&ObjectOptions {
|
||||
version_id: Some(delete_marker_version_id.to_string()),
|
||||
versioned: BucketVersioningSys::prefix_enabled(&bucket, &dobj.delete_object.object_name).await,
|
||||
version_suspended: BucketVersioningSys::prefix_suspended(&bucket, &dobj.delete_object.object_name).await,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
match source_marker_state {
|
||||
Ok(info) if info.delete_marker && info.version_id == Some(delete_marker_version_id) => {}
|
||||
Ok(_) => {
|
||||
warn!(
|
||||
bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
version_id = %delete_marker_version_id,
|
||||
"skipping stale delete-marker replication because source version is no longer a delete marker"
|
||||
);
|
||||
return;
|
||||
}
|
||||
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {
|
||||
warn!(
|
||||
bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
version_id = %delete_marker_version_id,
|
||||
"skipping stale delete-marker replication because source version no longer exists"
|
||||
);
|
||||
return;
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
version_id = %delete_marker_version_id,
|
||||
error = %err,
|
||||
"failed to verify source delete-marker state before replication"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let dsc = match parse_replicate_decision(
|
||||
&bucket,
|
||||
&dobj
|
||||
@@ -1529,7 +1578,6 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let ns_lock = match storage
|
||||
.new_ns_lock(&bucket, format!("/[replicate]/{}", dobj.delete_object.object_name).as_str())
|
||||
.await
|
||||
@@ -1653,7 +1701,33 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
|
||||
}
|
||||
}
|
||||
|
||||
let (replication_status, prev_status) = if dobj.delete_object.version_id.is_none() {
|
||||
let is_version_purge = is_version_delete_replication(&dobj.delete_object);
|
||||
|
||||
if !is_version_purge && dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
|
||||
let bucket_clone = bucket.clone();
|
||||
let dobj_clone = dobj.clone();
|
||||
let dsc_clone = dsc.clone();
|
||||
let storage_clone = storage.clone();
|
||||
tokio::spawn(async move {
|
||||
for _ in 0..5 {
|
||||
if let Some(delete_marker_version_id) = dobj_clone.delete_object.delete_marker_version_id
|
||||
&& source_delete_marker_missing(
|
||||
&*storage_clone,
|
||||
&bucket_clone,
|
||||
&dobj_clone.delete_object.object_name,
|
||||
delete_marker_version_id,
|
||||
)
|
||||
.await
|
||||
{
|
||||
replicate_delete_marker_purge_to_targets(&bucket_clone, &dobj_clone, &dsc_clone).await;
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(TokioDuration::from_secs(1)).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let (replication_status, prev_status) = if !is_version_purge {
|
||||
(
|
||||
rinfos.replication_status(),
|
||||
dobj.delete_object
|
||||
@@ -1741,6 +1815,65 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
|
||||
}
|
||||
}
|
||||
|
||||
async fn source_delete_marker_missing<S: StorageAPI>(
|
||||
storage: &S,
|
||||
bucket: &str,
|
||||
object_name: &str,
|
||||
delete_marker_version_id: Uuid,
|
||||
) -> bool {
|
||||
match storage
|
||||
.get_object_info(
|
||||
bucket,
|
||||
object_name,
|
||||
&ObjectOptions {
|
||||
version_id: Some(delete_marker_version_id.to_string()),
|
||||
versioned: BucketVersioningSys::prefix_enabled(bucket, object_name).await,
|
||||
version_suspended: BucketVersioningSys::prefix_suspended(bucket, object_name).await,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(info) => !info.delete_marker || info.version_id != Some(delete_marker_version_id),
|
||||
Err(err) => is_err_object_not_found(&err) || is_err_version_not_found(&err),
|
||||
}
|
||||
}
|
||||
|
||||
async fn replicate_delete_marker_purge_to_targets(bucket: &str, dobj: &DeletedObjectReplicationInfo, dsc: &ReplicateDecision) {
|
||||
let Some(delete_marker_version_id) = dobj.delete_object.delete_marker_version_id else {
|
||||
return;
|
||||
};
|
||||
|
||||
for tgt_entry in dsc.targets_map.values() {
|
||||
if !tgt_entry.replicate {
|
||||
continue;
|
||||
}
|
||||
if !dobj.target_arn.is_empty() && dobj.target_arn != tgt_entry.arn {
|
||||
continue;
|
||||
}
|
||||
let Some(tgt_client) = BucketTargetSys::get().get_remote_target_client(bucket, &tgt_entry.arn).await else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let _ = tgt_client
|
||||
.remove_object(
|
||||
&tgt_client.bucket,
|
||||
&dobj.delete_object.object_name,
|
||||
Some(delete_marker_version_id.to_string()),
|
||||
RemoveObjectOptions {
|
||||
force_delete: false,
|
||||
governance_bypass: false,
|
||||
replication_delete_marker: false,
|
||||
replication_mtime: dobj.delete_object.delete_marker_mtime,
|
||||
replication_status: ReplicationStatusType::Replica,
|
||||
replication_request: true,
|
||||
replication_validity_check: false,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn replicate_force_delete_to_targets<S: StorageAPI>(dobj: &DeletedObjectReplicationInfo, storage: Arc<S>) {
|
||||
let bucket = &dobj.bucket;
|
||||
let object_name = &dobj.delete_object.object_name;
|
||||
@@ -1924,6 +2057,14 @@ async fn replicate_force_delete_to_targets<S: StorageAPI>(dobj: &DeletedObjectRe
|
||||
}
|
||||
}
|
||||
|
||||
fn is_version_delete_replication(dobj: &DeletedObject) -> bool {
|
||||
dobj.version_id.is_some() || (dobj.delete_marker_version_id.is_some() && !dobj.delete_marker)
|
||||
}
|
||||
|
||||
fn is_retryable_delete_replication_head_error(is_not_found: bool, code: Option<&str>) -> bool {
|
||||
!is_not_found && !matches!(code, Some("MethodNotAllowed" | "405"))
|
||||
}
|
||||
|
||||
async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
|
||||
let version_id = if let Some(version_id) = &dobj.delete_object.delete_marker_version_id {
|
||||
version_id.to_owned()
|
||||
@@ -1941,7 +2082,8 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
|
||||
rinfo.endpoint = tgt_client.endpoint.clone();
|
||||
rinfo.secure = tgt_client.secure;
|
||||
|
||||
if dobj.delete_object.version_id.is_none()
|
||||
let is_version_purge = is_version_delete_replication(&dobj.delete_object);
|
||||
if !is_version_purge
|
||||
&& rinfo.prev_replication_status == ReplicationStatusType::Completed
|
||||
&& dobj.op_type != ReplicationType::ExistingObject
|
||||
{
|
||||
@@ -1949,12 +2091,12 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
|
||||
return rinfo;
|
||||
}
|
||||
|
||||
if dobj.delete_object.version_id.is_some() && rinfo.version_purge_status == VersionPurgeStatusType::Complete {
|
||||
if is_version_purge && rinfo.version_purge_status == VersionPurgeStatusType::Complete {
|
||||
return rinfo;
|
||||
}
|
||||
|
||||
if BucketTargetSys::get().is_offline(&tgt_client.to_url()).await {
|
||||
if dobj.delete_object.version_id.is_none() {
|
||||
if !is_version_purge {
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
} else {
|
||||
rinfo.version_purge_status = VersionPurgeStatusType::Failed;
|
||||
@@ -1968,18 +2110,29 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
|
||||
Some(version_id.to_string())
|
||||
};
|
||||
|
||||
if dobj.delete_object.delete_marker_version_id.is_some()
|
||||
&& let Err(e) = tgt_client
|
||||
if dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
|
||||
match tgt_client
|
||||
.head_object(&tgt_client.bucket, &dobj.delete_object.object_name, version_id.clone())
|
||||
.await
|
||||
&& let SdkError::ServiceError(service_err) = &e
|
||||
&& !service_err.err().is_not_found()
|
||||
{
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some(e.to_string());
|
||||
|
||||
return rinfo;
|
||||
};
|
||||
{
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
let non_retryable = matches!(
|
||||
&e,
|
||||
SdkError::ServiceError(service_err)
|
||||
if is_retryable_delete_replication_head_error(
|
||||
service_err.err().is_not_found(),
|
||||
service_err.err().code(),
|
||||
)
|
||||
);
|
||||
if non_retryable {
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some(e.to_string());
|
||||
return rinfo;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match tgt_client
|
||||
.remove_object(
|
||||
@@ -1989,7 +2142,7 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
|
||||
RemoveObjectOptions {
|
||||
force_delete: false,
|
||||
governance_bypass: false,
|
||||
replication_delete_marker: dobj.delete_object.delete_marker_version_id.is_some(),
|
||||
replication_delete_marker: dobj.delete_object.delete_marker,
|
||||
replication_mtime: dobj.delete_object.delete_marker_mtime,
|
||||
replication_status: ReplicationStatusType::Replica,
|
||||
replication_request: true,
|
||||
@@ -1999,7 +2152,7 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
if dobj.delete_object.version_id.is_none() {
|
||||
if !is_version_purge {
|
||||
rinfo.replication_status = ReplicationStatusType::Completed;
|
||||
} else {
|
||||
rinfo.version_purge_status = VersionPurgeStatusType::Complete;
|
||||
@@ -2007,7 +2160,7 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
|
||||
}
|
||||
Err(e) => {
|
||||
rinfo.error = Some(e.to_string());
|
||||
if dobj.delete_object.version_id.is_none() {
|
||||
if !is_version_purge {
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
} else {
|
||||
rinfo.version_purge_status = VersionPurgeStatusType::Failed;
|
||||
@@ -3450,6 +3603,54 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_version_delete_replication_for_delete_marker_version_purge() {
|
||||
let dobj = DeletedObject {
|
||||
delete_marker: false,
|
||||
delete_marker_version_id: Some(Uuid::new_v4()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
is_version_delete_replication(&dobj),
|
||||
"delete-marker version purges must be tracked as version purge replication, not delete-marker creation replication"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_version_delete_replication_for_delete_marker_creation() {
|
||||
let dobj = DeletedObject {
|
||||
delete_marker: true,
|
||||
delete_marker_version_id: Some(Uuid::new_v4()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
!is_version_delete_replication(&dobj),
|
||||
"delete-marker creation should remain on the delete-marker replication path"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_retryable_delete_replication_head_error_allows_delete_marker_head_responses() {
|
||||
assert!(
|
||||
!is_retryable_delete_replication_head_error(false, Some("405")),
|
||||
"numeric 405 responses should not block delete-marker purge replication"
|
||||
);
|
||||
assert!(
|
||||
!is_retryable_delete_replication_head_error(false, Some("MethodNotAllowed")),
|
||||
"MethodNotAllowed responses should not block delete-marker purge replication"
|
||||
);
|
||||
assert!(
|
||||
!is_retryable_delete_replication_head_error(true, Some("NoSuchKey")),
|
||||
"not-found responses should not block delete-marker purge replication"
|
||||
);
|
||||
assert!(
|
||||
is_retryable_delete_replication_head_error(false, Some("AccessDenied")),
|
||||
"unexpected head errors should still fail fast"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_heal_replicate_object_info_failed_object_returns_heal_roi() {
|
||||
let oi = ObjectInfo {
|
||||
|
||||
@@ -12,6 +12,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::Arc;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::bucket::lifecycle::lifecycle;
|
||||
use crate::bucket::versioning::VersioningApi;
|
||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||
@@ -19,7 +22,14 @@ use crate::store::ECStore;
|
||||
use crate::store_api::{ObjectOperations, ObjectOptions, ObjectToDelete};
|
||||
use rustfs_lock::MAX_DELETE_LIST;
|
||||
|
||||
pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) {
|
||||
pub async fn delete_object_versions(api: &Arc<ECStore>, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) {
|
||||
let version_suspended = match BucketVersioningSys::get(bucket).await {
|
||||
Ok(vc) => vc.suspended(),
|
||||
Err(err) => {
|
||||
warn!(bucket, error = ?err, "failed to get versioning config during lifecycle noncurrent version cleanup");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let mut remaining = to_del;
|
||||
loop {
|
||||
let mut to_del = remaining;
|
||||
@@ -29,15 +39,29 @@ pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[Object
|
||||
} else {
|
||||
remaining = &[];
|
||||
}
|
||||
let vc = BucketVersioningSys::get(bucket).await.expect("err!");
|
||||
let _deleted_objs = api.delete_objects(
|
||||
bucket,
|
||||
to_del.to_vec(),
|
||||
ObjectOptions {
|
||||
//prefix_enabled_fn: vc.prefix_enabled(""),
|
||||
version_suspended: vc.suspended(),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let (_deleted_objs, errors) = api
|
||||
.delete_objects(
|
||||
bucket,
|
||||
to_del.to_vec(),
|
||||
ObjectOptions {
|
||||
version_suspended,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
for (i, err) in errors.iter().enumerate() {
|
||||
if let Some(e) = err {
|
||||
let obj_name = to_del.get(i).map(|o| o.object_name.as_str()).unwrap_or("<unknown>");
|
||||
let vid = to_del
|
||||
.get(i)
|
||||
.and_then(|o| o.version_id)
|
||||
.map(|v| v.to_string())
|
||||
.unwrap_or_default();
|
||||
warn!(bucket, object = obj_name, version_id = %vid, error = ?e, "failed to delete noncurrent version during lifecycle cleanup");
|
||||
}
|
||||
}
|
||||
if remaining.is_empty() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+191
-34
@@ -1771,13 +1771,7 @@ impl ObjectOperations for SetDisks {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let dsc = if opts
|
||||
.delete_replication
|
||||
.as_ref()
|
||||
.map(|v| v.replica_status == ReplicationStatusType::Replica)
|
||||
== Some(true)
|
||||
|| opts.version_purge_status() == VersionPurgeStatusType::Complete
|
||||
{
|
||||
let dsc = if should_preserve_delete_replication_state(&opts) {
|
||||
ReplicateDecision::default()
|
||||
} else {
|
||||
check_replicate_delete(bucket, &otd, &goi, &opts, gerr.map(|e| e.to_string())).await
|
||||
@@ -1792,33 +1786,7 @@ impl ObjectOperations for SetDisks {
|
||||
.unwrap_or_default();
|
||||
}
|
||||
|
||||
let mut mark_delete = goi.version_id.is_some();
|
||||
|
||||
let mut delete_marker = opts.versioned;
|
||||
|
||||
if opts.version_id.is_some() {
|
||||
// Decommission/rebalance may recreate a delete marker on a new pool before that
|
||||
// exact version exists there, so we must still treat it as a mark-delete write.
|
||||
if opts.data_movement && opts.delete_marker && !version_found {
|
||||
mark_delete = true;
|
||||
}
|
||||
|
||||
if version_found && opts.delete_marker_replication_status() == ReplicationStatusType::Replica {
|
||||
mark_delete = false;
|
||||
}
|
||||
|
||||
if opts.version_purge_status().is_empty() && opts.delete_marker_replication_status().is_empty() {
|
||||
mark_delete = false;
|
||||
}
|
||||
|
||||
if opts.version_purge_status() == VersionPurgeStatusType::Complete {
|
||||
mark_delete = false;
|
||||
}
|
||||
|
||||
if version_found && (!goi.version_purge_status.is_empty() || !goi.delete_marker) {
|
||||
delete_marker = false;
|
||||
}
|
||||
}
|
||||
let (mark_delete, mut delete_marker) = resolve_delete_version_state(&opts, &goi, version_found);
|
||||
|
||||
let mod_time = if let Some(mt) = opts.mod_time {
|
||||
mt
|
||||
@@ -2427,6 +2395,63 @@ impl ObjectOperations for SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
fn should_preserve_delete_replication_state(opts: &ObjectOptions) -> bool {
|
||||
opts.delete_replication.as_ref().is_some_and(|state| {
|
||||
state.replica_status == ReplicationStatusType::Replica
|
||||
|| (!state.replicate_decision_str.is_empty()
|
||||
&& (!state.composite_replication_status().is_empty() || !state.composite_version_purge_status().is_empty()))
|
||||
}) || opts.version_purge_status() == VersionPurgeStatusType::Complete
|
||||
}
|
||||
|
||||
fn resolve_delete_version_state(opts: &ObjectOptions, goi: &ObjectInfo, version_found: bool) -> (bool, bool) {
|
||||
let mut mark_delete = goi.version_id.is_some();
|
||||
let mut delete_marker = opts.versioned;
|
||||
|
||||
if opts.version_id.is_some() {
|
||||
// Decommission/rebalance may recreate a delete marker on a new pool before that
|
||||
// exact version exists there, so we must still treat it as a mark-delete write.
|
||||
if opts.data_movement && opts.delete_marker && !version_found {
|
||||
mark_delete = true;
|
||||
}
|
||||
|
||||
let delete_marker_version_purge = version_found && goi.delete_marker && !opts.version_purge_status().is_empty();
|
||||
|
||||
if version_found && opts.delete_marker_replication_status() == ReplicationStatusType::Replica {
|
||||
mark_delete = false;
|
||||
}
|
||||
|
||||
if opts.version_purge_status().is_empty() && opts.delete_marker_replication_status().is_empty() {
|
||||
mark_delete = false;
|
||||
}
|
||||
|
||||
if opts.version_purge_status() == VersionPurgeStatusType::Complete {
|
||||
mark_delete = false;
|
||||
}
|
||||
|
||||
let replica_delete_marker_version_purge =
|
||||
version_found && goi.delete_marker && opts.delete_marker_replication_status() == ReplicationStatusType::Replica;
|
||||
|
||||
if delete_marker_version_purge {
|
||||
mark_delete = false;
|
||||
}
|
||||
|
||||
if !version_found && !opts.delete_marker && opts.delete_marker_replication_status() == ReplicationStatusType::Replica {
|
||||
delete_marker = false;
|
||||
}
|
||||
|
||||
if version_found
|
||||
&& (!goi.version_purge_status.is_empty()
|
||||
|| !goi.delete_marker
|
||||
|| replica_delete_marker_version_purge
|
||||
|| delete_marker_version_purge)
|
||||
{
|
||||
delete_marker = false;
|
||||
}
|
||||
}
|
||||
|
||||
(mark_delete, delete_marker)
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
#[tracing::instrument(skip(self, fi, opts))]
|
||||
pub(crate) async fn decommission_tiered_object(
|
||||
@@ -4337,6 +4362,7 @@ mod tests {
|
||||
use crate::store_api::{CompletePart, ObjectInfo};
|
||||
use crate::store_init::save_format_file;
|
||||
use rustfs_filemeta::ErasureInfo;
|
||||
use rustfs_filemeta::ReplicationState;
|
||||
use rustfs_lock::client::local::LocalClient;
|
||||
use rustfs_lock::{LockError, LockInfo, LockResponse, LockStats};
|
||||
use serial_test::serial;
|
||||
@@ -4564,6 +4590,137 @@ mod tests {
|
||||
assert!(is_min_allowed_part_size(100 * 1024 * 1024)); // 100MB - allowed
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_delete_version_state_clears_delete_marker_for_replica_marker_version_purge() {
|
||||
let opts = ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
delete_replication: Some(ReplicationState {
|
||||
replica_status: ReplicationStatusType::Replica,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let current = ObjectInfo {
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
delete_marker: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (mark_delete, delete_marker) = resolve_delete_version_state(&opts, ¤t, true);
|
||||
|
||||
assert!(!mark_delete);
|
||||
assert!(
|
||||
!delete_marker,
|
||||
"replica purge of an existing delete marker version must remove that version, not preserve delete-marker semantics"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_delete_version_state_keeps_delete_marker_for_replica_marker_creation() {
|
||||
let opts = ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
delete_marker: true,
|
||||
delete_replication: Some(ReplicationState {
|
||||
replica_status: ReplicationStatusType::Replica,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (mark_delete, delete_marker) = resolve_delete_version_state(&opts, &ObjectInfo::default(), false);
|
||||
|
||||
assert!(!mark_delete);
|
||||
assert!(delete_marker);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_delete_version_state_skips_marker_creation_for_replica_purge_when_version_missing() {
|
||||
let opts = ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
delete_replication: Some(ReplicationState {
|
||||
replica_status: ReplicationStatusType::Replica,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (mark_delete, delete_marker) = resolve_delete_version_state(&opts, &ObjectInfo::default(), false);
|
||||
|
||||
assert!(
|
||||
!mark_delete,
|
||||
"replica delete-marker purges should not schedule mark-delete writes when the target version is absent"
|
||||
);
|
||||
assert!(
|
||||
!delete_marker,
|
||||
"replica delete-marker purges must become no-ops when the marker version has not arrived on the target yet"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_preserve_delete_replication_state_for_completed_delete_marker_replication_update() {
|
||||
let opts = ObjectOptions {
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
delete_replication: Some(ReplicationState {
|
||||
replicate_decision_str: "target=true;false;target;".to_string(),
|
||||
replication_status_internal: Some("target=COMPLETED;".to_string()),
|
||||
targets: rustfs_filemeta::replication_statuses_map("target=COMPLETED;"),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
should_preserve_delete_replication_state(&opts),
|
||||
"source delete-marker replication status updates must not be re-evaluated as fresh delete replication requests"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_not_preserve_delete_replication_state_for_new_version_delete_request() {
|
||||
let opts = ObjectOptions {
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
!should_preserve_delete_replication_state(&opts),
|
||||
"fresh versioned deletes still need replication eligibility checks"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_delete_version_state_removes_source_delete_marker_version_during_purge_replication() {
|
||||
let opts = ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
delete_replication: Some(ReplicationState {
|
||||
version_purge_status_internal: Some("target=PENDING;".to_string()),
|
||||
purge_targets: rustfs_filemeta::version_purge_statuses_map("target=PENDING;"),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let current = ObjectInfo {
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
delete_marker: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (mark_delete, delete_marker) = resolve_delete_version_state(&opts, ¤t, true);
|
||||
|
||||
assert!(
|
||||
!mark_delete,
|
||||
"source delete-marker version purge should delete the local marker instead of rewriting it with purge metadata"
|
||||
);
|
||||
assert!(
|
||||
!delete_marker,
|
||||
"source delete-marker version purge should not leave delete-marker semantics behind locally"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_complete_multipart_md5() {
|
||||
// Test MD5 calculation for multipart upload
|
||||
|
||||
@@ -15,7 +15,9 @@
|
||||
#![allow(clippy::map_entry)]
|
||||
|
||||
use crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
|
||||
use crate::bucket::lifecycle::bucket_lifecycle_ops::{enqueue_transition_immediate, init_background_expiry};
|
||||
use crate::bucket::lifecycle::bucket_lifecycle_ops::{
|
||||
enqueue_immediate_expiry, enqueue_transition_immediate, init_background_expiry,
|
||||
};
|
||||
use crate::bucket::metadata_sys::{self, set_bucket_metadata};
|
||||
use crate::bucket::utils::check_abort_multipart_args;
|
||||
use crate::bucket::utils::check_complete_multipart_args;
|
||||
@@ -134,7 +136,8 @@ async fn enqueue_transition_after_write(result: Result<ObjectInfo>, src: LcEvent
|
||||
match result {
|
||||
Ok(oi) => {
|
||||
if should_enqueue_transition_immediately(&oi) {
|
||||
enqueue_transition_immediate(&oi, src).await;
|
||||
enqueue_transition_immediate(&oi, src.clone()).await;
|
||||
enqueue_immediate_expiry(&oi, src).await;
|
||||
}
|
||||
Ok(oi)
|
||||
}
|
||||
|
||||
@@ -446,6 +446,11 @@ impl ObjectInfo {
|
||||
.replication_state_internal
|
||||
.as_ref()
|
||||
.and_then(|v| v.version_purge_status_internal.clone());
|
||||
let replication_decision = fi
|
||||
.replication_state_internal
|
||||
.as_ref()
|
||||
.map(|v| v.replicate_decision_str.clone())
|
||||
.unwrap_or_default();
|
||||
|
||||
let mut replication_status = fi.replication_status();
|
||||
if replication_status.is_empty()
|
||||
@@ -542,6 +547,7 @@ impl ObjectInfo {
|
||||
replication_status,
|
||||
version_purge_status_internal,
|
||||
version_purge_status,
|
||||
replication_decision,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
@@ -1010,6 +1016,7 @@ pub struct ObjectInfoOrErr {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_filemeta::ReplicationState;
|
||||
|
||||
#[test]
|
||||
fn get_actual_size_prefers_actual_size_field() {
|
||||
@@ -1059,6 +1066,21 @@ mod tests {
|
||||
assert_eq!(info.get_actual_size().unwrap(), 77);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_file_info_preserves_replication_decision() {
|
||||
let fi = rustfs_filemeta::FileInfo {
|
||||
replication_state_internal: Some(ReplicationState {
|
||||
replicate_decision_str: "arn=true;false;arn:replication::1:dest;rule-id".to_string(),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let info = ObjectInfo::from_file_info(&fi, "bucket", "object", true);
|
||||
|
||||
assert_eq!(info.replication_decision, "arn=true;false;arn:replication::1:dest;rule-id");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_actual_size_uses_compressed_parts_actual_size_when_metadata_missing() {
|
||||
let user_defined = {
|
||||
|
||||
@@ -382,6 +382,11 @@ impl FileMeta {
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
pub fn delete_version(&mut self, fi: &FileInfo) -> Result<Option<Uuid>> {
|
||||
let vid = Some(fi.version_id.unwrap_or(Uuid::nil()));
|
||||
let target_is_delete_marker = self
|
||||
.versions
|
||||
.iter()
|
||||
.find(|ver| ver.header.version_id == vid)
|
||||
.is_some_and(|ver| ver.header.version_type == VersionType::Delete);
|
||||
|
||||
let mut ventry = FileMetaVersion::default();
|
||||
if fi.deleted {
|
||||
@@ -416,6 +421,10 @@ impl FileMeta {
|
||||
}
|
||||
}
|
||||
|
||||
if target_is_delete_marker && !fi.deleted && !fi.version_purge_status().is_empty() {
|
||||
update_version = false;
|
||||
}
|
||||
|
||||
if fi.deleted {
|
||||
if !fi.delete_marker_replication_status().is_empty()
|
||||
&& let Some(delete_marker) = ventry.delete_marker.as_mut()
|
||||
@@ -1659,6 +1668,45 @@ mod test {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_version_removes_delete_marker_during_version_purge_replication() {
|
||||
let version_id = Uuid::new_v4();
|
||||
let mut fm = FileMeta::new();
|
||||
fm.add_version_filemata(FileMetaVersion {
|
||||
version_type: VersionType::Delete,
|
||||
legacy_object: None,
|
||||
object: None,
|
||||
delete_marker: Some(MetaDeleteMarker {
|
||||
version_id: Some(version_id),
|
||||
mod_time: Some(OffsetDateTime::now_utc()),
|
||||
meta_sys: HashMap::new(),
|
||||
}),
|
||||
write_version: 1,
|
||||
uses_legacy_checksum: false,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let fi = FileInfo {
|
||||
deleted: false,
|
||||
mark_deleted: false,
|
||||
version_id: Some(version_id),
|
||||
replication_state_internal: Some(ReplicationState {
|
||||
version_purge_status_internal: Some("target=PENDING;".to_string()),
|
||||
purge_targets: version_purge_statuses_map("target=PENDING;"),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let result = fm.delete_version(&fi).unwrap();
|
||||
|
||||
assert!(result.is_none());
|
||||
assert!(
|
||||
fm.versions.is_empty(),
|
||||
"delete-marker version purge should remove the local marker instead of rewriting purge metadata onto it"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_data_integrity_validation() {
|
||||
// Test data integrity checks
|
||||
|
||||
@@ -55,6 +55,7 @@ metrics = { workspace = true }
|
||||
[dev-dependencies]
|
||||
tracing-subscriber = { workspace = true }
|
||||
serial_test = { workspace = true }
|
||||
temp-env = { workspace = true }
|
||||
uuid = { workspace = true, features = ["v4", "serde"] }
|
||||
tokio = { workspace = true, features = ["test-util"] }
|
||||
|
||||
|
||||
@@ -22,10 +22,8 @@ use crate::{DataUsageInfo, ScannerError};
|
||||
use chrono::{DateTime, Utc};
|
||||
use rustfs_common::heal_channel::HealScanMode;
|
||||
use rustfs_common::metrics::{CurrentCycle, Metric, Metrics, emit_scan_cycle_complete, global_metrics};
|
||||
use rustfs_config::DEFAULT_SCANNER_SPEED;
|
||||
use rustfs_config::ENV_SCANNER_SPEED;
|
||||
use rustfs_config::ENV_SCANNER_START_DELAY_SECS;
|
||||
use rustfs_config::ScannerSpeed;
|
||||
use rustfs_config::{DEFAULT_SCANNER_SPEED, ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS};
|
||||
use rustfs_ecstore::StorageAPI as _;
|
||||
use rustfs_ecstore::config::com::{read_config, save_config};
|
||||
use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
|
||||
@@ -40,11 +38,15 @@ use tracing::{debug, error, info, instrument, warn};
|
||||
|
||||
const ENV_SCANNER_START_DELAY_SECS_DEPRECATED: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS";
|
||||
|
||||
/// Returns the base cycle interval. If `RUSTFS_SCANNER_START_DELAY_SECS`
|
||||
/// is set (or `RUSTFS_DATA_SCANNER_START_DELAY_SECS` as deprecated alias),
|
||||
/// it takes precedence; otherwise the value is derived from the
|
||||
/// `RUSTFS_SCANNER_SPEED` preset.
|
||||
/// Returns the base cycle interval.
|
||||
/// Priority order:
|
||||
/// 1. RUSTFS_SCANNER_CYCLE (if set, overrides everything)
|
||||
/// 2. RUSTFS_SCANNER_START_DELAY_SECS (for backward compatibility)
|
||||
/// 3. RUSTFS_SCANNER_SPEED preset
|
||||
fn cycle_interval() -> Duration {
|
||||
if let Some(secs) = rustfs_utils::get_env_opt_u64(ENV_SCANNER_CYCLE) {
|
||||
return Duration::from_secs(secs);
|
||||
}
|
||||
if let Some(secs) = scanner_start_delay_secs() {
|
||||
return Duration::from_secs(secs);
|
||||
}
|
||||
@@ -181,6 +183,7 @@ fn get_lock_acquire_timeout() -> Duration {
|
||||
|
||||
#[instrument(skip_all)]
|
||||
async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>, cycle_info: &mut CurrentCycle) {
|
||||
SCANNER_SLEEPER.refresh_from_env();
|
||||
info!("Start run data scanner cycle");
|
||||
cycle_info.current = cycle_info.next;
|
||||
let now = Instant::now();
|
||||
@@ -356,6 +359,7 @@ pub async fn store_data_usage_in_backend(
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serial_test::serial;
|
||||
use temp_env::{with_var, with_var_unset};
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
@@ -376,6 +380,42 @@ mod tests {
|
||||
assert!(delay <= Duration::from_secs(132));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_cycle_interval_prefers_explicit_cycle_override() {
|
||||
with_var(ENV_SCANNER_SPEED, Some("slowest"), || {
|
||||
with_var(ENV_SCANNER_CYCLE, Some("42"), || {
|
||||
assert_eq!(cycle_interval(), Duration::from_secs(42));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_cycle_interval_supports_minio_speed_alias() {
|
||||
with_var_unset(ENV_SCANNER_SPEED, || {
|
||||
with_var_unset(ENV_SCANNER_CYCLE, || {
|
||||
with_var_unset(ENV_SCANNER_START_DELAY_SECS, || {
|
||||
with_var("MINIO_SCANNER_SPEED", Some("slowest"), || {
|
||||
assert_eq!(cycle_interval(), Duration::from_secs(30 * 60));
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_cycle_interval_supports_minio_cycle_alias() {
|
||||
with_var_unset(ENV_SCANNER_CYCLE, || {
|
||||
with_var_unset(ENV_SCANNER_START_DELAY_SECS, || {
|
||||
with_var("MINIO_SCANNER_CYCLE", Some("90"), || {
|
||||
assert_eq!(cycle_interval(), Duration::from_secs(90));
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_randomized_cycle_delay_handles_small_start_delay() {
|
||||
|
||||
@@ -30,7 +30,7 @@ use rustfs_common::heal_channel::{
|
||||
};
|
||||
use rustfs_common::metrics::{IlmAction, Metric, Metrics, UpdateCurrentPathFn, current_path_updater};
|
||||
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
|
||||
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_rule;
|
||||
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{GLOBAL_ExpiryState, apply_expiry_rule};
|
||||
use rustfs_ecstore::bucket::lifecycle::evaluator::Evaluator;
|
||||
use rustfs_ecstore::bucket::lifecycle::{
|
||||
bucket_lifecycle_ops::apply_transition_rule,
|
||||
@@ -444,8 +444,14 @@ impl ScannerItem {
|
||||
}
|
||||
}
|
||||
|
||||
if !to_delete_objs.is_empty() {
|
||||
// TODO: enqueueNoncurrentVersions
|
||||
if !to_delete_objs.is_empty()
|
||||
&& let Some(event) = noncurrent_events.first().cloned()
|
||||
{
|
||||
GLOBAL_ExpiryState
|
||||
.write()
|
||||
.await
|
||||
.enqueue_by_newer_noncurrent(&self.bucket, to_delete_objs, event)
|
||||
.await;
|
||||
}
|
||||
self.alert_excessive_versions(remaining_versions, cumulative_size);
|
||||
}
|
||||
|
||||
@@ -21,6 +21,13 @@ use tokio::time::Duration;
|
||||
|
||||
const MIN_SLEEP: Duration = Duration::from_millis(1);
|
||||
|
||||
fn scanner_env_config() -> (ScannerSpeed, bool) {
|
||||
let speed_str = rustfs_utils::get_env_str(ENV_SCANNER_SPEED, DEFAULT_SCANNER_SPEED);
|
||||
let speed = ScannerSpeed::from_env_str(&speed_str);
|
||||
let idle_mode = rustfs_utils::get_env_bool(ENV_SCANNER_IDLE_MODE, DEFAULT_SCANNER_IDLE_MODE);
|
||||
(speed, idle_mode)
|
||||
}
|
||||
|
||||
/// When `true` (default), the scanner throttles itself between operations.
|
||||
/// When `false`, all sleeps are skipped and the scanner runs at full speed.
|
||||
pub static SCANNER_IDLE_MODE: AtomicBool = AtomicBool::new(DEFAULT_SCANNER_IDLE_MODE);
|
||||
@@ -28,10 +35,7 @@ pub static SCANNER_IDLE_MODE: AtomicBool = AtomicBool::new(DEFAULT_SCANNER_IDLE_
|
||||
/// Global scanner sleeper initialized from the `RUSTFS_SCANNER_SPEED` and
|
||||
/// `RUSTFS_SCANNER_IDLE_MODE` environment variables.
|
||||
pub static SCANNER_SLEEPER: LazyLock<DynamicSleeper> = LazyLock::new(|| {
|
||||
let speed_str = rustfs_utils::get_env_str(ENV_SCANNER_SPEED, DEFAULT_SCANNER_SPEED);
|
||||
let speed = ScannerSpeed::from_env_str(&speed_str);
|
||||
|
||||
let idle_mode = rustfs_utils::get_env_bool(ENV_SCANNER_IDLE_MODE, DEFAULT_SCANNER_IDLE_MODE);
|
||||
let (speed, idle_mode) = scanner_env_config();
|
||||
SCANNER_IDLE_MODE.store(idle_mode, Ordering::Relaxed);
|
||||
|
||||
DynamicSleeper::new(speed)
|
||||
@@ -104,6 +108,13 @@ impl DynamicSleeper {
|
||||
let mut m = self.inner.max_sleep.write().unwrap_or_else(|e| e.into_inner());
|
||||
*m = speed.max_sleep();
|
||||
}
|
||||
|
||||
/// Reload speed and idle-mode settings from the current environment.
|
||||
pub fn refresh_from_env(&self) {
|
||||
let (speed, idle_mode) = scanner_env_config();
|
||||
self.update(speed);
|
||||
SCANNER_IDLE_MODE.store(idle_mode, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
/// A timer returned by [`DynamicSleeper::timer`]. Records the instant it
|
||||
@@ -138,6 +149,7 @@ impl SleepTimer {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serial_test::serial;
|
||||
use temp_env::with_var;
|
||||
|
||||
#[test]
|
||||
fn test_scanner_speed_presets() {
|
||||
@@ -166,6 +178,26 @@ mod tests {
|
||||
assert_eq!(max_sleep, Duration::from_secs(15));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_refresh_from_env_applies_speed_and_idle_mode_for_next_cycle() {
|
||||
let prev_mode = SCANNER_IDLE_MODE.load(Ordering::Relaxed);
|
||||
SCANNER_IDLE_MODE.store(true, Ordering::Relaxed);
|
||||
|
||||
let s = DynamicSleeper::new(ScannerSpeed::Fastest);
|
||||
with_var(ENV_SCANNER_SPEED, Some("slow"), || {
|
||||
with_var(ENV_SCANNER_IDLE_MODE, Some("false"), || {
|
||||
s.refresh_from_env();
|
||||
let (factor, max_sleep) = s.read_params();
|
||||
assert_eq!(factor, 10.0);
|
||||
assert_eq!(max_sleep, Duration::from_secs(15));
|
||||
assert!(!SCANNER_IDLE_MODE.load(Ordering::Relaxed));
|
||||
});
|
||||
});
|
||||
|
||||
SCANNER_IDLE_MODE.store(prev_mode, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn test_fastest_never_sleeps() {
|
||||
|
||||
@@ -487,6 +487,89 @@ async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) -> usi
|
||||
.len()
|
||||
}
|
||||
|
||||
async fn object_version_count(disk_path: &Path, bucket: &str, object: &str) -> usize {
|
||||
let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("failed to open local disk");
|
||||
let data = disk
|
||||
.read_metadata(bucket, &path_join_buf(&[object, STORAGE_FORMAT_FILE]))
|
||||
.await
|
||||
.expect("failed to read object metadata");
|
||||
let meta = FileMeta::load(&data).expect("failed to load file metadata");
|
||||
meta.get_file_info_versions(bucket, object, false)
|
||||
.expect("failed to decode file info versions")
|
||||
.versions
|
||||
.len()
|
||||
}
|
||||
|
||||
async fn wait_for_version_count(disk_path: &Path, bucket: &str, object: &str, expected: usize, timeout: Duration) -> bool {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
|
||||
loop {
|
||||
if object_version_count(disk_path, bucket, object).await == expected {
|
||||
return true;
|
||||
}
|
||||
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return false;
|
||||
}
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn scan_object_with_lifecycle(disk_path: &Path, bucket: &str, object: &str) {
|
||||
let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("failed to open local disk");
|
||||
let metadata_path = disk_path.join(bucket).join(object).join(STORAGE_FORMAT_FILE);
|
||||
let relative_path = metadata_path.to_string_lossy().to_string();
|
||||
let (_, scanner_path) = path2_bucket_object_with_base_path(disk_path.to_string_lossy().as_ref(), relative_path.as_str());
|
||||
let file_type = fs::metadata(&metadata_path)
|
||||
.await
|
||||
.expect("failed to stat object metadata")
|
||||
.file_type();
|
||||
let lifecycle = metadata_sys::get(bucket)
|
||||
.await
|
||||
.expect("failed to load bucket metadata")
|
||||
.lifecycle_config
|
||||
.clone()
|
||||
.map(Arc::new);
|
||||
let item = ScannerItem {
|
||||
path: scanner_path.clone(),
|
||||
bucket: bucket.to_string(),
|
||||
prefix: object.to_string(),
|
||||
object_name: STORAGE_FORMAT_FILE.to_string(),
|
||||
file_type,
|
||||
lifecycle,
|
||||
replication: None,
|
||||
heal_enabled: false,
|
||||
heal_bitrot: false,
|
||||
debug: false,
|
||||
};
|
||||
disk.get_size(item).await.expect("scanner get_size should succeed");
|
||||
}
|
||||
|
||||
async fn scan_object_metadata(disk_path: &Path, bucket: &str, object: &str) {
|
||||
let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
|
||||
endpoint.set_pool_index(0);
|
||||
@@ -1131,4 +1214,255 @@ mod serial_tests {
|
||||
"deleted object should remain absent after scanner cleanup"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
async fn test_scanner_expires_zero_day_current_version() {
|
||||
let (disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
||||
|
||||
let bucket_name = format!("test-zero-day-expire-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object_name = "test/object.txt";
|
||||
|
||||
create_test_bucket(&ecstore, bucket_name.as_str()).await;
|
||||
set_bucket_lifecycle(bucket_name.as_str())
|
||||
.await
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await;
|
||||
|
||||
assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await);
|
||||
|
||||
scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;
|
||||
|
||||
assert!(
|
||||
wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(3)).await,
|
||||
"scanner should delete zero-day current version after enqueueing expiry"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
async fn test_put_object_immediately_enqueues_zero_day_current_expiry() {
|
||||
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
||||
|
||||
let bucket_name = format!("test-put-zero-day-expire-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object_name = "expire-now.txt";
|
||||
|
||||
create_test_bucket(&ecstore, bucket_name.as_str()).await;
|
||||
|
||||
let lifecycle_xml = format!(
|
||||
r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<LifecycleConfiguration>
|
||||
<Rule>
|
||||
<ID>test-rule</ID>
|
||||
<Status>Enabled</Status>
|
||||
<Filter>
|
||||
<Prefix>{object_name}</Prefix>
|
||||
</Filter>
|
||||
<Expiration>
|
||||
<Days>0</Days>
|
||||
</Expiration>
|
||||
</Rule>
|
||||
</LifecycleConfiguration>"#
|
||||
);
|
||||
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
|
||||
.await
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
|
||||
upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await;
|
||||
|
||||
assert!(
|
||||
wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(2)).await,
|
||||
"put_object should enqueue zero-day current expiry without waiting for scanner"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
async fn test_scanner_expires_zero_day_noncurrent_version() {
|
||||
let (disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
||||
|
||||
let bucket_name = format!("test-zero-day-noncurrent-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object_name = "test/object.txt";
|
||||
|
||||
create_test_lock_bucket(&ecstore, bucket_name.as_str()).await;
|
||||
|
||||
let mut reader = PutObjReader::from_vec(b"v1".to_vec());
|
||||
ecstore
|
||||
.put_object(
|
||||
bucket_name.as_str(),
|
||||
object_name,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("failed to upload v1");
|
||||
let mut reader = PutObjReader::from_vec(b"v2".to_vec());
|
||||
ecstore
|
||||
.put_object(
|
||||
bucket_name.as_str(),
|
||||
object_name,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("failed to upload v2");
|
||||
|
||||
assert_eq!(object_version_count(&disk_paths[0], bucket_name.as_str(), object_name).await, 2);
|
||||
|
||||
let lifecycle_xml = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<LifecycleConfiguration>
|
||||
<Rule>
|
||||
<ID>test-rule</ID>
|
||||
<Status>Enabled</Status>
|
||||
<Filter>
|
||||
<Prefix>test/</Prefix>
|
||||
</Filter>
|
||||
<NoncurrentVersionExpiration>
|
||||
<NoncurrentDays>0</NoncurrentDays>
|
||||
</NoncurrentVersionExpiration>
|
||||
</Rule>
|
||||
</LifecycleConfiguration>"#;
|
||||
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
|
||||
.await
|
||||
.expect("Failed to set noncurrent lifecycle configuration");
|
||||
|
||||
rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
|
||||
|
||||
scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;
|
||||
|
||||
assert!(
|
||||
wait_for_version_count(&disk_paths[0], bucket_name.as_str(), object_name, 1, Duration::from_secs(3)).await,
|
||||
"scanner should delete zero-day noncurrent versions after enqueueing expiry"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
async fn test_put_object_immediately_enqueues_zero_day_noncurrent_expiry() {
|
||||
let (disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
||||
|
||||
let bucket_name = format!("test-put-zero-day-noncurrent-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object_name = "test/object.txt";
|
||||
|
||||
create_test_lock_bucket(&ecstore, bucket_name.as_str()).await;
|
||||
|
||||
let lifecycle_xml = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<LifecycleConfiguration>
|
||||
<Rule>
|
||||
<ID>test-rule</ID>
|
||||
<Status>Enabled</Status>
|
||||
<Filter>
|
||||
<Prefix>test/</Prefix>
|
||||
</Filter>
|
||||
<NoncurrentVersionExpiration>
|
||||
<NoncurrentDays>0</NoncurrentDays>
|
||||
</NoncurrentVersionExpiration>
|
||||
</Rule>
|
||||
</LifecycleConfiguration>"#;
|
||||
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
|
||||
.await
|
||||
.expect("Failed to set noncurrent lifecycle configuration");
|
||||
|
||||
let mut reader = PutObjReader::from_vec(b"v1".to_vec());
|
||||
ecstore
|
||||
.put_object(
|
||||
bucket_name.as_str(),
|
||||
object_name,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("failed to upload v1");
|
||||
let mut reader = PutObjReader::from_vec(b"v2".to_vec());
|
||||
ecstore
|
||||
.put_object(
|
||||
bucket_name.as_str(),
|
||||
object_name,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("failed to upload v2");
|
||||
|
||||
assert!(
|
||||
wait_for_version_count(&disk_paths[0], bucket_name.as_str(), object_name, 1, Duration::from_secs(2)).await,
|
||||
"put_object should enqueue zero-day noncurrent expiry without waiting for scanner"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
async fn test_background_scanner_expires_zero_day_current_version() {
|
||||
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
||||
|
||||
let bucket_name = format!("test-bg-zero-day-expire-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object_name = "test/object.txt";
|
||||
|
||||
create_test_bucket(&ecstore, bucket_name.as_str()).await;
|
||||
set_bucket_lifecycle(bucket_name.as_str())
|
||||
.await
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await;
|
||||
|
||||
let ctx = CancellationToken::new();
|
||||
init_data_scanner(ctx.clone(), ecstore.clone()).await;
|
||||
|
||||
let deleted = wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(12)).await;
|
||||
|
||||
ctx.cancel();
|
||||
|
||||
assert!(deleted, "background scanner should delete zero-day current version after startup delay");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
async fn test_background_scanner_expires_zero_day_current_version_for_exact_key_prefix() {
|
||||
let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
|
||||
|
||||
let bucket_name = format!("test-bg-zero-day-exact-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object_name = "expire-now.txt";
|
||||
|
||||
create_test_bucket(&ecstore, bucket_name.as_str()).await;
|
||||
|
||||
let lifecycle_xml = format!(
|
||||
r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<LifecycleConfiguration>
|
||||
<Rule>
|
||||
<ID>test-rule</ID>
|
||||
<Status>Enabled</Status>
|
||||
<Filter>
|
||||
<Prefix>{object_name}</Prefix>
|
||||
</Filter>
|
||||
<Expiration>
|
||||
<Days>0</Days>
|
||||
</Expiration>
|
||||
</Rule>
|
||||
</LifecycleConfiguration>"#
|
||||
);
|
||||
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
|
||||
.await
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await;
|
||||
|
||||
let ctx = CancellationToken::new();
|
||||
init_data_scanner(ctx.clone(), ecstore.clone()).await;
|
||||
|
||||
let deleted = wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(12)).await;
|
||||
|
||||
ctx.cancel();
|
||||
|
||||
assert!(deleted, "background scanner should delete zero-day exact-key lifecycle targets");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -173,6 +173,8 @@ const EXTERNAL_COMPATIBLE_SUFFIXES: &[&str] = &[
|
||||
"REGION",
|
||||
"ROOT_PASSWORD",
|
||||
"ROOT_USER",
|
||||
"SCANNER_CYCLE",
|
||||
"SCANNER_SPEED",
|
||||
"SECRET_KEY",
|
||||
"SECRET_KEY_FILE",
|
||||
"STORAGE_CLASS_INLINE_BLOCK",
|
||||
|
||||
@@ -34,7 +34,9 @@ use http::StatusCode;
|
||||
use metrics::counter;
|
||||
use rustfs_config::RUSTFS_REGION;
|
||||
use rustfs_ecstore::bucket::{
|
||||
lifecycle::bucket_lifecycle_ops::{enqueue_transition_for_existing_objects, validate_transition_tier},
|
||||
lifecycle::bucket_lifecycle_ops::{
|
||||
enqueue_expiry_for_existing_objects, enqueue_transition_for_existing_objects, validate_transition_tier,
|
||||
},
|
||||
metadata::{
|
||||
BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG,
|
||||
BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_REPLICATION_CONFIG, BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG,
|
||||
@@ -494,6 +496,13 @@ fn lifecycle_has_transition_rules(config: &BucketLifecycleConfiguration) -> bool
|
||||
})
|
||||
}
|
||||
|
||||
fn lifecycle_has_expiry_rules(config: &BucketLifecycleConfiguration) -> bool {
|
||||
config.rules.iter().any(|rule| {
|
||||
rule.status == ExpirationStatus::from_static(ExpirationStatus::ENABLED)
|
||||
&& (rule.expiration.is_some() || rule.del_marker_expiration.is_some() || rule.noncurrent_version_expiration.is_some())
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct DefaultBucketUsecase {
|
||||
context: Option<Arc<AppContext>>,
|
||||
@@ -1342,6 +1351,18 @@ impl DefaultBucketUsecase {
|
||||
});
|
||||
}
|
||||
|
||||
if lifecycle_has_expiry_rules(&input_cfg)
|
||||
&& let Some(store) = new_object_layer_fn()
|
||||
{
|
||||
let bucket_name = bucket.clone();
|
||||
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
|
||||
spawn_background_with_context(request_context, async move {
|
||||
if let Err(err) = enqueue_expiry_for_existing_objects(store, &bucket_name).await {
|
||||
warn!(bucket = %bucket_name, error = ?err, "failed to enqueue expiry for existing objects");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Ok(S3Response::new(PutBucketLifecycleConfigurationOutput::default()))
|
||||
}
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::{multipart_usecase::DefaultMultipartUsecase, object_usecase::DefaultObjectUsecase};
|
||||
use crate::app::bucket_usecase::DefaultBucketUsecase;
|
||||
use crate::storage::ecfs::FS;
|
||||
use bytes::Bytes;
|
||||
use futures::stream;
|
||||
@@ -182,6 +183,40 @@ async fn set_bucket_lifecycle_transition_with_tier(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn expiration_lifecycle_configuration(prefix: &str) -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
expiration: Some(LifecycleExpiration {
|
||||
date: Some(Timestamp::from(
|
||||
time::OffsetDateTime::now_utc()
|
||||
.replace_time(time::Time::MIDNIGHT)
|
||||
.saturating_sub(time::Duration::days(1)),
|
||||
)),
|
||||
days: None,
|
||||
expired_object_delete_marker: None,
|
||||
..Default::default()
|
||||
}),
|
||||
filter: Some(LifecycleRuleFilter {
|
||||
and: None,
|
||||
object_size_greater_than: None,
|
||||
object_size_less_than: None,
|
||||
prefix: Some(prefix.to_string()),
|
||||
tag: None,
|
||||
..Default::default()
|
||||
}),
|
||||
id: Some("expire-existing".to_string()),
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: None,
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct MockWarmBackend {
|
||||
objects: Arc<Mutex<HashMap<String, Vec<u8>>>>,
|
||||
@@ -326,6 +361,24 @@ async fn wait_for_object_absence(ecstore: &Arc<ECStore>, bucket: &str, object: &
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_delete_marker(ecstore: &Arc<ECStore>, bucket: &str, object: &str, timeout: Duration) -> bool {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
|
||||
loop {
|
||||
if let Ok(info) = ecstore.get_object_info(bucket, object, &ObjectOptions::default()).await
|
||||
&& info.delete_marker
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return false;
|
||||
}
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
}
|
||||
|
||||
fn build_request<T>(input: T, method: Method) -> S3Request<T> {
|
||||
S3Request {
|
||||
input,
|
||||
@@ -583,3 +636,37 @@ async fn lifecycle_transition_marks_dirty_disks_for_capacity_manager() {
|
||||
.collect();
|
||||
assert_eq!(actual_paths, expected_paths);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "requires isolated global object layer state"]
|
||||
async fn put_bucket_lifecycle_configuration_expires_existing_objects() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let bucket = format!("test-api-expire-existing-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "test/existing.txt";
|
||||
let payload = b"expire existing object after lifecycle update";
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
|
||||
|
||||
let req = build_request(
|
||||
PutBucketLifecycleConfigurationInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.lifecycle_configuration(Some(expiration_lifecycle_configuration("test/")))
|
||||
.build()
|
||||
.unwrap(),
|
||||
Method::PUT,
|
||||
);
|
||||
|
||||
usecase
|
||||
.execute_put_bucket_lifecycle_configuration(req)
|
||||
.await
|
||||
.expect("Failed to update lifecycle configuration");
|
||||
|
||||
assert!(
|
||||
wait_for_delete_marker(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT).await,
|
||||
"existing object should be lifecycle-deleted after lifecycle update"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -58,8 +58,8 @@ use rustfs_ecstore::bucket::{
|
||||
},
|
||||
quota::QuotaOperation,
|
||||
replication::{
|
||||
DeletedObjectReplicationInfo, check_replicate_delete, get_must_replicate_options, must_replicate, schedule_replication,
|
||||
schedule_replication_delete,
|
||||
DeletedObjectReplicationInfo, ObjectOpts as ReplicationObjectOpts, ReplicationConfigurationExt, check_replicate_delete,
|
||||
get_must_replicate_options, must_replicate, schedule_replication, schedule_replication_delete,
|
||||
},
|
||||
tagging::decode_tags,
|
||||
versioning::VersioningApi,
|
||||
@@ -75,8 +75,9 @@ use rustfs_ecstore::store_api::{
|
||||
HTTPRangeSpec, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, PutObjReader,
|
||||
};
|
||||
use rustfs_filemeta::{
|
||||
REPLICATE_INCOMING_DELETE, ReplicationStatusType, ReplicationType, RestoreStatusOps, VersionPurgeStatusType,
|
||||
parse_restore_obj_status,
|
||||
REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateTargetDecision, ReplicationState, ReplicationStatusType,
|
||||
ReplicationType, RestoreStatusOps, VersionPurgeStatusType, parse_restore_obj_status, replication_statuses_map,
|
||||
version_purge_statuses_map,
|
||||
};
|
||||
use rustfs_io_metrics;
|
||||
use rustfs_notify::EventArgsBuilder;
|
||||
@@ -460,6 +461,100 @@ fn build_put_object_expiration_header(event: &lifecycle::Event) -> Option<String
|
||||
Some(format!("expiry-date=\"{}\", rule-id=\"{}\"", expiry_date, event.rule_id))
|
||||
}
|
||||
|
||||
fn delete_replication_state_from_config(
|
||||
config: &ReplicationConfiguration,
|
||||
obj_info: &ObjectInfo,
|
||||
version_id: Option<Uuid>,
|
||||
replica: bool,
|
||||
) -> Option<ReplicationState> {
|
||||
let opts = ReplicationObjectOpts {
|
||||
name: obj_info.name.clone(),
|
||||
user_tags: obj_info.user_tags.clone(),
|
||||
version_id,
|
||||
delete_marker: obj_info.delete_marker,
|
||||
op_type: ReplicationType::Delete,
|
||||
replica,
|
||||
..Default::default()
|
||||
};
|
||||
let target_arns = config.filter_target_arns(&opts);
|
||||
if target_arns.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut decision = ReplicateDecision::new();
|
||||
for target_arn in target_arns {
|
||||
let mut target_opts = opts.clone();
|
||||
target_opts.target_arn = target_arn.clone();
|
||||
decision.set(ReplicateTargetDecision::new(target_arn, config.replicate(&target_opts), false));
|
||||
}
|
||||
if !decision.replicate_any() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let pending_status = decision.pending_status();
|
||||
let mut state = ReplicationState {
|
||||
replicate_decision_str: decision.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
if version_id.is_some() {
|
||||
state.version_purge_status_internal = pending_status.clone();
|
||||
state.purge_targets = version_purge_statuses_map(pending_status.as_deref().unwrap_or_default());
|
||||
} else {
|
||||
state.replication_status_internal = pending_status.clone();
|
||||
state.targets = replication_statuses_map(pending_status.as_deref().unwrap_or_default());
|
||||
}
|
||||
Some(state)
|
||||
}
|
||||
|
||||
async fn enrich_delete_replication_state_if_needed(
|
||||
bucket: &str,
|
||||
delete_object: &mut rustfs_ecstore::store_api::DeletedObject,
|
||||
obj_info: &ObjectInfo,
|
||||
) {
|
||||
let Some(replication_state) = delete_object.replication_state.as_ref() else {
|
||||
return;
|
||||
};
|
||||
if !replication_state.replicate_decision_str.is_empty()
|
||||
&& (!replication_state.targets.is_empty() || !replication_state.purge_targets.is_empty())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
let Ok((config, _)) = metadata_sys::get_replication_config(bucket).await else {
|
||||
return;
|
||||
};
|
||||
let version_id = if delete_object.delete_marker {
|
||||
None
|
||||
} else if delete_object.delete_marker_version_id.is_some() {
|
||||
delete_object.delete_marker_version_id
|
||||
} else {
|
||||
delete_object.version_id
|
||||
};
|
||||
if let Some(local_state) = delete_replication_state_from_config(
|
||||
&config,
|
||||
obj_info,
|
||||
version_id,
|
||||
obj_info.replication_status == ReplicationStatusType::Replica,
|
||||
) {
|
||||
delete_object.replication_state = Some(local_state);
|
||||
}
|
||||
}
|
||||
|
||||
fn should_schedule_delete_replication(
|
||||
opts: &ObjectOptions,
|
||||
replication_source: &ObjectInfo,
|
||||
deleted_delete_marker_version: bool,
|
||||
) -> bool {
|
||||
if opts.replication_request {
|
||||
return false;
|
||||
}
|
||||
|
||||
replication_source.replication_status == ReplicationStatusType::Replica
|
||||
|| replication_source.replication_status == ReplicationStatusType::Pending
|
||||
|| replication_source.version_purge_status == VersionPurgeStatusType::Pending
|
||||
|| (deleted_delete_marker_version && replication_source.replication_status == ReplicationStatusType::Completed)
|
||||
}
|
||||
|
||||
const AMZ_SNOWBALL_EXTRACT_COMPAT: &str = "X-Amz-Snowball-Auto-Extract";
|
||||
#[cfg(test)]
|
||||
const AMZ_SNOWBALL_PREFIX_INTERNAL: &str = "X-Amz-Meta-Rustfs-Snowball-Prefix";
|
||||
@@ -1904,6 +1999,15 @@ impl DefaultObjectUsecase {
|
||||
None
|
||||
};
|
||||
|
||||
// x-amz-restore: extract from object metadata
|
||||
let restore = info.user_defined.get(X_AMZ_RESTORE.as_str()).and_then(|v| {
|
||||
let rs = parse_restore_obj_status(v).ok()?;
|
||||
Some(rs.to_string2())
|
||||
});
|
||||
|
||||
// x-amz-expiration: predict from lifecycle configuration
|
||||
let expiration = resolve_put_object_expiration(bucket, &info).await;
|
||||
|
||||
let output = GetObjectOutput {
|
||||
body,
|
||||
content_length: Some(response_content_length),
|
||||
@@ -1925,6 +2029,8 @@ impl DefaultObjectUsecase {
|
||||
checksum_crc64nvme: checksums.crc64nvme,
|
||||
checksum_type: checksums.checksum_type,
|
||||
version_id: output_version_id,
|
||||
restore,
|
||||
expiration,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -3126,25 +3232,35 @@ impl DefaultObjectUsecase {
|
||||
return result;
|
||||
}
|
||||
|
||||
if obj_info.replication_status == ReplicationStatusType::Replica
|
||||
|| obj_info.replication_status == ReplicationStatusType::Pending
|
||||
|| obj_info.version_purge_status == VersionPurgeStatusType::Pending
|
||||
{
|
||||
schedule_replication_delete(DeletedObjectReplicationInfo {
|
||||
let deleted_replication_info = existing_object_info.as_ref().filter(|_| opts.version_id.is_some());
|
||||
let replication_source = deleted_replication_info.unwrap_or(&obj_info);
|
||||
let deleted_delete_marker_version = deleted_replication_info.is_some_and(|info| info.delete_marker);
|
||||
|
||||
if should_schedule_delete_replication(&opts, replication_source, deleted_delete_marker_version) {
|
||||
let mut deleted_object = DeletedObjectReplicationInfo {
|
||||
delete_object: rustfs_ecstore::store_api::DeletedObject {
|
||||
delete_marker: obj_info.delete_marker,
|
||||
delete_marker_version_id: if obj_info.delete_marker { obj_info.version_id } else { None },
|
||||
delete_marker: replication_source.delete_marker && !deleted_delete_marker_version,
|
||||
delete_marker_version_id: if replication_source.delete_marker {
|
||||
replication_source.version_id
|
||||
} else {
|
||||
None
|
||||
},
|
||||
object_name: key.clone(),
|
||||
version_id: if obj_info.delete_marker { None } else { obj_info.version_id },
|
||||
delete_marker_mtime: obj_info.mod_time,
|
||||
replication_state: Some(obj_info.replication_state()),
|
||||
version_id: if replication_source.delete_marker {
|
||||
None
|
||||
} else {
|
||||
replication_source.version_id
|
||||
},
|
||||
delete_marker_mtime: replication_source.mod_time,
|
||||
replication_state: Some(replication_source.replication_state()),
|
||||
..Default::default()
|
||||
},
|
||||
bucket: bucket.clone(),
|
||||
event_type: REPLICATE_INCOMING_DELETE.to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
};
|
||||
enrich_delete_replication_state_if_needed(&bucket, &mut deleted_object.delete_object, replication_source).await;
|
||||
schedule_replication_delete(deleted_object).await;
|
||||
}
|
||||
|
||||
let delete_marker = obj_info.delete_marker;
|
||||
@@ -3299,6 +3415,8 @@ impl DefaultObjectUsecase {
|
||||
req.input.sse_customer_key_md5.as_ref(),
|
||||
)?;
|
||||
|
||||
// Compute x-amz-expiration header from lifecycle prediction (before info is partially moved)
|
||||
let expiration_header = resolve_put_object_expiration(&bucket, &info).await;
|
||||
let event_info = info.clone();
|
||||
let content_type = {
|
||||
if let Some(content_type) = &info.content_type {
|
||||
@@ -3411,6 +3529,13 @@ impl DefaultObjectUsecase {
|
||||
checksum_crc64nvme,
|
||||
checksum_type,
|
||||
storage_class,
|
||||
// x-amz-restore from object metadata
|
||||
restore: metadata_map.get(X_AMZ_RESTORE.as_str()).and_then(|v| {
|
||||
let rs = parse_restore_obj_status(v).ok()?;
|
||||
Some(rs.to_string2())
|
||||
}),
|
||||
// x-amz-expiration from lifecycle prediction
|
||||
expiration: expiration_header,
|
||||
// metadata: object_metadata,
|
||||
..Default::default()
|
||||
};
|
||||
@@ -4197,6 +4322,10 @@ fn object_attributes_requested(object_attributes: &[ObjectAttributes], name: &'s
|
||||
mod tests {
|
||||
use super::*;
|
||||
use http::{Extensions, HeaderMap, HeaderName, HeaderValue, Method, Uri};
|
||||
use s3s::dto::{
|
||||
DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ExistingObjectReplication,
|
||||
ExistingObjectReplicationStatus, ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus,
|
||||
};
|
||||
|
||||
fn build_request<T>(input: T, method: Method) -> S3Request<T> {
|
||||
S3Request {
|
||||
@@ -4606,6 +4735,44 @@ mod tests {
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_schedule_delete_replication_skips_replica_requests() {
|
||||
let opts = ObjectOptions {
|
||||
replication_request: true,
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let replication_source = ObjectInfo {
|
||||
delete_marker: true,
|
||||
replication_status: ReplicationStatusType::Completed,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
!should_schedule_delete_replication(&opts, &replication_source, true),
|
||||
"replica delete requests on target sites must not enqueue a second replication delete task"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_schedule_delete_replication_keeps_delete_marker_version_purge_from_source() {
|
||||
let opts = ObjectOptions {
|
||||
replication_request: false,
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let replication_source = ObjectInfo {
|
||||
delete_marker: true,
|
||||
replication_status: ReplicationStatusType::Completed,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
should_schedule_delete_replication(&opts, &replication_source, true),
|
||||
"source-side delete-marker version purge still needs replication scheduling"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_get_object_attributes_returns_internal_error_when_store_uninitialized() {
|
||||
let input = GetObjectAttributesInput::builder()
|
||||
@@ -4772,4 +4939,89 @@ mod tests {
|
||||
let err = usecase.execute_restore_object(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_state_from_config_tracks_downstream_delete_marker_targets() {
|
||||
let arn = "arn:aws:s3:::target-bucket".to_string();
|
||||
let config = ReplicationConfiguration {
|
||||
role: arn.clone(),
|
||||
rules: vec![ReplicationRule {
|
||||
delete_marker_replication: Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
|
||||
}),
|
||||
delete_replication: None,
|
||||
destination: Destination {
|
||||
bucket: arn.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
existing_object_replication: Some(ExistingObjectReplication {
|
||||
status: ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::ENABLED),
|
||||
}),
|
||||
filter: None,
|
||||
id: Some("rule-1".to_string()),
|
||||
prefix: Some("test/".to_string()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
}],
|
||||
};
|
||||
let obj_info = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "test/object.txt".to_string(),
|
||||
delete_marker: true,
|
||||
replication_status: ReplicationStatusType::Replica,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let state = delete_replication_state_from_config(&config, &obj_info, None, true)
|
||||
.expect("replica delete marker should be forwarded to downstream targets");
|
||||
let pending = format!("{arn}=PENDING;");
|
||||
|
||||
assert_eq!(state.replication_status_internal.as_deref(), Some(pending.as_str()));
|
||||
assert_eq!(state.replicate_decision_str, format!("{arn}=true;false;{arn};"));
|
||||
assert!(state.targets.contains_key(&arn));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_state_from_config_tracks_delete_marker_version_purges() {
|
||||
let arn = "arn:aws:s3:::target-bucket".to_string();
|
||||
let config = ReplicationConfiguration {
|
||||
role: arn.clone(),
|
||||
rules: vec![ReplicationRule {
|
||||
delete_marker_replication: Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
|
||||
}),
|
||||
delete_replication: None,
|
||||
destination: Destination {
|
||||
bucket: arn.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
existing_object_replication: Some(ExistingObjectReplication {
|
||||
status: ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::ENABLED),
|
||||
}),
|
||||
filter: None,
|
||||
id: Some("rule-1".to_string()),
|
||||
prefix: Some("test/".to_string()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
}],
|
||||
};
|
||||
let obj_info = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "test/object.txt".to_string(),
|
||||
delete_marker: true,
|
||||
replication_status: ReplicationStatusType::Completed,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let version_id = Some(Uuid::new_v4());
|
||||
let state = delete_replication_state_from_config(&config, &obj_info, version_id, false)
|
||||
.expect("delete-marker version purge should honor delete-marker replication rules");
|
||||
let pending = format!("{arn}=PENDING;");
|
||||
|
||||
assert_eq!(state.version_purge_status_internal.as_deref(), Some(pending.as_str()));
|
||||
assert_eq!(state.replicate_decision_str, format!("{arn}=true;false;{arn};"));
|
||||
assert!(state.purge_targets.contains_key(&arn));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user