gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-27 16:48:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ecstore): add RUSTFS_ILM_DEBUG_DAY_SECS lifecycle time-acceleration for tests (#4677)

Browse Source 
feat(ecstore): add RUSTFS_ILM_DEBUG_DAY_SECS lifecycle time-acceleration for tests (backlog#1148 ilm-5)

Model a Ceph rgw_lc_debug_interval-style switch: when RUSTFS_ILM_DEBUG_DAY_SECS
is set to a positive integer N, lifecycle Days-based rule evaluation treats one
'day' as N seconds instead of 86400, so Days>=1 expiration/transition/noncurrent
rules become exercisable in seconds. Unset => byte-identical to production.

All Days->deadline conversions funnel through expected_expiry_time(); the new
ilm_day_secs() helper (OnceLock-cached in production, env-read under cfg(test))
rescales both the day offset and the default rounding boundary. An explicit
RUSTFS_ILM_PROCESS_TIME still wins for the rounding boundary. Absolute Date-based
rules are deliberately NOT rescaled. Activation emits a WARN; test/debug only.

Refs rustfs/backlog#1148 (ilm-5), rustfs/backlog#1155.
This commit is contained in:
Zhengchao An
2026-07-10 21:57:19 +08:00
committed by GitHub
parent 6fed0a4067
commit 40875026df
2 changed files with 363 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/config/src/constants/env.rs
+18
View File
@@ -41,6 +41,24 @@ pub const ENV_ILM_PROCESS_TIME_DEPRECATED: &str = "_RUSTFS_ILM_PROCESS_TIME";
/// Default ILM process boundary in seconds (24h).
pub const DEFAULT_ILM_PROCESS_TIME_SECS: i32 = 86400;
/// **TEST/DEBUG ONLY** — lifecycle "day" length override, in seconds.
///
/// Modeled on Ceph RGW's `rgw_lc_debug_interval`. When set to a positive
/// integer `N`, lifecycle rule evaluation treats one Days-based "day" as `N`
/// seconds instead of [`DEFAULT_ILM_DAY_SECS`] (86400), so `Days`-based
/// expiration/transition/noncurrent rules become exercisable in seconds under
/// test. When unset (or set to `0`/an invalid value), behavior is byte-for-byte
/// identical to production.
///
/// This accelerates data deletion and MUST NEVER be set in a production
/// deployment. Enabling it emits a `WARN` log. It only rescales relative
/// `Days` deadlines; absolute `Date`-based rules are unaffected.
pub const ENV_ILM_DEBUG_DAY_SECS: &str = "RUSTFS_ILM_DEBUG_DAY_SECS";
/// Number of seconds in one lifecycle "day" (86400) used as the default when
/// [`ENV_ILM_DEBUG_DAY_SECS`] is unset.
pub const DEFAULT_ILM_DAY_SECS: u32 = 86400;
/// Medium-drawn lines separator
/// This is used to separate words in environment variable names.
pub const ENV_WORD_DELIMITER_DASH: &str = "-";
crates/lifecycle/src/core.rs
+345 -8
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_config::{DEFAULT_ILM_PROCESS_TIME_SECS, ENV_ILM_PROCESS_TIME, ENV_ILM_PROCESS_TIME_DEPRECATED};
use rustfs_config::{DEFAULT_ILM_DAY_SECS, ENV_ILM_DEBUG_DAY_SECS, ENV_ILM_PROCESS_TIME, ENV_ILM_PROCESS_TIME_DEPRECATED};
use rustfs_replication::{ReplicationStatusType, VersionPurgeStatusType};
use s3s::dto::{
BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter,
@@ -22,7 +22,7 @@ use std::collections::HashMap;
use std::sync::Arc;
use time::macros::offset;
use time::{self, Duration, OffsetDateTime};
use tracing::debug;
use tracing::{debug, warn};
use uuid::Uuid;
use crate::rule::{Filter, NoncurrentVersionTransitionOps, TransitionOps};
@@ -32,6 +32,7 @@ pub const TRANSITION_PENDING: &str = "pending";
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
const EVENT_LIFECYCLE_EXPIRY_COMPUTED: &str = "lifecycle_expiry_computed";
const EVENT_LIFECYCLE_DEBUG_DAY_SECS: &str = "lifecycle_debug_day_secs";
const ERR_LIFECYCLE_NO_RULE: &str = "Lifecycle configuration should have at least one rule";
const ERR_LIFECYCLE_DUPLICATE_ID: &str = "Rule ID must be unique. Found same ID for more than one rule";
const _ERR_XML_NOT_WELL_FORMED: &str =
@@ -829,12 +830,18 @@ pub fn expected_expiry_time(mod_time: OffsetDateTime, days: i32) -> OffsetDateTi
);
return OffsetDateTime::UNIX_EPOCH; // Return epoch time to ensure immediate expiry
}
// One "day" is normally 86400 seconds; RUSTFS_ILM_DEBUG_DAY_SECS (test/debug
// only) can shrink it so Days-based rules become testable in seconds. Unset =>
// ilm_day_secs() == 86400, byte-identical to `Duration::days(days)`.
let offset_secs = i64::from(days).saturating_mul(i64::from(ilm_day_secs()));
let t = mod_time
.to_offset(offset!(-0:00:00))
.saturating_add(Duration::days(i64::from(days)));
.saturating_add(Duration::seconds(offset_secs));
// Round up to the next processing boundary per S3-compatible Days semantics.
// Canonical key: RUSTFS_ILM_PROCESS_TIME; deprecated alias: _RUSTFS_ILM_PROCESS_TIME.
// When RUSTFS_ILM_PROCESS_TIME is unset the boundary defaults to the (possibly
// debug-accelerated) day length, so RUSTFS_ILM_DEBUG_DAY_SECS scales rounding too.
// TODO(GA): Remove ENV_ILM_PROCESS_TIME_DEPRECATED compatibility during GA release.
let process_interval_secs = ilm_process_interval_secs();
@@ -850,14 +857,83 @@ pub fn expected_expiry_time(mod_time: OffsetDateTime, days: i32) -> OffsetDateTi
}
fn ilm_process_interval_secs() -> u32 {
let configured = std::env::var(ENV_ILM_PROCESS_TIME)
// An explicit RUSTFS_ILM_PROCESS_TIME (or its deprecated alias) always wins.
// Otherwise fall back to the lifecycle day length so the debug day-length
// override rescales the rounding boundary consistently with the deadline.
std::env::var(ENV_ILM_PROCESS_TIME)
.ok()
.or_else(|| std::env::var(ENV_ILM_PROCESS_TIME_DEPRECATED).ok())
.and_then(|value| value.parse::<i32>().ok())
.filter(|value| *value > 0)
.unwrap_or(DEFAULT_ILM_PROCESS_TIME_SECS);
.and_then(|value| u32::try_from(value).ok())
.unwrap_or_else(ilm_day_secs)
}
u32::try_from(configured).unwrap_or_else(|_| u32::try_from(DEFAULT_ILM_PROCESS_TIME_SECS).unwrap_or(1))
/// Number of seconds treated as one lifecycle "day".
///
/// **TEST/DEBUG ONLY override.** Returns [`DEFAULT_ILM_DAY_SECS`] (86400) unless
/// the `RUSTFS_ILM_DEBUG_DAY_SECS` environment variable is set to a positive
/// integer, modeled on Ceph RGW's `rgw_lc_debug_interval`. When set, one
/// Days-based lifecycle "day" is compressed to that many seconds, making
/// expiration / transition / noncurrent Days rules exercisable in seconds.
///
/// A missing / zero / non-numeric value falls back to 86400, so with the
/// variable unset the behavior is byte-for-byte identical to production.
/// Enabling it emits a `WARN` because it accelerates data deletion and must
/// never be set in a production deployment. Only relative `Days` deadlines are
/// rescaled; absolute `Date`-based rules are unaffected.
///
/// In production the value is parsed once (via `OnceLock`) so there is no
/// per-object environment read on the scanner evaluation path. Under `#[cfg(test)]`
/// the environment is re-read on every call so `temp_env`-style tests remain
/// deterministic and cannot poison each other across a shared process.
fn ilm_day_secs() -> u32 {
#[cfg(test)]
{
resolve_ilm_day_secs()
}
#[cfg(not(test))]
{
use std::sync::OnceLock;
static CACHE: OnceLock<u32> = OnceLock::new();
*CACHE.get_or_init(resolve_ilm_day_secs)
}
}
fn resolve_ilm_day_secs() -> u32 {
match std::env::var(ENV_ILM_DEBUG_DAY_SECS) {
Ok(raw) => parse_ilm_day_secs(&raw),
Err(_) => DEFAULT_ILM_DAY_SECS,
}
}
/// Parse a `RUSTFS_ILM_DEBUG_DAY_SECS` value. Positive integers activate the
/// override (with a `WARN`); anything else warns and falls back to 86400.
fn parse_ilm_day_secs(raw: &str) -> u32 {
match raw.trim().parse::<u32>() {
Ok(secs) if secs > 0 => {
warn!(
event = EVENT_LIFECYCLE_DEBUG_DAY_SECS,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
day_secs = secs,
"RUSTFS_ILM_DEBUG_DAY_SECS is set: lifecycle Days are accelerated to {secs}s/day. \
TEST/DEBUG ONLY - this accelerates object deletion and must never be enabled in production."
);
secs
}
_ => {
warn!(
event = EVENT_LIFECYCLE_DEBUG_DAY_SECS,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
raw = raw,
fallback_day_secs = DEFAULT_ILM_DAY_SECS,
"Ignoring invalid RUSTFS_ILM_DEBUG_DAY_SECS value; falling back to 86400s/day."
);
DEFAULT_ILM_DAY_SECS
}
}
}
pub async fn abort_incomplete_multipart_upload_due(
@@ -979,8 +1055,12 @@ mod tests {
use time::macros::datetime;
fn with_default_ilm_process_time(test: impl FnOnce()) {
temp_env::with_var_unset(ENV_ILM_PROCESS_TIME, || {
temp_env::with_var_unset(ENV_ILM_PROCESS_TIME_DEPRECATED, test);
// Also neutralize the debug day-length override so these boundary tests
// stay hermetic against an ambient RUSTFS_ILM_DEBUG_DAY_SECS.
temp_env::with_var_unset(ENV_ILM_DEBUG_DAY_SECS, || {
temp_env::with_var_unset(ENV_ILM_PROCESS_TIME, || {
temp_env::with_var_unset(ENV_ILM_PROCESS_TIME_DEPRECATED, test);
});
});
}
@@ -2801,6 +2881,263 @@ mod tests {
});
}
// --- ilm-5 tests: RUSTFS_ILM_DEBUG_DAY_SECS time-acceleration switch ---
// (a) Default path (env unset) is byte-identical: one day == 86400s.
#[test]
#[serial]
fn ilm_day_secs_defaults_to_86400_when_unset() {
temp_env::with_var_unset(ENV_ILM_DEBUG_DAY_SECS, || {
assert_eq!(ilm_day_secs(), DEFAULT_ILM_DAY_SECS);
assert_eq!(ilm_day_secs(), 86400);
});
}
// (b) Pure parse: positive integers activate the override.
#[test]
fn parse_ilm_day_secs_accepts_positive_values() {
assert_eq!(parse_ilm_day_secs("1"), 1);
assert_eq!(parse_ilm_day_secs("2"), 2);
assert_eq!(parse_ilm_day_secs(" 10 "), 10);
assert_eq!(parse_ilm_day_secs("3600"), 3600);
}
// (c) Pure parse: invalid / zero / negative fall back to 86400.
#[test]
fn parse_ilm_day_secs_rejects_invalid_and_zero() {
assert_eq!(parse_ilm_day_secs("0"), DEFAULT_ILM_DAY_SECS);
assert_eq!(parse_ilm_day_secs("-5"), DEFAULT_ILM_DAY_SECS);
assert_eq!(parse_ilm_day_secs("not-a-number"), DEFAULT_ILM_DAY_SECS);
assert_eq!(parse_ilm_day_secs(""), DEFAULT_ILM_DAY_SECS);
assert_eq!(parse_ilm_day_secs("1.5"), DEFAULT_ILM_DAY_SECS);
}
// (b) End-to-end env read scales the day length.
#[test]
#[serial]
fn ilm_day_secs_scales_when_env_set() {
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("2"), || {
assert_eq!(ilm_day_secs(), 2);
});
}
// (c) Invalid env value falls back to 86400.
#[test]
#[serial]
fn ilm_day_secs_falls_back_on_invalid_env() {
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("bogus"), || {
assert_eq!(ilm_day_secs(), DEFAULT_ILM_DAY_SECS);
});
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("0"), || {
assert_eq!(ilm_day_secs(), DEFAULT_ILM_DAY_SECS);
});
}
// Deadline math scales: with a 1s day and PROCESS_TIME unset, a Days=1 rule is
// due 1s after mod_time (rounded up to the next 1s boundary => same instant).
#[test]
#[serial]
fn expected_expiry_time_scales_with_debug_day_secs() {
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("1"), || {
temp_env::with_var_unset(ENV_ILM_PROCESS_TIME, || {
temp_env::with_var_unset(ENV_ILM_PROCESS_TIME_DEPRECATED, || {
// 1 "day" == 1s; boundary also defaults to 1s so no extra rounding.
assert_eq!(expected_expiry_time(mod_time, 1), mod_time + Duration::seconds(1));
assert_eq!(expected_expiry_time(mod_time, 5), mod_time + Duration::seconds(5));
});
});
});
}
// days == 0 still yields the immediate-expiry sentinel regardless of the switch.
#[test]
#[serial]
fn expected_expiry_time_zero_days_ignores_debug_day_secs() {
let mod_time = datetime!(2025-06-01 12:00:00 UTC);
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("2"), || {
assert_eq!(expected_expiry_time(mod_time, 0), OffsetDateTime::UNIX_EPOCH);
});
}
// (③) Interaction with an explicit RUSTFS_ILM_PROCESS_TIME: the deadline offset
// uses the accelerated day length, but the rounding boundary honors PROCESS_TIME.
#[test]
#[serial]
fn expected_expiry_time_debug_day_secs_respects_explicit_process_time() {
let mod_time = datetime!(2025-01-15 10:30:00 UTC);
// day == 10s, but round up to the next 60s (PROCESS_TIME) boundary.
temp_env::with_var(ENV_ILM_DEBUG_DAY_SECS, Some("10"), || {
temp_env::with_var(ENV_ILM_PROCESS_TIME, Some("60"), || {
temp_env::with_var_unset(ENV_ILM_PROCESS_TIME_DEPRECATED, || {
// mod_time + 30s = 10:30:30, rounded up to next minute => 10:31:00.
let result = expected_expiry_time(mod_time, 3);
assert_eq!(result, datetime!(2025-01-15 10:31:00 UTC));
});
});
});
}
// (③) With the switch unset, an explicit PROCESS_TIME behaves exactly as before.
#[test]
#[serial]
fn expected_expiry_time_unset_debug_day_secs_matches_legacy_process_time() {
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
temp_env::with_var_unset(ENV_ILM_DEBUG_DAY_SECS, || {
temp_env::with_var(ENV_ILM_PROCESS_TIME, Some("3600"), || {
temp_env::with_var_unset(ENV_ILM_PROCESS_TIME_DEPRECATED, || {
assert_eq!(expected_expiry_time(mod_time, 1), datetime!(2025-01-16 11:00:00 UTC));
});
});
});
}
/// Run an async body inside a synchronous `temp_env` scope so the overridden
/// environment stays set across every `.await`. Uses a fresh current-thread
/// runtime (no outer runtime => `block_on` is safe) and keeps these tests off
/// the optional `async_closure` temp-env feature.
fn block_on_with_env<F: std::future::Future>(vars: &[(&str, Option<&str>)], fut_fn: impl FnOnce() -> F) -> F::Output {
temp_env::with_vars(vars, || {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("build runtime");
rt.block_on(fut_fn())
})
}
// The abort-incomplete-multipart deadline path also scales through the switch.
#[test]
#[serial]
fn abort_incomplete_multipart_due_scales_with_debug_day_secs() {
use s3s::dto::AbortIncompleteMultipartUpload;
let initiated = datetime!(2025-01-15 10:30:45 UTC);
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: None,
abort_incomplete_multipart_upload: Some(AbortIncompleteMultipartUpload {
days_after_initiation: Some(3),
}),
del_marker_expiration: None,
filter: None,
id: Some("abort-mpu".to_string()),
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
}],
};
let opts = ObjectOpts {
name: "obj".to_string(),
mod_time: Some(initiated),
is_latest: true,
..Default::default()
};
block_on_with_env(
&[
(ENV_ILM_DEBUG_DAY_SECS, Some("1")),
(ENV_ILM_PROCESS_TIME, None),
(ENV_ILM_PROCESS_TIME_DEPRECATED, None),
],
|| async {
let (due, id) = abort_incomplete_multipart_upload_due(&lc, &opts).await.expect("due");
assert_eq!(due, initiated + Duration::seconds(3));
assert_eq!(id, "abort-mpu");
},
);
}
// (⑤ evaluator seam) A Days=1 rule fires under RUSTFS_ILM_DEBUG_DAY_SECS=1 once
// `now` advances a few seconds past a mod_time only ~seconds in the past.
#[test]
#[serial]
fn eval_inner_expires_days_one_rule_under_debug_day_secs() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
days: Some(1),
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("expire-days".to_string()),
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
}],
};
let mod_time = datetime!(2025-01-15 10:30:00 UTC);
let opts = ObjectOpts {
name: "obj".to_string(),
mod_time: Some(mod_time),
is_latest: true,
..Default::default()
};
block_on_with_env(
&[
(ENV_ILM_DEBUG_DAY_SECS, Some("1")),
(ENV_ILM_PROCESS_TIME, None),
(ENV_ILM_PROCESS_TIME_DEPRECATED, None),
],
|| async {
// Only 2s after mod_time: with a real day this would NOT be due, but a
// 1s day makes the Days=1 deadline (mod_time + 1s) already in the past.
let now = mod_time + Duration::seconds(2);
let event = lc.eval_inner(&opts, now, 0).await;
assert_eq!(event.action, IlmAction::DeleteAction);
assert_eq!(event.rule_id, "expire-days");
assert_eq!(event.due, Some(mod_time + Duration::seconds(1)));
},
);
}
// Absolute Date-based rules must NOT scale with the switch (regression guard).
#[test]
#[serial]
fn eval_inner_date_rule_ignores_debug_day_secs() {
let expiry_date = datetime!(2025-06-01 00:00:00 UTC);
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
date: Some(expiry_date.into()),
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("expire-date".to_string()),
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
}],
};
let opts = ObjectOpts {
name: "obj".to_string(),
mod_time: Some(datetime!(2025-01-15 10:30:00 UTC)),
is_latest: true,
..Default::default()
};
block_on_with_env(&[(ENV_ILM_DEBUG_DAY_SECS, Some("1"))], || async {
// A few seconds after mod_time; the absolute June date is still far away.
let now = datetime!(2025-01-15 10:30:05 UTC);
let event = lc.eval_inner(&opts, now, 0).await;
assert_eq!(event.action, IlmAction::NoneAction);
});
}
// --- TASK-007 tests: Legacy Prefix/Filter conflict ---
#[tokio::test]