mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-21 11:56:38 +00:00
fix: 2827 lifecycle days next midnight (#2833)
Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
@@ -34,6 +34,12 @@ pub const ENV_NOTIFY_ENABLE: &str = "RUSTFS_NOTIFY_ENABLE";
|
|||||||
pub const DEFAULT_AUDIT_ENABLE: bool = false;
|
pub const DEFAULT_AUDIT_ENABLE: bool = false;
|
||||||
/// Default global notify switch (disabled by default).
|
/// Default global notify switch (disabled by default).
|
||||||
pub const DEFAULT_NOTIFY_ENABLE: bool = false;
|
pub const DEFAULT_NOTIFY_ENABLE: bool = false;
|
||||||
|
/// Canonical ILM process boundary env key (seconds).
|
||||||
|
pub const ENV_ILM_PROCESS_TIME: &str = "RUSTFS_ILM_PROCESS_TIME";
|
||||||
|
/// Deprecated ILM process boundary env key kept for compatibility.
|
||||||
|
pub const ENV_ILM_PROCESS_TIME_DEPRECATED: &str = "_RUSTFS_ILM_PROCESS_TIME";
|
||||||
|
/// Default ILM process boundary in seconds (24h).
|
||||||
|
pub const DEFAULT_ILM_PROCESS_TIME_SECS: i32 = 86400;
|
||||||
|
|
||||||
/// Medium-drawn lines separator
|
/// Medium-drawn lines separator
|
||||||
/// This is used to separate words in environment variable names.
|
/// This is used to separate words in environment variable names.
|
||||||
|
|||||||
@@ -12,6 +12,7 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
|
use rustfs_config::{DEFAULT_ILM_PROCESS_TIME_SECS, ENV_ILM_PROCESS_TIME, ENV_ILM_PROCESS_TIME_DEPRECATED};
|
||||||
use rustfs_filemeta::{ReplicationStatusType, VersionPurgeStatusType};
|
use rustfs_filemeta::{ReplicationStatusType, VersionPurgeStatusType};
|
||||||
use s3s::dto::{
|
use s3s::dto::{
|
||||||
BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter,
|
BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter,
|
||||||
@@ -19,7 +20,6 @@ use s3s::dto::{
|
|||||||
};
|
};
|
||||||
use std::cmp::Ordering;
|
use std::cmp::Ordering;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::env;
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use time::macros::offset;
|
use time::macros::offset;
|
||||||
use time::{self, Duration, OffsetDateTime};
|
use time::{self, Duration, OffsetDateTime};
|
||||||
@@ -805,16 +805,29 @@ pub fn expected_expiry_time(mod_time: OffsetDateTime, days: i32) -> OffsetDateTi
|
|||||||
.to_offset(offset!(-0:00:00))
|
.to_offset(offset!(-0:00:00))
|
||||||
.saturating_add(Duration::days(days as i64));
|
.saturating_add(Duration::days(days as i64));
|
||||||
|
|
||||||
// Truncate to midnight UTC per S3 standard, unless overridden by env var.
|
// Round up to the next processing boundary per S3-compatible Days semantics.
|
||||||
// _RUSTFS_ILM_PROCESS_TIME controls the truncation granularity in seconds.
|
// Canonical key: RUSTFS_ILM_PROCESS_TIME; deprecated alias: _RUSTFS_ILM_PROCESS_TIME.
|
||||||
let truncation_secs = env::var("_RUSTFS_ILM_PROCESS_TIME")
|
// TODO(GA): Remove ENV_ILM_PROCESS_TIME_DEPRECATED compatibility during GA release.
|
||||||
.ok()
|
let process_interval_secs = rustfs_utils::get_env_i32_with_aliases(
|
||||||
.and_then(|v| v.parse::<u32>().ok())
|
ENV_ILM_PROCESS_TIME,
|
||||||
.unwrap_or(86400); // default: truncate to midnight (24h)
|
&[ENV_ILM_PROCESS_TIME_DEPRECATED],
|
||||||
|
DEFAULT_ILM_PROCESS_TIME_SECS,
|
||||||
|
);
|
||||||
|
let process_interval_secs = if process_interval_secs > 0 {
|
||||||
|
process_interval_secs as u32
|
||||||
|
} else {
|
||||||
|
DEFAULT_ILM_PROCESS_TIME_SECS as u32
|
||||||
|
};
|
||||||
|
|
||||||
let unix_secs = t.unix_timestamp();
|
let boundary_nanos = i128::from(process_interval_secs) * 1_000_000_000;
|
||||||
let truncated_secs = (unix_secs / truncation_secs as i64) * truncation_secs as i64;
|
let timestamp_nanos = t.unix_timestamp_nanos();
|
||||||
OffsetDateTime::from_unix_timestamp(truncated_secs).unwrap_or(t)
|
let remainder = timestamp_nanos.rem_euclid(boundary_nanos);
|
||||||
|
let rounded_nanos = if remainder == 0 {
|
||||||
|
timestamp_nanos
|
||||||
|
} else {
|
||||||
|
timestamp_nanos + (boundary_nanos - remainder)
|
||||||
|
};
|
||||||
|
OffsetDateTime::from_unix_timestamp_nanos(rounded_nanos).unwrap_or(t)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn abort_incomplete_multipart_upload_due(
|
pub async fn abort_incomplete_multipart_upload_due(
|
||||||
@@ -1336,7 +1349,7 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
#[serial]
|
||||||
async fn eval_inner_expires_latest_object_after_days_due() {
|
async fn eval_inner_expires_latest_object_after_days_due() {
|
||||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
|
let base_time = datetime!(2025-01-15 10:30:45 UTC);
|
||||||
let lc = BucketLifecycleConfiguration {
|
let lc = BucketLifecycleConfiguration {
|
||||||
expiry_updated_at: None,
|
expiry_updated_at: None,
|
||||||
rules: vec![LifecycleRule {
|
rules: vec![LifecycleRule {
|
||||||
@@ -1362,11 +1375,11 @@ mod tests {
|
|||||||
is_latest: true,
|
is_latest: true,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
let event = lc.eval_inner(&opts, base_time + Duration::days(2), 0).await;
|
let event = lc.eval_inner(&opts, datetime!(2025-01-17 00:00:00 UTC), 0).await;
|
||||||
|
|
||||||
assert_eq!(event.action, IlmAction::DeleteAction);
|
assert_eq!(event.action, IlmAction::DeleteAction);
|
||||||
assert_eq!(event.rule_id, "expire-days");
|
assert_eq!(event.rule_id, "expire-days");
|
||||||
assert_eq!(event.due, Some(expected_expiry_time(base_time, 1)));
|
assert_eq!(event.due, Some(datetime!(2025-01-17 00:00:00 UTC)));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -2135,19 +2148,19 @@ mod tests {
|
|||||||
.expect("expected days-based expiration to pass on locked bucket");
|
.expect("expected days-based expiration to pass on locked bucket");
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- TASK-003 tests: Midnight UTC truncation ---
|
// --- TASK-003 tests: Round up to next UTC processing boundary ---
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn expected_expiry_time_truncates_to_midnight_utc() {
|
fn expected_expiry_time_rounds_up_to_next_midnight_utc() {
|
||||||
// Object created at 2025-01-15T10:30:45Z, expire in 30 days
|
// Object created at 2025-01-15T10:30:45Z, expire in 30 days
|
||||||
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
|
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
|
||||||
let result = expected_expiry_time(mod_time, 30);
|
let result = expected_expiry_time(mod_time, 30);
|
||||||
|
|
||||||
// Should be truncated to midnight: 2025-02-14T00:00:00Z
|
// Should round up to the next midnight: 2025-02-15T00:00:00Z
|
||||||
assert_eq!(result.hour(), 0);
|
assert_eq!(result.hour(), 0);
|
||||||
assert_eq!(result.minute(), 0);
|
assert_eq!(result.minute(), 0);
|
||||||
assert_eq!(result.second(), 0);
|
assert_eq!(result.second(), 0);
|
||||||
assert_eq!(result, datetime!(2025-02-14 00:00:00 UTC));
|
assert_eq!(result, datetime!(2025-02-15 00:00:00 UTC));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -2158,17 +2171,37 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn expected_expiry_time_truncates_already_midnight() {
|
fn expected_expiry_time_preserves_exact_midnight_boundary() {
|
||||||
let mod_time = datetime!(2025-03-01 00:00:00 UTC);
|
let mod_time = datetime!(2025-03-01 00:00:00 UTC);
|
||||||
let result = expected_expiry_time(mod_time, 1);
|
let result = expected_expiry_time(mod_time, 1);
|
||||||
assert_eq!(result, datetime!(2025-03-02 00:00:00 UTC));
|
assert_eq!(result, datetime!(2025-03-02 00:00:00 UTC));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn expected_expiry_time_truncates_end_of_day() {
|
fn expected_expiry_time_rounds_end_of_day_to_following_midnight() {
|
||||||
let mod_time = datetime!(2025-06-15 23:59:59 UTC);
|
let mod_time = datetime!(2025-06-15 23:59:59 UTC);
|
||||||
let result = expected_expiry_time(mod_time, 1);
|
let result = expected_expiry_time(mod_time, 1);
|
||||||
assert_eq!(result, datetime!(2025-06-16 00:00:00 UTC));
|
assert_eq!(result, datetime!(2025-06-17 00:00:00 UTC));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial]
|
||||||
|
fn expected_expiry_time_uses_default_boundary_when_process_time_is_zero_or_invalid() {
|
||||||
|
let mod_time = datetime!(2025-01-15 10:30:45 UTC);
|
||||||
|
|
||||||
|
temp_env::with_var(ENV_ILM_PROCESS_TIME, Some("0"), || {
|
||||||
|
temp_env::with_var_unset(ENV_ILM_PROCESS_TIME_DEPRECATED, || {
|
||||||
|
let result = expected_expiry_time(mod_time, 30);
|
||||||
|
assert_eq!(result, datetime!(2025-02-15 00:00:00 UTC));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
temp_env::with_var(ENV_ILM_PROCESS_TIME, Some("not-a-number"), || {
|
||||||
|
temp_env::with_var_unset(ENV_ILM_PROCESS_TIME_DEPRECATED, || {
|
||||||
|
let result = expected_expiry_time(mod_time, 30);
|
||||||
|
assert_eq!(result, datetime!(2025-02-15 00:00:00 UTC));
|
||||||
|
});
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- TASK-007 tests: Legacy Prefix/Filter conflict ---
|
// --- TASK-007 tests: Legacy Prefix/Filter conflict ---
|
||||||
|
|||||||
@@ -365,6 +365,23 @@ pub fn get_env_opt_u16(key: &str) -> Option<u16> {
|
|||||||
pub fn get_env_i32(key: &str, default: i32) -> i32 {
|
pub fn get_env_i32(key: &str, default: i32) -> i32 {
|
||||||
parse_env_value(key).unwrap_or(default)
|
parse_env_value(key).unwrap_or(default)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Retrieve an i32 environment variable with deprecated aliases and a default fallback.
|
||||||
|
///
|
||||||
|
/// Canonical `key` takes precedence over deprecated aliases when both are present.
|
||||||
|
pub fn get_env_i32_with_aliases(key: &str, deprecated: &[&str], default: i32) -> i32 {
|
||||||
|
let Some((used_key, value)) = resolve_env_with_aliases(key, deprecated) else {
|
||||||
|
return default;
|
||||||
|
};
|
||||||
|
|
||||||
|
value.parse::<i32>().unwrap_or_else(|_| {
|
||||||
|
log_once(&format!("env_invalid_i32:{used_key}"), || {
|
||||||
|
format!("Invalid i32 value for {used_key}: {value}. Using default behavior.")
|
||||||
|
});
|
||||||
|
default
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
|
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
|
||||||
/// 32-bit type: signed i32
|
/// 32-bit type: signed i32
|
||||||
///
|
///
|
||||||
@@ -664,7 +681,8 @@ pub fn apply_external_env_compat() -> ExternalEnvCompatReport {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{
|
use super::{
|
||||||
apply_external_env_compat, build_external_env_compat_report_from_entries, get_env_bool_with_aliases, get_env_str,
|
apply_external_env_compat, build_external_env_compat_report_from_entries, get_env_bool_with_aliases,
|
||||||
|
get_env_i32_with_aliases, get_env_str,
|
||||||
};
|
};
|
||||||
|
|
||||||
fn source_key(suffix: &str) -> String {
|
fn source_key(suffix: &str) -> String {
|
||||||
@@ -761,6 +779,33 @@ mod tests {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn i32_alias_value_is_used_when_canonical_missing() {
|
||||||
|
temp_env::with_var_unset("RUSTFS_TEST_I32", || {
|
||||||
|
temp_env::with_var("RUSTFS_TEST_I32_LEGACY", Some("12"), || {
|
||||||
|
assert_eq!(get_env_i32_with_aliases("RUSTFS_TEST_I32", &["RUSTFS_TEST_I32_LEGACY"], 8), 12);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn i32_canonical_value_takes_precedence_over_alias() {
|
||||||
|
temp_env::with_var("RUSTFS_TEST_I32", Some("9"), || {
|
||||||
|
temp_env::with_var("RUSTFS_TEST_I32_LEGACY", Some("12"), || {
|
||||||
|
assert_eq!(get_env_i32_with_aliases("RUSTFS_TEST_I32", &["RUSTFS_TEST_I32_LEGACY"], 8), 9);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn i32_invalid_alias_value_falls_back_to_default() {
|
||||||
|
temp_env::with_var_unset("RUSTFS_TEST_I32", || {
|
||||||
|
temp_env::with_var("RUSTFS_TEST_I32_LEGACY", Some("not-an-i32"), || {
|
||||||
|
assert_eq!(get_env_i32_with_aliases("RUSTFS_TEST_I32", &["RUSTFS_TEST_I32_LEGACY"], 8), 8);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn apply_external_env_compat_copies_missing_rustfs_keys() {
|
fn apply_external_env_compat_copies_missing_rustfs_keys() {
|
||||||
temp_env::with_var("MINIO_ROOT_USER", Some("compat-admin"), || {
|
temp_env::with_var("MINIO_ROOT_USER", Some("compat-admin"), || {
|
||||||
|
|||||||
Reference in New Issue
Block a user