mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-21 07:26:37 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 7cf04018da |
@@ -1370,15 +1370,6 @@ jobs:
|
||||
# the intermediate 4-object plateau requires 4*lc_interval < 5*debug_day, i.e.
|
||||
# debug_day > 8. A smaller value lets the Days=5 rule fire inside the Days=1
|
||||
# poll window so the count collapses straight to 2 and the test fails (#4764).
|
||||
#
|
||||
# RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES=1 is also required. A compacted directory
|
||||
# is only re-descended (and its objects re-evaluated for ILM) once every
|
||||
# DATA_USAGE_UPDATE_DIR_CYCLES scanner cycles (default 16). At the accelerated
|
||||
# RUSTFS_SCANNER_CYCLE=2 that is ~32s between evaluations, so a Days=1 object
|
||||
# that becomes due at debug_day (10s) is not actually expired until the next
|
||||
# ~32s boundary (~42s), landing just past the 4*lc_interval (40s) poll window
|
||||
# and making the count stall at 6 (#4767). Forcing every-cycle re-descent
|
||||
# evaluates ILM within ~2s of the due time, well inside the poll window.
|
||||
s3-lifecycle-behavior-tests:
|
||||
name: S3 Lifecycle Behavior Tests
|
||||
# Also gated on e2e-tests, matching s3-implemented-tests: when the e2e smoke
|
||||
@@ -1420,7 +1411,6 @@ jobs:
|
||||
RUSTFS_ILM_PROCESS_TIME=1 \
|
||||
RUSTFS_SCANNER_ENABLED=true \
|
||||
RUSTFS_SCANNER_CYCLE=2 \
|
||||
RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES=1 \
|
||||
RUSTFS_SCANNER_START_DELAY_SECS=0 \
|
||||
RUSTFS_API_STALE_UPLOADS_CLEANUP_INTERVAL=2s \
|
||||
S3_PORT="${S3_PORT}" \
|
||||
|
||||
@@ -133,16 +133,6 @@ impl From<S3Error> for legacy_s3s::S3Error {
|
||||
}
|
||||
}
|
||||
|
||||
impl S3Error {
|
||||
/// The pinned legacy serializer's XML response. Call sites stay on this type.
|
||||
/// DELETE BY T4.2 with the rest of this file.
|
||||
pub fn to_http_response(self) -> Result<http::Response<crate::Body>, Self> {
|
||||
let response = legacy_s3s::S3Error::from(self).to_http_response().map_err(Self::from)?;
|
||||
let (parts, body) = response.into_parts();
|
||||
Ok(http::Response::from_parts(parts, crate::Body::from(body)))
|
||||
}
|
||||
}
|
||||
|
||||
impl From<legacy_s3s::S3Error> for S3Error {
|
||||
fn from(error: legacy_s3s::S3Error) -> Self {
|
||||
let code: S3ErrorCode = error.code().clone().into();
|
||||
|
||||
@@ -2704,7 +2704,7 @@ impl FolderScanner {
|
||||
}
|
||||
}
|
||||
FolderScanSource::Existing => {
|
||||
// Usage sampling is not proof that a Deep check ran.
|
||||
// An unchanged subtree can contain objects that have aged into a lifecycle rule.
|
||||
if self.scan_mode != HealScanMode::Deep
|
||||
&& !forward_sweep
|
||||
&& !self
|
||||
@@ -2715,6 +2715,10 @@ impl FolderScanner {
|
||||
.is_some_and(|config| config.has_active_rules("", true))
|
||||
&& !into.compacted
|
||||
&& self.old_cache.is_compacted(&h)
|
||||
&& !active_life_cycle.as_ref().is_some_and(|config| {
|
||||
let (_, prefix) = path2_bucket_object_with_base_path(&self.root, &folder_item.name);
|
||||
config.has_active_rules(&prefix)
|
||||
})
|
||||
{
|
||||
let next_cycle = self.old_cache.info.next_cycle as u32;
|
||||
if !h.mod_(next_cycle, data_usage_update_dir_cycles()) {
|
||||
|
||||
@@ -80,6 +80,98 @@ async fn disk_scan_withdraws_persisted_lifecycle_policy() {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn compacted_prefix_expiration_between_usage_samples() {
|
||||
let (_temp_dir, store) = setup_local_scanner_store(1, 4).await;
|
||||
let bucket = "compacted-expiration";
|
||||
let expired = "photos/vehicle/old.jpg";
|
||||
let fresh = "photos/vehicle/new.jpg";
|
||||
let unrelated = "archive/old.jpg";
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let old = now - time::Duration::days(4);
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
for (key, mod_time) in [(expired, old), (fresh, now), (unrelated, old)] {
|
||||
let mut reader = ScannerPutObjReader::from_vec(vec![1]);
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
key,
|
||||
&mut reader,
|
||||
&ScannerObjectOptions {
|
||||
mod_time: Some(mod_time),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write dated object");
|
||||
}
|
||||
let disks = scanner_set_disk_inventory(&store.pools[0].disk_set[0]).await;
|
||||
let scan = async |cache| {
|
||||
let ctx = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, Default::default());
|
||||
let outcome = disks[0]
|
||||
.clone()
|
||||
.nsscanner_disk(
|
||||
ctx,
|
||||
budget.clone(),
|
||||
disks.clone(),
|
||||
cache,
|
||||
None,
|
||||
ScannerDiskScanOptions {
|
||||
scan_mode: HealScanMode::Normal,
|
||||
prefix_scan_scope: None,
|
||||
checkpoint_tx: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("scan bucket");
|
||||
let ScannerDiskScanOutcome::Complete(cache) = outcome else { panic!("expected complete scan") };
|
||||
(cache, budget.progress().0)
|
||||
};
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.info.name = bucket.into();
|
||||
let (mut cache, scanned) = scan(cache).await;
|
||||
assert_eq!(scanned, 3);
|
||||
let prefixes = ["photos", "archive"].map(|prefix| rustfs_data_usage::hash_path(&format!("{bucket}/{prefix}")));
|
||||
assert!(prefixes.iter().all(|prefix| cache.is_compacted(prefix)));
|
||||
let xml = br#"<LifecycleConfiguration><Rule><ID>photos</ID><Status>Enabled</Status><Filter><Prefix>photos/vehicle/</Prefix></Filter><Expiration><Days>2</Days></Expiration></Rule></LifecycleConfiguration>"#;
|
||||
update_bucket_metadata(bucket, BUCKET_LIFECYCLE_CONFIG, xml.to_vec())
|
||||
.await
|
||||
.expect("configure expiration");
|
||||
init_background_expiry(store.clone()).await;
|
||||
let (_, scanned) = temp_env::async_with_vars([("RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES", None::<&str>)], async {
|
||||
let interval = crate::scanner_folder::data_usage_update_dir_cycles();
|
||||
let off_cycle = (1..=interval)
|
||||
.find(|cycle| prefixes.iter().all(|prefix| !prefix.mod_(*cycle, interval)))
|
||||
.expect("cycle that samples neither compacted prefix");
|
||||
cache.info.next_cycle = u64::from(off_cycle);
|
||||
scan(cache).await
|
||||
})
|
||||
.await;
|
||||
assert_eq!(scanned, 2, "visit the matching subtree and reuse the unrelated prefix");
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
loop {
|
||||
match store.get_object_info(bucket, expired, &ScannerObjectOptions::default()).await {
|
||||
Err(Error::ObjectNotFound(_, _)) => break,
|
||||
Ok(_) => tokio::time::sleep(Duration::from_millis(10)).await,
|
||||
Err(err) => panic!("read expired object: {err}"),
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("expiry worker must delete the old matching object");
|
||||
for key in [fresh, unrelated] {
|
||||
store
|
||||
.get_object_info(bucket, key, &ScannerObjectOptions::default())
|
||||
.await
|
||||
.expect("retain non-expiring object");
|
||||
}
|
||||
}
|
||||
|
||||
// Lifecycle expiry goes through the process-global store, which the first
|
||||
// store created in the process claims. Nextest runs this test in its own
|
||||
// process; the lifecycle scenarios share the store that owns expiry processing.
|
||||
|
||||
@@ -171,7 +171,7 @@ These have no persistent key and are read from the environment only.
|
||||
| `RUSTFS_SCANNER_ALERT_COOLDOWN_SECS` | `86400` (`DEFAULT_SCANNER_ALERT_COOLDOWN_SECS`, `scanner_folder.rs`) | Per-(kind, bucket, object) cooldown between S3 excess-alert events; `0` emits every cycle. See [Scanner Excess Alerts](scanner-excess-alerts.md). |
|
||||
| `RUSTFS_SCANNER_DEEP_VERIFY_COOLDOWN_SECS` | `60` (`DEFAULT_SCANNER_DEEP_VERIFY_COOLDOWN_SECS`, `scanner_folder.rs`) | Objects modified within this window are skipped by deep (bitrot) verification in the current cycle. |
|
||||
| `RUSTFS_HEAL_OBJECT_SELECT_PROB` | `1024` (`DEFAULT_HEAL_OBJECT_SELECT_PROB`, `scanner_folder.rs`) | Sampling divisor for scanner-originated heal checks: roughly one object in N per cycle is selected for a low-priority heal check. |
|
||||
| `RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES` | `16` (`DATA_USAGE_UPDATE_DIR_CYCLES`, `scanner_folder.rs`) | Every N cycles a compacted directory is re-descended instead of reusing its cached usage. `1` forces re-descent every cycle (used by lifecycle e2e lanes). |
|
||||
| `RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES` | `16` (`DATA_USAGE_UPDATE_DIR_CYCLES`, `scanner_folder.rs`) | Revisits compacted directories every N cycles for usage accounting. `1` revisits them every cycle. A scanner walk also visits prefixes covered by active lifecycle rules between usage samples. |
|
||||
| `RUSTFS_DATA_USAGE_FAILED_OBJECT_TTL_SECS` | `86400` (`DEFAULT_FAILED_OBJECT_TTL_SECS`, `scanner_folder.rs`) | Retention of per-bucket failed-object entries in the usage cache. |
|
||||
| `RUSTFS_DATA_USAGE_FAILED_OBJECTS_MAX` | `10000` (`DEFAULT_FAILED_OBJECTS_MAX`, `scanner_folder.rs`) | Cap on retained failed-object entries per bucket. |
|
||||
|
||||
|
||||
@@ -13921,18 +13921,18 @@ mod tests {
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/../crates/ecstore/tests/fixtures/bucket-config-goldens/lifecycle/minio.in.xml"
|
||||
));
|
||||
// Answer of s3s 5761ddfe (`Timestamp::parse(DateTime)` on this document's
|
||||
// ExpiryUpdatedAt). The owned axis must keep that instant.
|
||||
let legacy_axis =
|
||||
parse_lifecycle_date("2026-07-07T15:58:57.337315Z").expect("the legacy expiry axis is the minio.in.xml value");
|
||||
let mut old_reader = s3s::xml::Deserializer::new(raw);
|
||||
let old: s3s::dto::BucketLifecycleConfiguration =
|
||||
s3s::xml::Deserialize::deserialize(&mut old_reader).expect("read the actual old lifecycle expiry axis");
|
||||
old_reader.expect_eof().expect("the old expiry reader consumes the document");
|
||||
let old_axis = OffsetDateTime::from(old.expiry_updated_at.expect("the old config contains its expiry axis"));
|
||||
let new = deserialize(raw).expect("read the persisted lifecycle expiry axis");
|
||||
let new_axis = local_lifecycle_staleness_axis(Some(&new), OffsetDateTime::UNIX_EPOCH)
|
||||
.expect("the parsed lifecycle axis is readable");
|
||||
let incoming = parse_lifecycle_date("2026-07-07T15:58:57.337100Z").expect("the incoming expiry axis is valid");
|
||||
assert_eq!(new_axis, legacy_axis, "the parsed axis must keep the legacy submillisecond instant");
|
||||
assert_eq!(
|
||||
is_stale_update(new_axis, Some(incoming)),
|
||||
is_stale_update(legacy_axis, Some(incoming)),
|
||||
is_stale_update(old_axis, Some(incoming)),
|
||||
"a peer update older than the full-precision local axis must remain stale"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1021,33 +1021,6 @@ impl<T> S3Router<T> {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl<T: OwnedOperation> S3Router<T> {
|
||||
/// Dispatches one legacy admin request. Callers keep their assertions and do not name s3s.
|
||||
pub(crate) async fn call_legacy(
|
||||
&self,
|
||||
method: Method,
|
||||
uri: &str,
|
||||
extensions: http::Extensions,
|
||||
) -> s3s::S3Result<S3Response<Body>> {
|
||||
S3Route::call(
|
||||
self,
|
||||
S3Request {
|
||||
input: Body::empty(),
|
||||
method,
|
||||
uri: uri.parse().expect("test URI"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions,
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> Default for S3Router<T> {
|
||||
fn default() -> Self {
|
||||
Self::new(false)
|
||||
|
||||
@@ -142,12 +142,14 @@ mod tests {
|
||||
};
|
||||
use crate::admin::operation::{AdminRequest, AdminResponse, Operation};
|
||||
use crate::admin::router::S3Router;
|
||||
use crate::app::{RequestEnvelope, RequestHead};
|
||||
use crate::app::RequestEnvelope;
|
||||
use hyper::{Method, StatusCode};
|
||||
use matchit::Params;
|
||||
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
|
||||
use rustfs_kms::KmsServiceManager;
|
||||
use rustfs_s3_types::{S3ErrorCode, s3_error};
|
||||
use s3s::route::S3Route;
|
||||
use s3s::{Body, S3Request};
|
||||
use std::sync::Arc;
|
||||
|
||||
struct UnreadyIam;
|
||||
@@ -190,18 +192,22 @@ mod tests {
|
||||
Arc::new(AppContext::new(context.object_store(), Arc::new(UnreadyIam), Arc::new(TestKms)))
|
||||
}
|
||||
|
||||
fn envelope(extensions: http::Extensions) -> RequestEnvelope {
|
||||
RequestEnvelope::from_parts(
|
||||
RequestHead {
|
||||
method: Method::GET,
|
||||
uri: "/context-probe".parse().expect("test URI"),
|
||||
headers: http::HeaderMap::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
},
|
||||
fn request(extensions: http::Extensions) -> S3Request<Body> {
|
||||
S3Request {
|
||||
input: Body::empty(),
|
||||
method: Method::GET,
|
||||
uri: "/context-probe".parse().expect("test URI"),
|
||||
headers: http::HeaderMap::new(),
|
||||
extensions,
|
||||
)
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn envelope(extensions: http::Extensions) -> RequestEnvelope {
|
||||
RequestEnvelope::from_s3s(&request(extensions))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -324,14 +330,14 @@ mod tests {
|
||||
.expect("register test route");
|
||||
|
||||
let err = router
|
||||
.call_legacy(Method::GET, "/context-probe", http::Extensions::new())
|
||||
.call(request(http::Extensions::new()))
|
||||
.await
|
||||
.expect_err("empty slot must fail closed");
|
||||
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable.into());
|
||||
|
||||
assert!(slot.install(distinct_context(&ambient)));
|
||||
let response = router
|
||||
.call_legacy(Method::GET, "/context-probe", http::Extensions::new())
|
||||
.call(request(http::Extensions::new()))
|
||||
.await
|
||||
.expect("installed slot must serve request");
|
||||
assert_eq!(response.status, Some(StatusCode::NO_CONTENT));
|
||||
|
||||
@@ -906,7 +906,9 @@ mod tests {
|
||||
.expect_err("full queue rejects immediately");
|
||||
assert_eq!(error.code(), &S3ErrorCode::SlowDown);
|
||||
assert_eq!(error.headers().expect("retry headers")[http::header::RETRY_AFTER], "1");
|
||||
let response = error.to_http_response().expect("serialize admission error");
|
||||
let response = s3s::S3Error::from(error)
|
||||
.to_http_response()
|
||||
.expect("serialize admission error");
|
||||
assert_eq!(response.status(), http::StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(response.headers()[http::header::RETRY_AFTER], "1");
|
||||
use http_body_util::BodyExt as _;
|
||||
|
||||
@@ -52,31 +52,38 @@ fn website_and_object_lambda_ranges_match_the_legacy_wire_reader() {
|
||||
|
||||
#[test]
|
||||
fn website_and_object_lambda_conditions_match_the_legacy_wire_reader() {
|
||||
// Answers measured on s3s 5761ddfe (`ETagCondition::parse_http_header` and
|
||||
// `Timestamp::parse(HttpDate)`). The owned checks must keep those answers.
|
||||
for (value, accepted) in [
|
||||
(b"*".as_slice(), true),
|
||||
(b"abc-2", true),
|
||||
(b"\"\"", true),
|
||||
(b"W/\"x\"", true),
|
||||
(b"\"a \tb\"", true),
|
||||
(b"\"a\"b\"", true),
|
||||
(b"", false),
|
||||
(b"bad etag", false),
|
||||
(b"w/\"x\"", false),
|
||||
(b"\"\xff\"", false),
|
||||
(b"\"\x7f\"", false),
|
||||
(b"\"x", false),
|
||||
use s3s::dto::{ETagCondition, Timestamp, TimestampFormat};
|
||||
for value in [
|
||||
b"*".as_slice(),
|
||||
b"abc-2",
|
||||
b"\"\"",
|
||||
b"W/\"x\"",
|
||||
b"\"a \tb\"",
|
||||
b"\"a\"b\"",
|
||||
b"",
|
||||
b"bad etag",
|
||||
b"w/\"x\"",
|
||||
b"\"\xff\"",
|
||||
b"\"\x7f\"",
|
||||
b"\"x",
|
||||
] {
|
||||
assert_eq!(GetObjectFields::valid_etag_condition(value), accepted, "{value:?}");
|
||||
assert_eq!(
|
||||
GetObjectFields::valid_etag_condition(value),
|
||||
ETagCondition::parse_http_header(value).is_ok(),
|
||||
"{value:?}"
|
||||
);
|
||||
}
|
||||
for (value, accepted) in [
|
||||
("Sun, 06 Nov 1994 08:49:37 GMT", true),
|
||||
("Sunday, 06-Nov-94 08:49:37 GMT", false),
|
||||
("2025-01-01", false),
|
||||
("Sun, 32 Nov 1994 08:49:37 GMT", false),
|
||||
for value in [
|
||||
"Sun, 06 Nov 1994 08:49:37 GMT",
|
||||
"Sunday, 06-Nov-94 08:49:37 GMT",
|
||||
"2025-01-01",
|
||||
"Sun, 32 Nov 1994 08:49:37 GMT",
|
||||
] {
|
||||
assert_eq!(GetObjectFields::valid_date_condition(value), accepted, "{value}");
|
||||
assert_eq!(
|
||||
GetObjectFields::valid_date_condition(value),
|
||||
Timestamp::parse(TimestampFormat::HttpDate, value).is_ok(),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -265,27 +265,22 @@ mod tests {
|
||||
] {
|
||||
let retention = stored_retention(&stored(value)).expect("representable retention instant");
|
||||
assert_eq!(retention.mode.as_ref().expect("mode").as_str(), "COMPLIANCE");
|
||||
// The response codec writes RetainUntilDate as ISO 8601.
|
||||
let serialized = retention
|
||||
.retain_until_date
|
||||
.expect("stored retention date")
|
||||
.render(rustfs_gateway_types::TimestampFormat::Iso8601)
|
||||
.expect("serialize the response timestamp");
|
||||
let returned = OffsetDateTime::parse(&serialized, &Rfc3339).expect("parse wire timestamp");
|
||||
let stored_at = OffsetDateTime::parse(value, &Rfc3339).expect("parse stored timestamp");
|
||||
assert_eq!(returned.unix_timestamp(), stored_at.unix_timestamp(), "stored date {value}");
|
||||
let output = get_object_retention::output_to_s3s(GetObjectRetentionOutput {
|
||||
retention: Some(retention),
|
||||
})
|
||||
.expect("convert the response");
|
||||
let seam = OffsetDateTime::from(
|
||||
output
|
||||
.retention
|
||||
.expect("retention")
|
||||
.retain_until_date
|
||||
.expect("stored retention date"),
|
||||
);
|
||||
assert_eq!(seam.unix_timestamp(), stored_at.unix_timestamp(), "stored date {value}");
|
||||
let mut serialized = Vec::new();
|
||||
output
|
||||
.retention
|
||||
.expect("retention")
|
||||
.retain_until_date
|
||||
.expect("stored retention date")
|
||||
.format(s3s::dto::TimestampFormat::DateTime, &mut serialized)
|
||||
.expect("serialize the response timestamp");
|
||||
let serialized = String::from_utf8(serialized).expect("UTF-8 timestamp");
|
||||
let returned = OffsetDateTime::parse(&serialized, &Rfc3339).expect("parse wire timestamp");
|
||||
let stored = OffsetDateTime::parse(value, &Rfc3339).expect("parse stored timestamp");
|
||||
assert_eq!(returned.unix_timestamp(), stored.unix_timestamp(), "stored date {value}");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1727,7 +1727,9 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn http_xml_error(error: S3Error) -> (StatusCode, String) {
|
||||
let response = error.to_http_response().expect("S3 error should serialize to HTTP");
|
||||
let response = s3s::S3Error::from(error)
|
||||
.to_http_response()
|
||||
.expect("S3 error should serialize to HTTP");
|
||||
let status = response.status();
|
||||
let body = http_body_util::BodyExt::collect(response.into_body())
|
||||
.await
|
||||
|
||||
@@ -55,41 +55,32 @@ mod tests {
|
||||
use http::{HeaderMap, HeaderValue, StatusCode};
|
||||
use rustfs_config::MI_B;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedDefaultRetention, PersistedFilterRule, PersistedLambdaFunctionConfiguration,
|
||||
PersistedNotificationConfigurationFilter, PersistedObjectLockConfiguration, PersistedObjectLockRule,
|
||||
PersistedFilterRule, PersistedLambdaFunctionConfiguration, PersistedNotificationConfigurationFilter,
|
||||
PersistedQueueConfiguration, PersistedS3KeyFilter, PersistedTopicConfiguration,
|
||||
};
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result, s3_error};
|
||||
use rustfs_s3_types::{S3ErrorCode, S3Result};
|
||||
use rustfs_utils::http::{
|
||||
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
|
||||
MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
|
||||
SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, contains_key_str, get_str, insert_str,
|
||||
};
|
||||
use rustfs_zip::CompressionFormat;
|
||||
use s3s::dto::{
|
||||
DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHold, ObjectLockLegalHoldStatus,
|
||||
ObjectLockRetention, ObjectLockRetentionMode, ObjectLockRule,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
|
||||
fn parse_object_lock_retention(retention: Option<RequestedObjectLockRetention>) -> S3Result<HashMap<String, String>> {
|
||||
parse_object_lock_retention_native(retention)
|
||||
fn parse_object_lock_retention(retention: Option<ObjectLockRetention>) -> S3Result<HashMap<String, String>> {
|
||||
parse_object_lock_retention_native(retention.map(|r| RequestedObjectLockRetention {
|
||||
mode: r.mode.map(|m| m.as_str().to_owned()),
|
||||
retain_until: r.retain_until_date.map(OffsetDateTime::from),
|
||||
}))
|
||||
}
|
||||
|
||||
fn parse_object_lock_legal_hold(legal_hold: Option<RequestedObjectLegalHold>) -> S3Result<HashMap<String, String>> {
|
||||
parse_object_lock_legal_hold_native(legal_hold)
|
||||
}
|
||||
|
||||
/// Bucket default retention already stored, the shape `apply_default_lock_retention_metadata` reads.
|
||||
/// `Enabled` / `COMPLIANCE` / `GOVERNANCE` are the spellings s3s 5761ddfe wrote for these fixtures.
|
||||
fn enabled_lock(retention: Option<(&str, i32)>) -> PersistedObjectLockConfiguration {
|
||||
PersistedObjectLockConfiguration {
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: retention.map(|(mode, days)| PersistedObjectLockRule {
|
||||
default_retention: Some(PersistedDefaultRetention {
|
||||
mode: Some(mode.to_owned()),
|
||||
days: Some(days),
|
||||
years: None,
|
||||
}),
|
||||
}),
|
||||
}
|
||||
fn parse_object_lock_legal_hold(legal_hold: Option<ObjectLockLegalHold>) -> S3Result<HashMap<String, String>> {
|
||||
parse_object_lock_legal_hold_native(legal_hold.map(|h| RequestedObjectLegalHold(h.status.map(|s| s.as_str().to_owned()))))
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -164,6 +155,8 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_s3_error_scenarios() {
|
||||
use s3s::{S3Error, S3ErrorCode, s3_error};
|
||||
|
||||
// Test that we can create expected S3 errors for common validation cases
|
||||
|
||||
// Test incomplete body error
|
||||
@@ -192,12 +185,27 @@ mod tests {
|
||||
assert_eq!(gz_format.extension(), "gz");
|
||||
}
|
||||
|
||||
/// The bucket default retention path reads the persisted shape; the request
|
||||
/// literals in these tests stay on the wire DTO and convert once here.
|
||||
fn persisted_lock(cfg: &ObjectLockConfiguration) -> rustfs_gateway_types::persistence::PersistedObjectLockConfiguration {
|
||||
crate::storage::persisted_from_s3s(cfg).expect("the test configuration converts")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_apply_default_lock_retention_metadata_applies_bucket_default() {
|
||||
let cfg = enabled_lock(Some(("COMPLIANCE", 1)));
|
||||
let cfg = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
rule: Some(ObjectLockRule {
|
||||
default_retention: Some(DefaultRetention {
|
||||
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)),
|
||||
days: Some(1),
|
||||
years: None,
|
||||
}),
|
||||
}),
|
||||
};
|
||||
let mut metadata = std::collections::HashMap::new();
|
||||
|
||||
assert!(apply_default_lock_retention_metadata(Some(&cfg), &mut metadata));
|
||||
assert!(apply_default_lock_retention_metadata(Some(&persisted_lock(&cfg)), &mut metadata));
|
||||
assert_eq!(metadata.get(AMZ_OBJECT_LOCK_MODE_LOWER), Some(&"COMPLIANCE".to_string()));
|
||||
let retain_until = metadata
|
||||
.get(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER)
|
||||
@@ -216,13 +224,22 @@ mod tests {
|
||||
fn test_apply_default_lock_retention_metadata_preserves_explicit_retention() {
|
||||
use std::collections::HashMap;
|
||||
|
||||
let cfg = enabled_lock(Some(("COMPLIANCE", 1)));
|
||||
let cfg = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
rule: Some(ObjectLockRule {
|
||||
default_retention: Some(DefaultRetention {
|
||||
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)),
|
||||
days: Some(1),
|
||||
years: None,
|
||||
}),
|
||||
}),
|
||||
};
|
||||
let mut metadata = HashMap::from([
|
||||
(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "GOVERNANCE".to_string()),
|
||||
(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2030-01-01T00:00:00Z".to_string()),
|
||||
]);
|
||||
|
||||
assert!(!apply_default_lock_retention_metadata(Some(&cfg), &mut metadata));
|
||||
assert!(!apply_default_lock_retention_metadata(Some(&persisted_lock(&cfg)), &mut metadata));
|
||||
assert_eq!(metadata.get(AMZ_OBJECT_LOCK_MODE_LOWER), Some(&"GOVERNANCE".to_string()));
|
||||
assert_eq!(
|
||||
metadata.get(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER),
|
||||
@@ -232,17 +249,30 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_apply_default_lock_retention_metadata_ignores_bucket_without_default() {
|
||||
let cfg = enabled_lock(None);
|
||||
let cfg = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
rule: None,
|
||||
};
|
||||
let mut metadata = std::collections::HashMap::new();
|
||||
|
||||
assert!(!apply_default_lock_retention_metadata(Some(&cfg), &mut metadata));
|
||||
assert!(!apply_default_lock_retention_metadata(Some(&persisted_lock(&cfg)), &mut metadata));
|
||||
assert!(metadata.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_apply_bucket_default_lock_retention_applies_authoritative_default() {
|
||||
let config = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
rule: Some(ObjectLockRule {
|
||||
default_retention: Some(DefaultRetention {
|
||||
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::GOVERNANCE)),
|
||||
days: Some(1),
|
||||
years: None,
|
||||
}),
|
||||
}),
|
||||
};
|
||||
let state = ObjectLockConfigState::Configured {
|
||||
config: enabled_lock(Some(("GOVERNANCE", 1))),
|
||||
config: persisted_lock(&config),
|
||||
updated_at: OffsetDateTime::now_utc(),
|
||||
};
|
||||
let mut metadata = std::collections::HashMap::new();
|
||||
@@ -256,10 +286,19 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn empty_retention_metadata_does_not_suppress_bucket_default() {
|
||||
let config = enabled_lock(Some(("GOVERNANCE", 1)));
|
||||
let config = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
rule: Some(ObjectLockRule {
|
||||
default_retention: Some(DefaultRetention {
|
||||
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::GOVERNANCE)),
|
||||
days: Some(1),
|
||||
years: None,
|
||||
}),
|
||||
}),
|
||||
};
|
||||
for name in [AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER] {
|
||||
let mut metadata = HashMap::from([(name.to_string(), String::new())]);
|
||||
assert!(apply_default_lock_retention_metadata(Some(&config), &mut metadata));
|
||||
assert!(apply_default_lock_retention_metadata(Some(&persisted_lock(&config)), &mut metadata));
|
||||
assert_eq!(metadata.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str), Some("GOVERNANCE"));
|
||||
assert!(!metadata[AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER].is_empty());
|
||||
}
|
||||
@@ -267,8 +306,18 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_apply_bucket_default_lock_retention_preserves_explicit_retention_path() {
|
||||
let config = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
rule: Some(ObjectLockRule {
|
||||
default_retention: Some(DefaultRetention {
|
||||
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)),
|
||||
days: Some(1),
|
||||
years: None,
|
||||
}),
|
||||
}),
|
||||
};
|
||||
let state = ObjectLockConfigState::Configured {
|
||||
config: enabled_lock(Some(("COMPLIANCE", 1))),
|
||||
config: persisted_lock(&config),
|
||||
updated_at: OffsetDateTime::now_utc(),
|
||||
};
|
||||
let mut metadata = HashMap::new();
|
||||
@@ -606,15 +655,15 @@ mod tests {
|
||||
// [1] Normal case: No retention specified (empty metadata)
|
||||
assert!(parse_object_lock_retention(None).is_ok());
|
||||
assert!(parse_object_lock_retention(None).unwrap().is_empty());
|
||||
let cleared = parse_object_lock_retention(Some(RequestedObjectLockRetention::default())).unwrap();
|
||||
let cleared = parse_object_lock_retention(Some(ObjectLockRetention::default())).unwrap();
|
||||
assert_eq!(cleared.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str), Some(""));
|
||||
assert_eq!(cleared.get(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER).map(String::as_str), Some(""));
|
||||
assert!(contains_key_str(&cleared, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP));
|
||||
|
||||
// [2] Normal case: Retention with valid COMPLIANCE mode (future date)
|
||||
let valid_compliance_retention = RequestedObjectLockRetention {
|
||||
mode: Some("COMPLIANCE".to_owned()),
|
||||
retain_until: Some(datetime!(2030-01-01 00:00:00 UTC)),
|
||||
let valid_compliance_retention = ObjectLockRetention {
|
||||
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)),
|
||||
retain_until_date: Some(datetime!(2030-01-01 00:00:00 UTC).into()),
|
||||
};
|
||||
let compliance_metadata = parse_object_lock_retention(Some(valid_compliance_retention)).unwrap();
|
||||
assert_eq!(compliance_metadata.get(AMZ_OBJECT_LOCK_MODE_LOWER).unwrap(), "COMPLIANCE");
|
||||
@@ -629,26 +678,26 @@ mod tests {
|
||||
assert!(retention_timestamp <= OffsetDateTime::now_utc());
|
||||
|
||||
// [3] Normal case: Retention with valid GOVERNANCE mode (future date)
|
||||
let valid_governance_retention = RequestedObjectLockRetention {
|
||||
mode: Some("GOVERNANCE".to_owned()),
|
||||
retain_until: Some(datetime!(2030-01-01 00:00:00 UTC)),
|
||||
let valid_governance_retention = ObjectLockRetention {
|
||||
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::GOVERNANCE)),
|
||||
retain_until_date: Some(datetime!(2030-01-01 00:00:00 UTC).into()),
|
||||
};
|
||||
let governance_metadata = parse_object_lock_retention(Some(valid_governance_retention)).unwrap();
|
||||
assert_eq!(governance_metadata.get(AMZ_OBJECT_LOCK_MODE_LOWER).unwrap(), "GOVERNANCE");
|
||||
|
||||
// A retention update is either empty or has both fields.
|
||||
let none_mode_retention = RequestedObjectLockRetention {
|
||||
let none_mode_retention = ObjectLockRetention {
|
||||
mode: None,
|
||||
retain_until: Some(datetime!(2030-01-01 00:00:00 UTC)),
|
||||
retain_until_date: Some(datetime!(2030-01-01 00:00:00 UTC).into()),
|
||||
};
|
||||
assert_eq!(
|
||||
parse_object_lock_retention(Some(none_mode_retention)).unwrap_err().code(),
|
||||
&S3ErrorCode::MalformedXML
|
||||
);
|
||||
|
||||
let none_date_retention = RequestedObjectLockRetention {
|
||||
mode: Some("COMPLIANCE".to_owned()),
|
||||
retain_until: None,
|
||||
let none_date_retention = ObjectLockRetention {
|
||||
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)),
|
||||
retain_until_date: None,
|
||||
};
|
||||
assert_eq!(
|
||||
parse_object_lock_retention(Some(none_date_retention)).unwrap_err().code(),
|
||||
@@ -656,9 +705,9 @@ mod tests {
|
||||
);
|
||||
|
||||
// [6] Error case: Retention with invalid mode (non COMPLIANCE/GOVERNANCE)
|
||||
let invalid_mode_retention = RequestedObjectLockRetention {
|
||||
mode: Some("INVALID_MODE".to_owned()),
|
||||
retain_until: Some(datetime!(2030-01-01 00:00:00 UTC)),
|
||||
let invalid_mode_retention = ObjectLockRetention {
|
||||
mode: Some(ObjectLockRetentionMode::from_static("INVALID_MODE")),
|
||||
retain_until_date: Some(datetime!(2030-01-01 00:00:00 UTC).into()),
|
||||
};
|
||||
let err = parse_object_lock_retention(Some(invalid_mode_retention)).unwrap_err();
|
||||
assert_eq!(err.code().as_str(), S3ErrorCode::MalformedXML.as_str());
|
||||
@@ -668,9 +717,9 @@ mod tests {
|
||||
);
|
||||
|
||||
// [7] Error case: Retention with past date should fail
|
||||
let past_date_retention = RequestedObjectLockRetention {
|
||||
mode: Some("COMPLIANCE".to_owned()),
|
||||
retain_until: Some(datetime!(2020-01-01 00:00:00 UTC)),
|
||||
let past_date_retention = ObjectLockRetention {
|
||||
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)),
|
||||
retain_until_date: Some(datetime!(2020-01-01 00:00:00 UTC).into()),
|
||||
};
|
||||
let err = parse_object_lock_retention(Some(past_date_retention)).unwrap_err();
|
||||
assert_eq!(err.code().as_str(), S3ErrorCode::InvalidArgument.as_str());
|
||||
@@ -681,9 +730,9 @@ mod tests {
|
||||
.to_offset(UtcOffset::from_hms(5, 30, 45).expect("offset with seconds should be constructible"));
|
||||
assert!(unsupported_rfc3339_date > OffsetDateTime::now_utc());
|
||||
|
||||
let unsupported_date_retention = RequestedObjectLockRetention {
|
||||
mode: Some("COMPLIANCE".to_owned()),
|
||||
retain_until: Some(unsupported_rfc3339_date),
|
||||
let unsupported_date_retention = ObjectLockRetention {
|
||||
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)),
|
||||
retain_until_date: Some(unsupported_rfc3339_date.into()),
|
||||
};
|
||||
let err = parse_object_lock_retention(Some(unsupported_date_retention)).unwrap_err();
|
||||
assert_eq!(err.code().as_str(), S3ErrorCode::InvalidArgument.as_str());
|
||||
@@ -697,7 +746,9 @@ mod tests {
|
||||
assert!(parse_object_lock_legal_hold(None).unwrap().is_empty());
|
||||
|
||||
// [2] Normal case: Legal hold with valid ON status
|
||||
let valid_on_legal_hold = RequestedObjectLegalHold(Some("ON".to_owned()));
|
||||
let valid_on_legal_hold = ObjectLockLegalHold {
|
||||
status: Some(ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::ON)),
|
||||
};
|
||||
let on_metadata = parse_object_lock_legal_hold(Some(valid_on_legal_hold)).unwrap();
|
||||
assert_eq!(on_metadata.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).unwrap(), "ON");
|
||||
let legal_hold_timestamp = get_str(&on_metadata, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP)
|
||||
@@ -706,16 +757,20 @@ mod tests {
|
||||
assert!(legal_hold_timestamp <= OffsetDateTime::now_utc());
|
||||
|
||||
// [3] Normal case: Legal hold with valid OFF status
|
||||
let valid_off_legal_hold = RequestedObjectLegalHold(Some("OFF".to_owned()));
|
||||
let valid_off_legal_hold = ObjectLockLegalHold {
|
||||
status: Some(ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF)),
|
||||
};
|
||||
let off_metadata = parse_object_lock_legal_hold(Some(valid_off_legal_hold)).unwrap();
|
||||
assert_eq!(off_metadata.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).unwrap(), "OFF");
|
||||
|
||||
let none_status_legal_hold = RequestedObjectLegalHold(None);
|
||||
let none_status_legal_hold = ObjectLockLegalHold { status: None };
|
||||
let err = parse_object_lock_legal_hold(Some(none_status_legal_hold)).expect_err("Status is required");
|
||||
assert_eq!(err.code().as_str(), S3ErrorCode::MalformedXML.as_str());
|
||||
|
||||
// [5] Error case: Legal hold with invalid status (non ON/OFF)
|
||||
let invalid_status_legal_hold = RequestedObjectLegalHold(Some("INVALID_STATUS".to_owned()));
|
||||
let invalid_status_legal_hold = ObjectLockLegalHold {
|
||||
status: Some(ObjectLockLegalHoldStatus::from_static("INVALID_STATUS")),
|
||||
};
|
||||
let err = parse_object_lock_legal_hold(Some(invalid_status_legal_hold)).unwrap_err();
|
||||
assert_eq!(err.code().as_str(), S3ErrorCode::MalformedXML.as_str());
|
||||
assert_eq!(
|
||||
@@ -1039,7 +1094,7 @@ mod tests {
|
||||
.source()
|
||||
.is_some_and(|cause| cause.is::<http::header::InvalidHeaderValue>())
|
||||
);
|
||||
let wire = crate::app::storage_api::s3::S3Error::from(error);
|
||||
let wire = s3s::S3Error::from(error);
|
||||
assert_eq!(wire.status_code(), Some(StatusCode::INTERNAL_SERVER_ERROR));
|
||||
assert!(
|
||||
wire.source()
|
||||
|
||||
@@ -20,10 +20,6 @@
|
||||
# RUSTFS_ILM_DEBUG_DAY_SECS=10 (1 "day" = 10s, matches s3-tests lc_debug_interval)
|
||||
# RUSTFS_SCANNER_ENABLED=true
|
||||
# RUSTFS_SCANNER_CYCLE=2 (scan every 2s, well under a debug-day)
|
||||
# RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES=1 (re-descend compacted dirs every cycle so
|
||||
# ILM is evaluated ~every 2s, not once per 16 cycles;
|
||||
# without this a Days=1 object due at 10s is not
|
||||
# expired until the next ~32s boundary — #4772)
|
||||
# RUSTFS_SCANNER_START_DELAY_SECS=0
|
||||
# RUSTFS_API_STALE_UPLOADS_CLEANUP_INTERVAL=2s (multipart abort lane)
|
||||
#
|
||||
|
||||
Reference in New Issue
Block a user