feat(s3): serve PutBucketLifecycleConfiguration through the gateway

Validate lifecycle status across both request adapters and preserve mutation error priority, ordered rule IDs, and stored dates. Forward the original bucket and request envelope at the legacy edge. Pin the gateway lifecycle date carrier.

The priority fixture covers seven refusal cases and one success case. All 39 scoped tests passed after source restoration. Fifteen fault switches triggered 16 expected failures covering priority, early conversion, metrics and audit, action binding, transition writes, date precision, offsets, and rule indices.

The tests and both Clippy modes passed on 0567078 with gateway 2a6b8d1. The gateway all-target check and prepush passed after rebasing onto 1467d32. Full-workspace and remote CI results remain unverified.
This commit is contained in:
overtrue
2026-11-15 13:08:53 +08:00
parent 1467d32695
commit 77da63397c
12 changed files with 872 additions and 260 deletions
Generated
+8 -8
View File
@@ -10268,7 +10268,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway"
version = "0.63.0"
source = "git+https://github.com/rustfs/gateway.git?rev=d75b6cebb77d43f0e314d250a1f6bd7c5437bf73#d75b6cebb77d43f0e314d250a1f6bd7c5437bf73"
source = "git+https://github.com/rustfs/gateway.git?rev=2a6b8d1f7663db978d8784ab3ea331cade19237b#2a6b8d1f7663db978d8784ab3ea331cade19237b"
dependencies = [
"arc-swap",
"bytes",
@@ -10315,7 +10315,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-core"
version = "0.60.0"
source = "git+https://github.com/rustfs/gateway.git?rev=d75b6cebb77d43f0e314d250a1f6bd7c5437bf73#d75b6cebb77d43f0e314d250a1f6bd7c5437bf73"
source = "git+https://github.com/rustfs/gateway.git?rev=2a6b8d1f7663db978d8784ab3ea331cade19237b#2a6b8d1f7663db978d8784ab3ea331cade19237b"
dependencies = [
"bytes",
"futures-core",
@@ -10336,7 +10336,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-http"
version = "0.1.4"
source = "git+https://github.com/rustfs/gateway.git?rev=d75b6cebb77d43f0e314d250a1f6bd7c5437bf73#d75b6cebb77d43f0e314d250a1f6bd7c5437bf73"
source = "git+https://github.com/rustfs/gateway.git?rev=2a6b8d1f7663db978d8784ab3ea331cade19237b#2a6b8d1f7663db978d8784ab3ea331cade19237b"
dependencies = [
"hmac 0.13.0",
"http 1.5.0",
@@ -10354,7 +10354,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-macros"
version = "0.1.4"
source = "git+https://github.com/rustfs/gateway.git?rev=d75b6cebb77d43f0e314d250a1f6bd7c5437bf73#d75b6cebb77d43f0e314d250a1f6bd7c5437bf73"
source = "git+https://github.com/rustfs/gateway.git?rev=2a6b8d1f7663db978d8784ab3ea331cade19237b#2a6b8d1f7663db978d8784ab3ea331cade19237b"
dependencies = [
"proc-macro2",
"quote",
@@ -10364,7 +10364,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-sig"
version = "0.15.0"
source = "git+https://github.com/rustfs/gateway.git?rev=d75b6cebb77d43f0e314d250a1f6bd7c5437bf73#d75b6cebb77d43f0e314d250a1f6bd7c5437bf73"
source = "git+https://github.com/rustfs/gateway.git?rev=2a6b8d1f7663db978d8784ab3ea331cade19237b#2a6b8d1f7663db978d8784ab3ea331cade19237b"
dependencies = [
"hmac 0.13.0",
"http 1.5.0",
@@ -10379,7 +10379,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-stream"
version = "0.2.0"
source = "git+https://github.com/rustfs/gateway.git?rev=d75b6cebb77d43f0e314d250a1f6bd7c5437bf73#d75b6cebb77d43f0e314d250a1f6bd7c5437bf73"
source = "git+https://github.com/rustfs/gateway.git?rev=2a6b8d1f7663db978d8784ab3ea331cade19237b#2a6b8d1f7663db978d8784ab3ea331cade19237b"
dependencies = [
"bitflags 2.13.2",
"bytes",
@@ -10390,7 +10390,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-types"
version = "0.40.0+aws.2026-10-02"
source = "git+https://github.com/rustfs/gateway.git?rev=d75b6cebb77d43f0e314d250a1f6bd7c5437bf73#d75b6cebb77d43f0e314d250a1f6bd7c5437bf73"
source = "git+https://github.com/rustfs/gateway.git?rev=2a6b8d1f7663db978d8784ab3ea331cade19237b#2a6b8d1f7663db978d8784ab3ea331cade19237b"
dependencies = [
"bytes",
"crc-fast",
@@ -10413,7 +10413,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-xml"
version = "0.4.0"
source = "git+https://github.com/rustfs/gateway.git?rev=d75b6cebb77d43f0e314d250a1f6bd7c5437bf73#d75b6cebb77d43f0e314d250a1f6bd7c5437bf73"
source = "git+https://github.com/rustfs/gateway.git?rev=2a6b8d1f7663db978d8784ab3ea331cade19237b#2a6b8d1f7663db978d8784ab3ea331cade19237b"
dependencies = [
"quick-xml 0.41.0",
]
+4 -4
View File
@@ -151,14 +151,14 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
# engine crates move onto from s3s (rustfs/backlog#2745). The one pin of the
# gateway in this workspace; no crate may enable a `compat-s3s*` feature except
# the s3s-facing edge of the rustfs binary.
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "d75b6cebb77d43f0e314d250a1f6bd7c5437bf73", version = "0.40.0", default-features = false }
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "d75b6cebb77d43f0e314d250a1f6bd7c5437bf73", version = "0.2.0", default-features = false }
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "2a6b8d1f7663db978d8784ab3ea331cade19237b", version = "0.40.0", default-features = false }
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "2a6b8d1f7663db978d8784ab3ea331cade19237b", version = "0.2.0", default-features = false }
# The gateway framework itself, consumed for its extension traits by the ring-2
# adapter. `default-features = false` keeps the `server` feature, and with it the
# server crate, out of the build.
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "d75b6cebb77d43f0e314d250a1f6bd7c5437bf73", version = "0.63.0", default-features = false }
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "2a6b8d1f7663db978d8784ab3ea331cade19237b", version = "0.63.0", default-features = false }
# Test-only: builds the `EffectiveHost` a host-resolution test queries with.
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "d75b6cebb77d43f0e314d250a1f6bd7c5437bf73", default-features = false }
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "2a6b8d1f7663db978d8784ab3ea331cade19237b", default-features = false }
# Async Runtime and Networking
async-channel = "2.5.0"
+1
View File
@@ -132,6 +132,7 @@ mod encryption;
#[cfg(test)]
mod guards;
mod lifecycle;
pub(crate) use lifecycle::{PreparedLifecycleConfiguration, validate_lifecycle_rule_status};
mod list_object_versions;
pub(crate) mod list_objects;
mod listing;
+174 -176
View File
@@ -15,16 +15,19 @@
//! Bucket lifecycle configuration use-case operations.
use super::*;
use crate::app::storage_api::s3::dto::{
DeleteBucketLifecycleInput, DeleteBucketLifecycleOutput, ExpirationStatus, LifecycleRule,
PutBucketLifecycleConfigurationInput, PutBucketLifecycleConfigurationOutput,
};
use crate::app::storage_api::s3::dto::{DeleteBucketLifecycleInput, DeleteBucketLifecycleOutput};
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
use rustfs_gateway_types::persistence::parse_lifecycle_dto;
use rustfs_gateway_types::persistence::{PersistedLifecycleRule, parse_lifecycle_dto};
const ERR_LIFECYCLE_RULE_STATUS: &str = "Rule status must be either Enabled or Disabled";
fn assign_lifecycle_rule_ids(rules: &mut [LifecycleRule]) {
pub(crate) enum PreparedLifecycleConfiguration {
Missing,
InvalidStatus,
Prepared(std::io::Result<PersistedLifecycleConfiguration>),
}
fn assign_lifecycle_rule_ids(rules: &mut [PersistedLifecycleRule]) {
let mut rule_ids: HashSet<String> = HashSet::new();
for rule in rules.iter() {
@@ -49,11 +52,11 @@ fn assign_lifecycle_rule_ids(rules: &mut [LifecycleRule]) {
}
}
fn validate_lifecycle_rule_status(rules: &[LifecycleRule]) -> std::result::Result<(), &'static str> {
for rule in rules {
if rule.status != ExpirationStatus::from_static(ExpirationStatus::ENABLED)
&& rule.status != ExpirationStatus::from_static(ExpirationStatus::DISABLED)
{
pub(crate) fn validate_lifecycle_rule_status<'a>(
statuses: impl IntoIterator<Item = &'a str>,
) -> std::result::Result<(), &'static str> {
for status in statuses {
if status != "Enabled" && status != "Disabled" {
return Err(ERR_LIFECYCLE_RULE_STATUS);
}
}
@@ -194,156 +197,155 @@ impl DefaultBucketUsecase {
})
}
#[instrument(level = "debug", skip(self, req))]
pub async fn execute_put_bucket_lifecycle_configuration(
/// Validates and persists a lifecycle configuration fenced to the authorized bucket incarnation.
///
/// # Errors
/// Returns the original configuration validation or storage error. Missing configuration is
/// `InvalidArgument`; invalid status is `MalformedXML`; conversion errors are `InternalError`.
#[instrument(level = "debug", skip(self, envelope, bucket, prepared_configuration))]
pub(crate) async fn execute_put_bucket_lifecycle_configuration(
&self,
req: S3Request<PutBucketLifecycleConfigurationInput>,
prepared_configuration: std::io::Result<Option<PersistedLifecycleConfiguration>>,
) -> S3Result<S3Response<PutBucketLifecycleConfigurationOutput>> {
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
let PutBucketLifecycleConfigurationInput {
bucket,
lifecycle_configuration,
..
} = req.input;
envelope: RequestEnvelope,
bucket: &str,
prepared_configuration: PreparedLifecycleConfiguration,
) -> rustfs_s3_types::S3Result<gateway_dto::PutBucketLifecycleConfigurationOutput> {
RequestEnvelope::audit_only_operation(
envelope,
S3Operation::PutBucketLifecycleConfiguration,
StatusCode::OK,
|envelope| async move {
let expected_incarnation_id = bucket_config_mutation_incarnation(&envelope, bucket)?;
let request_context = envelope.request_context().cloned();
let prepared_configuration = match prepared_configuration {
PreparedLifecycleConfiguration::Missing => return Err(rustfs_s3_types::s3_error!(InvalidArgument)),
PreparedLifecycleConfiguration::InvalidStatus => {
return Err(rustfs_s3_types::s3_error!(MalformedXML, "Malformed XML: {ERR_LIFECYCLE_RULE_STATUS}"));
}
PreparedLifecycleConfiguration::Prepared(result) => result,
};
let Some(mut wire_cfg) = lifecycle_configuration else { return Err(s3_error!(InvalidArgument)) };
assign_lifecycle_rule_ids(&mut wire_cfg.rules);
if let Err(err) = validate_lifecycle_rule_status(&wire_cfg.rules) {
return Err(S3Error::with_message(S3ErrorCode::MalformedXML, format!("Malformed XML: {err}")));
}
let rcfg = match metadata_sys::get_object_lock_config(bucket).await {
Ok((cfg, _)) => cfg,
Err(StorageError::ConfigNotFound) => ObjectLockConfiguration::default(),
Err(err) => {
warn!(
component = LOG_COMPONENT_APP,
subsystem = LOG_SUBSYSTEM_BUCKET,
event = "bucket_object_lock_config_load_failed",
bucket = %bucket,
error = ?err,
"Failed to load bucket object lock configuration"
);
return Err(ApiError::from(err).into());
}
};
let rcfg = match metadata_sys::get_object_lock_config(&bucket).await {
Ok((cfg, _)) => cfg,
Err(StorageError::ConfigNotFound) => ObjectLockConfiguration::default(),
Err(err) => {
warn!(
component = LOG_COMPONENT_APP,
subsystem = LOG_SUBSYSTEM_BUCKET,
event = "bucket_object_lock_config_load_failed",
bucket = %bucket,
error = ?err,
"Failed to load bucket object lock configuration"
);
return Err(ApiError::from(err).into());
}
};
// Keep conversion failures behind the request-boundary status and Object Lock checks.
let Some(mut input_cfg) = prepared_configuration.map_err(to_internal_error)? else {
return Err(rustfs_s3_types::s3_error!(InvalidArgument).into());
};
if input_cfg.rules.len() != wire_cfg.rules.len() {
return Err(S3Error::with_message(
S3ErrorCode::InternalError,
"lifecycle conversion changed the rule count",
));
}
for (rule, wire_rule) in input_cfg.rules.iter_mut().zip(&wire_cfg.rules) {
rule.id.clone_from(&wire_rule.id);
}
if let Err(err) = validate_lifecycle_config(&input_cfg, &rcfg).await {
return Err(lifecycle_validation_error(&err));
}
if let Err(err) = validate_transition_tier(&input_cfg).await {
return Err(s3_error!(InvalidArgument, "{err}"));
}
// Stamp the expiry axis only when the expiry subset can have changed
// (MinIO: HasExpiry() || expiryRuleRemoved). Site-replication peers
// judge lc-config staleness on this axis; a transition-only edit that
// advanced it would let this site's stale expiry subset shadow — and
// roll back — a newer peer expiry edit fleet-wide.
let previous_expiry_updated_at = match metadata_sys::get_lifecycle_config(&bucket).await {
Ok((previous, _)) => {
if lifecycle_rules_have_expiry(&input_cfg) || lifecycle_rules_have_expiry(&previous) {
Some(
time::OffsetDateTime::now_utc()
.format(&time::format_description::well_known::Rfc3339)
.map_err(to_internal_error)?,
)
} else {
previous.expiry_updated_at
// Keep conversion failures behind the request-boundary status and Object Lock checks.
let mut input_cfg = prepared_configuration.map_err(to_internal_error)?;
assign_lifecycle_rule_ids(&mut input_cfg.rules);
if let Err(err) = validate_lifecycle_config(&input_cfg, &rcfg).await {
return Err(lifecycle_validation_error(&err).into());
}
}
// No previous config (or unreadable): stamping is the
// conservative pre-existing behavior.
Err(_) => {
if lifecycle_rules_have_expiry(&input_cfg) {
Some(
time::OffsetDateTime::now_utc()
.format(&time::format_description::well_known::Rfc3339)
.map_err(to_internal_error)?,
)
} else {
None
if let Err(err) = validate_transition_tier(&input_cfg).await {
return Err(rustfs_s3_types::s3_error!(InvalidArgument, "{err}"));
}
}
};
input_cfg.expiry_updated_at = previous_expiry_updated_at;
let data = serialize_config(&input_cfg)?;
update_bucket_config_for_incarnation(&bucket, BUCKET_LIFECYCLE_CONFIG, data, expected_incarnation_id)
.await
.map_err(ApiError::from)?;
notify_bucket_metadata_reload(bucket.clone(), "put bucket lifecycle", request_context, true).await;
// Stamp the expiry axis only when the expiry subset can have changed
// (MinIO: HasExpiry() || expiryRuleRemoved). Site-replication peers
// judge lc-config staleness on this axis; a transition-only edit that
// advanced it would let this site's stale expiry subset shadow — and
// roll back — a newer peer expiry edit fleet-wide.
let previous_expiry_updated_at = match metadata_sys::get_lifecycle_config(bucket).await {
Ok((previous, _)) => {
if lifecycle_rules_have_expiry(&input_cfg) || lifecycle_rules_have_expiry(&previous) {
Some(
time::OffsetDateTime::now_utc()
.format(&time::format_description::well_known::Rfc3339)
.map_err(to_internal_error)?,
)
} else {
previous.expiry_updated_at
}
}
// No previous config (or unreadable): stamping is the
// conservative pre-existing behavior.
Err(_) => {
if lifecycle_rules_have_expiry(&input_cfg) {
Some(
time::OffsetDateTime::now_utc()
.format(&time::format_description::well_known::Rfc3339)
.map_err(to_internal_error)?,
)
} else {
None
}
}
};
input_cfg.expiry_updated_at = previous_expiry_updated_at;
let data = serialize_config(&input_cfg)?;
update_bucket_config_for_incarnation(bucket, BUCKET_LIFECYCLE_CONFIG, data, expected_incarnation_id)
.await
.map_err(ApiError::from)?;
let mut item = sr_bucket_meta_item(bucket.clone(), "lc-config");
item.expiry_lc_config =
Some(serialize_config(&input_cfg).and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?);
// The item travels with the expiry axis, not the wall clock: a site
// whose expiry knowledge is old (or absent — UNIX_EPOCH) must not
// out-rank newer peer expiry state at the receivers.
item.expiry_updated_at = input_cfg
.expiry_updated_at
.as_deref()
.map(|value| time::OffsetDateTime::parse(value, &time::format_description::well_known::Rfc3339))
.transpose()
.map_err(to_internal_error)?
.or(Some(time::OffsetDateTime::UNIX_EPOCH));
if let Err(err) = site_replication_bucket_meta_hook(item).await {
warn!(bucket = %bucket, error = ?err, "site replication bucket lifecycle hook failed");
}
notify_bucket_metadata_reload(bucket.to_owned(), "put bucket lifecycle", request_context, true).await;
if lifecycle_has_transition_rules(&input_cfg)
&& let Some(store) = self.object_store()
{
let bucket_name = bucket.clone();
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
spawn_background_with_context(request_context, async move {
if let Err(err) = enqueue_transition_for_existing_objects(store, &bucket_name).await {
warn!(bucket = %bucket_name, error = ?err, "failed to enqueue transition for existing objects");
let mut item = sr_bucket_meta_item(bucket.to_owned(), "lc-config");
item.expiry_lc_config =
Some(serialize_config(&input_cfg).and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?);
// The item travels with the expiry axis, not the wall clock: a site
// whose expiry knowledge is old (or absent — UNIX_EPOCH) must not
// out-rank newer peer expiry state at the receivers.
item.expiry_updated_at = input_cfg
.expiry_updated_at
.as_deref()
.map(|value| time::OffsetDateTime::parse(value, &time::format_description::well_known::Rfc3339))
.transpose()
.map_err(to_internal_error)?
.or(Some(time::OffsetDateTime::UNIX_EPOCH));
if let Err(err) = site_replication_bucket_meta_hook(item).await {
warn!(bucket = %bucket, error = ?err, "site replication bucket lifecycle hook failed");
}
});
}
if lifecycle_has_expiry_rules(&input_cfg)
&& let Some(store) = self.object_store()
{
let bucket_name = bucket.clone();
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
spawn_background_with_context(request_context, async move {
if let Err(err) = enqueue_expiry_for_existing_objects(store, &bucket_name).await {
warn!(bucket = %bucket_name, error = ?err, "failed to enqueue expiry for existing objects");
if lifecycle_has_transition_rules(&input_cfg)
&& let Some(store) = self.object_store()
{
let bucket_name = bucket.to_owned();
let request_context = envelope.request_context().cloned();
spawn_background_with_context(request_context, async move {
if let Err(err) = enqueue_transition_for_existing_objects(store, &bucket_name).await {
warn!(bucket = %bucket_name, error = ?err, "failed to enqueue transition for existing objects");
}
});
}
});
}
if lifecycle_has_abort_multipart_rules(&input_cfg)
&& let Some(store) = self.object_store()
{
let bucket_name = bucket.clone();
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
spawn_background_with_context(request_context, async move {
let deleted = run_stale_multipart_upload_cleanup_once(store).await;
debug!(bucket = %bucket_name, deleted, "completed lifecycle abort multipart cleanup trigger");
});
}
if lifecycle_has_expiry_rules(&input_cfg)
&& let Some(store) = self.object_store()
{
let bucket_name = bucket.to_owned();
let request_context = envelope.request_context().cloned();
spawn_background_with_context(request_context, async move {
if let Err(err) = enqueue_expiry_for_existing_objects(store, &bucket_name).await {
warn!(bucket = %bucket_name, error = ?err, "failed to enqueue expiry for existing objects");
}
});
}
Ok(S3Response::new(PutBucketLifecycleConfigurationOutput::default()))
if lifecycle_has_abort_multipart_rules(&input_cfg)
&& let Some(store) = self.object_store()
{
let bucket_name = bucket.to_owned();
let request_context = envelope.request_context().cloned();
spawn_background_with_context(request_context, async move {
let deleted = run_stale_multipart_upload_cleanup_once(store).await;
debug!(bucket = %bucket_name, deleted, "completed lifecycle abort multipart cleanup trigger");
});
}
Ok(gateway_dto::PutBucketLifecycleConfigurationOutput::default())
},
)
.await
}
}
@@ -353,7 +355,8 @@ mod tests {
use crate::app::bucket_usecase::test_support::build_request;
use crate::app::storage_api::s3::dto;
use crate::app::storage_api::s3::dto::{
BucketLifecycleConfiguration, LifecycleExpiration, NoncurrentVersionTransition, Transition, TransitionStorageClass,
BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, NoncurrentVersionTransition,
PutBucketLifecycleConfigurationInput, Transition, TransitionStorageClass,
};
use http::Method;
@@ -371,7 +374,7 @@ mod tests {
#[test]
fn normalize_lifecycle_rules_generate_rule_ids_for_missing_values() {
let mut rules = vec![
let rules = vec![
LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
@@ -419,6 +422,12 @@ mod tests {
},
];
let mut rules = crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s(BucketLifecycleConfiguration {
rules,
..Default::default()
})
.unwrap()
.rules;
assign_lifecycle_rule_ids(&mut rules);
assert_eq!(rules[0].id.as_deref(), Some("rule-0"));
@@ -492,24 +501,15 @@ mod tests {
}
#[test]
fn validate_lifecycle_rule_status_rejects_invalid_status() {
let rules = vec![LifecycleRule {
status: ExpirationStatus::from_static("enabled"),
expiration: Some(LifecycleExpiration {
days: Some(30),
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: None,
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
}];
assert_eq!(validate_lifecycle_rule_status(&rules).unwrap_err(), ERR_LIFECYCLE_RULE_STATUS);
fn validate_lifecycle_rule_status_checks_exact_values() {
assert!(validate_lifecycle_rule_status(["Enabled", "Disabled"]).is_ok());
for invalid in ["enabled", "unknown"] {
assert_eq!(
validate_lifecycle_rule_status(["Enabled", invalid]),
Err(ERR_LIFECYCLE_RULE_STATUS),
"{invalid}"
);
}
}
#[test]
@@ -604,16 +604,14 @@ mod tests {
let req = build_request(input, Method::PUT);
let usecase = DefaultBucketUsecase::without_context();
let prepared = req
.input
.lifecycle_configuration
.clone()
.map(crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s)
.transpose();
let err = usecase
.execute_put_bucket_lifecycle_configuration(req, prepared)
.execute_put_bucket_lifecycle_configuration(
RequestEnvelope::from_s3s(&req),
&req.input.bucket,
PreparedLifecycleConfiguration::Missing,
)
.await
.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
}
}
@@ -16,6 +16,7 @@
use super::common::*;
use super::*;
use crate::app::storage_api::s3::S3 as _;
fn expiration_lifecycle_configuration(prefix: &str) -> BucketLifecycleConfiguration {
BucketLifecycleConfiguration {
@@ -105,7 +106,6 @@ fn del_marker_expiration_lifecycle_configuration(days: i32) -> BucketLifecycleCo
async fn compensation_driven_delete_marker_still_honors_lifecycle_cleanup() {
let (_disk_paths, ecstore) = setup_test_env().await;
let usecase = DefaultObjectUsecase::from_global();
let bucket_usecase = DefaultBucketUsecase::from_global();
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
let backend = register_mock_tier(&tier_name).await;
@@ -162,14 +162,8 @@ async fn compensation_driven_delete_marker_still_honors_lifecycle_cleanup() {
.unwrap(),
Method::PUT,
);
let prepared = req
.input
.lifecycle_configuration
.clone()
.map(crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s)
.transpose();
bucket_usecase
.execute_put_bucket_lifecycle_configuration(req, prepared)
FS::new()
.put_bucket_lifecycle_configuration(req)
.await
.expect("Failed to update lifecycle configuration for delete marker cleanup");
@@ -188,7 +182,6 @@ async fn compensation_driven_delete_marker_still_honors_lifecycle_cleanup() {
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
async fn put_bucket_lifecycle_configuration_expires_existing_objects() {
let (_disk_paths, ecstore) = setup_test_env().await;
let usecase = DefaultBucketUsecase::from_global();
let bucket = format!("test-api-expire-existing-{}", &Uuid::new_v4().simple().to_string()[..8]);
let object = "test/existing.txt";
@@ -206,14 +199,8 @@ async fn put_bucket_lifecycle_configuration_expires_existing_objects() {
Method::PUT,
);
let prepared = req
.input
.lifecycle_configuration
.clone()
.map(crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s)
.transpose();
usecase
.execute_put_bucket_lifecycle_configuration(req, prepared)
FS::new()
.put_bucket_lifecycle_configuration(req)
.await
.expect("Failed to update lifecycle configuration");
@@ -228,7 +215,6 @@ async fn put_bucket_lifecycle_configuration_expires_existing_objects() {
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
async fn put_bucket_lifecycle_configuration_allows_expired_delete_marker_on_object_lock_bucket() {
let (_disk_paths, ecstore) = setup_test_env().await;
let usecase = DefaultBucketUsecase::from_global();
let bucket = format!("test-lock-expired-del-marker-{}", &Uuid::new_v4().simple().to_string()[..8]);
create_test_bucket(&ecstore, bucket.as_str()).await;
@@ -245,14 +231,8 @@ async fn put_bucket_lifecycle_configuration_allows_expired_delete_marker_on_obje
Method::PUT,
);
let prepared = req
.input
.lifecycle_configuration
.clone()
.map(crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s)
.transpose();
usecase
.execute_put_bucket_lifecycle_configuration(req, prepared)
FS::new()
.put_bucket_lifecycle_configuration(req)
.await
.expect("ExpiredObjectDeleteMarker should be accepted on object-lock bucket");
}
@@ -262,7 +242,6 @@ async fn put_bucket_lifecycle_configuration_allows_expired_delete_marker_on_obje
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
async fn put_bucket_lifecycle_configuration_rejects_del_marker_expiration_on_object_lock_bucket() {
let (_disk_paths, ecstore) = setup_test_env().await;
let usecase = DefaultBucketUsecase::from_global();
let bucket = format!("test-lock-del-marker-exp-{}", &Uuid::new_v4().simple().to_string()[..8]);
create_test_bucket(&ecstore, bucket.as_str()).await;
@@ -279,17 +258,12 @@ async fn put_bucket_lifecycle_configuration_rejects_del_marker_expiration_on_obj
Method::PUT,
);
let prepared = req
.input
.lifecycle_configuration
.clone()
.map(crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s)
.transpose();
let err = usecase
.execute_put_bucket_lifecycle_configuration(req, prepared)
let err: rustfs_s3_types::S3Error = FS::new()
.put_bucket_lifecycle_configuration(req)
.await
.expect_err("DelMarkerExpiration must be rejected on object-lock bucket");
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
.expect_err("DelMarkerExpiration must be rejected on object-lock bucket")
.into();
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
let message = err.message().unwrap_or_default();
assert!(
message.contains(
@@ -304,7 +278,6 @@ async fn put_bucket_lifecycle_configuration_rejects_del_marker_expiration_on_obj
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
async fn put_bucket_lifecycle_configuration_rejects_zero_day_del_marker_expiration_on_object_lock_bucket() {
let (_disk_paths, ecstore) = setup_test_env().await;
let usecase = DefaultBucketUsecase::from_global();
let bucket = format!("test-lock-del-marker-exp0-{}", &Uuid::new_v4().simple().to_string()[..8]);
create_test_bucket(&ecstore, bucket.as_str()).await;
@@ -321,17 +294,12 @@ async fn put_bucket_lifecycle_configuration_rejects_zero_day_del_marker_expirati
Method::PUT,
);
let prepared = req
.input
.lifecycle_configuration
.clone()
.map(crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s)
.transpose();
let err = usecase
.execute_put_bucket_lifecycle_configuration(req, prepared)
let err: rustfs_s3_types::S3Error = FS::new()
.put_bucket_lifecycle_configuration(req)
.await
.expect_err("DelMarkerExpiration with zero days must be rejected on object-lock bucket");
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
.expect_err("DelMarkerExpiration with zero days must be rejected on object-lock bucket")
.into();
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
let message = err.message().unwrap_or_default();
assert!(
message.contains("Days must be a positive integer with DelMarkerExpiration"),
@@ -348,7 +316,6 @@ async fn put_bucket_lifecycle_configuration_rejects_zero_day_expiration() {
// PutBucketLifecycleConfiguration handler, not merely at the crates/lifecycle
// io::Error layer. No object lock required.
let (_disk_paths, ecstore) = setup_test_env().await;
let usecase = DefaultBucketUsecase::from_global();
let bucket = format!("test-zero-day-exp-{}", &Uuid::new_v4().simple().to_string()[..8]);
create_test_bucket(&ecstore, bucket.as_str()).await;
@@ -384,20 +351,199 @@ async fn put_bucket_lifecycle_configuration_rejects_zero_day_expiration() {
Method::PUT,
);
let prepared = req
.input
.lifecycle_configuration
.clone()
.map(crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s)
.transpose();
let err = usecase
.execute_put_bucket_lifecycle_configuration(req, prepared)
let err: rustfs_s3_types::S3Error = FS::new()
.put_bucket_lifecycle_configuration(req)
.await
.expect_err("zero-day expiration must be rejected with InvalidArgument");
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
.expect_err("zero-day expiration must be rejected with InvalidArgument")
.into();
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
let message = err.message().unwrap_or_default();
assert!(
message.contains("'Days' for Expiration action must be a positive integer"),
"unexpected error message: {message}"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
async fn put_bucket_lifecycle_configuration_preserves_error_priority() {
use crate::app::storage_api::bucket_usecase::{error::StorageError, helper::MutationCompletionProbe};
use crate::app::storage_api::owned_parity::{RequestContext, parity_test_support};
use crate::app::storage_api::test::{get_global_bucket_metadata_sys, set_bucket_metadata};
use crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s;
use rustfs_s3_ops::S3Operation;
let (_disk_paths, store) = setup_test_env().await;
let suffix = Uuid::new_v4().simple().to_string();
let readable = format!("lifecycle-readable-{suffix}");
let unreadable = format!("lifecycle-unreadable-{suffix}");
create_test_bucket(&store, &readable).await;
create_test_bucket(&store, &unreadable).await;
let sys = get_global_bucket_metadata_sys().unwrap();
let cached = sys.read().await.get(&unreadable).await.unwrap();
let mut metadata = (*cached).clone();
metadata.object_lock_config_xml = b"<ObjectLockConfiguration>".to_vec();
metadata.object_lock_config = None;
set_bucket_metadata(unreadable.clone(), metadata).await.unwrap();
let storage_error = metadata_sys::get_object_lock_config(&unreadable).await.unwrap_err();
let lookup_error = crate::app::storage_api::s3::S3Error::from(crate::error::ApiError::from(storage_error));
let lookup = lookup_error.message().unwrap();
assert!(lookup.contains(OBJECT_LOCK_CONFIG) && lookup.contains(&unreadable), "{lookup}");
assert!(matches!(
metadata_sys::get_object_lock_config(&readable).await,
Err(StorageError::ConfigNotFound)
));
let configuration = |status, bad_date| {
let mut configuration = expiration_lifecycle_configuration("dated/");
let date = time::macros::datetime!(2099-01-01 00:00 UTC);
let date = if bad_date {
date.to_offset(time::UtcOffset::from_hms(0, 0, 1).unwrap())
} else {
date
};
configuration.rules[0].expiration.as_mut().unwrap().date = Some(Timestamp::from(date));
configuration.rules[0].id = None;
let mut days_rule = del_marker_expiration_lifecycle_configuration(30).rules.remove(0);
days_rule.id = Some("rule-0".to_owned());
days_rule.status = ExpirationStatus::from_static(status);
configuration.rules.push(days_rule);
configuration
};
let bad_date = lifecycle_configuration_from_s3s(configuration("enabled", true))
.unwrap_err()
.to_string();
assert!(bad_date.contains("offset_second"), "{bad_date}");
assert!(lifecycle_configuration_from_s3s(configuration("Disabled", false)).is_ok());
let malformed = "Malformed XML: Rule status must be either Enabled or Disabled";
let mismatch = "bucket config mutation snapshot does not match request bucket";
let cases = [
(
"incarnation before status",
&unreadable,
Some(("enabled", true)),
true,
S3ErrorCode::InternalError,
Some(mismatch),
),
("missing before lock lookup", &unreadable, None, false, S3ErrorCode::InvalidArgument, None),
(
"status before lock lookup",
&unreadable,
Some(("enabled", true)),
false,
S3ErrorCode::MalformedXML,
Some(malformed),
),
(
"later status before first date",
&readable,
Some(("enabled", true)),
false,
S3ErrorCode::MalformedXML,
Some(malformed),
),
(
"lock lookup before date",
&unreadable,
Some(("Disabled", true)),
false,
S3ErrorCode::ServiceUnavailable,
Some(lookup),
),
(
"date after lock lookup",
&readable,
Some(("Disabled", true)),
false,
S3ErrorCode::InternalError,
Some(bad_date.as_str()),
),
(
"real lock lookup failure",
&unreadable,
Some(("Disabled", false)),
false,
S3ErrorCode::ServiceUnavailable,
Some(lookup),
),
];
let fs = FS::new();
let handled_count = || {
rustfs_io_metrics::s3_op_metrics_snapshot()
.into_iter()
.find(|metric| metric.op == S3Operation::PutBucketLifecycleConfiguration.as_str())
.map_or(0, |metric| metric.total)
};
let request = |bucket: &str, config: Option<(&'static str, bool)>| {
let mut request = build_request(
PutBucketLifecycleConfigurationInput::builder()
.bucket(bucket.to_owned())
.lifecycle_configuration(config.map(|(status, bad_date)| configuration(status, bad_date)))
.build()
.unwrap(),
Method::PUT,
);
request.uri = format!("/{bucket}?lifecycle").parse().unwrap();
request.extensions.insert(RequestContext {
request_id: "lifecycle-priority-context".to_owned(),
x_amz_request_id: "lifecycle-priority-context".to_owned(),
trace_id: None,
span_id: None,
start_time: std::time::Instant::now(),
});
request
};
let assert_audit = |probe: &MutationCompletionProbe| {
let audits = probe.audits.lock().unwrap();
assert_eq!(audits.len(), 1, "each attempt is audited once");
assert_eq!(audits[0].request_id.as_deref(), Some("lifecycle-priority-context"));
};
for (name, bucket, config, mismatched_snapshot, code, message) in cases {
let before = handled_count();
let probe = MutationCompletionProbe::install(bucket);
let mut request = request(bucket, config);
if mismatched_snapshot {
request
.extensions
.insert(parity_test_support::bucket_config_mutation(&readable));
}
let error = fs.put_bucket_lifecycle_configuration(request).await.unwrap_err();
assert_eq!(error.code(), &code, "{name}: {error}");
if let Some(message) = message {
assert_eq!(error.message(), Some(message), "{name}");
}
if code == S3ErrorCode::ServiceUnavailable {
assert_eq!(error.status_code(), Some(http::StatusCode::SERVICE_UNAVAILABLE), "{name}");
assert!(
error.source().and_then(|err| err.downcast_ref::<StorageError>()).is_some(),
"{name} keeps the storage cause"
);
}
assert_audit(&probe);
assert_eq!(handled_count() - before, 1, "{name} counts once");
}
assert!(matches!(
metadata_sys::get_lifecycle_config(&readable).await,
Err(StorageError::ConfigNotFound)
));
let before = handled_count();
let probe = MutationCompletionProbe::install(&readable);
fs.put_bucket_lifecycle_configuration(request(&readable, Some(("Disabled", false))))
.await
.unwrap();
assert_audit(&probe);
assert_eq!(handled_count() - before, 1, "success counts once");
let (stored, _) = metadata_sys::get_lifecycle_config(&readable).await.unwrap();
let rules = stored.rules;
assert_eq!(rules.len(), 2);
assert_eq!((rules[0].id.as_deref(), rules[0].status.as_str()), (Some("rule-0-1"), "Enabled"));
assert_eq!((rules[1].id.as_deref(), rules[1].status.as_str()), (Some("rule-0"), "Disabled"));
assert_eq!(rules[0].expiration.as_ref().unwrap().date.as_deref(), Some("2099-01-01T00:00:00.000Z"));
assert_eq!(rules[1].expiration.as_ref().unwrap().days, Some(30));
}
@@ -14,6 +14,8 @@
//! The gateway operations of migration batch A3 (rustfs/backlog#2765): bucket lifecycle, replication and object lock configuration.
use super::access::authorize_bucket_config_mutation;
use super::lifecycle_input::prepare_lifecycle_configuration;
use super::{OpRow, Served, row};
use crate::app::RequestEnvelope;
use crate::app::bucket_usecase::DefaultBucketUsecase;
@@ -27,6 +29,7 @@ use std::sync::Arc;
pub(super) const ROWS: &[OpRow] = &[
row::<dto::GetBucketLifecycleConfiguration>(),
row::<dto::PutBucketLifecycleConfiguration>(),
row::<dto::GetBucketReplication>(),
row::<dto::GetObjectLockConfiguration>(),
];
@@ -78,3 +81,22 @@ impl Served for dto::GetObjectLockConfiguration {
Ok(Resp::new(output))
}
}
impl Served for dto::PutBucketLifecycleConfiguration {
const METRIC: S3Operation = S3Operation::PutBucketLifecycleConfiguration;
async fn authorize(envelope: &mut RequestEnvelope, request: &Req<Self>) -> S3Result<()> {
let action = Action::S3Action(S3Action::PutBucketLifecycleAction);
authorize_bucket_config_mutation(envelope, request.input().bucket.as_str(), action).await
}
async fn execute(context: Option<Arc<AppContext>>, envelope: RequestEnvelope, request: Req<Self>) -> S3Result<Resp<Self>> {
let dates = request.context().lifecycle_request_dates().cloned();
let input = request.into_input();
let prepared = prepare_lifecycle_configuration(input.lifecycle_configuration, dates.as_deref());
let output = DefaultBucketUsecase::with_context(context)
.execute_put_bucket_lifecycle_configuration(envelope, input.bucket.as_str(), prepared)
.await?;
Ok(Resp::new(output))
}
}
@@ -0,0 +1,189 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Converts gateway lifecycle input to the native persisted shape before validation.
//! It preserves request-local dates and their precision for validation in the use case.
use crate::app::bucket_usecase::{PreparedLifecycleConfiguration, validate_lifecycle_rule_status};
use rustfs_gateway::LifecycleRequestDates;
use rustfs_gateway_types::persistence::{
PersistedAbortIncompleteMultipartUpload, PersistedDelMarkerExpiration, PersistedLifecycleAnd,
PersistedLifecycleConfiguration, PersistedLifecycleExpiration, PersistedLifecycleFilter, PersistedLifecycleRule,
PersistedLifecycleTag, PersistedNoncurrentVersionExpiration, PersistedNoncurrentVersionTransition, PersistedTransition,
};
use rustfs_gateway_types::{Timestamp, dto};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
fn date_from_gateway(value: Option<Timestamp>, source_date: Option<&str>) -> std::io::Result<Option<String>> {
value
.map(|value| {
if let Some(source_date) = source_date {
return Ok(source_date.to_owned());
}
let nanos = i128::from(value.secs()) * 1_000_000_000 + i128::from(value.subsec_nanos());
OffsetDateTime::from_unix_timestamp_nanos(nanos)
.map_err(std::io::Error::other)?
.format(&Rfc3339)
.map_err(std::io::Error::other)
})
.transpose()
}
pub(super) fn prepare_lifecycle_configuration(
configuration: Option<dto::BucketLifecycleConfiguration>,
dates: Option<&LifecycleRequestDates>,
) -> PreparedLifecycleConfiguration {
let Some(configuration) = configuration else {
return PreparedLifecycleConfiguration::Missing;
};
if validate_lifecycle_rule_status(configuration.rules.iter().map(|rule| rule.status.as_str())).is_err() {
return PreparedLifecycleConfiguration::InvalidStatus;
}
PreparedLifecycleConfiguration::Prepared(lifecycle_configuration_from_gateway(configuration, dates))
}
fn lifecycle_configuration_from_gateway(
value: dto::BucketLifecycleConfiguration,
dates: Option<&LifecycleRequestDates>,
) -> std::io::Result<PersistedLifecycleConfiguration> {
Ok(PersistedLifecycleConfiguration {
expiry_updated_at: date_from_gateway(value.expiry_updated_at, dates.and_then(LifecycleRequestDates::expiry_updated_at))?,
rules: value
.rules
.into_iter()
.enumerate()
.map(|(index, rule)| rule_from_gateway(rule, dates, index))
.collect::<std::io::Result<_>>()?,
})
}
fn rule_from_gateway(
value: dto::LifecycleRule,
dates: Option<&LifecycleRequestDates>,
rule_index: usize,
) -> std::io::Result<PersistedLifecycleRule> {
Ok(PersistedLifecycleRule {
abort_incomplete_multipart_upload: value.abort_incomplete_multipart_upload.map(|v| {
PersistedAbortIncompleteMultipartUpload {
days_after_initiation: v.days_after_initiation,
}
}),
del_marker_expiration: value
.del_marker_expiration
.map(|v| PersistedDelMarkerExpiration { days: v.days }),
expiration: value
.expiration
.map(|v| -> std::io::Result<PersistedLifecycleExpiration> {
Ok(PersistedLifecycleExpiration {
date: date_from_gateway(v.date, dates.and_then(|dates| dates.expiration(rule_index)))?,
days: v.days,
expired_object_all_versions: v.expired_object_all_versions,
expired_object_delete_marker: v.expired_object_delete_marker,
})
})
.transpose()?,
filter: value.filter.map(|v| PersistedLifecycleFilter {
and: v.and.map(|v| PersistedLifecycleAnd {
object_size_greater_than: v.object_size_greater_than,
object_size_less_than: v.object_size_less_than,
prefix: v.prefix,
tags: (!v.tags.is_empty()).then(|| v.tags.into_iter().map(tag_from_gateway).collect()),
}),
object_size_greater_than: v.object_size_greater_than,
object_size_less_than: v.object_size_less_than,
prefix: v.prefix,
tag: v.tag.map(tag_from_gateway),
}),
id: value.id,
noncurrent_version_expiration: value
.noncurrent_version_expiration
.map(|v| PersistedNoncurrentVersionExpiration {
newer_noncurrent_versions: v.newer_noncurrent_versions,
noncurrent_days: v.noncurrent_days,
}),
noncurrent_version_transitions: (!value.noncurrent_version_transitions.is_empty()).then(|| {
value
.noncurrent_version_transitions
.into_iter()
.map(|v| PersistedNoncurrentVersionTransition {
newer_noncurrent_versions: v.newer_noncurrent_versions,
noncurrent_days: v.noncurrent_days,
storage_class: v.storage_class.map(|v| v.as_str().to_owned()),
})
.collect()
}),
prefix: value.prefix,
status: value.status.as_str().to_owned(),
transitions: (!value.transitions.is_empty())
.then(|| {
value
.transitions
.into_iter()
.enumerate()
.map(|(index, v)| -> std::io::Result<PersistedTransition> {
Ok(PersistedTransition {
date: date_from_gateway(v.date, dates.and_then(|dates| dates.transition(rule_index, index)))?,
days: v.days,
storage_class: v.storage_class.map(|v| v.as_str().to_owned()),
})
})
.collect::<std::io::Result<_>>()
})
.transpose()?,
})
}
fn tag_from_gateway(value: dto::Tag) -> PersistedLifecycleTag {
PersistedLifecycleTag {
key: value.key,
value: value.value,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn prepare_lifecycle_configuration_checks_all_statuses_before_converting_dates() {
let mut configuration = dto::BucketLifecycleConfiguration {
rules: vec![
dto::LifecycleRule {
expiration: Some(dto::LifecycleExpiration {
date: Some(Timestamp::from_secs(i64::MAX)),
..Default::default()
}),
status: "Enabled".into(),
..Default::default()
},
dto::LifecycleRule {
status: "enabled".into(),
..Default::default()
},
],
..Default::default()
};
assert!(lifecycle_configuration_from_gateway(configuration.clone(), None).is_err());
assert!(matches!(
prepare_lifecycle_configuration(Some(configuration.clone()), None),
PreparedLifecycleConfiguration::InvalidStatus
));
configuration.rules[1].status = "Disabled".into();
assert!(matches!(
prepare_lifecycle_configuration(Some(configuration), None),
PreparedLifecycleConfiguration::Prepared(Err(_))
));
}
}
@@ -34,6 +34,7 @@ mod bucket_access;
mod bucket_configs;
mod bucket_core;
mod bucket_replication_lifecycle;
mod lifecycle_input;
mod listing_multipart;
mod object_commit;
mod object_meta;
+1
View File
@@ -4437,6 +4437,7 @@ impl S3Access for FS {
/// parity tests, which live next to the envelope's facts impl.
#[cfg(test)]
pub(crate) mod parity_test_support {
pub(crate) use super::envelope_payloads::bucket_config_mutation;
use super::{Action, Arc, CreateBucketInput, S3Request, S3Result};
use super::{AuthorizationRequest, AuthzFacts, AuthzPrincipal, AuthzTarget, ECStore, HashMap};
@@ -20,6 +20,7 @@
//! owns this file and is the only one that edits it.
use crate::admin::handlers::site_replication::site_replication_bucket_meta_hook;
use crate::app::RequestEnvelope;
use crate::app::storage_api::object_usecase::bucket::replication::OperatorRuleContract;
use crate::error::ApiError;
use crate::storage::StorageVersioningConfigExt as _;
@@ -27,7 +28,7 @@ use crate::storage::access::bucket_config_mutation_incarnation;
use crate::storage::ecfs::*;
use crate::storage::helper::audit_only_operation;
use crate::storage::persisted_from_s3s;
use crate::storage::s3_api;
use crate::storage::s3_api::{self, PreparedLifecycleConfiguration, validate_lifecycle_rule_status};
use crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s;
use crate::storage::s3_compat::replication::replication_configuration_from_s3s;
use crate::storage::storage_api::ecfs_consumer::contract::bucket::{BucketOperations, BucketOptions};
@@ -327,16 +328,27 @@ impl FS {
req: S3Request<PutBucketLifecycleConfigurationInput>,
) -> S3Result<S3Response<PutBucketLifecycleConfigurationOutput>> {
let usecase = s3_api::bucket_usecase_for(self);
let prepared = req
.input
.lifecycle_configuration
.clone()
.map(lifecycle_configuration_from_s3s)
.transpose();
audit_only_operation(req, S3Operation::PutBucketLifecycleConfiguration, |req| {
usecase.execute_put_bucket_lifecycle_configuration(req, prepared)
})
.await
let envelope = RequestEnvelope::from_s3s(&req);
let PutBucketLifecycleConfigurationInput {
bucket,
lifecycle_configuration,
..
} = req.input;
let prepared = match lifecycle_configuration {
None => PreparedLifecycleConfiguration::Missing,
Some(configuration) => {
if validate_lifecycle_rule_status(configuration.rules.iter().map(|rule| rule.status.as_str())).is_err() {
PreparedLifecycleConfiguration::InvalidStatus
} else {
PreparedLifecycleConfiguration::Prepared(lifecycle_configuration_from_s3s(configuration))
}
}
};
record_s3_op(S3Operation::PutBucketLifecycleConfiguration);
usecase
.execute_put_bucket_lifecycle_configuration(envelope, &bucket, prepared)
.await?;
Ok(S3Response::new(PutBucketLifecycleConfigurationOutput::default()))
}
pub(in crate::storage::ecfs) async fn put_bucket_replication_edge(
+1
View File
@@ -18,6 +18,7 @@
//! until each helper is moved with dedicated small refactor steps.
use crate::app::bucket_usecase::DefaultBucketUsecase;
pub(crate) use crate::app::bucket_usecase::{PreparedLifecycleConfiguration, validate_lifecycle_rule_status};
use crate::app::multipart_usecase::DefaultMultipartUsecase;
pub(crate) use crate::app::object::reject_veeam_sosapi_mutation;
pub(crate) use crate::app::object::shared::mark_on_demand_migration_list_local_only;
@@ -19,7 +19,8 @@
//! these tests do not depend on another increment being served.
use super::{
GATEWAY, LEGACY, MatrixRow, Outcome, Refusal, USER_SECRET_KEY, create_bucket_in_the_store, outcome, s3_client, with_stack,
GATEWAY, LEGACY, MatrixRow, Outcome, Refusal, USER_SECRET_KEY, create_bucket_in_the_store, create_user_allowed_on_one_bucket,
outcome, s3_client, with_stack,
};
use aws_sdk_s3::Client;
use futures::future::BoxFuture;
@@ -32,6 +33,12 @@ use rustfs_policy::policy::Policy;
use rustfs_s3_ops::S3Operation;
pub(crate) const MATRIX: &[MatrixRow] = &[
MatrixRow {
operation: "PutBucketLifecycleConfiguration",
metric: S3Operation::PutBucketLifecycleConfiguration,
has_error_body: true,
call: put_lifecycle_matrix,
},
MatrixRow {
operation: "GetBucketLifecycleConfiguration",
metric: S3Operation::GetBucketLifecycleConfiguration,
@@ -277,3 +284,237 @@ fn the_gateway_stack_answers_the_configuration_reads_as_stored() {
fn the_legacy_stack_answers_the_configuration_reads_as_stored() {
with_stack(LEGACY, the_configuration_reads_answer_what_is_stored);
}
/// The ordinary and the reserved-shaped bucket the mutations write to.
const MUTATED_BUCKETS: [&str; 2] = ["a3-mutated", "sthree-a3-mutated"];
/// A bucket the writer below is allowed on, so its refusals elsewhere are about the resource.
const WRITER_BUCKET: &str = "a3-writer-own";
const WRITER_USER: &str = "gatewaystacka3writer";
/// The tier the lifecycle documents transition to; registered as a mock backend.
const TIER: &str = "WARM";
/// Replaces the body the SDK serialized with `document`, as a client that writes its own XML
/// sends it: the SDK's checksum headers go, a Content-MD5 of `document` is set, and the request
/// is signed after the change.
fn send_document(request: &mut aws_smithy_runtime_api::http::Request, document: &'static str) {
use md5::Digest as _;
for name in [
"content-md5",
"x-amz-sdk-checksum-algorithm",
"x-amz-checksum-crc32",
"x-amz-checksum-crc32c",
] {
request.headers_mut().remove(name);
}
let md5 = base64_simd::STANDARD.encode_to_string(md5::Md5::digest(document.as_bytes()));
request.headers_mut().insert("content-md5", md5);
request.headers_mut().insert("content-length", document.len().to_string());
*request.body_mut() = aws_smithy_types::body::SdkBody::from(document);
}
fn lifecycle_document() -> aws_sdk_s3::types::BucketLifecycleConfiguration {
use aws_sdk_s3::types::{BucketLifecycleConfiguration, ExpirationStatus, LifecycleRule, LifecycleRuleFilter};
let rule = LifecycleRule::builder()
.status(ExpirationStatus::Disabled)
.filter(LifecycleRuleFilter::builder().prefix("placeholder/").build())
.build()
.expect("placeholder rule");
BucketLifecycleConfiguration::builder()
.rules(rule)
.build()
.expect("placeholder configuration")
}
/// A PutBucketLifecycleConfiguration carrying `document` as its body.
async fn put_lifecycle(client: &Client, bucket: &str, document: &'static str) -> Outcome {
let put = client
.put_bucket_lifecycle_configuration()
.bucket(bucket)
.lifecycle_configuration(lifecycle_document())
.customize()
.mutate_request(move |request| send_document(request, document));
outcome(put.send().await)
}
/// The lifecycle bytes the store holds for `bucket`, without the expiry stamp: the stamp is the
/// wall clock of the write whenever the document holds an expiry action.
async fn stored_lifecycle(bucket: &str) -> String {
let metadata = metadata_sys::get(bucket).await.expect("bucket metadata");
let stored = String::from_utf8(metadata.lifecycle_config_xml.clone()).expect("stored lifecycle is UTF-8");
match (stored.find("<ExpiryUpdatedAt>"), stored.find("</ExpiryUpdatedAt>")) {
(Some(start), Some(end)) => format!("{}{}", &stored[..start], &stored[end + "</ExpiryUpdatedAt>".len()..]),
_ => stored,
}
}
/// Every member a stored lifecycle rule can carry without an expiry action, two rules without an
/// ID, a dated transition, and a Filter with every predicate.
const EVERY_MEMBER: &str = "<LifecycleConfiguration><Rule><Status>Enabled</Status><Filter><And><Prefix>logs/</Prefix>\
<Tag><Key>tier</Key><Value>cold</Value></Tag><ObjectSizeGreaterThan>1</ObjectSizeGreaterThan><ObjectSizeLessThan>99\
</ObjectSizeLessThan></And></Filter><Transition><Days>10</Days><StorageClass>WARM</StorageClass></Transition>\
<NoncurrentVersionTransition><NoncurrentDays>3</NoncurrentDays><NewerNoncurrentVersions>2</NewerNoncurrentVersions>\
<StorageClass>WARM</StorageClass></NoncurrentVersionTransition>\
</Rule><Rule><ID>rule-1</ID><Status>Disabled</Status><Prefix>old/</Prefix><Transition><Date>2099-01-01T00:00:00Z</Date>\
<StorageClass>WARM</StorageClass></Transition><AbortIncompleteMultipartUpload><DaysAfterInitiation>3</DaysAfterInitiation>\
</AbortIncompleteMultipartUpload></Rule><Rule><Status>Disabled</Status><Filter><Tag><Key>k</Key><Value>v</Value></Tag>\
</Filter><Transition><Days>1</Days><StorageClass>WARM</StorageClass></Transition></Rule><Rule><ID>markers</ID>\
<Status>Disabled</Status><Filter><Prefix>markers/</Prefix></Filter><DelMarkerExpiration><Days>4</Days>\
</DelMarkerExpiration></Rule></LifecycleConfiguration>";
/// A midnight in the local time of a +08:00 offset.
const OFFSET_MIDNIGHT: &str = "<LifecycleConfiguration><Rule><ID>offset</ID><Status>Disabled</Status><Filter><Prefix>a/\
</Prefix></Filter><Expiration><Date>2099-01-01T00:00:00+08:00</Date></Expiration></Rule></LifecycleConfiguration>";
/// A midnight in UTC, written in the local time of a +08:00 offset.
const OFFSET_UTC_MIDNIGHT: &str = "<LifecycleConfiguration><Rule><ID>offset</ID><Status>Disabled</Status><Filter><Prefix>a/\
</Prefix></Filter><Expiration><Date>2099-01-01T08:00:00+08:00</Date></Expiration></Rule></LifecycleConfiguration>";
const OFFSET_TRANSITION: &str = "<LifecycleConfiguration><Rule><ID>days</ID><Status>Disabled</Status><Filter><Prefix>new/</Prefix>\
</Filter><Transition><Days>10</Days><StorageClass>WARM</StorageClass></Transition></Rule><Rule><ID>date</ID><Status>Disabled</Status>\
<Filter><Prefix>old/</Prefix></Filter><Transition><Date>1970-01-01T00:00:00-08:00</Date><StorageClass>WARM</StorageClass>\
</Transition></Rule></LifecycleConfiguration>";
const NEGATIVE_TRANSITION_INSTANT: &str = "<LifecycleConfiguration><Rule><ID>date</ID><Status>Disabled</Status><Filter><Prefix>old/</Prefix>\
</Filter><Transition><Date>1970-01-01T00:00:00+08:00</Date><StorageClass>WARM</StorageClass></Transition></Rule></LifecycleConfiguration>";
const OFFSET_SUB_MILLISECOND: &str = "<LifecycleConfiguration><Rule><ID>fraction</ID><Status>Disabled</Status><Filter><Prefix>a/\
</Prefix></Filter><Expiration><Date>2099-01-01T00:00:00.0001+08:00</Date></Expiration></Rule></LifecycleConfiguration>";
const SUB_MILLISECOND: &str = "<LifecycleConfiguration><Rule><ID>fraction</ID><Status>Disabled</Status><Filter><Prefix>a/\
</Prefix></Filter><Expiration><Date>2099-01-01T00:00:00.0001Z</Date></Expiration></Rule></LifecycleConfiguration>";
const MALFORMED: &str = "<LifecycleConfiguration><Rule><Status>Enabled</Status>";
const LOWERCASE_STATUS: &str = "<LifecycleConfiguration><Rule><ID>r</ID><Status>enabled</Status><Filter><Prefix>a/</Prefix>\
</Filter><Expiration><Days>1</Days></Expiration></Rule></LifecycleConfiguration>";
const ZERO_DAYS: &str = "<LifecycleConfiguration><Rule><ID>r</ID><Status>Enabled</Status><Filter><Prefix>a/</Prefix>\
</Filter><Expiration><Days>0</Days></Expiration></Rule></LifecycleConfiguration>";
const UNKNOWN_TIER: &str = "<LifecycleConfiguration><Rule><ID>r</ID><Status>Enabled</Status><Filter><Prefix>a/</Prefix>\
</Filter><Transition><Days>1</Days><StorageClass>NOPE</StorageClass></Transition></Rule></LifecycleConfiguration>";
const TWO_PREDICATES: &str = "<LifecycleConfiguration><Rule><ID>r</ID><Status>Enabled</Status><Filter><Prefix>a/</Prefix>\
<Tag><Key>k</Key><Value>v</Value></Tag></Filter><Expiration><Days>1</Days></Expiration></Rule></LifecycleConfiguration>";
const NO_RULE: &str = "<LifecycleConfiguration></LifecycleConfiguration>";
const EVERY_MEMBER_STORED: &str = "<LifecycleConfiguration><Rule><Filter><And><ObjectSizeGreaterThan>1</ObjectSizeGreaterThan><ObjectSizeLessThan>99</ObjectSizeLessThan><Prefix>logs/</Prefix><Tag><Key>tier</Key><Value>cold</Value></Tag></And></Filter><ID>rule-0</ID><NoncurrentVersionTransition><NewerNoncurrentVersions>2</NewerNoncurrentVersions><NoncurrentDays>3</NoncurrentDays><StorageClass>WARM</StorageClass></NoncurrentVersionTransition><Status>Enabled</Status><Transition><Days>10</Days><StorageClass>WARM</StorageClass></Transition></Rule><Rule><AbortIncompleteMultipartUpload><DaysAfterInitiation>3</DaysAfterInitiation></AbortIncompleteMultipartUpload><ID>rule-1</ID><Prefix>old/</Prefix><Status>Disabled</Status><Transition><Date>2099-01-01T00:00:00.000Z</Date><StorageClass>WARM</StorageClass></Transition></Rule><Rule><Filter><Tag><Key>k</Key><Value>v</Value></Tag></Filter><ID>rule-2</ID><Status>Disabled</Status><Transition><Days>1</Days><StorageClass>WARM</StorageClass></Transition></Rule><Rule><DelMarkerExpiration><Days>4</Days></DelMarkerExpiration><Filter><Prefix>markers/</Prefix></Filter><ID>markers</ID><Status>Disabled</Status></Rule></LifecycleConfiguration>";
const OFFSET_MIDNIGHT_STORED: &str = "<LifecycleConfiguration><Rule><Expiration><Date>2099-01-01T00:00:00.000Z</Date></Expiration><Filter><Prefix>a/</Prefix></Filter><ID>offset</ID><Status>Disabled</Status></Rule></LifecycleConfiguration>";
const OFFSET_TRANSITION_STORED: &str = "<LifecycleConfiguration><Rule><Filter><Prefix>new/</Prefix></Filter><ID>days</ID><Status>Disabled</Status><Transition><Days>10</Days><StorageClass>WARM</StorageClass></Transition></Rule><Rule><Filter><Prefix>old/</Prefix></Filter><ID>date</ID><Status>Disabled</Status><Transition><Date>1970-01-01T00:00:00.000Z</Date><StorageClass>WARM</StorageClass></Transition></Rule></LifecycleConfiguration>";
const VALID_DAY: &str = "<LifecycleConfiguration><Rule><ID>expiry</ID><Status>Disabled</Status><Filter><Prefix>logs/</Prefix></Filter><Expiration><Days>30</Days></Expiration></Rule></LifecycleConfiguration>";
async fn lifecycle_put_preserves_responses_and_storage(server: RustFSServer) {
for bucket in MUTATED_BUCKETS.into_iter().chain([WRITER_BUCKET]) {
create_bucket_in_the_store(bucket).await;
}
rustfs_ecstore::api::tier::test_util::register_mock_tier(&rustfs_ecstore::api::runtime::global_tier_config_mgr(), TIER).await;
create_user_allowed_on_one_bucket(WRITER_USER, "s3:PutBucketLifecycle", WRITER_BUCKET).await;
let root = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
let writer = s3_client(&server.endpoint(), WRITER_USER, USER_SECRET_KEY);
assert_eq!(
answer(put_lifecycle(&root, MISSING_BUCKET, EVERY_MEMBER).await),
refused("NoSuchBucket", 404)
);
for bucket in MUTATED_BUCKETS {
for (label, document, code) in [
("malformed", MALFORMED, "MalformedXML"),
("lowercase", LOWERCASE_STATUS, "MalformedXML"),
("zero", ZERO_DAYS, "InvalidArgument"),
("unknown tier", UNKNOWN_TIER, "InvalidArgument"),
("two predicates", TWO_PREDICATES, "MalformedXML"),
("no rule", NO_RULE, "MalformedXML"),
("offset utc midnight", OFFSET_UTC_MIDNIGHT, "InvalidArgument"),
("sub-millisecond", SUB_MILLISECOND, "InvalidArgument"),
("offset sub-millisecond", OFFSET_SUB_MILLISECOND, "InvalidArgument"),
("negative transition instant", NEGATIVE_TRANSITION_INSTANT, "InvalidArgument"),
] {
let before = stored_lifecycle(bucket).await;
assert_eq!(
answer(put_lifecycle(&root, bucket, document).await),
refused(code, 400),
"{bucket} {label}"
);
assert_eq!(stored_lifecycle(bucket).await, before, "{bucket} {label} must not store");
}
let before = stored_lifecycle(bucket).await;
assert_eq!(answer(put_lifecycle(&writer, bucket, EVERY_MEMBER).await), refused("AccessDenied", 403));
assert_eq!(stored_lifecycle(bucket).await, before, "a wrong-resource refusal must not store");
assert_eq!(put_lifecycle(&root, bucket, OFFSET_MIDNIGHT).await, Ok(()));
assert_eq!(stored_lifecycle(bucket).await, OFFSET_MIDNIGHT_STORED);
assert_eq!(put_lifecycle(&root, bucket, OFFSET_TRANSITION).await, Ok(()));
assert_eq!(stored_lifecycle(bucket).await, OFFSET_TRANSITION_STORED);
assert_eq!(put_lifecycle(&root, bucket, EVERY_MEMBER).await, Ok(()));
assert_eq!(stored_lifecycle(bucket).await, EVERY_MEMBER_STORED);
}
assert_eq!(put_lifecycle(&writer, WRITER_BUCKET, EVERY_MEMBER).await, Ok(()));
assert_eq!(stored_lifecycle(WRITER_BUCKET).await, EVERY_MEMBER_STORED);
server.shutdown().await;
}
#[test]
fn legacy_lifecycle_put_preserves_responses_and_storage() {
with_stack(LEGACY, lifecycle_put_preserves_responses_and_storage);
}
#[test]
fn gateway_lifecycle_put_preserves_responses_and_storage() {
with_stack(GATEWAY, lifecycle_put_preserves_responses_and_storage);
}
#[test]
fn gateway_lifecycle_put_accepts_owned_configuration() {
with_stack(GATEWAY, |server| async move {
create_bucket_in_the_store(WRITER_BUCKET).await;
let client = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
assert_eq!(put_lifecycle(&client, WRITER_BUCKET, VALID_DAY).await, Ok(()));
server.shutdown().await;
});
}
fn put_lifecycle_matrix<'a>(client: &'a Client, bucket: &'a str) -> BoxFuture<'a, Outcome> {
Box::pin(put_lifecycle(client, bucket, VALID_DAY))
}
async fn lifecycle_put_authorization_uses_action_and_bucket(server: RustFSServer) {
for bucket in [WRITER_BUCKET, MUTATED_BUCKETS[0]] {
create_bucket_in_the_store(bucket).await;
}
create_user_allowed_on_one_bucket(WRITER_USER, "s3:PutBucketLifecycle", WRITER_BUCKET).await;
let writer = s3_client(&server.endpoint(), WRITER_USER, USER_SECRET_KEY);
assert_eq!(put_lifecycle(&writer, WRITER_BUCKET, VALID_DAY).await, Ok(()));
assert!(!stored_lifecycle(WRITER_BUCKET).await.is_empty());
assert_eq!(
answer(put_lifecycle(&writer, MUTATED_BUCKETS[0], VALID_DAY).await),
refused("AccessDenied", 403)
);
assert!(stored_lifecycle(MUTATED_BUCKETS[0]).await.is_empty());
server.shutdown().await;
}
#[test]
fn gateway_lifecycle_put_authorization_uses_action_and_bucket() {
with_stack(GATEWAY, lifecycle_put_authorization_uses_action_and_bucket);
}
#[test]
fn legacy_lifecycle_put_authorization_uses_action_and_bucket() {
with_stack(LEGACY, lifecycle_put_authorization_uses_action_and_bucket);
}
#[test]
fn gateway_lifecycle_put_keeps_transition_fields() {
with_stack(GATEWAY, |server| async move {
create_bucket_in_the_store(WRITER_BUCKET).await;
rustfs_ecstore::api::tier::test_util::register_mock_tier(&rustfs_ecstore::api::runtime::global_tier_config_mgr(), TIER)
.await;
let root = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
assert_eq!(put_lifecycle(&root, WRITER_BUCKET, EVERY_MEMBER).await, Ok(()));
assert_eq!(stored_lifecycle(WRITER_BUCKET).await, EVERY_MEMBER_STORED);
server.shutdown().await;
});
}
#[test]
fn gateway_lifecycle_put_rejects_submillisecond_date() {
with_stack(GATEWAY, |server| async move {
create_bucket_in_the_store(WRITER_BUCKET).await;
let root = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
assert_eq!(
answer(put_lifecycle(&root, WRITER_BUCKET, SUB_MILLISECOND).await),
refused("InvalidArgument", 400)
);
assert!(stored_lifecycle(WRITER_BUCKET).await.is_empty());
server.shutdown().await;
});
}