fix(object-lock): materialize default retention metadata (#2824)

Signed-off-by: houseme <housemecn@gmail.com>
Co-authored-by: cxymds <Cxymds@qq.com>
Co-authored-by: loverustfs <hello@rustfs.com>
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
GatewayJ
2026-05-06 21:48:12 +08:00
committed by GitHub
parent 9f07029373
commit 9e93d3f47a
5 changed files with 743 additions and 33 deletions
+2
View File
@@ -505,6 +505,7 @@ impl DefaultMultipartUsecase {
metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags);
}
let has_explicit_object_lock_retention = object_lock_mode.is_some() || object_lock_retain_until_date.is_some();
if let Some(object_lock_metadata) = build_put_like_object_lock_metadata(
&bucket,
object_lock_legal_hold_status,
@@ -515,6 +516,7 @@ impl DefaultMultipartUsecase {
{
metadata.extend(object_lock_metadata);
}
apply_bucket_default_lock_retention(&bucket, &mut metadata, has_explicit_object_lock_retention).await?;
let encryption_request = PrepareEncryptionRequest {
bucket: &bucket,
+47 -7
View File
@@ -920,6 +920,11 @@ async fn apply_put_request_object_lock_opts(
Ok(())
}
// Shared across Object Lock validation paths to keep the client-facing
// InvalidRequest message consistent.
pub(crate) const ERR_OBJECT_LOCK_RETENTION_HEADERS_MUST_BE_PAIRED: &str =
"x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied";
pub(crate) async fn build_put_like_object_lock_metadata(
bucket: &str,
object_lock_legal_hold_status: Option<ObjectLockLegalHoldStatus>,
@@ -930,20 +935,22 @@ pub(crate) async fn build_put_like_object_lock_metadata(
return Ok(None);
}
validate_bucket_object_lock_enabled(bucket).await?;
let retention = match (object_lock_mode, object_lock_retain_until_date) {
(Some(mode), retain_until_date) => Some(ObjectLockRetention {
(Some(mode), Some(retain_until_date)) => Some(ObjectLockRetention {
mode: Some(ObjectLockRetentionMode::from(mode.as_str().to_string())),
retain_until_date,
}),
(None, Some(retain_until_date)) => Some(ObjectLockRetention {
mode: None,
retain_until_date: Some(retain_until_date),
}),
(Some(_), None) | (None, Some(_)) => {
return Err(S3Error::with_message(
S3ErrorCode::InvalidRequest,
ERR_OBJECT_LOCK_RETENTION_HEADERS_MUST_BE_PAIRED.to_string(),
));
}
(None, None) => None,
};
validate_bucket_object_lock_enabled(bucket).await?;
let mut eval_metadata = parse_object_lock_retention(retention)?;
eval_metadata.extend(parse_object_lock_legal_hold(
object_lock_legal_hold_status.map(|status| ObjectLockLegalHold { status: Some(status) }),
@@ -1792,6 +1799,7 @@ impl DefaultObjectUsecase {
)?;
let mut metadata = metadata.unwrap_or_default();
let has_explicit_object_lock_retention = object_lock_mode.is_some() || object_lock_retain_until_date.is_some();
apply_put_request_metadata(
&mut metadata,
&req.headers,
@@ -1806,6 +1814,7 @@ impl DefaultObjectUsecase {
tagging,
storage_class.clone(),
)?;
apply_bucket_default_lock_retention(&bucket, &mut metadata, has_explicit_object_lock_retention).await?;
let mut opts: ObjectOptions = put_opts(&bucket, &key, version_id.clone(), &req.headers, metadata.clone())
.await
@@ -2759,6 +2768,8 @@ impl DefaultObjectUsecase {
}
}
let has_explicit_object_lock_retention = object_lock_mode.is_some() || object_lock_retain_until_date.is_some();
remove_object_lock_metadata_for_copy(&mut src_info.user_defined);
if let Some(object_lock_metadata) = build_put_like_object_lock_metadata(
&bucket,
object_lock_legal_hold_status,
@@ -2769,6 +2780,7 @@ impl DefaultObjectUsecase {
{
src_info.user_defined.extend(object_lock_metadata);
}
apply_bucket_default_lock_retention(&bucket, &mut src_info.user_defined, has_explicit_object_lock_retention).await?;
let mut reader = match decryption_material {
Some(material) => {
@@ -4292,6 +4304,7 @@ impl DefaultObjectUsecase {
.ok()
.and_then(|modified_at_secs| OffsetDateTime::from_unix_timestamp(modified_at_secs as i64).ok());
let mut metadata = HashMap::new();
let has_explicit_object_lock_retention = object_lock_mode.is_some() || object_lock_retain_until_date.is_some();
apply_put_request_metadata(
&mut metadata,
&req.headers,
@@ -4306,6 +4319,7 @@ impl DefaultObjectUsecase {
tagging.clone(),
storage_class.clone(),
)?;
apply_bucket_default_lock_retention(&bucket, &mut metadata, has_explicit_object_lock_retention).await?;
let mut opts = put_opts(&bucket, &fpath, None, &req.headers, metadata.clone())
.await
.map_err(ApiError::from)?;
@@ -4510,6 +4524,32 @@ mod tests {
}
}
#[tokio::test]
async fn build_put_like_object_lock_metadata_rejects_mode_without_retain_until_date() {
let err = build_put_like_object_lock_metadata(
"test-bucket",
None,
Some(ObjectLockMode::from_static(ObjectLockMode::GOVERNANCE)),
None,
)
.await
.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
assert_eq!(err.message(), Some(ERR_OBJECT_LOCK_RETENTION_HEADERS_MUST_BE_PAIRED));
}
#[tokio::test]
async fn build_put_like_object_lock_metadata_rejects_retain_until_date_without_mode() {
let retain_until = Timestamp::from(OffsetDateTime::now_utc().add(time::Duration::days(1)));
let err = build_put_like_object_lock_metadata("test-bucket", None, None, Some(retain_until))
.await
.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
assert_eq!(err.message(), Some(ERR_OBJECT_LOCK_RETENTION_HEADERS_MUST_BE_PAIRED));
}
#[test]
fn is_put_object_extract_requested_accepts_meta_header() {
let mut headers = HeaderMap::new();
+87 -16
View File
@@ -30,7 +30,7 @@ use rustfs_targets::EventName;
use rustfs_targets::arn::{TargetID, TargetIDError};
use rustfs_utils::http::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, insert_str,
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, contains_key_str, insert_str, remove_str,
};
use s3s::dto::{
Delimiter, LambdaFunctionConfiguration, NotificationConfigurationFilter, ObjectLockConfiguration, ObjectLockEnabled,
@@ -50,21 +50,41 @@ use tracing::{debug, warn};
pub const RFC1123: &[FormatItem<'_>] =
format_description!("[weekday repr:short], [day] [month repr:short] [year] [hour]:[minute]:[second] GMT");
fn format_object_lock_timestamp(timestamp: OffsetDateTime) -> String {
timestamp.format(&Rfc3339).unwrap_or_default()
}
fn has_object_lock_retention_metadata(metadata: &HashMap<String, String>) -> bool {
metadata.contains_key(AMZ_OBJECT_LOCK_MODE_LOWER) || metadata.contains_key(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER)
}
pub(crate) fn remove_object_lock_retention_metadata(metadata: &mut HashMap<String, String>) -> bool {
let removed_mode = metadata.remove(AMZ_OBJECT_LOCK_MODE_LOWER).is_some();
let removed_retain_until_date = metadata.remove(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER).is_some();
let removed_timestamp = contains_key_str(metadata, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP);
remove_str(metadata, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP);
removed_mode || removed_retain_until_date || removed_timestamp
}
fn remove_object_lock_legal_hold_metadata(metadata: &mut HashMap<String, String>) -> bool {
let removed_legal_hold = metadata.remove(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).is_some();
let removed_timestamp = contains_key_str(metadata, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP);
remove_str(metadata, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP);
removed_legal_hold || removed_timestamp
}
pub(crate) fn remove_object_lock_metadata_for_copy(metadata: &mut HashMap<String, String>) -> bool {
let removed_retention = remove_object_lock_retention_metadata(metadata);
let removed_legal_hold = remove_object_lock_legal_hold_metadata(metadata);
removed_retention || removed_legal_hold
}
/// Apply bucket default Object Lock retention to object metadata if no explicit retention is set.
///
/// This function implements S3-compatible behavior where objects uploaded to a bucket with
/// default retention configuration automatically inherit the bucket's default retention policy.
/// The retention is only applied if:
/// 1. The bucket has Object Lock enabled
/// 2. The bucket has a default retention rule configured
/// 3. The object metadata does not already contain explicit retention headers
///
/// # Arguments
/// * `object_lock_config` - Optional bucket Object Lock configuration. If None, no retention is applied.
/// * `metadata` - Mutable reference to object metadata HashMap. Retention headers are inserted here.
#[allow(dead_code)]
pub(crate) fn apply_lock_retention(object_lock_config: Option<ObjectLockConfiguration>, metadata: &mut HashMap<String, String>) {
if metadata.contains_key(AMZ_OBJECT_LOCK_MODE_LOWER) || metadata.contains_key(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER) {
if has_object_lock_retention_metadata(metadata) {
return;
}
@@ -87,9 +107,60 @@ pub(crate) fn apply_lock_retention(object_lock_config: Option<ObjectLockConfigur
if let Ok(date_str) = retain_until.format(&Rfc3339) {
metadata.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), mode.as_str().to_string());
metadata.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), date_str);
insert_str(metadata, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, format_object_lock_timestamp(now));
}
}
pub(crate) fn apply_default_lock_retention_metadata(
object_lock_configuration: Option<ObjectLockConfiguration>,
metadata: &mut HashMap<String, String>,
) -> bool {
if has_object_lock_retention_metadata(metadata) {
return false;
}
let mut default_retention_metadata = HashMap::new();
apply_lock_retention(object_lock_configuration, &mut default_retention_metadata);
if default_retention_metadata.is_empty() {
return false;
}
metadata.extend(default_retention_metadata);
true
}
pub(crate) async fn apply_bucket_default_lock_retention(
bucket: &str,
metadata: &mut HashMap<String, String>,
has_explicit_retention: bool,
) -> S3Result<()> {
if has_explicit_retention {
return Ok(());
}
if has_object_lock_retention_metadata(metadata) {
return Ok(());
}
let object_lock_configuration = match metadata_sys::get_object_lock_config(bucket).await {
Ok((cfg, _created)) => Some(cfg),
Err(err) => {
if err == StorageError::ConfigNotFound {
None
} else {
warn!("get_object_lock_config err {:?}", err);
return Err(S3Error::with_message(
S3ErrorCode::InternalError,
"Failed to load Object Lock configuration".to_string(),
));
}
}
};
apply_default_lock_retention_metadata(object_lock_configuration, metadata);
Ok(())
}
/// Calculate adaptive buffer size with workload profile support.
///
/// This enhanced version supports different workload profiles for optimal performance
@@ -301,7 +372,7 @@ pub(crate) fn parse_object_lock_retention(retention: Option<ObjectLockRetention>
insert_str(
&mut eval_metadata,
SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
format!("{}.{:09}Z", now.format(&Rfc3339).unwrap(), now.nanosecond()),
format_object_lock_timestamp(now),
);
}
Ok(eval_metadata)
@@ -328,7 +399,7 @@ pub(crate) fn parse_object_lock_legal_hold(legal_hold: Option<ObjectLockLegalHol
insert_str(
&mut eval_metadata,
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
format!("{}.{:09}Z", now.format(&Rfc3339).unwrap(), now.nanosecond()),
format_object_lock_timestamp(now),
);
}
Ok(eval_metadata)
+126 -8
View File
@@ -19,10 +19,11 @@ mod tests {
use crate::storage::ecfs::{FS, validate_object_lock_configuration_input};
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
use crate::storage::{
apply_cors_headers, check_preconditions, get_adaptive_buffer_size_with_profile, get_buffer_size_opt_in, is_etag_equal,
matches_origin_pattern, parse_etag, parse_object_lock_legal_hold, parse_object_lock_retention,
process_lambda_configurations, process_queue_configurations, process_topic_configurations,
validate_bucket_object_lock_enabled, validate_list_object_unordered_with_delimiter,
apply_cors_headers, apply_default_lock_retention_metadata, check_preconditions, get_adaptive_buffer_size_with_profile,
get_buffer_size_opt_in, is_etag_equal, matches_origin_pattern, parse_etag, parse_object_lock_legal_hold,
parse_object_lock_retention, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
remove_object_lock_metadata_for_copy, remove_object_lock_retention_metadata, validate_bucket_object_lock_enabled,
validate_list_object_unordered_with_delimiter,
};
use http::{Extensions, HeaderMap, HeaderValue, Method, StatusCode, Uri};
use rustfs_config::MI_B;
@@ -30,8 +31,8 @@ mod tests {
use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
use rustfs_ecstore::store_api::ObjectInfo;
use rustfs_utils::http::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
contains_key_str,
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, contains_key_str, get_str, insert_str,
};
use rustfs_zip::CompressionFormat;
use s3s::dto::{
@@ -43,7 +44,7 @@ mod tests {
TopicConfiguration,
};
use s3s::{S3, S3Error, S3ErrorCode, S3Request, s3_error};
use time::OffsetDateTime;
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
fn build_request<T>(input: T, method: Method) -> S3Request<T> {
S3Request {
@@ -408,6 +409,116 @@ mod tests {
assert_eq!(err.code(), &S3ErrorCode::Custom("InvalidRetentionPeriod".into()));
}
#[test]
fn test_apply_default_lock_retention_metadata_applies_bucket_default() {
use std::collections::HashMap;
let cfg = ObjectLockConfiguration {
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
rule: Some(ObjectLockRule {
default_retention: Some(DefaultRetention {
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)),
days: Some(1),
years: None,
}),
}),
};
let mut metadata = HashMap::new();
assert!(apply_default_lock_retention_metadata(Some(cfg), &mut metadata));
assert_eq!(metadata.get(AMZ_OBJECT_LOCK_MODE_LOWER), Some(&"COMPLIANCE".to_string()));
let retain_until = metadata
.get(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER)
.and_then(|value| OffsetDateTime::parse(value, &Rfc3339).ok())
.expect("default retention should write a valid retain-until date");
assert!(retain_until > OffsetDateTime::now_utc());
let retention_timestamp = get_str(&metadata, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP)
.and_then(|value| OffsetDateTime::parse(&value, &Rfc3339).ok())
.expect("default retention should write a valid internal timestamp");
assert!(retention_timestamp <= OffsetDateTime::now_utc());
}
#[test]
fn test_apply_default_lock_retention_metadata_preserves_explicit_retention() {
use std::collections::HashMap;
let cfg = ObjectLockConfiguration {
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
rule: Some(ObjectLockRule {
default_retention: Some(DefaultRetention {
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)),
days: Some(1),
years: None,
}),
}),
};
let mut metadata = HashMap::from([
(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "GOVERNANCE".to_string()),
(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2030-01-01T00:00:00Z".to_string()),
]);
assert!(!apply_default_lock_retention_metadata(Some(cfg), &mut metadata));
assert_eq!(metadata.get(AMZ_OBJECT_LOCK_MODE_LOWER), Some(&"GOVERNANCE".to_string()));
assert_eq!(
metadata.get(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER),
Some(&"2030-01-01T00:00:00Z".to_string())
);
}
#[test]
fn test_apply_default_lock_retention_metadata_ignores_bucket_without_default() {
use std::collections::HashMap;
let cfg = ObjectLockConfiguration {
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
rule: None,
};
let mut metadata = HashMap::new();
assert!(!apply_default_lock_retention_metadata(Some(cfg), &mut metadata));
assert!(metadata.is_empty());
}
#[test]
fn test_remove_object_lock_retention_metadata_clears_only_retention_fields() {
use std::collections::HashMap;
let mut metadata = HashMap::from([
(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "GOVERNANCE".to_string()),
(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2030-01-01T00:00:00Z".to_string()),
(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "ON".to_string()),
]);
insert_str(&mut metadata, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, "2026-01-01T00:00:00Z".to_string());
assert!(remove_object_lock_retention_metadata(&mut metadata));
assert!(!metadata.contains_key(AMZ_OBJECT_LOCK_MODE_LOWER));
assert!(!metadata.contains_key(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER));
assert!(!contains_key_str(&metadata, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP));
assert_eq!(metadata.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER), Some(&"ON".to_string()));
}
#[test]
fn test_remove_object_lock_metadata_for_copy_clears_retention_and_legal_hold() {
use std::collections::HashMap;
let mut metadata = HashMap::from([
(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "GOVERNANCE".to_string()),
(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2030-01-01T00:00:00Z".to_string()),
(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "ON".to_string()),
("content-type".to_string(), "application/octet-stream".to_string()),
]);
insert_str(&mut metadata, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, "2026-01-01T00:00:00Z".to_string());
insert_str(&mut metadata, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, "2026-01-01T00:00:00Z".to_string());
assert!(remove_object_lock_metadata_for_copy(&mut metadata));
assert!(!metadata.contains_key(AMZ_OBJECT_LOCK_MODE_LOWER));
assert!(!metadata.contains_key(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER));
assert!(!metadata.contains_key(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER));
assert!(!contains_key_str(&metadata, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP));
assert!(!contains_key_str(&metadata, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP));
assert_eq!(metadata.get("content-type"), Some(&"application/octet-stream".to_string()));
}
#[tokio::test]
#[ignore = "requires isolated global object layer state"]
async fn test_get_object_tagging_returns_internal_error_when_store_uninitialized() {
@@ -736,6 +847,10 @@ mod tests {
"2030-01-01T00:00:00Z"
);
assert!(contains_key_str(&compliance_metadata, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP));
let retention_timestamp = get_str(&compliance_metadata, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP)
.and_then(|value| OffsetDateTime::parse(&value, &Rfc3339).ok())
.expect("explicit retention should write a valid internal timestamp");
assert!(retention_timestamp <= OffsetDateTime::now_utc());
// [3] Normal case: Retention with valid GOVERNANCE mode (future date)
let valid_governance_retention = ObjectLockRetention {
@@ -795,7 +910,10 @@ mod tests {
};
let on_metadata = parse_object_lock_legal_hold(Some(valid_on_legal_hold)).unwrap();
assert_eq!(on_metadata.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).unwrap(), "ON");
assert!(contains_key_str(&on_metadata, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP));
let legal_hold_timestamp = get_str(&on_metadata, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP)
.and_then(|value| OffsetDateTime::parse(&value, &Rfc3339).ok())
.expect("legal hold should write a valid internal timestamp");
assert!(legal_hold_timestamp <= OffsetDateTime::now_utc());
// [3] Normal case: Legal hold with valid OFF status
let valid_off_legal_hold = ObjectLockLegalHold {