gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-19 09:05:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(ecstore): read persisted object-metadata keys from the filemeta authority (A3b) (#7770)

Browse Source 
Migrate ecstore and rustfs-lifecycle consumers of the persisted xl.meta
meta_user keys (object lock, restore, replication status, storage class,
SSE read) from s3s::header / rustfs_utils header constants to
rustfs_filemeta::metadata_keys. HTTP header production is unchanged and
every lookup keeps its previous exact / case-insensitive mode.

- SSE key: persisted only as lowercase; the mixed-case spelling is
  outbound replication user metadata. Keep the case-insensitive read and
  pin it against the pre-A3a fixture.
- warm tier: strip promoted keys case-insensitively so the persisted
  X-Amz-Replication-Status is no longer forwarded to the tier.
- fix stale HashReader::add_checksum_from_s3s call in ecstore tests.

Refs rustfs/backlog#1735
This commit is contained in:
Chris
2026-09-14 02:12:36 +08:00
committed by GitHub
parent c1190c24e6
commit d99460ced0
26 changed files with 633 additions and 281 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+9 -9
View File
@@ -82,6 +82,7 @@ use rustfs_config::{
ENV_TRANSITION_WORKERS, ENV_TRANSITION_WORKERS_ABSOLUTE_MAX,
};
use rustfs_data_usage::TierStats;
use rustfs_filemeta::metadata_keys;
use rustfs_filemeta::{
FileInfo, FileInfoOpts, NULL_VERSION_ID, RestoreStatusOps, TRANSITION_COMPLETE, get_file_info, is_restored_object_on_disk,
};
@@ -93,7 +94,6 @@ use s3s::dto::{
BucketLifecycleConfiguration, ExpirationStatus, ObjectLockConfiguration, RestoreRequest, RestoreRequestType, RestoreStatus,
Timestamp,
};
use s3s::header::X_AMZ_RESTORE;
use sha2::{Digest, Sha256};
use std::any::Any;
use std::collections::{BTreeMap, HashMap, HashSet};
@@ -5174,7 +5174,7 @@ pub async fn put_restore_opts(
}
let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1));
meta.insert(
X_AMZ_RESTORE.as_str().to_string(),
metadata_keys::RESTORE.to_string(),
RestoreStatus {
is_restore_in_progress: Some(false),
restore_expiry_date: Some(Timestamp::from(restore_expiry)),
@@ -5912,6 +5912,7 @@ mod tests {
use rustfs_config::ENV_MAX_EXPIRY_WORKERS;
use rustfs_config::ENV_TRANSITION_WORKERS_ABSOLUTE_MAX;
use rustfs_data_usage::TierStats;
use rustfs_filemeta::metadata_keys;
use rustfs_filemeta::{FileInfo, FileMeta};
#[cfg(feature = "test-util")]
use rustfs_s3_client::transition_api::ReaderImpl;
@@ -5921,7 +5922,6 @@ mod tests {
NoncurrentVersionExpiration, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule,
OutputLocation, RestoreRequest, RestoreRequestType, S3Location, Timestamp, Transition, TransitionStorageClass,
};
use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
use serial_test::serial;
use sha2::{Digest, Sha256};
use std::collections::HashMap;
@@ -12173,7 +12173,7 @@ mod tests {
"locked historical null",
ObjectInfo {
user_defined: Arc::new(HashMap::from([(
X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(),
metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(),
"ON".to_string(),
)])),
..historical_null.clone()
@@ -12951,7 +12951,7 @@ mod tests {
let lc = latest_expiration_lifecycle();
let object = current_object_with_metadata(
ReplicationStatusType::Completed,
HashMap::from([(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), "ON".to_string())]),
HashMap::from([(metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(), "ON".to_string())]),
);
let event = eval_action_from_lifecycle(&lc, None, &object).await;
@@ -12969,10 +12969,10 @@ mod tests {
ReplicationStatusType::Completed,
HashMap::from([
(
X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_MODE.to_string(),
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
),
(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(), retain_until),
(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), retain_until),
]),
);
@@ -12994,10 +12994,10 @@ mod tests {
ReplicationStatusType::Completed,
HashMap::from([
(
X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_MODE.to_string(),
ObjectLockRetentionMode::COMPLIANCE.to_string(),
),
(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(), retain_until),
(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), retain_until),
]),
);
object.transitioned_object.status = TRANSITION_COMPLETE.to_string();
crates/ecstore/src/bucket/lifecycle/object_lock_boundary.rs
+1 -1
View File
@@ -37,7 +37,7 @@ mod tests {
#[test]
fn is_object_locked_by_metadata_preserves_object_lock_parser_behavior() {
let mut user_defined = HashMap::new();
user_defined.insert("x-amz-object-lock-legal-hold".to_string(), "ON".to_string());
user_defined.insert(rustfs_filemeta::metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(), "ON".to_string());
assert!(is_object_locked_by_metadata(&user_defined, false));
assert!(!is_object_locked_by_metadata(&user_defined, true));
crates/ecstore/src/bucket/object_lock/objectlock.rs
+4 -6
View File
@@ -13,9 +13,7 @@
// limitations under the License.
use super::types::{LegalHoldStatus, ObjectLegalHold, ObjectRetention, RetentionMode};
use rustfs_utils::http::headers::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
};
use rustfs_filemeta::metadata_keys;
use std::collections::HashMap;
use time::{OffsetDateTime, format_description};
@@ -35,7 +33,7 @@ pub fn utc_now_ntp() -> OffsetDateTime {
pub fn get_object_retention_meta(meta: &HashMap<String, String>) -> ObjectRetention {
// The persisted metadata keys are the lowercase wire header names.
let mode_str = meta.get(AMZ_OBJECT_LOCK_MODE_LOWER);
let mode_str = meta.get(metadata_keys::OBJECT_LOCK_MODE);
let Some(mode_str) = mode_str else {
return ObjectRetention::default();
@@ -46,7 +44,7 @@ pub fn get_object_retention_meta(meta: &HashMap<String, String>) -> ObjectRetent
return ObjectRetention::default();
};
let till_str = meta.get(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER);
let till_str = meta.get(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE);
let retain_until_date =
till_str.and_then(|s| OffsetDateTime::parse(s, &format_description::well_known::Iso8601::DEFAULT).ok());
@@ -58,7 +56,7 @@ pub fn get_object_retention_meta(meta: &HashMap<String, String>) -> ObjectRetent
}
pub fn get_object_legalhold_meta(meta: &HashMap<String, String>) -> ObjectLegalHold {
let hold_str = meta.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER);
let hold_str = meta.get(metadata_keys::OBJECT_LOCK_LEGAL_HOLD);
ObjectLegalHold {
status: hold_str.and_then(|s| parse_legalhold_status(s)),
crates/ecstore/src/bucket/object_lock/objectlock_sys.rs
+22 -27
View File
@@ -19,9 +19,7 @@ use crate::bucket::object_lock::objectlock;
use crate::bucket::object_lock::types::{DefaultRetention, LegalHoldStatus, RetentionMode};
use crate::error::{Error, Result, StorageError};
use crate::object_api::{ObjectInfo, ObjectOptions};
use rustfs_utils::http::headers::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
};
use rustfs_filemeta::metadata_keys;
use std::sync::Arc;
use time::OffsetDateTime;
@@ -317,7 +315,7 @@ fn persisted_lock_value<'a>(obj_info: &'a ObjectInfo, key: &str) -> Option<&'a S
/// Whether the version's persisted legal hold is ON. Any other non-empty
/// value than ON/OFF is malformed metadata and fails closed.
fn legal_hold_locks(obj_info: &ObjectInfo) -> Result<bool> {
let Some(status) = persisted_lock_value(obj_info, AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER) else {
let Some(status) = persisted_lock_value(obj_info, metadata_keys::OBJECT_LOCK_LEGAL_HOLD) else {
return Ok(false);
};
match LegalHoldStatus::parse(status) {
@@ -335,8 +333,8 @@ fn active_retention(
default_retention: Option<&DefaultRetention>,
obj_info: &ObjectInfo,
) -> Result<Option<(RetentionMode, OffsetDateTime)>> {
let mode = persisted_lock_value(obj_info, AMZ_OBJECT_LOCK_MODE_LOWER);
let retain_until = persisted_lock_value(obj_info, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER);
let mode = persisted_lock_value(obj_info, metadata_keys::OBJECT_LOCK_MODE);
let retain_until = persisted_lock_value(obj_info, metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE);
match (mode, retain_until) {
(None, None) => {}
(Some(mode), Some(retain_until)) => {
@@ -413,9 +411,6 @@ pub async fn check_object_lock_for_deletion(
mod tests {
use super::*;
use crate::bucket::metadata_sys::configured_object_lock_state_for_tests;
use rustfs_utils::http::headers::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
};
use time::{Date, Month, PrimitiveDateTime, Time};
fn make_datetime(year: i32, month: u8, day: u8) -> OffsetDateTime {
@@ -509,7 +504,7 @@ mod tests {
#[test]
fn deletion_rejects_incomplete_persisted_retention_metadata() {
let mut user_defined = std::collections::HashMap::new();
user_defined.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), RetentionMode::COMPLIANCE.to_string());
user_defined.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), RetentionMode::COMPLIANCE.to_string());
let obj_info = ObjectInfo {
user_defined: Arc::new(user_defined),
..Default::default()
@@ -535,10 +530,10 @@ mod tests {
for (case, mode, retain_until, expected) in cases {
let mut user_defined = std::collections::HashMap::new();
if let Some(mode) = mode {
user_defined.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), mode.to_string());
user_defined.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), mode.to_string());
}
if let Some(retain_until) = retain_until {
user_defined.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), retain_until.to_string());
user_defined.insert(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), retain_until.to_string());
}
let obj_info = ObjectInfo {
user_defined: Arc::new(user_defined),
@@ -579,14 +574,14 @@ mod tests {
/// source timestamp of every category that currently locks the version.
#[test]
fn replication_write_passes_worm_gate_only_with_every_locking_category_timestamp() {
let hold = [(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "ON")];
let hold = [(metadata_keys::OBJECT_LOCK_LEGAL_HOLD, "ON")];
let retention = [
(AMZ_OBJECT_LOCK_MODE_LOWER, "GOVERNANCE"),
(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, "2099-01-01T00:00:00Z"),
(metadata_keys::OBJECT_LOCK_MODE, "GOVERNANCE"),
(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE, "2099-01-01T00:00:00Z"),
];
let expired = [
(AMZ_OBJECT_LOCK_MODE_LOWER, "COMPLIANCE"),
(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, "2000-01-01T00:00:00Z"),
(metadata_keys::OBJECT_LOCK_MODE, "COMPLIANCE"),
(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE, "2000-01-01T00:00:00Z"),
];
let absent = ObjectLockConfigState::ConfirmedAbsent;
let passes = |state: &ObjectLockConfigState, entries: &[&[(&str, &str)]], opts: &ObjectOptions| {
@@ -606,7 +601,7 @@ mod tests {
// Expired retention and a released hold no longer lock anything.
assert!(passes(
&absent,
&[&expired, &[(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "OFF")]],
&[&expired, &[(metadata_keys::OBJECT_LOCK_LEGAL_HOLD, "OFF")]],
&replication_opts(false, false)
));
@@ -651,7 +646,7 @@ mod tests {
);
// Default retention plus a legal hold: both categories need a timestamp.
let held = lock_object_info(lock_metadata(&[&[(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "ON")]]));
let held = lock_object_info(lock_metadata(&[&[(metadata_keys::OBJECT_LOCK_LEGAL_HOLD, "ON")]]));
assert!(!replication_write_may_pass_worm_gate(&state, &held, &replication_opts(false, true)).expect("judged"));
assert!(!replication_write_may_pass_worm_gate(&state, &held, &replication_opts(true, false)).expect("judged"));
assert!(replication_write_may_pass_worm_gate(&state, &held, &replication_opts(true, true)).expect("judged"));
@@ -672,7 +667,7 @@ mod tests {
assert!(replication_write_may_pass_worm_gate(&state, &delete_marker, &tagging_only).expect("judged"));
// Cleared (empty) explicit keys fall back to the bucket default.
let cleared = lock_object_info(lock_metadata(&[&[(AMZ_OBJECT_LOCK_MODE_LOWER, "")]]));
let cleared = lock_object_info(lock_metadata(&[&[(metadata_keys::OBJECT_LOCK_MODE, "")]]));
assert!(!replication_write_may_pass_worm_gate(&state, &cleared, &tagging_only).expect("judged"));
}
}
@@ -690,7 +685,7 @@ mod tests {
.expect_err("fabricated bucket lock metadata must not be judged");
assert!(err.to_string().contains("not authoritative"));
let malformed = lock_object_info(lock_metadata(&[&[(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "MAYBE")]]));
let malformed = lock_object_info(lock_metadata(&[&[(metadata_keys::OBJECT_LOCK_LEGAL_HOLD, "MAYBE")]]));
let err = replication_write_may_pass_worm_gate(&ObjectLockConfigState::ConfirmedAbsent, &malformed, &opts)
.expect_err("malformed legal hold must not be judged");
assert!(err.to_string().contains("legal-hold"));
@@ -738,15 +733,15 @@ mod tests {
let cases: [(&str, &[&str]); 3] = [
(
"cleared retention",
&[AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER],
&[metadata_keys::OBJECT_LOCK_MODE, metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE],
),
("cleared legal hold", &[AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER]),
("cleared legal hold", &[metadata_keys::OBJECT_LOCK_LEGAL_HOLD]),
(
"all cleared",
&[
AMZ_OBJECT_LOCK_MODE_LOWER,
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER,
metadata_keys::OBJECT_LOCK_MODE,
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE,
metadata_keys::OBJECT_LOCK_LEGAL_HOLD,
],
),
];
@@ -766,7 +761,7 @@ mod tests {
#[test]
fn deletion_rejects_invalid_persisted_legal_hold_metadata() {
let mut user_defined = std::collections::HashMap::new();
user_defined.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "INVALID".to_string());
user_defined.insert(metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(), "INVALID".to_string());
let obj_info = ObjectInfo {
user_defined: Arc::new(user_defined),
..Default::default()
crates/ecstore/src/bucket/replication/replication_filemeta_boundary.rs
+13
View File
@@ -14,6 +14,19 @@
#[cfg(test)]
pub(crate) use rustfs_filemeta::ObjectPartInfo;
/// Persisted object-metadata keys (`meta_user`); filemeta owns the on-disk spelling.
pub(crate) use rustfs_filemeta::metadata_keys;
/// `FileInfo.metadata` of the xl.meta fixture written before `metadata_keys` existed.
#[cfg(test)]
pub(crate) fn pre_metadata_keys_fixture_metadata() -> std::collections::HashMap<String, String> {
rustfs_filemeta::FileMeta::load(&rustfs_filemeta::test_data::create_pre_metadata_keys_xlmeta().expect("decode fixture hex"))
.expect("load fixture xl.meta")
.into_fileinfo("bucket", "object", "0b1e5a3a-1735-4a3a-8000-00000000a3a0", false, false, false)
.expect("fixture version to FileInfo")
.metadata
}
pub use rustfs_replication::{MrfOpKind, MrfReplicateEntry};
pub(crate) use rustfs_replication::{
REPLICATE_EXISTING, REPLICATE_HEAL_DELETE, ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction,
crates/ecstore/src/bucket/replication/replication_object_config.rs
+2 -2
View File
@@ -14,8 +14,8 @@
use std::{collections::HashMap, fmt, sync::Arc};
use super::replication_filemeta_boundary::metadata_keys;
use crate::bucket::metadata::BucketMetadata;
use rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS;
use s3s::dto::{BucketVersioningStatus, ReplicationConfiguration, ReplicationRuleStatus, VersioningConfiguration};
use serde::{Deserialize, Serialize};
use time::OffsetDateTime;
@@ -334,7 +334,7 @@ impl ReplicationConfig {
}
let mut user_defined = (*oi.user_defined).clone();
user_defined.remove(AMZ_BUCKET_REPLICATION_STATUS);
user_defined.remove(metadata_keys::REPLICATION_STATUS);
let dsc = must_replicate(
oi.bucket.as_str(),
crates/ecstore/src/bucket/replication/replication_resyncer.rs
+6 -5
View File
@@ -19,6 +19,7 @@ use super::replication_error_boundary::{Error, Result, is_err_object_not_found,
use super::replication_event_sink::{EventArgs, send_event, send_local_event};
#[cfg(test)]
use super::replication_filemeta_boundary::ReplicationGenerationSnapshot;
use super::replication_filemeta_boundary::metadata_keys;
use super::replication_filemeta_boundary::{
REPLICATE_EXISTING, ReplicateDecision, ReplicateObjectInfo, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction,
ReplicationState, ReplicationStatusType, ReplicationType, VersionPurgeStatusType, get_replication_state,
@@ -85,9 +86,9 @@ use metrics::counter;
use rmp_serde;
use rustfs_s3_types::EventName;
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE,
AMZ_TAGGING_DIRECTIVE, SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TARGET_VERSION_ARN_PREFIX,
has_internal_suffix, insert_str, replication_target_versions,
AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_TAGGING_DIRECTIVE,
SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TARGET_VERSION_ARN_PREFIX, has_internal_suffix,
insert_str, replication_target_versions,
};
use rustfs_utils::{DEFAULT_SIP_HASH_KEY, get_env_usize, sip_hash};
#[cfg(test)]
@@ -202,7 +203,7 @@ fn metadata_requires_existing_target(op_type: ReplicationType, object_info: &Obj
op_type == ReplicationType::Metadata
&& object_info
.user_defined
.get(AMZ_BUCKET_REPLICATION_STATUS)
.get(metadata_keys::REPLICATION_STATUS)
.is_some_and(|status| status.eq_ignore_ascii_case(ReplicationStatusType::Replica.as_str()))
}
@@ -6269,7 +6270,7 @@ mod tests {
version_id: roi.version_id,
etag: Some("source-etag".to_string()),
user_defined: Arc::new(HashMap::from([(
AMZ_BUCKET_REPLICATION_STATUS.to_string(),
metadata_keys::REPLICATION_STATUS.to_string(),
ReplicationStatusType::Replica.as_str().to_string(),
)])),
..Default::default()
crates/ecstore/src/bucket/replication/replication_target_boundary.rs
+48 -1
View File
@@ -52,6 +52,7 @@ pub use rustfs_replication::{VersionIdentityCapability, version_identity_capabil
use super::replication_config_store::ReplicationConfigStore;
use super::replication_error_boundary::{Error, Result};
use super::replication_filemeta_boundary::metadata_keys;
use super::replication_filemeta_boundary::{ReplicationAction, ReplicationStatusType, ReplicationType};
use super::replication_storage_boundary::ObjectInfo;
use super::replication_tagging_boundary::ReplicationTagFilter;
@@ -94,7 +95,7 @@ fn metadata_value<'a>(metadata: &'a HashMap<String, String>, name: &str) -> Opti
fn classify_replication_source_encryption(metadata: &HashMap<String, String>) -> ReplicationSourceEncryption {
let is_ssec = replication_object_is_ssec_encrypted(metadata);
let sse = metadata_value(metadata, AMZ_SERVER_SIDE_ENCRYPTION);
let sse = metadata_value(metadata, metadata_keys::SERVER_SIDE_ENCRYPTION);
let kms_key_id = metadata_value(metadata, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID);
let kms_context = metadata_value(metadata, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT);
@@ -577,6 +578,52 @@ mod tests {
use time::Duration;
use uuid::Uuid;
/// backlog#1735 A3b SSE key-case finding. Every RustFS writer persists the
/// SSE intent as lowercase `x-amz-server-side-encryption`
/// (`encryption_material_to_metadata`; header-derived keys come from a
/// lowercase `http::HeaderMap`). The mixed-case `X-Amz-Server-Side-Encryption`
/// spelling only appears in outbound replication user metadata, which the
/// S3 client re-lowercases on the wire. This reader has always matched
/// the key ASCII-case-insensitively; keep that, so the old lowercase bytes
/// and any title-case spelling both classify, while any non-case drift of
/// the key reads as "no SSE intent".
#[test]
fn replication_sse_classification_reads_pre_module_xlmeta_key() {
let metadata = super::super::replication_filemeta_boundary::pre_metadata_keys_fixture_metadata();
assert_eq!(metadata.get("x-amz-server-side-encryption").map(String::as_str), Some("AES256"));
assert_eq!(classify_replication_source_encryption(&metadata), ReplicationSourceEncryption::SseS3);
let sse_only = |key: String| HashMap::from([(key, "AES256".to_string())]);
let key = metadata_keys::SERVER_SIDE_ENCRYPTION;
for spelling in [
key.to_string(),
"X-Amz-Server-Side-Encryption".to_string(),
key.to_ascii_uppercase(),
] {
assert_eq!(
classify_replication_source_encryption(&sse_only(spelling.clone())),
ReplicationSourceEncryption::SseS3,
"{spelling:?}"
);
}
for idx in 0..key.len() {
let mut bytes = key.as_bytes().to_vec();
bytes[idx] = if bytes[idx] == b'z' {
b'y'
} else if bytes[idx].is_ascii_alphabetic() {
b'z'
} else {
b'_'
};
let mutated = String::from_utf8(bytes).expect("ascii");
assert_eq!(
classify_replication_source_encryption(&sse_only(mutated.clone())),
ReplicationSourceEncryption::Plaintext,
"{mutated:?} must not read as the SSE key"
);
}
}
/// Serialize an object-level checksum record the way
/// `complete_multipart_upload` persists it for a **full-object** checksum:
/// the record carries the plain algorithm type, without the MULTIPART
crates/ecstore/src/data_movement/mod.rs
+17 -17
View File
@@ -2110,8 +2110,8 @@ async fn migrate_object_inner(
mod tests {
use super::*;
use crate::bucket::replication::{ReplicationStatusType, VersionPurgeStatusType};
use rustfs_filemeta::metadata_keys;
use rustfs_rio::{Checksum, ChecksumType};
use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
use std::collections::HashMap;
use std::io::Cursor;
use std::sync::atomic::AtomicUsize;
@@ -2169,16 +2169,16 @@ mod tests {
assert_eq!(source.version_purge_status_internal, target.version_purge_status_internal);
assert_eq!(source.version_purge_status, target.version_purge_status);
assert_eq!(
source.user_defined.get(X_AMZ_OBJECT_LOCK_MODE.as_str()),
target.user_defined.get(X_AMZ_OBJECT_LOCK_MODE.as_str())
source.user_defined.get(metadata_keys::OBJECT_LOCK_MODE),
target.user_defined.get(metadata_keys::OBJECT_LOCK_MODE)
);
assert_eq!(
source.user_defined.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str()),
target.user_defined.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str())
source.user_defined.get(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE),
target.user_defined.get(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE)
);
assert_eq!(
source.user_defined.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()),
target.user_defined.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str())
source.user_defined.get(metadata_keys::OBJECT_LOCK_LEGAL_HOLD),
target.user_defined.get(metadata_keys::OBJECT_LOCK_LEGAL_HOLD)
);
assert_eq!(source.parts.len(), target.parts.len());
for (source_part, target_part) in source.parts.iter().zip(target.parts.iter()) {
@@ -2818,13 +2818,13 @@ mod tests {
let mod_time = OffsetDateTime::UNIX_EPOCH;
let metadata = Arc::new(HashMap::from([
("x-amz-meta-key".to_string(), "value".to_string()),
(rustfs_utils::http::AMZ_STORAGE_CLASS.to_string(), "STANDARD_IA".to_string()),
(X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(), "GOVERNANCE".to_string()),
(rustfs_filemeta::metadata_keys::STORAGE_CLASS.to_string(), "STANDARD_IA".to_string()),
(metadata_keys::OBJECT_LOCK_MODE.to_string(), "GOVERNANCE".to_string()),
(
X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
"2030-01-01T00:00:00Z".to_string(),
),
(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), "ON".to_string()),
(metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(), "ON".to_string()),
]));
let part = ObjectPartInfo {
number: 1,
@@ -2864,12 +2864,12 @@ mod tests {
rustfs_utils::http::SUFFIX_REPLICATION_STATUS.to_string(),
"arn:minio:target=PENDING;".to_string(),
),
(X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(), "COMPLIANCE".to_string()),
(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string()),
(
X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
"2031-01-01T00:00:00Z".to_string(),
),
(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), "ON".to_string()),
(metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(), "ON".to_string()),
])),
..Default::default()
};
@@ -2882,15 +2882,15 @@ mod tests {
Some(&"arn:minio:target=PENDING;".to_string())
);
assert_eq!(
new_multipart_opts.user_defined.get(X_AMZ_OBJECT_LOCK_MODE.as_str()),
new_multipart_opts.user_defined.get(metadata_keys::OBJECT_LOCK_MODE),
Some(&"COMPLIANCE".to_string())
);
assert_eq!(
put_opts.user_defined.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str()),
put_opts.user_defined.get(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE),
Some(&"2031-01-01T00:00:00Z".to_string())
);
assert_eq!(
new_multipart_opts.user_defined.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()),
new_multipart_opts.user_defined.get(metadata_keys::OBJECT_LOCK_LEGAL_HOLD),
Some(&"ON".to_string())
);
}
crates/ecstore/src/io_support/rio.rs
+1 -1
View File
@@ -560,7 +560,7 @@ mod tests {
let mut headers = HeaderMap::new();
headers.insert("x-amz-trailer", HeaderValue::from_static("x-amz-checksum-crc32"));
reader
.add_checksum_from_s3s(&headers, None, false)
.add_checksum(&headers, None, false)
.expect("attach trailing checksum metadata");
let transformed = WritePlan::new()
crates/ecstore/src/object_api/mod.rs
+5 -3
View File
@@ -34,13 +34,12 @@ use crate::store::utils::clean_metadata;
use crate::{bucket::lifecycle::bucket_lifecycle_audit::LcAuditEvent, bucket::lifecycle::lifecycle::TransitionOptions};
use bytes::Bytes;
use http::{HeaderMap, HeaderValue};
use rustfs_filemeta::metadata_keys;
use rustfs_filemeta::{FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, RestoreStatusOps as _, parse_restore_obj_status};
use rustfs_rio::Checksum;
use rustfs_utils::CompressionAlgorithm;
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, AMZ_RESTORE, AMZ_STORAGE_CLASS, SUFFIX_PLAINTEXT_CHECKSUM, get_consistent_str,
};
use rustfs_utils::http::{SUFFIX_PLAINTEXT_CHECKSUM, get_consistent_str};
use rustfs_utils::path::decode_dir_object;
use std::collections::HashMap;
use std::fmt::Debug;
@@ -106,6 +105,9 @@ mod object_mutation_hook;
mod readers;
mod types;
#[cfg(test)]
mod persisted_metadata_keys_tests;
#[cfg(test)]
pub(crate) use body_cache_hook::clear_get_object_body_cache_hook;
pub use body_cache_hook::{
crates/ecstore/src/object_api/persisted_metadata_keys_tests.rs
+192
View File
@@ -0,0 +1,192 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Old-bytes coverage for the ecstore/lifecycle readers of the persisted
//! `meta_user` keys (backlog#1735 A3b).
//!
//! The fixture is xl.meta written by the code that predates
//! `rustfs_filemeta::metadata_keys`, with each key taken from its historical
//! source. Every ecstore reader migrated to the authority must still find its
//! key in those bytes, and a single-character drift of any key (ASCII case
//! included, because these lookups are exact) must make the reader miss.
use super::*;
use crate::bucket::object_lock::objectlock::{get_object_legalhold_meta, get_object_retention_meta};
use crate::bucket::object_lock::objectlock_sys::{ObjectLockBlockReason, check_object_lock_for_deletion_with_default_retention};
use crate::bucket::object_lock::types::{LegalHoldStatus, RetentionMode};
use crate::bucket::replication::ReplicationStatusType;
use crate::store::utils::remove_standard_storage_class;
use rustfs_filemeta::FileMeta;
use rustfs_filemeta::metadata_keys;
use rustfs_filemeta::test_data::create_pre_metadata_keys_xlmeta;
const FIXTURE_VERSION_ID: &str = "0b1e5a3a-1735-4a3a-8000-00000000a3a0";
fn fixture_file_info() -> FileInfo {
FileMeta::load(&create_pre_metadata_keys_xlmeta().expect("decode fixture hex"))
.expect("load fixture xl.meta")
.into_fileinfo("bucket", "object", FIXTURE_VERSION_ID, false, false, false)
.expect("fixture version to FileInfo")
}
fn object_info(fi: &FileInfo) -> ObjectInfo {
ObjectInfo::from_file_info(fi, "bucket", "object", true)
}
/// Replaces the byte at `idx` with a different one of the same class:
/// ASCII letters flip case, anything else becomes `_`.
fn mutate_at(key: &str, idx: usize) -> String {
let mut bytes = key.as_bytes().to_vec();
let b = bytes[idx];
bytes[idx] = if b.is_ascii_lowercase() {
b.to_ascii_uppercase()
} else if b.is_ascii_uppercase() {
b.to_ascii_lowercase()
} else {
b'_'
};
String::from_utf8(bytes).expect("ascii mutation stays utf-8")
}
/// The fixture version with `key` stored under `mutated` instead.
fn file_info_with_key_renamed(key: &str, mutated: &str) -> FileInfo {
let mut fi = fixture_file_info();
let value = fi.metadata.remove(key).expect("fixture carries the key");
fi.metadata.insert(mutated.to_string(), value);
fi
}
fn locked_by_object_lock(oi: &ObjectInfo) -> bool {
let retention = get_object_retention_meta(&oi.user_defined);
let hold = get_object_legalhold_meta(&oi.user_defined);
retention.mode.is_some()
|| hold.status.is_some()
|| rustfs_lifecycle::object_lock::is_object_locked_by_metadata(&oi.user_defined, false)
|| check_object_lock_for_deletion_with_default_retention(None, oi, false)
.expect("object-lock check")
.is_some()
}
#[test]
fn pre_module_xlmeta_object_lock_is_enforced_by_ecstore_and_lifecycle() {
let oi = object_info(&fixture_file_info());
let retention = get_object_retention_meta(&oi.user_defined);
assert!(matches!(retention.mode, Some(RetentionMode::Compliance)));
assert_eq!(retention.retain_until_date.map(|date| date.year()), Some(2099));
assert!(matches!(get_object_legalhold_meta(&oi.user_defined).status, Some(LegalHoldStatus::On)));
assert!(rustfs_lifecycle::object_lock::is_object_locked_by_metadata(&oi.user_defined, false));
assert!(matches!(
check_object_lock_for_deletion_with_default_retention(None, &oi, true),
Ok(Some(ObjectLockBlockReason::LegalHold))
));
let mut without_hold = fixture_file_info();
without_hold.metadata.remove(metadata_keys::OBJECT_LOCK_LEGAL_HOLD);
assert!(matches!(
check_object_lock_for_deletion_with_default_retention(None, &object_info(&without_hold), true),
Ok(Some(ObjectLockBlockReason::Retention {
mode: RetentionMode::Compliance,
..
}))
));
}
#[test]
fn pre_module_xlmeta_restore_replication_and_storage_class_read_back() {
let fi = fixture_file_info();
let oi = object_info(&fi);
assert!(!oi.restore_ongoing);
assert_eq!(oi.restore_expires.map(|date| date.year()), Some(9999));
let mut replica = fixture_file_info();
replica.metadata.insert(
metadata_keys::REPLICATION_STATUS.to_string(),
ReplicationStatusType::Replica.as_str().to_string(),
);
assert_eq!(object_info(&replica).replication_status, ReplicationStatusType::Replica);
let mut metadata = fi.metadata;
assert_eq!(metadata.get(metadata_keys::STORAGE_CLASS).map(String::as_str), Some("GLACIER"));
remove_standard_storage_class(&mut metadata);
assert!(metadata.contains_key(metadata_keys::STORAGE_CLASS), "non-STANDARD class must stay");
metadata.insert(
metadata_keys::STORAGE_CLASS.to_string(),
crate::config::storageclass::STANDARD.to_string(),
);
remove_standard_storage_class(&mut metadata);
assert!(!metadata.contains_key(metadata_keys::STORAGE_CLASS), "STANDARD must be dropped");
}
/// Per-character mutation over the persisted keys the migrated ecstore and
/// lifecycle readers look up exactly: each drifted spelling must be missed.
#[test]
fn single_character_key_mutation_is_missed_by_ecstore_readers() {
// Keep a single object-lock category per object so each mutation is
// judged on its own: the legal hold alone, then the retention pair alone.
for key in [
metadata_keys::OBJECT_LOCK_LEGAL_HOLD,
metadata_keys::OBJECT_LOCK_MODE,
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE,
] {
for idx in 0..key.len() {
let mutated = mutate_at(key, idx);
let mut fi = file_info_with_key_renamed(key, &mutated);
if key == metadata_keys::OBJECT_LOCK_LEGAL_HOLD {
fi.metadata.remove(metadata_keys::OBJECT_LOCK_MODE);
fi.metadata.remove(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE);
assert!(!locked_by_object_lock(&object_info(&fi)), "legal hold under {mutated:?} must not lock");
} else {
fi.metadata.remove(metadata_keys::OBJECT_LOCK_LEGAL_HOLD);
let oi = object_info(&fi);
let retention = get_object_retention_meta(&oi.user_defined);
if key == metadata_keys::OBJECT_LOCK_MODE {
assert!(retention.mode.is_none(), "mode under {mutated:?} must not be read");
} else {
assert!(retention.retain_until_date.is_none(), "retain-until under {mutated:?} must not be read");
}
assert!(
!rustfs_lifecycle::object_lock::is_object_locked_by_metadata(&oi.user_defined, false),
"retention with {mutated:?} must not read as locked"
);
}
}
}
for idx in 0..metadata_keys::RESTORE.len() {
let mutated = mutate_at(metadata_keys::RESTORE, idx);
let oi = object_info(&file_info_with_key_renamed(metadata_keys::RESTORE, &mutated));
assert!(oi.restore_expires.is_none(), "restore status under {mutated:?} must not be read");
}
for idx in 0..metadata_keys::REPLICATION_STATUS.len() {
let mutated = mutate_at(metadata_keys::REPLICATION_STATUS, idx);
let mut fi = fixture_file_info();
fi.metadata.remove(metadata_keys::REPLICATION_STATUS);
fi.metadata
.insert(mutated.clone(), ReplicationStatusType::Replica.as_str().to_string());
assert_ne!(
object_info(&fi).replication_status,
ReplicationStatusType::Replica,
"replica status under {mutated:?} must not be read"
);
}
for idx in 0..metadata_keys::STORAGE_CLASS.len() {
let mutated = mutate_at(metadata_keys::STORAGE_CLASS, idx);
let mut metadata = HashMap::from([(mutated.clone(), crate::config::storageclass::STANDARD.to_string())]);
remove_standard_storage_class(&mut metadata);
assert!(metadata.contains_key(&mutated), "storage class under {mutated:?} must not be matched");
}
}
crates/ecstore/src/object_api/types.rs
+7 -8
View File
@@ -1562,8 +1562,7 @@ impl ObjectInfo {
.user_defined
.iter()
.filter(|(key, _)| {
!rustfs_utils::http::is_internal_key(key)
&& !key.eq_ignore_ascii_case(rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS)
!rustfs_utils::http::is_internal_key(key) && !key.eq_ignore_ascii_case(metadata_keys::REPLICATION_STATUS)
})
.collect::<Vec<_>>();
user_metadata.sort_unstable_by(|left, right| left.0.cmp(right.0).then_with(|| left.1.cmp(right.1)));
@@ -1783,7 +1782,7 @@ impl ObjectInfo {
let mut replication_status = replication_status_from_filemeta(fi.replication_status());
if replication_status.is_empty()
&& let Some(status) = fi.metadata.get(AMZ_BUCKET_REPLICATION_STATUS).cloned()
&& let Some(status) = fi.metadata.get(metadata_keys::REPLICATION_STATUS).cloned()
&& status == ReplicationStatusType::Replica.as_str()
{
replication_status = ReplicationStatusType::Replica;
@@ -1811,7 +1810,7 @@ impl ObjectInfo {
let storage_class = Some(
storageclass::effective_class(
fi.metadata.get(AMZ_STORAGE_CLASS).map(String::as_str),
fi.metadata.get(metadata_keys::STORAGE_CLASS).map(String::as_str),
(fi.transition_status == rustfs_filemeta::TRANSITION_COMPLETE && !fi.transition_tier.is_empty())
.then_some(fi.transition_tier.as_str()),
)
@@ -1820,7 +1819,7 @@ impl ObjectInfo {
let mut restore_ongoing = false;
let mut restore_expires = None;
if let Some(restore_status) = fi.metadata.get(AMZ_RESTORE).cloned()
if let Some(restore_status) = fi.metadata.get(metadata_keys::RESTORE).cloned()
&& let Ok(restore_status) = parse_restore_obj_status(&restore_status)
{
restore_ongoing = restore_status.on_going();
@@ -2725,7 +2724,7 @@ mod tests {
storageclass::GLACIER,
] {
let fi = FileInfo {
metadata: HashMap::from([(AMZ_STORAGE_CLASS.to_string(), legacy_label.to_string())]),
metadata: HashMap::from([(metadata_keys::STORAGE_CLASS.to_string(), legacy_label.to_string())]),
..Default::default()
};
@@ -2742,7 +2741,7 @@ mod tests {
#[test]
fn from_file_info_preserves_transitioned_tier_storage_class() {
let fi = FileInfo {
metadata: HashMap::from([(AMZ_STORAGE_CLASS.to_string(), storageclass::STANDARD_IA.to_string())]),
metadata: HashMap::from([(metadata_keys::STORAGE_CLASS.to_string(), storageclass::STANDARD_IA.to_string())]),
transition_tier: "WARM-TIER".to_string(),
transition_status: TRANSITION_COMPLETE.to_string(),
..Default::default()
@@ -2757,7 +2756,7 @@ mod tests {
#[test]
fn from_file_info_ignores_a_tier_name_without_a_completed_transition() {
let fi = FileInfo {
metadata: HashMap::from([(AMZ_STORAGE_CLASS.to_string(), storageclass::STANDARD_IA.to_string())]),
metadata: HashMap::from([(metadata_keys::STORAGE_CLASS.to_string(), storageclass::STANDARD_IA.to_string())]),
transition_tier: "WARM-TIER".to_string(),
..Default::default()
};
crates/ecstore/src/services/tier/warm_backend.rs
+52 -23
View File
@@ -35,6 +35,7 @@ use crate::services::tier::{
};
use bytes::Bytes;
use http::StatusCode;
use rustfs_filemeta::metadata_keys;
use rustfs_s3_client::credentials::{Credentials, SignatureType, Static, Value};
use rustfs_s3_client::transition_api::{BucketLookupType, Options, TransitionClient, TransitionClientTimeouts, TransitionCore};
use rustfs_s3_client::{
@@ -48,10 +49,6 @@ use rustfs_utils::egress::validate_outbound_url;
use rustfs_utils::http::headers::{
CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _,
};
use s3s::header::{
X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_REPLICATION_STATUS,
X_AMZ_STORAGE_CLASS,
};
use s3s::{
S3ErrorCode,
dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus},
@@ -253,36 +250,39 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
opts.expires = expires;
}
if let Some(mode) = metadata.lookup(X_AMZ_OBJECT_LOCK_MODE.as_str()) {
if let Some(mode) = metadata.lookup(metadata_keys::OBJECT_LOCK_MODE) {
opts.mode = ObjectLockRetentionMode::from(mode.to_ascii_uppercase());
}
if let Some(retain_until_date) = metadata
.lookup(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str())
.lookup(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE)
.and_then(parse_http_timestamp)
{
opts.retain_until_date = retain_until_date;
}
if let Some(legalhold) = metadata.lookup(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()) {
if let Some(legalhold) = metadata.lookup(metadata_keys::OBJECT_LOCK_LEGAL_HOLD) {
opts.legalhold = ObjectLockLegalHoldStatus::from(legalhold.to_ascii_uppercase());
}
for key in [
// Promoted keys are read above through `lookup`, which accepts more than
// one spelling, so strip every ASCII-case spelling here. The replication
// status is persisted as `X-Amz-Replication-Status`; the former exact
// lowercase removal left that spelling in the forwarded user metadata.
const PROMOTED_KEYS: [&str; 11] = [
CONTENT_TYPE,
CONTENT_ENCODING,
CONTENT_LANGUAGE,
CONTENT_DISPOSITION,
CACHE_CONTROL,
EXPIRES,
X_AMZ_OBJECT_LOCK_MODE.as_str(),
X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str(),
X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str(),
X_AMZ_REPLICATION_STATUS.as_str(),
X_AMZ_STORAGE_CLASS.as_str(),
] {
metadata.remove(key);
}
metadata_keys::OBJECT_LOCK_MODE,
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE,
metadata_keys::OBJECT_LOCK_LEGAL_HOLD,
metadata_keys::REPLICATION_STATUS,
metadata_keys::STORAGE_CLASS,
];
metadata.retain(|key, _| !PROMOTED_KEYS.iter().any(|promoted| key.eq_ignore_ascii_case(promoted)));
for suffix in [
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
@@ -2068,9 +2068,18 @@ mod tests {
#[test]
fn build_transition_put_options_preserves_object_lock_headers_when_present() {
let mut metadata = HashMap::new();
metadata.insert(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), "2026-03-23T00:00:00Z".to_string());
metadata.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.to_string(), ObjectLockLegalHoldStatus::ON.to_string());
metadata.insert(X_AMZ_OBJECT_LOCK_MODE.to_string(), ObjectLockRetentionMode::GOVERNANCE.to_string());
metadata.insert(
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
"2026-03-23T00:00:00Z".to_string(),
);
metadata.insert(
metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(),
ObjectLockLegalHoldStatus::ON.to_string(),
);
metadata.insert(
metadata_keys::OBJECT_LOCK_MODE.to_string(),
ObjectLockRetentionMode::GOVERNANCE.to_string(),
);
let opts = build_transition_put_options("COLD".to_string(), metadata);
@@ -2084,15 +2093,35 @@ mod tests {
let mut metadata = HashMap::new();
metadata.insert("name".to_string(), "object".to_string());
metadata.insert(CONTENT_TYPE.to_string(), "text/plain".to_string());
metadata.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.to_string(), ObjectLockLegalHoldStatus::ON.to_string());
metadata.insert(X_AMZ_REPLICATION_STATUS.to_string(), "PENDING".to_string());
metadata.insert(
metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(),
ObjectLockLegalHoldStatus::ON.to_string(),
);
metadata.insert("x-amz-replication-status".to_string(), "PENDING".to_string());
let opts = build_transition_put_options("COLD".to_string(), metadata);
assert_eq!(opts.user_metadata.get("name"), Some(&"object".to_string()));
assert!(!opts.user_metadata.contains_key(CONTENT_TYPE));
assert!(!opts.user_metadata.contains_key(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()));
assert!(!opts.user_metadata.contains_key(X_AMZ_REPLICATION_STATUS.as_str()));
assert!(!opts.user_metadata.contains_key(metadata_keys::OBJECT_LOCK_LEGAL_HOLD));
assert!(!opts.user_metadata.contains_key("x-amz-replication-status"));
}
/// Object metadata read back from xl.meta carries the replication status
/// under its persisted mixed-case key; it must not be forwarded to the
/// tier as user metadata either.
#[test]
fn build_transition_put_options_filters_persisted_replication_status_key() {
let metadata = HashMap::from([
("name".to_string(), "object".to_string()),
(metadata_keys::REPLICATION_STATUS.to_string(), "COMPLETED".to_string()),
(metadata_keys::STORAGE_CLASS.to_string(), "STANDARD".to_string()),
]);
let opts = build_transition_put_options("COLD".to_string(), metadata);
assert_eq!(opts.user_metadata.len(), 1, "{:?}", opts.user_metadata);
assert_eq!(opts.user_metadata.get("name"), Some(&"object".to_string()));
}
#[test]
crates/ecstore/src/set_disk/metadata.rs
+2 -1
View File
@@ -23,6 +23,7 @@ use crate::disk::DiskOption;
use crate::disk::endpoint::Endpoint;
#[cfg(test)]
use crate::disk::new_disk;
use rustfs_filemeta::metadata_keys;
use rustfs_utils::http;
use sha2::Digest;
@@ -586,7 +587,7 @@ impl SetDisks {
}
fn is_replication_quorum_metadata_key(name: &str) -> bool {
if name.eq_ignore_ascii_case(http::AMZ_BUCKET_REPLICATION_STATUS) {
if name.eq_ignore_ascii_case(metadata_keys::REPLICATION_STATUS) {
return true;
}
crates/ecstore/src/set_disk/mod.rs
+22 -26
View File
@@ -121,6 +121,7 @@ use http::HeaderMap;
use md5::{Digest as Md5Digest, Md5};
use regex::Regex;
use rustfs_config::MI_B;
use rustfs_filemeta::metadata_keys;
use rustfs_filemeta::{
FileInfo, FileMeta, FileMetaShallowVersion, MetaCacheEntries, MetaCacheEntry, ObjectPartInfo, RawFileInfo,
merge_file_meta_versions,
@@ -147,7 +148,6 @@ use rustfs_s3_types::EventName;
#[cfg(test)]
use rustfs_utils::http::SSEC_ALGORITHM_HEADER;
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
use rustfs_utils::http::headers::{
CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES,
};
@@ -161,7 +161,6 @@ use rustfs_utils::{
crypto::hex,
path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, path_join_buf},
};
use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_RESTORE};
use sha2::Sha256;
use std::hash::{BuildHasher, Hash, Hasher};
use std::mem::{self};
@@ -213,7 +212,7 @@ pub(super) fn require_restore_operation_id(metadata: &HashMap<String, String>, e
}
pub(super) fn restore_commit_operation_id_from_metadata(metadata: &HashMap<String, String>) -> Result<Option<Uuid>> {
if !metadata.contains_key(X_AMZ_RESTORE.as_str()) {
if !metadata.contains_key(metadata_keys::RESTORE) {
return Ok(None);
}
restore_operation_id_from_metadata(metadata)
@@ -5917,7 +5916,7 @@ impl SetDisks {
}
let disks = self.disks.read().await.clone();
let storage_class = opts.user_defined.get(AMZ_STORAGE_CLASS).map(String::as_str);
let storage_class = opts.user_defined.get(metadata_keys::STORAGE_CLASS).map(String::as_str);
let layout = resolve_write_layout(
&storage_class_config,
self.pool_index,
@@ -10365,10 +10364,7 @@ mod tests {
);
assert!(meta_a.replication_state_internal.is_some());
assert_eq!(
meta_a
.metadata
.get(rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS)
.map(String::as_str),
meta_a.metadata.get(metadata_keys::REPLICATION_STATUS).map(String::as_str),
Some("COMPLETED")
);
@@ -11659,11 +11655,11 @@ mod tests {
let mut user_defined = HashMap::new();
user_defined.insert(
X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_MODE.to_string(),
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
);
user_defined.insert(
X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
existing_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
);
@@ -11694,11 +11690,11 @@ mod tests {
let mut user_defined = HashMap::new();
user_defined.insert(
X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_MODE.to_string(),
s3s::dto::ObjectLockRetentionMode::GOVERNANCE.to_string(),
);
user_defined.insert(
X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
existing_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
);
@@ -11724,11 +11720,11 @@ mod tests {
let retain_until = OffsetDateTime::now_utc() + Duration::from_secs(60 * 60 * 24 * 60);
let mut user_defined = HashMap::new();
user_defined.insert(
X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_MODE.to_string(),
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
);
user_defined.insert(
X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
retain_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
);
@@ -11769,11 +11765,11 @@ mod tests {
restore_expires: Some(restore_expiry),
user_defined: Arc::new(HashMap::from([
(
X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_MODE.to_string(),
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
),
(
X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
retain_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
),
])),
@@ -11843,11 +11839,11 @@ mod tests {
let retain_until = OffsetDateTime::now_utc() + Duration::from_secs(60 * 60 * 24 * 60);
let mut user_defined = HashMap::new();
user_defined.insert(
X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_MODE.to_string(),
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
);
user_defined.insert(
X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
retain_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
);
@@ -11871,11 +11867,11 @@ mod tests {
let retain_until = OffsetDateTime::now_utc() + Duration::from_secs(60 * 60 * 24 * 60);
let mut user_defined = HashMap::new();
user_defined.insert(
X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_MODE.to_string(),
s3s::dto::ObjectLockRetentionMode::GOVERNANCE.to_string(),
);
user_defined.insert(
X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
retain_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
);
ObjectInfo {
@@ -11924,11 +11920,11 @@ mod tests {
let retain_until = OffsetDateTime::now_utc() + Duration::from_secs(60 * 60 * 24 * 60);
let mut user_defined = HashMap::new();
user_defined.insert(
X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_MODE.to_string(),
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
);
user_defined.insert(
X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
retain_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
);
let obj_info = ObjectInfo {
@@ -11947,7 +11943,7 @@ mod tests {
#[tokio::test]
async fn test_check_object_lock_delete_blocks_replicated_legal_hold_version_purge() {
let mut user_defined = HashMap::new();
user_defined.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), "ON".to_string());
user_defined.insert(metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(), "ON".to_string());
let obj_info = ObjectInfo {
user_defined: Arc::new(user_defined),
..Default::default()
@@ -14560,7 +14556,7 @@ mod tests {
let object = "object.txt";
let mod_time = OffsetDateTime::from_unix_timestamp(1_717_171_717).expect("fixed timestamp should parse");
let mut user_defined = HashMap::new();
user_defined.insert(AMZ_STORAGE_CLASS.to_string(), storageclass::STANDARD.to_string());
user_defined.insert(metadata_keys::STORAGE_CLASS.to_string(), storageclass::STANDARD.to_string());
user_defined.insert(SUFFIX_COMPRESSION.to_string(), "zstd".to_string());
let mut eval_metadata = HashMap::new();
eval_metadata.insert("x-amz-meta-evaluated".to_string(), "yes".to_string());
@@ -14584,7 +14580,7 @@ mod tests {
assert_eq!(written.etag.as_deref(), Some("preserved-etag"));
assert_eq!(written.mod_time, Some(mod_time));
assert_eq!(written.user_defined.get("x-amz-meta-evaluated").map(String::as_str), Some("yes"));
assert!(!written.user_defined.contains_key(AMZ_STORAGE_CLASS));
assert!(!written.user_defined.contains_key(metadata_keys::STORAGE_CLASS));
let info = set_disks
.get_object_info(bucket, object, &opts)
@@ -14593,7 +14589,7 @@ mod tests {
assert_eq!(info.etag.as_deref(), Some("preserved-etag"));
assert_eq!(info.mod_time, Some(mod_time));
assert_eq!(info.user_defined.get("x-amz-meta-evaluated").map(String::as_str), Some("yes"));
assert!(!info.user_defined.contains_key(AMZ_STORAGE_CLASS));
assert!(!info.user_defined.contains_key(metadata_keys::STORAGE_CLASS));
}
#[tokio::test]
crates/ecstore/src/set_disk/ops/multipart.rs
+28 -29
View File
@@ -29,23 +29,22 @@ use super::super::MetadataCacheInvalidationProbe;
#[cfg(test)]
use super::super::capacity_scope_from_disks;
use super::super::{
AMZ_STORAGE_CLASS, Arc, Bytes, CompletePart, Cursor, DATA_MOVEMENT_MULTIPART_PREFIX, DiskError, DiskStore,
EVENT_SET_DISK_MULTIPART, Error, FileInfo, GLOBAL_MIN_PART_SIZE, HashAlgorithm, HashMap, HashReader, HashSet,
HealChannelPriority, Instant, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, ListMultipartsInfo, ListPartsInfo,
MAX_PARTS_COUNT, MULTIPART_WRITE_QUORUM_RENAME_PART, MULTIPART_WRITE_QUORUM_UPLOAD_METADATA,
MULTIPART_WRITE_QUORUM_WRITER_SETUP, MultipartInfo, MultipartUploadResult, MultipartWriteQuorumContext, NamespaceLockFence,
OBJECT_OP_IGNORED_ERRS, ObjectInfo, ObjectLockDiagGuard, ObjectOptions, ObjectPartInfo, OffsetDateTime, PartInfo,
PutObjReader, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET, RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY,
Result, SLASH_SEPARATOR, SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, SUFFIX_BUCKET_INCARNATION_ID,
SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK, SetDisks,
SmallWritePath, StorageError, Uuid, WriteLayout, check_object_lock_for_deletion_with_state,
classify_multipart_part_write_path, coding, complete_multipart_part_error, complete_multipart_part_error_result,
complete_part_checksum, completed_multipart_object_part, contains_key_str, create_bitrot_writer, debug, disk, error,
get_complete_multipart_md5, get_header_map, get_str, insert_str, is_err_object_not_found, is_err_version_not_found,
is_min_allowed_part_size, log_multipart_write_quorum_failure, parts_after_marker, path_join_buf,
record_compression_total_memory, reduce_read_quorum_errs, reduce_write_quorum_errs, remove_header_map, resolve_write_layout,
restore_commit_operation_id_from_metadata, should_persist_encryption_original_size, strip_internal_multipart_metadata,
to_object_err, warn,
Arc, Bytes, CompletePart, Cursor, DATA_MOVEMENT_MULTIPART_PREFIX, DiskError, DiskStore, EVENT_SET_DISK_MULTIPART, Error,
FileInfo, GLOBAL_MIN_PART_SIZE, HashAlgorithm, HashMap, HashReader, HashSet, HealChannelPriority, Instant,
LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, ListMultipartsInfo, ListPartsInfo, MAX_PARTS_COUNT,
MULTIPART_WRITE_QUORUM_RENAME_PART, MULTIPART_WRITE_QUORUM_UPLOAD_METADATA, MULTIPART_WRITE_QUORUM_WRITER_SETUP,
MultipartInfo, MultipartUploadResult, MultipartWriteQuorumContext, NamespaceLockFence, OBJECT_OP_IGNORED_ERRS, ObjectInfo,
ObjectLockDiagGuard, ObjectOptions, ObjectPartInfo, OffsetDateTime, PartInfo, PutObjReader, RUSTFS_META_MULTIPART_BUCKET,
RUSTFS_META_TMP_BUCKET, RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, Result, SLASH_SEPARATOR,
SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, SUFFIX_BUCKET_INCARNATION_ID, SUFFIX_COMPRESSION_SIZE,
SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK, SetDisks, SmallWritePath, StorageError,
Uuid, WriteLayout, check_object_lock_for_deletion_with_state, classify_multipart_part_write_path, coding,
complete_multipart_part_error, complete_multipart_part_error_result, complete_part_checksum, completed_multipart_object_part,
contains_key_str, create_bitrot_writer, debug, disk, error, get_complete_multipart_md5, get_header_map, get_str, insert_str,
is_err_object_not_found, is_err_version_not_found, is_min_allowed_part_size, log_multipart_write_quorum_failure,
parts_after_marker, path_join_buf, record_compression_total_memory, reduce_read_quorum_errs, reduce_write_quorum_errs,
remove_header_map, resolve_write_layout, restore_commit_operation_id_from_metadata, should_persist_encryption_original_size,
strip_internal_multipart_metadata, to_object_err, warn,
};
use super::bitrot_self_verify::{BitrotSelfVerifyTarget, drop_failed_writer_disks, verify_written_bitrot_shards};
#[cfg(test)]
@@ -81,6 +80,7 @@ use crate::storage_api_contracts::object::ObjectOperations;
use futures::{StreamExt, stream};
#[cfg(test)]
use http::HeaderMap;
use rustfs_filemeta::metadata_keys;
use rustfs_rio::EtagResolvable;
use rustfs_rio::TryGetIndex;
#[cfg(test)]
@@ -1913,7 +1913,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
// Extract storage class from metadata, default to STANDARD if not found
let storage_class = fi
.metadata
.get(AMZ_STORAGE_CLASS)
.get(metadata_keys::STORAGE_CLASS)
.cloned()
.unwrap_or_else(|| storageclass::STANDARD.to_string());
@@ -2103,10 +2103,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
user_defined.insert("etag".to_owned(), etag.clone());
}
if let Some(sc) = user_defined.get(AMZ_STORAGE_CLASS)
if let Some(sc) = user_defined.get(metadata_keys::STORAGE_CLASS)
&& sc == storageclass::STANDARD
{
let _ = user_defined.remove(AMZ_STORAGE_CLASS);
let _ = user_defined.remove(metadata_keys::STORAGE_CLASS);
}
let WriteLayout {
@@ -2118,7 +2118,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
self.pool_index,
disks.len(),
self.default_parity_count,
user_defined.get(AMZ_STORAGE_CLASS).map(String::as_str),
user_defined.get(metadata_keys::STORAGE_CLASS).map(String::as_str),
opts.max_parity,
)?;
@@ -2160,10 +2160,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
// TODO(backlog): detect content-type from part data when header is missing
}
if let Some(sc) = user_defined.get(AMZ_STORAGE_CLASS)
if let Some(sc) = user_defined.get(metadata_keys::STORAGE_CLASS)
&& sc == storageclass::STANDARD
{
let _ = user_defined.remove(AMZ_STORAGE_CLASS);
let _ = user_defined.remove(metadata_keys::STORAGE_CLASS);
}
if let Some(checksum) = &opts.want_checksum {
@@ -2398,7 +2398,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
opts.replication_request || opts.delete_marker_replication_status() == ReplicationStatusType::Replica;
if !authorized_inbound_replica {
fi.metadata
.retain(|key, _| !key.eq_ignore_ascii_case(rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS));
.retain(|key, _| !key.eq_ignore_ascii_case(metadata_keys::REPLICATION_STATUS));
for suffix in [
rustfs_utils::http::SUFFIX_REPLICA_STATUS,
rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP,
@@ -8783,8 +8783,7 @@ mod tests {
rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP,
"foreign-replica-time".to_string(),
);
create_replication_metadata
.insert(rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS.to_string(), "REPLICA".to_string());
create_replication_metadata.insert(metadata_keys::REPLICATION_STATUS.to_string(), "REPLICA".to_string());
let (upload_id, parts) = stage_upload_with_create_opts(
&set_disks,
bucket,
@@ -8855,7 +8854,7 @@ mod tests {
completed
.user_defined
.iter()
.filter(|(key, _)| key.eq_ignore_ascii_case(rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS))
.filter(|(key, _)| key.eq_ignore_ascii_case(metadata_keys::REPLICATION_STATUS))
.all(|(_, value)| value != "REPLICA")
);
@@ -8911,7 +8910,7 @@ mod tests {
rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP,
"authorized-inbound-time".to_string(),
);
inbound_replica_metadata.insert(rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS.to_string(), "REPLICA".to_string());
inbound_replica_metadata.insert(metadata_keys::REPLICATION_STATUS.to_string(), "REPLICA".to_string());
let (upload_id, parts) = stage_upload_with_create_opts(
&set_disks,
bucket,
@@ -8952,7 +8951,7 @@ mod tests {
inbound
.user_defined
.iter()
.find(|(key, _)| key.eq_ignore_ascii_case(rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS))
.find(|(key, _)| key.eq_ignore_ascii_case(metadata_keys::REPLICATION_STATUS))
.map(|(_, value)| value.as_str()),
Some("REPLICA")
);
crates/ecstore/src/set_disk/ops/object.rs
+158 -83
View File
@@ -20,27 +20,27 @@
//! SetDisks core (io_primitives) via inherent calls.
use crate::core::pools::DecommissionCapacityAdmission;
use rustfs_filemeta::metadata_keys;
#[cfg(test)]
use super::super::MetadataCacheInvalidationProbe;
use super::super::{
AMZ_OBJECT_TAGGING, AMZ_STORAGE_CLASS, Arc, AsyncWrite, AtomicU64, BufReader, Bytes, CACHE_CONTROL, CONTENT_DISPOSITION,
CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, CompletePart, Cursor, DeleteAccounting, DeleteOptions, DeletedObject,
DiskError, DiskStore, EVENT_SET_DISK_COMMIT_TAIL_SLOW, EVENT_SET_DISK_PUT_OBJECT_STAGE_SUMMARY, EVENT_SET_DISK_WRITE,
EXPIRES, Error, EventArgs, EventName, FastLockGuard, FileInfo, FileInfoVersions,
GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, GET_OBJECT_PATH_BODY_CACHE, GET_OBJECT_PATH_CODEC_STREAMING,
GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_EMPTY, GET_OBJECT_PATH_INLINE_DIRECT, GET_OBJECT_PATH_INTERNAL_META,
GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_REMOTE_TRANSITION, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, GET_STAGE_EMIT,
GET_STAGE_INLINE_PREPARE, GET_STAGE_LOCK_ACQUIRE, GET_STAGE_METADATA, GET_STAGE_OBJECT_INFO, GET_STAGE_PATH_DECISION,
GET_STAGE_READER_SETUP, GenericError, GetCodecStreamingDecision, GetCodecStreamingFallbackReason, GetDirectMemoryDecision,
GetObjectReader, HTTPRangeSpec, HashAlgorithm, HashMap, HashReader, HashSet, HeaderMap, HealChannelPriority, InstanceContext,
Instant, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, OBJECT_OP_IGNORED_ERRS, ObjectApiError, ObjectInfo, ObjectKey,
AMZ_OBJECT_TAGGING, Arc, AsyncWrite, AtomicU64, BufReader, Bytes, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING,
CONTENT_LANGUAGE, CONTENT_TYPE, CompletePart, Cursor, DeleteAccounting, DeleteOptions, DeletedObject, DiskError, DiskStore,
EVENT_SET_DISK_COMMIT_TAIL_SLOW, EVENT_SET_DISK_PUT_OBJECT_STAGE_SUMMARY, EVENT_SET_DISK_WRITE, EXPIRES, Error, EventArgs,
EventName, FastLockGuard, FileInfo, FileInfoVersions, GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART,
GET_OBJECT_PATH_BODY_CACHE, GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_EMPTY,
GET_OBJECT_PATH_INLINE_DIRECT, GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX,
GET_OBJECT_PATH_REMOTE_TRANSITION, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, GET_STAGE_EMIT, GET_STAGE_INLINE_PREPARE,
GET_STAGE_LOCK_ACQUIRE, GET_STAGE_METADATA, GET_STAGE_OBJECT_INFO, GET_STAGE_PATH_DECISION, GET_STAGE_READER_SETUP,
GenericError, GetCodecStreamingDecision, GetCodecStreamingFallbackReason, GetDirectMemoryDecision, GetObjectReader,
HTTPRangeSpec, HashAlgorithm, HashMap, HashReader, HashSet, HeaderMap, HealChannelPriority, InstanceContext, Instant,
LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, OBJECT_OP_IGNORED_ERRS, ObjectApiError, ObjectInfo, ObjectKey,
ObjectLockConfigSnapshot, ObjectLockConfigState, ObjectOptions, ObjectReader, ObjectToDelete, OffsetDateTime, Ordering, Pin,
PutObjReader, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET, ReadPathPlan, ReaderImpl, ReplicateDecision,
ReplicationObjectBridge, Result, SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS, SLASH_SEPARATOR, SUFFIX_ACTUAL_SIZE,
SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK, SetDisks,
SmallWritePath, StorageError, TRANSITION_COMPLETE, UpdateMetadataOpts, Uuid, WriteLayout, X_AMZ_OBJECT_LOCK_LEGAL_HOLD,
X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_RESTORE, adaptive_duplex_buffer_size,
SmallWritePath, StorageError, TRANSITION_COMPLETE, UpdateMetadataOpts, Uuid, WriteLayout, adaptive_duplex_buffer_size,
build_get_object_info, build_inline_bitrot_readers, build_inline_bitrot_readers_from_refs, can_try_inline_data_shards_direct,
check_object_lock_delete, check_object_lock_for_deletion_with_state, check_object_lock_retention_update,
classify_get_codec_streaming_object_class, classify_put_write_path, classify_storage_error,
@@ -1831,9 +1831,9 @@ mod duration_metrics_tests {
}
fn is_restore_control_metadata(key: &str) -> bool {
key.eq_ignore_ascii_case(X_AMZ_RESTORE.as_str())
|| key.eq_ignore_ascii_case(rustfs_utils::http::headers::AMZ_RESTORE_EXPIRY_DAYS)
|| key.eq_ignore_ascii_case(rustfs_utils::http::headers::AMZ_RESTORE_REQUEST_DATE)
key.eq_ignore_ascii_case(metadata_keys::RESTORE)
|| key.eq_ignore_ascii_case(metadata_keys::RESTORE_EXPIRY_DAYS)
|| key.eq_ignore_ascii_case(metadata_keys::RESTORE_REQUEST_DATE)
|| rustfs_utils::http::internal_key_strip_suffix_prefix(key, SUFFIX_RESTORE_OPERATION_ID)
.is_some_and(|remainder| remainder.is_empty())
|| rustfs_utils::http::internal_key_strip_suffix_prefix(key, SUFFIX_RESTORE_WORKER_LOCK)
@@ -1861,6 +1861,43 @@ fn restore_metadata_update_preserves_protected_metadata(
mod restore_metadata_update_tests {
use super::*;
/// The restore keys in pre-`metadata_keys` xl.meta are exactly the ones
/// a restore metadata update may change; every other persisted key is
/// protected (backlog#1735 A3b).
#[test]
fn pre_module_xlmeta_restore_keys_are_the_restore_control_keys() {
let fi = rustfs_filemeta::FileMeta::load(
&rustfs_filemeta::test_data::create_pre_metadata_keys_xlmeta().expect("decode fixture hex"),
)
.expect("load fixture xl.meta")
.into_fileinfo("bucket", "object", "0b1e5a3a-1735-4a3a-8000-00000000a3a0", false, false, false)
.expect("fixture version to FileInfo");
let mut control: Vec<&str> = fi
.metadata
.keys()
.map(String::as_str)
.filter(|key| is_restore_control_metadata(key))
.collect();
control.sort_unstable();
assert_eq!(
control,
["X-Amz-Restore-Expiry-Days", "X-Amz-Restore-Request-Date", "x-amz-restore"],
"restore control keys in the pre-module bytes"
);
for key in [
metadata_keys::OBJECT_LOCK_LEGAL_HOLD,
metadata_keys::OBJECT_LOCK_MODE,
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE,
metadata_keys::SERVER_SIDE_ENCRYPTION,
metadata_keys::STORAGE_CLASS,
metadata_keys::REPLICATION_STATUS,
] {
assert!(fi.metadata.contains_key(key), "{key:?}");
assert!(!is_restore_control_metadata(key), "{key:?} must stay protected");
}
}
#[test]
fn restore_metadata_update_cannot_change_retention_or_user_metadata() {
let mut existing = HashMap::from([
@@ -1869,7 +1906,7 @@ mod restore_metadata_update_tests {
("x-amz-object-lock-mode".to_string(), "COMPLIANCE".to_string()),
]);
let mut replacement = existing.clone();
replacement.insert(X_AMZ_RESTORE.as_str().to_string(), "ongoing-request=\"true\"".to_string());
replacement.insert(metadata_keys::RESTORE.to_string(), "ongoing-request=\"true\"".to_string());
rustfs_utils::http::metadata_compat::insert_str(
&mut replacement,
SUFFIX_RESTORE_OPERATION_ID,
@@ -1889,9 +1926,9 @@ mod restore_metadata_update_tests {
replacement.insert("x-amz-meta-owner".to_string(), "mallory".to_string());
assert!(!restore_metadata_update_preserves_protected_metadata(&existing, &replacement));
existing.insert(X_AMZ_RESTORE.as_str().to_string(), "ongoing-request=\"false\"".to_string());
existing.insert(metadata_keys::RESTORE.to_string(), "ongoing-request=\"false\"".to_string());
replacement.clone_from(&existing);
replacement.remove(X_AMZ_RESTORE.as_str());
replacement.remove(metadata_keys::RESTORE);
assert!(restore_metadata_update_preserves_protected_metadata(&existing, &replacement));
}
}
@@ -3161,9 +3198,7 @@ pub(in crate::set_disk) fn merge_replication_metadata_lww(
existing: &HashMap<String, String>,
opts: &ObjectOptions,
) -> bool {
use rustfs_utils::http::headers::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING,
};
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::metadata_compat::{
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, get_str,
remove_str,
@@ -3175,12 +3210,12 @@ pub(in crate::set_disk) fn merge_replication_metadata_lww(
(
opts.replication_retention_timestamp,
SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
&[AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER],
&[metadata_keys::OBJECT_LOCK_MODE, metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE],
),
(
opts.replication_legalhold_timestamp,
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
&[AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER],
&[metadata_keys::OBJECT_LOCK_LEGAL_HOLD],
),
];
@@ -3486,7 +3521,7 @@ impl SetDisks {
self.pool_index,
disks.len(),
self.default_parity_count,
user_defined.get(AMZ_STORAGE_CLASS).map(String::as_str),
user_defined.get(metadata_keys::STORAGE_CLASS).map(String::as_str),
opts.max_parity,
)?;
@@ -3793,10 +3828,10 @@ impl SetDisks {
fi.checksum = Some(content_hash.to_bytes(&[]));
}
if let Some(sc) = user_defined.get(AMZ_STORAGE_CLASS)
if let Some(sc) = user_defined.get(metadata_keys::STORAGE_CLASS)
&& sc == storageclass::STANDARD
{
let _ = user_defined.remove(AMZ_STORAGE_CLASS);
let _ = user_defined.remove(metadata_keys::STORAGE_CLASS);
}
let mod_time = opts.mod_time;
@@ -7546,7 +7581,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
&& src_info
.user_defined
.keys()
.any(|key| key.eq_ignore_ascii_case(X_AMZ_RESTORE.as_str()))
.any(|key| key.eq_ignore_ascii_case(metadata_keys::RESTORE))
&& restore_metadata_update_preserves_protected_metadata(&fi.metadata, src_info.user_defined.as_ref());
if let Some(dst_version_id) = dst_opts.version_id.as_deref()
&& !is_meta_bucketname(dst_bucket)
@@ -9236,9 +9271,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
CONTENT_DISPOSITION,
CACHE_CONTROL,
EXPIRES,
X_AMZ_OBJECT_LOCK_MODE.as_str(),
X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str(),
X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str(),
metadata_keys::OBJECT_LOCK_MODE,
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE,
metadata_keys::OBJECT_LOCK_LEGAL_HOLD,
] {
if let Some(value) = fi.metadata.lookup(header).filter(|value| !value.is_empty()) {
transition_meta.insert(header.to_ascii_lowercase(), value.to_string());
@@ -9686,7 +9721,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
let mut restore_commit_metadata = if let Some(expected_operation_id) = expected_operation_id {
let mut metadata = HashMap::new();
metadata.insert(X_AMZ_RESTORE.as_str().to_string(), "ongoing-request=\"false\"".to_string());
metadata.insert(metadata_keys::RESTORE.to_string(), "ongoing-request=\"false\"".to_string());
rustfs_utils::http::metadata_compat::insert_str(
&mut metadata,
SUFFIX_RESTORE_OPERATION_ID,
@@ -10673,9 +10708,7 @@ mod replication_lww_tests {
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
use super::*;
use rustfs_utils::http::headers::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING,
};
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::{
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, get_str,
insert_str,
@@ -10853,8 +10886,11 @@ mod replication_lww_tests {
let mut inbound = HashMap::new();
inbound.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
insert_str(&mut inbound, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "COMPLIANCE".to_string());
inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2028-01-01T00:00:00Z".to_string());
inbound.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string());
inbound.insert(
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
"2028-01-01T00:00:00Z".to_string(),
);
insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_NEW.to_string());
let opts = ObjectOptions {
replication_request: true,
@@ -10867,7 +10903,7 @@ mod replication_lww_tests {
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(info.user_tags.as_str(), "site=local", "the stale tagging category must keep local values");
assert_eq!(
info.user_defined.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str),
info.user_defined.get(metadata_keys::OBJECT_LOCK_MODE).map(String::as_str),
Some("COMPLIANCE"),
"the newer retention category must be applied in the same write"
);
@@ -10887,12 +10923,12 @@ mod replication_lww_tests {
// LWW-reachable divergence is a stale inbound ON resurrecting a hold
// that was released more recently on this site.)
let mut local = HashMap::new();
local.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "OFF".to_string());
local.insert(metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(), "OFF".to_string());
insert_str(&mut local, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_LOCAL.to_string());
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
let mut inbound = HashMap::new();
inbound.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "ON".to_string());
inbound.insert(metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(), "ON".to_string());
insert_str(&mut inbound, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_OLD.to_string());
let opts = ObjectOptions {
replication_request: true,
@@ -10903,7 +10939,9 @@ mod replication_lww_tests {
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str),
info.user_defined
.get(metadata_keys::OBJECT_LOCK_LEGAL_HOLD)
.map(String::as_str),
Some("OFF"),
"a stale inbound legal hold must not resurrect a hold released more recently"
);
@@ -10963,8 +11001,11 @@ mod replication_lww_tests {
// path's eval_metadata stomped the metadata key with receiver-now
// (simulated by T_NEW here).
let mut inbound = HashMap::new();
inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "GOVERNANCE".to_string());
inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2028-01-01T00:00:00Z".to_string());
inbound.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), "GOVERNANCE".to_string());
inbound.insert(
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
"2028-01-01T00:00:00Z".to_string(),
);
insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_NEW.to_string());
let opts = ObjectOptions {
replication_request: true,
@@ -10979,7 +11020,10 @@ mod replication_lww_tests {
Some(T_LOCAL),
"the stored category timestamp must be the source-authored time, not the receiver's clock"
);
assert_eq!(info.user_defined.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str), Some("GOVERNANCE"));
assert_eq!(
info.user_defined.get(metadata_keys::OBJECT_LOCK_MODE).map(String::as_str),
Some("GOVERNANCE")
);
}
#[tokio::test]
@@ -10991,7 +11035,7 @@ mod replication_lww_tests {
make_bucket(&disk_stores, bucket).await;
let mut inbound = HashMap::new();
inbound.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "OFF".to_string());
inbound.insert(metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(), "OFF".to_string());
insert_str(&mut inbound, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_OLD.to_string());
let mut evaluated = inbound.clone();
insert_str(&mut evaluated, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_NEW.to_string());
@@ -11045,10 +11089,13 @@ mod replication_lww_tests {
/// plus an active COMPLIANCE retention (no retention timestamp).
async fn seed_locked_version(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str, hold_timestamp: &str) {
let mut local = HashMap::new();
local.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "ON".to_string());
local.insert(metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(), "ON".to_string());
insert_str(&mut local, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, hold_timestamp.to_string());
local.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "COMPLIANCE".to_string());
local.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2099-01-01T00:00:00Z".to_string());
local.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string());
local.insert(
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
"2099-01-01T00:00:00Z".to_string(),
);
put_version(set_disks, bucket, object, version_id, &versioned_opts(version_id, local)).await;
}
@@ -11057,10 +11104,13 @@ mod replication_lww_tests {
/// category the source version has.
fn inbound_legal_hold_release_opts(version_id: &str, timestamp: &str) -> ObjectOptions {
let mut inbound = HashMap::new();
inbound.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "OFF".to_string());
inbound.insert(metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(), "OFF".to_string());
insert_str(&mut inbound, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, timestamp.to_string());
inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "COMPLIANCE".to_string());
inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2099-01-01T00:00:00Z".to_string());
inbound.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string());
inbound.insert(
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
"2099-01-01T00:00:00Z".to_string(),
);
insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_OLD.to_string());
ObjectOptions {
replication_request: true,
@@ -11094,13 +11144,15 @@ mod replication_lww_tests {
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str),
info.user_defined
.get(metadata_keys::OBJECT_LOCK_LEGAL_HOLD)
.map(String::as_str),
Some("OFF"),
"a newer source-side legal hold release must be applied to the locked replica"
);
assert_eq!(get_str(&info.user_defined, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP).as_deref(), Some(T_NEW));
assert_eq!(
info.user_defined.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str),
info.user_defined.get(metadata_keys::OBJECT_LOCK_MODE).map(String::as_str),
Some("COMPLIANCE"),
"the untouched retention category must survive the write"
);
@@ -11128,7 +11180,9 @@ mod replication_lww_tests {
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str),
info.user_defined
.get(metadata_keys::OBJECT_LOCK_LEGAL_HOLD)
.map(String::as_str),
Some("ON"),
"a stale inbound release must not lift a hold applied more recently on this site"
);
@@ -11154,8 +11208,11 @@ mod replication_lww_tests {
let mut inbound = HashMap::new();
inbound.insert(AMZ_OBJECT_TAGGING.to_string(), "k=v".to_string());
insert_str(&mut inbound, SUFFIX_TAGGING_TIMESTAMP, T_NEW.to_string());
inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "COMPLIANCE".to_string());
inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2099-01-01T00:00:00Z".to_string());
inbound.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string());
inbound.insert(
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
"2099-01-01T00:00:00Z".to_string(),
);
let opts = ObjectOptions {
replication_request: true,
replication_tagging_timestamp: Some(parse_ts(T_NEW)),
@@ -11171,7 +11228,12 @@ mod replication_lww_tests {
assert!(matches!(err, StorageError::PrefixAccessDenied(_, _)), "unexpected error: {err}");
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str), Some("ON"));
assert_eq!(
info.user_defined
.get(metadata_keys::OBJECT_LOCK_LEGAL_HOLD)
.map(String::as_str),
Some("ON")
);
}
fn default_retention_snapshot(mode: &'static str) -> Arc<ObjectLockConfigSnapshot> {
@@ -11209,7 +11271,7 @@ mod replication_lww_tests {
seed_local_tagged_version(&set_disks, bucket, object, &version_id).await;
let seeded = version_info(&set_disks, bucket, object, &version_id).await;
assert!(
!seeded.user_defined.contains_key(AMZ_OBJECT_LOCK_MODE_LOWER),
!seeded.user_defined.contains_key(metadata_keys::OBJECT_LOCK_MODE),
"the seeded version must be protected by the bucket default only"
);
@@ -11254,7 +11316,7 @@ mod replication_lww_tests {
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
let mut local = HashMap::new();
local.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "MAYBE".to_string());
local.insert(metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(), "MAYBE".to_string());
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
let mut reader = PutObjReader::from_vec(b"lww-body".to_vec());
@@ -11265,7 +11327,12 @@ mod replication_lww_tests {
assert!(!matches!(err, StorageError::PrefixAccessDenied(_, _)), "unexpected error: {err}");
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str), Some("MAYBE"));
assert_eq!(
info.user_defined
.get(metadata_keys::OBJECT_LOCK_LEGAL_HOLD)
.map(String::as_str),
Some("MAYBE")
);
}
/// The bypass is scoped to authorized replication writes: the same
@@ -11291,7 +11358,12 @@ mod replication_lww_tests {
assert!(matches!(err, StorageError::PrefixAccessDenied(_, _)), "unexpected error: {err}");
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str), Some("ON"));
assert_eq!(
info.user_defined
.get(metadata_keys::OBJECT_LOCK_LEGAL_HOLD)
.map(String::as_str),
Some("ON")
);
}
}
@@ -13223,7 +13295,10 @@ mod transition_commit_failure_tests {
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_WORKER_LOCK,
rustfs_utils::http::metadata_compat::RESTORE_WORKER_LOCK_PROTOCOL_V1.to_string(),
);
metadata.insert(s3s::header::X_AMZ_RESTORE.as_str().to_string(), format!("ongoing-request=\"{ongoing}\""));
metadata.insert(
rustfs_filemeta::metadata_keys::RESTORE.to_string(),
format!("ongoing-request=\"{ongoing}\""),
);
metadata
}
@@ -13416,7 +13491,7 @@ mod transition_commit_failure_tests {
.await
.expect("failed restore cleanup should leave the transitioned object readable");
assert!(
!cleaned.user_defined.contains_key(s3s::header::X_AMZ_RESTORE.as_str()),
!cleaned.user_defined.contains_key(rustfs_filemeta::metadata_keys::RESTORE),
"{point:?}: every post-snapshot failure must clean the public ongoing marker"
);
assert!(
@@ -13492,7 +13567,7 @@ mod transition_commit_failure_tests {
);
let restore_header = restored
.user_defined
.get(s3s::header::X_AMZ_RESTORE.as_str())
.get(rustfs_filemeta::metadata_keys::RESTORE)
.expect("successful multipart restore must persist restore status");
let restore_status = parse_restore_obj_status(restore_header).expect("successful restore status must be valid");
assert!(!restore_status.on_going(), "successful multipart restore must not remain in progress");
@@ -13978,7 +14053,7 @@ mod transition_commit_failure_tests {
}
return;
}
assert!(!cleaned.user_defined.contains_key(s3s::header::X_AMZ_RESTORE.as_str()));
assert!(!cleaned.user_defined.contains_key(rustfs_filemeta::metadata_keys::RESTORE));
assert!(
rustfs_utils::http::get_str(cleaned.user_defined.as_ref(), rustfs_utils::http::SUFFIX_RESTORE_OPERATION_ID,)
.is_none()
@@ -14084,7 +14159,7 @@ mod transition_commit_failure_tests {
assert_eq!(restored.version_id, Some(version_id), "restore must preserve the selected {case} version");
let restore_header = restored
.user_defined
.get(s3s::header::X_AMZ_RESTORE.as_str())
.get(rustfs_filemeta::metadata_keys::RESTORE)
.expect("restored version must carry its completed restore status");
let restore_status = parse_restore_obj_status(restore_header).expect("restore status must be valid");
assert!(!restore_status.on_going(), "restored {case} version must not remain in progress");
@@ -14744,7 +14819,7 @@ mod transition_commit_failure_tests {
assert!(
current_operation_b
.user_defined
.contains_key(s3s::header::X_AMZ_RESTORE.as_str()),
.contains_key(rustfs_filemeta::metadata_keys::RESTORE),
"stale cleanup for operation A must not remove operation B's restore header"
);
assert_eq!(
@@ -14779,7 +14854,7 @@ mod transition_commit_failure_tests {
.await
.expect("cleaned object metadata should remain readable");
assert!(
!cleaned.user_defined.contains_key(s3s::header::X_AMZ_RESTORE.as_str()),
!cleaned.user_defined.contains_key(rustfs_filemeta::metadata_keys::RESTORE),
"matching cleanup must remove the restore header"
);
assert!(
@@ -14848,7 +14923,7 @@ mod transition_commit_failure_tests {
let restore_status = parse_restore_obj_status(
current
.user_defined
.get(s3s::header::X_AMZ_RESTORE.as_str())
.get(rustfs_filemeta::metadata_keys::RESTORE)
.expect("restore header must remain pending"),
)
.expect("restore header should remain parseable");
@@ -14912,7 +14987,7 @@ mod transition_commit_failure_tests {
.await
.expect("restore metadata should remain readable");
assert!(
current.user_defined.contains_key(s3s::header::X_AMZ_RESTORE.as_str()),
current.user_defined.contains_key(rustfs_filemeta::metadata_keys::RESTORE),
"lost no_lock cleanup must not remove the restore header"
);
assert_eq!(
@@ -15130,7 +15205,7 @@ mod transition_commit_failure_tests {
parse_restore_obj_status(
current
.user_defined
.get(s3s::header::X_AMZ_RESTORE.as_str())
.get(rustfs_filemeta::metadata_keys::RESTORE)
.expect("operation B restore header should remain pending"),
)
.expect("operation B restore header should parse")
@@ -15798,7 +15873,7 @@ mod transition_upload_integrity_tests {
rustfs_filemeta::parse_restore_obj_status(
current
.user_defined
.get(s3s::header::X_AMZ_RESTORE.as_str())
.get(rustfs_filemeta::metadata_keys::RESTORE)
.expect("pending restore header should remain"),
)
.expect("restore header should parse")
@@ -19739,26 +19814,26 @@ mod put_object_tmp_cleanup_tests {
"compliance",
HashMap::from([
(
X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_MODE.to_string(),
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
),
(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(), retain_until.clone()),
(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), retain_until.clone()),
]),
),
(
"governance",
HashMap::from([
(
X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_MODE.to_string(),
s3s::dto::ObjectLockRetentionMode::GOVERNANCE.to_string(),
),
(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(), retain_until),
(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), retain_until),
]),
),
(
"legal-hold",
HashMap::from([(
X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(),
metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(),
s3s::dto::ObjectLockLegalHoldStatus::ON.to_string(),
)]),
),
@@ -20059,9 +20134,9 @@ mod put_object_tmp_cleanup_tests {
object,
&ObjectOptions {
eval_metadata: Some(HashMap::from([
(X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(), String::new()),
(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(), String::new()),
(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), String::new()),
(metadata_keys::OBJECT_LOCK_MODE.to_string(), String::new()),
(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), String::new()),
(metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(), String::new()),
])),
..version_opts.clone()
},
@@ -20128,7 +20203,7 @@ mod put_object_tmp_cleanup_tests {
version_id: Some(destination_version.clone()),
versioned: true,
eval_metadata: Some(HashMap::from([(
X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(),
metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(),
s3s::dto::ObjectLockLegalHoldStatus::ON.to_string(),
)])),
..Default::default()
@@ -21089,11 +21164,11 @@ mod delete_objects_lock_gating_tests {
let retain_until = OffsetDateTime::now_utc() + Duration::from_secs(60 * 60 * 24 * 30);
let mut user_defined = HashMap::new();
user_defined.insert(
X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_MODE.to_string(),
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
);
user_defined.insert(
X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
retain_until
.format(&time::format_description::well_known::Rfc3339)
.expect("retain-until date should format"),
crates/ecstore/src/set_disk/replication.rs
+6 -6
View File
@@ -14,12 +14,12 @@
use super::{
Error, FileInfo, NamespaceLockFence, ObjectInfo, ObjectOptions, OffsetDateTime, Result, SetDisks, StorageError,
UpdateMetadataOpts, Uuid, X_AMZ_RESTORE, get_raw_etag, restore_operation_id_from_metadata,
UpdateMetadataOpts, Uuid, get_raw_etag, restore_operation_id_from_metadata,
};
use crate::bucket::lifecycle::lifecycle;
use crate::core::pools::DecommissionCapacityAdmission;
use rustfs_filemeta::RestoreStatusOps;
use rustfs_utils::http::headers::{AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE};
use rustfs_filemeta::metadata_keys;
use s3s::dto::{RestoreStatus, Timestamp};
#[cfg(all(test, feature = "test-util"))]
use std::sync::Arc;
@@ -213,7 +213,7 @@ impl SetDisks {
let restore_expiry =
lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), opts.transition.restore_request.days.unwrap_or(1));
fi.metadata.insert(
X_AMZ_RESTORE.as_str().to_string(),
metadata_keys::RESTORE.to_string(),
RestoreStatus {
is_restore_in_progress: Some(false),
restore_expiry_date: Some(Timestamp::from(restore_expiry)),
@@ -317,9 +317,9 @@ impl SetDisks {
return Ok(());
}
ensure_restore_metadata_lock_held(bucket, object, opts, "restore_cleanup_metadata")?;
fi.metadata.remove(X_AMZ_RESTORE.as_str());
fi.metadata.remove(AMZ_RESTORE_EXPIRY_DAYS);
fi.metadata.remove(AMZ_RESTORE_REQUEST_DATE);
fi.metadata.remove(metadata_keys::RESTORE);
fi.metadata.remove(metadata_keys::RESTORE_EXPIRY_DAYS);
fi.metadata.remove(metadata_keys::RESTORE_REQUEST_DATE);
rustfs_utils::http::metadata_compat::remove_str(
&mut fi.metadata,
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_OPERATION_ID,
crates/ecstore/src/set_disk/transition_matrix_tests.rs
+1 -1
View File
@@ -145,7 +145,7 @@ async fn transition_and_restore_reclaim_prior_metadata_generations() {
let restore_status = parse_restore_obj_status(
restored_info
.user_defined
.get(s3s::header::X_AMZ_RESTORE.as_str())
.get(rustfs_filemeta::metadata_keys::RESTORE)
.expect("completed restore header should be present"),
)
.expect("completed restore header should parse");
crates/ecstore/src/store/init.rs
+7 -7
View File
@@ -5665,7 +5665,7 @@ mod tests {
metadata_acknowledged_target,
&ObjectOptions {
eval_metadata: Some(HashMap::from([(
s3s::header::X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(),
rustfs_filemeta::metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(),
s3s::dto::ObjectLockLegalHoldStatus::OFF.to_string(),
)])),
..target_version_opts.clone()
@@ -5694,7 +5694,7 @@ mod tests {
assert_eq!(
metadata_acknowledged
.user_defined
.get(s3s::header::X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str())
.get(rustfs_filemeta::metadata_keys::OBJECT_LOCK_LEGAL_HOLD)
.map(String::as_str),
Some("OFF")
);
@@ -5724,9 +5724,9 @@ mod tests {
("governance-target.bin", s3s::dto::ObjectLockRetentionMode::GOVERNANCE),
] {
let retained_metadata = HashMap::from([
(s3s::header::X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(), mode.to_string()),
(rustfs_filemeta::metadata_keys::OBJECT_LOCK_MODE.to_string(), mode.to_string()),
(
s3s::header::X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(),
rustfs_filemeta::metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
retain_until.clone(),
),
]);
@@ -12441,7 +12441,7 @@ mod tests {
.await
.expect("restored transitioned source should remain readable");
assert!(
restored.user_defined.contains_key(s3s::header::X_AMZ_RESTORE.as_str()),
restored.user_defined.contains_key(rustfs_filemeta::metadata_keys::RESTORE),
"restore completion metadata must be present before the delete regression"
);
}
@@ -16748,11 +16748,11 @@ mod tests {
&ObjectOptions {
user_defined: HashMap::from([
(
s3s::header::X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
rustfs_filemeta::metadata_keys::OBJECT_LOCK_MODE.to_string(),
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
),
(
s3s::header::X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(),
rustfs_filemeta::metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
"2099-01-01T00:00:00Z".to_string(),
),
]),
crates/ecstore/src/store/utils.rs
+4 -3
View File
@@ -15,7 +15,8 @@
use crate::config::storageclass::STANDARD;
use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
use regex::Regex;
use rustfs_utils::http::headers::{AMZ_OBJECT_TAGGING, AMZ_STORAGE_CLASS};
use rustfs_filemeta::metadata_keys;
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use std::collections::HashMap;
use std::io::{Error, Result};
use std::sync::LazyLock;
@@ -32,8 +33,8 @@ pub fn clean_metadata(metadata: &mut HashMap<String, String>) {
}
pub fn remove_standard_storage_class(metadata: &mut HashMap<String, String>) {
if metadata.get(AMZ_STORAGE_CLASS) == Some(&STANDARD.to_string()) {
metadata.remove(AMZ_STORAGE_CLASS);
if metadata.get(metadata_keys::STORAGE_CLASS) == Some(&STANDARD.to_string()) {
metadata.remove(metadata_keys::STORAGE_CLASS);
}
}
crates/lifecycle/src/evaluator.rs
+6 -6
View File
@@ -206,12 +206,12 @@ mod tests {
use std::sync::Arc;
use rustfs_scanner_metrics::metrics::IlmAction;
use rustfs_storage_api::metadata_keys;
use s3s::dto::{
BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleExpiration, LifecycleRule,
NoncurrentVersionExpiration, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule,
Transition, TransitionStorageClass,
};
use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
use time::OffsetDateTime;
use uuid::Uuid;
@@ -251,7 +251,7 @@ mod tests {
for object in &mut objects {
object
.user_defined
.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), "ON".to_string());
.insert(metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(), "ON".to_string());
}
let locked = evaluator
.eval(&objects)
@@ -485,7 +485,7 @@ mod tests {
fn locked_current_object_opts(replication_status: ReplicationStatusType) -> ObjectOpts {
let mut user_defined = HashMap::new();
user_defined.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), "ON".to_string());
user_defined.insert(metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(), "ON".to_string());
ObjectOpts {
user_defined,
@@ -507,10 +507,10 @@ mod tests {
.expect("future retain-until date should format");
let mut user_defined = HashMap::new();
user_defined.insert(
X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_MODE.to_string(),
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
);
user_defined.insert(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(), retain_until);
user_defined.insert(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), retain_until);
ObjectOpts {
user_defined,
@@ -717,7 +717,7 @@ mod tests {
.eval(&version_group(
Some(Uuid::nil()),
ReplicationStatusType::Completed,
HashMap::from([(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), "ON".to_string())]),
HashMap::from([(metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(), "ON".to_string())]),
))
.await
.expect("locked null-version lifecycle evaluation should fail closed without aborting evaluation");
crates/lifecycle/src/object_lock.rs
+14 -14
View File
@@ -14,8 +14,8 @@
use std::collections::HashMap;
use rustfs_storage_api::metadata_keys;
use s3s::dto::{ObjectLockConfiguration, ObjectLockRetentionMode};
use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
use time::{OffsetDateTime, format_description};
pub fn is_object_locked_by_metadata(user_defined: &HashMap<String, String>, is_delete_marker: bool) -> bool {
@@ -24,13 +24,13 @@ pub fn is_object_locked_by_metadata(user_defined: &HashMap<String, String>, is_d
}
if user_defined
.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str())
.get(metadata_keys::OBJECT_LOCK_LEGAL_HOLD)
.is_some_and(|value| value.eq_ignore_ascii_case("ON"))
{
return true;
}
let Some(mode) = user_defined.get(X_AMZ_OBJECT_LOCK_MODE.as_str()) else {
let Some(mode) = user_defined.get(metadata_keys::OBJECT_LOCK_MODE) else {
return false;
};
if !is_retention_mode(mode) {
@@ -38,7 +38,7 @@ pub fn is_object_locked_by_metadata(user_defined: &HashMap<String, String>, is_d
}
user_defined
.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str())
.get(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE)
.and_then(|value| OffsetDateTime::parse(value, &format_description::well_known::Iso8601::DEFAULT).ok())
.is_some_and(|retain_until| retain_until.unix_timestamp() > OffsetDateTime::now_utc().unix_timestamp())
}
@@ -85,22 +85,22 @@ pub fn is_object_locked(
}
fn has_explicit_lock_metadata(user_defined: &HashMap<String, String>) -> bool {
user_defined.contains_key(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str())
|| user_defined.contains_key(X_AMZ_OBJECT_LOCK_MODE.as_str())
|| user_defined.contains_key(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str())
user_defined.contains_key(metadata_keys::OBJECT_LOCK_LEGAL_HOLD)
|| user_defined.contains_key(metadata_keys::OBJECT_LOCK_MODE)
|| user_defined.contains_key(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE)
}
fn explicit_lock_metadata_is_well_formed(user_defined: &HashMap<String, String>) -> bool {
if user_defined
.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str())
.get(metadata_keys::OBJECT_LOCK_LEGAL_HOLD)
.is_some_and(|value| !value.eq_ignore_ascii_case("ON") && !value.eq_ignore_ascii_case("OFF"))
{
return false;
}
match (
user_defined.get(X_AMZ_OBJECT_LOCK_MODE.as_str()),
user_defined.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str()),
user_defined.get(metadata_keys::OBJECT_LOCK_MODE),
user_defined.get(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE),
) {
(None, None) => true,
(Some(mode), Some(retain_until)) => {
@@ -142,7 +142,7 @@ mod tests {
#[test]
fn is_object_locked_by_metadata_preserves_object_lock_parser_behavior() {
let mut user_defined = HashMap::new();
user_defined.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), "ON".to_string());
user_defined.insert(metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(), "ON".to_string());
assert!(is_object_locked_by_metadata(&user_defined, false));
assert!(!is_object_locked_by_metadata(&user_defined, true));
@@ -231,11 +231,11 @@ mod tests {
let config = default_retention_config(30);
let mut user_defined = HashMap::new();
user_defined.insert(
X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_MODE.to_string(),
ObjectLockRetentionMode::GOVERNANCE.to_string(),
);
user_defined.insert(
X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
(OffsetDateTime::now_utc() - Duration::days(1))
.format(&format_description::well_known::Iso8601::DEFAULT)
.expect("expired retention date should format"),
@@ -249,7 +249,7 @@ mod tests {
let config = default_retention_config(1);
let mut user_defined = HashMap::new();
user_defined.insert(
X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
metadata_keys::OBJECT_LOCK_MODE.to_string(),
ObjectLockRetentionMode::GOVERNANCE.to_string(),
);
let created = OffsetDateTime::now_utc() - Duration::days(2);
crates/storage-api/src/lib.rs
+4
View File
@@ -101,6 +101,10 @@ pub use object::{VersionMarker, WalkOptions, WalkVersionsSortOrder};
pub use observability::{
MemorySamplingState, ObservabilitySnapshot, ObservabilitySnapshotProvider, PlatformSupport, UserspaceProfilingCapability,
};
/// Object-metadata keys persisted in xl.meta `meta_user`. filemeta owns the
/// on-disk spelling; contract consumers that read or write persisted object
/// metadata (lifecycle object-lock checks) take the keys from here.
pub use rustfs_filemeta::metadata_keys;
pub use topology::{
DiskCapabilities, TopologyCapabilities, TopologyDisk, TopologyLabels, TopologyPool, TopologySet, TopologySnapshot,
TopologySnapshotProvider,
scripts/check_s3s_footprint.sh
+2 -2
View File
@@ -61,14 +61,14 @@ cd "$(dirname "$0")/.."
# 213 -> 212 on 2026-09-14: rustfs/backlog#1735 A4 moved rio's trailer
# handle behind rustfs_rio::TrailerSource; the only adapter imports s3s through
# the app storage_api shim, so crates/rio no longer references s3s.
S3S_IMPORT_FILES_BASELINE=212
S3S_IMPORT_FILES_BASELINE=210
S3_ERROR_LINES_BASELINE=1588
# ecstore-scoped ratchet (rustfs/backlog#1842): the storage engine must not
# know S3 wire/DTO types (ARCHITECTURE.md invariant 4). The S3-*consuming*
# client was extracted to crates/s3-client, where s3s usage is legitimate;
# this counter ratchets the remaining serving-side s3s references out of
# crates/ecstore. Baseline verified on 2026-08-26.
S3S_ECSTORE_FILES_BASELINE=39
S3S_ECSTORE_FILES_BASELINE=37
S3S_PATH_PATTERN='(^|[^"[:alnum:]_])s3s::'
E2E_TEST_GLOB='--glob=!crates/e2e_test/**'