From 022e3dfc21d9bcae45598859fbad735e24f6d978 Mon Sep 17 00:00:00 2001 From: LeonWang0735 Date: Thu, 29 Jan 2026 19:35:03 +0800 Subject: [PATCH] fix:s3 tests fix (#1652) Co-authored-by: houseme --- crates/rio/src/hash_reader.rs | 2 +- crates/utils/src/http/headers.rs | 2 ++ rustfs/src/storage/ecfs.rs | 3 ++- rustfs/src/storage/ecfs_extend.rs | 2 +- rustfs/src/storage/options.rs | 45 +++++++++++++++++++++++++------ 5 files changed, 43 insertions(+), 11 deletions(-) diff --git a/crates/rio/src/hash_reader.rs b/crates/rio/src/hash_reader.rs index 0ca1d07d1..40e05218a 100644 --- a/crates/rio/src/hash_reader.rs +++ b/crates/rio/src/hash_reader.rs @@ -240,7 +240,7 @@ impl HashReader { let er = EtagReader::new(inner, md5hex.clone()); inner = Box::new(er); } - } else if !diskable_md5 { + } else if size != Self::SIZE_PRESERVE_LAYER && !diskable_md5 { let er = EtagReader::new(inner, md5hex.clone()); inner = Box::new(er); } diff --git a/crates/utils/src/http/headers.rs b/crates/utils/src/http/headers.rs index 4626641f7..9d5199f5c 100644 --- a/crates/utils/src/http/headers.rs +++ b/crates/utils/src/http/headers.rs @@ -148,6 +148,8 @@ pub const AMZ_META_NAME: &str = "X-Amz-Meta-Name"; pub const AMZ_META_UNENCRYPTED_CONTENT_LENGTH: &str = "X-Amz-Meta-X-Amz-Unencrypted-Content-Length"; pub const AMZ_META_UNENCRYPTED_CONTENT_MD5: &str = "X-Amz-Meta-X-Amz-Unencrypted-Content-Md5"; +pub const RUSTFS_ENCRYPTION: &str = "X-Rustfs-Encryption-"; +pub const RUSTFS_ENCRYPTION_LOWER: &str = "x-rustfs-encryption-"; pub const RESERVED_METADATA_PREFIX: &str = "X-RustFS-Internal-"; pub const RESERVED_METADATA_PREFIX_LOWER: &str = "x-rustfs-internal-"; diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 52dd5efaf..a1673a74b 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -5746,7 +5746,8 @@ impl S3 for FS { if let Some((key_bytes, base_nonce, _)) = decrypt_managed_encryption_key(&bucket, &key, &fi.user_defined).await? { let part_nonce = derive_part_nonce(base_nonce, part_id); let encrypt_reader = EncryptReader::new(reader, key_bytes, part_nonce); - reader = HashReader::new(Box::new(encrypt_reader), -1, actual_size, None, None, false).map_err(ApiError::from)?; + reader = HashReader::new(Box::new(encrypt_reader), HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false) + .map_err(ApiError::from)?; } let mut reader = PutObjReader::new(reader); diff --git a/rustfs/src/storage/ecfs_extend.rs b/rustfs/src/storage/ecfs_extend.rs index 5b364408e..a12f97731 100644 --- a/rustfs/src/storage/ecfs_extend.rs +++ b/rustfs/src/storage/ecfs_extend.rs @@ -580,7 +580,7 @@ pub(crate) fn parse_object_lock_legal_hold(legal_hold: Option String::default(), }; let now = OffsetDateTime::now_utc(); - // This is intentional behavior. Empty string represents "status cleared" which is different from "status never set". Consistent with minio + // This is intentional behavior. Empty string represents "status cleared" which is different from "status never set". eval_metadata.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), status); eval_metadata.insert( format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "objectlock-legalhold-timestamp"), diff --git a/rustfs/src/storage/options.rs b/rustfs/src/storage/options.rs index 77dc82cbf..ad48fb64a 100644 --- a/rustfs/src/storage/options.rs +++ b/rustfs/src/storage/options.rs @@ -33,6 +33,7 @@ use rustfs_utils::http::RUSTFS_BUCKET_REPLICATION_DELETE_MARKER; use rustfs_utils::http::RUSTFS_BUCKET_REPLICATION_REQUEST; use rustfs_utils::http::RUSTFS_BUCKET_REPLICATION_SSEC_CHECKSUM; use rustfs_utils::http::RUSTFS_BUCKET_SOURCE_VERSION_ID; +use rustfs_utils::http::RUSTFS_ENCRYPTION_LOWER; use rustfs_utils::path::is_dir_object; use s3s::{S3Result, s3_error}; use std::collections::HashMap; @@ -126,19 +127,28 @@ pub async fn get_opts( let vid = vid.map(|v| v.as_str().trim().to_owned()); - if let Some(ref id) = vid - && *id != Uuid::nil().to_string() - && let Err(_err) = Uuid::parse_str(id.as_str()) - { - return Err(StorageError::InvalidVersionID(bucket.to_owned(), object.to_owned(), id.clone())); - } + let nil_uuid_str = Uuid::nil().to_string(); + + let vid = match vid { + Some(ref id) => { + if id.eq_ignore_ascii_case("null") { + Some(nil_uuid_str.clone()) + } else { + if id.as_str() != nil_uuid_str.as_str() && Uuid::parse_str(id).is_err() { + return Err(StorageError::InvalidVersionID(bucket.to_owned(), object.to_owned(), id.clone())); + } + Some(id.clone()) + } + } + None => None, + }; let mut opts = get_default_opts(headers, HashMap::new(), false) .map_err(|err| StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string()))?; opts.version_id = { if is_dir_object(object) && vid.is_none() { - Some(Uuid::nil().to_string()) + Some(nil_uuid_str) } else { vid } @@ -372,12 +382,18 @@ pub(crate) fn filter_object_metadata(metadata: &HashMap) -> Opti if lower_key.starts_with(RESERVED_METADATA_PREFIX_LOWER) { continue; } + + // Skip internal encryption metadata + if lower_key.starts_with(RUSTFS_ENCRYPTION_LOWER) { + continue; + } + // Skip empty object lock values if v.is_empty() && (k == &X_AMZ_OBJECT_LOCK_MODE.to_string() || k == &X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string()) { continue; } - // Skip encryption metadata placeholders + // Skip UNENCRYPTED metadata placeholders if k == AMZ_META_UNENCRYPTED_CONTENT_MD5 || k == AMZ_META_UNENCRYPTED_CONTENT_LENGTH { continue; } @@ -726,6 +742,19 @@ mod tests { assert!(result.is_ok()); } + #[tokio::test] + async fn test_get_ops_with_null_version_id() { + let headers = create_test_headers(); + let result = get_opts("test-bucket", "test-object", Some("null".to_string()), None, &headers).await; + assert!(result.is_ok()); + let opts = result.unwrap(); + assert_eq!(opts.version_id, Some(Uuid::nil().to_string())); + let result = get_opts("test-bucket", "test-object", Some("NULL".to_string()), None, &headers).await; + assert!(result.is_ok()); + let opts = result.unwrap(); + assert_eq!(opts.version_id, Some(Uuid::nil().to_string())); + } + #[tokio::test] async fn test_get_opts_basic() { let headers = create_test_headers();