From 939a69f9c2e10c1c3179baec72d28a82a825e25a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=89=E6=AD=A3=E8=B6=85?= Date: Sun, 29 Mar 2026 19:19:45 +0800 Subject: [PATCH] test(s3): complete snowball auto-extract compatibility (#2324) --- .../src/snowball_auto_extract_test.rs | 137 ++++++++++++++++++ crates/utils/src/http/headers.rs | 20 ++- rustfs/src/app/object_usecase.rs | 93 ++++++++++-- 3 files changed, 234 insertions(+), 16 deletions(-) diff --git a/crates/e2e_test/src/snowball_auto_extract_test.rs b/crates/e2e_test/src/snowball_auto_extract_test.rs index 743d654b6..2b64a7953 100644 --- a/crates/e2e_test/src/snowball_auto_extract_test.rs +++ b/crates/e2e_test/src/snowball_auto_extract_test.rs @@ -107,6 +107,97 @@ mod tests { Ok(()) } + #[tokio::test] + #[serial] + async fn snowball_auto_extract_supports_standard_headers_with_combined_extract_options() + -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let client = env.create_s3_client(); + let bucket = "snowball-standard-options"; + let extracted_prefix = "/tenant-standard/"; + + client.create_bucket().bucket(bucket).send().await?; + + let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new())); + + let mut dir_header = tokio_tar::Header::new_gnu(); + dir_header.set_entry_type(tokio_tar::EntryType::Directory); + dir_header.set_size(0); + dir_header.set_mode(0o755); + dir_header.set_cksum(); + builder + .append_data(&mut dir_header, "ignored-dir/", Cursor::new(Vec::new())) + .await?; + + let mut valid_header = tokio_tar::Header::new_gnu(); + valid_header.set_size(b"standard-body".len() as u64); + valid_header.set_mode(0o644); + valid_header.set_cksum(); + builder + .append_data(&mut valid_header, "valid.txt", Cursor::new(b"standard-body".as_slice())) + .await?; + + let long_name = format!("{}.txt", "a".repeat(1100)); + let mut invalid_header = tokio_tar::Header::new_gnu(); + invalid_header.set_size(b"ignored-body".len() as u64); + invalid_header.set_mode(0o644); + invalid_header.set_cksum(); + builder + .append_data(&mut invalid_header, long_name, Cursor::new(b"ignored-body".as_slice())) + .await?; + + let archive = builder.into_inner().await?.into_inner(); + + client + .put_object() + .bucket(bucket) + .key("fixture.tar") + .body(ByteStream::from(archive)) + .customize() + .mutate_request(move |req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + req.headers_mut().insert("x-amz-meta-snowball-prefix", extracted_prefix); + req.headers_mut().insert("x-amz-meta-snowball-ignore-dirs", "true"); + req.headers_mut().insert("x-amz-meta-snowball-ignore-errors", "true"); + }) + .send() + .await?; + + let valid = client + .get_object() + .bucket(bucket) + .key("tenant-standard/valid.txt") + .send() + .await?; + assert_eq!(valid.body.collect().await?.into_bytes().as_ref(), b"standard-body"); + + let dir_err = client + .head_object() + .bucket(bucket) + .key("tenant-standard/ignored-dir/") + .send() + .await + .expect_err("directory marker should be skipped when standard ignore-dirs=true"); + let dir_service_err = dir_err.into_service_error(); + assert_eq!(dir_service_err.code(), Some("NotFound")); + + let listed = client + .list_objects_v2() + .bucket(bucket) + .prefix("tenant-standard/") + .send() + .await?; + let keys: Vec<_> = listed.contents().iter().filter_map(|entry| entry.key()).collect(); + assert_eq!(keys, vec!["tenant-standard/valid.txt"]); + + env.stop_server(); + Ok(()) + } + #[tokio::test] #[serial] async fn snowball_auto_extract_ignores_directories_when_requested() -> Result<(), Box> { @@ -181,4 +272,50 @@ mod tests { env.stop_server(); Ok(()) } + + #[tokio::test] + #[serial] + async fn snowball_auto_extract_prefers_exact_minio_prefix_over_suffix_fallback() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let client = env.create_s3_client(); + let bucket = "snowball-prefix-precedence"; + let archive = build_test_archive().await?; + + client.create_bucket().bucket(bucket).send().await?; + + client + .put_object() + .bucket(bucket) + .key("fixture.tar") + .body(ByteStream::from(archive)) + .customize() + .mutate_request(|req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + req.headers_mut() + .insert("x-amz-meta-acme-snowball-prefix", "/tenant-fallback/"); + req.headers_mut().insert("x-amz-meta-minio-snowball-prefix", "/tenant-exact/"); + }) + .send() + .await?; + + let exact = client.get_object().bucket(bucket).key("tenant-exact/root.txt").send().await?; + assert_eq!(exact.body.collect().await?.into_bytes().as_ref(), b"root payload\n"); + + let fallback_err = client + .head_object() + .bucket(bucket) + .key("tenant-fallback/root.txt") + .send() + .await + .expect_err("fallback suffix header should not override exact MinIO prefix"); + let fallback_service_err = fallback_err.into_service_error(); + assert_eq!(fallback_service_err.code(), Some("NotFound")); + + env.stop_server(); + Ok(()) + } } diff --git a/crates/utils/src/http/headers.rs b/crates/utils/src/http/headers.rs index a0488a463..db9fead1e 100644 --- a/crates/utils/src/http/headers.rs +++ b/crates/utils/src/http/headers.rs @@ -75,9 +75,27 @@ pub const AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER: &str = "x-amz-object-lock-legal-hold pub const AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE: &str = "X-Amz-Bypass-Governance-Retention"; pub const AMZ_BUCKET_REPLICATION_STATUS: &str = "X-Amz-Replication-Status"; -// AmzSnowballExtract will trigger unpacking of an archive content +// Snowball auto-extract compatibility headers. +// +// Supported external request headers: +// - X-Amz-Meta-Snowball-Auto-Extract +// - X-Amz-Snowball-Auto-Extract +// - X-Amz-Meta-Snowball-Prefix +// - X-Amz-Meta-Snowball-Ignore-Dirs +// - X-Amz-Meta-Snowball-Ignore-Errors +// - X-Amz-Meta-Minio-Snowball-Prefix +// - X-Amz-Meta-Minio-Snowball-Ignore-Dirs +// - X-Amz-Meta-Minio-Snowball-Ignore-Errors +// +// Internal compatibility headers: +// - X-Amz-Meta-Rustfs-Snowball-Prefix +// - X-Amz-Meta-Rustfs-Snowball-Ignore-Dirs +// - X-Amz-Meta-Rustfs-Snowball-Ignore-Errors pub const AMZ_SNOWBALL_EXTRACT: &str = "X-Amz-Meta-Snowball-Auto-Extract"; pub const AMZ_SNOWBALL_EXTRACT_ALT: &str = "X-Amz-Snowball-Auto-Extract"; +pub const AMZ_SNOWBALL_PREFIX: &str = "X-Amz-Meta-Snowball-Prefix"; +pub const AMZ_SNOWBALL_IGNORE_DIRS: &str = "X-Amz-Meta-Snowball-Ignore-Dirs"; +pub const AMZ_SNOWBALL_IGNORE_ERRORS: &str = "X-Amz-Meta-Snowball-Ignore-Errors"; pub const AMZ_MINIO_SNOWBALL_PREFIX: &str = "X-Amz-Meta-Minio-Snowball-Prefix"; pub const AMZ_MINIO_SNOWBALL_IGNORE_DIRS: &str = "X-Amz-Meta-Minio-Snowball-Ignore-Dirs"; pub const AMZ_MINIO_SNOWBALL_IGNORE_ERRORS: &str = "X-Amz-Meta-Minio-Snowball-Ignore-Errors"; diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 9623cf9bf..1d00abc1d 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -100,7 +100,7 @@ use rustfs_utils::http::{ AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING, AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE, AMZ_RUSTFS_SNOWBALL_IGNORE_DIRS, AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, AMZ_RUSTFS_SNOWBALL_PREFIX, AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, AMZ_SNOWBALL_EXTRACT, AMZ_SNOWBALL_EXTRACT_ALT, - AMZ_STORAGE_CLASS, AMZ_TAG_COUNT, + AMZ_SNOWBALL_IGNORE_DIRS, AMZ_SNOWBALL_IGNORE_ERRORS, AMZ_SNOWBALL_PREFIX, AMZ_STORAGE_CLASS, AMZ_TAG_COUNT, }, insert_str, remove_str, }; @@ -340,6 +340,17 @@ const AMZ_META_PREFIX_LOWER: &str = "x-amz-meta-"; const SNOWBALL_PREFIX_SUFFIX_LOWER: &str = "snowball-prefix"; const SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER: &str = "snowball-ignore-dirs"; const SNOWBALL_IGNORE_ERRORS_SUFFIX_LOWER: &str = "snowball-ignore-errors"; +const SNOWBALL_PREFIX_HEADER_KEYS: &[&str] = &[AMZ_MINIO_SNOWBALL_PREFIX, AMZ_SNOWBALL_PREFIX, AMZ_RUSTFS_SNOWBALL_PREFIX]; +const SNOWBALL_IGNORE_DIRS_HEADER_KEYS: &[&str] = &[ + AMZ_MINIO_SNOWBALL_IGNORE_DIRS, + AMZ_SNOWBALL_IGNORE_DIRS, + AMZ_RUSTFS_SNOWBALL_IGNORE_DIRS, +]; +const SNOWBALL_IGNORE_ERRORS_HEADER_KEYS: &[&str] = &[ + AMZ_MINIO_SNOWBALL_IGNORE_ERRORS, + AMZ_SNOWBALL_IGNORE_ERRORS, + AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, +]; #[derive(Debug, Clone, Default, PartialEq, Eq)] struct PutObjectExtractOptions { @@ -359,15 +370,23 @@ fn is_put_object_extract_requested(headers: &HeaderMap) -> bool { header_value_is_true(headers, AMZ_SNOWBALL_EXTRACT) || header_value_is_true(headers, AMZ_SNOWBALL_EXTRACT_ALT) } -fn snowball_meta_value_by_suffix(headers: &HeaderMap, preferred_key: &str, suffix_lower: &str) -> Option { - if let Some(preferred) = headers.get(preferred_key).and_then(|value| value.to_str().ok()) { - return Some(preferred.trim().to_string()); - } +fn trimmed_header_value(headers: &HeaderMap, key: &str) -> Option { + headers + .get(key) + .and_then(|value| value.to_str().ok()) + .map(|value| value.trim().to_string()) +} +fn is_exact_snowball_meta_key(key: &str, exact_keys: &[&str]) -> bool { + exact_keys.iter().any(|exact_key| key.eq_ignore_ascii_case(exact_key)) +} + +fn snowball_meta_value_by_suffix(headers: &HeaderMap, suffix_lower: &str, exact_keys: &[&str]) -> Option { for (name, value) in headers { - let key = name.as_str().to_ascii_lowercase(); + let key = name.as_str(); if key.starts_with(AMZ_META_PREFIX_LOWER) && key.ends_with(suffix_lower) + && !is_exact_snowball_meta_key(key, exact_keys) && let Ok(parsed) = value.to_str() { return Some(parsed.trim().to_string()); @@ -377,8 +396,18 @@ fn snowball_meta_value_by_suffix(headers: &HeaderMap, preferred_key: &str, suffi None } -fn snowball_meta_flag_by_suffix(headers: &HeaderMap, preferred_key: &str, suffix_lower: &str) -> bool { - snowball_meta_value_by_suffix(headers, preferred_key, suffix_lower).is_some_and(|value| value.eq_ignore_ascii_case("true")) +fn snowball_meta_value(headers: &HeaderMap, exact_keys: &[&str], suffix_lower: &str) -> Option { + for key in exact_keys { + if let Some(value) = trimmed_header_value(headers, key) { + return Some(value); + } + } + + snowball_meta_value_by_suffix(headers, suffix_lower, exact_keys) +} + +fn snowball_meta_flag(headers: &HeaderMap, exact_keys: &[&str], suffix_lower: &str) -> bool { + snowball_meta_value(headers, exact_keys, suffix_lower).is_some_and(|value| value.eq_ignore_ascii_case("true")) } fn normalize_snowball_prefix(prefix: &str) -> Option { @@ -661,14 +690,10 @@ fn delete_creates_delete_marker(opts: &ObjectOptions) -> bool { } fn resolve_put_object_extract_options(headers: &HeaderMap) -> PutObjectExtractOptions { - let prefix = snowball_meta_value_by_suffix(headers, AMZ_MINIO_SNOWBALL_PREFIX, SNOWBALL_PREFIX_SUFFIX_LOWER) - .or_else(|| snowball_meta_value_by_suffix(headers, AMZ_RUSTFS_SNOWBALL_PREFIX, SNOWBALL_PREFIX_SUFFIX_LOWER)) + let prefix = snowball_meta_value(headers, SNOWBALL_PREFIX_HEADER_KEYS, SNOWBALL_PREFIX_SUFFIX_LOWER) .and_then(|value| normalize_snowball_prefix(&value)); - let ignore_dirs = snowball_meta_flag_by_suffix(headers, AMZ_MINIO_SNOWBALL_IGNORE_DIRS, SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER) - || snowball_meta_flag_by_suffix(headers, AMZ_RUSTFS_SNOWBALL_IGNORE_DIRS, SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER); - let ignore_errors = - snowball_meta_flag_by_suffix(headers, AMZ_MINIO_SNOWBALL_IGNORE_ERRORS, SNOWBALL_IGNORE_ERRORS_SUFFIX_LOWER) - || snowball_meta_flag_by_suffix(headers, AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, SNOWBALL_IGNORE_ERRORS_SUFFIX_LOWER); + let ignore_dirs = snowball_meta_flag(headers, SNOWBALL_IGNORE_DIRS_HEADER_KEYS, SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER); + let ignore_errors = snowball_meta_flag(headers, SNOWBALL_IGNORE_ERRORS_HEADER_KEYS, SNOWBALL_IGNORE_ERRORS_SUFFIX_LOWER); PutObjectExtractOptions { prefix, @@ -4634,6 +4659,19 @@ mod tests { assert!(options.ignore_errors); } + #[test] + fn resolve_put_object_extract_options_accepts_standard_headers() { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_SNOWBALL_PREFIX, HeaderValue::from_static(" /standard/prefix/ ")); + headers.insert(AMZ_SNOWBALL_IGNORE_DIRS, HeaderValue::from_static(" true ")); + headers.insert(AMZ_SNOWBALL_IGNORE_ERRORS, HeaderValue::from_static("TRUE")); + + let options = resolve_put_object_extract_options(&headers); + assert_eq!(options.prefix.as_deref(), Some("standard/prefix")); + assert!(options.ignore_dirs); + assert!(options.ignore_errors); + } + #[test] fn resolve_put_object_extract_options_accepts_suffix_compatible_headers() { let mut headers = HeaderMap::new(); @@ -4646,6 +4684,31 @@ mod tests { assert!(options.ignore_dirs); assert!(options.ignore_errors); } + + #[test] + fn resolve_put_object_extract_options_prefers_exact_headers_over_suffix_fallback() { + let mut headers = HeaderMap::new(); + headers.insert("x-amz-meta-acme-snowball-prefix", HeaderValue::from_static("/fallback/prefix/")); + headers.insert(AMZ_RUSTFS_SNOWBALL_PREFIX, HeaderValue::from_static("/internal/prefix/")); + headers.insert(AMZ_SNOWBALL_PREFIX, HeaderValue::from_static("/standard/prefix/")); + headers.insert(AMZ_MINIO_SNOWBALL_PREFIX, HeaderValue::from_static("/minio/prefix/")); + + let options = resolve_put_object_extract_options(&headers); + assert_eq!(options.prefix.as_deref(), Some("minio/prefix")); + } + + #[test] + fn resolve_put_object_extract_options_exact_flags_override_suffix_fallback() { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_SNOWBALL_IGNORE_DIRS, HeaderValue::from_static("false")); + headers.insert("x-amz-meta-acme-snowball-ignore-dirs", HeaderValue::from_static("true")); + headers.insert(AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, HeaderValue::from_static("false")); + headers.insert("x-amz-meta-acme-snowball-ignore-errors", HeaderValue::from_static("true")); + + let options = resolve_put_object_extract_options(&headers); + assert!(!options.ignore_dirs); + assert!(!options.ignore_errors); + } #[tokio::test] async fn execute_put_object_rejects_post_object_sse_kms_from_input() { let input = PutObjectInput::builder()