From f98664ea4af4952f2aea648eb657552ec0cbe1f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=89=E6=AD=A3=E8=B6=85?= Date: Sat, 28 Mar 2026 22:56:42 +0800 Subject: [PATCH] test(s3): complete snowball auto-extract coverage (#2313) --- crates/e2e_test/src/lib.rs | 3 + .../src/snowball_auto_extract_test.rs | 184 ++++++++++++++++++ crates/utils/src/http/headers.rs | 7 + rustfs/src/app/object_usecase.rs | 55 +++--- 4 files changed, 217 insertions(+), 32 deletions(-) create mode 100644 crates/e2e_test/src/snowball_auto_extract_test.rs diff --git a/crates/e2e_test/src/lib.rs b/crates/e2e_test/src/lib.rs index b1266a8a3..97dd2461d 100644 --- a/crates/e2e_test/src/lib.rs +++ b/crates/e2e_test/src/lib.rs @@ -104,3 +104,6 @@ mod object_lambda_test; // Replication extension end-to-end regression tests #[cfg(test)] mod replication_extension_test; + +#[cfg(test)] +mod snowball_auto_extract_test; diff --git a/crates/e2e_test/src/snowball_auto_extract_test.rs b/crates/e2e_test/src/snowball_auto_extract_test.rs new file mode 100644 index 000000000..743d654b6 --- /dev/null +++ b/crates/e2e_test/src/snowball_auto_extract_test.rs @@ -0,0 +1,184 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#[cfg(test)] +mod tests { + use crate::common::{RustFSTestEnvironment, init_logging}; + use aws_sdk_s3::error::ProvideErrorMetadata; + use aws_sdk_s3::primitives::ByteStream; + use serial_test::serial; + use std::error::Error; + use std::io::Cursor; + + async fn build_test_archive() -> Result, Box> { + let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new())); + + for dir in ["dir/", "empty-dir/"] { + let mut header = tokio_tar::Header::new_gnu(); + header.set_entry_type(tokio_tar::EntryType::Directory); + header.set_size(0); + header.set_mode(0o755); + header.set_cksum(); + builder.append_data(&mut header, dir, Cursor::new(Vec::new())).await?; + } + + for (path, data) in [ + ("dir/file.txt", b"nested payload\n".as_slice()), + ("root.txt", b"root payload\n".as_slice()), + ] { + let mut header = tokio_tar::Header::new_gnu(); + header.set_size(data.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder.append_data(&mut header, path, Cursor::new(data)).await?; + } + + Ok(builder.into_inner().await?.into_inner()) + } + + async fn build_archive_with_invalid_entry() -> Result, Box> { + let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new())); + + let mut valid_header = tokio_tar::Header::new_gnu(); + valid_header.set_size(b"valid-body".len() as u64); + valid_header.set_mode(0o644); + valid_header.set_cksum(); + builder + .append_data(&mut valid_header, "valid.txt", Cursor::new(b"valid-body".as_slice())) + .await?; + + let long_name = format!("{}.txt", "a".repeat(1100)); + let mut invalid_header = tokio_tar::Header::new_gnu(); + invalid_header.set_size(b"ignored-body".len() as u64); + invalid_header.set_mode(0o644); + invalid_header.set_cksum(); + builder + .append_data(&mut invalid_header, long_name, Cursor::new(b"ignored-body".as_slice())) + .await?; + + Ok(builder.into_inner().await?.into_inner()) + } + + #[tokio::test] + #[serial] + async fn snowball_auto_extract_supports_minio_prefix_and_directory_markers() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let client = env.create_s3_client(); + let bucket = "snowball-prefix-test"; + let archive = build_test_archive().await?; + + client.create_bucket().bucket(bucket).send().await?; + + client + .put_object() + .bucket(bucket) + .key("fixture.tar") + .metadata("Snowball-Auto-Extract", "true") + .metadata("Minio-Snowball-Prefix", "/tenant-a/") + .body(ByteStream::from(archive)) + .send() + .await?; + + let root = client.get_object().bucket(bucket).key("tenant-a/root.txt").send().await?; + assert_eq!(root.body.collect().await?.into_bytes().as_ref(), b"root payload\n"); + + let nested = client.get_object().bucket(bucket).key("tenant-a/dir/file.txt").send().await?; + assert_eq!(nested.body.collect().await?.into_bytes().as_ref(), b"nested payload\n"); + + let dir_marker = client.head_object().bucket(bucket).key("tenant-a/empty-dir/").send().await?; + assert_eq!(dir_marker.content_length(), Some(0)); + + env.stop_server(); + Ok(()) + } + + #[tokio::test] + #[serial] + async fn snowball_auto_extract_ignores_directories_when_requested() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let client = env.create_s3_client(); + let bucket = "snowball-ignore-dirs-default"; + let archive = build_test_archive().await?; + + client.create_bucket().bucket(bucket).send().await?; + + client + .put_object() + .bucket(bucket) + .key("fixture.tar") + .metadata("Snowball-Auto-Extract", "true") + .metadata("Minio-Snowball-Prefix", "tenant-b") + .metadata("Minio-Snowball-Ignore-Dirs", "true") + .body(ByteStream::from(archive)) + .send() + .await?; + + let err = client + .head_object() + .bucket(bucket) + .key("tenant-b/empty-dir/") + .send() + .await + .expect_err("directory marker should be skipped when ignore-dirs=true"); + let service_err = err.into_service_error(); + assert_eq!(service_err.code(), Some("NotFound")); + + env.stop_server(); + Ok(()) + } + + #[tokio::test] + #[serial] + async fn snowball_auto_extract_ignores_invalid_entries_when_requested() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let client = env.create_s3_client(); + let bucket = "snowball-ignore-errors"; + let archive = build_archive_with_invalid_entry().await?; + + client.create_bucket().bucket(bucket).send().await?; + + client + .put_object() + .bucket(bucket) + .key("fixture.tar") + .metadata("Snowball-Auto-Extract", "true") + .metadata("Minio-Snowball-Prefix", "tenant-c") + .metadata("Minio-Snowball-Ignore-Errors", "true") + .body(ByteStream::from(archive)) + .send() + .await?; + + let valid = client.get_object().bucket(bucket).key("tenant-c/valid.txt").send().await?; + assert_eq!(valid.body.collect().await?.into_bytes().as_ref(), b"valid-body"); + + let listed = client.list_objects_v2().bucket(bucket).prefix("tenant-c/").send().await?; + let keys: Vec<_> = listed.contents().iter().filter_map(|entry| entry.key()).collect(); + assert_eq!(keys, vec!["tenant-c/valid.txt"]); + + env.stop_server(); + Ok(()) + } +} diff --git a/crates/utils/src/http/headers.rs b/crates/utils/src/http/headers.rs index 3c3b1d1e8..a0488a463 100644 --- a/crates/utils/src/http/headers.rs +++ b/crates/utils/src/http/headers.rs @@ -77,6 +77,13 @@ pub const AMZ_BUCKET_REPLICATION_STATUS: &str = "X-Amz-Replication-Status"; // AmzSnowballExtract will trigger unpacking of an archive content pub const AMZ_SNOWBALL_EXTRACT: &str = "X-Amz-Meta-Snowball-Auto-Extract"; +pub const AMZ_SNOWBALL_EXTRACT_ALT: &str = "X-Amz-Snowball-Auto-Extract"; +pub const AMZ_MINIO_SNOWBALL_PREFIX: &str = "X-Amz-Meta-Minio-Snowball-Prefix"; +pub const AMZ_MINIO_SNOWBALL_IGNORE_DIRS: &str = "X-Amz-Meta-Minio-Snowball-Ignore-Dirs"; +pub const AMZ_MINIO_SNOWBALL_IGNORE_ERRORS: &str = "X-Amz-Meta-Minio-Snowball-Ignore-Errors"; +pub const AMZ_RUSTFS_SNOWBALL_PREFIX: &str = "X-Amz-Meta-Rustfs-Snowball-Prefix"; +pub const AMZ_RUSTFS_SNOWBALL_IGNORE_DIRS: &str = "X-Amz-Meta-Rustfs-Snowball-Ignore-Dirs"; +pub const AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS: &str = "X-Amz-Meta-Rustfs-Snowball-Ignore-Errors"; // Object lock enabled pub const AMZ_OBJECT_LOCK_ENABLED: &str = "x-amz-bucket-object-lock-enabled"; diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 79f86a921..9623cf9bf 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -95,10 +95,12 @@ use rustfs_utils::http::{ AMZ_BUCKET_REPLICATION_STATUS, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE, AMZ_WEBSITE_REDIRECT_LOCATION, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, headers::{ - AMZ_DECODED_CONTENT_LENGTH, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE, - AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, - AMZ_OBJECT_TAGGING, AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE, AMZ_SERVER_SIDE_ENCRYPTION, - AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, AMZ_SNOWBALL_EXTRACT, AMZ_STORAGE_CLASS, AMZ_TAG_COUNT, + AMZ_DECODED_CONTENT_LENGTH, AMZ_MINIO_SNOWBALL_IGNORE_DIRS, AMZ_MINIO_SNOWBALL_IGNORE_ERRORS, AMZ_MINIO_SNOWBALL_PREFIX, + AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE, AMZ_OBJECT_LOCK_MODE_LOWER, + AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING, AMZ_RESTORE_EXPIRY_DAYS, + AMZ_RESTORE_REQUEST_DATE, AMZ_RUSTFS_SNOWBALL_IGNORE_DIRS, AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, AMZ_RUSTFS_SNOWBALL_PREFIX, + AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, AMZ_SNOWBALL_EXTRACT, AMZ_SNOWBALL_EXTRACT_ALT, + AMZ_STORAGE_CLASS, AMZ_TAG_COUNT, }, insert_str, remove_str, }; @@ -334,10 +336,6 @@ fn build_put_object_expiration_header(event: &lifecycle::Event) -> Option bool { } fn is_put_object_extract_requested(headers: &HeaderMap) -> bool { - header_value_is_true(headers, AMZ_SNOWBALL_EXTRACT) || header_value_is_true(headers, AMZ_SNOWBALL_EXTRACT_COMPAT) + header_value_is_true(headers, AMZ_SNOWBALL_EXTRACT) || header_value_is_true(headers, AMZ_SNOWBALL_EXTRACT_ALT) } fn snowball_meta_value_by_suffix(headers: &HeaderMap, preferred_key: &str, suffix_lower: &str) -> Option { @@ -393,7 +391,7 @@ fn normalize_snowball_prefix(prefix: &str) -> Option { } fn normalize_extract_entry_key(path: &str, prefix: Option<&str>, is_dir: bool) -> String { - let path = path.trim_matches('/'); + let path = path.trim_start_matches("./").trim_start_matches('/'); let mut key = match prefix { Some(prefix) if !path.is_empty() => format!("{prefix}/{path}"), Some(prefix) => prefix.to_string(), @@ -663,11 +661,14 @@ fn delete_creates_delete_marker(opts: &ObjectOptions) -> bool { } fn resolve_put_object_extract_options(headers: &HeaderMap) -> PutObjectExtractOptions { - let prefix = snowball_meta_value_by_suffix(headers, AMZ_SNOWBALL_PREFIX_INTERNAL, SNOWBALL_PREFIX_SUFFIX_LOWER) + let prefix = snowball_meta_value_by_suffix(headers, AMZ_MINIO_SNOWBALL_PREFIX, SNOWBALL_PREFIX_SUFFIX_LOWER) + .or_else(|| snowball_meta_value_by_suffix(headers, AMZ_RUSTFS_SNOWBALL_PREFIX, SNOWBALL_PREFIX_SUFFIX_LOWER)) .and_then(|value| normalize_snowball_prefix(&value)); - let ignore_dirs = snowball_meta_flag_by_suffix(headers, AMZ_SNOWBALL_IGNORE_DIRS_INTERNAL, SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER); + let ignore_dirs = snowball_meta_flag_by_suffix(headers, AMZ_MINIO_SNOWBALL_IGNORE_DIRS, SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER) + || snowball_meta_flag_by_suffix(headers, AMZ_RUSTFS_SNOWBALL_IGNORE_DIRS, SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER); let ignore_errors = - snowball_meta_flag_by_suffix(headers, AMZ_SNOWBALL_IGNORE_ERRORS_INTERNAL, SNOWBALL_IGNORE_ERRORS_SUFFIX_LOWER); + snowball_meta_flag_by_suffix(headers, AMZ_MINIO_SNOWBALL_IGNORE_ERRORS, SNOWBALL_IGNORE_ERRORS_SUFFIX_LOWER) + || snowball_meta_flag_by_suffix(headers, AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, SNOWBALL_IGNORE_ERRORS_SUFFIX_LOWER); PutObjectExtractOptions { prefix, @@ -4518,7 +4519,7 @@ fn object_attributes_requested(object_attributes: &[ObjectAttributes], name: &'s #[cfg(test)] mod tests { use super::*; - use http::{Extensions, HeaderMap, HeaderName, HeaderValue, Method, Uri}; + use http::{Extensions, HeaderMap, HeaderValue, Method, Uri}; fn build_request(input: T, method: Method) -> S3Request { S3Request { @@ -4576,7 +4577,7 @@ mod tests { #[test] fn is_put_object_extract_requested_accepts_compat_header_case_insensitive() { let mut headers = HeaderMap::new(); - headers.insert(AMZ_SNOWBALL_EXTRACT_COMPAT, HeaderValue::from_static(" TRUE ")); + headers.insert(AMZ_SNOWBALL_EXTRACT_ALT, HeaderValue::from_static(" TRUE ")); assert!(is_put_object_extract_requested(&headers)); } @@ -4599,7 +4600,7 @@ mod tests { #[test] fn normalize_extract_entry_key_applies_prefix_and_directory_suffix() { assert_eq!( - normalize_extract_entry_key("nested/path.txt", Some("imports"), false), + normalize_extract_entry_key("./nested/path.txt", Some("imports"), false), "imports/nested/path.txt" ); assert_eq!(normalize_extract_entry_key("nested/dir/", Some("imports"), true), "imports/nested/dir/"); @@ -4623,9 +4624,9 @@ mod tests { #[test] fn resolve_put_object_extract_options_accepts_internal_headers() { let mut headers = HeaderMap::new(); - headers.insert(AMZ_SNOWBALL_PREFIX_INTERNAL, HeaderValue::from_static("/internal/prefix/")); - headers.insert(AMZ_SNOWBALL_IGNORE_DIRS_INTERNAL, HeaderValue::from_static("true")); - headers.insert(AMZ_SNOWBALL_IGNORE_ERRORS_INTERNAL, HeaderValue::from_static("TRUE")); + headers.insert(AMZ_RUSTFS_SNOWBALL_PREFIX, HeaderValue::from_static("/internal/prefix/")); + headers.insert(AMZ_RUSTFS_SNOWBALL_IGNORE_DIRS, HeaderValue::from_static("true")); + headers.insert(AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, HeaderValue::from_static("TRUE")); let options = resolve_put_object_extract_options(&headers); assert_eq!(options.prefix.as_deref(), Some("internal/prefix")); @@ -4636,25 +4637,15 @@ mod tests { #[test] fn resolve_put_object_extract_options_accepts_suffix_compatible_headers() { let mut headers = HeaderMap::new(); - headers.insert( - HeaderName::from_static("x-amz-meta-acme-snowball-prefix"), - HeaderValue::from_static(" /partner/import "), - ); - headers.insert( - HeaderName::from_static("x-amz-meta-acme-snowball-ignore-dirs"), - HeaderValue::from_static(" true "), - ); - headers.insert( - HeaderName::from_static("x-amz-meta-acme-snowball-ignore-errors"), - HeaderValue::from_static("TRUE"), - ); + headers.insert("x-amz-meta-acme-snowball-prefix", HeaderValue::from_static(" /partner/import ")); + headers.insert("x-amz-meta-acme-snowball-ignore-dirs", HeaderValue::from_static(" true ")); + headers.insert("x-amz-meta-acme-snowball-ignore-errors", HeaderValue::from_static("TRUE")); let options = resolve_put_object_extract_options(&headers); assert_eq!(options.prefix.as_deref(), Some("partner/import")); assert!(options.ignore_dirs); assert!(options.ignore_errors); } - #[tokio::test] async fn execute_put_object_rejects_post_object_sse_kms_from_input() { let input = PutObjectInput::builder()