gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 16:28:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(s3): complete snowball auto-extract coverage (#2313)

Browse Source
This commit is contained in:
安正超
2026-03-28 22:56:42 +08:00
committed by GitHub
parent 9536ed8d38
commit f98664ea4a
4 changed files with 217 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/e2e_test/src/lib.rs
+3
View File
@@ -104,3 +104,6 @@ mod object_lambda_test;
// Replication extension end-to-end regression tests
#[cfg(test)]
mod replication_extension_test;
#[cfg(test)]
mod snowball_auto_extract_test;
crates/e2e_test/src/snowball_auto_extract_test.rs
+184
View File
@@ -0,0 +1,184 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(test)]
mod tests {
use crate::common::{RustFSTestEnvironment, init_logging};
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::primitives::ByteStream;
use serial_test::serial;
use std::error::Error;
use std::io::Cursor;
async fn build_test_archive() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
for dir in ["dir/", "empty-dir/"] {
let mut header = tokio_tar::Header::new_gnu();
header.set_entry_type(tokio_tar::EntryType::Directory);
header.set_size(0);
header.set_mode(0o755);
header.set_cksum();
builder.append_data(&mut header, dir, Cursor::new(Vec::new())).await?;
}
for (path, data) in [
("dir/file.txt", b"nested payload\n".as_slice()),
("root.txt", b"root payload\n".as_slice()),
] {
let mut header = tokio_tar::Header::new_gnu();
header.set_size(data.len() as u64);
header.set_mode(0o644);
header.set_cksum();
builder.append_data(&mut header, path, Cursor::new(data)).await?;
}
Ok(builder.into_inner().await?.into_inner())
}
async fn build_archive_with_invalid_entry() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
let mut valid_header = tokio_tar::Header::new_gnu();
valid_header.set_size(b"valid-body".len() as u64);
valid_header.set_mode(0o644);
valid_header.set_cksum();
builder
.append_data(&mut valid_header, "valid.txt", Cursor::new(b"valid-body".as_slice()))
.await?;
let long_name = format!("{}.txt", "a".repeat(1100));
let mut invalid_header = tokio_tar::Header::new_gnu();
invalid_header.set_size(b"ignored-body".len() as u64);
invalid_header.set_mode(0o644);
invalid_header.set_cksum();
builder
.append_data(&mut invalid_header, long_name, Cursor::new(b"ignored-body".as_slice()))
.await?;
Ok(builder.into_inner().await?.into_inner())
}
#[tokio::test]
#[serial]
async fn snowball_auto_extract_supports_minio_prefix_and_directory_markers() -> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
let client = env.create_s3_client();
let bucket = "snowball-prefix-test";
let archive = build_test_archive().await?;
client.create_bucket().bucket(bucket).send().await?;
client
.put_object()
.bucket(bucket)
.key("fixture.tar")
.metadata("Snowball-Auto-Extract", "true")
.metadata("Minio-Snowball-Prefix", "/tenant-a/")
.body(ByteStream::from(archive))
.send()
.await?;
let root = client.get_object().bucket(bucket).key("tenant-a/root.txt").send().await?;
assert_eq!(root.body.collect().await?.into_bytes().as_ref(), b"root payload\n");
let nested = client.get_object().bucket(bucket).key("tenant-a/dir/file.txt").send().await?;
assert_eq!(nested.body.collect().await?.into_bytes().as_ref(), b"nested payload\n");
let dir_marker = client.head_object().bucket(bucket).key("tenant-a/empty-dir/").send().await?;
assert_eq!(dir_marker.content_length(), Some(0));
env.stop_server();
Ok(())
}
#[tokio::test]
#[serial]
async fn snowball_auto_extract_ignores_directories_when_requested() -> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
let client = env.create_s3_client();
let bucket = "snowball-ignore-dirs-default";
let archive = build_test_archive().await?;
client.create_bucket().bucket(bucket).send().await?;
client
.put_object()
.bucket(bucket)
.key("fixture.tar")
.metadata("Snowball-Auto-Extract", "true")
.metadata("Minio-Snowball-Prefix", "tenant-b")
.metadata("Minio-Snowball-Ignore-Dirs", "true")
.body(ByteStream::from(archive))
.send()
.await?;
let err = client
.head_object()
.bucket(bucket)
.key("tenant-b/empty-dir/")
.send()
.await
.expect_err("directory marker should be skipped when ignore-dirs=true");
let service_err = err.into_service_error();
assert_eq!(service_err.code(), Some("NotFound"));
env.stop_server();
Ok(())
}
#[tokio::test]
#[serial]
async fn snowball_auto_extract_ignores_invalid_entries_when_requested() -> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
let client = env.create_s3_client();
let bucket = "snowball-ignore-errors";
let archive = build_archive_with_invalid_entry().await?;
client.create_bucket().bucket(bucket).send().await?;
client
.put_object()
.bucket(bucket)
.key("fixture.tar")
.metadata("Snowball-Auto-Extract", "true")
.metadata("Minio-Snowball-Prefix", "tenant-c")
.metadata("Minio-Snowball-Ignore-Errors", "true")
.body(ByteStream::from(archive))
.send()
.await?;
let valid = client.get_object().bucket(bucket).key("tenant-c/valid.txt").send().await?;
assert_eq!(valid.body.collect().await?.into_bytes().as_ref(), b"valid-body");
let listed = client.list_objects_v2().bucket(bucket).prefix("tenant-c/").send().await?;
let keys: Vec<_> = listed.contents().iter().filter_map(|entry| entry.key()).collect();
assert_eq!(keys, vec!["tenant-c/valid.txt"]);
env.stop_server();
Ok(())
}
}
crates/utils/src/http/headers.rs
+7
View File
@@ -77,6 +77,13 @@ pub const AMZ_BUCKET_REPLICATION_STATUS: &str = "X-Amz-Replication-Status";
// AmzSnowballExtract will trigger unpacking of an archive content
pub const AMZ_SNOWBALL_EXTRACT: &str = "X-Amz-Meta-Snowball-Auto-Extract";
pub const AMZ_SNOWBALL_EXTRACT_ALT: &str = "X-Amz-Snowball-Auto-Extract";
pub const AMZ_MINIO_SNOWBALL_PREFIX: &str = "X-Amz-Meta-Minio-Snowball-Prefix";
pub const AMZ_MINIO_SNOWBALL_IGNORE_DIRS: &str = "X-Amz-Meta-Minio-Snowball-Ignore-Dirs";
pub const AMZ_MINIO_SNOWBALL_IGNORE_ERRORS: &str = "X-Amz-Meta-Minio-Snowball-Ignore-Errors";
pub const AMZ_RUSTFS_SNOWBALL_PREFIX: &str = "X-Amz-Meta-Rustfs-Snowball-Prefix";
pub const AMZ_RUSTFS_SNOWBALL_IGNORE_DIRS: &str = "X-Amz-Meta-Rustfs-Snowball-Ignore-Dirs";
pub const AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS: &str = "X-Amz-Meta-Rustfs-Snowball-Ignore-Errors";
// Object lock enabled
pub const AMZ_OBJECT_LOCK_ENABLED: &str = "x-amz-bucket-object-lock-enabled";
rustfs/src/app/object_usecase.rs
+23 -32
View File
@@ -95,10 +95,12 @@ use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE, AMZ_WEBSITE_REDIRECT_LOCATION, SUFFIX_ACTUAL_SIZE,
SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP,
headers::{
AMZ_DECODED_CONTENT_LENGTH, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE,
AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
AMZ_OBJECT_TAGGING, AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE, AMZ_SERVER_SIDE_ENCRYPTION,
AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, AMZ_SNOWBALL_EXTRACT, AMZ_STORAGE_CLASS, AMZ_TAG_COUNT,
AMZ_DECODED_CONTENT_LENGTH, AMZ_MINIO_SNOWBALL_IGNORE_DIRS, AMZ_MINIO_SNOWBALL_IGNORE_ERRORS, AMZ_MINIO_SNOWBALL_PREFIX,
AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE, AMZ_OBJECT_LOCK_MODE_LOWER,
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING, AMZ_RESTORE_EXPIRY_DAYS,
AMZ_RESTORE_REQUEST_DATE, AMZ_RUSTFS_SNOWBALL_IGNORE_DIRS, AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, AMZ_RUSTFS_SNOWBALL_PREFIX,
AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, AMZ_SNOWBALL_EXTRACT, AMZ_SNOWBALL_EXTRACT_ALT,
AMZ_STORAGE_CLASS, AMZ_TAG_COUNT,
},
insert_str, remove_str,
};
@@ -334,10 +336,6 @@ fn build_put_object_expiration_header(event: &lifecycle::Event) -> Option<String
Some(format!("expiry-date=\"{}\", rule-id=\"{}\"", expiry_date, event.rule_id))
}
const AMZ_SNOWBALL_EXTRACT_COMPAT: &str = "X-Amz-Snowball-Auto-Extract";
const AMZ_SNOWBALL_PREFIX_INTERNAL: &str = "X-Amz-Meta-Rustfs-Snowball-Prefix";
const AMZ_SNOWBALL_IGNORE_DIRS_INTERNAL: &str = "X-Amz-Meta-Rustfs-Snowball-Ignore-Dirs";
const AMZ_SNOWBALL_IGNORE_ERRORS_INTERNAL: &str = "X-Amz-Meta-Rustfs-Snowball-Ignore-Errors";
const AMZ_META_PREFIX_LOWER: &str = "x-amz-meta-";
const SNOWBALL_PREFIX_SUFFIX_LOWER: &str = "snowball-prefix";
const SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER: &str = "snowball-ignore-dirs";
@@ -358,7 +356,7 @@ fn header_value_is_true(headers: &HeaderMap, key: &str) -> bool {
}
fn is_put_object_extract_requested(headers: &HeaderMap) -> bool {
header_value_is_true(headers, AMZ_SNOWBALL_EXTRACT) || header_value_is_true(headers, AMZ_SNOWBALL_EXTRACT_COMPAT)
header_value_is_true(headers, AMZ_SNOWBALL_EXTRACT) || header_value_is_true(headers, AMZ_SNOWBALL_EXTRACT_ALT)
}
fn snowball_meta_value_by_suffix(headers: &HeaderMap, preferred_key: &str, suffix_lower: &str) -> Option<String> {
@@ -393,7 +391,7 @@ fn normalize_snowball_prefix(prefix: &str) -> Option<String> {
}
fn normalize_extract_entry_key(path: &str, prefix: Option<&str>, is_dir: bool) -> String {
let path = path.trim_matches('/');
let path = path.trim_start_matches("./").trim_start_matches('/');
let mut key = match prefix {
Some(prefix) if !path.is_empty() => format!("{prefix}/{path}"),
Some(prefix) => prefix.to_string(),
@@ -663,11 +661,14 @@ fn delete_creates_delete_marker(opts: &ObjectOptions) -> bool {
}
fn resolve_put_object_extract_options(headers: &HeaderMap) -> PutObjectExtractOptions {
let prefix = snowball_meta_value_by_suffix(headers, AMZ_SNOWBALL_PREFIX_INTERNAL, SNOWBALL_PREFIX_SUFFIX_LOWER)
let prefix = snowball_meta_value_by_suffix(headers, AMZ_MINIO_SNOWBALL_PREFIX, SNOWBALL_PREFIX_SUFFIX_LOWER)
.or_else(|| snowball_meta_value_by_suffix(headers, AMZ_RUSTFS_SNOWBALL_PREFIX, SNOWBALL_PREFIX_SUFFIX_LOWER))
.and_then(|value| normalize_snowball_prefix(&value));
let ignore_dirs = snowball_meta_flag_by_suffix(headers, AMZ_SNOWBALL_IGNORE_DIRS_INTERNAL, SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER);
let ignore_dirs = snowball_meta_flag_by_suffix(headers, AMZ_MINIO_SNOWBALL_IGNORE_DIRS, SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER)
|| snowball_meta_flag_by_suffix(headers, AMZ_RUSTFS_SNOWBALL_IGNORE_DIRS, SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER);
let ignore_errors =
snowball_meta_flag_by_suffix(headers, AMZ_SNOWBALL_IGNORE_ERRORS_INTERNAL, SNOWBALL_IGNORE_ERRORS_SUFFIX_LOWER);
snowball_meta_flag_by_suffix(headers, AMZ_MINIO_SNOWBALL_IGNORE_ERRORS, SNOWBALL_IGNORE_ERRORS_SUFFIX_LOWER)
|| snowball_meta_flag_by_suffix(headers, AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, SNOWBALL_IGNORE_ERRORS_SUFFIX_LOWER);
PutObjectExtractOptions {
prefix,
@@ -4518,7 +4519,7 @@ fn object_attributes_requested(object_attributes: &[ObjectAttributes], name: &'s
#[cfg(test)]
mod tests {
use super::*;
use http::{Extensions, HeaderMap, HeaderName, HeaderValue, Method, Uri};
use http::{Extensions, HeaderMap, HeaderValue, Method, Uri};
fn build_request<T>(input: T, method: Method) -> S3Request<T> {
S3Request {
@@ -4576,7 +4577,7 @@ mod tests {
#[test]
fn is_put_object_extract_requested_accepts_compat_header_case_insensitive() {
let mut headers = HeaderMap::new();
headers.insert(AMZ_SNOWBALL_EXTRACT_COMPAT, HeaderValue::from_static(" TRUE "));
headers.insert(AMZ_SNOWBALL_EXTRACT_ALT, HeaderValue::from_static(" TRUE "));
assert!(is_put_object_extract_requested(&headers));
}
@@ -4599,7 +4600,7 @@ mod tests {
#[test]
fn normalize_extract_entry_key_applies_prefix_and_directory_suffix() {
assert_eq!(
normalize_extract_entry_key("nested/path.txt", Some("imports"), false),
normalize_extract_entry_key("./nested/path.txt", Some("imports"), false),
"imports/nested/path.txt"
);
assert_eq!(normalize_extract_entry_key("nested/dir/", Some("imports"), true), "imports/nested/dir/");
@@ -4623,9 +4624,9 @@ mod tests {
#[test]
fn resolve_put_object_extract_options_accepts_internal_headers() {
let mut headers = HeaderMap::new();
headers.insert(AMZ_SNOWBALL_PREFIX_INTERNAL, HeaderValue::from_static("/internal/prefix/"));
headers.insert(AMZ_SNOWBALL_IGNORE_DIRS_INTERNAL, HeaderValue::from_static("true"));
headers.insert(AMZ_SNOWBALL_IGNORE_ERRORS_INTERNAL, HeaderValue::from_static("TRUE"));
headers.insert(AMZ_RUSTFS_SNOWBALL_PREFIX, HeaderValue::from_static("/internal/prefix/"));
headers.insert(AMZ_RUSTFS_SNOWBALL_IGNORE_DIRS, HeaderValue::from_static("true"));
headers.insert(AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, HeaderValue::from_static("TRUE"));
let options = resolve_put_object_extract_options(&headers);
assert_eq!(options.prefix.as_deref(), Some("internal/prefix"));
@@ -4636,25 +4637,15 @@ mod tests {
#[test]
fn resolve_put_object_extract_options_accepts_suffix_compatible_headers() {
let mut headers = HeaderMap::new();
headers.insert(
HeaderName::from_static("x-amz-meta-acme-snowball-prefix"),
HeaderValue::from_static(" /partner/import "),
);
headers.insert(
HeaderName::from_static("x-amz-meta-acme-snowball-ignore-dirs"),
HeaderValue::from_static(" true "),
);
headers.insert(
HeaderName::from_static("x-amz-meta-acme-snowball-ignore-errors"),
HeaderValue::from_static("TRUE"),
);
headers.insert("x-amz-meta-acme-snowball-prefix", HeaderValue::from_static(" /partner/import "));
headers.insert("x-amz-meta-acme-snowball-ignore-dirs", HeaderValue::from_static(" true "));
headers.insert("x-amz-meta-acme-snowball-ignore-errors", HeaderValue::from_static("TRUE"));
let options = resolve_put_object_extract_options(&headers);
assert_eq!(options.prefix.as_deref(), Some("partner/import"));
assert!(options.ignore_dirs);
assert!(options.ignore_errors);
}
#[tokio::test]
async fn execute_put_object_rejects_post_object_sse_kms_from_input() {
let input = PutObjectInput::builder()