test(s3): complete snowball auto-extract compatibility (#2324)

crates/e2e_test/src/snowball_auto_extract_test.rs

@@ -107,6 +107,97 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn snowball_auto_extract_supports_standard_headers_with_combined_extract_options()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = env.create_s3_client();
+        let bucket = "snowball-standard-options";
+        let extracted_prefix = "/tenant-standard/";
+
+        client.create_bucket().bucket(bucket).send().await?;
+
+        let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
+
+        let mut dir_header = tokio_tar::Header::new_gnu();
+        dir_header.set_entry_type(tokio_tar::EntryType::Directory);
+        dir_header.set_size(0);
+        dir_header.set_mode(0o755);
+        dir_header.set_cksum();
+        builder
+            .append_data(&mut dir_header, "ignored-dir/", Cursor::new(Vec::new()))
+            .await?;
+
+        let mut valid_header = tokio_tar::Header::new_gnu();
+        valid_header.set_size(b"standard-body".len() as u64);
+        valid_header.set_mode(0o644);
+        valid_header.set_cksum();
+        builder
+            .append_data(&mut valid_header, "valid.txt", Cursor::new(b"standard-body".as_slice()))
+            .await?;
+
+        let long_name = format!("{}.txt", "a".repeat(1100));
+        let mut invalid_header = tokio_tar::Header::new_gnu();
+        invalid_header.set_size(b"ignored-body".len() as u64);
+        invalid_header.set_mode(0o644);
+        invalid_header.set_cksum();
+        builder
+            .append_data(&mut invalid_header, long_name, Cursor::new(b"ignored-body".as_slice()))
+            .await?;
+
+        let archive = builder.into_inner().await?.into_inner();
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("fixture.tar")
+            .body(ByteStream::from(archive))
+            .customize()
+            .mutate_request(move |req| {
+                req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
+                req.headers_mut().insert("x-amz-meta-snowball-prefix", extracted_prefix);
+                req.headers_mut().insert("x-amz-meta-snowball-ignore-dirs", "true");
+                req.headers_mut().insert("x-amz-meta-snowball-ignore-errors", "true");
+            })
+            .send()
+            .await?;
+
+        let valid = client
+            .get_object()
+            .bucket(bucket)
+            .key("tenant-standard/valid.txt")
+            .send()
+            .await?;
+        assert_eq!(valid.body.collect().await?.into_bytes().as_ref(), b"standard-body");
+
+        let dir_err = client
+            .head_object()
+            .bucket(bucket)
+            .key("tenant-standard/ignored-dir/")
+            .send()
+            .await
+            .expect_err("directory marker should be skipped when standard ignore-dirs=true");
+        let dir_service_err = dir_err.into_service_error();
+        assert_eq!(dir_service_err.code(), Some("NotFound"));
+
+        let listed = client
+            .list_objects_v2()
+            .bucket(bucket)
+            .prefix("tenant-standard/")
+            .send()
+            .await?;
+        let keys: Vec<_> = listed.contents().iter().filter_map(|entry| entry.key()).collect();
+        assert_eq!(keys, vec!["tenant-standard/valid.txt"]);
+
+        env.stop_server();
+        Ok(())
+    }
+
     #[tokio::test]
     #[serial]
     async fn snowball_auto_extract_ignores_directories_when_requested() -> Result<(), Box<dyn Error + Send + Sync>> {
@@ -181,4 +272,50 @@ mod tests {
         env.stop_server();
         Ok(())
     }
+
+    #[tokio::test]
+    #[serial]
+    async fn snowball_auto_extract_prefers_exact_minio_prefix_over_suffix_fallback() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = env.create_s3_client();
+        let bucket = "snowball-prefix-precedence";
+        let archive = build_test_archive().await?;
+
+        client.create_bucket().bucket(bucket).send().await?;
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("fixture.tar")
+            .body(ByteStream::from(archive))
+            .customize()
+            .mutate_request(|req| {
+                req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
+                req.headers_mut()
+                    .insert("x-amz-meta-acme-snowball-prefix", "/tenant-fallback/");
+                req.headers_mut().insert("x-amz-meta-minio-snowball-prefix", "/tenant-exact/");
+            })
+            .send()
+            .await?;
+
+        let exact = client.get_object().bucket(bucket).key("tenant-exact/root.txt").send().await?;
+        assert_eq!(exact.body.collect().await?.into_bytes().as_ref(), b"root payload\n");
+
+        let fallback_err = client
+            .head_object()
+            .bucket(bucket)
+            .key("tenant-fallback/root.txt")
+            .send()
+            .await
+            .expect_err("fallback suffix header should not override exact MinIO prefix");
+        let fallback_service_err = fallback_err.into_service_error();
+        assert_eq!(fallback_service_err.code(), Some("NotFound"));
+
+        env.stop_server();
+        Ok(())
+    }
 }

crates/utils/src/http/headers.rs

@@ -75,9 +75,27 @@ pub const AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER: &str = "x-amz-object-lock-legal-hold
 pub const AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE: &str = "X-Amz-Bypass-Governance-Retention";
 pub const AMZ_BUCKET_REPLICATION_STATUS: &str = "X-Amz-Replication-Status";
 
-// AmzSnowballExtract will trigger unpacking of an archive content
+// Snowball auto-extract compatibility headers.
+//
+// Supported external request headers:
+// - X-Amz-Meta-Snowball-Auto-Extract
+// - X-Amz-Snowball-Auto-Extract
+// - X-Amz-Meta-Snowball-Prefix
+// - X-Amz-Meta-Snowball-Ignore-Dirs
+// - X-Amz-Meta-Snowball-Ignore-Errors
+// - X-Amz-Meta-Minio-Snowball-Prefix
+// - X-Amz-Meta-Minio-Snowball-Ignore-Dirs
+// - X-Amz-Meta-Minio-Snowball-Ignore-Errors
+//
+// Internal compatibility headers:
+// - X-Amz-Meta-Rustfs-Snowball-Prefix
+// - X-Amz-Meta-Rustfs-Snowball-Ignore-Dirs
+// - X-Amz-Meta-Rustfs-Snowball-Ignore-Errors
 pub const AMZ_SNOWBALL_EXTRACT: &str = "X-Amz-Meta-Snowball-Auto-Extract";
 pub const AMZ_SNOWBALL_EXTRACT_ALT: &str = "X-Amz-Snowball-Auto-Extract";
+pub const AMZ_SNOWBALL_PREFIX: &str = "X-Amz-Meta-Snowball-Prefix";
+pub const AMZ_SNOWBALL_IGNORE_DIRS: &str = "X-Amz-Meta-Snowball-Ignore-Dirs";
+pub const AMZ_SNOWBALL_IGNORE_ERRORS: &str = "X-Amz-Meta-Snowball-Ignore-Errors";
 pub const AMZ_MINIO_SNOWBALL_PREFIX: &str = "X-Amz-Meta-Minio-Snowball-Prefix";
 pub const AMZ_MINIO_SNOWBALL_IGNORE_DIRS: &str = "X-Amz-Meta-Minio-Snowball-Ignore-Dirs";
 pub const AMZ_MINIO_SNOWBALL_IGNORE_ERRORS: &str = "X-Amz-Meta-Minio-Snowball-Ignore-Errors";

rustfs/src/app/object_usecase.rs

@@ -100,7 +100,7 @@ use rustfs_utils::http::{
         AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING, AMZ_RESTORE_EXPIRY_DAYS,
         AMZ_RESTORE_REQUEST_DATE, AMZ_RUSTFS_SNOWBALL_IGNORE_DIRS, AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, AMZ_RUSTFS_SNOWBALL_PREFIX,
         AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, AMZ_SNOWBALL_EXTRACT, AMZ_SNOWBALL_EXTRACT_ALT,
-        AMZ_STORAGE_CLASS, AMZ_TAG_COUNT,
+        AMZ_SNOWBALL_IGNORE_DIRS, AMZ_SNOWBALL_IGNORE_ERRORS, AMZ_SNOWBALL_PREFIX, AMZ_STORAGE_CLASS, AMZ_TAG_COUNT,
     },
     insert_str, remove_str,
 };
@@ -340,6 +340,17 @@ const AMZ_META_PREFIX_LOWER: &str = "x-amz-meta-";
 const SNOWBALL_PREFIX_SUFFIX_LOWER: &str = "snowball-prefix";
 const SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER: &str = "snowball-ignore-dirs";
 const SNOWBALL_IGNORE_ERRORS_SUFFIX_LOWER: &str = "snowball-ignore-errors";
+const SNOWBALL_PREFIX_HEADER_KEYS: &[&str] = &[AMZ_MINIO_SNOWBALL_PREFIX, AMZ_SNOWBALL_PREFIX, AMZ_RUSTFS_SNOWBALL_PREFIX];
+const SNOWBALL_IGNORE_DIRS_HEADER_KEYS: &[&str] = &[
+    AMZ_MINIO_SNOWBALL_IGNORE_DIRS,
+    AMZ_SNOWBALL_IGNORE_DIRS,
+    AMZ_RUSTFS_SNOWBALL_IGNORE_DIRS,
+];
+const SNOWBALL_IGNORE_ERRORS_HEADER_KEYS: &[&str] = &[
+    AMZ_MINIO_SNOWBALL_IGNORE_ERRORS,
+    AMZ_SNOWBALL_IGNORE_ERRORS,
+    AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS,
+];
 
 #[derive(Debug, Clone, Default, PartialEq, Eq)]
 struct PutObjectExtractOptions {
@@ -359,15 +370,23 @@ fn is_put_object_extract_requested(headers: &HeaderMap) -> bool {
     header_value_is_true(headers, AMZ_SNOWBALL_EXTRACT) || header_value_is_true(headers, AMZ_SNOWBALL_EXTRACT_ALT)
 }
 
-fn snowball_meta_value_by_suffix(headers: &HeaderMap, preferred_key: &str, suffix_lower: &str) -> Option<String> {
-    if let Some(preferred) = headers.get(preferred_key).and_then(|value| value.to_str().ok()) {
-        return Some(preferred.trim().to_string());
-    }
+fn trimmed_header_value(headers: &HeaderMap, key: &str) -> Option<String> {
+    headers
+        .get(key)
+        .and_then(|value| value.to_str().ok())
+        .map(|value| value.trim().to_string())
+}
 
+fn is_exact_snowball_meta_key(key: &str, exact_keys: &[&str]) -> bool {
+    exact_keys.iter().any(|exact_key| key.eq_ignore_ascii_case(exact_key))
+}
+
+fn snowball_meta_value_by_suffix(headers: &HeaderMap, suffix_lower: &str, exact_keys: &[&str]) -> Option<String> {
     for (name, value) in headers {
-        let key = name.as_str().to_ascii_lowercase();
+        let key = name.as_str();
         if key.starts_with(AMZ_META_PREFIX_LOWER)
             && key.ends_with(suffix_lower)
+            && !is_exact_snowball_meta_key(key, exact_keys)
             && let Ok(parsed) = value.to_str()
         {
             return Some(parsed.trim().to_string());
@@ -377,8 +396,18 @@ fn snowball_meta_value_by_suffix(headers: &HeaderMap, preferred_key: &str, suffi
     None
 }
 
-fn snowball_meta_flag_by_suffix(headers: &HeaderMap, preferred_key: &str, suffix_lower: &str) -> bool {
-    snowball_meta_value_by_suffix(headers, preferred_key, suffix_lower).is_some_and(|value| value.eq_ignore_ascii_case("true"))
+fn snowball_meta_value(headers: &HeaderMap, exact_keys: &[&str], suffix_lower: &str) -> Option<String> {
+    for key in exact_keys {
+        if let Some(value) = trimmed_header_value(headers, key) {
+            return Some(value);
+        }
+    }
+
+    snowball_meta_value_by_suffix(headers, suffix_lower, exact_keys)
+}
+
+fn snowball_meta_flag(headers: &HeaderMap, exact_keys: &[&str], suffix_lower: &str) -> bool {
+    snowball_meta_value(headers, exact_keys, suffix_lower).is_some_and(|value| value.eq_ignore_ascii_case("true"))
 }
 
 fn normalize_snowball_prefix(prefix: &str) -> Option<String> {
@@ -661,14 +690,10 @@ fn delete_creates_delete_marker(opts: &ObjectOptions) -> bool {
 }
 
 fn resolve_put_object_extract_options(headers: &HeaderMap) -> PutObjectExtractOptions {
-    let prefix = snowball_meta_value_by_suffix(headers, AMZ_MINIO_SNOWBALL_PREFIX, SNOWBALL_PREFIX_SUFFIX_LOWER)
-        .or_else(|| snowball_meta_value_by_suffix(headers, AMZ_RUSTFS_SNOWBALL_PREFIX, SNOWBALL_PREFIX_SUFFIX_LOWER))
+    let prefix = snowball_meta_value(headers, SNOWBALL_PREFIX_HEADER_KEYS, SNOWBALL_PREFIX_SUFFIX_LOWER)
         .and_then(|value| normalize_snowball_prefix(&value));
-    let ignore_dirs = snowball_meta_flag_by_suffix(headers, AMZ_MINIO_SNOWBALL_IGNORE_DIRS, SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER)
-        || snowball_meta_flag_by_suffix(headers, AMZ_RUSTFS_SNOWBALL_IGNORE_DIRS, SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER);
-    let ignore_errors =
-        snowball_meta_flag_by_suffix(headers, AMZ_MINIO_SNOWBALL_IGNORE_ERRORS, SNOWBALL_IGNORE_ERRORS_SUFFIX_LOWER)
-            || snowball_meta_flag_by_suffix(headers, AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, SNOWBALL_IGNORE_ERRORS_SUFFIX_LOWER);
+    let ignore_dirs = snowball_meta_flag(headers, SNOWBALL_IGNORE_DIRS_HEADER_KEYS, SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER);
+    let ignore_errors = snowball_meta_flag(headers, SNOWBALL_IGNORE_ERRORS_HEADER_KEYS, SNOWBALL_IGNORE_ERRORS_SUFFIX_LOWER);
 
     PutObjectExtractOptions {
         prefix,
@@ -4634,6 +4659,19 @@ mod tests {
         assert!(options.ignore_errors);
     }
 
+    #[test]
+    fn resolve_put_object_extract_options_accepts_standard_headers() {
+        let mut headers = HeaderMap::new();
+        headers.insert(AMZ_SNOWBALL_PREFIX, HeaderValue::from_static(" /standard/prefix/ "));
+        headers.insert(AMZ_SNOWBALL_IGNORE_DIRS, HeaderValue::from_static(" true "));
+        headers.insert(AMZ_SNOWBALL_IGNORE_ERRORS, HeaderValue::from_static("TRUE"));
+
+        let options = resolve_put_object_extract_options(&headers);
+        assert_eq!(options.prefix.as_deref(), Some("standard/prefix"));
+        assert!(options.ignore_dirs);
+        assert!(options.ignore_errors);
+    }
+
     #[test]
     fn resolve_put_object_extract_options_accepts_suffix_compatible_headers() {
         let mut headers = HeaderMap::new();
@@ -4646,6 +4684,31 @@ mod tests {
         assert!(options.ignore_dirs);
         assert!(options.ignore_errors);
     }
+
+    #[test]
+    fn resolve_put_object_extract_options_prefers_exact_headers_over_suffix_fallback() {
+        let mut headers = HeaderMap::new();
+        headers.insert("x-amz-meta-acme-snowball-prefix", HeaderValue::from_static("/fallback/prefix/"));
+        headers.insert(AMZ_RUSTFS_SNOWBALL_PREFIX, HeaderValue::from_static("/internal/prefix/"));
+        headers.insert(AMZ_SNOWBALL_PREFIX, HeaderValue::from_static("/standard/prefix/"));
+        headers.insert(AMZ_MINIO_SNOWBALL_PREFIX, HeaderValue::from_static("/minio/prefix/"));
+
+        let options = resolve_put_object_extract_options(&headers);
+        assert_eq!(options.prefix.as_deref(), Some("minio/prefix"));
+    }
+
+    #[test]
+    fn resolve_put_object_extract_options_exact_flags_override_suffix_fallback() {
+        let mut headers = HeaderMap::new();
+        headers.insert(AMZ_SNOWBALL_IGNORE_DIRS, HeaderValue::from_static("false"));
+        headers.insert("x-amz-meta-acme-snowball-ignore-dirs", HeaderValue::from_static("true"));
+        headers.insert(AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, HeaderValue::from_static("false"));
+        headers.insert("x-amz-meta-acme-snowball-ignore-errors", HeaderValue::from_static("true"));
+
+        let options = resolve_put_object_extract_options(&headers);
+        assert!(!options.ignore_dirs);
+        assert!(!options.ignore_errors);
+    }
     #[tokio::test]
     async fn execute_put_object_rejects_post_object_sse_kms_from_input() {
         let input = PutObjectInput::builder()