diff --git a/Cargo.lock b/Cargo.lock index 599f2284f..90317f143 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3081,14 +3081,19 @@ name = "e2e_test" version = "0.0.5" dependencies = [ "anyhow", + "astral-tokio-tar", + "async-compression", "async-trait", "aws-config", "aws-sdk-s3", + "aws-smithy-http-client", "base64 0.22.1", "bytes", "chrono", "flatbuffers", + "flate2", "futures", + "http 1.4.0", "md5", "rand 0.10.0", "rcgen", @@ -3100,18 +3105,24 @@ dependencies = [ "rustfs-lock", "rustfs-madmin", "rustfs-protos", + "rustfs-signer", "rustls", + "s3s", "serde", "serde_json", "serial_test", "sha2 0.11.0-rc.5", "suppaftp", + "time", "tokio", "tokio-stream", "tonic", "tracing", "tracing-subscriber", + "urlencoding", "uuid", + "walkdir", + "zstd", ] [[package]] @@ -7325,6 +7336,7 @@ dependencies = [ "rustfs-s3select-api", "rustfs-s3select-query", "rustfs-scanner", + "rustfs-signer", "rustfs-targets", "rustfs-trusted-proxies", "rustfs-utils", @@ -7728,6 +7740,7 @@ version = "0.0.5" dependencies = [ "anyhow", "aws-sdk-s3", + "aws-smithy-http-client", "clap", "mime_guess", "rmcp", @@ -8329,7 +8342,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "s3s" version = "0.14.0-dev" -source = "git+https://github.com/rustfs/s3s?rev=d9556e3c0036bd3f2b330966009cbaa5aebf19a3#d9556e3c0036bd3f2b330966009cbaa5aebf19a3" +source = "git+https://github.com/rustfs/s3s?rev=b296762bc9e7fa608f1bc44f5cd625d606e0dd31#b296762bc9e7fa608f1bc44f5cd625d606e0dd31" dependencies = [ "arc-swap", "arrayvec", diff --git a/Cargo.toml b/Cargo.toml index 5fa3d1058..c0ae594fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -243,7 +243,7 @@ rumqttc = { version = "0.25.1" } rustix = { version = "1.1.4", features = ["fs"] } rust-embed = { version = "8.11.0" } rustc-hash = { version = "2.1.1" } -s3s = { git = "https://github.com/rustfs/s3s", rev = "d9556e3c0036bd3f2b330966009cbaa5aebf19a3", features = ["minio"] } +s3s = { git = "https://github.com/rustfs/s3s", rev = "b296762bc9e7fa608f1bc44f5cd625d606e0dd31", features = ["minio"] } serial_test = "3.4.0" shadow-rs = { version = "1.7.1", default-features = false } siphasher = "1.0.2" diff --git a/crates/e2e_test/Cargo.toml b/crates/e2e_test/Cargo.toml index e5646cc0c..25c6ca0e9 100644 --- a/crates/e2e_test/Cargo.toml +++ b/crates/e2e_test/Cargo.toml @@ -46,17 +46,28 @@ bytes.workspace = true serial_test = { workspace = true } aws-sdk-s3.workspace = true aws-config = { workspace = true } +aws-smithy-http-client.workspace = true +async-compression = { workspace = true, features = ["tokio", "bzip2", "xz"] } async-trait = { workspace = true } +flate2.workspace = true +http.workspace = true reqwest = { workspace = true } +rustfs-signer.workspace = true tracing = { workspace = true } tracing-subscriber = { workspace = true } uuid = { workspace = true } +urlencoding.workspace = true +walkdir.workspace = true base64 = { workspace = true } rand = { workspace = true } chrono = { workspace = true } md5 = { workspace = true } sha2 = { workspace = true } +astral-tokio-tar = { workspace = true } +s3s.workspace = true +zstd.workspace = true +time.workspace = true suppaftp = { workspace = true, features = ["tokio", "rustls-aws-lc-rs"] } rcgen.workspace = true anyhow.workspace = true -rustls.workspace = true \ No newline at end of file +rustls.workspace = true diff --git a/crates/e2e_test/src/anonymous_access_test.rs b/crates/e2e_test/src/anonymous_access_test.rs index 4d12557c9..eb10d1063 100644 --- a/crates/e2e_test/src/anonymous_access_test.rs +++ b/crates/e2e_test/src/anonymous_access_test.rs @@ -16,7 +16,7 @@ //! Verifies that anonymous access works correctly with bucket policies //! when PublicAccessBlock configuration is missing or explicitly set. -use crate::common::{RustFSTestEnvironment, init_logging}; +use crate::common::{RustFSTestEnvironment, init_logging, local_http_client}; use aws_sdk_s3::types::PublicAccessBlockConfiguration; use serial_test::serial; use tracing::info; @@ -67,7 +67,7 @@ async fn anonymous_get_object( key: &str, ) -> Result { let url = format!("{}/{}/{}", env.url, bucket_name, key); - reqwest::Client::new().get(&url).send().await + local_http_client().get(&url).send().await } /// Issue #2036: Anonymous GetObject should succeed when bucket policy allows it diff --git a/crates/e2e_test/src/bucket_logging_test.rs b/crates/e2e_test/src/bucket_logging_test.rs new file mode 100644 index 000000000..461c52995 --- /dev/null +++ b/crates/e2e_test/src/bucket_logging_test.rs @@ -0,0 +1,544 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! End-to-end tests for S3 dummy-compat bucket APIs. + +#[cfg(test)] +mod tests { + use crate::common::{RustFSTestEnvironment, init_logging}; + use aws_sdk_s3::error::ProvideErrorMetadata; + use aws_sdk_s3::types::{ + AccelerateConfiguration, BucketAccelerateStatus, BucketLoggingStatus, IndexDocument, LoggingEnabled, Payer, + RequestPaymentConfiguration, WebsiteConfiguration, + }; + use serial_test::serial; + use std::path::PathBuf; + use std::process::Command; + use tracing::info; + + fn awscurl_binary_path() -> PathBuf { + std::env::var_os("AWSCURL_PATH") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from("awscurl")) + } + + fn awscurl_available() -> bool { + Command::new(awscurl_binary_path()).arg("--version").output().is_ok() + } + + fn execute_s3_awscurl( + method: &str, + url: &str, + access_key: &str, + secret_key: &str, + ) -> Result> { + let output = Command::new(awscurl_binary_path()) + .args([ + "--service", + "s3", + "--region", + "us-east-1", + "--access_key", + access_key, + "--secret_key", + secret_key, + "-i", + "-X", + method, + url, + ]) + .output()?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + let stdout = String::from_utf8_lossy(&output.stdout); + return Err(format!("awscurl failed: stderr='{stderr}', stdout='{stdout}'").into()); + } + Ok(String::from_utf8_lossy(&output.stdout).to_string()) + } + + fn parse_status(raw: &str) -> Option { + raw.lines() + .filter_map(|line| { + if line.starts_with("HTTP/") { + line.split_whitespace().nth(1)?.parse::().ok() + } else { + None + } + }) + .next_back() + } + + fn parse_body(raw: &str) -> String { + if let Some(pos) = raw.rfind("\r\n\r\n") { + return raw[pos + 4..].to_string(); + } + if let Some(pos) = raw.rfind("\n\n") { + return raw[pos + 2..].to_string(); + } + String::new() + } + + fn parse_headers(raw: &str) -> String { + let start = raw.rfind("HTTP/").unwrap_or(0); + let tail = &raw[start..]; + if let Some(pos) = tail.find("\r\n\r\n") { + return tail[..pos].to_string(); + } + if let Some(pos) = tail.find("\n\n") { + return tail[..pos].to_string(); + } + tail.to_string() + } + + #[tokio::test] + #[serial] + async fn test_dummy_bucket_compatibility_endpoints() { + init_logging(); + info!("Starting test: dummy-compat bucket APIs should match S3-compatible behavior"); + + let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); + env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); + + let client = env.create_s3_client(); + let bucket = "test-get-bucket-logging"; + + client + .create_bucket() + .bucket(bucket) + .send() + .await + .expect("Failed to create bucket"); + + let result = client.get_bucket_logging().bucket(bucket).send().await; + assert!( + result.is_ok(), + "GetBucketLogging should return success for existing bucket, got: {:?}", + result.err() + ); + + let output = result.unwrap(); + assert!( + output.logging_enabled().is_none(), + "Default GetBucketLogging should return empty logging configuration" + ); + + let put_logging = client + .put_bucket_logging() + .bucket(bucket) + .bucket_logging_status( + BucketLoggingStatus::builder() + .logging_enabled( + LoggingEnabled::builder() + .target_bucket(bucket) + .target_prefix("logs/") + .build() + .expect("failed to build LoggingEnabled"), + ) + .build(), + ) + .send() + .await; + assert!( + put_logging.is_ok(), + "PutBucketLogging should return success for existing bucket, got: {:?}", + put_logging.err() + ); + + let output_after_put = client + .get_bucket_logging() + .bucket(bucket) + .send() + .await + .expect("GetBucketLogging should succeed after PutBucketLogging"); + let logging_after_put = output_after_put + .logging_enabled() + .expect("GetBucketLogging should return persisted logging_enabled"); + assert_eq!( + logging_after_put.target_bucket(), + bucket, + "GetBucketLogging should preserve target bucket" + ); + assert_eq!( + logging_after_put.target_prefix(), + "logs/", + "GetBucketLogging should preserve target prefix" + ); + + let accelerate = client + .get_bucket_accelerate_configuration() + .bucket(bucket) + .send() + .await + .expect("GetBucketAccelerateConfiguration should succeed"); + assert!( + accelerate.status().is_none(), + "Default GetBucketAccelerateConfiguration should return empty status" + ); + + let payment = client + .get_bucket_request_payment() + .bucket(bucket) + .send() + .await + .expect("GetBucketRequestPayment should succeed"); + assert_eq!( + payment.payer().map(|p| p.as_str()), + Some("BucketOwner"), + "GetBucketRequestPayment should return BucketOwner by default" + ); + + let put_accelerate = client + .put_bucket_accelerate_configuration() + .bucket(bucket) + .accelerate_configuration( + AccelerateConfiguration::builder() + .status(BucketAccelerateStatus::Suspended) + .build(), + ) + .send() + .await; + assert!( + put_accelerate.is_ok(), + "PutBucketAccelerateConfiguration should return success for existing bucket, got: {:?}", + put_accelerate.err() + ); + + let put_request_payment = client + .put_bucket_request_payment() + .bucket(bucket) + .request_payment_configuration( + RequestPaymentConfiguration::builder() + .payer(Payer::Requester) + .build() + .expect("failed to build RequestPaymentConfiguration"), + ) + .send() + .await; + assert!( + put_request_payment.is_ok(), + "PutBucketRequestPayment should return success for existing bucket, got: {:?}", + put_request_payment.err() + ); + + let accelerate_after_put = client + .get_bucket_accelerate_configuration() + .bucket(bucket) + .send() + .await + .expect("GetBucketAccelerateConfiguration should succeed after put"); + assert_eq!( + accelerate_after_put.status().map(|s| s.as_str()), + Some("Suspended"), + "GetBucketAccelerateConfiguration should preserve put status" + ); + + let payment_after_put = client + .get_bucket_request_payment() + .bucket(bucket) + .send() + .await + .expect("GetBucketRequestPayment should succeed after put"); + assert_eq!( + payment_after_put.payer().map(|p| p.as_str()), + Some("Requester"), + "GetBucketRequestPayment should preserve put payer" + ); + + let put_website = client + .put_bucket_website() + .bucket(bucket) + .website_configuration( + WebsiteConfiguration::builder() + .index_document( + IndexDocument::builder() + .suffix("index.html") + .build() + .expect("failed to build IndexDocument"), + ) + .build(), + ) + .send() + .await; + assert!( + put_website.is_ok(), + "PutBucketWebsite should return success for existing bucket, got: {:?}", + put_website.err() + ); + + let website = client.get_bucket_website().bucket(bucket).send().await; + assert!(website.is_ok(), "GetBucketWebsite should return persisted website configuration"); + let website_output = website.unwrap(); + assert_eq!( + website_output.index_document().map(|doc| doc.suffix()), + Some("index.html"), + "GetBucketWebsite should preserve index document suffix" + ); + + client + .delete_bucket_website() + .bucket(bucket) + .send() + .await + .expect("DeleteBucketWebsite should return success"); + + let website_after_delete = client.get_bucket_website().bucket(bucket).send().await; + assert!( + website_after_delete.is_err(), + "GetBucketWebsite should return NoSuchWebsiteConfiguration after deletion" + ); + let website_err = website_after_delete.err().unwrap(); + let website_code = website_err.as_service_error().and_then(|e| e.code()); + assert!( + matches!(website_code, Some("NoSuchWebsiteConfiguration")), + "Unexpected GetBucketWebsite error code: {:?}, err: {:?}", + website_code, + website_err + ); + + env.stop_server(); + } + + #[tokio::test] + #[serial] + async fn test_dummy_bucket_compatibility_endpoints_no_such_bucket() { + init_logging(); + info!("Starting test: dummy-compat bucket APIs should return NoSuchBucket for missing bucket"); + + let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); + env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); + + let client = env.create_s3_client(); + let missing_bucket = "test-dummy-bucket-missing"; + + let get_logging = client.get_bucket_logging().bucket(missing_bucket).send().await; + assert!(get_logging.is_err(), "GetBucketLogging should fail for missing bucket"); + let get_logging_err = get_logging.err().unwrap(); + let get_logging_code = get_logging_err.as_service_error().and_then(|e| e.code()); + assert!( + matches!(get_logging_code, Some("NoSuchBucket")), + "Unexpected GetBucketLogging error code: {:?}, err: {:?}", + get_logging_code, + get_logging_err + ); + + let put_logging = client + .put_bucket_logging() + .bucket(missing_bucket) + .bucket_logging_status(BucketLoggingStatus::builder().build()) + .send() + .await; + assert!(put_logging.is_err(), "PutBucketLogging should fail for missing bucket"); + let put_logging_err = put_logging.err().unwrap(); + let put_logging_code = put_logging_err.as_service_error().and_then(|e| e.code()); + assert!( + matches!(put_logging_code, Some("NoSuchBucket")), + "Unexpected PutBucketLogging error code: {:?}, err: {:?}", + put_logging_code, + put_logging_err + ); + + let get_accelerate = client + .get_bucket_accelerate_configuration() + .bucket(missing_bucket) + .send() + .await; + assert!(get_accelerate.is_err(), "GetBucketAccelerateConfiguration should fail for missing bucket"); + let get_accelerate_err = get_accelerate.err().unwrap(); + let get_accelerate_code = get_accelerate_err.as_service_error().and_then(|e| e.code()); + assert!( + matches!(get_accelerate_code, Some("NoSuchBucket")), + "Unexpected GetBucketAccelerateConfiguration error code: {:?}, err: {:?}", + get_accelerate_code, + get_accelerate_err + ); + + let get_request_payment = client.get_bucket_request_payment().bucket(missing_bucket).send().await; + assert!(get_request_payment.is_err(), "GetBucketRequestPayment should fail for missing bucket"); + let get_request_payment_err = get_request_payment.err().unwrap(); + let get_request_payment_code = get_request_payment_err.as_service_error().and_then(|e| e.code()); + assert!( + matches!(get_request_payment_code, Some("NoSuchBucket")), + "Unexpected GetBucketRequestPayment error code: {:?}, err: {:?}", + get_request_payment_code, + get_request_payment_err + ); + + let put_accelerate = client + .put_bucket_accelerate_configuration() + .bucket(missing_bucket) + .accelerate_configuration( + AccelerateConfiguration::builder() + .status(BucketAccelerateStatus::Suspended) + .build(), + ) + .send() + .await; + assert!(put_accelerate.is_err(), "PutBucketAccelerateConfiguration should fail for missing bucket"); + let put_accelerate_err = put_accelerate.err().unwrap(); + let put_accelerate_code = put_accelerate_err.as_service_error().and_then(|e| e.code()); + assert!( + matches!(put_accelerate_code, Some("NoSuchBucket")), + "Unexpected PutBucketAccelerateConfiguration error code: {:?}, err: {:?}", + put_accelerate_code, + put_accelerate_err + ); + + let put_request_payment = client + .put_bucket_request_payment() + .bucket(missing_bucket) + .request_payment_configuration( + RequestPaymentConfiguration::builder() + .payer(Payer::BucketOwner) + .build() + .expect("failed to build RequestPaymentConfiguration"), + ) + .send() + .await; + assert!(put_request_payment.is_err(), "PutBucketRequestPayment should fail for missing bucket"); + let put_request_payment_err = put_request_payment.err().unwrap(); + let put_request_payment_code = put_request_payment_err.as_service_error().and_then(|e| e.code()); + assert!( + matches!(put_request_payment_code, Some("NoSuchBucket")), + "Unexpected PutBucketRequestPayment error code: {:?}, err: {:?}", + put_request_payment_code, + put_request_payment_err + ); + + let put_website = client + .put_bucket_website() + .bucket(missing_bucket) + .website_configuration( + WebsiteConfiguration::builder() + .index_document( + IndexDocument::builder() + .suffix("index.html") + .build() + .expect("failed to build IndexDocument"), + ) + .build(), + ) + .send() + .await; + assert!(put_website.is_err(), "PutBucketWebsite should fail for missing bucket"); + let put_website_err = put_website.err().unwrap(); + let put_website_code = put_website_err.as_service_error().and_then(|e| e.code()); + assert!( + matches!(put_website_code, Some("NoSuchBucket")), + "Unexpected PutBucketWebsite error code: {:?}, err: {:?}", + put_website_code, + put_website_err + ); + + let get_website = client.get_bucket_website().bucket(missing_bucket).send().await; + assert!(get_website.is_err(), "GetBucketWebsite should fail for missing bucket"); + let get_website_err = get_website.err().unwrap(); + let get_website_code = get_website_err.as_service_error().and_then(|e| e.code()); + assert!( + matches!(get_website_code, Some("NoSuchBucket")), + "Unexpected GetBucketWebsite error code: {:?}, err: {:?}", + get_website_code, + get_website_err + ); + + let delete_website = client.delete_bucket_website().bucket(missing_bucket).send().await; + assert!(delete_website.is_err(), "DeleteBucketWebsite should fail for missing bucket"); + let delete_website_err = delete_website.err().unwrap(); + let delete_website_code = delete_website_err.as_service_error().and_then(|e| e.code()); + assert!( + matches!(delete_website_code, Some("NoSuchBucket")), + "Unexpected DeleteBucketWebsite error code: {:?}, err: {:?}", + delete_website_code, + delete_website_err + ); + + env.stop_server(); + } + + #[tokio::test] + #[serial] + async fn test_dummy_bucket_endpoints_http_contracts() { + init_logging(); + info!("Starting test: dummy-compat bucket API HTTP contracts"); + if !awscurl_available() { + info!("Skipping test_dummy_bucket_endpoints_http_contracts: awscurl binary not found"); + return; + } + + let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); + env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); + + let client = env.create_s3_client(); + let bucket = "test-dummy-bucket-http-contracts"; + + client + .create_bucket() + .bucket(bucket) + .send() + .await + .expect("Failed to create bucket"); + + let logging_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?logging=", env.url), &env.access_key, &env.secret_key) + .expect("GetBucketLogging HTTP request failed"); + assert_eq!(parse_status(&logging_raw), Some(200), "GetBucketLogging should return 200"); + let logging_body = parse_body(&logging_raw); + assert!( + logging_body.contains("BucketOwner"), + "GetBucketRequestPayment should return BucketOwner payer, got: {payment_body}" + ); + + let website_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?website=", env.url), &env.access_key, &env.secret_key) + .expect("GetBucketWebsite HTTP request failed"); + assert_eq!( + parse_status(&website_raw), + Some(404), + "GetBucketWebsite should return 404 when website config is absent" + ); + let website_content_type = parse_headers(&website_raw).to_ascii_lowercase(); + assert!( + website_content_type.contains("content-type:") && website_content_type.contains("xml"), + "GetBucketWebsite error response should be XML, got content-type: {website_content_type}" + ); + let website_body = parse_body(&website_raw); + assert!( + website_body.contains("NoSuchWebsiteConfiguration"), + "GetBucketWebsite should return NoSuchWebsiteConfiguration code, got: {website_body}" + ); + + let delete_raw = + execute_s3_awscurl("DELETE", &format!("{}/{bucket}?website=", env.url), &env.access_key, &env.secret_key) + .expect("DeleteBucketWebsite HTTP request failed"); + assert_eq!(parse_status(&delete_raw), Some(204), "DeleteBucketWebsite should return 204"); + + env.stop_server(); + } +} diff --git a/crates/e2e_test/src/bucket_policy_check_test.rs b/crates/e2e_test/src/bucket_policy_check_test.rs index c0b18ec65..9b958f343 100644 --- a/crates/e2e_test/src/bucket_policy_check_test.rs +++ b/crates/e2e_test/src/bucket_policy_check_test.rs @@ -54,6 +54,10 @@ fn create_user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: #[serial] async fn test_bucket_policy_authenticated_user() -> Result<(), Box> { init_logging(); + if !crate::common::awscurl_available() { + info!("Skipping test_bucket_policy_authenticated_user because awscurl is not available"); + return Ok(()); + } info!("Starting test_bucket_policy_authenticated_user..."); let mut env = RustFSTestEnvironment::new().await?; diff --git a/crates/e2e_test/src/common.rs b/crates/e2e_test/src/common.rs index aab8bf283..c6a268335 100644 --- a/crates/e2e_test/src/common.rs +++ b/crates/e2e_test/src/common.rs @@ -23,7 +23,11 @@ use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::{Client, Config}; -use std::path::PathBuf; +use aws_smithy_http_client::Builder as SmithyHttpClientBuilder; +use reqwest::Client as HttpClient; +use std::ffi::OsStr; +use std::fs as stdfs; +use std::path::{Path, PathBuf}; use std::process::{Child, Command}; use std::sync::Once; use std::time::Duration; @@ -32,11 +36,29 @@ use tokio::net::TcpStream; use tokio::time::sleep; use tracing::{error, info, warn}; use uuid::Uuid; +use walkdir::WalkDir; // Common constants for all E2E tests pub const DEFAULT_ACCESS_KEY: &str = "rustfsadmin"; pub const DEFAULT_SECRET_KEY: &str = "rustfsadmin"; pub const TEST_BUCKET: &str = "e2e-test-bucket"; + +fn build_test_s3_config(endpoint_url: &str, access_key: &str, secret_key: &str, provider_name: &'static str) -> Config { + let credentials = Credentials::new(access_key, secret_key, None, None, provider_name); + let mut config = Config::builder() + .credentials_provider(credentials) + .region(Region::new("us-east-1")) + .endpoint_url(endpoint_url) + .force_path_style(true) + .behavior_version_latest(); + + if endpoint_url.starts_with("http://") { + config = config.http_client(SmithyHttpClientBuilder::new().build_http()); + } + + config.build() +} + pub fn workspace_root() -> PathBuf { let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); path.pop(); // e2e_test @@ -44,29 +66,125 @@ pub fn workspace_root() -> PathBuf { path } +pub fn local_http_client() -> HttpClient { + HttpClient::builder() + .no_proxy() + .build() + .expect("failed to build local reqwest client") +} + /// Resolve the RustFS binary relative to the workspace. -/// Always builds the binary to ensure it's up to date. pub fn rustfs_binary_path() -> PathBuf { + rustfs_binary_path_with_features(requested_rustfs_build_features().as_deref()) +} + +/// Resolve the RustFS binary relative to the workspace, optionally requesting build features. +pub fn rustfs_binary_path_with_features(requested_features: Option<&str>) -> PathBuf { if let Some(path) = std::env::var_os("CARGO_BIN_EXE_rustfs") { return PathBuf::from(path); } - // Always build the binary to ensure it's up to date - info!("Building RustFS binary to ensure it's up to date..."); - build_rustfs_binary(); - let mut binary_path = workspace_root(); binary_path.push("target"); let profile_dir = if cfg!(debug_assertions) { "debug" } else { "release" }; binary_path.push(profile_dir); binary_path.push(format!("rustfs{}", std::env::consts::EXE_SUFFIX)); + let features_match = binary_features_match(&binary_path, requested_features); + let source_is_newer = workspace_sources_newer_than_binary(&binary_path); + let can_reuse_inside_e2e = running_inside_e2e_test_binary() && requested_features.is_none() && features_match; + if binary_path.is_file() && features_match && (!source_is_newer || can_reuse_inside_e2e) { + if source_is_newer { + warn!( + "RustFS binary at {:?} appears older than workspace sources; reusing it inside cargo test to avoid nested builds", + binary_path + ); + } + info!("Using existing RustFS binary at {:?}", binary_path); + return binary_path; + } + + info!("Building RustFS binary to ensure it's up to date..."); + build_rustfs_binary(requested_features); + info!("Using RustFS binary at {:?}", binary_path); binary_path } +fn workspace_sources_newer_than_binary(binary_path: &PathBuf) -> bool { + let Ok(binary_meta) = std::fs::metadata(binary_path) else { + return true; + }; + let Ok(binary_modified) = binary_meta.modified() else { + return true; + }; + + let workspace = workspace_root(); + let watch_roots = [ + workspace.join("Cargo.toml"), + workspace.join("Cargo.lock"), + workspace.join("rustfs"), + workspace.join("crates"), + ]; + + watch_roots.iter().any(|path| path_is_newer_than(binary_modified, path)) +} + +fn running_inside_e2e_test_binary() -> bool { + std::env::var("CARGO_PKG_NAME").is_ok_and(|value| value == "e2e_test") +} + +fn requested_rustfs_build_features() -> Option { + std::env::var("RUSTFS_BUILD_FEATURES") + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) +} + +fn rustfs_binary_features_stamp_path(binary_path: &Path) -> PathBuf { + binary_path.with_extension("features") +} + +fn binary_features_match(binary_path: &Path, requested_features: Option<&str>) -> bool { + let stamp_path = rustfs_binary_features_stamp_path(binary_path); + let recorded = stdfs::read_to_string(stamp_path).ok().map(|value| value.trim().to_string()); + + match requested_features { + Some(features) => recorded.as_deref() == Some(features), + None => recorded.as_deref().is_none_or(str::is_empty), + } +} + +fn path_is_newer_than(binary_modified: std::time::SystemTime, path: &Path) -> bool { + if path.is_file() { + return std::fs::metadata(path) + .and_then(|meta| meta.modified()) + .map(|modified| modified > binary_modified) + .unwrap_or(false); + } + + if !path.is_dir() { + return false; + } + + WalkDir::new(path) + .into_iter() + .filter_entry(|entry| { + let name = entry.file_name(); + name != OsStr::new("target") && name != OsStr::new(".git") + }) + .filter_map(Result::ok) + .filter(|entry| entry.file_type().is_file()) + .any(|entry| { + std::fs::metadata(entry.path()) + .and_then(|meta| meta.modified()) + .map(|modified| modified > binary_modified) + .unwrap_or(false) + }) +} + /// Build the RustFS binary using cargo -fn build_rustfs_binary() { +fn build_rustfs_binary(requested_features: Option<&str>) { let workspace = workspace_root(); info!("Building RustFS binary from workspace: {:?}", workspace); @@ -81,11 +199,8 @@ fn build_rustfs_binary() { let mut cmd = Command::new("cargo"); cmd.current_dir(&workspace).args(["build", "--bin", "rustfs"]); - // Read features from environment variable for e2e tests - if let Ok(features) = std::env::var("RUSTFS_BUILD_FEATURES") - && !features.is_empty() - { - cmd.arg("--features").arg(&features); + if let Some(features) = requested_features { + cmd.arg("--features").arg(features); info!("Building with features: {}", features); } @@ -105,6 +220,15 @@ fn build_rustfs_binary() { panic!("Failed to build RustFS binary. Error: {stderr}"); } + let mut binary_path = workspace; + binary_path.push("target"); + binary_path.push(if cfg!(debug_assertions) { "debug" } else { "release" }); + binary_path.push(format!("rustfs{}", std::env::consts::EXE_SUFFIX)); + let stamp_path = rustfs_binary_features_stamp_path(&binary_path); + if let Err(err) = stdfs::write(&stamp_path, requested_features.unwrap_or_default()) { + warn!("Failed to write RustFS feature stamp {:?}: {}", stamp_path, err); + } + info!("✅ RustFS binary built successfully"); } @@ -114,6 +238,17 @@ fn awscurl_binary_path() -> PathBuf { .unwrap_or_else(|| PathBuf::from("awscurl")) } +pub fn awscurl_available() -> bool { + let path = awscurl_binary_path(); + if path.components().count() > 1 || path.is_absolute() { + return path.is_file(); + } + + std::env::var_os("PATH") + .map(|paths| std::env::split_paths(&paths).any(|dir| dir.join(&path).is_file())) + .unwrap_or(false) +} + // Global initialization static INIT: Once = Once::new(); @@ -183,24 +318,22 @@ impl RustFSTestEnvironment { /// Kill any existing RustFS processes pub async fn cleanup_existing_processes(&self) -> Result<(), Box> { - info!("Cleaning up any existing RustFS processes"); - let binary_path = rustfs_binary_path(); - let binary_name = binary_path.to_string_lossy(); - let output = Command::new("pkill").args(["-f", &binary_name]).output(); + info!("Cleaning up any existing RustFS processes for {}", self.address); - if let Ok(output) = output - && output.status.success() - { - info!("Killed existing RustFS processes: {}", binary_name); - sleep(Duration::from_millis(1000)).await; + for pattern in [&self.address, &self.temp_dir] { + let output = Command::new("pkill").args(["-f", pattern]).output(); + + if let Ok(output) = output + && output.status.success() + { + info!("Killed existing RustFS processes matching: {}", pattern); + sleep(Duration::from_millis(250)).await; + } } Ok(()) } - /// Start RustFS server with basic configuration - pub async fn start_rustfs_server(&mut self, extra_args: Vec<&str>) -> Result<(), Box> { - self.cleanup_existing_processes().await?; - + fn build_start_args<'a>(&'a self, extra_args: Vec<&'a str>) -> Vec<&'a str> { let mut args = vec![ "--address", &self.address, @@ -210,16 +343,29 @@ impl RustFSTestEnvironment { &self.secret_key, ]; - // Add extra arguments args.extend(extra_args); - - // Add temp directory as the last argument args.push(&self.temp_dir); + args + } + + async fn start_rustfs_server_inner( + &mut self, + extra_args: Vec<&str>, + cleanup_existing: bool, + ) -> Result<(), Box> { + if cleanup_existing { + self.cleanup_existing_processes().await?; + } + + let args = self.build_start_args(extra_args); info!("Starting RustFS server with args: {:?}", args); let binary_path = rustfs_binary_path(); - let process = Command::new(&binary_path).args(&args).spawn()?; + let process = Command::new(&binary_path) + .env("RUST_LOG", "rustfs=info,rustfs_notify=debug") + .args(&args) + .spawn()?; self.process = Some(process); @@ -229,18 +375,40 @@ impl RustFSTestEnvironment { Ok(()) } - /// Wait for RustFS server to be ready by checking TCP connectivity + /// Start RustFS server with basic configuration + pub async fn start_rustfs_server(&mut self, extra_args: Vec<&str>) -> Result<(), Box> { + self.start_rustfs_server_inner(extra_args, true).await + } + + /// Start RustFS server without cleaning up other running RustFS processes. + /// + /// This is useful for tests that need multiple independent RustFS instances + /// alive at the same time. + pub async fn start_rustfs_server_without_cleanup( + &mut self, + extra_args: Vec<&str>, + ) -> Result<(), Box> { + self.start_rustfs_server_inner(extra_args, false).await + } + + /// Wait for RustFS server to be ready. + /// + /// A listening TCP port is not sufficient here: the process may accept + /// connections before the S3 stack is fully initialized, which causes + /// early requests to fail intermittently. Treat readiness as "S3 API + /// responds successfully" instead. pub async fn wait_for_server_ready(&self) -> Result<(), Box> { info!("Waiting for RustFS server to be ready on {}", self.address); + let client = self.create_s3_client(); - for i in 0..30 { - if TcpStream::connect(&self.address).await.is_ok() { + for i in 0..60 { + if TcpStream::connect(&self.address).await.is_ok() && client.list_buckets().send().await.is_ok() { info!("✅ RustFS server is ready after {} attempts", i + 1); return Ok(()); } - if i == 29 { - return Err("RustFS server failed to become ready within 30 seconds".into()); + if i == 59 { + return Err("RustFS server failed to become ready within 60 seconds".into()); } sleep(Duration::from_secs(1)).await; @@ -251,16 +419,7 @@ impl RustFSTestEnvironment { /// Create an AWS S3 client configured for this RustFS instance pub fn create_s3_client(&self) -> Client { - let credentials = Credentials::new(&self.access_key, &self.secret_key, None, None, "e2e-test"); - let config = Config::builder() - .credentials_provider(credentials) - .region(Region::new("us-east-1")) - .endpoint_url(&self.url) - .force_path_style(true) - .behavior_version_latest() - .build(); - - Client::from_conf(config) + Client::from_conf(build_test_s3_config(&self.url, &self.access_key, &self.secret_key, "e2e-test")) } /// Create test bucket @@ -493,6 +652,7 @@ impl RustFSTestClusterEnvironment { .env("RUSTFS_ACCESS_KEY", &self.access_key) .env("RUSTFS_SECRET_KEY", &self.secret_key) .env("RUSTFS_CONSOLE_ENABLE", "false") + .env("RUST_LOG", "rustfs=info,rustfs_notify=debug") .current_dir(&node.data_dir) .spawn()?; @@ -503,7 +663,9 @@ impl RustFSTestClusterEnvironment { self.wait_for_node_ready(&node.address, i).await?; } - self.wait_for_service_ready().await?; + for node_idx in 0..self.nodes.len() { + self.wait_for_node_service_ready(node_idx).await?; + } Ok(()) } @@ -523,17 +685,17 @@ impl RustFSTestClusterEnvironment { Err(format!("Node {} failed to become ready", idx).into()) } - /// Wait for the entire cluster's S3-compatible service to be ready (internal helper method). + /// Wait for a specific node's S3-compatible service to be ready (internal helper method). /// - /// Verifies service availability by calling the S3 `list_buckets` API, retries up to 120 times - /// with a 1-second interval between attempts. Fails if the API call remains unsuccessful after all retries. - async fn wait_for_service_ready(&self) -> Result<(), Box> { - let client = self.create_s3_client(0)?; + /// Verifies service availability by calling the S3 `list_buckets` API against the requested node, + /// retries up to 120 times with a 1-second interval between attempts. + async fn wait_for_node_service_ready(&self, node_idx: usize) -> Result<(), Box> { + let client = self.create_s3_client(node_idx)?; for attempt in 0..120 { match client.list_buckets().send().await { Ok(_) => { - info!("Cluster service ready after {} attempts", attempt + 1); + info!("Cluster node {} service ready after {} attempts", node_idx, attempt + 1); return Ok(()); } Err(_) => { @@ -541,7 +703,8 @@ impl RustFSTestClusterEnvironment { } } } - Err("Cluster service failed to become ready".into()) + + Err(format!("Cluster node {} service failed to become ready", node_idx).into()) } /// Create an S3 client configured to communicate with a specific cluster node. @@ -562,15 +725,12 @@ impl RustFSTestClusterEnvironment { if node_idx >= self.nodes.len() { return Err("node_idx is invalid".into()); } - let credentials = Credentials::new(&self.access_key, &self.secret_key, None, None, "cluster-test"); - let config = Config::builder() - .credentials_provider(credentials) - .region(Region::new("us-east-1")) - .endpoint_url(&self.nodes[node_idx].url) - .force_path_style(true) - .behavior_version_latest() - .build(); - Ok(Client::from_conf(config)) + Ok(Client::from_conf(build_test_s3_config( + &self.nodes[node_idx].url, + &self.access_key, + &self.secret_key, + "cluster-test", + ))) } /// Create S3 clients for all nodes in the RustFS cluster and collect them into a vector. diff --git a/crates/e2e_test/src/kms/common.rs b/crates/e2e_test/src/kms/common.rs index 455233056..98c520358 100644 --- a/crates/e2e_test/src/kms/common.rs +++ b/crates/e2e_test/src/kms/common.rs @@ -22,11 +22,13 @@ //! - KMS backend configuration (Local and Vault) //! - SSE encryption testing utilities -use crate::common::{RustFSTestEnvironment, awscurl_get, awscurl_post, init_logging as common_init_logging}; +use crate::common::{ + RustFSTestEnvironment, awscurl_available, awscurl_get, awscurl_post, init_logging as common_init_logging, local_http_client, +}; use aws_sdk_s3::Client; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::ServerSideEncryption; -use base64::Engine; +use base64::{Engine, engine::general_purpose::STANDARD as BASE64}; use serde_json; use std::process::{Child, Command}; use std::time::Duration; @@ -51,6 +53,19 @@ pub fn init_logging() { // Additional KMS-specific logging configuration can be added here if needed } +pub fn skip_if_kms_admin_tool_unavailable(test_name: &str) -> bool { + if awscurl_available() { + return false; + } + + info!("Skipping {} because awscurl is not available in PATH", test_name); + true +} + +pub fn sse_customer_key_md5_base64(key: &str) -> String { + BASE64.encode(md5::compute(key).0) +} + // KMS-specific helper functions /// Configure KMS backend via admin API pub async fn configure_kms( @@ -133,10 +148,10 @@ pub async fn create_key_with_specific_id(key_dir: &str, key_id: &str) -> Result< "usage": "EncryptDecrypt", "status": "Active", "metadata": HashMap::::new(), - "created_at": chrono::Utc::now().to_rfc3339(), + "created_at": format!("{}[UTC]", chrono::Utc::now().to_rfc3339()), "rotated_at": serde_json::Value::Null, "created_by": "e2e-test", - "encrypted_key_material": key_data.to_vec(), + "encrypted_key_material": BASE64.encode(key_data), "nonce": Vec::::new() }); @@ -155,7 +170,7 @@ pub async fn test_sse_c_encryption(s3_client: &Client, bucket: &str) -> Result<( let test_key = "01234567890123456789012345678901"; // 32-byte key let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key); - let test_key_md5 = format!("{:x}", md5::compute(test_key)); + let test_key_md5 = sse_customer_key_md5_base64(test_key); let test_data = b"Hello, KMS SSE-C World!"; let object_key = "test-sse-c-object"; @@ -272,6 +287,10 @@ pub async fn test_kms_key_management( access_key: &str, secret_key: &str, ) -> Result<(), Box> { + if skip_if_kms_admin_tool_unavailable("test_kms_key_management") { + return Ok(()); + } + info!("Testing KMS key management APIs"); // Test CreateKey @@ -324,8 +343,8 @@ pub async fn test_error_scenarios(s3_client: &Client, bucket: &str) -> Result<() let wrong_key = "98765432109876543210987654321098"; let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key); let wrong_key_b64 = base64::engine::general_purpose::STANDARD.encode(wrong_key); - let test_key_md5 = format!("{:x}", md5::compute(test_key)); - let wrong_key_md5 = format!("{:x}", md5::compute(wrong_key)); + let test_key_md5 = sse_customer_key_md5_base64(test_key); + let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key); let test_data = b"Test data for error scenarios"; let object_key = "test-error-object"; @@ -406,7 +425,7 @@ impl VaultTestEnvironment { let port_check = TcpStream::connect(VAULT_ADDRESS).await.is_ok(); if port_check { // Additional check by making a health request - if let Ok(response) = reqwest::get(&format!("{VAULT_URL}/v1/sys/health")).await + if let Ok(response) = local_http_client().get(format!("{VAULT_URL}/v1/sys/health")).send().await && response.status().is_success() { info!("Vault server is ready after {} seconds", i); @@ -426,7 +445,7 @@ impl VaultTestEnvironment { /// Setup Vault transit secrets engine pub async fn setup_vault_transit(&self) -> Result<(), Box> { - let client = reqwest::Client::new(); + let client = local_http_client(); info!("Enabling Vault transit secrets engine"); @@ -687,7 +706,7 @@ pub async fn test_multipart_upload_with_config( /// Create a standard SSE-C encryption configuration for testing pub fn create_sse_c_config() -> EncryptionType { let key = "01234567890123456789012345678901"; // 32-byte key - let key_md5 = format!("{:x}", md5::compute(key)); + let key_md5 = sse_customer_key_md5_base64(key); EncryptionType::SSEC { key: key.to_string(), key_md5, diff --git a/crates/e2e_test/src/kms/encryption_metadata_test.rs b/crates/e2e_test/src/kms/encryption_metadata_test.rs index 8f1555508..bfa669b6e 100644 --- a/crates/e2e_test/src/kms/encryption_metadata_test.rs +++ b/crates/e2e_test/src/kms/encryption_metadata_test.rs @@ -26,27 +26,20 @@ use serial_test::serial; use std::collections::{HashMap, VecDeque}; use tracing::info; -fn assert_encryption_metadata(metadata: &HashMap, expected_size: usize) { +fn assert_managed_encryption_metadata_hidden(metadata: Option<&HashMap>) { + let Some(metadata) = metadata else { return }; + for key in [ "x-rustfs-encryption-key", "x-rustfs-encryption-iv", "x-rustfs-encryption-context", "x-rustfs-encryption-original-size", ] { - assert!(metadata.contains_key(key), "expected managed encryption metadata '{key}' to be present"); assert!( - !metadata.get(key).unwrap().is_empty(), - "managed encryption metadata '{key}' should not be empty" + !metadata.contains_key(key), + "managed encryption metadata '{key}' should not be exposed to clients" ); } - - let size_value = metadata - .get("x-rustfs-encryption-original-size") - .expect("managed encryption metadata should include original size"); - let parsed_size: usize = size_value - .parse() - .expect("x-rustfs-encryption-original-size should be numeric"); - assert_eq!(parsed_size, expected_size, "recorded original size should match uploaded payload length"); } fn assert_storage_encrypted(storage_root: &std::path::Path, bucket: &str, key: &str, plaintext: &[u8]) { @@ -142,10 +135,7 @@ async fn test_head_reports_managed_metadata_for_sse_s3() -> Result<(), Box Result<(), &default_key_id, "source object should maintain the configured KMS key id" ); - let source_metadata = head_source - .metadata() - .expect("source object should include managed encryption metadata"); - assert_encryption_metadata(source_metadata, payload.len()); + assert_managed_encryption_metadata_hidden(head_source.metadata()); let dest_key = "metadata-sse-kms-object-copy"; let copy_source = format!("{TEST_BUCKET}/{source_key}"); @@ -238,10 +225,7 @@ async fn test_head_reports_managed_metadata_for_sse_kms_and_copy() -> Result<(), &default_key_id, "copied object should keep the default KMS key id" ); - let dest_metadata = head_dest - .metadata() - .expect("copied object should include managed encryption metadata"); - assert_encryption_metadata(dest_metadata, payload.len()); + assert_managed_encryption_metadata_hidden(head_dest.metadata()); let copied_body = s3_client .get_object() @@ -358,10 +342,7 @@ async fn test_multipart_upload_writes_encrypted_data() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box> { init_logging(); + if skip_if_kms_admin_tool_unavailable("test_local_kms_end_to_end") { + return Ok(()); + } info!("Starting Local KMS End-to-End Test"); // Create LocalKMS test environment @@ -140,8 +146,8 @@ async fn test_local_kms_key_isolation() { let key2 = "98765432109876543210987654321098"; let key1_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key1); let key2_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key2); - let key1_md5 = format!("{:x}", md5::compute(key1)); - let key2_md5 = format!("{:x}", md5::compute(key2)); + let key1_md5 = sse_customer_key_md5_base64(key1); + let key2_md5 = sse_customer_key_md5_base64(key2); let data1 = b"Data encrypted with key 1"; let data2 = b"Data encrypted with key 2"; @@ -562,7 +568,7 @@ async fn test_multipart_upload_with_sse_c( // SSE-C encryption key let encryption_key = "01234567890123456789012345678901"; let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, encryption_key); - let key_md5 = format!("{:x}", md5::compute(encryption_key)); + let key_md5 = sse_customer_key_md5_base64(encryption_key); // Generate test data let test_data: Vec = (0..total_size).map(|i| ((i * 3) % 256) as u8).collect(); diff --git a/crates/e2e_test/src/kms/kms_vault_test.rs b/crates/e2e_test/src/kms/kms_vault_test.rs index eb9b2a2f4..2377afb8e 100644 --- a/crates/e2e_test/src/kms/kms_vault_test.rs +++ b/crates/e2e_test/src/kms/kms_vault_test.rs @@ -19,14 +19,14 @@ //! multipart upload behaviour. use crate::common::{TEST_BUCKET, init_logging}; -use md5::compute; use serial_test::serial; use tokio::time::{Duration, sleep}; use tracing::{error, info}; use super::common::{ - VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, start_kms, test_all_multipart_encryption_types, test_error_scenarios, - test_kms_key_management, test_sse_c_encryption, test_sse_kms_encryption, test_sse_s3_encryption, + VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, skip_if_kms_admin_tool_unavailable, sse_customer_key_md5_base64, + start_kms, test_all_multipart_encryption_types, test_error_scenarios, test_kms_key_management, test_sse_c_encryption, + test_sse_kms_encryption, test_sse_s3_encryption, }; /// Helper that brings up Vault, configures RustFS, and starts the KMS service. @@ -65,6 +65,9 @@ impl VaultKmsTestContext { #[serial] async fn test_vault_kms_end_to_end() -> Result<(), Box> { init_logging(); + if skip_if_kms_admin_tool_unavailable("test_vault_kms_end_to_end") { + return Ok(()); + } info!("Starting Vault KMS End-to-End Test with default key {}", VAULT_KEY_NAME); let context = VaultKmsTestContext::new().await?; @@ -118,6 +121,9 @@ async fn test_vault_kms_end_to_end() -> Result<(), Box Result<(), Box> { init_logging(); + if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_isolation") { + return Ok(()); + } info!("Starting Vault KMS SSE-C key isolation test"); let context = VaultKmsTestContext::new().await?; @@ -133,8 +139,8 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box Result<(), Box Result<(), Box> { init_logging(); + if skip_if_kms_admin_tool_unavailable("test_vault_kms_large_file") { + return Ok(()); + } info!("Starting Vault KMS large file SSE-S3 test"); let context = VaultKmsTestContext::new().await?; @@ -264,6 +273,9 @@ async fn test_vault_kms_large_file() -> Result<(), Box Result<(), Box> { init_logging(); + if skip_if_kms_admin_tool_unavailable("test_vault_kms_multipart_upload") { + return Ok(()); + } info!("Starting Vault KMS multipart upload encryption suite"); let context = VaultKmsTestContext::new().await?; @@ -292,6 +304,9 @@ async fn test_vault_kms_multipart_upload() -> Result<(), Box Result<(), Box> { init_logging(); + if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_operations") { + return Ok(()); + } info!("Starting Vault KMS key operations test (CRUD)"); let context = VaultKmsTestContext::new().await?; diff --git a/crates/e2e_test/src/kms/multipart_encryption_test.rs b/crates/e2e_test/src/kms/multipart_encryption_test.rs index b744f48ab..22ab6c8d5 100644 --- a/crates/e2e_test/src/kms/multipart_encryption_test.rs +++ b/crates/e2e_test/src/kms/multipart_encryption_test.rs @@ -21,7 +21,7 @@ //! 3. Test the saving and reading of encrypted metadata //! 4. Test the complete sharded upload encryption process -use super::common::LocalKMSTestEnvironment; +use super::common::{LocalKMSTestEnvironment, sse_customer_key_md5_base64}; use crate::common::{TEST_BUCKET, init_logging}; use serial_test::serial; use tracing::{debug, info}; @@ -504,7 +504,7 @@ async fn test_multipart_encryption_type( let (sse_c_key, sse_c_md5) = if matches!(encryption_type, EncryptionType::SSEC) { let key = "01234567890123456789012345678901"; let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key); - let key_md5 = format!("{:x}", md5::compute(key)); + let key_md5 = sse_customer_key_md5_base64(key); (Some(key_b64), Some(key_md5)) } else { (None, None) diff --git a/crates/e2e_test/src/lib.rs b/crates/e2e_test/src/lib.rs index 16d403eac..b1266a8a3 100644 --- a/crates/e2e_test/src/lib.rs +++ b/crates/e2e_test/src/lib.rs @@ -88,3 +88,19 @@ mod checksum_upload_test; // Group deletion tests #[cfg(test)] mod group_delete_test; + +// S3 dummy-compat bucket API tests +#[cfg(test)] +mod bucket_logging_test; + +// Multipart control API auth regression tests +#[cfg(test)] +mod multipart_auth_test; + +// Object lambda end-to-end regression tests +#[cfg(test)] +mod object_lambda_test; + +// Replication extension end-to-end regression tests +#[cfg(test)] +mod replication_extension_test; diff --git a/crates/e2e_test/src/multipart_auth_test.rs b/crates/e2e_test/src/multipart_auth_test.rs new file mode 100644 index 000000000..0d40fd2a6 --- /dev/null +++ b/crates/e2e_test/src/multipart_auth_test.rs @@ -0,0 +1,6086 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Regression coverage for anonymous access on multipart control APIs. + +use crate::common::{RustFSTestEnvironment, init_logging, local_http_client}; +use async_compression::tokio::write::{BzEncoder, XzEncoder}; +use aws_sdk_s3::error::SdkError; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::{ + ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule, +}; +use base64::Engine; +use chrono::{Duration as ChronoDuration, Utc}; +use flate2::{Compression, write::GzEncoder}; +use http::HeaderValue; +use serial_test::serial; +use std::collections::HashMap; +use std::io::Cursor; +use std::io::Write; +use tokio::io::AsyncWriteExt; +use uuid::Uuid; + +fn encode_post_policy(conditions: Vec) -> String { + let expiration = (Utc::now() + ChronoDuration::hours(1)) + .format("%Y-%m-%dT%H:%M:%S.000Z") + .to_string(); + let policy = serde_json::json!({ + "expiration": expiration, + "conditions": conditions, + }); + + base64::engine::general_purpose::STANDARD.encode(policy.to_string()) +} + +fn sse_customer_key_md5_base64(key: &str) -> String { + base64::engine::general_purpose::STANDARD.encode(md5::compute(key).0) +} + +async fn make_tar(files: &[(&str, &[u8])], dirs: &[&str]) -> Vec { + let buf = Cursor::new(Vec::new()); + let mut builder = tokio_tar::Builder::new(buf); + + for &dir in dirs { + let mut header = tokio_tar::Header::new_gnu(); + header.set_entry_type(tokio_tar::EntryType::Directory); + header.set_size(0); + header.set_mode(0o755); + header.set_cksum(); + builder + .append_data(&mut header, dir, Cursor::new(&[] as &[u8])) + .await + .expect("directory entry should be appended"); + } + + for &(name, data) in files { + let mut header = tokio_tar::Header::new_gnu(); + header.set_size(data.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data(&mut header, name, Cursor::new(data)) + .await + .expect("file entry should be appended"); + } + + builder.into_inner().await.expect("tar builder should finalize").into_inner() +} + +fn build_pax_record(key: &str, value: &str) -> Vec { + let payload = format!("{key}={value}\n"); + let mut len = payload.len() + 3; + loop { + let record = format!("{len} {payload}"); + if record.len() == len { + return record.into_bytes(); + } + len = record.len(); + } +} + +async fn make_tar_with_pax_entry(path: &str, data: &[u8], mtime: Option, pax: &HashMap<&str, String>) -> Vec { + let buf = Cursor::new(Vec::new()); + let mut builder = tokio_tar::Builder::new(buf); + + if !pax.is_empty() { + let mut pax_payload = Vec::new(); + for (key, value) in pax { + pax_payload.extend(build_pax_record(key, value)); + } + + let mut pax_header = tokio_tar::Header::new_gnu(); + pax_header.set_entry_type(tokio_tar::EntryType::XHeader); + pax_header.set_size(pax_payload.len() as u64); + pax_header.set_mode(0o644); + pax_header.set_cksum(); + builder + .append_data(&mut pax_header, "PaxHeaders.X/entry", Cursor::new(pax_payload)) + .await + .expect("pax header entry should be appended"); + } + + let mut header = tokio_tar::Header::new_gnu(); + header.set_size(data.len() as u64); + header.set_mode(0o644); + if let Some(mtime) = mtime { + header.set_mtime(mtime); + } + header.set_cksum(); + builder + .append_data(&mut header, path, Cursor::new(data)) + .await + .expect("file entry should be appended"); + + builder.into_inner().await.expect("tar builder should finalize").into_inner() +} + +fn gzip_bytes(data: &[u8]) -> Vec { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(data).expect("gzip encoder should accept input"); + encoder.finish().expect("gzip encoder should finish") +} + +fn zstd_bytes(data: &[u8]) -> Vec { + let mut encoder = zstd::Encoder::new(Vec::new(), 0).expect("zstd encoder should initialize"); + encoder.write_all(data).expect("zstd encoder should accept input"); + encoder.finish().expect("zstd encoder should finish") +} + +async fn bzip2_bytes(data: &[u8]) -> Vec { + let cursor = Cursor::new(Vec::new()); + let mut encoder = BzEncoder::new(cursor); + encoder.write_all(data).await.expect("bzip2 encoder should accept input"); + encoder.shutdown().await.expect("bzip2 encoder should finish"); + encoder.into_inner().into_inner() +} + +async fn xz_bytes(data: &[u8]) -> Vec { + let cursor = Cursor::new(Vec::new()); + let mut encoder = XzEncoder::new(cursor); + encoder.write_all(data).await.expect("xz encoder should accept input"); + encoder.shutdown().await.expect("xz encoder should finish"); + encoder.into_inner().into_inner() +} + +fn assert_s3_error_code( + result: Result>, + code: &str, +) { + let err = result.expect_err("request should fail"); + match err { + SdkError::ServiceError(service_err) => { + let s3_err = service_err.into_err(); + assert_eq!(s3_err.meta().code(), Some(code), "unexpected S3 error: {s3_err:?}"); + } + other_err => panic!("Expected service error {code}, got: {other_err:?}"), + } +} + +async fn allow_anonymous_put_object( + client: &aws_sdk_s3::Client, + bucket: &str, +) -> Result<(), Box> { + let policy_json = serde_json::json!({ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowAnonymousPutObject", + "Effect": "Allow", + "Principal": "*", + "Action": ["s3:PutObject"], + "Resource": [format!("arn:aws:s3:::{}/*", bucket)] + } + ] + }) + .to_string(); + + client.put_bucket_policy().bucket(bucket).policy(policy_json).send().await?; + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_multipart_control_apis_require_auth() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-multipart-auth"; + let key = "multipart-target"; + let source_key = "copy-source"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + admin_client + .put_object() + .bucket(bucket) + .key(source_key) + .body(ByteStream::from_static(b"copy-source-data")) + .send() + .await?; + + let http = local_http_client(); + let base = format!("{}/{}/{}", env.url, bucket, key); + let upload_id = "dummy-upload-id"; + + let abort_resp = http.delete(format!("{base}?uploadId={upload_id}")).send().await?; + assert_eq!( + abort_resp.status(), + reqwest::StatusCode::FORBIDDEN, + "anonymous AbortMultipartUpload should be rejected" + ); + + let list_parts_resp = http.get(format!("{base}?uploadId={upload_id}")).send().await?; + assert_eq!( + list_parts_resp.status(), + reqwest::StatusCode::FORBIDDEN, + "anonymous ListParts should be rejected" + ); + + let complete_body = r#" + + + 1 + "dummy-etag" + +"#; + let complete_resp = http + .post(format!("{base}?uploadId={upload_id}")) + .header(reqwest::header::CONTENT_TYPE, "application/xml") + .body(complete_body) + .send() + .await?; + assert_eq!( + complete_resp.status(), + reqwest::StatusCode::FORBIDDEN, + "anonymous CompleteMultipartUpload should be rejected" + ); + + let copy_source = format!("/{bucket}/{source_key}"); + let upload_part_copy_resp = http + .put(format!("{base}?uploadId={upload_id}&partNumber=1")) + .header("x-amz-copy-source", copy_source) + .send() + .await?; + assert_eq!( + upload_part_copy_resp.status(), + reqwest::StatusCode::FORBIDDEN, + "anonymous UploadPartCopy should be rejected" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_requires_auth() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-auth"; + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + + let post_form = reqwest::multipart::Form::new().text("key", "post-object.txt").part( + "file", + reqwest::multipart::Part::bytes(b"post-object-body".to_vec()) + .file_name("post.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + assert_eq!( + post_resp.status(), + reqwest::StatusCode::FORBIDDEN, + "anonymous PostObject should be rejected" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_honors_success_action_status() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy"; + let object_key = "post-policy-object.txt"; + let expected_body = b"anonymous-post-body".to_vec(); + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("success_action_status", "201") + .part( + "file", + reqwest::multipart::Part::bytes(expected_body.clone()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!( + status, + reqwest::StatusCode::CREATED, + "PostObject should honor success_action_status=201 when upload is allowed" + ); + assert!( + response_body.contains(""), + "201 response should contain PostResponse XML, got: {response_body}" + ); + assert!( + response_body.contains(&format!("{bucket}")), + "201 response should include bucket in XML, got: {response_body}" + ); + assert!( + response_body.contains(&format!("{object_key}")), + "201 response should include object key in XML, got: {response_body}" + ); + assert!( + response_body.contains(""), + "201 response should include ETag in XML, got: {response_body}" + ); + + let get_out = admin_client.get_object().bucket(bucket).key(object_key).send().await?; + let uploaded = get_out.body.collect().await?.into_bytes(); + assert_eq!(uploaded.as_ref(), expected_body.as_slice(), "uploaded object body should match"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_honors_success_action_redirect() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-redirect"; + let object_key = "post-redirect-object.txt"; + let expected_body = b"anonymous-post-redirect-body".to_vec(); + let redirect_target = "https://example.com/upload/callback?origin=test"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("success_action_redirect", redirect_target.to_string()) + .part( + "file", + reqwest::multipart::Part::bytes(expected_body.clone()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let http = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .build()?; + + let post_resp = http + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + assert_eq!( + post_resp.status(), + reqwest::StatusCode::SEE_OTHER, + "PostObject should return redirect status when success_action_redirect is set" + ); + + let location = post_resp + .headers() + .get(reqwest::header::LOCATION) + .and_then(|v| v.to_str().ok()) + .ok_or("missing redirect location header")?; + assert!( + location.starts_with(redirect_target), + "redirect location should start with requested target, got: {location}" + ); + assert!( + location.contains("bucket="), + "redirect location should include bucket query parameter, got: {location}" + ); + assert!( + location.contains("key="), + "redirect location should include key query parameter, got: {location}" + ); + assert!( + location.to_ascii_lowercase().contains("etag="), + "redirect location should include etag query parameter, got: {location}" + ); + + let get_out = admin_client.get_object().bucket(bucket).key(object_key).send().await?; + let uploaded = get_out.body.collect().await?.into_bytes(); + assert_eq!(uploaded.as_ref(), expected_body.as_slice(), "uploaded object body should match"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_defaults_to_no_content() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-default-status"; + let object_key = "post-default-object.txt"; + let expected_body = b"anonymous-post-default-body".to_vec(); + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let post_form = reqwest::multipart::Form::new().text("key", object_key.to_string()).part( + "file", + reqwest::multipart::Part::bytes(expected_body.clone()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!( + status, + reqwest::StatusCode::NO_CONTENT, + "PostObject should default to 204 when no success_action_status is provided" + ); + assert!(response_body.is_empty(), "204 response should not contain a body, got: {response_body}"); + + let get_out = admin_client.get_object().bucket(bucket).key(object_key).send().await?; + let uploaded = get_out.body.collect().await?.into_bytes(); + assert_eq!(uploaded.as_ref(), expected_body.as_slice(), "uploaded object body should match"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_sse_kms() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-sse-kms"; + let object_key = "post-sse-kms-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("x-amz-server-side-encryption", "aws:kms") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-sse-kms-body".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!( + status, + reqwest::StatusCode::NOT_IMPLEMENTED, + "PostObject should reject SSE-KMS form uploads" + ); + assert!( + response_body.contains("NotImplemented"), + "response should contain NotImplemented code, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_sse_kms_with_key_id_outside_policy_conditions() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-sse-kms-keyid"; + let object_key = "post-sse-kms-keyid-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-server-side-encryption": "aws:kms" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-server-side-encryption", "aws:kms") + .text("x-amz-server-side-encryption-aws-kms-key-id", "test-key") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-sse-kms-body".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!( + status, + reqwest::StatusCode::NOT_IMPLEMENTED, + "SSE-KMS key id should not fail policy validation before runtime rejection" + ); + assert!( + response_body.contains("NotImplemented"), + "response should contain NotImplemented code, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_sse_kms_with_context_outside_policy_conditions() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-sse-kms-context"; + let object_key = "post-sse-kms-context-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-server-side-encryption": "aws:kms" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-server-side-encryption", "aws:kms") + .text("x-amz-server-side-encryption-context", "e30=") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-sse-kms-context-body".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!( + status, + reqwest::StatusCode::NOT_IMPLEMENTED, + "SSE-KMS context should not fail policy validation before runtime rejection" + ); + assert!( + response_body.contains("NotImplemented"), + "response should contain NotImplemented code, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_sse_kms_key_id_policy_mismatch() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-sse-kms-keyid-mismatch"; + let object_key = "post-sse-kms-keyid-mismatch-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-server-side-encryption": "aws:kms" }), + serde_json::json!({ "x-amz-server-side-encryption-aws-kms-key-id": "expected-key" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-server-side-encryption", "aws:kms") + .text("x-amz-server-side-encryption-aws-kms-key-id", "other-key") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-sse-kms-keyid-mismatch-body".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!( + response_body.contains("InvalidPolicyDocument"), + "response should contain InvalidPolicyDocument code, got: {response_body}" + ); + assert!( + response_body_lower.contains("aws-kms-key-id"), + "response should mention the conflicting kms key id field, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_sse_kms_context_policy_mismatch() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-sse-kms-context-mismatch"; + let object_key = "post-sse-kms-context-mismatch-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-server-side-encryption": "aws:kms" }), + serde_json::json!({ "x-amz-server-side-encryption-context": "e30=" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-server-side-encryption", "aws:kms") + .text("x-amz-server-side-encryption-context", "eyJrIjoiYiJ9") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-sse-kms-context-mismatch-body".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!( + response_body.contains("InvalidPolicyDocument"), + "response should contain InvalidPolicyDocument code, got: {response_body}" + ); + assert!( + response_body_lower.contains("server-side-encryption-context"), + "response should mention the conflicting kms context field, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_sse_kms_with_bucket_key_enabled_outside_policy_conditions() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-sse-kms-bucket-key"; + let object_key = "post-sse-kms-bucket-key-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-server-side-encryption": "aws:kms" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-server-side-encryption", "aws:kms") + .text("x-amz-server-side-encryption-bucket-key-enabled", "true") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-sse-kms-bucket-key-body".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!( + status, + reqwest::StatusCode::NOT_IMPLEMENTED, + "SSE-KMS bucket-key-enabled should not fail policy validation before runtime rejection" + ); + assert!( + response_body.contains("NotImplemented"), + "response should contain NotImplemented code, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_sse_kms_bucket_key_enabled_policy_mismatch() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-sse-kms-bucket-key-mismatch"; + let object_key = "post-sse-kms-bucket-key-mismatch-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-server-side-encryption": "aws:kms" }), + serde_json::json!({ "x-amz-server-side-encryption-bucket-key-enabled": "false" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-server-side-encryption", "aws:kms") + .text("x-amz-server-side-encryption-bucket-key-enabled", "true") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-sse-kms-bucket-key-mismatch-body".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!( + response_body.contains("InvalidPolicyDocument"), + "response should contain InvalidPolicyDocument code, got: {response_body}" + ); + assert!( + response_body_lower.contains("bucket-key-enabled"), + "response should mention the conflicting bucket-key-enabled field, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_accepts_sse_s3() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-sse-s3"; + let object_key = "post-sse-s3-object.txt"; + let expected_body = b"post-sse-s3-body".to_vec(); + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-server-side-encryption": "AES256" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-server-side-encryption", "AES256") + .part( + "file", + reqwest::multipart::Part::bytes(expected_body.clone()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::NO_CONTENT); + assert!(response_body.is_empty(), "204 response should not contain a body, got: {response_body}"); + + let head = admin_client.head_object().bucket(bucket).key(object_key).send().await?; + assert_eq!(head.server_side_encryption().map(|value| value.as_str()), Some("AES256")); + + let uploaded = admin_client.get_object().bucket(bucket).key(object_key).send().await?; + let uploaded = uploaded.body.collect().await?.into_bytes(); + assert_eq!(uploaded.as_ref(), expected_body.as_slice()); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_uses_bucket_default_sse_s3() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-default-sse-s3"; + let object_key = "post-default-sse-s3-object.txt"; + let expected_body = b"post-default-sse-s3-body".to_vec(); + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let encryption_config = ServerSideEncryptionConfiguration::builder() + .rules( + ServerSideEncryptionRule::builder() + .apply_server_side_encryption_by_default( + ServerSideEncryptionByDefault::builder() + .sse_algorithm(ServerSideEncryption::Aes256) + .build() + .expect("default encryption rule should build"), + ) + .build(), + ) + .build() + .expect("bucket encryption config should build"); + + admin_client + .put_bucket_encryption() + .bucket(bucket) + .server_side_encryption_configuration(encryption_config) + .send() + .await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .part( + "file", + reqwest::multipart::Part::bytes(expected_body.clone()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + assert_eq!(post_resp.status(), reqwest::StatusCode::NO_CONTENT); + + let head = admin_client.head_object().bucket(bucket).key(object_key).send().await?; + assert_eq!(head.server_side_encryption().map(|value| value.as_str()), Some("AES256")); + + let uploaded = admin_client.get_object().bucket(bucket).key(object_key).send().await?; + let uploaded = uploaded.body.collect().await?.into_bytes(); + assert_eq!(uploaded.as_ref(), expected_body.as_slice()); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_uses_bucket_default_sse_kms() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-default-sse-kms"; + let object_key = "post-default-sse-kms-object.txt"; + let expected_body = b"post-default-sse-kms-body".to_vec(); + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let encryption_config = ServerSideEncryptionConfiguration::builder() + .rules( + ServerSideEncryptionRule::builder() + .apply_server_side_encryption_by_default( + ServerSideEncryptionByDefault::builder() + .sse_algorithm(ServerSideEncryption::AwsKms) + .kms_master_key_id("test-key") + .build() + .expect("default encryption rule should build"), + ) + .build(), + ) + .build() + .expect("bucket encryption config should build"); + + admin_client + .put_bucket_encryption() + .bucket(bucket) + .server_side_encryption_configuration(encryption_config) + .send() + .await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .part( + "file", + reqwest::multipart::Part::bytes(expected_body.clone()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + assert_eq!(post_resp.status(), reqwest::StatusCode::NO_CONTENT); + + let head = admin_client.head_object().bucket(bucket).key(object_key).send().await?; + assert_eq!(head.server_side_encryption().map(|value| value.as_str()), Some("aws:kms")); + + let uploaded = admin_client.get_object().bucket(bucket).key(object_key).send().await?; + let uploaded = uploaded.body.collect().await?.into_bytes(); + assert_eq!(uploaded.as_ref(), expected_body.as_slice()); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_sse_s3_policy_mismatch() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-sse-s3-reject"; + let object_key = "post-sse-s3-reject-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-server-side-encryption": "AES256" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-server-side-encryption", "aws:kms") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-sse-s3-mismatch".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!( + response_body.contains("InvalidPolicyDocument"), + "response should contain InvalidPolicyDocument code, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_sse_s3_missing_from_policy_conditions() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-sse-s3-missing"; + let object_key = "post-sse-s3-missing-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-server-side-encryption", "AES256") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-sse-s3-missing".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::FORBIDDEN); + assert!( + response_body.contains("AccessDenied"), + "response should contain AccessDenied code, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_accepts_storage_class_exact_policy_match() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-storage-class"; + let object_key = "post-storage-class-object.txt"; + let expected_body = b"post-storage-class-body".to_vec(); + let storage_class = "STANDARD_IA"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-storage-class": storage_class }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-storage-class", storage_class) + .part( + "file", + reqwest::multipart::Part::bytes(expected_body.clone()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + assert_eq!(post_resp.status(), reqwest::StatusCode::NO_CONTENT); + + let head = admin_client.head_object().bucket(bucket).key(object_key).send().await?; + assert_eq!(head.storage_class().map(|value| value.as_str()), Some(storage_class)); + + let uploaded = admin_client.get_object().bucket(bucket).key(object_key).send().await?; + let uploaded = uploaded.body.collect().await?.into_bytes(); + assert_eq!(uploaded.as_ref(), expected_body.as_slice()); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_storage_class_missing_from_policy_conditions() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-storage-class-missing"; + let object_key = "post-storage-class-missing-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-storage-class", "STANDARD_IA") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-storage-class-missing".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::FORBIDDEN); + assert!( + response_body.contains("AccessDenied"), + "response should contain AccessDenied code, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_storage_class_policy_mismatch() -> Result<(), Box> +{ + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-storage-class-mismatch"; + let object_key = "post-storage-class-mismatch-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-storage-class": "STANDARD_IA" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-storage-class", "ONEZONE_IA") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-storage-class-mismatch".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!( + response_body.contains("InvalidPolicyDocument"), + "response should contain InvalidPolicyDocument code, got: {response_body}" + ); + assert!( + response_body_lower.contains("storage-class"), + "response should mention storage class mismatch, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_invalid_storage_class_value() -> Result<(), Box> +{ + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-storage-class-invalid"; + let object_key = "post-storage-class-invalid-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-storage-class": "INVALID" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-storage-class", "INVALID") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-storage-class-invalid".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!( + response_body.contains("InvalidStorageClass"), + "response should contain InvalidStorageClass code, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_checksum_algorithm_missing_from_policy_conditions() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-checksum-missing"; + let object_key = "post-checksum-missing-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key) + .text("policy", policy) + .text("x-amz-checksum-algorithm", "SHA256") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-checksum-missing".to_vec()) + .file_name("checksum.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::FORBIDDEN); + assert!( + response_body.contains("AccessDenied"), + "response should contain AccessDenied code, got: {response_body}" + ); + assert!( + response_body_lower.contains("x-amz-checksum-algorithm"), + "response should mention x-amz-checksum-algorithm, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_checksum_algorithm_policy_mismatch() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-checksum-mismatch"; + let object_key = "post-checksum-mismatch-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-checksum-algorithm": "SHA256" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key) + .text("policy", policy) + .text("x-amz-checksum-algorithm", "CRC32") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-checksum-mismatch".to_vec()) + .file_name("checksum.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!( + response_body.contains("InvalidPolicyDocument"), + "response should contain InvalidPolicyDocument code, got: {response_body}" + ); + assert!( + response_body_lower.contains("x-amz-checksum-algorithm"), + "response should mention x-amz-checksum-algorithm mismatch, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_checksum_auxiliary_fields_missing_from_policy_conditions() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let admin_client = env.create_s3_client(); + + for (bucket, field_name, field_value) in [ + ("anon-post-checksum-crc32-missing", "x-amz-checksum-crc32", "AAAAAA=="), + ("anon-post-checksum-crc32c-missing", "x-amz-checksum-crc32c", "AAAAAA=="), + ("anon-post-checksum-sha1-missing", "x-amz-checksum-sha1", "ZmFrZXNoYTE="), + ("anon-post-checksum-sha256-missing", "x-amz-checksum-sha256", "ZmFrZXNoYTI1Ng=="), + ("anon-post-checksum-mode-missing", "x-amz-checksum-mode", "ENABLED"), + ] { + let object_key = format!("uploads/{field_name}.txt"); + + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.clone()) + .text("policy", policy) + .text(field_name, field_value) + .part( + "file", + reqwest::multipart::Part::bytes(format!("post-{field_name}").into_bytes()) + .file_name("checksum.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "unexpected status for {field_name}"); + assert!( + response_body.contains("AccessDenied"), + "response should contain AccessDenied for {field_name}, got: {response_body}" + ); + assert!( + response_body_lower.contains(field_name), + "response should mention {field_name}, got: {response_body}" + ); + } + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_allows_sse_c_fields_outside_policy_conditions() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-sse-c-ignore"; + let object_key = "sse-c-object.txt"; + let expected_body = b"anonymous-post-sse-c".to_vec(); + let customer_key = "01234567890123456789012345678901"; + let customer_key_b64 = base64::engine::general_purpose::STANDARD.encode(customer_key); + let customer_key_md5 = sse_customer_key_md5_base64(customer_key); + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key) + .text("policy", policy) + .text("x-amz-server-side-encryption-customer-algorithm", "AES256") + .text("x-amz-server-side-encryption-customer-key", customer_key_b64.clone()) + .text("x-amz-server-side-encryption-customer-key-md5", customer_key_md5.clone()) + .part( + "file", + reqwest::multipart::Part::bytes(expected_body.clone()) + .file_name("sse-c.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + assert_eq!( + post_resp.status(), + reqwest::StatusCode::NO_CONTENT, + "SSE-C form fields should be accepted outside policy conditions" + ); + + let head_resp = admin_client + .head_object() + .bucket(bucket) + .key(object_key) + .sse_customer_algorithm("AES256") + .sse_customer_key(customer_key_b64) + .sse_customer_key_md5(customer_key_md5.clone()) + .send() + .await?; + assert_eq!(head_resp.sse_customer_algorithm(), Some("AES256")); + + let get_resp = admin_client + .get_object() + .bucket(bucket) + .key(object_key) + .sse_customer_algorithm("AES256") + .sse_customer_key(base64::engine::general_purpose::STANDARD.encode(customer_key)) + .sse_customer_key_md5(customer_key_md5) + .send() + .await?; + let actual_body = get_resp.body.collect().await?.into_bytes().to_vec(); + assert_eq!(actual_body, expected_body); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_sse_c_exact_policy_mismatch() -> Result<(), Box> +{ + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-sse-c-mismatch"; + let object_key = "sse-c-mismatch-object.txt"; + let policy_key = "01234567890123456789012345678901"; + let request_key = "abcdefghijklmnopqrstuvwxyzABCDEF"; + let policy_key_b64 = base64::engine::general_purpose::STANDARD.encode(policy_key); + let request_key_b64 = base64::engine::general_purpose::STANDARD.encode(request_key); + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-server-side-encryption-customer-algorithm": "AES256" }), + serde_json::json!({ "x-amz-server-side-encryption-customer-key": policy_key_b64 }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key) + .text("policy", policy) + .text("x-amz-server-side-encryption-customer-algorithm", "AES256") + .text("x-amz-server-side-encryption-customer-key", request_key_b64) + .text("x-amz-server-side-encryption-customer-key-md5", sse_customer_key_md5_base64(request_key)) + .part( + "file", + reqwest::multipart::Part::bytes(b"sse-c-policy-mismatch".to_vec()) + .file_name("sse-c.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!( + response_body.contains("InvalidPolicyDocument"), + "response should contain InvalidPolicyDocument code, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_duplicate_key_form_values() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-duplicate-key"; + let object_key = "duplicate-key-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("key", "other-object.txt") + .text("policy", policy) + .part( + "file", + reqwest::multipart::Part::bytes(b"duplicate-key".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!( + response_body.contains("InvalidPolicyDocument"), + "response should contain InvalidPolicyDocument code, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_invalid_success_action_status() -> Result<(), Box> +{ + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-invalid-status"; + let object_key = "post-invalid-status-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("success_action_status", "202") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-invalid-status-body".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!( + status, + reqwest::StatusCode::BAD_REQUEST, + "PostObject should reject unsupported success_action_status values" + ); + assert!( + response_body.contains("MalformedPOSTRequest"), + "response should contain MalformedPOSTRequest code, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_invalid_success_action_redirect() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-invalid-redirect"; + let object_key = "post-invalid-redirect-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("success_action_redirect", "://invalid-url") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-invalid-redirect-body".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!( + status, + reqwest::StatusCode::BAD_REQUEST, + "PostObject should reject malformed success_action_redirect values" + ); + assert!( + response_body.contains("MalformedPOSTRequest"), + "response should contain MalformedPOSTRequest code, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_form_fields_missing_from_policy_conditions() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-fields"; + let object_key = "post-policy-field-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("success_action_status", "201") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-body".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::FORBIDDEN); + assert!( + response_body.contains("AccessDenied"), + "response should contain AccessDenied code, got: {response_body}" + ); + assert!( + response_body.contains("success_action_status"), + "response should mention the missing field, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_accepts_form_fields_covered_by_policy_conditions() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-covered"; + let object_key = "post-policy-covered-object.txt"; + let expected_body = b"post-policy-covered-body".to_vec(); + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["starts-with", "$success_action_status", ""]), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("success_action_status", "201") + .part( + "file", + reqwest::multipart::Part::bytes(expected_body.clone()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::CREATED); + assert!( + response_body.contains(""), + "201 response should contain PostResponse XML, got: {response_body}" + ); + + let get_out = admin_client.get_object().bucket(bucket).key(object_key).send().await?; + let uploaded = get_out.body.collect().await?.into_bytes(); + assert_eq!(uploaded.as_ref(), expected_body.as_slice()); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_starts_with_policy_mismatch() -> Result<(), Box> +{ + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-starts-with"; + let object_key = "unexpected/upload.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!(["starts-with", "$key", "uploads/"]), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-body".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!( + response_body.contains("InvalidPolicyDocument"), + "response should contain InvalidPolicyDocument code, got: {response_body}" + ); + assert!( + response_body_lower.contains("starts-with"), + "response should mention the starts-with condition, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_content_length_range_violation() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-content-length"; + let object_key = "uploads/content-length-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 5]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .part( + "file", + reqwest::multipart::Part::bytes(b"payload-too-large".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!( + response_body.contains("EntityTooLarge"), + "response should contain EntityTooLarge code, got: {response_body}" + ); + assert!( + response_body.contains("maximum allowed object size"), + "response should mention the size limit, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_success_action_status_policy_mismatch() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-status-mismatch"; + let object_key = "uploads/status-mismatch-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "success_action_status": "201" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("success_action_status", "204") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-body".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!( + response_body.contains("InvalidPolicyDocument"), + "response should contain InvalidPolicyDocument code, got: {response_body}" + ); + assert!( + response_body_lower.contains("success_action_status"), + "response should mention the conflicting status field, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_accepts_success_action_status_exact_policy_match() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-status-accept"; + let object_key = "uploads/success-action-status-accept.txt"; + let expected_body = b"post-policy-success-action-status-accept".to_vec(); + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "success_action_status": "201" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("success_action_status", "201") + .part( + "file", + reqwest::multipart::Part::bytes(expected_body.clone()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::CREATED); + assert!( + response_body.contains(""), + "201 response should contain PostResponse XML, got: {response_body}" + ); + + let get_out = admin_client.get_object().bucket(bucket).key(object_key).send().await?; + let uploaded = get_out.body.collect().await?.into_bytes(); + assert_eq!(uploaded.as_ref(), expected_body.as_slice()); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_success_action_redirect_policy_mismatch() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-redirect-mismatch"; + let object_key = "uploads/redirect-mismatch-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "success_action_redirect": "https://example.com/success" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("success_action_redirect", "https://example.com/other") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-body".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!( + response_body.contains("InvalidPolicyDocument"), + "response should contain InvalidPolicyDocument code, got: {response_body}" + ); + assert!( + response_body_lower.contains("success_action_redirect"), + "response should mention the conflicting redirect field, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_accepts_success_action_redirect_exact_policy_match() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-redirect-accept"; + let object_key = "uploads/success-action-redirect-accept.txt"; + let expected_body = b"post-policy-success-action-redirect-accept".to_vec(); + let redirect_target = "https://example.com/upload/success"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "success_action_redirect": redirect_target }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("success_action_redirect", redirect_target.to_string()) + .part( + "file", + reqwest::multipart::Part::bytes(expected_body.clone()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let http = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .build()?; + + let post_resp = http + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + assert_eq!(post_resp.status(), reqwest::StatusCode::SEE_OTHER); + + let location = post_resp + .headers() + .get(reqwest::header::LOCATION) + .and_then(|v| v.to_str().ok()) + .ok_or("missing redirect location header")?; + assert!( + location.starts_with(redirect_target), + "redirect location should start with requested target, got: {location}" + ); + + let get_out = admin_client.get_object().bucket(bucket).key(object_key).send().await?; + let uploaded = get_out.body.collect().await?.into_bytes(); + assert_eq!(uploaded.as_ref(), expected_body.as_slice()); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_success_action_redirect_missing_from_policy_conditions() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-success-redirect-missing"; + let object_key = "uploads/success-redirect-missing.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("success_action_redirect", "https://example.com/success") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-success-redirect-missing".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::FORBIDDEN); + assert!(response_body.contains("AccessDenied")); + assert!( + response_body_lower.contains("success_action_redirect"), + "response should mention success_action_redirect, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_accepts_metadata_field_covered_by_starts_with() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-meta-accept"; + let object_key = "uploads/meta-object.txt"; + let metadata_value = "alpha-demo"; + let expected_body = b"post-policy-meta-body".to_vec(); + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["starts-with", "$x-amz-meta-project", "alpha-"]), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-meta-project", metadata_value) + .part( + "file", + reqwest::multipart::Part::bytes(expected_body.clone()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::NO_CONTENT); + assert!(response_body.is_empty(), "204 response should not contain a body, got: {response_body}"); + + let head = admin_client.head_object().bucket(bucket).key(object_key).send().await?; + let metadata = head.metadata().expect("head_object should expose uploaded metadata"); + assert_eq!(metadata.get("project").map(String::as_str), Some(metadata_value)); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_accepts_content_type_field_exact_policy_match() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-content-type-accept"; + let object_key = "uploads/content-type-accept.txt"; + let content_type = "text/plain"; + let expected_body = b"post-policy-content-type-accept".to_vec(); + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "Content-Type": content_type }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("Content-Type", content_type) + .part( + "file", + reqwest::multipart::Part::bytes(expected_body.clone()) + .file_name("upload.txt") + .mime_str(content_type)?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::NO_CONTENT); + assert!(response_body.is_empty(), "204 response should not contain a body, got: {response_body}"); + + let head = admin_client.head_object().bucket(bucket).key(object_key).send().await?; + assert_eq!(head.content_type(), Some(content_type)); + + let get_out = admin_client.get_object().bucket(bucket).key(object_key).send().await?; + let uploaded = get_out.body.collect().await?.into_bytes(); + assert_eq!(uploaded.as_ref(), expected_body.as_slice()); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_accepts_content_type_field_covered_by_starts_with() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-content-type-accept"; + let object_key = "uploads/content-type-object.txt"; + let content_type = "image/png"; + let expected_body = b"post-policy-content-type-body".to_vec(); + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["starts-with", "$Content-Type", "image/"]), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("Content-Type", content_type) + .part( + "file", + reqwest::multipart::Part::bytes(expected_body.clone()) + .file_name("upload.txt") + .mime_str(content_type)?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::NO_CONTENT); + assert!(response_body.is_empty(), "204 response should not contain a body, got: {response_body}"); + + let head = admin_client.head_object().bucket(bucket).key(object_key).send().await?; + assert_eq!(head.content_type(), Some(content_type)); + + let get_out = admin_client.get_object().bucket(bucket).key(object_key).send().await?; + let uploaded = get_out.body.collect().await?.into_bytes(); + assert_eq!(uploaded.as_ref(), expected_body.as_slice()); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_accepts_content_disposition_field_exact_policy_match() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-disposition-accept"; + let object_key = "uploads/disposition-object.txt"; + let content_disposition = "attachment; filename=\"upload.txt\""; + let expected_body = b"post-policy-disposition-body".to_vec(); + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "Content-Disposition": content_disposition }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("Content-Disposition", content_disposition) + .part( + "file", + reqwest::multipart::Part::bytes(expected_body.clone()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::NO_CONTENT); + assert!(response_body.is_empty(), "204 response should not contain a body, got: {response_body}"); + + let head = admin_client.head_object().bucket(bucket).key(object_key).send().await?; + assert_eq!(head.content_disposition(), Some(content_disposition)); + + let get_out = admin_client.get_object().bucket(bucket).key(object_key).send().await?; + let uploaded = get_out.body.collect().await?.into_bytes(); + assert_eq!(uploaded.as_ref(), expected_body.as_slice()); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_content_disposition_policy_mismatch() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-content-disposition-reject"; + let object_key = "uploads/content-disposition-reject.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "Content-Disposition": "attachment; filename=\"payload.bin\"" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("Content-Disposition", "inline") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-content-disposition-mismatch".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!(response_body.contains("InvalidPolicyDocument")); + assert!( + response_body_lower.contains("content-disposition"), + "response should mention content-disposition mismatch, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_accepts_cache_control_field_exact_policy_match() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-cache-control-accept"; + let object_key = "uploads/cache-control-object.txt"; + let cache_control = "max-age=60"; + let expected_body = b"post-policy-cache-control-body".to_vec(); + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "Cache-Control": cache_control }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("Cache-Control", cache_control) + .part( + "file", + reqwest::multipart::Part::bytes(expected_body.clone()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::NO_CONTENT); + assert!(response_body.is_empty(), "204 response should not contain a body, got: {response_body}"); + + let head = admin_client.head_object().bucket(bucket).key(object_key).send().await?; + assert_eq!(head.cache_control(), Some(cache_control)); + + let get_out = admin_client.get_object().bucket(bucket).key(object_key).send().await?; + let uploaded = get_out.body.collect().await?.into_bytes(); + assert_eq!(uploaded.as_ref(), expected_body.as_slice()); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_cache_control_policy_mismatch() -> Result<(), Box> +{ + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-cache-control-reject"; + let object_key = "uploads/cache-control-reject.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "Cache-Control": "max-age=60" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("Cache-Control", "max-age=120") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-cache-control-mismatch".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!(response_body.contains("InvalidPolicyDocument")); + assert!( + response_body_lower.contains("cache-control"), + "response should mention cache-control mismatch, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_cache_control_missing_from_policy_conditions() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-cache-control-missing"; + let object_key = "uploads/cache-control-missing.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("Cache-Control", "max-age=60") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-cache-control-missing".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::FORBIDDEN); + assert!(response_body.contains("AccessDenied")); + assert!( + response_body_lower.contains("cache-control"), + "response should mention cache-control, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_accepts_content_language_field_exact_policy_match() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-content-language-accept"; + let object_key = "uploads/content-language-object.txt"; + let content_language = "en-US"; + let expected_body = b"post-policy-content-language-body".to_vec(); + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "Content-Language": content_language }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("Content-Language", content_language) + .part( + "file", + reqwest::multipart::Part::bytes(expected_body.clone()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::NO_CONTENT); + assert!(response_body.is_empty(), "204 response should not contain a body, got: {response_body}"); + + let head = admin_client.head_object().bucket(bucket).key(object_key).send().await?; + assert_eq!(head.content_language(), Some(content_language)); + + let get_out = admin_client.get_object().bucket(bucket).key(object_key).send().await?; + let uploaded = get_out.body.collect().await?.into_bytes(); + assert_eq!(uploaded.as_ref(), expected_body.as_slice()); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_content_language_policy_mismatch() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-content-language-reject"; + let object_key = "uploads/content-language-reject.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "Content-Language": "en-US" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("Content-Language", "fr-FR") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-content-language-mismatch".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!(response_body.contains("InvalidPolicyDocument")); + assert!( + response_body_lower.contains("content-language"), + "response should mention content-language mismatch, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_content_language_missing_from_policy_conditions() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-content-language-missing"; + let object_key = "uploads/content-language-missing.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("Content-Language", "en-US") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-content-language-missing".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::FORBIDDEN); + assert!(response_body.contains("AccessDenied")); + assert!( + response_body_lower.contains("content-language"), + "response should mention content-language, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_accepts_content_encoding_field_exact_policy_match() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-content-encoding-accept"; + let object_key = "uploads/content-encoding-object.txt"; + let content_encoding = "gzip"; + let expected_body = b"post-policy-content-encoding-body".to_vec(); + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "Content-Encoding": content_encoding }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("Content-Encoding", content_encoding) + .part( + "file", + reqwest::multipart::Part::bytes(expected_body.clone()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::NO_CONTENT); + assert!(response_body.is_empty(), "204 response should not contain a body, got: {response_body}"); + + let head = admin_client.head_object().bucket(bucket).key(object_key).send().await?; + assert_eq!(head.content_encoding(), Some(content_encoding)); + + let get_out = admin_client.get_object().bucket(bucket).key(object_key).send().await?; + let uploaded = get_out.body.collect().await?.into_bytes(); + assert_eq!(uploaded.as_ref(), expected_body.as_slice()); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_content_encoding_policy_mismatch() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-content-encoding-reject"; + let object_key = "uploads/content-encoding-reject.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "Content-Encoding": "gzip" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("Content-Encoding", "br") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-content-encoding-mismatch".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!(response_body.contains("InvalidPolicyDocument")); + assert!( + response_body_lower.contains("content-encoding"), + "response should mention content-encoding mismatch, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_content_encoding_missing_from_policy_conditions() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-content-encoding-missing"; + let object_key = "uploads/content-encoding-missing.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("Content-Encoding", "gzip") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-content-encoding-missing".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::FORBIDDEN); + assert!(response_body.contains("AccessDenied")); + assert!( + response_body_lower.contains("content-encoding"), + "response should mention content-encoding, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_accepts_website_redirect_location_exact_policy_match() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-website-redirect-accept"; + let object_key = "uploads/website-redirect-object.txt"; + let website_redirect_location = "/docs/landing.html"; + let expected_body = b"post-policy-website-redirect-body".to_vec(); + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-website-redirect-location": website_redirect_location }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-website-redirect-location", website_redirect_location) + .part( + "file", + reqwest::multipart::Part::bytes(expected_body.clone()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::NO_CONTENT); + assert!(response_body.is_empty(), "204 response should not contain a body, got: {response_body}"); + + let head = admin_client.head_object().bucket(bucket).key(object_key).send().await?; + assert_eq!(head.website_redirect_location(), Some(website_redirect_location)); + + let get_out = admin_client.get_object().bucket(bucket).key(object_key).send().await?; + let uploaded = get_out.body.collect().await?.into_bytes(); + assert_eq!(uploaded.as_ref(), expected_body.as_slice()); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_website_redirect_location_missing_from_policy_conditions() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-website-redirect-missing"; + let object_key = "uploads/website-redirect-missing.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-website-redirect-location", "/docs/landing.html") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-website-redirect-missing".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::FORBIDDEN); + assert!(response_body.contains("AccessDenied")); + assert!( + response_body_lower.contains("x-amz-website-redirect-location"), + "response should mention x-amz-website-redirect-location, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_website_redirect_location_policy_mismatch() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-website-redirect-reject"; + let object_key = "uploads/website-redirect-reject-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-website-redirect-location": "/docs/landing.html" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-website-redirect-location", "/docs/other.html") + .part( + "file", + reqwest::multipart::Part::bytes(b"website-redirect-mismatch".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!(response_body.contains("InvalidPolicyDocument")); + assert!( + response_body_lower.contains("x-amz-website-redirect-location"), + "response should mention x-amz-website-redirect-location mismatch, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_accepts_expires_field_exact_policy_match() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-expires-accept"; + let object_key = "uploads/expires-object.txt"; + let expires = "Wed, 21 Oct 2037 07:28:00 GMT"; + let expected_body = b"post-policy-expires-body".to_vec(); + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "Expires": expires }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("Expires", expires) + .part( + "file", + reqwest::multipart::Part::bytes(expected_body.clone()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::NO_CONTENT); + assert!(response_body.is_empty(), "204 response should not contain a body, got: {response_body}"); + + let head = admin_client.head_object().bucket(bucket).key(object_key).send().await?; + assert_eq!(head.expires_string(), Some(expires)); + + let get_out = admin_client.get_object().bucket(bucket).key(object_key).send().await?; + let uploaded = get_out.body.collect().await?.into_bytes(); + assert_eq!(uploaded.as_ref(), expected_body.as_slice()); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_expires_field_policy_mismatch() -> Result<(), Box> +{ + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-expires-reject"; + let object_key = "uploads/expires-reject-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "Expires": "Wed, 21 Oct 2037 07:28:00 GMT" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("Expires", "Wed, 21 Oct 2037 08:28:00 GMT") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-expires-mismatch".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!(response_body.contains("InvalidPolicyDocument")); + assert!( + response_body_lower.contains("expires"), + "response should mention Expires mismatch, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_expires_field_missing_from_policy_conditions() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-expires-missing"; + let object_key = "uploads/expires-missing-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("Expires", "Wed, 21 Oct 2037 07:28:00 GMT") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-expires-missing".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::FORBIDDEN); + assert!(response_body.contains("AccessDenied")); + assert!( + response_body_lower.contains("expires"), + "response should mention Expires, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_accepts_object_lock_retention_fields() -> Result<(), Box> +{ + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-object-lock-retention"; + let object_key = "uploads/object-lock-retention.txt"; + let retain_until = "2037-10-21T07:28:00Z"; + let expected_body = b"post-policy-object-lock-retention-body".to_vec(); + + let admin_client = env.create_s3_client(); + admin_client + .create_bucket() + .bucket(bucket) + .object_lock_enabled_for_bucket(true) + .send() + .await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-object-lock-mode": "GOVERNANCE" }), + serde_json::json!({ "x-amz-object-lock-retain-until-date": retain_until }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-object-lock-mode", "GOVERNANCE") + .text("x-amz-object-lock-retain-until-date", retain_until) + .part( + "file", + reqwest::multipart::Part::bytes(expected_body.clone()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::NO_CONTENT); + assert!(response_body.is_empty(), "204 response should not contain a body, got: {response_body}"); + + let retention = admin_client + .get_object_retention() + .bucket(bucket) + .key(object_key) + .send() + .await?; + let retention = retention.retention().expect("retention should be present"); + assert_eq!(retention.mode().map(|value| value.as_str()), Some("GOVERNANCE")); + let retain_until_out = retention + .retain_until_date() + .expect("retain_until_date should be present") + .fmt(aws_sdk_s3::primitives::DateTimeFormat::DateTime)?; + assert_eq!(retain_until_out, retain_until); + + let get_out = admin_client.get_object().bucket(bucket).key(object_key).send().await?; + let uploaded = get_out.body.collect().await?.into_bytes(); + assert_eq!(uploaded.as_ref(), expected_body.as_slice()); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_object_lock_retention_policy_mismatch() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-object-lock-retention-reject"; + let object_key = "uploads/object-lock-retention-reject.txt"; + + let admin_client = env.create_s3_client(); + admin_client + .create_bucket() + .bucket(bucket) + .object_lock_enabled_for_bucket(true) + .send() + .await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-object-lock-mode": "GOVERNANCE" }), + serde_json::json!({ "x-amz-object-lock-retain-until-date": "2037-10-21T07:28:00Z" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-object-lock-mode", "GOVERNANCE") + .text("x-amz-object-lock-retain-until-date", "2037-10-21T08:28:00Z") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-object-lock-retention-mismatch".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!(response_body.contains("InvalidPolicyDocument")); + assert!( + response_body_lower.contains("x-amz-object-lock-retain-until-date"), + "response should mention x-amz-object-lock-retain-until-date mismatch, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_object_lock_mode_policy_mismatch() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-object-lock-mode-reject"; + let object_key = "uploads/object-lock-mode-reject.txt"; + + let admin_client = env.create_s3_client(); + admin_client + .create_bucket() + .bucket(bucket) + .object_lock_enabled_for_bucket(true) + .send() + .await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-object-lock-mode": "GOVERNANCE" }), + serde_json::json!({ "x-amz-object-lock-retain-until-date": "2037-10-21T07:28:00Z" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-object-lock-mode", "COMPLIANCE") + .text("x-amz-object-lock-retain-until-date", "2037-10-21T07:28:00Z") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-object-lock-mode-mismatch".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!(response_body.contains("InvalidPolicyDocument")); + assert!( + response_body_lower.contains("x-amz-object-lock-mode"), + "response should mention x-amz-object-lock-mode mismatch, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_object_lock_retention_missing_from_policy_conditions() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-object-lock-retention-missing"; + let object_key = "uploads/object-lock-retention-missing.txt"; + + let admin_client = env.create_s3_client(); + admin_client + .create_bucket() + .bucket(bucket) + .object_lock_enabled_for_bucket(true) + .send() + .await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-object-lock-mode", "GOVERNANCE") + .text("x-amz-object-lock-retain-until-date", "2037-10-21T07:28:00Z") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-object-lock-retention-missing".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::FORBIDDEN); + assert!(response_body.contains("AccessDenied")); + assert!( + response_body_lower.contains("x-amz-object-lock-mode") + || response_body_lower.contains("x-amz-object-lock-retain-until-date"), + "response should mention object lock retention fields, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_accepts_object_lock_legal_hold_field() -> Result<(), Box> +{ + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-object-lock-legal-hold"; + let object_key = "uploads/object-lock-legal-hold.txt"; + let expected_body = b"post-policy-object-lock-legal-hold-body".to_vec(); + + let admin_client = env.create_s3_client(); + admin_client + .create_bucket() + .bucket(bucket) + .object_lock_enabled_for_bucket(true) + .send() + .await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-object-lock-legal-hold": "ON" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-object-lock-legal-hold", "ON") + .part( + "file", + reqwest::multipart::Part::bytes(expected_body.clone()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::NO_CONTENT); + assert!(response_body.is_empty(), "204 response should not contain a body, got: {response_body}"); + + let legal_hold = admin_client + .get_object_legal_hold() + .bucket(bucket) + .key(object_key) + .send() + .await?; + assert_eq!( + legal_hold + .legal_hold() + .and_then(|value| value.status()) + .map(|value| value.as_str()), + Some("ON") + ); + + let get_out = admin_client.get_object().bucket(bucket).key(object_key).send().await?; + let uploaded = get_out.body.collect().await?.into_bytes(); + assert_eq!(uploaded.as_ref(), expected_body.as_slice()); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_object_lock_legal_hold_policy_mismatch() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-object-lock-legal-hold-reject"; + let object_key = "uploads/object-lock-legal-hold-reject.txt"; + + let admin_client = env.create_s3_client(); + admin_client + .create_bucket() + .bucket(bucket) + .object_lock_enabled_for_bucket(true) + .send() + .await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-object-lock-legal-hold": "ON" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-object-lock-legal-hold", "OFF") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-object-lock-legal-hold-mismatch".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!(response_body.contains("InvalidPolicyDocument")); + assert!( + response_body_lower.contains("x-amz-object-lock-legal-hold"), + "response should mention x-amz-object-lock-legal-hold mismatch, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_object_lock_legal_hold_missing_from_policy_conditions() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-object-lock-legal-hold-missing"; + let object_key = "uploads/object-lock-legal-hold-missing.txt"; + + let admin_client = env.create_s3_client(); + admin_client + .create_bucket() + .bucket(bucket) + .object_lock_enabled_for_bucket(true) + .send() + .await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-object-lock-legal-hold", "ON") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-object-lock-legal-hold-missing".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::FORBIDDEN); + assert!(response_body.contains("AccessDenied")); + assert!( + response_body_lower.contains("x-amz-object-lock-legal-hold"), + "response should mention x-amz-object-lock-legal-hold, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_accepts_tagging_field_exact_policy_match() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-tagging-accept"; + let object_key = "uploads/tagging-object.txt"; + let tagging = "project=alpha&env=test"; + let expected_body = b"post-policy-tagging-body".to_vec(); + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-tagging": tagging }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-tagging", tagging) + .part( + "file", + reqwest::multipart::Part::bytes(expected_body.clone()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::NO_CONTENT); + assert!(response_body.is_empty(), "204 response should not contain a body, got: {response_body}"); + + let tagging_output = admin_client + .get_object_tagging() + .bucket(bucket) + .key(object_key) + .send() + .await?; + let tag_set = tagging_output.tag_set(); + assert_eq!(tag_set.len(), 2); + assert!(tag_set.iter().any(|tag| tag.key() == "project" && tag.value() == "alpha")); + assert!(tag_set.iter().any(|tag| tag.key() == "env" && tag.value() == "test")); + + let get_out = admin_client.get_object().bucket(bucket).key(object_key).send().await?; + let uploaded = get_out.body.collect().await?.into_bytes(); + assert_eq!(uploaded.as_ref(), expected_body.as_slice()); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_tagging_field_policy_mismatch() -> Result<(), Box> +{ + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-tagging-reject"; + let object_key = "uploads/tagging-reject-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-tagging": "project=alpha&env=test" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-tagging", "project=alpha&env=prod") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-tagging-mismatch".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!(response_body.contains("InvalidPolicyDocument")); + assert!( + response_body_lower.contains("x-amz-tagging"), + "response should mention x-amz-tagging mismatch, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_tagging_field_missing_from_policy_conditions() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-tagging-missing"; + let object_key = "uploads/tagging-missing-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-tagging", "project=alpha&env=test") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-tagging-missing".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::FORBIDDEN); + assert!(response_body.contains("AccessDenied")); + assert!( + response_body_lower.contains("x-amz-tagging"), + "response should mention x-amz-tagging, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_metadata_field_missing_from_policy_conditions() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-meta-reject"; + let object_key = "uploads/meta-reject-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-meta-project", "alpha-demo") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-body".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::FORBIDDEN); + assert!( + response_body.contains("AccessDenied"), + "response should contain AccessDenied code, got: {response_body}" + ); + assert!( + response_body_lower.contains("x-amz-meta-project"), + "response should mention the missing metadata field, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_metadata_field_exact_policy_mismatch() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-meta-exact-mismatch"; + let object_key = "uploads/meta-exact-mismatch-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-meta-project": "alpha-demo" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-meta-project", "beta-demo") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-body".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!( + response_body.contains("InvalidPolicyDocument"), + "response should contain InvalidPolicyDocument code, got: {response_body}" + ); + assert!( + response_body_lower.contains("x-amz-meta-project"), + "response should mention the conflicting metadata field, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_accepts_metadata_field_exact_policy_match() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-meta-exact-accept"; + let object_key = "uploads/meta-exact-accept-object.txt"; + let metadata_value = "alpha-demo"; + let expected_body = b"post-policy-meta-exact-body".to_vec(); + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-meta-project": metadata_value }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-meta-project", metadata_value) + .part( + "file", + reqwest::multipart::Part::bytes(expected_body.clone()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::NO_CONTENT); + assert!(response_body.is_empty(), "204 response should not contain a body, got: {response_body}"); + + let head = admin_client.head_object().bucket(bucket).key(object_key).send().await?; + let metadata = head.metadata().expect("head_object should expose uploaded metadata"); + assert_eq!(metadata.get("project").map(String::as_str), Some(metadata_value)); + + let get_out = admin_client.get_object().bucket(bucket).key(object_key).send().await?; + let uploaded = get_out.body.collect().await?.into_bytes(); + assert_eq!(uploaded.as_ref(), expected_body.as_slice()); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_allows_x_ignore_fields_outside_policy_conditions() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-ignore"; + let object_key = "post-policy-ignore-object.txt"; + let expected_body = b"post-policy-ignore-body".to_vec(); + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-ignore-trace-id", "trace-123") + .part( + "file", + reqwest::multipart::Part::bytes(expected_body.clone()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::NO_CONTENT); + assert!(response_body.is_empty(), "204 response should not contain a body, got: {response_body}"); + + let get_out = admin_client.get_object().bucket(bucket).key(object_key).send().await?; + let uploaded = get_out.body.collect().await?.into_bytes(); + assert_eq!(uploaded.as_ref(), expected_body.as_slice()); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_metadata_field_missing_from_policy_conditions_for_new_key() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-meta-name-missing"; + let object_key = "uploads/meta-name-missing.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-meta-name", "demo-name") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-meta-name-missing".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::FORBIDDEN); + assert!(response_body.contains("AccessDenied")); + assert!( + response_body_lower.contains("x-amz-meta-name"), + "response should mention x-amz-meta-name, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_metadata_uuid_exact_policy_mismatch() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-meta-uuid-mismatch"; + let object_key = "uploads/meta-uuid-mismatch.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-meta-uuid": "14365123651274" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-meta-uuid", "151274") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-meta-uuid-mismatch".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!(response_body.contains("InvalidPolicyDocument")); + assert!( + response_body_lower.contains("x-amz-meta-uuid"), + "response should mention x-amz-meta-uuid mismatch, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_sigv4_algorithm_policy_mismatch() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-sigv4-algorithm-mismatch"; + let object_key = "uploads/sigv4-algorithm-mismatch.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-algorithm": "AWS4-HMAC-SHA256" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-algorithm", "incorrect") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-sigv4-algorithm-mismatch".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!(response_body.contains("InvalidPolicyDocument")); + assert!( + response_body_lower.contains("x-amz-algorithm"), + "response should mention x-amz-algorithm mismatch, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_sigv4_credential_policy_mismatch() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-sigv4-credential-mismatch"; + let object_key = "uploads/sigv4-credential-mismatch.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-credential": "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-credential", "incorrect") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-sigv4-credential-mismatch".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!(response_body.contains("InvalidPolicyDocument")); + assert!( + response_body_lower.contains("x-amz-credential"), + "response should mention x-amz-credential mismatch, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_sigv4_date_policy_mismatch() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-sigv4-date-mismatch"; + let object_key = "uploads/sigv4-date-mismatch.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "x-amz-date": "20160727T000000Z" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("x-amz-date", "20160728T000000Z") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-sigv4-date-mismatch".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!(response_body.contains("InvalidPolicyDocument")); + assert!( + response_body_lower.contains("x-amz-date"), + "response should mention x-amz-date mismatch, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_mismatched_bucket_form_field() -> Result<(), Box> +{ + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-bucket-mismatch"; + let object_key = "post-policy-bucket-mismatch-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("bucket", "different-bucket") + .text("key", object_key.to_string()) + .text("policy", policy) + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-body".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!( + response_body.contains("InvalidPolicyDocument"), + "response should contain InvalidPolicyDocument code, got: {response_body}" + ); + assert!( + response_body.contains("different-bucket"), + "response should mention the conflicting bucket field, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_multiple_bucket_values() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-many-bucket-values"; + let object_key = "uploads/many-bucket-values.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("bucket", bucket.to_string()) + .text("bucket", "anotherbucket") + .text("key", object_key.to_string()) + .text("policy", policy) + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-many-bucket-values".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!(response_body.contains("InvalidPolicyDocument")); + assert!( + response_body.contains("anotherbucket") || response_body.contains("multiple values"), + "response should mention duplicated bucket values, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_extra_content_disposition_field() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-extra-disposition"; + let object_key = "post-policy-extra-disposition-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("Content-Disposition", "attachment; filename=\"payload.bin\"") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-body".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::FORBIDDEN); + assert!( + response_body.contains("AccessDenied"), + "response should contain AccessDenied code, got: {response_body}" + ); + assert!( + response_body_lower.contains("content-disposition"), + "response should mention the extra field, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_content_type_policy_mismatch() -> Result<(), Box> +{ + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-content-type"; + let object_key = "post-policy-content-type-object.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!({ "Content-Type": "image/jpeg" }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("Content-Type", "application/octet-stream") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-body".to_vec()) + .file_name("upload.txt") + .mime_str("application/octet-stream")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST); + assert!( + response_body.contains("InvalidPolicyDocument"), + "response should contain InvalidPolicyDocument code, got: {response_body}" + ); + assert!( + response_body_lower.contains("content-type"), + "response should mention the conflicting field, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_anonymous_post_object_rejects_content_type_missing_from_policy_conditions() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "anon-post-policy-content-type-missing"; + let object_key = "uploads/content-type-missing.txt"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + allow_anonymous_put_object(&admin_client, bucket).await?; + + let policy = encode_post_policy(vec![ + serde_json::json!({ "bucket": bucket }), + serde_json::json!({ "key": object_key }), + serde_json::json!(["content-length-range", 0, 1024]), + ]); + + let post_form = reqwest::multipart::Form::new() + .text("key", object_key.to_string()) + .text("policy", policy) + .text("Content-Type", "text/plain") + .part( + "file", + reqwest::multipart::Part::bytes(b"post-policy-content-type-missing".to_vec()) + .file_name("upload.txt") + .mime_str("text/plain")?, + ); + + let post_resp = local_http_client() + .post(format!("{}/{}", env.url, bucket)) + .multipart(post_form) + .send() + .await?; + + let status = post_resp.status(); + let response_body = post_resp.text().await?; + let response_body_lower = response_body.to_ascii_lowercase(); + + assert_eq!(status, reqwest::StatusCode::FORBIDDEN); + assert!(response_body.contains("AccessDenied")); + assert!( + response_body_lower.contains("content-type"), + "response should mention content-type, got: {response_body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_expands_tar_entries_with_prefix_headers() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "signed-extract-upload"; + let archive_key = "batch.tar"; + let extracted_prefix = "imports/run-01"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + + let tar_bytes = make_tar(&[("alpha.txt", b"alpha-body"), ("nested/beta.txt", b"beta-body")], &["ignored/"]).await; + + admin_client + .put_object() + .bucket(bucket) + .key(archive_key) + .body(ByteStream::from(tar_bytes)) + .customize() + .mutate_request(move |req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + req.headers_mut().insert("x-amz-meta-acme-snowball-prefix", extracted_prefix); + req.headers_mut().insert("x-amz-meta-acme-snowball-ignore-dirs", "true"); + }) + .send() + .await?; + + let alpha = admin_client + .get_object() + .bucket(bucket) + .key(format!("{extracted_prefix}/alpha.txt")) + .send() + .await?; + let alpha_body = alpha.body.collect().await?.into_bytes(); + assert_eq!(alpha_body.as_ref(), b"alpha-body"); + + let beta = admin_client + .get_object() + .bucket(bucket) + .key(format!("{extracted_prefix}/nested/beta.txt")) + .send() + .await?; + let beta_body = beta.body.collect().await?.into_bytes(); + assert_eq!(beta_body.as_ref(), b"beta-body"); + + let ignored_dir = admin_client + .get_object() + .bucket(bucket) + .key(format!("{extracted_prefix}/ignored/")) + .send() + .await + .expect_err("directory marker should be skipped when ignore-dirs is enabled"); + match ignored_dir { + SdkError::ServiceError(service_err) => { + let s3_err = service_err.into_err(); + assert!( + s3_err.is_no_such_key() || s3_err.meta().code() == Some("NoSuchVersion"), + "Error should be NoSuchKey or NoSuchVersion, got: {s3_err:?}" + ); + } + other_err => panic!("Expected ServiceError with missing-object code, got: {other_err:?}"), + } + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_preserves_request_metadata_on_extracted_objects() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "signed-extract-request-metadata"; + let archive_key = "metadata.tar"; + let extracted_prefix = "imports/metadata"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + + let tar_bytes = make_tar(&[("alpha.txt", b"alpha-body")], &[]).await; + + admin_client + .put_object() + .bucket(bucket) + .key(archive_key) + .cache_control("max-age=60") + .tagging("project=archive&env=test") + .body(ByteStream::from(tar_bytes)) + .customize() + .mutate_request(move |req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + req.headers_mut().insert("x-amz-meta-acme-snowball-prefix", extracted_prefix); + }) + .send() + .await?; + + let head = admin_client + .head_object() + .bucket(bucket) + .key(format!("{extracted_prefix}/alpha.txt")) + .send() + .await?; + assert_eq!(head.cache_control(), Some("max-age=60")); + + let tagging = admin_client + .get_object_tagging() + .bucket(bucket) + .key(format!("{extracted_prefix}/alpha.txt")) + .send() + .await?; + + let mut tags = tagging + .tag_set() + .iter() + .map(|tag| (tag.key().to_string(), tag.value().to_string())) + .collect::>(); + tags.sort(); + assert_eq!( + tags, + vec![ + ("env".to_string(), "test".to_string()), + ("project".to_string(), "archive".to_string()) + ] + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_preserves_sse_s3_and_redirect() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "signed-extract-sse-s3-redirect"; + let archive_key = "encrypted-metadata.tar"; + let extracted_prefix = "imports/encrypted"; + let redirect_location = "/docs/extracted.html"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + + let tar_bytes = make_tar(&[("alpha.txt", b"alpha-body")], &[]).await; + + admin_client + .put_object() + .bucket(bucket) + .key(archive_key) + .server_side_encryption(aws_sdk_s3::types::ServerSideEncryption::Aes256) + .website_redirect_location(redirect_location) + .body(ByteStream::from(tar_bytes)) + .customize() + .mutate_request(move |req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + req.headers_mut().insert("x-amz-meta-acme-snowball-prefix", extracted_prefix); + }) + .send() + .await?; + + let head = admin_client + .head_object() + .bucket(bucket) + .key(format!("{extracted_prefix}/alpha.txt")) + .send() + .await?; + + assert_eq!(head.server_side_encryption().map(|value| value.as_str()), Some("AES256")); + assert_eq!(head.website_redirect_location(), Some(redirect_location)); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_preserves_storage_class() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "signed-extract-storage-class"; + let archive_key = "storage-class.tar"; + let extracted_prefix = "imports/storage-class"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + + let tar_bytes = make_tar(&[("alpha.txt", b"alpha-body")], &[]).await; + + admin_client + .put_object() + .bucket(bucket) + .key(archive_key) + .storage_class(aws_sdk_s3::types::StorageClass::StandardIa) + .body(ByteStream::from(tar_bytes)) + .customize() + .mutate_request(move |req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + req.headers_mut().insert("x-amz-meta-acme-snowball-prefix", extracted_prefix); + }) + .send() + .await?; + + let head = admin_client + .head_object() + .bucket(bucket) + .key(format!("{extracted_prefix}/alpha.txt")) + .send() + .await?; + + assert_eq!(head.storage_class().map(|value| value.as_str()), Some("STANDARD_IA")); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_rejects_invalid_storage_class() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "signed-extract-storage-class-invalid"; + let archive_key = "storage-class-invalid.tar"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + + let tar_bytes = make_tar(&[("alpha.txt", b"alpha-body")], &[]).await; + + let result = admin_client + .put_object() + .bucket(bucket) + .key(archive_key) + .body(ByteStream::from(tar_bytes)) + .customize() + .mutate_request(|req| { + req.headers_mut() + .insert("x-amz-meta-snowball-auto-extract", HeaderValue::from_static("true")); + req.headers_mut() + .insert("x-amz-storage-class", HeaderValue::from_static("INVALID")); + }) + .send() + .await; + + assert_s3_error_code(result, "InvalidStorageClass"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_uses_bucket_default_sse_s3() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "signed-extract-default-sse-s3"; + let archive_key = "default-encryption.tar"; + let extracted_prefix = "imports/default-encryption"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + + let encryption_config = ServerSideEncryptionConfiguration::builder() + .rules( + ServerSideEncryptionRule::builder() + .apply_server_side_encryption_by_default( + ServerSideEncryptionByDefault::builder() + .sse_algorithm(ServerSideEncryption::Aes256) + .build() + .expect("default encryption rule should build"), + ) + .build(), + ) + .build() + .expect("bucket encryption config should build"); + + admin_client + .put_bucket_encryption() + .bucket(bucket) + .server_side_encryption_configuration(encryption_config) + .send() + .await?; + + let tar_bytes = make_tar(&[("alpha.txt", b"alpha-body")], &[]).await; + + admin_client + .put_object() + .bucket(bucket) + .key(archive_key) + .body(ByteStream::from(tar_bytes)) + .customize() + .mutate_request(move |req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + req.headers_mut().insert("x-amz-meta-acme-snowball-prefix", extracted_prefix); + }) + .send() + .await?; + + let head = admin_client + .head_object() + .bucket(bucket) + .key(format!("{extracted_prefix}/alpha.txt")) + .send() + .await?; + + assert_eq!(head.server_side_encryption().map(|value| value.as_str()), Some("AES256")); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_rejects_bucket_default_sse_kms() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "signed-extract-default-sse-kms"; + let archive_key = "default-encryption-kms.tar"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + + let encryption_config = ServerSideEncryptionConfiguration::builder() + .rules( + ServerSideEncryptionRule::builder() + .apply_server_side_encryption_by_default( + ServerSideEncryptionByDefault::builder() + .sse_algorithm(ServerSideEncryption::AwsKms) + .kms_master_key_id("test-key") + .build() + .expect("default encryption rule should build"), + ) + .build(), + ) + .build() + .expect("bucket encryption config should build"); + + admin_client + .put_bucket_encryption() + .bucket(bucket) + .server_side_encryption_configuration(encryption_config) + .send() + .await?; + + let tar_bytes = make_tar(&[("alpha.txt", b"alpha-body")], &[]).await; + + let result = admin_client + .put_object() + .bucket(bucket) + .key(archive_key) + .body(ByteStream::from(tar_bytes)) + .customize() + .mutate_request(|req| { + req.headers_mut() + .insert("x-amz-meta-snowball-auto-extract", HeaderValue::from_static("true")); + }) + .send() + .await; + + assert_s3_error_code(result, "NotImplemented"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_preserves_sse_c() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "extract-sse-c"; + let archive_key = "bundle.tar"; + let extracted_key = "nested/file.txt"; + let expected_body = b"extract-sse-c-body".to_vec(); + let customer_key = "01234567890123456789012345678901"; + let customer_key_b64 = base64::engine::general_purpose::STANDARD.encode(customer_key); + let customer_key_md5 = sse_customer_key_md5_base64(customer_key); + + let client = env.create_s3_client(); + client.create_bucket().bucket(bucket).send().await?; + + let archive = make_tar(&[(extracted_key, expected_body.as_slice())], &[]).await; + + client + .put_object() + .bucket(bucket) + .key(archive_key) + .body(ByteStream::from(archive)) + .sse_customer_algorithm("AES256") + .sse_customer_key(customer_key_b64.clone()) + .sse_customer_key_md5(customer_key_md5.clone()) + .customize() + .mutate_request(|req| { + req.headers_mut() + .insert("x-amz-meta-snowball-auto-extract", HeaderValue::from_static("true")); + req.headers_mut() + .insert("x-amz-meta-rustfs-snowball-prefix", HeaderValue::from_static("extract-root")); + }) + .send() + .await?; + + let extracted = client + .head_object() + .bucket(bucket) + .key("extract-root/nested/file.txt") + .sse_customer_algorithm("AES256") + .sse_customer_key(customer_key_b64.clone()) + .sse_customer_key_md5(customer_key_md5.clone()) + .send() + .await?; + assert_eq!(extracted.sse_customer_algorithm(), Some("AES256")); + + let fetched = client + .get_object() + .bucket(bucket) + .key("extract-root/nested/file.txt") + .sse_customer_algorithm("AES256") + .sse_customer_key(customer_key_b64) + .sse_customer_key_md5(customer_key_md5) + .send() + .await?; + let actual_body = fetched.body.collect().await?.into_bytes().to_vec(); + assert_eq!(actual_body, expected_body); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_preserves_object_lock_legal_hold() -> Result<(), Box> +{ + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "signed-extract-object-lock-hold"; + let archive_key = "legal-hold.tar"; + let extracted_prefix = "imports/legal-hold"; + + let admin_client = env.create_s3_client(); + admin_client + .create_bucket() + .bucket(bucket) + .object_lock_enabled_for_bucket(true) + .send() + .await?; + + let tar_bytes = make_tar(&[("alpha.txt", b"alpha-body")], &[]).await; + + admin_client + .put_object() + .bucket(bucket) + .key(archive_key) + .object_lock_legal_hold_status(aws_sdk_s3::types::ObjectLockLegalHoldStatus::On) + .body(ByteStream::from(tar_bytes)) + .customize() + .mutate_request(move |req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + req.headers_mut().insert("x-amz-meta-acme-snowball-prefix", extracted_prefix); + }) + .send() + .await?; + + let legal_hold = admin_client + .get_object_legal_hold() + .bucket(bucket) + .key(format!("{extracted_prefix}/alpha.txt")) + .send() + .await?; + + assert_eq!( + legal_hold + .legal_hold() + .and_then(|value| value.status()) + .map(|value| value.as_str()), + Some("ON") + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_preserves_object_lock_retention() -> Result<(), Box> +{ + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "signed-extract-object-lock-retention"; + let archive_key = "retention.tar"; + let extracted_prefix = "imports/retention"; + let retain_until = aws_sdk_s3::primitives::DateTime::from_secs(2_143_623_680); + let retain_until_expected = retain_until.fmt(aws_sdk_s3::primitives::DateTimeFormat::DateTime)?; + + let admin_client = env.create_s3_client(); + admin_client + .create_bucket() + .bucket(bucket) + .object_lock_enabled_for_bucket(true) + .send() + .await?; + + let tar_bytes = make_tar(&[("alpha.txt", b"alpha-body")], &[]).await; + + admin_client + .put_object() + .bucket(bucket) + .key(archive_key) + .object_lock_mode(aws_sdk_s3::types::ObjectLockMode::Governance) + .object_lock_retain_until_date(retain_until) + .body(ByteStream::from(tar_bytes)) + .customize() + .mutate_request(move |req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + req.headers_mut().insert("x-amz-meta-acme-snowball-prefix", extracted_prefix); + }) + .send() + .await?; + + let retention = admin_client + .get_object_retention() + .bucket(bucket) + .key(format!("{extracted_prefix}/alpha.txt")) + .send() + .await?; + let retention = retention.retention().expect("retention should be present"); + + assert_eq!(retention.mode().map(|value| value.as_str()), Some("GOVERNANCE")); + assert_eq!( + retention + .retain_until_date() + .expect("retain_until_date should be present") + .fmt(aws_sdk_s3::primitives::DateTimeFormat::DateTime)?, + retain_until_expected + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_returns_archive_etag() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "signed-extract-etag"; + let archive_key = "bundle.tar"; + + let client = env.create_s3_client(); + client.create_bucket().bucket(bucket).send().await?; + + let archive = make_tar(&[("alpha.txt", b"alpha-body")], &[]).await; + let expected_etag = format!("\"{:x}\"", md5::compute(&archive)); + + let response = client + .put_object() + .bucket(bucket) + .key(archive_key) + .body(ByteStream::from(archive)) + .customize() + .mutate_request(|req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + }) + .send() + .await?; + + assert_eq!(response.e_tag(), Some(expected_etag.as_str())); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_preserves_entry_mtime() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "signed-extract-mtime"; + let archive_key = "bundle.tar"; + let extracted_key = "mtime/file.txt"; + let modified_at_secs = 1_704_000_123_u64; + + let client = env.create_s3_client(); + client.create_bucket().bucket(bucket).send().await?; + + let archive = make_tar_with_pax_entry(extracted_key, b"mtime-body", Some(modified_at_secs), &HashMap::new()).await; + + client + .put_object() + .bucket(bucket) + .key(archive_key) + .body(ByteStream::from(archive)) + .customize() + .mutate_request(|req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + }) + .send() + .await?; + + let head = client.head_object().bucket(bucket).key(extracted_key).send().await?; + assert_eq!(head.last_modified().expect("last_modified should exist").secs(), modified_at_secs as i64); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_preserves_pax_metadata_and_version_id() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "signed-extract-pax"; + let archive_key = "bundle.tar"; + let extracted_key = "pax/alpha.txt"; + let expected_version_id = Uuid::new_v4().to_string(); + + let client = env.create_s3_client(); + client.create_bucket().bucket(bucket).send().await?; + client + .put_bucket_versioning() + .bucket(bucket) + .versioning_configuration( + aws_sdk_s3::types::VersioningConfiguration::builder() + .status(aws_sdk_s3::types::BucketVersioningStatus::Enabled) + .build(), + ) + .send() + .await?; + + let mut pax = HashMap::new(); + pax.insert("minio.metadata.project", "alpha-demo".to_string()); + pax.insert("minio.metadata.x-amz-meta-owner", "ops".to_string()); + pax.insert("minio.versionId", expected_version_id.clone()); + let archive = make_tar_with_pax_entry(extracted_key, b"pax-body", None, &pax).await; + + client + .put_object() + .bucket(bucket) + .key(archive_key) + .body(ByteStream::from(archive)) + .customize() + .mutate_request(|req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + }) + .send() + .await?; + + let head = client.head_object().bucket(bucket).key(extracted_key).send().await?; + let metadata = head.metadata().expect("head_object should expose metadata"); + assert_eq!(metadata.get("project").map(String::as_str), Some("alpha-demo")); + assert_eq!(metadata.get("owner").map(String::as_str), Some("ops")); + assert_eq!(head.version_id(), Some(expected_version_id.as_str())); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_accepts_compat_header() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "signed-extract-compat"; + let archive_key = "compat.tar"; + let extracted_prefix = "imports/compat"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + + let tar_bytes = make_tar(&[("gamma.txt", b"gamma-body")], &[]).await; + + admin_client + .put_object() + .bucket(bucket) + .key(archive_key) + .body(ByteStream::from(tar_bytes)) + .customize() + .mutate_request(move |req| { + req.headers_mut().insert("x-amz-snowball-auto-extract", "true"); + req.headers_mut().insert("x-amz-meta-acme-snowball-prefix", extracted_prefix); + }) + .send() + .await?; + + let gamma = admin_client + .get_object() + .bucket(bucket) + .key(format!("{extracted_prefix}/gamma.txt")) + .send() + .await?; + let gamma_body = gamma.body.collect().await?.into_bytes(); + assert_eq!(gamma_body.as_ref(), b"gamma-body"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_preserves_directory_markers_by_default() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "signed-extract-dirs"; + let archive_key = "dirs.tar"; + let extracted_prefix = "imports/tree"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + + let tar_bytes = make_tar(&[("nested/file.txt", b"file-body")], &["empty/", "nested/"]).await; + + admin_client + .put_object() + .bucket(bucket) + .key(archive_key) + .body(ByteStream::from(tar_bytes)) + .customize() + .mutate_request(move |req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + req.headers_mut().insert("x-amz-meta-acme-snowball-prefix", extracted_prefix); + }) + .send() + .await?; + + let empty_dir = admin_client + .get_object() + .bucket(bucket) + .key(format!("{extracted_prefix}/empty/")) + .send() + .await?; + let empty_dir_body = empty_dir.body.collect().await?.into_bytes(); + assert!(empty_dir_body.is_empty(), "directory marker object should be empty"); + + let nested_dir = admin_client + .get_object() + .bucket(bucket) + .key(format!("{extracted_prefix}/nested/")) + .send() + .await?; + let nested_dir_body = nested_dir.body.collect().await?.into_bytes(); + assert!(nested_dir_body.is_empty(), "nested directory marker object should be empty"); + + let nested_file = admin_client + .get_object() + .bucket(bucket) + .key(format!("{extracted_prefix}/nested/file.txt")) + .send() + .await?; + let nested_file_body = nested_file.body.collect().await?.into_bytes(); + assert_eq!(nested_file_body.as_ref(), b"file-body"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_expands_tar_gz_archive() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "signed-extract-tar-gz"; + let archive_key = "bundle.tar.gz"; + let extracted_prefix = "imports/gzip"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + + let tar_bytes = make_tar(&[("delta.txt", b"delta-body"), ("nested/epsilon.txt", b"epsilon-body")], &[]).await; + let tar_gz_bytes = gzip_bytes(&tar_bytes); + + admin_client + .put_object() + .bucket(bucket) + .key(archive_key) + .body(ByteStream::from(tar_gz_bytes)) + .customize() + .mutate_request(move |req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + req.headers_mut().insert("x-amz-meta-acme-snowball-prefix", extracted_prefix); + }) + .send() + .await?; + + let delta = admin_client + .get_object() + .bucket(bucket) + .key(format!("{extracted_prefix}/delta.txt")) + .send() + .await?; + let delta_body = delta.body.collect().await?.into_bytes(); + assert_eq!(delta_body.as_ref(), b"delta-body"); + + let epsilon = admin_client + .get_object() + .bucket(bucket) + .key(format!("{extracted_prefix}/nested/epsilon.txt")) + .send() + .await?; + let epsilon_body = epsilon.body.collect().await?.into_bytes(); + assert_eq!(epsilon_body.as_ref(), b"epsilon-body"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_expands_tgz_archive() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "signed-extract-tgz"; + let archive_key = "bundle.tgz"; + let extracted_prefix = "imports/tgz"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + + let tar_bytes = make_tar(&[("phi.txt", b"phi-body"), ("nested/psi.txt", b"psi-body")], &[]).await; + let tgz_bytes = gzip_bytes(&tar_bytes); + + admin_client + .put_object() + .bucket(bucket) + .key(archive_key) + .body(ByteStream::from(tgz_bytes)) + .customize() + .mutate_request(move |req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + req.headers_mut().insert("x-amz-meta-acme-snowball-prefix", extracted_prefix); + }) + .send() + .await?; + + let phi = admin_client + .get_object() + .bucket(bucket) + .key(format!("{extracted_prefix}/phi.txt")) + .send() + .await?; + let phi_body = phi.body.collect().await?.into_bytes(); + assert_eq!(phi_body.as_ref(), b"phi-body"); + + let psi = admin_client + .get_object() + .bucket(bucket) + .key(format!("{extracted_prefix}/nested/psi.txt")) + .send() + .await?; + let psi_body = psi.body.collect().await?.into_bytes(); + assert_eq!(psi_body.as_ref(), b"psi-body"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_expands_tbz2_archive() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "signed-extract-tbz2"; + let archive_key = "bundle.tbz2"; + let extracted_prefix = "imports/tbz2"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + + let tar_bytes = make_tar(&[("rho.txt", b"rho-body"), ("nested/tau.txt", b"tau-body")], &[]).await; + let tbz2_bytes = bzip2_bytes(&tar_bytes).await; + + admin_client + .put_object() + .bucket(bucket) + .key(archive_key) + .body(ByteStream::from(tbz2_bytes)) + .customize() + .mutate_request(move |req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + req.headers_mut().insert("x-amz-meta-acme-snowball-prefix", extracted_prefix); + }) + .send() + .await?; + + let rho = admin_client + .get_object() + .bucket(bucket) + .key(format!("{extracted_prefix}/rho.txt")) + .send() + .await?; + let rho_body = rho.body.collect().await?.into_bytes(); + assert_eq!(rho_body.as_ref(), b"rho-body"); + + let tau = admin_client + .get_object() + .bucket(bucket) + .key(format!("{extracted_prefix}/nested/tau.txt")) + .send() + .await?; + let tau_body = tau.body.collect().await?.into_bytes(); + assert_eq!(tau_body.as_ref(), b"tau-body"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_expands_txz_archive() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "signed-extract-txz"; + let archive_key = "bundle.txz"; + let extracted_prefix = "imports/txz"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + + let tar_bytes = make_tar(&[("chi.txt", b"chi-body"), ("nested/upsilon.txt", b"upsilon-body")], &[]).await; + let txz_bytes = xz_bytes(&tar_bytes).await; + + admin_client + .put_object() + .bucket(bucket) + .key(archive_key) + .body(ByteStream::from(txz_bytes)) + .customize() + .mutate_request(move |req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + req.headers_mut().insert("x-amz-meta-acme-snowball-prefix", extracted_prefix); + }) + .send() + .await?; + + let chi = admin_client + .get_object() + .bucket(bucket) + .key(format!("{extracted_prefix}/chi.txt")) + .send() + .await?; + let chi_body = chi.body.collect().await?.into_bytes(); + assert_eq!(chi_body.as_ref(), b"chi-body"); + + let upsilon = admin_client + .get_object() + .bucket(bucket) + .key(format!("{extracted_prefix}/nested/upsilon.txt")) + .send() + .await?; + let upsilon_body = upsilon.body.collect().await?.into_bytes(); + assert_eq!(upsilon_body.as_ref(), b"upsilon-body"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_skips_invalid_entry_when_ignore_errors_enabled() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "signed-extract-ignore-errors"; + let archive_key = "bundle.tar"; + let extracted_prefix = "imports/ignore-errors"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + + let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new())); + + let mut valid_header = tokio_tar::Header::new_gnu(); + valid_header.set_size(b"valid-body".len() as u64); + valid_header.set_mode(0o644); + valid_header.set_cksum(); + builder + .append_data(&mut valid_header, "valid.txt", Cursor::new(b"valid-body".as_slice())) + .await + .expect("valid tar entry should be appended"); + + let long_name = format!("{}.txt", "a".repeat(1100)); + let mut invalid_header = tokio_tar::Header::new_gnu(); + invalid_header.set_size(b"ignored-body".len() as u64); + invalid_header.set_mode(0o644); + invalid_header.set_cksum(); + builder + .append_data(&mut invalid_header, long_name, Cursor::new(b"ignored-body".as_slice())) + .await + .expect("long-name tar entry should be appended"); + + let tar_bytes = builder.into_inner().await.expect("tar builder should finalize").into_inner(); + + admin_client + .put_object() + .bucket(bucket) + .key(archive_key) + .body(ByteStream::from(tar_bytes)) + .customize() + .mutate_request(move |req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + req.headers_mut().insert("x-amz-meta-acme-snowball-prefix", extracted_prefix); + req.headers_mut().insert("x-amz-meta-acme-snowball-ignore-errors", "true"); + }) + .send() + .await?; + + let valid = admin_client + .get_object() + .bucket(bucket) + .key(format!("{extracted_prefix}/valid.txt")) + .send() + .await?; + let valid_body = valid.body.collect().await?.into_bytes(); + assert_eq!(valid_body.as_ref(), b"valid-body"); + + let listed = admin_client + .list_objects_v2() + .bucket(bucket) + .prefix(format!("{extracted_prefix}/")) + .send() + .await?; + let keys: Vec<_> = listed.contents().iter().filter_map(|entry| entry.key()).collect(); + assert_eq!(keys, vec![format!("{extracted_prefix}/valid.txt")]); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_normalizes_prefix_header_value() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "signed-extract-prefix-normalize"; + let archive_key = "bundle.tar"; + let extracted_prefix = " /batch/incoming/ "; + let normalized_prefix = "batch/incoming"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + + let tar_bytes = make_tar(&[("theta.txt", b"theta-body")], &[]).await; + + admin_client + .put_object() + .bucket(bucket) + .key(archive_key) + .body(ByteStream::from(tar_bytes)) + .customize() + .mutate_request(move |req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + req.headers_mut().insert("x-amz-meta-snowball-prefix", extracted_prefix); + }) + .send() + .await?; + + let theta = admin_client + .get_object() + .bucket(bucket) + .key(format!("{normalized_prefix}/theta.txt")) + .send() + .await?; + let theta_body = theta.body.collect().await?.into_bytes(); + assert_eq!(theta_body.as_ref(), b"theta-body"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_expands_tzst_archive() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "signed-extract-tzst"; + let archive_key = "bundle.tzst"; + let extracted_prefix = "imports/tzst"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + + let tar_bytes = make_tar(&[("omega.txt", b"omega-body"), ("nested/sigma.txt", b"sigma-body")], &[]).await; + let tzst_bytes = zstd_bytes(&tar_bytes); + + admin_client + .put_object() + .bucket(bucket) + .key(archive_key) + .body(ByteStream::from(tzst_bytes)) + .customize() + .mutate_request(move |req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + req.headers_mut().insert("x-amz-meta-acme-snowball-prefix", extracted_prefix); + }) + .send() + .await?; + + let omega = admin_client + .get_object() + .bucket(bucket) + .key(format!("{extracted_prefix}/omega.txt")) + .send() + .await?; + let omega_body = omega.body.collect().await?.into_bytes(); + assert_eq!(omega_body.as_ref(), b"omega-body"); + + let sigma = admin_client + .get_object() + .bucket(bucket) + .key(format!("{extracted_prefix}/nested/sigma.txt")) + .send() + .await?; + let sigma_body = sigma.body.collect().await?.into_bytes(); + assert_eq!(sigma_body.as_ref(), b"sigma-body"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_rejects_missing_archive_extension() -> Result<(), Box> +{ + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "signed-extract-no-ext"; + let archive_key = "bundle"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + + let tar_bytes = make_tar(&[("plain.txt", b"plain-body")], &[]).await; + + let result = admin_client + .put_object() + .bucket(bucket) + .key(archive_key) + .body(ByteStream::from(tar_bytes)) + .customize() + .mutate_request(move |req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + }) + .send() + .await; + + assert_s3_error_code(result, "InvalidArgument"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_rejects_invalid_tar_gz_payload() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "signed-extract-bad-gzip"; + let archive_key = "broken.tar.gz"; + + let admin_client = env.create_s3_client(); + admin_client.create_bucket().bucket(bucket).send().await?; + + let result = admin_client + .put_object() + .bucket(bucket) + .key(archive_key) + .body(ByteStream::from_static(b"not-a-gzip-stream")) + .customize() + .mutate_request(move |req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + }) + .send() + .await; + + assert_s3_error_code(result, "InvalidArgument"); + + Ok(()) +} diff --git a/crates/e2e_test/src/object_lambda_test.rs b/crates/e2e_test/src/object_lambda_test.rs new file mode 100644 index 000000000..f4ed795a9 --- /dev/null +++ b/crates/e2e_test/src/object_lambda_test.rs @@ -0,0 +1,985 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, local_http_client}; +use aws_sdk_s3::primitives::ByteStream; +use http::header::{CONTENT_TYPE, HOST}; +use reqwest::StatusCode; +use rustfs_signer::constants::UNSIGNED_PAYLOAD; +use rustfs_signer::{pre_sign_v4, sign_v4}; +use s3s::Body; +use serial_test::serial; +use std::collections::HashMap; +use std::error::Error; +use time::OffsetDateTime; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpListener; +use tokio::sync::oneshot; +use tokio::time::{Duration, timeout}; + +#[derive(Debug)] +struct CapturedWebhookRequest { + headers: HashMap, + payload: serde_json::Value, +} + +struct WebhookResponseSpec { + status_line: String, + body: Vec, + headers: Vec<(String, String)>, + include_auth_headers: bool, + auth_route_override: Option, + auth_token_override: Option, +} + +fn find_header_terminator(buf: &[u8]) -> Option { + buf.windows(4).position(|window| window == b"\r\n\r\n") +} + +async fn read_http_request( + stream: &mut tokio::net::TcpStream, +) -> Result<(HashMap, Vec), Box> { + let mut buffer = Vec::new(); + let mut chunk = [0_u8; 4096]; + + let header_end = loop { + let read = stream.read(&mut chunk).await?; + if read == 0 { + return Err("webhook request ended before headers were fully received".into()); + } + buffer.extend_from_slice(&chunk[..read]); + if let Some(pos) = find_header_terminator(&buffer) { + break pos; + } + }; + + let header_bytes = &buffer[..header_end]; + let header_text = std::str::from_utf8(header_bytes)?; + let mut lines = header_text.split("\r\n"); + let _request_line = lines.next().ok_or("missing request line")?; + let mut headers = HashMap::new(); + for line in lines { + if line.is_empty() { + continue; + } + let (name, value) = line.split_once(':').ok_or("invalid header line")?; + headers.insert(name.trim().to_ascii_lowercase(), value.trim().to_string()); + } + + let content_length = headers + .get("content-length") + .ok_or("missing content-length header")? + .parse::()?; + let body_offset = header_end + 4; + while buffer.len().saturating_sub(body_offset) < content_length { + let read = stream.read(&mut chunk).await?; + if read == 0 { + return Err("webhook request ended before body was fully received".into()); + } + buffer.extend_from_slice(&chunk[..read]); + } + + Ok((headers, buffer[body_offset..body_offset + content_length].to_vec())) +} + +async fn spawn_object_lambda_webhook_server() -> Result< + ( + String, + oneshot::Receiver, + tokio::task::JoinHandle>>, + ), + Box, +> { + spawn_object_lambda_webhook_server_with_response(WebhookResponseSpec { + status_line: "200 OK".to_string(), + body: b"transformed through object lambda".to_vec(), + headers: vec![("content-type".to_string(), "text/plain".to_string())], + include_auth_headers: true, + auth_route_override: None, + auth_token_override: None, + }) + .await +} + +async fn spawn_object_lambda_webhook_server_with_response( + response_spec: WebhookResponseSpec, +) -> Result< + ( + String, + oneshot::Receiver, + tokio::task::JoinHandle>>, + ), + Box, +> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let address = listener.local_addr()?; + let webhook_url = format!("http://{address}/transform"); + let (request_tx, request_rx) = oneshot::channel(); + + let handle = tokio::spawn(async move { + loop { + let (mut stream, _) = listener.accept().await?; + let Ok(Ok((headers, body))) = timeout(Duration::from_secs(2), read_http_request(&mut stream)).await else { + continue; + }; + let payload: serde_json::Value = serde_json::from_slice(&body)?; + + let output_route = payload["getObjectContext"]["outputRoute"] + .as_str() + .ok_or("missing outputRoute in webhook payload")? + .to_string(); + let output_token = payload["getObjectContext"]["outputToken"] + .as_str() + .ok_or("missing outputToken in webhook payload")? + .to_string(); + + let _ = request_tx.send(CapturedWebhookRequest { headers, payload }); + + let mut response_head = format!( + "HTTP/1.1 {}\r\ncontent-length: {}\r\nconnection: close\r\n", + response_spec.status_line, + response_spec.body.len() + ); + for (name, value) in &response_spec.headers { + response_head.push_str(&format!("{name}: {value}\r\n")); + } + if response_spec.include_auth_headers { + let auth_route = response_spec.auth_route_override.as_deref().unwrap_or(&output_route); + let auth_token = response_spec.auth_token_override.as_deref().unwrap_or(&output_token); + response_head.push_str(&format!("x-amz-request-route: {auth_route}\r\n")); + response_head.push_str(&format!("x-amz-request-token: {auth_token}\r\n")); + } + response_head.push_str("\r\n"); + stream.write_all(response_head.as_bytes()).await?; + stream.write_all(&response_spec.body).await?; + stream.shutdown().await?; + + return Ok(()); + } + }); + + Ok((webhook_url, request_rx, handle)) +} + +async fn presigned_get_request( + url: &str, + access_key: &str, + secret_key: &str, +) -> Result> { + let uri = url.parse::()?; + let authority = uri.authority().ok_or("request URL missing authority")?.to_string(); + let signed = pre_sign_v4( + http::Request::builder() + .method(http::Method::GET) + .uri(uri) + .header(HOST, authority) + .body(Body::empty())?, + access_key, + secret_key, + "", + "us-east-1", + 600, + OffsetDateTime::now_utc(), + ); + + Ok(local_http_client().get(signed.uri().to_string()).send().await?) +} + +async fn signed_request( + method: http::Method, + url: &str, + access_key: &str, + secret_key: &str, + body: Option>, + content_type: Option<&str>, +) -> Result> { + let uri = url.parse::()?; + let authority = uri.authority().ok_or("request URL missing authority")?.to_string(); + let mut request = http::Request::builder().method(method.clone()).uri(uri); + request = request.header(HOST, authority); + request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD); + if let Some(content_type) = content_type { + request = request.header(CONTENT_TYPE, content_type); + } + + let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default(); + let signed = sign_v4(request.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1"); + + let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?; + let client = local_http_client(); + let mut request_builder = client.request(reqwest_method, url); + for (name, value) in signed.headers() { + request_builder = request_builder.header(name, value); + } + if let Some(body) = body { + request_builder = request_builder.body(body); + } + + Ok(request_builder.send().await?) +} + +async fn configure_webhook_target( + env: &RustFSTestEnvironment, + target_name: &str, + endpoint: &str, + auth_token: &str, +) -> Result<(), Box> { + configure_webhook_target_with_key_values( + env, + target_name, + vec![ + ("endpoint", endpoint.to_string()), + ("auth_token", auth_token.to_string()), + ("queue_dir", format!("{}/notify-queue", env.temp_dir)), + ], + ) + .await +} + +async fn configure_webhook_target_with_key_values( + env: &RustFSTestEnvironment, + target_name: &str, + key_values: Vec<(&str, String)>, +) -> Result<(), Box> { + let queue_dir = format!("{}/notify-queue", env.temp_dir); + tokio::fs::create_dir_all(&queue_dir).await?; + let mut key_values = key_values + .into_iter() + .map(|(key, value)| serde_json::json!({ "key": key, "value": value })) + .collect::>(); + if !key_values.iter().any(|entry| entry["key"].as_str() == Some("queue_dir")) { + key_values.push(serde_json::json!({ "key": "queue_dir", "value": queue_dir })); + } + let response = send_configure_webhook_target_request(env, target_name, key_values).await?; + if response.status() != StatusCode::OK { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(format!("failed to configure object lambda webhook target: {status} {body}").into()); + } + + Ok(()) +} + +async fn send_configure_webhook_target_request( + env: &RustFSTestEnvironment, + target_name: &str, + key_values: Vec, +) -> Result> { + let payload = serde_json::json!({ "key_values": key_values }); + let url = format!("{}/rustfs/admin/v3/target/notify_webhook/{}", env.url, target_name); + signed_request( + http::Method::PUT, + &url, + &env.access_key, + &env.secret_key, + Some(payload.to_string().into_bytes()), + Some("application/json"), + ) + .await +} + +async fn list_notification_targets(env: &RustFSTestEnvironment) -> Result> { + let url = format!("{}/rustfs/admin/v3/target/list", env.url); + let response = signed_request(http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await?; + let status = response.status(); + let body = response.bytes().await?; + if status != StatusCode::OK { + return Err(format!("failed to list notification targets: {status} {}", String::from_utf8_lossy(body.as_ref())).into()); + } + + Ok(serde_json::from_slice(&body)?) +} + +async fn list_target_arns(env: &RustFSTestEnvironment) -> Result, Box> { + let url = format!("{}/rustfs/admin/v3/target/arns", env.url); + let response = signed_request(http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await?; + let status = response.status(); + let body = response.bytes().await?; + if status != StatusCode::OK { + return Err(format!("failed to list target arns: {status} {}", String::from_utf8_lossy(body.as_ref())).into()); + } + + Ok(serde_json::from_slice(&body)?) +} + +async fn wait_for_target_visibility( + env: &RustFSTestEnvironment, + target_name: &str, +) -> Result<(serde_json::Value, Vec), Box> { + let mut last_targets = serde_json::Value::Null; + let mut last_arns = Vec::new(); + + for _ in 0..20 { + last_targets = list_notification_targets(env).await?; + last_arns = list_target_arns(env).await?; + + let listed = last_targets["notification_endpoints"] + .as_array() + .into_iter() + .flatten() + .any(|entry| { + entry["account_id"].as_str() == Some(target_name) + && entry["service"] + .as_str() + .is_some_and(|service| service == "webhook" || service.starts_with("webhook-")) + }); + + if listed { + return Ok((last_targets, last_arns)); + } + + tokio::time::sleep(Duration::from_millis(250)).await; + } + + Err(format!("target {target_name} did not become visible in admin APIs; targets={last_targets}, arns={last_arns:?}").into()) +} + +async fn read_persisted_server_config(env: &RustFSTestEnvironment) -> String { + let path = format!("{}/.rustfs.sys/config/config.json", env.temp_dir); + match tokio::fs::read_to_string(&path).await { + Ok(content) => content, + Err(err) => format!("failed to read persisted config at {path}: {err}"), + } +} + +async fn read_listen_notification_event( + response: reqwest::Response, + expected_key: &str, +) -> Result> { + let mut response = response; + let mut pending = String::new(); + loop { + let chunk = timeout(Duration::from_secs(12), response.chunk()).await??; + let Some(chunk) = chunk else { + return Err("listen_notification stream ended before payload".into()); + }; + if chunk.is_empty() { + continue; + } + pending.push_str(&String::from_utf8(chunk.to_vec())?); + + while let Some(newline) = pending.find('\n') { + let line = pending.drain(..=newline).collect::(); + let payload = line.trim(); + if payload.is_empty() { + continue; + } + + let json: serde_json::Value = serde_json::from_str(payload)?; + let Some(records) = json["Records"].as_array() else { + continue; + }; + if records.is_empty() { + continue; + } + + let has_expected_key = records.iter().any(|record| { + let Some(object_key) = record["s3"]["object"]["key"].as_str() else { + return false; + }; + let decoded = urlencoding::decode(object_key) + .map(|decoded| decoded.into_owned()) + .unwrap_or_else(|_| object_key.to_string()); + decoded == expected_key + }); + if has_expected_key { + return Ok(payload.to_string()); + } + } + } +} + +#[tokio::test] +#[serial] +async fn test_get_object_lambda_accepts_presigned_requests() -> Result<(), Box> { + init_logging(); + + let (webhook_url, request_rx, webhook_handle) = spawn_object_lambda_webhook_server().await?; + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "object-lambda-e2e-presigned"; + let key = "input.txt"; + let object_body = b"hello presigned object lambda"; + let lambda_arn = "arn:rustfs:s3-object-lambda:us-east-1:transformer:webhook"; + + let client = env.create_s3_client(); + client.create_bucket().bucket(bucket).send().await?; + client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from_static(object_body)) + .send() + .await?; + + configure_webhook_target(&env, "transformer", &webhook_url, "secret-token").await?; + wait_for_target_visibility(&env, "transformer").await?; + + let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn)); + let response = presigned_get_request(&lambda_url, &env.access_key, &env.secret_key).await?; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.text().await?, "transformed through object lambda"); + + let captured = timeout(Duration::from_secs(10), request_rx).await??; + assert_eq!(captured.payload["configuration"]["accessPointArn"].as_str(), Some(lambda_arn)); + + webhook_handle.await??; + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_get_object_lambda_accepts_named_webhook_target_arn() -> Result<(), Box> { + init_logging(); + + let (webhook_url, request_rx, webhook_handle) = spawn_object_lambda_webhook_server().await?; + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "object-lambda-e2e-named-target"; + let key = "input.txt"; + let lambda_arn = "arn:rustfs:s3-object-lambda:us-east-1:transformer:webhook-preview"; + + let client = env.create_s3_client(); + client.create_bucket().bucket(bucket).send().await?; + client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from_static(b"hello object lambda")) + .send() + .await?; + + configure_webhook_target(&env, "transformer", &webhook_url, "secret-token").await?; + wait_for_target_visibility(&env, "transformer").await?; + + let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn)); + let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.text().await?, "transformed through object lambda"); + + let captured = timeout(Duration::from_secs(10), request_rx).await??; + assert_eq!(captured.payload["configuration"]["accessPointArn"].as_str(), Some(lambda_arn)); + + webhook_handle.await??; + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_get_object_lambda_invokes_runtime_webhook_target() -> Result<(), Box> { + init_logging(); + + let (webhook_url, request_rx, webhook_handle) = spawn_object_lambda_webhook_server().await?; + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "object-lambda-e2e"; + let key = "input.txt"; + let object_body = b"hello object lambda"; + let lambda_arn = "arn:rustfs:s3-object-lambda:us-east-1:transformer:webhook"; + + let client = env.create_s3_client(); + client.create_bucket().bucket(bucket).send().await?; + client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from_static(object_body)) + .send() + .await?; + + configure_webhook_target(&env, "transformer", &webhook_url, "secret-token").await?; + let (visible_targets, visible_arns) = wait_for_target_visibility(&env, "transformer").await?; + + let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn)); + let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?; + if response.status() != StatusCode::OK { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + let persisted_config = read_persisted_server_config(&env).await; + return Err(format!( + "object lambda request failed: {status} {body}; visible_targets={visible_targets}; visible_arns={visible_arns:?}; persisted_config={persisted_config}" + ) + .into()); + } + assert_eq!( + response.headers().get(CONTENT_TYPE).and_then(|value| value.to_str().ok()), + Some("text/plain") + ); + assert_eq!(response.text().await?, "transformed through object lambda"); + + let captured = timeout(Duration::from_secs(10), request_rx).await??; + assert_eq!(captured.headers.get("authorization").map(String::as_str), Some("Bearer secret-token")); + assert_eq!(captured.headers.get("x-rustfs-object-lambda-bucket").map(String::as_str), Some(bucket)); + assert_eq!(captured.headers.get("x-rustfs-object-lambda-key").map(String::as_str), Some(key)); + + assert_eq!(captured.payload["configuration"]["accessPointArn"].as_str(), Some(lambda_arn)); + let expected_request_url = format!("/{bucket}/{key}?lambdaArn={}", urlencoding::encode(lambda_arn)); + assert_eq!(captured.payload["userRequest"]["url"].as_str(), Some(expected_request_url.as_str())); + + let input_s3_url = captured.payload["getObjectContext"]["inputS3Url"] + .as_str() + .ok_or("missing inputS3Url in object lambda payload")?; + assert!(!input_s3_url.contains("lambdaArn=")); + + let source_response = local_http_client().get(input_s3_url).send().await?; + assert_eq!(source_response.status(), StatusCode::OK); + assert_eq!(source_response.bytes().await?.as_ref(), object_body); + + webhook_handle.await??; + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_get_object_lambda_passthroughs_non_success_webhook_response() -> Result<(), Box> { + init_logging(); + + let (webhook_url, _request_rx, webhook_handle) = spawn_object_lambda_webhook_server_with_response(WebhookResponseSpec { + status_line: "418 I'm a teapot".to_string(), + body: b"lambda upstream rejected".to_vec(), + headers: vec![ + ("content-type".to_string(), "text/plain".to_string()), + ("x-rustfs-debug".to_string(), "passthrough".to_string()), + ("x-amz-request-route".to_string(), "should-not-leak".to_string()), + ("x-amz-request-token".to_string(), "should-not-leak".to_string()), + ], + include_auth_headers: false, + auth_route_override: None, + auth_token_override: None, + }) + .await?; + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "object-lambda-e2e-failure"; + let key = "input.txt"; + let lambda_arn = "arn:rustfs:s3-object-lambda:us-east-1:transformer:webhook"; + + let client = env.create_s3_client(); + client.create_bucket().bucket(bucket).send().await?; + client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from_static(b"hello object lambda")) + .send() + .await?; + + configure_webhook_target(&env, "transformer", &webhook_url, "secret-token").await?; + wait_for_target_visibility(&env, "transformer").await?; + + let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn)); + let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?; + assert_eq!(response.status(), StatusCode::IM_A_TEAPOT); + assert_eq!( + response.headers().get("content-type").and_then(|value| value.to_str().ok()), + Some("text/plain") + ); + assert_eq!( + response.headers().get("x-rustfs-debug").and_then(|value| value.to_str().ok()), + Some("passthrough") + ); + assert!(response.headers().get("x-amz-request-route").is_none()); + assert!(response.headers().get("x-amz-request-token").is_none()); + assert_eq!(response.text().await?, "lambda upstream rejected"); + + webhook_handle.await??; + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_get_object_lambda_rejects_success_response_without_auth_headers() -> Result<(), Box> { + init_logging(); + + let (webhook_url, _request_rx, webhook_handle) = spawn_object_lambda_webhook_server_with_response(WebhookResponseSpec { + status_line: "200 OK".to_string(), + body: b"missing auth headers".to_vec(), + headers: vec![("content-type".to_string(), "text/plain".to_string())], + include_auth_headers: false, + auth_route_override: None, + auth_token_override: None, + }) + .await?; + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "object-lambda-e2e-missing-auth"; + let key = "input.txt"; + let lambda_arn = "arn:rustfs:s3-object-lambda:us-east-1:transformer:webhook"; + + let client = env.create_s3_client(); + client.create_bucket().bucket(bucket).send().await?; + client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from_static(b"hello object lambda")) + .send() + .await?; + + configure_webhook_target(&env, "transformer", &webhook_url, "secret-token").await?; + wait_for_target_visibility(&env, "transformer").await?; + + let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn)); + let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let body = response.text().await?; + assert!(body.contains("authorization headers"), "unexpected error body: {body}"); + + webhook_handle.await??; + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_get_object_lambda_rejects_success_response_with_mismatched_auth_headers() -> Result<(), Box> +{ + init_logging(); + + let (webhook_url, _request_rx, webhook_handle) = spawn_object_lambda_webhook_server_with_response(WebhookResponseSpec { + status_line: "200 OK".to_string(), + body: b"mismatched auth headers".to_vec(), + headers: vec![("content-type".to_string(), "text/plain".to_string())], + include_auth_headers: true, + auth_route_override: Some("wrong-route".to_string()), + auth_token_override: Some("wrong-token".to_string()), + }) + .await?; + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "object-lambda-e2e-mismatched-auth"; + let key = "input.txt"; + let lambda_arn = "arn:rustfs:s3-object-lambda:us-east-1:transformer:webhook"; + + let client = env.create_s3_client(); + client.create_bucket().bucket(bucket).send().await?; + client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from_static(b"hello object lambda")) + .send() + .await?; + + configure_webhook_target(&env, "transformer", &webhook_url, "secret-token").await?; + wait_for_target_visibility(&env, "transformer").await?; + + let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn)); + let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let body = response.text().await?; + assert!(body.contains("authorization headers"), "unexpected error body: {body}"); + + webhook_handle.await??; + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_get_object_lambda_rejects_unsupported_target_type() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "object-lambda-e2e-unsupported-target"; + let key = "input.txt"; + let lambda_arn = "arn:rustfs:s3-object-lambda:us-east-1:transformer:mqtt"; + + let client = env.create_s3_client(); + client.create_bucket().bucket(bucket).send().await?; + client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from_static(b"hello object lambda")) + .send() + .await?; + + let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn)); + let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?; + let status = response.status(); + let body = response.text().await?; + + assert_eq!(status, StatusCode::NOT_IMPLEMENTED); + assert!(body.contains("NotImplemented"), "unexpected error body: {body}"); + assert!( + body.to_ascii_lowercase().contains("target type is not supported"), + "unexpected error body: {body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_get_object_lambda_rejects_unconfigured_target() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "object-lambda-e2e-missing-target"; + let key = "input.txt"; + let lambda_arn = "arn:rustfs:s3-object-lambda:us-east-1:transformer:webhook"; + + let client = env.create_s3_client(); + client.create_bucket().bucket(bucket).send().await?; + client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from_static(b"hello object lambda")) + .send() + .await?; + + let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn)); + let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?; + let status = response.status(); + let body = response.text().await?; + + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!(body.contains("InvalidRequest"), "unexpected error body: {body}"); + assert!( + body.to_ascii_lowercase().contains("target is not configured"), + "unexpected error body: {body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "object-lambda-e2e-disabled-target"; + let key = "input.txt"; + let lambda_arn = "arn:rustfs:s3-object-lambda:us-east-1:transformer:webhook"; + + let client = env.create_s3_client(); + client.create_bucket().bucket(bucket).send().await?; + client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from_static(b"hello object lambda")) + .send() + .await?; + + configure_webhook_target_with_key_values( + &env, + "transformer", + vec![ + ("endpoint", "http://127.0.0.1:9/transform".to_string()), + ("auth_token", "secret-token".to_string()), + ("enable", "off".to_string()), + ], + ) + .await?; + wait_for_target_visibility(&env, "transformer").await?; + + let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn)); + let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?; + let status = response.status(); + let body = response.text().await?; + + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!(body.contains("InvalidRequest"), "unexpected error body: {body}"); + assert!(body.to_ascii_lowercase().contains("target is disabled"), "unexpected error body: {body}"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_configure_object_lambda_target_rejects_invalid_endpoint() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "object-lambda-e2e-invalid-endpoint"; + + let client = env.create_s3_client(); + client.create_bucket().bucket(bucket).send().await?; + client + .put_object() + .bucket(bucket) + .key("input.txt") + .body(ByteStream::from_static(b"hello object lambda")) + .send() + .await?; + + let response = send_configure_webhook_target_request( + &env, + "transformer", + vec![ + serde_json::json!({ "key": "endpoint", "value": "://invalid-endpoint" }), + serde_json::json!({ "key": "auth_token", "value": "secret-token" }), + serde_json::json!({ "key": "queue_dir", "value": format!("{}/notify-queue", env.temp_dir) }), + ], + ) + .await?; + let status = response.status(); + let body = response.text().await?; + + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!(body.contains("InvalidArgument"), "unexpected error body: {body}"); + assert!( + body.to_ascii_lowercase().contains("invalid endpoint url"), + "unexpected error body: {body}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_configure_object_lambda_notify_webhook_rejects_response_header_timeout_key() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let response = send_configure_webhook_target_request( + &env, + "transformer", + vec![ + serde_json::json!({ "key": "endpoint", "value": "http://127.0.0.1:9/transform" }), + serde_json::json!({ "key": "auth_token", "value": "secret-token" }), + serde_json::json!({ "key": "response_header_timeout", "value": "not-a-duration" }), + serde_json::json!({ "key": "queue_dir", "value": format!("{}/notify-queue", env.temp_dir) }), + ], + ) + .await?; + let status = response.status(); + let body = response.text().await?; + + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!(body.contains("InvalidArgument"), "unexpected error body: {body}"); + assert!( + body.to_ascii_lowercase().contains("response_header_timeout"), + "unexpected error body: {body}" + ); + assert!(body.to_ascii_lowercase().contains("not allowed"), "unexpected error body: {body}"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_listen_notification_emits_after_put_object() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "listen-notification-e2e"; + let key = "logs/app.json"; + let client = env.create_s3_client(); + + client.create_bucket().bucket(bucket).send().await?; + + let listen_url = format!( + "{}/{bucket}?events={}&prefix={}&suffix={}&ping=1", + env.url, + urlencoding::encode("s3:ObjectCreated:Put"), + urlencoding::encode("logs/"), + urlencoding::encode(".json"), + ); + let response = signed_request(http::Method::GET, &listen_url, &env.access_key, &env.secret_key, None, None).await?; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers().get("content-type").and_then(|value| value.to_str().ok()), + Some("text/event-stream") + ); + + let read_task = tokio::spawn(read_listen_notification_event(response, key)); + + client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from_static(b"listen notification body")) + .send() + .await?; + + let payload = timeout(Duration::from_secs(12), read_task).await???; + assert!(!payload.is_empty(), "listen_notification payload should not be empty"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_listen_notification_fans_in_remote_node_events() -> Result<(), Box> { + init_logging(); + + let mut cluster = RustFSTestClusterEnvironment::new(2).await?; + cluster.start().await?; + + let bucket = "listen-notification-cluster"; + let key = "logs/cluster.json"; + let node0_client = cluster.create_s3_client(0)?; + let node1_client = cluster.create_s3_client(1)?; + + node0_client.create_bucket().bucket(bucket).send().await?; + + let listen_url = format!( + "{}/{bucket}?events={}&prefix={}&suffix={}&ping=1", + cluster.nodes[0].url, + urlencoding::encode("s3:ObjectCreated:Put"), + urlencoding::encode("logs/"), + urlencoding::encode(".json"), + ); + let response = signed_request(http::Method::GET, &listen_url, &cluster.access_key, &cluster.secret_key, None, None).await?; + assert_eq!(response.status(), StatusCode::OK); + + let read_task = tokio::spawn(read_listen_notification_event(response, key)); + + node1_client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from_static(b"cluster listen notification body")) + .send() + .await?; + + let payload = timeout(Duration::from_secs(12), read_task).await???; + assert!(!payload.is_empty(), "listen_notification cluster payload should not be empty"); + + Ok(()) +} diff --git a/crates/e2e_test/src/protocols/ftps_core.rs b/crates/e2e_test/src/protocols/ftps_core.rs index 67fe350d1..5cb6661fc 100644 --- a/crates/e2e_test/src/protocols/ftps_core.rs +++ b/crates/e2e_test/src/protocols/ftps_core.rs @@ -14,11 +14,13 @@ //! Core FTPS tests -use crate::common::rustfs_binary_path; +use crate::common::rustfs_binary_path_with_features; use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment}; use anyhow::Result; use rcgen::generate_simple_self_signed; -use rustls::{ClientConfig, RootCertStore, pki_types::CertificateDer, pki_types::pem::PemObject}; +use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; +use rustls::pki_types::{CertificateDer, ServerName, UnixTime}; +use rustls::{ClientConfig, DigitallySignedStruct, Error as RustlsError, SignatureScheme}; use std::io::Cursor; use std::path::PathBuf; use std::sync::Arc; @@ -31,6 +33,46 @@ use tracing::info; const FTPS_PORT: u16 = 9021; const FTPS_ADDRESS: &str = "127.0.0.1:9021"; +#[derive(Debug)] +struct AcceptAnyServerCertVerifier; + +impl ServerCertVerifier for AcceptAnyServerCertVerifier { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp_response: &[u8], + _now: UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + _message: &[u8], + _cert: &CertificateDer<'_>, + _dss: &DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn verify_tls13_signature( + &self, + _message: &[u8], + _cert: &CertificateDer<'_>, + _dss: &DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn supported_verify_schemes(&self) -> Vec { + rustls::crypto::aws_lc_rs::default_provider() + .signature_verification_algorithms + .supported_schemes() + } +} + /// Test FTPS: put, ls, mkdir, rmdir, delete operations pub async fn test_ftps_core_operations() -> Result<()> { let env = ProtocolTestEnvironment::new().map_err(|e| anyhow::anyhow!("{}", e))?; @@ -58,7 +100,7 @@ pub async fn test_ftps_core_operations() -> Result<()> { // Start server manually info!("Starting FTPS server on {}", FTPS_ADDRESS); - let binary_path = rustfs_binary_path(); + let binary_path = rustfs_binary_path_with_features(Some("ftps,webdav")); let mut server_process = Command::new(&binary_path) .env("RUSTFS_FTPS_ENABLE", "true") .env("RUSTFS_FTPS_ADDRESS", FTPS_ADDRESS) @@ -78,19 +120,9 @@ pub async fn test_ftps_core_operations() -> Result<()> { .install_default() .map_err(|e| anyhow::anyhow!("Failed to install crypto provider: {:?}", e))?; - // Create a simple rustls config that accepts any certificate for testing - let mut root_store = RootCertStore::empty(); - // Add the self-signed certificate to the trust store for e2e - // Note: In a real environment, you'd use proper root certificates - let cert_pem = default_cert.cert.pem(); - let cert_der = CertificateDer::pem_reader_iter(&mut Cursor::new(cert_pem)) - .collect::, _>>() - .map_err(|e| anyhow::anyhow!("Failed to parse cert: {}", e))?; - - root_store.add_parsable_certificates(cert_der); - let config = ClientConfig::builder() - .with_root_certificates(root_store) + .dangerous() + .with_custom_certificate_verifier(Arc::new(AcceptAnyServerCertVerifier)) .with_no_client_auth(); // Wrap in suppaftp's RustlsConnector diff --git a/crates/e2e_test/src/protocols/webdav_core.rs b/crates/e2e_test/src/protocols/webdav_core.rs index 7b7471b21..db5e8506c 100644 --- a/crates/e2e_test/src/protocols/webdav_core.rs +++ b/crates/e2e_test/src/protocols/webdav_core.rs @@ -14,7 +14,7 @@ //! Core WebDAV tests -use crate::common::rustfs_binary_path; +use crate::common::rustfs_binary_path_with_features; use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment}; use anyhow::Result; use base64::Engine; @@ -47,7 +47,7 @@ pub async fn test_webdav_core_operations() -> Result<()> { // Start server manually info!("Starting WebDAV server on {}", WEBDAV_ADDRESS); - let binary_path = rustfs_binary_path(); + let binary_path = rustfs_binary_path_with_features(Some("ftps,webdav")); let mut server_process = Command::new(&binary_path) .env("RUSTFS_WEBDAV_ENABLE", "true") .env("RUSTFS_WEBDAV_ADDRESS", WEBDAV_ADDRESS) diff --git a/crates/e2e_test/src/quota_test.rs b/crates/e2e_test/src/quota_test.rs index 000c16002..8f3cb0018 100644 --- a/crates/e2e_test/src/quota_test.rs +++ b/crates/e2e_test/src/quota_test.rs @@ -17,6 +17,15 @@ use aws_sdk_s3::Client; use serial_test::serial; use tracing::{debug, info}; +fn skip_without_awscurl() -> bool { + if crate::common::awscurl_available() { + return false; + } + + info!("Skipping quota test because awscurl is not available"); + true +} + /// Test environment setup for quota tests pub struct QuotaTestEnv { pub env: RustFSTestEnvironment, @@ -233,6 +242,9 @@ mod integration_tests { #[serial] async fn test_quota_basic_operations() -> Result<(), Box> { init_logging(); + if skip_without_awscurl() { + return Ok(()); + } let env = QuotaTestEnv::new().await?; // Create test bucket @@ -269,6 +281,9 @@ mod integration_tests { #[serial] async fn test_quota_update_and_clear() -> Result<(), Box> { init_logging(); + if skip_without_awscurl() { + return Ok(()); + } let env = QuotaTestEnv::new().await?; env.create_bucket().await?; @@ -302,6 +317,9 @@ mod integration_tests { #[serial] async fn test_quota_delete_operations() -> Result<(), Box> { init_logging(); + if skip_without_awscurl() { + return Ok(()); + } let env = QuotaTestEnv::new().await?; env.create_bucket().await?; @@ -336,6 +354,9 @@ mod integration_tests { #[serial] async fn test_quota_usage_tracking() -> Result<(), Box> { init_logging(); + if skip_without_awscurl() { + return Ok(()); + } let env = QuotaTestEnv::new().await?; env.create_bucket().await?; @@ -372,6 +393,9 @@ mod integration_tests { #[serial] async fn test_quota_statistics() -> Result<(), Box> { init_logging(); + if skip_without_awscurl() { + return Ok(()); + } let env = QuotaTestEnv::new().await?; env.create_bucket().await?; @@ -403,6 +427,9 @@ mod integration_tests { #[serial] async fn test_quota_check_api() -> Result<(), Box> { init_logging(); + if skip_without_awscurl() { + return Ok(()); + } let env = QuotaTestEnv::new().await?; env.create_bucket().await?; @@ -441,6 +468,9 @@ mod integration_tests { #[serial] async fn test_quota_multiple_buckets() -> Result<(), Box> { init_logging(); + if skip_without_awscurl() { + return Ok(()); + } let env = QuotaTestEnv::new().await?; // Create two buckets in the same environment @@ -479,6 +509,9 @@ mod integration_tests { #[serial] async fn test_quota_error_handling() -> Result<(), Box> { init_logging(); + if skip_without_awscurl() { + return Ok(()); + } let env = QuotaTestEnv::new().await?; env.create_bucket().await?; @@ -512,6 +545,9 @@ mod integration_tests { #[serial] async fn test_quota_http_endpoints() -> Result<(), Box> { init_logging(); + if skip_without_awscurl() { + return Ok(()); + } let env = QuotaTestEnv::new().await?; env.create_bucket().await?; @@ -575,6 +611,9 @@ mod integration_tests { #[serial] async fn test_quota_normal_user_permissions() -> Result<(), Box> { init_logging(); + if skip_without_awscurl() { + return Ok(()); + } let env = QuotaTestEnv::new().await?; env.create_bucket().await?; @@ -628,6 +667,9 @@ mod integration_tests { #[serial] async fn test_quota_copy_operations() -> Result<(), Box> { init_logging(); + if skip_without_awscurl() { + return Ok(()); + } let env = QuotaTestEnv::new().await?; env.create_bucket().await?; @@ -671,6 +713,9 @@ mod integration_tests { #[serial] async fn test_quota_batch_delete() -> Result<(), Box> { init_logging(); + if skip_without_awscurl() { + return Ok(()); + } let env = QuotaTestEnv::new().await?; env.create_bucket().await?; @@ -727,6 +772,9 @@ mod integration_tests { #[serial] async fn test_quota_multipart_upload() -> Result<(), Box> { init_logging(); + if skip_without_awscurl() { + return Ok(()); + } let env = QuotaTestEnv::new().await?; env.create_bucket().await?; diff --git a/crates/e2e_test/src/reliant/grpc_lock_server.rs b/crates/e2e_test/src/reliant/grpc_lock_server.rs index c199a945d..c1a927124 100644 --- a/crates/e2e_test/src/reliant/grpc_lock_server.rs +++ b/crates/e2e_test/src/reliant/grpc_lock_server.rs @@ -551,6 +551,13 @@ impl NodeService for MinimalLockNodeService { Err(Status::unimplemented("lock-only test server")) } + async fn get_live_events( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("lock-only test server")) + } + async fn start_profiling( &self, _request: Request, diff --git a/crates/e2e_test/src/replication_extension_test.rs b/crates/e2e_test/src/replication_extension_test.rs new file mode 100644 index 000000000..3fb31735f --- /dev/null +++ b/crates/e2e_test/src/replication_extension_test.rs @@ -0,0 +1,802 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::common::{RustFSTestEnvironment, init_logging, local_http_client}; +use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration}; +use http::header::{CONTENT_TYPE, HOST}; +use reqwest::StatusCode; +use rustfs_signer::constants::UNSIGNED_PAYLOAD; +use rustfs_signer::sign_v4; +use s3s::Body; +use serial_test::serial; +use std::error::Error; + +async fn signed_request( + method: http::Method, + url: &str, + access_key: &str, + secret_key: &str, + body: Option>, + content_type: Option<&str>, +) -> Result> { + let uri = url.parse::()?; + let authority = uri.authority().ok_or("request URL missing authority")?.to_string(); + let mut request = http::Request::builder().method(method.clone()).uri(uri); + request = request.header(HOST, authority); + request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD); + if let Some(content_type) = content_type { + request = request.header(CONTENT_TYPE, content_type); + } + + let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default(); + let signed = sign_v4(request.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1"); + + let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?; + let client = local_http_client(); + let mut request_builder = client.request(reqwest_method, url); + for (name, value) in signed.headers() { + request_builder = request_builder.header(name, value); + } + if let Some(body) = body { + request_builder = request_builder.body(body); + } + + Ok(request_builder.send().await?) +} + +async fn set_replication_target( + source_env: &RustFSTestEnvironment, + source_bucket: &str, + target_env: &RustFSTestEnvironment, + target_bucket: &str, +) -> Result> { + let body = serde_json::json!({ + "endpoint": target_env.address, + "credentials": { + "accessKey": target_env.access_key, + "secretKey": target_env.secret_key + }, + "targetbucket": target_bucket, + "secure": false, + "type": "replication" + }); + let url = format!( + "{}/rustfs/admin/v3/set-remote-target?bucket={}", + source_env.url, + urlencoding::encode(source_bucket) + ); + let response = signed_request( + http::Method::PUT, + &url, + &source_env.access_key, + &source_env.secret_key, + Some(body.to_string().into_bytes()), + Some("application/json"), + ) + .await?; + + if response.status() != StatusCode::OK { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(format!("set remote target failed: {status} {body}").into()); + } + + let body = response.bytes().await?; + let arn: String = serde_json::from_slice(&body)?; + Ok(arn) +} + +async fn send_set_replication_target_request( + source_env: &RustFSTestEnvironment, + source_bucket: &str, + update: bool, + body: serde_json::Value, +) -> Result> { + let mut url = format!( + "{}/rustfs/admin/v3/set-remote-target?bucket={}", + source_env.url, + urlencoding::encode(source_bucket) + ); + if update { + url.push_str("&update=true"); + } + signed_request( + http::Method::PUT, + &url, + &source_env.access_key, + &source_env.secret_key, + Some(body.to_string().into_bytes()), + Some("application/json"), + ) + .await +} + +async fn put_bucket_replication( + env: &RustFSTestEnvironment, + bucket: &str, + target_arn: &str, +) -> Result<(), Box> { + let body = format!( + r#" + + + rule-1 + 1 + Enabled + + Enabled + + + Enabled + + + {target_arn} + + +"# + ); + let url = format!("{}/{bucket}?replication", env.url); + let response = signed_request( + http::Method::PUT, + &url, + &env.access_key, + &env.secret_key, + Some(body.into_bytes()), + Some("application/xml"), + ) + .await?; + + if response.status() != StatusCode::OK { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(format!("put bucket replication failed: {status} {body}").into()); + } + + Ok(()) +} + +async fn enable_bucket_versioning(env: &RustFSTestEnvironment, bucket: &str) -> Result<(), Box> { + let client = env.create_s3_client(); + client + .put_bucket_versioning() + .bucket(bucket) + .versioning_configuration( + VersioningConfiguration::builder() + .status(BucketVersioningStatus::Enabled) + .build(), + ) + .send() + .await?; + Ok(()) +} + +async fn run_replication_check( + env: &RustFSTestEnvironment, + bucket: &str, +) -> Result> { + let url = format!("{}/{bucket}?replication-check", env.url); + signed_request(http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await +} + +async fn remove_replication_target( + env: &RustFSTestEnvironment, + bucket: &str, + arn: &str, +) -> Result> { + let url = format!( + "{}/rustfs/admin/v3/remove-remote-target?bucket={}&arn={}", + env.url, + urlencoding::encode(bucket), + urlencoding::encode(arn) + ); + signed_request(http::Method::DELETE, &url, &env.access_key, &env.secret_key, None, None).await +} + +async fn remove_replication_target_request( + env: &RustFSTestEnvironment, + bucket: Option<&str>, + arn: Option<&str>, +) -> Result> { + let mut url = format!("{}/rustfs/admin/v3/remove-remote-target", env.url); + let mut separator = '?'; + + if let Some(bucket) = bucket { + url.push(separator); + separator = '&'; + url.push_str("bucket="); + url.push_str(&urlencoding::encode(bucket)); + } + + if let Some(arn) = arn { + url.push(separator); + url.push_str("arn="); + url.push_str(&urlencoding::encode(arn)); + } + + signed_request(http::Method::DELETE, &url, &env.access_key, &env.secret_key, None, None).await +} + +async fn list_replication_targets_request( + env: &RustFSTestEnvironment, + bucket: Option<&str>, +) -> Result> { + let mut url = format!("{}/rustfs/admin/v3/list-remote-targets", env.url); + if let Some(bucket) = bucket { + url.push_str("?bucket="); + url.push_str(&urlencoding::encode(bucket)); + } + signed_request(http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await +} + +async fn build_replication_pair( + enable_target_versioning: bool, +) -> Result<(RustFSTestEnvironment, RustFSTestEnvironment, String), Box> { + let mut source_env = RustFSTestEnvironment::new().await?; + source_env.start_rustfs_server(vec![]).await?; + + let mut target_env = RustFSTestEnvironment::new().await?; + target_env.start_rustfs_server_without_cleanup(vec![]).await?; + + let source_bucket = "replication-check-src"; + let target_bucket = "replication-check-dst"; + + let source_client = source_env.create_s3_client(); + let target_client = target_env.create_s3_client(); + + source_client.create_bucket().bucket(source_bucket).send().await?; + target_client.create_bucket().bucket(target_bucket).send().await?; + + enable_bucket_versioning(&source_env, source_bucket).await?; + if enable_target_versioning { + enable_bucket_versioning(&target_env, target_bucket).await?; + } + + let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?; + put_bucket_replication(&source_env, source_bucket, &target_arn).await?; + + Ok((source_env, target_env, source_bucket.to_string())) +} + +#[tokio::test] +#[serial] +async fn test_replication_check_succeeds_with_remote_target() -> Result<(), Box> { + init_logging(); + + let (_source_env, _target_env, source_bucket) = build_replication_pair(true).await?; + let response = run_replication_check(&_source_env, &source_bucket).await?; + + assert_eq!(response.status(), StatusCode::OK); + assert!(response.text().await?.is_empty()); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_replication_check_rejects_target_without_object_lock() -> Result<(), Box> { + init_logging(); + + let mut source_env = RustFSTestEnvironment::new().await?; + source_env.start_rustfs_server(vec![]).await?; + + let mut target_env = RustFSTestEnvironment::new().await?; + target_env.start_rustfs_server_without_cleanup(vec![]).await?; + + let source_bucket = "replication-check-lock-src"; + let target_bucket = "replication-check-lock-dst"; + + let source_client = source_env.create_s3_client(); + let target_client = target_env.create_s3_client(); + + source_client + .create_bucket() + .bucket(source_bucket) + .object_lock_enabled_for_bucket(true) + .send() + .await?; + target_client.create_bucket().bucket(target_bucket).send().await?; + + enable_bucket_versioning(&source_env, source_bucket).await?; + enable_bucket_versioning(&target_env, target_bucket).await?; + + let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?; + put_bucket_replication(&source_env, source_bucket, &target_arn).await?; + + let response = run_replication_check(&source_env, source_bucket).await?; + let status = response.status(); + let body = response.text().await?; + + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!(body.contains("InvalidRequest"), "unexpected response: {body}"); + assert!(body.to_ascii_lowercase().contains("object lock"), "unexpected response: {body}"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_set_remote_target_rejects_unversioned_source_bucket() -> Result<(), Box> { + init_logging(); + + let mut source_env = RustFSTestEnvironment::new().await?; + source_env.start_rustfs_server(vec![]).await?; + + let mut target_env = RustFSTestEnvironment::new().await?; + target_env.start_rustfs_server_without_cleanup(vec![]).await?; + + let source_bucket = "replication-check-unversioned-src"; + let target_bucket = "replication-check-unversioned-dst"; + + let source_client = source_env.create_s3_client(); + let target_client = target_env.create_s3_client(); + + source_client.create_bucket().bucket(source_bucket).send().await?; + target_client.create_bucket().bucket(target_bucket).send().await?; + + enable_bucket_versioning(&target_env, target_bucket).await?; + + let err = set_replication_target(&source_env, source_bucket, &target_env, target_bucket) + .await + .expect_err("unversioned source bucket should be rejected during remote target setup"); + let err = err.to_string(); + + assert!(err.contains("400 Bad Request"), "unexpected set remote target error: {err}"); + assert!(err.contains("InvalidRequest"), "unexpected set remote target error: {err}"); + assert!( + err.to_ascii_lowercase().contains("not versioned"), + "unexpected set remote target error: {err}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_replication_check_rejects_unversioned_source_bucket() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "replication-check-source-unversioned"; + let client = env.create_s3_client(); + client.create_bucket().bucket(bucket).send().await?; + + let response = run_replication_check(&env, bucket).await?; + let status = response.status(); + let body = response.text().await?; + + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!(body.contains("InvalidRequest"), "unexpected response: {body}"); + assert!(body.to_ascii_lowercase().contains("versioning"), "unexpected response: {body}"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_replication_check_rejects_missing_replication_config() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "replication-check-missing-config"; + let client = env.create_s3_client(); + client.create_bucket().bucket(bucket).send().await?; + enable_bucket_versioning(&env, bucket).await?; + + let response = run_replication_check(&env, bucket).await?; + let status = response.status(); + let body = response.text().await?; + + assert_eq!(status, StatusCode::NOT_FOUND); + assert!(body.contains("ReplicationConfigurationNotFoundError"), "unexpected response: {body}"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_replication_check_rejects_invalid_bucket() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let response = run_replication_check(&env, "replication-check-no-such-bucket").await?; + let status = response.status(); + let body = response.text().await?; + + assert_eq!(status, StatusCode::NOT_FOUND); + assert!(body.contains("NoSuchBucket"), "unexpected response: {body}"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_set_remote_target_rejects_same_bucket_on_same_deployment() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "replication-check-same-target"; + let client = env.create_s3_client(); + client.create_bucket().bucket(bucket).send().await?; + enable_bucket_versioning(&env, bucket).await?; + + let body = serde_json::json!({ + "endpoint": env.address, + "credentials": { + "accessKey": env.access_key, + "secretKey": env.secret_key + }, + "targetbucket": bucket, + "secure": false, + "type": "replication" + }); + let url = format!("{}/rustfs/admin/v3/set-remote-target?bucket={}", env.url, urlencoding::encode(bucket)); + let response = signed_request( + http::Method::PUT, + &url, + &env.access_key, + &env.secret_key, + Some(body.to_string().into_bytes()), + Some("application/json"), + ) + .await?; + + let status = response.status(); + let body = response.text().await?; + + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!(body.contains("IncorrectEndpoint"), "unexpected response: {body}"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_set_remote_target_rejects_unversioned_target_bucket() -> Result<(), Box> { + init_logging(); + + let mut source_env = RustFSTestEnvironment::new().await?; + source_env.start_rustfs_server(vec![]).await?; + + let mut target_env = RustFSTestEnvironment::new().await?; + target_env.start_rustfs_server_without_cleanup(vec![]).await?; + + let source_bucket = "replication-check-src"; + let target_bucket = "replication-check-dst"; + + let source_client = source_env.create_s3_client(); + let target_client = target_env.create_s3_client(); + + source_client.create_bucket().bucket(source_bucket).send().await?; + target_client.create_bucket().bucket(target_bucket).send().await?; + enable_bucket_versioning(&source_env, source_bucket).await?; + + let err = set_replication_target(&source_env, source_bucket, &target_env, target_bucket) + .await + .expect_err("unversioned target bucket should be rejected during remote target setup"); + assert!(err.to_string().contains("not versioned"), "unexpected set remote target error: {err}"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_set_remote_target_update_requires_arn() -> Result<(), Box> { + init_logging(); + + let mut source_env = RustFSTestEnvironment::new().await?; + source_env.start_rustfs_server(vec![]).await?; + + let mut target_env = RustFSTestEnvironment::new().await?; + target_env.start_rustfs_server_without_cleanup(vec![]).await?; + + let source_bucket = "replication-update-needs-arn-src"; + let target_bucket = "replication-update-needs-arn-dst"; + + let source_client = source_env.create_s3_client(); + let target_client = target_env.create_s3_client(); + + source_client.create_bucket().bucket(source_bucket).send().await?; + target_client.create_bucket().bucket(target_bucket).send().await?; + + enable_bucket_versioning(&source_env, source_bucket).await?; + enable_bucket_versioning(&target_env, target_bucket).await?; + + let response = send_set_replication_target_request( + &source_env, + source_bucket, + true, + serde_json::json!({ + "endpoint": target_env.address, + "credentials": { + "accessKey": target_env.access_key, + "secretKey": target_env.secret_key + }, + "targetbucket": target_bucket, + "secure": false, + "type": "replication" + }), + ) + .await?; + + let status = response.status(); + let body = response.text().await?; + + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!(body.contains("InvalidRequest"), "unexpected response: {body}"); + assert!(body.to_ascii_lowercase().contains("arn is empty"), "unexpected response: {body}"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_set_remote_target_update_rejects_missing_target() -> Result<(), Box> { + init_logging(); + + let mut source_env = RustFSTestEnvironment::new().await?; + source_env.start_rustfs_server(vec![]).await?; + + let mut target_env = RustFSTestEnvironment::new().await?; + target_env.start_rustfs_server_without_cleanup(vec![]).await?; + + let source_bucket = "replication-update-missing-target-src"; + let target_bucket = "replication-update-missing-target-dst"; + + let source_client = source_env.create_s3_client(); + let target_client = target_env.create_s3_client(); + + source_client.create_bucket().bucket(source_bucket).send().await?; + target_client.create_bucket().bucket(target_bucket).send().await?; + + enable_bucket_versioning(&source_env, source_bucket).await?; + enable_bucket_versioning(&target_env, target_bucket).await?; + + let response = send_set_replication_target_request( + &source_env, + source_bucket, + true, + serde_json::json!({ + "endpoint": target_env.address, + "credentials": { + "accessKey": target_env.access_key, + "secretKey": target_env.secret_key + }, + "targetbucket": target_bucket, + "secure": false, + "type": "replication", + "arn": "arn:aws:s3:us-east-1:123456789012:replication::missing-target" + }), + ) + .await?; + + let status = response.status(); + let body = response.text().await?; + + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!(body.contains("InvalidRequest"), "unexpected response: {body}"); + assert!(body.to_ascii_lowercase().contains("target not found"), "unexpected response: {body}"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_set_remote_target_rejects_invalid_target_url() -> Result<(), Box> { + init_logging(); + + let mut source_env = RustFSTestEnvironment::new().await?; + source_env.start_rustfs_server(vec![]).await?; + + let bucket = "replication-invalid-target-url-src"; + let source_client = source_env.create_s3_client(); + source_client.create_bucket().bucket(bucket).send().await?; + enable_bucket_versioning(&source_env, bucket).await?; + + let response = send_set_replication_target_request( + &source_env, + bucket, + false, + serde_json::json!({ + "endpoint": "://invalid-target-url", + "credentials": { + "accessKey": "replication", + "secretKey": "replication" + }, + "targetbucket": "target-bucket", + "secure": false, + "type": "replication" + }), + ) + .await?; + + let status = response.status(); + let body = response.text().await?; + + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!(body.contains("InvalidRequest"), "unexpected response: {body}"); + assert!(body.to_ascii_lowercase().contains("invalid target url"), "unexpected response: {body}"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_list_remote_targets_rejects_empty_bucket() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let response = list_replication_targets_request(&env, Some("")).await?; + let status = response.status(); + let body = response.text().await?; + + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!(body.contains("InvalidRequest"), "unexpected response: {body}"); + assert!(body.to_ascii_lowercase().contains("bucket is required"), "unexpected response: {body}"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_list_remote_targets_rejects_invalid_bucket() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let response = list_replication_targets_request(&env, Some("missing-replication-target-bucket")).await?; + let status = response.status(); + let body = response.text().await?; + + assert_eq!(status, StatusCode::NOT_FOUND); + assert!(body.contains("NoSuchBucket"), "unexpected response: {body}"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_remove_remote_target_rejects_missing_target() -> Result<(), Box> { + init_logging(); + + let mut source_env = RustFSTestEnvironment::new().await?; + source_env.start_rustfs_server(vec![]).await?; + + let mut target_env = RustFSTestEnvironment::new().await?; + target_env.start_rustfs_server_without_cleanup(vec![]).await?; + + let bucket = "replication-remove-missing-target"; + let target_bucket = "replication-remove-missing-target-dst"; + + let source_client = source_env.create_s3_client(); + let target_client = target_env.create_s3_client(); + + source_client.create_bucket().bucket(bucket).send().await?; + target_client.create_bucket().bucket(target_bucket).send().await?; + + enable_bucket_versioning(&source_env, bucket).await?; + enable_bucket_versioning(&target_env, target_bucket).await?; + + let arn = set_replication_target(&source_env, bucket, &target_env, target_bucket).await?; + + let first_remove = remove_replication_target(&source_env, bucket, &arn).await?; + assert_eq!(first_remove.status(), StatusCode::NO_CONTENT); + + let response = remove_replication_target(&source_env, bucket, &arn).await?; + let status = response.status(); + let body = response.text().await?; + + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!(body.contains("InvalidRequest"), "unexpected response: {body}"); + assert!(body.to_ascii_lowercase().contains("not found"), "unexpected response: {body}"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_remove_remote_target_rejects_missing_arn() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "replication-remove-missing-arn"; + let client = env.create_s3_client(); + client.create_bucket().bucket(bucket).send().await?; + enable_bucket_versioning(&env, bucket).await?; + + let response = remove_replication_target_request(&env, Some(bucket), None).await?; + let status = response.status(); + let body = response.text().await?; + + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!(body.contains("InvalidRequest"), "unexpected response: {body}"); + assert!(body.to_ascii_lowercase().contains("arn is required"), "unexpected response: {body}"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_remove_remote_target_rejects_invalid_bucket() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let response = remove_replication_target_request( + &env, + Some("missing-replication-remove-bucket"), + Some("arn:aws:s3:us-east-1:123456789012:replication::missing"), + ) + .await?; + let status = response.status(); + let body = response.text().await?; + + assert_eq!(status, StatusCode::NOT_FOUND); + assert!(body.contains("NoSuchBucket"), "unexpected response: {body}"); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_remove_remote_target_rejects_target_used_by_replication() -> Result<(), Box> { + init_logging(); + + let (source_env, _target_env, source_bucket) = build_replication_pair(true).await?; + let targets_url = format!( + "{}/rustfs/admin/v3/list-remote-targets?bucket={}", + source_env.url, + urlencoding::encode(&source_bucket) + ); + let targets_response = signed_request( + http::Method::GET, + &targets_url, + &source_env.access_key, + &source_env.secret_key, + None, + None, + ) + .await?; + assert_eq!(targets_response.status(), StatusCode::OK); + let targets: Vec = targets_response.json().await?; + let arn = targets + .first() + .and_then(|target| target.get("arn")) + .and_then(|arn| arn.as_str()) + .ok_or("replication target arn missing")? + .to_string(); + + let response = remove_replication_target(&source_env, &source_bucket, &arn).await?; + let status = response.status(); + let body = response.text().await?; + + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!(body.contains("InvalidRequest"), "unexpected response: {body}"); + assert!(body.to_ascii_lowercase().contains("removal disallowed"), "unexpected response: {body}"); + + Ok(()) +} diff --git a/crates/e2e_test/src/version_id_regression_test.rs b/crates/e2e_test/src/version_id_regression_test.rs index 5833a84ce..1ff2e34ef 100644 --- a/crates/e2e_test/src/version_id_regression_test.rs +++ b/crates/e2e_test/src/version_id_regression_test.rs @@ -454,12 +454,12 @@ mod tests { Ok(()) } - /// Test 7: PutObject should return "null" version_id when versioning is Suspended + /// Test 7: PutObject should omit version_id when versioning is Suspended #[tokio::test] #[serial] - async fn test_put_object_returns_null_version_id_with_suspended_versioning() { + async fn test_put_object_omits_version_id_with_suspended_versioning() { init_logging(); - info!("🧪 TEST: PutObject returns null version_id with versioning suspended"); + info!("🧪 TEST: PutObject omits version_id with versioning suspended"); let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); @@ -489,13 +489,130 @@ mod tests { info!("📥 PutObject response - version_id: {:?}", output.version_id); - // When suspended, version_id must be "null" + // When suspended, version_id must be omitted assert_eq!( - output.version_id.as_deref(), - Some("null"), - "❌ FAILED: version_id should be 'null' when versioning is suspended" + output.version_id, None, + "❌ FAILED: version_id should be omitted when versioning is suspended" ); - info!("✅ PASSED: PutObject correctly returns 'null' version_id"); + info!("✅ PASSED: PutObject correctly omits version_id"); + } + + /// Test 8: CopyObject should omit version_id when versioning is Suspended + #[tokio::test] + #[serial] + async fn test_copy_object_omits_version_id_with_suspended_versioning() { + init_logging(); + info!("🧪 TEST: CopyObject omits version_id with versioning suspended"); + + let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); + env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); + + let client = create_s3_client(&env); + let bucket = "test-copy-suspended-version-id"; + + create_bucket(&client, bucket).await.expect("Failed to create bucket"); + suspend_versioning(&client, bucket) + .await + .expect("Failed to suspend versioning"); + + let source_key = "source-file.txt"; + let dest_key = "dest-file.txt"; + let content = b"Content to copy into suspended bucket"; + + client + .put_object() + .bucket(bucket) + .key(source_key) + .body(ByteStream::from_static(content)) + .send() + .await + .expect("Failed to create source object"); + + let result = client + .copy_object() + .bucket(bucket) + .key(dest_key) + .copy_source(format!("{}/{}", bucket, source_key)) + .send() + .await; + + assert!(result.is_ok(), "CopyObject failed: {:?}", result.err()); + let output = result.unwrap(); + + info!("📥 CopyObject response - version_id: {:?}", output.version_id); + assert_eq!( + output.version_id, None, + "❌ FAILED: version_id should be omitted when versioning is suspended" + ); + + info!("✅ PASSED: CopyObject correctly omits version_id"); + } + + /// Test 9: CompleteMultipartUpload should omit version_id when versioning is Suspended + #[tokio::test] + #[serial] + async fn test_multipart_upload_omits_version_id_with_suspended_versioning() { + init_logging(); + info!("🧪 TEST: CompleteMultipartUpload omits version_id with versioning suspended"); + + let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); + env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); + + let client = create_s3_client(&env); + let bucket = "test-multipart-suspended-version-id"; + + create_bucket(&client, bucket).await.expect("Failed to create bucket"); + suspend_versioning(&client, bucket) + .await + .expect("Failed to suspend versioning"); + + let key = "multipart-file.txt"; + let content = b"Part 1 content for suspended multipart upload test"; + + let create_result = client + .create_multipart_upload() + .bucket(bucket) + .key(key) + .send() + .await + .expect("Failed to create multipart upload"); + + let upload_id = create_result.upload_id().expect("No upload_id returned"); + + let upload_part_result = client + .upload_part() + .bucket(bucket) + .key(key) + .upload_id(upload_id) + .part_number(1) + .body(ByteStream::from_static(content)) + .send() + .await + .expect("Failed to upload part"); + + let etag = upload_part_result.e_tag().expect("No etag returned").to_string(); + let completed_part = CompletedPart::builder().part_number(1).e_tag(etag).build(); + let completed_upload = CompletedMultipartUpload::builder().parts(completed_part).build(); + + let result = client + .complete_multipart_upload() + .bucket(bucket) + .key(key) + .upload_id(upload_id) + .multipart_upload(completed_upload) + .send() + .await; + + assert!(result.is_ok(), "CompleteMultipartUpload failed: {:?}", result.err()); + let output = result.unwrap(); + + info!("📥 CompleteMultipartUpload response - version_id: {:?}", output.version_id); + assert_eq!( + output.version_id, None, + "❌ FAILED: version_id should be omitted when versioning is suspended" + ); + + info!("✅ PASSED: CompleteMultipartUpload correctly omits version_id"); } } diff --git a/crates/ecstore/src/bucket/bucket_target_sys.rs b/crates/ecstore/src/bucket/bucket_target_sys.rs index a29208556..2895e32fd 100644 --- a/crates/ecstore/src/bucket/bucket_target_sys.rs +++ b/crates/ecstore/src/bucket/bucket_target_sys.rs @@ -1134,11 +1134,20 @@ pub struct S3ClientError { } impl S3ClientError { pub fn new(value: impl Into) -> Self { + Self::with_metadata(value, None, None, None) + } + + pub fn with_metadata( + error: impl Into, + status_code: Option, + code: Option, + message: Option, + ) -> Self { S3ClientError { - error: value.into(), - status_code: None, - code: None, - message: None, + error: error.into(), + status_code, + code, + message, } } @@ -1154,16 +1163,16 @@ impl S3ClientError { impl From for S3ClientError { fn from(value: T) -> Self { - S3ClientError { - error: format!( - "{}: {}", - value.code().map(String::from).unwrap_or("unknown code".into()), - value.message().map(String::from).unwrap_or("missing reason".into()), - ), - status_code: None, - code: None, - message: None, - } + let code = value.code().map(String::from); + let message = value.message().map(String::from); + let error = match (code.as_deref(), message.as_deref()) { + (Some(code), Some(message)) => format!("{code}: {message}"), + (Some(code), None) => code.to_string(), + (None, Some(message)) => message.to_string(), + (None, None) => "unknown remote error".to_string(), + }; + + S3ClientError::with_metadata(error, None, code, message) } } @@ -1207,10 +1216,15 @@ impl TargetClient { other ); let message = other.meta().meta(); - Err(S3ClientError::new(format!( - "failed to check bucket exists for bucket:{bucket} please check the bucket name and credentials, error:{:?}", - message - ))) + Err(S3ClientError::with_metadata( + format!( + "failed to check bucket exists for bucket:{bucket} please check the bucket name and credentials, error:{:?}", + message + ), + None, + message.code().map(ToOwned::to_owned), + message.message().map(ToOwned::to_owned), + )) } }, SdkError::DispatchFailure(e) => Err(S3ClientError::new(format!( diff --git a/crates/ecstore/src/bucket/lifecycle/lifecycle.rs b/crates/ecstore/src/bucket/lifecycle/lifecycle.rs index 68d705ca7..7832d6893 100644 --- a/crates/ecstore/src/bucket/lifecycle/lifecycle.rs +++ b/crates/ecstore/src/bucket/lifecycle/lifecycle.rs @@ -1337,6 +1337,7 @@ mod tests { assert_eq!(event.action, IlmAction::TransitionAction); assert_eq!(event.rule_id, "transition-date"); assert_eq!(event.storage_class, "WARM"); + assert_eq!(event.due, Some(transition_date)); } #[tokio::test] diff --git a/crates/ecstore/src/bucket/metadata.rs b/crates/ecstore/src/bucket/metadata.rs index b07c871eb..d4aafc053 100644 --- a/crates/ecstore/src/bucket/metadata.rs +++ b/crates/ecstore/src/bucket/metadata.rs @@ -25,9 +25,9 @@ use crate::store::ECStore; use byteorder::{BigEndian, ByteOrder, LittleEndian}; use rustfs_policy::policy::BucketPolicy; use s3s::dto::{ - BucketLifecycleConfiguration, CORSConfiguration, NotificationConfiguration, ObjectLockConfiguration, - PublicAccessBlockConfiguration, ReplicationConfiguration, ServerSideEncryptionConfiguration, Tagging, - VersioningConfiguration, + AccelerateConfiguration, BucketLifecycleConfiguration, BucketLoggingStatus, CORSConfiguration, NotificationConfiguration, + ObjectLockConfiguration, PublicAccessBlockConfiguration, ReplicationConfiguration, RequestPaymentConfiguration, + ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration, WebsiteConfiguration, }; use serde::Serializer; use std::collections::HashMap; @@ -238,6 +238,10 @@ pub const BUCKET_VERSIONING_CONFIG: &str = "versioning.xml"; pub const BUCKET_REPLICATION_CONFIG: &str = "replication.xml"; pub const BUCKET_TARGETS_FILE: &str = "bucket-targets.json"; pub const BUCKET_CORS_CONFIG: &str = "cors.xml"; +pub const BUCKET_LOGGING_CONFIG: &str = "logging.xml"; +pub const BUCKET_WEBSITE_CONFIG: &str = "website.xml"; +pub const BUCKET_ACCELERATE_CONFIG: &str = "accelerate.xml"; +pub const BUCKET_REQUEST_PAYMENT_CONFIG: &str = "request-payment.xml"; pub const BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG: &str = "public-access-block.xml"; pub const BUCKET_ACL_CONFIG: &str = "bucket-acl.json"; @@ -258,6 +262,10 @@ pub struct BucketMetadata { pub bucket_targets_config_json: Vec, pub bucket_targets_config_meta_json: Vec, pub cors_config_xml: Vec, + pub logging_config_xml: Vec, + pub website_config_xml: Vec, + pub accelerate_config_xml: Vec, + pub request_payment_config_xml: Vec, pub public_access_block_config_xml: Vec, pub bucket_acl_config_json: Vec, @@ -273,6 +281,10 @@ pub struct BucketMetadata { pub bucket_targets_config_updated_at: OffsetDateTime, pub bucket_targets_config_meta_updated_at: OffsetDateTime, pub cors_config_updated_at: OffsetDateTime, + pub logging_config_updated_at: OffsetDateTime, + pub website_config_updated_at: OffsetDateTime, + pub accelerate_config_updated_at: OffsetDateTime, + pub request_payment_config_updated_at: OffsetDateTime, pub public_access_block_config_updated_at: OffsetDateTime, pub bucket_acl_config_updated_at: OffsetDateTime, @@ -290,6 +302,10 @@ pub struct BucketMetadata { pub bucket_target_config: Option, pub bucket_target_config_meta: Option>, pub cors_config: Option, + pub logging_config: Option, + pub website_config: Option, + pub accelerate_config: Option, + pub request_payment_config: Option, pub public_access_block_config: Option, pub bucket_acl_config: Option, } @@ -312,6 +328,10 @@ impl Default for BucketMetadata { bucket_targets_config_json: Default::default(), bucket_targets_config_meta_json: Default::default(), cors_config_xml: Default::default(), + logging_config_xml: Default::default(), + website_config_xml: Default::default(), + accelerate_config_xml: Default::default(), + request_payment_config_xml: Default::default(), public_access_block_config_xml: Default::default(), bucket_acl_config_json: Default::default(), policy_config_updated_at: OffsetDateTime::UNIX_EPOCH, @@ -326,6 +346,10 @@ impl Default for BucketMetadata { bucket_targets_config_updated_at: OffsetDateTime::UNIX_EPOCH, bucket_targets_config_meta_updated_at: OffsetDateTime::UNIX_EPOCH, cors_config_updated_at: OffsetDateTime::UNIX_EPOCH, + logging_config_updated_at: OffsetDateTime::UNIX_EPOCH, + website_config_updated_at: OffsetDateTime::UNIX_EPOCH, + accelerate_config_updated_at: OffsetDateTime::UNIX_EPOCH, + request_payment_config_updated_at: OffsetDateTime::UNIX_EPOCH, public_access_block_config_updated_at: OffsetDateTime::UNIX_EPOCH, bucket_acl_config_updated_at: OffsetDateTime::UNIX_EPOCH, new_field_updated_at: OffsetDateTime::UNIX_EPOCH, @@ -341,6 +365,10 @@ impl Default for BucketMetadata { bucket_target_config: Default::default(), bucket_target_config_meta: Default::default(), cors_config: Default::default(), + logging_config: Default::default(), + website_config: Default::default(), + accelerate_config: Default::default(), + request_payment_config: Default::default(), public_access_block_config: Default::default(), bucket_acl_config: Default::default(), } @@ -411,11 +439,19 @@ impl BucketMetadata { "BucketTargetsConfigUpdatedAt" => self.bucket_targets_config_updated_at = read_msgp_time_value(rd)?, "BucketTargetsConfigMetaUpdatedAt" => self.bucket_targets_config_meta_updated_at = read_msgp_time_value(rd)?, "CorsConfigXML" | "CorsConfigXml" => self.cors_config_xml = read_msgp_bin(rd)?, + "LoggingConfigXML" | "LoggingConfigXml" => self.logging_config_xml = read_msgp_bin(rd)?, + "WebsiteConfigXML" | "WebsiteConfigXml" => self.website_config_xml = read_msgp_bin(rd)?, + "AccelerateConfigXML" | "AccelerateConfigXml" => self.accelerate_config_xml = read_msgp_bin(rd)?, + "RequestPaymentConfigXML" | "RequestPaymentConfigXml" => self.request_payment_config_xml = read_msgp_bin(rd)?, "PublicAccessBlockConfigXML" | "PublicAccessBlockConfigXml" => { self.public_access_block_config_xml = read_msgp_bin(rd)? } "BucketAclConfigJSON" | "BucketAclConfigJson" => self.bucket_acl_config_json = read_msgp_bin(rd)?, "CorsConfigUpdatedAt" => self.cors_config_updated_at = read_msgp_time_value(rd)?, + "LoggingConfigUpdatedAt" => self.logging_config_updated_at = read_msgp_time_value(rd)?, + "WebsiteConfigUpdatedAt" => self.website_config_updated_at = read_msgp_time_value(rd)?, + "AccelerateConfigUpdatedAt" => self.accelerate_config_updated_at = read_msgp_time_value(rd)?, + "RequestPaymentConfigUpdatedAt" => self.request_payment_config_updated_at = read_msgp_time_value(rd)?, "PublicAccessBlockConfigUpdatedAt" => self.public_access_block_config_updated_at = read_msgp_time_value(rd)?, "BucketAclConfigUpdatedAt" => self.bucket_acl_config_updated_at = read_msgp_time_value(rd)?, other => { @@ -430,8 +466,8 @@ impl BucketMetadata { /// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility. pub fn encode_to(&self, wr: &mut W) -> Result<()> { - // Map size: MinIO fields (25) + RustFS extensions (6) - let map_len: u32 = 31; + // Map size: MinIO fields (25) + RustFS extensions (14) + let map_len: u32 = 39; rmp::encode::write_map_len(wr, map_len)?; // MinIO field order (same as Go struct) @@ -481,10 +517,22 @@ impl BucketMetadata { // RustFS extensions write_bin_field(wr, "CorsConfigXML", &self.cors_config_xml)?; + write_bin_field(wr, "LoggingConfigXML", &self.logging_config_xml)?; + write_bin_field(wr, "WebsiteConfigXML", &self.website_config_xml)?; + write_bin_field(wr, "AccelerateConfigXML", &self.accelerate_config_xml)?; + write_bin_field(wr, "RequestPaymentConfigXML", &self.request_payment_config_xml)?; write_bin_field(wr, "PublicAccessBlockConfigXML", &self.public_access_block_config_xml)?; write_bin_field(wr, "BucketAclConfigJSON", &self.bucket_acl_config_json)?; rmp::encode::write_str(wr, "CorsConfigUpdatedAt")?; write_msgp_time(wr, self.cors_config_updated_at)?; + rmp::encode::write_str(wr, "LoggingConfigUpdatedAt")?; + write_msgp_time(wr, self.logging_config_updated_at)?; + rmp::encode::write_str(wr, "WebsiteConfigUpdatedAt")?; + write_msgp_time(wr, self.website_config_updated_at)?; + rmp::encode::write_str(wr, "AccelerateConfigUpdatedAt")?; + write_msgp_time(wr, self.accelerate_config_updated_at)?; + rmp::encode::write_str(wr, "RequestPaymentConfigUpdatedAt")?; + write_msgp_time(wr, self.request_payment_config_updated_at)?; rmp::encode::write_str(wr, "PublicAccessBlockConfigUpdatedAt")?; write_msgp_time(wr, self.public_access_block_config_updated_at)?; rmp::encode::write_str(wr, "BucketAclConfigUpdatedAt")?; @@ -569,6 +617,18 @@ impl BucketMetadata { if self.public_access_block_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.public_access_block_config_updated_at = self.created } + if self.logging_config_updated_at == OffsetDateTime::UNIX_EPOCH { + self.logging_config_updated_at = self.created + } + if self.website_config_updated_at == OffsetDateTime::UNIX_EPOCH { + self.website_config_updated_at = self.created + } + if self.accelerate_config_updated_at == OffsetDateTime::UNIX_EPOCH { + self.accelerate_config_updated_at = self.created + } + if self.request_payment_config_updated_at == OffsetDateTime::UNIX_EPOCH { + self.request_payment_config_updated_at = self.created + } if self.bucket_acl_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.bucket_acl_config_updated_at = self.created } @@ -625,6 +685,22 @@ impl BucketMetadata { self.cors_config_xml = data; self.cors_config_updated_at = updated; } + BUCKET_LOGGING_CONFIG => { + self.logging_config_xml = data; + self.logging_config_updated_at = updated; + } + BUCKET_WEBSITE_CONFIG => { + self.website_config_xml = data; + self.website_config_updated_at = updated; + } + BUCKET_ACCELERATE_CONFIG => { + self.accelerate_config_xml = data; + self.accelerate_config_updated_at = updated; + } + BUCKET_REQUEST_PAYMENT_CONFIG => { + self.request_payment_config_xml = data; + self.request_payment_config_updated_at = updated; + } BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG => { self.public_access_block_config_xml = data; self.public_access_block_config_updated_at = updated; @@ -741,6 +817,33 @@ impl BucketMetadata { { tracing::warn!(bucket = %self.name, config = "cors", error = %e, "parse_all_configs: failed to parse"); } + if !self.logging_config_xml.is_empty() + && let Err(e) = deserialize::(&self.logging_config_xml).map(|c| self.logging_config = Some(c)) + { + tracing::warn!(bucket = %self.name, config = "logging", error = %e, "parse_all_configs: failed to parse"); + } + if !self.website_config_xml.is_empty() + && let Err(e) = deserialize::(&self.website_config_xml).map(|c| self.website_config = Some(c)) + { + tracing::warn!(bucket = %self.name, config = "website", error = %e, "parse_all_configs: failed to parse"); + } + if !self.accelerate_config_xml.is_empty() + && let Err(e) = + deserialize::(&self.accelerate_config_xml).map(|c| self.accelerate_config = Some(c)) + { + tracing::warn!(bucket = %self.name, config = "accelerate", error = %e, "parse_all_configs: failed to parse"); + } + if !self.request_payment_config_xml.is_empty() + && let Err(e) = deserialize::(&self.request_payment_config_xml) + .map(|c| self.request_payment_config = Some(c)) + { + tracing::warn!( + bucket = %self.name, + config = "request_payment", + error = %e, + "parse_all_configs: failed to parse" + ); + } if !self.public_access_block_config_xml.is_empty() && let Err(e) = deserialize::(&self.public_access_block_config_xml) .map(|c| self.public_access_block_config = Some(c)) diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs index b6ae15fc3..9ed77603f 100644 --- a/crates/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -28,8 +28,9 @@ use rustfs_common::heal_channel::HealOpts; use rustfs_policy::policy::BucketPolicy; use s3s::dto::ReplicationConfiguration; use s3s::dto::{ - BucketLifecycleConfiguration, CORSConfiguration, NotificationConfiguration, ObjectLockConfiguration, - PublicAccessBlockConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration, + AccelerateConfiguration, BucketLifecycleConfiguration, BucketLoggingStatus, CORSConfiguration, NotificationConfiguration, + ObjectLockConfiguration, PublicAccessBlockConfiguration, RequestPaymentConfiguration, ServerSideEncryptionConfiguration, + Tagging, VersioningConfiguration, WebsiteConfiguration, }; use std::collections::HashSet; use std::sync::OnceLock; @@ -193,6 +194,34 @@ pub async fn get_versioning_config(bucket: &str) -> Result<(VersioningConfigurat bucket_meta_sys.get_versioning_config(bucket).await } +pub async fn get_website_config(bucket: &str) -> Result<(WebsiteConfiguration, OffsetDateTime)> { + let bucket_meta_sys_lock = get_bucket_metadata_sys()?; + let bucket_meta_sys = bucket_meta_sys_lock.read().await; + + bucket_meta_sys.get_website_config(bucket).await +} + +pub async fn get_logging_config(bucket: &str) -> Result<(BucketLoggingStatus, OffsetDateTime)> { + let bucket_meta_sys_lock = get_bucket_metadata_sys()?; + let bucket_meta_sys = bucket_meta_sys_lock.read().await; + + bucket_meta_sys.get_logging_config(bucket).await +} + +pub async fn get_accelerate_config(bucket: &str) -> Result<(AccelerateConfiguration, OffsetDateTime)> { + let bucket_meta_sys_lock = get_bucket_metadata_sys()?; + let bucket_meta_sys = bucket_meta_sys_lock.read().await; + + bucket_meta_sys.get_accelerate_config(bucket).await +} + +pub async fn get_request_payment_config(bucket: &str) -> Result<(RequestPaymentConfiguration, OffsetDateTime)> { + let bucket_meta_sys_lock = get_bucket_metadata_sys()?; + let bucket_meta_sys = bucket_meta_sys_lock.read().await; + + bucket_meta_sys.get_request_payment_config(bucket).await +} + pub async fn get_config_from_disk(bucket: &str) -> Result { let bucket_meta_sys_lock = get_bucket_metadata_sys()?; let bucket_meta_sys = bucket_meta_sys_lock.read().await; @@ -587,6 +616,46 @@ impl BucketMetadataSys { } } + pub async fn get_website_config(&self, bucket: &str) -> Result<(WebsiteConfiguration, OffsetDateTime)> { + let (bm, _) = self.get_config(bucket).await?; + + if let Some(config) = &bm.website_config { + Ok((config.clone(), bm.website_config_updated_at)) + } else { + Err(Error::ConfigNotFound) + } + } + + pub async fn get_logging_config(&self, bucket: &str) -> Result<(BucketLoggingStatus, OffsetDateTime)> { + let (bm, _) = self.get_config(bucket).await?; + + if let Some(config) = &bm.logging_config { + Ok((config.clone(), bm.logging_config_updated_at)) + } else { + Err(Error::ConfigNotFound) + } + } + + pub async fn get_accelerate_config(&self, bucket: &str) -> Result<(AccelerateConfiguration, OffsetDateTime)> { + let (bm, _) = self.get_config(bucket).await?; + + if let Some(config) = &bm.accelerate_config { + Ok((config.clone(), bm.accelerate_config_updated_at)) + } else { + Err(Error::ConfigNotFound) + } + } + + pub async fn get_request_payment_config(&self, bucket: &str) -> Result<(RequestPaymentConfiguration, OffsetDateTime)> { + let (bm, _) = self.get_config(bucket).await?; + + if let Some(config) = &bm.request_payment_config { + Ok((config.clone(), bm.request_payment_config_updated_at)) + } else { + Err(Error::ConfigNotFound) + } + } + pub async fn created_at(&self, bucket: &str) -> Result { let bm = match self.get_config(bucket).await { Ok((bm, _)) => bm.created, diff --git a/crates/ecstore/src/bucket/replication/replication_pool.rs b/crates/ecstore/src/bucket/replication/replication_pool.rs index 8d7a1734a..20422b5ec 100644 --- a/crates/ecstore/src/bucket/replication/replication_pool.rs +++ b/crates/ecstore/src/bucket/replication/replication_pool.rs @@ -21,7 +21,7 @@ use crate::bucket::replication::replicate_delete; use crate::bucket::replication::replicate_object; use crate::bucket::replication::replication_resyncer::{ BucketReplicationResyncStatus, DeletedObjectReplicationInfo, REPLICATION_DIR, RESYNC_FILE_NAME, ReplicationConfig, - ReplicationResyncer, decode_resync_file, get_heal_replicate_object_info, + ReplicationResyncer, TargetReplicationResyncStatus, decode_resync_file, get_heal_replicate_object_info, save_resync_status, }; use crate::bucket::replication::replication_state::ReplicationStats; use crate::config::com::read_config; @@ -763,6 +763,63 @@ impl ReplicationPool { Ok(()) } + pub async fn get_bucket_resync_status(&self, bucket: &str) -> Result { + if let Some(status) = self.resyncer.status_map.read().await.get(bucket).cloned() { + return Ok(status); + } + + let status = load_bucket_resync_metadata(bucket, self.storage.clone()).await?; + self.resyncer + .status_map + .write() + .await + .insert(bucket.to_string(), status.clone()); + Ok(status) + } + + pub async fn start_bucket_resync(self: Arc, opts: ResyncOpts) -> Result<(), EcstoreError> { + let now = OffsetDateTime::now_utc(); + let bucket_status = { + let mut status_map = self.resyncer.status_map.write().await; + let bucket_status = status_map.entry(opts.bucket.clone()).or_insert_with(|| { + let mut status = BucketReplicationResyncStatus::new(); + status.id = 0; + status + }); + + bucket_status.last_update = Some(now); + bucket_status.targets_map.insert( + opts.arn.clone(), + TargetReplicationResyncStatus { + start_time: Some(now), + last_update: Some(now), + resync_id: opts.resync_id.clone(), + resync_before_date: opts.resync_before, + resync_status: ResyncStatusType::ResyncPending, + failed_size: 0, + failed_count: 0, + replicated_size: 0, + replicated_count: 0, + bucket: opts.bucket.clone(), + object: String::new(), + error: None, + }, + ); + + bucket_status.clone() + }; + + save_resync_status(&opts.bucket, &bucket_status, self.storage.clone()).await?; + + let resyncer = self.resyncer.clone(); + let storage = self.storage.clone(); + tokio::spawn(async move { + resyncer.resync_bucket(CancellationToken::new(), storage, false, opts).await; + }); + + Ok(()) + } + /// Start the resync routine that runs in a loop async fn start_resync_routine(self: Arc, buckets: Vec, cancellation_token: CancellationToken) { // Run the replication resync in a loop @@ -891,6 +948,8 @@ pub trait ReplicationPoolTrait: std::fmt::Debug { async fn queue_replica_task(&self, ri: ReplicateObjectInfo); async fn queue_replica_delete_task(&self, ri: DeletedObjectReplicationInfo); async fn resize(&self, priority: ReplicationPriority, max_workers: usize, max_l_workers: usize); + async fn get_bucket_resync_status(&self, bucket: &str) -> Result; + async fn start_bucket_resync(self: Arc, opts: ResyncOpts) -> Result<(), EcstoreError>; async fn init_resync( self: Arc, cancellation_token: CancellationToken, @@ -913,6 +972,14 @@ impl ReplicationPoolTrait for ReplicationPool { self.resize(priority, max_workers, max_l_workers).await; } + async fn get_bucket_resync_status(&self, bucket: &str) -> Result { + self.get_bucket_resync_status(bucket).await + } + + async fn start_bucket_resync(self: Arc, opts: ResyncOpts) -> Result<(), EcstoreError> { + self.start_bucket_resync(opts).await + } + async fn init_resync( self: Arc, cancellation_token: CancellationToken, diff --git a/crates/ecstore/src/bucket/replication/replication_resyncer.rs b/crates/ecstore/src/bucket/replication/replication_resyncer.rs index 32e3ebf63..3eec82917 100644 --- a/crates/ecstore/src/bucket/replication/replication_resyncer.rs +++ b/crates/ecstore/src/bucket/replication/replication_resyncer.rs @@ -901,7 +901,11 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC } } -async fn save_resync_status(bucket: &str, status: &BucketReplicationResyncStatus, api: Arc) -> Result<()> { +pub(crate) async fn save_resync_status( + bucket: &str, + status: &BucketReplicationResyncStatus, + api: Arc, +) -> Result<()> { let data = encode_resync_file(status)?; let config_file = path_join_buf(&[BUCKET_META_PREFIX, bucket, REPLICATION_DIR, RESYNC_FILE_NAME]); diff --git a/crates/ecstore/src/event_notification.rs b/crates/ecstore/src/event_notification.rs index 4a1a6f68a..c2990f477 100644 --- a/crates/ecstore/src/event_notification.rs +++ b/crates/ecstore/src/event_notification.rs @@ -21,6 +21,7 @@ use crate::store::ECStore; use crate::store_api::ObjectInfo; use std::collections::HashMap; use std::sync::Arc; +use std::sync::OnceLock; use std::sync::atomic::Ordering; use tokio::sync::RwLock; use tracing::warn; @@ -82,4 +83,50 @@ pub struct EventArgs { impl EventArgs {} -pub fn send_event(args: EventArgs) {} +type EventDispatchHook = Arc; + +static EVENT_DISPATCH_HOOK: OnceLock = OnceLock::new(); + +pub fn register_event_dispatch_hook(hook: F) -> bool +where + F: Fn(EventArgs) + Send + Sync + 'static, +{ + EVENT_DISPATCH_HOOK.set(Arc::new(hook)).is_ok() +} + +pub fn send_event(args: EventArgs) { + if let Some(hook) = EVENT_DISPATCH_HOOK.get() { + hook(args); + return; + } + + warn!( + event_name = args.event_name, + bucket = args.bucket_name, + "event send() dropped because no event dispatch hook is registered" + ); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + static DISPATCH_COUNT: AtomicUsize = AtomicUsize::new(0); + + #[test] + fn send_event_dispatches_to_registered_hook() { + let _ = register_event_dispatch_hook(|_args| { + DISPATCH_COUNT.fetch_add(1, Ordering::Relaxed); + }); + let before = DISPATCH_COUNT.load(Ordering::Relaxed); + + send_event(EventArgs { + event_name: "s3:ObjectCreated:Put".to_string(), + bucket_name: "demo".to_string(), + ..Default::default() + }); + + assert_eq!(DISPATCH_COUNT.load(Ordering::Relaxed), before + 1); + } +} diff --git a/crates/ecstore/src/rpc/peer_rest_client.rs b/crates/ecstore/src/rpc/peer_rest_client.rs index ac8639341..18a66d42e 100644 --- a/crates/ecstore/src/rpc/peer_rest_client.rs +++ b/crates/ecstore/src/rpc/peer_rest_client.rs @@ -29,9 +29,9 @@ use rustfs_madmin::{ use rustfs_protos::evict_failed_connection; use rustfs_protos::proto_gen::node_service::{ DeleteBucketMetadataRequest, DeletePolicyRequest, DeleteServiceAccountRequest, DeleteUserRequest, GetCpusRequest, - GetMemInfoRequest, GetMetricsRequest, GetNetInfoRequest, GetOsInfoRequest, GetPartitionsRequest, GetProcInfoRequest, - GetSeLinuxInfoRequest, GetSysConfigRequest, GetSysErrorsRequest, LoadBucketMetadataRequest, LoadGroupRequest, - LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest, + GetLiveEventsRequest, GetMemInfoRequest, GetMetricsRequest, GetNetInfoRequest, GetOsInfoRequest, GetPartitionsRequest, + GetProcInfoRequest, GetSeLinuxInfoRequest, GetSysConfigRequest, GetSysErrorsRequest, LoadBucketMetadataRequest, + LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ServerInfoRequest, SignalServiceRequest, StartProfilingRequest, StopRebalanceRequest, node_service_client::NodeServiceClient, @@ -48,6 +48,13 @@ pub const PEER_RESTSIGNAL: &str = "signal"; pub const PEER_RESTSUB_SYS: &str = "sub-sys"; pub const PEER_RESTDRY_RUN: &str = "dry-run"; +#[derive(Clone, Debug)] +pub struct PeerLiveEventsBatch { + pub events: Vec, + pub next_sequence: u64, + pub truncated: bool, +} + #[derive(Clone, Debug)] pub struct PeerRestClient { pub host: XHost, @@ -333,6 +340,25 @@ impl PeerRestClient { Ok(realtime_metrics) } + pub async fn get_live_events(&self, after_sequence: u64, limit: u32) -> Result { + let mut client = self.get_client().await?; + let request = Request::new(GetLiveEventsRequest { after_sequence, limit }); + + let response = client.get_live_events(request).await?.into_inner(); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(Error::other("")); + } + + Ok(PeerLiveEventsBatch { + events: response.events.to_vec(), + next_sequence: response.next_sequence, + truncated: response.truncated, + }) + } + pub async fn get_proc_info(&self) -> Result { let mut client = self.get_client().await?; let request = Request::new(GetProcInfoRequest {}); diff --git a/crates/ecstore/src/set_disk.rs b/crates/ecstore/src/set_disk.rs index 0e6e6d2ae..1237d8cc6 100644 --- a/crates/ecstore/src/set_disk.rs +++ b/crates/ecstore/src/set_disk.rs @@ -701,6 +701,11 @@ impl ObjectIO for SetDisks { } let mut user_defined = opts.user_defined.clone(); + if let Some(eval_metadata) = &opts.eval_metadata { + for (key, value) in eval_metadata { + user_defined.insert(key.clone(), value.clone()); + } + } let sc_parity_drives = { if let Some(sc) = GLOBAL_STORAGE_CLASS.get() { diff --git a/crates/ecstore/src/store.rs b/crates/ecstore/src/store.rs index b77c09b4e..07e16baa4 100644 --- a/crates/ecstore/src/store.rs +++ b/crates/ecstore/src/store.rs @@ -14,7 +14,8 @@ #![allow(clippy::map_entry)] -use crate::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry; +use crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc; +use crate::bucket::lifecycle::bucket_lifecycle_ops::{enqueue_transition_immediate, init_background_expiry}; use crate::bucket::metadata_sys::{self, set_bucket_metadata}; use crate::bucket::utils::check_abort_multipart_args; use crate::bucket::utils::check_complete_multipart_args; @@ -27,7 +28,7 @@ use crate::bucket::utils::check_new_multipart_args; use crate::bucket::utils::check_object_args; use crate::bucket::utils::check_put_object_args; use crate::bucket::utils::check_put_object_part_args; -use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict}; +use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname}; use crate::config::GLOBAL_STORAGE_CLASS; use crate::config::storageclass; use crate::disk::endpoint::{Endpoint, EndpointType}; @@ -129,6 +130,22 @@ async fn has_xlmeta_files(path: &std::path::Path) -> bool { false } +async fn enqueue_transition_after_write(result: Result, src: LcEventSrc) -> Result { + match result { + Ok(oi) => { + if should_enqueue_transition_immediately(&oi) { + enqueue_transition_immediate(&oi, src).await; + } + Ok(oi) + } + Err(err) => Err(err), + } +} + +fn should_enqueue_transition_immediately(oi: &ObjectInfo) -> bool { + !is_meta_bucketname(&oi.bucket) +} + const MAX_UPLOADS_LIST: usize = 10000; mod bucket; @@ -243,7 +260,7 @@ impl ObjectIO for ECStore { } #[instrument(level = "debug", skip(self, data))] async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result { - self.handle_put_object(bucket, object, data, opts).await + enqueue_transition_after_write(self.handle_put_object(bucket, object, data, opts).await, LcEventSrc::S3PutObject).await } } @@ -301,8 +318,12 @@ impl ObjectOperations for ECStore { src_opts: &ObjectOptions, dst_opts: &ObjectOptions, ) -> Result { - self.handle_copy_object(src_bucket, src_object, dst_bucket, dst_object, src_info, src_opts, dst_opts) - .await + enqueue_transition_after_write( + self.handle_copy_object(src_bucket, src_object, dst_bucket, dst_object, src_info, src_opts, dst_opts) + .await, + LcEventSrc::S3CopyObject, + ) + .await } #[instrument(skip(self))] @@ -520,8 +541,12 @@ impl MultipartOperations for ECStore { uploaded_parts: Vec, opts: &ObjectOptions, ) -> Result { - self.handle_complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts) - .await + enqueue_transition_after_write( + self.handle_complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts) + .await, + LcEventSrc::S3CompleteMultipartUpload, + ) + .await } } @@ -709,6 +734,17 @@ mod tests { assert!(disks.is_empty() || !disks.is_empty()); } + #[test] + fn test_should_not_enqueue_transition_for_internal_metadata_bucket() { + let oi = ObjectInfo { + bucket: RUSTFS_META_BUCKET.to_string(), + name: format!("{BUCKET_META_PREFIX}/bucket/.metadata.bin"), + ..Default::default() + }; + + assert!(!should_enqueue_transition_immediately(&oi)); + } + // Test that we can create the basic structures without global state #[test] fn test_pool_available_space_creation() { diff --git a/crates/kms/src/backends/local.rs b/crates/kms/src/backends/local.rs index ab9ba8251..6ed9e7aa0 100644 --- a/crates/kms/src/backends/local.rs +++ b/crates/kms/src/backends/local.rs @@ -57,7 +57,9 @@ struct StoredMasterKey { status: KeyStatus, description: Option, metadata: HashMap, + #[serde(with = "crate::time_serde::zoned")] created_at: Zoned, + #[serde(with = "crate::time_serde::option_zoned")] rotated_at: Option, created_by: Option, /// Encrypted key material (32 bytes encoded in base64 for AES-256) @@ -840,6 +842,7 @@ impl KmsBackend for LocalKmsBackend { #[cfg(test)] mod tests { use super::*; + use std::collections::HashMap; use tempfile::TempDir; async fn create_test_client() -> (LocalKmsClient, TempDir) { @@ -943,4 +946,39 @@ mod tests { // Note: Direct decryption of encrypt() results is not implemented in this simple version // In a real implementation, encrypt() would create a different envelope format } + + #[tokio::test] + async fn test_load_master_key_accepts_legacy_rfc3339_timestamp() { + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + let config = LocalConfig { + key_dir: temp_dir.path().to_path_buf(), + master_key: None, + file_permissions: Some(0o600), + }; + let client = LocalKmsClient::new(config).await.expect("Failed to create client"); + + let stored_key = serde_json::json!({ + "key_id": "legacy-key", + "version": 1u32, + "algorithm": "AES_256", + "usage": "EncryptDecrypt", + "status": "Active", + "description": serde_json::Value::Null, + "metadata": HashMap::::new(), + "created_at": "2024-01-01T00:00:00+00:00", + "rotated_at": serde_json::Value::Null, + "created_by": "legacy-test", + "encrypted_key_material": BASE64.encode([7u8; 32]), + "nonce": Vec::::new() + }); + + let key_path = client.master_key_path("legacy-key"); + fs::write(&key_path, serde_json::to_vec_pretty(&stored_key).expect("serialize test key")) + .await + .expect("write legacy key"); + + let key_info = client.load_master_key("legacy-key").await.expect("legacy key should load"); + assert_eq!(key_info.key_id, "legacy-key"); + assert_eq!(key_info.created_at.time_zone().iana_name(), Some("UTC")); + } } diff --git a/crates/kms/src/encryption/dek.rs b/crates/kms/src/encryption/dek.rs index 72d370f2c..a2753b65d 100644 --- a/crates/kms/src/encryption/dek.rs +++ b/crates/kms/src/encryption/dek.rs @@ -40,6 +40,7 @@ pub struct DataKeyEnvelope { pub encrypted_key: Vec, pub nonce: Vec, pub encryption_context: HashMap, + #[serde(with = "crate::time_serde::zoned")] pub created_at: Zoned, } @@ -311,4 +312,21 @@ mod tests { assert_eq!(deserialized.key_id, "test-key-id"); assert_eq!(deserialized.master_key_id, "master-key-id"); } + + #[tokio::test] + async fn test_data_key_envelope_accepts_legacy_rfc3339_timestamp() { + let envelope_json = r#"{ + "key_id": "test-key-id", + "master_key_id": "master-key-id", + "key_spec": "AES_256", + "encrypted_key": [1, 2, 3, 4], + "nonce": [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], + "encryption_context": {"bucket": "test-bucket"}, + "created_at": "2024-01-01T00:00:00+00:00" + }"#; + + let deserialized: DataKeyEnvelope = serde_json::from_str(envelope_json).expect("Should deserialize legacy format"); + assert_eq!(deserialized.key_id, "test-key-id"); + assert_eq!(deserialized.master_key_id, "master-key-id"); + } } diff --git a/crates/kms/src/lib.rs b/crates/kms/src/lib.rs index b882c35f5..7b8de98cc 100644 --- a/crates/kms/src/lib.rs +++ b/crates/kms/src/lib.rs @@ -65,6 +65,7 @@ mod error; pub mod manager; pub mod service; pub mod service_manager; +mod time_serde; pub mod types; // Re-export public API diff --git a/crates/kms/src/service.rs b/crates/kms/src/service.rs index 2a6ef720f..c405c9992 100644 --- a/crates/kms/src/service.rs +++ b/crates/kms/src/service.rs @@ -49,6 +49,8 @@ pub struct ObjectEncryptionService { kms_manager: KmsManager, } +const INTERNAL_ENCRYPTION_KEY_ID_HEADER: &str = "x-rustfs-encryption-key-id"; + /// Result of object encryption #[derive(Debug, Clone)] pub struct EncryptionResult { @@ -604,12 +606,13 @@ impl ObjectEncryptionService { headers.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()); } else if metadata.algorithm == "AES256" { headers.insert("x-amz-server-side-encryption".to_string(), "AES256".to_string()); - // For SSE-S3, we still need to store the key ID for internal use - headers.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), metadata.key_id.clone()); } else { headers.insert("x-amz-server-side-encryption".to_string(), "aws:kms".to_string()); headers.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), metadata.key_id.clone()); } + if metadata.key_id != "sse-c" { + headers.insert(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), metadata.key_id.clone()); + } // Internal headers for decryption headers.insert( @@ -653,8 +656,14 @@ impl ObjectEncryptionService { let key_id = if algorithm == "AES256" && headers.contains_key("x-amz-server-side-encryption-customer-algorithm") { "sse-c".to_string() + } else if let Some(key_id) = headers.get(INTERNAL_ENCRYPTION_KEY_ID_HEADER) { + key_id.clone() } else if let Some(kms_key_id) = headers.get("x-amz-server-side-encryption-aws-kms-key-id") { kms_key_id.clone() + } else if algorithm == "AES256" { + self.get_default_key_id() + .cloned() + .ok_or_else(|| KmsError::validation_error("Missing key ID"))? } else { return Err(KmsError::validation_error("Missing key ID")); }; @@ -821,6 +830,8 @@ mod tests { let headers = service.metadata_to_headers(&metadata); assert!(headers.contains_key("x-amz-server-side-encryption")); assert!(headers.contains_key("x-rustfs-encryption-iv")); + assert!(headers.contains_key(INTERNAL_ENCRYPTION_KEY_ID_HEADER)); + assert!(!headers.contains_key("x-amz-server-side-encryption-aws-kms-key-id")); // Convert back to metadata let parsed_metadata = service.headers_to_metadata(&headers).expect("Failed to parse headers"); diff --git a/crates/kms/src/time_serde.rs b/crates/kms/src/time_serde.rs new file mode 100644 index 000000000..3aa042156 --- /dev/null +++ b/crates/kms/src/time_serde.rs @@ -0,0 +1,73 @@ +use jiff::{Timestamp, Zoned, tz::TimeZone}; +use serde::{Deserialize, Deserializer, Serializer}; + +pub(crate) mod zoned { + use super::*; + + pub(crate) fn serialize(value: &Zoned, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&value.to_string()) + } + + pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + parse_zoned_compat(&value).map_err(serde::de::Error::custom) + } +} + +pub(crate) mod option_zoned { + use super::*; + + pub(crate) fn serialize(value: &Option, serializer: S) -> Result + where + S: Serializer, + { + match value { + Some(value) => serializer.serialize_some(&value.to_string()), + None => serializer.serialize_none(), + } + } + + pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let value = Option::::deserialize(deserializer)?; + value + .map(|value| parse_zoned_compat(&value).map_err(serde::de::Error::custom)) + .transpose() + } +} + +fn parse_zoned_compat(value: &str) -> Result { + if let Ok(zoned) = value.parse::() { + return Ok(zoned); + } + + let timestamp = value + .parse::() + .map_err(|err| format!("failed to parse legacy timestamp '{value}': {err}"))?; + Ok(timestamp.to_zoned(TimeZone::UTC)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_zoned_compat_accepts_current_zoned_format() { + let zoned = parse_zoned_compat("2024-01-01T00:00:00+00:00[UTC]").expect("current format should parse"); + assert_eq!(zoned.time_zone().iana_name(), Some("UTC")); + } + + #[test] + fn parse_zoned_compat_accepts_legacy_rfc3339_format() { + let zoned = parse_zoned_compat("2024-01-01T00:00:00+00:00").expect("legacy format should parse"); + assert_eq!(zoned.time_zone().iana_name(), Some("UTC")); + } +} diff --git a/crates/mcp/Cargo.toml b/crates/mcp/Cargo.toml index 9846b03d9..676309762 100644 --- a/crates/mcp/Cargo.toml +++ b/crates/mcp/Cargo.toml @@ -32,6 +32,7 @@ path = "src/main.rs" [dependencies] # AWS SDK for S3 operations aws-sdk-s3.workspace = true +aws-smithy-http-client.workspace = true # Async runtime and utilities tokio = { workspace = true, features = ["io-std", "io-util", "macros", "signal"] } diff --git a/crates/mcp/src/s3_client.rs b/crates/mcp/src/s3_client.rs index 7d9a14893..ac7ec0898 100644 --- a/crates/mcp/src/s3_client.rs +++ b/crates/mcp/src/s3_client.rs @@ -16,6 +16,7 @@ use anyhow::{Context, Result}; use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::{Client, Config as S3Config}; +use aws_smithy_http_client::Builder as SmithyHttpClientBuilder; use serde::{Deserialize, Serialize}; use std::path::Path; use tokio::io::AsyncWriteExt; @@ -133,6 +134,14 @@ impl S3Client { .region(Region::new(config.region.clone())) .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest()); + if config + .endpoint_url + .as_deref() + .is_some_and(|endpoint| endpoint.starts_with("http://")) + { + config_builder = config_builder.http_client(SmithyHttpClientBuilder::new().build_http()); + } + // Set force path style if custom endpoint or explicitly requested let should_force_path_style = config.endpoint_url.is_some() || config.force_path_style; if should_force_path_style { diff --git a/crates/mcp/src/server.rs b/crates/mcp/src/server.rs index 1088761c9..46ff8b9ae 100644 --- a/crates/mcp/src/server.rs +++ b/crates/mcp/src/server.rs @@ -629,6 +629,8 @@ mod tests { let config = Config { access_key_id: Some("test_key".to_string()), secret_access_key: Some("test_secret".to_string()), + endpoint_url: Some("http://127.0.0.1:9000".to_string()), + force_path_style: true, ..Config::default() }; diff --git a/crates/notify/src/global.rs b/crates/notify/src/global.rs index 4d70a571c..5280fd0d0 100644 --- a/crates/notify/src/global.rs +++ b/crates/notify/src/global.rs @@ -78,12 +78,6 @@ pub mod notifier_global { return; } - // Check if any subscribers are interested in the event - if !notification_sys.has_subscriber(&args.bucket_name, &args.event_name).await { - // error!("No subscribers for event: {} in bucket: {}", args.event_name, args.bucket_name); - return; - } - // Create an event and send it let event = Arc::new(Event::new(args)); notification_sys.send_event(event).await; diff --git a/crates/notify/src/integration.rs b/crates/notify/src/integration.rs index 70e8a49d7..89657f4c4 100644 --- a/crates/notify/src/integration.rs +++ b/crates/notify/src/integration.rs @@ -25,12 +25,62 @@ use rustfs_targets::arn::TargetID; use rustfs_targets::store::{Key, Store}; use rustfs_targets::target::EntityTarget; use rustfs_targets::{StoreError, Target}; +use std::collections::VecDeque; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::{Duration, Instant}; -use tokio::sync::{RwLock, Semaphore, mpsc}; +use tokio::sync::{RwLock, Semaphore, broadcast, mpsc}; use tracing::{debug, error, info, warn}; +const MAX_RECENT_LIVE_EVENTS: usize = 1024; + +#[derive(Clone)] +pub struct LiveEventBatch { + pub events: Vec>, + pub next_sequence: u64, + pub truncated: bool, +} + +#[derive(Default)] +struct LiveEventHistory { + next_sequence: u64, + events: VecDeque<(u64, Arc)>, +} + +impl LiveEventHistory { + fn record(&mut self, event: Arc) { + self.next_sequence = self.next_sequence.saturating_add(1); + self.events.push_back((self.next_sequence, event)); + while self.events.len() > MAX_RECENT_LIVE_EVENTS { + self.events.pop_front(); + } + } + + fn snapshot_since(&self, after_sequence: u64, limit: usize) -> LiveEventBatch { + let mut events = Vec::new(); + let mut next_sequence = after_sequence; + let mut truncated = false; + + for (sequence, event) in self.events.iter() { + if *sequence <= after_sequence { + continue; + } + if events.len() >= limit { + truncated = true; + break; + } + next_sequence = *sequence; + events.push(event.clone()); + } + + LiveEventBatch { + events, + next_sequence, + truncated, + } + } +} + /// Notify the system of monitoring indicators pub struct NotificationMetrics { /// The number of events currently being processed @@ -108,6 +158,10 @@ pub struct NotificationSystem { metrics: Arc, /// Subscriber view subscriber_view: NotificationSystemSubscriberView, + /// Live event fan-out for in-process streaming consumers. + live_event_sender: broadcast::Sender>, + /// Recent live event history for peer fan-in consumers. + live_event_history: Arc>, } impl NotificationSystem { @@ -115,6 +169,7 @@ impl NotificationSystem { pub fn new(config: Config) -> Self { let concurrency_limiter = rustfs_utils::get_env_usize(ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY); + let (live_event_sender, _) = broadcast::channel(1024); NotificationSystem { subscriber_view: NotificationSystemSubscriberView::new(), notifier: Arc::new(EventNotifier::new()), @@ -123,6 +178,8 @@ impl NotificationSystem { stream_cancellers: Arc::new(RwLock::new(HashMap::new())), concurrency_limiter: Arc::new(Semaphore::new(concurrency_limiter)), // Limit the maximum number of concurrent processing events to 20 metrics: Arc::new(NotificationMetrics::new()), + live_event_sender, + live_event_history: Arc::new(RwLock::new(LiveEventHistory::default())), } } @@ -216,6 +273,21 @@ impl NotificationSystem { self.notifier.has_subscriber(bucket, event).await } + /// Returns true when at least one in-process consumer is subscribed to live events. + pub fn has_live_listeners(&self) -> bool { + self.live_event_sender.receiver_count() > 0 + } + + /// Subscribes to the in-process live event stream. + pub fn subscribe_live_events(&self) -> broadcast::Receiver> { + self.live_event_sender.subscribe() + } + + pub async fn recent_live_events_since(&self, after_sequence: u64, limit: usize) -> LiveEventBatch { + let history = self.live_event_history.read().await; + history.snapshot_since(after_sequence, limit.max(1)) + } + async fn update_config_and_reload(&self, mut modifier: F) -> Result<(), NotificationError> where F: FnMut(&mut Config) -> bool, // The closure returns a boolean value indicating whether the configuration has been changed @@ -500,6 +572,8 @@ impl NotificationSystem { /// Sends an event pub async fn send_event(&self, event: Arc) { + self.live_event_history.write().await.record(event.clone()); + let _ = self.live_event_sender.send(event.clone()); self.notifier.send(event).await; } @@ -558,3 +632,37 @@ pub async fn load_config_from_file(path: &str, system: &NotificationSystem) -> R .map_err(|e| NotificationError::Configuration(format!("Failed to parse config: {e}")))?; system.reload_config(config).await } + +#[cfg(test)] +mod tests { + use super::*; + use rustfs_s3_common::EventName; + + #[test] + fn live_event_history_snapshots_from_sequence() { + let mut history = LiveEventHistory::default(); + history.record(Arc::new(Event::new_test_event("bucket", "one", EventName::ObjectCreatedPut))); + history.record(Arc::new(Event::new_test_event("bucket", "two", EventName::ObjectCreatedPut))); + + let batch = history.snapshot_since(1, 16); + + assert_eq!(batch.next_sequence, 2); + assert!(!batch.truncated); + assert_eq!(batch.events.len(), 1); + assert_eq!(batch.events[0].s3.object.key, "two"); + } + + #[test] + fn live_event_history_marks_truncation() { + let mut history = LiveEventHistory::default(); + history.record(Arc::new(Event::new_test_event("bucket", "one", EventName::ObjectCreatedPut))); + history.record(Arc::new(Event::new_test_event("bucket", "two", EventName::ObjectCreatedPut))); + + let batch = history.snapshot_since(0, 1); + + assert_eq!(batch.next_sequence, 1); + assert!(batch.truncated); + assert_eq!(batch.events.len(), 1); + assert_eq!(batch.events[0].s3.object.key, "one"); + } +} diff --git a/crates/protos/src/generated/proto_gen/node_service.rs b/crates/protos/src/generated/proto_gen/node_service.rs index 51cc40308..0efecb167 100644 --- a/crates/protos/src/generated/proto_gen/node_service.rs +++ b/crates/protos/src/generated/proto_gen/node_service.rs @@ -796,6 +796,26 @@ pub struct GetMetricsResponse { pub error_info: ::core::option::Option<::prost::alloc::string::String>, } #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetLiveEventsRequest { + #[prost(uint64, tag = "1")] + pub after_sequence: u64, + #[prost(uint32, tag = "2")] + pub limit: u32, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetLiveEventsResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(bytes = "bytes", tag = "2")] + pub events: ::prost::bytes::Bytes, + #[prost(uint64, tag = "3")] + pub next_sequence: u64, + #[prost(bool, tag = "4")] + pub truncated: bool, + #[prost(string, optional, tag = "5")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetProcInfoRequest {} #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetProcInfoResponse { @@ -1921,6 +1941,21 @@ pub mod node_service_client { .insert(GrpcMethod::new("node_service.NodeService", "GetMetrics")); self.inner.unary(req, path, codec).await } + pub async fn get_live_events( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/GetLiveEvents"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "GetLiveEvents")); + self.inner.unary(req, path, codec).await + } pub async fn get_proc_info( &mut self, request: impl tonic::IntoRequest, @@ -2521,6 +2556,10 @@ pub mod node_service_server { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + async fn get_live_events( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; async fn get_proc_info( &self, request: tonic::Request, @@ -4097,6 +4136,34 @@ pub mod node_service_server { }; Box::pin(fut) } + "/node_service.NodeService/GetLiveEvents" => { + #[allow(non_camel_case_types)] + struct GetLiveEventsSvc(pub Arc); + impl tonic::server::UnaryService for GetLiveEventsSvc { + type Response = super::GetLiveEventsResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::get_live_events(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = GetLiveEventsSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } "/node_service.NodeService/GetProcInfo" => { #[allow(non_camel_case_types)] struct GetProcInfoSvc(pub Arc); diff --git a/crates/protos/src/node.proto b/crates/protos/src/node.proto index 23024f09e..c3a20fd62 100644 --- a/crates/protos/src/node.proto +++ b/crates/protos/src/node.proto @@ -773,6 +773,19 @@ message LoadTransitionTierConfigResponse { optional string error_info = 2; } +message GetLiveEventsRequest { + uint64 after_sequence = 1; + uint32 limit = 2; +} + +message GetLiveEventsResponse { + bool success = 1; + bytes events = 2; + uint64 next_sequence = 3; + bool truncated = 4; + optional string error_info = 5; +} + /* -------------------------------------------------------------------- */ service NodeService { @@ -865,4 +878,5 @@ service NodeService { rpc StopRebalance(StopRebalanceRequest) returns (StopRebalanceResponse) {}; rpc LoadRebalanceMeta(LoadRebalanceMetaRequest) returns (LoadRebalanceMetaResponse) {}; rpc LoadTransitionTierConfig(LoadTransitionTierConfigRequest) returns (LoadTransitionTierConfigResponse) {}; + rpc GetLiveEvents(GetLiveEventsRequest) returns (GetLiveEventsResponse) {}; } diff --git a/crates/rio/src/encrypt_reader.rs b/crates/rio/src/encrypt_reader.rs index 009e1b810..4f1f39664 100644 --- a/crates/rio/src/encrypt_reader.rs +++ b/crates/rio/src/encrypt_reader.rs @@ -290,18 +290,20 @@ where Poll::Ready(Ok(())) => { let n = temp_buf.filled().len(); if n == 0 { - *this.finished = true; - return Poll::Ready(Ok(())); + if *this.header_read == 0 { + *this.finished = true; + return Poll::Ready(Ok(())); + } + return Poll::Ready(Err(Error::new( + std::io::ErrorKind::UnexpectedEof, + "unexpected EOF while reading encrypted block header", + ))); } this.header_buf[*this.header_read..*this.header_read + n].copy_from_slice(&temp_buf.filled()[..n]); *this.header_read += n; } Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), } - - if *this.header_read < 8 { - return Poll::Pending; - } } if !*this.header_done && *this.header_read == 8 { @@ -374,7 +376,10 @@ where Poll::Ready(Ok(())) => { let n = temp_buf.filled().len(); if n == 0 { - break; + return Poll::Ready(Err(Error::new( + std::io::ErrorKind::UnexpectedEof, + "unexpected EOF while reading encrypted block payload", + ))); } *this.ciphertext_read += n; } @@ -483,12 +488,50 @@ fn derive_part_nonce(base: &[u8; 12], part_number: usize) -> [u8; 12] { #[cfg(test)] mod tests { use std::io::Cursor; + use std::pin::Pin; + use std::task::{Context, Poll}; - use crate::WarpReader; + use crate::{HardLimitReader, WarpReader}; use super::*; + use futures::StreamExt; use rand::{Rng, RngExt}; - use tokio::io::{AsyncReadExt, BufReader}; + use tokio::io::{AsyncRead, AsyncReadExt, BufReader, ReadBuf}; + use tokio_util::io::ReaderStream; + + struct ChunkedCursor { + inner: Cursor>, + max_chunk: usize, + } + + impl ChunkedCursor { + fn new(data: Vec, max_chunk: usize) -> Self { + Self { + inner: Cursor::new(data), + max_chunk, + } + } + } + + impl AsyncRead for ChunkedCursor { + fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + if self.max_chunk == 0 || buf.remaining() == 0 { + return Poll::Ready(Ok(())); + } + + let remaining = self.inner.get_ref().len() as u64 - self.inner.position(); + if remaining == 0 { + return Poll::Ready(Ok(())); + } + + let to_read = remaining.min(self.max_chunk as u64).min(buf.remaining() as u64) as usize; + let start = self.inner.position() as usize; + let end = start + to_read; + buf.put_slice(&self.inner.get_ref()[start..end]); + self.inner.set_position(end as u64); + Poll::Ready(Ok(())) + } + } #[tokio::test] async fn test_encrypt_decrypt_reader_aes256gcm() { @@ -569,6 +612,86 @@ mod tests { assert_eq!(&decrypted, &data); } + #[tokio::test] + async fn test_decrypt_reader_large_with_small_chunks() { + let size = 1024 * 1024; + let mut data = vec![0u8; size]; + rand::rng().fill(&mut data[..]); + let mut key = [0u8; 32]; + let mut nonce = [0u8; 12]; + rand::rng().fill_bytes(&mut key); + rand::rng().fill_bytes(&mut nonce); + + let reader = Cursor::new(data.clone()); + let mut encrypt_reader = EncryptReader::new(WarpReader::new(reader), key, nonce); + let mut encrypted = Vec::new(); + encrypt_reader.read_to_end(&mut encrypted).await.unwrap(); + + let reader = ChunkedCursor::new(encrypted, 3); + let mut decrypt_reader = DecryptReader::new(WarpReader::new(reader), key, nonce); + let mut decrypted = Vec::new(); + decrypt_reader.read_to_end(&mut decrypted).await.unwrap(); + + assert_eq!(decrypted, data); + } + + #[tokio::test] + async fn test_decrypt_reader_large_through_reader_stream() { + let size = 1024 * 1024; + let mut data = vec![0u8; size]; + rand::rng().fill(&mut data[..]); + let mut key = [0u8; 32]; + let mut nonce = [0u8; 12]; + rand::rng().fill_bytes(&mut key); + rand::rng().fill_bytes(&mut nonce); + + let reader = Cursor::new(data.clone()); + let mut encrypt_reader = EncryptReader::new(WarpReader::new(reader), key, nonce); + let mut encrypted = Vec::new(); + encrypt_reader.read_to_end(&mut encrypted).await.unwrap(); + + let reader = ChunkedCursor::new(encrypted, 8192); + let decrypt_reader = DecryptReader::new(WarpReader::new(reader), key, nonce); + let mut stream = ReaderStream::with_capacity(Box::new(decrypt_reader), 262_144); + + let mut decrypted = Vec::new(); + while let Some(chunk) = stream.next().await { + let bytes = chunk.unwrap(); + decrypted.extend_from_slice(&bytes); + } + + assert_eq!(decrypted, data); + } + + #[tokio::test] + async fn test_decrypt_reader_large_through_hard_limit_reader_stream() { + let size = 1024 * 1024; + let mut data = vec![0u8; size]; + rand::rng().fill(&mut data[..]); + let mut key = [0u8; 32]; + let mut nonce = [0u8; 12]; + rand::rng().fill_bytes(&mut key); + rand::rng().fill_bytes(&mut nonce); + + let reader = Cursor::new(data.clone()); + let mut encrypt_reader = EncryptReader::new(WarpReader::new(reader), key, nonce); + let mut encrypted = Vec::new(); + encrypt_reader.read_to_end(&mut encrypted).await.unwrap(); + + let reader = ChunkedCursor::new(encrypted, 8192); + let decrypt_reader = DecryptReader::new(WarpReader::new(reader), key, nonce); + let limit_reader = HardLimitReader::new(Box::new(decrypt_reader), size as i64); + let mut stream = ReaderStream::with_capacity(Box::new(limit_reader), 262_144); + + let mut decrypted = Vec::new(); + while let Some(chunk) = stream.next().await { + let bytes = chunk.unwrap(); + decrypted.extend_from_slice(&bytes); + } + + assert_eq!(decrypted, data); + } + #[tokio::test] async fn test_decrypt_reader_multipart_segments() { let mut key = [0u8; 32]; diff --git a/crates/zip/src/lib.rs b/crates/zip/src/lib.rs index 08c9f9089..5bf2d1789 100644 --- a/crates/zip/src/lib.rs +++ b/crates/zip/src/lib.rs @@ -45,12 +45,12 @@ impl CompressionFormat { /// Identify compression format from file extension pub fn from_extension(ext: &str) -> Self { match ext.to_lowercase().as_str() { - "gz" | "gzip" => CompressionFormat::Gzip, - "bz2" | "bzip2" => CompressionFormat::Bzip2, + "gz" | "gzip" | "tgz" => CompressionFormat::Gzip, + "bz2" | "bzip2" | "tbz" | "tbz2" => CompressionFormat::Bzip2, "zip" => CompressionFormat::Zip, - "xz" => CompressionFormat::Xz, + "xz" | "txz" => CompressionFormat::Xz, "zlib" => CompressionFormat::Zlib, - "zst" | "zstd" => CompressionFormat::Zstd, + "zst" | "zstd" | "tzst" => CompressionFormat::Zstd, "tar" => CompressionFormat::Tar, _ => CompressionFormat::Unknown, } @@ -301,17 +301,23 @@ mod tests { // Test supported compression format recognition assert_eq!(CompressionFormat::from_extension("gz"), CompressionFormat::Gzip); assert_eq!(CompressionFormat::from_extension("gzip"), CompressionFormat::Gzip); + assert_eq!(CompressionFormat::from_extension("tgz"), CompressionFormat::Gzip); assert_eq!(CompressionFormat::from_extension("bz2"), CompressionFormat::Bzip2); assert_eq!(CompressionFormat::from_extension("bzip2"), CompressionFormat::Bzip2); + assert_eq!(CompressionFormat::from_extension("tbz"), CompressionFormat::Bzip2); + assert_eq!(CompressionFormat::from_extension("tbz2"), CompressionFormat::Bzip2); assert_eq!(CompressionFormat::from_extension("zip"), CompressionFormat::Zip); assert_eq!(CompressionFormat::from_extension("xz"), CompressionFormat::Xz); + assert_eq!(CompressionFormat::from_extension("txz"), CompressionFormat::Xz); assert_eq!(CompressionFormat::from_extension("zlib"), CompressionFormat::Zlib); assert_eq!(CompressionFormat::from_extension("zst"), CompressionFormat::Zstd); assert_eq!(CompressionFormat::from_extension("zstd"), CompressionFormat::Zstd); + assert_eq!(CompressionFormat::from_extension("tzst"), CompressionFormat::Zstd); assert_eq!(CompressionFormat::from_extension("tar"), CompressionFormat::Tar); // Test case insensitivity assert_eq!(CompressionFormat::from_extension("GZ"), CompressionFormat::Gzip); + assert_eq!(CompressionFormat::from_extension("TGZ"), CompressionFormat::Gzip); assert_eq!(CompressionFormat::from_extension("ZIP"), CompressionFormat::Zip); // Test unknown formats diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index a2814d28d..f9c87667f 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -89,6 +89,7 @@ reqwest = { workspace = true } socket2 = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "signal", "process", "io-util"] } tokio-rustls = { workspace = true } +aws-sdk-s3 = { workspace = true } tokio-stream.workspace = true tokio-util.workspace = true tonic = { workspace = true } @@ -99,6 +100,7 @@ tower-http = { workspace = true, features = ["trace", "compression-full", "cors" bytes = { workspace = true } flatbuffers.workspace = true rmp-serde.workspace = true +rustfs-signer.workspace = true serde.workspace = true serde_json.workspace = true serde_urlencoded = { workspace = true } @@ -167,7 +169,6 @@ pprof = { workspace = true } uuid = { workspace = true, features = ["v4"] } serial_test = { workspace = true } tempfile = { workspace = true } -aws-sdk-s3 = { workspace = true } aws-config = { workspace = true } anyhow = { workspace = true } tokio = { workspace = true, features = ["test-util"] } diff --git a/rustfs/src/admin/handlers/replication.rs b/rustfs/src/admin/handlers/replication.rs index 9f6367a5d..6c770c582 100644 --- a/rustfs/src/admin/handlers/replication.rs +++ b/rustfs/src/admin/handlers/replication.rs @@ -23,7 +23,7 @@ use hyper::{Method, StatusCode}; use matchit::Params; use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; use rustfs_credentials::Credentials; -use rustfs_ecstore::bucket::bucket_target_sys::BucketTargetSys; +use rustfs_ecstore::bucket::bucket_target_sys::{BucketTargetError, BucketTargetSys}; use rustfs_ecstore::bucket::metadata::BUCKET_TARGETS_FILE; use rustfs_ecstore::bucket::metadata_sys; use rustfs_ecstore::bucket::metadata_sys::get_replication_config; @@ -53,6 +53,22 @@ fn extract_query_params(uri: &Uri) -> HashMap { params } +fn map_bucket_target_error(err: BucketTargetError) -> S3Error { + match err { + BucketTargetError::BucketRemoteTargetNotFound { .. } + | BucketTargetError::BucketRemoteArnTypeInvalid { .. } + | BucketTargetError::BucketRemoteAlreadyExists { .. } + | BucketTargetError::BucketRemoteArnInvalid { .. } + | BucketTargetError::RemoteTargetConnectionErr { .. } + | BucketTargetError::BucketReplicationSourceNotVersioned { .. } + | BucketTargetError::BucketRemoteTargetNotVersioned { .. } + | BucketTargetError::BucketRemoteRemoveDisallowed { .. } => { + S3Error::with_message(S3ErrorCode::InvalidRequest, err.to_string()) + } + BucketTargetError::Io(io_err) => S3Error::with_message(S3ErrorCode::InternalError, io_err.to_string()), + } +} + pub fn register_replication_route(r: &mut S3Router) -> std::io::Result<()> { r.insert( Method::GET, @@ -200,7 +216,7 @@ impl Operation for SetRemoteTargetHandler { })?; let Ok(target_url) = remote_target.url() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Invalid target url".to_string())); + return Err(s3_error!(InvalidRequest, "invalid target url")); }; let same_target = rustfs_utils::net::is_local_host( @@ -232,7 +248,7 @@ impl Operation for SetRemoteTargetHandler { } if remote_target.arn.is_empty() { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "ARN is empty".to_string())); + return Err(S3Error::with_message(S3ErrorCode::InvalidRequest, "ARN is empty".to_string())); } if update { @@ -240,7 +256,7 @@ impl Operation for SetRemoteTargetHandler { .get_remote_bucket_target_by_arn(bucket, &remote_target.arn) .await else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Target not found".to_string())); + return Err(S3Error::with_message(S3ErrorCode::InvalidRequest, "Target not found".to_string())); }; target.credentials = remote_target.credentials; @@ -262,7 +278,7 @@ impl Operation for SetRemoteTargetHandler { bucket_target_sys .set_target(bucket, &remote_target, update) .await - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))?; + .map_err(map_bucket_target_error)?; let targets = bucket_target_sys.list_bucket_targets(bucket).await.map_err(|e| { error!("Failed to list bucket targets: {}", e); @@ -302,20 +318,17 @@ impl Operation for ListRemoteTargetHandler { if let Some(bucket) = queries.get("bucket") { if bucket.is_empty() { error!("bucket parameter is empty"); - return Ok(S3Response::new(( - StatusCode::BAD_REQUEST, - Body::from("Bucket parameter is required".to_string()), - ))); + return Err(s3_error!(InvalidRequest, "bucket is required")); } let Some(store) = new_object_layer_fn() else { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not initialized".to_string())); }; - if let Err(err) = store.get_bucket_info(bucket, &BucketOptions::default()).await { - error!("Error fetching bucket info: {:?}", err); - return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from("Invalid bucket".to_string())))); - } + store + .get_bucket_info(bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; let sys = BucketTargetSys::get(); let targets = sys.list_targets(bucket, "").await; @@ -355,31 +368,31 @@ impl Operation for RemoveRemoteTargetHandler { debug!("remove remote target called"); let queries = extract_query_params(&req.uri); let Some(bucket) = queries.get("bucket") else { - return Ok(S3Response::new(( - StatusCode::BAD_REQUEST, - Body::from("Bucket parameter is required".to_string()), - ))); + return Err(s3_error!(InvalidRequest, "bucket is required")); }; + if bucket.is_empty() { + return Err(s3_error!(InvalidRequest, "bucket is required")); + } let Some(arn_str) = queries.get("arn") else { - return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from("ARN is required".to_string())))); + return Err(s3_error!(InvalidRequest, "arn is required")); + }; + if arn_str.is_empty() { + return Err(s3_error!(InvalidRequest, "arn is required")); }; let Some(store) = new_object_layer_fn() else { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not initialized".to_string())); }; - if let Err(err) = store.get_bucket_info(bucket, &BucketOptions::default()).await { - error!("Error fetching bucket info: {:?}", err); - return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from("Invalid bucket".to_string())))); - } + store + .get_bucket_info(bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; let sys = BucketTargetSys::get(); - sys.remove_target(bucket, arn_str).await.map_err(|e| { - error!("Failed to remove target: {}", e); - S3Error::with_message(S3ErrorCode::InternalError, "Failed to remove target".to_string()) - })?; + sys.remove_target(bucket, arn_str).await.map_err(map_bucket_target_error)?; let targets = sys.list_bucket_targets(bucket).await.map_err(|e| { error!("Failed to list bucket targets: {}", e); diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index 0197723be..3551c4365 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -14,7 +14,17 @@ use crate::admin::console::{is_console_path, make_console_server}; use crate::admin::handlers::oidc::is_oidc_path; +use crate::app::object_usecase::DefaultObjectUsecase; +use crate::auth::{check_key_valid, get_session_token}; +use crate::error::ApiError; +use crate::license::license_check; use crate::server::{ADMIN_PREFIX, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_ADMIN_PREFIX, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH}; +use crate::storage::access::{ReqInfo, authorize_request}; +use aws_sdk_s3::primitives::ByteStream as AwsByteStream; +use bytes::Bytes; +use futures::{Stream, StreamExt}; +use http::HeaderValue; +use http::header::HeaderName; use hyper::HeaderMap; use hyper::Method; use hyper::StatusCode; @@ -22,14 +32,2175 @@ use hyper::Uri; use hyper::http::Extensions; use matchit::Params; use matchit::Router; +use reqwest::Url; +use rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS; +use rustfs_config::{ + ENABLE_KEY, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, + WEBHOOK_SKIP_TLS_VERIFY, +}; +use rustfs_ecstore::bucket::bandwidth::monitor::BandwidthDetails; +use rustfs_ecstore::bucket::bucket_target_sys::{ + BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError, TargetClient, +}; +use rustfs_ecstore::bucket::metadata::BUCKET_TARGETS_FILE; +use rustfs_ecstore::bucket::metadata_sys; +use rustfs_ecstore::bucket::replication::{ + BucketReplicationResyncStatus, BucketStats, GLOBAL_REPLICATION_STATS, ObjectOpts, ReplicationConfigurationExt, ResyncOpts, + get_global_replication_pool, +}; +use rustfs_ecstore::bucket::target::{BucketTarget, BucketTargetType, BucketTargets}; +use rustfs_ecstore::bucket::versioning::VersioningApi; +use rustfs_ecstore::bucket::versioning_sys::BucketVersioningSys; +use rustfs_ecstore::config::com::read_config_without_migrate; +use rustfs_ecstore::config::{Config, get_global_server_config}; +use rustfs_ecstore::global::GLOBAL_BOOT_TIME; +use rustfs_ecstore::notification_sys::get_global_notification_sys; +use rustfs_ecstore::rpc::PeerRestClient; +use rustfs_ecstore::store_api::{BucketOperations, BucketOptions}; +use rustfs_ecstore::{ + global::{get_global_bucket_monitor, get_global_deployment_id, get_global_region}, + new_object_layer_fn, +}; +use rustfs_filemeta::{ReplicationStatusType, ReplicationType}; +use rustfs_madmin::utils::parse_duration; +use rustfs_notify::{Event as NotificationEvent, notification_system}; +use rustfs_policy::policy::action::{Action, S3Action}; +use rustfs_s3_common::EventName; +use rustfs_signer::pre_sign_v4; +use rustfs_utils::http::{ + SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK, SUFFIX_SOURCE_REPLICATION_REQUEST, + SUFFIX_SOURCE_VERSION_ID, get_source_scheme, insert_header, +}; use s3s::Body; +use s3s::S3Error; +use s3s::S3ErrorCode; use s3s::S3Request; use s3s::S3Response; use s3s::S3Result; +use s3s::StdError; +use s3s::dto::{GetObjectInput, GetObjectOutput, IfMatch, IfNoneMatch, Range, StreamingBlob, Timestamp, TimestampFormat}; use s3s::header; use s3s::route::S3Route; use s3s::s3_error; +use s3s::stream::{ByteStream, DynByteStream}; +use std::collections::{HashMap, HashSet}; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::time::SystemTime; +use time::{OffsetDateTime, format_description::well_known::Rfc3339}; +use tokio::sync::{broadcast, mpsc}; +use tokio::time::Duration; +use tokio_stream::wrappers::ReceiverStream; use tower::Service; +use tracing::{error, warn}; +use url::form_urlencoded; +use uuid::Uuid; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ReplicationExtRoute { + MetricsV1, + MetricsV2, + Check, + ResetStart, + ResetStatus, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ReplicationExtRequest { + bucket: String, + route: ReplicationExtRoute, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum MiscExtRoute { + ObjectLambda { bucket: String, object: String }, + ListenNotification { bucket: Option }, +} + +#[derive(Debug, Clone, serde::Serialize, Default)] +struct ReplicationResetResponse { + #[serde(rename = "Targets")] + targets: Vec, +} + +#[derive(Debug, Clone, serde::Serialize, Default)] +struct ReplicationResetTarget { + #[serde(rename = "Arn")] + arn: String, + #[serde(rename = "ResetID")] + reset_id: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ReplicationResetStartRequest { + arn: String, + reset_id: String, + reset_before: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +struct ReplicationResetStatusRequest { + arn: Option, +} + +#[derive(Debug, Clone, serde::Serialize, Default)] +struct ReplicationResetStatusResponse { + #[serde(rename = "Targets")] + targets: Vec, +} + +#[derive(Debug, Clone, serde::Serialize, Default)] +struct ReplicationResetStatusTarget { + #[serde(rename = "Arn")] + arn: String, + #[serde(rename = "ResetID")] + reset_id: String, + #[serde( + rename = "ResetBeforeDate", + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + reset_before_date: Option, + #[serde( + rename = "StartTime", + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + start_time: Option, + #[serde( + rename = "EndTime", + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + end_time: Option, + #[serde(rename = "Status")] + status: String, + #[serde(rename = "ReplicatedCount")] + replicated_count: i64, + #[serde(rename = "ReplicatedSize")] + replicated_size: i64, + #[serde(rename = "FailedCount")] + failed_count: i64, + #[serde(rename = "FailedSize")] + failed_size: i64, + #[serde(rename = "Bucket", skip_serializing_if = "String::is_empty")] + bucket: String, + #[serde(rename = "Object", skip_serializing_if = "String::is_empty")] + object: String, + #[serde(rename = "Error", skip_serializing_if = "Option::is_none")] + error: Option, +} + +#[derive(Debug, Clone, serde::Serialize, Default)] +struct ReplicationCheckTargetStatus { + #[serde(rename = "Arn")] + arn: String, + #[serde(rename = "Endpoint")] + endpoint: String, + #[serde(rename = "Bucket")] + bucket: String, + #[serde(rename = "Status")] + status: String, + #[serde(rename = "Error", skip_serializing_if = "Option::is_none")] + error: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ReplicationCheckFailureContext { + BucketCheck, + VersioningCheck, + ReplicateObject, + ReplicateDeleteMarker, + DeleteObjectVersion, + ObjectLockCheck, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ListenNotificationFilter { + bucket: Option, + event_mask: u64, + prefix: Option, + suffix: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ObjectLambdaWebhookConfig { + endpoint: Url, + auth_token: String, + client_cert: String, + client_key: String, + client_ca: String, + skip_tls_verify: bool, + response_header_timeout: Option, +} + +const LAMBDA_WEBHOOK_SUB_SYS: &str = "lambda_webhook"; +const WEBHOOK_RESPONSE_HEADER_TIMEOUT: &str = "response_header_timeout"; +const OBJECT_LAMBDA_PRESIGN_EXPIRES_SECS: i64 = 3600; + +fn parse_query_pairs(uri: &Uri) -> Vec<(String, String)> { + uri.query() + .map(|query| { + form_urlencoded::parse(query.as_bytes()) + .map(|(k, v)| (k.into_owned(), v.into_owned())) + .collect() + }) + .unwrap_or_default() +} + +fn query_value_exact(uri: &Uri, key: &str) -> Option { + parse_query_pairs(uri) + .into_iter() + .find_map(|(k, v)| if k == key { Some(v) } else { None }) +} + +fn query_values_exact(uri: &Uri, key: &str) -> Vec { + parse_query_pairs(uri) + .into_iter() + .filter_map(|(k, v)| if k == key { Some(v) } else { None }) + .collect() +} + +fn is_valid_filter_rule_value(value: &str) -> bool { + if value.len() > 1024 || value.contains('\\') { + return false; + } + !value.split('/').any(|segment| segment == "." || segment == "..") +} + +fn extract_bucket_for_bucket_level_path(path: &str) -> Option { + let bucket = path.strip_prefix('/')?; + if bucket.is_empty() || bucket.contains('/') { + return None; + } + Some(bucket.to_string()) +} + +fn extract_bucket_object_path(path: &str) -> Option<(String, String)> { + let path = path.strip_prefix('/')?; + let (bucket, object) = path.split_once('/')?; + if bucket.is_empty() || object.is_empty() { + return None; + } + Some((bucket.to_string(), object.to_string())) +} + +fn parse_replication_extension_request(method: &Method, uri: &Uri) -> Option { + let bucket = extract_bucket_for_bucket_level_path(uri.path())?; + + if method == Method::PUT && query_value_exact(uri, "replication-reset").as_deref() == Some("") { + return Some(ReplicationExtRequest { + bucket, + route: ReplicationExtRoute::ResetStart, + }); + } + + if method == Method::GET { + if query_value_exact(uri, "replication-reset-status").as_deref() == Some("") { + return Some(ReplicationExtRequest { + bucket, + route: ReplicationExtRoute::ResetStatus, + }); + } + if let Some(value) = query_value_exact(uri, "replication-metrics") { + if value == "2" { + return Some(ReplicationExtRequest { + bucket, + route: ReplicationExtRoute::MetricsV2, + }); + } + if value.is_empty() { + return Some(ReplicationExtRequest { + bucket, + route: ReplicationExtRoute::MetricsV1, + }); + } + } + if query_value_exact(uri, "replication-check").as_deref() == Some("") { + return Some(ReplicationExtRequest { + bucket, + route: ReplicationExtRoute::Check, + }); + } + } + + None +} + +fn parse_misc_extension_request(method: &Method, uri: &Uri) -> Option { + if method != Method::GET { + return None; + } + + if query_value_exact(uri, "lambdaArn").is_some() + && let Some((bucket, object)) = extract_bucket_object_path(uri.path()) + { + return Some(MiscExtRoute::ObjectLambda { bucket, object }); + } + + if query_value_exact(uri, "events").is_some() { + if uri.path() == "/" { + return Some(MiscExtRoute::ListenNotification { bucket: None }); + } + if let Some(bucket) = extract_bucket_for_bucket_level_path(uri.path()) { + return Some(MiscExtRoute::ListenNotification { bucket: Some(bucket) }); + } + } + + None +} + +fn validate_object_lambda_query(uri: &Uri) -> S3Result<()> { + let lambda_arns = query_values_exact(uri, "lambdaArn"); + if lambda_arns.len() != 1 || lambda_arns[0].trim().is_empty() { + return Err(s3_error!(InvalidRequest, "lambdaArn query parameter must be provided exactly once")); + } + + let lambda_arn = lambda_arns[0].trim(); + let arn_parts = lambda_arn.split(':').collect::>(); + let is_valid_arn = arn_parts.len() >= 6 && arn_parts[0] == "arn" && !arn_parts[1].is_empty() && !arn_parts[2].is_empty(); + if !is_valid_arn { + return Err(s3_error!(InvalidRequest, "lambdaArn query parameter must be a valid ARN string")); + } + Ok(()) +} + +fn validate_listen_notification_query(uri: &Uri) -> S3Result<()> { + let events = query_values_exact(uri, "events"); + if events.is_empty() { + return Err(s3_error!(InvalidArgument, "events query parameter is required")); + } + + for event in events { + EventName::parse(&event).map_err(|_| s3_error!(InvalidArgument, "invalid event in events query parameter"))?; + } + + let prefixes = query_values_exact(uri, "prefix"); + if prefixes.len() > 1 { + return Err(s3_error!(InvalidArgument, "prefix query parameter must not be repeated")); + } + if let Some(prefix) = prefixes.first() + && !is_valid_filter_rule_value(prefix) + { + return Err(s3_error!(InvalidArgument, "invalid prefix filter value")); + } + + let suffixes = query_values_exact(uri, "suffix"); + if suffixes.len() > 1 { + return Err(s3_error!(InvalidArgument, "suffix query parameter must not be repeated")); + } + if let Some(suffix) = suffixes.first() + && !is_valid_filter_rule_value(suffix) + { + return Err(s3_error!(InvalidArgument, "invalid suffix filter value")); + } + + let pings = query_values_exact(uri, "ping"); + if pings.len() > 1 { + return Err(s3_error!(InvalidArgument, "ping query parameter must not be repeated")); + } + if let Some(ping) = pings.first() { + let ping_interval = ping + .parse::() + .map_err(|_| s3_error!(InvalidArgument, "ping query parameter must be a positive integer"))?; + if ping_interval == 0 { + return Err(s3_error!(InvalidArgument, "ping query parameter must be greater than zero")); + } + } + + Ok(()) +} + +fn parse_listen_notification_filter(uri: &Uri, bucket: Option<&str>) -> S3Result { + let mut event_mask = 0_u64; + for event in query_values_exact(uri, "events") { + event_mask |= EventName::parse(&event) + .map_err(|_| s3_error!(InvalidArgument, "invalid event in events query parameter"))? + .mask(); + } + + Ok(ListenNotificationFilter { + bucket: bucket.map(str::to_string), + event_mask, + prefix: query_value_exact(uri, "prefix").filter(|value| !value.is_empty()), + suffix: query_value_exact(uri, "suffix").filter(|value| !value.is_empty()), + }) +} + +fn validate_misc_extension_request(uri: &Uri, route: &MiscExtRoute) -> S3Result<()> { + match route { + MiscExtRoute::ObjectLambda { .. } => validate_object_lambda_query(uri), + MiscExtRoute::ListenNotification { .. } => validate_listen_notification_query(uri), + } +} + +fn query_pairs_without_key(uri: &Uri, excluded_key: &str) -> Vec<(String, String)> { + parse_query_pairs(uri) + .into_iter() + .filter(|(key, _)| key != excluded_key) + .collect() +} + +fn uri_without_query_key(uri: &Uri, excluded_key: &str) -> S3Result { + let filtered = query_pairs_without_key(uri, excluded_key); + let mut parts = uri.clone().into_parts(); + parts.path_and_query = if filtered.is_empty() { + Some( + uri.path() + .parse() + .map_err(|_| s3_error!(InvalidRequest, "failed to rebuild request URI"))?, + ) + } else { + let query = form_urlencoded::Serializer::new(String::new()) + .extend_pairs(filtered.iter().map(|(key, value)| (key.as_str(), value.as_str()))) + .finish(); + Some( + format!("{}?{}", uri.path(), query) + .parse() + .map_err(|_| s3_error!(InvalidRequest, "failed to rebuild request URI"))?, + ) + }; + Uri::from_parts(parts).map_err(|_| s3_error!(InvalidRequest, "failed to rebuild request URI")) +} + +fn parse_optional_header(headers: &HeaderMap, name: HeaderName) -> S3Result> { + headers + .get(name) + .map(|value| { + value + .to_str() + .map(|parsed| parsed.to_string()) + .map_err(|_| s3_error!(InvalidRequest, "request header contains invalid utf-8")) + }) + .transpose() +} + +fn parse_optional_timestamp_header(headers: &HeaderMap, name: HeaderName) -> S3Result> { + parse_optional_header(headers, name)? + .map(|value| { + Timestamp::parse(TimestampFormat::HttpDate, &value) + .map_err(|_| s3_error!(InvalidRequest, "request timestamp header is invalid")) + }) + .transpose() +} + +fn parse_optional_etag_condition_header(headers: &HeaderMap, name: HeaderName) -> S3Result> +where + T: std::str::FromStr, +{ + parse_optional_header(headers, name)? + .map(|value| { + value + .parse::() + .map_err(|_| s3_error!(InvalidRequest, "request etag condition header is invalid")) + }) + .transpose() +} + +fn build_object_lambda_get_request(req: &S3Request, bucket: &str, object: &str) -> S3Result> { + let filtered_uri = uri_without_query_key(&req.uri, "lambdaArn")?; + let part_number = query_value_exact(&filtered_uri, "partNumber") + .filter(|value| !value.is_empty()) + .map(|value| { + value + .parse::() + .map_err(|_| s3_error!(InvalidArgument, "partNumber query parameter must be a positive integer")) + }) + .transpose()?; + let version_id = query_value_exact(&filtered_uri, "versionId").filter(|value| !value.is_empty()); + let range = parse_optional_header(&req.headers, http::header::RANGE)? + .map(|value| Range::parse(&value).map_err(|_| s3_error!(InvalidArgument, "Range header is invalid"))) + .transpose()?; + + let mut builder = GetObjectInput::builder() + .bucket(bucket.to_string()) + .key(object.to_string()) + .part_number(part_number) + .version_id(version_id) + .range(range) + .if_match(parse_optional_etag_condition_header::(&req.headers, http::header::IF_MATCH)?) + .if_none_match(parse_optional_etag_condition_header::( + &req.headers, + http::header::IF_NONE_MATCH, + )?) + .if_modified_since(parse_optional_timestamp_header(&req.headers, http::header::IF_MODIFIED_SINCE)?) + .if_unmodified_since(parse_optional_timestamp_header(&req.headers, http::header::IF_UNMODIFIED_SINCE)?); + + builder = builder.sse_customer_algorithm(parse_optional_header( + &req.headers, + HeaderName::from_static("x-amz-server-side-encryption-customer-algorithm"), + )?); + builder = builder.sse_customer_key(parse_optional_header( + &req.headers, + HeaderName::from_static("x-amz-server-side-encryption-customer-key"), + )?); + builder = builder.sse_customer_key_md5(parse_optional_header( + &req.headers, + HeaderName::from_static("x-amz-server-side-encryption-customer-key-md5"), + )?); + + let input = builder + .build() + .map_err(|err| s3_error!(InvalidRequest, "failed to build object lambda get request: {err}"))?; + + Ok(S3Request { + input, + method: req.method.clone(), + uri: filtered_uri, + headers: req.headers.clone(), + extensions: req.extensions.clone(), + credentials: req.credentials.clone(), + region: req.region.clone(), + service: req.service.clone(), + trailing_headers: req.trailing_headers.clone(), + }) +} + +fn parse_object_lambda_arn(uri: &Uri) -> S3Result { + let lambda_arn = query_value_exact(uri, "lambdaArn") + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| s3_error!(InvalidRequest, "lambdaArn query parameter must be provided exactly once"))?; + + lambda_arn + .parse::() + .map_err(|_| s3_error!(InvalidRequest, "lambdaArn query parameter must reference a supported target ARN")) +} + +fn config_enable_is_on(value: &str) -> bool { + matches!(value.trim().to_ascii_lowercase().as_str(), "on" | "true" | "yes" | "1") +} + +fn resolve_object_lambda_webhook_config_from_server_config( + config: &Config, + arn: &rustfs_targets::arn::ARN, +) -> S3Result { + let target_name = arn.target_id.name.to_ascii_lowercase(); + if target_name != "webhook" && !target_name.starts_with("webhook-") { + return Err(s3_error!(NotImplemented, "object lambda target type is not supported")); + } + + let subsystem = config + .0 + .get(LAMBDA_WEBHOOK_SUB_SYS) + .or_else(|| config.0.get(NOTIFY_WEBHOOK_SUB_SYS)) + .ok_or_else(|| s3_error!(InvalidRequest, "object lambda webhook subsystem is not configured"))?; + let kvs = subsystem + .get(&arn.target_id.id) + .ok_or_else(|| s3_error!(InvalidRequest, "object lambda target is not configured"))?; + + if !config_enable_is_on(&kvs.get(ENABLE_KEY)) { + return Err(s3_error!(InvalidRequest, "object lambda target is disabled")); + } + + let endpoint = kvs.lookup(WEBHOOK_ENDPOINT).unwrap_or_default(); + if endpoint.trim().is_empty() { + return Err(s3_error!(InvalidRequest, "object lambda target endpoint is empty")); + } + + let response_header_timeout = match kvs.lookup(WEBHOOK_RESPONSE_HEADER_TIMEOUT) { + Some(value) if value.trim().is_empty() => None, + Some(value) => Some( + parse_duration(&value) + .map_err(|_| s3_error!(InvalidRequest, "object lambda target response_header_timeout is invalid"))?, + ), + None => None, + }; + + Ok(ObjectLambdaWebhookConfig { + endpoint: Url::parse(&endpoint).map_err(|_| s3_error!(InvalidRequest, "object lambda target endpoint is invalid"))?, + auth_token: kvs.lookup(WEBHOOK_AUTH_TOKEN).unwrap_or_default(), + client_cert: kvs.lookup(WEBHOOK_CLIENT_CERT).unwrap_or_default(), + client_key: kvs.lookup(WEBHOOK_CLIENT_KEY).unwrap_or_default(), + client_ca: kvs.lookup(WEBHOOK_CLIENT_CA).unwrap_or_default(), + skip_tls_verify: config_enable_is_on(&kvs.lookup(WEBHOOK_SKIP_TLS_VERIFY).unwrap_or_default()), + response_header_timeout, + }) +} + +async fn load_current_server_config() -> S3Result { + if let Some(system) = notification_system() { + return Ok(system.config.read().await.clone()); + } + + if let Some(store) = new_object_layer_fn() { + match read_config_without_migrate(store).await { + Ok(config) => return Ok(config), + Err(err) => { + warn!("failed to reload current server config for object lambda request: {err}"); + } + } + } + + let config = get_global_server_config().ok_or_else(|| s3_error!(InternalError, "server config is not initialized"))?; + Ok(config) +} + +async fn resolve_object_lambda_webhook_config(uri: &Uri) -> S3Result { + let config = load_current_server_config().await?; + let arn = parse_object_lambda_arn(uri)?; + resolve_object_lambda_webhook_config_from_server_config(&config, &arn) +} + +fn build_object_lambda_http_client(config: &ObjectLambdaWebhookConfig) -> S3Result { + let mut builder = reqwest::Client::builder().user_agent(rustfs_utils::get_user_agent(rustfs_utils::ServiceType::Basis)); + + if let Some(timeout) = config.response_header_timeout { + builder = builder.timeout(timeout); + } + + if config.skip_tls_verify { + builder = builder.danger_accept_invalid_certs(true); + } else if !config.client_ca.is_empty() { + let ca_pem = std::fs::read(&config.client_ca) + .map_err(|e| s3_error!(InternalError, "failed to read object lambda client_ca: {e}"))?; + let ca = reqwest::Certificate::from_pem(&ca_pem) + .map_err(|e| s3_error!(InternalError, "failed to parse object lambda client_ca: {e}"))?; + builder = builder.add_root_certificate(ca); + } + + if !config.client_cert.is_empty() || !config.client_key.is_empty() { + if config.client_cert.is_empty() || config.client_key.is_empty() { + return Err(s3_error!( + InvalidRequest, + "object lambda client_cert and client_key must be configured together" + )); + } + + let cert = std::fs::read(&config.client_cert) + .map_err(|e| s3_error!(InternalError, "failed to read object lambda client_cert: {e}"))?; + let key = std::fs::read(&config.client_key) + .map_err(|e| s3_error!(InternalError, "failed to read object lambda client_key: {e}"))?; + let identity = reqwest::Identity::from_pem(&[cert, key].concat()) + .map_err(|e| s3_error!(InternalError, "failed to build object lambda client identity: {e}"))?; + builder = builder.identity(identity); + } + + builder + .build() + .map_err(|e| s3_error!(InternalError, "failed to build object lambda http client: {e}")) +} + +fn extract_request_scheme(headers: &HeaderMap, uri: &Uri) -> String { + get_source_scheme(headers) + .and_then(|value| { + value + .split(',') + .next() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) + }) + .or_else(|| uri.scheme_str().map(str::to_owned)) + .unwrap_or_else(|| "http".to_string()) + .to_ascii_lowercase() +} + +fn extract_request_host(headers: &HeaderMap, uri: &Uri) -> Option { + headers + .get(http::header::HOST) + .and_then(|value| value.to_str().ok()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) + .or_else(|| uri.authority().map(|authority| authority.as_str().to_string())) +} + +fn build_object_lambda_source_url(req: &S3Request) -> S3Result { + let credentials = req + .credentials + .as_ref() + .ok_or_else(|| s3_error!(AccessDenied, "object lambda source URL requires authenticated credentials"))?; + let host = extract_request_host(&req.headers, &req.uri) + .ok_or_else(|| s3_error!(InvalidRequest, "object lambda source URL requires a valid host header"))?; + let scheme = extract_request_scheme(&req.headers, &req.uri); + let filtered_uri = uri_without_query_key(&req.uri, "lambdaArn")?; + let path_and_query = filtered_uri + .path_and_query() + .map(|value| value.as_str().to_string()) + .unwrap_or_else(|| filtered_uri.path().to_string()); + let source_uri = format!("{scheme}://{host}{path_and_query}") + .parse::() + .map_err(|e| s3_error!(InvalidRequest, "failed to construct object lambda source URL: {e}"))?; + let region = req + .region + .clone() + .or_else(get_global_region) + .map(|value| value.as_str().to_string()) + .unwrap_or_else(|| "us-east-1".to_string()); + let session_token = get_session_token(&req.uri, &req.headers).unwrap_or_default().to_string(); + + let presigned = pre_sign_v4( + http::Request::builder() + .method(Method::GET) + .uri(source_uri) + .header(http::header::HOST, host) + .body(Body::default()) + .map_err(|e| s3_error!(InvalidRequest, "failed to build object lambda source request: {e}"))?, + &credentials.access_key, + credentials.secret_key.expose(), + &session_token, + ®ion, + OBJECT_LAMBDA_PRESIGN_EXPIRES_SECS, + OffsetDateTime::now_utc(), + ); + + Ok(presigned.uri().to_string()) +} + +fn build_object_lambda_event_payload( + req: &S3Request, + lambda_arn: &str, + input_s3_url: &str, + output_route: &str, + output_token: &str, +) -> S3Result> { + let request_headers = req + .headers + .iter() + .filter_map(|(name, value)| value.to_str().ok().map(|value| (name.to_string(), value.to_string()))) + .collect::>(); + + serde_json::to_vec(&serde_json::json!({ + "getObjectContext": { + "inputS3Url": input_s3_url, + "outputRoute": output_route, + "outputToken": output_token, + }, + "configuration": { + "accessPointArn": lambda_arn, + }, + "userRequest": { + "url": req.uri.to_string(), + "headers": request_headers, + }, + "protocolVersion": "rustfs-object-lambda-1.0", + })) + .map_err(|e| s3_error!(InternalError, "failed to serialize object lambda payload: {e}")) +} + +fn validate_object_lambda_response_auth_headers(headers: &HeaderMap, output_route: &str, output_token: &str) -> S3Result<()> { + let route = headers + .get("x-amz-request-route") + .and_then(|value| value.to_str().ok()) + .map(str::trim); + let token = headers + .get("x-amz-request-token") + .and_then(|value| value.to_str().ok()) + .map(str::trim); + + if route == Some(output_route) && token == Some(output_token) { + return Ok(()); + } + + Err(s3_error!( + InvalidRequest, + "object lambda target response is missing or contains invalid authorization headers" + )) +} + +fn format_timestamp_http_date(value: &Timestamp) -> S3Result { + let mut buf = Vec::new(); + value + .format(TimestampFormat::HttpDate, &mut buf) + .map_err(|_| s3_error!(InternalError, "failed to format timestamp header"))?; + String::from_utf8(buf).map_err(|_| s3_error!(InternalError, "failed to format timestamp header")) +} + +fn insert_string_header(headers: &mut HeaderMap, name: HeaderName, value: String) -> S3Result<()> { + let header_value = + HeaderValue::from_str(&value).map_err(|_| s3_error!(InternalError, "failed to build response header value"))?; + headers.insert(name, header_value); + Ok(()) +} + +fn build_get_object_response_headers(output: &GetObjectOutput, base_headers: &HeaderMap) -> S3Result { + let mut headers = base_headers.clone(); + + if let Some(accept_ranges) = &output.accept_ranges { + insert_string_header(&mut headers, http::header::ACCEPT_RANGES, accept_ranges.clone())?; + } + if let Some(cache_control) = &output.cache_control { + insert_string_header(&mut headers, http::header::CACHE_CONTROL, cache_control.clone())?; + } + if let Some(content_disposition) = &output.content_disposition { + insert_string_header(&mut headers, http::header::CONTENT_DISPOSITION, content_disposition.clone())?; + } + if let Some(content_encoding) = &output.content_encoding { + insert_string_header(&mut headers, http::header::CONTENT_ENCODING, content_encoding.clone())?; + } + if let Some(content_language) = &output.content_language { + insert_string_header(&mut headers, http::header::CONTENT_LANGUAGE, content_language.clone())?; + } + if let Some(content_length) = output.content_length { + insert_string_header(&mut headers, http::header::CONTENT_LENGTH, content_length.to_string())?; + } + if let Some(content_range) = &output.content_range { + insert_string_header(&mut headers, http::header::CONTENT_RANGE, content_range.clone())?; + } + if let Some(content_type) = &output.content_type { + insert_string_header(&mut headers, http::header::CONTENT_TYPE, content_type.to_string())?; + } + if let Some(etag) = &output.e_tag { + headers.insert( + http::header::ETAG, + etag.to_http_header().map_err(|_| s3_error!(InternalError, "invalid etag"))?, + ); + } + if let Some(last_modified) = &output.last_modified { + insert_string_header(&mut headers, http::header::LAST_MODIFIED, format_timestamp_http_date(last_modified)?)?; + } + if let Some(expires) = &output.expires { + insert_string_header(&mut headers, http::header::EXPIRES, format_timestamp_http_date(expires)?)?; + } + if let Some(version_id) = &output.version_id { + insert_string_header(&mut headers, HeaderName::from_static("x-amz-version-id"), version_id.clone())?; + } + if let Some(server_side_encryption) = &output.server_side_encryption { + insert_string_header( + &mut headers, + HeaderName::from_static("x-amz-server-side-encryption"), + server_side_encryption.as_str().to_string(), + )?; + } + if let Some(sse_customer_algorithm) = &output.sse_customer_algorithm { + insert_string_header( + &mut headers, + HeaderName::from_static("x-amz-server-side-encryption-customer-algorithm"), + sse_customer_algorithm.clone(), + )?; + } + if let Some(sse_customer_key_md5) = &output.sse_customer_key_md5 { + insert_string_header( + &mut headers, + HeaderName::from_static("x-amz-server-side-encryption-customer-key-md5"), + sse_customer_key_md5.clone(), + )?; + } + if let Some(sse_kms_key_id) = &output.ssekms_key_id { + insert_string_header( + &mut headers, + HeaderName::from_static("x-amz-server-side-encryption-aws-kms-key-id"), + sse_kms_key_id.clone(), + )?; + } + if let Some(checksum_crc32) = &output.checksum_crc32 { + insert_string_header(&mut headers, HeaderName::from_static("x-amz-checksum-crc32"), checksum_crc32.clone())?; + } + if let Some(checksum_crc32c) = &output.checksum_crc32c { + insert_string_header(&mut headers, HeaderName::from_static("x-amz-checksum-crc32c"), checksum_crc32c.clone())?; + } + if let Some(checksum_crc64nvme) = &output.checksum_crc64nvme { + insert_string_header( + &mut headers, + HeaderName::from_static("x-amz-checksum-crc64nvme"), + checksum_crc64nvme.clone(), + )?; + } + if let Some(checksum_sha1) = &output.checksum_sha1 { + insert_string_header(&mut headers, HeaderName::from_static("x-amz-checksum-sha1"), checksum_sha1.clone())?; + } + if let Some(checksum_sha256) = &output.checksum_sha256 { + insert_string_header(&mut headers, HeaderName::from_static("x-amz-checksum-sha256"), checksum_sha256.clone())?; + } + if let Some(checksum_type) = &output.checksum_type { + insert_string_header( + &mut headers, + HeaderName::from_static("x-amz-checksum-type"), + checksum_type.as_str().to_string(), + )?; + } + if let Some(storage_class) = &output.storage_class { + insert_string_header( + &mut headers, + HeaderName::from_static("x-amz-storage-class"), + storage_class.as_str().to_string(), + )?; + } + if let Some(tag_count) = output.tag_count { + insert_string_header(&mut headers, HeaderName::from_static("x-amz-tagging-count"), tag_count.to_string())?; + } + if let Some(expiration) = &output.expiration { + insert_string_header(&mut headers, HeaderName::from_static("x-amz-expiration"), expiration.clone())?; + } + if let Some(restore) = &output.restore { + insert_string_header(&mut headers, HeaderName::from_static("x-amz-restore"), restore.clone())?; + } + + if let Some(metadata) = &output.metadata { + for (key, value) in metadata { + let header_name = format!("x-amz-meta-{key}"); + if let Ok(parsed_name) = HeaderName::from_bytes(header_name.as_bytes()) { + let parsed_value = HeaderValue::from_str(value) + .map_err(|_| s3_error!(InternalError, "failed to build metadata response header"))?; + headers.insert(parsed_name, parsed_value); + } + } + } + + Ok(headers) +} + +#[cfg_attr(not(test), allow(dead_code))] +fn convert_get_object_response(resp: S3Response) -> S3Result> { + let headers = build_get_object_response_headers(&resp.output, &resp.headers)?; + + let body = resp.output.body.map(Body::from).unwrap_or_else(|| Body::from(String::new())); + + Ok(S3Response { + output: body, + status: resp.status, + headers, + extensions: resp.extensions, + }) +} + +fn clear_object_lambda_variant_headers(headers: &mut HeaderMap) { + for name in [ + http::header::ACCEPT_RANGES, + http::header::CACHE_CONTROL, + http::header::CONTENT_DISPOSITION, + http::header::CONTENT_ENCODING, + http::header::CONTENT_LANGUAGE, + http::header::CONTENT_LENGTH, + http::header::CONTENT_RANGE, + http::header::CONTENT_TYPE, + http::header::ETAG, + http::header::LAST_MODIFIED, + http::header::EXPIRES, + HeaderName::from_static("x-amz-checksum-crc32"), + HeaderName::from_static("x-amz-checksum-crc32c"), + HeaderName::from_static("x-amz-checksum-crc64nvme"), + HeaderName::from_static("x-amz-checksum-sha1"), + HeaderName::from_static("x-amz-checksum-sha256"), + HeaderName::from_static("x-amz-checksum-type"), + HeaderName::from_static("x-amz-tagging-count"), + HeaderName::from_static("x-amz-request-route"), + HeaderName::from_static("x-amz-request-token"), + ] { + headers.remove(name); + } + + let metadata_headers = headers + .keys() + .filter(|name| name.as_str().starts_with("x-amz-meta-")) + .cloned() + .collect::>(); + for name in metadata_headers { + headers.remove(name); + } +} + +fn is_disallowed_object_lambda_response_header(name: &HeaderName) -> bool { + matches!( + name.as_str(), + "connection" + | "keep-alive" + | "proxy-authenticate" + | "proxy-authorization" + | "te" + | "trailer" + | "transfer-encoding" + | "upgrade" + ) +} + +fn build_object_lambda_passthrough_response( + mut response_headers: HeaderMap, + lambda_headers: &HeaderMap, + status: StatusCode, + body: Body, +) -> S3Response { + clear_object_lambda_variant_headers(&mut response_headers); + for (name, value) in lambda_headers { + if !is_disallowed_object_lambda_response_header(name) && name != "x-amz-request-route" && name != "x-amz-request-token" { + response_headers.insert(name.clone(), value.clone()); + } + } + + S3Response { + output: body, + status: Some(status), + headers: response_headers, + extensions: Extensions::new(), + } +} + +async fn invoke_object_lambda_target( + req: &S3Request, + bucket: &str, + object: &str, + get_resp: S3Response, +) -> S3Result> { + let lambda_config = resolve_object_lambda_webhook_config(&req.uri).await?; + let client = build_object_lambda_http_client(&lambda_config)?; + let lambda_arn = query_value_exact(&req.uri, "lambdaArn") + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| s3_error!(InvalidRequest, "lambdaArn query parameter must be provided exactly once"))?; + let input_s3_url = build_object_lambda_source_url(req)?; + let output_route = Uuid::new_v4().to_string(); + let output_token = Uuid::new_v4().to_string(); + let event_payload = build_object_lambda_event_payload(req, &lambda_arn, &input_s3_url, &output_route, &output_token)?; + + let S3Response { + output, + headers: upstream_headers, + .. + } = get_resp; + + let response_headers = build_get_object_response_headers(&output, &upstream_headers)?; + + let mut request_builder = client + .post(lambda_config.endpoint) + .header("x-rustfs-object-lambda-bucket", bucket) + .header("x-rustfs-object-lambda-key", object) + .header("x-rustfs-object-lambda-request-uri", req.uri.to_string()) + .header(http::header::CONTENT_TYPE, "application/json") + .body(event_payload); + + if !lambda_config.auth_token.is_empty() { + let tokens = lambda_config.auth_token.split_whitespace().collect::>(); + request_builder = match tokens.as_slice() { + [scheme, token] if !scheme.is_empty() && !token.is_empty() => { + request_builder.header(reqwest::header::AUTHORIZATION, lambda_config.auth_token) + } + [token] if !token.is_empty() => request_builder.header(reqwest::header::AUTHORIZATION, format!("Bearer {token}")), + _ => request_builder, + }; + } + + if let Some(version_id) = output.version_id.as_deref() { + request_builder = request_builder.header("x-rustfs-object-lambda-version-id", version_id); + } + + let lambda_response = request_builder + .send() + .await + .map_err(|e| s3_error!(InternalError, "object lambda target request failed: {e}"))?; + + let status = lambda_response.status(); + let lambda_headers = lambda_response.headers().clone(); + if status.is_success() { + validate_object_lambda_response_auth_headers(&lambda_headers, &output_route, &output_token)?; + } + let body = Body::from(StreamingBlob::wrap(lambda_response.bytes_stream())); + Ok(build_object_lambda_passthrough_response(response_headers, &lambda_headers, status, body)) +} + +struct ListenNotificationStream { + inner: ReceiverStream>, +} + +struct PeerLiveEventCursor { + client: PeerRestClient, + next_sequence: u64, +} + +const LISTEN_NOTIFICATION_PEER_BATCH_LIMIT: u32 = 128; +const LISTEN_NOTIFICATION_PEER_POLL_INTERVAL: Duration = Duration::from_millis(250); + +impl Stream for ListenNotificationStream { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = Pin::into_inner(self); + this.inner.poll_next_unpin(cx) + } +} + +impl ByteStream for ListenNotificationStream {} + +fn listen_notification_keepalive_plan(uri: &Uri) -> (Duration, Bytes) { + if let Some(ping_seconds) = query_value_exact(uri, "ping").and_then(|v| v.parse::().ok()) { + return (Duration::from_secs(ping_seconds), Bytes::from_static(b"{\"Records\":[]}\n")); + } + + (Duration::from_millis(500), Bytes::from_static(b" ")) +} + +fn event_matches_listen_notification(event: &NotificationEvent, filter: &ListenNotificationFilter) -> bool { + if let Some(bucket) = &filter.bucket + && event.s3.bucket.name != *bucket + { + return false; + } + + if filter.event_mask != 0 && event.event_name.mask() & filter.event_mask == 0 { + return false; + } + + let object_key = urlencoding::decode(&event.s3.object.key) + .map(|decoded| decoded.into_owned()) + .unwrap_or_else(|_| event.s3.object.key.clone()); + + if let Some(prefix) = &filter.prefix + && !object_key.starts_with(prefix) + { + return false; + } + + if let Some(suffix) = &filter.suffix + && !object_key.ends_with(suffix) + { + return false; + } + + true +} + +fn serialize_listen_notification_event(event: &NotificationEvent) -> S3Result { + #[derive(serde::Serialize)] + struct ListenNotificationEnvelope<'a> { + #[serde(rename = "Records")] + records: [&'a NotificationEvent; 1], + } + + serde_json::to_vec(&ListenNotificationEnvelope { records: [event] }) + .map(|mut payload| { + payload.push(b'\n'); + Bytes::from(payload) + }) + .map_err(|e| s3_error!(InternalError, "failed to serialize notification event: {e}")) +} + +fn list_remote_live_event_peers() -> Vec { + get_global_notification_sys() + .map(|system| { + system + .peer_clients + .iter() + .flatten() + .cloned() + .map(|client| PeerLiveEventCursor { + client, + next_sequence: 0, + }) + .collect() + }) + .unwrap_or_default() +} + +fn deserialize_peer_live_events(payload: &[u8]) -> Result, serde_json::Error> { + serde_json::from_slice(payload) +} + +async fn fan_in_remote_live_events( + peers: &mut [PeerLiveEventCursor], + filter: &ListenNotificationFilter, + tx: &mpsc::Sender>, +) -> bool { + for peer in peers.iter_mut() { + loop { + let batch = match tokio::time::timeout( + Duration::from_secs(2), + peer.client + .get_live_events(peer.next_sequence, LISTEN_NOTIFICATION_PEER_BATCH_LIMIT), + ) + .await + { + Ok(Ok(batch)) => batch, + Ok(Err(err)) => { + warn!("failed to fetch live events from peer {}: {err}", peer.client.host); + break; + } + Err(_) => { + warn!("timed out fetching live events from peer {}", peer.client.host); + break; + } + }; + + peer.next_sequence = batch.next_sequence.max(peer.next_sequence); + + if !batch.events.is_empty() { + match deserialize_peer_live_events(&batch.events) { + Ok(events) => { + for event in events { + if !event_matches_listen_notification(&event, filter) { + continue; + } + match serialize_listen_notification_event(&event) { + Ok(serialized) => { + if tx.send(Ok(serialized)).await.is_err() { + return false; + } + } + Err(err) => { + warn!("failed to serialize remote listen notification event: {err}"); + } + } + } + } + Err(err) => { + warn!("failed to decode live events from peer {}: {err}", peer.client.host); + } + } + } + + if !batch.truncated { + break; + } + } + } + + true +} + +fn build_listen_notification_response(uri: &Uri, bucket: Option<&str>) -> S3Result> { + let (interval_duration, payload) = listen_notification_keepalive_plan(uri); + let filter = parse_listen_notification_filter(uri, bucket)?; + let mut live_events = notification_system().map(|system| system.subscribe_live_events()); + let mut peer_live_events = list_remote_live_event_peers(); + + let (tx, rx) = mpsc::channel(16); + let stream: DynByteStream = Box::pin(ListenNotificationStream { + inner: ReceiverStream::new(rx), + }); + + tokio::spawn(async move { + let mut ticker = tokio::time::interval(interval_duration); + let mut peer_ticker = tokio::time::interval(LISTEN_NOTIFICATION_PEER_POLL_INTERVAL); + // Skip the immediate first tick so behavior starts after interval duration. + ticker.tick().await; + peer_ticker.tick().await; + loop { + if let Some(events_rx) = live_events.as_mut() { + tokio::select! { + _ = tx.closed() => break, + _ = ticker.tick() => { + if tx.send(Ok(payload.clone())).await.is_err() { + break; + } + } + event = events_rx.recv() => { + match event { + Ok(event) => { + if !event_matches_listen_notification(&event, &filter) { + continue; + } + match serialize_listen_notification_event(&event) { + Ok(serialized) => { + if tx.send(Ok(serialized)).await.is_err() { + break; + } + } + Err(err) => { + warn!("failed to serialize listen notification event: {err}"); + } + } + } + Err(broadcast::error::RecvError::Lagged(skipped)) => { + warn!("listen notification stream lagged and skipped {skipped} events"); + } + Err(broadcast::error::RecvError::Closed) => break, + } + } + _ = peer_ticker.tick(), if !peer_live_events.is_empty() => { + if !fan_in_remote_live_events(&mut peer_live_events, &filter, &tx).await { + break; + } + } + } + } else { + tokio::select! { + _ = tx.closed() => break, + _ = ticker.tick() => { + if tx.send(Ok(payload.clone())).await.is_err() { + break; + } + } + _ = peer_ticker.tick(), if !peer_live_events.is_empty() => { + if !fan_in_remote_live_events(&mut peer_live_events, &filter, &tx).await { + break; + } + } + } + } + } + }); + + let mut resp = S3Response::with_status(Body::from(stream), StatusCode::OK); + resp.headers + .insert(header::CONTENT_TYPE, HeaderValue::from_static("text/event-stream")); + resp.headers + .insert(header::CACHE_CONTROL, HeaderValue::from_static("no-cache")); + resp.headers.insert("x-accel-buffering", HeaderValue::from_static("no")); + Ok(resp) +} + +async fn ensure_replication_bucket_exists(bucket: &str) -> S3Result<()> { + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init")); + }; + + store + .get_bucket_info(bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + + Ok(()) +} + +async fn ensure_replication_config_exists(bucket: &str) -> S3Result<()> { + match metadata_sys::get_replication_config(bucket).await { + Ok(_) => Ok(()), + Err(rustfs_ecstore::error::StorageError::ConfigNotFound) => Err(s3_error!(ReplicationConfigurationNotFoundError)), + Err(err) => Err(ApiError::from(err).into()), + } +} + +async fn build_replication_metrics_response(bucket: &str, route: ReplicationExtRoute) -> S3Result> { + let bucket_stats = match GLOBAL_REPLICATION_STATS.get() { + Some(stats) => stats.get_latest_replication_stats(bucket).await, + None => BucketStats::default(), + }; + let bucket_stats = apply_replication_metrics_bandwidth_report(bucket_stats, collect_replication_metrics_bandwidth(bucket)); + let bucket_stats = apply_replication_metrics_runtime_fields(bucket_stats, route, replication_metrics_uptime_seconds()); + + let body = serialize_replication_metrics_body(&bucket_stats, route)?; + + let mut resp = S3Response::with_status(Body::from(body), StatusCode::OK); + resp.headers + .insert(header::CONTENT_TYPE, HeaderValue::from_static("application/json")); + Ok(resp) +} + +fn replication_metrics_uptime_seconds() -> i64 { + GLOBAL_BOOT_TIME + .get() + .and_then(|boot_time| SystemTime::now().duration_since(*boot_time).ok()) + .map(|uptime| uptime.as_secs() as i64) + .unwrap_or_default() +} + +fn collect_replication_metrics_bandwidth(bucket: &str) -> HashMap { + get_global_bucket_monitor() + .map(|monitor| { + monitor + .get_report(|name| name == bucket) + .bucket_stats + .into_iter() + .filter_map(|(opts, details)| { + if opts.replication_arn.is_empty() { + None + } else { + Some((opts.replication_arn, details)) + } + }) + .collect() + }) + .unwrap_or_default() +} + +fn apply_replication_metrics_bandwidth_report( + mut bucket_stats: BucketStats, + bandwidth_report: HashMap, +) -> BucketStats { + for (arn, details) in bandwidth_report { + let stat = bucket_stats.replication_stats.stats.entry(arn).or_default(); + stat.bandwidth_limit_bytes_per_sec = details.limit_bytes_per_sec; + stat.current_bandwidth_bytes_per_sec = details.current_bandwidth_bytes_per_sec; + } + + bucket_stats +} + +fn apply_replication_metrics_runtime_fields( + mut bucket_stats: BucketStats, + route: ReplicationExtRoute, + uptime_seconds: i64, +) -> BucketStats { + if route == ReplicationExtRoute::MetricsV2 { + bucket_stats.uptime = uptime_seconds; + } + bucket_stats +} + +fn serialize_replication_metrics_body(bucket_stats: &BucketStats, route: ReplicationExtRoute) -> S3Result> { + match route { + ReplicationExtRoute::MetricsV1 => { + serde_json::to_vec(&bucket_stats.replication_stats).map_err(|e| s3_error!(InternalError, "{e}")) + } + ReplicationExtRoute::MetricsV2 => serde_json::to_vec(bucket_stats).map_err(|e| s3_error!(InternalError, "{e}")), + ReplicationExtRoute::Check | ReplicationExtRoute::ResetStart | ReplicationExtRoute::ResetStatus => { + Err(s3_error!(InternalError, "invalid route for metrics response")) + } + } +} + +async fn authorize_replication_extension_request(req: &mut S3Request, ext_req: &ReplicationExtRequest) -> S3Result<()> { + let Some(input_cred) = req.credentials.as_ref() else { + return Err(s3_error!(AccessDenied, "Signature is required")); + }; + + let (cred, is_owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + + req.extensions.insert(ReqInfo { + cred: Some(cred), + is_owner, + bucket: Some(ext_req.bucket.clone()), + object: None, + version_id: None, + region: get_global_region(), + }); + + license_check().map_err(|er| match er.kind() { + std::io::ErrorKind::PermissionDenied => s3_error!(AccessDenied, "{er}"), + _ => { + error!("license check failed due to unexpected error: {er}"); + s3_error!(InternalError, "License validation failed") + } + })?; + + let action = match ext_req.route { + ReplicationExtRoute::MetricsV1 | ReplicationExtRoute::MetricsV2 | ReplicationExtRoute::Check => { + Action::S3Action(S3Action::GetReplicationConfigurationAction) + } + ReplicationExtRoute::ResetStart | ReplicationExtRoute::ResetStatus => { + Action::S3Action(S3Action::ResetBucketReplicationStateAction) + } + }; + authorize_request(req, action).await +} + +fn parse_reset_start_target(uri: &Uri) -> S3Result { + let arn = query_value_exact(uri, "arn").filter(|v| !v.is_empty()).unwrap_or_default(); + + let now = OffsetDateTime::now_utc(); + let reset_before = match query_value_exact(uri, "older-than").filter(|v| !v.is_empty()) { + Some(older_than) => { + let duration = parse_duration(&older_than) + .map_err(|err| s3_error!(InvalidRequest, "invalid older-than query parameter: {err}"))?; + let duration = time::Duration::try_from(duration) + .map_err(|err| s3_error!(InvalidRequest, "invalid older-than query parameter: {err}"))?; + Some(now - duration) + } + None => Some(now), + }; + + let reset_id = query_value_exact(uri, "reset-id") + .filter(|v| !v.is_empty()) + .unwrap_or_else(|| Uuid::new_v4().to_string()); + + Ok(ReplicationResetStartRequest { + arn, + reset_id, + reset_before, + }) +} + +fn collect_resettable_replication_target_arns(config: &s3s::dto::ReplicationConfiguration) -> Vec { + let mut arns = Vec::new(); + let mut seen = HashSet::new(); + + for rule in &config.rules { + if rule.status == s3s::dto::ReplicationRuleStatus::from_static(s3s::dto::ReplicationRuleStatus::DISABLED) { + continue; + } + + let existing_object_enabled = rule.existing_object_replication.as_ref().is_some_and(|status| { + status.status + == s3s::dto::ExistingObjectReplicationStatus::from_static(s3s::dto::ExistingObjectReplicationStatus::ENABLED) + }); + if !existing_object_enabled { + continue; + } + + let arn = if config.role.is_empty() { + rule.destination.bucket.clone() + } else { + config.role.clone() + }; + + if seen.insert(arn.clone()) { + arns.push(arn); + } + + if !config.role.is_empty() { + break; + } + } + + arns +} + +fn resolve_replication_reset_target_arn(config: &s3s::dto::ReplicationConfiguration, requested_arn: &str) -> S3Result { + let resettable_arns = collect_resettable_replication_target_arns(config); + + if requested_arn.is_empty() { + return match resettable_arns.as_slice() { + [] => Err(s3_error!( + InvalidRequest, + "replication reset requires a target with existing object replication enabled" + )), + [arn] => Ok(arn.clone()), + _ => Err(s3_error!( + InvalidRequest, + "arn query parameter is required when multiple replication targets are configured" + )), + }; + } + + let (has_arn, existing_object_enabled) = config.has_existing_object_replication(requested_arn); + if !has_arn { + return Err(s3_error!(InvalidRequest, "replication reset arn is not configured for this bucket")); + } + if !existing_object_enabled { + return Err(s3_error!( + InvalidRequest, + "replication reset requires existing object replication to be enabled for the target" + )); + } + + Ok(requested_arn.to_string()) +} + +fn build_replication_reset_response(targets: Vec) -> S3Result> { + let data = serde_json::to_vec(&ReplicationResetResponse { targets }).map_err(|e| s3_error!(InternalError, "{e}"))?; + let mut resp = S3Response::with_status(Body::from(data), StatusCode::OK); + resp.headers + .insert(header::CONTENT_TYPE, HeaderValue::from_static("application/json")); + Ok(resp) +} + +fn apply_replication_reset_to_targets(targets: &mut BucketTargets, reset: &ReplicationResetStartRequest) -> S3Result<()> { + let Some(target) = targets.targets.iter_mut().find(|target| target.arn == reset.arn) else { + return Err(s3_error!(InvalidRequest, "replication reset arn is not configured for this bucket")); + }; + + target.reset_id = reset.reset_id.clone(); + target.reset_before_date = reset.reset_before; + Ok(()) +} + +fn parse_reset_status_target(uri: &Uri) -> ReplicationResetStatusRequest { + ReplicationResetStatusRequest { + arn: query_value_exact(uri, "arn").filter(|v| !v.is_empty()), + } +} + +fn build_replication_reset_status_targets( + status: &BucketReplicationResyncStatus, + arn_filter: Option<&str>, +) -> Vec { + let mut targets = status + .targets_map + .iter() + .filter(|(arn, _)| arn_filter.is_none_or(|filter| *arn == filter)) + .map(|(arn, target)| ReplicationResetStatusTarget { + arn: arn.clone(), + reset_id: target.resync_id.clone(), + reset_before_date: target.resync_before_date, + start_time: target.start_time, + end_time: target.last_update, + status: target.resync_status.to_string(), + replicated_count: target.replicated_count, + replicated_size: target.replicated_size, + failed_count: target.failed_count, + failed_size: target.failed_size, + bucket: target.bucket.clone(), + object: target.object.clone(), + error: target.error.clone(), + }) + .collect::>(); + targets.sort_by(|left, right| left.arn.cmp(&right.arn)); + targets +} + +fn build_replication_reset_status_response( + status: BucketReplicationResyncStatus, + arn_filter: Option<&str>, +) -> S3Result> { + let data = serde_json::to_vec(&ReplicationResetStatusResponse { + targets: build_replication_reset_status_targets(&status, arn_filter), + }) + .map_err(|e| s3_error!(InternalError, "{e}"))?; + let mut resp = S3Response::with_status(Body::from(data), StatusCode::OK); + resp.headers + .insert(header::CONTENT_TYPE, HeaderValue::from_static("application/json")); + Ok(resp) +} + +fn build_replication_check_response(mut targets: Vec) -> S3Result> { + targets.sort_by(|left, right| left.arn.cmp(&right.arn)); + + if let Some(target) = targets.into_iter().find(|target| target.status != "OK") { + let detail = target.error.unwrap_or_else(|| target.status.to_lowercase()); + return Err(s3_error!( + InvalidRequest, + "replication check failed for target {} (bucket {}): {}", + target.arn, + target.bucket, + detail + )); + } + + Ok(S3Response::with_status(Body::empty(), StatusCode::OK)) +} + +fn format_replication_check_client_error(err: &S3ClientError, context: ReplicationCheckFailureContext) -> String { + if err.code.as_deref() == Some("AccessDenied") { + return match context { + ReplicationCheckFailureContext::ReplicateObject => { + "s3:ReplicateObject permissions missing for replication user".to_string() + } + ReplicationCheckFailureContext::ReplicateDeleteMarker => { + "s3:ReplicateDelete permissions missing for replication user".to_string() + } + ReplicationCheckFailureContext::DeleteObjectVersion => { + "s3:ReplicateDelete/s3:DeleteObject permissions missing for replication user".to_string() + } + ReplicationCheckFailureContext::BucketCheck => "target bucket check failed: access denied".to_string(), + ReplicationCheckFailureContext::VersioningCheck => "target bucket versioning check failed: access denied".to_string(), + ReplicationCheckFailureContext::ObjectLockCheck => "target object lock check failed: access denied".to_string(), + }; + } + + let context = match context { + ReplicationCheckFailureContext::BucketCheck => "target bucket check failed", + ReplicationCheckFailureContext::VersioningCheck => "target bucket versioning check failed", + ReplicationCheckFailureContext::ReplicateObject => "target replicate object check failed", + ReplicationCheckFailureContext::ReplicateDeleteMarker => "target replicate delete-marker check failed", + ReplicationCheckFailureContext::DeleteObjectVersion => "target delete object version check failed", + ReplicationCheckFailureContext::ObjectLockCheck => "target object lock check failed", + }; + + match (err.code.as_deref(), err.message.as_deref()) { + (Some("NoSuchBucket" | "NotFound"), _) => format!("{context}: target bucket does not exist"), + (Some(code), Some(message)) if !message.is_empty() => format!("{context}: {code}: {message}"), + (Some(code), _) => format!("{context}: {code}"), + (None, Some(message)) if !message.is_empty() => format!("{context}: {message}"), + _ => format!("{context}: {}", err.error), + } +} + +fn is_object_lock_not_enabled_error(err: &S3ClientError) -> bool { + matches!( + err.code.as_deref(), + Some("ObjectLockConfigurationNotFoundError" | "ObjectLockConfigurationNotFound") + ) || err.message.as_deref().is_some_and(|message| { + message.contains("Object Lock configuration does not exist") + || message.contains("Object Lock is not enabled for this bucket") + }) +} + +fn validate_replication_check_config_targets( + targets: &BucketTargets, + config: &s3s::dto::ReplicationConfiguration, +) -> S3Result<()> { + let configured_arns = targets + .targets + .iter() + .filter(|target| target.target_type == BucketTargetType::ReplicationService) + .map(|target| target.arn.as_str()) + .collect::>(); + + for rule in &config.rules { + if rule.status == s3s::dto::ReplicationRuleStatus::from_static(s3s::dto::ReplicationRuleStatus::DISABLED) { + continue; + } + + let configured_arn = if config.role.is_empty() { + rule.destination.bucket.as_str() + } else { + config.role.as_str() + }; + + if configured_arns.contains(configured_arn) { + continue; + } + + return Err(s3_error!( + InvalidRequest, + "replication config with rule ID {} has a stale target", + rule.id.clone().unwrap_or_default() + )); + } + + Ok(()) +} + +fn filter_replication_check_targets(targets: BucketTargets, config: &s3s::dto::ReplicationConfiguration) -> Vec { + let referenced_arns = config + .filter_target_arns(&ObjectOpts { + op_type: ReplicationType::All, + ..Default::default() + }) + .into_iter() + .collect::>(); + + targets + .targets + .into_iter() + .filter(|target| target.target_type == BucketTargetType::ReplicationService) + .filter(|target| referenced_arns.is_empty() || referenced_arns.contains(&target.arn)) + .collect() +} + +async fn check_replication_target(bucket: &str, target: &BucketTarget) -> ReplicationCheckTargetStatus { + let mut result = ReplicationCheckTargetStatus { + arn: target.arn.clone(), + endpoint: target.endpoint.clone(), + bucket: target.target_bucket.clone(), + status: "OK".to_string(), + error: None, + }; + + if target.target_bucket == bucket + && !target.deployment_id.is_empty() + && get_global_deployment_id().as_deref() == Some(target.deployment_id.as_str()) + { + result.status = "FAILED".to_string(); + result.error = Some("target bucket must not match source bucket on the same deployment".to_string()); + return result; + } + + let target_client = match resolve_replication_target_client(bucket, target).await { + Ok(client) => client, + Err(err) => { + result.status = "FAILED".to_string(); + result.error = Some(err); + return result; + } + }; + + match target_client.bucket_exists(&target.target_bucket).await { + Ok(true) => {} + Ok(false) => { + result.status = "FAILED".to_string(); + result.error = Some("target bucket does not exist".to_string()); + return result; + } + Err(err) => { + result.status = "FAILED".to_string(); + result.error = Some(format_replication_check_client_error(&err, ReplicationCheckFailureContext::BucketCheck)); + return result; + } + } + + match target_client.get_bucket_versioning(&target.target_bucket).await { + Ok(Some(_)) => {} + Ok(None) => { + result.status = "FAILED".to_string(); + result.error = Some(format!("target bucket {} is not versioned", target.target_bucket)); + return result; + } + Err(err) => { + result.status = "FAILED".to_string(); + result.error = Some(format_replication_check_client_error( + &err, + ReplicationCheckFailureContext::VersioningCheck, + )); + return result; + } + } + + let probe_key = format!(".rustfs-replication-check-{}", Uuid::new_v4()); + let (probe_version_id, probe_time) = + match put_replication_probe_object(&target_client, &target.target_bucket, &probe_key).await { + Ok(output) => output, + Err(err) => { + result.status = "FAILED".to_string(); + result.error = Some(format_replication_check_client_error( + &err, + ReplicationCheckFailureContext::ReplicateObject, + )); + return result; + } + }; + + if let Err(err) = delete_replication_probe_object( + &target_client, + &target.target_bucket, + &probe_key, + probe_version_id.as_deref(), + build_replication_probe_remove_options(probe_time, true), + ) + .await + { + result.status = "FAILED".to_string(); + result.error = Some(format_replication_check_client_error( + &err, + ReplicationCheckFailureContext::ReplicateDeleteMarker, + )); + return result; + } + + if let Err(err) = delete_replication_probe_object( + &target_client, + &target.target_bucket, + &probe_key, + probe_version_id.as_deref(), + build_replication_probe_remove_options(probe_time, false), + ) + .await + { + result.status = "FAILED".to_string(); + result.error = Some(format_replication_check_client_error( + &err, + ReplicationCheckFailureContext::DeleteObjectVersion, + )); + return result; + } + + result +} + +async fn resolve_replication_target_client(bucket: &str, target: &BucketTarget) -> Result, String> { + let target_sys = BucketTargetSys::get(); + match target_sys.get_remote_target_client(bucket, &target.arn).await { + Some(client) => Ok(client), + None => target_sys + .get_remote_target_client_internal(target) + .await + .map(Arc::new) + .map_err(|err| err.to_string()), + } +} + +fn build_replication_probe_put_options(now: OffsetDateTime) -> PutObjectOptions { + PutObjectOptions { + internal: rustfs_ecstore::bucket::bucket_target_sys::AdvancedPutOptions { + source_version_id: Uuid::new_v4().to_string(), + replication_status: ReplicationStatusType::Replica, + source_mtime: now, + replication_request: true, + replication_validity_check: true, + ..Default::default() + }, + ..Default::default() + } +} + +fn build_replication_probe_remove_options(now: OffsetDateTime, replication_delete_marker: bool) -> RemoveObjectOptions { + RemoveObjectOptions { + force_delete: false, + governance_bypass: false, + replication_delete_marker, + replication_mtime: Some(now), + replication_status: ReplicationStatusType::Replica, + replication_request: true, + replication_validity_check: true, + } +} + +async fn put_replication_probe_object( + target_client: &TargetClient, + target_bucket: &str, + probe_key: &str, +) -> Result<(Option, OffsetDateTime), S3ClientError> { + let now = OffsetDateTime::now_utc(); + let options = build_replication_probe_put_options(now); + let mut headers = HeaderMap::new(); + insert_header(&mut headers, SUFFIX_SOURCE_VERSION_ID, &options.internal.source_version_id); + insert_header( + &mut headers, + SUFFIX_SOURCE_MTIME, + options.internal.source_mtime.format(&Rfc3339).unwrap_or_default(), + ); + insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); + insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true"); + headers.insert( + HeaderName::from_static("x-amz-replication-status"), + HeaderValue::from_static(ReplicationStatusType::Replica.as_str()), + ); + + target_client + .client + .put_object() + .bucket(target_bucket) + .key(probe_key) + .content_length(8) + .body(AwsByteStream::from_static(b"aaaaaaaa")) + .customize() + .map_request(move |mut req| { + for (key, value) in headers.clone() { + req.headers_mut().insert(key.unwrap(), value); + } + Result::<_, std::io::Error>::Ok(req) + }) + .send() + .await + .map(|output| (output.version_id().map(ToOwned::to_owned), now)) + .map_err(S3ClientError::from) +} + +async fn delete_replication_probe_object( + target_client: &TargetClient, + target_bucket: &str, + probe_key: &str, + version_id: Option<&str>, + options: RemoveObjectOptions, +) -> Result<(), S3ClientError> { + let mut headers = HeaderMap::new(); + if options.replication_delete_marker { + insert_header(&mut headers, SUFFIX_SOURCE_DELETEMARKER, "true"); + } + if let Some(replication_mtime) = options.replication_mtime { + insert_header(&mut headers, SUFFIX_SOURCE_MTIME, replication_mtime.format(&Rfc3339).unwrap_or_default()); + } + headers.insert( + HeaderName::from_static("x-amz-replication-status"), + HeaderValue::from_static(options.replication_status.as_str()), + ); + if options.replication_request { + insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); + } + if options.replication_validity_check { + insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true"); + } + + target_client + .client + .delete_object() + .bucket(target_bucket) + .key(probe_key) + .set_version_id(version_id.map(ToOwned::to_owned)) + .customize() + .map_request(move |mut req| { + for (key, value) in headers.clone() { + req.headers_mut().insert(key.unwrap(), value); + } + Result::<_, std::io::Error>::Ok(req) + }) + .send() + .await + .map(|_| ()) + .map_err(S3ClientError::from) +} + +async fn source_bucket_requires_object_lock(bucket: &str) -> S3Result { + match metadata_sys::get_object_lock_config(bucket).await { + Ok((config, _)) => Ok(config + .object_lock_enabled + .as_ref() + .is_some_and(|state| state.as_str() == s3s::dto::ObjectLockEnabled::ENABLED)), + Err(rustfs_ecstore::error::StorageError::ConfigNotFound) => Ok(false), + Err(err) => Err(ApiError::from(err).into()), + } +} + +async fn run_replication_check(bucket: &str) -> S3Result> { + if !BucketVersioningSys::enabled(bucket).await { + return Err(s3_error!( + InvalidRequest, + "replication validation requires bucket versioning to be enabled" + )); + } + + let source_requires_object_lock = source_bucket_requires_object_lock(bucket).await?; + let (config, _) = metadata_sys::get_replication_config(bucket).await.map_err(ApiError::from)?; + let targets = metadata_sys::list_bucket_targets(bucket).await.map_err(ApiError::from)?; + validate_replication_check_config_targets(&targets, &config)?; + let replication_targets = filter_replication_check_targets(targets, &config); + + if replication_targets.is_empty() { + return Err(s3_error!( + InvalidRequest, + "replication check requires at least one configured replication target" + )); + } + + let mut statuses = Vec::with_capacity(replication_targets.len()); + for target in &replication_targets { + let mut status = check_replication_target(bucket, target).await; + if status.status == "OK" && source_requires_object_lock { + let target_lock_enabled = match target_client_object_lock_enabled(bucket, target).await { + Ok(enabled) => enabled, + Err(err) => { + status.status = "FAILED".to_string(); + status.error = Some(format_replication_check_client_error( + &err, + ReplicationCheckFailureContext::ObjectLockCheck, + )); + false + } + }; + if status.status == "OK" && !target_lock_enabled { + status.status = "FAILED".to_string(); + status.error = Some(format!("target bucket {} is not object lock enabled", target.target_bucket)); + } + } + statuses.push(status); + } + + build_replication_check_response(statuses) +} + +async fn target_client_object_lock_enabled(bucket: &str, target: &BucketTarget) -> Result { + let target_client = resolve_replication_target_client(bucket, target) + .await + .map_err(S3ClientError::new)?; + + match target_client + .client + .get_object_lock_configuration() + .bucket(&target.target_bucket) + .send() + .await + { + Ok(res) => Ok(res + .object_lock_configuration() + .and_then(|cfg| cfg.object_lock_enabled()) + .is_some_and(|state| state.as_str() == "Enabled")), + Err(err) => { + let err = S3ClientError::from(err); + if is_object_lock_not_enabled_error(&err) { + Ok(false) + } else { + Err(err) + } + } + } +} + +async fn start_replication_resync(bucket: &str, reset: &ReplicationResetStartRequest) -> S3Result { + let (config, _) = metadata_sys::get_replication_config(bucket).await.map_err(ApiError::from)?; + let resolved_arn = resolve_replication_reset_target_arn(&config, &reset.arn)?; + let mut resolved_reset = reset.clone(); + resolved_reset.arn = resolved_arn.clone(); + + let mut targets = metadata_sys::list_bucket_targets(bucket).await.map_err(ApiError::from)?; + apply_replication_reset_to_targets(&mut targets, &resolved_reset)?; + + let json_targets = serde_json::to_vec(&targets).map_err(|e| s3_error!(InternalError, "{e}"))?; + metadata_sys::update(bucket, BUCKET_TARGETS_FILE, json_targets) + .await + .map_err(ApiError::from)?; + BucketTargetSys::get().update_all_targets(bucket, Some(&targets)).await; + + let Some(pool) = get_global_replication_pool() else { + return Err(s3_error!(InternalError, "replication pool is not initialized")); + }; + + pool.start_bucket_resync(ResyncOpts { + bucket: bucket.to_string(), + arn: resolved_arn.clone(), + resync_id: reset.reset_id.clone(), + resync_before: reset.reset_before, + }) + .await + .map_err(|e| s3_error!(InternalError, "{e}"))?; + + Ok(ReplicationResetTarget { + arn: resolved_arn, + reset_id: reset.reset_id.clone(), + }) +} + +async fn load_replication_resync_status(bucket: &str) -> S3Result { + let Some(pool) = get_global_replication_pool() else { + return Err(s3_error!(InternalError, "replication pool is not initialized")); + }; + + pool.get_bucket_resync_status(bucket) + .await + .map_err(|e| s3_error!(InternalError, "{e}")) +} + +async fn handle_replication_extension_request( + req: &mut S3Request, + ext_req: &ReplicationExtRequest, +) -> S3Result> { + authorize_replication_extension_request(req, ext_req).await?; + ensure_replication_bucket_exists(&ext_req.bucket).await?; + + match ext_req.route { + ReplicationExtRoute::MetricsV1 | ReplicationExtRoute::MetricsV2 => { + ensure_replication_config_exists(&ext_req.bucket).await?; + build_replication_metrics_response(&ext_req.bucket, ext_req.route).await + } + ReplicationExtRoute::Check => { + let (versioning, _) = metadata_sys::get_versioning_config(&ext_req.bucket) + .await + .map_err(ApiError::from)?; + if !versioning.enabled() && !BucketVersioningSys::enabled(&ext_req.bucket).await { + return Err(s3_error!( + InvalidRequest, + "replication validation requires bucket versioning to be enabled" + )); + } + ensure_replication_config_exists(&ext_req.bucket).await?; + run_replication_check(&ext_req.bucket).await + } + ReplicationExtRoute::ResetStatus => { + ensure_replication_config_exists(&ext_req.bucket).await?; + let status_req = parse_reset_status_target(&req.uri); + let status = load_replication_resync_status(&ext_req.bucket).await?; + build_replication_reset_status_response(status, status_req.arn.as_deref()) + } + ReplicationExtRoute::ResetStart => { + ensure_replication_config_exists(&ext_req.bucket).await?; + let target = parse_reset_start_target(&req.uri)?; + let target = start_replication_resync(&ext_req.bucket, &target).await?; + build_replication_reset_response(vec![target]) + } + } +} + +async fn authorize_misc_extension_request(req: &mut S3Request, route: &MiscExtRoute) -> S3Result<()> { + let Some(input_cred) = req.credentials.as_ref() else { + return Err(s3_error!(AccessDenied, "Signature is required")); + }; + + let (cred, is_owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + + let (bucket, object, action) = match route { + MiscExtRoute::ObjectLambda { bucket, object } => { + (Some(bucket.clone()), Some(object.clone()), Action::S3Action(S3Action::GetObjectAction)) + } + MiscExtRoute::ListenNotification { bucket: Some(bucket) } => { + (Some(bucket.clone()), None, Action::S3Action(S3Action::ListenBucketNotificationAction)) + } + MiscExtRoute::ListenNotification { bucket: None } => (None, None, Action::S3Action(S3Action::ListenNotificationAction)), + }; + + req.extensions.insert(ReqInfo { + cred: Some(cred), + is_owner, + bucket, + object, + version_id: None, + region: get_global_region(), + }); + + license_check().map_err(|er| match er.kind() { + std::io::ErrorKind::PermissionDenied => s3_error!(AccessDenied, "{er}"), + _ => { + error!("license check failed due to unexpected error: {er}"); + s3_error!(InternalError, "License validation failed") + } + })?; + + authorize_request(req, action).await +} + +async fn handle_misc_extension_request(req: &mut S3Request, route: &MiscExtRoute) -> S3Result> { + authorize_misc_extension_request(req, route).await?; + validate_misc_extension_request(&req.uri, route)?; + + match route { + MiscExtRoute::ObjectLambda { bucket, object } => { + let get_req = build_object_lambda_get_request(req, bucket, object)?; + let usecase = DefaultObjectUsecase::from_global(); + let get_resp = usecase.execute_get_object(get_req).await?; + invoke_object_lambda_target(req, bucket, object, get_resp).await + } + MiscExtRoute::ListenNotification { bucket } => { + if let Some(bucket_name) = bucket { + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init")); + }; + store + .get_bucket_info(bucket_name, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + } + build_listen_notification_response(&req.uri, bucket.as_deref()) + } + } +} pub struct S3Router { router: Router, @@ -111,6 +2282,10 @@ where T: Operation, { fn is_match(&self, method: &Method, uri: &Uri, headers: &HeaderMap, _: &mut Extensions) -> bool { + if parse_replication_extension_request(method, uri).is_some() || parse_misc_extension_request(method, uri).is_some() { + return true; + } + let path = uri.path(); // Profiling endpoints @@ -141,6 +2316,15 @@ where // check_access before call async fn check_access(&self, req: &mut S3Request) -> S3Result<()> { + if parse_replication_extension_request(&req.method, &req.uri).is_some() + || parse_misc_extension_request(&req.method, &req.uri).is_some() + { + return match req.credentials { + Some(_) => Ok(()), + None => Err(s3_error!(AccessDenied, "Signature is required")), + }; + } + // Allow unauthenticated access to health check let path = req.uri.path(); @@ -195,7 +2379,14 @@ where } } - async fn call(&self, req: S3Request) -> S3Result> { + async fn call(&self, mut req: S3Request) -> S3Result> { + if let Some(ext_req) = parse_replication_extension_request(&req.method, &req.uri) { + return handle_replication_extension_request(&mut req, &ext_req).await; + } + if let Some(ext_req) = parse_misc_extension_request(&req.method, &req.uri) { + return handle_misc_extension_request(&mut req, &ext_req).await; + } + // Console requests should be handled by console router first (including OPTIONS) // Console has its own CORS layer configured if self.console_enabled && is_console_path(req.uri.path()) { @@ -246,6 +2437,11 @@ impl Operation for AdminOperation { #[cfg(test)] mod tests { use super::*; + use http::HeaderMap; + use http::Method; + use http::Uri; + use s3s::S3Request; + use time::macros::datetime; #[test] fn canonicalize_admin_path_maps_compat_prefix_to_rustfs_prefix() { @@ -259,6 +2455,1383 @@ mod tests { assert!(is_admin_path("/minio/admin/v3/info")); assert!(!is_admin_path("/bucket/object")); } + + #[test] + fn parse_replication_extension_request_matches_metrics_and_check() { + let metrics: Uri = "/demo-bucket?replication-metrics".parse().expect("uri should parse"); + let metrics_v2: Uri = "/demo-bucket?replication-metrics=2".parse().expect("uri should parse"); + let check: Uri = "/demo-bucket?replication-check".parse().expect("uri should parse"); + let reset_status: Uri = "/demo-bucket?replication-reset-status".parse().expect("uri should parse"); + let reset_start: Uri = "/demo-bucket?replication-reset".parse().expect("uri should parse"); + + let m = parse_replication_extension_request(&Method::GET, &metrics).expect("metrics route should parse"); + assert_eq!(m.bucket, "demo-bucket"); + assert_eq!(m.route, ReplicationExtRoute::MetricsV1); + + let v2 = parse_replication_extension_request(&Method::GET, &metrics_v2).expect("metrics v2 route should parse"); + assert_eq!(v2.bucket, "demo-bucket"); + assert_eq!(v2.route, ReplicationExtRoute::MetricsV2); + + let c = parse_replication_extension_request(&Method::GET, &check).expect("check route should parse"); + assert_eq!(c.bucket, "demo-bucket"); + assert_eq!(c.route, ReplicationExtRoute::Check); + + let rs = parse_replication_extension_request(&Method::GET, &reset_status).expect("reset status route should parse"); + assert_eq!(rs.bucket, "demo-bucket"); + assert_eq!(rs.route, ReplicationExtRoute::ResetStatus); + + let r = parse_replication_extension_request(&Method::PUT, &reset_start).expect("reset start route should parse"); + assert_eq!(r.bucket, "demo-bucket"); + assert_eq!(r.route, ReplicationExtRoute::ResetStart); + } + + #[test] + fn parse_replication_extension_request_rejects_object_level_and_invalid_query_values() { + let object_level: Uri = "/demo-bucket/path/file?replication-metrics" + .parse() + .expect("uri should parse"); + let invalid_value: Uri = "/demo-bucket?replication-metrics=1".parse().expect("uri should parse"); + let wrong_method: Uri = "/demo-bucket?replication-check".parse().expect("uri should parse"); + let wrong_method_reset: Uri = "/demo-bucket?replication-reset".parse().expect("uri should parse"); + let wrong_method_status: Uri = "/demo-bucket?replication-reset-status".parse().expect("uri should parse"); + + assert!(parse_replication_extension_request(&Method::GET, &object_level).is_none()); + assert!(parse_replication_extension_request(&Method::GET, &invalid_value).is_none()); + assert!(parse_replication_extension_request(&Method::PUT, &wrong_method).is_none()); + assert!(parse_replication_extension_request(&Method::GET, &wrong_method_reset).is_none()); + assert!(parse_replication_extension_request(&Method::PUT, &wrong_method_status).is_none()); + } + + #[test] + fn parse_reset_start_target_defaults_reset_before_and_supports_older_than() { + let no_window: Uri = "/demo-bucket?replication-reset".parse().expect("uri should parse"); + let before_default = OffsetDateTime::now_utc(); + let parsed_default = parse_reset_start_target(&no_window).expect("default reset request should parse"); + let after_default = OffsetDateTime::now_utc(); + + assert!(parsed_default.arn.is_empty()); + assert!(!parsed_default.reset_id.is_empty()); + let reset_before = parsed_default.reset_before.expect("default reset window should be set"); + assert!(reset_before >= before_default && reset_before <= after_default); + + let older_than: Uri = "/demo-bucket?replication-reset&arn=arn:target&reset-id=rid-1&older-than=1h" + .parse() + .expect("uri should parse"); + let before_window = OffsetDateTime::now_utc(); + let parsed_window = parse_reset_start_target(&older_than).expect("older-than reset request should parse"); + let after_window = OffsetDateTime::now_utc(); + + assert_eq!(parsed_window.reset_id, "rid-1"); + let reset_before = parsed_window.reset_before.expect("older-than reset window should be set"); + assert!(reset_before <= after_window - time::Duration::minutes(59)); + assert!(reset_before >= before_window - time::Duration::hours(1) - time::Duration::seconds(1)); + } + + #[test] + fn resolve_replication_reset_target_arn_uses_single_existing_object_target_by_default() { + let config = s3s::dto::ReplicationConfiguration { + role: String::new(), + rules: vec![s3s::dto::ReplicationRule { + delete_marker_replication: None, + delete_replication: None, + destination: s3s::dto::Destination { + bucket: "arn:replication:a".to_string(), + ..Default::default() + }, + existing_object_replication: Some(s3s::dto::ExistingObjectReplication { + status: s3s::dto::ExistingObjectReplicationStatus::from_static( + s3s::dto::ExistingObjectReplicationStatus::ENABLED, + ), + }), + filter: None, + id: Some("rule-a".to_string()), + prefix: Some(String::new()), + priority: None, + source_selection_criteria: None, + status: s3s::dto::ReplicationRuleStatus::from_static(s3s::dto::ReplicationRuleStatus::ENABLED), + }], + }; + + let resolved = resolve_replication_reset_target_arn(&config, "").expect("single target should resolve"); + assert_eq!(resolved, "arn:replication:a"); + } + + #[test] + fn resolve_replication_reset_target_arn_requires_arn_for_multiple_targets() { + let config = s3s::dto::ReplicationConfiguration { + role: String::new(), + rules: vec![ + s3s::dto::ReplicationRule { + delete_marker_replication: None, + delete_replication: None, + destination: s3s::dto::Destination { + bucket: "arn:replication:a".to_string(), + ..Default::default() + }, + existing_object_replication: Some(s3s::dto::ExistingObjectReplication { + status: s3s::dto::ExistingObjectReplicationStatus::from_static( + s3s::dto::ExistingObjectReplicationStatus::ENABLED, + ), + }), + filter: None, + id: Some("rule-a".to_string()), + prefix: Some(String::new()), + priority: None, + source_selection_criteria: None, + status: s3s::dto::ReplicationRuleStatus::from_static(s3s::dto::ReplicationRuleStatus::ENABLED), + }, + s3s::dto::ReplicationRule { + delete_marker_replication: None, + delete_replication: None, + destination: s3s::dto::Destination { + bucket: "arn:replication:b".to_string(), + ..Default::default() + }, + existing_object_replication: Some(s3s::dto::ExistingObjectReplication { + status: s3s::dto::ExistingObjectReplicationStatus::from_static( + s3s::dto::ExistingObjectReplicationStatus::ENABLED, + ), + }), + filter: None, + id: Some("rule-b".to_string()), + prefix: Some(String::new()), + priority: None, + source_selection_criteria: None, + status: s3s::dto::ReplicationRuleStatus::from_static(s3s::dto::ReplicationRuleStatus::ENABLED), + }, + ], + }; + + let err = resolve_replication_reset_target_arn(&config, "").expect_err("multiple targets should require arn"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + assert!(err.message().unwrap_or_default().contains("arn query parameter is required")); + } + + #[test] + fn resolve_replication_reset_target_arn_rejects_target_without_existing_object_replication() { + let config = s3s::dto::ReplicationConfiguration { + role: String::new(), + rules: vec![s3s::dto::ReplicationRule { + delete_marker_replication: None, + delete_replication: None, + destination: s3s::dto::Destination { + bucket: "arn:replication:a".to_string(), + ..Default::default() + }, + existing_object_replication: Some(s3s::dto::ExistingObjectReplication { + status: s3s::dto::ExistingObjectReplicationStatus::from_static( + s3s::dto::ExistingObjectReplicationStatus::DISABLED, + ), + }), + filter: None, + id: Some("rule-a".to_string()), + prefix: Some(String::new()), + priority: None, + source_selection_criteria: None, + status: s3s::dto::ReplicationRuleStatus::from_static(s3s::dto::ReplicationRuleStatus::ENABLED), + }], + }; + + let err = resolve_replication_reset_target_arn(&config, "arn:replication:a") + .expect_err("target without existing object replication should fail"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + assert!(err.message().unwrap_or_default().contains("existing object replication")); + } + + #[test] + fn apply_replication_reset_to_targets_updates_matching_target() { + let mut targets = BucketTargets { + targets: vec![rustfs_ecstore::bucket::target::BucketTarget { + arn: "arn:target".to_string(), + ..Default::default() + }], + }; + let reset = ReplicationResetStartRequest { + arn: "arn:target".to_string(), + reset_id: "rid-1".to_string(), + reset_before: Some(OffsetDateTime::now_utc()), + }; + + apply_replication_reset_to_targets(&mut targets, &reset).expect("target update should succeed"); + + assert_eq!(targets.targets[0].reset_id, "rid-1"); + assert_eq!(targets.targets[0].reset_before_date, reset.reset_before); + } + + #[test] + fn build_replication_reset_status_response_serializes_sorted_targets() { + let mut status = BucketReplicationResyncStatus::new(); + status.targets_map.insert( + "arn:z".to_string(), + rustfs_ecstore::bucket::replication::TargetReplicationResyncStatus { + resync_id: "rid-z".to_string(), + last_update: Some(datetime!(2025-01-03 00:00 UTC)), + resync_status: rustfs_ecstore::bucket::replication::ResyncStatusType::ResyncFailed, + failed_count: 2, + failed_size: 4, + bucket: "bucket-z".to_string(), + error: Some("boom".to_string()), + ..Default::default() + }, + ); + status.targets_map.insert( + "arn:a".to_string(), + rustfs_ecstore::bucket::replication::TargetReplicationResyncStatus { + resync_id: "rid-a".to_string(), + last_update: Some(datetime!(2025-01-02 00:00 UTC)), + resync_status: rustfs_ecstore::bucket::replication::ResyncStatusType::ResyncCompleted, + replicated_count: 3, + replicated_size: 9, + bucket: "bucket-a".to_string(), + ..Default::default() + }, + ); + + let response = build_replication_reset_status_response(status, None).expect("status response should build"); + let bytes = futures::executor::block_on(http_body_util::BodyExt::collect(response.output)) + .expect("body should read") + .to_bytes(); + let payload: serde_json::Value = serde_json::from_slice(&bytes).expect("response must be json"); + + assert_eq!(payload["Targets"][0]["Arn"], "arn:a"); + assert_eq!(payload["Targets"][0]["Bucket"], "bucket-a"); + assert_eq!(payload["Targets"][0]["Status"], "Completed"); + assert_eq!(payload["Targets"][0]["EndTime"], "2025-01-02T00:00:00Z"); + assert_eq!(payload["Targets"][1]["Arn"], "arn:z"); + assert_eq!(payload["Targets"][1]["Bucket"], "bucket-z"); + assert_eq!(payload["Targets"][1]["Status"], "Failed"); + assert_eq!(payload["Targets"][1]["EndTime"], "2025-01-03T00:00:00Z"); + assert_eq!(payload["Targets"][1]["Error"], "boom"); + } + + #[test] + fn build_replication_reset_status_response_filters_targets_by_arn() { + let mut status = BucketReplicationResyncStatus::new(); + status.targets_map.insert( + "arn:z".to_string(), + rustfs_ecstore::bucket::replication::TargetReplicationResyncStatus { + resync_id: "rid-z".to_string(), + last_update: Some(datetime!(2025-02-03 00:00 UTC)), + resync_status: rustfs_ecstore::bucket::replication::ResyncStatusType::ResyncFailed, + failed_count: 2, + failed_size: 4, + bucket: "bucket-z".to_string(), + error: Some("boom".to_string()), + ..Default::default() + }, + ); + status.targets_map.insert( + "arn:a".to_string(), + rustfs_ecstore::bucket::replication::TargetReplicationResyncStatus { + resync_id: "rid-a".to_string(), + last_update: Some(datetime!(2025-02-02 00:00 UTC)), + resync_status: rustfs_ecstore::bucket::replication::ResyncStatusType::ResyncCompleted, + replicated_count: 3, + replicated_size: 9, + bucket: "bucket-a".to_string(), + ..Default::default() + }, + ); + + let response = build_replication_reset_status_response(status, Some("arn:z")).expect("status response should build"); + let bytes = futures::executor::block_on(http_body_util::BodyExt::collect(response.output)) + .expect("body should read") + .to_bytes(); + let payload: serde_json::Value = serde_json::from_slice(&bytes).expect("response must be json"); + + assert_eq!(payload["Targets"].as_array().map(Vec::len), Some(1)); + assert_eq!(payload["Targets"][0]["Arn"], "arn:z"); + assert_eq!(payload["Targets"][0]["Bucket"], "bucket-z"); + assert_eq!(payload["Targets"][0]["Status"], "Failed"); + assert_eq!(payload["Targets"][0]["EndTime"], "2025-02-03T00:00:00Z"); + assert_eq!(payload["Targets"][0]["Error"], "boom"); + } + + #[test] + fn build_replication_check_response_returns_empty_body_on_success() { + let response = build_replication_check_response(vec![ + ReplicationCheckTargetStatus { + arn: "arn:a".to_string(), + endpoint: "remote-a:9000".to_string(), + bucket: "bucket-a".to_string(), + status: "OK".to_string(), + error: None, + }, + ReplicationCheckTargetStatus { + arn: "arn:z".to_string(), + endpoint: "remote-z:9000".to_string(), + bucket: "bucket-z".to_string(), + status: "OK".to_string(), + error: None, + }, + ]) + .expect("response should build"); + + let bytes = futures::executor::block_on(http_body_util::BodyExt::collect(response.output)) + .expect("body should read") + .to_bytes(); + assert!(bytes.is_empty()); + } + + #[test] + fn build_replication_check_response_surfaces_first_failure() { + let err = build_replication_check_response(vec![ + ReplicationCheckTargetStatus { + arn: "arn:z".to_string(), + endpoint: "remote-z:9000".to_string(), + bucket: "bucket-z".to_string(), + status: "FAILED".to_string(), + error: Some("boom".to_string()), + }, + ReplicationCheckTargetStatus { + arn: "arn:a".to_string(), + endpoint: "remote-a:9000".to_string(), + bucket: "bucket-a".to_string(), + status: "OK".to_string(), + error: None, + }, + ]) + .expect_err("failed target should surface as request error"); + + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + assert!(err.message().unwrap_or_default().contains("arn:z")); + } + + #[test] + fn build_replication_check_response_rejects_empty_target_list_at_runtime_boundary() { + let config = s3s::dto::ReplicationConfiguration { + role: String::new(), + rules: vec![], + }; + let replication_targets = filter_replication_check_targets(BucketTargets::default(), &config); + + assert!(replication_targets.is_empty()); + } + + #[test] + fn format_replication_check_client_error_prefers_structured_access_denied() { + let err = S3ClientError::with_metadata( + "AccessDenied: denied", + None, + Some("AccessDenied".to_string()), + Some("denied".to_string()), + ); + + let formatted = format_replication_check_client_error(&err, ReplicationCheckFailureContext::BucketCheck); + assert_eq!(formatted, "target bucket check failed: access denied"); + } + + #[test] + fn format_replication_check_client_error_uses_remote_code_and_message() { + let err = S3ClientError::with_metadata( + "InvalidRequest: bucket versioning is suspended", + None, + Some("InvalidRequest".to_string()), + Some("bucket versioning is suspended".to_string()), + ); + + let formatted = format_replication_check_client_error(&err, ReplicationCheckFailureContext::VersioningCheck); + assert_eq!( + formatted, + "target bucket versioning check failed: InvalidRequest: bucket versioning is suspended" + ); + } + + #[test] + fn format_replication_check_client_error_maps_replicate_permission_failures() { + let err = S3ClientError::with_metadata( + "AccessDenied: denied", + None, + Some("AccessDenied".to_string()), + Some("denied".to_string()), + ); + + let replicate_object = format_replication_check_client_error(&err, ReplicationCheckFailureContext::ReplicateObject); + assert_eq!(replicate_object, "s3:ReplicateObject permissions missing for replication user"); + + let replicate_delete = format_replication_check_client_error(&err, ReplicationCheckFailureContext::ReplicateDeleteMarker); + assert_eq!(replicate_delete, "s3:ReplicateDelete permissions missing for replication user"); + + let delete_object = format_replication_check_client_error(&err, ReplicationCheckFailureContext::DeleteObjectVersion); + assert_eq!( + delete_object, + "s3:ReplicateDelete/s3:DeleteObject permissions missing for replication user" + ); + } + + #[test] + fn is_object_lock_not_enabled_error_recognizes_missing_configuration() { + let code_only = S3ClientError::with_metadata( + "ObjectLockConfigurationNotFoundError: missing", + None, + Some("ObjectLockConfigurationNotFoundError".to_string()), + Some("missing".to_string()), + ); + assert!(is_object_lock_not_enabled_error(&code_only)); + + let message_only = S3ClientError::with_metadata( + "Object Lock is not enabled for this bucket", + None, + None, + Some("Object Lock is not enabled for this bucket".to_string()), + ); + assert!(is_object_lock_not_enabled_error(&message_only)); + + let access_denied = S3ClientError::with_metadata( + "AccessDenied: denied", + None, + Some("AccessDenied".to_string()), + Some("denied".to_string()), + ); + assert!(!is_object_lock_not_enabled_error(&access_denied)); + } + + #[test] + fn filter_replication_check_targets_only_keeps_configured_replication_targets() { + let targets = BucketTargets { + targets: vec![ + BucketTarget { + arn: "arn:replication:a".to_string(), + target_type: BucketTargetType::ReplicationService, + ..Default::default() + }, + BucketTarget { + arn: "arn:replication:b".to_string(), + target_type: BucketTargetType::ReplicationService, + ..Default::default() + }, + BucketTarget { + arn: "arn:ilm:c".to_string(), + target_type: BucketTargetType::IlmService, + ..Default::default() + }, + ], + }; + let config = s3s::dto::ReplicationConfiguration { + role: String::new(), + rules: vec![s3s::dto::ReplicationRule { + delete_marker_replication: None, + delete_replication: None, + destination: s3s::dto::Destination { + bucket: "arn:replication:b".to_string(), + ..Default::default() + }, + existing_object_replication: None, + filter: None, + id: None, + prefix: Some(String::new()), + priority: None, + source_selection_criteria: None, + status: s3s::dto::ReplicationRuleStatus::from_static(s3s::dto::ReplicationRuleStatus::ENABLED), + }], + }; + + let filtered = filter_replication_check_targets(targets, &config); + + assert_eq!(filtered.len(), 1); + assert_eq!(filtered[0].arn, "arn:replication:b"); + } + + #[test] + fn validate_replication_check_config_targets_rejects_stale_enabled_rule_target() { + let targets = BucketTargets { + targets: vec![BucketTarget { + arn: "arn:replication:a".to_string(), + target_type: BucketTargetType::ReplicationService, + ..Default::default() + }], + }; + let config = s3s::dto::ReplicationConfiguration { + role: String::new(), + rules: vec![s3s::dto::ReplicationRule { + delete_marker_replication: None, + delete_replication: None, + destination: s3s::dto::Destination { + bucket: "arn:replication:missing".to_string(), + ..Default::default() + }, + existing_object_replication: None, + filter: None, + id: Some("rule-stale".to_string()), + prefix: Some(String::new()), + priority: None, + source_selection_criteria: None, + status: s3s::dto::ReplicationRuleStatus::from_static(s3s::dto::ReplicationRuleStatus::ENABLED), + }], + }; + + let err = validate_replication_check_config_targets(&targets, &config).expect_err("stale target should be rejected"); + + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + assert!(err.message().unwrap_or_default().contains("rule-stale")); + } + + #[test] + fn serialize_replication_metrics_body_v1_returns_replication_stats_only() { + let mut stats = BucketStats { + uptime: 99, + ..Default::default() + }; + stats.replication_stats.replica_count = 7; + stats.proxy_stats.put_total = 3; + + let body = + serialize_replication_metrics_body(&stats, ReplicationExtRoute::MetricsV1).expect("metrics v1 body should serialize"); + let payload: serde_json::Value = serde_json::from_slice(&body).expect("body should be json"); + + assert_eq!(payload["replica_count"], 7); + assert!(payload.get("uptime").is_none()); + assert!(payload.get("proxy_stats").is_none()); + } + + #[test] + fn apply_replication_metrics_bandwidth_report_updates_existing_target_stats() { + let mut stats = BucketStats::default(); + stats + .replication_stats + .stats + .entry("arn:replication:a".to_string()) + .or_default() + .replicated_count = 3; + + let bandwidth_report = HashMap::from([( + "arn:replication:a".to_string(), + BandwidthDetails { + limit_bytes_per_sec: 2048, + current_bandwidth_bytes_per_sec: 1536.5, + }, + )]); + + let updated = apply_replication_metrics_bandwidth_report(stats, bandwidth_report); + let stat = updated + .replication_stats + .stats + .get("arn:replication:a") + .expect("target stats should exist"); + + assert_eq!(stat.replicated_count, 3); + assert_eq!(stat.bandwidth_limit_bytes_per_sec, 2048); + assert_eq!(stat.current_bandwidth_bytes_per_sec, 1536.5); + } + + #[test] + fn apply_replication_metrics_bandwidth_report_creates_missing_target_stats() { + let bandwidth_report = HashMap::from([( + "arn:replication:b".to_string(), + BandwidthDetails { + limit_bytes_per_sec: 4096, + current_bandwidth_bytes_per_sec: 1024.25, + }, + )]); + + let updated = apply_replication_metrics_bandwidth_report(BucketStats::default(), bandwidth_report); + let stat = updated + .replication_stats + .stats + .get("arn:replication:b") + .expect("target stats should be created from bandwidth report"); + + assert_eq!(stat.bandwidth_limit_bytes_per_sec, 4096); + assert_eq!(stat.current_bandwidth_bytes_per_sec, 1024.25); + } + + #[test] + fn serialize_replication_metrics_body_v2_returns_full_bucket_stats() { + let mut stats = BucketStats { + uptime: 99, + ..Default::default() + }; + stats.replication_stats.replica_count = 7; + stats.proxy_stats.put_total = 3; + + let body = + serialize_replication_metrics_body(&stats, ReplicationExtRoute::MetricsV2).expect("metrics v2 body should serialize"); + let payload: serde_json::Value = serde_json::from_slice(&body).expect("body should be json"); + + assert_eq!(payload["uptime"], 99); + assert_eq!(payload["replication_stats"]["replica_count"], 7); + assert_eq!(payload["proxy_stats"]["put_total"], 3); + } + + #[test] + fn apply_replication_metrics_runtime_fields_only_overrides_v2_uptime() { + let stats = BucketStats { + uptime: 99, + ..Default::default() + }; + + let v1 = apply_replication_metrics_runtime_fields(stats.clone(), ReplicationExtRoute::MetricsV1, 42); + let v2 = apply_replication_metrics_runtime_fields(stats, ReplicationExtRoute::MetricsV2, 42); + + assert_eq!(v1.uptime, 99); + assert_eq!(v2.uptime, 42); + } + + #[test] + fn build_replication_probe_put_options_sets_replication_flags() { + let now = OffsetDateTime::from_unix_timestamp(42).expect("timestamp should build"); + let options = build_replication_probe_put_options(now); + + assert_eq!(options.internal.replication_status, ReplicationStatusType::Replica); + assert!(options.internal.replication_request); + assert!(options.internal.replication_validity_check); + assert_eq!(options.internal.source_mtime, now); + assert!(!options.internal.source_version_id.is_empty()); + } + + #[test] + fn build_replication_probe_remove_options_sets_replication_flags() { + let now = OffsetDateTime::from_unix_timestamp(42).expect("timestamp should build"); + let options = build_replication_probe_remove_options(now, true); + + assert!(options.replication_delete_marker); + assert_eq!(options.replication_status, ReplicationStatusType::Replica); + assert!(options.replication_request); + assert!(options.replication_validity_check); + assert_eq!(options.replication_mtime, Some(now)); + } + + #[test] + fn parse_misc_extension_request_matches_object_lambda_and_listen_notification() { + let object_lambda: Uri = "/demo-bucket/path/to/object.txt?lambdaArn=arn%3Atarget" + .parse() + .expect("uri should parse"); + let listen_bucket: Uri = "/demo-bucket?events=s3:ObjectCreated:*".parse().expect("uri should parse"); + let listen_root: Uri = "/?events=s3:ObjectRemoved:*".parse().expect("uri should parse"); + + let object_route = parse_misc_extension_request(&Method::GET, &object_lambda).expect("object lambda route should parse"); + assert_eq!( + object_route, + MiscExtRoute::ObjectLambda { + bucket: "demo-bucket".to_string(), + object: "path/to/object.txt".to_string() + } + ); + + let listen_bucket_route = + parse_misc_extension_request(&Method::GET, &listen_bucket).expect("bucket listen route should parse"); + assert_eq!( + listen_bucket_route, + MiscExtRoute::ListenNotification { + bucket: Some("demo-bucket".to_string()) + } + ); + + let listen_root_route = parse_misc_extension_request(&Method::GET, &listen_root).expect("root listen route should parse"); + assert_eq!(listen_root_route, MiscExtRoute::ListenNotification { bucket: None }); + } + + #[test] + fn parse_misc_extension_request_rejects_invalid_paths_or_methods() { + let bucket_without_object: Uri = "/demo-bucket?lambdaArn=arn%3Atarget".parse().expect("uri should parse"); + let wrong_method_lambda: Uri = "/demo-bucket/object?lambdaArn=arn%3Atarget" + .parse() + .expect("uri should parse"); + let object_level_listen: Uri = "/demo-bucket/object?events=s3:ObjectCreated:*" + .parse() + .expect("uri should parse"); + + assert!(parse_misc_extension_request(&Method::GET, &bucket_without_object).is_none()); + assert!(parse_misc_extension_request(&Method::PUT, &wrong_method_lambda).is_none()); + assert!(parse_misc_extension_request(&Method::GET, &object_level_listen).is_none()); + } + + #[test] + fn validate_listen_notification_query_accepts_valid_values() { + let uri: Uri = "/demo-bucket?events=s3:ObjectCreated:*&prefix=logs/&suffix=.json&ping=3" + .parse() + .expect("uri should parse"); + + assert!(validate_listen_notification_query(&uri).is_ok()); + } + + #[test] + fn validate_listen_notification_query_rejects_invalid_event_or_duplicate_filters() { + let invalid_event: Uri = "/demo-bucket?events=invalid-event".parse().expect("uri should parse"); + let duplicate_prefix: Uri = "/demo-bucket?events=s3:ObjectCreated:*&prefix=a&prefix=b" + .parse() + .expect("uri should parse"); + let invalid_ping: Uri = "/demo-bucket?events=s3:ObjectCreated:*&ping=0" + .parse() + .expect("uri should parse"); + + assert_eq!( + validate_listen_notification_query(&invalid_event) + .expect_err("invalid event should fail") + .code(), + &S3ErrorCode::InvalidArgument + ); + assert_eq!( + validate_listen_notification_query(&duplicate_prefix) + .expect_err("duplicate prefix should fail") + .code(), + &S3ErrorCode::InvalidArgument + ); + assert_eq!( + validate_listen_notification_query(&invalid_ping) + .expect_err("invalid ping should fail") + .code(), + &S3ErrorCode::InvalidArgument + ); + } + + #[test] + fn validate_object_lambda_query_rejects_missing_empty_or_invalid_arn() { + let missing: Uri = "/demo-bucket/object.txt".parse().expect("uri should parse"); + let empty: Uri = "/demo-bucket/object.txt?lambdaArn=".parse().expect("uri should parse"); + let duplicated: Uri = "/demo-bucket/object.txt?lambdaArn=a&lambdaArn=b" + .parse() + .expect("uri should parse"); + let invalid_format: Uri = "/demo-bucket/object.txt?lambdaArn=not-an-arn" + .parse() + .expect("uri should parse"); + + assert_eq!( + validate_object_lambda_query(&missing) + .expect_err("missing lambdaArn should fail") + .code(), + &S3ErrorCode::InvalidRequest + ); + assert_eq!( + validate_object_lambda_query(&empty) + .expect_err("empty lambdaArn should fail") + .code(), + &S3ErrorCode::InvalidRequest + ); + assert_eq!( + validate_object_lambda_query(&duplicated) + .expect_err("duplicated lambdaArn should fail") + .code(), + &S3ErrorCode::InvalidRequest + ); + assert_eq!( + validate_object_lambda_query(&invalid_format) + .expect_err("invalid lambdaArn should fail") + .code(), + &S3ErrorCode::InvalidRequest + ); + } + + #[test] + fn validate_object_lambda_query_accepts_arn() { + let valid: Uri = "/demo-bucket/object.txt?lambdaArn=arn%3Aacme%3As3-object-lambda%3A%3Atransformer%3Awebhook" + .parse() + .expect("uri should parse"); + + assert!(validate_object_lambda_query(&valid).is_ok()); + } + + #[test] + fn resolve_object_lambda_webhook_config_from_server_config_accepts_enabled_webhook_target() { + let arn = "arn:acme:s3-object-lambda::transformer:webhook" + .parse::() + .expect("arn should parse"); + let config = rustfs_ecstore::config::Config(std::collections::HashMap::from([( + LAMBDA_WEBHOOK_SUB_SYS.to_string(), + std::collections::HashMap::from([( + "transformer".to_string(), + rustfs_ecstore::config::KVS(vec![ + rustfs_ecstore::config::KV { + key: ENABLE_KEY.to_string(), + value: "on".to_string(), + hidden_if_empty: false, + }, + rustfs_ecstore::config::KV { + key: WEBHOOK_ENDPOINT.to_string(), + value: "https://example.com/transform".to_string(), + hidden_if_empty: false, + }, + rustfs_ecstore::config::KV { + key: WEBHOOK_AUTH_TOKEN.to_string(), + value: "secret-token".to_string(), + hidden_if_empty: true, + }, + ]), + )]), + )])); + + let resolved = resolve_object_lambda_webhook_config_from_server_config(&config, &arn).expect("config should resolve"); + + assert_eq!(resolved.endpoint.as_str(), "https://example.com/transform"); + assert_eq!(resolved.auth_token, "secret-token"); + assert!(!resolved.skip_tls_verify); + assert!(resolved.response_header_timeout.is_none()); + } + + #[test] + fn resolve_object_lambda_webhook_config_from_server_config_accepts_named_webhook_target() { + let arn = "arn:acme:s3-object-lambda::transformer:webhook-csv" + .parse::() + .expect("arn should parse"); + let config = rustfs_ecstore::config::Config(std::collections::HashMap::from([( + LAMBDA_WEBHOOK_SUB_SYS.to_string(), + std::collections::HashMap::from([( + "transformer".to_string(), + rustfs_ecstore::config::KVS(vec![ + rustfs_ecstore::config::KV { + key: ENABLE_KEY.to_string(), + value: "on".to_string(), + hidden_if_empty: false, + }, + rustfs_ecstore::config::KV { + key: WEBHOOK_ENDPOINT.to_string(), + value: "https://example.com/transform-csv".to_string(), + hidden_if_empty: false, + }, + ]), + )]), + )])); + + let resolved = resolve_object_lambda_webhook_config_from_server_config(&config, &arn).expect("config should resolve"); + assert_eq!(resolved.endpoint.as_str(), "https://example.com/transform-csv"); + } + + #[test] + fn resolve_object_lambda_webhook_config_from_server_config_parses_response_header_timeout() { + let arn = "arn:acme:s3-object-lambda::transformer:webhook" + .parse::() + .expect("arn should parse"); + let config = rustfs_ecstore::config::Config(std::collections::HashMap::from([( + LAMBDA_WEBHOOK_SUB_SYS.to_string(), + std::collections::HashMap::from([( + "transformer".to_string(), + rustfs_ecstore::config::KVS(vec![ + rustfs_ecstore::config::KV { + key: ENABLE_KEY.to_string(), + value: "on".to_string(), + hidden_if_empty: false, + }, + rustfs_ecstore::config::KV { + key: WEBHOOK_ENDPOINT.to_string(), + value: "https://example.com/transform".to_string(), + hidden_if_empty: false, + }, + rustfs_ecstore::config::KV { + key: WEBHOOK_RESPONSE_HEADER_TIMEOUT.to_string(), + value: "2s".to_string(), + hidden_if_empty: false, + }, + ]), + )]), + )])); + + let resolved = resolve_object_lambda_webhook_config_from_server_config(&config, &arn).expect("config should resolve"); + assert_eq!(resolved.response_header_timeout, Some(Duration::from_secs(2))); + } + + #[test] + fn resolve_object_lambda_webhook_config_from_server_config_accepts_notify_webhook_fallback() { + let arn = "arn:acme:s3-object-lambda::transformer:webhook" + .parse::() + .expect("arn should parse"); + let config = rustfs_ecstore::config::Config(std::collections::HashMap::from([( + NOTIFY_WEBHOOK_SUB_SYS.to_string(), + std::collections::HashMap::from([( + "transformer".to_string(), + rustfs_ecstore::config::KVS(vec![ + rustfs_ecstore::config::KV { + key: ENABLE_KEY.to_string(), + value: "on".to_string(), + hidden_if_empty: false, + }, + rustfs_ecstore::config::KV { + key: WEBHOOK_ENDPOINT.to_string(), + value: "https://example.com/notify-transform".to_string(), + hidden_if_empty: false, + }, + ]), + )]), + )])); + + let resolved = resolve_object_lambda_webhook_config_from_server_config(&config, &arn).expect("config should resolve"); + assert_eq!(resolved.endpoint.as_str(), "https://example.com/notify-transform"); + } + + #[test] + fn resolve_object_lambda_webhook_config_from_server_config_rejects_invalid_response_header_timeout() { + let arn = "arn:acme:s3-object-lambda::transformer:webhook" + .parse::() + .expect("arn should parse"); + let config = rustfs_ecstore::config::Config(std::collections::HashMap::from([( + LAMBDA_WEBHOOK_SUB_SYS.to_string(), + std::collections::HashMap::from([( + "transformer".to_string(), + rustfs_ecstore::config::KVS(vec![ + rustfs_ecstore::config::KV { + key: ENABLE_KEY.to_string(), + value: "on".to_string(), + hidden_if_empty: false, + }, + rustfs_ecstore::config::KV { + key: WEBHOOK_ENDPOINT.to_string(), + value: "https://example.com/transform".to_string(), + hidden_if_empty: false, + }, + rustfs_ecstore::config::KV { + key: WEBHOOK_RESPONSE_HEADER_TIMEOUT.to_string(), + value: "definitely-not-a-duration".to_string(), + hidden_if_empty: false, + }, + ]), + )]), + )])); + + let err = + resolve_object_lambda_webhook_config_from_server_config(&config, &arn).expect_err("invalid timeout should fail"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + } + + #[test] + fn resolve_object_lambda_webhook_config_from_server_config_rejects_unsupported_or_disabled_targets() { + let unsupported = "arn:acme:s3-object-lambda::transformer:mqtt" + .parse::() + .expect("arn should parse"); + let empty_config = rustfs_ecstore::config::Config(std::collections::HashMap::new()); + let unsupported_err = resolve_object_lambda_webhook_config_from_server_config(&empty_config, &unsupported) + .expect_err("unsupported target type should fail"); + assert_eq!(unsupported_err.code(), &S3ErrorCode::NotImplemented); + + let webhook = "arn:acme:s3-object-lambda::transformer:webhook" + .parse::() + .expect("arn should parse"); + let disabled_config = rustfs_ecstore::config::Config(std::collections::HashMap::from([( + LAMBDA_WEBHOOK_SUB_SYS.to_string(), + std::collections::HashMap::from([( + "transformer".to_string(), + rustfs_ecstore::config::KVS(vec![ + rustfs_ecstore::config::KV { + key: ENABLE_KEY.to_string(), + value: "off".to_string(), + hidden_if_empty: false, + }, + rustfs_ecstore::config::KV { + key: WEBHOOK_ENDPOINT.to_string(), + value: "https://example.com/transform".to_string(), + hidden_if_empty: false, + }, + ]), + )]), + )])); + + let disabled_err = resolve_object_lambda_webhook_config_from_server_config(&disabled_config, &webhook) + .expect_err("disabled target should fail"); + assert_eq!(disabled_err.code(), &S3ErrorCode::InvalidRequest); + } + + #[test] + fn clear_object_lambda_variant_headers_removes_original_object_payload_headers() { + let mut headers = HeaderMap::new(); + headers.insert(http::header::CONTENT_LENGTH, HeaderValue::from_static("7")); + headers.insert(http::header::CONTENT_TYPE, HeaderValue::from_static("text/plain")); + headers.insert("x-amz-meta-demo", HeaderValue::from_static("value")); + headers.insert("x-amz-version-id", HeaderValue::from_static("v1")); + + clear_object_lambda_variant_headers(&mut headers); + + assert!(headers.get(http::header::CONTENT_LENGTH).is_none()); + assert!(headers.get(http::header::CONTENT_TYPE).is_none()); + assert!(headers.get("x-amz-meta-demo").is_none()); + assert_eq!(headers.get("x-amz-version-id").and_then(|value| value.to_str().ok()), Some("v1")); + } + + #[test] + fn build_object_lambda_passthrough_response_preserves_target_status_and_filters_headers() { + let mut upstream_headers = HeaderMap::new(); + upstream_headers.insert(http::header::CONTENT_LENGTH, HeaderValue::from_static("7")); + upstream_headers.insert(http::header::CONTENT_TYPE, HeaderValue::from_static("text/plain")); + upstream_headers.insert("x-amz-meta-demo", HeaderValue::from_static("value")); + upstream_headers.insert("x-amz-version-id", HeaderValue::from_static("v1")); + + let mut lambda_headers = HeaderMap::new(); + lambda_headers.insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/json")); + lambda_headers.insert("x-rustfs-lambda-error", HeaderValue::from_static("upstream")); + lambda_headers.insert(http::header::CONNECTION, HeaderValue::from_static("keep-alive")); + lambda_headers.insert("x-amz-request-route", HeaderValue::from_static("route-token")); + lambda_headers.insert("x-amz-request-token", HeaderValue::from_static("request-token")); + + let response = build_object_lambda_passthrough_response( + upstream_headers, + &lambda_headers, + StatusCode::BAD_GATEWAY, + Body::from("lambda failed".to_string()), + ); + + assert_eq!(response.status, Some(StatusCode::BAD_GATEWAY)); + assert_eq!( + response + .headers + .get(http::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()), + Some("application/json") + ); + assert_eq!( + response + .headers + .get("x-rustfs-lambda-error") + .and_then(|value| value.to_str().ok()), + Some("upstream") + ); + assert!(response.headers.get(http::header::CONTENT_LENGTH).is_none()); + assert!(response.headers.get("x-amz-meta-demo").is_none()); + assert!(response.headers.get(http::header::CONNECTION).is_none()); + assert!(response.headers.get("x-amz-request-route").is_none()); + assert!(response.headers.get("x-amz-request-token").is_none()); + assert_eq!(response.headers.get("x-amz-version-id").and_then(|value| value.to_str().ok()), Some("v1")); + } + + #[test] + fn build_object_lambda_source_url_presigns_request_without_lambda_arn() { + let req = S3Request { + input: Body::from(String::new()), + method: Method::GET, + uri: "/demo-bucket/object.txt?lambdaArn=arn%3Aacme%3As3-object-lambda%3A%3Atransformer%3Awebhook&versionId=v1" + .parse() + .expect("uri should parse"), + headers: HeaderMap::from_iter([(http::header::HOST, HeaderValue::from_static("localhost:9000"))]), + extensions: http::Extensions::new(), + credentials: Some(s3s::auth::Credentials { + access_key: "rustfsadmin".to_string(), + secret_key: s3s::auth::SecretKey::from("rustfssecret"), + }), + region: get_global_region(), + service: None, + trailing_headers: None, + }; + + let source_url = build_object_lambda_source_url(&req).expect("source url should build"); + let source_url = Url::parse(&source_url).expect("source url should parse"); + let query_pairs = source_url.query_pairs().collect::>(); + + assert_eq!(source_url.scheme(), "http"); + assert_eq!(source_url.host_str(), Some("localhost")); + assert_eq!(source_url.port_or_known_default(), Some(9000)); + assert_eq!(source_url.path(), "/demo-bucket/object.txt"); + assert_eq!(query_pairs.get("versionId").map(|value| value.as_ref()), Some("v1")); + assert!(!query_pairs.contains_key("lambdaArn")); + let expires = query_pairs.get("X-Amz-Expires").and_then(|value| value.parse::().ok()); + assert_eq!(expires, Some(3600)); + assert_eq!(query_pairs.get("X-Amz-Algorithm").map(|value| value.as_ref()), Some("AWS4-HMAC-SHA256")); + assert!(query_pairs.contains_key("X-Amz-Signature")); + } + + #[test] + fn build_object_lambda_event_payload_contains_required_context() { + let req = S3Request { + input: Body::from(String::new()), + method: Method::GET, + uri: "/demo-bucket/object.txt?lambdaArn=arn%3Aacme%3As3-object-lambda%3A%3Atransformer%3Awebhook" + .parse() + .expect("uri should parse"), + headers: HeaderMap::from_iter([(http::header::HOST, HeaderValue::from_static("localhost:9000"))]), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + + let payload = build_object_lambda_event_payload( + &req, + "arn:acme:s3-object-lambda::transformer:webhook", + "https://example.com/source", + "route-123", + "token-456", + ) + .expect("payload should serialize"); + let payload: serde_json::Value = serde_json::from_slice(&payload).expect("payload should be json"); + + assert_eq!(payload["getObjectContext"]["inputS3Url"], "https://example.com/source"); + assert_eq!(payload["getObjectContext"]["outputRoute"], "route-123"); + assert_eq!(payload["getObjectContext"]["outputToken"], "token-456"); + assert_eq!( + payload["configuration"]["accessPointArn"], + "arn:acme:s3-object-lambda::transformer:webhook" + ); + assert_eq!( + payload["userRequest"]["url"], + "/demo-bucket/object.txt?lambdaArn=arn%3Aacme%3As3-object-lambda%3A%3Atransformer%3Awebhook" + ); + } + + #[test] + fn validate_object_lambda_response_auth_headers_rejects_missing_or_mismatched_values() { + let mut matching = HeaderMap::new(); + matching.insert("x-amz-request-route", HeaderValue::from_static("route-123")); + matching.insert("x-amz-request-token", HeaderValue::from_static("token-456")); + assert!(validate_object_lambda_response_auth_headers(&matching, "route-123", "token-456").is_ok()); + + let missing = HeaderMap::new(); + let err = validate_object_lambda_response_auth_headers(&missing, "route-123", "token-456") + .expect_err("missing auth headers should fail"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + + let mut mismatched = HeaderMap::new(); + mismatched.insert("x-amz-request-route", HeaderValue::from_static("route-123")); + mismatched.insert("x-amz-request-token", HeaderValue::from_static("wrong-token")); + let err = validate_object_lambda_response_auth_headers(&mismatched, "route-123", "token-456") + .expect_err("mismatched auth headers should fail"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + } + + #[test] + fn build_object_lambda_get_request_removes_lambda_arn_and_preserves_request_inputs() { + let mut req = S3Request { + input: Body::from(String::new()), + method: Method::GET, + uri: "/demo-bucket/object.txt?lambdaArn=arn%3Aacme%3As3-object-lambda%3A%3Atransformer%3Awebhook&versionId=v1&partNumber=7" + .parse() + .expect("uri should parse"), + headers: HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + req.headers + .insert(http::header::RANGE, HeaderValue::from_static("bytes=5-10")); + req.headers + .insert(http::header::IF_MATCH, HeaderValue::from_static("\"abc\"")); + + let bridged = build_object_lambda_get_request(&req, "demo-bucket", "object.txt").expect("bridge request should build"); + + assert_eq!(bridged.uri.path(), "/demo-bucket/object.txt"); + assert_eq!(bridged.uri.query(), Some("versionId=v1&partNumber=7")); + assert_eq!(bridged.input.bucket, "demo-bucket"); + assert_eq!(bridged.input.key, "object.txt"); + assert_eq!(bridged.input.version_id.as_deref(), Some("v1")); + assert_eq!(bridged.input.part_number, Some(7)); + assert_eq!( + bridged.input.range, + Some(Range::Int { + first: 5, + last: Some(10) + }) + ); + assert!(bridged.input.if_match.is_some()); + } + + #[test] + fn build_object_lambda_get_request_rejects_invalid_range_header() { + let mut req = S3Request { + input: Body::from(String::new()), + method: Method::GET, + uri: "/demo-bucket/object.txt?lambdaArn=arn%3Aacme%3As3-object-lambda%3A%3Atransformer%3Awebhook" + .parse() + .expect("uri should parse"), + headers: HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + req.headers + .insert(http::header::RANGE, HeaderValue::from_static("bytes=10-5")); + + let err = build_object_lambda_get_request(&req, "demo-bucket", "object.txt").expect_err("invalid range must fail"); + assert_eq!(err.code(), &S3ErrorCode::InvalidArgument); + } + + #[test] + fn convert_get_object_response_maps_core_headers() { + let mut resp = S3Response::new(GetObjectOutput { + body: Some(Body::from("payload".to_string()).into()), + content_length: Some(7), + content_type: Some("text/plain".to_string()), + accept_ranges: Some("bytes".to_string()), + version_id: Some("v1".to_string()), + metadata: Some(std::collections::HashMap::from([("custom-key".to_string(), "custom-value".to_string())])), + ..Default::default() + }); + resp.status = Some(StatusCode::OK); + + let converted = convert_get_object_response(resp).expect("response conversion should succeed"); + + assert_eq!(converted.status, Some(StatusCode::OK)); + assert_eq!( + converted + .headers + .get(http::header::CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()), + Some("7") + ); + assert_eq!( + converted + .headers + .get(http::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()), + Some("text/plain") + ); + assert_eq!( + converted + .headers + .get(http::header::ACCEPT_RANGES) + .and_then(|value| value.to_str().ok()), + Some("bytes") + ); + assert_eq!( + converted + .headers + .get("x-amz-version-id") + .and_then(|value| value.to_str().ok()), + Some("v1") + ); + assert_eq!( + converted + .headers + .get("x-amz-meta-custom-key") + .and_then(|value| value.to_str().ok()), + Some("custom-value") + ); + } + + #[tokio::test] + async fn check_access_rejects_anonymous_replication_extension_request() { + let router: S3Router = S3Router::new(false); + let mut req = S3Request { + input: Body::from(String::new()), + method: Method::GET, + uri: "/demo-bucket?replication-metrics".parse().expect("uri should parse"), + headers: HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + + let err = router + .check_access(&mut req) + .await + .expect_err("anonymous extension request must be denied"); + assert_eq!(err.code(), &S3ErrorCode::AccessDenied); + } + + #[tokio::test] + async fn check_access_rejects_anonymous_misc_extension_request() { + let router: S3Router = S3Router::new(false); + let mut req = S3Request { + input: Body::from(String::new()), + method: Method::GET, + uri: "/demo-bucket/path/object.txt?lambdaArn=arn%3Atarget" + .parse() + .expect("uri should parse"), + headers: HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + + let err = router + .check_access(&mut req) + .await + .expect_err("anonymous extension request must be denied"); + assert_eq!(err.code(), &S3ErrorCode::AccessDenied); + } + + #[test] + fn listen_notification_keepalive_plan_defaults_to_space_keepalive() { + let uri: Uri = "/demo-bucket?events=s3:ObjectCreated:Put".parse().expect("uri should parse"); + let (interval, payload) = listen_notification_keepalive_plan(&uri); + assert_eq!(interval, Duration::from_millis(500)); + assert_eq!(payload, Bytes::from_static(b" ")); + } + + #[test] + fn listen_notification_keepalive_plan_uses_empty_record_payload_when_ping_is_present() { + let uri: Uri = "/demo-bucket?events=s3:ObjectCreated:Put&ping=3" + .parse() + .expect("uri should parse"); + let (interval, payload) = listen_notification_keepalive_plan(&uri); + assert_eq!(interval, Duration::from_secs(3)); + assert_eq!(payload, Bytes::from_static(b"{\"Records\":[]}\n")); + } + + #[test] + fn parse_listen_notification_filter_expands_event_mask_and_filters() { + let uri: Uri = "/demo-bucket?events=s3:ObjectCreated:*&events=s3:ObjectRemoved:Delete&prefix=logs/&suffix=.json" + .parse() + .expect("uri should parse"); + + let filter = parse_listen_notification_filter(&uri, Some("demo-bucket")).expect("filter should parse"); + + assert_eq!(filter.bucket.as_deref(), Some("demo-bucket")); + assert_eq!(filter.prefix.as_deref(), Some("logs/")); + assert_eq!(filter.suffix.as_deref(), Some(".json")); + assert_ne!(filter.event_mask & EventName::ObjectCreatedPut.mask(), 0); + assert_ne!(filter.event_mask & EventName::ObjectRemovedDelete.mask(), 0); + assert_eq!(filter.event_mask & EventName::ObjectAccessedGet.mask(), 0); + } + + #[test] + fn event_matches_listen_notification_respects_bucket_event_and_object_filters() { + let filter = ListenNotificationFilter { + bucket: Some("demo-bucket".to_string()), + event_mask: EventName::ObjectCreatedPut.mask() | EventName::ObjectCreatedPost.mask(), + prefix: Some("logs/".to_string()), + suffix: Some(".json".to_string()), + }; + + let matched = NotificationEvent::new_test_event("demo-bucket", "logs/app.json", EventName::ObjectCreatedPut); + assert!(event_matches_listen_notification(&matched, &filter)); + + let wrong_bucket = NotificationEvent::new_test_event("other-bucket", "logs/app.json", EventName::ObjectCreatedPut); + assert!(!event_matches_listen_notification(&wrong_bucket, &filter)); + + let wrong_event = NotificationEvent::new_test_event("demo-bucket", "logs/app.json", EventName::ObjectRemovedDelete); + assert!(!event_matches_listen_notification(&wrong_event, &filter)); + + let wrong_prefix = NotificationEvent::new_test_event("demo-bucket", "archive/app.json", EventName::ObjectCreatedPut); + assert!(!event_matches_listen_notification(&wrong_prefix, &filter)); + + let wrong_suffix = NotificationEvent::new_test_event("demo-bucket", "logs/app.txt", EventName::ObjectCreatedPut); + assert!(!event_matches_listen_notification(&wrong_suffix, &filter)); + } + + #[test] + fn event_matches_listen_notification_decodes_object_key_before_filtering() { + let filter = ListenNotificationFilter { + bucket: Some("demo-bucket".to_string()), + event_mask: EventName::ObjectCreatedPut.mask(), + prefix: Some("logs/".to_string()), + suffix: Some(".json".to_string()), + }; + + let encoded = NotificationEvent::new_test_event("demo-bucket", "logs%2Fapp.json", EventName::ObjectCreatedPut); + assert!(event_matches_listen_notification(&encoded, &filter)); + } + + #[test] + fn serialize_listen_notification_event_wraps_records_payload() { + let event = NotificationEvent::new_test_event("demo-bucket", "logs/app.json", EventName::ObjectCreatedPut); + + let payload = serialize_listen_notification_event(&event).expect("payload should serialize"); + let body = std::str::from_utf8(payload.as_ref()).expect("payload should be utf-8"); + + assert!(body.contains("\"Records\":[")); + assert!(body.contains("\"name\":\"demo-bucket\"")); + assert!(body.contains("\"eventName\":\"ObjectCreatedPut\"") || body.contains("s3:ObjectCreated:Put")); + assert!(body.ends_with('\n')); + } + + #[tokio::test] + async fn build_listen_notification_response_sets_event_stream_headers() { + let uri: Uri = "/demo-bucket?events=s3:ObjectCreated:Put&ping=1" + .parse() + .expect("uri should parse"); + + let resp = build_listen_notification_response(&uri, Some("demo-bucket")).expect("response should build"); + + assert_eq!( + resp.headers.get(header::CONTENT_TYPE).and_then(|v| v.to_str().ok()), + Some("text/event-stream") + ); + assert_eq!(resp.headers.get(header::CACHE_CONTROL).and_then(|v| v.to_str().ok()), Some("no-cache")); + assert_eq!(resp.headers.get("x-accel-buffering").and_then(|v| v.to_str().ok()), Some("no")); + } } #[allow(dead_code)] diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index 800efa3de..1143e6cfc 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -34,6 +34,7 @@ use rustfs_ecstore::bucket::{ metadata_sys, quota::QuotaOperation, replication::{get_must_replicate_options, must_replicate, schedule_replication}, + versioning_sys::BucketVersioningSys, }; use rustfs_ecstore::client::object_api_utils::to_s3s_etag; use rustfs_ecstore::compress::is_compressible; @@ -315,10 +316,13 @@ impl DefaultMultipartUsecase { server_side_encryption ); - let ssekms_key_id = multipart_info - .user_defined - .get("x-amz-server-side-encryption-aws-kms-key-id") - .cloned(); + let ssekms_key_id = match server_side_encryption.as_ref() { + Some(sse) if sse.as_str() == ServerSideEncryption::AWS_KMS => multipart_info + .user_defined + .get("x-amz-server-side-encryption-aws-kms-key-id") + .cloned(), + _ => None, + }; info!( "TDD: Extracted encryption info - SSE: {:?}, KMS Key: {:?}", @@ -367,7 +371,12 @@ impl DefaultMultipartUsecase { let manager = get_concurrency_manager(); let mpu_bucket = bucket.clone(); let mpu_key = key.clone(); - let mpu_version = obj_info.version_id.map(|v| v.to_string()); + let raw_mpu_version = obj_info.version_id.map(|v| v.to_string()); + let mpu_version = if BucketVersioningSys::prefix_enabled(&bucket, &key).await { + raw_mpu_version.clone() + } else { + None + }; let mpu_version_clone = mpu_version.clone(); let mpu_version_for_event = mpu_version.clone(); tokio::spawn(async move { @@ -495,6 +504,14 @@ impl DefaultMultipartUsecase { .. } = req.input.clone(); + let server_side_encryption = server_side_encryption.or(extract_server_side_encryption_from_headers(&req.headers)?); + let ssekms_key_id = ssekms_key_id.or_else(|| { + req.headers + .get("x-amz-server-side-encryption-aws-kms-key-id") + .and_then(|value| value.to_str().ok()) + .map(ToOwned::to_owned) + }); + // Validate storage class if provided if let Some(ref storage_class) = storage_class && !is_valid_storage_class(storage_class.as_str()) @@ -708,10 +725,13 @@ impl DefaultMultipartUsecase { .map_err(|e| ApiError::from(StorageError::other(format!("Invalid server-side encryption: {e}")))) }) .transpose()?; - let key_id = fi - .user_defined - .get("x-amz-server-side-encryption-aws-kms-key-id") - .map(|s| s.to_string()); + let key_id = match sse.as_ref() { + Some(sse) if sse.as_str() == ServerSideEncryption::AWS_KMS => fi + .user_defined + .get("x-amz-server-side-encryption-aws-kms-key-id") + .map(|s| s.to_string()), + _ => None, + }; (sse, key_id) }; let part_key = fi.user_defined.get("x-rustfs-encryption-key").cloned(); @@ -1045,6 +1065,7 @@ impl DefaultMultipartUsecase { sse_customer_key_md5: copy_source_sse_customer_key_md5.as_ref(), part_number: None, parts: &src_info.parts, + etag: src_info.etag.as_deref(), }; if let Some(material) = sse_decryption(src_decryption_request).await? { @@ -1073,10 +1094,13 @@ impl DefaultMultipartUsecase { .map_err(|e| ApiError::from(StorageError::other(format!("Invalid server-side encryption: {e}")))) }) .transpose()?; - let ssekms_key_id = mp_info - .user_defined - .get("x-amz-server-side-encryption-aws-kms-key-id") - .map(|s| s.to_string()); + let ssekms_key_id = match server_side_encryption.as_ref() { + Some(sse) if sse.as_str() == ServerSideEncryption::AWS_KMS => mp_info + .user_defined + .get("x-amz-server-side-encryption-aws-kms-key-id") + .map(|s| s.to_string()), + _ => None, + }; let part_key = mp_info.user_defined.get("x-rustfs-encryption-key").cloned(); let part_nonce = mp_info.user_defined.get("x-rustfs-encryption-iv").cloned(); let encryption_request = EncryptionRequest { diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index c633cc0f6..e22e1e3ee 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -17,7 +17,7 @@ use crate::app::context::{AppContext, default_notify_interface, get_global_app_context}; use crate::config::RustFSBufferConfig; use crate::error::ApiError; -use crate::storage::access::{authorize_request, has_bypass_governance_header, req_info_mut}; +use crate::storage::access::{PostObjectRequestMarker, authorize_request, has_bypass_governance_header, req_info_mut}; use crate::storage::concurrency::{ CachedGetObject, ConcurrencyManager, GetObjectGuard, get_concurrency_aware_buffer_size, get_concurrency_manager, }; @@ -26,7 +26,7 @@ use crate::storage::head_prefix::{head_prefix_not_found_message, probe_prefix_ha use crate::storage::helper::OperationHelper; use crate::storage::options::{ copy_dst_opts, copy_src_opts, del_opts, extract_metadata, extract_metadata_from_mime_with_object_name, - filter_object_metadata, get_content_sha256_with_query, get_opts, put_opts, + filter_object_metadata, get_content_sha256_with_query, get_opts, normalize_content_encoding_for_storage, put_opts, }; use crate::storage::s3_api::multipart::parse_list_parts_params; use crate::storage::s3_api::{acl, restore, select}; @@ -38,7 +38,9 @@ use datafusion::arrow::{ }; use futures::StreamExt; use http::{HeaderMap, HeaderValue, StatusCode}; +use md5::Context as Md5Context; use metrics::{counter, histogram}; +use pin_project_lite::pin_project; use rustfs_ecstore::bucket::quota::checker::QuotaChecker; use rustfs_ecstore::bucket::{ lifecycle::{ @@ -84,12 +86,13 @@ use rustfs_s3select_api::{ use rustfs_s3select_query::get_global_db; use rustfs_targets::EventName; use rustfs_utils::http::{ - AMZ_BUCKET_REPLICATION_STATUS, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, - SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, + AMZ_BUCKET_REPLICATION_STATUS, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE, AMZ_WEBSITE_REDIRECT_LOCATION, SUFFIX_ACTUAL_SIZE, + SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, headers::{ AMZ_DECODED_CONTENT_LENGTH, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, - AMZ_OBJECT_TAGGING, AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE, AMZ_STORAGE_CLASS, AMZ_TAG_COUNT, + AMZ_OBJECT_TAGGING, AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE, AMZ_SERVER_SIDE_ENCRYPTION, + AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, AMZ_SNOWBALL_EXTRACT, AMZ_STORAGE_CLASS, AMZ_TAG_COUNT, }, insert_str, remove_str, }; @@ -107,8 +110,9 @@ use std::convert::Infallible; use std::ops::Add; use std::path::Path; use std::str::FromStr; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; +use tokio::io::{AsyncRead, ReadBuf}; use tokio::sync::RwLock; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; @@ -117,6 +121,54 @@ use tokio_util::io::{ReaderStream, StreamReader}; use tracing::{debug, error, info, instrument, warn}; use uuid::Uuid; +pin_project! { + struct ExtractArchiveEtagReader { + #[pin] + inner: R, + md5: Md5Context, + finished: bool, + etag: Arc>>, + } +} + +impl ExtractArchiveEtagReader { + fn new(inner: R, etag: Arc>>) -> Self { + Self { + inner, + md5: Md5Context::new(), + finished: false, + etag, + } + } +} + +impl AsyncRead for ExtractArchiveEtagReader { + fn poll_read( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> std::task::Poll> { + let this = self.project(); + let before = buf.filled().len(); + match this.inner.poll_read(cx, buf) { + std::task::Poll::Pending => std::task::Poll::Pending, + std::task::Poll::Ready(Ok(())) => { + let filled = &buf.filled()[before..]; + if !filled.is_empty() { + this.md5.consume(filled); + } else if !*this.finished { + *this.finished = true; + if let Ok(mut etag) = this.etag.lock() { + *etag = Some(format!("{:x}", this.md5.clone().finalize())); + } + } + std::task::Poll::Ready(Ok(())) + } + std::task::Poll::Ready(Err(err)) => std::task::Poll::Ready(Err(err)), + } + } +} + async fn maybe_enqueue_transition_immediate(obj_info: &ObjectInfo, src: LcEventSrc) { enqueue_transition_immediate(obj_info, src).await; } @@ -195,6 +247,237 @@ fn build_put_object_expiration_header(event: &lifecycle::Event) -> Option, + ignore_dirs: bool, + ignore_errors: bool, +} + +fn header_value_is_true(headers: &HeaderMap, key: &str) -> bool { + headers + .get(key) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.trim().eq_ignore_ascii_case("true")) +} + +fn is_put_object_extract_requested(headers: &HeaderMap) -> bool { + header_value_is_true(headers, AMZ_SNOWBALL_EXTRACT) || header_value_is_true(headers, AMZ_SNOWBALL_EXTRACT_COMPAT) +} + +fn snowball_meta_value_by_suffix(headers: &HeaderMap, preferred_key: &str, suffix_lower: &str) -> Option { + if let Some(preferred) = headers.get(preferred_key).and_then(|value| value.to_str().ok()) { + return Some(preferred.trim().to_string()); + } + + for (name, value) in headers { + let key = name.as_str().to_ascii_lowercase(); + if key.starts_with(AMZ_META_PREFIX_LOWER) + && key.ends_with(suffix_lower) + && let Ok(parsed) = value.to_str() + { + return Some(parsed.trim().to_string()); + } + } + + None +} + +fn snowball_meta_flag_by_suffix(headers: &HeaderMap, preferred_key: &str, suffix_lower: &str) -> bool { + snowball_meta_value_by_suffix(headers, preferred_key, suffix_lower).is_some_and(|value| value.eq_ignore_ascii_case("true")) +} + +fn normalize_snowball_prefix(prefix: &str) -> Option { + let normalized = prefix.trim().trim_matches('/'); + if normalized.is_empty() { + return None; + } + + Some(normalized.to_string()) +} + +fn normalize_extract_entry_key(path: &str, prefix: Option<&str>, is_dir: bool) -> String { + let path = path.trim_matches('/'); + let mut key = match prefix { + Some(prefix) if !path.is_empty() => format!("{prefix}/{path}"), + Some(prefix) => prefix.to_string(), + None => path.to_string(), + }; + + if is_dir && !key.ends_with('/') { + key.push('/'); + } + + key +} + +fn map_extract_archive_error(err: impl std::fmt::Display) -> S3Error { + s3_error!(InvalidArgument, "Failed to process archive entry: {}", err) +} + +async fn apply_extract_entry_pax_extensions( + entry: &mut tokio_tar::Entry>, + metadata: &mut HashMap, + opts: &mut ObjectOptions, +) -> S3Result<()> +where + R: AsyncRead + Send + Unpin + 'static, +{ + let Some(extensions) = entry.pax_extensions().await.map_err(map_extract_archive_error)? else { + return Ok(()); + }; + + for ext in extensions { + let ext = ext.map_err(map_extract_archive_error)?; + let key = ext.key().map_err(map_extract_archive_error)?; + let value = ext.value().map_err(map_extract_archive_error)?; + + if let Some(meta_key) = key.strip_prefix("minio.metadata.") { + let meta_key = meta_key.strip_prefix("x-amz-meta-").unwrap_or(meta_key); + if !meta_key.is_empty() { + metadata.insert(meta_key.to_string(), value.to_string()); + } + continue; + } + + if key == "minio.versionId" && !value.is_empty() { + opts.version_id = Some(value.to_string()); + } + } + + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn apply_put_request_metadata( + metadata: &mut HashMap, + headers: &HeaderMap, + object_name: &str, + cache_control: Option, + content_disposition: Option, + content_encoding: Option, + content_language: Option, + content_type: Option, + expires: Option, + website_redirect_location: Option, + tagging: Option, + storage_class: Option, +) -> S3Result<()> { + if let Some(cache_control) = cache_control { + metadata.insert("cache-control".to_string(), cache_control.to_string()); + } + if let Some(content_disposition) = content_disposition { + metadata.insert("content-disposition".to_string(), content_disposition.to_string()); + } + if let Some(content_encoding) = content_encoding + && let Some(normalized_content_encoding) = normalize_content_encoding_for_storage(&content_encoding) + { + metadata.insert("content-encoding".to_string(), normalized_content_encoding); + } + if let Some(content_language) = content_language { + metadata.insert("content-language".to_string(), content_language.to_string()); + } + if let Some(content_type) = content_type { + metadata.insert("content-type".to_string(), content_type.to_string()); + } + if let Some(expires) = expires { + let mut formatted = Vec::new(); + expires + .format(TimestampFormat::HttpDate, &mut formatted) + .map_err(|e| ApiError::from(StorageError::other(format!("Invalid expires timestamp: {e}"))))?; + metadata.insert("expires".to_string(), String::from_utf8_lossy(&formatted).into_owned()); + } + if let Some(website_redirect_location) = website_redirect_location { + metadata.insert(AMZ_WEBSITE_REDIRECT_LOCATION.to_string(), website_redirect_location.to_string()); + } + if let Some(tags) = tagging { + metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags.to_string()); + } + if let Some(storage_class) = storage_class { + metadata.insert(AMZ_STORAGE_CLASS.to_string(), storage_class.as_str().to_string()); + } + + extract_metadata_from_mime_with_object_name(headers, metadata, true, Some(object_name)); + Ok(()) +} + +async fn apply_put_request_object_lock_opts( + bucket: &str, + object_lock_legal_hold_status: Option, + object_lock_mode: Option, + object_lock_retain_until_date: Option, + opts: &mut ObjectOptions, +) -> S3Result<()> { + if object_lock_legal_hold_status.is_none() && object_lock_mode.is_none() && object_lock_retain_until_date.is_none() { + return Ok(()); + } + + validate_bucket_object_lock_enabled(bucket).await?; + + let retention = match (object_lock_mode, object_lock_retain_until_date) { + (Some(mode), retain_until_date) => Some(ObjectLockRetention { + mode: Some(ObjectLockRetentionMode::from(mode.as_str().to_string())), + retain_until_date, + }), + (None, Some(retain_until_date)) => Some(ObjectLockRetention { + mode: None, + retain_until_date: Some(retain_until_date), + }), + (None, None) => None, + }; + + let mut eval_metadata = parse_object_lock_retention(retention)?; + eval_metadata.extend(parse_object_lock_legal_hold( + object_lock_legal_hold_status.map(|status| ObjectLockLegalHold { status: Some(status) }), + )?); + + if !eval_metadata.is_empty() { + opts.eval_metadata = Some(eval_metadata); + } + + Ok(()) +} + +fn resolve_put_object_extract_options(headers: &HeaderMap) -> PutObjectExtractOptions { + let prefix = snowball_meta_value_by_suffix(headers, AMZ_SNOWBALL_PREFIX_INTERNAL, SNOWBALL_PREFIX_SUFFIX_LOWER) + .and_then(|value| normalize_snowball_prefix(&value)); + let ignore_dirs = snowball_meta_flag_by_suffix(headers, AMZ_SNOWBALL_IGNORE_DIRS_INTERNAL, SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER); + let ignore_errors = + snowball_meta_flag_by_suffix(headers, AMZ_SNOWBALL_IGNORE_ERRORS_INTERNAL, SNOWBALL_IGNORE_ERRORS_SUFFIX_LOWER); + + PutObjectExtractOptions { + prefix, + ignore_dirs, + ignore_errors, + } +} + +fn is_sse_kms_requested(input: &PutObjectInput, headers: &HeaderMap) -> bool { + input + .server_side_encryption + .as_ref() + .is_some_and(|sse| sse.as_str().eq_ignore_ascii_case(ServerSideEncryption::AWS_KMS)) + || input.ssekms_key_id.is_some() + || headers + .get(AMZ_SERVER_SIDE_ENCRYPTION) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.trim().eq_ignore_ascii_case(ServerSideEncryption::AWS_KMS)) + || headers.contains_key(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID) +} + +fn is_post_object_sse_kms_requested(input: &PutObjectInput, headers: &HeaderMap) -> bool { + is_sse_kms_requested(input, headers) +} + async fn resolve_put_object_expiration(bucket: &str, obj_info: &ObjectInfo) -> Option { let Ok((lifecycle_config, _)) = metadata_sys::get_lifecycle_config(bucket).await else { debug!("resolve_put_object_expiration: lifecycle config not found for bucket {bucket}"); @@ -268,35 +551,48 @@ impl DefaultObjectUsecase { }); } + fn put_object_execution_context(req: &S3Request) -> (EventName, QuotaOperation, &'static str) { + if req.extensions.get::().is_some() { + (EventName::ObjectCreatedPost, QuotaOperation::PostObject, "POST") + } else { + (EventName::ObjectCreatedPut, QuotaOperation::PutObject, "PUT") + } + } + #[instrument(level = "debug", skip(self, _fs, req))] pub async fn execute_put_object(&self, _fs: &FS, req: S3Request) -> S3Result> { if let Some(context) = &self.context { let _ = context.object_store(); } - let mut helper = OperationHelper::new(&req, EventName::ObjectCreatedPut, S3Operation::PutObject); - if req - .headers - .get("X-Amz-Meta-Snowball-Auto-Extract") - .is_some_and(|v| v.to_str().unwrap_or_default() == "true") + let (event_name, quota_operation, request_method_name) = Self::put_object_execution_context(&req); + let mut helper = OperationHelper::new(&req, event_name, S3Operation::PutObject); + if req.extensions.get::().is_some() && is_post_object_sse_kms_requested(&req.input, &req.headers) { + return Err(s3_error!(NotImplemented, "SSE-KMS is not supported for POST object uploads")); + } + if let Some(ref storage_class) = req.input.storage_class + && !is_valid_storage_class(storage_class.as_str()) + { + return Err(s3_error!(InvalidStorageClass)); + } + if is_put_object_extract_requested(&req.headers) { return self.execute_put_object_extract(req).await; } let input = req.input; - // Save SSE-C parameters before moving input - if let Some(ref storage_class) = input.storage_class - && !is_valid_storage_class(storage_class.as_str()) - { - return Err(s3_error!(InvalidStorageClass)); - } let PutObjectInput { body, bucket, + cache_control, key, content_length, + content_disposition, + content_encoding, + content_language, content_type, + expires, tagging, metadata, version_id, @@ -306,6 +602,11 @@ impl DefaultObjectUsecase { sse_customer_key_md5, ssekms_key_id, content_md5, + object_lock_legal_hold_status, + object_lock_mode, + object_lock_retain_until_date, + storage_class, + website_redirect_location, .. } = input; @@ -319,11 +620,10 @@ impl DefaultObjectUsecase { let server_side_encryption = server_side_encryption.or(extract_server_side_encryption_from_headers(&req.headers)?); // Validate object key - validate_object_key(&key, "PUT")?; + validate_object_key(&key, request_method_name)?; if let Some(size) = content_length { - self.check_bucket_quota(&bucket, QuotaOperation::PutObject, size as u64) - .await?; + self.check_bucket_quota(&bucket, quota_operation, size as u64).await?; } let Some(body) = body else { return Err(s3_error!(IncompleteBody)) }; @@ -402,19 +702,32 @@ impl DefaultObjectUsecase { )?; let mut metadata = metadata.unwrap_or_default(); - if let Some(content_type) = content_type { - metadata.insert("content-type".to_string(), content_type.to_string()); - } - - extract_metadata_from_mime_with_object_name(&req.headers, &mut metadata, true, Some(&key)); - - if let Some(tags) = tagging { - metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags.to_string()); - } + apply_put_request_metadata( + &mut metadata, + &req.headers, + &key, + cache_control, + content_disposition, + content_encoding, + content_language, + content_type, + expires, + website_redirect_location, + tagging, + storage_class.clone(), + )?; let mut opts: ObjectOptions = put_opts(&bucket, &key, version_id.clone(), &req.headers, metadata.clone()) .await .map_err(ApiError::from)?; + apply_put_request_object_lock_opts( + &bucket, + object_lock_legal_hold_status, + object_lock_mode, + object_lock_retain_until_date, + &mut opts, + ) + .await?; let mut reader: Box = Box::new(WarpReader::new(body)); @@ -528,7 +841,6 @@ impl DefaultObjectUsecase { Self::spawn_cache_invalidation(bucket.clone(), key.clone(), raw_version.clone()); - // Per S3 spec: only return VersionId when versioning is Enabled (not Suspended or default) let put_version = if BucketVersioningSys::prefix_enabled(&bucket, &key).await { raw_version } else { @@ -576,7 +888,8 @@ impl DefaultObjectUsecase { ..Default::default() }; - // TODO fix response for POST Policy (multipart/form-data), wait s3s crate update, fix issue #1564 + // For browser-based POST uploads (multipart/form-data), response status/body handling + // is decided by s3s PostObject serializer (success_action_status / redirect semantics). let result = Ok(S3Response::new(output)); let _ = helper.complete(&result); @@ -727,7 +1040,7 @@ impl DefaultObjectUsecase { .map_err(ApiError::from)?; // When Object Lock is enabled, automatically enable versioning if not already enabled. - // This matches AWS S3 and MinIO behavior. + // This matches S3-compatible behavior. let versioning_config = BucketVersioningSys::get(&bucket).await.map_err(ApiError::from)?; if !versioning_config.enabled() { let enable_versioning_config = VersioningConfiguration { @@ -1342,6 +1655,7 @@ impl DefaultObjectUsecase { sse_customer_key_md5: req.input.sse_customer_key_md5.as_ref(), part_number: None, parts: &info.parts, + etag: info.etag.as_deref(), }; let (server_side_encryption, sse_customer_algorithm, sse_customer_key_md5, ssekms_key_id, encryption_applied) = @@ -1455,13 +1769,39 @@ impl DefaultObjectUsecase { response_content_length as usize, ))) } else if encryption_applied { - // For encrypted objects (SSE-C or managed SSE), avoid bytes_stream length limiting - // because DecryptReader may need to consume the full encrypted stream. - info!( - "Encrypted object: Using unlimited stream for decryption with buffer size {}", - optimal_buffer_size - ); - Some(StreamingBlob::wrap(ReaderStream::with_capacity(final_stream, optimal_buffer_size))) + let seekable_object_size_threshold = rustfs_config::DEFAULT_OBJECT_SEEK_SUPPORT_THRESHOLD; + let should_buffer_encrypted_object = response_content_length > 0 + && response_content_length <= seekable_object_size_threshold as i64 + && part_number.is_none() + && rs.is_none(); + + if should_buffer_encrypted_object { + let mut buf = Vec::with_capacity(response_content_length as usize); + if let Err(e) = tokio::io::AsyncReadExt::read_to_end(&mut final_stream, &mut buf).await { + error!("Failed to read decrypted object into memory: {}", e); + return Err(ApiError::from(StorageError::other(format!("Failed to read decrypted object: {e}"))).into()); + } + + if buf.len() != response_content_length as usize { + warn!( + "Encrypted object size mismatch during read: expected={} actual={}", + response_content_length, + buf.len() + ); + } + + let mem_reader = InMemoryAsyncReader::new(buf); + Some(StreamingBlob::wrap(bytes_stream( + ReaderStream::with_capacity(Box::new(mem_reader), optimal_buffer_size), + response_content_length as usize, + ))) + } else { + info!( + "Encrypted object: Using unlimited stream for decryption with buffer size {}", + optimal_buffer_size + ); + Some(StreamingBlob::wrap(ReaderStream::with_capacity(final_stream, optimal_buffer_size))) + } } else { let seekable_object_size_threshold = rustfs_config::DEFAULT_OBJECT_SEEK_SUPPORT_THRESHOLD; @@ -1489,7 +1829,7 @@ impl DefaultObjectUsecase { ); } - // Create seekable in-memory reader (similar to MinIO SDK's bytes.Reader) + // Create seekable in-memory reader (similar to common S3 SDK bytes readers) let mem_reader = InMemoryAsyncReader::new(buf); Some(StreamingBlob::wrap(bytes_stream( ReaderStream::with_capacity(Box::new(mem_reader), optimal_buffer_size), @@ -2223,6 +2563,7 @@ impl DefaultObjectUsecase { sse_customer_key_md5: copy_source_sse_customer_key_md5.as_ref(), part_number: None, parts: &src_info.parts, + etag: src_info.etag.as_deref(), }; if let Some(material) = sse_decryption(decryption_request).await? { @@ -2321,8 +2662,13 @@ impl DefaultObjectUsecase { rustfs_ecstore::data_usage::increment_bucket_usage_memory(&bucket, oi.size as u64).await; } - let dest_version = oi.version_id.map(|v| v.to_string()); - Self::spawn_cache_invalidation(bucket.clone(), key.clone(), dest_version.clone()); + let raw_dest_version = oi.version_id.map(|v| v.to_string()); + Self::spawn_cache_invalidation(bucket.clone(), key.clone(), raw_dest_version.clone()); + let dest_version = if BucketVersioningSys::prefix_enabled(&bucket, &key).await { + raw_dest_version + } else { + None + }; // warn!("copy_object oi {:?}", &oi); let object_info = oi.clone(); @@ -3091,6 +3437,7 @@ impl DefaultObjectUsecase { let cache_control = metadata_map.get("cache-control").cloned(); let content_disposition = metadata_map.get("content-disposition").cloned(); let content_language = metadata_map.get("content-language").cloned(); + let website_redirect_location = metadata_map.get(AMZ_WEBSITE_REDIRECT_LOCATION).cloned(); let expires = info.expires.map(Timestamp::from); // Calculate tag count from user_tags already in ObjectInfo @@ -3108,6 +3455,7 @@ impl DefaultObjectUsecase { cache_control, content_disposition, content_language, + website_redirect_location, expires, last_modified, e_tag: info.etag.map(|etag| to_s3s_etag(&etag)), @@ -3473,6 +3821,17 @@ impl DefaultObjectUsecase { #[instrument(level = "debug", skip(self, req))] pub async fn execute_put_object_extract(&self, req: S3Request) -> S3Result> { let helper = OperationHelper::new(&req, EventName::ObjectCreatedPut, S3Operation::PutObject).suppress_event(); + let auth_method = req.method.clone(); + let auth_uri = req.uri.clone(); + let auth_headers = req.headers.clone(); + let auth_extensions = req.extensions.clone(); + let auth_credentials = req.credentials.clone(); + let auth_region = req.region.clone(); + let auth_service = req.service.clone(); + let auth_trailing_headers = req.trailing_headers.clone(); + if is_sse_kms_requested(&req.input, &req.headers) { + return Err(s3_error!(NotImplemented, "SSE-KMS is not supported for extract uploads")); + } let input = req.input; let PutObjectInput { @@ -3480,12 +3839,72 @@ impl DefaultObjectUsecase { bucket, key, version_id, + cache_control, + content_disposition, + content_encoding, content_length, + content_language, + content_type, content_md5, + expires, + object_lock_legal_hold_status, + object_lock_mode, + object_lock_retain_until_date, + server_side_encryption, + sse_customer_algorithm, + sse_customer_key, + sse_customer_key_md5, + ssekms_key_id, + storage_class, + tagging, + website_redirect_location, .. } = input; let event_version_id = version_id; + let (h_algo, h_key, h_md5) = extract_ssec_params_from_headers(&req.headers)?; + let sse_customer_algorithm = sse_customer_algorithm.or(h_algo); + let sse_customer_key = sse_customer_key.or(h_key); + let sse_customer_key_md5 = sse_customer_key_md5.or(h_md5); + + let original_sse = server_side_encryption.or(extract_server_side_encryption_from_headers(&req.headers)?); + let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok(); + let mut effective_sse = original_sse.or_else(|| { + bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { + config.rules.first().and_then(|rule| { + rule.apply_server_side_encryption_by_default + .as_ref() + .map(|sse| match sse.sse_algorithm.as_str() { + "AES256" => ServerSideEncryption::from_static(ServerSideEncryption::AES256), + "aws:kms" => ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS), + _ => ServerSideEncryption::from_static(ServerSideEncryption::AES256), + }) + }) + }) + }); + let mut effective_kms_key_id = ssekms_key_id.or_else(|| { + bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { + config.rules.first().and_then(|rule| { + rule.apply_server_side_encryption_by_default + .as_ref() + .and_then(|sse| sse.kms_master_key_id.clone()) + }) + }) + }); + if effective_sse + .as_ref() + .is_some_and(|sse| sse.as_str().eq_ignore_ascii_case(ServerSideEncryption::AWS_KMS)) + { + return Err(s3_error!(NotImplemented, "SSE-KMS is not supported for extract uploads")); + } + validate_sse_headers_for_write( + effective_sse.as_ref(), + effective_kms_key_id.as_ref(), + sse_customer_algorithm.as_ref(), + sse_customer_key.as_ref(), + sse_customer_key_md5.as_ref(), + true, + )?; let Some(body) = body else { return Err(s3_error!(IncompleteBody)) }; let size = match content_length { @@ -3501,6 +3920,12 @@ impl DefaultObjectUsecase { } } }; + if size == -1 { + return Err(s3_error!(UnexpectedContent)); + } + validate_object_key(&key, "PUT")?; + self.check_bucket_quota(&bucket, QuotaOperation::PutObject, size as u64) + .await?; // Apply adaptive buffer sizing based on file size for optimal streaming performance. // Uses workload profile configuration (enabled by default) to select appropriate buffer size. @@ -3531,16 +3956,19 @@ impl DefaultObjectUsecase { let reader: Box = Box::new(WarpReader::new(body)); - let mut hreader = HashReader::new(reader, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?; + let mut archive_reader = HashReader::new(reader, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?; - if let Err(err) = hreader.add_checksum_from_s3s(&req.headers, req.trailing_headers.clone(), false) { + if let Err(err) = archive_reader.add_checksum_from_s3s(&req.headers, req.trailing_headers.clone(), false) { return Err(ApiError::from(err).into()); } - let decoder = CompressionFormat::from_extension(&ext).get_decoder(hreader).map_err(|e| { - error!("get_decoder err {:?}", e); - s3_error!(InvalidArgument, "get_decoder err") - })?; + let archive_etag = Arc::new(Mutex::new(None)); + let decoder = CompressionFormat::from_extension(&ext) + .get_decoder(ExtractArchiveEtagReader::new(archive_reader, archive_etag.clone())) + .map_err(|e| { + error!("get_decoder err {:?}", e); + s3_error!(InvalidArgument, "get_decoder err") + })?; let mut ar = Archive::new(decoder); let mut entries = ar.entries().map_err(|e| { @@ -3552,11 +3980,7 @@ impl DefaultObjectUsecase { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; - let prefix = req - .headers - .get("X-Amz-Meta-Rustfs-Snowball-Prefix") - .map(|v| v.to_str().unwrap_or_default()) - .unwrap_or_default(); + let extract_options = resolve_put_object_extract_options(&req.headers); let version_id = match event_version_id { Some(v) => v.to_string(), None => String::new(), @@ -3567,88 +3991,190 @@ impl DefaultObjectUsecase { .as_ref() .map(|context| context.notify()) .unwrap_or_else(default_notify_interface); + let req_params = extract_params_header(&req.headers); + let host = get_request_host(&req.headers); + let port = get_request_port(&req.headers); + let user_agent = get_request_user_agent(&req.headers); while let Some(entry) = entries.next().await { - let f = match entry { + let mut f = match entry { Ok(f) => f, Err(e) => { + if extract_options.ignore_errors { + warn!("Skipping archive entry because read failed and ignore-errors is enabled: {e}"); + continue; + } error!("Failed to read archive entry: {}", e); return Err(s3_error!(InvalidArgument, "Failed to read archive entry: {:?}", e)); } }; - if f.header().entry_type().is_dir() { - continue; + let fpath = match f.path() { + Ok(path) => path, + Err(e) => { + if extract_options.ignore_errors { + warn!("Skipping archive entry because path decode failed and ignore-errors is enabled: {e}"); + continue; + } + return Err(s3_error!(InvalidArgument, "Failed to decode archive entry path")); + } + }; + + let is_dir = f.header().entry_type().is_dir(); + let fpath = normalize_extract_entry_key(&fpath.to_string_lossy(), extract_options.prefix.as_deref(), is_dir); + + let mut auth_req = S3Request { + input: PutObjectInput::default(), + method: auth_method.clone(), + uri: auth_uri.clone(), + headers: auth_headers.clone(), + extensions: auth_extensions.clone(), + credentials: auth_credentials.clone(), + region: auth_region.clone(), + service: auth_service.clone(), + trailing_headers: auth_trailing_headers.clone(), + }; + { + let req_info = req_info_mut(&mut auth_req)?; + req_info.bucket = Some(bucket.clone()); + req_info.object = Some(fpath.clone()); + req_info.version_id = None; + } + authorize_request(&mut auth_req, Action::S3Action(S3Action::PutObjectAction)).await?; + + let mut size = f.header().size().unwrap_or_default() as i64; + let archive_entry_mod_time = f + .header() + .mtime() + .ok() + .and_then(|modified_at_secs| OffsetDateTime::from_unix_timestamp(modified_at_secs as i64).ok()); + let mut metadata = HashMap::new(); + apply_put_request_metadata( + &mut metadata, + &req.headers, + &fpath, + cache_control.clone(), + content_disposition.clone(), + content_encoding.clone(), + content_language.clone(), + content_type.clone(), + expires.clone(), + website_redirect_location.clone(), + tagging.clone(), + storage_class.clone(), + )?; + let mut opts = put_opts(&bucket, &fpath, None, &req.headers, metadata.clone()) + .await + .map_err(ApiError::from)?; + apply_extract_entry_pax_extensions(&mut f, &mut metadata, &mut opts).await?; + if archive_entry_mod_time.is_some() { + opts.mod_time = archive_entry_mod_time; } - if let Ok(fpath) = f.path() { - let mut fpath = fpath.to_string_lossy().to_string(); + debug!("Extracting file: {}, size: {} bytes", fpath, size); - if !prefix.is_empty() { - fpath = format!("{prefix}/{fpath}"); + let mut reader: Box = if is_dir { + if extract_options.ignore_dirs { + debug!("Skipping directory entry during archive extract: {}", fpath); + continue; } + size = 0; + Box::new(WarpReader::new(std::io::Cursor::new(Vec::new()))) + } else { + Box::new(WarpReader::new(f)) + }; - let mut size = f.header().size().unwrap_or_default() as i64; + let actual_size = size; - debug!("Extracting file: {}, size: {} bytes", fpath, size); - - let mut reader: Box = Box::new(WarpReader::new(f)); - - let mut metadata = HashMap::new(); - - let actual_size = size; - - if is_compressible(&HeaderMap::new(), &fpath) && size > MIN_COMPRESSIBLE_SIZE as i64 { - insert_str(&mut metadata, SUFFIX_COMPRESSION, CompressionAlgorithm::default().to_string()); - insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string()); - - let hrd = HashReader::new(reader, size, actual_size, None, None, false).map_err(ApiError::from)?; - - reader = Box::new(CompressReader::new(hrd, CompressionAlgorithm::default())); - size = HashReader::SIZE_PRESERVE_LAYER; - } + if !is_dir && is_compressible(&HeaderMap::new(), &fpath) && size > MIN_COMPRESSIBLE_SIZE as i64 { + insert_str(&mut metadata, SUFFIX_COMPRESSION, CompressionAlgorithm::default().to_string()); + insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string()); let hrd = HashReader::new(reader, size, actual_size, None, None, false).map_err(ApiError::from)?; - let mut reader = PutObjReader::new(hrd); - let obj_info = store - .put_object(&bucket, &fpath, &mut reader, &ObjectOptions::default()) - .await + reader = Box::new(CompressReader::new(hrd, CompressionAlgorithm::default())); + size = HashReader::SIZE_PRESERVE_LAYER; + } + + let mut hrd = HashReader::new(reader, size, actual_size, None, None, false).map_err(ApiError::from)?; + apply_put_request_object_lock_opts( + &bucket, + object_lock_legal_hold_status.clone(), + object_lock_mode.clone(), + object_lock_retain_until_date.clone(), + &mut opts, + ) + .await?; + if let Some(material) = sse_encryption(EncryptionRequest { + bucket: &bucket, + key: &fpath, + server_side_encryption: effective_sse.clone(), + ssekms_key_id: effective_kms_key_id.clone(), + sse_customer_algorithm: sse_customer_algorithm.clone(), + sse_customer_key: sse_customer_key.clone(), + sse_customer_key_md5: sse_customer_key_md5.clone(), + content_size: actual_size, + part_number: None, + part_key: None, + part_nonce: None, + }) + .await? + { + effective_sse = Some(material.server_side_encryption.clone()); + effective_kms_key_id = material.kms_key_id.clone(); + + let encrypted_reader = material.wrap_reader(hrd); + hrd = HashReader::new(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false) .map_err(ApiError::from)?; - maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await; - - let manager = get_concurrency_manager(); - let fpath_clone = fpath.clone(); - let bucket_clone = bucket.clone(); - tokio::spawn(async move { - manager.invalidate_cache_versioned(&bucket_clone, &fpath_clone, None).await; - }); - - let e_tag = obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)); - - let output = PutObjectOutput { - e_tag, - ..Default::default() - }; - - let event_args = rustfs_notify::EventArgs { - event_name: EventName::ObjectCreatedPut, - bucket_name: bucket.clone(), - object: obj_info.clone(), - req_params: extract_params_header(&req.headers), - resp_elements: extract_resp_elements(&S3Response::new(output.clone())), - version_id: version_id.clone(), - host: get_request_host(&req.headers), - port: get_request_port(&req.headers), - user_agent: get_request_user_agent(&req.headers), - }; - - let notify = notify.clone(); - tokio::spawn(async move { - notify.notify(event_args).await; - }); + let encryption_metadata = material.metadata; + metadata.extend(encryption_metadata.clone()); + opts.user_defined.extend(encryption_metadata); } + opts.user_defined.extend(metadata); + let mut reader = PutObjReader::new(hrd); + + let obj_info = match store.put_object(&bucket, &fpath, &mut reader, &opts).await { + Ok(info) => info, + Err(e) => { + if extract_options.ignore_errors { + warn!("Skipping archive entry because object write failed and ignore-errors is enabled: {e}"); + continue; + } + return Err(ApiError::from(e).into()); + } + }; + + let manager = get_concurrency_manager(); + let fpath_clone = fpath.clone(); + let bucket_clone = bucket.clone(); + tokio::spawn(async move { + manager.invalidate_cache_versioned(&bucket_clone, &fpath_clone, None).await; + }); + + let e_tag = obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)); + + let output = PutObjectOutput { + e_tag, + ..Default::default() + }; + + let event_args = rustfs_notify::EventArgs { + event_name: EventName::ObjectCreatedPut, + bucket_name: bucket.clone(), + object: obj_info.clone(), + req_params: req_params.clone(), + resp_elements: extract_resp_elements(&S3Response::new(output.clone())), + version_id: version_id.clone(), + host: host.clone(), + port, + user_agent: user_agent.clone(), + }; + + let notify = notify.clone(); + tokio::spawn(async move { + notify.notify(event_args).await; + }); } let mut checksums = PutObjectChecksums { @@ -3669,7 +4195,22 @@ impl DefaultObjectUsecase { checksums.crc32, checksums.crc32c, checksums.sha1, checksums.sha256, checksums.crc64nvme, ); + drop(entries); + let mut decoder = match ar.into_inner() { + Ok(decoder) => decoder, + Err(_) => return Err(s3_error!(InvalidArgument, "Failed to finalize archive reader")), + }; + tokio::io::copy(&mut decoder, &mut tokio::io::sink()) + .await + .map_err(map_extract_archive_error)?; + let archive_etag = archive_etag + .lock() + .ok() + .and_then(|etag| etag.clone()) + .map(|etag| to_s3s_etag(&etag)); + let output = PutObjectOutput { + e_tag: archive_etag, checksum_crc32: checksums.crc32, checksum_crc32c: checksums.crc32c, checksum_sha1: checksums.sha1, @@ -3695,7 +4236,7 @@ fn object_attributes_requested(object_attributes: &[ObjectAttributes], name: &'s #[cfg(test)] mod tests { use super::*; - use http::{Extensions, HeaderMap, Method, Uri}; + use http::{Extensions, HeaderMap, HeaderName, HeaderValue, Method, Uri}; fn build_request(input: T, method: Method) -> S3Request { S3Request { @@ -3711,6 +4252,224 @@ mod tests { } } + #[test] + fn put_object_execution_context_defaults_to_put() { + let input = PutObjectInput::builder() + .bucket("test-bucket".to_string()) + .key("test-key".to_string()) + .build() + .unwrap(); + let req = build_request(input, Method::PUT); + + let (event_name, quota_operation, method_name) = DefaultObjectUsecase::put_object_execution_context(&req); + assert_eq!(event_name, EventName::ObjectCreatedPut); + assert!(matches!(quota_operation, QuotaOperation::PutObject)); + assert_eq!(method_name, "PUT"); + } + + #[test] + fn put_object_execution_context_uses_post_marker() { + let input = PutObjectInput::builder() + .bucket("test-bucket".to_string()) + .key("test-key".to_string()) + .build() + .unwrap(); + let mut req = build_request(input, Method::POST); + req.extensions.insert(PostObjectRequestMarker); + + let (event_name, quota_operation, method_name) = DefaultObjectUsecase::put_object_execution_context(&req); + assert_eq!(event_name, EventName::ObjectCreatedPost); + assert!(matches!(quota_operation, QuotaOperation::PostObject)); + assert_eq!(method_name, "POST"); + } + + #[test] + fn is_put_object_extract_requested_accepts_meta_header() { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("true")); + + assert!(is_put_object_extract_requested(&headers)); + } + + #[test] + fn is_put_object_extract_requested_accepts_compat_header_case_insensitive() { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_SNOWBALL_EXTRACT_COMPAT, HeaderValue::from_static(" TRUE ")); + + assert!(is_put_object_extract_requested(&headers)); + } + + #[test] + fn is_put_object_extract_requested_rejects_missing_or_false_value() { + let mut headers = HeaderMap::new(); + assert!(!is_put_object_extract_requested(&headers)); + + headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("false")); + assert!(!is_put_object_extract_requested(&headers)); + } + + #[test] + fn normalize_snowball_prefix_trims_slashes_and_whitespace() { + assert_eq!(normalize_snowball_prefix(" /batch/incoming/ "), Some("batch/incoming".to_string())); + assert_eq!(normalize_snowball_prefix("///"), None); + } + + #[test] + fn normalize_extract_entry_key_applies_prefix_and_directory_suffix() { + assert_eq!( + normalize_extract_entry_key("nested/path.txt", Some("imports"), false), + "imports/nested/path.txt" + ); + assert_eq!(normalize_extract_entry_key("nested/dir/", Some("imports"), true), "imports/nested/dir/"); + assert_eq!(normalize_extract_entry_key("top-level", None, false), "top-level"); + } + + #[test] + fn resolve_put_object_extract_options_defaults_when_headers_missing() { + let headers = HeaderMap::new(); + let options = resolve_put_object_extract_options(&headers); + assert_eq!( + options, + PutObjectExtractOptions { + prefix: None, + ignore_dirs: false, + ignore_errors: false + } + ); + } + + #[test] + fn resolve_put_object_extract_options_accepts_internal_headers() { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_SNOWBALL_PREFIX_INTERNAL, HeaderValue::from_static("/internal/prefix/")); + headers.insert(AMZ_SNOWBALL_IGNORE_DIRS_INTERNAL, HeaderValue::from_static("true")); + headers.insert(AMZ_SNOWBALL_IGNORE_ERRORS_INTERNAL, HeaderValue::from_static("TRUE")); + + let options = resolve_put_object_extract_options(&headers); + assert_eq!(options.prefix.as_deref(), Some("internal/prefix")); + assert!(options.ignore_dirs); + assert!(options.ignore_errors); + } + + #[test] + fn resolve_put_object_extract_options_accepts_suffix_compatible_headers() { + let mut headers = HeaderMap::new(); + headers.insert( + HeaderName::from_static("x-amz-meta-acme-snowball-prefix"), + HeaderValue::from_static(" /partner/import "), + ); + headers.insert( + HeaderName::from_static("x-amz-meta-acme-snowball-ignore-dirs"), + HeaderValue::from_static(" true "), + ); + headers.insert( + HeaderName::from_static("x-amz-meta-acme-snowball-ignore-errors"), + HeaderValue::from_static("TRUE"), + ); + + let options = resolve_put_object_extract_options(&headers); + assert_eq!(options.prefix.as_deref(), Some("partner/import")); + assert!(options.ignore_dirs); + assert!(options.ignore_errors); + } + + #[tokio::test] + async fn execute_put_object_rejects_post_object_sse_kms_from_input() { + let input = PutObjectInput::builder() + .bucket("test-bucket".to_string()) + .key("test-key".to_string()) + .server_side_encryption(Some(ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS))) + .build() + .unwrap(); + + let mut req = build_request(input, Method::POST); + req.extensions.insert(PostObjectRequestMarker); + + let usecase = DefaultObjectUsecase::without_context(); + let fs = FS::new(); + + let err = usecase.execute_put_object(&fs, req).await.unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::NotImplemented); + } + + #[tokio::test] + async fn execute_put_object_rejects_extract_sse_kms() { + let input = PutObjectInput::builder() + .bucket("test-bucket".to_string()) + .key("archive.tar".to_string()) + .server_side_encryption(Some(ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS))) + .build() + .unwrap(); + + let mut req = build_request(input, Method::PUT); + req.headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("true")); + + let usecase = DefaultObjectUsecase::without_context(); + let fs = FS::new(); + + let err = usecase.execute_put_object(&fs, req).await.unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::NotImplemented); + } + + #[tokio::test] + async fn execute_put_object_extract_rejects_invalid_storage_class() { + let input = PutObjectInput::builder() + .bucket("test-bucket".to_string()) + .key("archive.tar".to_string()) + .storage_class(Some(StorageClass::from_static("INVALID"))) + .build() + .unwrap(); + + let mut req = build_request(input, Method::PUT); + req.headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("true")); + + let usecase = DefaultObjectUsecase::without_context(); + let fs = FS::new(); + + let err = usecase.execute_put_object(&fs, req).await.unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::InvalidStorageClass); + } + + #[tokio::test] + async fn execute_put_object_rejects_post_object_sse_kms_from_headers() { + let input = PutObjectInput::builder() + .bucket("test-bucket".to_string()) + .key("test-key".to_string()) + .build() + .unwrap(); + + let mut req = build_request(input, Method::POST); + req.extensions.insert(PostObjectRequestMarker); + req.headers + .insert(AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static("aws:kms")); + + let usecase = DefaultObjectUsecase::without_context(); + let fs = FS::new(); + + let err = usecase.execute_put_object(&fs, req).await.unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::NotImplemented); + } + + #[tokio::test] + async fn execute_put_object_rejects_post_object_sse_kms_key_id_header() { + let input = PutObjectInput::builder() + .bucket("test-bucket".to_string()) + .key("test-key".to_string()) + .build() + .unwrap(); + + let mut req = build_request(input, Method::POST); + req.extensions.insert(PostObjectRequestMarker); + req.headers + .insert(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, HeaderValue::from_static("test-kms-key-id")); + + let usecase = DefaultObjectUsecase::without_context(); + let fs = FS::new(); + + let err = usecase.execute_put_object(&fs, req).await.unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::NotImplemented); + } + #[tokio::test] async fn execute_put_object_rejects_invalid_storage_class() { let input = PutObjectInput::builder() diff --git a/rustfs/src/server/event.rs b/rustfs/src/server/event.rs index 9fee1d1ee..64c1be00f 100644 --- a/rustfs/src/server/event.rs +++ b/rustfs/src/server/event.rs @@ -13,12 +13,54 @@ // limitations under the License. use crate::app::context::resolve_server_config; +use rustfs_ecstore::event_notification::{EventArgs as EcstoreEventArgs, register_event_dispatch_hook}; +use rustfs_notify::EventArgs as NotifyEventArgs; +use rustfs_s3_common::EventName; +use tokio::spawn; use tracing::{error, info, instrument, warn}; fn server_config_from_context() -> Option { resolve_server_config() } +fn convert_ecstore_event_args(args: EcstoreEventArgs) -> NotifyEventArgs { + let version_id = args.object.version_id.map(|v| v.to_string()).unwrap_or_default(); + let (host, port) = match args.host.rsplit_once(':') { + Some((host, port)) => match port.parse::() { + Ok(port) => (host.to_string(), port), + Err(_) => (args.host, 0), + }, + None => (args.host, 0), + }; + let req_params = args.req_params.into_iter().collect(); + let resp_elements = args.resp_elements.into_iter().collect(); + + NotifyEventArgs { + event_name: EventName::from(args.event_name.as_str()), + bucket_name: args.bucket_name, + object: args.object, + req_params, + resp_elements, + version_id, + host, + port, + user_agent: args.user_agent, + } +} + +fn install_ecstore_event_dispatch_hook() { + let installed = register_event_dispatch_hook(|args| { + let notify_args = convert_ecstore_event_args(args); + spawn(async move { + rustfs_notify::notifier_global::notify(notify_args).await; + }); + }); + + if !installed { + warn!("ECStore event dispatch hook was already registered"); + } +} + /// Shuts down the event notifier system gracefully pub(crate) async fn shutdown_event_notifier() { info!("Shutting down event notifier system..."); @@ -67,6 +109,7 @@ pub(crate) async fn init_event_notifier() { if let Err(e) = rustfs_notify::initialize(server_config).await { error!("Failed to initialize event notifier system: {}", e); } else { + install_ecstore_event_dispatch_hook(); info!( target: "rustfs::main::init_event_notifier", "Event notifier system initialized successfully." diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index eebf27e3c..341697336 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -29,6 +29,7 @@ use rustfs_utils::http::AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE; use s3s::access::{S3Access, S3AccessContext}; use s3s::{S3Error, S3ErrorCode, S3Request, S3Result, dto::*, s3_error}; use std::collections::HashMap; +use url::Url; #[derive(Default, Clone, Debug)] pub(crate) struct ReqInfo { @@ -41,6 +42,9 @@ pub(crate) struct ReqInfo { pub region: Option, } +#[derive(Clone, Debug)] +pub(crate) struct PostObjectRequestMarker; + pub(crate) fn req_info_ref(req: &S3Request) -> S3Result<&ReqInfo> { req.extensions .get::() @@ -359,6 +363,35 @@ fn put_bucket_policy_authorize_action() -> Action { Action::S3Action(S3Action::PutBucketPolicyAction) } +fn post_object_authorize_action() -> Action { + Action::S3Action(S3Action::PutObjectAction) +} + +fn complete_multipart_upload_authorize_action() -> Action { + Action::S3Action(S3Action::PutObjectAction) +} + +fn list_parts_authorize_action() -> Action { + Action::S3Action(S3Action::ListMultipartUploadPartsAction) +} + +fn validate_post_object_success_controls(input: &PostObjectInput) -> S3Result<()> { + if let Some(status) = input.success_action_status + && !matches!(status, 200 | 201 | 204) + { + return Err(s3_error!(MalformedPOSTRequest, "success_action_status must be one of 200, 201, or 204")); + } + + if let Some(redirect) = input.success_action_redirect.as_deref().map(str::trim) + && !redirect.is_empty() + && Url::parse(redirect).is_err() + { + return Err(s3_error!(MalformedPOSTRequest, "success_action_redirect must be a valid absolute URL")); + } + + Ok(()) +} + #[async_trait::async_trait] impl S3Access for FS { // /// Checks whether the current request has accesses to the resources. @@ -437,15 +470,23 @@ impl S3Access for FS { /// Checks whether the AbortMultipartUpload request has accesses to the resources. /// /// This method returns `Ok(())` by default. - async fn abort_multipart_upload(&self, _req: &mut S3Request) -> S3Result<()> { - Ok(()) + async fn abort_multipart_upload(&self, req: &mut S3Request) -> S3Result<()> { + let req_info = ext_req_info_mut(&mut req.extensions)?; + req_info.bucket = Some(req.input.bucket.clone()); + req_info.object = Some(req.input.key.clone()); + + authorize_request(req, Action::S3Action(S3Action::AbortMultipartUploadAction)).await } /// Checks whether the CompleteMultipartUpload request has accesses to the resources. /// /// This method returns `Ok(())` by default. - async fn complete_multipart_upload(&self, _req: &mut S3Request) -> S3Result<()> { - Ok(()) + async fn complete_multipart_upload(&self, req: &mut S3Request) -> S3Result<()> { + let req_info = ext_req_info_mut(&mut req.extensions)?; + req_info.bucket = Some(req.input.bucket.clone()); + req_info.object = Some(req.input.key.clone()); + + authorize_request(req, complete_multipart_upload_authorize_action()).await } /// Checks whether the CopyObject request has accesses to the resources. @@ -617,8 +658,11 @@ impl S3Access for FS { /// Checks whether the DeleteBucketWebsite request has accesses to the resources. /// /// This method returns `Ok(())` by default. - async fn delete_bucket_website(&self, _req: &mut S3Request) -> S3Result<()> { - Ok(()) + async fn delete_bucket_website(&self, req: &mut S3Request) -> S3Result<()> { + let req_info = ext_req_info_mut(&mut req.extensions)?; + req_info.bucket = Some(req.input.bucket.clone()); + + authorize_request(req, Action::S3Action(S3Action::GetBucketPolicyAction)).await } /// Checks whether the DeleteObject request has accesses to the resources. @@ -708,9 +752,12 @@ impl S3Access for FS { /// This method returns `Ok(())` by default. async fn get_bucket_accelerate_configuration( &self, - _req: &mut S3Request, + req: &mut S3Request, ) -> S3Result<()> { - Ok(()) + let req_info = ext_req_info_mut(&mut req.extensions)?; + req_info.bucket = Some(req.input.bucket.clone()); + + authorize_request(req, Action::S3Action(S3Action::GetBucketPolicyAction)).await } /// Checks whether the GetBucketAcl request has accesses to the resources. @@ -866,8 +913,11 @@ impl S3Access for FS { /// Checks whether the GetBucketRequestPayment request has accesses to the resources. /// /// This method returns `Ok(())` by default. - async fn get_bucket_request_payment(&self, _req: &mut S3Request) -> S3Result<()> { - Ok(()) + async fn get_bucket_request_payment(&self, req: &mut S3Request) -> S3Result<()> { + let req_info = ext_req_info_mut(&mut req.extensions)?; + req_info.bucket = Some(req.input.bucket.clone()); + + authorize_request(req, Action::S3Action(S3Action::GetBucketPolicyAction)).await } /// Checks whether the GetBucketTagging request has accesses to the resources. @@ -893,8 +943,11 @@ impl S3Access for FS { /// Checks whether the GetBucketWebsite request has accesses to the resources. /// /// This method returns `Ok(())` by default. - async fn get_bucket_website(&self, _req: &mut S3Request) -> S3Result<()> { - Ok(()) + async fn get_bucket_website(&self, req: &mut S3Request) -> S3Result<()> { + let req_info = ext_req_info_mut(&mut req.extensions)?; + req_info.bucket = Some(req.input.bucket.clone()); + + authorize_request(req, Action::S3Action(S3Action::GetBucketPolicyAction)).await } /// Checks whether the GetObject request has accesses to the resources. @@ -1142,8 +1195,25 @@ impl S3Access for FS { /// Checks whether the ListParts request has accesses to the resources. /// /// This method returns `Ok(())` by default. - async fn list_parts(&self, _req: &mut S3Request) -> S3Result<()> { - Ok(()) + async fn list_parts(&self, req: &mut S3Request) -> S3Result<()> { + let req_info = ext_req_info_mut(&mut req.extensions)?; + req_info.bucket = Some(req.input.bucket.clone()); + req_info.object = Some(req.input.key.clone()); + + authorize_request(req, list_parts_authorize_action()).await + } + + /// Checks whether the PostObject request has accesses to the resources. + async fn post_object(&self, req: &mut S3Request) -> S3Result<()> { + validate_post_object_success_controls(&req.input)?; + + let req_info = ext_req_info_mut(&mut req.extensions)?; + req_info.bucket = Some(req.input.bucket.clone()); + req_info.object = Some(req.input.key.clone()); + req_info.version_id = req.input.version_id.clone(); + req.extensions.insert(PostObjectRequestMarker); + + authorize_request(req, post_object_authorize_action()).await } /// Checks whether the PutBucketAccelerateConfiguration request has accesses to the resources. @@ -1151,9 +1221,12 @@ impl S3Access for FS { /// This method returns `Ok(())` by default. async fn put_bucket_accelerate_configuration( &self, - _req: &mut S3Request, + req: &mut S3Request, ) -> S3Result<()> { - Ok(()) + let req_info = ext_req_info_mut(&mut req.extensions)?; + req_info.bucket = Some(req.input.bucket.clone()); + + authorize_request(req, Action::S3Action(S3Action::PutBucketPolicyAction)).await } /// Checks whether the PutBucketAcl request has accesses to the resources. @@ -1286,8 +1359,11 @@ impl S3Access for FS { /// Checks whether the PutBucketRequestPayment request has accesses to the resources. /// /// This method returns `Ok(())` by default. - async fn put_bucket_request_payment(&self, _req: &mut S3Request) -> S3Result<()> { - Ok(()) + async fn put_bucket_request_payment(&self, req: &mut S3Request) -> S3Result<()> { + let req_info = ext_req_info_mut(&mut req.extensions)?; + req_info.bucket = Some(req.input.bucket.clone()); + + authorize_request(req, Action::S3Action(S3Action::PutBucketPolicyAction)).await } /// Checks whether the PutBucketTagging request has accesses to the resources. @@ -1313,8 +1389,11 @@ impl S3Access for FS { /// Checks whether the PutBucketWebsite request has accesses to the resources. /// /// This method returns `Ok(())` by default. - async fn put_bucket_website(&self, _req: &mut S3Request) -> S3Result<()> { - Ok(()) + async fn put_bucket_website(&self, req: &mut S3Request) -> S3Result<()> { + let req_info = ext_req_info_mut(&mut req.extensions)?; + req_info.bucket = Some(req.input.bucket.clone()); + + authorize_request(req, Action::S3Action(S3Action::PutBucketPolicyAction)).await } /// Checks whether the PutObject request has accesses to the resources. @@ -1444,8 +1523,35 @@ impl S3Access for FS { /// Checks whether the UploadPartCopy request has accesses to the resources. /// /// This method returns `Ok(())` by default. - async fn upload_part_copy(&self, _req: &mut S3Request) -> S3Result<()> { - Ok(()) + async fn upload_part_copy(&self, req: &mut S3Request) -> S3Result<()> { + { + let (src_bucket, src_key, version_id) = match &req.input.copy_source { + CopySource::AccessPoint { .. } => return Err(s3_error!(NotImplemented)), + CopySource::Outpost { .. } => return Err(s3_error!(NotImplemented)), + CopySource::Bucket { bucket, key, version_id } => { + (bucket.to_string(), key.to_string(), version_id.as_ref().map(|v| v.to_string())) + } + }; + + let req_info = ext_req_info_mut(&mut req.extensions)?; + req_info.bucket = Some(src_bucket.clone()); + req_info.object = Some(src_key.clone()); + req_info.version_id = version_id.clone(); + + let tag_conds = self + .fetch_tag_conditions(&src_bucket, &src_key, version_id.as_deref(), "upload_part_copy_src") + .await?; + req.extensions.insert(tag_conds); + + authorize_request(req, Action::S3Action(S3Action::GetObjectAction)).await?; + } + + let req_info = ext_req_info_mut(&mut req.extensions)?; + req_info.bucket = Some(req.input.bucket.clone()); + req_info.object = Some(req.input.key.clone()); + req_info.version_id = None; + + authorize_request(req, Action::S3Action(S3Action::PutObjectAction)).await } /// Checks whether the WriteGetObjectResponse request has accesses to the resources. @@ -1459,6 +1565,7 @@ impl S3Access for FS { #[cfg(test)] mod tests { use super::*; + use http::{HeaderMap, Method, Uri}; use std::collections::HashMap; #[test] @@ -1471,6 +1578,74 @@ mod tests { assert_eq!(put_bucket_policy_authorize_action(), Action::S3Action(S3Action::PutBucketPolicyAction)); } + #[test] + fn post_object_uses_put_object_action() { + assert_eq!(post_object_authorize_action(), Action::S3Action(S3Action::PutObjectAction)); + } + + #[test] + fn complete_multipart_upload_uses_put_object_action() { + assert_eq!(complete_multipart_upload_authorize_action(), Action::S3Action(S3Action::PutObjectAction)); + } + + #[test] + fn list_parts_uses_list_multipart_upload_parts_action() { + assert_eq!(list_parts_authorize_action(), Action::S3Action(S3Action::ListMultipartUploadPartsAction)); + } + + #[test] + fn validate_post_object_success_controls_accepts_supported_status_codes() { + for status in [200, 201, 204] { + let input = PostObjectInput::builder() + .bucket("test-bucket".to_string()) + .key("test-key".to_string()) + .success_action_status(Some(status)) + .build() + .expect("post object input should build"); + assert!( + validate_post_object_success_controls(&input).is_ok(), + "status {status} should be accepted" + ); + } + } + + #[test] + fn validate_post_object_success_controls_rejects_invalid_status_code() { + let input = PostObjectInput::builder() + .bucket("test-bucket".to_string()) + .key("test-key".to_string()) + .success_action_status(Some(202)) + .build() + .expect("post object input should build"); + + let err = validate_post_object_success_controls(&input).expect_err("status 202 should be rejected"); + assert_eq!(err.code(), &S3ErrorCode::MalformedPOSTRequest); + } + + #[test] + fn validate_post_object_success_controls_accepts_empty_redirect() { + let input = PostObjectInput::builder() + .bucket("test-bucket".to_string()) + .key("test-key".to_string()) + .success_action_redirect(Some("".to_string())) + .build() + .expect("post object input should build"); + assert!(validate_post_object_success_controls(&input).is_ok()); + } + + #[test] + fn validate_post_object_success_controls_rejects_invalid_redirect() { + let input = PostObjectInput::builder() + .bucket("test-bucket".to_string()) + .key("test-key".to_string()) + .success_action_redirect(Some("://invalid-url".to_string())) + .build() + .expect("post object input should build"); + + let err = validate_post_object_success_controls(&input).expect_err("invalid redirect should be rejected"); + assert_eq!(err.code(), &S3ErrorCode::MalformedPOSTRequest); + } + /// Object tag conditions must use keys like ExistingObjectTag/ so that /// bucket policy conditions (e.g. s3:ExistingObjectTag/security) are evaluated correctly. #[test] @@ -1515,4 +1690,34 @@ mod tests { &Action::S3Action(S3Action::DeleteBucketPolicyAction) )); } + + #[tokio::test] + async fn post_object_marks_request_extensions() { + let input = PostObjectInput::builder() + .bucket("test-bucket".to_string()) + .key("test-key".to_string()) + .build() + .expect("post object input should build"); + + let mut req = S3Request { + input, + method: Method::POST, + uri: Uri::from_static("/"), + headers: HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + req.extensions.insert(ReqInfo::default()); + + let fs = FS::new(); + let _ = fs.post_object(&mut req).await; + + assert!( + req.extensions.get::().is_some(), + "post object request should carry the marker for downstream handling" + ); + } } diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 2b7e19502..c3b72d34c 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -16,8 +16,13 @@ use crate::app::bucket_usecase::DefaultBucketUsecase; use crate::app::multipart_usecase::DefaultMultipartUsecase; use crate::app::object_usecase::DefaultObjectUsecase; use rustfs_ecstore::{ - bucket::tagging::decode_tags_to_map, - error::{is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found}, + bucket::{ + metadata::{BUCKET_ACCELERATE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG, BUCKET_WEBSITE_CONFIG}, + metadata_sys, + tagging::decode_tags_to_map, + utils::serialize, + }, + error::{StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found}, new_object_layer_fn, store_api::{BucketOperations, BucketOptions, ObjectOperations, ObjectOptions}, }; @@ -265,6 +270,26 @@ impl S3 for FS { usecase.execute_delete_bucket_tagging(req).await } + async fn delete_bucket_website( + &self, + req: S3Request, + ) -> S3Result> { + let Some(store) = new_object_layer_fn() else { + return Err(s3_error!(InternalError, "Not init")); + }; + + store + .get_bucket_info(&req.input.bucket, &BucketOptions::default()) + .await + .map_err(crate::error::ApiError::from)?; + + metadata_sys::delete(&req.input.bucket, BUCKET_WEBSITE_CONFIG) + .await + .map_err(crate::error::ApiError::from)?; + + Ok(S3Response::new(DeleteBucketWebsiteOutput::default())) + } + #[instrument(level = "debug", skip(self))] async fn delete_public_access_block( &self, @@ -303,6 +328,29 @@ impl S3 for FS { usecase.execute_get_bucket_acl(req).await } + async fn get_bucket_accelerate_configuration( + &self, + req: S3Request, + ) -> S3Result> { + let Some(store) = new_object_layer_fn() else { + return Err(s3_error!(InternalError, "Not init")); + }; + + store + .get_bucket_info(&req.input.bucket, &BucketOptions::default()) + .await + .map_err(crate::error::ApiError::from)?; + + match metadata_sys::get_accelerate_config(&req.input.bucket).await { + Ok((accelerate, _)) => Ok(S3Response::new(GetBucketAccelerateConfigurationOutput { + status: accelerate.status, + ..Default::default() + })), + Err(StorageError::ConfigNotFound) => Ok(S3Response::new(GetBucketAccelerateConfigurationOutput::default())), + Err(err) => Err(crate::error::ApiError::from(err).into()), + } + } + #[instrument(level = "debug", skip(self))] async fn get_bucket_cors(&self, req: S3Request) -> S3Result> { record_s3_op(S3Operation::GetBucketCors, &req.input.bucket); @@ -370,6 +418,30 @@ impl S3 for FS { usecase.execute_get_bucket_replication(req).await } + async fn get_bucket_request_payment( + &self, + req: S3Request, + ) -> S3Result> { + let Some(store) = new_object_layer_fn() else { + return Err(s3_error!(InternalError, "Not init")); + }; + + store + .get_bucket_info(&req.input.bucket, &BucketOptions::default()) + .await + .map_err(crate::error::ApiError::from)?; + + match metadata_sys::get_request_payment_config(&req.input.bucket).await { + Ok((payment, _)) => Ok(S3Response::new(GetBucketRequestPaymentOutput { + payer: Some(payment.payer), + })), + Err(StorageError::ConfigNotFound) => Ok(S3Response::new(GetBucketRequestPaymentOutput { + payer: Some(Payer::from_static(Payer::BUCKET_OWNER)), + })), + Err(err) => Err(crate::error::ApiError::from(err).into()), + } + } + #[instrument(level = "debug", skip(self))] async fn get_bucket_tagging(&self, req: S3Request) -> S3Result> { record_s3_op(S3Operation::GetBucketTagging, &req.input.bucket); @@ -397,6 +469,28 @@ impl S3 for FS { usecase.execute_get_bucket_versioning(req).await } + async fn get_bucket_website(&self, req: S3Request) -> S3Result> { + let Some(store) = new_object_layer_fn() else { + return Err(s3_error!(InternalError, "Not init")); + }; + + store + .get_bucket_info(&req.input.bucket, &BucketOptions::default()) + .await + .map_err(crate::error::ApiError::from)?; + + match metadata_sys::get_website_config(&req.input.bucket).await { + Ok((website, _)) => Ok(S3Response::new(GetBucketWebsiteOutput { + error_document: website.error_document, + index_document: website.index_document, + redirect_all_requests_to: website.redirect_all_requests_to, + routing_rules: website.routing_rules, + })), + Err(StorageError::ConfigNotFound) => Err(s3_error!(NoSuchWebsiteConfiguration)), + Err(err) => Err(crate::error::ApiError::from(err).into()), + } + } + /// Get bucket notification #[instrument( level = "debug", @@ -528,6 +622,27 @@ impl S3 for FS { usecase.execute_put_bucket_acl(req).await } + async fn put_bucket_accelerate_configuration( + &self, + req: S3Request, + ) -> S3Result> { + let Some(store) = new_object_layer_fn() else { + return Err(s3_error!(InternalError, "Not init")); + }; + store + .get_bucket_info(&req.input.bucket, &BucketOptions::default()) + .await + .map_err(crate::error::ApiError::from)?; + + let accelerate_config = serialize(&req.input.accelerate_configuration) + .map_err(|err| S3Error::with_message(S3ErrorCode::MalformedXML, format!("{err}")))?; + metadata_sys::update(&req.input.bucket, BUCKET_ACCELERATE_CONFIG, accelerate_config) + .await + .map_err(crate::error::ApiError::from)?; + + Ok(S3Response::new(PutBucketAccelerateConfigurationOutput::default())) + } + #[instrument(level = "debug", skip(self))] async fn put_bucket_cors(&self, req: S3Request) -> S3Result> { let usecase = DefaultBucketUsecase::from_global(); @@ -543,7 +658,14 @@ impl S3 for FS { .get_bucket_info(&req.input.bucket, &BucketOptions::default()) .await .map_err(crate::error::ApiError::from)?; - Err(s3_error!(NotImplemented, "GetBucketLogging is not implemented yet")) + + match metadata_sys::get_logging_config(&req.input.bucket).await { + Ok((logging, _)) => Ok(S3Response::new(GetBucketLoggingOutput { + logging_enabled: logging.logging_enabled, + })), + Err(StorageError::ConfigNotFound) => Ok(S3Response::new(GetBucketLoggingOutput::default())), + Err(err) => Err(crate::error::ApiError::from(err).into()), + } } async fn put_bucket_logging(&self, req: S3Request) -> S3Result> { @@ -555,7 +677,14 @@ impl S3 for FS { .get_bucket_info(&req.input.bucket, &BucketOptions::default()) .await .map_err(crate::error::ApiError::from)?; - Err(s3_error!(NotImplemented, "PutBucketLogging is not implemented yet")) + + let logging_config = serialize(&req.input.bucket_logging_status) + .map_err(|err| S3Error::with_message(S3ErrorCode::MalformedXML, format!("{err}")))?; + metadata_sys::update(&req.input.bucket, BUCKET_LOGGING_CONFIG, logging_config) + .await + .map_err(crate::error::ApiError::from)?; + + Ok(S3Response::new(PutBucketLoggingOutput::default())) } async fn put_bucket_encryption( @@ -596,6 +725,27 @@ impl S3 for FS { usecase.execute_put_bucket_replication(req).await } + async fn put_bucket_request_payment( + &self, + req: S3Request, + ) -> S3Result> { + let Some(store) = new_object_layer_fn() else { + return Err(s3_error!(InternalError, "Not init")); + }; + store + .get_bucket_info(&req.input.bucket, &BucketOptions::default()) + .await + .map_err(crate::error::ApiError::from)?; + + let payment_config = serialize(&req.input.request_payment_configuration) + .map_err(|err| S3Error::with_message(S3ErrorCode::MalformedXML, format!("{err}")))?; + metadata_sys::update(&req.input.bucket, BUCKET_REQUEST_PAYMENT_CONFIG, payment_config) + .await + .map_err(crate::error::ApiError::from)?; + + Ok(S3Response::new(PutBucketRequestPaymentOutput::default())) + } + #[instrument(level = "debug", skip(self))] async fn put_public_access_block( &self, @@ -620,6 +770,24 @@ impl S3 for FS { usecase.execute_put_bucket_versioning(req).await } + async fn put_bucket_website(&self, req: S3Request) -> S3Result> { + let Some(store) = new_object_layer_fn() else { + return Err(s3_error!(InternalError, "Not init")); + }; + store + .get_bucket_info(&req.input.bucket, &BucketOptions::default()) + .await + .map_err(crate::error::ApiError::from)?; + + let website_config = serialize(&req.input.website_configuration) + .map_err(|err| S3Error::with_message(S3ErrorCode::MalformedXML, format!("{err}")))?; + metadata_sys::update(&req.input.bucket, BUCKET_WEBSITE_CONFIG, website_config) + .await + .map_err(crate::error::ApiError::from)?; + + Ok(S3Response::new(PutBucketWebsiteOutput::default())) + } + #[instrument(level = "debug", skip(self, req))] async fn put_object(&self, req: S3Request) -> S3Result> { let usecase = DefaultObjectUsecase::from_global(); diff --git a/rustfs/src/storage/options.rs b/rustfs/src/storage/options.rs index f2fe73be1..6ad860be6 100644 --- a/rustfs/src/storage/options.rs +++ b/rustfs/src/storage/options.rs @@ -334,7 +334,7 @@ pub fn extract_metadata_from_mime(headers: &HeaderMap, metadata: &m /// request-side transfer encoding for SigV4 streaming and must not be stored or returned. /// If the only value is "aws-chunked", returns None (do not persist). Otherwise returns /// the value with "aws-chunked" stripped, or None if nothing remains. -fn normalize_content_encoding_for_storage(value: &str) -> Option { +pub(crate) fn normalize_content_encoding_for_storage(value: &str) -> Option { let trimmed = value.trim(); if trimmed.is_empty() { return None; diff --git a/rustfs/src/storage/rpc/event.rs b/rustfs/src/storage/rpc/event.rs new file mode 100644 index 000000000..7a574e0e3 --- /dev/null +++ b/rustfs/src/storage/rpc/event.rs @@ -0,0 +1,60 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use rustfs_notify::notification_system; + +impl NodeService { + pub(super) async fn handle_get_live_events( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + let limit = usize::try_from(request.limit).unwrap_or(0).clamp(1, 256); + + let Some(system) = notification_system() else { + return Ok(Response::new(GetLiveEventsResponse { + success: true, + events: Bytes::new(), + next_sequence: request.after_sequence, + truncated: false, + error_info: None, + })); + }; + + let batch = system.recent_live_events_since(request.after_sequence, limit).await; + let events = batch.events.into_iter().map(|event| (*event).clone()).collect::>(); + + let payload = match serde_json::to_vec(&events) { + Ok(payload) => payload, + Err(err) => { + return Ok(Response::new(GetLiveEventsResponse { + success: false, + events: Bytes::new(), + next_sequence: request.after_sequence, + truncated: false, + error_info: Some(format!("failed to serialize live events: {err}")), + })); + } + }; + + Ok(Response::new(GetLiveEventsResponse { + success: true, + events: payload.into(), + next_sequence: batch.next_sequence, + truncated: batch.truncated, + error_info: None, + })) + } +} diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 35355a40b..97e9eb1df 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -56,6 +56,8 @@ type ResponseStream = Pin> + Send>>; mod bucket; #[path = "disk.rs"] mod disk; +#[path = "event.rs"] +mod event; #[path = "health.rs"] mod health; #[path = "lock.rs"] @@ -429,6 +431,10 @@ impl Node for NodeService { self.handle_get_metrics(request).await } + async fn get_live_events(&self, request: Request) -> Result, Status> { + self.handle_get_live_events(request).await + } + async fn get_proc_info(&self, _request: Request) -> Result, Status> { self.handle_get_proc_info(_request).await } diff --git a/rustfs/src/storage/sse.rs b/rustfs/src/storage/sse.rs index 1a0ce86ce..2ca6e7da9 100644 --- a/rustfs/src/storage/sse.rs +++ b/rustfs/src/storage/sse.rs @@ -96,6 +96,8 @@ use std::sync::{Arc, OnceLock}; use tokio::io::AsyncRead; use tracing::{debug, error}; +const INTERNAL_ENCRYPTION_KEY_ID_HEADER: &str = "x-rustfs-encryption-key-id"; + use crate::error::ApiError; use crate::storage::readers::InMemoryAsyncReader; use rustfs_ecstore::bucket::metadata_sys; @@ -159,6 +161,15 @@ async fn prepare_sse_configuration( server_side_encryption: Option, ssekms_key_id: Option, ) -> Result, ApiError> { + if let Some(server_side_encryption) = server_side_encryption.clone() + && server_side_encryption.as_str() == ServerSideEncryption::AES256 + { + return Ok(Some(SseConfiguration { + effective_sse: server_side_encryption, + effective_kms_key_id: None, + })); + } + if let Some(server_side_encryption) = server_side_encryption.clone() && let Some(ssekms_key_id) = ssekms_key_id { @@ -192,7 +203,7 @@ async fn prepare_sse_configuration( debug!("effective_sse={:?} (original={:?})", effective_sse, server_side_encryption); - let effective_kms_key_id = ssekms_key_id.or_else(|| { + let effective_kms_key_id = resolve_effective_kms_key_id(effective_sse.as_ref(), ssekms_key_id, || { bucket_sse_config.rules.first().and_then(|rule| { rule.apply_server_side_encryption_by_default .as_ref() @@ -226,6 +237,21 @@ async fn prepare_sse_configuration( } } +fn resolve_effective_kms_key_id( + effective_sse: Option<&ServerSideEncryption>, + requested_kms_key_id: Option, + bucket_default_kms_key_id: F, +) -> Option +where + F: FnOnce() -> Option, +{ + if effective_sse.is_none_or(|sse| sse.as_str() != ServerSideEncryption::AWS_KMS) { + return requested_kms_key_id; + } + + requested_kms_key_id.or_else(bucket_default_kms_key_id) +} + #[derive(Debug, Clone)] pub enum SseTypeV2 { SseS3(ServerSideEncryption), @@ -524,6 +550,8 @@ pub struct DecryptionRequest<'a> { pub part_number: Option, /// Parts information for multipart objects pub parts: &'a [ObjectPartInfo], + /// Object-level ETag, used to distinguish multipart objects from single-part objects. + pub etag: Option<&'a str>, } /// Unified encryption material returned by `apply_encryption()` @@ -568,6 +596,14 @@ pub struct DecryptionMaterial { pub parts: Vec, } +fn is_multipart_object(etag: Option<&str>, parts: &[ObjectPartInfo]) -> bool { + if parts.len() > 1 { + return true; + } + + etag.map(|etag| etag.trim_matches('"').len() != 32).unwrap_or(false) +} + /// Type of encryption used #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SSEType { @@ -815,7 +851,7 @@ pub async fn sse_prepare_encryption(request: PrepareEncryptionRequest<'_>) -> Re /// } /// ``` pub async fn sse_decryption(request: DecryptionRequest<'_>) -> Result, ApiError> { - let is_multipart = request.parts.len() > 1; + let is_multipart = is_multipart_object(request.etag, request.parts); // Check for SSE-C encryption if request @@ -1021,7 +1057,7 @@ async fn apply_managed_encryption_material( context = context.with_size(content_size as u64); } - // Determine KMS key ID to use + // Determine KMS key ID to use for internal key wrapping. let mut kms_key_candidate = kms_key_id.clone().map(|s| s.to_string()); if kms_key_candidate.is_none() { // Try to get default key from KMS service (if available) @@ -1030,11 +1066,17 @@ async fn apply_managed_encryption_material( } } - let kms_key_to_use = kms_key_candidate.clone().ok_or_else(|| { - ApiError::from(StorageError::other( - "No KMS key available for managed server-side encryption (required for SSE-KMS)", - )) - })?; + let kms_key_to_use = match (encryption_type, kms_key_candidate.clone()) { + (SSEType::SseS3, Some(kms_key_id)) => kms_key_id, + (SSEType::SseS3, None) => "default".to_string(), + (SSEType::SseKms, Some(kms_key_id)) => kms_key_id, + (SSEType::SseKms, None) => { + return Err(ApiError::from(StorageError::other( + "No KMS key available for managed server-side encryption (required for SSE-KMS)", + ))); + } + _ => unreachable!("managed SSE branch only supports SSE-S3 or SSE-KMS"), + }; let provider = get_sse_dek_provider().await?; @@ -1074,7 +1116,7 @@ async fn apply_managed_encryption_material( (data_key, encrypted_data_key) }; - let algorithm = DEFAULT_SSE_ALGORITHM.to_string(); + let algorithm = server_side_encryption.as_str().to_string(); let encryption_metadata = EncryptionMetadata { algorithm: algorithm.clone(), @@ -1103,13 +1145,15 @@ async fn apply_managed_encryption_material( metadata.insert("x-rustfs-encryption-iv".to_string(), BASE64_STANDARD.encode(&encryption_metadata.iv)); metadata.insert("x-rustfs-encryption-algorithm".to_string(), encryption_metadata.algorithm.clone()); metadata.insert("x-amz-server-side-encryption".to_string(), server_side_encryption.as_str().to_string()); - - // if kms_key is changed, we need to update the metadata - if kms_key_id.is_none() { - metadata.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), kms_key_to_use.clone()); - } } + if matches!(encryption_type, SSEType::SseKms) { + metadata.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), kms_key_to_use.clone()); + } else { + metadata.remove("x-amz-server-side-encryption-aws-kms-key-id"); + } + metadata.insert(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), kms_key_to_use.clone()); + metadata.insert( "x-rustfs-encryption-original-size".to_string(), encryption_metadata.original_size.to_string(), @@ -1118,7 +1162,7 @@ async fn apply_managed_encryption_material( Ok(EncryptionMaterial { sse_type: encryption_type, server_side_encryption, - kms_key_id: Some(kms_key_to_use), + kms_key_id: matches!(encryption_type, SSEType::SseKms).then_some(kms_key_to_use), algorithm, key_bytes: data_key.plaintext_key, @@ -1182,7 +1226,8 @@ async fn apply_managed_decryption_material( // Extract KMS key ID from metadata (optional, used for provider context) let kms_key_id = metadata - .get("x-amz-server-side-encryption-aws-kms-key-id") + .get(INTERNAL_ENCRYPTION_KEY_ID_HEADER) + .or_else(|| metadata.get("x-amz-server-side-encryption-aws-kms-key-id")) .cloned() .unwrap_or_else(|| "default".to_string()); @@ -2179,6 +2224,80 @@ mod tests { assert_eq!(err.code, S3ErrorCode::InvalidArgument); } + #[test] + fn test_resolve_effective_kms_key_id_ignores_bucket_default_for_explicit_sse_s3() { + let effective_sse = ServerSideEncryption::from_static(ServerSideEncryption::AES256); + + let kms_key_id = resolve_effective_kms_key_id(Some(&effective_sse), None, || Some("bucket-default".to_string())); + + assert_eq!(kms_key_id, None); + } + + #[test] + fn test_resolve_effective_kms_key_id_uses_bucket_default_for_sse_kms() { + let effective_sse = ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS); + + let kms_key_id = resolve_effective_kms_key_id(Some(&effective_sse), None, || Some("bucket-default".to_string())); + + assert_eq!(kms_key_id.as_deref(), Some("bucket-default")); + } + + #[tokio::test] + async fn test_sse_encryption_persists_aws_kms_header_for_kms_objects() { + let request = EncryptionRequest { + bucket: "test-bucket", + key: "test-key", + server_side_encryption: Some("aws:kms".to_string().into()), + ssekms_key_id: Some("test-key".to_string()), + sse_customer_algorithm: None, + sse_customer_key: None, + sse_customer_key_md5: None, + content_size: 1024, + part_number: None, + part_key: None, + part_nonce: None, + }; + + let material = sse_encryption(request).await.expect("kms encryption should succeed"); + let metadata = material.expect("managed kms encryption should return material").metadata; + + assert_eq!(metadata.get("x-amz-server-side-encryption").map(String::as_str), Some("aws:kms")); + assert_eq!( + metadata + .get("x-amz-server-side-encryption-aws-kms-key-id") + .map(String::as_str), + Some("test-key") + ); + } + + #[tokio::test] + async fn test_sse_encryption_omits_kms_header_for_sse_s3_objects() { + let request = EncryptionRequest { + bucket: "test-bucket", + key: "test-key", + server_side_encryption: Some(ServerSideEncryption::from_static(ServerSideEncryption::AES256)), + ssekms_key_id: None, + sse_customer_algorithm: None, + sse_customer_key: None, + sse_customer_key_md5: None, + content_size: 1024, + part_number: None, + part_key: None, + part_nonce: None, + }; + + let material = sse_encryption(request).await.expect("sse-s3 encryption should succeed"); + let material = material.expect("managed sse-s3 encryption should return material"); + + assert_eq!(material.kms_key_id, None); + assert_eq!(material.metadata.get("x-amz-server-side-encryption").map(String::as_str), Some("AES256")); + assert!(!material.metadata.contains_key("x-amz-server-side-encryption-aws-kms-key-id")); + assert_eq!( + material.metadata.get(INTERNAL_ENCRYPTION_KEY_ID_HEADER).map(String::as_str), + Some("default") + ); + } + #[test] fn test_strip_managed_encryption_metadata() { let mut metadata = HashMap::new(); @@ -2193,6 +2312,32 @@ mod tests { assert!(metadata.contains_key("content-type")); } + #[test] + fn test_is_multipart_object_treats_single_part_multipart_etag_as_multipart() { + let metadata = HashMap::from([("etag".to_string(), "0123456789abcdef0123456789abcdef-1".to_string())]); + let parts = vec![ObjectPartInfo { + number: 1, + size: 128, + actual_size: 64, + ..Default::default() + }]; + + assert!(is_multipart_object(metadata.get("etag").map(String::as_str), &parts)); + } + + #[test] + fn test_is_multipart_object_keeps_regular_single_part_object_as_non_multipart() { + let metadata = HashMap::from([("etag".to_string(), "0123456789abcdef0123456789abcdef".to_string())]); + let parts = vec![ObjectPartInfo { + number: 1, + size: 128, + actual_size: 64, + ..Default::default() + }]; + + assert!(!is_multipart_object(metadata.get("etag").map(String::as_str), &parts)); + } + #[test] fn test_verify_ssec_key_match_success() { let md5 = "test_md5".to_string(); diff --git a/scripts/s3-tests/compare_dual_targets.py b/scripts/s3-tests/compare_dual_targets.py new file mode 100644 index 000000000..6ba851f7f --- /dev/null +++ b/scripts/s3-tests/compare_dual_targets.py @@ -0,0 +1,417 @@ +#!/usr/bin/env python3 +"""Send one S3 request to two endpoints and summarize response differences.""" + +from __future__ import annotations + +import argparse +import base64 +import datetime as dt +import hashlib +import hmac +import http.client +import json +import os +import pathlib +import ssl +import sys +import urllib.parse +from dataclasses import dataclass +from difflib import unified_diff +from typing import Iterable + + +DEFAULT_IGNORE_HEADERS = { + "date", + "server", + "x-amz-id-2", + "x-amz-request-id", + "x-rustfs-deployment-id", +} + + +@dataclass +class Endpoint: + label: str + url: str + + +@dataclass +class SignedRequest: + method: str + endpoint: Endpoint + path_and_query: str + headers: list[tuple[str, str]] + body: bytes + + +@dataclass +class ResponseSnapshot: + label: str + url: str + status: int + reason: str + headers: list[tuple[str, str]] + body: bytes + + def normalized_headers(self, ignored: set[str]) -> dict[str, list[str]]: + values: dict[str, list[str]] = {} + for name, value in self.headers: + key = name.lower() + if key in ignored: + continue + values.setdefault(key, []).append(value) + for key in values: + values[key].sort() + return dict(sorted(values.items())) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--left-url", required=True, help="Reference endpoint base URL, for example http://127.0.0.1:9000") + parser.add_argument("--right-url", required=True, help="Candidate endpoint base URL, for example http://127.0.0.1:9001") + parser.add_argument("--left-label", default="reference", help="Label used in reports for the left endpoint") + parser.add_argument("--right-label", default="candidate", help="Label used in reports for the right endpoint") + parser.add_argument("--method", default="GET", help="HTTP method") + parser.add_argument("--path", required=True, help="Absolute request path with optional query, for example /bucket/object?versionId=1") + parser.add_argument("--header", action="append", default=[], help="Extra header in 'Name: Value' form") + parser.add_argument("--body-file", help="Read request body from file") + parser.add_argument("--body", default="", help="Inline request body string") + parser.add_argument("--content-type", help="Convenience setter for Content-Type") + parser.add_argument("--region", default=os.getenv("AWS_REGION", "us-east-1"), help="Signing region") + parser.add_argument("--service", default="s3", help="Signing service name") + parser.add_argument("--access-key", default=os.getenv("AWS_ACCESS_KEY_ID", ""), help="Access key for SigV4 signing") + parser.add_argument("--secret-key", default=os.getenv("AWS_SECRET_ACCESS_KEY", ""), help="Secret key for SigV4 signing") + parser.add_argument("--session-token", default=os.getenv("AWS_SESSION_TOKEN", ""), help="Optional session token") + parser.add_argument("--unsigned", action="store_true", help="Send request without SigV4 signing") + parser.add_argument("--timeout", type=float, default=30.0, help="Per-request timeout in seconds") + parser.add_argument("--insecure", action="store_true", help="Disable TLS certificate verification") + parser.add_argument( + "--ignore-header", + action="append", + default=[], + help="Response header name to ignore during comparison. Can be provided multiple times.", + ) + parser.add_argument("--output-dir", default="artifacts/s3-compare/latest", help="Directory for snapshots and summary") + return parser.parse_args() + + +def parse_header(raw: str) -> tuple[str, str]: + name, sep, value = raw.partition(":") + if not sep: + raise ValueError(f"invalid header {raw!r}, expected 'Name: Value'") + name = name.strip() + value = value.strip() + if not name: + raise ValueError(f"invalid header {raw!r}, empty name") + return name, value + + +def load_body(args: argparse.Namespace) -> bytes: + if args.body_file: + return pathlib.Path(args.body_file).read_bytes() + return args.body.encode() + + +def payload_hash(body: bytes) -> str: + return hashlib.sha256(body).hexdigest() + + +def sign(key: bytes, value: str) -> bytes: + return hmac.new(key, value.encode(), hashlib.sha256).digest() + + +def derive_signing_key(secret_key: str, date_stamp: str, region: str, service: str) -> bytes: + key_date = sign(("AWS4" + secret_key).encode(), date_stamp) + key_region = sign(key_date, region) + key_service = sign(key_region, service) + return sign(key_service, "aws4_request") + + +def canonical_query(query: str) -> str: + pairs = urllib.parse.parse_qsl(query, keep_blank_values=True) + encoded = [ + ( + urllib.parse.quote(key, safe="-_.~"), + urllib.parse.quote(value, safe="-_.~"), + ) + for key, value in pairs + ] + encoded.sort() + return "&".join(f"{key}={value}" for key, value in encoded) + + +def canonical_uri(path: str) -> str: + segments = path.split("/") + return "/".join(urllib.parse.quote(segment, safe="-_.~/") for segment in segments) or "/" + + +def build_signed_request( + endpoint: Endpoint, + method: str, + raw_path: str, + headers: list[tuple[str, str]], + body: bytes, + args: argparse.Namespace, +) -> SignedRequest: + parsed = urllib.parse.urlsplit(endpoint.url) + if not parsed.scheme or not parsed.netloc: + raise ValueError(f"invalid endpoint URL: {endpoint.url}") + if not raw_path.startswith("/"): + raise ValueError("--path must start with '/'") + + path, _, query = raw_path.partition("?") + amz_date = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + date_stamp = amz_date[:8] + + request_headers: list[tuple[str, str]] = [] + for name, value in headers: + request_headers.append((name, value)) + + if not any(name.lower() == "host" for name, _ in request_headers): + request_headers.append(("Host", parsed.netloc)) + if not any(name.lower() == "x-amz-content-sha256" for name, _ in request_headers): + request_headers.append(("x-amz-content-sha256", payload_hash(body))) + if not any(name.lower() == "x-amz-date" for name, _ in request_headers): + request_headers.append(("x-amz-date", amz_date)) + if args.session_token and not any(name.lower() == "x-amz-security-token" for name, _ in request_headers): + request_headers.append(("x-amz-security-token", args.session_token)) + + if args.unsigned: + return SignedRequest(method=method, endpoint=endpoint, path_and_query=raw_path, headers=request_headers, body=body) + + if not args.access_key or not args.secret_key: + raise ValueError("SigV4 signing requires --access-key and --secret-key, or set --unsigned") + + normalized = [(name.lower().strip(), " ".join(value.strip().split())) for name, value in request_headers] + normalized.sort() + canonical_headers = "".join(f"{name}:{value}\n" for name, value in normalized) + signed_headers = ";".join(name for name, _ in normalized) + + canonical_request = "\n".join( + [ + method, + canonical_uri(path), + canonical_query(query), + canonical_headers, + signed_headers, + payload_hash(body), + ] + ) + scope = f"{date_stamp}/{args.region}/{args.service}/aws4_request" + string_to_sign = "\n".join( + [ + "AWS4-HMAC-SHA256", + amz_date, + scope, + hashlib.sha256(canonical_request.encode()).hexdigest(), + ] + ) + signature = hmac.new( + derive_signing_key(args.secret_key, date_stamp, args.region, args.service), + string_to_sign.encode(), + hashlib.sha256, + ).hexdigest() + authorization = ( + "AWS4-HMAC-SHA256 " + f"Credential={args.access_key}/{scope}, " + f"SignedHeaders={signed_headers}, " + f"Signature={signature}" + ) + request_headers.append(("Authorization", authorization)) + + return SignedRequest(method=method, endpoint=endpoint, path_and_query=raw_path, headers=request_headers, body=body) + + +def send_request(request: SignedRequest, timeout: float, insecure: bool) -> ResponseSnapshot: + parsed = urllib.parse.urlsplit(request.endpoint.url) + if parsed.scheme == "https": + context = ssl.create_default_context() + if insecure: + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + conn: http.client.HTTPConnection = http.client.HTTPSConnection(parsed.hostname, parsed.port or 443, timeout=timeout, context=context) + elif parsed.scheme == "http": + conn = http.client.HTTPConnection(parsed.hostname, parsed.port or 80, timeout=timeout) + else: + raise ValueError(f"unsupported URL scheme: {parsed.scheme}") + + conn.request(request.method, request.path_and_query, body=request.body, headers=dict(request.headers)) + response = conn.getresponse() + body = response.read() + headers = response.getheaders() + conn.close() + return ResponseSnapshot( + label=request.endpoint.label, + url=request.endpoint.url, + status=response.status, + reason=response.reason, + headers=headers, + body=body, + ) + + +def body_as_text(body: bytes) -> str | None: + try: + return body.decode("utf-8") + except UnicodeDecodeError: + return None + + +def summarize_diff(left: ResponseSnapshot, right: ResponseSnapshot, ignored_headers: set[str]) -> dict: + header_diff = { + "left_only": {}, + "right_only": {}, + "different": {}, + } + left_headers = left.normalized_headers(ignored_headers) + right_headers = right.normalized_headers(ignored_headers) + left_keys = set(left_headers) + right_keys = set(right_headers) + + for key in sorted(left_keys - right_keys): + header_diff["left_only"][key] = left_headers[key] + for key in sorted(right_keys - left_keys): + header_diff["right_only"][key] = right_headers[key] + for key in sorted(left_keys & right_keys): + if left_headers[key] != right_headers[key]: + header_diff["different"][key] = { + left.label: left_headers[key], + right.label: right_headers[key], + } + + left_text = body_as_text(left.body) + right_text = body_as_text(right.body) + if left_text is not None and right_text is not None: + body_diff = { + "kind": "text", + "equal": left_text == right_text, + "unified_diff": list( + unified_diff( + left_text.splitlines(), + right_text.splitlines(), + fromfile=left.label, + tofile=right.label, + lineterm="", + ) + ), + } + else: + body_diff = { + "kind": "binary", + "equal": left.body == right.body, + left.label: { + "size": len(left.body), + "sha256": hashlib.sha256(left.body).hexdigest(), + }, + right.label: { + "size": len(right.body), + "sha256": hashlib.sha256(right.body).hexdigest(), + }, + } + + return { + "status_equal": left.status == right.status, + "status": { + left.label: {"code": left.status, "reason": left.reason}, + right.label: {"code": right.status, "reason": right.reason}, + }, + "headers_equal": not any(header_diff.values()), + "header_diff": header_diff, + "body_equal": body_diff["equal"], + "body_diff": body_diff, + } + + +def write_snapshot(output_dir: pathlib.Path, response: ResponseSnapshot) -> dict: + endpoint_dir = output_dir / response.label + endpoint_dir.mkdir(parents=True, exist_ok=True) + body_file = endpoint_dir / "body.bin" + body_file.write_bytes(response.body) + headers_file = endpoint_dir / "headers.json" + headers_file.write_text(json.dumps(response.headers, indent=2, ensure_ascii=False) + "\n") + summary = { + "label": response.label, + "url": response.url, + "status": response.status, + "reason": response.reason, + "headers_file": str(headers_file), + "body_file": str(body_file), + "body_sha256": hashlib.sha256(response.body).hexdigest(), + "body_base64_preview": base64.b64encode(response.body[:96]).decode(), + } + (endpoint_dir / "response.json").write_text(json.dumps(summary, indent=2, ensure_ascii=False) + "\n") + return summary + + +def print_report(diff: dict, left: ResponseSnapshot, right: ResponseSnapshot) -> None: + print(f"Compared {left.label} <-> {right.label}") + print(f"Status: {left.status} vs {right.status}") + print(f"Headers equal: {diff['headers_equal']}") + print(f"Body equal: {diff['body_equal']}") + + header_diff = diff["header_diff"] + if header_diff["left_only"] or header_diff["right_only"] or header_diff["different"]: + print("Header differences detected:") + if header_diff["left_only"]: + print(f" left only: {sorted(header_diff['left_only'])}") + if header_diff["right_only"]: + print(f" right only: {sorted(header_diff['right_only'])}") + if header_diff["different"]: + print(f" changed: {sorted(header_diff['different'])}") + + body_diff = diff["body_diff"] + if body_diff["kind"] == "text" and not body_diff["equal"]: + preview = body_diff["unified_diff"][:40] + if preview: + print("Body diff preview:") + for line in preview: + print(line) + + +def main() -> int: + args = parse_args() + try: + extra_headers = [parse_header(raw) for raw in args.header] + if args.content_type and not any(name.lower() == "content-type" for name, _ in extra_headers): + extra_headers.append(("Content-Type", args.content_type)) + + body = load_body(args) + output_dir = pathlib.Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + left_endpoint = Endpoint(label=args.left_label, url=args.left_url.rstrip("/")) + right_endpoint = Endpoint(label=args.right_label, url=args.right_url.rstrip("/")) + method = args.method.upper() + + requests = [ + build_signed_request(left_endpoint, method, args.path, extra_headers, body, args), + build_signed_request(right_endpoint, method, args.path, extra_headers, body, args), + ] + responses = [send_request(request, args.timeout, args.insecure) for request in requests] + left, right = responses + + ignored_headers = {name.lower() for name in DEFAULT_IGNORE_HEADERS} + ignored_headers.update(name.lower() for name in args.ignore_header) + + report = { + "request": { + "method": method, + "path": args.path, + "headers": requests[0].headers, + "body_sha256": payload_hash(body), + "body_size": len(body), + "signed": not args.unsigned, + }, + "responses": [write_snapshot(output_dir, response) for response in responses], + } + report["diff"] = summarize_diff(left, right, ignored_headers) + (output_dir / "summary.json").write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n") + print_report(report["diff"], left, right) + return 0 if report["diff"]["status_equal"] and report["diff"]["headers_equal"] and report["diff"]["body_equal"] else 1 + except Exception as exc: # noqa: BLE001 + print(f"compare_dual_targets.py failed: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main())