ci(e2e): stabilize full-gate tooling (#5805)

crates/e2e_test/src/bucket_logging_test.rs

@@ -16,91 +16,17 @@
 
 #[cfg(test)]
 mod tests {
-    use crate::common::{RustFSTestEnvironment, init_logging};
+    use crate::common::{RustFSTestEnvironment, init_logging, signed_s3_request};
     use aws_sdk_s3::error::ProvideErrorMetadata;
     use aws_sdk_s3::types::{
         AccelerateConfiguration, BucketAccelerateStatus, BucketLoggingStatus, IndexDocument, LoggingEnabled, Payer,
         RequestPaymentConfiguration, WebsiteConfiguration,
     };
+    use http::Method;
+    use http::header::CONTENT_TYPE;
     use serial_test::serial;
-    use std::path::PathBuf;
-    use std::process::Command;
     use tracing::info;
 
-    fn awscurl_binary_path() -> PathBuf {
-        std::env::var_os("AWSCURL_PATH")
-            .map(PathBuf::from)
-            .unwrap_or_else(|| PathBuf::from("awscurl"))
-    }
-
-    fn awscurl_available() -> bool {
-        Command::new(awscurl_binary_path()).arg("--version").output().is_ok()
-    }
-
-    fn execute_s3_awscurl(
-        method: &str,
-        url: &str,
-        access_key: &str,
-        secret_key: &str,
-    ) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
-        let output = Command::new(awscurl_binary_path())
-            .args([
-                "--service",
-                "s3",
-                "--region",
-                "us-east-1",
-                "--access_key",
-                access_key,
-                "--secret_key",
-                secret_key,
-                "-i",
-                "-X",
-                method,
-                url,
-            ])
-            .output()?;
-        if !output.status.success() {
-            let stderr = String::from_utf8_lossy(&output.stderr);
-            let stdout = String::from_utf8_lossy(&output.stdout);
-            return Err(format!("awscurl failed: stderr='{stderr}', stdout='{stdout}'").into());
-        }
-        Ok(String::from_utf8_lossy(&output.stdout).to_string())
-    }
-
-    fn parse_status(raw: &str) -> Option<u16> {
-        raw.lines()
-            .filter_map(|line| {
-                if line.starts_with("HTTP/") {
-                    line.split_whitespace().nth(1)?.parse::<u16>().ok()
-                } else {
-                    None
-                }
-            })
-            .next_back()
-    }
-
-    fn parse_body(raw: &str) -> String {
-        if let Some(pos) = raw.rfind("\r\n\r\n") {
-            return raw[pos + 4..].to_string();
-        }
-        if let Some(pos) = raw.rfind("\n\n") {
-            return raw[pos + 2..].to_string();
-        }
-        String::new()
-    }
-
-    fn parse_headers(raw: &str) -> String {
-        let start = raw.rfind("HTTP/").unwrap_or(0);
-        let tail = &raw[start..];
-        if let Some(pos) = tail.find("\r\n\r\n") {
-            return tail[..pos].to_string();
-        }
-        if let Some(pos) = tail.find("\n\n") {
-            return tail[..pos].to_string();
-        }
-        tail.to_string()
-    }
-
     #[tokio::test]
     #[serial]
     async fn test_dummy_bucket_compatibility_endpoints() {
@@ -470,10 +396,6 @@ mod tests {
     async fn test_dummy_bucket_endpoints_http_contracts() {
         init_logging();
         info!("Starting test: dummy-compat bucket API HTTP contracts");
-        if !awscurl_available() {
-            info!("Skipping test_dummy_bucket_endpoints_http_contracts: awscurl binary not found");
-            return;
-        }
 
         let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
         env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
@@ -488,56 +410,112 @@ mod tests {
             .await
             .expect("Failed to create bucket");
 
-        let logging_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?logging=", env.url), &env.access_key, &env.secret_key)
-            .expect("GetBucketLogging HTTP request failed");
-        assert_eq!(parse_status(&logging_raw), Some(200), "GetBucketLogging should return 200");
-        let logging_body = parse_body(&logging_raw);
+        let logging_response = signed_s3_request(
+            Method::GET,
+            &format!("{}/{bucket}?logging=", env.url),
+            None,
+            None,
+            &env.access_key,
+            &env.secret_key,
+        )
+        .await
+        .expect("GetBucketLogging HTTP request failed");
+        assert_eq!(logging_response.status(), 200, "GetBucketLogging should return 200");
+        let logging_body = logging_response
+            .text()
+            .await
+            .expect("Failed to read GetBucketLogging response body");
         assert!(
             logging_body.contains("<BucketLoggingStatus"),
             "GetBucketLogging response should contain BucketLoggingStatus XML, got: {logging_body}"
         );
 
-        let accel_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?accelerate=", env.url), &env.access_key, &env.secret_key)
-            .expect("GetBucketAccelerateConfiguration HTTP request failed");
-        assert_eq!(parse_status(&accel_raw), Some(200), "GetBucketAccelerateConfiguration should return 200");
-        let accel_body = parse_body(&accel_raw);
+        let accel_response = signed_s3_request(
+            Method::GET,
+            &format!("{}/{bucket}?accelerate=", env.url),
+            None,
+            None,
+            &env.access_key,
+            &env.secret_key,
+        )
+        .await
+        .expect("GetBucketAccelerateConfiguration HTTP request failed");
+        assert_eq!(accel_response.status(), 200, "GetBucketAccelerateConfiguration should return 200");
+        let accel_body = accel_response
+            .text()
+            .await
+            .expect("Failed to read GetBucketAccelerateConfiguration response body");
         assert!(
             accel_body.contains("<AccelerateConfiguration"),
             "GetBucketAccelerateConfiguration response should contain AccelerateConfiguration XML, got: {accel_body}"
         );
 
-        let payment_raw =
-            execute_s3_awscurl("GET", &format!("{}/{bucket}?requestPayment=", env.url), &env.access_key, &env.secret_key)
-                .expect("GetBucketRequestPayment HTTP request failed");
-        assert_eq!(parse_status(&payment_raw), Some(200), "GetBucketRequestPayment should return 200");
-        let payment_body = parse_body(&payment_raw);
+        let payment_response = signed_s3_request(
+            Method::GET,
+            &format!("{}/{bucket}?requestPayment=", env.url),
+            None,
+            None,
+            &env.access_key,
+            &env.secret_key,
+        )
+        .await
+        .expect("GetBucketRequestPayment HTTP request failed");
+        assert_eq!(payment_response.status(), 200, "GetBucketRequestPayment should return 200");
+        let payment_body = payment_response
+            .text()
+            .await
+            .expect("Failed to read GetBucketRequestPayment response body");
         assert!(
             payment_body.contains("<Payer>BucketOwner</Payer>"),
             "GetBucketRequestPayment should return BucketOwner payer, got: {payment_body}"
         );
 
-        let website_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?website=", env.url), &env.access_key, &env.secret_key)
-            .expect("GetBucketWebsite HTTP request failed");
+        let website_response = signed_s3_request(
+            Method::GET,
+            &format!("{}/{bucket}?website=", env.url),
+            None,
+            None,
+            &env.access_key,
+            &env.secret_key,
+        )
+        .await
+        .expect("GetBucketWebsite HTTP request failed");
         assert_eq!(
-            parse_status(&website_raw),
-            Some(404),
+            website_response.status(),
+            404,
             "GetBucketWebsite should return 404 when website config is absent"
         );
-        let website_content_type = parse_headers(&website_raw).to_ascii_lowercase();
+        let website_content_type = website_response
+            .headers()
+            .get(CONTENT_TYPE)
+            .expect("GetBucketWebsite response should include Content-Type")
+            .to_str()
+            .expect("GetBucketWebsite Content-Type should be valid ASCII")
+            .to_ascii_lowercase();
         assert!(
-            website_content_type.contains("content-type:") && website_content_type.contains("xml"),
+            website_content_type.contains("xml"),
             "GetBucketWebsite error response should be XML, got content-type: {website_content_type}"
         );
-        let website_body = parse_body(&website_raw);
+        let website_body = website_response
+            .text()
+            .await
+            .expect("Failed to read GetBucketWebsite response body");
         assert!(
             website_body.contains("<Code>NoSuchWebsiteConfiguration</Code>"),
             "GetBucketWebsite should return NoSuchWebsiteConfiguration code, got: {website_body}"
         );
 
-        let delete_raw =
-            execute_s3_awscurl("DELETE", &format!("{}/{bucket}?website=", env.url), &env.access_key, &env.secret_key)
-                .expect("DeleteBucketWebsite HTTP request failed");
-        assert_eq!(parse_status(&delete_raw), Some(204), "DeleteBucketWebsite should return 204");
+        let delete_response = signed_s3_request(
+            Method::DELETE,
+            &format!("{}/{bucket}?website=", env.url),
+            None,
+            None,
+            &env.access_key,
+            &env.secret_key,
+        )
+        .await
+        .expect("DeleteBucketWebsite HTTP request failed");
+        assert_eq!(delete_response.status(), 204, "DeleteBucketWebsite should return 204");
 
         env.stop_server();
     }

crates/e2e_test/src/common.rs

@@ -128,6 +128,38 @@ pub fn local_http_client() -> HttpClient {
         .expect("failed to build local reqwest client")
 }
 
+pub(crate) async fn signed_s3_request(
+    method: http::Method,
+    url: &str,
+    body: Option<String>,
+    content_type: Option<&str>,
+    access_key: &str,
+    secret_key: &str,
+) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("S3 URL missing authority")?.to_string();
+    let mut request = http::Request::builder()
+        .method(method.clone())
+        .uri(uri)
+        .header(HOST, authority)
+        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+    if let Some(content_type) = content_type {
+        request = request.header(CONTENT_TYPE, content_type);
+    }
+
+    let content_length = i64::try_from(body.as_ref().map_or(0, String::len)).map_err(|_| "S3 request body is too large")?;
+    let signed = sign_v4(request.body(Body::empty())?, content_length, access_key, secret_key, "", "us-east-1");
+
+    let mut request = local_http_client().request(method, url);
+    for (name, value) in signed.headers() {
+        request = request.header(name, value);
+    }
+    if let Some(body) = body {
+        request = request.body(body);
+    }
+    Ok(request.send().await?)
+}
+
 /// Signs and sends an admin HTTP request with the given credentials.
 pub(crate) async fn admin_request(
     base_url: &str,
@@ -138,28 +170,8 @@ pub(crate) async fn admin_request(
     secret_key: &str,
 ) -> Result<(StatusCode, String), Box<dyn std::error::Error + Send + Sync>> {
     let url = format!("{base_url}{path_and_query}");
-    let uri = url.parse::<http::Uri>()?;
-    let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
-    let mut request = http::Request::builder()
-        .method(method.clone())
-        .uri(uri)
-        .header(HOST, authority)
-        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
-    if body.is_some() {
-        request = request.header(CONTENT_TYPE, "application/json");
-    }
-
-    let content_length = i64::try_from(body.as_ref().map_or(0, String::len)).map_err(|_| "admin request body is too large")?;
-    let signed = sign_v4(request.body(Body::empty())?, content_length, access_key, secret_key, "", "us-east-1");
-
-    let mut request = local_http_client().request(method, &url);
-    for (name, value) in signed.headers() {
-        request = request.header(name, value);
-    }
-    if let Some(body) = body {
-        request = request.body(body);
-    }
-    let response = request.send().await?;
+    let content_type = body.as_ref().map(|_| "application/json");
+    let response = signed_s3_request(method, &url, body, content_type, access_key, secret_key).await?;
     let status = response.status();
     let body = response.text().await?;
     Ok((status, body))

crates/e2e_test/src/list_buckets_iam_filter_test.rs

@@ -16,6 +16,7 @@ use crate::common::{RustFSTestEnvironment, admin_ok, build_test_s3_config, build
 use aws_sdk_s3::Client;
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use serial_test::serial;
+use tokio::time::{Duration, Instant};
 
 fn user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str, session_token: Option<&str>) -> Client {
     Client::from_conf(build_test_s3_config(
@@ -201,8 +202,25 @@ async fn list_buckets_filters_with_iam_bucket_resources() -> Result<(), Box<dyn
         bucket_names(benchmark_client.list_buckets().send().await?.buckets()),
         vec!["benchmark-artifacts", "benchmark-location-only", "benchmark-test1"]
     );
-    let audit_log =
-        tokio::fs::read_to_string(env.capture_log_path.as_deref().expect("server log path should be configured")).await?;
+    let log_path = env.capture_log_path.as_deref().expect("server log path should be configured");
+    let deadline = Instant::now() + Duration::from_secs(5);
+    let audit_log = loop {
+        let audit_log = tokio::fs::read_to_string(log_path).await?;
+        if [
+            "iam_implicit_deny",
+            "s3_authorization_denied",
+            "ListAllMyBucketsAction",
+            "benchmark",
+            "DEBUG",
+        ]
+        .iter()
+        .all(|field| audit_log.contains(field))
+            || Instant::now() >= deadline
+        {
+            break audit_log;
+        }
+        tokio::time::sleep(Duration::from_millis(50)).await;
+    };
     assert_eq!(audit_log.matches("iam_implicit_deny").count(), 1, "{audit_log}");
     for field in ["s3_authorization_denied", "ListAllMyBucketsAction", "benchmark", "DEBUG"] {
         assert!(audit_log.contains(field), "missing {field} in {audit_log}");