ci(e2e): stabilize full-gate tooling (#5805)

.config/nextest.toml

@@ -30,6 +30,7 @@
 [test-groups]
 ecstore-serial-flaky = { max-threads = 1 }
 embedded-test-ports = { max-threads = 1 }
+e2e-vault = { max-threads = 1 }
 
 # Reliability / fault-injection e2e tests each spawn a single-node 4-disk RustFS
 # server and manipulate its disk directories at runtime (crates/e2e_test:
@@ -96,6 +97,12 @@ test-group = 'e2e-reliability'
 filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)'
 test-group = 'e2e-inline-boundaries'
 
+# Vault KMS tests share the fixed dev-server port 8200. serial_test's #[serial]
+# does not cross nextest process boundaries, so keep these tests in one group.
+[[profile.default.overrides]]
+filter = 'package(e2e_test) & test(/^kms::kms_vault_test::/)'
+test-group = 'e2e-vault'
+
 # ---------------------------------------------------------------------------
 # ci profile — the strict CI gate (ci.yml `cargo nextest run --profile ci`)
 # ---------------------------------------------------------------------------
@@ -342,9 +349,9 @@ path = "junit.xml"
 #
 # Each e2e test spawns its own single-node rustfs server on a random port with
 # an isolated temp dir (crates/e2e_test/src/common.rs), so the set is
-# parallel-safe — the same property e2e-smoke relies on. The exception is the
+# parallel-safe — the same property e2e-smoke relies on. The exceptions are the
-# 4-disk reliability / degraded-read fault-injection tests, serialized below
+# 4-disk reliability / degraded-read fault-injection tests and the fixed-port
-# (identical to the ci profile) so several 4-disk servers never run at once.
+# Vault tests, both serialized below.
 # KNOWN-FAILURE EXCLUSIONS (characterization run 29381309848, 2026-07-15:
 # 341 ran / 32 failed on the suites' first automated run ever). Deterministic
 # product failures cannot be quarantined away with retries, so each family is
@@ -380,3 +387,7 @@ test-group = 'e2e-reliability'
 [[profile.e2e-full.overrides]]
 filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)'
 test-group = 'e2e-inline-boundaries'
+
+[[profile.e2e-full.overrides]]
+filter = 'package(e2e_test) & test(/^kms::kms_vault_test::/)'
+test-group = 'e2e-vault'

.github/workflows/ci.yml

@@ -777,12 +777,25 @@ jobs:
 
       - name: Install awscurl
         run: |
-          python3 -m pip install --user --upgrade pip awscurl
+          python3 -m pip install --user --upgrade pip "awscurl==0.44"
           echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
 
       - name: Verify awscurl
         run: test -x "$AWSCURL_PATH"
 
+      - name: Install Vault
+        run: |
+          VAULT_VERSION="1.17.6"
+          VAULT_ARCHIVE="vault_${VAULT_VERSION}_linux_amd64.zip"
+          curl -fsSLo "$RUNNER_TEMP/$VAULT_ARCHIVE" "https://releases.hashicorp.com/vault/${VAULT_VERSION}/${VAULT_ARCHIVE}"
+          echo "0cddc1fbbb88583b5ba5b845f9f8fae47c6fb39a6d48cd543c6ba6fd3ac1a669  $RUNNER_TEMP/$VAULT_ARCHIVE" | sha256sum --check --status
+          unzip -q "$RUNNER_TEMP/$VAULT_ARCHIVE" -d "$RUNNER_TEMP/vault-bin"
+          echo "RUSTFS_TEST_VAULT_BIN=$RUNNER_TEMP/vault-bin/vault" >> "$GITHUB_ENV"
+
+      - name: Verify Vault
+        run: |
+          "$RUSTFS_TEST_VAULT_BIN" version
+
       # Download after the cache restore so the freshly built binary from the
       # build job always wins over anything restored into target/debug.
       - name: Download debug binary

crates/e2e_test/src/bucket_logging_test.rs

@@ -16,91 +16,17 @@
 
 #[cfg(test)]
 mod tests {
-    use crate::common::{RustFSTestEnvironment, init_logging};
+    use crate::common::{RustFSTestEnvironment, init_logging, signed_s3_request};
     use aws_sdk_s3::error::ProvideErrorMetadata;
     use aws_sdk_s3::types::{
         AccelerateConfiguration, BucketAccelerateStatus, BucketLoggingStatus, IndexDocument, LoggingEnabled, Payer,
         RequestPaymentConfiguration, WebsiteConfiguration,
     };
+    use http::Method;
+    use http::header::CONTENT_TYPE;
     use serial_test::serial;
-    use std::path::PathBuf;
-    use std::process::Command;
     use tracing::info;
 
-    fn awscurl_binary_path() -> PathBuf {
-        std::env::var_os("AWSCURL_PATH")
-            .map(PathBuf::from)
-            .unwrap_or_else(|| PathBuf::from("awscurl"))
-    }
-
-    fn awscurl_available() -> bool {
-        Command::new(awscurl_binary_path()).arg("--version").output().is_ok()
-    }
-
-    fn execute_s3_awscurl(
-        method: &str,
-        url: &str,
-        access_key: &str,
-        secret_key: &str,
-    ) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
-        let output = Command::new(awscurl_binary_path())
-            .args([
-                "--service",
-                "s3",
-                "--region",
-                "us-east-1",
-                "--access_key",
-                access_key,
-                "--secret_key",
-                secret_key,
-                "-i",
-                "-X",
-                method,
-                url,
-            ])
-            .output()?;
-        if !output.status.success() {
-            let stderr = String::from_utf8_lossy(&output.stderr);
-            let stdout = String::from_utf8_lossy(&output.stdout);
-            return Err(format!("awscurl failed: stderr='{stderr}', stdout='{stdout}'").into());
-        }
-        Ok(String::from_utf8_lossy(&output.stdout).to_string())
-    }
-
-    fn parse_status(raw: &str) -> Option<u16> {
-        raw.lines()
-            .filter_map(|line| {
-                if line.starts_with("HTTP/") {
-                    line.split_whitespace().nth(1)?.parse::<u16>().ok()
-                } else {
-                    None
-                }
-            })
-            .next_back()
-    }
-
-    fn parse_body(raw: &str) -> String {
-        if let Some(pos) = raw.rfind("\r\n\r\n") {
-            return raw[pos + 4..].to_string();
-        }
-        if let Some(pos) = raw.rfind("\n\n") {
-            return raw[pos + 2..].to_string();
-        }
-        String::new()
-    }
-
-    fn parse_headers(raw: &str) -> String {
-        let start = raw.rfind("HTTP/").unwrap_or(0);
-        let tail = &raw[start..];
-        if let Some(pos) = tail.find("\r\n\r\n") {
-            return tail[..pos].to_string();
-        }
-        if let Some(pos) = tail.find("\n\n") {
-            return tail[..pos].to_string();
-        }
-        tail.to_string()
-    }
-
     #[tokio::test]
     #[serial]
     async fn test_dummy_bucket_compatibility_endpoints() {
@@ -470,10 +396,6 @@ mod tests {
     async fn test_dummy_bucket_endpoints_http_contracts() {
         init_logging();
         info!("Starting test: dummy-compat bucket API HTTP contracts");
-        if !awscurl_available() {
-            info!("Skipping test_dummy_bucket_endpoints_http_contracts: awscurl binary not found");
-            return;
-        }
 
         let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
         env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
@@ -488,56 +410,112 @@ mod tests {
             .await
             .expect("Failed to create bucket");
 
-        let logging_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?logging=", env.url), &env.access_key, &env.secret_key)
+        let logging_response = signed_s3_request(
-            .expect("GetBucketLogging HTTP request failed");
+            Method::GET,
-        assert_eq!(parse_status(&logging_raw), Some(200), "GetBucketLogging should return 200");
+            &format!("{}/{bucket}?logging=", env.url),
-        let logging_body = parse_body(&logging_raw);
+            None,
+            None,
+            &env.access_key,
+            &env.secret_key,
+        )
+        .await
+        .expect("GetBucketLogging HTTP request failed");
+        assert_eq!(logging_response.status(), 200, "GetBucketLogging should return 200");
+        let logging_body = logging_response
+            .text()
+            .await
+            .expect("Failed to read GetBucketLogging response body");
         assert!(
             logging_body.contains("<BucketLoggingStatus"),
             "GetBucketLogging response should contain BucketLoggingStatus XML, got: {logging_body}"
         );
 
-        let accel_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?accelerate=", env.url), &env.access_key, &env.secret_key)
+        let accel_response = signed_s3_request(
-            .expect("GetBucketAccelerateConfiguration HTTP request failed");
+            Method::GET,
-        assert_eq!(parse_status(&accel_raw), Some(200), "GetBucketAccelerateConfiguration should return 200");
+            &format!("{}/{bucket}?accelerate=", env.url),
-        let accel_body = parse_body(&accel_raw);
+            None,
+            None,
+            &env.access_key,
+            &env.secret_key,
+        )
+        .await
+        .expect("GetBucketAccelerateConfiguration HTTP request failed");
+        assert_eq!(accel_response.status(), 200, "GetBucketAccelerateConfiguration should return 200");
+        let accel_body = accel_response
+            .text()
+            .await
+            .expect("Failed to read GetBucketAccelerateConfiguration response body");
         assert!(
             accel_body.contains("<AccelerateConfiguration"),
             "GetBucketAccelerateConfiguration response should contain AccelerateConfiguration XML, got: {accel_body}"
         );
 
-        let payment_raw =
+        let payment_response = signed_s3_request(
-            execute_s3_awscurl("GET", &format!("{}/{bucket}?requestPayment=", env.url), &env.access_key, &env.secret_key)
+            Method::GET,
-                .expect("GetBucketRequestPayment HTTP request failed");
+            &format!("{}/{bucket}?requestPayment=", env.url),
-        assert_eq!(parse_status(&payment_raw), Some(200), "GetBucketRequestPayment should return 200");
+            None,
-        let payment_body = parse_body(&payment_raw);
+            None,
+            &env.access_key,
+            &env.secret_key,
+        )
+        .await
+        .expect("GetBucketRequestPayment HTTP request failed");
+        assert_eq!(payment_response.status(), 200, "GetBucketRequestPayment should return 200");
+        let payment_body = payment_response
+            .text()
+            .await
+            .expect("Failed to read GetBucketRequestPayment response body");
         assert!(
             payment_body.contains("<Payer>BucketOwner</Payer>"),
             "GetBucketRequestPayment should return BucketOwner payer, got: {payment_body}"
         );
 
-        let website_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?website=", env.url), &env.access_key, &env.secret_key)
+        let website_response = signed_s3_request(
-            .expect("GetBucketWebsite HTTP request failed");
+            Method::GET,
+            &format!("{}/{bucket}?website=", env.url),
+            None,
+            None,
+            &env.access_key,
+            &env.secret_key,
+        )
+        .await
+        .expect("GetBucketWebsite HTTP request failed");
         assert_eq!(
-            parse_status(&website_raw),
+            website_response.status(),
-            Some(404),
+            404,
             "GetBucketWebsite should return 404 when website config is absent"
         );
-        let website_content_type = parse_headers(&website_raw).to_ascii_lowercase();
+        let website_content_type = website_response
+            .headers()
+            .get(CONTENT_TYPE)
+            .expect("GetBucketWebsite response should include Content-Type")
+            .to_str()
+            .expect("GetBucketWebsite Content-Type should be valid ASCII")
+            .to_ascii_lowercase();
         assert!(
-            website_content_type.contains("content-type:") && website_content_type.contains("xml"),
+            website_content_type.contains("xml"),
             "GetBucketWebsite error response should be XML, got content-type: {website_content_type}"
         );
-        let website_body = parse_body(&website_raw);
+        let website_body = website_response
+            .text()
+            .await
+            .expect("Failed to read GetBucketWebsite response body");
         assert!(
             website_body.contains("<Code>NoSuchWebsiteConfiguration</Code>"),
             "GetBucketWebsite should return NoSuchWebsiteConfiguration code, got: {website_body}"
         );
 
-        let delete_raw =
+        let delete_response = signed_s3_request(
-            execute_s3_awscurl("DELETE", &format!("{}/{bucket}?website=", env.url), &env.access_key, &env.secret_key)
+            Method::DELETE,
-                .expect("DeleteBucketWebsite HTTP request failed");
+            &format!("{}/{bucket}?website=", env.url),
-        assert_eq!(parse_status(&delete_raw), Some(204), "DeleteBucketWebsite should return 204");
+            None,
+            None,
+            &env.access_key,
+            &env.secret_key,
+        )
+        .await
+        .expect("DeleteBucketWebsite HTTP request failed");
+        assert_eq!(delete_response.status(), 204, "DeleteBucketWebsite should return 204");
 
         env.stop_server();
     }

crates/e2e_test/src/common.rs

@@ -128,6 +128,38 @@ pub fn local_http_client() -> HttpClient {
         .expect("failed to build local reqwest client")
 }
 
+pub(crate) async fn signed_s3_request(
+    method: http::Method,
+    url: &str,
+    body: Option<String>,
+    content_type: Option<&str>,
+    access_key: &str,
+    secret_key: &str,
+) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("S3 URL missing authority")?.to_string();
+    let mut request = http::Request::builder()
+        .method(method.clone())
+        .uri(uri)
+        .header(HOST, authority)
+        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+    if let Some(content_type) = content_type {
+        request = request.header(CONTENT_TYPE, content_type);
+    }
+
+    let content_length = i64::try_from(body.as_ref().map_or(0, String::len)).map_err(|_| "S3 request body is too large")?;
+    let signed = sign_v4(request.body(Body::empty())?, content_length, access_key, secret_key, "", "us-east-1");
+
+    let mut request = local_http_client().request(method, url);
+    for (name, value) in signed.headers() {
+        request = request.header(name, value);
+    }
+    if let Some(body) = body {
+        request = request.body(body);
+    }
+    Ok(request.send().await?)
+}
+
 /// Signs and sends an admin HTTP request with the given credentials.
 pub(crate) async fn admin_request(
     base_url: &str,
@@ -138,28 +170,8 @@ pub(crate) async fn admin_request(
     secret_key: &str,
 ) -> Result<(StatusCode, String), Box<dyn std::error::Error + Send + Sync>> {
     let url = format!("{base_url}{path_and_query}");
-    let uri = url.parse::<http::Uri>()?;
+    let content_type = body.as_ref().map(|_| "application/json");
-    let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
+    let response = signed_s3_request(method, &url, body, content_type, access_key, secret_key).await?;
-    let mut request = http::Request::builder()
-        .method(method.clone())
-        .uri(uri)
-        .header(HOST, authority)
-        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
-    if body.is_some() {
-        request = request.header(CONTENT_TYPE, "application/json");
-    }
-
-    let content_length = i64::try_from(body.as_ref().map_or(0, String::len)).map_err(|_| "admin request body is too large")?;
-    let signed = sign_v4(request.body(Body::empty())?, content_length, access_key, secret_key, "", "us-east-1");
-
-    let mut request = local_http_client().request(method, &url);
-    for (name, value) in signed.headers() {
-        request = request.header(name, value);
-    }
-    if let Some(body) = body {
-        request = request.body(body);
-    }
-    let response = request.send().await?;
     let status = response.status();
     let body = response.text().await?;
     Ok((status, body))

crates/e2e_test/src/list_buckets_iam_filter_test.rs

@@ -16,6 +16,7 @@ use crate::common::{RustFSTestEnvironment, admin_ok, build_test_s3_config, build
 use aws_sdk_s3::Client;
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use serial_test::serial;
+use tokio::time::{Duration, Instant};
 
 fn user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str, session_token: Option<&str>) -> Client {
     Client::from_conf(build_test_s3_config(
@@ -201,8 +202,25 @@ async fn list_buckets_filters_with_iam_bucket_resources() -> Result<(), Box<dyn
         bucket_names(benchmark_client.list_buckets().send().await?.buckets()),
         vec!["benchmark-artifacts", "benchmark-location-only", "benchmark-test1"]
     );
-    let audit_log =
+    let log_path = env.capture_log_path.as_deref().expect("server log path should be configured");
-        tokio::fs::read_to_string(env.capture_log_path.as_deref().expect("server log path should be configured")).await?;
+    let deadline = Instant::now() + Duration::from_secs(5);
+    let audit_log = loop {
+        let audit_log = tokio::fs::read_to_string(log_path).await?;
+        if [
+            "iam_implicit_deny",
+            "s3_authorization_denied",
+            "ListAllMyBucketsAction",
+            "benchmark",
+            "DEBUG",
+        ]
+        .iter()
+        .all(|field| audit_log.contains(field))
+            || Instant::now() >= deadline
+        {
+            break audit_log;
+        }
+        tokio::time::sleep(Duration::from_millis(50)).await;
+    };
     assert_eq!(audit_log.matches("iam_implicit_deny").count(), 1, "{audit_log}");
     for field in ["s3_authorization_denied", "ListAllMyBucketsAction", "benchmark", "DEBUG"] {
         assert!(audit_log.contains(field), "missing {field} in {audit_log}");