From ba5641237c447028d03131f79fa507e04acadeb9 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 7 Aug 2026 23:04:33 +0800 Subject: [PATCH] ci(e2e): stabilize full-gate tooling (#5805) --- .config/nextest.toml | 17 +- .github/workflows/ci.yml | 15 +- crates/e2e_test/src/bucket_logging_test.rs | 188 ++++++++---------- crates/e2e_test/src/common.rs | 56 ++++-- .../src/list_buckets_iam_filter_test.rs | 22 +- 5 files changed, 165 insertions(+), 133 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index d17398000..bc9a4d5b3 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -30,6 +30,7 @@ [test-groups] ecstore-serial-flaky = { max-threads = 1 } embedded-test-ports = { max-threads = 1 } +e2e-vault = { max-threads = 1 } # Reliability / fault-injection e2e tests each spawn a single-node 4-disk RustFS # server and manipulate its disk directories at runtime (crates/e2e_test: @@ -96,6 +97,12 @@ test-group = 'e2e-reliability' filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)' test-group = 'e2e-inline-boundaries' +# Vault KMS tests share the fixed dev-server port 8200. serial_test's #[serial] +# does not cross nextest process boundaries, so keep these tests in one group. +[[profile.default.overrides]] +filter = 'package(e2e_test) & test(/^kms::kms_vault_test::/)' +test-group = 'e2e-vault' + # --------------------------------------------------------------------------- # ci profile — the strict CI gate (ci.yml `cargo nextest run --profile ci`) # --------------------------------------------------------------------------- @@ -342,9 +349,9 @@ path = "junit.xml" # # Each e2e test spawns its own single-node rustfs server on a random port with # an isolated temp dir (crates/e2e_test/src/common.rs), so the set is -# parallel-safe — the same property e2e-smoke relies on. The exception is the -# 4-disk reliability / degraded-read fault-injection tests, serialized below -# (identical to the ci profile) so several 4-disk servers never run at once. +# parallel-safe — the same property e2e-smoke relies on. The exceptions are the +# 4-disk reliability / degraded-read fault-injection tests and the fixed-port +# Vault tests, both serialized below. # KNOWN-FAILURE EXCLUSIONS (characterization run 29381309848, 2026-07-15: # 341 ran / 32 failed on the suites' first automated run ever). Deterministic # product failures cannot be quarantined away with retries, so each family is @@ -380,3 +387,7 @@ test-group = 'e2e-reliability' [[profile.e2e-full.overrides]] filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)' test-group = 'e2e-inline-boundaries' + +[[profile.e2e-full.overrides]] +filter = 'package(e2e_test) & test(/^kms::kms_vault_test::/)' +test-group = 'e2e-vault' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 06f393a72..981eda359 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -777,12 +777,25 @@ jobs: - name: Install awscurl run: | - python3 -m pip install --user --upgrade pip awscurl + python3 -m pip install --user --upgrade pip "awscurl==0.44" echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV" - name: Verify awscurl run: test -x "$AWSCURL_PATH" + - name: Install Vault + run: | + VAULT_VERSION="1.17.6" + VAULT_ARCHIVE="vault_${VAULT_VERSION}_linux_amd64.zip" + curl -fsSLo "$RUNNER_TEMP/$VAULT_ARCHIVE" "https://releases.hashicorp.com/vault/${VAULT_VERSION}/${VAULT_ARCHIVE}" + echo "0cddc1fbbb88583b5ba5b845f9f8fae47c6fb39a6d48cd543c6ba6fd3ac1a669 $RUNNER_TEMP/$VAULT_ARCHIVE" | sha256sum --check --status + unzip -q "$RUNNER_TEMP/$VAULT_ARCHIVE" -d "$RUNNER_TEMP/vault-bin" + echo "RUSTFS_TEST_VAULT_BIN=$RUNNER_TEMP/vault-bin/vault" >> "$GITHUB_ENV" + + - name: Verify Vault + run: | + "$RUSTFS_TEST_VAULT_BIN" version + # Download after the cache restore so the freshly built binary from the # build job always wins over anything restored into target/debug. - name: Download debug binary diff --git a/crates/e2e_test/src/bucket_logging_test.rs b/crates/e2e_test/src/bucket_logging_test.rs index 461c52995..79d2e6843 100644 --- a/crates/e2e_test/src/bucket_logging_test.rs +++ b/crates/e2e_test/src/bucket_logging_test.rs @@ -16,91 +16,17 @@ #[cfg(test)] mod tests { - use crate::common::{RustFSTestEnvironment, init_logging}; + use crate::common::{RustFSTestEnvironment, init_logging, signed_s3_request}; use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::types::{ AccelerateConfiguration, BucketAccelerateStatus, BucketLoggingStatus, IndexDocument, LoggingEnabled, Payer, RequestPaymentConfiguration, WebsiteConfiguration, }; + use http::Method; + use http::header::CONTENT_TYPE; use serial_test::serial; - use std::path::PathBuf; - use std::process::Command; use tracing::info; - fn awscurl_binary_path() -> PathBuf { - std::env::var_os("AWSCURL_PATH") - .map(PathBuf::from) - .unwrap_or_else(|| PathBuf::from("awscurl")) - } - - fn awscurl_available() -> bool { - Command::new(awscurl_binary_path()).arg("--version").output().is_ok() - } - - fn execute_s3_awscurl( - method: &str, - url: &str, - access_key: &str, - secret_key: &str, - ) -> Result> { - let output = Command::new(awscurl_binary_path()) - .args([ - "--service", - "s3", - "--region", - "us-east-1", - "--access_key", - access_key, - "--secret_key", - secret_key, - "-i", - "-X", - method, - url, - ]) - .output()?; - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - let stdout = String::from_utf8_lossy(&output.stdout); - return Err(format!("awscurl failed: stderr='{stderr}', stdout='{stdout}'").into()); - } - Ok(String::from_utf8_lossy(&output.stdout).to_string()) - } - - fn parse_status(raw: &str) -> Option { - raw.lines() - .filter_map(|line| { - if line.starts_with("HTTP/") { - line.split_whitespace().nth(1)?.parse::().ok() - } else { - None - } - }) - .next_back() - } - - fn parse_body(raw: &str) -> String { - if let Some(pos) = raw.rfind("\r\n\r\n") { - return raw[pos + 4..].to_string(); - } - if let Some(pos) = raw.rfind("\n\n") { - return raw[pos + 2..].to_string(); - } - String::new() - } - - fn parse_headers(raw: &str) -> String { - let start = raw.rfind("HTTP/").unwrap_or(0); - let tail = &raw[start..]; - if let Some(pos) = tail.find("\r\n\r\n") { - return tail[..pos].to_string(); - } - if let Some(pos) = tail.find("\n\n") { - return tail[..pos].to_string(); - } - tail.to_string() - } - #[tokio::test] #[serial] async fn test_dummy_bucket_compatibility_endpoints() { @@ -470,10 +396,6 @@ mod tests { async fn test_dummy_bucket_endpoints_http_contracts() { init_logging(); info!("Starting test: dummy-compat bucket API HTTP contracts"); - if !awscurl_available() { - info!("Skipping test_dummy_bucket_endpoints_http_contracts: awscurl binary not found"); - return; - } let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); @@ -488,56 +410,112 @@ mod tests { .await .expect("Failed to create bucket"); - let logging_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?logging=", env.url), &env.access_key, &env.secret_key) - .expect("GetBucketLogging HTTP request failed"); - assert_eq!(parse_status(&logging_raw), Some(200), "GetBucketLogging should return 200"); - let logging_body = parse_body(&logging_raw); + let logging_response = signed_s3_request( + Method::GET, + &format!("{}/{bucket}?logging=", env.url), + None, + None, + &env.access_key, + &env.secret_key, + ) + .await + .expect("GetBucketLogging HTTP request failed"); + assert_eq!(logging_response.status(), 200, "GetBucketLogging should return 200"); + let logging_body = logging_response + .text() + .await + .expect("Failed to read GetBucketLogging response body"); assert!( logging_body.contains("BucketOwner"), "GetBucketRequestPayment should return BucketOwner payer, got: {payment_body}" ); - let website_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?website=", env.url), &env.access_key, &env.secret_key) - .expect("GetBucketWebsite HTTP request failed"); + let website_response = signed_s3_request( + Method::GET, + &format!("{}/{bucket}?website=", env.url), + None, + None, + &env.access_key, + &env.secret_key, + ) + .await + .expect("GetBucketWebsite HTTP request failed"); assert_eq!( - parse_status(&website_raw), - Some(404), + website_response.status(), + 404, "GetBucketWebsite should return 404 when website config is absent" ); - let website_content_type = parse_headers(&website_raw).to_ascii_lowercase(); + let website_content_type = website_response + .headers() + .get(CONTENT_TYPE) + .expect("GetBucketWebsite response should include Content-Type") + .to_str() + .expect("GetBucketWebsite Content-Type should be valid ASCII") + .to_ascii_lowercase(); assert!( - website_content_type.contains("content-type:") && website_content_type.contains("xml"), + website_content_type.contains("xml"), "GetBucketWebsite error response should be XML, got content-type: {website_content_type}" ); - let website_body = parse_body(&website_raw); + let website_body = website_response + .text() + .await + .expect("Failed to read GetBucketWebsite response body"); assert!( website_body.contains("NoSuchWebsiteConfiguration"), "GetBucketWebsite should return NoSuchWebsiteConfiguration code, got: {website_body}" ); - let delete_raw = - execute_s3_awscurl("DELETE", &format!("{}/{bucket}?website=", env.url), &env.access_key, &env.secret_key) - .expect("DeleteBucketWebsite HTTP request failed"); - assert_eq!(parse_status(&delete_raw), Some(204), "DeleteBucketWebsite should return 204"); + let delete_response = signed_s3_request( + Method::DELETE, + &format!("{}/{bucket}?website=", env.url), + None, + None, + &env.access_key, + &env.secret_key, + ) + .await + .expect("DeleteBucketWebsite HTTP request failed"); + assert_eq!(delete_response.status(), 204, "DeleteBucketWebsite should return 204"); env.stop_server(); } diff --git a/crates/e2e_test/src/common.rs b/crates/e2e_test/src/common.rs index a5b5476f1..c8fc50a7c 100644 --- a/crates/e2e_test/src/common.rs +++ b/crates/e2e_test/src/common.rs @@ -128,6 +128,38 @@ pub fn local_http_client() -> HttpClient { .expect("failed to build local reqwest client") } +pub(crate) async fn signed_s3_request( + method: http::Method, + url: &str, + body: Option, + content_type: Option<&str>, + access_key: &str, + secret_key: &str, +) -> Result> { + let uri = url.parse::()?; + let authority = uri.authority().ok_or("S3 URL missing authority")?.to_string(); + let mut request = http::Request::builder() + .method(method.clone()) + .uri(uri) + .header(HOST, authority) + .header("x-amz-content-sha256", UNSIGNED_PAYLOAD); + if let Some(content_type) = content_type { + request = request.header(CONTENT_TYPE, content_type); + } + + let content_length = i64::try_from(body.as_ref().map_or(0, String::len)).map_err(|_| "S3 request body is too large")?; + let signed = sign_v4(request.body(Body::empty())?, content_length, access_key, secret_key, "", "us-east-1"); + + let mut request = local_http_client().request(method, url); + for (name, value) in signed.headers() { + request = request.header(name, value); + } + if let Some(body) = body { + request = request.body(body); + } + Ok(request.send().await?) +} + /// Signs and sends an admin HTTP request with the given credentials. pub(crate) async fn admin_request( base_url: &str, @@ -138,28 +170,8 @@ pub(crate) async fn admin_request( secret_key: &str, ) -> Result<(StatusCode, String), Box> { let url = format!("{base_url}{path_and_query}"); - let uri = url.parse::()?; - let authority = uri.authority().ok_or("admin URL missing authority")?.to_string(); - let mut request = http::Request::builder() - .method(method.clone()) - .uri(uri) - .header(HOST, authority) - .header("x-amz-content-sha256", UNSIGNED_PAYLOAD); - if body.is_some() { - request = request.header(CONTENT_TYPE, "application/json"); - } - - let content_length = i64::try_from(body.as_ref().map_or(0, String::len)).map_err(|_| "admin request body is too large")?; - let signed = sign_v4(request.body(Body::empty())?, content_length, access_key, secret_key, "", "us-east-1"); - - let mut request = local_http_client().request(method, &url); - for (name, value) in signed.headers() { - request = request.header(name, value); - } - if let Some(body) = body { - request = request.body(body); - } - let response = request.send().await?; + let content_type = body.as_ref().map(|_| "application/json"); + let response = signed_s3_request(method, &url, body, content_type, access_key, secret_key).await?; let status = response.status(); let body = response.text().await?; Ok((status, body)) diff --git a/crates/e2e_test/src/list_buckets_iam_filter_test.rs b/crates/e2e_test/src/list_buckets_iam_filter_test.rs index f82f30e9a..ca748be1a 100644 --- a/crates/e2e_test/src/list_buckets_iam_filter_test.rs +++ b/crates/e2e_test/src/list_buckets_iam_filter_test.rs @@ -16,6 +16,7 @@ use crate::common::{RustFSTestEnvironment, admin_ok, build_test_s3_config, build use aws_sdk_s3::Client; use aws_sdk_s3::error::ProvideErrorMetadata; use serial_test::serial; +use tokio::time::{Duration, Instant}; fn user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str, session_token: Option<&str>) -> Client { Client::from_conf(build_test_s3_config( @@ -201,8 +202,25 @@ async fn list_buckets_filters_with_iam_bucket_resources() -> Result<(), Box= deadline + { + break audit_log; + } + tokio::time::sleep(Duration::from_millis(50)).await; + }; assert_eq!(audit_log.matches("iam_implicit_deny").count(), 1, "{audit_log}"); for field in ["s3_authorization_denied", "ListAllMyBucketsAction", "benchmark", "DEBUG"] { assert!(audit_log.contains(field), "missing {field} in {audit_log}");