From 22741603f59e83c4352b76c2657a20633390a95f Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 28 Aug 2026 08:12:45 +0800 Subject: [PATCH] test(e2e): finish the helper consolidation onto common.rs (#6766) - common.rs gains an AdminTransport knob (Signed | Awscurl) with admin_execute_at plus three family wrappers: admin_create_user_via, admin_add_canned_policy_via, admin_attach_user_policy_via; the existing admin_create_user now delegates over the Signed transport. - Deleted the four signed admin request clones in admin_mfa_test, admin_auth_test, reliant/tiering, and inline_fast_path_cluster_test; each keeps a thin local wrapper over common::admin_request so call sites keep their Option<&str> body shape. - Deduped the notification_webhook signer onto common::signed_request and the webdav_core signer plus its three admin helpers onto the shared _via helpers. - Consolidated the S3-client-with-credentials builders: admin_auth s3_client_with, existing_object_tag user_client/sts_session_client, bucket_policy_check create_user_client, and the create_user_s3_client copies in group_delete_test and replication_extension_test now delegate to create_s3_client_with_credentials / build_test_s3_config; replication_extension admin_add_canned_policy and admin_attach_policy_to_user route through the _via helpers on the Signed transport. - The awscurl-gated suites (existing_object_tag_policy, bucket_policy_check, policy/policy_variables) keep going through the external awscurl binary via AdminTransport::Awscurl, preserving their wire behavior. Part of rustfs/backlog#1846 (cluster 2). --- crates/e2e_test/src/admin_auth_test.rs | 62 ++------ crates/e2e_test/src/admin_mfa_test.rs | 29 +--- .../e2e_test/src/bucket_policy_check_test.rs | 28 +--- crates/e2e_test/src/common.rs | 136 +++++++++++++++--- .../src/existing_object_tag_policy_test.rs | 62 ++++---- crates/e2e_test/src/group_delete_test.rs | 13 +- .../src/inline_fast_path_cluster_test.rs | 34 +---- .../e2e_test/src/notification_webhook_test.rs | 38 +---- .../src/policy/policy_variables_test.rs | 41 +++--- crates/e2e_test/src/protocols/webdav_core.rs | 113 ++++----------- crates/e2e_test/src/reliant/tiering.rs | 36 +---- .../src/replication_extension_test.rs | 53 ++----- 12 files changed, 238 insertions(+), 407 deletions(-) diff --git a/crates/e2e_test/src/admin_auth_test.rs b/crates/e2e_test/src/admin_auth_test.rs index 1a70bce0c..40e05d4d8 100644 --- a/crates/e2e_test/src/admin_auth_test.rs +++ b/crates/e2e_test/src/admin_auth_test.rs @@ -31,14 +31,9 @@ #[cfg(test)] mod tests { - use crate::common::{RustFSTestEnvironment, init_logging, local_http_client, rustfs_binary_path}; - use aws_sdk_s3::config::{Credentials, Region}; + use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path}; + use aws_sdk_s3::Client; use aws_sdk_s3::error::ProvideErrorMetadata; - use aws_sdk_s3::{Client, Config}; - use http::header::HOST; - use rustfs_signer::constants::UNSIGNED_PAYLOAD; - use rustfs_signer::sign_v4; - use s3s::Body; use std::error::Error; use std::io::Read; use std::process::{Command, Stdio}; @@ -87,10 +82,10 @@ mod tests { } /// Send a SigV4-signed request to `path` (optionally with a JSON `body`) and - /// return `(status, body)`. Uses the `UNSIGNED_PAYLOAD` content hash so a - /// request body can be attached without the caller pre-hashing it — the - /// server verifies the signature against the same sentinel, exactly as the - /// AWS SDKs / MinIO client do for streaming/unsigned payloads. + /// return `(status, body)`. + /// + /// Thin wrapper over [`crate::common::admin_request`], kept local so the + /// call sites below keep their `Option<&str>` body shape. async fn signed_request( base_url: &str, method: http::Method, @@ -99,47 +94,13 @@ mod tests { access_key: &str, secret_key: &str, ) -> Result<(reqwest::StatusCode, String), Box> { - let url = format!("{base_url}{path}"); - let uri = url.parse::()?; - let authority = uri.authority().ok_or("missing authority")?.to_string(); - let body_bytes = body.map(|b| b.as_bytes().to_vec()).unwrap_or_default(); - - // The signature is computed over `UNSIGNED_PAYLOAD`, so the body bytes do - // not participate in the SigV4 hash — sign over an empty body and attach - // the real payload to the wire request below. - let request = http::Request::builder() - .method(method.clone()) - .uri(uri) - .header(HOST, authority) - .header("x-amz-content-sha256", UNSIGNED_PAYLOAD); - let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1"); - - let client = local_http_client(); - let mut rb = client.request(method, url.as_str()); - for (name, value) in signed.headers() { - rb = rb.header(name, value); - } - if !body_bytes.is_empty() { - rb = rb.body(body_bytes); - } - let resp = rb.send().await?; - let status = resp.status(); - let text = resp.text().await?; - Ok((status, text)) + crate::common::admin_request(base_url, method, path, body.map(str::to_string), access_key, secret_key).await } /// Build an S3 client bound to explicit credentials (used to exercise the S3 /// data plane with rotated / stale root credentials). fn s3_client_with(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client { - let credentials = Credentials::new(access_key, secret_key, None, None, "sec4-admin-auth"); - let config = Config::builder() - .credentials_provider(credentials) - .region(Region::new("us-east-1")) - .endpoint_url(&env.url) - .force_path_style(true) - .behavior_version_latest() - .build(); - Client::from_conf(config) + env.create_s3_client_with_credentials(access_key, secret_key) } /// Create a non-admin IAM user via the admin `add-user` API using the root @@ -151,12 +112,7 @@ mod tests { access_key: &str, secret_key: &str, ) -> Result<(), Box> { - let path = format!("/rustfs/admin/v3/add-user?accessKey={access_key}"); - let body = serde_json::json!({ "secretKey": secret_key, "status": "enabled" }).to_string(); - let (status, resp) = - signed_request(&env.url, http::Method::PUT, &path, Some(&body), &env.access_key, &env.secret_key).await?; - assert!(status.is_success(), "add-user should succeed (status={status}, body={resp})"); - Ok(()) + crate::common::admin_create_user(env, access_key, secret_key).await } /// A fully authenticated but non-admin credential must be rejected with diff --git a/crates/e2e_test/src/admin_mfa_test.rs b/crates/e2e_test/src/admin_mfa_test.rs index dc84a3484..e2b0f1dc5 100644 --- a/crates/e2e_test/src/admin_mfa_test.rs +++ b/crates/e2e_test/src/admin_mfa_test.rs @@ -59,8 +59,8 @@ mod tests { /// One signed admin request, returning the status and the raw body. /// - /// Signs with `UNSIGNED_PAYLOAD` so the body does not participate in the - /// hash, matching how the other admin e2e tests drive these routes. + /// Thin wrapper over [`crate::common::admin_request`], kept local so the + /// call sites below keep their `Option<&str>` body shape. async fn signed_request( base_url: &str, method: http::Method, @@ -69,30 +69,7 @@ mod tests { access_key: &str, secret_key: &str, ) -> Result<(reqwest::StatusCode, String), Box> { - let url = format!("{base_url}{path}"); - let uri = url.parse::()?; - let authority = uri.authority().ok_or("missing authority")?.to_string(); - let body_bytes = body.map(|b| b.as_bytes().to_vec()).unwrap_or_default(); - - let request = http::Request::builder() - .method(method.clone()) - .uri(uri) - .header(HOST, authority) - .header("x-amz-content-sha256", UNSIGNED_PAYLOAD); - let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1"); - - let client = local_http_client(); - let mut builder = client.request(method, url.as_str()); - for (name, value) in signed.headers() { - builder = builder.header(name, value); - } - if !body_bytes.is_empty() { - builder = builder.body(body_bytes); - } - let response = builder.send().await?; - let status = response.status(); - let text = response.text().await?; - Ok((status, text)) + crate::common::admin_request(base_url, method, path, body.map(str::to_string), access_key, secret_key).await } /// A SigV4-signed `AssumeRole` form POST, optionally carrying a second factor. diff --git a/crates/e2e_test/src/bucket_policy_check_test.rs b/crates/e2e_test/src/bucket_policy_check_test.rs index 51800e195..5542f2c4f 100644 --- a/crates/e2e_test/src/bucket_policy_check_test.rs +++ b/crates/e2e_test/src/bucket_policy_check_test.rs @@ -15,39 +15,23 @@ //! Regression test for Issue #1423 //! Verifies that Bucket Policies are honored for Authenticated Users. -use crate::common::{RustFSTestEnvironment, init_logging}; -use aws_sdk_s3::config::{Credentials, Region}; +use crate::common::{AdminTransport, RustFSTestEnvironment, admin_create_user_via, init_logging}; +use aws_sdk_s3::Client; use aws_sdk_s3::error::ProvideErrorMetadata; -use aws_sdk_s3::{Client, Config}; use tracing::info; +/// This suite deliberately drives the admin API through the external `awscurl` +/// binary, so user creation pins `AdminTransport::Awscurl`. async fn create_user( env: &RustFSTestEnvironment, username: &str, password: &str, ) -> Result<(), Box> { - let create_user_body = serde_json::json!({ - "secretKey": password, - "status": "enabled" - }) - .to_string(); - - let create_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username); - crate::common::awscurl_put(&create_user_url, &create_user_body, &env.access_key, &env.secret_key).await?; - Ok(()) + admin_create_user_via(AdminTransport::Awscurl, &env.url, &env.access_key, &env.secret_key, username, password).await } fn create_user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client { - let credentials = Credentials::new(access_key, secret_key, None, None, "test-user"); - let config = Config::builder() - .credentials_provider(credentials) - .region(Region::new("us-east-1")) - .endpoint_url(&env.url) - .force_path_style(true) - .behavior_version_latest() - .build(); - - Client::from_conf(config) + env.create_s3_client_with_credentials(access_key, secret_key) } #[tokio::test] diff --git a/crates/e2e_test/src/common.rs b/crates/e2e_test/src/common.rs index 93a533113..0ec379070 100644 --- a/crates/e2e_test/src/common.rs +++ b/crates/e2e_test/src/common.rs @@ -1744,30 +1744,128 @@ pub(crate) async fn admin_create_user( username: &str, secret_key: &str, ) -> Result<(), Box> { - let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username); - let body = serde_json::json!({ - "secretKey": secret_key, - "status": "enabled" - }); - let response = signed_request( - http::Method::PUT, - &url, - &env.access_key, - &env.secret_key, - Some(body.to_string().into_bytes()), - Some("application/json"), - ) - .await?; + admin_create_user_via(AdminTransport::Signed, &env.url, &env.access_key, &env.secret_key, username, secret_key).await +} - if response.status() != reqwest::StatusCode::OK { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - return Err(format!("create user failed: {status} {body}").into()); +/// Transport used by the shared admin-API helpers: in-process SigV4 signing +/// via [`signed_request`], or the external `awscurl` binary (an independent +/// SigV4 implementation exercised by the awscurl-gated suites). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum AdminTransport { + Signed, + Awscurl, +} + +/// Execute an admin-API request against `base_url` with admin credentials over +/// the chosen transport, failing on any non-success response. +pub(crate) async fn admin_execute_at( + transport: AdminTransport, + method: http::Method, + base_url: &str, + admin_access_key: &str, + admin_secret_key: &str, + path_and_query: &str, + body: Option<&str>, +) -> Result<(), Box> { + let url = format!("{base_url}{path_and_query}"); + match transport { + AdminTransport::Signed => { + let content_type = match body { + Some(body) if !body.is_empty() => Some("application/json"), + _ => None, + }; + let response = signed_request( + method.clone(), + &url, + admin_access_key, + admin_secret_key, + body.map(|body| body.as_bytes().to_vec()), + content_type, + ) + .await?; + if !response.status().is_success() { + let status = response.status(); + let text = response.text().await.unwrap_or_default(); + return Err(format!("{method} {path_and_query} failed: {status} {text}").into()); + } + } + AdminTransport::Awscurl => { + execute_awscurl(&url, method.as_str(), body, admin_access_key, admin_secret_key).await?; + } } - Ok(()) } +/// Create a new IAM user via the admin API over the chosen transport. +pub(crate) async fn admin_create_user_via( + transport: AdminTransport, + base_url: &str, + admin_access_key: &str, + admin_secret_key: &str, + username: &str, + secret_key: &str, +) -> Result<(), Box> { + let path = format!("/rustfs/admin/v3/add-user?accessKey={username}"); + let body = serde_json::json!({"secretKey": secret_key, "status": "enabled"}).to_string(); + admin_execute_at( + transport, + http::Method::PUT, + base_url, + admin_access_key, + admin_secret_key, + &path, + Some(&body), + ) + .await +} + +/// Install a canned policy via the admin API over the chosen transport. +pub(crate) async fn admin_add_canned_policy_via( + transport: AdminTransport, + base_url: &str, + admin_access_key: &str, + admin_secret_key: &str, + policy_name: &str, + policy_json: &str, +) -> Result<(), Box> { + let path = format!("/rustfs/admin/v3/add-canned-policy?name={policy_name}"); + admin_execute_at( + transport, + http::Method::PUT, + base_url, + admin_access_key, + admin_secret_key, + &path, + Some(policy_json), + ) + .await +} + +/// Attach a canned policy to a user via the admin API over the chosen transport. +pub(crate) async fn admin_attach_user_policy_via( + transport: AdminTransport, + base_url: &str, + admin_access_key: &str, + admin_secret_key: &str, + policy_name: &str, + username: &str, +) -> Result<(), Box> { + let path = format!("/rustfs/admin/v3/set-user-or-group-policy?policyName={policy_name}&userOrGroup={username}&isGroup=false"); + // `Some("")` preserves the historical wire shape on both transports: awscurl + // keeps sending `-d ''` and the signed path attaches an empty body with no + // content type. + admin_execute_at( + transport, + http::Method::PUT, + base_url, + admin_access_key, + admin_secret_key, + &path, + Some(""), + ) + .await +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/e2e_test/src/existing_object_tag_policy_test.rs b/crates/e2e_test/src/existing_object_tag_policy_test.rs index baa798ed5..57a21f48e 100644 --- a/crates/e2e_test/src/existing_object_tag_policy_test.rs +++ b/crates/e2e_test/src/existing_object_tag_policy_test.rs @@ -16,37 +16,29 @@ //! session policy** (`Policy` parameter) via `awscurl --service sts` with explicit //! `Content-Type: application/x-www-form-urlencoded` on `POST /`. -use crate::common::{RustFSTestEnvironment, awscurl_delete, awscurl_post_sts_form_urlencoded, awscurl_put, init_logging}; -use aws_sdk_s3::config::{Credentials, Region}; +use crate::common::{ + AdminTransport, RustFSTestEnvironment, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, + awscurl_delete, awscurl_post_sts_form_urlencoded, build_test_s3_config, init_logging, +}; +use aws_sdk_s3::Client; use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{Delete, ObjectIdentifier, Tag, Tagging}; -use aws_sdk_s3::{Client, Config}; use tracing::info; use uuid::Uuid; fn user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client { - let credentials = Credentials::new(access_key, secret_key, None, None, "e2e-existing-tag"); - let config = Config::builder() - .credentials_provider(credentials) - .region(Region::new("us-east-1")) - .endpoint_url(&env.url) - .force_path_style(true) - .behavior_version_latest() - .build(); - Client::from_conf(config) + env.create_s3_client_with_credentials(access_key, secret_key) } fn sts_session_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str, session_token: &str) -> Client { - let credentials = Credentials::new(access_key, secret_key, Some(session_token.into()), None, "e2e-sts-session"); - let config = Config::builder() - .credentials_provider(credentials) - .region(Region::new("us-east-1")) - .endpoint_url(&env.url) - .force_path_style(true) - .behavior_version_latest() - .build(); - Client::from_conf(config) + Client::from_conf(build_test_s3_config( + &env.url, + access_key, + secret_key, + Some(session_token), + "e2e-sts-session", + )) } fn extract_xml_tag(xml: &str, tag: &str) -> Option { @@ -77,15 +69,16 @@ async fn assume_role_with_session_policy( parse_assume_role_credentials(&xml) } +// This suite deliberately drives the admin API through the external `awscurl` +// binary (an independent SigV4 implementation), so the wrappers below pin +// `AdminTransport::Awscurl`. + async fn admin_create_user( env: &RustFSTestEnvironment, username: &str, password: &str, ) -> Result<(), Box> { - let body = serde_json::json!({ "secretKey": password, "status": "enabled" }).to_string(); - let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username); - awscurl_put(&url, &body, &env.access_key, &env.secret_key).await?; - Ok(()) + admin_create_user_via(AdminTransport::Awscurl, &env.url, &env.access_key, &env.secret_key, username, password).await } async fn admin_add_canned_policy( @@ -93,9 +86,15 @@ async fn admin_add_canned_policy( policy_name: &str, policy_json: &str, ) -> Result<(), Box> { - let url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name); - awscurl_put(&url, policy_json, &env.access_key, &env.secret_key).await?; - Ok(()) + admin_add_canned_policy_via( + AdminTransport::Awscurl, + &env.url, + &env.access_key, + &env.secret_key, + policy_name, + policy_json, + ) + .await } async fn admin_attach_policy_to_user( @@ -103,12 +102,7 @@ async fn admin_attach_policy_to_user( policy_name: &str, username: &str, ) -> Result<(), Box> { - let url = format!( - "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false", - env.url, policy_name, username - ); - awscurl_put(&url, "", &env.access_key, &env.secret_key).await?; - Ok(()) + admin_attach_user_policy_via(AdminTransport::Awscurl, &env.url, &env.access_key, &env.secret_key, policy_name, username).await } async fn admin_remove_user(env: &RustFSTestEnvironment, username: &str) { diff --git a/crates/e2e_test/src/group_delete_test.rs b/crates/e2e_test/src/group_delete_test.rs index d6aaec32f..f8d1956e2 100644 --- a/crates/e2e_test/src/group_delete_test.rs +++ b/crates/e2e_test/src/group_delete_test.rs @@ -15,20 +15,11 @@ //! E2E tests for group management (fixes #2028). use crate::common::{RustFSTestEnvironment, admin_ok, admin_request, init_logging}; -use aws_sdk_s3::config::{Credentials, Region}; -use aws_sdk_s3::{Client, Config}; +use aws_sdk_s3::Client; use tracing::info; fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client { - let credentials = Credentials::new(access_key, secret_key, None, None, "e2e-group-test"); - let config = Config::builder() - .credentials_provider(credentials) - .region(Region::new("us-east-1")) - .endpoint_url(&env.url) - .force_path_style(true) - .behavior_version_latest() - .build(); - Client::from_conf(config) + env.create_s3_client_with_credentials(access_key, secret_key) } #[tokio::test(flavor = "multi_thread")] diff --git a/crates/e2e_test/src/inline_fast_path_cluster_test.rs b/crates/e2e_test/src/inline_fast_path_cluster_test.rs index 5583a1b84..e618d6cf9 100644 --- a/crates/e2e_test/src/inline_fast_path_cluster_test.rs +++ b/crates/e2e_test/src/inline_fast_path_cluster_test.rs @@ -21,7 +21,7 @@ //! One S3 GET can select readers on multiple EC nodes, so the counter tracks //! distributed reader selection rather than HTTP request count. -use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, local_http_client}; +use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging}; use aws_sdk_s3::Client; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{ @@ -30,7 +30,7 @@ use aws_sdk_s3::types::{ }; use bytes::Bytes; use flate2::read::GzDecoder; -use http::header::{CONTENT_ENCODING, HOST}; +use http::header::CONTENT_ENCODING; use http::{Method, Request, Response, StatusCode}; use http_body_util::{BodyExt, Full}; use hyper::body::Incoming; @@ -42,9 +42,6 @@ use opentelemetry_proto::tonic::metrics::v1::{ Metric, NumberDataPoint, ResourceMetrics, ScopeMetrics, Sum, metric, number_data_point, }; use prost::Message; -use rustfs_signer::constants::UNSIGNED_PAYLOAD; -use rustfs_signer::sign_v4; -use s3s::Body; use std::collections::BTreeMap; use std::convert::Infallible; use std::error::Error; @@ -1262,6 +1259,8 @@ async fn put_two_part_multipart(client: &Client, bucket: &str, key: &str) -> Tes Ok((body, part2, complete.e_tag().map(str::to_owned))) } +/// Thin wrapper over [`crate::common::admin_request`], kept local so the call +/// sites below keep their `Option<&str>` body shape. async fn signed_admin_request( base_url: &str, method: Method, @@ -1270,30 +1269,7 @@ async fn signed_admin_request( access_key: &str, secret_key: &str, ) -> TestResult<(reqwest::StatusCode, String)> { - let url = format!("{base_url}{path}"); - let uri = url.parse::()?; - let authority = uri.authority().ok_or("request URL missing authority")?.to_string(); - let body_bytes = body.map(|value| value.as_bytes().to_vec()).unwrap_or_default(); - - let request = http::Request::builder() - .method(method.clone()) - .uri(uri) - .header(HOST, authority) - .header("x-amz-content-sha256", UNSIGNED_PAYLOAD); - let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1"); - - let client = local_http_client(); - let mut request_builder = client.request(method, url.as_str()); - for (name, value) in signed.headers() { - request_builder = request_builder.header(name, value); - } - if !body_bytes.is_empty() { - request_builder = request_builder.body(body_bytes); - } - let response = request_builder.send().await?; - let status = response.status(); - let text = response.text().await?; - Ok((status, text)) + crate::common::admin_request(base_url, method, path, body.map(str::to_string), access_key, secret_key).await } fn unique_tier_name() -> String { diff --git a/crates/e2e_test/src/notification_webhook_test.rs b/crates/e2e_test/src/notification_webhook_test.rs index 1eec17343..1a115cc5e 100644 --- a/crates/e2e_test/src/notification_webhook_test.rs +++ b/crates/e2e_test/src/notification_webhook_test.rs @@ -38,14 +38,10 @@ use aws_sdk_s3::types::{ NotificationConfiguration, NotificationConfigurationFilter, ObjectIdentifier, QueueConfiguration, S3KeyFilter, VersioningConfiguration, }; -use http::header::{CONTENT_TYPE, HOST}; use local_ip_address::local_ip; use reqwest::StatusCode; -use rustfs_signer::constants::UNSIGNED_PAYLOAD; -use rustfs_signer::sign_v4; use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS; use rustfs_utils::http::headers::{AMZ_REQUEST_ID, REQUEST_ID_HEADER}; -use s3s::Body; use serde_json::Value; use std::error::Error; use std::io::Cursor; @@ -415,42 +411,16 @@ async fn collect_until( // Admin target configuration (signed admin HTTP) // --------------------------------------------------------------------------- +/// Thin wrapper over [`crate::common::signed_request`] with this suite's +/// root credentials; a `Some` body is always JSON here. async fn signed_admin_request( env: &RustFSTestEnvironment, method: http::Method, url: &str, body: Option>, ) -> Result { - let uri = url.parse::()?; - let authority = uri.authority().ok_or("admin URL missing authority")?.to_string(); - let mut builder = http::Request::builder() - .method(method.clone()) - .uri(uri) - .header(HOST, authority) - .header("x-amz-content-sha256", UNSIGNED_PAYLOAD); - if body.is_some() { - builder = builder.header(CONTENT_TYPE, "application/json"); - } - - let content_len = body.as_ref().map(|b| b.len() as i64).unwrap_or_default(); - let signed = sign_v4( - builder.body(Body::empty())?, - content_len, - &env.access_key, - &env.secret_key, - "", - "us-east-1", - ); - - let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?; - let mut request = crate::common::local_http_client().request(reqwest_method, url); - for (name, value) in signed.headers() { - request = request.header(name, value); - } - if let Some(body) = body { - request = request.body(body); - } - Ok(request.send().await?) + let content_type = body.is_some().then_some("application/json"); + crate::common::signed_request(method, url, &env.access_key, &env.secret_key, body, content_type).await } async fn enable_notify_module(env: &RustFSTestEnvironment) -> TestResult { diff --git a/crates/e2e_test/src/policy/policy_variables_test.rs b/crates/e2e_test/src/policy/policy_variables_test.rs index 0dd096046..5ad60263d 100644 --- a/crates/e2e_test/src/policy/policy_variables_test.rs +++ b/crates/e2e_test/src/policy/policy_variables_test.rs @@ -15,28 +15,24 @@ //! Tests for AWS IAM policy variables with single-value, multi-value, and nested scenarios use crate::common::{ - RustFSTestEnvironment, awscurl_delete, awscurl_put, build_test_s3_config, build_test_sts_client, init_logging, + AdminTransport, RustFSTestEnvironment, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, + awscurl_delete, awscurl_put, build_test_s3_config, build_test_sts_client, init_logging, }; use aws_sdk_s3::Client; use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::primitives::ByteStream; use tracing::info; -/// Helper function to create a regular user with given credentials +/// Helper function to create a regular user with given credentials. +/// +/// This suite deliberately drives the admin API through the external `awscurl` +/// binary, so the shared helpers are pinned to `AdminTransport::Awscurl`. async fn create_user( env: &RustFSTestEnvironment, username: &str, password: &str, ) -> Result<(), Box> { - let create_user_body = serde_json::json!({ - "secretKey": password, - "status": "enabled" - }) - .to_string(); - - let create_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username); - awscurl_put(&create_user_url, &create_user_body, &env.access_key, &env.secret_key).await?; - Ok(()) + admin_create_user_via(AdminTransport::Awscurl, &env.url, &env.access_key, &env.secret_key, username, password).await } /// Helper function to create and attach a policy @@ -46,18 +42,17 @@ async fn create_and_attach_policy( username: &str, policy_document: serde_json::Value, ) -> Result<(), Box> { - let policy_string = policy_document.to_string(); - - // Create policy - let add_policy_url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name); - awscurl_put(&add_policy_url, &policy_string, &env.access_key, &env.secret_key).await?; - - // Attach policy to user - let attach_policy_url = format!( - "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false", - env.url, policy_name, username - ); - awscurl_put(&attach_policy_url, "", &env.access_key, &env.secret_key).await?; + admin_add_canned_policy_via( + AdminTransport::Awscurl, + &env.url, + &env.access_key, + &env.secret_key, + policy_name, + &policy_document.to_string(), + ) + .await?; + admin_attach_user_policy_via(AdminTransport::Awscurl, &env.url, &env.access_key, &env.secret_key, policy_name, username) + .await?; Ok(()) } diff --git a/crates/e2e_test/src/protocols/webdav_core.rs b/crates/e2e_test/src/protocols/webdav_core.rs index 39136987d..aaf730f36 100644 --- a/crates/e2e_test/src/protocols/webdav_core.rs +++ b/crates/e2e_test/src/protocols/webdav_core.rs @@ -31,15 +31,11 @@ //! //! Advisory: -use crate::common::local_http_client; use crate::common::rustfs_binary_path_with_features; +use crate::common::{AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via}; use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment}; use anyhow::Result; -use http::header::{CONTENT_TYPE, HOST}; use reqwest::Client; -use rustfs_signer::constants::UNSIGNED_PAYLOAD; -use rustfs_signer::sign_v4; -use s3s::Body; use tokio::process::Command; use tracing::info; @@ -67,92 +63,43 @@ fn basic_auth_header_for(access_key: &str, secret_key: &str) -> String { format!("Basic {}", encoded) } -async fn signed_admin_request( - method: http::Method, - url: &str, - body: Option>, - content_type: Option<&str>, -) -> Result { - let uri = url.parse::()?; - let authority = uri - .authority() - .ok_or_else(|| anyhow::anyhow!("request URL missing authority"))? - .to_string(); - let mut request = http::Request::builder().method(method.clone()).uri(uri); - request = request.header(HOST, authority); - request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD); - if let Some(content_type) = content_type { - request = request.header(CONTENT_TYPE, content_type); - } - - let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default(); - let signed = sign_v4( - request.body(Body::empty())?, - content_len, +async fn admin_create_user(base_url: &str, username: &str, secret_key: &str) -> Result<()> { + admin_create_user_via( + AdminTransport::Signed, + base_url, DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, - "", - "us-east-1", - ); - - let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?; - let mut request_builder = local_http_client().request(reqwest_method, url); - for (name, value) in signed.headers() { - request_builder = request_builder.header(name, value); - } - if let Some(body) = body { - request_builder = request_builder.body(body); - } - - Ok(request_builder.send().await?) -} - -async fn admin_create_user(base_url: &str, username: &str, secret_key: &str) -> Result<()> { - let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", base_url, username); - let body = serde_json::json!({ - "secretKey": secret_key, - "status": "enabled" - }); - let response = - signed_admin_request(http::Method::PUT, &url, Some(body.to_string().into_bytes()), Some("application/json")).await?; - - if response.status() != reqwest::StatusCode::OK { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - anyhow::bail!("create user failed: {status} {body}"); - } - - Ok(()) + username, + secret_key, + ) + .await + .map_err(|e| anyhow::anyhow!(e)) } async fn admin_add_canned_policy(base_url: &str, policy_name: &str, policy: &serde_json::Value) -> Result<()> { - let url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", base_url, policy_name); - let response = - signed_admin_request(http::Method::PUT, &url, Some(policy.to_string().into_bytes()), Some("application/json")).await?; - - if response.status() != reqwest::StatusCode::OK { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - anyhow::bail!("add canned policy failed: {status} {body}"); - } - - Ok(()) + admin_add_canned_policy_via( + AdminTransport::Signed, + base_url, + DEFAULT_ACCESS_KEY, + DEFAULT_SECRET_KEY, + policy_name, + &policy.to_string(), + ) + .await + .map_err(|e| anyhow::anyhow!(e)) } async fn admin_attach_policy_to_user(base_url: &str, policy_name: &str, username: &str) -> Result<()> { - let url = format!( - "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false", - base_url, policy_name, username - ); - let response = signed_admin_request(http::Method::PUT, &url, Some(Vec::new()), None).await?; - - if response.status() != reqwest::StatusCode::OK { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - anyhow::bail!("attach policy failed: {status} {body}"); - } - - Ok(()) + admin_attach_user_policy_via( + AdminTransport::Signed, + base_url, + DEFAULT_ACCESS_KEY, + DEFAULT_SECRET_KEY, + policy_name, + username, + ) + .await + .map_err(|e| anyhow::anyhow!(e)) } /// Test WebDAV: MKCOL (create bucket), PUT, GET, DELETE, PROPFIND operations diff --git a/crates/e2e_test/src/reliant/tiering.rs b/crates/e2e_test/src/reliant/tiering.rs index 401e3752c..0557dfac2 100644 --- a/crates/e2e_test/src/reliant/tiering.rs +++ b/crates/e2e_test/src/reliant/tiering.rs @@ -46,7 +46,7 @@ //! retry serves the object locally until expiry, and expiry leaves the //! remote object available for a second restore. -use crate::common::{RustFSTestEnvironment, local_http_client}; +use crate::common::RustFSTestEnvironment; use aws_sdk_s3::Client; use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::primitives::ByteStream; @@ -56,10 +56,6 @@ use aws_sdk_s3::types::{ VersioningConfiguration, }; use http::Method; -use http::header::HOST; -use rustfs_signer::constants::UNSIGNED_PAYLOAD; -use rustfs_signer::sign_v4; -use s3s::Body; use serde::Deserialize; use std::time::{Duration as StdDuration, Instant}; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; @@ -131,9 +127,8 @@ fn payload() -> Vec { /// Sign and send an admin request in-process (no `awscurl`). /// -/// Mirrors the shared admin-API e2e pattern: the SigV4 signature is computed -/// over `UNSIGNED_PAYLOAD`, so the JSON body rides on the wire without being -/// pre-hashed. Returns the response status and body text. +/// Thin wrapper over [`crate::common::admin_request`], kept local so the call +/// sites below keep their `Option<&str>` body shape. async fn signed_admin_request( base_url: &str, method: Method, @@ -142,30 +137,7 @@ async fn signed_admin_request( access_key: &str, secret_key: &str, ) -> Result<(reqwest::StatusCode, String), Box> { - let url = format!("{base_url}{path}"); - let uri = url.parse::()?; - let authority = uri.authority().ok_or("request URL missing authority")?.to_string(); - let body_bytes = body.map(|b| b.as_bytes().to_vec()).unwrap_or_default(); - - let request = http::Request::builder() - .method(method.clone()) - .uri(uri) - .header(HOST, authority) - .header("x-amz-content-sha256", UNSIGNED_PAYLOAD); - let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1"); - - let client = local_http_client(); - let mut request_builder = client.request(method, url.as_str()); - for (name, value) in signed.headers() { - request_builder = request_builder.header(name, value); - } - if !body_bytes.is_empty() { - request_builder = request_builder.body(body_bytes); - } - let response = request_builder.send().await?; - let status = response.status(); - let text = response.text().await?; - Ok((status, text)) + crate::common::admin_request(base_url, method, path, body.map(str::to_string), access_key, secret_key).await } /// Wire `hot` -> `cold` as a `TierType::RustFS` remote tier via `AddTier`. diff --git a/crates/e2e_test/src/replication_extension_test.rs b/crates/e2e_test/src/replication_extension_test.rs index 98b9a5d8c..20927a55f 100644 --- a/crates/e2e_test/src/replication_extension_test.rs +++ b/crates/e2e_test/src/replication_extension_test.rs @@ -13,8 +13,9 @@ // limitations under the License. use crate::common::{ - RustFSTestEnvironment, admin_create_user, awscurl_post_sts_form_urlencoded, init_logging, local_http_client, - replication_fast_env, rustfs_binary_path, signed_request, signed_request_with_client, signed_request_with_session_token, + AdminTransport, RustFSTestEnvironment, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user, + awscurl_post_sts_form_urlencoded, init_logging, local_http_client, replication_fast_env, rustfs_binary_path, signed_request, + signed_request_with_client, signed_request_with_session_token, }; use crate::fake_s3_target::{ FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, FaultAction as FakeTargetFault, Operation as FakeTargetOperation, @@ -25,7 +26,7 @@ use crate::kms::common::{ sse_customer_key_md5_base64, }; use crate::storage_api::replication_extension::BucketTargetSys; -use aws_sdk_s3::config::{Credentials, Region}; +use aws_sdk_s3::Client; use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput; use aws_sdk_s3::primitives::ByteStream; @@ -33,7 +34,6 @@ use aws_sdk_s3::types::{ BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption, VersioningConfiguration, }; -use aws_sdk_s3::{Client, Config}; use base64_simd::STANDARD as BASE64_STANDARD; use bytes::Bytes; use flate2::read::GzDecoder; @@ -895,15 +895,7 @@ async fn wait_for_replicated_object_over_https( } fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client { - let credentials = Credentials::new(access_key, secret_key, None, None, "e2e-site-replication"); - let config = Config::builder() - .credentials_provider(credentials) - .region(Region::new("us-east-1")) - .endpoint_url(&env.url) - .force_path_style(true) - .behavior_version_latest() - .build(); - Client::from_conf(config) + env.create_s3_client_with_credentials(access_key, secret_key) } async fn admin_add_canned_policy( @@ -911,24 +903,15 @@ async fn admin_add_canned_policy( policy_name: &str, policy: &serde_json::Value, ) -> Result<(), Box> { - let url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name); - let response = signed_request( - http::Method::PUT, - &url, + admin_add_canned_policy_via( + AdminTransport::Signed, + &env.url, &env.access_key, &env.secret_key, - Some(policy.to_string().into_bytes()), - Some("application/json"), + policy_name, + &policy.to_string(), ) - .await?; - - if response.status() != StatusCode::OK { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - return Err(format!("add canned policy failed: {status} {body}").into()); - } - - Ok(()) + .await } async fn admin_attach_policy_to_user( @@ -936,19 +919,7 @@ async fn admin_attach_policy_to_user( policy_name: &str, username: &str, ) -> Result<(), Box> { - let url = format!( - "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false", - env.url, policy_name, username - ); - let response = signed_request(http::Method::PUT, &url, &env.access_key, &env.secret_key, Some(Vec::new()), None).await?; - - if response.status() != StatusCode::OK { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - return Err(format!("attach policy to user failed: {status} {body}").into()); - } - - Ok(()) + admin_attach_user_policy_via(AdminTransport::Signed, &env.url, &env.access_key, &env.secret_key, policy_name, username).await } async fn admin_update_group_members(