gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-31 09:18:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(e2e): finish the helper consolidation onto common.rs (#6766)

Browse Source 
- common.rs gains an AdminTransport knob (Signed | Awscurl) with admin_execute_at plus three family wrappers: admin_create_user_via, admin_add_canned_policy_via, admin_attach_user_policy_via; the existing admin_create_user now delegates over the Signed transport.
- Deleted the four signed admin request clones in admin_mfa_test, admin_auth_test, reliant/tiering, and inline_fast_path_cluster_test; each keeps a thin local wrapper over common::admin_request so call sites keep their Option<&str> body shape.
- Deduped the notification_webhook signer onto common::signed_request and the webdav_core signer plus its three admin helpers onto the shared _via helpers.
- Consolidated the S3-client-with-credentials builders: admin_auth s3_client_with, existing_object_tag user_client/sts_session_client, bucket_policy_check create_user_client, and the create_user_s3_client copies in group_delete_test and replication_extension_test now delegate to create_s3_client_with_credentials / build_test_s3_config; replication_extension admin_add_canned_policy and admin_attach_policy_to_user route through the _via helpers on the Signed transport.
- The awscurl-gated suites (existing_object_tag_policy, bucket_policy_check, policy/policy_variables) keep going through the external awscurl binary via AdminTransport::Awscurl, preserving their wire behavior.

Part of rustfs/backlog#1846 (cluster 2).
This commit is contained in:
Zhengchao An
2026-08-28 08:12:45 +08:00
committed by GitHub
parent 3c89c71f66
commit 22741603f5
12 changed files with 238 additions and 407 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/e2e_test/src/admin_auth_test.rs
+9 -53
View File
@@ -31,14 +31,9 @@
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client, rustfs_binary_path}; use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path};
use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::Client;
use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::{Client, Config};
use http::header::HOST;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use s3s::Body;
use std::error::Error; use std::error::Error;
use std::io::Read; use std::io::Read;
use std::process::{Command, Stdio}; use std::process::{Command, Stdio};
@@ -87,10 +82,10 @@ mod tests {
} }
/// Send a SigV4-signed request to `path` (optionally with a JSON `body`) and /// Send a SigV4-signed request to `path` (optionally with a JSON `body`) and
/// return `(status, body)`. Uses the `UNSIGNED_PAYLOAD` content hash so a /// return `(status, body)`.
/// request body can be attached without the caller pre-hashing it — the ///
/// server verifies the signature against the same sentinel, exactly as the /// Thin wrapper over [`crate::common::admin_request`], kept local so the
/// AWS SDKs / MinIO client do for streaming/unsigned payloads. /// call sites below keep their `Option<&str>` body shape.
async fn signed_request( async fn signed_request(
base_url: &str, base_url: &str,
method: http::Method, method: http::Method,
@@ -99,47 +94,13 @@ mod tests {
access_key: &str, access_key: &str,
secret_key: &str, secret_key: &str,
) -> Result<(reqwest::StatusCode, String), Box<dyn Error + Send + Sync>> { ) -> Result<(reqwest::StatusCode, String), Box<dyn Error + Send + Sync>> {
let url = format!("{base_url}{path}"); crate::common::admin_request(base_url, method, path, body.map(str::to_string), access_key, secret_key).await
let uri = url.parse::<http::Uri>()?;
let authority = uri.authority().ok_or("missing authority")?.to_string();
let body_bytes = body.map(|b| b.as_bytes().to_vec()).unwrap_or_default();
// The signature is computed over `UNSIGNED_PAYLOAD`, so the body bytes do
// not participate in the SigV4 hash — sign over an empty body and attach
// the real payload to the wire request below.
let request = http::Request::builder()
.method(method.clone())
.uri(uri)
.header(HOST, authority)
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
let client = local_http_client();
let mut rb = client.request(method, url.as_str());
for (name, value) in signed.headers() {
rb = rb.header(name, value);
}
if !body_bytes.is_empty() {
rb = rb.body(body_bytes);
}
let resp = rb.send().await?;
let status = resp.status();
let text = resp.text().await?;
Ok((status, text))
} }
/// Build an S3 client bound to explicit credentials (used to exercise the S3 /// Build an S3 client bound to explicit credentials (used to exercise the S3
/// data plane with rotated / stale root credentials). /// data plane with rotated / stale root credentials).
fn s3_client_with(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client { fn s3_client_with(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
let credentials = Credentials::new(access_key, secret_key, None, None, "sec4-admin-auth"); env.create_s3_client_with_credentials(access_key, secret_key)
let config = Config::builder()
.credentials_provider(credentials)
.region(Region::new("us-east-1"))
.endpoint_url(&env.url)
.force_path_style(true)
.behavior_version_latest()
.build();
Client::from_conf(config)
} }
/// Create a non-admin IAM user via the admin `add-user` API using the root /// Create a non-admin IAM user via the admin `add-user` API using the root
@@ -151,12 +112,7 @@ mod tests {
access_key: &str, access_key: &str,
secret_key: &str, secret_key: &str,
) -> Result<(), Box<dyn Error + Send + Sync>> { ) -> Result<(), Box<dyn Error + Send + Sync>> {
let path = format!("/rustfs/admin/v3/add-user?accessKey={access_key}"); crate::common::admin_create_user(env, access_key, secret_key).await
let body = serde_json::json!({ "secretKey": secret_key, "status": "enabled" }).to_string();
let (status, resp) =
signed_request(&env.url, http::Method::PUT, &path, Some(&body), &env.access_key, &env.secret_key).await?;
assert!(status.is_success(), "add-user should succeed (status={status}, body={resp})");
Ok(())
} }
/// A fully authenticated but non-admin credential must be rejected with /// A fully authenticated but non-admin credential must be rejected with
crates/e2e_test/src/admin_mfa_test.rs
+3 -26
View File
@@ -59,8 +59,8 @@ mod tests {
/// One signed admin request, returning the status and the raw body. /// One signed admin request, returning the status and the raw body.
/// ///
/// Signs with `UNSIGNED_PAYLOAD` so the body does not participate in the /// Thin wrapper over [`crate::common::admin_request`], kept local so the
/// hash, matching how the other admin e2e tests drive these routes. /// call sites below keep their `Option<&str>` body shape.
async fn signed_request( async fn signed_request(
base_url: &str, base_url: &str,
method: http::Method, method: http::Method,
@@ -69,30 +69,7 @@ mod tests {
access_key: &str, access_key: &str,
secret_key: &str, secret_key: &str,
) -> Result<(reqwest::StatusCode, String), Box<dyn Error + Send + Sync>> { ) -> Result<(reqwest::StatusCode, String), Box<dyn Error + Send + Sync>> {
let url = format!("{base_url}{path}"); crate::common::admin_request(base_url, method, path, body.map(str::to_string), access_key, secret_key).await
let uri = url.parse::<http::Uri>()?;
let authority = uri.authority().ok_or("missing authority")?.to_string();
let body_bytes = body.map(|b| b.as_bytes().to_vec()).unwrap_or_default();
let request = http::Request::builder()
.method(method.clone())
.uri(uri)
.header(HOST, authority)
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
let client = local_http_client();
let mut builder = client.request(method, url.as_str());
for (name, value) in signed.headers() {
builder = builder.header(name, value);
}
if !body_bytes.is_empty() {
builder = builder.body(body_bytes);
}
let response = builder.send().await?;
let status = response.status();
let text = response.text().await?;
Ok((status, text))
} }
/// A SigV4-signed `AssumeRole` form POST, optionally carrying a second factor. /// A SigV4-signed `AssumeRole` form POST, optionally carrying a second factor.
crates/e2e_test/src/bucket_policy_check_test.rs
+6 -22
View File
@@ -15,39 +15,23 @@
//! Regression test for Issue #1423 //! Regression test for Issue #1423
//! Verifies that Bucket Policies are honored for Authenticated Users. //! Verifies that Bucket Policies are honored for Authenticated Users.
use crate::common::{RustFSTestEnvironment, init_logging}; use crate::common::{AdminTransport, RustFSTestEnvironment, admin_create_user_via, init_logging};
use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::Client;
use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::{Client, Config};
use tracing::info; use tracing::info;
/// This suite deliberately drives the admin API through the external `awscurl`
/// binary, so user creation pins `AdminTransport::Awscurl`.
async fn create_user( async fn create_user(
env: &RustFSTestEnvironment, env: &RustFSTestEnvironment,
username: &str, username: &str,
password: &str, password: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> { ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let create_user_body = serde_json::json!({ admin_create_user_via(AdminTransport::Awscurl, &env.url, &env.access_key, &env.secret_key, username, password).await
"secretKey": password,
"status": "enabled"
})
.to_string();
let create_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
crate::common::awscurl_put(&create_user_url, &create_user_body, &env.access_key, &env.secret_key).await?;
Ok(())
} }
fn create_user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client { fn create_user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
let credentials = Credentials::new(access_key, secret_key, None, None, "test-user"); env.create_s3_client_with_credentials(access_key, secret_key)
let config = Config::builder()
.credentials_provider(credentials)
.region(Region::new("us-east-1"))
.endpoint_url(&env.url)
.force_path_style(true)
.behavior_version_latest()
.build();
Client::from_conf(config)
} }
#[tokio::test] #[tokio::test]
crates/e2e_test/src/common.rs
+117 -19
View File
@@ -1744,30 +1744,128 @@ pub(crate) async fn admin_create_user(
username: &str, username: &str,
secret_key: &str, secret_key: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> { ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username); admin_create_user_via(AdminTransport::Signed, &env.url, &env.access_key, &env.secret_key, username, secret_key).await
let body = serde_json::json!({ }
"secretKey": secret_key,
"status": "enabled"
});
let response = signed_request(
http::Method::PUT,
&url,
&env.access_key,
&env.secret_key,
Some(body.to_string().into_bytes()),
Some("application/json"),
)
.await?;
if response.status() != reqwest::StatusCode::OK { /// Transport used by the shared admin-API helpers: in-process SigV4 signing
let status = response.status(); /// via [`signed_request`], or the external `awscurl` binary (an independent
let body = response.text().await.unwrap_or_default(); /// SigV4 implementation exercised by the awscurl-gated suites).
return Err(format!("create user failed: {status} {body}").into()); #[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum AdminTransport {
Signed,
Awscurl,
}
/// Execute an admin-API request against `base_url` with admin credentials over
/// the chosen transport, failing on any non-success response.
pub(crate) async fn admin_execute_at(
transport: AdminTransport,
method: http::Method,
base_url: &str,
admin_access_key: &str,
admin_secret_key: &str,
path_and_query: &str,
body: Option<&str>,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let url = format!("{base_url}{path_and_query}");
match transport {
AdminTransport::Signed => {
let content_type = match body {
Some(body) if !body.is_empty() => Some("application/json"),
_ => None,
};
let response = signed_request(
method.clone(),
&url,
admin_access_key,
admin_secret_key,
body.map(|body| body.as_bytes().to_vec()),
content_type,
)
.await?;
if !response.status().is_success() {
let status = response.status();
let text = response.text().await.unwrap_or_default();
return Err(format!("{method} {path_and_query} failed: {status} {text}").into());
}
}
AdminTransport::Awscurl => {
execute_awscurl(&url, method.as_str(), body, admin_access_key, admin_secret_key).await?;
}
} }
Ok(()) Ok(())
} }
/// Create a new IAM user via the admin API over the chosen transport.
pub(crate) async fn admin_create_user_via(
transport: AdminTransport,
base_url: &str,
admin_access_key: &str,
admin_secret_key: &str,
username: &str,
secret_key: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let path = format!("/rustfs/admin/v3/add-user?accessKey={username}");
let body = serde_json::json!({"secretKey": secret_key, "status": "enabled"}).to_string();
admin_execute_at(
transport,
http::Method::PUT,
base_url,
admin_access_key,
admin_secret_key,
&path,
Some(&body),
)
.await
}
/// Install a canned policy via the admin API over the chosen transport.
pub(crate) async fn admin_add_canned_policy_via(
transport: AdminTransport,
base_url: &str,
admin_access_key: &str,
admin_secret_key: &str,
policy_name: &str,
policy_json: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let path = format!("/rustfs/admin/v3/add-canned-policy?name={policy_name}");
admin_execute_at(
transport,
http::Method::PUT,
base_url,
admin_access_key,
admin_secret_key,
&path,
Some(policy_json),
)
.await
}
/// Attach a canned policy to a user via the admin API over the chosen transport.
pub(crate) async fn admin_attach_user_policy_via(
transport: AdminTransport,
base_url: &str,
admin_access_key: &str,
admin_secret_key: &str,
policy_name: &str,
username: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let path = format!("/rustfs/admin/v3/set-user-or-group-policy?policyName={policy_name}&userOrGroup={username}&isGroup=false");
// `Some("")` preserves the historical wire shape on both transports: awscurl
// keeps sending `-d ''` and the signed path attaches an empty body with no
// content type.
admin_execute_at(
transport,
http::Method::PUT,
base_url,
admin_access_key,
admin_secret_key,
&path,
Some(""),
)
.await
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
crates/e2e_test/src/existing_object_tag_policy_test.rs
+28 -34
View File
@@ -16,37 +16,29 @@
//! session policy** (`Policy` parameter) via `awscurl --service sts` with explicit //! session policy** (`Policy` parameter) via `awscurl --service sts` with explicit
//! `Content-Type: application/x-www-form-urlencoded` on `POST /`. //! `Content-Type: application/x-www-form-urlencoded` on `POST /`.
use crate::common::{RustFSTestEnvironment, awscurl_delete, awscurl_post_sts_form_urlencoded, awscurl_put, init_logging}; use crate::common::{
use aws_sdk_s3::config::{Credentials, Region}; AdminTransport, RustFSTestEnvironment, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via,
awscurl_delete, awscurl_post_sts_form_urlencoded, build_test_s3_config, init_logging,
};
use aws_sdk_s3::Client;
use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{Delete, ObjectIdentifier, Tag, Tagging}; use aws_sdk_s3::types::{Delete, ObjectIdentifier, Tag, Tagging};
use aws_sdk_s3::{Client, Config};
use tracing::info; use tracing::info;
use uuid::Uuid; use uuid::Uuid;
fn user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client { fn user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
let credentials = Credentials::new(access_key, secret_key, None, None, "e2e-existing-tag"); env.create_s3_client_with_credentials(access_key, secret_key)
let config = Config::builder()
.credentials_provider(credentials)
.region(Region::new("us-east-1"))
.endpoint_url(&env.url)
.force_path_style(true)
.behavior_version_latest()
.build();
Client::from_conf(config)
} }
fn sts_session_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str, session_token: &str) -> Client { fn sts_session_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str, session_token: &str) -> Client {
let credentials = Credentials::new(access_key, secret_key, Some(session_token.into()), None, "e2e-sts-session"); Client::from_conf(build_test_s3_config(
let config = Config::builder() &env.url,
.credentials_provider(credentials) access_key,
.region(Region::new("us-east-1")) secret_key,
.endpoint_url(&env.url) Some(session_token),
.force_path_style(true) "e2e-sts-session",
.behavior_version_latest() ))
.build();
Client::from_conf(config)
} }
fn extract_xml_tag(xml: &str, tag: &str) -> Option<String> { fn extract_xml_tag(xml: &str, tag: &str) -> Option<String> {
@@ -77,15 +69,16 @@ async fn assume_role_with_session_policy(
parse_assume_role_credentials(&xml) parse_assume_role_credentials(&xml)
} }
// This suite deliberately drives the admin API through the external `awscurl`
// binary (an independent SigV4 implementation), so the wrappers below pin
// `AdminTransport::Awscurl`.
async fn admin_create_user( async fn admin_create_user(
env: &RustFSTestEnvironment, env: &RustFSTestEnvironment,
username: &str, username: &str,
password: &str, password: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> { ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let body = serde_json::json!({ "secretKey": password, "status": "enabled" }).to_string(); admin_create_user_via(AdminTransport::Awscurl, &env.url, &env.access_key, &env.secret_key, username, password).await
let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
awscurl_put(&url, &body, &env.access_key, &env.secret_key).await?;
Ok(())
} }
async fn admin_add_canned_policy( async fn admin_add_canned_policy(
@@ -93,9 +86,15 @@ async fn admin_add_canned_policy(
policy_name: &str, policy_name: &str,
policy_json: &str, policy_json: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> { ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name); admin_add_canned_policy_via(
awscurl_put(&url, policy_json, &env.access_key, &env.secret_key).await?; AdminTransport::Awscurl,
Ok(()) &env.url,
&env.access_key,
&env.secret_key,
policy_name,
policy_json,
)
.await
} }
async fn admin_attach_policy_to_user( async fn admin_attach_policy_to_user(
@@ -103,12 +102,7 @@ async fn admin_attach_policy_to_user(
policy_name: &str, policy_name: &str,
username: &str, username: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> { ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let url = format!( admin_attach_user_policy_via(AdminTransport::Awscurl, &env.url, &env.access_key, &env.secret_key, policy_name, username).await
"{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false",
env.url, policy_name, username
);
awscurl_put(&url, "", &env.access_key, &env.secret_key).await?;
Ok(())
} }
async fn admin_remove_user(env: &RustFSTestEnvironment, username: &str) { async fn admin_remove_user(env: &RustFSTestEnvironment, username: &str) {
crates/e2e_test/src/group_delete_test.rs
+2 -11
View File
@@ -15,20 +15,11 @@
//! E2E tests for group management (fixes #2028). //! E2E tests for group management (fixes #2028).
use crate::common::{RustFSTestEnvironment, admin_ok, admin_request, init_logging}; use crate::common::{RustFSTestEnvironment, admin_ok, admin_request, init_logging};
use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::Client;
use aws_sdk_s3::{Client, Config};
use tracing::info; use tracing::info;
fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client { fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
let credentials = Credentials::new(access_key, secret_key, None, None, "e2e-group-test"); env.create_s3_client_with_credentials(access_key, secret_key)
let config = Config::builder()
.credentials_provider(credentials)
.region(Region::new("us-east-1"))
.endpoint_url(&env.url)
.force_path_style(true)
.behavior_version_latest()
.build();
Client::from_conf(config)
} }
#[tokio::test(flavor = "multi_thread")] #[tokio::test(flavor = "multi_thread")]
crates/e2e_test/src/inline_fast_path_cluster_test.rs
+5 -29
View File
@@ -21,7 +21,7 @@
//! One S3 GET can select readers on multiple EC nodes, so the counter tracks //! One S3 GET can select readers on multiple EC nodes, so the counter tracks
//! distributed reader selection rather than HTTP request count. //! distributed reader selection rather than HTTP request count.
use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, local_http_client}; use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging};
use aws_sdk_s3::Client; use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{ use aws_sdk_s3::types::{
@@ -30,7 +30,7 @@ use aws_sdk_s3::types::{
}; };
use bytes::Bytes; use bytes::Bytes;
use flate2::read::GzDecoder; use flate2::read::GzDecoder;
use http::header::{CONTENT_ENCODING, HOST}; use http::header::CONTENT_ENCODING;
use http::{Method, Request, Response, StatusCode}; use http::{Method, Request, Response, StatusCode};
use http_body_util::{BodyExt, Full}; use http_body_util::{BodyExt, Full};
use hyper::body::Incoming; use hyper::body::Incoming;
@@ -42,9 +42,6 @@ use opentelemetry_proto::tonic::metrics::v1::{
Metric, NumberDataPoint, ResourceMetrics, ScopeMetrics, Sum, metric, number_data_point, Metric, NumberDataPoint, ResourceMetrics, ScopeMetrics, Sum, metric, number_data_point,
}; };
use prost::Message; use prost::Message;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use s3s::Body;
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::convert::Infallible; use std::convert::Infallible;
use std::error::Error; use std::error::Error;
@@ -1262,6 +1259,8 @@ async fn put_two_part_multipart(client: &Client, bucket: &str, key: &str) -> Tes
Ok((body, part2, complete.e_tag().map(str::to_owned))) Ok((body, part2, complete.e_tag().map(str::to_owned)))
} }
/// Thin wrapper over [`crate::common::admin_request`], kept local so the call
/// sites below keep their `Option<&str>` body shape.
async fn signed_admin_request( async fn signed_admin_request(
base_url: &str, base_url: &str,
method: Method, method: Method,
@@ -1270,30 +1269,7 @@ async fn signed_admin_request(
access_key: &str, access_key: &str,
secret_key: &str, secret_key: &str,
) -> TestResult<(reqwest::StatusCode, String)> { ) -> TestResult<(reqwest::StatusCode, String)> {
let url = format!("{base_url}{path}"); crate::common::admin_request(base_url, method, path, body.map(str::to_string), access_key, secret_key).await
let uri = url.parse::<http::Uri>()?;
let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
let body_bytes = body.map(|value| value.as_bytes().to_vec()).unwrap_or_default();
let request = http::Request::builder()
.method(method.clone())
.uri(uri)
.header(HOST, authority)
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
let client = local_http_client();
let mut request_builder = client.request(method, url.as_str());
for (name, value) in signed.headers() {
request_builder = request_builder.header(name, value);
}
if !body_bytes.is_empty() {
request_builder = request_builder.body(body_bytes);
}
let response = request_builder.send().await?;
let status = response.status();
let text = response.text().await?;
Ok((status, text))
} }
fn unique_tier_name() -> String { fn unique_tier_name() -> String {
crates/e2e_test/src/notification_webhook_test.rs
+4 -34
View File
@@ -38,14 +38,10 @@ use aws_sdk_s3::types::{
NotificationConfiguration, NotificationConfigurationFilter, ObjectIdentifier, QueueConfiguration, S3KeyFilter, NotificationConfiguration, NotificationConfigurationFilter, ObjectIdentifier, QueueConfiguration, S3KeyFilter,
VersioningConfiguration, VersioningConfiguration,
}; };
use http::header::{CONTENT_TYPE, HOST};
use local_ip_address::local_ip; use local_ip_address::local_ip;
use reqwest::StatusCode; use reqwest::StatusCode;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS; use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
use rustfs_utils::http::headers::{AMZ_REQUEST_ID, REQUEST_ID_HEADER}; use rustfs_utils::http::headers::{AMZ_REQUEST_ID, REQUEST_ID_HEADER};
use s3s::Body;
use serde_json::Value; use serde_json::Value;
use std::error::Error; use std::error::Error;
use std::io::Cursor; use std::io::Cursor;
@@ -415,42 +411,16 @@ async fn collect_until(
// Admin target configuration (signed admin HTTP) // Admin target configuration (signed admin HTTP)
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
/// Thin wrapper over [`crate::common::signed_request`] with this suite's
/// root credentials; a `Some` body is always JSON here.
async fn signed_admin_request( async fn signed_admin_request(
env: &RustFSTestEnvironment, env: &RustFSTestEnvironment,
method: http::Method, method: http::Method,
url: &str, url: &str,
body: Option<Vec<u8>>, body: Option<Vec<u8>>,
) -> Result<reqwest::Response, BoxError> { ) -> Result<reqwest::Response, BoxError> {
let uri = url.parse::<http::Uri>()?; let content_type = body.is_some().then_some("application/json");
let authority = uri.authority().ok_or("admin URL missing authority")?.to_string(); crate::common::signed_request(method, url, &env.access_key, &env.secret_key, body, content_type).await
let mut builder = http::Request::builder()
.method(method.clone())
.uri(uri)
.header(HOST, authority)
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
if body.is_some() {
builder = builder.header(CONTENT_TYPE, "application/json");
}
let content_len = body.as_ref().map(|b| b.len() as i64).unwrap_or_default();
let signed = sign_v4(
builder.body(Body::empty())?,
content_len,
&env.access_key,
&env.secret_key,
"",
"us-east-1",
);
let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
let mut request = crate::common::local_http_client().request(reqwest_method, url);
for (name, value) in signed.headers() {
request = request.header(name, value);
}
if let Some(body) = body {
request = request.body(body);
}
Ok(request.send().await?)
} }
async fn enable_notify_module(env: &RustFSTestEnvironment) -> TestResult { async fn enable_notify_module(env: &RustFSTestEnvironment) -> TestResult {
crates/e2e_test/src/policy/policy_variables_test.rs
+18 -23
View File
@@ -15,28 +15,24 @@
//! Tests for AWS IAM policy variables with single-value, multi-value, and nested scenarios //! Tests for AWS IAM policy variables with single-value, multi-value, and nested scenarios
use crate::common::{ use crate::common::{
RustFSTestEnvironment, awscurl_delete, awscurl_put, build_test_s3_config, build_test_sts_client, init_logging, AdminTransport, RustFSTestEnvironment, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via,
awscurl_delete, awscurl_put, build_test_s3_config, build_test_sts_client, init_logging,
}; };
use aws_sdk_s3::Client; use aws_sdk_s3::Client;
use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::primitives::ByteStream;
use tracing::info; use tracing::info;
/// Helper function to create a regular user with given credentials /// Helper function to create a regular user with given credentials.
///
/// This suite deliberately drives the admin API through the external `awscurl`
/// binary, so the shared helpers are pinned to `AdminTransport::Awscurl`.
async fn create_user( async fn create_user(
env: &RustFSTestEnvironment, env: &RustFSTestEnvironment,
username: &str, username: &str,
password: &str, password: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> { ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let create_user_body = serde_json::json!({ admin_create_user_via(AdminTransport::Awscurl, &env.url, &env.access_key, &env.secret_key, username, password).await
"secretKey": password,
"status": "enabled"
})
.to_string();
let create_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
awscurl_put(&create_user_url, &create_user_body, &env.access_key, &env.secret_key).await?;
Ok(())
} }
/// Helper function to create and attach a policy /// Helper function to create and attach a policy
@@ -46,18 +42,17 @@ async fn create_and_attach_policy(
username: &str, username: &str,
policy_document: serde_json::Value, policy_document: serde_json::Value,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> { ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let policy_string = policy_document.to_string(); admin_add_canned_policy_via(
AdminTransport::Awscurl,
// Create policy &env.url,
let add_policy_url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name); &env.access_key,
awscurl_put(&add_policy_url, &policy_string, &env.access_key, &env.secret_key).await?; &env.secret_key,
policy_name,
// Attach policy to user &policy_document.to_string(),
let attach_policy_url = format!( )
"{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false", .await?;
env.url, policy_name, username admin_attach_user_policy_via(AdminTransport::Awscurl, &env.url, &env.access_key, &env.secret_key, policy_name, username)
); .await?;
awscurl_put(&attach_policy_url, "", &env.access_key, &env.secret_key).await?;
Ok(()) Ok(())
} }
crates/e2e_test/src/protocols/webdav_core.rs
+30 -83
View File
@@ -31,15 +31,11 @@
//! //!
//! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-3p3x-734c-h5vx> //! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-3p3x-734c-h5vx>
use crate::common::local_http_client;
use crate::common::rustfs_binary_path_with_features; use crate::common::rustfs_binary_path_with_features;
use crate::common::{AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via};
use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment}; use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment};
use anyhow::Result; use anyhow::Result;
use http::header::{CONTENT_TYPE, HOST};
use reqwest::Client; use reqwest::Client;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use s3s::Body;
use tokio::process::Command; use tokio::process::Command;
use tracing::info; use tracing::info;
@@ -67,92 +63,43 @@ fn basic_auth_header_for(access_key: &str, secret_key: &str) -> String {
format!("Basic {}", encoded) format!("Basic {}", encoded)
} }
async fn signed_admin_request( async fn admin_create_user(base_url: &str, username: &str, secret_key: &str) -> Result<()> {
method: http::Method, admin_create_user_via(
url: &str, AdminTransport::Signed,
body: Option<Vec<u8>>, base_url,
content_type: Option<&str>,
) -> Result<reqwest::Response> {
let uri = url.parse::<http::Uri>()?;
let authority = uri
.authority()
.ok_or_else(|| anyhow::anyhow!("request URL missing authority"))?
.to_string();
let mut request = http::Request::builder().method(method.clone()).uri(uri);
request = request.header(HOST, authority);
request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
if let Some(content_type) = content_type {
request = request.header(CONTENT_TYPE, content_type);
}
let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
let signed = sign_v4(
request.body(Body::empty())?,
content_len,
DEFAULT_ACCESS_KEY, DEFAULT_ACCESS_KEY,
DEFAULT_SECRET_KEY, DEFAULT_SECRET_KEY,
"", username,
"us-east-1", secret_key,
); )
.await
let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?; .map_err(|e| anyhow::anyhow!(e))
let mut request_builder = local_http_client().request(reqwest_method, url);
for (name, value) in signed.headers() {
request_builder = request_builder.header(name, value);
}
if let Some(body) = body {
request_builder = request_builder.body(body);
}
Ok(request_builder.send().await?)
}
async fn admin_create_user(base_url: &str, username: &str, secret_key: &str) -> Result<()> {
let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", base_url, username);
let body = serde_json::json!({
"secretKey": secret_key,
"status": "enabled"
});
let response =
signed_admin_request(http::Method::PUT, &url, Some(body.to_string().into_bytes()), Some("application/json")).await?;
if response.status() != reqwest::StatusCode::OK {
let status = response.status();
let body = response.text().await.unwrap_or_default();
anyhow::bail!("create user failed: {status} {body}");
}
Ok(())
} }
async fn admin_add_canned_policy(base_url: &str, policy_name: &str, policy: &serde_json::Value) -> Result<()> { async fn admin_add_canned_policy(base_url: &str, policy_name: &str, policy: &serde_json::Value) -> Result<()> {
let url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", base_url, policy_name); admin_add_canned_policy_via(
let response = AdminTransport::Signed,
signed_admin_request(http::Method::PUT, &url, Some(policy.to_string().into_bytes()), Some("application/json")).await?; base_url,
DEFAULT_ACCESS_KEY,
if response.status() != reqwest::StatusCode::OK { DEFAULT_SECRET_KEY,
let status = response.status(); policy_name,
let body = response.text().await.unwrap_or_default(); &policy.to_string(),
anyhow::bail!("add canned policy failed: {status} {body}"); )
} .await
.map_err(|e| anyhow::anyhow!(e))
Ok(())
} }
async fn admin_attach_policy_to_user(base_url: &str, policy_name: &str, username: &str) -> Result<()> { async fn admin_attach_policy_to_user(base_url: &str, policy_name: &str, username: &str) -> Result<()> {
let url = format!( admin_attach_user_policy_via(
"{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false", AdminTransport::Signed,
base_url, policy_name, username base_url,
); DEFAULT_ACCESS_KEY,
let response = signed_admin_request(http::Method::PUT, &url, Some(Vec::new()), None).await?; DEFAULT_SECRET_KEY,
policy_name,
if response.status() != reqwest::StatusCode::OK { username,
let status = response.status(); )
let body = response.text().await.unwrap_or_default(); .await
anyhow::bail!("attach policy failed: {status} {body}"); .map_err(|e| anyhow::anyhow!(e))
}
Ok(())
} }
/// Test WebDAV: MKCOL (create bucket), PUT, GET, DELETE, PROPFIND operations /// Test WebDAV: MKCOL (create bucket), PUT, GET, DELETE, PROPFIND operations
crates/e2e_test/src/reliant/tiering.rs
+4 -32
View File
@@ -46,7 +46,7 @@
//! retry serves the object locally until expiry, and expiry leaves the //! retry serves the object locally until expiry, and expiry leaves the
//! remote object available for a second restore. //! remote object available for a second restore.
use crate::common::{RustFSTestEnvironment, local_http_client}; use crate::common::RustFSTestEnvironment;
use aws_sdk_s3::Client; use aws_sdk_s3::Client;
use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::primitives::ByteStream;
@@ -56,10 +56,6 @@ use aws_sdk_s3::types::{
VersioningConfiguration, VersioningConfiguration,
}; };
use http::Method; use http::Method;
use http::header::HOST;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use s3s::Body;
use serde::Deserialize; use serde::Deserialize;
use std::time::{Duration as StdDuration, Instant}; use std::time::{Duration as StdDuration, Instant};
use time::{OffsetDateTime, format_description::well_known::Rfc3339}; use time::{OffsetDateTime, format_description::well_known::Rfc3339};
@@ -131,9 +127,8 @@ fn payload() -> Vec<u8> {
/// Sign and send an admin request in-process (no `awscurl`). /// Sign and send an admin request in-process (no `awscurl`).
/// ///
/// Mirrors the shared admin-API e2e pattern: the SigV4 signature is computed /// Thin wrapper over [`crate::common::admin_request`], kept local so the call
/// over `UNSIGNED_PAYLOAD`, so the JSON body rides on the wire without being /// sites below keep their `Option<&str>` body shape.
/// pre-hashed. Returns the response status and body text.
async fn signed_admin_request( async fn signed_admin_request(
base_url: &str, base_url: &str,
method: Method, method: Method,
@@ -142,30 +137,7 @@ async fn signed_admin_request(
access_key: &str, access_key: &str,
secret_key: &str, secret_key: &str,
) -> Result<(reqwest::StatusCode, String), Box<dyn std::error::Error + Send + Sync>> { ) -> Result<(reqwest::StatusCode, String), Box<dyn std::error::Error + Send + Sync>> {
let url = format!("{base_url}{path}"); crate::common::admin_request(base_url, method, path, body.map(str::to_string), access_key, secret_key).await
let uri = url.parse::<http::Uri>()?;
let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
let body_bytes = body.map(|b| b.as_bytes().to_vec()).unwrap_or_default();
let request = http::Request::builder()
.method(method.clone())
.uri(uri)
.header(HOST, authority)
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
let client = local_http_client();
let mut request_builder = client.request(method, url.as_str());
for (name, value) in signed.headers() {
request_builder = request_builder.header(name, value);
}
if !body_bytes.is_empty() {
request_builder = request_builder.body(body_bytes);
}
let response = request_builder.send().await?;
let status = response.status();
let text = response.text().await?;
Ok((status, text))
} }
/// Wire `hot` -> `cold` as a `TierType::RustFS` remote tier via `AddTier`. /// Wire `hot` -> `cold` as a `TierType::RustFS` remote tier via `AddTier`.
crates/e2e_test/src/replication_extension_test.rs
+12 -41
View File
@@ -13,8 +13,9 @@
// limitations under the License. // limitations under the License.
use crate::common::{ use crate::common::{
RustFSTestEnvironment, admin_create_user, awscurl_post_sts_form_urlencoded, init_logging, local_http_client, AdminTransport, RustFSTestEnvironment, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user,
replication_fast_env, rustfs_binary_path, signed_request, signed_request_with_client, signed_request_with_session_token, awscurl_post_sts_form_urlencoded, init_logging, local_http_client, replication_fast_env, rustfs_binary_path, signed_request,
signed_request_with_client, signed_request_with_session_token,
}; };
use crate::fake_s3_target::{ use crate::fake_s3_target::{
FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, FaultAction as FakeTargetFault, Operation as FakeTargetOperation, FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, FaultAction as FakeTargetFault, Operation as FakeTargetOperation,
@@ -25,7 +26,7 @@ use crate::kms::common::{
sse_customer_key_md5_base64, sse_customer_key_md5_base64,
}; };
use crate::storage_api::replication_extension::BucketTargetSys; use crate::storage_api::replication_extension::BucketTargetSys;
use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::Client;
use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput; use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::primitives::ByteStream;
@@ -33,7 +34,6 @@ use aws_sdk_s3::types::{
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption, BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
VersioningConfiguration, VersioningConfiguration,
}; };
use aws_sdk_s3::{Client, Config};
use base64_simd::STANDARD as BASE64_STANDARD; use base64_simd::STANDARD as BASE64_STANDARD;
use bytes::Bytes; use bytes::Bytes;
use flate2::read::GzDecoder; use flate2::read::GzDecoder;
@@ -895,15 +895,7 @@ async fn wait_for_replicated_object_over_https(
} }
fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client { fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
let credentials = Credentials::new(access_key, secret_key, None, None, "e2e-site-replication"); env.create_s3_client_with_credentials(access_key, secret_key)
let config = Config::builder()
.credentials_provider(credentials)
.region(Region::new("us-east-1"))
.endpoint_url(&env.url)
.force_path_style(true)
.behavior_version_latest()
.build();
Client::from_conf(config)
} }
async fn admin_add_canned_policy( async fn admin_add_canned_policy(
@@ -911,24 +903,15 @@ async fn admin_add_canned_policy(
policy_name: &str, policy_name: &str,
policy: &serde_json::Value, policy: &serde_json::Value,
) -> Result<(), Box<dyn Error + Send + Sync>> { ) -> Result<(), Box<dyn Error + Send + Sync>> {
let url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name); admin_add_canned_policy_via(
let response = signed_request( AdminTransport::Signed,
http::Method::PUT, &env.url,
&url,
&env.access_key, &env.access_key,
&env.secret_key, &env.secret_key,
Some(policy.to_string().into_bytes()), policy_name,
Some("application/json"), &policy.to_string(),
) )
.await?; .await
if response.status() != StatusCode::OK {
let status = response.status();
let body = response.text().await.unwrap_or_default();
return Err(format!("add canned policy failed: {status} {body}").into());
}
Ok(())
} }
async fn admin_attach_policy_to_user( async fn admin_attach_policy_to_user(
@@ -936,19 +919,7 @@ async fn admin_attach_policy_to_user(
policy_name: &str, policy_name: &str,
username: &str, username: &str,
) -> Result<(), Box<dyn Error + Send + Sync>> { ) -> Result<(), Box<dyn Error + Send + Sync>> {
let url = format!( admin_attach_user_policy_via(AdminTransport::Signed, &env.url, &env.access_key, &env.secret_key, policy_name, username).await
"{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false",
env.url, policy_name, username
);
let response = signed_request(http::Method::PUT, &url, &env.access_key, &env.secret_key, Some(Vec::new()), None).await?;
if response.status() != StatusCode::OK {
let status = response.status();
let body = response.text().await.unwrap_or_default();
return Err(format!("attach policy to user failed: {status} {body}").into());
}
Ok(())
} }
async fn admin_update_group_members( async fn admin_update_group_members(