mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-31 01:09:23 +00:00
test(e2e): finish the helper consolidation onto common.rs (#6766)
- common.rs gains an AdminTransport knob (Signed | Awscurl) with admin_execute_at plus three family wrappers: admin_create_user_via, admin_add_canned_policy_via, admin_attach_user_policy_via; the existing admin_create_user now delegates over the Signed transport. - Deleted the four signed admin request clones in admin_mfa_test, admin_auth_test, reliant/tiering, and inline_fast_path_cluster_test; each keeps a thin local wrapper over common::admin_request so call sites keep their Option<&str> body shape. - Deduped the notification_webhook signer onto common::signed_request and the webdav_core signer plus its three admin helpers onto the shared _via helpers. - Consolidated the S3-client-with-credentials builders: admin_auth s3_client_with, existing_object_tag user_client/sts_session_client, bucket_policy_check create_user_client, and the create_user_s3_client copies in group_delete_test and replication_extension_test now delegate to create_s3_client_with_credentials / build_test_s3_config; replication_extension admin_add_canned_policy and admin_attach_policy_to_user route through the _via helpers on the Signed transport. - The awscurl-gated suites (existing_object_tag_policy, bucket_policy_check, policy/policy_variables) keep going through the external awscurl binary via AdminTransport::Awscurl, preserving their wire behavior. Part of rustfs/backlog#1846 (cluster 2).
This commit is contained in:
@@ -31,14 +31,9 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client, rustfs_binary_path};
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::{Client, Config};
|
||||
use http::header::HOST;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
use std::io::Read;
|
||||
use std::process::{Command, Stdio};
|
||||
@@ -87,10 +82,10 @@ mod tests {
|
||||
}
|
||||
|
||||
/// Send a SigV4-signed request to `path` (optionally with a JSON `body`) and
|
||||
/// return `(status, body)`. Uses the `UNSIGNED_PAYLOAD` content hash so a
|
||||
/// request body can be attached without the caller pre-hashing it — the
|
||||
/// server verifies the signature against the same sentinel, exactly as the
|
||||
/// AWS SDKs / MinIO client do for streaming/unsigned payloads.
|
||||
/// return `(status, body)`.
|
||||
///
|
||||
/// Thin wrapper over [`crate::common::admin_request`], kept local so the
|
||||
/// call sites below keep their `Option<&str>` body shape.
|
||||
async fn signed_request(
|
||||
base_url: &str,
|
||||
method: http::Method,
|
||||
@@ -99,47 +94,13 @@ mod tests {
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(reqwest::StatusCode, String), Box<dyn Error + Send + Sync>> {
|
||||
let url = format!("{base_url}{path}");
|
||||
let uri = url.parse::<http::Uri>()?;
|
||||
let authority = uri.authority().ok_or("missing authority")?.to_string();
|
||||
let body_bytes = body.map(|b| b.as_bytes().to_vec()).unwrap_or_default();
|
||||
|
||||
// The signature is computed over `UNSIGNED_PAYLOAD`, so the body bytes do
|
||||
// not participate in the SigV4 hash — sign over an empty body and attach
|
||||
// the real payload to the wire request below.
|
||||
let request = http::Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(uri)
|
||||
.header(HOST, authority)
|
||||
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
|
||||
let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
|
||||
|
||||
let client = local_http_client();
|
||||
let mut rb = client.request(method, url.as_str());
|
||||
for (name, value) in signed.headers() {
|
||||
rb = rb.header(name, value);
|
||||
}
|
||||
if !body_bytes.is_empty() {
|
||||
rb = rb.body(body_bytes);
|
||||
}
|
||||
let resp = rb.send().await?;
|
||||
let status = resp.status();
|
||||
let text = resp.text().await?;
|
||||
Ok((status, text))
|
||||
crate::common::admin_request(base_url, method, path, body.map(str::to_string), access_key, secret_key).await
|
||||
}
|
||||
|
||||
/// Build an S3 client bound to explicit credentials (used to exercise the S3
|
||||
/// data plane with rotated / stale root credentials).
|
||||
fn s3_client_with(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
|
||||
let credentials = Credentials::new(access_key, secret_key, None, None, "sec4-admin-auth");
|
||||
let config = Config::builder()
|
||||
.credentials_provider(credentials)
|
||||
.region(Region::new("us-east-1"))
|
||||
.endpoint_url(&env.url)
|
||||
.force_path_style(true)
|
||||
.behavior_version_latest()
|
||||
.build();
|
||||
Client::from_conf(config)
|
||||
env.create_s3_client_with_credentials(access_key, secret_key)
|
||||
}
|
||||
|
||||
/// Create a non-admin IAM user via the admin `add-user` API using the root
|
||||
@@ -151,12 +112,7 @@ mod tests {
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let path = format!("/rustfs/admin/v3/add-user?accessKey={access_key}");
|
||||
let body = serde_json::json!({ "secretKey": secret_key, "status": "enabled" }).to_string();
|
||||
let (status, resp) =
|
||||
signed_request(&env.url, http::Method::PUT, &path, Some(&body), &env.access_key, &env.secret_key).await?;
|
||||
assert!(status.is_success(), "add-user should succeed (status={status}, body={resp})");
|
||||
Ok(())
|
||||
crate::common::admin_create_user(env, access_key, secret_key).await
|
||||
}
|
||||
|
||||
/// A fully authenticated but non-admin credential must be rejected with
|
||||
|
||||
@@ -59,8 +59,8 @@ mod tests {
|
||||
|
||||
/// One signed admin request, returning the status and the raw body.
|
||||
///
|
||||
/// Signs with `UNSIGNED_PAYLOAD` so the body does not participate in the
|
||||
/// hash, matching how the other admin e2e tests drive these routes.
|
||||
/// Thin wrapper over [`crate::common::admin_request`], kept local so the
|
||||
/// call sites below keep their `Option<&str>` body shape.
|
||||
async fn signed_request(
|
||||
base_url: &str,
|
||||
method: http::Method,
|
||||
@@ -69,30 +69,7 @@ mod tests {
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(reqwest::StatusCode, String), Box<dyn Error + Send + Sync>> {
|
||||
let url = format!("{base_url}{path}");
|
||||
let uri = url.parse::<http::Uri>()?;
|
||||
let authority = uri.authority().ok_or("missing authority")?.to_string();
|
||||
let body_bytes = body.map(|b| b.as_bytes().to_vec()).unwrap_or_default();
|
||||
|
||||
let request = http::Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(uri)
|
||||
.header(HOST, authority)
|
||||
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
|
||||
let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
|
||||
|
||||
let client = local_http_client();
|
||||
let mut builder = client.request(method, url.as_str());
|
||||
for (name, value) in signed.headers() {
|
||||
builder = builder.header(name, value);
|
||||
}
|
||||
if !body_bytes.is_empty() {
|
||||
builder = builder.body(body_bytes);
|
||||
}
|
||||
let response = builder.send().await?;
|
||||
let status = response.status();
|
||||
let text = response.text().await?;
|
||||
Ok((status, text))
|
||||
crate::common::admin_request(base_url, method, path, body.map(str::to_string), access_key, secret_key).await
|
||||
}
|
||||
|
||||
/// A SigV4-signed `AssumeRole` form POST, optionally carrying a second factor.
|
||||
|
||||
@@ -15,39 +15,23 @@
|
||||
//! Regression test for Issue #1423
|
||||
//! Verifies that Bucket Policies are honored for Authenticated Users.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use crate::common::{AdminTransport, RustFSTestEnvironment, admin_create_user_via, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::{Client, Config};
|
||||
use tracing::info;
|
||||
|
||||
/// This suite deliberately drives the admin API through the external `awscurl`
|
||||
/// binary, so user creation pins `AdminTransport::Awscurl`.
|
||||
async fn create_user(
|
||||
env: &RustFSTestEnvironment,
|
||||
username: &str,
|
||||
password: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let create_user_body = serde_json::json!({
|
||||
"secretKey": password,
|
||||
"status": "enabled"
|
||||
})
|
||||
.to_string();
|
||||
|
||||
let create_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
|
||||
crate::common::awscurl_put(&create_user_url, &create_user_body, &env.access_key, &env.secret_key).await?;
|
||||
Ok(())
|
||||
admin_create_user_via(AdminTransport::Awscurl, &env.url, &env.access_key, &env.secret_key, username, password).await
|
||||
}
|
||||
|
||||
fn create_user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
|
||||
let credentials = Credentials::new(access_key, secret_key, None, None, "test-user");
|
||||
let config = Config::builder()
|
||||
.credentials_provider(credentials)
|
||||
.region(Region::new("us-east-1"))
|
||||
.endpoint_url(&env.url)
|
||||
.force_path_style(true)
|
||||
.behavior_version_latest()
|
||||
.build();
|
||||
|
||||
Client::from_conf(config)
|
||||
env.create_s3_client_with_credentials(access_key, secret_key)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
+117
-19
@@ -1744,30 +1744,128 @@ pub(crate) async fn admin_create_user(
|
||||
username: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
|
||||
let body = serde_json::json!({
|
||||
"secretKey": secret_key,
|
||||
"status": "enabled"
|
||||
});
|
||||
let response = signed_request(
|
||||
http::Method::PUT,
|
||||
&url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
Some(body.to_string().into_bytes()),
|
||||
Some("application/json"),
|
||||
)
|
||||
.await?;
|
||||
admin_create_user_via(AdminTransport::Signed, &env.url, &env.access_key, &env.secret_key, username, secret_key).await
|
||||
}
|
||||
|
||||
if response.status() != reqwest::StatusCode::OK {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
return Err(format!("create user failed: {status} {body}").into());
|
||||
/// Transport used by the shared admin-API helpers: in-process SigV4 signing
|
||||
/// via [`signed_request`], or the external `awscurl` binary (an independent
|
||||
/// SigV4 implementation exercised by the awscurl-gated suites).
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum AdminTransport {
|
||||
Signed,
|
||||
Awscurl,
|
||||
}
|
||||
|
||||
/// Execute an admin-API request against `base_url` with admin credentials over
|
||||
/// the chosen transport, failing on any non-success response.
|
||||
pub(crate) async fn admin_execute_at(
|
||||
transport: AdminTransport,
|
||||
method: http::Method,
|
||||
base_url: &str,
|
||||
admin_access_key: &str,
|
||||
admin_secret_key: &str,
|
||||
path_and_query: &str,
|
||||
body: Option<&str>,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let url = format!("{base_url}{path_and_query}");
|
||||
match transport {
|
||||
AdminTransport::Signed => {
|
||||
let content_type = match body {
|
||||
Some(body) if !body.is_empty() => Some("application/json"),
|
||||
_ => None,
|
||||
};
|
||||
let response = signed_request(
|
||||
method.clone(),
|
||||
&url,
|
||||
admin_access_key,
|
||||
admin_secret_key,
|
||||
body.map(|body| body.as_bytes().to_vec()),
|
||||
content_type,
|
||||
)
|
||||
.await?;
|
||||
if !response.status().is_success() {
|
||||
let status = response.status();
|
||||
let text = response.text().await.unwrap_or_default();
|
||||
return Err(format!("{method} {path_and_query} failed: {status} {text}").into());
|
||||
}
|
||||
}
|
||||
AdminTransport::Awscurl => {
|
||||
execute_awscurl(&url, method.as_str(), body, admin_access_key, admin_secret_key).await?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Create a new IAM user via the admin API over the chosen transport.
|
||||
pub(crate) async fn admin_create_user_via(
|
||||
transport: AdminTransport,
|
||||
base_url: &str,
|
||||
admin_access_key: &str,
|
||||
admin_secret_key: &str,
|
||||
username: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let path = format!("/rustfs/admin/v3/add-user?accessKey={username}");
|
||||
let body = serde_json::json!({"secretKey": secret_key, "status": "enabled"}).to_string();
|
||||
admin_execute_at(
|
||||
transport,
|
||||
http::Method::PUT,
|
||||
base_url,
|
||||
admin_access_key,
|
||||
admin_secret_key,
|
||||
&path,
|
||||
Some(&body),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Install a canned policy via the admin API over the chosen transport.
|
||||
pub(crate) async fn admin_add_canned_policy_via(
|
||||
transport: AdminTransport,
|
||||
base_url: &str,
|
||||
admin_access_key: &str,
|
||||
admin_secret_key: &str,
|
||||
policy_name: &str,
|
||||
policy_json: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let path = format!("/rustfs/admin/v3/add-canned-policy?name={policy_name}");
|
||||
admin_execute_at(
|
||||
transport,
|
||||
http::Method::PUT,
|
||||
base_url,
|
||||
admin_access_key,
|
||||
admin_secret_key,
|
||||
&path,
|
||||
Some(policy_json),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Attach a canned policy to a user via the admin API over the chosen transport.
|
||||
pub(crate) async fn admin_attach_user_policy_via(
|
||||
transport: AdminTransport,
|
||||
base_url: &str,
|
||||
admin_access_key: &str,
|
||||
admin_secret_key: &str,
|
||||
policy_name: &str,
|
||||
username: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let path = format!("/rustfs/admin/v3/set-user-or-group-policy?policyName={policy_name}&userOrGroup={username}&isGroup=false");
|
||||
// `Some("")` preserves the historical wire shape on both transports: awscurl
|
||||
// keeps sending `-d ''` and the signed path attaches an empty body with no
|
||||
// content type.
|
||||
admin_execute_at(
|
||||
transport,
|
||||
http::Method::PUT,
|
||||
base_url,
|
||||
admin_access_key,
|
||||
admin_secret_key,
|
||||
&path,
|
||||
Some(""),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -16,37 +16,29 @@
|
||||
//! session policy** (`Policy` parameter) via `awscurl --service sts` with explicit
|
||||
//! `Content-Type: application/x-www-form-urlencoded` on `POST /`.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, awscurl_delete, awscurl_post_sts_form_urlencoded, awscurl_put, init_logging};
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use crate::common::{
|
||||
AdminTransport, RustFSTestEnvironment, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via,
|
||||
awscurl_delete, awscurl_post_sts_form_urlencoded, build_test_s3_config, init_logging,
|
||||
};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{Delete, ObjectIdentifier, Tag, Tagging};
|
||||
use aws_sdk_s3::{Client, Config};
|
||||
use tracing::info;
|
||||
use uuid::Uuid;
|
||||
|
||||
fn user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
|
||||
let credentials = Credentials::new(access_key, secret_key, None, None, "e2e-existing-tag");
|
||||
let config = Config::builder()
|
||||
.credentials_provider(credentials)
|
||||
.region(Region::new("us-east-1"))
|
||||
.endpoint_url(&env.url)
|
||||
.force_path_style(true)
|
||||
.behavior_version_latest()
|
||||
.build();
|
||||
Client::from_conf(config)
|
||||
env.create_s3_client_with_credentials(access_key, secret_key)
|
||||
}
|
||||
|
||||
fn sts_session_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str, session_token: &str) -> Client {
|
||||
let credentials = Credentials::new(access_key, secret_key, Some(session_token.into()), None, "e2e-sts-session");
|
||||
let config = Config::builder()
|
||||
.credentials_provider(credentials)
|
||||
.region(Region::new("us-east-1"))
|
||||
.endpoint_url(&env.url)
|
||||
.force_path_style(true)
|
||||
.behavior_version_latest()
|
||||
.build();
|
||||
Client::from_conf(config)
|
||||
Client::from_conf(build_test_s3_config(
|
||||
&env.url,
|
||||
access_key,
|
||||
secret_key,
|
||||
Some(session_token),
|
||||
"e2e-sts-session",
|
||||
))
|
||||
}
|
||||
|
||||
fn extract_xml_tag(xml: &str, tag: &str) -> Option<String> {
|
||||
@@ -77,15 +69,16 @@ async fn assume_role_with_session_policy(
|
||||
parse_assume_role_credentials(&xml)
|
||||
}
|
||||
|
||||
// This suite deliberately drives the admin API through the external `awscurl`
|
||||
// binary (an independent SigV4 implementation), so the wrappers below pin
|
||||
// `AdminTransport::Awscurl`.
|
||||
|
||||
async fn admin_create_user(
|
||||
env: &RustFSTestEnvironment,
|
||||
username: &str,
|
||||
password: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let body = serde_json::json!({ "secretKey": password, "status": "enabled" }).to_string();
|
||||
let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
|
||||
awscurl_put(&url, &body, &env.access_key, &env.secret_key).await?;
|
||||
Ok(())
|
||||
admin_create_user_via(AdminTransport::Awscurl, &env.url, &env.access_key, &env.secret_key, username, password).await
|
||||
}
|
||||
|
||||
async fn admin_add_canned_policy(
|
||||
@@ -93,9 +86,15 @@ async fn admin_add_canned_policy(
|
||||
policy_name: &str,
|
||||
policy_json: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
|
||||
awscurl_put(&url, policy_json, &env.access_key, &env.secret_key).await?;
|
||||
Ok(())
|
||||
admin_add_canned_policy_via(
|
||||
AdminTransport::Awscurl,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
policy_name,
|
||||
policy_json,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn admin_attach_policy_to_user(
|
||||
@@ -103,12 +102,7 @@ async fn admin_attach_policy_to_user(
|
||||
policy_name: &str,
|
||||
username: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let url = format!(
|
||||
"{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false",
|
||||
env.url, policy_name, username
|
||||
);
|
||||
awscurl_put(&url, "", &env.access_key, &env.secret_key).await?;
|
||||
Ok(())
|
||||
admin_attach_user_policy_via(AdminTransport::Awscurl, &env.url, &env.access_key, &env.secret_key, policy_name, username).await
|
||||
}
|
||||
|
||||
async fn admin_remove_user(env: &RustFSTestEnvironment, username: &str) {
|
||||
|
||||
@@ -15,20 +15,11 @@
|
||||
//! E2E tests for group management (fixes #2028).
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, admin_ok, admin_request, init_logging};
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use aws_sdk_s3::{Client, Config};
|
||||
use aws_sdk_s3::Client;
|
||||
use tracing::info;
|
||||
|
||||
fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
|
||||
let credentials = Credentials::new(access_key, secret_key, None, None, "e2e-group-test");
|
||||
let config = Config::builder()
|
||||
.credentials_provider(credentials)
|
||||
.region(Region::new("us-east-1"))
|
||||
.endpoint_url(&env.url)
|
||||
.force_path_style(true)
|
||||
.behavior_version_latest()
|
||||
.build();
|
||||
Client::from_conf(config)
|
||||
env.create_s3_client_with_credentials(access_key, secret_key)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
//! One S3 GET can select readers on multiple EC nodes, so the counter tracks
|
||||
//! distributed reader selection rather than HTTP request count.
|
||||
|
||||
use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
@@ -30,7 +30,7 @@ use aws_sdk_s3::types::{
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use flate2::read::GzDecoder;
|
||||
use http::header::{CONTENT_ENCODING, HOST};
|
||||
use http::header::CONTENT_ENCODING;
|
||||
use http::{Method, Request, Response, StatusCode};
|
||||
use http_body_util::{BodyExt, Full};
|
||||
use hyper::body::Incoming;
|
||||
@@ -42,9 +42,6 @@ use opentelemetry_proto::tonic::metrics::v1::{
|
||||
Metric, NumberDataPoint, ResourceMetrics, ScopeMetrics, Sum, metric, number_data_point,
|
||||
};
|
||||
use prost::Message;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::collections::BTreeMap;
|
||||
use std::convert::Infallible;
|
||||
use std::error::Error;
|
||||
@@ -1262,6 +1259,8 @@ async fn put_two_part_multipart(client: &Client, bucket: &str, key: &str) -> Tes
|
||||
Ok((body, part2, complete.e_tag().map(str::to_owned)))
|
||||
}
|
||||
|
||||
/// Thin wrapper over [`crate::common::admin_request`], kept local so the call
|
||||
/// sites below keep their `Option<&str>` body shape.
|
||||
async fn signed_admin_request(
|
||||
base_url: &str,
|
||||
method: Method,
|
||||
@@ -1270,30 +1269,7 @@ async fn signed_admin_request(
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> TestResult<(reqwest::StatusCode, String)> {
|
||||
let url = format!("{base_url}{path}");
|
||||
let uri = url.parse::<http::Uri>()?;
|
||||
let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
|
||||
let body_bytes = body.map(|value| value.as_bytes().to_vec()).unwrap_or_default();
|
||||
|
||||
let request = http::Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(uri)
|
||||
.header(HOST, authority)
|
||||
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
|
||||
let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
|
||||
|
||||
let client = local_http_client();
|
||||
let mut request_builder = client.request(method, url.as_str());
|
||||
for (name, value) in signed.headers() {
|
||||
request_builder = request_builder.header(name, value);
|
||||
}
|
||||
if !body_bytes.is_empty() {
|
||||
request_builder = request_builder.body(body_bytes);
|
||||
}
|
||||
let response = request_builder.send().await?;
|
||||
let status = response.status();
|
||||
let text = response.text().await?;
|
||||
Ok((status, text))
|
||||
crate::common::admin_request(base_url, method, path, body.map(str::to_string), access_key, secret_key).await
|
||||
}
|
||||
|
||||
fn unique_tier_name() -> String {
|
||||
|
||||
@@ -38,14 +38,10 @@ use aws_sdk_s3::types::{
|
||||
NotificationConfiguration, NotificationConfigurationFilter, ObjectIdentifier, QueueConfiguration, S3KeyFilter,
|
||||
VersioningConfiguration,
|
||||
};
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use local_ip_address::local_ip;
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
|
||||
use rustfs_utils::http::headers::{AMZ_REQUEST_ID, REQUEST_ID_HEADER};
|
||||
use s3s::Body;
|
||||
use serde_json::Value;
|
||||
use std::error::Error;
|
||||
use std::io::Cursor;
|
||||
@@ -415,42 +411,16 @@ async fn collect_until(
|
||||
// Admin target configuration (signed admin HTTP)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Thin wrapper over [`crate::common::signed_request`] with this suite's
|
||||
/// root credentials; a `Some` body is always JSON here.
|
||||
async fn signed_admin_request(
|
||||
env: &RustFSTestEnvironment,
|
||||
method: http::Method,
|
||||
url: &str,
|
||||
body: Option<Vec<u8>>,
|
||||
) -> Result<reqwest::Response, BoxError> {
|
||||
let uri = url.parse::<http::Uri>()?;
|
||||
let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
|
||||
let mut builder = http::Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(uri)
|
||||
.header(HOST, authority)
|
||||
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
|
||||
if body.is_some() {
|
||||
builder = builder.header(CONTENT_TYPE, "application/json");
|
||||
}
|
||||
|
||||
let content_len = body.as_ref().map(|b| b.len() as i64).unwrap_or_default();
|
||||
let signed = sign_v4(
|
||||
builder.body(Body::empty())?,
|
||||
content_len,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
"",
|
||||
"us-east-1",
|
||||
);
|
||||
|
||||
let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
|
||||
let mut request = crate::common::local_http_client().request(reqwest_method, url);
|
||||
for (name, value) in signed.headers() {
|
||||
request = request.header(name, value);
|
||||
}
|
||||
if let Some(body) = body {
|
||||
request = request.body(body);
|
||||
}
|
||||
Ok(request.send().await?)
|
||||
let content_type = body.is_some().then_some("application/json");
|
||||
crate::common::signed_request(method, url, &env.access_key, &env.secret_key, body, content_type).await
|
||||
}
|
||||
|
||||
async fn enable_notify_module(env: &RustFSTestEnvironment) -> TestResult {
|
||||
|
||||
@@ -15,28 +15,24 @@
|
||||
//! Tests for AWS IAM policy variables with single-value, multi-value, and nested scenarios
|
||||
|
||||
use crate::common::{
|
||||
RustFSTestEnvironment, awscurl_delete, awscurl_put, build_test_s3_config, build_test_sts_client, init_logging,
|
||||
AdminTransport, RustFSTestEnvironment, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via,
|
||||
awscurl_delete, awscurl_put, build_test_s3_config, build_test_sts_client, init_logging,
|
||||
};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use tracing::info;
|
||||
|
||||
/// Helper function to create a regular user with given credentials
|
||||
/// Helper function to create a regular user with given credentials.
|
||||
///
|
||||
/// This suite deliberately drives the admin API through the external `awscurl`
|
||||
/// binary, so the shared helpers are pinned to `AdminTransport::Awscurl`.
|
||||
async fn create_user(
|
||||
env: &RustFSTestEnvironment,
|
||||
username: &str,
|
||||
password: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let create_user_body = serde_json::json!({
|
||||
"secretKey": password,
|
||||
"status": "enabled"
|
||||
})
|
||||
.to_string();
|
||||
|
||||
let create_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
|
||||
awscurl_put(&create_user_url, &create_user_body, &env.access_key, &env.secret_key).await?;
|
||||
Ok(())
|
||||
admin_create_user_via(AdminTransport::Awscurl, &env.url, &env.access_key, &env.secret_key, username, password).await
|
||||
}
|
||||
|
||||
/// Helper function to create and attach a policy
|
||||
@@ -46,18 +42,17 @@ async fn create_and_attach_policy(
|
||||
username: &str,
|
||||
policy_document: serde_json::Value,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let policy_string = policy_document.to_string();
|
||||
|
||||
// Create policy
|
||||
let add_policy_url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
|
||||
awscurl_put(&add_policy_url, &policy_string, &env.access_key, &env.secret_key).await?;
|
||||
|
||||
// Attach policy to user
|
||||
let attach_policy_url = format!(
|
||||
"{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false",
|
||||
env.url, policy_name, username
|
||||
);
|
||||
awscurl_put(&attach_policy_url, "", &env.access_key, &env.secret_key).await?;
|
||||
admin_add_canned_policy_via(
|
||||
AdminTransport::Awscurl,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
policy_name,
|
||||
&policy_document.to_string(),
|
||||
)
|
||||
.await?;
|
||||
admin_attach_user_policy_via(AdminTransport::Awscurl, &env.url, &env.access_key, &env.secret_key, policy_name, username)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -31,15 +31,11 @@
|
||||
//!
|
||||
//! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-3p3x-734c-h5vx>
|
||||
|
||||
use crate::common::local_http_client;
|
||||
use crate::common::rustfs_binary_path_with_features;
|
||||
use crate::common::{AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via};
|
||||
use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment};
|
||||
use anyhow::Result;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use reqwest::Client;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use tokio::process::Command;
|
||||
use tracing::info;
|
||||
|
||||
@@ -67,92 +63,43 @@ fn basic_auth_header_for(access_key: &str, secret_key: &str) -> String {
|
||||
format!("Basic {}", encoded)
|
||||
}
|
||||
|
||||
async fn signed_admin_request(
|
||||
method: http::Method,
|
||||
url: &str,
|
||||
body: Option<Vec<u8>>,
|
||||
content_type: Option<&str>,
|
||||
) -> Result<reqwest::Response> {
|
||||
let uri = url.parse::<http::Uri>()?;
|
||||
let authority = uri
|
||||
.authority()
|
||||
.ok_or_else(|| anyhow::anyhow!("request URL missing authority"))?
|
||||
.to_string();
|
||||
let mut request = http::Request::builder().method(method.clone()).uri(uri);
|
||||
request = request.header(HOST, authority);
|
||||
request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
|
||||
if let Some(content_type) = content_type {
|
||||
request = request.header(CONTENT_TYPE, content_type);
|
||||
}
|
||||
|
||||
let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
|
||||
let signed = sign_v4(
|
||||
request.body(Body::empty())?,
|
||||
content_len,
|
||||
async fn admin_create_user(base_url: &str, username: &str, secret_key: &str) -> Result<()> {
|
||||
admin_create_user_via(
|
||||
AdminTransport::Signed,
|
||||
base_url,
|
||||
DEFAULT_ACCESS_KEY,
|
||||
DEFAULT_SECRET_KEY,
|
||||
"",
|
||||
"us-east-1",
|
||||
);
|
||||
|
||||
let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
|
||||
let mut request_builder = local_http_client().request(reqwest_method, url);
|
||||
for (name, value) in signed.headers() {
|
||||
request_builder = request_builder.header(name, value);
|
||||
}
|
||||
if let Some(body) = body {
|
||||
request_builder = request_builder.body(body);
|
||||
}
|
||||
|
||||
Ok(request_builder.send().await?)
|
||||
}
|
||||
|
||||
async fn admin_create_user(base_url: &str, username: &str, secret_key: &str) -> Result<()> {
|
||||
let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", base_url, username);
|
||||
let body = serde_json::json!({
|
||||
"secretKey": secret_key,
|
||||
"status": "enabled"
|
||||
});
|
||||
let response =
|
||||
signed_admin_request(http::Method::PUT, &url, Some(body.to_string().into_bytes()), Some("application/json")).await?;
|
||||
|
||||
if response.status() != reqwest::StatusCode::OK {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
anyhow::bail!("create user failed: {status} {body}");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
username,
|
||||
secret_key,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!(e))
|
||||
}
|
||||
|
||||
async fn admin_add_canned_policy(base_url: &str, policy_name: &str, policy: &serde_json::Value) -> Result<()> {
|
||||
let url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", base_url, policy_name);
|
||||
let response =
|
||||
signed_admin_request(http::Method::PUT, &url, Some(policy.to_string().into_bytes()), Some("application/json")).await?;
|
||||
|
||||
if response.status() != reqwest::StatusCode::OK {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
anyhow::bail!("add canned policy failed: {status} {body}");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
admin_add_canned_policy_via(
|
||||
AdminTransport::Signed,
|
||||
base_url,
|
||||
DEFAULT_ACCESS_KEY,
|
||||
DEFAULT_SECRET_KEY,
|
||||
policy_name,
|
||||
&policy.to_string(),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!(e))
|
||||
}
|
||||
|
||||
async fn admin_attach_policy_to_user(base_url: &str, policy_name: &str, username: &str) -> Result<()> {
|
||||
let url = format!(
|
||||
"{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false",
|
||||
base_url, policy_name, username
|
||||
);
|
||||
let response = signed_admin_request(http::Method::PUT, &url, Some(Vec::new()), None).await?;
|
||||
|
||||
if response.status() != reqwest::StatusCode::OK {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
anyhow::bail!("attach policy failed: {status} {body}");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
admin_attach_user_policy_via(
|
||||
AdminTransport::Signed,
|
||||
base_url,
|
||||
DEFAULT_ACCESS_KEY,
|
||||
DEFAULT_SECRET_KEY,
|
||||
policy_name,
|
||||
username,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!(e))
|
||||
}
|
||||
|
||||
/// Test WebDAV: MKCOL (create bucket), PUT, GET, DELETE, PROPFIND operations
|
||||
|
||||
@@ -46,7 +46,7 @@
|
||||
//! retry serves the object locally until expiry, and expiry leaves the
|
||||
//! remote object available for a second restore.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, local_http_client};
|
||||
use crate::common::RustFSTestEnvironment;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
@@ -56,10 +56,6 @@ use aws_sdk_s3::types::{
|
||||
VersioningConfiguration,
|
||||
};
|
||||
use http::Method;
|
||||
use http::header::HOST;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use serde::Deserialize;
|
||||
use std::time::{Duration as StdDuration, Instant};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
@@ -131,9 +127,8 @@ fn payload() -> Vec<u8> {
|
||||
|
||||
/// Sign and send an admin request in-process (no `awscurl`).
|
||||
///
|
||||
/// Mirrors the shared admin-API e2e pattern: the SigV4 signature is computed
|
||||
/// over `UNSIGNED_PAYLOAD`, so the JSON body rides on the wire without being
|
||||
/// pre-hashed. Returns the response status and body text.
|
||||
/// Thin wrapper over [`crate::common::admin_request`], kept local so the call
|
||||
/// sites below keep their `Option<&str>` body shape.
|
||||
async fn signed_admin_request(
|
||||
base_url: &str,
|
||||
method: Method,
|
||||
@@ -142,30 +137,7 @@ async fn signed_admin_request(
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(reqwest::StatusCode, String), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let url = format!("{base_url}{path}");
|
||||
let uri = url.parse::<http::Uri>()?;
|
||||
let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
|
||||
let body_bytes = body.map(|b| b.as_bytes().to_vec()).unwrap_or_default();
|
||||
|
||||
let request = http::Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(uri)
|
||||
.header(HOST, authority)
|
||||
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
|
||||
let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
|
||||
|
||||
let client = local_http_client();
|
||||
let mut request_builder = client.request(method, url.as_str());
|
||||
for (name, value) in signed.headers() {
|
||||
request_builder = request_builder.header(name, value);
|
||||
}
|
||||
if !body_bytes.is_empty() {
|
||||
request_builder = request_builder.body(body_bytes);
|
||||
}
|
||||
let response = request_builder.send().await?;
|
||||
let status = response.status();
|
||||
let text = response.text().await?;
|
||||
Ok((status, text))
|
||||
crate::common::admin_request(base_url, method, path, body.map(str::to_string), access_key, secret_key).await
|
||||
}
|
||||
|
||||
/// Wire `hot` -> `cold` as a `TierType::RustFS` remote tier via `AddTier`.
|
||||
|
||||
@@ -13,8 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::common::{
|
||||
RustFSTestEnvironment, admin_create_user, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
|
||||
replication_fast_env, rustfs_binary_path, signed_request, signed_request_with_client, signed_request_with_session_token,
|
||||
AdminTransport, RustFSTestEnvironment, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user,
|
||||
awscurl_post_sts_form_urlencoded, init_logging, local_http_client, replication_fast_env, rustfs_binary_path, signed_request,
|
||||
signed_request_with_client, signed_request_with_session_token,
|
||||
};
|
||||
use crate::fake_s3_target::{
|
||||
FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, FaultAction as FakeTargetFault, Operation as FakeTargetOperation,
|
||||
@@ -25,7 +26,7 @@ use crate::kms::common::{
|
||||
sse_customer_key_md5_base64,
|
||||
};
|
||||
use crate::storage_api::replication_extension::BucketTargetSys;
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
@@ -33,7 +34,6 @@ use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
|
||||
VersioningConfiguration,
|
||||
};
|
||||
use aws_sdk_s3::{Client, Config};
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use bytes::Bytes;
|
||||
use flate2::read::GzDecoder;
|
||||
@@ -895,15 +895,7 @@ async fn wait_for_replicated_object_over_https(
|
||||
}
|
||||
|
||||
fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
|
||||
let credentials = Credentials::new(access_key, secret_key, None, None, "e2e-site-replication");
|
||||
let config = Config::builder()
|
||||
.credentials_provider(credentials)
|
||||
.region(Region::new("us-east-1"))
|
||||
.endpoint_url(&env.url)
|
||||
.force_path_style(true)
|
||||
.behavior_version_latest()
|
||||
.build();
|
||||
Client::from_conf(config)
|
||||
env.create_s3_client_with_credentials(access_key, secret_key)
|
||||
}
|
||||
|
||||
async fn admin_add_canned_policy(
|
||||
@@ -911,24 +903,15 @@ async fn admin_add_canned_policy(
|
||||
policy_name: &str,
|
||||
policy: &serde_json::Value,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
|
||||
let response = signed_request(
|
||||
http::Method::PUT,
|
||||
&url,
|
||||
admin_add_canned_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
Some(policy.to_string().into_bytes()),
|
||||
Some("application/json"),
|
||||
policy_name,
|
||||
&policy.to_string(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
if response.status() != StatusCode::OK {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
return Err(format!("add canned policy failed: {status} {body}").into());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
.await
|
||||
}
|
||||
|
||||
async fn admin_attach_policy_to_user(
|
||||
@@ -936,19 +919,7 @@ async fn admin_attach_policy_to_user(
|
||||
policy_name: &str,
|
||||
username: &str,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let url = format!(
|
||||
"{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false",
|
||||
env.url, policy_name, username
|
||||
);
|
||||
let response = signed_request(http::Method::PUT, &url, &env.access_key, &env.secret_key, Some(Vec::new()), None).await?;
|
||||
|
||||
if response.status() != StatusCode::OK {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
return Err(format!("attach policy to user failed: {status} {body}").into());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
admin_attach_user_policy_via(AdminTransport::Signed, &env.url, &env.access_key, &env.secret_key, policy_name, username).await
|
||||
}
|
||||
|
||||
async fn admin_update_group_members(
|
||||
|
||||
Reference in New Issue
Block a user