mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-28 17:18:58 +00:00
fix(sts): return Query API-compatible responses (#5282)
* fix(sts): return Query API-compatible responses * fix(sts): resolve review and CI failures * fix(sts): harden query response conversion * test(e2e): stabilize transition conflict coverage * fix(ilm): retry vanished transition admission CAS * test(e2e): narrow transition overlap coverage --------- Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
@@ -209,7 +209,7 @@ test-group = 'ecstore-serial-flaky'
|
||||
[profile.e2e-smoke]
|
||||
default-filter = """
|
||||
package(e2e_test) & (
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools)_test::|^fake_s3_target::/)
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat)_test::|^fake_s3_target::/)
|
||||
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
|
||||
| test(/^reliant::lifecycle::/)
|
||||
| test(/^reliant::tiering::/)
|
||||
|
||||
Generated
+1
@@ -3662,6 +3662,7 @@ dependencies = [
|
||||
"async-trait",
|
||||
"aws-config",
|
||||
"aws-sdk-s3",
|
||||
"aws-sdk-sts",
|
||||
"aws-smithy-http-client",
|
||||
"base64 0.23.0",
|
||||
"bytes",
|
||||
|
||||
@@ -228,6 +228,7 @@ atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.10.1" }
|
||||
aws-credential-types = { version = "1.3.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.140.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.110.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.2.0" }
|
||||
aws-smithy-runtime-api = { version = "1.14.0" }
|
||||
aws-smithy-types = { version = "1.6.1" }
|
||||
|
||||
@@ -50,6 +50,7 @@ rustfs-filemeta.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
serial_test = { workspace = true }
|
||||
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
|
||||
aws-config = { workspace = true }
|
||||
aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }
|
||||
async-compression = { workspace = true, features = ["tokio", "bzip2", "xz"] }
|
||||
|
||||
@@ -101,6 +101,9 @@ mod admin_auth_test;
|
||||
#[cfg(test)]
|
||||
mod existing_object_tag_policy_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod sts_query_compat_test;
|
||||
|
||||
// Regression tests for Issue #2036: anonymous access with PublicAccessBlock
|
||||
#[cfg(test)]
|
||||
mod anonymous_access_test;
|
||||
|
||||
@@ -74,7 +74,6 @@ const MANUAL_ASYNC_STATUS_BUCKET: &str = "ilm7-manual-async-status";
|
||||
const MANUAL_CONTINUATION_BUCKET: &str = "ilm7-manual-continuation";
|
||||
const MANUAL_ASYNC_LIMIT_BUCKET: &str = "ilm7-manual-async-limit";
|
||||
const MANUAL_ASYNC_CONFLICT_BUCKET: &str = "ilm7-manual-async-conflict";
|
||||
const MANUAL_ASYNC_SAME_SCOPE_CONFLICT_BUCKET: &str = "ilm7-manual-async-same-scope-conflict";
|
||||
const MANUAL_ASYNC_PARALLEL_BUCKET_A: &str = "ilm7-manual-async-parallel-a";
|
||||
const MANUAL_ASYNC_PARALLEL_BUCKET_B: &str = "ilm7-manual-async-parallel-b";
|
||||
const MANUAL_TIER_FAILURE_BUCKET: &str = "ilm7-manual-tier-failure";
|
||||
@@ -86,7 +85,6 @@ const MANUAL_CONTINUATION_PREFIX: &str = "manual-continuation/";
|
||||
const MANUAL_ASYNC_LIMIT_PREFIX: &str = "manual-async-limit/";
|
||||
const MANUAL_ASYNC_CONFLICT_PREFIX: &str = "manual-async-conflict/";
|
||||
const MANUAL_ASYNC_CONFLICT_NESTED_PREFIX: &str = "manual-async-conflict/nested/";
|
||||
const MANUAL_ASYNC_SAME_SCOPE_CONFLICT_PREFIX: &str = "manual-async-same-scope-conflict/";
|
||||
const MANUAL_ASYNC_PARALLEL_PREFIX: &str = "manual-async-parallel/";
|
||||
const MANUAL_TIER_FAILURE_PREFIX: &str = "manual-tier-failure/";
|
||||
const MANUAL_WORKER_FAILURE_PREFIX: &str = "manual-worker-failure/";
|
||||
@@ -97,6 +95,7 @@ const MANUAL_ASYNC_PARALLEL_OBJECTS: usize = 64;
|
||||
const MANUAL_ACTIVE_CANCEL_OBJECTS: usize = 512;
|
||||
const MANUAL_RESTART_CANCEL_OBJECTS: usize = 512;
|
||||
const MANUAL_ACTIVE_CANCEL_RUNNING_TIMEOUT: StdDuration = StdDuration::from_secs(15);
|
||||
const MANUAL_ASYNC_CONFLICT_TERMINAL_TIMEOUT: StdDuration = StdDuration::from_secs(90);
|
||||
const MANUAL_RESTART_RECOVERY_TIMEOUT: StdDuration = StdDuration::from_secs(80);
|
||||
const OBJECT_KEY: &str = "tier/鲁A12345/report.bin";
|
||||
const MANUAL_DUE_KEY: &str = "manual-due/report.bin";
|
||||
@@ -1267,7 +1266,7 @@ async fn test_manual_transition_async_limit_reports_terminal_partial() -> TestRe
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn test_manual_transition_async_overlapping_scope_conflict_reports_active_job() -> TestResult {
|
||||
async fn test_manual_transition_async_scope_conflicts_report_active_job() -> TestResult {
|
||||
let mut cold = RustFSTestEnvironment::new().await?;
|
||||
cold.access_key = "manualasyncconflictcoldtieradmin".to_string();
|
||||
cold.secret_key = "manualasyncconflictcoldtiersecret".to_string();
|
||||
@@ -1296,34 +1295,14 @@ async fn test_manual_transition_async_overlapping_scope_conflict_reports_active_
|
||||
.await?;
|
||||
|
||||
let before_remote_count = cold_tier_object_count(&cold_client).await?;
|
||||
let (parent, nested) = tokio::join!(
|
||||
manual_transition_async_run_raw(
|
||||
&hot,
|
||||
MANUAL_ASYNC_CONFLICT_BUCKET,
|
||||
MANUAL_ASYNC_CONFLICT_PREFIX,
|
||||
false,
|
||||
MANUAL_ASYNC_CONFLICT_OBJECTS as u64,
|
||||
),
|
||||
manual_transition_async_run_raw(
|
||||
&hot,
|
||||
MANUAL_ASYNC_CONFLICT_BUCKET,
|
||||
MANUAL_ASYNC_CONFLICT_NESTED_PREFIX,
|
||||
false,
|
||||
MANUAL_ASYNC_CONFLICT_OBJECTS as u64,
|
||||
)
|
||||
);
|
||||
let responses = [parent?, nested?];
|
||||
let accepted = responses
|
||||
.iter()
|
||||
.find(|(status, _)| *status == reqwest::StatusCode::ACCEPTED)
|
||||
.ok_or("one concurrent overlapping-scope async run must be accepted")?;
|
||||
let conflict = responses
|
||||
.iter()
|
||||
.find(|(status, _)| *status == reqwest::StatusCode::CONFLICT)
|
||||
.ok_or("one concurrent overlapping-scope async run must report conflict")?;
|
||||
|
||||
let accepted: ManualTransitionRunResponse = serde_json::from_str(&accepted.1)?;
|
||||
let conflict: ManualTransitionJobConflictResponse = serde_json::from_str(&conflict.1)?;
|
||||
let accepted = manual_transition_async_run(
|
||||
&hot,
|
||||
MANUAL_ASYNC_CONFLICT_BUCKET,
|
||||
MANUAL_ASYNC_CONFLICT_PREFIX,
|
||||
false,
|
||||
MANUAL_ASYNC_CONFLICT_OBJECTS as u64,
|
||||
)
|
||||
.await?;
|
||||
let job_id = accepted
|
||||
.job_id
|
||||
.as_deref()
|
||||
@@ -1341,13 +1320,25 @@ async fn test_manual_transition_async_overlapping_scope_conflict_reports_active_
|
||||
assert_eq!(accepted.state, "accepted");
|
||||
assert_eq!(accepted.mode, "durable_job");
|
||||
assert_eq!(accepted.report.bucket, MANUAL_ASYNC_CONFLICT_BUCKET);
|
||||
assert!(
|
||||
matches!(
|
||||
accepted.report.prefix.as_str(),
|
||||
MANUAL_ASYNC_CONFLICT_PREFIX | MANUAL_ASYNC_CONFLICT_NESTED_PREFIX
|
||||
),
|
||||
"overlapping async winner prefix: {accepted:#?}"
|
||||
assert_eq!(accepted.report.prefix, MANUAL_ASYNC_CONFLICT_PREFIX);
|
||||
let active = wait_for_manual_transition_job_running(&hot, status_endpoint, MANUAL_ACTIVE_CANCEL_RUNNING_TIMEOUT).await?;
|
||||
assert_eq!(active.job_id, job_id);
|
||||
|
||||
let (conflict_status, conflict_body) = manual_transition_async_run_raw(
|
||||
&hot,
|
||||
MANUAL_ASYNC_CONFLICT_BUCKET,
|
||||
MANUAL_ASYNC_CONFLICT_NESTED_PREFIX,
|
||||
false,
|
||||
MANUAL_ASYNC_CONFLICT_OBJECTS as u64,
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
conflict_status,
|
||||
reqwest::StatusCode::CONFLICT,
|
||||
"active async run must reject nested prefix {}: {conflict_body}",
|
||||
MANUAL_ASYNC_CONFLICT_NESTED_PREFIX
|
||||
);
|
||||
let conflict: ManualTransitionJobConflictResponse = serde_json::from_str(&conflict_body)?;
|
||||
assert_eq!(conflict.state, "conflict");
|
||||
assert_eq!(conflict.mode, "durable_job");
|
||||
assert_eq!(conflict.active_job_id, job_id);
|
||||
@@ -1355,7 +1346,7 @@ async fn test_manual_transition_async_overlapping_scope_conflict_reports_active_
|
||||
assert_eq!(conflict.cancel_endpoint, status_endpoint);
|
||||
assert!(!conflict.scope_key.is_empty());
|
||||
|
||||
let terminal = wait_for_manual_transition_job_terminal(&hot, status_endpoint, StdDuration::from_secs(30)).await?;
|
||||
let terminal = wait_for_manual_transition_job_terminal(&hot, status_endpoint, MANUAL_ASYNC_CONFLICT_TERMINAL_TIMEOUT).await?;
|
||||
assert_eq!(terminal.job_id, job_id);
|
||||
assert!(!terminal.report.dry_run);
|
||||
assert_eq!(terminal.report.bucket, MANUAL_ASYNC_CONFLICT_BUCKET);
|
||||
@@ -1376,116 +1367,6 @@ async fn test_manual_transition_async_overlapping_scope_conflict_reports_active_
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn test_manual_transition_async_same_scope_conflict_reports_active_job() -> TestResult {
|
||||
let mut cold = RustFSTestEnvironment::new().await?;
|
||||
cold.access_key = "manualasyncsameconflictcoldadmin".to_string();
|
||||
cold.secret_key = "manualasyncsameconflictcoldsecret".to_string();
|
||||
cold.start_rustfs_server_without_cleanup(vec![]).await?;
|
||||
let cold_client = cold.create_s3_client();
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
|
||||
hot_client
|
||||
.create_bucket()
|
||||
.bucket(MANUAL_ASYNC_SAME_SCOPE_CONFLICT_BUCKET)
|
||||
.send()
|
||||
.await?;
|
||||
for idx in 0..50 {
|
||||
let key = format!("{MANUAL_ASYNC_SAME_SCOPE_CONFLICT_PREFIX}obj-{idx:02}");
|
||||
put_single_part_object(
|
||||
&hot_client,
|
||||
MANUAL_ASYNC_SAME_SCOPE_CONFLICT_BUCKET,
|
||||
&key,
|
||||
b"async same-scope conflict payload",
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
put_lifecycle_transition_rule(
|
||||
&hot_client,
|
||||
MANUAL_ASYNC_SAME_SCOPE_CONFLICT_BUCKET,
|
||||
"manual-async-same-scope-conflict",
|
||||
MANUAL_ASYNC_SAME_SCOPE_CONFLICT_PREFIX,
|
||||
0,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let (first, second) = tokio::join!(
|
||||
manual_transition_async_run_raw(
|
||||
&hot,
|
||||
MANUAL_ASYNC_SAME_SCOPE_CONFLICT_BUCKET,
|
||||
MANUAL_ASYNC_SAME_SCOPE_CONFLICT_PREFIX,
|
||||
false,
|
||||
50
|
||||
),
|
||||
manual_transition_async_run_raw(
|
||||
&hot,
|
||||
MANUAL_ASYNC_SAME_SCOPE_CONFLICT_BUCKET,
|
||||
MANUAL_ASYNC_SAME_SCOPE_CONFLICT_PREFIX,
|
||||
false,
|
||||
50
|
||||
)
|
||||
);
|
||||
let responses = [first?, second?];
|
||||
let accepted = responses
|
||||
.iter()
|
||||
.find(|(status, _)| *status == reqwest::StatusCode::ACCEPTED)
|
||||
.ok_or("one concurrent same-scope async run must be accepted")?;
|
||||
let conflict = responses
|
||||
.iter()
|
||||
.find(|(status, _)| *status == reqwest::StatusCode::CONFLICT)
|
||||
.ok_or("one concurrent same-scope async run must report conflict")?;
|
||||
|
||||
let accepted: ManualTransitionRunResponse = serde_json::from_str(&accepted.1)?;
|
||||
let conflict: ManualTransitionJobConflictResponse = serde_json::from_str(&conflict.1)?;
|
||||
let job_id = accepted
|
||||
.job_id
|
||||
.as_deref()
|
||||
.ok_or("accepted same-scope async response must include job_id")?;
|
||||
let status_endpoint = accepted
|
||||
.status_endpoint
|
||||
.as_deref()
|
||||
.ok_or("accepted same-scope async response must include status_endpoint")?;
|
||||
let cancel_endpoint = accepted
|
||||
.cancel_endpoint
|
||||
.as_deref()
|
||||
.ok_or("accepted same-scope async response must include cancel_endpoint")?;
|
||||
assert_eq!(cancel_endpoint, status_endpoint);
|
||||
|
||||
assert_eq!(accepted.state, "accepted");
|
||||
assert_eq!(accepted.mode, "durable_job");
|
||||
assert_eq!(accepted.report.bucket, MANUAL_ASYNC_SAME_SCOPE_CONFLICT_BUCKET);
|
||||
assert_eq!(accepted.report.prefix, MANUAL_ASYNC_SAME_SCOPE_CONFLICT_PREFIX);
|
||||
assert_eq!(conflict.state, "conflict");
|
||||
assert_eq!(conflict.mode, "durable_job");
|
||||
assert_eq!(conflict.active_job_id, job_id);
|
||||
assert_eq!(conflict.status_endpoint, status_endpoint);
|
||||
assert_eq!(conflict.cancel_endpoint, status_endpoint);
|
||||
assert!(!conflict.scope_key.is_empty());
|
||||
|
||||
let terminal = wait_for_manual_transition_job_terminal(&hot, status_endpoint, StdDuration::from_secs(30)).await?;
|
||||
assert_eq!(terminal.job_id, job_id);
|
||||
assert_eq!(terminal.report.bucket, MANUAL_ASYNC_SAME_SCOPE_CONFLICT_BUCKET);
|
||||
assert_eq!(terminal.report.prefix, MANUAL_ASYNC_SAME_SCOPE_CONFLICT_PREFIX);
|
||||
assert_conflict_winner_report(&terminal.status, &terminal.report, 50, "terminal same-scope conflict winner response");
|
||||
assert_eq!(
|
||||
terminal.report.transition_failed, 0,
|
||||
"terminal same-scope conflict winner response: {terminal:#?}"
|
||||
);
|
||||
assert_eq!(
|
||||
terminal.report.tier_failure, 0,
|
||||
"terminal same-scope conflict winner response: {terminal:#?}"
|
||||
);
|
||||
assert!(cold_tier_object_count(&cold_client).await? <= 50);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn test_manual_transition_async_different_buckets_admit_concurrently() -> TestResult {
|
||||
let mut cold = RustFSTestEnvironment::new().await?;
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use aws_sdk_sts::config::retry::RetryConfig;
|
||||
use aws_sdk_sts::config::{Credentials, Region};
|
||||
use aws_sdk_sts::error::ProvideErrorMetadata;
|
||||
use aws_sdk_sts::operation::RequestId;
|
||||
use aws_sdk_sts::{Client, Config};
|
||||
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use serial_test::serial;
|
||||
use std::error::Error;
|
||||
|
||||
type BoxError = Box<dyn Error + Send + Sync>;
|
||||
type TestResult = Result<(), BoxError>;
|
||||
|
||||
fn sts_client(url: &str, access_key: &str, secret_key: &str, session_token: Option<&str>) -> Client {
|
||||
let mut config = Config::builder()
|
||||
.credentials_provider(Credentials::new(
|
||||
access_key,
|
||||
secret_key,
|
||||
session_token.map(str::to_owned),
|
||||
None,
|
||||
"e2e-sts-query-compat",
|
||||
))
|
||||
.region(Region::new("us-east-1"))
|
||||
.endpoint_url(url)
|
||||
.retry_config(RetryConfig::standard().with_max_attempts(1))
|
||||
.behavior_version_latest();
|
||||
if url.starts_with("http://") {
|
||||
config = config.http_client(SmithyHttpClientBuilder::new().build_http());
|
||||
}
|
||||
Client::from_conf(config.build())
|
||||
}
|
||||
|
||||
async fn create_root_service_account(env: &RustFSTestEnvironment) -> Result<(String, String), BoxError> {
|
||||
let path = "/rustfs/admin/v3/add-service-accounts";
|
||||
let url = format!("{}{path}", env.url);
|
||||
let uri = url.parse::<http::Uri>()?;
|
||||
let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
|
||||
let body = serde_json::json!({ "targetUser": env.access_key.clone() }).to_string();
|
||||
let request = http::Request::builder()
|
||||
.method(http::Method::PUT)
|
||||
.uri(uri)
|
||||
.header(HOST, authority)
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
|
||||
.body(Body::empty())?;
|
||||
let content_length = i64::try_from(body.len()).map_err(|_| "service account request body is too large")?;
|
||||
let signed = sign_v4(request, content_length, &env.access_key, &env.secret_key, "", "us-east-1");
|
||||
let mut request = local_http_client().put(&url);
|
||||
for (name, value) in signed.headers() {
|
||||
request = request.header(name, value);
|
||||
}
|
||||
let response = request.body(body).send().await?;
|
||||
let status = response.status();
|
||||
let body = response.text().await?;
|
||||
if !status.is_success() {
|
||||
return Err(format!("create service account failed: {status} {body}").into());
|
||||
}
|
||||
|
||||
let response: serde_json::Value = serde_json::from_str(&body)?;
|
||||
let access_key = response["credentials"]["accessKey"]
|
||||
.as_str()
|
||||
.ok_or("service account response should contain credentials.accessKey")?
|
||||
.to_owned();
|
||||
let secret_key = response["credentials"]["secretKey"]
|
||||
.as_str()
|
||||
.ok_or("service account response should contain credentials.secretKey")?
|
||||
.to_owned();
|
||||
Ok((access_key, secret_key))
|
||||
}
|
||||
|
||||
async fn assert_chaining_denied(client: &Client, credential_kind: &str) -> TestResult {
|
||||
let error = client
|
||||
.assume_role()
|
||||
.role_arn("arn:aws:iam::123456789012:role/test")
|
||||
.role_session_name("sts-query-compat-e2e")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("credential chaining must be denied");
|
||||
let service_error = error
|
||||
.as_service_error()
|
||||
.ok_or_else(|| format!("{credential_kind} denial should deserialize as an STS service error: {error:?}"))?;
|
||||
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(403));
|
||||
assert_eq!(service_error.code(), Some("AccessDenied"));
|
||||
assert_eq!(service_error.message(), Some("Access Denied"));
|
||||
assert!(
|
||||
error.request_id().is_some_and(|request_id| !request_id.is_empty()),
|
||||
"{credential_kind} denial should include a request ID"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_sts_query_responses_are_aws_sdk_compatible() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let assumed = sts_client(&env.url, &env.access_key, &env.secret_key, None)
|
||||
.assume_role()
|
||||
.role_arn("arn:aws:iam::123456789012:role/test")
|
||||
.role_session_name("sts-query-compat-e2e")
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
assumed.request_id().is_some_and(|request_id| !request_id.is_empty()),
|
||||
"successful AssumeRole should include a request ID"
|
||||
);
|
||||
let temporary = assumed
|
||||
.credentials()
|
||||
.ok_or("successful AssumeRole response should contain credentials")?;
|
||||
|
||||
let invalid_signature = sts_client(&env.url, &env.access_key, "incorrect-secret-key", None)
|
||||
.assume_role()
|
||||
.role_arn("arn:aws:iam::123456789012:role/test")
|
||||
.role_session_name("sts-query-invalid-signature")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("an invalid signature must be rejected");
|
||||
assert_eq!(invalid_signature.raw_response().map(|response| response.status().as_u16()), Some(403));
|
||||
let invalid_signature_service_error = invalid_signature
|
||||
.as_service_error()
|
||||
.ok_or_else(|| format!("invalid signature should deserialize as an STS service error: {invalid_signature:?}"))?;
|
||||
assert_eq!(invalid_signature_service_error.code(), Some("SignatureDoesNotMatch"));
|
||||
assert!(
|
||||
invalid_signature_service_error
|
||||
.message()
|
||||
.is_some_and(|message| message.starts_with("The request signature we calculated does not match")),
|
||||
"signature rejection should preserve the canonical error message"
|
||||
);
|
||||
assert!(
|
||||
invalid_signature
|
||||
.request_id()
|
||||
.is_some_and(|request_id| !request_id.is_empty()),
|
||||
"signature rejection should include a request ID"
|
||||
);
|
||||
|
||||
assert_chaining_denied(
|
||||
&sts_client(
|
||||
&env.url,
|
||||
temporary.access_key_id(),
|
||||
temporary.secret_access_key(),
|
||||
Some(temporary.session_token()),
|
||||
),
|
||||
"temporary credential",
|
||||
)
|
||||
.await?;
|
||||
|
||||
let (service_access_key, service_secret_key) = create_root_service_account(&env).await?;
|
||||
assert_chaining_denied(&sts_client(&env.url, &service_access_key, &service_secret_key, None), "service account").await?;
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_sts_query_rate_limit_error_is_aws_sdk_compatible() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&[
|
||||
("RUSTFS_API_RATE_LIMIT_ENABLE", "true"),
|
||||
("RUSTFS_API_RATE_LIMIT_RPM", "60"),
|
||||
("RUSTFS_API_RATE_LIMIT_BURST", "1"),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
|
||||
let client = sts_client(&env.url, &env.access_key, &env.secret_key, None);
|
||||
let mut throttled = None;
|
||||
let request = || {
|
||||
client
|
||||
.assume_role()
|
||||
.role_arn("arn:aws:iam::123456789012:role/test")
|
||||
.role_session_name("sts-query-rate-limit")
|
||||
.send()
|
||||
};
|
||||
let (first, second, third, fourth) = tokio::join!(request(), request(), request(), request());
|
||||
for result in [first, second, third, fourth] {
|
||||
if let Err(error) = result
|
||||
&& error.raw_response().map(|response| response.status().as_u16()) == Some(429)
|
||||
{
|
||||
throttled = Some(error);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let error = throttled.ok_or("at least one concurrent STS request should be throttled at burst one")?;
|
||||
let service_error = error
|
||||
.as_service_error()
|
||||
.ok_or_else(|| format!("rate limit response should deserialize as an STS service error: {error:?}"))?;
|
||||
assert_eq!(service_error.code(), Some("TooManyRequests"));
|
||||
assert!(
|
||||
service_error
|
||||
.message()
|
||||
.is_some_and(|message| message.starts_with("Request rate limit exceeded")),
|
||||
"rate limit response should preserve the server message"
|
||||
);
|
||||
assert!(
|
||||
error.request_id().is_some_and(|request_id| !request_id.is_empty()),
|
||||
"rate limit response should include a request ID"
|
||||
);
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
@@ -9483,6 +9483,48 @@ mod tests {
|
||||
assert_eq!(current.lease_id, second.lease_id);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn manual_transition_scope_missing_current_is_retryable_cas_miss() {
|
||||
let (_paths, ecstore) = setup_test_env().await;
|
||||
let options = ManualTransitionRunOptions {
|
||||
prefix: "logs/".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let first = ManualTransitionJobRecord::new(Uuid::new_v4(), "manual-replace-race-bucket", &options, "first-owner");
|
||||
save_manual_transition_scope_admission_if_absent(ecstore.clone(), &ManualTransitionScopeAdmission::from_job(&first))
|
||||
.await
|
||||
.expect("first scope admission should save");
|
||||
let (_loaded, etag) = load_manual_transition_scope_admission_with_etag(ecstore.clone(), &first.scope_key)
|
||||
.await
|
||||
.expect("first scope admission should load with an ETag");
|
||||
let object = manual_transition_scope_record_object_name(&first.scope_key).expect("scope admission path should encode");
|
||||
config_boundary::delete_config(ecstore.clone(), &object)
|
||||
.await
|
||||
.expect("current scope admission should be deleted");
|
||||
|
||||
let replacement =
|
||||
ManualTransitionJobRecord::new(Uuid::new_v4(), "manual-replace-race-bucket", &options, "replacement-owner");
|
||||
let stale_replace = save_manual_transition_scope_admission_if_current(
|
||||
ecstore.clone(),
|
||||
&ManualTransitionScopeAdmission::from_job(&replacement),
|
||||
&etag,
|
||||
)
|
||||
.await
|
||||
.expect_err("replacing a disappeared scope admission must report a CAS miss");
|
||||
assert_eq!(stale_replace, Error::PreconditionFailed);
|
||||
|
||||
let claim =
|
||||
claim_manual_transition_scope_admission(ecstore.clone(), &ManualTransitionScopeAdmission::from_job(&replacement))
|
||||
.await
|
||||
.expect("replacement claim should recover through the create path");
|
||||
assert_eq!(claim, ManualTransitionScopeAdmissionClaim::Claimed);
|
||||
let current = load_manual_transition_scope_admission(ecstore, &replacement.scope_key)
|
||||
.await
|
||||
.expect("replacement scope admission should be saved");
|
||||
assert_eq!(current.job_id, replacement.job_id);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn manual_transition_admission_reclaims_stale_scope_records() {
|
||||
|
||||
@@ -1466,7 +1466,7 @@ pub async fn save_manual_transition_scope_admission_if_current(
|
||||
admission.validate().map_err(manual_transition_job_store_error)?;
|
||||
let object = manual_transition_scope_record_object_name(&admission.scope_key).map_err(manual_transition_job_store_error)?;
|
||||
let data = serde_json::to_vec(admission).map_err(Error::other)?;
|
||||
config_boundary::save_config_with_opts(
|
||||
match config_boundary::save_config_with_opts(
|
||||
api,
|
||||
&object,
|
||||
data,
|
||||
@@ -1480,6 +1480,12 @@ pub async fn save_manual_transition_scope_admission_if_current(
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Err(Error::ObjectNotFound(bucket, current_object)) if bucket == RUSTFS_META_BUCKET && current_object == object => {
|
||||
Err(Error::PreconditionFailed)
|
||||
}
|
||||
result => result,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn claim_manual_transition_scope_admission(
|
||||
|
||||
@@ -162,13 +162,13 @@ async fn handle_assume_role(
|
||||
|
||||
let session_token = get_session_token(&uri, &headers);
|
||||
if session_token.is_some() {
|
||||
return Err(s3_error!(InvalidRequest, "AccessDenied1"));
|
||||
return Err(s3_error!(AccessDenied, "Access Denied"));
|
||||
}
|
||||
|
||||
let (cred, owner) = check_key_valid(get_session_token(&uri, &headers).unwrap_or_default(), &user.access_key).await?;
|
||||
|
||||
if cred.is_temp() || cred.is_service_account() {
|
||||
return Err(s3_error!(InvalidRequest, "AccessDenied"));
|
||||
return Err(s3_error!(AccessDenied, "Access Denied"));
|
||||
}
|
||||
|
||||
let Ok(iam_store) = crate::admin::runtime_sources::current_ready_iam_handle() else {
|
||||
|
||||
+40
-25
@@ -38,6 +38,7 @@ use crate::error::ApiError;
|
||||
use crate::license::license_check;
|
||||
use crate::server::{
|
||||
ADMIN_PREFIX, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_ADMIN_PREFIX, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH, is_admin_path,
|
||||
is_sts_query_request,
|
||||
};
|
||||
use crate::storage::storage_api::lock_bucket_targets_metadata;
|
||||
use aws_sdk_s3::primitives::ByteStream as AwsByteStream;
|
||||
@@ -2769,15 +2770,7 @@ where
|
||||
}
|
||||
|
||||
// AssumeRole
|
||||
if method == Method::POST
|
||||
&& path == "/"
|
||||
&& headers
|
||||
.get(header::CONTENT_TYPE)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(|ct| ct.split(';').next().unwrap_or("").trim().to_lowercase())
|
||||
.map(|ct| ct == "application/x-www-form-urlencoded")
|
||||
.unwrap_or(false)
|
||||
{
|
||||
if is_sts_query_request(method, uri, headers) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -2827,22 +2820,7 @@ where
|
||||
// The handler dispatches on the Action parameter: AssumeRole will reject if
|
||||
// credentials are missing, AssumeRoleWithWebIdentity will validate the JWT.
|
||||
// Require application/x-www-form-urlencoded Content-Type to narrow the bypass.
|
||||
if req.method == Method::POST
|
||||
&& path == "/"
|
||||
&& req.credentials.is_none()
|
||||
&& req
|
||||
.headers
|
||||
.get(header::CONTENT_TYPE)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(|ct| {
|
||||
ct.split(';')
|
||||
.next()
|
||||
.unwrap_or("")
|
||||
.trim()
|
||||
.eq_ignore_ascii_case("application/x-www-form-urlencoded")
|
||||
})
|
||||
.unwrap_or(false)
|
||||
{
|
||||
if req.credentials.is_none() && is_sts_query_request(&req.method, &req.uri, &req.headers) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -4794,6 +4772,43 @@ mod tests {
|
||||
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn check_access_limits_anonymous_sts_exemption_to_form_content_type() {
|
||||
let router: S3Router<AdminOperation> = S3Router::new(false);
|
||||
let request = |content_type: Option<&'static str>| {
|
||||
let mut headers = HeaderMap::new();
|
||||
if let Some(content_type) = content_type {
|
||||
headers.insert(http::header::CONTENT_TYPE, HeaderValue::from_static(content_type));
|
||||
}
|
||||
S3Request {
|
||||
input: Body::from(String::from("Action=AssumeRoleWithWebIdentity")),
|
||||
method: Method::POST,
|
||||
uri: Uri::from_static("/"),
|
||||
headers,
|
||||
extensions: http::Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
}
|
||||
};
|
||||
|
||||
for content_type in [None, Some("application/json")] {
|
||||
let mut req = request(content_type);
|
||||
let err = router
|
||||
.check_access(&mut req)
|
||||
.await
|
||||
.expect_err("anonymous STS request without form content type must be denied");
|
||||
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
|
||||
}
|
||||
|
||||
let mut req = request(Some("application/x-www-form-urlencoded"));
|
||||
router
|
||||
.check_access(&mut req)
|
||||
.await
|
||||
.expect("anonymous form-encoded STS request should reach identity validation");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn check_access_allows_object_zip_download_token_navigation() {
|
||||
let router: S3Router<AdminOperation> = S3Router::new(false);
|
||||
|
||||
@@ -407,6 +407,8 @@ pub(crate) enum PathCategory {
|
||||
InternodeRpc,
|
||||
/// Health/probe paths — skip compression (tiny responses)
|
||||
Probe,
|
||||
/// STS Query API — response compatibility is applied outside compression.
|
||||
StsQueryApi,
|
||||
}
|
||||
|
||||
impl PathCategory {
|
||||
@@ -544,7 +546,14 @@ where
|
||||
}
|
||||
|
||||
fn call(&mut self, req: Request<ReqBody>) -> Self::Future {
|
||||
let category = PathCategory::classify(req.uri().path());
|
||||
let category = if req.method() == http::Method::POST
|
||||
&& req.uri().path() == "/"
|
||||
&& req.extensions().get::<crate::server::layer::StsQueryRequest>().is_some()
|
||||
{
|
||||
PathCategory::StsQueryApi
|
||||
} else {
|
||||
PathCategory::classify(req.uri().path())
|
||||
};
|
||||
InjectCategoryFut {
|
||||
inner: self.inner.call(req),
|
||||
category,
|
||||
@@ -785,5 +794,6 @@ mod tests {
|
||||
assert!(!PathCategory::Console.should_evaluate_compression());
|
||||
assert!(!PathCategory::InternodeRpc.should_evaluate_compression());
|
||||
assert!(!PathCategory::Probe.should_evaluate_compression());
|
||||
assert!(!PathCategory::StsQueryApi.should_evaluate_compression());
|
||||
}
|
||||
}
|
||||
|
||||
+24
-22
@@ -24,8 +24,8 @@ use crate::server::{
|
||||
layer::{
|
||||
BodylessStatusFixLayer, ConditionalCorsLayer, DoubleSlashListBucketsCompatLayer, EmptyBodyContentLengthCompatLayer,
|
||||
HeadRequestBodyFixLayer, IcebergRestErrorCompatLayer, ObjectAttributesEtagFixLayer, PublicHealthEndpointLayer,
|
||||
RedirectLayer, RequestContextLayer, RequestLoggingLayer, S3ErrorMessageCompatLayer, VirtualHostStyleHintLayer,
|
||||
redact_sensitive_uri_query,
|
||||
RedirectLayer, RequestContextLayer, RequestLoggingLayer, S3ErrorMessageCompatLayer, StsQueryApiCompatLayer,
|
||||
VirtualHostStyleHintLayer, redact_sensitive_uri_query,
|
||||
},
|
||||
rate_limit::{RateLimitLayer, api_rate_limit_layer_from_env},
|
||||
tls_material::{
|
||||
@@ -1390,26 +1390,27 @@ fn process_connection(
|
||||
// 3. TrustedProxyLayer — conditional, parses X-Forwarded-For
|
||||
// 4. SetRequestIdLayer — generates X-Request-ID
|
||||
// 5. RequestContextLayer — creates RequestContext in extensions
|
||||
// 6. EmptyBodyContentLengthCompatLayer — adds Content-Length: 0 for known empty-body API routes
|
||||
// 7. CatchPanicLayer — panic → 500
|
||||
// 8. RateLimitLayer — conditional (external stack only), per-client 429 throttling
|
||||
// 9. ReadinessGateLayer — blocks until ready
|
||||
// 10. KeystoneAuthLayer — X-Auth-Token validation
|
||||
// 11. TraceLayer — request span creation + metrics
|
||||
// 12. RequestLoggingLayer — single completion event per request
|
||||
// 13. PropagateRequestIdLayer — X-Request-ID → response
|
||||
// 14. CompressionLayer — response compression (whitelist, path-aware)
|
||||
// 15. PathCategoryInjectionLayer — injects path category for compression predicate
|
||||
// 16. S3ErrorMessageCompatLayer — missing S3 error message compatibility
|
||||
// 17. IcebergRestErrorCompatLayer — Iceberg REST JSON error compatibility
|
||||
// 18. ObjectAttributesEtagFixLayer — ETag fix for GetObjectAttributes
|
||||
// 19. ConditionalCorsLayer — S3 API CORS
|
||||
// 20. RedirectLayer — console redirect (conditional)
|
||||
// 21. BodylessStatusFixLayer — clears body for 1xx/204/205/304 responses
|
||||
// 22. HeadRequestBodyFixLayer — strips actual body bytes from HEAD responses
|
||||
// 23. PublicHealthEndpointLayer — handles public health before s3s host parsing
|
||||
// 24. VirtualHostStyleHintLayer — actionable error for unroutable virtual-hosted-style (conditional)
|
||||
// 25. DoubleSlashListBucketsCompatLayer — rewrites `GET //` to `GET /` for ListBuckets (MinIO browser compat)
|
||||
// 6. StsQueryApiCompatLayer — route-scoped STS envelopes, including outer short-circuit errors
|
||||
// 7. EmptyBodyContentLengthCompatLayer — adds Content-Length: 0 for known empty-body API routes
|
||||
// 8. CatchPanicLayer — panic → 500
|
||||
// 9. RateLimitLayer — conditional (external stack only), per-client 429 throttling
|
||||
// 10. ReadinessGateLayer — blocks until ready
|
||||
// 11. KeystoneAuthLayer — X-Auth-Token validation
|
||||
// 12. TraceLayer — request span creation + metrics
|
||||
// 13. RequestLoggingLayer — single completion event per request
|
||||
// 14. PropagateRequestIdLayer — X-Request-ID → response
|
||||
// 15. CompressionLayer — response compression (whitelist, path-aware)
|
||||
// 16. PathCategoryInjectionLayer — injects path category for compression predicate
|
||||
// 17. S3ErrorMessageCompatLayer — missing S3 error message compatibility
|
||||
// 18. IcebergRestErrorCompatLayer — Iceberg REST JSON error compatibility
|
||||
// 19. ObjectAttributesEtagFixLayer — ETag fix for GetObjectAttributes
|
||||
// 20. ConditionalCorsLayer — S3 API CORS
|
||||
// 21. RedirectLayer — console redirect (conditional)
|
||||
// 22. BodylessStatusFixLayer — clears body for 1xx/204/205/304 responses
|
||||
// 23. HeadRequestBodyFixLayer — strips actual body bytes from HEAD responses
|
||||
// 24. PublicHealthEndpointLayer — handles public health before s3s host parsing
|
||||
// 25. VirtualHostStyleHintLayer — actionable error for unroutable virtual-hosted-style (conditional)
|
||||
// 26. DoubleSlashListBucketsCompatLayer — rewrites `GET //` to `GET /` for ListBuckets (MinIO browser compat)
|
||||
// ─────────────────────────────────────────────────────────────
|
||||
// Batch 1 intentionally keeps the external and internode stacks behaviorally
|
||||
// identical while giving each path family a named construction boundary.
|
||||
@@ -1431,6 +1432,7 @@ fn process_connection(
|
||||
.option_layer(trusted_proxy_layer.clone())
|
||||
.layer(SetRequestIdLayer::x_request_id(MakeRequestUuid))
|
||||
.layer(InternodeRequestContextLiteLayer)
|
||||
.layer(StsQueryApiCompatLayer)
|
||||
.layer(EmptyBodyContentLengthCompatLayer)
|
||||
.layer(CatchPanicLayer::new())
|
||||
// Per-client API rate limit (backlog#1191): rejects over-limit
|
||||
|
||||
+559
-3
@@ -28,16 +28,19 @@ use crate::storage_api::server::layer::apply_cors_headers;
|
||||
use crate::storage_api::server::layer::request_context::{
|
||||
RequestContext, extract_request_id_from_headers, extract_trace_context_ids_from_headers, spawn_traced,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use futures::future::Either;
|
||||
use http::{HeaderMap, HeaderValue, Method, Request as HttpRequest, Response, StatusCode, Uri};
|
||||
use http_body::Body;
|
||||
use http_body_util::BodyExt;
|
||||
use http_body_util::{BodyExt, Full};
|
||||
use hyper::body::Incoming;
|
||||
use quick_xml::events::Event;
|
||||
use rustfs_trusted_proxies::ClientInfo;
|
||||
use rustfs_utils::get_env_opt_str;
|
||||
use rustfs_utils::http::headers::AMZ_REQUEST_ID;
|
||||
use s3s::S3ErrorCode;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::borrow::Cow;
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
@@ -56,6 +59,9 @@ const LOG_SUBSYSTEM_HTTP: &str = "http";
|
||||
const REDACTED_QUERY_VALUE: &str = "redacted";
|
||||
const OBJECT_ZIP_DOWNLOADS_PATH: &str = "/v3/object-zip-downloads/";
|
||||
const HTTP_REQUEST_INFLIGHT_WARN_THRESHOLD: Duration = Duration::from_secs(5);
|
||||
const STS_RESPONSE_METADATA_TAG: &str = "ResponseMetadata";
|
||||
const STS_REQUEST_ID_TAG: &str = "RequestId";
|
||||
const STS_SUCCESS_RESPONSE_TAGS: [&str; 2] = ["AssumeRoleResponse", "AssumeRoleWithWebIdentityResponse"];
|
||||
|
||||
pub(crate) fn redact_sensitive_uri_query(uri: &http::Uri) -> String {
|
||||
let path = uri.path();
|
||||
@@ -601,12 +607,14 @@ where
|
||||
}
|
||||
|
||||
fn call(&mut self, req: HttpRequest<Incoming>) -> Self::Future {
|
||||
let is_sts_query =
|
||||
req.method() == Method::POST && req.uri().path() == "/" && req.extensions().get::<StsQueryRequest>().is_some();
|
||||
let mut inner = self.inner.clone();
|
||||
|
||||
Box::pin(async move {
|
||||
let response = inner.call(req).await?;
|
||||
let (parts, body) = response.into_parts();
|
||||
let should_fix = parts.status == StatusCode::FORBIDDEN && is_xml_response(&parts.headers);
|
||||
let should_fix = !is_sts_query && parts.status == StatusCode::FORBIDDEN && is_xml_response(&parts.headers);
|
||||
|
||||
let response = match body {
|
||||
HybridBody::Rest { rest_body } => {
|
||||
@@ -629,6 +637,81 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct StsQueryApiCompatLayer;
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub(crate) struct StsQueryRequest;
|
||||
|
||||
pub(crate) fn is_sts_query_request(method: &Method, uri: &Uri, headers: &HeaderMap) -> bool {
|
||||
method == Method::POST
|
||||
&& uri.path() == "/"
|
||||
&& headers
|
||||
.get(http::header::CONTENT_TYPE)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.split(';').next())
|
||||
.is_some_and(|value| value.trim().eq_ignore_ascii_case("application/x-www-form-urlencoded"))
|
||||
}
|
||||
|
||||
impl<S> Layer<S> for StsQueryApiCompatLayer {
|
||||
type Service = StsQueryApiCompatService<S>;
|
||||
|
||||
fn layer(&self, inner: S) -> Self::Service {
|
||||
StsQueryApiCompatService { inner }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct StsQueryApiCompatService<S> {
|
||||
inner: S,
|
||||
}
|
||||
|
||||
type StsBoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
type StsBoxBody = tower_http::body::UnsyncBoxBody<Bytes, StsBoxError>;
|
||||
|
||||
impl<S, RequestBody> Service<HttpRequest<RequestBody>> for StsQueryApiCompatService<S>
|
||||
where
|
||||
S: Service<HttpRequest<RequestBody>, Response = Response<StsBoxBody>> + Send + 'static,
|
||||
S::Future: Send + 'static,
|
||||
S::Error: From<StsBoxError> + Send + 'static,
|
||||
{
|
||||
type Response = Response<StsBoxBody>;
|
||||
type Error = S::Error;
|
||||
type Future = Either<S::Future, Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>>;
|
||||
|
||||
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
self.inner.poll_ready(cx)
|
||||
}
|
||||
|
||||
fn call(&mut self, mut req: HttpRequest<RequestBody>) -> Self::Future {
|
||||
if !is_sts_query_request(req.method(), req.uri(), req.headers()) {
|
||||
return Either::Left(self.inner.call(req));
|
||||
}
|
||||
|
||||
req.extensions_mut().insert(StsQueryRequest);
|
||||
let (request_id, request_id_header) = sts_response_request_id(&req);
|
||||
let future = self.inner.call(req);
|
||||
|
||||
Either::Right(Box::pin(async move {
|
||||
let response = future.await?;
|
||||
let (mut parts, body) = response.into_parts();
|
||||
parts.headers.insert(AMZ_REQUEST_ID, request_id_header);
|
||||
|
||||
let (body, changed) = convert_sts_query_response(body, parts.status, &request_id).await?;
|
||||
if changed {
|
||||
parts.headers.remove(http::header::CONTENT_LENGTH);
|
||||
parts
|
||||
.headers
|
||||
.insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/xml"));
|
||||
}
|
||||
let body = tower_http::body::UnsyncBoxBody::new(Full::from(body).map_err(|error| -> StsBoxError { match error {} }));
|
||||
let response = Response::from_parts(parts, body);
|
||||
|
||||
Ok(response)
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct IcebergRestErrorCompatLayer;
|
||||
|
||||
@@ -1338,6 +1421,153 @@ fn is_xml_response(headers: &HeaderMap) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
fn sts_response_request_id<B>(req: &HttpRequest<B>) -> (String, HeaderValue) {
|
||||
let request_id = req
|
||||
.extensions()
|
||||
.get::<RequestContext>()
|
||||
.map(|context| context.request_id.as_str())
|
||||
.filter(|request_id| !request_id.trim().is_empty())
|
||||
.unwrap_or("sts-request-id")
|
||||
.to_owned();
|
||||
|
||||
match HeaderValue::from_str(&request_id) {
|
||||
Ok(header) => (request_id, header),
|
||||
Err(_) => ("sts-request-id".to_owned(), HeaderValue::from_static("sts-request-id")),
|
||||
}
|
||||
}
|
||||
|
||||
async fn convert_sts_query_response<RestBody>(
|
||||
body: RestBody,
|
||||
status: StatusCode,
|
||||
request_id: &str,
|
||||
) -> Result<(Bytes, bool), RestBody::Error>
|
||||
where
|
||||
RestBody: Body<Data = Bytes>,
|
||||
{
|
||||
let bytes = BodyExt::collect(body).await?.to_bytes();
|
||||
let Ok(xml) = std::str::from_utf8(&bytes) else {
|
||||
return Ok((bytes, false));
|
||||
};
|
||||
|
||||
let converted = if status.is_success() {
|
||||
add_sts_response_metadata(xml, request_id)
|
||||
} else {
|
||||
Some(Bytes::from(wrap_sts_error_response(xml, status, request_id)))
|
||||
};
|
||||
|
||||
match converted {
|
||||
Some(xml) => Ok((xml, true)),
|
||||
None => Ok((bytes, false)),
|
||||
}
|
||||
}
|
||||
|
||||
fn add_sts_response_metadata(xml: &str, request_id: &str) -> Option<Bytes> {
|
||||
let mut reader = quick_xml::Reader::from_str(xml);
|
||||
let mut depth = 0_usize;
|
||||
let mut root_prefix = None;
|
||||
|
||||
loop {
|
||||
let event_start = usize::try_from(reader.buffer_position()).ok()?;
|
||||
match reader.read_event() {
|
||||
Ok(Event::Start(element)) => {
|
||||
if depth == 0 {
|
||||
if !is_sts_success_response_tag(element.local_name().as_ref()) {
|
||||
return None;
|
||||
}
|
||||
root_prefix = element
|
||||
.name()
|
||||
.prefix()
|
||||
.map(|prefix| std::str::from_utf8(prefix.as_ref()).map(str::to_owned))
|
||||
.transpose()
|
||||
.ok()?;
|
||||
} else if depth == 1 && element.local_name().as_ref() == STS_RESPONSE_METADATA_TAG.as_bytes() {
|
||||
return None;
|
||||
}
|
||||
depth += 1;
|
||||
}
|
||||
Ok(Event::Empty(element)) => {
|
||||
if depth == 0 {
|
||||
return None;
|
||||
}
|
||||
if depth == 1 && element.local_name().as_ref() == STS_RESPONSE_METADATA_TAG.as_bytes() {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
Ok(Event::End(element)) => {
|
||||
if depth == 0 {
|
||||
return None;
|
||||
}
|
||||
if depth == 1 {
|
||||
if !is_sts_success_response_tag(element.local_name().as_ref()) {
|
||||
return None;
|
||||
}
|
||||
let prefix = root_prefix.as_deref().map(|prefix| format!("{prefix}:")).unwrap_or_default();
|
||||
let metadata = format!(
|
||||
"<{prefix}{STS_RESPONSE_METADATA_TAG}>\
|
||||
<{prefix}{STS_REQUEST_ID_TAG}>{request_id}</{prefix}{STS_REQUEST_ID_TAG}>\
|
||||
</{prefix}{STS_RESPONSE_METADATA_TAG}>",
|
||||
request_id = xml_escape(request_id),
|
||||
);
|
||||
let capacity = xml.len().checked_add(metadata.len())?;
|
||||
let mut converted = BytesMut::with_capacity(capacity);
|
||||
converted.extend_from_slice(&xml.as_bytes()[..event_start]);
|
||||
converted.extend_from_slice(metadata.as_bytes());
|
||||
converted.extend_from_slice(&xml.as_bytes()[event_start..]);
|
||||
return Some(converted.freeze());
|
||||
}
|
||||
depth -= 1;
|
||||
}
|
||||
Ok(Event::Eof) | Err(_) => return None,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_sts_success_response_tag(tag: &[u8]) -> bool {
|
||||
STS_SUCCESS_RESPONSE_TAGS.iter().any(|candidate| candidate.as_bytes() == tag)
|
||||
}
|
||||
|
||||
fn wrap_sts_error_response(xml: &str, status: StatusCode, request_id: &str) -> String {
|
||||
let parsed = quick_xml::de::from_str::<SerializedS3Error>(xml).ok();
|
||||
let (code, message) = parsed
|
||||
.as_ref()
|
||||
.map(|error| {
|
||||
let message = match error.message.as_deref() {
|
||||
Some(message) => Cow::Borrowed(message),
|
||||
None if error.code == S3ErrorCode::SignatureDoesNotMatch.as_str() => {
|
||||
Cow::Owned(ApiError::error_code_to_message(&S3ErrorCode::SignatureDoesNotMatch))
|
||||
}
|
||||
None => Cow::Borrowed(error.code.as_str()),
|
||||
};
|
||||
(error.code.as_str(), message)
|
||||
})
|
||||
.unwrap_or_else(|| {
|
||||
let (code, message) = sts_error_for_status(status);
|
||||
(code, Cow::Borrowed(message))
|
||||
});
|
||||
let error_type = if status.is_server_error() { "Receiver" } else { "Sender" };
|
||||
|
||||
format!(
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
|
||||
<ErrorResponse xmlns=\"https://sts.amazonaws.com/doc/2011-06-15/\">\
|
||||
<Error><Type>{error_type}</Type><Code>{code}</Code><Message>{message}</Message></Error>\
|
||||
<RequestId>{request_id}</RequestId></ErrorResponse>",
|
||||
code = xml_escape(code),
|
||||
message = xml_escape(&message),
|
||||
request_id = xml_escape(request_id),
|
||||
)
|
||||
}
|
||||
|
||||
fn sts_error_for_status(status: StatusCode) -> (&'static str, &'static str) {
|
||||
match status {
|
||||
StatusCode::BAD_REQUEST => ("InvalidRequest", "Invalid Request"),
|
||||
StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => ("AccessDenied", "Access Denied"),
|
||||
StatusCode::TOO_MANY_REQUESTS => ("TooManyRequests", "Request rate limit exceeded"),
|
||||
StatusCode::SERVICE_UNAVAILABLE => ("ServiceUnavailable", "Service Unavailable"),
|
||||
_ => ("InternalError", "Internal Server Error"),
|
||||
}
|
||||
}
|
||||
|
||||
async fn fix_object_attributes_etag_in_xml<RestBody>(body: RestBody) -> Result<RestBody, RestBody::Error>
|
||||
where
|
||||
RestBody: Body<Data = Bytes> + From<Bytes>,
|
||||
@@ -1780,6 +2010,7 @@ fn rewrite_double_slash_root(uri: &Uri) -> Option<Uri> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::server::compress::{HttpCompressionConfig, PathAwareHttpCompressionPredicate, PathCategoryInjectionLayer};
|
||||
use crate::server::{FAVICON_PATH, LICENSE, RemoteAddr, VERSION};
|
||||
use futures::future::{Ready, ready};
|
||||
use http::Request;
|
||||
@@ -2884,6 +3115,331 @@ mod tests {
|
||||
assert_eq!(bytes, input);
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct FixedStsResponse {
|
||||
status: StatusCode,
|
||||
body: Bytes,
|
||||
upstream_request_id: Option<&'static str>,
|
||||
}
|
||||
|
||||
impl<RequestBody: Send + 'static> Service<Request<RequestBody>> for FixedStsResponse {
|
||||
type Response = Response<StsBoxBody>;
|
||||
type Error = StsBoxError;
|
||||
type Future = Ready<Result<Self::Response, Self::Error>>;
|
||||
|
||||
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn call(&mut self, _req: Request<RequestBody>) -> Self::Future {
|
||||
let body = tower_http::body::UnsyncBoxBody::new(
|
||||
Full::from(self.body.clone()).map_err(|error| -> StsBoxError { match error {} }),
|
||||
);
|
||||
let mut response = Response::builder()
|
||||
.status(self.status)
|
||||
.header(http::header::CONTENT_TYPE, "application/xml")
|
||||
.header(http::header::CONTENT_LENGTH, self.body.len());
|
||||
if let Some(request_id) = self.upstream_request_id {
|
||||
response = response.header(AMZ_REQUEST_ID, request_id);
|
||||
}
|
||||
ready(Ok(response.body(body).expect("fixed STS response")))
|
||||
}
|
||||
}
|
||||
|
||||
async fn call_sts_compat_layer(
|
||||
status: StatusCode,
|
||||
body: &'static [u8],
|
||||
upstream_request_id: Option<&'static str>,
|
||||
) -> (HeaderMap, String) {
|
||||
let mut service = StsQueryApiCompatLayer.layer(FixedStsResponse {
|
||||
status,
|
||||
body: Bytes::from_static(body),
|
||||
upstream_request_id,
|
||||
});
|
||||
let mut request = Request::builder()
|
||||
.method(Method::POST)
|
||||
.uri("/")
|
||||
.header(http::header::CONTENT_TYPE, "application/x-www-form-urlencoded")
|
||||
.body(())
|
||||
.expect("STS request");
|
||||
request.extensions_mut().insert(RequestContext {
|
||||
request_id: "server-request-id".to_string(),
|
||||
x_amz_request_id: "untrusted-upstream-id".to_string(),
|
||||
trace_id: None,
|
||||
span_id: None,
|
||||
start_time: Instant::now(),
|
||||
});
|
||||
|
||||
let response = service.call(request).await.expect("STS compatibility response");
|
||||
let headers = response.headers().clone();
|
||||
let body = BodyExt::collect(response.into_body())
|
||||
.await
|
||||
.expect("collect STS response")
|
||||
.to_bytes();
|
||||
(headers, String::from_utf8(body.to_vec()).expect("STS response should be UTF-8 XML"))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sts_compat_layer_uses_canonical_request_id_for_success_header_and_xml() {
|
||||
let (headers, body) = call_sts_compat_layer(
|
||||
StatusCode::OK,
|
||||
b"<AssumeRoleResponse><AssumeRoleResult/></AssumeRoleResponse>",
|
||||
Some("upstream-request-id"),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
headers.get(AMZ_REQUEST_ID).and_then(|value| value.to_str().ok()),
|
||||
Some("server-request-id")
|
||||
);
|
||||
assert!(!headers.contains_key(http::header::CONTENT_LENGTH));
|
||||
assert!(body.contains("<ResponseMetadata><RequestId>server-request-id</RequestId></ResponseMetadata>"));
|
||||
assert!(!body.contains("untrusted-upstream-id"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sts_compat_layer_uses_same_request_id_for_error_header_and_xml() {
|
||||
let (headers, body) = call_sts_compat_layer(
|
||||
StatusCode::FORBIDDEN,
|
||||
b"<Error><Code>AccessDenied</Code><Message>Access Denied</Message></Error>",
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
headers.get(AMZ_REQUEST_ID).and_then(|value| value.to_str().ok()),
|
||||
Some("server-request-id")
|
||||
);
|
||||
assert!(body.contains("<ErrorResponse xmlns=\"https://sts.amazonaws.com/doc/2011-06-15/\">"));
|
||||
assert!(body.contains("<RequestId>server-request-id</RequestId>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sts_response_request_id_rejects_empty_context_id() {
|
||||
let mut request = Request::new(());
|
||||
request.extensions_mut().insert(RequestContext {
|
||||
request_id: String::new(),
|
||||
x_amz_request_id: String::new(),
|
||||
trace_id: None,
|
||||
span_id: None,
|
||||
start_time: Instant::now(),
|
||||
});
|
||||
|
||||
let (request_id, header) = sts_response_request_id(&request);
|
||||
assert_eq!(request_id, "sts-request-id");
|
||||
assert_eq!(header, HeaderValue::from_static("sts-request-id"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sts_query_route_match_requires_post_root_and_form_content_type() {
|
||||
let uri = Uri::from_static("/");
|
||||
let mut headers = HeaderMap::new();
|
||||
|
||||
assert!(!is_sts_query_request(&Method::POST, &uri, &headers));
|
||||
|
||||
headers.insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
assert!(!is_sts_query_request(&Method::POST, &uri, &headers));
|
||||
|
||||
headers.insert(
|
||||
http::header::CONTENT_TYPE,
|
||||
HeaderValue::from_static("application/x-www-form-urlencoded; charset=utf-8"),
|
||||
);
|
||||
assert!(is_sts_query_request(&Method::POST, &uri, &headers));
|
||||
assert!(!is_sts_query_request(&Method::GET, &uri, &headers));
|
||||
assert!(!is_sts_query_request(&Method::POST, &Uri::from_static("/bucket"), &headers));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sts_success_response_gets_response_metadata() {
|
||||
let xml = "<AssumeRoleResponse xmlns=\"https://sts.amazonaws.com/doc/2011-06-15/\">\
|
||||
<AssumeRoleResult><Credentials/></AssumeRoleResult></AssumeRoleResponse>";
|
||||
let converted = add_sts_response_metadata(xml, "request-123").expect("AssumeRole response should be converted");
|
||||
let converted = std::str::from_utf8(&converted).expect("converted response should be UTF-8 XML");
|
||||
|
||||
assert!(converted.contains("<Credentials/>"));
|
||||
assert!(converted.contains("<ResponseMetadata><RequestId>request-123</RequestId></ResponseMetadata>"));
|
||||
assert!(converted.ends_with("</AssumeRoleResponse>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sts_web_identity_success_response_gets_response_metadata() {
|
||||
let xml = "<AssumeRoleWithWebIdentityResponse xmlns=\"https://sts.amazonaws.com/doc/2011-06-15/\">\
|
||||
<AssumeRoleWithWebIdentityResult><Credentials/></AssumeRoleWithWebIdentityResult>\
|
||||
</AssumeRoleWithWebIdentityResponse>";
|
||||
let converted =
|
||||
add_sts_response_metadata(xml, "request-123").expect("AssumeRoleWithWebIdentity response should be converted");
|
||||
let converted = std::str::from_utf8(&converted).expect("converted response should be UTF-8 XML");
|
||||
|
||||
assert!(converted.contains("<Credentials/>"));
|
||||
assert!(converted.contains("<ResponseMetadata><RequestId>request-123</RequestId></ResponseMetadata>"));
|
||||
assert!(converted.ends_with("</AssumeRoleWithWebIdentityResponse>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sts_namespaced_success_response_gets_namespaced_metadata() {
|
||||
let xml = "<sts:AssumeRoleResponse xmlns:sts=\"https://sts.amazonaws.com/doc/2011-06-15/\">\
|
||||
<sts:AssumeRoleResult/></sts:AssumeRoleResponse>";
|
||||
let converted = add_sts_response_metadata(xml, "request-123").expect("namespaced response should be converted");
|
||||
let converted = std::str::from_utf8(&converted).expect("converted response should be UTF-8 XML");
|
||||
|
||||
assert!(converted.contains("<sts:ResponseMetadata><sts:RequestId>request-123</sts:RequestId></sts:ResponseMetadata>"));
|
||||
assert!(converted.ends_with("</sts:AssumeRoleResponse>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sts_success_response_with_existing_metadata_is_unchanged() {
|
||||
let xml = "<AssumeRoleResponse xmlns=\"https://sts.amazonaws.com/doc/2011-06-15/\">\
|
||||
<AssumeRoleResult/>\
|
||||
<ResponseMetadata source=\"upstream\"><RequestId>existing</RequestId></ResponseMetadata>\
|
||||
</AssumeRoleResponse>";
|
||||
|
||||
assert!(add_sts_response_metadata(xml, "request-123").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sts_error_response_uses_query_api_envelope() {
|
||||
let converted = wrap_sts_error_response(
|
||||
"<Error><Code>AccessDenied</Code><Message>Access & Denied</Message></Error>",
|
||||
StatusCode::FORBIDDEN,
|
||||
"request-456",
|
||||
);
|
||||
|
||||
assert!(converted.contains("<ErrorResponse xmlns=\"https://sts.amazonaws.com/doc/2011-06-15/\">"));
|
||||
assert!(converted.contains("<Type>Sender</Type>"));
|
||||
assert!(converted.contains("<Code>AccessDenied</Code>"));
|
||||
assert!(converted.contains("<Message>Access & Denied</Message>"));
|
||||
assert!(converted.contains("<RequestId>request-456</RequestId>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sts_signature_error_response_fills_missing_message() {
|
||||
let converted =
|
||||
wrap_sts_error_response("<Error><Code>SignatureDoesNotMatch</Code></Error>", StatusCode::FORBIDDEN, "request-456");
|
||||
let expected = ApiError::error_code_to_message(&S3ErrorCode::SignatureDoesNotMatch);
|
||||
|
||||
assert!(converted.contains(&format!("<Message>{expected}</Message>")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sts_compat_layer_wraps_non_xml_short_circuit_errors() {
|
||||
let (headers, body) =
|
||||
call_sts_compat_layer(StatusCode::SERVICE_UNAVAILABLE, b"Service not ready: waiting for iam", None).await;
|
||||
|
||||
assert_eq!(
|
||||
headers.get(AMZ_REQUEST_ID).and_then(|value| value.to_str().ok()),
|
||||
Some("server-request-id")
|
||||
);
|
||||
assert!(body.contains("<Type>Receiver</Type>"));
|
||||
assert!(body.contains("<Code>ServiceUnavailable</Code>"));
|
||||
assert!(body.contains("<Message>Service Unavailable</Message>"));
|
||||
assert!(body.contains("<RequestId>server-request-id</RequestId>"));
|
||||
assert!(!body.contains("waiting for iam"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sts_compat_layer_replaces_upstream_and_duplicate_error_request_ids() {
|
||||
let (headers, body) = call_sts_compat_layer(
|
||||
StatusCode::FORBIDDEN,
|
||||
b"<Error><Code>AccessDenied</Code><Message>Access Denied</Message>\
|
||||
<RequestId>body-request-id-1</RequestId><RequestId>body-request-id-2</RequestId></Error>",
|
||||
Some("header-request-id"),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
headers.get(AMZ_REQUEST_ID).and_then(|value| value.to_str().ok()),
|
||||
Some("server-request-id")
|
||||
);
|
||||
assert_eq!(body.matches("<RequestId>").count(), 1);
|
||||
assert!(body.contains("<RequestId>server-request-id</RequestId>"));
|
||||
for stale_request_id in ["header-request-id", "body-request-id-1", "body-request-id-2"] {
|
||||
assert!(!body.contains(stale_request_id));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sts_compat_layer_falls_back_for_empty_and_malformed_error_xml() {
|
||||
for (body, status, expected_code, expected_message) in [
|
||||
(
|
||||
b"".as_slice(),
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"ServiceUnavailable",
|
||||
"Service Unavailable",
|
||||
),
|
||||
(
|
||||
b"<Error><Code>AccessDenied".as_slice(),
|
||||
StatusCode::BAD_GATEWAY,
|
||||
"InternalError",
|
||||
"Internal Server Error",
|
||||
),
|
||||
] {
|
||||
let (headers, body) = call_sts_compat_layer(status, body, None).await;
|
||||
|
||||
assert_eq!(
|
||||
headers.get(AMZ_REQUEST_ID).and_then(|value| value.to_str().ok()),
|
||||
Some("server-request-id")
|
||||
);
|
||||
assert!(body.contains("<Type>Receiver</Type>"));
|
||||
assert!(body.contains(&format!("<Code>{expected_code}</Code>")));
|
||||
assert!(body.contains(&format!("<Message>{expected_message}</Message>")));
|
||||
assert_eq!(body.matches("<RequestId>").count(), 1);
|
||||
assert!(body.contains("<RequestId>server-request-id</RequestId>"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sts_compat_layer_prevents_pre_conversion_compression() {
|
||||
let xml = format!(
|
||||
"<AssumeRoleResponse><AssumeRoleResult>{}</AssumeRoleResult></AssumeRoleResponse>",
|
||||
"x".repeat(2048)
|
||||
);
|
||||
let mut service = tower::ServiceBuilder::new()
|
||||
.layer(StsQueryApiCompatLayer)
|
||||
.layer(tower_http::catch_panic::CatchPanicLayer::new())
|
||||
.layer(
|
||||
tower_http::compression::CompressionLayer::new().compress_when(PathAwareHttpCompressionPredicate::new(
|
||||
HttpCompressionConfig {
|
||||
enabled: true,
|
||||
extensions: Vec::new(),
|
||||
mime_patterns: vec!["application/xml".to_owned()],
|
||||
min_size: 0,
|
||||
},
|
||||
)),
|
||||
)
|
||||
.layer(PathCategoryInjectionLayer)
|
||||
.service(FixedStsResponse {
|
||||
status: StatusCode::OK,
|
||||
body: Bytes::from(xml),
|
||||
upstream_request_id: None,
|
||||
});
|
||||
let mut request = Request::builder()
|
||||
.method(Method::POST)
|
||||
.uri("/")
|
||||
.header(http::header::CONTENT_TYPE, "application/x-www-form-urlencoded")
|
||||
.header(http::header::ACCEPT_ENCODING, "gzip")
|
||||
.body(())
|
||||
.expect("STS request");
|
||||
request.extensions_mut().insert(RequestContext {
|
||||
request_id: "server-request-id".to_string(),
|
||||
x_amz_request_id: String::new(),
|
||||
trace_id: None,
|
||||
span_id: None,
|
||||
start_time: Instant::now(),
|
||||
});
|
||||
|
||||
let response = service.call(request).await.expect("STS compatibility response");
|
||||
assert!(
|
||||
!response.headers().contains_key(http::header::CONTENT_ENCODING),
|
||||
"STS XML must stay uncompressed until the compatibility envelope is complete"
|
||||
);
|
||||
let body = BodyExt::collect(response.into_body())
|
||||
.await
|
||||
.expect("collect STS response")
|
||||
.to_bytes();
|
||||
let body = String::from_utf8(body.to_vec()).expect("STS response should be UTF-8 XML");
|
||||
assert!(body.contains("<ResponseMetadata><RequestId>server-request-id</RequestId></ResponseMetadata>"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn iceberg_rest_error_conversion_returns_standard_json_envelope() {
|
||||
let body = Full::from(Bytes::from_static(
|
||||
|
||||
@@ -58,7 +58,7 @@ pub(crate) use health::{
|
||||
pub(crate) use health::{HealthProbe, build_health_response_parts, collect_probe_readiness, probe_from_path};
|
||||
pub(crate) use http::HeaderMapCarrier;
|
||||
pub(crate) use http::active_http_requests;
|
||||
pub(crate) use layer::RequestContextLayer;
|
||||
pub(crate) use layer::{RequestContextLayer, is_sts_query_request};
|
||||
pub(crate) use module_switch::{
|
||||
MODULE_SWITCHES_SIGNAL_SUBSYSTEM, ModuleSwitchSnapshot, ModuleSwitchSource, PersistedModuleSwitches,
|
||||
current_module_switch_snapshot, refresh_persisted_module_switches_from, refresh_persisted_module_switches_from_store,
|
||||
|
||||
@@ -164,15 +164,26 @@ pub fn extract_request_id_from_headers(headers: &HeaderMap) -> String {
|
||||
let request_id = headers
|
||||
.get(REQUEST_ID_HEADER)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.map(String::from)
|
||||
.or_else(|| headers.get(AMZ_REQUEST_ID).and_then(|v| v.to_str().ok()).map(String::from))
|
||||
.unwrap_or_else(generate_fallback_request_id);
|
||||
.or_else(|| {
|
||||
headers
|
||||
.get(AMZ_REQUEST_ID)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.map(String::from)
|
||||
});
|
||||
|
||||
if !headers.contains_key(REQUEST_ID_HEADER) && !headers.contains_key(AMZ_REQUEST_ID) {
|
||||
counter!("rustfs_log_chain_fallback_request_id_total", "source" => "headers_missing").increment(1);
|
||||
if request_id.is_none() {
|
||||
let source = if headers.contains_key(REQUEST_ID_HEADER) || headers.contains_key(AMZ_REQUEST_ID) {
|
||||
"headers_empty_or_invalid"
|
||||
} else {
|
||||
"headers_missing"
|
||||
};
|
||||
counter!("rustfs_log_chain_fallback_request_id_total", "source" => source).increment(1);
|
||||
}
|
||||
|
||||
request_id
|
||||
request_id.unwrap_or_else(generate_fallback_request_id)
|
||||
}
|
||||
|
||||
/// Spawn a request-internal task that inherits the current tracing span.
|
||||
@@ -283,6 +294,21 @@ mod tests {
|
||||
assert_eq!(id, "x-req-789");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_request_id_ignores_empty_header_values() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-request-id", http::HeaderValue::from_static(""));
|
||||
headers.insert("x-amz-request-id", http::HeaderValue::from_static("amz-req-000"));
|
||||
assert_eq!(extract_request_id_from_headers(&headers), "amz-req-000");
|
||||
|
||||
headers.insert(
|
||||
"x-amz-request-id",
|
||||
http::HeaderValue::from_bytes(b" \t").expect("optional whitespace is a valid header value"),
|
||||
);
|
||||
let id = extract_request_id_from_headers(&headers);
|
||||
assert!(id.starts_with("req-"), "empty request ID headers must generate a fallback ID");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_trace_context_ids_from_traceparent_header() {
|
||||
global::set_text_map_propagator(TraceContextPropagator::new());
|
||||
|
||||
Reference in New Issue
Block a user