fix(sts): return Query API-compatible responses (#5282)

* fix(sts): return Query API-compatible responses

* fix(sts): resolve review and CI failures

* fix(sts): harden query response conversion

* test(e2e): stabilize transition conflict coverage

* fix(ilm): retry vanished transition admission CAS

* test(e2e): narrow transition overlap coverage

---------

Co-authored-by: houseme <housemecn@gmail.com>

crates/e2e_test/src/lib.rs

@@ -101,6 +101,9 @@ mod admin_auth_test;
 #[cfg(test)]
 mod existing_object_tag_policy_test;
 
+#[cfg(test)]
+mod sts_query_compat_test;
+
 // Regression tests for Issue #2036: anonymous access with PublicAccessBlock
 #[cfg(test)]
 mod anonymous_access_test;

crates/e2e_test/src/reliant/tiering.rs

@@ -74,7 +74,6 @@ const MANUAL_ASYNC_STATUS_BUCKET: &str = "ilm7-manual-async-status";
 const MANUAL_CONTINUATION_BUCKET: &str = "ilm7-manual-continuation";
 const MANUAL_ASYNC_LIMIT_BUCKET: &str = "ilm7-manual-async-limit";
 const MANUAL_ASYNC_CONFLICT_BUCKET: &str = "ilm7-manual-async-conflict";
-const MANUAL_ASYNC_SAME_SCOPE_CONFLICT_BUCKET: &str = "ilm7-manual-async-same-scope-conflict";
 const MANUAL_ASYNC_PARALLEL_BUCKET_A: &str = "ilm7-manual-async-parallel-a";
 const MANUAL_ASYNC_PARALLEL_BUCKET_B: &str = "ilm7-manual-async-parallel-b";
 const MANUAL_TIER_FAILURE_BUCKET: &str = "ilm7-manual-tier-failure";
@@ -86,7 +85,6 @@ const MANUAL_CONTINUATION_PREFIX: &str = "manual-continuation/";
 const MANUAL_ASYNC_LIMIT_PREFIX: &str = "manual-async-limit/";
 const MANUAL_ASYNC_CONFLICT_PREFIX: &str = "manual-async-conflict/";
 const MANUAL_ASYNC_CONFLICT_NESTED_PREFIX: &str = "manual-async-conflict/nested/";
-const MANUAL_ASYNC_SAME_SCOPE_CONFLICT_PREFIX: &str = "manual-async-same-scope-conflict/";
 const MANUAL_ASYNC_PARALLEL_PREFIX: &str = "manual-async-parallel/";
 const MANUAL_TIER_FAILURE_PREFIX: &str = "manual-tier-failure/";
 const MANUAL_WORKER_FAILURE_PREFIX: &str = "manual-worker-failure/";
@@ -97,6 +95,7 @@ const MANUAL_ASYNC_PARALLEL_OBJECTS: usize = 64;
 const MANUAL_ACTIVE_CANCEL_OBJECTS: usize = 512;
 const MANUAL_RESTART_CANCEL_OBJECTS: usize = 512;
 const MANUAL_ACTIVE_CANCEL_RUNNING_TIMEOUT: StdDuration = StdDuration::from_secs(15);
+const MANUAL_ASYNC_CONFLICT_TERMINAL_TIMEOUT: StdDuration = StdDuration::from_secs(90);
 const MANUAL_RESTART_RECOVERY_TIMEOUT: StdDuration = StdDuration::from_secs(80);
 const OBJECT_KEY: &str = "tier/鲁A12345/report.bin";
 const MANUAL_DUE_KEY: &str = "manual-due/report.bin";
@@ -1267,7 +1266,7 @@ async fn test_manual_transition_async_limit_reports_terminal_partial() -> TestRe
 }
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-async fn test_manual_transition_async_overlapping_scope_conflict_reports_active_job() -> TestResult {
+async fn test_manual_transition_async_scope_conflicts_report_active_job() -> TestResult {
     let mut cold = RustFSTestEnvironment::new().await?;
     cold.access_key = "manualasyncconflictcoldtieradmin".to_string();
     cold.secret_key = "manualasyncconflictcoldtiersecret".to_string();
@@ -1296,34 +1295,14 @@ async fn test_manual_transition_async_overlapping_scope_conflict_reports_active_
     .await?;
 
     let before_remote_count = cold_tier_object_count(&cold_client).await?;
-    let (parent, nested) = tokio::join!(
-        manual_transition_async_run_raw(
-            &hot,
-            MANUAL_ASYNC_CONFLICT_BUCKET,
-            MANUAL_ASYNC_CONFLICT_PREFIX,
-            false,
-            MANUAL_ASYNC_CONFLICT_OBJECTS as u64,
-        ),
-        manual_transition_async_run_raw(
-            &hot,
-            MANUAL_ASYNC_CONFLICT_BUCKET,
-            MANUAL_ASYNC_CONFLICT_NESTED_PREFIX,
-            false,
-            MANUAL_ASYNC_CONFLICT_OBJECTS as u64,
-        )
-    );
-    let responses = [parent?, nested?];
-    let accepted = responses
-        .iter()
-        .find(|(status, _)| *status == reqwest::StatusCode::ACCEPTED)
-        .ok_or("one concurrent overlapping-scope async run must be accepted")?;
-    let conflict = responses
-        .iter()
-        .find(|(status, _)| *status == reqwest::StatusCode::CONFLICT)
-        .ok_or("one concurrent overlapping-scope async run must report conflict")?;
-
-    let accepted: ManualTransitionRunResponse = serde_json::from_str(&accepted.1)?;
-    let conflict: ManualTransitionJobConflictResponse = serde_json::from_str(&conflict.1)?;
+    let accepted = manual_transition_async_run(
+        &hot,
+        MANUAL_ASYNC_CONFLICT_BUCKET,
+        MANUAL_ASYNC_CONFLICT_PREFIX,
+        false,
+        MANUAL_ASYNC_CONFLICT_OBJECTS as u64,
+    )
+    .await?;
     let job_id = accepted
         .job_id
         .as_deref()
@@ -1341,13 +1320,25 @@ async fn test_manual_transition_async_overlapping_scope_conflict_reports_active_
     assert_eq!(accepted.state, "accepted");
     assert_eq!(accepted.mode, "durable_job");
     assert_eq!(accepted.report.bucket, MANUAL_ASYNC_CONFLICT_BUCKET);
-    assert!(
-        matches!(
-            accepted.report.prefix.as_str(),
-            MANUAL_ASYNC_CONFLICT_PREFIX | MANUAL_ASYNC_CONFLICT_NESTED_PREFIX
-        ),
-        "overlapping async winner prefix: {accepted:#?}"
+    assert_eq!(accepted.report.prefix, MANUAL_ASYNC_CONFLICT_PREFIX);
+    let active = wait_for_manual_transition_job_running(&hot, status_endpoint, MANUAL_ACTIVE_CANCEL_RUNNING_TIMEOUT).await?;
+    assert_eq!(active.job_id, job_id);
+
+    let (conflict_status, conflict_body) = manual_transition_async_run_raw(
+        &hot,
+        MANUAL_ASYNC_CONFLICT_BUCKET,
+        MANUAL_ASYNC_CONFLICT_NESTED_PREFIX,
+        false,
+        MANUAL_ASYNC_CONFLICT_OBJECTS as u64,
+    )
+    .await?;
+    assert_eq!(
+        conflict_status,
+        reqwest::StatusCode::CONFLICT,
+        "active async run must reject nested prefix {}: {conflict_body}",
+        MANUAL_ASYNC_CONFLICT_NESTED_PREFIX
     );
+    let conflict: ManualTransitionJobConflictResponse = serde_json::from_str(&conflict_body)?;
     assert_eq!(conflict.state, "conflict");
     assert_eq!(conflict.mode, "durable_job");
     assert_eq!(conflict.active_job_id, job_id);
@@ -1355,7 +1346,7 @@ async fn test_manual_transition_async_overlapping_scope_conflict_reports_active_
     assert_eq!(conflict.cancel_endpoint, status_endpoint);
     assert!(!conflict.scope_key.is_empty());
 
-    let terminal = wait_for_manual_transition_job_terminal(&hot, status_endpoint, StdDuration::from_secs(30)).await?;
+    let terminal = wait_for_manual_transition_job_terminal(&hot, status_endpoint, MANUAL_ASYNC_CONFLICT_TERMINAL_TIMEOUT).await?;
     assert_eq!(terminal.job_id, job_id);
     assert!(!terminal.report.dry_run);
     assert_eq!(terminal.report.bucket, MANUAL_ASYNC_CONFLICT_BUCKET);
@@ -1376,116 +1367,6 @@ async fn test_manual_transition_async_overlapping_scope_conflict_reports_active_
     Ok(())
 }
 
-#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-async fn test_manual_transition_async_same_scope_conflict_reports_active_job() -> TestResult {
-    let mut cold = RustFSTestEnvironment::new().await?;
-    cold.access_key = "manualasyncsameconflictcoldadmin".to_string();
-    cold.secret_key = "manualasyncsameconflictcoldsecret".to_string();
-    cold.start_rustfs_server_without_cleanup(vec![]).await?;
-    let cold_client = cold.create_s3_client();
-    cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
-
-    let mut hot = RustFSTestEnvironment::new().await?;
-    hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
-        .await?;
-    let hot_client = hot.create_s3_client();
-    add_rustfs_tier(&hot, &cold).await?;
-
-    hot_client
-        .create_bucket()
-        .bucket(MANUAL_ASYNC_SAME_SCOPE_CONFLICT_BUCKET)
-        .send()
-        .await?;
-    for idx in 0..50 {
-        let key = format!("{MANUAL_ASYNC_SAME_SCOPE_CONFLICT_PREFIX}obj-{idx:02}");
-        put_single_part_object(
-            &hot_client,
-            MANUAL_ASYNC_SAME_SCOPE_CONFLICT_BUCKET,
-            &key,
-            b"async same-scope conflict payload",
-        )
-        .await?;
-    }
-    put_lifecycle_transition_rule(
-        &hot_client,
-        MANUAL_ASYNC_SAME_SCOPE_CONFLICT_BUCKET,
-        "manual-async-same-scope-conflict",
-        MANUAL_ASYNC_SAME_SCOPE_CONFLICT_PREFIX,
-        0,
-    )
-    .await?;
-
-    let (first, second) = tokio::join!(
-        manual_transition_async_run_raw(
-            &hot,
-            MANUAL_ASYNC_SAME_SCOPE_CONFLICT_BUCKET,
-            MANUAL_ASYNC_SAME_SCOPE_CONFLICT_PREFIX,
-            false,
-            50
-        ),
-        manual_transition_async_run_raw(
-            &hot,
-            MANUAL_ASYNC_SAME_SCOPE_CONFLICT_BUCKET,
-            MANUAL_ASYNC_SAME_SCOPE_CONFLICT_PREFIX,
-            false,
-            50
-        )
-    );
-    let responses = [first?, second?];
-    let accepted = responses
-        .iter()
-        .find(|(status, _)| *status == reqwest::StatusCode::ACCEPTED)
-        .ok_or("one concurrent same-scope async run must be accepted")?;
-    let conflict = responses
-        .iter()
-        .find(|(status, _)| *status == reqwest::StatusCode::CONFLICT)
-        .ok_or("one concurrent same-scope async run must report conflict")?;
-
-    let accepted: ManualTransitionRunResponse = serde_json::from_str(&accepted.1)?;
-    let conflict: ManualTransitionJobConflictResponse = serde_json::from_str(&conflict.1)?;
-    let job_id = accepted
-        .job_id
-        .as_deref()
-        .ok_or("accepted same-scope async response must include job_id")?;
-    let status_endpoint = accepted
-        .status_endpoint
-        .as_deref()
-        .ok_or("accepted same-scope async response must include status_endpoint")?;
-    let cancel_endpoint = accepted
-        .cancel_endpoint
-        .as_deref()
-        .ok_or("accepted same-scope async response must include cancel_endpoint")?;
-    assert_eq!(cancel_endpoint, status_endpoint);
-
-    assert_eq!(accepted.state, "accepted");
-    assert_eq!(accepted.mode, "durable_job");
-    assert_eq!(accepted.report.bucket, MANUAL_ASYNC_SAME_SCOPE_CONFLICT_BUCKET);
-    assert_eq!(accepted.report.prefix, MANUAL_ASYNC_SAME_SCOPE_CONFLICT_PREFIX);
-    assert_eq!(conflict.state, "conflict");
-    assert_eq!(conflict.mode, "durable_job");
-    assert_eq!(conflict.active_job_id, job_id);
-    assert_eq!(conflict.status_endpoint, status_endpoint);
-    assert_eq!(conflict.cancel_endpoint, status_endpoint);
-    assert!(!conflict.scope_key.is_empty());
-
-    let terminal = wait_for_manual_transition_job_terminal(&hot, status_endpoint, StdDuration::from_secs(30)).await?;
-    assert_eq!(terminal.job_id, job_id);
-    assert_eq!(terminal.report.bucket, MANUAL_ASYNC_SAME_SCOPE_CONFLICT_BUCKET);
-    assert_eq!(terminal.report.prefix, MANUAL_ASYNC_SAME_SCOPE_CONFLICT_PREFIX);
-    assert_conflict_winner_report(&terminal.status, &terminal.report, 50, "terminal same-scope conflict winner response");
-    assert_eq!(
-        terminal.report.transition_failed, 0,
-        "terminal same-scope conflict winner response: {terminal:#?}"
-    );
-    assert_eq!(
-        terminal.report.tier_failure, 0,
-        "terminal same-scope conflict winner response: {terminal:#?}"
-    );
-    assert!(cold_tier_object_count(&cold_client).await? <= 50);
-
-    Ok(())
-}
-
 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
 async fn test_manual_transition_async_different_buckets_admit_concurrently() -> TestResult {
     let mut cold = RustFSTestEnvironment::new().await?;

crates/e2e_test/src/sts_query_compat_test.rs

@@ -0,0 +1,229 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
+use aws_sdk_sts::config::retry::RetryConfig;
+use aws_sdk_sts::config::{Credentials, Region};
+use aws_sdk_sts::error::ProvideErrorMetadata;
+use aws_sdk_sts::operation::RequestId;
+use aws_sdk_sts::{Client, Config};
+use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
+use http::header::{CONTENT_TYPE, HOST};
+use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+use rustfs_signer::sign_v4;
+use s3s::Body;
+use serial_test::serial;
+use std::error::Error;
+
+type BoxError = Box<dyn Error + Send + Sync>;
+type TestResult = Result<(), BoxError>;
+
+fn sts_client(url: &str, access_key: &str, secret_key: &str, session_token: Option<&str>) -> Client {
+    let mut config = Config::builder()
+        .credentials_provider(Credentials::new(
+            access_key,
+            secret_key,
+            session_token.map(str::to_owned),
+            None,
+            "e2e-sts-query-compat",
+        ))
+        .region(Region::new("us-east-1"))
+        .endpoint_url(url)
+        .retry_config(RetryConfig::standard().with_max_attempts(1))
+        .behavior_version_latest();
+    if url.starts_with("http://") {
+        config = config.http_client(SmithyHttpClientBuilder::new().build_http());
+    }
+    Client::from_conf(config.build())
+}
+
+async fn create_root_service_account(env: &RustFSTestEnvironment) -> Result<(String, String), BoxError> {
+    let path = "/rustfs/admin/v3/add-service-accounts";
+    let url = format!("{}{path}", env.url);
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
+    let body = serde_json::json!({ "targetUser": env.access_key.clone() }).to_string();
+    let request = http::Request::builder()
+        .method(http::Method::PUT)
+        .uri(uri)
+        .header(HOST, authority)
+        .header(CONTENT_TYPE, "application/json")
+        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
+        .body(Body::empty())?;
+    let content_length = i64::try_from(body.len()).map_err(|_| "service account request body is too large")?;
+    let signed = sign_v4(request, content_length, &env.access_key, &env.secret_key, "", "us-east-1");
+    let mut request = local_http_client().put(&url);
+    for (name, value) in signed.headers() {
+        request = request.header(name, value);
+    }
+    let response = request.body(body).send().await?;
+    let status = response.status();
+    let body = response.text().await?;
+    if !status.is_success() {
+        return Err(format!("create service account failed: {status} {body}").into());
+    }
+
+    let response: serde_json::Value = serde_json::from_str(&body)?;
+    let access_key = response["credentials"]["accessKey"]
+        .as_str()
+        .ok_or("service account response should contain credentials.accessKey")?
+        .to_owned();
+    let secret_key = response["credentials"]["secretKey"]
+        .as_str()
+        .ok_or("service account response should contain credentials.secretKey")?
+        .to_owned();
+    Ok((access_key, secret_key))
+}
+
+async fn assert_chaining_denied(client: &Client, credential_kind: &str) -> TestResult {
+    let error = client
+        .assume_role()
+        .role_arn("arn:aws:iam::123456789012:role/test")
+        .role_session_name("sts-query-compat-e2e")
+        .send()
+        .await
+        .expect_err("credential chaining must be denied");
+    let service_error = error
+        .as_service_error()
+        .ok_or_else(|| format!("{credential_kind} denial should deserialize as an STS service error: {error:?}"))?;
+
+    assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(403));
+    assert_eq!(service_error.code(), Some("AccessDenied"));
+    assert_eq!(service_error.message(), Some("Access Denied"));
+    assert!(
+        error.request_id().is_some_and(|request_id| !request_id.is_empty()),
+        "{credential_kind} denial should include a request ID"
+    );
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_sts_query_responses_are_aws_sdk_compatible() -> TestResult {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let assumed = sts_client(&env.url, &env.access_key, &env.secret_key, None)
+        .assume_role()
+        .role_arn("arn:aws:iam::123456789012:role/test")
+        .role_session_name("sts-query-compat-e2e")
+        .send()
+        .await?;
+    assert!(
+        assumed.request_id().is_some_and(|request_id| !request_id.is_empty()),
+        "successful AssumeRole should include a request ID"
+    );
+    let temporary = assumed
+        .credentials()
+        .ok_or("successful AssumeRole response should contain credentials")?;
+
+    let invalid_signature = sts_client(&env.url, &env.access_key, "incorrect-secret-key", None)
+        .assume_role()
+        .role_arn("arn:aws:iam::123456789012:role/test")
+        .role_session_name("sts-query-invalid-signature")
+        .send()
+        .await
+        .expect_err("an invalid signature must be rejected");
+    assert_eq!(invalid_signature.raw_response().map(|response| response.status().as_u16()), Some(403));
+    let invalid_signature_service_error = invalid_signature
+        .as_service_error()
+        .ok_or_else(|| format!("invalid signature should deserialize as an STS service error: {invalid_signature:?}"))?;
+    assert_eq!(invalid_signature_service_error.code(), Some("SignatureDoesNotMatch"));
+    assert!(
+        invalid_signature_service_error
+            .message()
+            .is_some_and(|message| message.starts_with("The request signature we calculated does not match")),
+        "signature rejection should preserve the canonical error message"
+    );
+    assert!(
+        invalid_signature
+            .request_id()
+            .is_some_and(|request_id| !request_id.is_empty()),
+        "signature rejection should include a request ID"
+    );
+
+    assert_chaining_denied(
+        &sts_client(
+            &env.url,
+            temporary.access_key_id(),
+            temporary.secret_access_key(),
+            Some(temporary.session_token()),
+        ),
+        "temporary credential",
+    )
+    .await?;
+
+    let (service_access_key, service_secret_key) = create_root_service_account(&env).await?;
+    assert_chaining_denied(&sts_client(&env.url, &service_access_key, &service_secret_key, None), "service account").await?;
+
+    env.stop_server();
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_sts_query_rate_limit_error_is_aws_sdk_compatible() -> TestResult {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server_with_env(
+        vec![],
+        &[
+            ("RUSTFS_API_RATE_LIMIT_ENABLE", "true"),
+            ("RUSTFS_API_RATE_LIMIT_RPM", "60"),
+            ("RUSTFS_API_RATE_LIMIT_BURST", "1"),
+        ],
+    )
+    .await?;
+
+    let client = sts_client(&env.url, &env.access_key, &env.secret_key, None);
+    let mut throttled = None;
+    let request = || {
+        client
+            .assume_role()
+            .role_arn("arn:aws:iam::123456789012:role/test")
+            .role_session_name("sts-query-rate-limit")
+            .send()
+    };
+    let (first, second, third, fourth) = tokio::join!(request(), request(), request(), request());
+    for result in [first, second, third, fourth] {
+        if let Err(error) = result
+            && error.raw_response().map(|response| response.status().as_u16()) == Some(429)
+        {
+            throttled = Some(error);
+            break;
+        }
+    }
+
+    let error = throttled.ok_or("at least one concurrent STS request should be throttled at burst one")?;
+    let service_error = error
+        .as_service_error()
+        .ok_or_else(|| format!("rate limit response should deserialize as an STS service error: {error:?}"))?;
+    assert_eq!(service_error.code(), Some("TooManyRequests"));
+    assert!(
+        service_error
+            .message()
+            .is_some_and(|message| message.starts_with("Request rate limit exceeded")),
+        "rate limit response should preserve the server message"
+    );
+    assert!(
+        error.request_id().is_some_and(|request_id| !request_id.is_empty()),
+        "rate limit response should include a request ID"
+    );
+
+    env.stop_server();
+    Ok(())
+}