feat(iam,admin): prepared IAM auth, ExistingObjectTag, admin permission checks (#2315)

Signed-off-by: GatewayJ <835269233@qq.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: GatewayJ <8352692332qq.com>

crates/e2e_test/src/common.rs

@@ -463,18 +463,18 @@ impl Drop for RustFSTestEnvironment {
     }
 }
 
-/// Utility function to execute awscurl commands
-pub async fn execute_awscurl(
+async fn execute_awscurl_with_service(
     url: &str,
     method: &str,
     body: Option<&str>,
     access_key: &str,
     secret_key: &str,
+    service: &str,
 ) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
     let mut args = vec![
         "--fail-with-body",
         "--service",
-        "s3",
+        service,
         "--region",
         "us-east-1",
         "--access_key",
@@ -490,7 +490,64 @@ pub async fn execute_awscurl(
         args.extend(&["-d", body_content]);
     }
 
-    info!("Executing awscurl: {} {}", method, url);
+    info!("Executing awscurl: {} {} (service={})", method, url, service);
+    let awscurl_path = awscurl_binary_path();
+    let output = Command::new(&awscurl_path).args(&args).output()?;
+
+    if !output.status.success() {
+        let stderr = String::from_utf8_lossy(&output.stderr);
+        let stdout = String::from_utf8_lossy(&output.stdout);
+        return Err(format!("awscurl failed: stderr='{stderr}', stdout='{stdout}'").into());
+    }
+
+    let response = String::from_utf8_lossy(&output.stdout).to_string();
+    Ok(response)
+}
+
+/// Utility function to execute awscurl commands (SigV4 service `s3` for admin APIs).
+pub async fn execute_awscurl(
+    url: &str,
+    method: &str,
+    body: Option<&str>,
+    access_key: &str,
+    secret_key: &str,
+) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
+    execute_awscurl_with_service(url, method, body, access_key, secret_key, "s3").await
+}
+
+/// `POST` with SigV4 `--service sts` and explicit `Content-Type: application/x-www-form-urlencoded`.
+///
+/// RustFS `AssumeRole` is handled on `POST /` by the admin router; `is_match` requires this
+/// content type so s3s routes to the custom handler instead of `Unknown operation`.
+pub async fn awscurl_post_sts_form_urlencoded(
+    url: &str,
+    body: &str,
+    access_key: &str,
+    secret_key: &str,
+) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
+    let args = vec![
+        "--fail-with-body",
+        "--service",
+        "sts",
+        "--region",
+        "us-east-1",
+        "--access_key",
+        access_key,
+        "--secret_key",
+        secret_key,
+        "-H",
+        "Content-Type: application/x-www-form-urlencoded",
+        "-X",
+        "POST",
+        url,
+        "-d",
+        body,
+    ];
+
+    info!(
+        "Executing awscurl: POST {} (service=sts, Content-Type=application/x-www-form-urlencoded)",
+        url
+    );
     let awscurl_path = awscurl_binary_path();
     let output = Command::new(&awscurl_path).args(&args).output()?;
 

crates/e2e_test/src/existing_object_tag_policy_test.rs

@@ -0,0 +1,364 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! E2E: `s3:ExistingObjectTag` with **IAM identity policy**, **bucket policy**, and **STS AssumeRole
+//! session policy** (`Policy` parameter) via `awscurl --service sts` with explicit
+//! `Content-Type: application/x-www-form-urlencoded` on `POST /`.
+
+use crate::common::{
+    RustFSTestEnvironment, awscurl_available, awscurl_delete, awscurl_post_sts_form_urlencoded, awscurl_put, init_logging,
+};
+use aws_sdk_s3::config::{Credentials, Region};
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{Tag, Tagging};
+use aws_sdk_s3::{Client, Config};
+use serial_test::serial;
+use tracing::info;
+use uuid::Uuid;
+
+fn user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
+    let credentials = Credentials::new(access_key, secret_key, None, None, "e2e-existing-tag");
+    let config = Config::builder()
+        .credentials_provider(credentials)
+        .region(Region::new("us-east-1"))
+        .endpoint_url(&env.url)
+        .force_path_style(true)
+        .behavior_version_latest()
+        .build();
+    Client::from_conf(config)
+}
+
+fn sts_session_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str, session_token: &str) -> Client {
+    let credentials = Credentials::new(access_key, secret_key, Some(session_token.into()), None, "e2e-sts-session");
+    let config = Config::builder()
+        .credentials_provider(credentials)
+        .region(Region::new("us-east-1"))
+        .endpoint_url(&env.url)
+        .force_path_style(true)
+        .behavior_version_latest()
+        .build();
+    Client::from_conf(config)
+}
+
+fn extract_xml_tag(xml: &str, tag: &str) -> Option<String> {
+    let open = format!("<{tag}>");
+    let close = format!("</{tag}>");
+    let start = xml.find(&open)? + open.len();
+    let end = xml[start..].find(&close)? + start;
+    Some(xml[start..end].to_string())
+}
+
+fn parse_assume_role_credentials(xml: &str) -> Result<(String, String, String), Box<dyn std::error::Error + Send + Sync>> {
+    let ak = extract_xml_tag(xml, "AccessKeyId").ok_or("missing AccessKeyId in AssumeRole response")?;
+    let sk = extract_xml_tag(xml, "SecretAccessKey").ok_or("missing SecretAccessKey in AssumeRole response")?;
+    let token = extract_xml_tag(xml, "SessionToken").ok_or("missing SessionToken in AssumeRole response")?;
+    Ok((ak, sk, token))
+}
+
+async fn assume_role_with_session_policy(
+    env: &RustFSTestEnvironment,
+    parent_ak: &str,
+    parent_sk: &str,
+    session_policy_json: &str,
+) -> Result<(String, String, String), Box<dyn std::error::Error + Send + Sync>> {
+    let policy_enc = urlencoding::encode(session_policy_json);
+    let body = format!("Action=AssumeRole&Version=2011-06-15&DurationSeconds=3600&Policy={}", policy_enc);
+    let url = format!("{}/", env.url.trim_end_matches('/'));
+    let xml = awscurl_post_sts_form_urlencoded(&url, &body, parent_ak, parent_sk).await?;
+    parse_assume_role_credentials(&xml)
+}
+
+async fn admin_create_user(
+    env: &RustFSTestEnvironment,
+    username: &str,
+    password: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let body = serde_json::json!({ "secretKey": password, "status": "enabled" }).to_string();
+    let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
+    awscurl_put(&url, &body, &env.access_key, &env.secret_key).await?;
+    Ok(())
+}
+
+async fn admin_add_canned_policy(
+    env: &RustFSTestEnvironment,
+    policy_name: &str,
+    policy_json: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
+    awscurl_put(&url, policy_json, &env.access_key, &env.secret_key).await?;
+    Ok(())
+}
+
+async fn admin_attach_policy_to_user(
+    env: &RustFSTestEnvironment,
+    policy_name: &str,
+    username: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let url = format!(
+        "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false",
+        env.url, policy_name, username
+    );
+    awscurl_put(&url, "", &env.access_key, &env.secret_key).await?;
+    Ok(())
+}
+
+async fn admin_remove_user(env: &RustFSTestEnvironment, username: &str) {
+    let url = format!("{}/rustfs/admin/v3/remove-user?accessKey={}", env.url, username);
+    let _ = awscurl_delete(&url, &env.access_key, &env.secret_key).await;
+}
+
+async fn admin_remove_policy(env: &RustFSTestEnvironment, policy_name: &str) {
+    let url = format!("{}/rustfs/admin/v3/remove-canned-policy?name={}", env.url, policy_name);
+    let _ = awscurl_delete(&url, &env.access_key, &env.secret_key).await;
+}
+
+async fn put_object_with_tagging_str(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    data: &[u8],
+    tagging: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from(data.to_vec()))
+        .tagging(tagging)
+        .send()
+        .await?;
+    Ok(())
+}
+
+async fn put_object_tag_kv(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    tag_key: &str,
+    tag_value: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let tag = Tag::builder()
+        .key(tag_key)
+        .value(tag_value)
+        .build()
+        .map_err(|e| format!("Tag build: {e}"))?;
+    let tagging = Tagging::builder()
+        .tag_set(tag)
+        .build()
+        .map_err(|e| format!("Tagging build: {e}"))?;
+    client
+        .put_object_tagging()
+        .bucket(bucket)
+        .key(key)
+        .tagging(tagging)
+        .send()
+        .await?;
+    Ok(())
+}
+
+async fn cleanup_bucket_and_object(admin: &Client, bucket: &str, key: &str) {
+    let _ = admin.delete_object().bucket(bucket).key(key).send().await;
+    let _ = admin.delete_bucket().bucket(bucket).send().await;
+}
+
+/// IAM identity policy: GetObject allowed only when `s3:ExistingObjectTag/security` == `public`.
+#[tokio::test]
+#[serial]
+async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+    if !awscurl_available() {
+        info!("Skipping test_e2e_iam_policy_existing_object_tag_get_object: awscurl not available");
+        return Ok(());
+    }
+
+    let suffix = Uuid::new_v4();
+    let user = format!("e2eiamtag-{suffix}");
+    let user_secret = "longSecretKeyForTest123!";
+    let policy_name = format!("e2e-iam-tag-pol-{suffix}");
+    let bucket = format!("e2e-iam-tag-bkt-{suffix}");
+    let key = "tagged-object.txt";
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let admin = env.create_s3_client();
+    admin_create_user(&env, &user, user_secret).await?;
+
+    let policy_doc = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Action": ["s3:GetObject"],
+            "Resource": [format!("arn:aws:s3:::{}/*", bucket)],
+            "Condition": { "StringEquals": { "s3:ExistingObjectTag/security": "public" } }
+        }]
+    })
+    .to_string();
+
+    admin_add_canned_policy(&env, &policy_name, &policy_doc).await?;
+    admin_attach_policy_to_user(&env, &policy_name, &user).await?;
+
+    admin.create_bucket().bucket(&bucket).send().await?;
+    put_object_with_tagging_str(&admin, &bucket, key, b"hello-iam-tag", "security=public").await?;
+
+    let uclient = user_client(&env, &user, user_secret);
+    let out = uclient.get_object().bucket(&bucket).key(key).send().await?;
+    let _ = out.body.collect().await?;
+
+    put_object_tag_kv(&admin, &bucket, key, "security", "private").await?;
+    let denied = uclient.get_object().bucket(&bucket).key(key).send().await;
+    assert!(
+        denied.is_err(),
+        "GetObject must be denied when ExistingObjectTag no longer matches IAM policy"
+    );
+
+    cleanup_bucket_and_object(&admin, &bucket, key).await;
+    admin_remove_user(&env, &user).await;
+    admin_remove_policy(&env, &policy_name).await;
+
+    info!("test_e2e_iam_policy_existing_object_tag_get_object passed");
+    Ok(())
+}
+
+/// Bucket policy: same `ExistingObjectTag` condition; user has no canned IAM policy attached.
+#[tokio::test]
+#[serial]
+async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+    if !awscurl_available() {
+        info!("Skipping test_e2e_bucket_policy_existing_object_tag_get_object: awscurl not available");
+        return Ok(());
+    }
+
+    let suffix = Uuid::new_v4();
+    let user = format!("e2ebptag-{suffix}");
+    let user_secret = "longSecretKeyForTest456!";
+    let bucket = format!("e2e-bp-tag-bkt-{suffix}");
+    let key = "obj.txt";
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let admin = env.create_s3_client();
+    admin_create_user(&env, &user, user_secret).await?;
+
+    admin.create_bucket().bucket(&bucket).send().await?;
+
+    let deny_before = user_client(&env, &user, user_secret)
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .send()
+        .await;
+    assert!(deny_before.is_err(), "without bucket policy, user must be denied");
+
+    let bp = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Principal": { "AWS": [user.clone()] },
+            "Action": ["s3:GetObject"],
+            "Resource": [format!("arn:aws:s3:::{}/*", bucket)],
+            "Condition": { "StringEquals": { "s3:ExistingObjectTag/security": "public" } }
+        }]
+    })
+    .to_string();
+
+    admin.put_bucket_policy().bucket(&bucket).policy(&bp).send().await?;
+    put_object_with_tagging_str(&admin, &bucket, key, b"data", "security=public").await?;
+
+    let uclient = user_client(&env, &user, user_secret);
+    let ok = uclient.get_object().bucket(&bucket).key(key).send().await?;
+    let _ = ok.body.collect().await?;
+
+    put_object_tag_kv(&admin, &bucket, key, "security", "private").await?;
+    let denied = uclient.get_object().bucket(&bucket).key(key).send().await;
+    assert!(denied.is_err(), "GetObject must fail when tag no longer satisfies bucket policy");
+
+    cleanup_bucket_and_object(&admin, &bucket, key).await;
+    admin_remove_user(&env, &user).await;
+
+    info!("test_e2e_bucket_policy_existing_object_tag_get_object passed");
+    Ok(())
+}
+
+/// STS `AssumeRole` with inline `Policy` (session policy): GetObject only when `ExistingObjectTag/security` is `public`.
+#[tokio::test]
+#[serial]
+async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+    if !awscurl_available() {
+        info!("Skipping test_e2e_sts_assume_role_session_policy_existing_object_tag: awscurl not available");
+        return Ok(());
+    }
+
+    let suffix = Uuid::new_v4();
+    let parent = format!("e2e-sts-par-{suffix}");
+    let parent_secret = "longSecretKeyForParentSts99!";
+    let policy_readwrite = format!("e2e-sts-rw-{suffix}");
+    let bucket = format!("e2e-sts-tag-bkt-{suffix}");
+    let key = "sts-obj.txt";
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let admin = env.create_s3_client();
+    admin_create_user(&env, &parent, parent_secret).await?;
+
+    let rw = serde_json::to_string(&serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Action": ["s3:*"],
+            "Resource": ["arn:aws:s3:::*"]
+        }]
+    }))?;
+    admin_add_canned_policy(&env, &policy_readwrite, &rw).await?;
+    admin_attach_policy_to_user(&env, &policy_readwrite, &parent).await?;
+
+    let parent_client = user_client(&env, &parent, parent_secret);
+    parent_client.create_bucket().bucket(&bucket).send().await?;
+    put_object_with_tagging_str(&parent_client, &bucket, key, b"sts-e2e-data", "security=public").await?;
+
+    let session_policy = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Action": ["s3:GetObject"],
+            "Resource": [format!("arn:aws:s3:::{}/*", bucket)],
+            "Condition": { "StringEquals": { "s3:ExistingObjectTag/security": "public" } }
+        }]
+    })
+    .to_string();
+
+    let (ak, sk, token) = assume_role_with_session_policy(&env, &parent, parent_secret, &session_policy).await?;
+
+    let session_client = sts_session_client(&env, &ak, &sk, &token);
+    let ok = session_client.get_object().bucket(&bucket).key(key).send().await?;
+    let _ = ok.body.collect().await?;
+
+    put_object_tag_kv(&parent_client, &bucket, key, "security", "private").await?;
+    let denied = session_client.get_object().bucket(&bucket).key(key).send().await;
+    assert!(
+        denied.is_err(),
+        "session policy must deny GetObject when ExistingObjectTag no longer matches"
+    );
+
+    cleanup_bucket_and_object(&admin, &bucket, key).await;
+    admin_remove_user(&env, &parent).await;
+    admin_remove_policy(&env, &policy_readwrite).await;
+
+    info!("test_e2e_sts_assume_role_session_policy_existing_object_tag passed");
+    Ok(())
+}

crates/e2e_test/src/lib.rs

@@ -40,6 +40,10 @@ mod quota_test;
 #[cfg(test)]
 mod bucket_policy_check_test;
 
+/// IAM / bucket / STS session policy with `s3:ExistingObjectTag` conditions (E2E).
+#[cfg(test)]
+mod existing_object_tag_policy_test;
+
 // Regression tests for Issue #2036: anonymous access with PublicAccessBlock
 #[cfg(test)]
 mod anonymous_access_test;

crates/e2e_test/src/policy/policy_variables_test.rs

@@ -14,7 +14,7 @@
 
 //! Tests for AWS IAM policy variables with single-value, multi-value, and nested scenarios
 
-use crate::common::{awscurl_put, init_logging};
+use crate::common::{awscurl_delete, awscurl_put, init_logging};
 use crate::policy::test_env::PolicyTestEnvironment;
 use aws_sdk_s3::primitives::ByteStream;
 use serial_test::serial;
@@ -113,11 +113,11 @@ async fn cleanup_user_and_policy(env: &PolicyTestEnvironment, username: &str, po
 
     // Remove user
     let remove_user_url = format!("{}/rustfs/admin/v3/remove-user?accessKey={}", env.url, username);
-    let _ = awscurl_put(&remove_user_url, "", &env.access_key, &env.secret_key).await;
+    let _ = awscurl_delete(&remove_user_url, &env.access_key, &env.secret_key).await;
 
     // Remove policy
     let remove_policy_url = format!("{}/rustfs/admin/v3/remove-canned-policy?name={}", env.url, policy_name);
-    let _ = awscurl_put(&remove_policy_url, "", &env.access_key, &env.secret_key).await;
+    let _ = awscurl_delete(&remove_policy_url, &env.access_key, &env.secret_key).await;
 }
 
 /// Test AWS policy variables with single-value scenarios