ci(test): enable local KMS for s3 compatibility tests (#7553)

Configure the s3-tests harness with a local KMS key so SSE-KMS cases run in CI without relying on an external KMS service.

Also move the anonymous POST default SSE-KMS regression onto the shared local KMS test environment.

Co-authored-by: zhi22915 <qiuzgang@gmail.com>

crates/e2e_test/src/multipart_auth_test.rs

@@ -15,6 +15,7 @@
 //! Regression coverage for anonymous access on multipart control APIs.
 
 use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
+use crate::kms::common::LocalKMSTestEnvironment;
 use async_compression::tokio::write::{BzEncoder, Lz4Encoder, XzEncoder};
 use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
 use aws_sdk_s3::operation::head_object::HeadObjectOutput;
@@ -1465,10 +1466,10 @@ async fn test_anonymous_post_object_uses_bucket_default_sse_s3() -> Result<(), B
 async fn test_anonymous_post_object_uses_bucket_default_sse_kms() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
 
-    let mut env = RustFSTestEnvironment::new().await?;
-    let master_key = local_sse_master_key_value();
-    env.start_rustfs_server_with_env(vec![], &[(LOCAL_SSE_MASTER_KEY_ENV, master_key.as_str())])
-        .await?;
+    let mut kms_env = LocalKMSTestEnvironment::new().await?;
+    let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
+    kms_env.wait_for_kms_ready().await?;
+    let env = &kms_env.base_env;
 
     let bucket = "anon-post-default-sse-kms";
     let object_key = "post-default-sse-kms-object.txt";
@@ -1484,7 +1485,7 @@ async fn test_anonymous_post_object_uses_bucket_default_sse_kms() -> Result<(),
                 .apply_server_side_encryption_by_default(
                     ServerSideEncryptionByDefault::builder()
                         .sse_algorithm(ServerSideEncryption::AwsKms)
-                        .kms_master_key_id("test-key")
+                        .kms_master_key_id(default_key_id)
                         .build()
                         .expect("default encryption rule should build"),
                 )

scripts/s3-tests/run.sh

@@ -239,6 +239,8 @@ DEPLOY_MODE="${DEPLOY_MODE:-build}"
 RUSTFS_BINARY="${RUSTFS_BINARY:-}"
 NO_CACHE="${NO_CACHE:-false}"
 S3TESTS_LOCAL_SSE_MASTER_KEY_DEFAULT="MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY="
+S3TESTS_ENABLE_LOCAL_KMS="${S3TESTS_ENABLE_LOCAL_KMS:-true}"
+S3_KMS_KEY_ID="${S3_KMS_KEY_ID:-rustfs-s3tests-default-key}"
 
 # Additional directories (SCRIPT_DIR and PROJECT_ROOT defined earlier)
 ARTIFACTS_DIR="${PROJECT_ROOT}/artifacts/s3tests-${TEST_MODE}"
@@ -252,6 +254,9 @@ else
 fi
 DATA_DIR="${DATA_BASE}/test-data/${CONTAINER_NAME}"
 RUSTFS_PID=""
+RUSTFS_KMS_ARGS=()
+S3TESTS_KMS_HOST_KEY_DIR="${S3TESTS_KMS_KEY_DIR:-${DATA_BASE}/kms-keys/${CONTAINER_NAME}}"
+S3TESTS_KMS_RUNTIME_KEY_DIR="${S3TESTS_KMS_HOST_KEY_DIR}"
 
 if [ "${DEPLOY_MODE}" != "existing" ] && [ -z "${RUSTFS_SSE_S3_MASTER_KEY:-}" ]; then
     export RUSTFS_SSE_S3_MASTER_KEY="${S3TESTS_LOCAL_SSE_MASTER_KEY_DEFAULT}"
@@ -282,6 +287,9 @@ Environment Variables:
   S3_ALT_ACCESS_KEY      - Alt user access key (default: rustfsalt)
   S3_ALT_SECRET_KEY      - Alt user secret key (default: rustfsalt)
   RUSTFS_SSE_S3_MASTER_KEY - Optional base64 32-byte key for local managed SSE fallback
+  S3TESTS_ENABLE_LOCAL_KMS - Enable local KMS for SSE-KMS cases (default: true)
+  S3_KMS_KEY_ID          - s3-tests KMS key id (default: rustfs-s3tests-default-key)
+  S3TESTS_KMS_KEY_DIR    - Host key directory for local KMS (default: DATA_ROOT/kms-keys)
   RUSTFS_SCANNER_ENABLED - Enable background scanner for harness service (default: false)
   MAXFAIL                - Stop after N failures, 0 = never stop (default: 1)
   XDIST                  - Enable parallel execution with N workers (default: 0)
@@ -345,6 +353,52 @@ cleanup() {
 
 trap cleanup EXIT
 
+prepare_s3tests_local_kms() {
+    if [ "${S3TESTS_ENABLE_LOCAL_KMS}" != "true" ]; then
+        return 0
+    fi
+    if [ "${DEPLOY_MODE}" = "existing" ]; then
+        log_warn "Skipping local KMS setup for DEPLOY_MODE=existing; set S3_KMS_KEY_ID only when the target service is KMS-enabled"
+        return 0
+    fi
+    if [ "${DEPLOY_MODE}" = "docker" ] && [ -z "${S3TESTS_KMS_KEY_DIR:-}" ]; then
+        S3TESTS_KMS_HOST_KEY_DIR="/tmp/${CONTAINER_NAME}/kms-keys"
+        S3TESTS_KMS_RUNTIME_KEY_DIR="/data/kms-keys"
+    fi
+
+    mkdir -p "${S3TESTS_KMS_HOST_KEY_DIR}"
+    cat > "${S3TESTS_KMS_HOST_KEY_DIR}/${S3_KMS_KEY_ID}.key" <<EOF
+{
+  "key_id": "${S3_KMS_KEY_ID}",
+  "version": 1,
+  "algorithm": "AES_256",
+  "usage": "EncryptDecrypt",
+  "status": "Active",
+  "metadata": {},
+  "created_at": "2026-01-01T00:00:00+00:00[UTC]",
+  "rotated_at": null,
+  "created_by": "s3-tests",
+  "encrypted_key_material": "${S3TESTS_LOCAL_SSE_MASTER_KEY_DEFAULT}",
+  "nonce": [],
+  "at_rest_protection": "plaintext-dev-only"
+}
+EOF
+    chmod 700 "${S3TESTS_KMS_HOST_KEY_DIR}"
+    chmod 600 "${S3TESTS_KMS_HOST_KEY_DIR}/${S3_KMS_KEY_ID}.key"
+    export RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS="true"
+    export RUSTFS_KMS_ENABLE="true"
+    export RUSTFS_KMS_BACKEND="local"
+    export RUSTFS_KMS_KEY_DIR="${S3TESTS_KMS_RUNTIME_KEY_DIR}"
+    export RUSTFS_KMS_DEFAULT_KEY_ID="${S3_KMS_KEY_ID}"
+    RUSTFS_KMS_ARGS=(
+        --kms-enable
+        --kms-backend local
+        --kms-key-dir "${S3TESTS_KMS_RUNTIME_KEY_DIR}"
+        --kms-default-key-id "${S3_KMS_KEY_ID}"
+    )
+    log_info "Using local KMS key '${S3_KMS_KEY_ID}' for the s3-tests harness"
+}
+
 # Parse command line arguments
 while [[ $# -gt 0 ]]; do
     case $1 in
@@ -375,6 +429,8 @@ if [ "${DEPLOY_MODE}" != "existing" ]; then
     fi
 fi
 
+prepare_s3tests_local_kms
+
 # Start RustFS based on deployment mode
 if [ "${DEPLOY_MODE}" = "existing" ]; then
     log_info "Using existing RustFS service at ${S3_HOST}:${S3_PORT}"
@@ -408,6 +464,7 @@ elif [ "${DEPLOY_MODE}" = "binary" ]; then
         --address "${S3_HOST}:${S3_PORT}" \
         --access-key "${S3_ACCESS_KEY}" \
         --secret-key "${S3_SECRET_KEY}" \
+        "${RUSTFS_KMS_ARGS[@]}" \
         "${DATA_DIR}/rustfs0" "${DATA_DIR}/rustfs1" "${DATA_DIR}/rustfs2" "${DATA_DIR}/rustfs3" \
         > "${ARTIFACTS_DIR}/rustfs-${TEST_MODE}/rustfs.log" 2>&1 &
 
@@ -472,6 +529,7 @@ elif [ "${DEPLOY_MODE}" = "build" ]; then
         --address "${S3_HOST}:${S3_PORT}" \
         --access-key "${S3_ACCESS_KEY}" \
         --secret-key "${S3_SECRET_KEY}" \
+        "${RUSTFS_KMS_ARGS[@]}" \
         "${DATA_DIR}/rustfs0" "${DATA_DIR}/rustfs1" "${DATA_DIR}/rustfs2" "${DATA_DIR}/rustfs3" \
         > "${ARTIFACTS_DIR}/rustfs-${TEST_MODE}/rustfs.log" 2>&1 &
 
@@ -506,6 +564,11 @@ elif [ "${DEPLOY_MODE}" = "docker" ]; then
         -e RUSTFS_ACCESS_KEY="${S3_ACCESS_KEY}" \
         -e RUSTFS_SECRET_KEY="${S3_SECRET_KEY}" \
         -e RUSTFS_SSE_S3_MASTER_KEY="${RUSTFS_SSE_S3_MASTER_KEY}" \
+        -e RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS="${RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS:-false}" \
+        -e RUSTFS_KMS_ENABLE="${RUSTFS_KMS_ENABLE:-false}" \
+        -e RUSTFS_KMS_BACKEND="${RUSTFS_KMS_BACKEND:-local}" \
+        -e RUSTFS_KMS_KEY_DIR="${RUSTFS_KMS_KEY_DIR:-}" \
+        -e RUSTFS_KMS_DEFAULT_KEY_ID="${RUSTFS_KMS_DEFAULT_KEY_ID:-}" \
         -e RUSTFS_SCANNER_ENABLED="${RUSTFS_SCANNER_ENABLED}" \
         -e RUSTFS_SCANNER_START_DELAY_SECS="${RUSTFS_SCANNER_START_DELAY_SECS}" \
         -e RUSTFS_SCANNER_CYCLE="${RUSTFS_SCANNER_CYCLE}" \
@@ -761,6 +824,11 @@ envsubst < "${TEMPLATE_PATH}" > "${CONF_OUTPUT_PATH}" || {
     log_error "Failed to generate s3tests config"
     exit 1
 }
+if [ -n "${S3_KMS_KEY_ID:-}" ]; then
+    tmp_conf="${CONF_OUTPUT_PATH}.tmp"
+    sed "s|^#kms_keyid = .*$|kms_keyid = ${S3_KMS_KEY_ID}|" "${CONF_OUTPUT_PATH}" > "${tmp_conf}"
+    mv "${tmp_conf}" "${CONF_OUTPUT_PATH}"
+fi
 
 # Step 7: Provision s3-tests alt user
 # Note: Main user (rustfsadmin) is a system user and doesn't need to be created via API