From c62384e58dc18cf207eb1c6b9998c8ca338f5b45 Mon Sep 17 00:00:00 2001 From: houseme Date: Wed, 9 Sep 2026 12:08:06 +0800 Subject: [PATCH] ci(test): enable local KMS for s3 compatibility tests (#7553) Configure the s3-tests harness with a local KMS key so SSE-KMS cases run in CI without relying on an external KMS service. Also move the anonymous POST default SSE-KMS regression onto the shared local KMS test environment. Co-authored-by: zhi22915 --- crates/e2e_test/src/multipart_auth_test.rs | 11 ++-- scripts/s3-tests/run.sh | 68 ++++++++++++++++++++++ 2 files changed, 74 insertions(+), 5 deletions(-) diff --git a/crates/e2e_test/src/multipart_auth_test.rs b/crates/e2e_test/src/multipart_auth_test.rs index d4606a8ff..613bf0839 100644 --- a/crates/e2e_test/src/multipart_auth_test.rs +++ b/crates/e2e_test/src/multipart_auth_test.rs @@ -15,6 +15,7 @@ //! Regression coverage for anonymous access on multipart control APIs. use crate::common::{RustFSTestEnvironment, init_logging, local_http_client}; +use crate::kms::common::LocalKMSTestEnvironment; use async_compression::tokio::write::{BzEncoder, Lz4Encoder, XzEncoder}; use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError}; use aws_sdk_s3::operation::head_object::HeadObjectOutput; @@ -1465,10 +1466,10 @@ async fn test_anonymous_post_object_uses_bucket_default_sse_s3() -> Result<(), B async fn test_anonymous_post_object_uses_bucket_default_sse_kms() -> Result<(), Box> { init_logging(); - let mut env = RustFSTestEnvironment::new().await?; - let master_key = local_sse_master_key_value(); - env.start_rustfs_server_with_env(vec![], &[(LOCAL_SSE_MASTER_KEY_ENV, master_key.as_str())]) - .await?; + let mut kms_env = LocalKMSTestEnvironment::new().await?; + let default_key_id = kms_env.start_rustfs_for_local_kms().await?; + kms_env.wait_for_kms_ready().await?; + let env = &kms_env.base_env; let bucket = "anon-post-default-sse-kms"; let object_key = "post-default-sse-kms-object.txt"; @@ -1484,7 +1485,7 @@ async fn test_anonymous_post_object_uses_bucket_default_sse_kms() -> Result<(), .apply_server_side_encryption_by_default( ServerSideEncryptionByDefault::builder() .sse_algorithm(ServerSideEncryption::AwsKms) - .kms_master_key_id("test-key") + .kms_master_key_id(default_key_id) .build() .expect("default encryption rule should build"), ) diff --git a/scripts/s3-tests/run.sh b/scripts/s3-tests/run.sh index bd197e0f5..593bec898 100755 --- a/scripts/s3-tests/run.sh +++ b/scripts/s3-tests/run.sh @@ -239,6 +239,8 @@ DEPLOY_MODE="${DEPLOY_MODE:-build}" RUSTFS_BINARY="${RUSTFS_BINARY:-}" NO_CACHE="${NO_CACHE:-false}" S3TESTS_LOCAL_SSE_MASTER_KEY_DEFAULT="MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=" +S3TESTS_ENABLE_LOCAL_KMS="${S3TESTS_ENABLE_LOCAL_KMS:-true}" +S3_KMS_KEY_ID="${S3_KMS_KEY_ID:-rustfs-s3tests-default-key}" # Additional directories (SCRIPT_DIR and PROJECT_ROOT defined earlier) ARTIFACTS_DIR="${PROJECT_ROOT}/artifacts/s3tests-${TEST_MODE}" @@ -252,6 +254,9 @@ else fi DATA_DIR="${DATA_BASE}/test-data/${CONTAINER_NAME}" RUSTFS_PID="" +RUSTFS_KMS_ARGS=() +S3TESTS_KMS_HOST_KEY_DIR="${S3TESTS_KMS_KEY_DIR:-${DATA_BASE}/kms-keys/${CONTAINER_NAME}}" +S3TESTS_KMS_RUNTIME_KEY_DIR="${S3TESTS_KMS_HOST_KEY_DIR}" if [ "${DEPLOY_MODE}" != "existing" ] && [ -z "${RUSTFS_SSE_S3_MASTER_KEY:-}" ]; then export RUSTFS_SSE_S3_MASTER_KEY="${S3TESTS_LOCAL_SSE_MASTER_KEY_DEFAULT}" @@ -282,6 +287,9 @@ Environment Variables: S3_ALT_ACCESS_KEY - Alt user access key (default: rustfsalt) S3_ALT_SECRET_KEY - Alt user secret key (default: rustfsalt) RUSTFS_SSE_S3_MASTER_KEY - Optional base64 32-byte key for local managed SSE fallback + S3TESTS_ENABLE_LOCAL_KMS - Enable local KMS for SSE-KMS cases (default: true) + S3_KMS_KEY_ID - s3-tests KMS key id (default: rustfs-s3tests-default-key) + S3TESTS_KMS_KEY_DIR - Host key directory for local KMS (default: DATA_ROOT/kms-keys) RUSTFS_SCANNER_ENABLED - Enable background scanner for harness service (default: false) MAXFAIL - Stop after N failures, 0 = never stop (default: 1) XDIST - Enable parallel execution with N workers (default: 0) @@ -345,6 +353,52 @@ cleanup() { trap cleanup EXIT +prepare_s3tests_local_kms() { + if [ "${S3TESTS_ENABLE_LOCAL_KMS}" != "true" ]; then + return 0 + fi + if [ "${DEPLOY_MODE}" = "existing" ]; then + log_warn "Skipping local KMS setup for DEPLOY_MODE=existing; set S3_KMS_KEY_ID only when the target service is KMS-enabled" + return 0 + fi + if [ "${DEPLOY_MODE}" = "docker" ] && [ -z "${S3TESTS_KMS_KEY_DIR:-}" ]; then + S3TESTS_KMS_HOST_KEY_DIR="/tmp/${CONTAINER_NAME}/kms-keys" + S3TESTS_KMS_RUNTIME_KEY_DIR="/data/kms-keys" + fi + + mkdir -p "${S3TESTS_KMS_HOST_KEY_DIR}" + cat > "${S3TESTS_KMS_HOST_KEY_DIR}/${S3_KMS_KEY_ID}.key" < "${ARTIFACTS_DIR}/rustfs-${TEST_MODE}/rustfs.log" 2>&1 & @@ -472,6 +529,7 @@ elif [ "${DEPLOY_MODE}" = "build" ]; then --address "${S3_HOST}:${S3_PORT}" \ --access-key "${S3_ACCESS_KEY}" \ --secret-key "${S3_SECRET_KEY}" \ + "${RUSTFS_KMS_ARGS[@]}" \ "${DATA_DIR}/rustfs0" "${DATA_DIR}/rustfs1" "${DATA_DIR}/rustfs2" "${DATA_DIR}/rustfs3" \ > "${ARTIFACTS_DIR}/rustfs-${TEST_MODE}/rustfs.log" 2>&1 & @@ -506,6 +564,11 @@ elif [ "${DEPLOY_MODE}" = "docker" ]; then -e RUSTFS_ACCESS_KEY="${S3_ACCESS_KEY}" \ -e RUSTFS_SECRET_KEY="${S3_SECRET_KEY}" \ -e RUSTFS_SSE_S3_MASTER_KEY="${RUSTFS_SSE_S3_MASTER_KEY}" \ + -e RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS="${RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS:-false}" \ + -e RUSTFS_KMS_ENABLE="${RUSTFS_KMS_ENABLE:-false}" \ + -e RUSTFS_KMS_BACKEND="${RUSTFS_KMS_BACKEND:-local}" \ + -e RUSTFS_KMS_KEY_DIR="${RUSTFS_KMS_KEY_DIR:-}" \ + -e RUSTFS_KMS_DEFAULT_KEY_ID="${RUSTFS_KMS_DEFAULT_KEY_ID:-}" \ -e RUSTFS_SCANNER_ENABLED="${RUSTFS_SCANNER_ENABLED}" \ -e RUSTFS_SCANNER_START_DELAY_SECS="${RUSTFS_SCANNER_START_DELAY_SECS}" \ -e RUSTFS_SCANNER_CYCLE="${RUSTFS_SCANNER_CYCLE}" \ @@ -761,6 +824,11 @@ envsubst < "${TEMPLATE_PATH}" > "${CONF_OUTPUT_PATH}" || { log_error "Failed to generate s3tests config" exit 1 } +if [ -n "${S3_KMS_KEY_ID:-}" ]; then + tmp_conf="${CONF_OUTPUT_PATH}.tmp" + sed "s|^#kms_keyid = .*$|kms_keyid = ${S3_KMS_KEY_ID}|" "${CONF_OUTPUT_PATH}" > "${tmp_conf}" + mv "${tmp_conf}" "${CONF_OUTPUT_PATH}" +fi # Step 7: Provision s3-tests alt user # Note: Main user (rustfsadmin) is a system user and doesn't need to be created via API