fix(ci): align S3 KMS fixtures and default key expectations (#7573)

scripts/s3-tests/README.md

@@ -238,6 +238,28 @@ DEPLOY_MODE=existing \
   ./scripts/s3-tests/run.sh
 ```
 
+### KMS Test Fixtures
+
+Managed deployments enable local KMS and provision two distinct public test keys.
+`S3_KMS_KEY_ID` selects the primary key (default `rustfs-s3tests-default-key`), and
+`S3_KMS_SECONDARY_KEY_ID` selects the second key (default
+`rustfs-s3tests-secondary-key`). Their IDs and key material differ so encrypted
+copy tests exercise a change of key. The generated config sets `kms_keyid` and
+`kms_keyid2` explicitly.
+
+`S3_KMS_DEFAULT_KEY_ID` defaults to the primary key for managed local KMS. Set it
+to an empty value to start without a default key. The generated
+`kms_default_keyid` setting fixes the expectation for an SSE-KMS PUT without a
+key ID: with a configured default, the patched upstream test requires a
+successful PUT, HEAD and GET with that key ID and matching object contents;
+without one, the request must fail. The test remains in the implemented gate.
+
+For `DEPLOY_MODE=existing`, the harness does not provision keys or infer the
+server's default. Set both key IDs to keys that already exist, and set
+`S3_KMS_DEFAULT_KEY_ID` only when the target has a default. Custom config
+templates should include `kms_keyid`, `kms_keyid2`, and `kms_default_keyid`, or
+the commented placeholders from the bundled template.
+
 ### Custom Configuration Files
 
 ```bash

scripts/s3-tests/patches/0004-sse-kms-default-key-expectation.patch

@@ -0,0 +1,61 @@
+diff --git a/s3tests/functional/__init__.py b/s3tests/functional/__init__.py
+index 5fcb1f4..09af01d 100644
+--- a/s3tests/functional/__init__.py
++++ b/s3tests/functional/__init__.py
+@@ -232,6 +232,8 @@ def configure():
+     except (configparser.NoSectionError, configparser.NoOptionError):
+         config.main_kms_keyid2 = 'testkey-2'
+
++    config.main_kms_default_keyid = cfg.get('s3 main', 'kms_default_keyid', fallback=None)
++
+     try:
+         config.main_api_name = cfg.get('s3 main',"api_name")
+     except (configparser.NoSectionError, configparser.NoOptionError):
+@@ -732,6 +734,9 @@ def get_main_kms_keyid():
+ def get_secondary_kms_keyid():
+     return config.main_kms_keyid2
+
++def get_default_kms_keyid():
++    return config.main_kms_default_keyid or None
++
+ def get_alt_aws_access_key():
+     return config.alt_access_key
+
+diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py
+index aa8879c..7e0c381 100644
+--- a/s3tests/functional/test_s3.py
++++ b/s3tests/functional/test_s3.py
+@@ -79,6 +79,7 @@ from . import (
+     get_objects_list,
+     get_main_kms_keyid,
+     get_secondary_kms_keyid,
++    get_default_kms_keyid,
+     get_svc_client,
+     get_cloud_storage_class,
+     get_cloud_retain_head_object,
+@@ -11282,7 +11283,24 @@ def test_sse_kms_no_key():
+     lf = (lambda **kwargs: kwargs['params']['headers'].update(sse_kms_client_headers))
+     client.meta.events.register('before-call.s3.PutObject', lf)
+
+-    e = assert_raises(ClientError, client.put_object, Bucket=bucket_name, Key=key, Body=data)
++    default_keyid = get_default_kms_keyid()
++    if default_keyid is None:
++        assert_raises(ClientError, client.put_object, Bucket=bucket_name, Key=key, Body=data)
++        return
++
++    # A configured default must encrypt the object, not silently store plaintext.
++    response = client.put_object(Bucket=bucket_name, Key=key, Body=data)
++    assert response['ServerSideEncryption'] == 'aws:kms'
++    assert response['SSEKMSKeyId'] == default_keyid
++
++    response = client.head_object(Bucket=bucket_name, Key=key)
++    assert response['ServerSideEncryption'] == 'aws:kms'
++    assert response['SSEKMSKeyId'] == default_keyid
++
++    response = client.get_object(Bucket=bucket_name, Key=key)
++    assert response['ServerSideEncryption'] == 'aws:kms'
++    assert response['SSEKMSKeyId'] == default_keyid
++    assert _get_body(response) == data
+
+
+ @pytest.mark.encryption

scripts/s3-tests/run.sh

@@ -241,6 +241,13 @@ NO_CACHE="${NO_CACHE:-false}"
 S3TESTS_LOCAL_SSE_MASTER_KEY_DEFAULT="MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY="
 S3TESTS_ENABLE_LOCAL_KMS="${S3TESTS_ENABLE_LOCAL_KMS:-true}"
 S3_KMS_KEY_ID="${S3_KMS_KEY_ID:-rustfs-s3tests-default-key}"
+S3_KMS_SECONDARY_KEY_ID="${S3_KMS_SECONDARY_KEY_ID:-rustfs-s3tests-secondary-key}"
+if [ "${S3TESTS_ENABLE_LOCAL_KMS}" = "true" ] && [ "${DEPLOY_MODE}" != "existing" ]; then
+    # An explicitly empty value exercises the missing-default-key path.
+    S3_KMS_DEFAULT_KEY_ID="${S3_KMS_DEFAULT_KEY_ID-${S3_KMS_KEY_ID}}"
+else
+    S3_KMS_DEFAULT_KEY_ID="${S3_KMS_DEFAULT_KEY_ID:-}"
+fi
 
 # Additional directories (SCRIPT_DIR and PROJECT_ROOT defined earlier)
 ARTIFACTS_DIR="${PROJECT_ROOT}/artifacts/s3tests-${TEST_MODE}"
@@ -289,6 +296,8 @@ Environment Variables:
   RUSTFS_SSE_S3_MASTER_KEY - Optional base64 32-byte key for local managed SSE fallback
   S3TESTS_ENABLE_LOCAL_KMS - Enable local KMS for SSE-KMS cases (default: true)
   S3_KMS_KEY_ID          - s3-tests KMS key id (default: rustfs-s3tests-default-key)
+  S3_KMS_SECONDARY_KEY_ID - Second KMS key id (default: rustfs-s3tests-secondary-key)
+  S3_KMS_DEFAULT_KEY_ID  - Expected default key (managed local KMS: primary key; existing: none; empty disables it)
   S3TESTS_KMS_KEY_DIR    - Host key directory for local KMS (default: DATA_ROOT/kms-keys)
   RUSTFS_SCANNER_ENABLED - Enable background scanner for harness service (default: false)
   MAXFAIL                - Stop after N failures, 0 = never stop (default: 1)
@@ -358,18 +367,30 @@ prepare_s3tests_local_kms() {
         return 0
     fi
     if [ "${DEPLOY_MODE}" = "existing" ]; then
-        log_warn "Skipping local KMS setup for DEPLOY_MODE=existing; set S3_KMS_KEY_ID only when the target service is KMS-enabled"
+        log_warn "Skipping local KMS setup for DEPLOY_MODE=existing; configure both test key IDs and the expected default for the target service"
         return 0
     fi
+    if [ "${S3_KMS_KEY_ID}" = "${S3_KMS_SECONDARY_KEY_ID}" ]; then
+        log_error "S3_KMS_KEY_ID and S3_KMS_SECONDARY_KEY_ID must differ for cross-key tests"
+        return 1
+    fi
     if [ "${DEPLOY_MODE}" = "docker" ] && [ -z "${S3TESTS_KMS_KEY_DIR:-}" ]; then
         S3TESTS_KMS_HOST_KEY_DIR="/tmp/${CONTAINER_NAME}/kms-keys"
         S3TESTS_KMS_RUNTIME_KEY_DIR="/data/kms-keys"
     fi
 
     mkdir -p "${S3TESTS_KMS_HOST_KEY_DIR}"
-    cat > "${S3TESTS_KMS_HOST_KEY_DIR}/${S3_KMS_KEY_ID}.key" <<EOF
+    chmod 700 "${S3TESTS_KMS_HOST_KEY_DIR}"
+    local key_id key_material
+    for key_id in "${S3_KMS_KEY_ID}" "${S3_KMS_SECONDARY_KEY_ID}"; do
+        # Public, distinct test fixtures stay stable when DATA_ROOT is reused.
+        key_material="${S3TESTS_LOCAL_SSE_MASTER_KEY_DEFAULT}"
+        if [ "${key_id}" = "${S3_KMS_SECONDARY_KEY_ID}" ]; then
+            key_material=$(python3 -c 'import base64, hashlib; print(base64.b64encode(hashlib.sha256(b"rustfs-s3-tests-secondary-fixture").digest()).decode())')
+        fi
+        cat > "${S3TESTS_KMS_HOST_KEY_DIR}/${key_id}.key" <<EOF
 {
-  "key_id": "${S3_KMS_KEY_ID}",
+  "key_id": "${key_id}",
   "version": 1,
   "algorithm": "AES_256",
   "usage": "EncryptDecrypt",
@@ -378,25 +399,29 @@ prepare_s3tests_local_kms() {
   "created_at": "2026-01-01T00:00:00+00:00[UTC]",
   "rotated_at": null,
   "created_by": "s3-tests",
-  "encrypted_key_material": "${S3TESTS_LOCAL_SSE_MASTER_KEY_DEFAULT}",
+  "encrypted_key_material": "${key_material}",
   "nonce": [],
   "at_rest_protection": "plaintext-dev-only"
 }
 EOF
-    chmod 700 "${S3TESTS_KMS_HOST_KEY_DIR}"
-    chmod 600 "${S3TESTS_KMS_HOST_KEY_DIR}/${S3_KMS_KEY_ID}.key"
+        chmod 600 "${S3TESTS_KMS_HOST_KEY_DIR}/${key_id}.key"
+    done
     export RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS="true"
     export RUSTFS_KMS_ENABLE="true"
     export RUSTFS_KMS_BACKEND="local"
     export RUSTFS_KMS_KEY_DIR="${S3TESTS_KMS_RUNTIME_KEY_DIR}"
-    export RUSTFS_KMS_DEFAULT_KEY_ID="${S3_KMS_KEY_ID}"
     RUSTFS_KMS_ARGS=(
         --kms-enable
         --kms-backend local
         --kms-key-dir "${S3TESTS_KMS_RUNTIME_KEY_DIR}"
-        --kms-default-key-id "${S3_KMS_KEY_ID}"
     )
-    log_info "Using local KMS key '${S3_KMS_KEY_ID}' for the s3-tests harness"
+    if [ -n "${S3_KMS_DEFAULT_KEY_ID}" ]; then
+        export RUSTFS_KMS_DEFAULT_KEY_ID="${S3_KMS_DEFAULT_KEY_ID}"
+        RUSTFS_KMS_ARGS+=(--kms-default-key-id "${S3_KMS_DEFAULT_KEY_ID}")
+    else
+        unset RUSTFS_KMS_DEFAULT_KEY_ID
+    fi
+    log_info "Using local KMS keys '${S3_KMS_KEY_ID}' and '${S3_KMS_SECONDARY_KEY_ID}' (default: '${S3_KMS_DEFAULT_KEY_ID}')"
 }
 
 # Parse command line arguments
@@ -568,7 +593,7 @@ elif [ "${DEPLOY_MODE}" = "docker" ]; then
         -e RUSTFS_KMS_ENABLE="${RUSTFS_KMS_ENABLE:-false}" \
         -e RUSTFS_KMS_BACKEND="${RUSTFS_KMS_BACKEND:-local}" \
         -e RUSTFS_KMS_KEY_DIR="${RUSTFS_KMS_KEY_DIR:-}" \
-        -e RUSTFS_KMS_DEFAULT_KEY_ID="${RUSTFS_KMS_DEFAULT_KEY_ID:-}" \
+        -e RUSTFS_KMS_DEFAULT_KEY_ID \
         -e RUSTFS_SCANNER_ENABLED="${RUSTFS_SCANNER_ENABLED}" \
         -e RUSTFS_SCANNER_START_DELAY_SECS="${RUSTFS_SCANNER_START_DELAY_SECS}" \
         -e RUSTFS_SCANNER_CYCLE="${RUSTFS_SCANNER_CYCLE}" \
@@ -826,7 +851,11 @@ envsubst < "${TEMPLATE_PATH}" > "${CONF_OUTPUT_PATH}" || {
 }
 if [ -n "${S3_KMS_KEY_ID:-}" ]; then
     tmp_conf="${CONF_OUTPUT_PATH}.tmp"
-    sed "s|^#kms_keyid = .*$|kms_keyid = ${S3_KMS_KEY_ID}|" "${CONF_OUTPUT_PATH}" > "${tmp_conf}"
+    sed \
+        -e "s|^#kms_keyid = .*$|kms_keyid = ${S3_KMS_KEY_ID}|" \
+        -e "s|^#kms_keyid2 = .*$|kms_keyid2 = ${S3_KMS_SECONDARY_KEY_ID}|" \
+        -e "s|^#kms_default_keyid =.*$|kms_default_keyid = ${S3_KMS_DEFAULT_KEY_ID}|" \
+        "${CONF_OUTPUT_PATH}" > "${tmp_conf}"
     mv "${tmp_conf}" "${CONF_OUTPUT_PATH}"
 fi