fix(s3): return proper HTTP 400 for SSE-C validation errors (#1998)

This commit is contained in:
安正超
2026-02-28 10:24:46 +08:00
committed by GitHub
parent af6c32efac
commit a24cbbb7a6
6 changed files with 395 additions and 85 deletions
+8
View File
@@ -2745,6 +2745,14 @@ impl DefaultObjectUsecase {
{
return Err(S3Error::new(S3ErrorCode::PreconditionFailed));
}
// Validate SSE-C: if the object was encrypted with a customer-provided key,
// the caller must supply the matching key even for HEAD requests (per S3 spec).
validate_ssec_for_read(
&info.user_defined,
req.input.sse_customer_key.as_ref(),
req.input.sse_customer_key_md5.as_ref(),
)?;
let event_info = info.clone();
let content_type = {
if let Some(content_type) = &info.content_type {
+1 -1
View File
@@ -36,5 +36,5 @@ mod sse_test;
pub(crate) use ecfs_extend::*;
pub(crate) use sse::{
DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, sse_decryption, sse_encryption, sse_prepare_encryption,
strip_managed_encryption_metadata,
strip_managed_encryption_metadata, validate_ssec_for_read,
};
+188 -23
View File
@@ -87,6 +87,7 @@ use rustfs_kms::{
types::{EncryptionMetadata, ObjectEncryptionContext},
};
use rustfs_rio::{DecryptReader, EncryptReader, HardLimitReader, Reader, WarpReader};
use s3s::S3ErrorCode;
use s3s::dto::ServerSideEncryption;
use std::collections::HashMap;
use std::sync::{Arc, OnceLock};
@@ -632,9 +633,10 @@ pub async fn sse_decryption(request: DecryptionRequest<'_>) -> Result<Option<Dec
let (key, key_md5) = match (request.sse_customer_key, request.sse_customer_key_md5) {
(Some(k), Some(md5)) => (k, md5),
_ => {
return Err(ApiError::from(StorageError::other(
"Object is encrypted with SSE-C but no customer key provided",
)));
return Err(ssec_invalid_request(
"The object was stored using a form of Server Side Encryption. \
The correct parameters must be provided to retrieve the object.",
));
}
};
@@ -1432,38 +1434,36 @@ pub(crate) async fn decrypt_multipart_managed_stream(
/// # Returns
/// `ValidatedSsecParams` with decoded key bytes
pub fn validate_ssec_params(params: SsecParams) -> Result<ValidatedSsecParams, ApiError> {
// Validate algorithm
if !SUPPORT_SSE_ALGORITHMS.contains(&params.algorithm.as_str()) {
return Err(ApiError::from(StorageError::other(format!(
"Unsupported SSE-C algorithm: {}. Only {} is supported",
return Err(ssec_invalid_request(&format!(
"Unsupported SSE-C algorithm: {}. Only {} is supported.",
params.algorithm, DEFAULT_SSE_ALGORITHM
))));
)));
}
// Decode Base64 key
let key_bytes = BASE64_STANDARD.decode(&params.key).map_err(|e| {
error!("Failed to decode SSE-C key: {}", e);
ApiError::from(StorageError::other("Invalid SSE-C key: not valid Base64"))
ssec_invalid_request("Invalid SSE-C key: not valid Base64.")
})?;
// Validate key length (must be 32 bytes for AES-256)
if key_bytes.len() != 32 {
return Err(ApiError::from(StorageError::other(format!(
"SSE-C key must be 32 bytes (256 bits), got {} bytes",
return Err(ssec_invalid_request(&format!(
"SSE-C key must be 32 bytes (256 bits), got {} bytes.",
key_bytes.len()
))));
)));
}
// Verify MD5 hash
let computed_md5 = BASE64_STANDARD.encode(md5::compute(&key_bytes).0);
if computed_md5 != params.key_md5 {
error!("SSE-C key MD5 mismatch: expected '{}', got '{}'", params.key_md5, computed_md5);
return Err(ApiError::from(StorageError::other("SSE-C key MD5 mismatch")));
return Err(ssec_invalid_request(
"The calculated MD5 hash of the key did not match the hash that was provided.",
));
}
let key_array: [u8; 32] = key_bytes
.try_into()
.map_err(|_| ApiError::from(StorageError::other("SSE-C key must be exactly 32 bytes")))?;
.map_err(|_| ssec_invalid_request("SSE-C key must be exactly 32 bytes."))?;
Ok(ValidatedSsecParams {
algorithm: params.algorithm,
@@ -1485,17 +1485,71 @@ pub fn generate_ssec_nonce(bucket: &str, key: &str) -> [u8; 12] {
nonce
}
/// Verify SSE-C key matches the stored metadata
/// Verify SSE-C key matches the stored metadata.
///
/// Used during GetObject to ensure the client provided the correct key.
/// Used during GetObject/HeadObject to ensure the client provided the correct key.
/// Returns 400 InvalidRequest on mismatch, consistent with AWS S3 behavior.
pub fn verify_ssec_key_match(provided_md5: &str, stored_md5: Option<&String>) -> Result<(), ApiError> {
match stored_md5 {
Some(stored) if stored == provided_md5 => Ok(()),
Some(stored) => Err(ApiError::from(StorageError::other(format!(
"SSE-C key MD5 mismatch: provided '{}' but expected '{}'",
provided_md5, stored
)))),
None => Err(ApiError::from(StorageError::other("Object has no stored SSE-C key MD5"))),
Some(_) => Err(ssec_invalid_request(
"The provided encryption parameters did not match the ones used originally to encrypt the object.",
)),
None => Err(ssec_invalid_request("Object has no stored SSE-C key metadata.")),
}
}
/// Validate that the SSE-C headers required for reading an SSE-C encrypted object
/// are present in the request. This is used by HeadObject which does not decrypt
/// the data but still must verify the caller holds the correct key.
///
/// Performs full validation: decodes the customer key, recomputes its MD5,
/// verifies the client-provided MD5 header matches the key, then compares
/// the computed MD5 against the stored metadata. This prevents a client from
/// bypassing validation by guessing/obtaining only the stored MD5 without
/// possessing the actual encryption key.
///
/// Returns `Ok(())` if either the object is not SSE-C encrypted, or valid SSE-C
/// headers are provided and the key matches. Returns 400 InvalidRequest otherwise.
pub fn validate_ssec_for_read(
metadata: &HashMap<String, String>,
sse_customer_key: Option<&SSECustomerKey>,
sse_customer_key_md5: Option<&SSECustomerKeyMD5>,
) -> Result<(), ApiError> {
let stored_algorithm = metadata.get("x-amz-server-side-encryption-customer-algorithm");
if stored_algorithm.is_none() {
return Ok(());
}
let (key, key_md5) = match (sse_customer_key, sse_customer_key_md5) {
(Some(k), Some(md5)) => (k, md5),
_ => {
return Err(ssec_invalid_request(
"The object was stored using a form of Server Side Encryption. \
The correct parameters must be provided to retrieve the object.",
));
}
};
// Full param validation: decode key, verify 32 bytes, recompute MD5
// from actual key bytes and compare to the client-provided MD5 header.
let algorithm = stored_algorithm.cloned().unwrap_or_else(|| DEFAULT_SSE_ALGORITHM.to_string());
let validated = validate_ssec_params(SsecParams {
algorithm,
key: key.to_string(),
key_md5: key_md5.clone(),
})?;
let stored_md5 = metadata.get("x-amz-server-side-encryption-customer-key-md5");
verify_ssec_key_match(&validated.key_md5, stored_md5)
}
/// Build an `ApiError` with `InvalidRequest` (HTTP 400) for SSE-C related errors.
fn ssec_invalid_request(message: &str) -> ApiError {
ApiError {
code: S3ErrorCode::InvalidRequest,
message: message.to_string(),
source: None,
}
}
@@ -1932,4 +1986,115 @@ mod tests {
let debug_str = format!("{:?}", SSEType::SseKms);
assert!(debug_str.contains("SseKms"));
}
#[test]
fn test_verify_ssec_key_match_returns_invalid_request() {
let stored = "stored_md5".to_string();
let err = verify_ssec_key_match("wrong_md5", Some(&stored)).unwrap_err();
assert_eq!(err.code, S3ErrorCode::InvalidRequest);
}
#[test]
fn test_verify_ssec_key_match_no_stored_returns_invalid_request() {
let err = verify_ssec_key_match("any_md5", None).unwrap_err();
assert_eq!(err.code, S3ErrorCode::InvalidRequest);
}
#[test]
fn test_validate_ssec_for_read_non_encrypted_object() {
let metadata = HashMap::new();
let result = validate_ssec_for_read(&metadata, None, None);
assert!(result.is_ok());
}
#[test]
fn test_validate_ssec_for_read_missing_customer_key() {
let mut metadata = HashMap::new();
metadata.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string());
metadata.insert(
"x-amz-server-side-encryption-customer-key-md5".to_string(),
"DWygnHRtgiJ77HCm+1rvHw==".to_string(),
);
let err = validate_ssec_for_read(&metadata, None, None).unwrap_err();
assert_eq!(err.code, S3ErrorCode::InvalidRequest);
}
#[test]
fn test_validate_ssec_for_read_wrong_key() {
// Key A is used to "encrypt" the object (stored MD5 is from key A).
let key_a = [42u8; 32];
let stored_md5 = BASE64_STANDARD.encode(md5::compute(key_a).0);
let mut metadata = HashMap::new();
metadata.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string());
metadata.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), stored_md5);
// Key B is a different key; its MD5 won't match stored MD5.
let key_b = [99u8; 32];
let key_b_b64 = BASE64_STANDARD.encode(key_b);
let key_b_md5 = BASE64_STANDARD.encode(md5::compute(key_b).0);
let err = validate_ssec_for_read(&metadata, Some(&key_b_b64), Some(&key_b_md5)).unwrap_err();
assert_eq!(err.code, S3ErrorCode::InvalidRequest);
}
#[test]
fn test_validate_ssec_for_read_correct_key() {
let key_bytes = [42u8; 32];
let key_b64 = BASE64_STANDARD.encode(key_bytes);
let key_md5 = BASE64_STANDARD.encode(md5::compute(key_bytes).0);
let mut metadata = HashMap::new();
metadata.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string());
metadata.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), key_md5.clone());
let result = validate_ssec_for_read(&metadata, Some(&key_b64), Some(&key_md5));
assert!(result.is_ok());
}
#[test]
fn test_validate_ssec_for_read_spoofed_md5() {
// A client provides the correct stored MD5 in the header but with a
// DIFFERENT key. The server must recompute MD5 from the key bytes and
// reject the request because the recomputed MD5 won't match the header.
let real_key = [42u8; 32];
let stored_md5 = BASE64_STANDARD.encode(md5::compute(real_key).0);
let mut metadata = HashMap::new();
metadata.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string());
metadata.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), stored_md5.clone());
// Attacker has a different key but tries to pass the stored MD5 as their header
let fake_key = [99u8; 32];
let fake_key_b64 = BASE64_STANDARD.encode(fake_key);
let err = validate_ssec_for_read(&metadata, Some(&fake_key_b64), Some(&stored_md5)).unwrap_err();
assert_eq!(err.code, S3ErrorCode::InvalidRequest);
}
#[test]
fn test_validate_ssec_params_returns_invalid_request_on_bad_algorithm() {
let key = BASE64_STANDARD.encode([42u8; 32]);
let key_md5 = BASE64_STANDARD.encode(md5::compute([42u8; 32]).0);
let params = SsecParams {
algorithm: "AES128".to_string(),
key,
key_md5,
};
let err = validate_ssec_params(params).unwrap_err();
assert_eq!(err.code, S3ErrorCode::InvalidRequest);
}
#[test]
fn test_validate_ssec_params_returns_invalid_request_on_bad_md5() {
let key = BASE64_STANDARD.encode([42u8; 32]);
let params = SsecParams {
algorithm: "AES256".to_string(),
key,
key_md5: BASE64_STANDARD.encode([99u8; 16]),
};
let err = validate_ssec_params(params).unwrap_err();
assert_eq!(err.code, S3ErrorCode::InvalidRequest);
}
}
+144
View File
@@ -0,0 +1,144 @@
# S3 Compatibility Fix Workflow
Step-by-step guide for identifying and fixing S3 API compatibility issues in RustFS.
## Prerequisites
- Rust toolchain installed (`cargo`, `rustc`)
- Python 3 for running ceph s3-tests
- Access to upstream remote (`git remote add upstream https://github.com/rustfs/rustfs.git`)
- Familiarity with the [run.sh](./run.sh) test runner
## Workflow
### Step 1: Sync with upstream
```bash
git checkout main
git fetch upstream
git merge upstream/main
```
### Step 2: Select candidate tests
Pick ~20 tests from `unimplemented_tests.txt` and write them to `selected_tests.txt`:
```bash
TESTEXPR=$(scripts/s3-tests/build_testexpr.sh selected_tests.txt) \
DEPLOY_MODE=build MAXFAIL=0 ./scripts/s3-tests/run.sh
```
Or run them directly:
```bash
TESTEXPR="test_foo or test_bar" DEPLOY_MODE=build MAXFAIL=0 ./scripts/s3-tests/run.sh
```
Review `artifacts/s3tests-single/pytest.log` for results.
### Step 3: Triage failures
From the test report, classify each failure:
| Category | Action |
|----------|--------|
| Feature not implemented (e.g., returns `501 Not Implemented`) | Skip — do not attempt |
| Incorrect HTTP status code or response body | Good candidate for a fix |
| Teardown/cleanup issue (test passes but cleanup fails) | Good candidate — usually simple |
| Complex multi-feature dependency | Defer to a later iteration |
Pick the **simplest** failure to fix. "Simplest" means: fewest code paths affected, clearest expected behavior, closest to existing implementation.
### Step 4: Deep analysis
Before writing any code:
1. **Read the test source** in `s3-tests/s3tests/functional/test_s3.py` to understand exactly what the test expects.
2. **Read the S3 API specification** for the operation being tested.
3. **Search the RustFS codebase** for the handler that serves this operation.
4. **Compare with MinIO** (`github.com/minio/minio`) — find the equivalent handler and see how it handles the same edge case. This is critical because RustFS was ported from MinIO's Go code to Rust.
5. **Identify the root cause** — is it a missing header, wrong status code, incorrect XML response, logic bug, etc.?
6. **Document your findings** before making changes.
### Step 5: Create a fix branch
```bash
git checkout main
git checkout -b fix/s3-compat-<short-description>
```
One branch per fix. Never combine unrelated fixes in a single branch.
### Step 6: Write tests first (when applicable)
If the fix involves logic changes, add unit tests before modifying the production code:
- Co-locate tests with their module (`#[cfg(test)] mod tests { ... }`)
- Use descriptive test names: `test_<operation>_<scenario>_<expected_outcome>`
- Cover both the happy path and the edge case being fixed
### Step 7: Implement the fix
Guidelines:
- **Reuse existing abstractions** — do not duplicate logic that already exists in helper functions or shared modules.
- **Follow s3s conventions** — RustFS's S3 layer is built on the `s3s` crate. Respect its traits, error types, and request/response patterns.
- **Name things clearly** — variable names, function names, and types should be self-explanatory.
- **Add comments only where intent is non-obvious** — explain *why*, not *what*.
- **Do not introduce security holes** — validate inputs, check permissions, handle errors properly.
- **Do not use `unwrap()` or `expect()` in production code** — use proper error handling with `Result` and `?`.
### Step 8: Verify the fix
Run the specific test(s) you fixed:
```bash
TESTEXPR="test_the_fixed_test" DEPLOY_MODE=build ./scripts/s3-tests/run.sh
```
Then run the full implemented test suite to confirm no regressions:
```bash
./scripts/s3-tests/run.sh
```
### Step 9: Run quality checks
```bash
make pre-commit
```
This runs:
- `cargo fmt --all --check`
- `cargo clippy --all-targets --all-features -- -D warnings`
- `cargo test --workspace --exclude e2e_test`
All three must pass before committing.
### Step 10: Commit and prepare PR
```bash
git add -A
git commit -m "fix(s3): <concise description of the fix>"
```
Write a PR description following `.github/pull_request_template.md`. The description must:
- Be written in English
- Use plain, natural language (no emoji, no marketing speak)
- Explain what was wrong, why, and how it was fixed
- Reference the specific s3-tests that now pass
### Step 11: Update test lists
Move the now-passing test(s) from `unimplemented_tests.txt` to `implemented_tests.txt`. Update the test count comment in `implemented_tests.txt`.
## Important Rules
1. **One branch, one fix** — never mix unrelated changes.
2. **Analyze before coding** — understand the root cause thoroughly before writing a fix.
3. **No shotgun debugging** — do not blindly try different return codes or response shapes hoping to pass the test.
4. **Every change must be justified** — if you cannot explain why a line changed, do not change it.
5. **Compare with MinIO** — when in doubt about the correct behavior, check MinIO's source code for the equivalent logic.
6. **Security first** — never skip input validation, permission checks, or error handling for the sake of passing a test.
+52 -1
View File
@@ -17,7 +17,10 @@
# - Metadata: User-defined metadata
# - Conditional GET: If-Match, If-None-Match, If-Modified-Since
#
# Total: 123 tests
# - SSE-C: Server-side encryption with customer-provided keys
# - Object ownership: Bucket ownership controls
#
# Total: 159 tests
test_basic_key_count
test_bucket_create_naming_bad_short_one
@@ -149,3 +152,51 @@ test_set_multipart_tagging
test_upload_part_copy_percent_encoded_key
test_api_error_from_storage_error_mappings
test_get_object_torrent
# SSE-C encryption tests
test_encryption_sse_c_method_head
test_encryption_sse_c_present
test_encryption_sse_c_other_key
# ListObjectsV2 delimiter and encoding tests
test_bucket_list_encoding_basic
test_bucket_listv2_delimiter_alt
test_bucket_listv2_delimiter_basic
test_bucket_listv2_delimiter_dot
test_bucket_listv2_delimiter_empty
test_bucket_listv2_delimiter_none
test_bucket_listv2_delimiter_not_exist
test_bucket_listv2_delimiter_percentage
test_bucket_listv2_delimiter_prefix_ends_with_delimiter
test_bucket_listv2_delimiter_unreadable
test_bucket_listv2_delimiter_whitespace
test_bucket_listv2_encoding_basic
# Multipart and tagging tests
test_abort_multipart_upload
test_multipart_resend_first_finishes_last
test_set_bucket_tagging
test_put_excess_key_tags
test_put_excess_tags
test_put_excess_val_tags
# PublicAccessBlock tests
test_put_get_delete_public_block
test_put_public_block
test_block_public_policy
test_block_public_policy_with_principal
test_get_public_block_deny_bucket_policy
test_get_undefined_public_block
# Bucket policy tests
test_bucketv2_policy
test_bucket_policy_acl
test_bucketv2_policy_acl
test_bucket_policy_another_bucket
test_bucket_policy_allow_notprincipal
test_bucket_policy_put_obj_acl
test_object_presigned_put_object_with_acl
test_object_put_acl_mtime
# Object ownership
test_create_bucket_no_ownership_controls
+2 -60
View File
@@ -16,9 +16,6 @@
# - STS: Security Token Service
# - Checksum: Full checksum validation
# - Conditional writes: If-Match/If-None-Match for writes
# - Object ownership: BucketOwnerEnforced/Preferred
#
# Total: all unimplemented S3 feature tests listed below (keep this comment in sync with the list)
test_bucket_create_delete_bucket_ownership
test_bucket_logging_owner
@@ -32,12 +29,9 @@ test_delete_bucket_encryption_kms
test_delete_bucket_encryption_s3
test_encryption_key_no_sse_c
test_encryption_sse_c_invalid_md5
test_encryption_sse_c_method_head
test_encryption_sse_c_multipart_bad_download
test_encryption_sse_c_no_key
test_encryption_sse_c_no_md5
test_encryption_sse_c_other_key
test_encryption_sse_c_present
test_get_bucket_encryption_kms
test_get_bucket_encryption_s3
test_get_versioned_object_attributes
@@ -104,21 +98,6 @@ test_versioning_obj_plain_null_version_overwrite_suspended
test_versioning_obj_plain_null_version_removal
test_versioning_obj_suspend_versions
# Teardown issues (list_object_versions on non-versioned buckets)
# These tests pass but have cleanup issues with list_object_versions
test_bucket_list_encoding_basic
test_bucket_listv2_delimiter_alt
test_bucket_listv2_delimiter_basic
test_bucket_listv2_delimiter_dot
test_bucket_listv2_delimiter_empty
test_bucket_listv2_delimiter_none
test_bucket_listv2_delimiter_not_exist
test_bucket_listv2_delimiter_percentage
test_bucket_listv2_delimiter_prefix_ends_with_delimiter
test_bucket_listv2_delimiter_unreadable
test_bucket_listv2_delimiter_whitespace
test_bucket_listv2_encoding_basic
# Checksum and atomic write tests (require x-amz-checksum-* support)
test_atomic_dual_write_1mb
test_atomic_dual_write_4mb
@@ -130,50 +109,13 @@ test_atomic_read_8mb
test_atomic_write_1mb
test_atomic_write_4mb
test_atomic_write_8mb
test_set_bucket_tagging
# Tests with implementation issues (need investigation)
test_bucket_policy_acl
# Tests with known issues (need further investigation)
test_bucket_policy_different_tenant
test_bucketv2_policy_acl
test_multipart_resend_first_finishes_last
# Multipart abort and policy issues
test_abort_multipart_upload
test_bucket_policy_multipart
# Tests with prefix conflicts or ACL/tenant dependencies
test_bucket_policy
test_bucket_policy_allow_notprincipal
test_bucket_policy_another_bucket
test_bucket_policy_put_obj_acl
test_bucket_policy_put_obj_grant
test_bucket_policy_tenanted_bucket
test_bucketv2_policy
test_object_presigned_put_object_with_acl
test_object_presigned_put_object_with_acl_tenant
test_object_put_acl_mtime
# ACL-dependent tests (PutBucketAcl not implemented)
test_block_public_object_canned_acls
test_block_public_put_bucket_acls
test_get_authpublic_acl_bucket_policy_status
test_get_nonpublicpolicy_acl_bucket_policy_status
test_get_public_acl_bucket_policy_status
test_get_publicpolicy_acl_bucket_policy_status
test_ignore_public_acls
# PublicAccessBlock and tag validation tests
test_block_public_policy
test_block_public_policy_with_principal
test_get_public_block_deny_bucket_policy
test_get_undefined_public_block
test_put_excess_key_tags
test_put_excess_tags
test_put_excess_val_tags
test_put_get_delete_public_block
test_put_public_block
# Object attributes and torrent tests
test_create_bucket_no_ownership_controls
# Object attributes
test_get_checksum_object_attributes