From 116119d93ae303391f3137dc74a9231a594efa8e Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 25 Aug 2026 04:32:19 +0800 Subject: [PATCH] test(e2e): require exact quota errors (#6544) --- crates/e2e_test/src/quota_test.rs | 159 ++++++++++++++++++++---------- 1 file changed, 105 insertions(+), 54 deletions(-) diff --git a/crates/e2e_test/src/quota_test.rs b/crates/e2e_test/src/quota_test.rs index d6f5b6e8e..7acb51e02 100644 --- a/crates/e2e_test/src/quota_test.rs +++ b/crates/e2e_test/src/quota_test.rs @@ -14,6 +14,7 @@ use crate::common::{RustFSTestEnvironment, admin_request, awscurl_delete, awscurl_get, awscurl_post, awscurl_put, init_logging}; use aws_sdk_s3::Client; +use aws_sdk_s3::error::ProvideErrorMetadata; use http::{Method, StatusCode}; use tokio::time::{Duration, sleep, timeout}; use tracing::{debug, info}; @@ -132,19 +133,13 @@ impl QuotaTestEnv { pub async fn object_exists(&self, key: &str) -> Result> { match self.client.head_object().bucket(&self.bucket_name).key(key).send().await { Ok(_) => Ok(true), - Err(e) => { - // Check for any 404-related errors and return false instead of propagating - let error_str = e.to_string(); - if error_str.contains("404") || error_str.contains("Not Found") || error_str.contains("NotFound") { + Err(error) => { + let status = error.raw_response().map(|response| response.status().as_u16()); + let code = error.as_service_error().and_then(ProvideErrorMetadata::code); + if status == Some(404) && matches!(code, Some("NotFound" | "NoSuchKey")) { Ok(false) } else { - // Also check the error code directly - if let Some(service_err) = e.as_service_error() - && service_err.is_not_found() - { - return Ok(false); - } - Err(e.into()) + Err(error.into()) } } } @@ -278,7 +273,46 @@ impl QuotaTestEnv { #[cfg(test)] mod integration_tests { use super::*; - use aws_sdk_s3::error::ProvideErrorMetadata; + + fn assert_error_response(status: StatusCode, body: &str, expected_status: StatusCode, expected_code: &str) { + assert_eq!(status, expected_status, "unexpected error status: {status} {body}"); + assert!( + body.contains(&format!("{expected_code}")), + "expected {expected_code}, got: {body}" + ); + } + + fn assert_quota_rejection(status: Option, service_error: Option<&E>, error: &impl std::fmt::Debug) + where + E: ProvideErrorMetadata + std::fmt::Debug, + { + assert_eq!(status, Some(400), "quota rejection must return HTTP 400: {error:?}"); + let service_error = service_error.expect("quota rejection must be an S3 service error"); + assert_eq!(service_error.code(), Some("InvalidRequest"), "unexpected quota error: {error:?}"); + assert!( + service_error + .message() + .is_some_and(|message| message.starts_with("Bucket quota exceeded")), + "operation must fail specifically at quota admission: {error:?}" + ); + } + + async fn assert_put_rejected_by_quota(env: &QuotaTestEnv, key: &str, size_bytes: usize) { + let error = env + .client + .put_object() + .bucket(&env.bucket_name) + .key(key) + .body(aws_sdk_s3::primitives::ByteStream::from(vec![0u8; size_bytes])) + .send() + .await + .expect_err("PUT above quota must be rejected"); + assert_quota_rejection( + error.raw_response().map(|response| response.status().as_u16()), + error.as_service_error(), + &error, + ); + } #[tokio::test] async fn test_quota_basic_operations() -> Result<(), Box> { @@ -304,8 +338,7 @@ mod integration_tests { assert!(env.object_exists("test2.txt").await?); // Try to upload 1KB more (should fail due to quota) - let upload_result = env.upload_object("test3.txt", 1024).await; - assert!(upload_result.is_err()); + assert_put_rejected_by_quota(&env, "test3.txt", 1024).await; assert!(!env.object_exists("test3.txt").await?); // Clean up @@ -356,10 +389,10 @@ mod integration_tests { let err = put_aws_chunked("over-quota.bin", 16 * 1024) .await .expect_err("declared aws-chunked PUT over quota must be rejected"); - let err_debug = format!("{err:?}"); - assert!( - !err_debug.contains("UnexpectedContent"), - "over-quota rejection must be the quota error, not UnexpectedContent: {err_debug}" + assert_quota_rejection( + err.raw_response().map(|response| response.status().as_u16()), + err.as_service_error(), + &err, ); assert!(!env.object_exists("over-quota.bin").await?); @@ -581,24 +614,35 @@ mod integration_tests { env.create_bucket().await?; // Test invalid quota type - let url = format!("{}/rustfs/admin/v3/quota/{}", env.env.url, env.bucket_name); + let quota_path = format!("/rustfs/admin/v3/quota/{}", env.bucket_name); let invalid_config = serde_json::json!({ "quota": 1024, "quota_type": "SOFT" // Invalid type }); - let response = awscurl_put(&url, &invalid_config.to_string(), &env.env.access_key, &env.env.secret_key).await; - assert!(response.is_err()); - let error_msg = response.unwrap_err().to_string(); - assert!(error_msg.contains("InvalidArgument")); + let (status, body) = admin_request( + &env.env.url, + Method::PUT, + "a_path, + Some(invalid_config.to_string()), + &env.env.access_key, + &env.env.secret_key, + ) + .await?; + assert_error_response(status, &body, StatusCode::BAD_REQUEST, "InvalidArgument"); // Test operations on non-existent bucket - let url = format!("{}/rustfs/admin/v3/quota/non-existent-bucket", env.env.url); - let response = awscurl_get(&url, &env.env.access_key, &env.env.secret_key).await; - assert!(response.is_err()); - let error_msg = response.unwrap_err().to_string(); - assert!(error_msg.contains("NoSuchBucket")); + let (status, body) = admin_request( + &env.env.url, + Method::GET, + "/rustfs/admin/v3/quota/non-existent-bucket", + None, + &env.env.access_key, + &env.env.secret_key, + ) + .await?; + assert_error_response(status, &body, StatusCode::NOT_FOUND, "NoSuchBucket"); env.cleanup_bucket().await?; @@ -652,10 +696,16 @@ mod integration_tests { "quota": 1024, "quota_type": "SOFT" }); - let response = awscurl_put(&url, &invalid_config.to_string(), &env.env.access_key, &env.env.secret_key).await; - assert!(response.is_err()); - let error_msg = response.unwrap_err().to_string(); - assert!(error_msg.contains("InvalidArgument")); + let (status, body) = admin_request( + &env.env.url, + Method::PUT, + &format!("/rustfs/admin/v3/quota/{}", env.bucket_name), + Some(invalid_config.to_string()), + &env.env.access_key, + &env.env.secret_key, + ) + .await?; + assert_error_response(status, &body, StatusCode::BAD_REQUEST, "InvalidArgument"); env.cleanup_bucket().await?; @@ -698,26 +748,21 @@ mod integration_tests { assert!(resp.contains("quota_limit")); // Normal user sets quota — should be denied - let set_error = awscurl_put( - &get_url, - &serde_json::json!({"quota": 2048, "quota_type": "HARD"}).to_string(), + let quota_path = format!("/rustfs/admin/v3/quota/{}", env.bucket_name); + let (status, body) = admin_request( + &env.env.url, + Method::PUT, + "a_path, + Some(serde_json::json!({"quota": 2048, "quota_type": "HARD"}).to_string()), normal_ak, normal_sk, ) - .await - .expect_err("normal user should not be able to set quota") - .to_string(); - assert!(set_error.contains("AccessDenied"), "quota denial must return AccessDenied: {set_error}"); + .await?; + assert_error_response(status, &body, StatusCode::FORBIDDEN, "AccessDenied"); // Normal user clears quota — should be denied - let delete_error = awscurl_delete(&get_url, normal_ak, normal_sk) - .await - .expect_err("normal user should not be able to clear quota") - .to_string(); - assert!( - delete_error.contains("AccessDenied"), - "quota deletion denial must return AccessDenied: {delete_error}" - ); + let (status, body) = admin_request(&env.env.url, Method::DELETE, "a_path, None, normal_ak, normal_sk).await?; + assert_error_response(status, &body, StatusCode::FORBIDDEN, "AccessDenied"); env.cleanup_bucket().await?; Ok(()) @@ -757,7 +802,12 @@ mod integration_tests { .send() .await; - assert!(copy_result.is_err()); + let copy_error = copy_result.expect_err("copy above quota must be rejected"); + assert_quota_rejection( + copy_error.raw_response().map(|response| response.status().as_u16()), + copy_error.as_service_error(), + ©_error, + ); assert!(!env.object_exists("copy2.txt").await?); env.cleanup_bucket().await?; @@ -780,8 +830,7 @@ mod integration_tests { env.upload_object("file2.txt", 1024 * 1024).await?; // Verify quota is full - let upload_result = env.upload_object("file3.txt", 1024).await; - assert!(upload_result.is_err()); + assert_put_rejected_by_quota(&env, "file3.txt", 1024).await; // Delete multiple objects using batch delete let objects = vec![ @@ -881,9 +930,7 @@ mod integration_tests { // Test 2: Multipart upload exceeds quota (should fail) // Upload 6MB filler (total now: 5MB + 6MB = 11MB > 10MB quota) - let upload_filler = env.upload_object("filler.txt", 6 * 1024 * 1024).await; - // This should fail due to quota - assert!(upload_filler.is_err()); + assert_put_rejected_by_quota(&env, "filler.txt", 6 * 1024 * 1024).await; // Verify filler doesn't exist assert!(!env.object_exists("filler.txt").await?); @@ -939,7 +986,11 @@ mod integration_tests { .await; let complete_error = complete_result.expect_err("multipart completion above quota must be rejected"); - assert_eq!(complete_error.as_service_error().and_then(|error| error.code()), Some("InvalidRequest")); + assert_quota_rejection( + complete_error.raw_response().map(|response| response.status().as_u16()), + complete_error.as_service_error(), + &complete_error, + ); assert!(!env.object_exists("over_quota.txt").await?); let staged_parts = env