diff --git a/crates/e2e_test/src/list_objects_v2_pagination_test.rs b/crates/e2e_test/src/list_objects_v2_pagination_test.rs index 9f06d7ba5..339e92cf1 100644 --- a/crates/e2e_test/src/list_objects_v2_pagination_test.rs +++ b/crates/e2e_test/src/list_objects_v2_pagination_test.rs @@ -368,6 +368,71 @@ mod tests { env.stop_server(); } + /// Test ListObjectsV2 caps max_keys above the service limit and still paginates. + #[tokio::test] + #[serial] + async fn test_list_objects_v2_max_keys_above_limit_returns_token() { + init_logging(); + info!("Starting test: ListObjectsV2 with max_keys above limit"); + + let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); + env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); + + let client = create_s3_client(&env); + let bucket = "test-max-keys-above-limit"; + + create_bucket(&client, bucket).await.expect("Failed to create bucket"); + + let object_count = 1002; + for i in 0..object_count { + let key = format!("object{:04}.txt", i); + client + .put_object() + .bucket(bucket) + .key(&key) + .body(ByteStream::from_static(b"test content")) + .send() + .await + .expect("Failed to put object"); + } + + let output = client + .list_objects_v2() + .bucket(bucket) + .max_keys(1001) + .send() + .await + .expect("Failed to list objects"); + + assert_eq!(output.contents().len(), 1000); + assert_eq!(output.key_count(), Some(1000)); + assert_eq!(output.max_keys(), Some(1000)); + assert!( + output.is_truncated().unwrap_or(false), + "IsTruncated should be true when more objects remain after capped max_keys" + ); + + let next_token = output + .next_continuation_token() + .expect("NextContinuationToken should be present when capped response is truncated") + .to_string(); + + let output = client + .list_objects_v2() + .bucket(bucket) + .max_keys(1001) + .continuation_token(next_token) + .send() + .await + .expect("Failed to list objects with continuation token"); + + assert_eq!(output.contents().len(), 2); + assert!(!output.is_truncated().unwrap_or(false)); + assert!(output.next_continuation_token().is_none()); + + env.stop_server(); + } + /// Test ListObjectsV2 with max_keys=0 /// /// S3 semantics: when max_keys is 0, the response should include no objects diff --git a/crates/ecstore/src/store_list_objects.rs b/crates/ecstore/src/store_list_objects.rs index eda4435ab..e890e00ca 100644 --- a/crates/ecstore/src/store_list_objects.rs +++ b/crates/ecstore/src/store_list_objects.rs @@ -50,6 +50,10 @@ const MAX_OBJECT_LIST: i32 = 1000; const METACACHE_SHARE_PREFIX: bool = false; +fn normalize_max_keys(max_keys: i32) -> i32 { + max_keys.min(MAX_OBJECT_LIST) +} + fn ensure_non_empty_listing_disks(bucket: &str, path: &str, disks: &[DiskStore]) -> Result<()> { if disks.is_empty() { warn!( @@ -267,6 +271,7 @@ impl ECStore { max_keys: i32, incl_deleted: bool, ) -> Result { + let max_keys = normalize_max_keys(max_keys); let effective_max_keys = if max_keys <= 0 { 0 } else { max_keys_plus_one(max_keys, true) }; let opts = ListPathOptions { bucket: bucket.to_owned(), @@ -390,6 +395,7 @@ impl ECStore { delimiter: Option, max_keys: i32, ) -> Result { + let max_keys = normalize_max_keys(max_keys); if marker.is_none() && version_marker.is_some() { return Err(StorageError::NotImplemented); } @@ -1410,9 +1416,18 @@ fn calc_common_counter(infos: &[DiskInfo], read_quorum: usize) -> u64 { #[cfg(test)] mod test { - use super::ListPathOptions; + use super::{ListPathOptions, MAX_OBJECT_LIST, max_keys_plus_one}; use uuid::Uuid; + #[test] + fn test_max_keys_plus_one_caps_before_lookahead() { + assert_eq!(max_keys_plus_one(999, true), 1000); + assert_eq!(max_keys_plus_one(MAX_OBJECT_LIST, true), MAX_OBJECT_LIST + 1); + assert_eq!(max_keys_plus_one(MAX_OBJECT_LIST + 1, true), MAX_OBJECT_LIST + 1); + assert_eq!(max_keys_plus_one(i32::MAX, true), MAX_OBJECT_LIST + 1); + assert_eq!(max_keys_plus_one(-1, true), MAX_OBJECT_LIST + 1); + } + /// Test that "null" version marker is handled correctly /// AWS S3 API uses "null" string to represent non-versioned objects #[test] diff --git a/rustfs/src/storage/s3_api/bucket.rs b/rustfs/src/storage/s3_api/bucket.rs index 761f96190..c40ca405f 100644 --- a/rustfs/src/storage/s3_api/bucket.rs +++ b/rustfs/src/storage/s3_api/bucket.rs @@ -23,6 +23,12 @@ use s3s::{S3Error, S3ErrorCode}; use tracing::debug; use urlencoding::encode; +const S3_MAX_KEYS: i32 = 1000; + +fn normalize_max_keys(max_keys: i32) -> i32 { + max_keys.min(S3_MAX_KEYS) +} + #[derive(Debug, PartialEq, Eq)] pub(crate) struct ListObjectVersionsParams { pub prefix: String, @@ -92,10 +98,11 @@ pub(crate) fn parse_list_object_versions_params( let delimiter = delimiter.filter(|v| !v.is_empty()); let key_marker = key_marker.filter(|v| !v.is_empty()); let version_id_marker = version_id_marker.filter(|v| !v.is_empty()); - let max_keys = max_keys.unwrap_or(1000); + let max_keys = max_keys.unwrap_or(S3_MAX_KEYS); if max_keys < 0 { return Err(S3Error::with_message(S3ErrorCode::InvalidArgument, "Invalid max keys".to_string())); } + let max_keys = normalize_max_keys(max_keys); Ok(ListObjectVersionsParams { prefix, @@ -120,10 +127,11 @@ pub(crate) fn parse_list_objects_v2_params( debug!("LIST objects with special characters in prefix: {:?}", prefix); } - let max_keys = max_keys.unwrap_or(1000); + let max_keys = max_keys.unwrap_or(S3_MAX_KEYS); if max_keys < 0 { return Err(S3Error::with_message(S3ErrorCode::InvalidArgument, "Invalid max keys".to_string())); } + let max_keys = normalize_max_keys(max_keys); let delimiter = delimiter.filter(|v| !v.is_empty()); @@ -553,6 +561,20 @@ mod tests { assert_eq!(parsed.decoded_continuation_token, None); } + #[test] + fn test_parse_list_objects_v2_params_caps_large_max_keys() { + let parsed = parse_list_objects_v2_params(None, None, Some(1001), None, None).expect("parse should succeed"); + + assert_eq!(parsed.max_keys, 1000); + } + + #[test] + fn test_parse_list_object_versions_params_caps_large_max_keys() { + let parsed = parse_list_object_versions_params(None, None, None, None, Some(1001)).expect("parse should succeed"); + + assert_eq!(parsed.max_keys, 1000); + } + #[test] fn test_parse_list_objects_v2_params_rejects_negative_max_keys() { let err =