fix: preserve pagination when max keys exceed limit (#2943)

Co-authored-by: houseme <housemecn@gmail.com>

crates/e2e_test/src/list_objects_v2_pagination_test.rs

@@ -368,6 +368,71 @@ mod tests {
         env.stop_server();
     }
 
+    /// Test ListObjectsV2 caps max_keys above the service limit and still paginates.
+    #[tokio::test]
+    #[serial]
+    async fn test_list_objects_v2_max_keys_above_limit_returns_token() {
+        init_logging();
+        info!("Starting test: ListObjectsV2 with max_keys above limit");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-max-keys-above-limit";
+
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+
+        let object_count = 1002;
+        for i in 0..object_count {
+            let key = format!("object{:04}.txt", i);
+            client
+                .put_object()
+                .bucket(bucket)
+                .key(&key)
+                .body(ByteStream::from_static(b"test content"))
+                .send()
+                .await
+                .expect("Failed to put object");
+        }
+
+        let output = client
+            .list_objects_v2()
+            .bucket(bucket)
+            .max_keys(1001)
+            .send()
+            .await
+            .expect("Failed to list objects");
+
+        assert_eq!(output.contents().len(), 1000);
+        assert_eq!(output.key_count(), Some(1000));
+        assert_eq!(output.max_keys(), Some(1000));
+        assert!(
+            output.is_truncated().unwrap_or(false),
+            "IsTruncated should be true when more objects remain after capped max_keys"
+        );
+
+        let next_token = output
+            .next_continuation_token()
+            .expect("NextContinuationToken should be present when capped response is truncated")
+            .to_string();
+
+        let output = client
+            .list_objects_v2()
+            .bucket(bucket)
+            .max_keys(1001)
+            .continuation_token(next_token)
+            .send()
+            .await
+            .expect("Failed to list objects with continuation token");
+
+        assert_eq!(output.contents().len(), 2);
+        assert!(!output.is_truncated().unwrap_or(false));
+        assert!(output.next_continuation_token().is_none());
+
+        env.stop_server();
+    }
+
     /// Test ListObjectsV2 with max_keys=0
     ///
     /// S3 semantics: when max_keys is 0, the response should include no objects

crates/ecstore/src/store_list_objects.rs

@@ -50,6 +50,10 @@ const MAX_OBJECT_LIST: i32 = 1000;
 
 const METACACHE_SHARE_PREFIX: bool = false;
 
+fn normalize_max_keys(max_keys: i32) -> i32 {
+    max_keys.min(MAX_OBJECT_LIST)
+}
+
 fn ensure_non_empty_listing_disks(bucket: &str, path: &str, disks: &[DiskStore]) -> Result<()> {
     if disks.is_empty() {
         warn!(
@@ -267,6 +271,7 @@ impl ECStore {
         max_keys: i32,
         incl_deleted: bool,
     ) -> Result<ListObjectsInfo> {
+        let max_keys = normalize_max_keys(max_keys);
         let effective_max_keys = if max_keys <= 0 { 0 } else { max_keys_plus_one(max_keys, true) };
         let opts = ListPathOptions {
             bucket: bucket.to_owned(),
@@ -390,6 +395,7 @@ impl ECStore {
         delimiter: Option<String>,
         max_keys: i32,
     ) -> Result<ListObjectVersionsInfo> {
+        let max_keys = normalize_max_keys(max_keys);
         if marker.is_none() && version_marker.is_some() {
             return Err(StorageError::NotImplemented);
         }
@@ -1410,9 +1416,18 @@ fn calc_common_counter(infos: &[DiskInfo], read_quorum: usize) -> u64 {
 
 #[cfg(test)]
 mod test {
-    use super::ListPathOptions;
+    use super::{ListPathOptions, MAX_OBJECT_LIST, max_keys_plus_one};
     use uuid::Uuid;
 
+    #[test]
+    fn test_max_keys_plus_one_caps_before_lookahead() {
+        assert_eq!(max_keys_plus_one(999, true), 1000);
+        assert_eq!(max_keys_plus_one(MAX_OBJECT_LIST, true), MAX_OBJECT_LIST + 1);
+        assert_eq!(max_keys_plus_one(MAX_OBJECT_LIST + 1, true), MAX_OBJECT_LIST + 1);
+        assert_eq!(max_keys_plus_one(i32::MAX, true), MAX_OBJECT_LIST + 1);
+        assert_eq!(max_keys_plus_one(-1, true), MAX_OBJECT_LIST + 1);
+    }
+
     /// Test that "null" version marker is handled correctly
     /// AWS S3 API uses "null" string to represent non-versioned objects
     #[test]

rustfs/src/storage/s3_api/bucket.rs

@@ -23,6 +23,12 @@ use s3s::{S3Error, S3ErrorCode};
 use tracing::debug;
 use urlencoding::encode;
 
+const S3_MAX_KEYS: i32 = 1000;
+
+fn normalize_max_keys(max_keys: i32) -> i32 {
+    max_keys.min(S3_MAX_KEYS)
+}
+
 #[derive(Debug, PartialEq, Eq)]
 pub(crate) struct ListObjectVersionsParams {
     pub prefix: String,
@@ -92,10 +98,11 @@ pub(crate) fn parse_list_object_versions_params(
     let delimiter = delimiter.filter(|v| !v.is_empty());
     let key_marker = key_marker.filter(|v| !v.is_empty());
     let version_id_marker = version_id_marker.filter(|v| !v.is_empty());
-    let max_keys = max_keys.unwrap_or(1000);
+    let max_keys = max_keys.unwrap_or(S3_MAX_KEYS);
     if max_keys < 0 {
         return Err(S3Error::with_message(S3ErrorCode::InvalidArgument, "Invalid max keys".to_string()));
     }
+    let max_keys = normalize_max_keys(max_keys);
 
     Ok(ListObjectVersionsParams {
         prefix,
@@ -120,10 +127,11 @@ pub(crate) fn parse_list_objects_v2_params(
         debug!("LIST objects with special characters in prefix: {:?}", prefix);
     }
 
-    let max_keys = max_keys.unwrap_or(1000);
+    let max_keys = max_keys.unwrap_or(S3_MAX_KEYS);
     if max_keys < 0 {
         return Err(S3Error::with_message(S3ErrorCode::InvalidArgument, "Invalid max keys".to_string()));
     }
+    let max_keys = normalize_max_keys(max_keys);
 
     let delimiter = delimiter.filter(|v| !v.is_empty());
 
@@ -553,6 +561,20 @@ mod tests {
         assert_eq!(parsed.decoded_continuation_token, None);
     }
 
+    #[test]
+    fn test_parse_list_objects_v2_params_caps_large_max_keys() {
+        let parsed = parse_list_objects_v2_params(None, None, Some(1001), None, None).expect("parse should succeed");
+
+        assert_eq!(parsed.max_keys, 1000);
+    }
+
+    #[test]
+    fn test_parse_list_object_versions_params_caps_large_max_keys() {
+        let parsed = parse_list_object_versions_params(None, None, None, None, Some(1001)).expect("parse should succeed");
+
+        assert_eq!(parsed.max_keys, 1000);
+    }
+
     #[test]
     fn test_parse_list_objects_v2_params_rejects_negative_max_keys() {
         let err =