fix(ecstore): harden object-prefix probe and parse markers before forward_past (#4600)

crates/e2e_test/src/list_objects_duplicates_test.rs

@@ -133,75 +133,6 @@ mod tests {
         env.stop_server();
     }
 
-    /// Test that a plain object and a same-named prefix coexist in a delimiter listing.
-    ///
-    /// Bug Reference: backlog#1042 (follow-up to backlog#880).
-    /// A plain object `a` (no trailing slash) and a sibling object `a/b` under the
-    /// same-named prefix must BOTH surface when listing with delimiter "/": `a` as a
-    /// Content and `a/` as a CommonPrefix (S3 delimiter semantics). On single-disk /
-    /// consistent deployments the source scan classifies `a` as an object and, before
-    /// the fix, never emitted the prefix `a/`, silently dropping the CommonPrefix.
-    #[tokio::test]
-    #[serial]
-    async fn test_list_objects_v2_object_and_same_named_prefix_coexist() {
-        init_logging();
-        info!("Starting test: object `a` and prefix `a/` must coexist under delimiter");
-
-        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
-        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
-
-        let client = create_s3_client(&env);
-        let bucket = "test-list-object-prefix-coexist";
-        create_bucket(&client, bucket).await.expect("Failed to create bucket");
-
-        // Plain object `a` (no trailing slash) — must land in Contents.
-        client
-            .put_object()
-            .bucket(bucket)
-            .key("a")
-            .body(ByteStream::from_static(b"top"))
-            .send()
-            .await
-            .expect("Failed to create object `a`");
-        // Sibling object under the same-named prefix — makes `a/` a CommonPrefix.
-        client
-            .put_object()
-            .bucket(bucket)
-            .key("a/b")
-            .body(ByteStream::from_static(b"child"))
-            .send()
-            .await
-            .expect("Failed to create object `a/b`");
-
-        let result = client
-            .list_objects_v2()
-            .bucket(bucket)
-            .delimiter("/")
-            .send()
-            .await
-            .expect("Failed to list objects");
-
-        let keys: Vec<String> = result
-            .contents()
-            .iter()
-            .filter_map(|o| o.key().map(ToOwned::to_owned))
-            .collect();
-        let prefixes: Vec<String> = result
-            .common_prefixes()
-            .iter()
-            .filter_map(|p| p.prefix().map(ToOwned::to_owned))
-            .collect();
-        info!("Contents: {:?}, CommonPrefixes: {:?}", keys, prefixes);
-
-        assert!(keys.iter().any(|k| k == "a"), "object `a` must appear in Contents, got {keys:?}");
-        assert!(
-            prefixes.iter().any(|p| p == "a/"),
-            "prefix `a/` must appear in CommonPrefixes, got {prefixes:?}"
-        );
-
-        env.stop_server();
-    }
-
     /// Test ensuring that ListObjectsV2 returns unique keys when an explicit directory marker
     /// exists under the requested prefix and delimiter is not provided.
     ///
@@ -267,4 +198,121 @@ mod tests {
 
         env.stop_server();
     }
+
+    /// Test ensuring that a plain object and a same-named prefix coexist in
+    /// delimiter listings on a single-disk deployment.
+    ///
+    /// Bug Reference: backlog#880 / backlog#1042
+    /// On a single disk, object `a` and its children `a/...` share one backing
+    /// directory, so the non-recursive scan used to classify `a` as an object
+    /// and never produce the prefix entry `a/`. Delimiter="/" listings then
+    /// returned Contents `a` but silently dropped CommonPrefix `a/`.
+    #[tokio::test]
+    #[serial]
+    async fn test_list_objects_v2_object_and_same_named_prefix_coexist() {
+        init_logging();
+        info!("Starting test: ListObjectsV2 should return both object `a` and CommonPrefix `a/`");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
+            .await
+            .expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-list-object-prefix-coexist";
+
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+
+        for (key, body) in [
+            ("a", ByteStream::from_static(b"object body")),
+            ("a/b", ByteStream::from_static(b"child body")),
+            ("plain", ByteStream::from_static(b"no children")),
+        ] {
+            client
+                .put_object()
+                .bucket(bucket)
+                .key(key)
+                .body(body)
+                .send()
+                .await
+                .unwrap_or_else(|err| panic!("Failed to create test object {key}: {err}"));
+        }
+
+        let result = client
+            .list_objects_v2()
+            .bucket(bucket)
+            .delimiter("/")
+            .send()
+            .await
+            .expect("Failed to list objects");
+
+        let keys: Vec<&str> = result.contents().iter().filter_map(|object| object.key()).collect();
+        let prefixes: Vec<&str> = result.common_prefixes().iter().filter_map(|prefix| prefix.prefix()).collect();
+
+        info!("Contents: {:?}, CommonPrefixes: {:?}", keys, prefixes);
+
+        assert_eq!(keys, vec!["a", "plain"], "objects `a` and `plain` must both stay in Contents");
+        assert_eq!(
+            prefixes,
+            vec!["a/"],
+            "prefix `a/` must be listed and `plain/` must not appear as a phantom prefix"
+        );
+
+        // Children are still reachable under the prefix.
+        let nested = client
+            .list_objects_v2()
+            .bucket(bucket)
+            .prefix("a/")
+            .delimiter("/")
+            .send()
+            .await
+            .expect("Failed to list objects under prefix");
+        let nested_keys: Vec<&str> = nested.contents().iter().filter_map(|object| object.key()).collect();
+        assert_eq!(nested_keys, vec!["a/b"]);
+
+        // Pagination must keep both entries across page boundaries: `a` sorts
+        // before `a/`, so a one-key page splits them.
+        let page1 = client
+            .list_objects_v2()
+            .bucket(bucket)
+            .delimiter("/")
+            .max_keys(1)
+            .send()
+            .await
+            .expect("Failed to list first page");
+        let page1_keys: Vec<&str> = page1.contents().iter().filter_map(|object| object.key()).collect();
+        assert_eq!(page1_keys, vec!["a"]);
+        assert_eq!(page1.is_truncated(), Some(true), "one-key first page must be truncated");
+
+        let mut token = page1.next_continuation_token().map(ToOwned::to_owned);
+        let mut remaining_keys = Vec::new();
+        let mut remaining_prefixes = Vec::new();
+        while let Some(continuation) = token {
+            let page = client
+                .list_objects_v2()
+                .bucket(bucket)
+                .delimiter("/")
+                .max_keys(1)
+                .continuation_token(continuation)
+                .send()
+                .await
+                .expect("Failed to list continuation page");
+            remaining_keys.extend(
+                page.contents()
+                    .iter()
+                    .filter_map(|object| object.key().map(ToOwned::to_owned)),
+            );
+            remaining_prefixes.extend(
+                page.common_prefixes()
+                    .iter()
+                    .filter_map(|prefix| prefix.prefix().map(ToOwned::to_owned)),
+            );
+            token = page.next_continuation_token().map(ToOwned::to_owned);
+        }
+
+        assert_eq!(remaining_prefixes, vec!["a/".to_string()], "prefix `a/` must survive pagination");
+        assert_eq!(remaining_keys, vec!["plain".to_string()]);
+
+        env.stop_server();
+    }
 }

crates/e2e_test/src/list_objects_v2_pagination_test.rs

@@ -1029,4 +1029,67 @@ mod tests {
 
         env.stop_server();
     }
+
+    /// Test that continuation pages do not skip keys sorting between the marker
+    /// and the marker plus the cursor tag.
+    ///
+    /// Bug Reference: backlog#1047
+    /// The V2 continuation token appends a `[rustfs_cache:...]` cursor tag to the
+    /// last returned key. The orchestration layer passed that raw string to
+    /// `forward_past`, so any key whose byte after the shared stem sorts below
+    /// `[` (0x5B) — `.`, `/`, `-`, digits, uppercase — was silently dropped on
+    /// the next page: with keys `a`, `a.txt`, `zz` and max_keys=1, page 2
+    /// returned `zz` and `a.txt` was never listed.
+    #[tokio::test]
+    #[serial]
+    async fn test_list_objects_v2_continuation_keeps_keys_after_marker_stem() {
+        init_logging();
+        info!("Starting test: continuation must not skip keys sorting below the cursor tag");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
+            .await
+            .expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-continuation-marker-stem";
+
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+
+        let expected_keys = ["a", "a.txt", "zz"];
+        for key in expected_keys {
+            client
+                .put_object()
+                .bucket(bucket)
+                .key(key)
+                .body(ByteStream::from_static(b"content"))
+                .send()
+                .await
+                .unwrap_or_else(|err| panic!("Failed to create test object {key}: {err}"));
+        }
+
+        let mut collected: Vec<String> = Vec::new();
+        let mut token: Option<String> = None;
+        loop {
+            let mut request = client.list_objects_v2().bucket(bucket).max_keys(1);
+            if let Some(continuation) = token.take() {
+                request = request.continuation_token(continuation);
+            }
+            let page = request.send().await.expect("Failed to list objects");
+            collected.extend(
+                page.contents()
+                    .iter()
+                    .filter_map(|object| object.key().map(ToOwned::to_owned)),
+            );
+            token = page.next_continuation_token().map(ToOwned::to_owned);
+            if token.is_none() {
+                break;
+            }
+            assert!(collected.len() <= expected_keys.len() + 1, "pagination did not converge: {collected:?}");
+        }
+
+        assert_eq!(collected, expected_keys, "every key must survive pagination in order");
+
+        env.stop_server();
+    }
 }