fix(ecstore): avoid duplicate keys in ListObjectsV2 (#2467)

crates/e2e_test/src/list_objects_duplicates_test.rs

@@ -132,4 +132,70 @@ mod tests {
         // Stop the RustFS server to ensure proper cleanup
         env.stop_server();
     }
+
+    /// Test ensuring that ListObjectsV2 returns unique keys when an explicit directory marker
+    /// exists under the requested prefix and delimiter is not provided.
+    ///
+    /// Bug Reference: Issue #2439
+    /// When both "marker/subdir/" and "marker/subdir/file.txt" exist, listing with
+    /// Prefix="marker/" must not duplicate "marker/subdir/file.txt" in Contents.
+    #[tokio::test]
+    #[serial]
+    async fn test_list_objects_v2_unique_contents_with_explicit_directory_markers() {
+        init_logging();
+        info!("Starting test: ListObjectsV2 should return unique keys with explicit directory markers");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-list-unique-contents";
+
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+
+        for (key, body) in [
+            ("marker/", ByteStream::from_static(b"")),
+            ("marker/subdir/", ByteStream::from_static(b"")),
+            ("marker/file.txt", ByteStream::from_static(b"content")),
+            ("marker/subdir/file.txt", ByteStream::from_static(b"nested")),
+        ] {
+            client
+                .put_object()
+                .bucket(bucket)
+                .key(key)
+                .body(body)
+                .send()
+                .await
+                .unwrap_or_else(|err| panic!("Failed to create test object {key}: {err}"));
+        }
+
+        let result = client
+            .list_objects_v2()
+            .bucket(bucket)
+            .prefix("marker/")
+            .send()
+            .await
+            .expect("Failed to list objects");
+
+        let keys: Vec<String> = result
+            .contents()
+            .iter()
+            .filter_map(|object| object.key().map(ToOwned::to_owned))
+            .collect();
+
+        info!("Contents: {:?}", keys);
+
+        assert_eq!(
+            keys,
+            vec![
+                "marker/".to_string(),
+                "marker/file.txt".to_string(),
+                "marker/subdir/".to_string(),
+                "marker/subdir/file.txt".to_string(),
+            ]
+        );
+        assert_eq!(result.key_count(), Some(4));
+
+        env.stop_server();
+    }
 }

crates/ecstore/src/disk/local.rs

@@ -1477,6 +1477,16 @@ impl LocalDisk {
         }
 
         let mut dir_stack: Vec<(String, bool)> = Vec::with_capacity(5);
+        // Explicit directory markers and real directories can resolve to the same logical path.
+        let schedule_dir = |dir_stack: &mut Vec<(String, bool)>, dir_name: String, skip_object: bool| {
+            if let Some((last_dir_name, existing_skip_object)) = dir_stack.last_mut()
+                && *last_dir_name == dir_name
+            {
+                *existing_skip_object |= skip_object;
+            } else {
+                dir_stack.push((dir_name, skip_object));
+            }
+        };
         prefix = "".to_owned();
 
         for entry in entries.iter() {
@@ -1545,7 +1555,7 @@ impl LocalDisk {
                         if !dir_name.ends_with(SLASH_SEPARATOR) {
                             dir_name.push_str(SLASH_SEPARATOR);
                         }
-                        dir_stack.push((dir_name, true));
+                        schedule_dir(&mut dir_stack, dir_name, true);
                     }
                 }
                 Err(err) => {
@@ -1554,7 +1564,7 @@ impl LocalDisk {
                         // If dirObject, but no metadata (which is unexpected) we skip it.
                         if !is_dir_obj && !is_empty_dir(self.get_object_path(&opts.bucket, &meta.name)?).await {
                             meta.name.push_str(SLASH_SEPARATOR);
-                            dir_stack.push((meta.name, false));
+                            schedule_dir(&mut dir_stack, meta.name, false);
                         }
                     }
 
@@ -3169,6 +3179,60 @@ mod test {
         assert!(names.contains(&"quux/thud".to_string()));
     }
 
+    #[tokio::test]
+    async fn test_scan_dir_deduplicates_explicit_dir_marker_recursion() {
+        use rustfs_filemeta::MetacacheReader;
+        use tempfile::tempdir;
+
+        let dir = tempdir().unwrap();
+        let bucket = "test-bucket";
+        let bucket_dir = dir.path().join(bucket);
+
+        fs::create_dir_all(bucket_dir.join("marker/file.txt")).await.unwrap();
+        fs::create_dir_all(bucket_dir.join("marker/subdir/file.txt")).await.unwrap();
+        fs::create_dir_all(bucket_dir.join(format!("marker/subdir{GLOBAL_DIR_SUFFIX}")))
+            .await
+            .unwrap();
+
+        fs::write(bucket_dir.join("marker/file.txt/xl.meta"), b"meta").await.unwrap();
+        fs::write(bucket_dir.join("marker/subdir/file.txt/xl.meta"), b"meta")
+            .await
+            .unwrap();
+        fs::write(bucket_dir.join(format!("marker/subdir{GLOBAL_DIR_SUFFIX}/xl.meta")), b"meta")
+            .await
+            .unwrap();
+
+        let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap();
+        let disk = LocalDisk::new(&endpoint, false).await.unwrap();
+
+        let (reader, mut writer) = tokio::io::duplex(4096);
+        let mut out = MetacacheWriter::new(&mut writer);
+        let opts = WalkDirOptions {
+            bucket: bucket.to_string(),
+            base_dir: "marker/".to_string(),
+            recursive: true,
+            ..Default::default()
+        };
+        let mut objs_returned = 0;
+
+        disk.scan_dir("marker/".to_string(), "".to_string(), &opts, &mut out, &mut objs_returned, false)
+            .await
+            .unwrap();
+        out.close().await.unwrap();
+
+        let mut reader = MetacacheReader::new(reader);
+        let entries = reader.read_all().await.unwrap();
+        let names: Vec<String> = entries
+            .into_iter()
+            .filter(|entry| !entry.metadata.is_empty())
+            .map(|entry| entry.name)
+            .collect();
+
+        assert_eq!(names.iter().filter(|name| *name == "marker/subdir/file.txt").count(), 1);
+        assert_eq!(names.iter().filter(|name| *name == "marker/subdir/").count(), 1);
+        assert_eq!(names.iter().filter(|name| *name == "marker/file.txt").count(), 1);
+    }
+
     #[tokio::test]
     async fn test_make_volume() {
         let p = "./testv0";