diff --git a/crates/e2e_test/src/list_objects_duplicates_test.rs b/crates/e2e_test/src/list_objects_duplicates_test.rs index 34df57ebd..c863bfb3c 100644 --- a/crates/e2e_test/src/list_objects_duplicates_test.rs +++ b/crates/e2e_test/src/list_objects_duplicates_test.rs @@ -132,4 +132,70 @@ mod tests { // Stop the RustFS server to ensure proper cleanup env.stop_server(); } + + /// Test ensuring that ListObjectsV2 returns unique keys when an explicit directory marker + /// exists under the requested prefix and delimiter is not provided. + /// + /// Bug Reference: Issue #2439 + /// When both "marker/subdir/" and "marker/subdir/file.txt" exist, listing with + /// Prefix="marker/" must not duplicate "marker/subdir/file.txt" in Contents. + #[tokio::test] + #[serial] + async fn test_list_objects_v2_unique_contents_with_explicit_directory_markers() { + init_logging(); + info!("Starting test: ListObjectsV2 should return unique keys with explicit directory markers"); + + let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); + env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); + + let client = create_s3_client(&env); + let bucket = "test-list-unique-contents"; + + create_bucket(&client, bucket).await.expect("Failed to create bucket"); + + for (key, body) in [ + ("marker/", ByteStream::from_static(b"")), + ("marker/subdir/", ByteStream::from_static(b"")), + ("marker/file.txt", ByteStream::from_static(b"content")), + ("marker/subdir/file.txt", ByteStream::from_static(b"nested")), + ] { + client + .put_object() + .bucket(bucket) + .key(key) + .body(body) + .send() + .await + .unwrap_or_else(|err| panic!("Failed to create test object {key}: {err}")); + } + + let result = client + .list_objects_v2() + .bucket(bucket) + .prefix("marker/") + .send() + .await + .expect("Failed to list objects"); + + let keys: Vec = result + .contents() + .iter() + .filter_map(|object| object.key().map(ToOwned::to_owned)) + .collect(); + + info!("Contents: {:?}", keys); + + assert_eq!( + keys, + vec![ + "marker/".to_string(), + "marker/file.txt".to_string(), + "marker/subdir/".to_string(), + "marker/subdir/file.txt".to_string(), + ] + ); + assert_eq!(result.key_count(), Some(4)); + + env.stop_server(); + } } diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index d023a121c..66c1786d5 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -1477,6 +1477,16 @@ impl LocalDisk { } let mut dir_stack: Vec<(String, bool)> = Vec::with_capacity(5); + // Explicit directory markers and real directories can resolve to the same logical path. + let schedule_dir = |dir_stack: &mut Vec<(String, bool)>, dir_name: String, skip_object: bool| { + if let Some((last_dir_name, existing_skip_object)) = dir_stack.last_mut() + && *last_dir_name == dir_name + { + *existing_skip_object |= skip_object; + } else { + dir_stack.push((dir_name, skip_object)); + } + }; prefix = "".to_owned(); for entry in entries.iter() { @@ -1545,7 +1555,7 @@ impl LocalDisk { if !dir_name.ends_with(SLASH_SEPARATOR) { dir_name.push_str(SLASH_SEPARATOR); } - dir_stack.push((dir_name, true)); + schedule_dir(&mut dir_stack, dir_name, true); } } Err(err) => { @@ -1554,7 +1564,7 @@ impl LocalDisk { // If dirObject, but no metadata (which is unexpected) we skip it. if !is_dir_obj && !is_empty_dir(self.get_object_path(&opts.bucket, &meta.name)?).await { meta.name.push_str(SLASH_SEPARATOR); - dir_stack.push((meta.name, false)); + schedule_dir(&mut dir_stack, meta.name, false); } } @@ -3169,6 +3179,60 @@ mod test { assert!(names.contains(&"quux/thud".to_string())); } + #[tokio::test] + async fn test_scan_dir_deduplicates_explicit_dir_marker_recursion() { + use rustfs_filemeta::MetacacheReader; + use tempfile::tempdir; + + let dir = tempdir().unwrap(); + let bucket = "test-bucket"; + let bucket_dir = dir.path().join(bucket); + + fs::create_dir_all(bucket_dir.join("marker/file.txt")).await.unwrap(); + fs::create_dir_all(bucket_dir.join("marker/subdir/file.txt")).await.unwrap(); + fs::create_dir_all(bucket_dir.join(format!("marker/subdir{GLOBAL_DIR_SUFFIX}"))) + .await + .unwrap(); + + fs::write(bucket_dir.join("marker/file.txt/xl.meta"), b"meta").await.unwrap(); + fs::write(bucket_dir.join("marker/subdir/file.txt/xl.meta"), b"meta") + .await + .unwrap(); + fs::write(bucket_dir.join(format!("marker/subdir{GLOBAL_DIR_SUFFIX}/xl.meta")), b"meta") + .await + .unwrap(); + + let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap(); + let disk = LocalDisk::new(&endpoint, false).await.unwrap(); + + let (reader, mut writer) = tokio::io::duplex(4096); + let mut out = MetacacheWriter::new(&mut writer); + let opts = WalkDirOptions { + bucket: bucket.to_string(), + base_dir: "marker/".to_string(), + recursive: true, + ..Default::default() + }; + let mut objs_returned = 0; + + disk.scan_dir("marker/".to_string(), "".to_string(), &opts, &mut out, &mut objs_returned, false) + .await + .unwrap(); + out.close().await.unwrap(); + + let mut reader = MetacacheReader::new(reader); + let entries = reader.read_all().await.unwrap(); + let names: Vec = entries + .into_iter() + .filter(|entry| !entry.metadata.is_empty()) + .map(|entry| entry.name) + .collect(); + + assert_eq!(names.iter().filter(|name| *name == "marker/subdir/file.txt").count(), 1); + assert_eq!(names.iter().filter(|name| *name == "marker/subdir/").count(), 1); + assert_eq!(names.iter().filter(|name| *name == "marker/file.txt").count(), 1); + } + #[tokio::test] async fn test_make_volume() { let p = "./testv0";