From 4d50c72e4cc9cc2cecde5f3e053f62a26c484f82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=89=E6=AD=A3=E8=B6=85?= Date: Fri, 5 Jun 2026 21:50:02 +0800 Subject: [PATCH] fix(ecstore): skip hidden metadata in walk limit (#3224) * fix(ecstore): skip hidden metadata in walk limit * fix(ecstore): match walk limit to visible versions * fix(ecstore): avoid decoding all versions for limit --- crates/ecstore/src/disk/local.rs | 149 ++++++++++++++++++++++++++++--- 1 file changed, 137 insertions(+), 12 deletions(-) diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 92de5493c..1521122fa 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -1451,12 +1451,9 @@ impl LocalDisk { let name = entry.trim_end_matches(SLASH_SEPARATOR); let name = decode_dir_object(format!("{}/{}", ¤t, &name).as_str()); - // if opts.limit > 0 - // && let Ok(meta) = FileMeta::load(&metadata) - // && !meta.all_hidden(true) - // { - *objs_returned += 1; - // } + if opts.limit <= 0 || metadata_counts_toward_limit(&metadata) { + *objs_returned += 1; + } out.write_obj(&MetaCacheEntry { name: name.clone(), @@ -1559,15 +1556,19 @@ impl LocalDisk { out.write_obj(&meta).await?; - // if let Ok(meta) = FileMeta::load(&meta.metadata) - // && !meta.all_hidden(true) - // { - *objs_returned += 1; - // } + let file_meta = if opts.limit > 0 || opts.recursive { + FileMeta::load(&res).ok() + } else { + None + }; + + if opts.limit <= 0 || file_meta.as_ref().is_none_or(file_meta_counts_toward_limit) { + *objs_returned += 1; + } if opts.recursive { let mut dir_to_skip = HashSet::new(); - if let Ok(file_meta) = FileMeta::load(&res) + if let Some(file_meta) = file_meta.as_ref() && let Ok(data_dirs) = file_meta.get_data_dirs() { for data_dir in data_dirs.iter().flatten() { @@ -1636,6 +1637,15 @@ fn is_root_path(path: impl AsRef) -> bool { path.as_ref().components().count() == 1 && path.as_ref().has_root() } +fn metadata_counts_toward_limit(metadata: &[u8]) -> bool { + FileMeta::load(metadata).map_or(true, |meta| file_meta_counts_toward_limit(&meta)) +} + +fn file_meta_counts_toward_limit(meta: &FileMeta) -> bool { + meta.into_fileinfo("", "", "", false, true, false) + .map_or_else(|_| !meta.all_hidden(true), |latest| !latest.deleted && !latest.tier_free_version()) +} + // Filter std::io::ErrorKind::NotFound async fn read_file_exists(path: impl AsRef) -> Result<(Bytes, Option)> { let p = path.as_ref(); @@ -3641,6 +3651,121 @@ mod test { assert_eq!(double_count as usize, double_names.len()); } + #[tokio::test] + async fn test_scan_dir_hidden_delete_markers_do_not_exhaust_limit() { + use rustfs_filemeta::MetacacheReader; + use tempfile::tempdir; + + fn delete_marker_metadata(version_id: &str) -> Vec { + let mut fm = FileMeta::default(); + fm.add_version(FileInfo { + deleted: true, + version_id: Some(Uuid::parse_str(version_id).expect("test version id should parse")), + mod_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }) + .expect("delete marker metadata should be valid"); + fm.marshal_msg().expect("delete marker metadata should encode") + } + + fn delete_marker_with_old_object_metadata(delete_version_id: &str, object_version_id: &str) -> Vec { + let mut fm = FileMeta::default(); + fm.add_version({ + let mut fi = FileInfo::new("hidden", 1, 1); + fi.version_id = Some(Uuid::parse_str(object_version_id).expect("test version id should parse")); + fi.mod_time = Some(OffsetDateTime::now_utc() - time::Duration::seconds(1)); + fi + }) + .expect("object metadata should be valid"); + fm.add_version(FileInfo { + deleted: true, + version_id: Some(Uuid::parse_str(delete_version_id).expect("test version id should parse")), + mod_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }) + .expect("delete marker metadata should be valid"); + fm.marshal_msg().expect("delete marker metadata should encode") + } + + fn object_metadata(version_id: &str) -> Vec { + let mut fm = FileMeta::default(); + let mut fi = FileInfo::new("visible", 1, 1); + fi.version_id = Some(Uuid::parse_str(version_id).expect("test version id should parse")); + fi.mod_time = Some(OffsetDateTime::now_utc()); + fm.add_version(fi).expect("object metadata should be valid"); + fm.marshal_msg().expect("object metadata should encode") + } + + let dir = tempdir().unwrap(); + let bucket = "test-bucket"; + let bucket_dir = dir.path().join(bucket); + + for (name, version_id) in [ + ("shard/aaa-trash-0000", "11111111-1111-1111-1111-111111111111"), + ("shard/aaa-trash-0001", "22222222-2222-2222-2222-222222222222"), + ("shard/aaa-trash-0002", "33333333-3333-3333-3333-333333333333"), + ] { + let object_dir = bucket_dir.join(name); + fs::create_dir_all(&object_dir).await.unwrap(); + fs::write(object_dir.join(STORAGE_FORMAT_FILE), delete_marker_metadata(version_id)) + .await + .unwrap(); + } + + let hidden_versioned_dir = bucket_dir.join("shard/aaa-trash-0003"); + fs::create_dir_all(&hidden_versioned_dir).await.unwrap(); + fs::write( + hidden_versioned_dir.join(STORAGE_FORMAT_FILE), + delete_marker_with_old_object_metadata( + "44444444-4444-4444-4444-444444444444", + "55555555-5555-5555-5555-555555555555", + ), + ) + .await + .unwrap(); + + let visible_dir = bucket_dir.join("shard/bbb-visible-0000"); + fs::create_dir_all(&visible_dir).await.unwrap(); + fs::write( + visible_dir.join(STORAGE_FORMAT_FILE), + object_metadata("66666666-6666-6666-6666-666666666666"), + ) + .await + .unwrap(); + + let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap(); + let disk = LocalDisk::new(&endpoint, false).await.unwrap(); + + let (reader, mut writer) = tokio::io::duplex(4096); + let mut out = MetacacheWriter::new(&mut writer); + let opts = WalkDirOptions { + bucket: bucket.to_string(), + base_dir: "".to_string(), + recursive: true, + limit: 1, + ..Default::default() + }; + let mut objs_returned = 0; + + disk.scan_dir("".to_string(), "".to_string(), &opts, &mut out, &mut objs_returned, false, None) + .await + .unwrap(); + out.close().await.unwrap(); + drop(out); + drop(writer); + + let mut reader = MetacacheReader::new(reader); + let has_visible_object = reader + .read_all() + .await + .unwrap() + .into_iter() + .any(|entry| !entry.metadata.is_empty() && entry.name == "shard/bbb-visible-0000"); + + assert!(has_visible_object); + assert_eq!(objs_returned, 1); + } + #[tokio::test] async fn test_walk_dir_ignore_multipart_dirs() { use rustfs_filemeta::MetacacheReader;