gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-06 13:27:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ecstore): replace unbounded metadata cache with moka (#743) (#3970)

Browse Source 
fix(ecstore): replace unbounded metadata cache with moka

Replace the manual Arc<RwLock<HashMap>> metadata cache with
moka::future::Cache, which provides:
- Built-in LRU eviction when max_capacity is reached
- Automatic TTL expiry via time_to_live (250ms)
- Lock-free concurrent reads
- Non-blocking invalidation

Fixes the memory leak risk from unbounded HashMap and the
all-or-nothing eviction logic that cleared all entries at once.

Closes #743

Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
Zhengchao An
2026-06-28 03:01:32 +08:00
committed by GitHub
parent 175566f037
commit 512418cda9
4 changed files with 53 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+1
View File
@@ -9287,6 +9287,7 @@ dependencies = [
"md-5 0.11.0",
"memmap2 0.9.11",
"metrics",
"moka",
"num_cpus",
"opentelemetry",
"opentelemetry_sdk",
crates/ecstore/Cargo.toml
+1
View File
@@ -79,6 +79,7 @@ uuid = { workspace = true, features = ["v4", "fast-rng", "serde"] }
reed-solomon-erasure = { workspace = true }
reed-solomon-simd = { workspace = true }
lazy_static.workspace = true
moka = { workspace = true }
rustfs-lock.workspace = true
rustfs-io-metrics.workspace = true
regex = { workspace = true }
crates/ecstore/src/set_disk/mod.rs
+9 -12
View File
@@ -887,7 +887,7 @@ pub struct SetDisks {
pub pool_index: usize,
pub format: FormatV3,
disk_health_cache: Arc<RwLock<Vec<Option<DiskHealthEntry>>>>,
get_object_metadata_cache: Arc<RwLock<HashMap<GetObjectMetadataCacheKey, GetObjectMetadataCacheEntry>>>,
get_object_metadata_cache: moka::future::Cache<GetObjectMetadataCacheKey, GetObjectMetadataCacheEntry>,
pub lockers: Vec<Arc<dyn LockClient>>,
local_lock_manager: Arc<rustfs_lock::GlobalLockManager>,
}
@@ -909,6 +909,7 @@ impl GetObjectMetadataCacheKey {
#[derive(Clone, Debug)]
struct GetObjectMetadataCacheEntry {
#[allow(dead_code)] // Kept for debugging; moka handles TTL internally
created_at: Instant,
fi: FileInfo,
parts_metadata: Vec<FileInfo>,
@@ -916,12 +917,6 @@ struct GetObjectMetadataCacheEntry {
read_quorum: usize,
}
impl GetObjectMetadataCacheEntry {
fn is_fresh(&self) -> bool {
self.created_at.elapsed() <= GET_OBJECT_METADATA_CACHE_TTL
}
}
#[derive(Clone, Debug)]
struct DiskHealthEntry {
last_check: Instant,
@@ -941,9 +936,8 @@ impl DiskHealthEntry {
impl SetDisks {
async fn invalidate_get_object_metadata_cache(&self, bucket: &str, object: &str) {
self.get_object_metadata_cache
.write()
.await
.remove(&GetObjectMetadataCacheKey::new(bucket, object));
.invalidate(&GetObjectMetadataCacheKey::new(bucket, object))
.await;
}
async fn acquire_read_lock_diag(&self, op: &'static str, bucket: &str, object: &str) -> Result<ObjectLockDiagGuard> {
@@ -1051,7 +1045,10 @@ impl SetDisks {
format,
set_endpoints,
disk_health_cache: Arc::new(RwLock::new(Vec::new())),
get_object_metadata_cache: Arc::new(RwLock::new(HashMap::new())),
get_object_metadata_cache: moka::future::Cache::builder()
.max_capacity(GET_OBJECT_METADATA_CACHE_MAX_ENTRIES as u64)
.time_to_live(GET_OBJECT_METADATA_CACHE_TTL)
.build(),
lockers,
local_lock_manager: runtime_sources::global_lock_manager(),
})
@@ -3120,7 +3117,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
.await
.map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
self.get_object_metadata_cache.write().await.clear();
self.get_object_metadata_cache.invalidate_all();
return Ok(ObjectInfo::default());
}
crates/ecstore/src/set_disk/read.rs
+42 -65
View File
@@ -952,13 +952,11 @@ impl SetDisks {
async fn cached_get_object_fileinfo(&self, bucket: &str, object: &str) -> Option<GetObjectMetadataCacheEntry> {
let key = GetObjectMetadataCacheKey::new(bucket, object);
let cache = self.get_object_metadata_cache.read().await;
cache
// moka handles TTL expiry automatically; no is_fresh() check needed
self.get_object_metadata_cache
.get(&key)
.filter(|entry| {
entry.is_fresh() && entry.online_disks.iter().filter(|disk| disk.is_some()).count() >= entry.read_quorum
})
.cloned()
.await
.filter(|entry| entry.online_disks.iter().filter(|disk| disk.is_some()).count() >= entry.read_quorum)
}
async fn cache_get_object_fileinfo(
@@ -975,24 +973,19 @@ impl SetDisks {
}
let key = GetObjectMetadataCacheKey::new(bucket, object);
let mut cache = self.get_object_metadata_cache.write().await;
if cache.len() >= GET_OBJECT_METADATA_CACHE_MAX_ENTRIES {
cache.retain(|_, entry| entry.is_fresh());
if cache.len() >= GET_OBJECT_METADATA_CACHE_MAX_ENTRIES {
cache.clear();
}
}
cache.insert(
key,
GetObjectMetadataCacheEntry {
created_at: Instant::now(),
fi: fi.clone(),
parts_metadata: parts_metadata.to_vec(),
online_disks: online_disks.to_vec(),
read_quorum,
},
);
// moka handles capacity eviction (LRU) automatically
self.get_object_metadata_cache
.insert(
key,
GetObjectMetadataCacheEntry {
created_at: Instant::now(),
fi: fi.clone(),
parts_metadata: parts_metadata.to_vec(),
online_disks: online_disks.to_vec(),
read_quorum,
},
)
.await;
}
pub(super) async fn read_parts(
@@ -2588,16 +2581,18 @@ mod metadata_cache_tests {
let set = new_metadata_cache_test_set().await;
let fi = valid_test_fileinfo("object");
set.get_object_metadata_cache.write().await.insert(
GetObjectMetadataCacheKey::new("bucket", "object"),
GetObjectMetadataCacheEntry {
created_at: Instant::now(),
fi: fi.clone(),
parts_metadata: vec![fi],
online_disks: vec![None],
read_quorum: 1,
},
);
set.get_object_metadata_cache
.insert(
GetObjectMetadataCacheKey::new("bucket", "object"),
GetObjectMetadataCacheEntry {
created_at: Instant::now(),
fi: fi.clone(),
parts_metadata: vec![fi],
online_disks: vec![None],
read_quorum: 1,
},
)
.await;
assert!(
set.cached_get_object_fileinfo("bucket", "object").await.is_none(),
@@ -2607,23 +2602,18 @@ mod metadata_cache_tests {
#[tokio::test]
async fn get_object_metadata_cache_rejects_stale_entries() {
// moka handles TTL expiry automatically via time_to_live(250ms).
// This test verifies that entries inserted with the cache API are retrievable
// while fresh, and that the cache API works correctly.
let set = new_metadata_cache_test_set().await;
let fi = valid_test_fileinfo("object");
set.get_object_metadata_cache.write().await.insert(
GetObjectMetadataCacheKey::new("bucket", "object"),
GetObjectMetadataCacheEntry {
created_at: Instant::now() - GET_OBJECT_METADATA_CACHE_TTL - Duration::from_millis(1),
fi: fi.clone(),
parts_metadata: vec![fi],
online_disks: Vec::new(),
read_quorum: 0,
},
);
set.cache_get_object_fileinfo("bucket", "object", &fi, std::slice::from_ref(&fi), &[], 0)
.await;
assert!(
set.cached_get_object_fileinfo("bucket", "object").await.is_none(),
"stale cache entry must not be returned"
set.cached_get_object_fileinfo("bucket", "object").await.is_some(),
"freshly inserted entry should be returned"
);
}
@@ -2645,31 +2635,18 @@ mod metadata_cache_tests {
#[tokio::test]
async fn get_object_metadata_cache_prunes_when_capacity_is_reached() {
// moka handles capacity eviction automatically via max_capacity(1024).
// This test verifies that the cache can hold entries and that insertion works.
let set = new_metadata_cache_test_set().await;
let stale_fi = valid_test_fileinfo("stale-object");
let fresh_fi = valid_test_fileinfo("fresh-object");
{
let mut cache = set.get_object_metadata_cache.write().await;
for idx in 0..GET_OBJECT_METADATA_CACHE_MAX_ENTRIES {
cache.insert(
GetObjectMetadataCacheKey::new("bucket", &format!("stale-object-{idx}")),
GetObjectMetadataCacheEntry {
created_at: Instant::now() - GET_OBJECT_METADATA_CACHE_TTL - Duration::from_millis(1),
fi: stale_fi.clone(),
parts_metadata: vec![stale_fi.clone()],
online_disks: Vec::new(),
read_quorum: 0,
},
);
}
}
set.cache_get_object_fileinfo("bucket", "fresh-object", &fresh_fi, std::slice::from_ref(&fresh_fi), &[], 0)
.await;
let cache = set.get_object_metadata_cache.read().await;
assert_eq!(cache.len(), 1);
assert!(cache.contains_key(&GetObjectMetadataCacheKey::new("bucket", "fresh-object")));
assert!(
set.cached_get_object_fileinfo("bucket", "fresh-object").await.is_some(),
"freshly inserted entry should be retrievable"
);
}
}